def auth_wget(url, chunk_size=1048576): """Returns the content of specified URL, which requires authentication. If the content is bigger than 1MB, then save it to file. """ opener = build_opener(X509CertOpen()) url_file = opener.open(Request(url)) size = int(url_file.headers["Content-Length"]) if size < 1048576: # if File size < 1MB filename = basename(url) #still download readed = url_file.read( ) ## and then check if its not an empty dir (parent directory) if filename != '': outfile = open(filename, 'wb') #then write File to local system outfile.write(readed) return readed filename = basename(url) file_id = selected_files.index(filename) if isfile("./%s" % filename): print('%d. Exists on disk. Skipping.' % (file_id + 1)) return print('%d. Downloading...' % (file_id + 1)) file = open(filename, 'wb') # progress = 0 chunk = url_file.read(chunk_size) while chunk: file.write(chunk) # progress += chunk_size chunk = url_file.read(chunk_size) print('%d. Done.' % (file_id + 1)) file.close()
def auth_wget(url, chunk_size=2097152): from os.path import basename #, isfile """Returns the content of specified URL, which requires authentication. If the content is bigger than 1MB, then save it to file. """ # NOTE 2097152 instead of 1048576 is needed because CMSSW_7_1_X > 1 MB opener = build_opener(X509CertOpen()) url_file = opener.open(Request(url)) size = int(url_file.headers["Content-Length"]) if size < 2097152: # if File size < 2MB filename = basename(url) #still download readed = url_file.read( ) ## and then check if its not an empty dir (parent directory) if filename != '': outfile = open(filename, 'wb') #then write File to local system outfile.write(readed) return readed filename = basename(url) #file_id = selected_files.index(filename) file = open(filename, 'wb') chunk = url_file.read(chunk_size) while chunk: file.write(chunk) chunk = url_file.read(chunk_size) file.close()
def auth_wget2(url, chunk_size=2097152): from os.path import basename, isfile from optparse import OptionParser from urllib2 import build_opener, Request """Returns the content of specified URL, which requires authentication. If the content is bigger than 1MB, then save it to file. """ try: from Utilities.RelMon.authentication import X509CertOpen except ImportError: from authentication import X509CertOpen opener = build_opener(X509CertOpen()) url_file = opener.open(Request(url)) size = int(url_file.headers["Content-Length"]) filename = basename(url) if isfile("./%s" % filename): print '%s. Exists on disk. Skipping.' % (filename) return file = open(filename, 'wb') chunk = url_file.read(chunk_size) while chunk: file.write(chunk) chunk = url_file.read(chunk_size) file.close()
def auth_wget(url, chunk_size=1048576): """Returns the content of specified URL, which requires authentication. If the content is bigger than 1MB, then save it to file. """ opener = build_opener(X509CertOpen()) url_file = opener.open(Request(url)) size = int(url_file.headers["Content-Length"]) if size < 1048576: return url_file.read() filename = basename(url) file_id = selected_files.index(filename) if isfile("./%s" % filename): print '%d. Exsits on disk. Skipping.' % (file_id + 1) return print '%d. Downloading...' % (file_id + 1) file = open(filename, 'wb') # progress = 0 chunk = url_file.read(chunk_size) while chunk: file.write(chunk) # progress += chunk_size chunk = url_file.read(chunk_size) print '%d. Done.' % (file_id + 1) file.close()
def get_page(url): """ Get the web page listing the rootfiles. Use the X509 auth. """ opener = build_opener(X509CertOpen()) datareq = Request(url) datareq.add_header('authenticated_wget', "The ultimate wgetter") filename = basename(url) return opener.open(datareq).read()
def __init__(self, server, is_private=False, ident="DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]): self.ident = ident self.server = server self.is_private = is_private self.DQMpwd = DQMcommunicator.base_dir self.prevDQMpwd = self.DQMpwd self.opener = None if not self.is_private: self.opener = build_opener(X509CertOpen())
def wget(url): """ Fetch the WHOLE file, not in bunches... To be optimised. """ opener = build_opener(X509CertOpen()) datareq = Request(url) datareq.add_header('authenticated_wget', "The ultimate wgetter") bin_content = None try: filename = basename(url) print "Checking existence of file %s on disk..." % filename if not isfile("./%s" % filename): bin_content = opener.open(datareq).read() else: print "File %s exists, skipping.." % filename except ValueError: print "Error: Unknown url %s" % url if bin_content != None: ofile = open(filename, 'wb') ofile.write(bin_content) ofile.close()
def auth_wget2(url, chunk_size=2097152): from os.path import basename, isfile from optparse import OptionParser """Returns the content of specified URL, which requires authentication. If the content is bigger than 1MB, then save it to file. """ opener = build_opener(X509CertOpen()) url_file = opener.open(Request(url)) filename = basename(url) if isfile("./%s" % filename): print('%s. Exists on disk. Skipping.' % (filename)) return file = open(filename, 'wb') chunk = url_file.read(chunk_size) while chunk: file.write(chunk) chunk = url_file.read(chunk_size) print('\rDownloaded: %s ' % (filename, )) file.close()