def auth_wget(url, chunk_size=1048576):
"""Returns the content of specified URL, which requires authentication.
If the content is bigger than 1MB, then save it to file.
"""
opener = build_opener(X509CertOpen())
url_file = opener.open(Request(url))
size = int(url_file.headers["Content-Length"])
if size < 1048576: # if File size < 1MB
filename = basename(url) #still download
readed = url_file.read(
) ## and then check if its not an empty dir (parent directory)
if filename != '':
outfile = open(filename, 'wb') #then write File to local system
outfile.write(readed)
return readed
filename = basename(url)
file_id = selected_files.index(filename)
if isfile("./%s" % filename):
print('%d. Exsits on disk. Skipping.' % (file_id + 1))
return
print('%d. Downloading...' % (file_id + 1))
file = open(filename, 'wb')
# progress = 0
chunk = url_file.read(chunk_size)
while chunk:
file.write(chunk)
# progress += chunk_size
chunk = url_file.read(chunk_size)
print('%d. Done.' % (file_id + 1))
file.close()
def auth_wget(url):
try:
opener = build_opener(X509CertOpen())
return opener.open(Request(url)).read()
except HTTPError, e:
print '\nError: DQM GUI is temporarily unavailable. Probably maintainance hours. '+\
'Please try again later. Original error message: ``%s``. \nExiting...\n' % (e,)
exit()
def auth_wget(url):
try:
opener = build_opener(X509CertOpen())
return opener.open(Request(url)).read()
except HTTPError as e:
print('\nError: DQM GUI is temporarily unavailable. Probably maintainance hours. '+\
'Please try again later. Original error message: ``%s``. \nExiting...\n' % (e,))
exit()
except BadStatusLine as e:
print('\nYou do not have permissions to access DQM GUI. Please check if your certificates '+\
'in ``~/.globus`` directory are configured correctly. Exitting...')
exit()
def auth_download_file(url, chunk_size=1048576):
filename = basename(url)
file_path = join(auth_download_file.work_dir, filename)
file = open(file_path, 'wb')
opener = build_opener(X509CertOpen())
url_file = opener.open(Request(url))
chunk = url_file.read(chunk_size)
while chunk:
file.write(chunk)
auth_download_file.q.put((1, )) # reports, that downloaded 1MB
chunk = url_file.read(chunk_size)
print '\rDownloaded: %s ' % (filename, )
file.close()
def get_size_to_download(work_path, files_with_urls):
"""Returns file list to download and total size to download."""
opener = build_opener(X509CertOpen())
size_to_download = 0
files_to_download = []
for filename, url in files_with_urls:
url_file = opener.open(Request(url))
size = int(url_file.headers["Content-Length"])
file_path = join(work_path, filename)
if exists(file_path) and getsize(file_path) / 1024 == size / 1024:
print "Exists on disk %s." % filename
else:
size_to_download += size
files_to_download.append(url)
return size_to_download, files_to_download
def wget(url):
""" Fetch the WHOLE file, not in bunches... To be optimised.
"""
opener=build_opener(X509CertOpen())
datareq = Request(url)
datareq.add_header('authenticated_wget', "The ultimate wgetter")
bin_content=None
try:
filename=basename(url)
print("Checking existence of file %s on disk..."%filename)
if not isfile("./%s"%filename):
bin_content=opener.open(datareq).read()
else:
print("File %s exists, skipping.." %filename)
except ValueError:
print("Error: Unknown url %s" %url)
if bin_content!=None:
ofile = open(filename, 'wb')
ofile.write(bin_content)
ofile.close()
