def auth_wget(url, chunk_size=1048576):
"""Returns the content of specified URL, which requires authentication.
If the content is bigger than 1MB, then save it to file.
"""
opener = build_opener(X509CertOpen())
url_file = opener.open(Request(url))
size = int(url_file.headers["Content-Length"])
if size < 1048576: # if File size < 1MB
filename = basename(url) #still download
readed = url_file.read(
) ## and then check if its not an empty dir (parent directory)
if filename != '':
outfile = open(filename,
'wb') #then write File to local system
outfile.write(readed)
return readed
filename = basename(url)
file_id = selected_files.index(filename)
if isfile("./%s" % filename):
print('%d. Exists on disk. Skipping.' % (file_id + 1))
return
print('%d. Downloading...' % (file_id + 1))
file = open(filename, 'wb')
# progress = 0
chunk = url_file.read(chunk_size)
while chunk:
file.write(chunk)
# progress += chunk_size
chunk = url_file.read(chunk_size)
print('%d. Done.' % (file_id + 1))
file.close()
def auth_wget(url, chunk_size=2097152):
from os.path import basename #, isfile
"""Returns the content of specified URL, which requires authentication.
If the content is bigger than 1MB, then save it to file.
"""
# NOTE 2097152 instead of 1048576 is needed because CMSSW_7_1_X > 1 MB
opener = build_opener(X509CertOpen())
url_file = opener.open(Request(url))
size = int(url_file.headers["Content-Length"])
if size < 2097152: # if File size < 2MB
filename = basename(url) #still download
readed = url_file.read(
) ## and then check if its not an empty dir (parent directory)
if filename != '':
outfile = open(filename,
'wb') #then write File to local system
outfile.write(readed)
return readed
filename = basename(url)
#file_id = selected_files.index(filename)
file = open(filename, 'wb')
chunk = url_file.read(chunk_size)
while chunk:
file.write(chunk)
chunk = url_file.read(chunk_size)
file.close()
def auth_wget2(url, chunk_size=2097152):
from os.path import basename, isfile
from optparse import OptionParser
from urllib2 import build_opener, Request
"""Returns the content of specified URL, which requires authentication.
If the content is bigger than 1MB, then save it to file.
"""
try:
from Utilities.RelMon.authentication import X509CertOpen
except ImportError:
from authentication import X509CertOpen
opener = build_opener(X509CertOpen())
url_file = opener.open(Request(url))
size = int(url_file.headers["Content-Length"])
filename = basename(url)
if isfile("./%s" % filename):
print '%s. Exists on disk. Skipping.' % (filename)
return
file = open(filename, 'wb')
chunk = url_file.read(chunk_size)
while chunk:
file.write(chunk)
chunk = url_file.read(chunk_size)
file.close()
def auth_wget(url, chunk_size=1048576):
"""Returns the content of specified URL, which requires authentication.
If the content is bigger than 1MB, then save it to file.
"""
opener = build_opener(X509CertOpen())
url_file = opener.open(Request(url))
size = int(url_file.headers["Content-Length"])
if size < 1048576:
return url_file.read()
filename = basename(url)
file_id = selected_files.index(filename)
if isfile("./%s" % filename):
print '%d. Exsits on disk. Skipping.' % (file_id + 1)
return
print '%d. Downloading...' % (file_id + 1)
file = open(filename, 'wb')
# progress = 0
chunk = url_file.read(chunk_size)
while chunk:
file.write(chunk)
# progress += chunk_size
chunk = url_file.read(chunk_size)
print '%d. Done.' % (file_id + 1)
file.close()
def get_page(url):
""" Get the web page listing the rootfiles. Use the X509 auth.
"""
opener = build_opener(X509CertOpen())
datareq = Request(url)
datareq.add_header('authenticated_wget', "The ultimate wgetter")
filename = basename(url)
return opener.open(datareq).read()
def __init__(self,
server,
is_private=False,
ident="DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]):
self.ident = ident
self.server = server
self.is_private = is_private
self.DQMpwd = DQMcommunicator.base_dir
self.prevDQMpwd = self.DQMpwd
self.opener = None
if not self.is_private:
self.opener = build_opener(X509CertOpen())
def wget(url):
""" Fetch the WHOLE file, not in bunches... To be optimised.
"""
opener = build_opener(X509CertOpen())
datareq = Request(url)
datareq.add_header('authenticated_wget', "The ultimate wgetter")
bin_content = None
try:
filename = basename(url)
print "Checking existence of file %s on disk..." % filename
if not isfile("./%s" % filename):
bin_content = opener.open(datareq).read()
else:
print "File %s exists, skipping.." % filename
except ValueError:
print "Error: Unknown url %s" % url
if bin_content != None:
ofile = open(filename, 'wb')
ofile.write(bin_content)
ofile.close()
def auth_wget2(url, chunk_size=2097152):
from os.path import basename, isfile
from optparse import OptionParser
"""Returns the content of specified URL, which requires authentication.
If the content is bigger than 1MB, then save it to file.
"""
opener = build_opener(X509CertOpen())
url_file = opener.open(Request(url))
filename = basename(url)
if isfile("./%s" % filename):
print('%s. Exists on disk. Skipping.' % (filename))
return
file = open(filename, 'wb')
chunk = url_file.read(chunk_size)
while chunk:
file.write(chunk)
chunk = url_file.read(chunk_size)
print('\rDownloaded: %s ' % (filename, ))
file.close()
