def download_gf_store(url=g_url_static,
site=g_default_site,
majorversion=1,
store_id=None,
force=False,
quiet=False):
url = fillurl(url, site, 'static', majorversion)
stores_url = ujoin(url, 'stores')
tlast = [time.time()]
if not quiet:
def status_callback(d):
i = d['nread_bytes_all_files']
n = d['ntotal_bytes_all_files']
tnow = time.time()
if n != 0 and ((tnow - tlast[0]) > 5 or i == n):
print('%s / %s [%.1f%%]' %
(util.human_bytesize(i), util.human_bytesize(n),
i * 100.0 / n))
tlast[0] = tnow
else:
def status_callback(d):
pass
wanted = ['config', 'extra/', 'index', 'phases/', 'traces/']
try:
if store_id is None:
print(
static(url=stores_url + '/',
format='text').read().decode('utf-8'))
else:
store_url = ujoin(stores_url, store_id)
stotal = rget(store_url,
store_id,
force=force,
method='calcsize',
entries_wanted=wanted)
rget(store_url,
store_id,
force=force,
stats=[0, stotal],
status_callback=status_callback,
entries_wanted=wanted)
except Exception as e:
raise DownloadError('download failed. Original error was: %s, %s' %
(type(e).__name__, e))
import shutil
shutil.rmtree(store_id)
def __init__(self, url):
DownloadError.__init__(self)
self._url = url
def _request(url, post=False, user=None, passwd=None,
allow_TLSv1=False, **kwargs):
timeout = float(kwargs.pop('timeout', g_timeout))
url_values = urlencode(kwargs)
if url_values:
url += '?' + url_values
logger.debug('Accessing URL %s' % url)
url_args = {
'timeout': timeout
}
if allow_TLSv1:
url_args['context'] = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
opener = None
req = Request(url)
if post:
if isinstance(post, str):
post = post.encode('utf8')
logger.debug('POST data: \n%s' % post.decode('utf8'))
req.data = post
req.add_header('Accept', '*/*')
itry = 0
while True:
itry += 1
try:
urlopen_ = opener.open if opener else urlopen
while True:
try:
resp = urlopen_(req, **url_args)
break
except TypeError:
del url_args['context'] # context not avail before 3.4.3
logger.debug('Response: %s' % resp.getcode())
if resp.getcode() == 204:
raise EmptyResult(url)
return resp
except HTTPError as e:
if e.code == 413:
raise RequestEntityTooLarge(url)
elif e.code == 401:
headers = getattr(e, 'headers', e.hdrs)
realm = get_realm_from_auth_header(headers)
if itry == 1 and user is not None:
auth_handler = HTTPDigestAuthHandler()
auth_handler.add_password(
realm=realm,
uri=url,
user=user,
passwd=passwd or '')
opener = build_opener(auth_handler)
continue
else:
logger.error(
'authentication failed for realm "%s" when '
'accessing url "%s"' % (realm, url))
raise DownloadError('Original error was: %s' % str(e))
else:
logger.error(
'error content returned by server:\n%s' % e.read())
raise DownloadError('Original error was: %s' % str(e))
break
def __init__(self, url):
DownloadError.__init__(self)
self._url = url
def _request(
url,
post=False,
user=None,
passwd=None,
allow_TLSv1=False,
timeout=g_timeout,
**kwargs):
url_values = urlencode(kwargs)
if url_values:
url += '?' + url_values
logger.debug('Accessing URL %s' % url)
url_args = {
'timeout': timeout
}
if allow_TLSv1:
url_args['context'] = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
opener = None
req = Request(url)
if post:
if isinstance(post, newstr):
post = post.encode('utf8')
logger.debug('POST data: \n%s' % post.decode('utf8'))
req.data = post
req.add_header('Accept', '*/*')
itry = 0
while True:
itry += 1
try:
urlopen_ = opener.open if opener else urlopen
while True:
try:
resp = urlopen_(req, **url_args)
break
except TypeError:
del url_args['context'] # context not avail before 3.4.3
logger.debug('Response: %s' % resp.getcode())
if resp.getcode() == 204:
raise EmptyResult(url)
return resp
except HTTPError as e:
if e.code == 413:
raise RequestEntityTooLarge(url)
elif e.code == 401:
headers = getattr(e, 'headers', e.hdrs)
realm = get_realm_from_auth_header(headers)
if itry == 1 and user is not None:
auth_handler = HTTPDigestAuthHandler()
auth_handler.add_password(
realm=realm,
uri=url,
user=user,
passwd=passwd or '')
opener = build_opener(auth_handler)
continue
else:
raise DownloadError(
'Authentication failed for realm "%s" when accessing '
'url "%s". Original error was: %s' % (
realm, url, str(e)))
else:
raise DownloadError(
'Error content returned by server (HTML stripped):\n%s\n'
' Original error was: %s' % (
indent(
strip_html(e.read()),
' ! '),
str(e)))
except socket.timeout:
raise Timeout(
'Timeout error. No response received within %i s. You '
'may want to retry with a longer timeout setting.' % timeout)
break