def init_http_request(url, byte_range=0):
LOG.debug('Trying to initialize http request object %s, byte range: %s' %
(url, byte_range))
retry = 0
while True:
if (CONF.http_max_retries == 0) or retry <= CONF.http_max_retries:
try:
response_obj = requests.get(
url,
stream=True,
timeout=CONF.http_request_timeout,
headers={'Range': 'bytes=%s-' % byte_range})
except (socket.timeout, urllib3.exceptions.DecodeError,
urllib3.exceptions.ProxyError,
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.TooManyRedirects) as e:
LOG.debug('Got non-critical error when accessing to %s '
'on %s attempt: %s' % (url, retry + 1, e))
else:
LOG.debug('Successful http request to %s on %s retry' %
(url, retry + 1))
break
retry += 1
time.sleep(CONF.http_retry_delay)
else:
raise errors.HttpUrlConnectionError(
'Exceeded maximum http request retries for %s' % url)
response_obj.raise_for_status()
return response_obj
def init_http_request(url, byte_range=0, proxies=None, noproxy_addrs=None):
LOG.debug("Trying to initialize http request object %s, byte range: %s",
url, byte_range)
if should_bypass_proxy(url, noproxy_addrs):
LOG.debug("Proxy will be bypassed for url %s", url)
proxies = None
retry = 0
while True:
if (CONF.http_max_retries == 0) or retry <= CONF.http_max_retries:
try:
response_obj = requests.get(
url,
stream=True,
timeout=CONF.http_request_timeout,
headers={'Range': 'bytes=%s-' % byte_range},
proxies=proxies)
except (socket.timeout, urllib3.exceptions.DecodeError,
urllib3.exceptions.ProxyError,
requests.exceptions.ConnectionError,
requests.exceptions.Timeout,
requests.exceptions.TooManyRedirects) as e:
LOG.debug(
"Got non-critical error when accessing to %s "
"on %s attempt: %s", url, retry + 1, e)
else:
LOG.debug("Successful http request to %s on %s retry", url,
retry + 1)
break
retry += 1
time.sleep(CONF.http_retry_delay)
else:
raise errors.HttpUrlConnectionError(
"Exceeded maximum http request retries for %s".format(url))
response_obj.raise_for_status()
return response_obj
def next(self):
while self.processed_bytes < self.length:
try:
data = self.response_obj.raw.read(CONF.data_chunk_size)
if not data:
raise errors.HttpUrlConnectionError(
'Could not receive data: URL=%s, range=%s' %
(self.url, self.processed_bytes))
except Exception as exc:
LOG.exception(exc)
self.response_obj = utils.init_http_request(
self.url, byte_range=self.processed_bytes)
continue
else:
self.processed_bytes += len(data)
return data
raise StopIteration()