def get(url):
request = Request(url)
attempt = 0
while True:
attempt += 1
try:
return urlopen(request, timeout=settings.HTTP_TIMEOUT).read()
except (URLError, OSError) as exc:
logger.debug(
'Error occurred during fetching %s',
url,
exc_info=exc,
)
if attempt == settings.HTTP_RETRIES:
raise FakeUserAgentError('Maximum amount of retries reached')
else:
logger.debug(
'Sleeping for %s secconds',
settings.HTTP_TIMEOUT,
)
sleep(settings.HTTP_TIMEOUT)
def get(url, verify_ssl=True):
attempt = 0
while True:
request = Request(url)
attempt += 1
try:
if urlopen_has_ssl_context:
if not verify_ssl:
context = ssl._create_unverified_context()
else:
context = None
with contextlib.closing(urlopen(
request,
timeout=settings.HTTP_TIMEOUT,
context=context,
)) as response:
return response.read()
else: # ssl context is not supported ;(
with contextlib.closing(urlopen(
request,
timeout=settings.HTTP_TIMEOUT,
)) as response:
return response.read()
except (URLError, OSError) as exc:
logger.debug(
'Error occurred during fetching %s',
url,
exc_info=exc,
)
if attempt == settings.HTTP_RETRIES:
raise FakeUserAgentError('Maximum amount of retries reached')
else:
logger.debug(
'Sleeping for %s seconds',
settings.HTTP_DELAY,
)
try:
response = requests.get(url)
return response.content
except:
pass
sleep(settings.HTTP_DELAY)
def get(url, verify_ssl=True):
attempt = 0
while True:
request = Request(url)
attempt += 1
try:
if urlopen_has_ssl_context:
if not verify_ssl:
context = ssl._create_unverified_context()
else:
context = None
with contextlib.closing(urlopen(
request,
timeout=settings.HTTP_TIMEOUT,
context=context,
)) as response:
return response.read()
else: # ssl context is not supported ;(
with contextlib.closing(urlopen(
request,
timeout=settings.HTTP_TIMEOUT,
)) as response:
return response.read()
except (URLError, OSError) as exc:
logger.debug(
'Error occurred during fetching %s',
url,
exc_info=exc,
)
if attempt == settings.HTTP_RETRIES:
raise FakeUserAgentError('Maximum amount of retries reached')
else:
logger.debug(
'Sleeping for %s seconds',
settings.HTTP_DELAY,
)
sleep(settings.HTTP_DELAY)