def exception_handler(req, ex):
msg = "Failed to download " + req.url
if isinstance(ex, requests.exceptions.Timeout):
msg = "Connection timed out: %(ex)s (%(url)s)" % {"ex": ex.__str__(), "url": req.url}
elif isinstance(ex, requests.exceptions.RequestException):
msg = "Error downloading: %(ex)s (%(url)s)" % {"ex": ex, "url": req.url}
else:
msg = " ".join([msg, str(ex)])
errors.append(req.url)
logger.error(msg)
def exception_handler(req, ex):
msg = 'Failed to download ' + req.url
if isinstance(ex, requests.exceptions.Timeout):
msg = 'Connection timed out: %(ex)s (%(url)s)' % {'ex': ex.__str__(), 'url': req.url}
elif isinstance(ex, requests.exceptions.RequestException):
msg = 'Error downloading: %(ex)s (%(url)s)' % {'ex': ex, 'url': req.url}
else:
msg = ' '.join([msg, str(ex)])
errors.append(req.url)
logger.error(msg)
def _download(self, url):
logger.info("Download started: " + url)
try:
r = requests.get(url)
logger.info("Download completed: " + url)
if r.status_code == 200:
return r.content
r.raise_for_status()
except requests.exceptions.Timeout as e:
logger.error("Connection timed out: " + e.__str__())
except requests.exceptions.RequestException as e:
logger.error("Error downloading: " + e.__str__())
return None
def _download(self, url):
logger.info('Download started: ' + url)
try:
r = requests.get(url)
logger.info('Download completed: ' + url)
if r.status_code == 200:
return r.content
r.raise_for_status()
except requests.exceptions.Timeout as e:
logger.error('Connection timed out: ' + e.__str__())
except requests.exceptions.RequestException as e:
logger.error('Error downloading: ' + e.__str__())
return None
