def response(self):
if not hasattr(self, '_response'):
headers = {}
if self.state.hidden:
headers['User-Agent'] = get_ua()
self._response = self.state.session.get(self.url, stream=True,
headers=headers)
return self._response
def response(self):
if not hasattr(self, '_response'):
headers = {}
if self.state.hidden:
headers['User-Agent'] = get_ua()
self._response = self.state.session.get(self.url,
stream=True,
headers=headers)
return self._response
def load(self):
headers = {}
if self.config.hidden:
headers['User-Agent'] = get_ua()
try:
return self.state.session.get(self.url,
stream=True,
headers=headers,
timeout=60)
except requests.RequestException as reqerr:
log.info("Crawl error: %r", reqerr)
time.sleep(BACKOFF_TIME)
return reqerr.response
