def response(self): if not hasattr(self, '_response'): headers = {} if self.state.hidden: headers['User-Agent'] = get_ua() self._response = self.state.session.get(self.url, stream=True, headers=headers) return self._response
def load(self): headers = {} if self.config.hidden: headers['User-Agent'] = get_ua() try: return self.state.session.get(self.url, stream=True, headers=headers, timeout=60) except requests.RequestException as reqerr: log.info("Crawl error: %r", reqerr) time.sleep(BACKOFF_TIME) return reqerr.response