def __init__(self, spider):
if hasattr(spider, 'download_delay'):
deprecate.attribute(spider, 'download_delay', 'DOWNLOAD_DELAY')
self._download_delay = spider.download_delay
else:
self._download_delay = spider.settings.getfloat('DOWNLOAD_DELAY')
if self._download_delay:
self.max_concurrent_requests = 1
else:
if hasattr(spider, 'max_concurrent_requests'):
deprecate.attribute(spider, 'max_concurrent_requests',
'CONCURRENT_REQUESTS_PER_SPIDER')
self.max_concurrent_requests = spider.max_concurrent_requests
else:
self.max_concurrent_requests = spider.settings.getint(
'CONCURRENT_REQUESTS_PER_SPIDER')
if self._download_delay and spider.settings.getbool(
'RANDOMIZE_DOWNLOAD_DELAY'):
# same policy as wget --random-wait
self.random_delay_interval = (0.5*self._download_delay, \
1.5*self._download_delay)
else:
self.random_delay_interval = None
self.active = set()
self.queue = []
self.transferring = set()
self.closing = False
self.lastseen = 0
self.next_request_calls = set()
def __init__(self, spider):
if hasattr(spider, 'download_delay'):
deprecate.attribute(spider, 'download_delay', 'DOWNLOAD_DELAY')
self._download_delay = spider.download_delay
else:
self._download_delay = spider.settings.getfloat('DOWNLOAD_DELAY')
if self._download_delay:
self.max_concurrent_requests = 1
else:
if hasattr(spider, 'max_concurrent_requests'):
deprecate.attribute(spider, 'max_concurrent_requests', 'CONCURRENT_REQUESTS_PER_SPIDER')
self.max_concurrent_requests = spider.max_concurrent_requests
else:
self.max_concurrent_requests = spider.settings.getint('CONCURRENT_REQUESTS_PER_SPIDER')
if self._download_delay and spider.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY'):
# same policy as wget --random-wait
self.random_delay_interval = (0.5*self._download_delay, \
1.5*self._download_delay)
else:
self.random_delay_interval = None
self.active = set()
self.queue = []
self.transferring = set()
self.closing = False
self.lastseen = 0
self.next_request_calls = set()
def _download_timeout(self, spider):
if hasattr(spider, 'download_timeout'):
deprecate.attribute(spider, 'download_timeout', 'DOWNLOAD_TIMEOUT')
return spider.download_timeout
return spider.settings.getint('DOWNLOAD_TIMEOUT')
def _user_agent(self, spider):
if hasattr(spider, "user_agent"):
deprecate.attribute(spider, "user_agent", "USER_AGENT")
return spider.user_agent
return spider.settings["USER_AGENT"]
def _user_agent(self, spider):
if hasattr(spider, 'user_agent'):
deprecate.attribute(spider, 'user_agent', 'USER_AGENT')
return spider.user_agent
return spider.settings['USER_AGENT']