Beispiel #1
0
    def __init__(self, spider):
        if hasattr(spider, 'download_delay'):
            deprecate.attribute(spider, 'download_delay', 'DOWNLOAD_DELAY')
            self._download_delay = spider.download_delay
        else:
            self._download_delay = spider.settings.getfloat('DOWNLOAD_DELAY')
        if self._download_delay:
            self.max_concurrent_requests = 1
        else:
            if hasattr(spider, 'max_concurrent_requests'):
                deprecate.attribute(spider, 'max_concurrent_requests',
                                    'CONCURRENT_REQUESTS_PER_SPIDER')
                self.max_concurrent_requests = spider.max_concurrent_requests
            else:
                self.max_concurrent_requests = spider.settings.getint(
                    'CONCURRENT_REQUESTS_PER_SPIDER')
        if self._download_delay and spider.settings.getbool(
                'RANDOMIZE_DOWNLOAD_DELAY'):
            # same policy as wget --random-wait
            self.random_delay_interval = (0.5*self._download_delay, \
                1.5*self._download_delay)
        else:
            self.random_delay_interval = None

        self.active = set()
        self.queue = []
        self.transferring = set()
        self.closing = False
        self.lastseen = 0
        self.next_request_calls = set()
Beispiel #2
0
    def __init__(self, spider):
        if hasattr(spider, 'download_delay'):
            deprecate.attribute(spider, 'download_delay', 'DOWNLOAD_DELAY')
            self._download_delay = spider.download_delay
        else:
            self._download_delay = spider.settings.getfloat('DOWNLOAD_DELAY')
        if self._download_delay:
            self.max_concurrent_requests = 1
        else:
            if hasattr(spider, 'max_concurrent_requests'):
                deprecate.attribute(spider, 'max_concurrent_requests', 'CONCURRENT_REQUESTS_PER_SPIDER')
                self.max_concurrent_requests = spider.max_concurrent_requests
            else:
                self.max_concurrent_requests = spider.settings.getint('CONCURRENT_REQUESTS_PER_SPIDER')
        if self._download_delay and spider.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY'):
            # same policy as wget --random-wait
            self.random_delay_interval = (0.5*self._download_delay, \
                1.5*self._download_delay)
        else:
            self.random_delay_interval = None

        self.active = set()
        self.queue = []
        self.transferring = set()
        self.closing = False
        self.lastseen = 0
        self.next_request_calls = set()
Beispiel #3
0
 def _download_timeout(self, spider):
     if hasattr(spider, 'download_timeout'):
         deprecate.attribute(spider, 'download_timeout', 'DOWNLOAD_TIMEOUT')
         return spider.download_timeout
     return spider.settings.getint('DOWNLOAD_TIMEOUT')
Beispiel #4
0
 def _user_agent(self, spider):
     if hasattr(spider, "user_agent"):
         deprecate.attribute(spider, "user_agent", "USER_AGENT")
         return spider.user_agent
     return spider.settings["USER_AGENT"]
Beispiel #5
0
 def _user_agent(self, spider):
     if hasattr(spider, 'user_agent'):
         deprecate.attribute(spider, 'user_agent', 'USER_AGENT')
         return spider.user_agent
     return spider.settings['USER_AGENT']