Пример #1
0
 def __init__(self, crawler):
     self.crawler = crawler
     self.stats_manager = StatsManager(crawler.stats)
     self._pending_requests = deque()
     self.redirect_enabled = crawler.settings.get('REDIRECT_ENABLED')
     settings = ScrapySettingsAdapter(crawler.settings)
     self.frontier = ScrapyFrontierManager(settings)
     self._delay_on_empty = self.frontier.manager.settings.get('DELAY_ON_EMPTY')
     self._delay_next_call = 0.0
Пример #2
0
 def __init__(self, crawler, manager=None):
     self.crawler = crawler
     self.stats_manager = StatsManager(crawler.stats)
     self._pending_requests = deque()
     self.redirect_enabled = crawler.settings.get('REDIRECT_ENABLED')
     settings = ScrapySettingsAdapter(crawler.settings)
     self.frontier = ScrapyFrontierManager(settings, manager)
     self._delay_on_empty = self.frontier.manager.settings.get('DELAY_ON_EMPTY')
     self._delay_next_call = 0.0
     self.logger = getLogger('frontera.contrib.scrapy.schedulers.FronteraScheduler')
Пример #3
0
    def __init__(self, crawler):
        self.crawler = crawler
        self.stats_manager = StatsManager(crawler.stats)
        self._pending_requests = deque()
        self.redirect_enabled = crawler.settings.get('REDIRECT_ENABLED')

        frontier_settings = crawler.settings.get('FRONTERA_SETTINGS', None)
        if not frontier_settings:
            log.msg('FRONTERA_SETTINGS not found! Using default Frontera settings...', log.WARNING)
        self.frontier = ScrapyFrontierManager(frontier_settings)

        self._delay_on_empty = self.frontier.manager.settings.get('DELAY_ON_EMPTY')
        self._delay_next_call = 0.0