def __init__(self, crawler):
self.crawler = crawler
self.stats_manager = StatsManager(crawler.stats)
self._pending_requests = deque()
self.redirect_enabled = crawler.settings.get('REDIRECT_ENABLED')
settings = ScrapySettingsAdapter(crawler.settings)
self.frontier = ScrapyFrontierManager(settings)
self._delay_on_empty = self.frontier.manager.settings.get('DELAY_ON_EMPTY')
self._delay_next_call = 0.0
def __init__(self, crawler, manager=None):
self.crawler = crawler
self.stats_manager = StatsManager(crawler.stats)
self._pending_requests = deque()
self.redirect_enabled = crawler.settings.get('REDIRECT_ENABLED')
settings = ScrapySettingsAdapter(crawler.settings)
self.frontier = ScrapyFrontierManager(settings, manager)
self._delay_on_empty = self.frontier.manager.settings.get('DELAY_ON_EMPTY')
self._delay_next_call = 0.0
self.logger = getLogger('frontera.contrib.scrapy.schedulers.FronteraScheduler')
def __init__(self, crawler):
self.crawler = crawler
self.stats_manager = StatsManager(crawler.stats)
self._pending_requests = deque()
self.redirect_enabled = crawler.settings.get('REDIRECT_ENABLED')
frontier_settings = crawler.settings.get('FRONTERA_SETTINGS', None)
if not frontier_settings:
log.msg('FRONTERA_SETTINGS not found! Using default Frontera settings...', log.WARNING)
self.frontier = ScrapyFrontierManager(frontier_settings)
self._delay_on_empty = self.frontier.manager.settings.get('DELAY_ON_EMPTY')
self._delay_next_call = 0.0
