def sw_setup_add_seeds(self): settings = Settings() settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed' settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus' settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100 settings.STRATEGY = 'tests.mocks.components.CrawlingStrategy' return StrategyWorker(settings, True)
def sw_setup_filtered_links(self): settings = Settings() settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed' settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus' settings.STRATEGY = 'tests.test_worker_strategy.FilteredLinksCrawlingStrategy' settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100 return StrategyWorker(settings, False)
def sw_setup_add_seeds(self): settings = Settings() settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed' settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus' settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100 settings.STRATEGY = 'tests.mocks.components.CrawlingStrategy' return StrategyWorker(settings, True)
def sw_setup_filtered_links(self): settings = Settings() settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed' settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus' settings.STRATEGY = 'tests.test_worker_strategy.FilteredLinksCrawlingStrategy' settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100 return StrategyWorker(settings, False)
def setUp(self): settings = Settings() settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed' settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus' settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100 self.sw = StrategyWorker(settings, CrawlingStrategy, None, None)