def sw_setup_add_seeds(self): settings = Settings() settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed' settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus' settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100 settings.STRATEGY = 'tests.mocks.components.CrawlingStrategy' return StrategyWorker(settings, True)
def sw_setup_filtered_links(self): settings = Settings() settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed' settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus' settings.STRATEGY = 'tests.test_worker_strategy.FilteredLinksCrawlingStrategy' settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100 return StrategyWorker(settings, False)
def dbw_setup(self, distributed=False): settings = Settings() settings.MAX_NEXT_REQUESTS = 64 settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus' if distributed: settings.BACKEND = 'tests.mocks.components.FakeDistributedBackend' else: settings.BACKEND = 'tests.mocks.components.FakeBackend' return DBWorker(settings, False, False, False, partitions="0")
def dbw_setup(self, distributed=False): settings = Settings() settings.MAX_NEXT_REQUESTS = 64 settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus' if distributed: settings.BACKEND = 'tests.mocks.components.FakeDistributedBackend' else: settings.BACKEND = 'tests.mocks.components.FakeBackend' return DBWorker(settings, True, True, False)
def setUp(self): settings = Settings() settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed' settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus' settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100 self.sw = StrategyWorker(settings, CrawlingStrategy, None, None)