Esempi in Python per Settings.SPIDER_LOG_CONSUMER_BATCH_SIZE

Esempio n. 1

 def sw_setup_add_seeds(self):
     settings = Settings()
     settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed'
     settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus'
     settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100
     settings.STRATEGY = 'tests.mocks.components.CrawlingStrategy'
     return StrategyWorker(settings, True)

Esempio n. 2

 def sw_setup_filtered_links(self):
     settings = Settings()
     settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed'
     settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus'
     settings.STRATEGY = 'tests.test_worker_strategy.FilteredLinksCrawlingStrategy'
     settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100
     return StrategyWorker(settings, False)

Esempio n. 3

File: test_worker_strategy.py Progetto: scrapinghub/frontera

 def sw_setup_add_seeds(self):
     settings = Settings()
     settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed'
     settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus'
     settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100
     settings.STRATEGY = 'tests.mocks.components.CrawlingStrategy'
     return StrategyWorker(settings, True)

Esempio n. 4

File: test_worker_strategy.py Progetto: scrapinghub/frontera

 def sw_setup_filtered_links(self):
     settings = Settings()
     settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed'
     settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus'
     settings.STRATEGY = 'tests.test_worker_strategy.FilteredLinksCrawlingStrategy'
     settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100
     return StrategyWorker(settings, False)

Esempio n. 5

File: test_worker_strategy.py Progetto: widy28/frontera

 def setUp(self):
     settings = Settings()
     settings.BACKEND = 'frontera.contrib.backends.sqlalchemy.Distributed'
     settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus'
     settings.SPIDER_LOG_CONSUMER_BATCH_SIZE = 100
     self.sw = StrategyWorker(settings, CrawlingStrategy, None, None)