def test_add_multiple_processors(self):
processors = Processors()
processor_1 = MagicMock()
processor_2 = MagicMock()
processor_3 = MagicMock()
processors.add_multiple([processor_1, processor_2, processor_3])
self.assertEqual(processors.count, 3)
self.assertEqual(processors._processors,
[processor_1, processor_2, processor_3])
def create_hackernews_api_crawler_job(config, session):
"""Create the job that will crawl hackernews
:param dict config: the job configuration
:param Session session: the sqlalchemy Session object to use
:rtype: HackernewsCrawlJob
:return: the job object
"""
logger.info("Initializing data processors")
processors = Processors()
processors.add(SQLAlchemyStorage(session))
processors.add_multiple(
[import_string(processor)() for processor in config["PROCESSORS"]])
processors.configure_all(config)
logger.info("Data processors initialized")
job = HackernewsCrawlJob(config, processors)
logger.info("Created job with id %s", job.id)
return job