def test_add_multiple_processors(self): processors = Processors() processor_1 = MagicMock() processor_2 = MagicMock() processor_3 = MagicMock() processors.add_multiple([processor_1, processor_2, processor_3]) self.assertEqual(processors.count, 3) self.assertEqual(processors._processors, [processor_1, processor_2, processor_3])
def create_hackernews_api_crawler_job(config, session): """Create the job that will crawl hackernews :param dict config: the job configuration :param Session session: the sqlalchemy Session object to use :rtype: HackernewsCrawlJob :return: the job object """ logger.info("Initializing data processors") processors = Processors() processors.add(SQLAlchemyStorage(session)) processors.add_multiple( [import_string(processor)() for processor in config["PROCESSORS"]]) processors.configure_all(config) logger.info("Data processors initialized") job = HackernewsCrawlJob(config, processors) logger.info("Created job with id %s", job.id) return job