def test_configure_processors(self): processors = Processors() processor = MagicMock() processors.add(processor) config = {} processors.configure_all(config) processor.configure.assert_called_once_with(config)
def create_hackernews_api_crawler_job(config, session): """Create the job that will crawl hackernews :param dict config: the job configuration :param Session session: the sqlalchemy Session object to use :rtype: HackernewsCrawlJob :return: the job object """ logger.info("Initializing data processors") processors = Processors() processors.add(SQLAlchemyStorage(session)) processors.add_multiple( [import_string(processor)() for processor in config["PROCESSORS"]]) processors.configure_all(config) logger.info("Data processors initialized") job = HackernewsCrawlJob(config, processors) logger.info("Created job with id %s", job.id) return job