def run(self, run_command): """ Run the crawler of a code project """ import_user_module("crawlers") BaseProject.run(self, run_command, user_crawlers)
def run(self, run_command): """ Runs the crawley. For this kind of project it needs to generate the crawlers and scrapers classes at runtime firts. """ scraper_classes = run_command.syncdb.generator.gen_scrapers() compiler = CrawlerCompiler(scraper_classes, run_command.settings) crawler_class = compiler.compile() BaseProject.run(self, run_command, [crawler_class])
def run(self, run_command): """ Runs the crawley. For this kind of project it needs to generate the crawlers and scrapers classes at runtime first. """ scraper_classes = run_command.syncdb.generator.gen_scrapers() config = self._get_config(run_command) compiler = CrawlerCompiler(scraper_classes, config) crawler_class = compiler.compile() BaseProject.run(self, run_command, [crawler_class])