def run(self, run_command):
"""
Run the crawler of a code project
"""
import_user_module("crawlers")
BaseProject.run(self, run_command, user_crawlers)
def run(self, run_command):
"""
Runs the crawley.
For this kind of project it needs to generate the crawlers and scrapers
classes at runtime firts.
"""
scraper_classes = run_command.syncdb.generator.gen_scrapers()
compiler = CrawlerCompiler(scraper_classes, run_command.settings)
crawler_class = compiler.compile()
BaseProject.run(self, run_command, [crawler_class])
def run(self, run_command):
"""
Runs the crawley.
For this kind of project it needs to generate the crawlers and scrapers
classes at runtime first.
"""
scraper_classes = run_command.syncdb.generator.gen_scrapers()
config = self._get_config(run_command)
compiler = CrawlerCompiler(scraper_classes, config)
crawler_class = compiler.compile()
BaseProject.run(self, run_command, [crawler_class])
