def execute(self, args, opts): task = None if opts.task_id: task = Task().load(id=opts.task_id) if opts.task_name: task = Task().next(name=opts.task_name) if task or len(args): if task: domain = task.domain else: domain = args[0] spider = spiders.fromdomain(domain) scrapymanager.configure() if opts.child: def _stop(): pass # monkeypatching stop command to prevent stoping prematurely in child mode scrapymanager.stop = _stop if not task.locked: task.lock() self.crawl(spider, task) scrapyengine.start() else: log.msg('You must specify atleast 1 domain', level=log.ERROR)
def start(self): """Start the scrapy server, without scheduling any domains""" scrapyengine.keep_alive = True scrapyengine.start() if self.control_reactor: reactor.run(installSignalHandlers=False)
def runonce(self, *args): """Run the engine until it finishes scraping all domains and then exit""" self.crawl(*args) scrapyengine.start() if self.control_reactor: reactor.run(installSignalHandlers=False)