def execute(self, args, opts):
task = None
if opts.task_id:
task = Task().load(id=opts.task_id)
if opts.task_name:
task = Task().next(name=opts.task_name)
if task or len(args):
if task:
domain = task.domain
else:
domain = args[0]
spider = spiders.fromdomain(domain)
scrapymanager.configure()
if opts.child:
def _stop():
pass
# monkeypatching stop command to prevent stoping prematurely in child mode
scrapymanager.stop = _stop
if not task.locked:
task.lock()
self.crawl(spider, task)
scrapyengine.start()
else:
log.msg('You must specify atleast 1 domain', level=log.ERROR)
def start(self):
"""Start the scrapy server, without scheduling any domains"""
scrapyengine.keep_alive = True
scrapyengine.start()
if self.control_reactor:
reactor.run(installSignalHandlers=False)
def runonce(self, *args):
"""Run the engine until it finishes scraping all domains and then exit"""
self.crawl(*args)
scrapyengine.start()
if self.control_reactor:
reactor.run(installSignalHandlers=False)
