def run(self, incremental=None, run_id=None):
"""Queue the execution of a particular crawler."""
state = {
'crawler': self.name,
'run_id': run_id,
'incremental': settings.INCREMENTAL
}
if incremental is not None:
state['incremental'] = incremental
# TaskRunner.execute(stage.name, state, {})
Queue.queue(self.init_stage, state, {})
if not settings.REDIS_HOST:
TaskRunner.run()
def run(self, incremental=None, run_id=None):
"""Queue the execution of a particular crawler."""
state = {
'crawler': self.name,
'run_id': run_id,
'incremental': settings.INCREMENTAL
}
if incremental is not None:
state['incremental'] = incremental
# Cancel previous runs:
self.cancel()
# Flush out previous events:
Event.delete(self)
Queue.queue(self.init_stage, state, {})
if not settings.REDIS_HOST:
TaskRunner.run()
def process():
"""Start the queue and process tasks as they come. Blocks while waiting"""
TaskRunner.run()
def run(crawler):
"""Run a specified crawler."""
crawler = get_crawler(crawler)
crawler.run()
if settings.DEBUG:
TaskRunner.run_sync()
def run(crawler):
"""Run a specified crawler."""
crawler = get_crawler(crawler)
crawler.run()
if is_sync_mode():
TaskRunner.run_sync()