def run_crawler(crawler,
threads=None,
continue_on_error=False,
flush=False,
flushall=False):
settings._crawler = crawler
settings.CONTINUE_ON_ERROR = continue_on_error
if flush:
crawler.flush()
if flushall:
conn.flushall()
crawler.run()
if threads is not None and threads > 1:
if settings.sls.REDIS_URL is None:
log.warning(
"REDIS_URL not set. Can't run in multithreaded mode without Redis. Exiting."
)
return
if settings.DATASTORE_URI.startswith("sqlite:///"):
log.warning(
"Can't run in multithreaded mode with sqlite database. Exiting."
)
return
worker = get_worker(num_threads=threads)
code = worker.run(blocking=False)
sys.exit(code)
def run(crawler):
"""Run a specified crawler."""
crawler = get_crawler(crawler)
crawler.run()
if is_sync_mode():
worker = get_worker()
worker.sync()
def process():
"""Start the queue and process tasks as they come. Blocks while waiting"""
worker = get_worker()
worker.run()