def run_crawler(crawler,
threads=None,
continue_on_error=False,
flush=False,
flushall=False):
settings._crawler = crawler
settings.CONTINUE_ON_ERROR = continue_on_error
if flush:
crawler.flush()
if flushall:
conn.flushall()
crawler.run()
if threads is not None and threads > 1:
if settings.sls.REDIS_URL is None:
log.warning(
"REDIS_URL not set. Can't run in multithreaded mode without Redis. Exiting."
)
return
if settings.DATASTORE_URI.startswith("sqlite:///"):
log.warning(
"Can't run in multithreaded mode with sqlite database. Exiting."
)
return
worker = get_worker(num_threads=threads)
code = worker.run(blocking=False)
sys.exit(code)
def test_operation_reporting(self, crawler, context):
conn.flushall()
assert len(crawler.runs) == 0
stage = list(crawler.stages)[0]
Crawl.operation_start(crawler, stage, context.run_id)
assert crawler.latest_runid == context.run_id
assert len(crawler.runs) == 1
assert crawler.op_count == 1
assert isinstance(crawler.last_run, datetime.datetime)
Crawl.operation_end(crawler, context.run_id)
crawler.flush()
assert crawler.op_count == 0
assert len(crawler.runs) == 0
def killthekitten():
"""Completely kill redis contents."""
from memorious.core import connect_redis
conn = connect_redis()
conn.flushall()