def run_crawler(crawler, threads=None, continue_on_error=False, flush=False, flushall=False): settings._crawler = crawler settings.CONTINUE_ON_ERROR = continue_on_error if flush: crawler.flush() if flushall: conn.flushall() crawler.run() if threads is not None and threads > 1: if settings.sls.REDIS_URL is None: log.warning( "REDIS_URL not set. Can't run in multithreaded mode without Redis. Exiting." ) return if settings.DATASTORE_URI.startswith("sqlite:///"): log.warning( "Can't run in multithreaded mode with sqlite database. Exiting." ) return worker = get_worker(num_threads=threads) code = worker.run(blocking=False) sys.exit(code)
def test_operation_reporting(self, crawler, context): conn.flushall() assert len(crawler.runs) == 0 stage = list(crawler.stages)[0] Crawl.operation_start(crawler, stage, context.run_id) assert crawler.latest_runid == context.run_id assert len(crawler.runs) == 1 assert crawler.op_count == 1 assert isinstance(crawler.last_run, datetime.datetime) Crawl.operation_end(crawler, context.run_id) crawler.flush() assert crawler.op_count == 0 assert len(crawler.runs) == 0
def killthekitten(): """Completely kill redis contents.""" from memorious.core import connect_redis conn = connect_redis() conn.flushall()