def timeout(cls, stage, rate_limit):
stages_on_timeout = make_key("memorious", "timeout_stages")
conn.sadd(stages_on_timeout, stage.namespaced_name)
stage_timeout_key = make_key("memorious", "timeout", stage.namespaced_name)
expiry = (rate_limit.interval * rate_limit.unit) / rate_limit.limit
conn.set(stage_timeout_key, "true", ex=math.ceil(expiry))
# Delay the current task without further adding to call count
rate_limit.comply(amount=0)
def operation_start(cls, crawler, stage, run_id):
if not conn.sismember(make_key(crawler, "runs"), run_id):
conn.sadd(make_key(crawler, "runs"), run_id)
conn.set(make_key("run", run_id, "start"), pack_now())
conn.incr(make_key("run", run_id))
conn.incr(make_key("run", run_id, "total_ops"))
conn.incr(make_key(crawler, stage))
conn.incr(make_key(crawler, "total_ops"))
conn.set(make_key(crawler, "last_run"), pack_now())
conn.set(make_key(crawler, "current_run"), run_id)
def abort_run(cls, crawler, run_id):
conn.sadd(make_key(crawler, "runs_abort"), run_id)
conn.expire(make_key(crawler, "runs_abort"), REDIS_LONG)
conn.setnx(make_key(crawler, "run", run_id, "end"), pack_now())
job = Job(conn, crawler.queue, run_id)
job.remove()
def abort_run(cls, crawler, run_id):
conn.sadd(make_key(crawler, "runs_abort"), run_id)
if conn.get(make_key("run", run_id, "end")) is None:
conn.set(make_key("run", run_id, "end"), pack_now())