def save(self):
session = pickle.dumps(self.session)
session = codecs.encode(session, 'base64')
key = sha1(session).hexdigest()[:15]
key = make_key(self.context.run_id, "session", key)
conn.set(key, session, ex=QUEUE_EXPIRE)
self.context.state[self.STATE_SESSION] = key
def operation_end(cls, crawler, run_id):
conn.set(make_key(crawler, "last_run"), pack_now(), ex=REDIS_LONG)
pending = conn.decr(make_key(crawler, "run", run_id))
if unpack_int(pending) == 0:
conn.set(make_key(crawler, "run", run_id, "end"),
pack_now(),
ex=REDIS_LONG)
def save(self):
session = pickle.dumps(self.session)
session = codecs.encode(session, 'base64')
key = sha1(session).hexdigest()[:15]
key = make_key(self.context.crawler, "session", self.context.run_id, key) # noqa
conn.set(key, session, ex=REDIS_SHORT)
self.context.state[self.STATE_SESSION] = key
def timeout(cls, stage, rate_limit):
stages_on_timeout = make_key("memorious", "timeout_stages")
conn.sadd(stages_on_timeout, stage.namespaced_name)
stage_timeout_key = make_key("memorious", "timeout", stage.namespaced_name)
expiry = (rate_limit.interval * rate_limit.unit) / rate_limit.limit
conn.set(stage_timeout_key, "true", ex=math.ceil(expiry))
# Delay the current task without further adding to call count
rate_limit.comply(amount=0)
def operation_start(cls, crawler, stage, run_id):
if not conn.sismember(make_key(crawler, "runs"), run_id):
conn.sadd(make_key(crawler, "runs"), run_id)
conn.set(make_key("run", run_id, "start"), pack_now())
conn.incr(make_key("run", run_id))
conn.incr(make_key("run", run_id, "total_ops"))
conn.incr(make_key(crawler, stage))
conn.incr(make_key(crawler, "total_ops"))
conn.set(make_key(crawler, "last_run"), pack_now())
conn.set(make_key(crawler, "current_run"), run_id)
def heartbeat(cls, crawler):
conn.set(make_key(crawler, "last_run"), pack_now())
def set_tag(self, key, value):
data = dump_json(value)
key = make_key(self.crawler, "tag", key)
return conn.set(key, data, ex=self.crawler.expire)
def set_schedule(cls, crawler, schedule):
key = make_key(crawler, "schedule")
conn.set(key, schedule)
def abort_run(cls, crawler, run_id):
conn.sadd(make_key(crawler, "runs_abort"), run_id)
if conn.get(make_key("run", run_id, "end")) is None:
conn.set(make_key("run", run_id, "end"), pack_now())
