def op_count(cls, crawler, stage=None):
"""Total operations performed for this crawler"""
if stage:
total_ops = conn.get(make_key(crawler, stage))
else:
total_ops = conn.get(make_key(crawler, "total_ops"))
return unpack_int(total_ops)
def runs(cls, crawler):
runs = []
for run_id in cls.run_ids(crawler):
start = conn.get(make_key(crawler, "run", run_id, "start"))
end = conn.get(make_key(crawler, "run", run_id, "end"))
total_ops = conn.get(make_key(crawler, "run", run_id, "total_ops"))
runs.append({
"run_id": run_id,
"total_ops": unpack_int(total_ops),
"start": unpack_datetime(start, datetime.utcnow()),
"end": unpack_datetime(end),
})
return runs
def timeout_expiration_check(self):
stages_on_timeout_key = make_key('memorious', 'timeout_stages')
stages_on_timeout = conn.smembers(stages_on_timeout_key)
for stage in stages_on_timeout:
key = make_key('memorious', 'timeout', stage)
if not conn.get(key):
conn.srem(stages_on_timeout_key, stage)
def load_session(self):
if self.STATE_SESSION not in self.context.state:
return
key = self.context.state.get(self.STATE_SESSION)
value = conn.get(make_key(self.context.run_id, "session", key))
if value is not None:
session = codecs.decode(bytes(value, 'utf-8'), 'base64')
return pickle.loads(session)
def load_session(self):
if self.STATE_SESSION not in self.context.state:
return
key = self.context.state.get(self.STATE_SESSION)
value = conn.get(key)
if value is not None:
session = codecs.decode(bytes(value, "utf-8"), "base64")
return pickle.loads(session)
def latest_runid(cls, crawler):
return conn.get(make_key(crawler, "current_run"))
def last_run(cls, crawler):
last_run = conn.get(make_key(crawler, "last_run"))
return unpack_datetime(last_run)
def get_tag(self, key):
value = conn.get(make_key(self.crawler, "tag", key))
if value is not None:
return load_json(value)
def get_schedule(cls, crawler):
key = make_key(crawler, "schedule")
return conn.get(key)
def get_stage_counts(cls, crawler, stage):
counts = {}
for level in cls.LEVELS:
key = make_key(crawler, "events", "count", stage, level)
counts[level] = unpack_int(conn.get(key))
return counts
def size(cls, crawler):
"""Total operations pending for this crawler"""
key = make_key('queue_pending', crawler)
return unpack_int(conn.get(key))
def abort_run(cls, crawler, run_id):
conn.sadd(make_key(crawler, "runs_abort"), run_id)
if conn.get(make_key("run", run_id, "end")) is None:
conn.set(make_key("run", run_id, "end"), pack_now())
