def op_count(cls, crawler, stage=None): """Total operations performed for this crawler""" if stage: total_ops = conn.get(make_key(crawler, stage)) else: total_ops = conn.get(make_key(crawler, "total_ops")) return unpack_int(total_ops)
def runs(cls, crawler): runs = [] for run_id in cls.run_ids(crawler): start = conn.get(make_key(crawler, "run", run_id, "start")) end = conn.get(make_key(crawler, "run", run_id, "end")) total_ops = conn.get(make_key(crawler, "run", run_id, "total_ops")) runs.append({ "run_id": run_id, "total_ops": unpack_int(total_ops), "start": unpack_datetime(start, datetime.utcnow()), "end": unpack_datetime(end), }) return runs
def timeout_expiration_check(self): stages_on_timeout_key = make_key('memorious', 'timeout_stages') stages_on_timeout = conn.smembers(stages_on_timeout_key) for stage in stages_on_timeout: key = make_key('memorious', 'timeout', stage) if not conn.get(key): conn.srem(stages_on_timeout_key, stage)
def load_session(self): if self.STATE_SESSION not in self.context.state: return key = self.context.state.get(self.STATE_SESSION) value = conn.get(make_key(self.context.run_id, "session", key)) if value is not None: session = codecs.decode(bytes(value, 'utf-8'), 'base64') return pickle.loads(session)
def load_session(self): if self.STATE_SESSION not in self.context.state: return key = self.context.state.get(self.STATE_SESSION) value = conn.get(key) if value is not None: session = codecs.decode(bytes(value, "utf-8"), "base64") return pickle.loads(session)
def latest_runid(cls, crawler): return conn.get(make_key(crawler, "current_run"))
def last_run(cls, crawler): last_run = conn.get(make_key(crawler, "last_run")) return unpack_datetime(last_run)
def get_tag(self, key): value = conn.get(make_key(self.crawler, "tag", key)) if value is not None: return load_json(value)
def get_schedule(cls, crawler): key = make_key(crawler, "schedule") return conn.get(key)
def get_stage_counts(cls, crawler, stage): counts = {} for level in cls.LEVELS: key = make_key(crawler, "events", "count", stage, level) counts[level] = unpack_int(conn.get(key)) return counts
def size(cls, crawler): """Total operations pending for this crawler""" key = make_key('queue_pending', crawler) return unpack_int(conn.get(key))
def abort_run(cls, crawler, run_id): conn.sadd(make_key(crawler, "runs_abort"), run_id) if conn.get(make_key("run", run_id, "end")) is None: conn.set(make_key("run", run_id, "end"), pack_now())