def get_crawler_runs(crawler): if not settings.REDIS_HOST: return [] conn = connect_redis() runs = [] for run_id in conn.smembers(crawler.name + ":runs"): runs.append({ 'run_id': run_id, 'total_ops': conn.get("run:" + run_id + ":total_ops"), 'start': parse_date(conn.get("run:" + run_id + ":start")), 'end': parse_date(conn.get("run:" + run_id + ":end")) }) return runs
def get_last_run(crawler): if not settings.REDIS_HOST: return None conn = connect_redis() last_run = conn.get(crawler.name + ":last_run") if last_run: return parse_date(last_run)
def last_modified(self): now = datetime.utcnow() last_modified_header = self.headers.get("Last-Modified") if last_modified_header is not None: # Tue, 15 Nov 1994 12:45:26 GMT last_modified = parse_date(last_modified_header) if last_modified < now + timedelta(seconds=30): return last_modified.strftime("%Y-%m-%dT%H:%M:%S%z") return None