def get_crawler_runs(crawler):
if not settings.REDIS_HOST:
return []
conn = connect_redis()
runs = []
for run_id in conn.smembers(crawler.name + ":runs"):
runs.append({
'run_id': run_id,
'total_ops': conn.get("run:" + run_id + ":total_ops"),
'start': parse_date(conn.get("run:" + run_id + ":start")),
'end': parse_date(conn.get("run:" + run_id + ":end"))
})
return runs
def get_last_run(crawler):
if not settings.REDIS_HOST:
return None
conn = connect_redis()
last_run = conn.get(crawler.name + ":last_run")
if last_run:
return parse_date(last_run)
def last_modified(self):
now = datetime.utcnow()
last_modified_header = self.headers.get("Last-Modified")
if last_modified_header is not None:
# Tue, 15 Nov 1994 12:45:26 GMT
last_modified = parse_date(last_modified_header)
if last_modified < now + timedelta(seconds=30):
return last_modified.strftime("%Y-%m-%dT%H:%M:%S%z")
return None