def start(crawlid, host, custom): if custom: from custom_redis.client import Redis else: from redis import Redis redis_conn = Redis(host) key = "crawlid:%s" % crawlid failed_pages = int(redis_conn.hget(key, "failed_download_pages") or 0) format(redis_conn.hgetall(key)) if failed_pages: print_if = raw_input("show the failed pages? y/n default n:") if print_if == "y": key_ = "failed_pages:%s" % crawlid p = redis_conn.hgetall(key_) format(p, True)
def start(crawlid, host, port, custom): if custom: from custom_redis.client import Redis else: from redis import Redis redis_conn = Redis(host, port) key = "crawlid:%s" % crawlid data = redis_conn.hgetall(key) failed_keys = [x for x in data.keys() if fnmatch.fnmatch( x.decode() if isinstance(x, bytes) else x, "failed_download_*")] format(data) for fk in failed_keys: fk = fk.decode() if isinstance(fk, bytes) else fk print_if = input("show the %s? y/n default n:" % fk.replace("_", " ")) if print_if == "y": key_ = "%s:%s" % (fk, crawlid) p = redis_conn.hgetall(key_) format(p, True)
def start(crawlid, host, custom): if custom: from custom_redis.client import Redis else: from redis import Redis redis_conn = Redis(host) key = "crawlid:%s" % crawlid data = redis_conn.hgetall(key) failed_keys = filter(lambda x: fnmatch.fnmatch(x, "failed_download_*"), data.keys()) format(data) for fk in failed_keys: print_if = raw_input("show the %s? y/n default n:" % fk.replace("_", " ")) if print_if == "y": key_ = "%s:%s" % (fk, crawlid) p = redis_conn.hgetall(key_) format(p, True)