def start(crawlid, host, custom):
if custom:
from custom_redis.client import Redis
else:
from redis import Redis
redis_conn = Redis(host)
key = "crawlid:%s" % crawlid
failed_pages = int(redis_conn.hget(key, "failed_download_pages") or 0)
format(redis_conn.hgetall(key))
if failed_pages:
print_if = raw_input("show the failed pages? y/n default n:")
if print_if == "y":
key_ = "failed_pages:%s" % crawlid
p = redis_conn.hgetall(key_)
format(p, True)
def start(crawlid, host, port, custom):
if custom:
from custom_redis.client import Redis
else:
from redis import Redis
redis_conn = Redis(host, port)
key = "crawlid:%s" % crawlid
data = redis_conn.hgetall(key)
failed_keys = [x for x in data.keys() if fnmatch.fnmatch(
x.decode() if isinstance(x, bytes) else x, "failed_download_*")]
format(data)
for fk in failed_keys:
fk = fk.decode() if isinstance(fk, bytes) else fk
print_if = input("show the %s? y/n default n:" % fk.replace("_", " "))
if print_if == "y":
key_ = "%s:%s" % (fk, crawlid)
p = redis_conn.hgetall(key_)
format(p, True)
def start(crawlid, host, custom):
if custom:
from custom_redis.client import Redis
else:
from redis import Redis
redis_conn = Redis(host)
key = "crawlid:%s" % crawlid
data = redis_conn.hgetall(key)
failed_keys = filter(lambda x: fnmatch.fnmatch(x, "failed_download_*"),
data.keys())
format(data)
for fk in failed_keys:
print_if = raw_input("show the %s? y/n default n:" %
fk.replace("_", " "))
if print_if == "y":
key_ = "%s:%s" % (fk, crawlid)
p = redis_conn.hgetall(key_)
format(p, True)