def fail_action(self, values):
print "come in fail_action"
#失败了就将记录type恢复为2，并累加fail_times
update_sql = """
update hainiu_queue set fail_times=fail_times+1,fail_ip='%s' where id=%s;
"""
update_sql1 = """
update hainiu_queue set type=2 where id =%s
"""
try:
d = DBUtil(config._OGC_DB)
id = values[0]
u = Util()
ip = u.get_local_ip()
sql = update_sql % (ip, id)
d.execute(sql)
d.execute_no_commit(sql)
#超过单机器尝试次数，工作状态置为不工作
if (self.try_num == Consumer._WORK_TRY_NUM):
sql = update_sql1 % id
d.execute_no_commit(sql)
d.commit()
except:
self.rl.error(sql)
self.rl.exception()
finally:
d.close()
def push_queue_items():
redis = RedisUtil()
rl = LogUtil().get_base_logger()
# redis_key = redis.kyes_limit_scan(pattern="down*", limit=10)
# print redis_key
# type为2意思是从redis中获取的待下载链接，为3意思是已经被消费者拿去下载了
insert_queue_sql = """
insert into hainiu_queue(type,params,action) values (2,'from redis','%s');
"""
try:
db = DBUtil(config._OGC_DB)
redis_len = len(redis.get_conn().keys())
page_size = 10
page_num = redis_len / page_size
# redis_len = len(redis.get_conn().keys("down*"))
# sum=0
for i in range(0, page_num):
redis_key = redis.kyes_limit_scan(pattern="down*",
limit=page_size * (i + 1),
cursor=0)
if len(redis_key) != 0:
redis_value = redis.get_values_batch_keys(redis_key)
for each in redis_value:
print each
sql = insert_queue_sql % (each)
db.execute_no_commit(sql)
db.commit()
redis.delete_batch(redis_key)
#避免后面无用的扫描
# if sum==redis_len:
# break
#下面是一下全取出来，没有分页的方法
# redis_key=redis.get_conn().keys("down*")
# print redis_key
# if len(redis_key) !=0:
# redis_value = redis.get_values_batch_keys(redis_key)
# for each in redis_value:
# print redis_value
# sql=insert_queue_sql%(each[5:])
# db.execute_no_commit(sql)
# db.commit()
# redis.delete_batch(redis_key)
except:
rl.exception()
rl.error(insert_queue_sql)
db.rollback()
finally:
db.close()
def fail_action(self, values):
update_sql = """
update hainiu_queue set fail_times=fail_times+1,fail_ip='%s' where id=%s;
"""
update_sql1 = """
update hainiu_queue set is_work=0 where id =%s
"""
try:
d = DBUtil(config._OGC_DB)
id = values[0]
u = Util()
ip = u.get_local_ip()
sql = update_sql % (ip, id)
d.execute_no_commit(sql)
if (self.try_num == Consumer.work_try_num):
sql = update_sql1 % id
d.execute_no_commit(sql)
d.commit()
except:
self.rl.error()
self.rl.exception()
finally:
d.close()
def fail_action(self,values):
update_sql="""
update hainiu_web_seed set fail_times=fail_times+1,fail_ip='%s' where id=%s;
"""
#超过尝试次数就把工作状态设为不工作状态
update_sql1="""
update hainiu_web_seed set status=0,last_crawl_time='' where id =%s
"""
try:
d=DBUtil(config._OGC_DB)
id=values[0]
u=Util
ip=u.get_local_ip()
sql=update_sql % (ip,id)
d.execute_no_commit(sql)
if(self.try_num==Consumer.work_try_num):
sql=update_sql1 % id
d.execute_no_commit(sql)
d.commit()
except:
self.rl.error(sql)
self.rl.exception()
finally:
d.close()
def fail_action(self,values):
#失败之后恢复type为0，以便让其他线程继续访问
update_sql="""
update hainiu_queue set fail_times=fail_times+1,fail_ip='%s' where id=%s;
"""
update_sql1="""
update hainiu_queue set type=0 where id =%s
"""
try:
d=DBUtil(config._OGC_DB)
id=values[0]
u=Util()
ip=u.get_local_ip()
sql=update_sql % (ip,id)
d.execute_no_commit(sql)
if(self.try_num==Consumer._WORK_TRY_NUM):
sql=update_sql1 % id
d.execute_no_commit(sql)
d.commit()
except:
self.rl.error(sql)
self.rl.exception()
finally:
d.close()