예제 #1
0
 def fail_action(self, values):
     print "come in fail_action"
     #失败了就将记录type恢复为2,并累加fail_times
     update_sql = """
             update hainiu_queue set fail_times=fail_times+1,fail_ip='%s' where id=%s;
             """
     update_sql1 = """
             update hainiu_queue set type=2 where id =%s
             """
     try:
         d = DBUtil(config._OGC_DB)
         id = values[0]
         u = Util()
         ip = u.get_local_ip()
         sql = update_sql % (ip, id)
         d.execute(sql)
         d.execute_no_commit(sql)
         #超过单机器尝试次数,工作状态置为不工作
         if (self.try_num == Consumer._WORK_TRY_NUM):
             sql = update_sql1 % id
             d.execute_no_commit(sql)
         d.commit()
     except:
         self.rl.error(sql)
         self.rl.exception()
     finally:
         d.close()
예제 #2
0
def push_queue_items():
    redis = RedisUtil()
    rl = LogUtil().get_base_logger()
    # redis_key = redis.kyes_limit_scan(pattern="down*", limit=10)
    # print redis_key
    # type为2意思是从redis中获取的待下载链接,为3意思是已经被消费者拿去下载了
    insert_queue_sql = """
            insert into hainiu_queue(type,params,action) values (2,'from redis','%s');
            """
    try:
        db = DBUtil(config._OGC_DB)
        redis_len = len(redis.get_conn().keys())
        page_size = 10
        page_num = redis_len / page_size
        # redis_len = len(redis.get_conn().keys("down*"))
        # sum=0
        for i in range(0, page_num):
            redis_key = redis.kyes_limit_scan(pattern="down*",
                                              limit=page_size * (i + 1),
                                              cursor=0)
            if len(redis_key) != 0:
                redis_value = redis.get_values_batch_keys(redis_key)
                for each in redis_value:
                    print each
                    sql = insert_queue_sql % (each)
                    db.execute_no_commit(sql)
                db.commit()
                redis.delete_batch(redis_key)
            #避免后面无用的扫描
            # if sum==redis_len:
            #     break
        #下面是一下全取出来,没有分页的方法
        # redis_key=redis.get_conn().keys("down*")
        # print redis_key
        # if len(redis_key) !=0:
        #     redis_value = redis.get_values_batch_keys(redis_key)
        #     for each in redis_value:
        #         print redis_value
        #         sql=insert_queue_sql%(each[5:])
        #         db.execute_no_commit(sql)
        #     db.commit()
        #     redis.delete_batch(redis_key)
    except:
        rl.exception()
        rl.error(insert_queue_sql)
        db.rollback()
    finally:
        db.close()
예제 #3
0
 def fail_action(self, values):
     update_sql = """
     update hainiu_queue set fail_times=fail_times+1,fail_ip='%s' where id=%s;
     """
     update_sql1 = """
     update hainiu_queue set is_work=0 where id =%s
     """
     try:
         d = DBUtil(config._OGC_DB)
         id = values[0]
         u = Util()
         ip = u.get_local_ip()
         sql = update_sql % (ip, id)
         d.execute_no_commit(sql)
         if (self.try_num == Consumer.work_try_num):
             sql = update_sql1 % id
             d.execute_no_commit(sql)
         d.commit()
     except:
         self.rl.error()
         self.rl.exception()
     finally:
         d.close()
예제 #4
0
 def fail_action(self,values):
     update_sql="""
     update hainiu_web_seed set fail_times=fail_times+1,fail_ip='%s' where id=%s;
     """
     #超过尝试次数就把工作状态设为不工作状态
     update_sql1="""
     update hainiu_web_seed set status=0,last_crawl_time='' where id =%s
     """
     try:
         d=DBUtil(config._OGC_DB)
         id=values[0]
         u=Util
         ip=u.get_local_ip()
         sql=update_sql % (ip,id)
         d.execute_no_commit(sql)
         if(self.try_num==Consumer.work_try_num):
             sql=update_sql1 % id
             d.execute_no_commit(sql)
         d.commit()
     except:
         self.rl.error(sql)
         self.rl.exception()
     finally:
         d.close()
예제 #5
0
 def fail_action(self,values):
     #失败之后恢复type为0,以便让其他线程继续访问
     update_sql="""
     update hainiu_queue set fail_times=fail_times+1,fail_ip='%s' where id=%s;
     """
     update_sql1="""
     update hainiu_queue set type=0 where id =%s
     """
     try:
         d=DBUtil(config._OGC_DB)
         id=values[0]
         u=Util()
         ip=u.get_local_ip()
         sql=update_sql % (ip,id)
         d.execute_no_commit(sql)
         if(self.try_num==Consumer._WORK_TRY_NUM):
             sql=update_sql1 % id
             d.execute_no_commit(sql)
         d.commit()
     except:
         self.rl.error(sql)
         self.rl.exception()
     finally:
         d.close()