def get_conn(self):
try:
return psycopg2.connect(database=PG_DB, user=PG_USER, password=PG_PWD,
host=PG_HOST, port=PG_PORT)
except Exception as e:
error_record('200')
pd_db_log.warning('200:Can not establish a connection to guangzhou pg DB: {}'.format(e.message))
def connection_test(self):
try:
self.R.set('test', 'test')
except:
error_record('101')
redis_db_log.warning(
'101:Can not establish a connection to local redis DB')
def task_failed(self, task_id):
"""move task from doing to failed"""
try:
self.R.smove(TASK_DOING, TASK_FAILED, task_id)
return True
except Exception as e:
error_record('107')
redis_db_log.warning('107:task failed error :{}'.format(e.message))
def task_finished(self, task_ids):
"""move task from doing to finished save task detail"""
p = self.R.pipeline(transaction=False)
try:
for task_id in task_ids:
p.smove(TASK_DOING, TASK_FINISH, task_id)
p.execute()
return True
redis_db_log.info('finish back {} tasks'.format(len(task_ids)))
except Exception as e:
error_record('106')
redis_db_log.warning('106:task finished error:{}'.format(
e.message))
def channel_2(self,data_list):
""" for table rc_cp_mi_info_phone_company"""
if not isinstance(data_list,list):
return
try:
self.cursor.execute("INSERT INTO hadoop.rc_cp_mi_info_phone_company"
"(phone,company,create_time)VALUES(%s,%s,%s)",tuple(data_list))
self.conn.commit()
return True
except Exception as e:
self.conn.rollback()
pd_db_log.warning('202:Error in insert data to pg table two:{}'.format(e.message))
if 'duplicate' not in e.message:
error_record('202')
def trim_failed_set(self):
"""trim failed set and back task to todo list"""
try:
count = self.R.scard(TASK_FAILED)
failed_list = [self.R.spop(TASK_FAILED) for i in xrange(count)]
back_list = filter(self.task_is_available, failed_list)
map(self.back_task_to_todo, back_list)
clear_redis_log.info('success clear failed set,'
'total:{},back{}'.format(
count, len(back_list)))
return True
except Exception as e:
error_record('002')
clear_redis_log.warning('002:trim_failed_set error :{}'.format(
e.message))
def channel_1(self,data_list):
""" for table rc_cp_mi_info_contact_phone"""
if not isinstance(data_list,list):
return
try:
self.cursor.execute("INSERT INTO hadoop.rc_cp_mi_info_contact_phone"
"(idnumber,c_phone,relation,c_name,comment,level"
",create_time)VALUES(%s,%s,%s,%s,%s,%s,%s)",tuple(data_list))
self.conn.commit()
return True
except Exception as e:
self.conn.rollback()
pd_db_log.warning('201:arror in insert data to pg table one:{}'.format(e.message))
if 'duplicate' not in e.message:
error_record('201')
def save_tasks_to_redis(self, tasks):
"""save tasks to todo list"""
if self.R.dbsize() >= REDIS_AMOUNT:
return 'full'
p = self.R.pipeline(transaction=False)
try:
for task in tasks:
task_id = task['task_id']
task['retry'] = 0
p.rpush(TASK_TODO, task_id)
p.hmset(task_id, task)
p.execute()
return True
except Exception as e:
error_record('103')
redis_db_log.warning('103:Save many tasks error:{}'.format(
e.message))
def trim_finish_set(self):
"""trim finish set to empty"""
p = self.R.pipeline(transaction=False)
try:
count = self.R.scard(TASK_FINISH)
if count <= TASK_AMOUNT:
return True
need_clear = count - TASK_AMOUNT
clear_list = [self.R.spop(TASK_FINISH) for i in xrange(need_clear)]
map(p.delete, clear_list)
p.execute()
clear_redis_log.info(
'success clear finish set,count:{}'.format(need_clear))
return True
except Exception as e:
error_record('001')
clear_redis_log.warning('001:trim_finsh_set error :{}'.format(
e.message))
def handle_bad_doing(self):
"""if task stay in doing-set over 1 day
back it to todo-list for another try"""
try:
count = 0
doing_list = self.R.smembers(TASK_DOING)
for task_id in doing_list:
date = self.R.hget(task_id, 'date')
if not isinstance(date, int):
date = int(date)
if abs(date - int(get_current_day())) > 0:
count += 1
self.R.srem(TASK_DOING, task_id)
self.back_task_to_todo(task_id)
clear_redis_log.info('success handle doing set,'
'back count:{}'.format(count))
return True
except Exception as e:
error_record('003')
clear_redis_log.warning('003:handle_bad_doing error :{}'.format(
e.message))
def get_task(self, count):
"""get tasks to do"""
def is_dict(item):
if isinstance(item, dict):
return True
p = self.R.pipeline(transaction=False)
try:
[p.lpop(TASK_TODO) for x in xrange(count)]
task_ids = p.execute()
for task_id in task_ids:
if not task_id:
continue
p.hincrby(task_id, 'retry', amount=1)
p.hset(task_id, 'date', get_current_day())
p.sadd(TASK_DOING, task_id)
p.hgetall(task_id)
tasks = filter(is_dict, p.execute())
redis_db_log.info('success get {} tasks'.format(len(tasks)))
return tasks
except Exception as e:
error_record('105')
redis_db_log.warning('105:get tasks error:{}'.format(e.message))
return []
def get_cursor(self):
try:
return self.conn.cursor()
except Exception as e:
error_record('200')
pd_db_log.warning('200:Can not establish a connection to guangzhou pg DB :{}'.format(e.message))