def tuangou_brand_taskrun(event_day):
#任务时间
print(u"任务时间为："+str(event_day))
#任务初始化
tuangou_brand=class_tuangou_brand(event_day)
mysql=mysqlLib()
now=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
param=("tuandan_pinpai",now,0,event_day)
n,last_id=mysql.add_task(param)
print n,last_id
mysql.close()
#多进程随机取样
tuangou_brand.batch_rand_sample()
#取样二次过滤
tuangou_brand.secondary_filter()
print tuangou_brand.sample_list
mysql=mysqlLib()
for sample in tuangou_brand.sample_list:
sample=sample.strip()
param=(last_id,sample,0)
mysql.add_sample(param)
mysql.close()
#线程池下发历史信息查询任务
tuangou_brand.batch_get_history_cuid_info()
print tuangou_brand.history_info_list,len(tuangou_brand.history_info_list)
fp=open(tuangou_brand.data_directory+tuangou_brand.event_day+tuangou_brand.history_info_filepath,"w")
for line in tuangou_brand.history_info_list:
fp.write(line)
fp.close()
mysql=mysqlLib()
for line in tuangou_brand.history_info_list:
line = line.strip()
line_list=re.split(r'\t',line.rstrip('\t'))
param=(last_id,line_list[0],line_list[1],line_list[2],line_list[3],line_list[4],line_list[5],line_list[6],line_list[7],line_list[8],)
mysql.add_history_info(param)
mysql.close()
#构造线程池，源数据文件批量下载
tuangou_brand.batch_download_upps_src_data()
#源数据文件解析
tuangou_brand.get_src_data_info()
#推荐信息查询
tuangou_brand.batch_get_recommend_cuid_info("tuangou","10af214253a015b8ffdfbac9f98077b4","userpreference",12)
print tuangou_brand.recommend_info_list
#推荐信息入库
mysql=mysqlLib()
for recommend_info in tuangou_brand.recommend_info_list:
recommend_info_list=recommend_info.split("\t")
param=(last_id,recommend_info_list[0],recommend_info_list[1])
mysql.add_recommend_info(param)
mysql.close()
#源数据信息入库
mysql=mysqlLib()
for cuid in tuangou_brand.src_data_info:
param=(str(tuangou_brand.src_data_info[cuid]),last_id,cuid)
mysql.update_recommend_info(param)
mysql.close()
#任务结束
mysql=mysqlLib()
now=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
param=(now,1,last_id)
mysql.update_task_endtime_status(param)
#任务结束，短信通知
msgSend(['18665817689','15220056030'],u'TASK FINISHED:tuangou_brand badcase mining.')
def map_category_taskrun(event_day):
#任务时间
print(u"任务时间为："+str(event_day))
#任务初始化
map_category=class_map_category(event_day)
mysql=mysqlLib()
now=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
param=("map_category",now,0,event_day)
n,last_id=mysql.add_task(param)
print n,last_id
mysql.close()
#多进程随机取样
map_category.batch_rand_sample()
#取样二次过滤
map_category.secondary_filter()
print map_category.sample_list
mysql=mysqlLib()
for sample in map_category.sample_list:
sample=sample.strip()
param=(last_id,sample,0)
mysql.add_sample(param)
mysql.close()
#线程池下发历史信息查询任务
map_category.batch_get_history_cuid_info()
print map_category.history_info_list,len(map_category.history_info_list)
fp=open(map_category.data_directory+map_category.event_day+map_category.history_info_filepath,"w")
for line in map_category.history_info_list:
fp.write(line)
fp.close()
filepath=map_category.data_directory+map_category.event_day+map_category.history_info_filepath
lines=get_file_lines(filepath)
mysql=mysqlLib()
for line in lines:
line=line.strip()
line_list=re.split(r'\t',line.rstrip('\t'))
#print(str(len(line_list)))
if(len(line_list) == 6):
param=(last_id,line_list[0],line_list[1],line_list[2],line_list[3],line_list[4],'','','browse_rawdata',line_list[5],)
if(len(line_list) == 7):
param=(last_id,line_list[0],line_list[1],line_list[2],line_list[3],line_list[4],'','',line_list[5],line_list[6],)
if(len(line_list) == 9):
param=(last_id,line_list[0],line_list[1],line_list[2],line_list[3],line_list[4],'','','traj_rawdata',line_list[8],)
mysql.add_history_info(param)
mysql.close()
'''mysql=mysqlLib()
for line in map_category.history_info_list:
line = line.strip()
line_list=re.split(r'\t',line.rstrip('\t'))
param=(last_id,line_list[0],line_list[1],line_list[2],line_list[3],line_list[4],line_list[5],line_list[6],line_list[7],line_list[8],)
mysql.add_history_info(param)
mysql.close()'''
#构造线程池，源数据文件批量下载
map_category.batch_download_upps_src_data()
#源数据文件解析
map_category.get_src_data_info()
#推荐信息查询
map_category.batch_get_recommend_cuid_info("tuangou","10af214253a015b8ffdfbac9f98077b4","userpreference",12)
print map_category.recommend_info_list
#推荐信息入库
mysql=mysqlLib()
for recommend_info in map_category.recommend_info_list:
recommend_info_list=recommend_info.split("\t")
param=(last_id,recommend_info_list[0],recommend_info_list[1])
mysql.add_recommend_info(param)
mysql.close()
#源数据信息入库
mysql=mysqlLib()
for cuid in map_category.src_data_info:
param=(str(map_category.src_data_info[cuid]),last_id,cuid)
mysql.update_recommend_info(param)
mysql.close()
#任务结束
mysql=mysqlLib()
now=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
param=(now,1,last_id)
mysql.update_task_endtime_status(param)
#任务结束，短信通知
msgSend(['18665817689','15220056030','15019478061'],u'TASK FINISHED:map_category badcase mining.')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from common.logLib import *
from common.msgSend import *
from common.queryUserPreference import *
import time
TM_INTERVAL = 60
Exception_INTERVAL = 3600
phonenum_list = ["18665817689", "18665910949", "18666662305", "13632979619"]
msg_content = "userPreference:getResponse failed"
cnt = 1
while 1:
status = queryUserPreference()
if status == 0 and cnt == 3:
cnt = 0
time.sleep(TM_INTERVAL)
elif status == -1 and cnt == 3:
logging.error(msg_content + str(cnt) + " times")
msgSend(phonenum_list, msg_content)
cnt = 0
time.sleep(Exception_INTERVAL)
elif status == -1 and cnt != 3:
logging.error(msg_content + str(cnt) + " times")
cnt += 1
time.sleep(TM_INTERVAL)
else:
time.sleep(TM_INTERVAL)