Exemple #1
0
def tuangou_brand_taskrun(event_day):
    #任务时间
    print(u"任务时间为:"+str(event_day))

    #任务初始化
    tuangou_brand=class_tuangou_brand(event_day)
    mysql=mysqlLib()
    now=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
    param=("tuandan_pinpai",now,0,event_day)
    n,last_id=mysql.add_task(param)
    print n,last_id
    mysql.close()

    #多进程随机取样
    tuangou_brand.batch_rand_sample()
    #取样二次过滤
    tuangou_brand.secondary_filter()
    print tuangou_brand.sample_list
    mysql=mysqlLib()
    for sample in tuangou_brand.sample_list:
        sample=sample.strip()
        param=(last_id,sample,0)
        mysql.add_sample(param)
    mysql.close()

    #线程池下发历史信息查询任务
    tuangou_brand.batch_get_history_cuid_info()
    print tuangou_brand.history_info_list,len(tuangou_brand.history_info_list)
    fp=open(tuangou_brand.data_directory+tuangou_brand.event_day+tuangou_brand.history_info_filepath,"w")
    for line in tuangou_brand.history_info_list:
        fp.write(line)
    fp.close()
    mysql=mysqlLib()
    for line in tuangou_brand.history_info_list:
        line = line.strip()
        line_list=re.split(r'\t',line.rstrip('\t'))
        param=(last_id,line_list[0],line_list[1],line_list[2],line_list[3],line_list[4],line_list[5],line_list[6],line_list[7],line_list[8],)
        mysql.add_history_info(param)
    mysql.close()

    #构造线程池,源数据文件批量下载
    tuangou_brand.batch_download_upps_src_data()
    #源数据文件解析
    tuangou_brand.get_src_data_info()
    #推荐信息查询
    tuangou_brand.batch_get_recommend_cuid_info("tuangou","10af214253a015b8ffdfbac9f98077b4","userpreference",12)
    print tuangou_brand.recommend_info_list
    #推荐信息入库
    mysql=mysqlLib()
    for recommend_info in tuangou_brand.recommend_info_list:
        recommend_info_list=recommend_info.split("\t")
        param=(last_id,recommend_info_list[0],recommend_info_list[1])
        mysql.add_recommend_info(param)
    mysql.close()
    #源数据信息入库
    mysql=mysqlLib()
    for cuid in tuangou_brand.src_data_info:
        param=(str(tuangou_brand.src_data_info[cuid]),last_id,cuid)
        mysql.update_recommend_info(param)
    mysql.close()


    #任务结束
    mysql=mysqlLib()
    now=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
    param=(now,1,last_id)
    mysql.update_task_endtime_status(param)

    #任务结束,短信通知
    msgSend(['18665817689','15220056030'],u'TASK FINISHED:tuangou_brand badcase mining.')
Exemple #2
0
def map_category_taskrun(event_day):
    #任务时间
    print(u"任务时间为:"+str(event_day))

    #任务初始化
    map_category=class_map_category(event_day)
    mysql=mysqlLib()
    now=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
    param=("map_category",now,0,event_day)
    n,last_id=mysql.add_task(param)
    print n,last_id
    mysql.close()

    #多进程随机取样
    map_category.batch_rand_sample()
    #取样二次过滤
    map_category.secondary_filter()
    print map_category.sample_list
    mysql=mysqlLib()
    for sample in map_category.sample_list:
        sample=sample.strip()
        param=(last_id,sample,0)
        mysql.add_sample(param)
    mysql.close()

    #线程池下发历史信息查询任务
    map_category.batch_get_history_cuid_info()
    print map_category.history_info_list,len(map_category.history_info_list)
    fp=open(map_category.data_directory+map_category.event_day+map_category.history_info_filepath,"w")
    for line in map_category.history_info_list:
        fp.write(line)
    fp.close()

    filepath=map_category.data_directory+map_category.event_day+map_category.history_info_filepath
    lines=get_file_lines(filepath)
    mysql=mysqlLib()
    for line in lines:
        line=line.strip()
        line_list=re.split(r'\t',line.rstrip('\t'))
        #print(str(len(line_list)))
        if(len(line_list) == 6):
            param=(last_id,line_list[0],line_list[1],line_list[2],line_list[3],line_list[4],'','','browse_rawdata',line_list[5],)
        if(len(line_list) == 7):
            param=(last_id,line_list[0],line_list[1],line_list[2],line_list[3],line_list[4],'','',line_list[5],line_list[6],)
        if(len(line_list) == 9):
            param=(last_id,line_list[0],line_list[1],line_list[2],line_list[3],line_list[4],'','','traj_rawdata',line_list[8],)
        mysql.add_history_info(param)
    mysql.close()
    '''mysql=mysqlLib()
    for line in map_category.history_info_list:
        line = line.strip()
        line_list=re.split(r'\t',line.rstrip('\t'))
        param=(last_id,line_list[0],line_list[1],line_list[2],line_list[3],line_list[4],line_list[5],line_list[6],line_list[7],line_list[8],)
        mysql.add_history_info(param)
    mysql.close()'''

    #构造线程池,源数据文件批量下载
    map_category.batch_download_upps_src_data()
    #源数据文件解析
    map_category.get_src_data_info()
    #推荐信息查询
    map_category.batch_get_recommend_cuid_info("tuangou","10af214253a015b8ffdfbac9f98077b4","userpreference",12)
    print map_category.recommend_info_list
    #推荐信息入库
    mysql=mysqlLib()
    for recommend_info in map_category.recommend_info_list:
        recommend_info_list=recommend_info.split("\t")
        param=(last_id,recommend_info_list[0],recommend_info_list[1])
        mysql.add_recommend_info(param)
    mysql.close()
    #源数据信息入库
    mysql=mysqlLib()
    for cuid in map_category.src_data_info:
        param=(str(map_category.src_data_info[cuid]),last_id,cuid)
        mysql.update_recommend_info(param)
    mysql.close()


    #任务结束
    mysql=mysqlLib()
    now=datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
    param=(now,1,last_id)
    mysql.update_task_endtime_status(param)

    #任务结束,短信通知
    msgSend(['18665817689','15220056030','15019478061'],u'TASK FINISHED:map_category badcase mining.')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from common.logLib import *
from common.msgSend import *
from common.queryUserPreference import *
import time

TM_INTERVAL = 60
Exception_INTERVAL = 3600
phonenum_list = ["18665817689", "18665910949", "18666662305", "13632979619"]
msg_content = "userPreference:getResponse failed"
cnt = 1

while 1:
    status = queryUserPreference()
    if status == 0 and cnt == 3:
        cnt = 0
        time.sleep(TM_INTERVAL)
    elif status == -1 and cnt == 3:
        logging.error(msg_content + str(cnt) + " times")
        msgSend(phonenum_list, msg_content)
        cnt = 0
        time.sleep(Exception_INTERVAL)
    elif status == -1 and cnt != 3:
        logging.error(msg_content + str(cnt) + " times")
        cnt += 1
        time.sleep(TM_INTERVAL)
    else:
        time.sleep(TM_INTERVAL)