Ejemplo n.º 1
0
def create_inverted_index():
    rdx_name_list = g_conf.get("ridx", "ridx_list").split(",")
    for rdx_name in rdx_name_list:
        ridx_obj = inverted_index_t()
        name_list = rdx_name.split("_")
        for name in name_list:
            if name == "idea":
                continue
            key_name = name + "_id"
        data_file = "../data/" + rdx_name + ".txt"
        fp = open(data_file, "r")
        file_obj = file_process_t()
        line_cnt = 0
        indexed_idea_dict = {}
        for ori_line in fp:
            data_list = ori_line.rstrip("\r\n").split("\t")
            line_cnt += 1
            if line_cnt == 1:
                file_obj.init_header(data_list)
                continue
            idea_id = file_obj.get_value("idea_id", data_list)
            if not (idea_id in g_valid_idea_dict):
                logging.debug("no need to indexed idea id[%s]" % (idea_id))
                continue
            key = file_obj.get_value(key_name, data_list)
            ridx_obj.insert_data(key, idea_id)
            indexed_idea_dict[idea_id] = 1
        for idea_id in g_valid_idea_dict:
            if not (idea_id in indexed_idea_dict) and rdx_name != "ban_idea":
                ridx_obj.insert_data("0", idea_id)
        fp.close()
        ridx_obj.dump("../index/" + rdx_name + ".ridx")
Ejemplo n.º 2
0
def process(input_file, output_sql):
    now_time_list = datetime.datetime.now().strftime("%H:%M:%S").split(":")
    date_today = datetime.datetime.now().strftime("%Y-%m-%d")
    hour = int(now_time_list[0])
    minute = int(now_time_list[1])
    second = int(now_time_list[2])
    total_second = hour * 3600 + minute * 60 + second
    ratio = float(total_second) / 86400

    input_fp = open(input_file, "r")
    sql_fp = open(output_sql, "w")
    file_obj = file_process_t()
    line_cnt = 0
    sql_fp.write("use mis;\n")
    for ori_line in input_fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        idea_id = file_obj.get_value("id", data_list)
        budget = float(file_obj.get_value("budget", data_list))
        available_budget = budget * ratio
        sql_fp.write(
            "insert into consumptions(date,idea_id,smooth) values('%s','%s','%f') on DUPLICATE KEY update smooth='%f';\n"
            % (date_today, idea_id, available_budget, available_budget))

    input_fp.close()
    sql_fp.close()
Ejemplo n.º 3
0
def create_inverted_index():
    rdx_name_list = g_conf.get("ridx", "ridx_list").split(",")
    for rdx_name in rdx_name_list:
        ridx_obj = inverted_index_t()
        name_list = rdx_name.split("_")
        for name in name_list:
            if name == "idea":
                continue
            key_name = name + "_id"
        data_file = "../data/" + rdx_name + ".txt"
        fp = open(data_file, "r")
        file_obj = file_process_t()
        line_cnt = 0
        indexed_idea_dict = {}
        for ori_line in fp:
            data_list = ori_line.rstrip("\r\n").split("\t")
            line_cnt += 1
            if line_cnt == 1:
                file_obj.init_header(data_list)
                continue
            idea_id = file_obj.get_value("idea_id", data_list)
            if not (idea_id in g_valid_idea_dict):
                logging.debug("no need to indexed idea id[%s]" % (idea_id))
                continue
            key = file_obj.get_value(key_name, data_list)
            ridx_obj.insert_data(key, idea_id)
            indexed_idea_dict[idea_id] = 1
        for idea_id in g_valid_idea_dict:
            if not (idea_id in indexed_idea_dict) and rdx_name != "ban_idea":
                ridx_obj.insert_data("0", idea_id)
        fp.close()
        ridx_obj.dump("../index/" + rdx_name + ".ridx")
Ejemplo n.º 4
0
def process(input_file,output_sql):
    now_time_list=datetime.datetime.now().strftime("%H:%M:%S").split(":")
    date_today=datetime.datetime.now().strftime("%Y-%m-%d")
    hour=int(now_time_list[0])
    minute=int(now_time_list[1])
    second=int(now_time_list[2])
    total_second=hour*3600+minute*60+second
    ratio=float(total_second)/86400
    
    input_fp=open(input_file,"r")
    sql_fp=open(output_sql,"w")
    file_obj=file_process_t()
    line_cnt=0
    sql_fp.write("use mis;\n")
    for ori_line in input_fp:
        data_list=ori_line.rstrip("\r\n").split("\t")
        line_cnt+=1
        if line_cnt==1:
            file_obj.init_header(data_list)
            continue
        idea_id=file_obj.get_value("id",data_list)
        budget=float(file_obj.get_value("budget",data_list))   
        available_budget=budget*ratio
        sql_fp.write("insert into consumptions(date,idea_id,smooth) values('%s','%s','%f') on DUPLICATE KEY update smooth='%f';\n" %(date_today,idea_id,available_budget,available_budget))

    input_fp.close()
    sql_fp.close()
Ejemplo n.º 5
0
def create_index():
    idea_fp = open(g_conf.get("file", "ideas"), "r")
    index_name_list = g_conf.get("index", "index_list").split(",")
    index_obj = index_t()
    line_cnt = 0
    file_obj = file_process_t()
    type_ridx = inverted_index_t()
    size_ridx = inverted_index_t()
    for ori_line in idea_fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        idea_id = file_obj.get_value("id", data_list)
        if not (idea_id in g_valid_idea_dict):
            logging.debug("idea id no budget[%s]" % (idea_id))
            continue
        type = file_obj.get_value("type", data_list)
        size_id = file_obj.get_value("size_id", data_list)
        type_ridx.insert_data(type, idea_id)
        size_ridx.insert_data(size_id, idea_id)
        for name in index_name_list:
            value = file_obj.get_value(name, data_list)
            index_obj.insert_data(idea_id, name, value)
    idea_fp.close()
    #dump type invert index
    type_ridx.dump("../index/type.ridx")
    size_ridx.dump("../index/size.ridx")
    file_obj = file_process_t()
    line_cnt = 0
    operate_fp = open(g_conf.get("file", "idea_operate"), "r")
    for ori_line in operate_fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        idea_id = file_obj.get_value("idea_id", data_list)
        if not (idea_id in g_valid_idea_dict):
            logging.debug("idea id no budget[%s]" % (idea_id))
            continue
        value = file_obj.get_value("basic_bid", data_list)
        index_obj.insert_data(idea_id, "basic_bid", value)
    operate_fp.close()
    index_obj.dump(g_conf.get("index", "idea_idx"))
Ejemplo n.º 6
0
def create_index():
    idea_fp = open(g_conf.get("file", "ideas"), "r")
    index_name_list = g_conf.get("index", "index_list").split(",")
    index_obj = index_t()
    line_cnt = 0
    file_obj = file_process_t()
    type_ridx = inverted_index_t()
    size_ridx = inverted_index_t()
    for ori_line in idea_fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        idea_id = file_obj.get_value("id", data_list)
        if not (idea_id in g_valid_idea_dict):
            logging.debug("idea id no budget[%s]" % (idea_id))
            continue
        type = file_obj.get_value("type", data_list)
        size_id = file_obj.get_value("size_id", data_list)
        type_ridx.insert_data(type, idea_id)
        size_ridx.insert_data(size_id, idea_id)
        for name in index_name_list:
            value = file_obj.get_value(name, data_list)
            index_obj.insert_data(idea_id, name, value)
    idea_fp.close()
    # dump type invert index
    type_ridx.dump("../index/type.ridx")
    size_ridx.dump("../index/size.ridx")
    file_obj = file_process_t()
    line_cnt = 0
    operate_fp = open(g_conf.get("file", "idea_operate"), "r")
    for ori_line in operate_fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        idea_id = file_obj.get_value("idea_id", data_list)
        if not (idea_id in g_valid_idea_dict):
            logging.debug("idea id no budget[%s]" % (idea_id))
            continue
        value = file_obj.get_value("basic_bid", data_list)
        index_obj.insert_data(idea_id, "basic_bid", value)
    operate_fp.close()
    index_obj.dump(g_conf.get("index", "idea_idx"))
Ejemplo n.º 7
0
def load_idea_info(idea_operator_file, idea_file):
    global g_idea_dict
    g_idea_dict = {}
    fp = open(idea_operator_file, "r")
    file_obj = file_process_t()
    line_cnt = 0
    for ori_line in fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        idea_id = file_obj.get_value("idea_id", data_list)
        charge_type = file_obj.get_value("charge_type", data_list)
        basic_price = float(file_obj.get_value("basic_price", data_list))
        g_idea_dict[idea_id] = {
            "charge_type": charge_type,
            "basic_price": basic_price
        }
    fp.close()

    fp = open(idea_file, "r")
    file_obj = file_process_t()
    line_cnt = 0
    for ori_line in fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        idea_id = file_obj.get_value("id", data_list)
        user_id = file_obj.get_value("user_id", data_list)
        plan_id = file_obj.get_value("plan_id", data_list)
        if idea_id in g_idea_dict:
            g_idea_dict[idea_id]["user_id"] = user_id
            g_idea_dict[idea_id]["plan_id"] = plan_id
        else:
            sub_dict = {}
            sub_dict = {
                "user_id": user_id,
                "plan_id": plan_id,
                "charge_type": "cpm"
            }
            g_idea_dict[idea_id] = sub_dict
    fp.close()
Ejemplo n.º 8
0
def load_idea_info(idea_operator_file,idea_file):
    global g_idea_dict
    g_idea_dict={}
    fp=open(idea_operator_file,"r")
    file_obj=file_process_t()
    line_cnt=0
    for ori_line in fp:
        data_list=ori_line.rstrip("\r\n").split("\t")
        line_cnt+=1
        if line_cnt==1:
            file_obj.init_header(data_list)
            continue
        idea_id=file_obj.get_value("idea_id",data_list)
        charge_type=file_obj.get_value("charge_type",data_list)
        basic_price=float(file_obj.get_value("basic_price",data_list))
        g_idea_dict[idea_id]={"charge_type":charge_type,"basic_price":basic_price}
    fp.close()   

    fp=open(idea_file,"r")
    file_obj=file_process_t()
    line_cnt=0
    for ori_line in fp:
        data_list=ori_line.rstrip("\r\n").split("\t")
        line_cnt+=1
        if line_cnt==1:
            file_obj.init_header(data_list)
            continue
        idea_id=file_obj.get_value("id",data_list)
        user_id=file_obj.get_value("user_id",data_list)
        plan_id=file_obj.get_value("plan_id",data_list)
        if idea_id in g_idea_dict:
            g_idea_dict[idea_id]["user_id"]=user_id
            g_idea_dict[idea_id]["plan_id"]=plan_id
        else:
            sub_dict={}
            sub_dict={"user_id":user_id,"plan_id":plan_id,"charge_type":"cpm"}
            g_idea_dict[idea_id]=sub_dict
    fp.close()   
Ejemplo n.º 9
0
 def __init__(self,idea_operator_file):
     self.idea_operator_dict={}
     fp=open(idea_operator_file,"r")
     line_cnt=0
     file_obj=file_process_t()
     for line in fp:
         data_list=line.rstrip("\r\n").split("\t")
         line_cnt+=1
         if line_cnt==1:
             file_obj.init_header(data_list)
             continue
         idea_id=file_obj.get_value("idea_id",data_list)
         basic_bid=float(file_obj.get_value("basic_bid",data_list))
         thr=float(file_obj.get_value("thr",data_list))
         self.idea_operator_dict[idea_id]={"basic_bid":basic_bid,"thr":thr}
     fp.close()
Ejemplo n.º 10
0
def process(input_file, output_file, date_str):
    input_fp = open(input_file, "r")
    output_fp = open(output_file, "w")
    date_time = datetime.datetime.strptime(date_str, "%Y-%m-%d-%H:%M:%S")
    date_day_str = datetime.datetime.strftime(date_time, "%Y-%m-%d")
    file_obj = file_process_t()
    line_cnt = 0
    cost_dict = {}
    price_dict = {}
    for ori_line in input_fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        idea_id = file_obj.get_value("idea_id", data_list)
        price = float(file_obj.get_value("price", data_list))
        cost = float(file_obj.get_value("cost", data_list))
        if idea_id in cost_dict:
            cost_dict[idea_id] += cost
        else:
            cost_dict[idea_id] = cost
        if idea_id in price_dict:
            price_dict[idea_id] += price
        else:
            price_dict[idea_id] = price

    output_fp.write("use mis;set names utf8;\n")
    for idea_id in cost_dict:
        cost = cost_dict[idea_id]
        if idea_id in price_dict:
            price = price_dict[idea_id]
        else:
            price = 0
        if not (idea_id in g_idea_dict):
            logging.warning("no such idea id[%s] in idea lib" % (idea_id))
            continue
        idea_info_dict = g_idea_dict[idea_id]
        plan_id = idea_info_dict["plan_id"]
        user_id = idea_info_dict["user_id"]
        output_fp.write(
            "insert into consumptions(idea_id,user_id,plan_id,price,cost,date) values('%s','%s','%s','%f','%f','%s') on DUPLICATE KEY update price=price+%f,cost=cost+%f,user_id='%s',plan_id='%s';\n"
            % (idea_id, user_id, plan_id, price, cost, date_day_str, price,
               cost, user_id, plan_id))

    input_fp.close()
    output_fp.close()
Ejemplo n.º 11
0
def process(input_file,output_file,date_str):
    input_fp=open(input_file,"r")
    output_fp=open(output_file,"w")
    date_time=datetime.datetime.strptime(date_str,"%Y-%m-%d-%H:%M:%S")
    date_day_str=datetime.datetime.strftime(date_time,"%Y-%m-%d")
    file_obj=file_process_t()
    line_cnt=0
    cost_dict={}
    price_dict={}
    for ori_line in input_fp:
        data_list=ori_line.rstrip("\r\n").split("\t")
        line_cnt+=1
        if line_cnt==1:
            file_obj.init_header(data_list)
            continue
        idea_id=file_obj.get_value("idea_id",data_list)
        price=float(file_obj.get_value("price",data_list))
        cost=float(file_obj.get_value("cost",data_list))
        if idea_id in cost_dict:
            cost_dict[idea_id]+=cost
        else:
            cost_dict[idea_id]=cost
        if idea_id in price_dict:
            price_dict[idea_id]+=price
        else:
            price_dict[idea_id]=price
        
    output_fp.write("use mis;set names utf8;\n")
    for idea_id in cost_dict:
        cost=cost_dict[idea_id]
        if idea_id in price_dict:
            price=price_dict[idea_id]
        else:
            price=0
        if not(idea_id in g_idea_dict):
            logging.warning("no such idea id[%s] in idea lib" %(idea_id))
            continue
        idea_info_dict=g_idea_dict[idea_id]
        plan_id=idea_info_dict["plan_id"]
        user_id=idea_info_dict["user_id"]    
        output_fp.write("insert into consumptions(idea_id,user_id,plan_id,price,cost,date) values('%s','%s','%s','%f','%f','%s') on DUPLICATE KEY update price=price+%f,cost=cost+%f,user_id='%s',plan_id='%s';\n" %(idea_id,user_id,plan_id,price,cost,date_day_str,price,cost,user_id,plan_id))
        
    input_fp.close()
    output_fp.close()
Ejemplo n.º 12
0
 def __init__(self, idea_operator_file):
     self.idea_operator_dict = {}
     fp = open(idea_operator_file, "r")
     line_cnt = 0
     file_obj = file_process_t()
     for line in fp:
         data_list = line.rstrip("\r\n").split("\t")
         line_cnt += 1
         if line_cnt == 1:
             file_obj.init_header(data_list)
             continue
         idea_id = file_obj.get_value("idea_id", data_list)
         basic_bid = float(file_obj.get_value("basic_bid", data_list))
         thr = float(file_obj.get_value("thr", data_list))
         self.idea_operator_dict[idea_id] = {
             "basic_bid": basic_bid,
             "thr": thr
         }
     fp.close()
Ejemplo n.º 13
0
    def __init__(self, conf):
        self.adx_id_dict = {}
        self.adx_id_file_dict = {}
        self.adx_id_dict["device"] = {}
        self.adx_id_file_dict["device"] = conf.get("file", "device")

        self.adx_id_dict["network"] = {}
        self.adx_id_file_dict["network"] = conf.get("file", "network")

        self.adx_id_dict["classification"] = {}
        self.adx_id_file_dict["classification"] = conf.get(
            "file", "classification")
        self.adx_id_dict["app"] = {}
        self.adx_id_file_dict["app"] = conf.get("file", "app")

        self.search_type_dict = {}
        self.type_size_dict = {}
        self.size_dict = {}
        self.size_invert_dict = {}
        for name in self.adx_id_file_dict:
            file_name = self.adx_id_file_dict[name]
            map_dict = self.adx_id_dict[name]
            fp = open(file_name, "r")
            file_obj = file_process_t()
            line_cnt = 0
            for ori_line in fp:
                data_list = ori_line.rstrip("\r\n").split("\t")
                line_cnt += 1
                if line_cnt == 1:
                    file_obj.init_header(data_list)
                    continue
                id = file_obj.get_value("id", data_list)
                adx = file_obj.get_value("adx", data_list)
                map_dict[adx] = id

        fp = open(conf.get("file", "sizes"), "r")
        file_obj = file_process_t()
        line_cnt = 0
        for ori_line in fp:
            data_list = ori_line.rstrip("\r\n").split("\t")
            line_cnt += 1
            if line_cnt == 1:
                file_obj.init_header(data_list)
                continue
            id = file_obj.get_value("id", data_list)
            width = file_obj.get_value("width", data_list)
            height = file_obj.get_value("height", data_list)
            is_full = file_obj.get_value("is_full", data_list)
            size_str = width + "x" + height
            self.size_invert_dict[size_str] = id
            self.size_dict[id] = {"size_str": size_str, "is_full": is_full}
        fp = open(conf.get("file", "type_size_join"), "r")
        file_obj = file_process_t()
        line_cnt = 0
        for ori_line in fp:
            data_list = ori_line.rstrip("\r\n").split("\t")
            line_cnt += 1
            if line_cnt == 1:
                file_obj.init_header(data_list)
                continue
            type_name = file_obj.get_value("name_en", data_list)
            ad_type = (type_name.split("_"))[0]
            device_id = file_obj.get_value("device_id", data_list)
            size_id = file_obj.get_value("size_id", data_list)
            combine_key = ad_type + "_" + device_id
            self.search_type_dict[combine_key] = type_name
            if type_name in self.type_size_dict:
                sub_dict = self.type_size_dict[type_name]
            else:
                sub_dict = {}
            sub_dict[size_id] = 1
            self.type_size_dict[type_name] = sub_dict

        fp = open(conf.get("file", "manual_operator"), "r")
        self.adx_id_dict["operator"] = {}
        file_obj = file_process_t()
        line_cnt = 0
        for ori_line in fp:
            data_list = ori_line.rstrip("\r\n").split("\t")
            line_cnt += 1
            if line_cnt == 1:
                file_obj.init_header(data_list)
                continue
            code = file_obj.get_value("code", data_list)
            id = file_obj.get_value("id", data_list)
            self.adx_id_dict["operator"][code] = id
        fp.close()
Ejemplo n.º 14
0
def create_valid_idea():
    global g_valid_idea_dict
    g_valid_idea_dict = {}
    valid_user_dict = {}
    valid_plan_dict = {}
    plan_consume_dict = {}
    idea_consume_dict = {}
    date_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    week_day = datetime.datetime.now().weekday()
    hour = int(datetime.datetime.now().strftime("%H"))
    hour_idx = str(week_day * 24 + hour)
    logging.debug("date:%s hour:%d hour_idx:%s" % (date_str, hour, hour_idx))
    #load user file
    user_file = g_conf.get("file", "users")
    fp = open(user_file, "r")
    file_obj = file_process_t()
    line_cnt = 0
    for ori_line in fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        user_id = file_obj.get_value("user_id", data_list)
        consume = float(file_obj.get_value("consumption_total", data_list))
        total = float(file_obj.get_value("total", data_list))
        balance = total - consume
        if balance <= 0:
            logging.debug("user id[%s] no budget" % (balance))
            continue
        valid_user_dict[user_id] = 1
    fp.close()

    #load consumption
    consumption_file = g_conf.get("file", "consumption")
    fp = open(consumption_file, "r")
    file_obj = file_process_t()
    line_cnt = 0
    idea_smooth_dict = {}
    for ori_line in fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        plan_id = file_obj.get_value("plan_id", data_list)
        idea_id = file_obj.get_value("idea_id", data_list)
        consume = float(file_obj.get_value("price", data_list))
        smooth = float(file_obj.get_value("smooth", data_list))
        idea_smooth_dict[idea_id] = smooth
        if plan_id in plan_consume_dict:
            plan_consume_dict[plan_id] += consume
        else:
            plan_consume_dict[plan_id] = consume
        if idea_id in idea_consume_dict:
            idea_consume_dict[idea_id] += consume
        else:
            idea_consume_dict[idea_id] = consume
    fp.close()

    #create valid plan
    plan_file = g_conf.get("file", "plans")
    fp = open(plan_file, "r")
    file_obj = file_process_t()
    line_cnt = 0
    for ori_line in fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        plan_id = file_obj.get_value("plan_id", data_list)
        user_id = file_obj.get_value("user_id", data_list)
        budget = float(file_obj.get_value("budget", data_list))
        start_time = file_obj.get_value("start_time", data_list)
        end_time = file_obj.get_value("end_time", data_list)
        if date_str < start_time or date_str > end_time:
            logging.debug("date not in date range idea_id[%s]" % (plan_id))
            continue
        if not (user_id in valid_user_dict):
            logging.debug("plan id[%s] user id[%s] no budget" %
                          (plan_id, user_id))
            continue
        if plan_id in plan_consume_dict:
            plan_consume = plan_consume_dict[plan_id]
        else:
            plan_consume = 0
        #print budget,plan_consume,plan_id
        if budget - plan_consume <= 0:
            logging.debug("plan[%s] no budget" % (plan_id))
            continue
        valid_plan_dict[plan_id] = 1
    fp.close()

    #create valid idea
    idea_file = g_conf.get("file", "ideas")
    fp = open(idea_file, "r")
    file_obj = file_process_t()
    line_cnt = 0
    for ori_line in fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        idea_id = file_obj.get_value("id", data_list)
        plan_id = file_obj.get_value("plan_id", data_list)
        budget = float(file_obj.get_value("budget", data_list))
        status = file_obj.get_value("status", data_list)
        if status != "3":
            continue
        time_range_list = file_obj.get_value("timerange", data_list)
        if time_range_list != None and time_range_list != "NULL" and len(
                time_range_list) > 0:
            time_range_list = time_range_list.split(",")
            dict_temp = {}
            for target_hour in time_range_list:
                dict_temp[target_hour] = 1
            if not (hour_idx in dict_temp):
                logging.debug("hour idx[%s] not in hour range" % (hour_idx))
                continue
        if not (plan_id in valid_plan_dict):
            logging.debug("plan id[%s] no budget" % (plan_id))
            continue

        if idea_id in idea_consume_dict:
            idea_consume = idea_consume_dict[idea_id]
        else:
            idea_consume = 0
        if idea_id in idea_smooth_dict:
            smooth = idea_smooth_dict[idea_id]
            if idea_consume > smooth:
                logging.debug(
                    "idea id[%s] no smooth budget[%f] idea consume[%f]" %
                    (idea_id, smooth, idea_consume))
                continue
        if budget - idea_consume <= 0:
            logging.debug("idea id[%s] no budget idea consume[%f]" %
                          (idea_id, idea_consume))
            continue
        g_valid_idea_dict[idea_id] = 1
    fp.close()
Ejemplo n.º 15
0
def create_valid_idea():
    global g_valid_idea_dict
    g_valid_idea_dict = {}
    valid_user_dict = {}
    valid_plan_dict = {}
    plan_consume_dict = {}
    idea_consume_dict = {}
    date_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    week_day = datetime.datetime.now().weekday()
    hour = int(datetime.datetime.now().strftime("%H"))
    hour_idx = str(week_day * 24 + hour)
    logging.debug("date:%s hour:%d hour_idx:%s" % (date_str, hour, hour_idx))
    # load user file
    user_file = g_conf.get("file", "users")
    fp = open(user_file, "r")
    file_obj = file_process_t()
    line_cnt = 0
    for ori_line in fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        user_id = file_obj.get_value("user_id", data_list)
        consume = float(file_obj.get_value("consumption_total", data_list))
        total = float(file_obj.get_value("total", data_list))
        balance = total - consume
        if balance <= 0:
            logging.debug("user id[%s] no budget" % (balance))
            continue
        valid_user_dict[user_id] = 1
    fp.close()

    # load consumption
    consumption_file = g_conf.get("file", "consumption")
    fp = open(consumption_file, "r")
    file_obj = file_process_t()
    line_cnt = 0
    idea_smooth_dict = {}
    for ori_line in fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        plan_id = file_obj.get_value("plan_id", data_list)
        idea_id = file_obj.get_value("idea_id", data_list)
        consume = float(file_obj.get_value("price", data_list))
        smooth = float(file_obj.get_value("smooth", data_list))
        idea_smooth_dict[idea_id] = smooth
        if plan_id in plan_consume_dict:
            plan_consume_dict[plan_id] += consume
        else:
            plan_consume_dict[plan_id] = consume
        if idea_id in idea_consume_dict:
            idea_consume_dict[idea_id] += consume
        else:
            idea_consume_dict[idea_id] = consume
    fp.close()

    # create valid plan
    plan_file = g_conf.get("file", "plans")
    fp = open(plan_file, "r")
    file_obj = file_process_t()
    line_cnt = 0
    for ori_line in fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        plan_id = file_obj.get_value("plan_id", data_list)
        user_id = file_obj.get_value("user_id", data_list)
        budget = float(file_obj.get_value("budget", data_list))
        start_time = file_obj.get_value("start_time", data_list)
        end_time = file_obj.get_value("end_time", data_list)
        if date_str < start_time or date_str > end_time:
            logging.debug("date not in date range idea_id[%s]" % (plan_id))
            continue
        if not (user_id in valid_user_dict):
            logging.debug("plan id[%s] user id[%s] no budget" % (plan_id, user_id))
            continue
        if plan_id in plan_consume_dict:
            plan_consume = plan_consume_dict[plan_id]
        else:
            plan_consume = 0
        # print budget,plan_consume,plan_id
        if budget - plan_consume <= 0:
            logging.debug("plan[%s] no budget" % (plan_id))
            continue
        valid_plan_dict[plan_id] = 1
    fp.close()

    # create valid idea
    idea_file = g_conf.get("file", "ideas")
    fp = open(idea_file, "r")
    file_obj = file_process_t()
    line_cnt = 0
    for ori_line in fp:
        data_list = ori_line.rstrip("\r\n").split("\t")
        line_cnt += 1
        if line_cnt == 1:
            file_obj.init_header(data_list)
            continue
        idea_id = file_obj.get_value("id", data_list)
        plan_id = file_obj.get_value("plan_id", data_list)
        budget = float(file_obj.get_value("budget", data_list))
        status = file_obj.get_value("status", data_list)
        if status != "3":
            continue
        time_range_list = file_obj.get_value("timerange", data_list)
        if time_range_list != None and time_range_list != "NULL" and len(time_range_list) > 0:
            time_range_list = time_range_list.split(",")
            dict_temp = {}
            for target_hour in time_range_list:
                dict_temp[target_hour] = 1
            if not (hour_idx in dict_temp):
                logging.debug("hour idx[%s] not in hour range" % (hour_idx))
                continue
        if not (plan_id in valid_plan_dict):
            logging.debug("plan id[%s] no budget" % (plan_id))
            continue

        if idea_id in idea_consume_dict:
            idea_consume = idea_consume_dict[idea_id]
        else:
            idea_consume = 0
        if idea_id in idea_smooth_dict:
            smooth = idea_smooth_dict[idea_id]
            if idea_consume > smooth:
                logging.debug("idea id[%s] no smooth budget[%f] idea consume[%f]" % (idea_id, smooth, idea_consume))
                continue
        if budget - idea_consume <= 0:
            logging.debug("idea id[%s] no budget idea consume[%f]" % (idea_id, idea_consume))
            continue
        g_valid_idea_dict[idea_id] = 1
    fp.close()
Ejemplo n.º 16
0
    def __init__(self,conf):
        self.adx_id_dict={}
        self.adx_id_file_dict={}
        self.adx_id_dict["device"]={}
        self.adx_id_file_dict["device"]=conf.get("file","device")
        
        self.adx_id_dict["network"]={}
        self.adx_id_file_dict["network"]=conf.get("file","network")

        self.adx_id_dict["classification"]={}
        self.adx_id_file_dict["classification"]=conf.get("file","classification")
        self.adx_id_dict["app"]={}
        self.adx_id_file_dict["app"]=conf.get("file","app")

        self.search_type_dict={}
        self.type_size_dict={}
        self.size_dict={}
        self.size_invert_dict={}
        for name in self.adx_id_file_dict:
            file_name=self.adx_id_file_dict[name]
            map_dict=self.adx_id_dict[name]
            fp=open(file_name,"r")
            file_obj=file_process_t()
            line_cnt=0
            for ori_line in fp:
                data_list=ori_line.rstrip("\r\n").split("\t")
                line_cnt+=1
                if line_cnt==1:
                    file_obj.init_header(data_list)
                    continue
                id=file_obj.get_value("id",data_list)
                adx=file_obj.get_value("adx",data_list)
                map_dict[adx]=id        

        fp=open(conf.get("file","sizes"),"r")
        file_obj=file_process_t()
        line_cnt=0
        for ori_line in fp:
            data_list=ori_line.rstrip("\r\n").split("\t")
            line_cnt+=1
            if line_cnt==1:
                file_obj.init_header(data_list)
                continue
            id=file_obj.get_value("id",data_list)
            width=file_obj.get_value("width",data_list)
            height=file_obj.get_value("height",data_list)
            is_full=file_obj.get_value("is_full",data_list)
            size_str=width+"x"+height
            self.size_invert_dict[size_str]=id
            self.size_dict[id]={"size_str":size_str,"is_full":is_full}
        fp=open(conf.get("file","type_size_join"),"r")
        file_obj=file_process_t()
        line_cnt=0
        for ori_line in fp:
            data_list=ori_line.rstrip("\r\n").split("\t")
            line_cnt+=1
            if line_cnt==1:
                file_obj.init_header(data_list)
                continue
            type_name=file_obj.get_value("name_en",data_list)
            ad_type=(type_name.split("_"))[0]
            device_id=file_obj.get_value("device_id",data_list)
            size_id=file_obj.get_value("size_id",data_list)
            combine_key=ad_type+"_"+device_id
            self.search_type_dict[combine_key]=type_name
            if type_name in self.type_size_dict:
                sub_dict=self.type_size_dict[type_name]
            else:
                sub_dict={}
            sub_dict[size_id]=1
            self.type_size_dict[type_name]=sub_dict    
            
        fp=open(conf.get("file","manual_operator"),"r")
        self.adx_id_dict["operator"]={}
        file_obj=file_process_t()
        line_cnt=0
        for ori_line in fp:
            data_list=ori_line.rstrip("\r\n").split("\t")
            line_cnt+=1
            if line_cnt==1:
                file_obj.init_header(data_list)
                continue
            code=file_obj.get_value("code",data_list)
            id=file_obj.get_value("id",data_list)
            self.adx_id_dict["operator"][code]=id
        fp.close()