Ejemplo n.º 1
0
def main():
    global app
    sql_file_name = toolkit.gen_sql_file_name()
    start_date, stop_date, task_type = parse_args()

    baidu_task_config = config.baidu_task_list[task_type]
    detail_log_file = baidu_task_config["detail_log_file"]
    app["statics_log_file"] = baidu_task_config["statics_log_file"]
    clean_sql_file = baidu_task_config["clean_sql_file"]

    init_logging(detail_log_file)
    current_date = start_date
    while current_date <= stop_date:
        data_str = current_date.strftime("%Y-%m-%d")
        main_loop(task_type, data_str, sql_file_name)
        current_date += datetime.timedelta(days=1)

    # 执行 SQL
    mysql_config = config.mysql_config
    toolkit.execute_mysql_sql(mysql_config['host'], mysql_config['port'],
                              mysql_config["user"], mysql_config["passwd"],
                              mysql_config["dbname"], sql_file_name)
    # END
    if clean_sql_file:
        os.remove(sql_file_name)
    else:
        logging.info("keep the tmp sql file @ %s" % sql_file_name)
    return
Ejemplo n.º 2
0
def main():
    global app
    default_task = config.google_play_default_task
    statics_log_file = default_task["statics_log_file"]
    app["statics_log_file"] = statics_log_file
    detail_log_file = default_task["detail_log_file"]
    clean_sql_file = default_task["clean_sql_file"]
    init_logging(detail_log_file)
    #
    # task_list = ["inner", "outer", "country", "organic"]
    task_list = config.google_play_task_list.keys()
    start_date, stop_date, task_type = parse_args(task_list)
    print start_date, stop_date, task_type
    # exit()

    # tmp sql file
    sql_file_name = toolkit.gen_sql_file_name()
    # loop by date
    time_step = config.google_play_task_list[task_type].get('time_step', 1)
    p_date = start_date
    while p_date <= stop_date:
        worker(task_type, sql_file_name, p_date)
        p_date += datetime.timedelta(days=time_step)

    # update year,month, year_month
    table_name = config.google_play_task_list[task_type]['table_name']
    update_time_sql_pattern = "\nUPDATE `{table_name}` " \
                              "SET `year` = YEAR(`date`), `month` = Month(`date`), " \
                              " `year_month` = DATE_FORMAT(`date`,'%Y%m') " \
                              "WHERE `year` IS NULL;\n"
    update_sql = update_time_sql_pattern.format(table_name=table_name)
    with open(sql_file_name, 'a') as f:
        f.write(update_sql)

    # 执行 SQL
    mysql_config = config.mysql_config
    toolkit.execute_mysql_sql(mysql_config['host'], mysql_config['port'],
                              mysql_config["user"], mysql_config["passwd"],
                              mysql_config["dbname"], sql_file_name)
    # END
    if clean_sql_file:
        os.remove(sql_file_name)
    else:
        logging.info("keep the tmp sql file @ %s" % sql_file_name)
    return