def main():
global app
sql_file_name = toolkit.gen_sql_file_name()
start_date, stop_date, task_type = parse_args()
baidu_task_config = config.baidu_task_list[task_type]
detail_log_file = baidu_task_config["detail_log_file"]
app["statics_log_file"] = baidu_task_config["statics_log_file"]
clean_sql_file = baidu_task_config["clean_sql_file"]
init_logging(detail_log_file)
current_date = start_date
while current_date <= stop_date:
data_str = current_date.strftime("%Y-%m-%d")
main_loop(task_type, data_str, sql_file_name)
current_date += datetime.timedelta(days=1)
# 执行 SQL
mysql_config = config.mysql_config
toolkit.execute_mysql_sql(mysql_config['host'], mysql_config['port'],
mysql_config["user"], mysql_config["passwd"],
mysql_config["dbname"], sql_file_name)
# END
if clean_sql_file:
os.remove(sql_file_name)
else:
logging.info("keep the tmp sql file @ %s" % sql_file_name)
return
def main():
global app
datetime_from, datetime_end, task_type = parse_args()
task_config = config.mopub_task_list[task_type]
detail_log_file = task_config["detail_log_file"]
clean_sql_file = task_config["clean_sql_file"]
app["statics_log_file"] = task_config["statics_log_file"]
init_logging(detail_log_file)
sql_file_name = gen_sql_file_name()
logging.info("main.sql_file_name.%s" % sql_file_name)
p = toolkit.datetime_to_timestamp(datetime_from)
g_stop_time = toolkit.datetime_to_timestamp(datetime_end)
while p <= g_stop_time:
main_loop(task_type, sql_file_name, p)
p += 1 * 24 * 3600
# 执行 SQL
mysql_config = config.mysql_config
toolkit.execute_mysql_sql(
mysql_config['host'],
mysql_config['port'],
mysql_config["user"],
mysql_config["passwd"],
mysql_config["dbname"],
sql_file_name)
# END
if clean_sql_file:
os.remove(sql_file_name)
else:
logging.info("keep the tmp sql file @ %s" % sql_file_name)
return
def update_revenue_app():
update_revenue_app = "update_revenue_app.sql"
mysql_config = config.mysql_config
toolkit.execute_mysql_sql(
mysql_config['host'],
mysql_config['port'],
mysql_config["user"],
mysql_config["passwd"],
mysql_config["dbname"],
update_revenue_app)
def main():
global app
default_task = config.google_play_default_task
statics_log_file = default_task["statics_log_file"]
app["statics_log_file"] = statics_log_file
detail_log_file = default_task["detail_log_file"]
clean_sql_file = default_task["clean_sql_file"]
init_logging(detail_log_file)
#
# task_list = ["inner", "outer", "country", "organic"]
task_list = config.google_play_task_list.keys()
start_date, stop_date, task_type = parse_args(task_list)
print start_date, stop_date, task_type
# exit()
# tmp sql file
sql_file_name = toolkit.gen_sql_file_name()
# loop by date
time_step = config.google_play_task_list[task_type].get('time_step', 1)
p_date = start_date
while p_date <= stop_date:
worker(task_type, sql_file_name, p_date)
p_date += datetime.timedelta(days=time_step)
# update year,month, year_month
table_name = config.google_play_task_list[task_type]['table_name']
update_time_sql_pattern = "\nUPDATE `{table_name}` " \
"SET `year` = YEAR(`date`), `month` = Month(`date`), " \
" `year_month` = DATE_FORMAT(`date`,'%Y%m') " \
"WHERE `year` IS NULL;\n"
update_sql = update_time_sql_pattern.format(table_name=table_name)
with open(sql_file_name, 'a') as f:
f.write(update_sql)
# 执行 SQL
mysql_config = config.mysql_config
toolkit.execute_mysql_sql(mysql_config['host'], mysql_config['port'],
mysql_config["user"], mysql_config["passwd"],
mysql_config["dbname"], sql_file_name)
# END
if clean_sql_file:
os.remove(sql_file_name)
else:
logging.info("keep the tmp sql file @ %s" % sql_file_name)
return
def main():
global app
datetime_from, datetime_end, task_type = parse_args()
task_config = config.facebook_task_list[task_type]
split_task = task_config["split_task"]
size_per_split = task_config["size_per_split"]
detail_log_file = task_config["detail_log_file"]
clean_sql_file = task_config["clean_sql_file"]
app["statics_log_file"] = task_config["statics_log_file"]
init_logging(detail_log_file)
sql_file_name = gen_sql_file_name()
logging.info("main.sql_file_name.%s" % sql_file_name)
if split_task:
start_time = toolkit.datetime_to_timestamp(datetime_from)
g_stop_time = toolkit.datetime_to_timestamp(datetime_end)
while start_time < g_stop_time:
stop_time = start_time + size_per_split * 24 * 3600
if stop_time > g_stop_time:
stop_time = g_stop_time
main_loop(task_type, sql_file_name, start_time, stop_time)
start_time += size_per_split * 24 * 3600
else:
start_time = toolkit.datetime_to_timestamp(datetime_from)
stop_time = toolkit.datetime_to_timestamp(datetime_end)
main_loop(task_type, sql_file_name, start_time, stop_time)
# 执行 SQL
mysql_config = config.mysql_config
toolkit.execute_mysql_sql(mysql_config['host'], mysql_config['port'],
mysql_config["user"], mysql_config["passwd"],
mysql_config["dbname"], sql_file_name)
# END
if clean_sql_file:
os.remove(sql_file_name)
else:
logging.info("keep the tmp sql file @ %s" % sql_file_name)
return