def main():
global app
sql_file_name = toolkit.gen_sql_file_name()
start_date, stop_date, task_type = parse_args()
baidu_task_config = config.baidu_task_list[task_type]
detail_log_file = baidu_task_config["detail_log_file"]
app["statics_log_file"] = baidu_task_config["statics_log_file"]
clean_sql_file = baidu_task_config["clean_sql_file"]
init_logging(detail_log_file)
current_date = start_date
while current_date <= stop_date:
data_str = current_date.strftime("%Y-%m-%d")
main_loop(task_type, data_str, sql_file_name)
current_date += datetime.timedelta(days=1)
# 执行 SQL
mysql_config = config.mysql_config
toolkit.execute_mysql_sql(mysql_config['host'], mysql_config['port'],
mysql_config["user"], mysql_config["passwd"],
mysql_config["dbname"], sql_file_name)
# END
if clean_sql_file:
os.remove(sql_file_name)
else:
logging.info("keep the tmp sql file @ %s" % sql_file_name)
return
def main():
global app
default_task = config.google_play_default_task
statics_log_file = default_task["statics_log_file"]
app["statics_log_file"] = statics_log_file
detail_log_file = default_task["detail_log_file"]
clean_sql_file = default_task["clean_sql_file"]
init_logging(detail_log_file)
#
# task_list = ["inner", "outer", "country", "organic"]
task_list = config.google_play_task_list.keys()
start_date, stop_date, task_type = parse_args(task_list)
print start_date, stop_date, task_type
# exit()
# tmp sql file
sql_file_name = toolkit.gen_sql_file_name()
# loop by date
time_step = config.google_play_task_list[task_type].get('time_step', 1)
p_date = start_date
while p_date <= stop_date:
worker(task_type, sql_file_name, p_date)
p_date += datetime.timedelta(days=time_step)
# update year,month, year_month
table_name = config.google_play_task_list[task_type]['table_name']
update_time_sql_pattern = "\nUPDATE `{table_name}` " \
"SET `year` = YEAR(`date`), `month` = Month(`date`), " \
" `year_month` = DATE_FORMAT(`date`,'%Y%m') " \
"WHERE `year` IS NULL;\n"
update_sql = update_time_sql_pattern.format(table_name=table_name)
with open(sql_file_name, 'a') as f:
f.write(update_sql)
# 执行 SQL
mysql_config = config.mysql_config
toolkit.execute_mysql_sql(mysql_config['host'], mysql_config['port'],
mysql_config["user"], mysql_config["passwd"],
mysql_config["dbname"], sql_file_name)
# END
if clean_sql_file:
os.remove(sql_file_name)
else:
logging.info("keep the tmp sql file @ %s" % sql_file_name)
return