def main():
tasks_id = task_db.CustomIndexTask.get_timely_tasks()
for task_id in tasks_id:
task = task_db.CustomIndexTask(task_id)
if task.if_run_now():
date = datetime.date.today()
date = date.strftime("%Y%m%d")
tools.run_main_cmd("custom_timely_import", [task.id, date])
def run(task_id, date, replace_ftp=None):
logging.info("[BEGIN]task id:%s, date:%s" % (task_id, date))
task_id = int(task_id)
task = task_db.CustomIndexTask(task_id)
if task.task_type != "index":
logging.fatal('task type is %s' % task.task_type)
exit(-1)
# 若启动时输入了ftp地址则覆盖任务默认地址
if replace_ftp:
ftp = replace_ftp
else:
ftp = task.path
try:
path = base.get_index(task_id, date, ftp)
except error.DownloadError:
if "%s" in ftp:
ftp = ftp % date
addr = '*****@*****.**' % task.owner
cc = '*****@*****.**'
title = u'%s 指标入库失败' % task.name
text = u'%s 指标【任务id:%s】入库失败\n日期:%s\nftp地址:%s' %\
(task.name, task.id, date, ftp)
logging.fatal('wget error')
tools.send_email(addr, title, text, cc=cc)
exit(-1)
else:
# 解析入库
save_index(path, task, date)
# 获取待summary待更新日期
date_list = get_summary_date(task_id, date)
logging.info("summary date list:%s" % date_list)
run_summary_cmd(date_list, task_id)
logging.info("build output!")
tools.run_main_cmd("custom_original_output", [task_id, date])
task.update_last_run_date(date)
logging.info("[END]task id:%s, date:%s" % (task_id, date))
def run_routine_task(hour):
tasks = task_db.CustomIndexTask.get_routine_tasks_by_hour(hour)
today = datetime.date.today().strftime("%Y%m%d")
for task in tasks:
task = task_db.CustomIndexTask(task)
date = datetime.date.today() - datetime.timedelta(days=task.time_delta)
date = date.strftime("%Y%m%d")
if task.task_type == "index":
if task.if_run_today():
tools.run_main_cmd("custom_original_import", [task.id, date])
task.update_last_run_date(today)
else:
tools.run_main_cmd("custom_summary_import", [task.id, date])
elif task.task_type == "detail":
tools.run_main_cmd("custom_detail_import", [task.id, date])
def run_summary_cmd(date_list, task_id):
for date in date_list:
tools.run_main_cmd("custom_summary_import", [task_id, date])
def run_unroutine_task():
tasks = task_db.CustomIndexTask.get_unroutine_tasks()
today = datetime.date.today().strftime("%Y%m%d")
for task in tasks:
tools.run_main_cmd("custom_summary_import", [task, today])