def _fetchNewCronTask(self):
"""
update cron info
:return:
"""
cron_list = models.CronConf().getCronTask(fetch_time=self.fetch_time)
for cron in cron_list:
if (cron.task_id not in self.crons_conf) or (
cron.start_time !=
self.crons_conf[cron.task_id].start_time):
self.crons_conf[cron.task_id] = cron
self.addTimer(cron_conf=cron)
self.fetch_time = datetime.now()
def run_timer(self, timer):
"""
:param task: timer info
:return: err
"""
log.logger.info(" begin run timer {} with time {}".format(
timer.task_id, timer.execute_time))
task_list = models.CronConf().getCronTask(timer.task_id)
err = None
for cron_conf in task_list:
if cron_conf.enable != State.Enabled:
err = " task {}, status {} is not enabled, stop running ".format(
cron_conf.task_id, cron_conf.enable)
log.logger.info(err)
continue
timer.etl_day = self.calcCronEtlDay(cron_conf, timer.execute_time)
self.create_instance(cron_conf, timer.etl_day)
return err
def rerun_task(self,
task_id,
date_list,
up_and_down=False,
run_up=False,
run_down=False,
force=False):
"""
rerun_task
:param task_id: task_id
:param date_list: range list
:param run_up:run upstream
:param run_down: run downstream
:return:
"""
kafka_utils.setup_kafka(config.G_Conf.Common.Broker)
# run job define
instance_list = []
msg = ""
run_type = ""
for loop in ["looop"]:
_ = loop
job = models.TaskDefine().get_job_by_task_id(
task_id_list=[task_id])
if job and len(job) > 0:
run_type = "job"
job_list = []
if run_up:
job_list = models.TaskDependency().get_all_upstream(
task_id)
elif run_down:
job_list = models.TaskDependency().get_all_downstream(
task_id)
elif up_and_down:
up_job = models.TaskDependency().get_all_upstream(task_id)
down_job = models.TaskDependency().get_all_downstream(
task_id)
if len(up_job) > 0:
job_list.extend(up_job)
if len(down_job) > 0:
job_list.extend(down_job)
else:
# run a job with force
if force:
for etl_day in date_list:
tmp = models.TaskInstance(
).direct_run_single_job_task(etl_day, job)
if tmp > 0:
instance_list.extend(tmp)
break
else:
# run single waiting dependency
pass
# add self
job_list.append(task_id)
need_run_job_list = models.TaskDefine().get_job_by_task_id(
task_id_list=job_list)
if need_run_job_list and len(need_run_job_list) > 0:
for etl_day in date_list:
models.TaskInstance().create_job_task_instance(
execute_date=etl_day, job_list=need_run_job_list)
msg = "generate {} TaskDefine task instance ".format(
len(need_run_job_list) * len(date_list))
break
# run cron task
cron = models.CronConf().get_cron_by_task_id(task_id=task_id)
if cron:
run_type = "cron"
for etl_day in date_list:
tmp = models.TaskInstance().create_cron_task_instance(
execute_date=etl_day, cron_list=[cron])
if len(tmp) > 0:
instance_list.extend(tmp)
break
for instance in instance_list:
err = kafka_utils.PushMsgWithRetry(
kafka_utils.TOPIC_DISPATCHER,
kafka_utils.TaskBeginMsg(instance_id=instance.id,
task_id=instance.task_id,
execute_date=instance.etl_day))
log.logger.info("push task to queue, instance {}, err {}".format(
instance, err))
if len(instance_list) > 0:
msg = "generate {} {} task instance ".format(
len(instance_list), run_type)
return msg