def _fetchNewCronTask(self): """ update cron info :return: """ cron_list = models.CronConf().getCronTask(fetch_time=self.fetch_time) for cron in cron_list: if (cron.task_id not in self.crons_conf) or ( cron.start_time != self.crons_conf[cron.task_id].start_time): self.crons_conf[cron.task_id] = cron self.addTimer(cron_conf=cron) self.fetch_time = datetime.now()
def run_timer(self, timer): """ :param task: timer info :return: err """ log.logger.info(" begin run timer {} with time {}".format( timer.task_id, timer.execute_time)) task_list = models.CronConf().getCronTask(timer.task_id) err = None for cron_conf in task_list: if cron_conf.enable != State.Enabled: err = " task {}, status {} is not enabled, stop running ".format( cron_conf.task_id, cron_conf.enable) log.logger.info(err) continue timer.etl_day = self.calcCronEtlDay(cron_conf, timer.execute_time) self.create_instance(cron_conf, timer.etl_day) return err
def rerun_task(self, task_id, date_list, up_and_down=False, run_up=False, run_down=False, force=False): """ rerun_task :param task_id: task_id :param date_list: range list :param run_up:run upstream :param run_down: run downstream :return: """ kafka_utils.setup_kafka(config.G_Conf.Common.Broker) # run job define instance_list = [] msg = "" run_type = "" for loop in ["looop"]: _ = loop job = models.TaskDefine().get_job_by_task_id( task_id_list=[task_id]) if job and len(job) > 0: run_type = "job" job_list = [] if run_up: job_list = models.TaskDependency().get_all_upstream( task_id) elif run_down: job_list = models.TaskDependency().get_all_downstream( task_id) elif up_and_down: up_job = models.TaskDependency().get_all_upstream(task_id) down_job = models.TaskDependency().get_all_downstream( task_id) if len(up_job) > 0: job_list.extend(up_job) if len(down_job) > 0: job_list.extend(down_job) else: # run a job with force if force: for etl_day in date_list: tmp = models.TaskInstance( ).direct_run_single_job_task(etl_day, job) if tmp > 0: instance_list.extend(tmp) break else: # run single waiting dependency pass # add self job_list.append(task_id) need_run_job_list = models.TaskDefine().get_job_by_task_id( task_id_list=job_list) if need_run_job_list and len(need_run_job_list) > 0: for etl_day in date_list: models.TaskInstance().create_job_task_instance( execute_date=etl_day, job_list=need_run_job_list) msg = "generate {} TaskDefine task instance ".format( len(need_run_job_list) * len(date_list)) break # run cron task cron = models.CronConf().get_cron_by_task_id(task_id=task_id) if cron: run_type = "cron" for etl_day in date_list: tmp = models.TaskInstance().create_cron_task_instance( execute_date=etl_day, cron_list=[cron]) if len(tmp) > 0: instance_list.extend(tmp) break for instance in instance_list: err = kafka_utils.PushMsgWithRetry( kafka_utils.TOPIC_DISPATCHER, kafka_utils.TaskBeginMsg(instance_id=instance.id, task_id=instance.task_id, execute_date=instance.etl_day)) log.logger.info("push task to queue, instance {}, err {}".format( instance, err)) if len(instance_list) > 0: msg = "generate {} {} task instance ".format( len(instance_list), run_type) return msg