def run(self):
current_time = DateUtil.get_now()
second = current_time.second
time.sleep(60 - second)
trigger_job = self.scheduler.add_job(self.sched_trigger_run, 'interval', minutes=1)
self.logger.info("添加处理时间触发任务:" + str(trigger_job))
run_job = self.scheduler.add_job(self.sched_job_run, 'interval', seconds=30)
self.logger.info("添加处理Job运行任务" + str(run_job))
self.scheduler.start()
def reslove_job_dep(self, job):
today = DateUtil.format_year_day(DateUtil.get_now())
job_name = job["job_name"]
dep_jobs = self.dboption.get_dependency_job_status(job_name) # job_name,status
self.logger.info("job_name:" + str(job_name) + " 依赖的 job数量:" + str(self.get_list_length(dep_jobs)))
should_run = True
for dep_job in dep_jobs:
dep_job_name = dep_job["job_name"]
dep_job_status = dep_job["job_status"]
dep_job_last_run_date = dep_job["last_run_date"] # 最后一次运行日期
self.logger.info("job_name:" + job_name + " 依赖的Job:" + dep_job_name + " 运行状态:" + str(dep_job_status)
+ " 最后运行时间:" + str(dep_job_last_run_date))
if (dep_job_last_run_date and dep_job_last_run_date != today) or dep_job_last_run_date is None:
should_run = False
self.logger.info("因Job:" + job_name + " 依赖Job:" + dep_job_name +
"最后运行时间:" + str(dep_job_last_run_date) + "不是:" + today + ",无法运行")
break
if dep_job_status != "Done":
should_run = False
self.logger.info("因Job:" + job_name + " 依赖Job:"
+ dep_job_name + " 没有运行完成状态:" + dep_job_status + ",无法运行")
break
return should_run
def sched_job_run(self):
self.logger.info("... interval run sched_job_run ....")
start_time = DateUtil.get_now()
# 当前运行的任务数
max_running_jobs = int(self.config.get("job.run.max"))
running_jobs = self.dboption.get_running_jobs()
if running_jobs is not None:
count_running_jobs = len(running_jobs)
if count_running_jobs > max_running_jobs:
self.logger.info("当前运行的Job 数量:" + str(count_running_jobs) + "大于系统的最大任务数:" + str(max_running_jobs))
return
else:
count_running_jobs = 0
self.logger.info(str(start_time) + " 当前RUNNING状态的Job 数量:" + str(count_running_jobs))
pending_jobs = self.dboption.get_pending_jobs()
if pending_jobs is None or len(pending_jobs) == 0:
count_pending_jobs = 0
self.logger.info("当前Pending状态的 Job 数量:" + str(count_pending_jobs))
return
else:
count_pending_jobs = len(pending_jobs)
self.logger.info("当前Pending状态的 Job 数量:" + str(count_pending_jobs))
require_jobs_count = max_running_jobs - count_running_jobs
if require_jobs_count > 0:
should_require_jobs = set()
require_time = 0
should_continue = True
while True:
if not should_continue:
break
if len(should_require_jobs) == require_jobs_count:
break
if require_time > 100: # 强制的判断方式有点问题,判断100次
break
self.logger.info("第" + str(require_time) + " 次运行")
self.logger.info("可以运行的job数量:" + str(len(should_require_jobs)) + " 需要运行的job数量:" + str(require_jobs_count))
pending_jobs = self.dboption.get_pending_jobs_by_require_time(require_time, max_running_jobs)
for job in pending_jobs:
job_name = job["job_name"]
self.logger.info("判断Job:" + job_name + "依赖是否全部运行完成")
should_run = self.reslove_job_dep(job)
if should_run: # FixMe 需要判断今天是否运行过
self.logger.info("job:" + job_name + " 依赖全部运行完成添加到可以运行的job列表中")
should_require_jobs.add(job_name)
if len(should_require_jobs) == require_jobs_count:
should_continue = False
break
if pending_jobs is None or len(pending_jobs) < require_jobs_count:
self.logger.info("当前Pending状态的 Job 数量 " + str(self.get_list_length(pending_jobs)) +
" 小于 需要运行的任务数 " + str(require_jobs_count) +",无需循环")
should_continue = False
require_time += 1
self.logger.info("需要运行的Job:" + str(should_require_jobs))
self.run_job_command(should_require_jobs)
end_time = DateUtil.get_now()
self.logger.info("依赖触发调度执行时长:" + str(end_time - start_time))
else:
self.logger.info("当前运行的Job 数量:" + str(count_running_jobs) + "大于系统的最大任务数")
def sched_trigger_run(self, current_time = None):
self.logger.info("... interval run sched_trigger_run ....")
start_time = DateUtil.get_now()
self.get_timetrigger_job(start_time)
end_time = DateUtil.get_now()
self.logger.info("时间触发调度运行时长:" + str(end_time - start_time))