def update_jobstatus(job_name,execute_time,execid,status=None,isUpdateStart=False):
job_status = JobStatus()
job_status.job_name = job_name
job_status.execute_time = execute_time
job_status.exec_id = execid
if status!=None:
job_status.status = status
job_status.update_jobstatus()
sysout('更新[%s]的执行状态[execute_time:%s][status:%s]' % (job_name,execute_time,status))
elif isUpdateStart:
job_status.update_starttime()
def get_extdep_jobs(base_time,ext_deps):
ext_jobs = json.loads(ext_deps)
query_jobs = []
for ext_job in ext_jobs['jobs']:
execute_time = get_exetime(base_time,ext_job['time_type'],ext_job['hour_diff'])
job_status = JobStatus()
job_status.job_name = ext_job['name']
job_status.execute_time = execute_time
print '>>>>>>>>>>>>外部依赖任务检查[%s][%s]' % (job_status.job_name,job_status.execute_time)
query_jobs.append(job_status)
return query_jobs
def update_jobstatus(job_name,execute_time,execid,status):
job_status = JobStatus()
job_status.job_name = job_name
job_status.execute_time = execute_time
job_status.exec_id = execid
job_status.status = status
job_status.update_jobstatus()
print '更新[%s]的执行状态[execute_time:%s][status:%s]' % (job_name,execute_time,status)
def post(self):
#任务
job_name = self.get_argument('job_name', '')
exec_time = self.get_argument('execute_time', '')
logging.info('----job name:[%s]' % job_name)
try:
json = JobStatus.get_jobstatus_json(job_name, exec_time)
self.write(json)
except Exception, e:
self.write('error:%s' % e)
def get_extdep_jobs(base_time,ext_deps):
ext_jobs = json.loads(ext_deps)
query_jobs = []
for ext_job in ext_jobs['jobs']:
execute_time = get_exetime(base_time,ext_job['time_type'],ext_job['hour_diff'])
#判断小时级任务天级依赖
scripts = Job.get_job_fromdb(ext_job['name']).server_script
if ext_job['time_type']=='day' and (scripts.find('&[last_hour]')>=0 or scripts.find('&[cur_hour]')>=0):
sysout('>>>>>>>>>>>>%s为小时级任务天级依赖，生成所有任务依赖' % ext_job['name'])
for i in range(24):
job_status = JobStatus()
job_status.job_name = ext_job['name']
job_status.execute_time = '%s%s' % (execute_time,'0%s' % i if i<10 else i)
sysout('>>>>>>>>>>>>添加外部依赖任务检查项[%s][%s]' % (job_status.job_name,job_status.execute_time))
query_jobs.append(job_status)
else:
job_status = JobStatus()
job_status.job_name = ext_job['name']
job_status.execute_time = execute_time
sysout('>>>>>>>>>>>>添加外部依赖任务检查项[%s][%s]' % (job_status.job_name,job_status.execute_time))
query_jobs.append(job_status)
return query_jobs
def exe_job(name,execid,start_time,param_dict_str=''):
job = Job.get_job_fromdb(name)
if job.server_host == '' or job.server_user=='' or job.server_script=='':
raise MyScheduleException('远程配置信息不全')
#1/初始化配置
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(job.server_host,22,job.server_user, "")
#2/初始化默认参数,替换脚本
param_dict = get_param_dict(start_time,param_dict_str)
#拼接脚本命令,cd目录,执行脚本,获取脚本执行状态
command = ' source /etc/bashrc;'
if job.server_dir !='':
work_dir,tmp_time = replace_param(job.server_dir,param_dict)
print 'work_dir:%s' % work_dir
command = 'cd %s;' % work_dir
command = command + job.server_script
if not job.server_script.endswith(';'):
command = command + ';'
command = command+'echo "script execute status ["$?"]";'
command,execute_time = replace_param(command,param_dict)
print '>>>>>>>>>>>>%s' % (command)
#####更新job执行状态,启动######
update_jobstatus(name,execute_time,execid,0)
#3/检查跨项目或时间维度的依赖任务,未完成的两分钟检查一次,并实时输出到日志中
file_path = '%s/../executor/executions/%s/home/hadoop/service/azkaban/files/%s/_job.%s.%s.log' % (CURRENTPATH,execid,job.project_name,execid,job.name)
if job.ext_dependencies != '':
query_jobs = get_extdep_jobs(execute_time,job.ext_dependencies)
f = open(file_path,'a')
f.write('>>>>>>>>>>>>当前任务的基准时间:%s\n' % execute_time)
f.write('>>>>>>>>>>>>依赖的外部任务配置:%s\n' % job.ext_dependencies)
f.close()
while True:
flag,mes = JobStatus.is_ready(query_jobs)
f = open(file_path,'a')
f.write('>>>>>>>>>>>>依赖的外部任务准备情况:%s\n' % mes)
f.close()
if flag:
break
else:
time.sleep(120)
#4/脚本执行,输出信息
stdin, stdout, stderr = ssh.exec_command(command)
error_flag = True
print '---------------out----------------'
for line in stdout.readlines():
print line.replace('\n','')
if line.find('script execute status [0]')>=0:
error_flag=False
print '---------------err-----------------'
for line in stderr.readlines():
print line.replace('\n','')
print '---------------err end-----------------'
ssh.close()
#5/出错发邮件及短信,并抛出异常
if error_flag:
#####更新job执行状态,失败######
update_jobstatus(name,execute_time,execid,-1)
exe_alarm(job,execid,start_time)
raise MyScheduleException('脚本执行过程出错')
#####更新job执行状态,成功######
update_jobstatus(name,execute_time,execid,1)
def process(name,execid,start_time,param_dict_str=''):
sysout('+++++++++++++任务初始化+++++++++++++++')
sysout('开始加载远程任务 %s' % name)
job = Job.get_job_fromdb(name)
if job.server_host == '' or job.server_user=='' or job.server_script=='':
raise MyScheduleException('远程配置信息不全')
sysout('任务加载成功 \n=====================\n%s\n=====================' % job)
#1/初始化配置
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(job.server_host,22,job.server_user, "")
#2/初始化默认参数,替换脚本
param_dict = get_param_dict(execid,param_dict_str)
#拼接脚本命令,cd目录,执行脚本,获取脚本执行状态
command = ' source /etc/bashrc;'
if job.server_dir !='':
work_dir,tmp_time = replace_param(job.server_dir,param_dict)
sysout('配置脚本执行目录 work_dir:%s' % work_dir)
command = 'cd %s;' % work_dir
#给每个命令设备stdout不缓存的参数
scripts = ';'.join(map(lambda x:'' if x.strip()=='' else 'stdbuf -o0 %s' % x , job.server_script.split(';')))
command = command + scripts
if not job.server_script.endswith(';'):
command = command + ';'
command = command+'echo "script execute status ["$?"]";'
command,execute_time = replace_param(command,param_dict)
sysout('生成完整执行脚本:%s' % (command))
sysout('+++++++++++++任务初始化完成+++++++++++++++')
#####更新job执行状态,启动######
update_jobstatus(name,execute_time,execid,0)
#3/检查跨项目或时间维度的依赖任务,未完成的两分钟检查一次,并实时输出到日志中
if job.ext_dependencies != '':
sysout('>>>>>>>>>>>>检查外部依赖完成情况')
sysout('>>>>>>>>>>>>当前任务的基准时间:%s' % execute_time)
sysout('>>>>>>>>>>>>依赖的外部任务配置:%s' % job.ext_dependencies)
query_jobs = get_extdep_jobs(execute_time,job.ext_dependencies)
wait_count = 1
while True:
flag,mes = JobStatus.is_ready(query_jobs)
sysout('>>>>>>>>>>>>第%s次检查,依赖的外部任务准备情况:%s\n' % (wait_count,mes))
if wait_count*120 > 86400:
update_jobstatus(name,execute_time,execid,-1)
raise MyScheduleException('依赖任务等待时间超过1天,请检查')
if flag:
update_jobstatus(name,execute_time,execid,None,True)
break
else:
wait_count += 1
time.sleep(120)
#4/脚本执行,输出信息
sysout('开始执行远程脚本:\n%s' % scripts)
stdin, stdout, stderr = ssh.exec_command(command)
error_flag = True
sysout('---------------out 远程脚本日志输出----------------')
while True:
line = stdout.readline()
if len(line) == 0:
break
sysout(line.strip())
#sysout('%s,%s' % (time.time(),line))
if line.find('script execute status [0]')>=0:
error_flag=False
sysout('---------------err 远程脚本错误输出-----------------')
for line in stderr.readlines():
sysout(line.replace('\n',''))
sysout('---------------err end-----------------')
ssh.close()
#5/出错发邮件及短信,并抛出异常
if error_flag:
#####更新job执行状态,失败######
update_jobstatus(name,execute_time,execid,-1)
return True
#####更新job执行状态,成功######
exe_success(job,execid,start_time)
update_jobstatus(name,execute_time,execid,1)
return False