def StartJob():
global _pool
Receive = os.listdir(Config.ETL_Ctrl_Receive)
Running = os.listdir(Config.ETL_Ctrl_Running)
QueueLs = os.listdir(Config.ETL_Ctrl_Queue)
# S04_S04_COMP_20171024.ctl
reg = re.compile(r'^(\w+?)_(\w+)_(\d{8})\.ctl$')
JobStats = GetJobStats()
for i in Receive:
if i in Running:
msg = r"作业已经在执行中:%s" % i
LogUtil.Etl_Log(msg)
os.remove(Config.ETL_Ctrl_Receive + os.sep + i)
if i in QueueLs:
msg = r"作业已经在队列中:%s" % i
LogUtil.Etl_Log(msg)
os.remove(Config.ETL_Ctrl_Receive + os.sep + i)
getList = reg.findall(i)[0]
if getList != []:
if getList[1] in JobStats[getList[0]]['Jobs']:
if checkRunning(JobStats[getList[0]]['Jobs'][getList[1]],
getList[2]):
FileUtil.move(Config.ETL_Ctrl_Receive + os.sep + i,
Config.ETL_Ctrl_Running + os.sep + i)
arg = JobStats[getList[0]]['Jobs'][getList[1]]
_pool.apply_async(RunJob, (
arg,
i,
))
else:
FileUtil.move(Config.ETL_Ctrl_Receive + os.sep + i,
Config.ETL_Ctrl_Queue + os.sep + i)
def updateJob(Job, dt, Last_Stats):
# Job:{'Name':
# ,'dt':
# ,'model':
# ,'system':}
# dt:20171025
# Last_Stats:20171024
try:
InfoPath = Config.ETL_Ctrl_JobInfo + os.sep + Job[
'system'] + os.sep + Job['Name']
if not os.path.exists(InfoPath):
LogUtil.Etl_Log(r'作业信息不存在，请检查：' + InfoPath)
f = open(InfoPath, 'r')
f_read = f.read()
f.close()
f_List = f_read.split(' ')
f_write = '%s %s %s %s' % (dt, f_List[1], f_List[2], Last_Stats)
f = open(InfoPath, 'w')
f.write(f_write)
f.close()
except Exception as e:
print "执行updateJob失败"
LogUtil.Etl_Log_Exception(e, 'StartJob.updateJob')
ExceptionExit()
finally:
if 'f' in vars():
f.close()
def RunJob(Job, CtlName):
# Job:{'Name':
# ,'dt':
# ,'model':
# ,'system':}
# CtlName:S04_S04_COMP_20171023.ctl
reg = re.compile(r'^(\w+?)_(\w+)_(\d{8})\.ctl$')
getList = reg.findall(CtlName)[0]
msg = r"开始执行作业：%s" % (getList[1] + '_' + getList[2])
print msg
LogUtil.Etl_Log(msg)
try:
for i in xrange(3):
time.sleep(10)
print Job['Name'] + ' ' + str(i)
FileUtil.move(Config.ETL_Ctrl_Running + os.sep + CtlName,
Config.ETL_Job_Done + os.sep + CtlName)
except Exception as e:
updateJob(Job, CtlName[len(CtlName) - 12:-4], 'FAILED')
LogUtil.Etl_Log_Exception(e, 'StartJob.RunJob:' + CtlName)
ExceptionExit()
msg = "作业执行完毕：%s %s" % (Job['Name'], CtlName[len(CtlName) - 12:-4])
print msg
LogUtil.Etl_Log(msg)
updateJob(Job, CtlName[len(CtlName) - 12:-4], 'DONE')
# ToDo 调起流作业
touchStream(CtlName)
def init():
global _pool
log = LogUtil()
log.Clear_Ctrl_Log()
log.Etl_Log(
'---------------------------------开始初始化---------------------------------'
)
log.Etl_Log('开始清理调度')
# ToDo 清理所有ETL_HOME下状态的文件
# 创建进程池
if _pool == None:
_pool = ProcessUtil.CreateProcessPool()
def GetJobInfo(etl_sys, jobName):
# etl_sys:S04
# jobName:S04_COMP
ReJobInfo = {}
try:
sys_dir = Config.ETL_Ctrl_JobInfo + os.sep + etl_sys
file_job = open(sys_dir + os.sep + jobName, 'r')
file_job_read = file_job.read()
file_job.close()
if file_job_read.strip() == '':
return False
[job_dt, job_stats, job_model, Last_job_st] = file_job_read.split(' ')
if job_stats.upper() != 'V':
ReJobInfo = {
'Name': jobName,
'dt': job_dt,
'model': job_model,
'system': etl_sys
}
return ReJobInfo
except Exception as e:
print "执行GetJobInfo失败"
LogUtil.Etl_Log_Exception(e, 'StartJob.GetJobInfo')
finally:
if 'file_job' in vars():
file_job.close()
def CheckQueue():
# 如果是数据库，可以设置优先级之类的来做排序，只取前n个作业判断是否可以执行。而后cp到Receive目录
# 这边第一版用的是文件来保存信息。优先级虽然也可以做，但是目测效率上依旧不会太好
# 遂放弃治疗。无视效率。不考虑优先级。所有check后可以run的作业全部扔到Receive目录
try:
JobList = os.listdir(Config.ETL_Ctrl_Queue)
reg = re.compile(r'^(\w+?)_(\w+)_(\d{8})\.ctl$')
for i in JobList:
if reg.match(i):
(etl_sys, job, dt) = reg.findall(i)[0]
Job = GetJobInfo(etl_sys, job)
if checkDependency(Job, dt):
FileUtil.move(Config.ETL_Ctrl_Queue + os.sep + i,
Config.ETL_Ctrl_Receive + os.sep + i)
else:
msg = r'依赖未完成，继续等待:' + etl_sys + '_' + job
LogUtil.Etl_Log(msg)
except Exception as e:
print "执行CheckQueue失败"
LogUtil.Etl_Log_Exception(e, 'StartJob.CheckQueue')
def checkRunning(Job, tx_dt):
# Job:{'Name':
# ,'dt':
# ,'model':
# ,'system':}
# tx_dt:20171025
try:
RunningList = os.listdir(Config.ETL_Ctrl_Running)
TrueList = []
# Running flag file's name like : S04_S04_COMP_20171024.ctl
RegexStr = re.compile(r'^\w+?_\w+_\d{8}\.ctl$')
msg = r"正在检查执行中的作业..."
LogUtil.Etl_Log(msg)
for i in RunningList:
if RegexStr.match(i):
TrueList.append(i)
msg = r"正在执行作业数量：[%s]" % len(TrueList)
LogUtil.Etl_Log(msg)
if len(TrueList) < Config.PARALLEL_JOB_NUM:
if checkDependency(Job, tx_dt):
msg = r"检查完毕，可以继续执行作业"
LogUtil.Etl_Log(msg)
return True
else:
msg = r"检查完毕，依赖作业未完成"
LogUtil.Etl_Log(msg)
return False
else:
msg = r"检查完毕，并行已满"
LogUtil.Etl_Log(msg)
return False
except Exception as e:
print "执行checkRunning失败"
LogUtil.Etl_Log_Exception(e, 'StartJob.checkRunning')
ExceptionExit()
def touchStream(CtlName):
# CtlName:S04_S04_COMP_20171023.ctl
try:
reg = re.compile(r'^(\w+?)_(\w+)_(\d{8})\.ctl$')
(etl_sys, etl_job, dt) = reg.findall(CtlName)[0]
streamDir = Config.ETL_Ctrl_Stream + os.sep + etl_sys + os.sep + etl_job
if os.path.exists(streamDir):
with open(streamDir, 'r') as f:
f_read = f.read()
[st_sys, st_job] = f_read.split(' ')
filename = '%s_%s_%s.ctl' % (st_sys, st_job, dt)
path = Config.ETL_Ctrl_Queue + os.sep + filename
FileUtil.touch(path)
else:
return False
except Exception as e:
print "执行touchStream失败"
LogUtil.Etl_Log_Exception(e, 'StartJob.touchStream')
exit(12)
def checkDependency(Job, tx_dt):
# Job:{'Name':
# ,'dt':
# ,'model':
# ,'system':}
# tx_dt:20171025
try:
if Job == None:
return False
dt = DatetimeUtil()
if dt.DateAdd(Job['dt'], 1) != tx_dt:
return False
if not os.path.exists(Config.ETL_Ctrl_Dependency + os.sep +
Job['system'] + os.sep + Job['Name']):
return True
with open(Config.ETL_Ctrl_Dependency + os.sep + Job['system'] +
os.sep + Job['Name']) as f:
f_read = f.read()
depdir = f_read.split('\n')
IsOk = 1
for i in depdir:
[depsys, depjob] = i.split(' ')
with open(Config.ETL_Ctrl_JobInfo + os.sep + depsys + os.sep +
depjob) as a:
a_read = a.read()
[job_dt, job_stats, job_model,
Last_job_st] = a_read.split(' ')
if not (job_stats.upper() != 'V' and Last_job_st.upper()
== 'DONE' and datetime.strptime(job_dt, '%Y%m%d')
>= datetime.strptime(tx_dt, '%Y%m%d')):
IsOk = 0
if IsOk == 1:
return True
else:
return False
except Exception as e:
print "执行checkDependency失败"
LogUtil.Etl_Log_Exception(e, 'StartJob.checkDependency')
ExceptionExit()
def GetJobStats():
JobStats = {}
try:
msg = r"-----开始检查作业状态-----"
#LogUtil.Etl_Log(msg)
for etl_sys in Config.SystemList:
sys_dir = Config.ETL_Ctrl_JobInfo + os.sep + etl_sys
if not os.path.exists(sys_dir):
continue
job_list = os.listdir(sys_dir)
job_info = {}
for job in job_list:
job_info[job] = GetJobInfo(etl_sys, job)
if job_info[job] == False:
continue
JobStats[etl_sys] = {
'Jobs': job_info,
}
msg = r'-----检查作业状态完毕-----'
#LogUtil.Etl_Log(msg)
return JobStats
except Exception as e:
print "执行GetJobStats失败"
LogUtil.Etl_Log_Exception(e, 'StartJob.GetJobStats')