def setMkdirDir():
try:
### 创建目录 ###
baseModel.existeFolder(metaInfo.LOGDIRPATH + '/' + metaInfo.getvDate())
baseModel.existeFolder(metaInfo.LOGDIRPATH + '/' +
metaInfo.getvDate() + '/' +
metaInfo.getvGroupId())
return "0"
except:
errors = traceback.format_exc()
print(errors)
return "-99999"
def run():
try:
#print(bigdog.STARTJOBRELATIONDICT)
if len(bigdog.STARTJOBRELATIONDICT) > 0:
manager = multiprocessing.Manager()
tmpStartJobRelation = manager.dict(bigdog.STARTJOBRELATIONDICT)
jobLog.deleteGroupLog()
jobLog.insertGroupLog()
producer = Producer.Producer(tmpStartJobRelation, workQueue,
metaInfo.getvGroupId(),
metaInfo.getvDate(),
metaInfo.getvSnapshot(), successQueue,
errorQueue, {},
producerOver) ### 生产者 ###
consumer = Consumer.Consumer(tmpStartJobRelation, workQueue,
metaInfo.getvGroupId(),
metaInfo.getvDate(),
metaInfo.getvSnapshot(), successQueue,
errorQueue, int(bigdog.GROUPINFO[1]),
producerOver) ### 消费者 ###
producer.start()
consumer.start()
consumer.join()
producer.join()
#jobLog.updateGroupLog(0)
### 运行统计信息 ##
groupRunInfo = jobLog.getGroupRunInfo()
#print(tmpStatus)
if isinstance(groupRunInfo, tuple):
confPv, relaPv, succPv, errPv = groupRunInfo
if errPv == 0:
jobLog.updateGroupLog(2, confPv, relaPv, succPv, errPv)
else:
jobLog.updateGroupLog(3, confPv, relaPv, succPv, errPv)
msgContents = 'bigdog[' + metaInfo.getvGroupId(
) + ':%s:%s]{confPv:%d,relaPv:%d,succPv:%d,errPv:%d}'
msgContents = msgContents % (metaInfo.getvDate(),
metaInfo.getvSnapshot(), confPv,
relaPv, succPv, errPv)
print(msgContents)
baseModel.sendMsgInfo(
'bigdog group[' + metaInfo.getvGroupId() + '] run status',
bigdog.GROUPINFO[-1], msgContents)
else:
pass
else:
print('配置信息有问题')
except:
errors = traceback.format_exc()
print('mainInfo[run] ->' + errors)
baseModel.setWriteContentList(metaInfo.getvLogFilePath(), errors, 'a')
return "-99999"
def runJob(self, jobId):
try:
#### statusId 1 开始 2 正常 3 错误 4 未执行
jobName, jobPath, executeTime, executeDay, retryCount, ruleName, mailList, statusId = bigdog.JOBINFODICT[
jobId]
if mailList is None:
mailList = bigdog.GROUPINFO[-1]
crontabTime = baseModel.getTimeFormat(time.time(),
forMat='%Y%m%d%H%M%S')
alarmIndex = 0
if retryCount is None:
retryCount = bigdog.GROUPINFO[2]
for i in range(retryCount): ### 异常后 重复多少次 ###
### 执行程序 ###
tmpLogName = jobPath.split(
'/')[-1][0:-2] + 'log.' + self.vSnapshot
tmpLogPathName = metaInfo.LOGDIRPATH + '/' + metaInfo.getvDate(
) + '/' + metaInfo.getvGroupId(
) + '/' + tmpLogName + '.' + str(i)
baseModel.rmFile(tmpLogPathName)
jobLog.deleteJobLog(jobId, i)
jobLog.insertJobLog(jobId, i, crontabTime, tmpLogPathName)
baseCmd = 'bash ' + jobPath + ' ' + self.vDate + ' ' + mailList + ' ' + self.vSnapshot + ' ' + tmpLogPathName + ' >>' + tmpLogPathName + ' 1>>' + tmpLogPathName + ' 2>>' + tmpLogPathName
print(baseCmd)
status = os.system(baseCmd)
if status == 0:
### 调整日志 ###
jobLog.updateJobLog(jobId, i, 2, status)
### 删除 执行成功 jobid ###
Consumer.deleteJobId(self, jobId)
break
else:
jobLog.updateJobLog(jobId, i, 3, status)
alarmIndex = i
### 是否Hive hive 日志解析###
if alarmIndex == retryCount - 1: ### 重复次数都错误 告警###
msgContent = 'bigdog[' + self.vDate + ':' + self.vSnapshot + ']{' + bigdog.GROUPINFO[
0] + '---' + jobPath + ' is execution error}-->' + tmpLogPathName
baseModel.sendMsgInfo(metaInfo.taskSubject, mailList,
msgContent)
### 依赖相关程序 下线 ###
Consumer.deleteFailtJobId(self, jobId, crontabTime)
else:
pass
#print('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx------->' + str(jobId))
except:
errors = traceback.format_exc()
print('Consumer[runJob] ->' + errors)
baseModel.setWriteContentList(metaInfo.getvLogFilePath(), errors,
'a')
return "-99999"
def getGroupRunInfo():
try:
dbType, dbUserName, dbUserPassWord, dbName, dbPort, dbHost, charset = dbConfig.DBINFO[
0:]
sqlName = dbConfig.LOGSQLDICT['groupRunInfo'].format(
metaInfo.getvSnapshot(), metaInfo.getvGroupId(),
metaInfo.getvDate())
result = dbConnet.select(dbType, dbUserName, dbUserPassWord, dbName,
dbPort, dbHost, charset, sqlName)
return result[0]
except:
errors = traceback.format_exc()
print('jobLog[updateGroupLog] ->' + errors)
baseModel.setWriteContentList(metaInfo.getvLogFilePath(), errors, 'a')
return "-99999"
def insertGroupLog():
try:
dbType, dbUserName, dbUserPassWord, dbName, dbPort, dbHost, charset = dbConfig.DBINFO[
0:]
sqlName = dbConfig.LOGSQLDICT['INSERTGROUPLOG'].format(
metaInfo.getvGroupId(), metaInfo.getvDate(),
metaInfo.getvSnapshot())
result = dbConnet.insertDeteleUpdate(dbType, dbUserName,
dbUserPassWord, dbName, dbPort,
dbHost, charset, sqlName)
if result == "0":
return "0"
else:
return "-99999"
except:
errors = traceback.format_exc()
print('jobLog[insertGroupLog] ->' + errors)
baseModel.setWriteContentList(metaInfo.getvLogFilePath(), errors, 'a')
return "-99999"
def updateJobLog(jobId, executeStep, vStatus, vErrorCode):
try:
dbType, dbUserName, dbUserPassWord, dbName, dbPort, dbHost, charset = dbConfig.DBINFO[
0:]
sqlName = dbConfig.LOGSQLDICT['UPDATELOG'].format(
vStatus, vErrorCode, metaInfo.getvSnapshot(),
metaInfo.getvGroupId(), jobId, metaInfo.getvDate(), executeStep)
#print(sqlName)
result = dbConnet.insertDeteleUpdate(dbType, dbUserName,
dbUserPassWord, dbName, dbPort,
dbHost, charset, sqlName)
if result == "0":
return "0"
else:
return "-99999"
except:
errors = traceback.format_exc()
#print('jobLog[updateJobLog] ->' + errors)
baseModel.setWriteContentList(metaInfo.getvLogFilePath(), errors, 'a')
return "-99999"
def getOraleInfo():
try:
global STARTJOBRELATIONDICT, JOBRELATIONDICT, JOBINFODICT, GROUPINFO, PUSHMAIL
dbType, dbUserName, dbUserPassWord, dbName, dbPort, dbHost, charset = dbConfig.DBINFO[
0:]
for key, value in dbConfig.SQLDICT.items():
if key == "JOBINFO":
sqlName = value.format(metaInfo.getvSnapshot(),
metaInfo.getvGroupId(),
metaInfo.getvDate())
#print(sqlName)
sqlResult = dbConnet.select(dbType, dbUserName, dbUserPassWord,
dbName, dbPort, dbHost, charset,
sqlName)
for i in sqlResult:
jobId, jobName, jobPath, executeTime, executeDay, retryCount, ruleName, mailList, statusId = i
if jobId not in JOBINFODICT.keys():
JOBINFODICT[jobId] = [
jobName, jobPath, executeTime, executeDay,
retryCount, ruleName, mailList, statusId
]
else:
print("调度有重复----->" + str(jobId))
elif key == "STARTJOBRELATION":
sqlName = value.format(metaInfo.getvGroupId())
#print(sqlName)
sqlResult = dbConnet.select(dbType, dbUserName, dbUserPassWord,
dbName, dbPort, dbHost, charset,
sqlName)
for i in sqlResult:
startJobId, jobList = i
if startJobId not in STARTJOBRELATIONDICT.keys():
STARTJOBRELATIONDICT[startJobId] = []
if jobList is not None:
for tmpI in jobList.split(','):
STARTJOBRELATIONDICT[startJobId].append(
int(tmpI))
else:
print("STARTJOBRELATIONDICT[依赖关系有重复]---->" +
str(startJobId))
#print(STARTJOBRELATIONDICT)
elif key == 'pushMailInfo':
sqlName = value
sqlResult = dbConnet.select(dbType, dbUserName, dbUserPassWord,
dbName, dbPort, dbHost, charset,
sqlName)
for i in sqlResult:
mailHost, mailUser, mailUserHead, mailUserPassword, mailPort, mailSubject = i
pc = passwordAes.prpcrypt('f$Jun%big@Dog!fisher.jie')
mailUserPassword = pc.decrypt(mailUserPassword)
PUSHMAIL = [
mailHost, mailUser, mailUserHead, mailUserPassword,
mailPort, mailSubject
] #passwordAes.aes_decrypt('f$Jun%big@Dog!fisher.jie',mailUserPassword), mailPort, mailSubject]
elif key == "JOBRELATION":
sqlName = value.format(metaInfo.getvGroupId())
#print(sqlName)
sqlResult = dbConnet.select(dbType, dbUserName, dbUserPassWord,
dbName, dbPort, dbHost, charset,
sqlName)
for i in sqlResult:
startJobId, jobList = i
if startJobId not in JOBRELATIONDICT.keys():
JOBRELATIONDICT[startJobId] = []
for tmpI in jobList.split(','):
JOBRELATIONDICT[startJobId].append(int(tmpI))
else:
print("JOBRELATION[依赖关系有重复]---->" + str(startJobId))
#print(JOBRELATIONDICT)
elif key == "GROUPINFO":
sqlName = value.format(metaInfo.getvGroupId())
#print(sqlName)
sqlResult = dbConnet.select(dbType, dbUserName, dbUserPassWord,
dbName, dbPort, dbHost, charset,
sqlName)
for i in sqlResult:
groupName, parallelNums, retryCount, mainList = i
GROUPINFO = [groupName, parallelNums, retryCount, mainList]
#print(GROUPINFO)
if metaInfo.getvJobId() is not None:
tmpJobId = int(metaInfo.getvJobId())
for key, value in JOBINFODICT.items():
JOBINFODICT.pop(key)
jobName, jobPath, executeTime, executeDay, retryCount, ruleName, mailList, statusId = value
JOBINFODICT[key] = [
jobName, jobPath, executeTime, executeDay, retryCount,
ruleName, mailList, 4
]
STARTJOBRELATIONDICT = {}
STARTJOBRELATIONDICT[tmpJobId] = []
getDeleteStartJobRela(metaInfo.getvGroupId(), metaInfo.getvDate(),
metaInfo.getvSnapshot(), tmpJobId)
print(STARTJOBRELATIONDICT)
print(JOBRELATIONDICT)
except:
errors = traceback.format_exc()
print(errors)