def run():
try:
#print(bigdog.STARTJOBRELATIONDICT)
if len(bigdog.STARTJOBRELATIONDICT) > 0:
manager = multiprocessing.Manager()
tmpStartJobRelation = manager.dict(bigdog.STARTJOBRELATIONDICT)
jobLog.deleteGroupLog()
jobLog.insertGroupLog()
producer = Producer.Producer(tmpStartJobRelation, workQueue,
metaInfo.getvGroupId(),
metaInfo.getvDate(),
metaInfo.getvSnapshot(), successQueue,
errorQueue, {},
producerOver) ### 生产者 ###
consumer = Consumer.Consumer(tmpStartJobRelation, workQueue,
metaInfo.getvGroupId(),
metaInfo.getvDate(),
metaInfo.getvSnapshot(), successQueue,
errorQueue, int(bigdog.GROUPINFO[1]),
producerOver) ### 消费者 ###
producer.start()
consumer.start()
consumer.join()
producer.join()
#jobLog.updateGroupLog(0)
### 运行统计信息 ##
groupRunInfo = jobLog.getGroupRunInfo()
#print(tmpStatus)
if isinstance(groupRunInfo, tuple):
confPv, relaPv, succPv, errPv = groupRunInfo
if errPv == 0:
jobLog.updateGroupLog(2, confPv, relaPv, succPv, errPv)
else:
jobLog.updateGroupLog(3, confPv, relaPv, succPv, errPv)
msgContents = 'bigdog[' + metaInfo.getvGroupId(
) + ':%s:%s]{confPv:%d,relaPv:%d,succPv:%d,errPv:%d}'
msgContents = msgContents % (metaInfo.getvDate(),
metaInfo.getvSnapshot(), confPv,
relaPv, succPv, errPv)
print(msgContents)
baseModel.sendMsgInfo(
'bigdog group[' + metaInfo.getvGroupId() + '] run status',
bigdog.GROUPINFO[-1], msgContents)
else:
pass
else:
print('配置信息有问题')
except:
errors = traceback.format_exc()
print('mainInfo[run] ->' + errors)
baseModel.setWriteContentList(metaInfo.getvLogFilePath(), errors, 'a')
return "-99999"
def getGroupRunInfo():
try:
dbType, dbUserName, dbUserPassWord, dbName, dbPort, dbHost, charset = dbConfig.DBINFO[
0:]
sqlName = dbConfig.LOGSQLDICT['groupRunInfo'].format(
metaInfo.getvSnapshot(), metaInfo.getvGroupId(),
metaInfo.getvDate())
result = dbConnet.select(dbType, dbUserName, dbUserPassWord, dbName,
dbPort, dbHost, charset, sqlName)
return result[0]
except:
errors = traceback.format_exc()
print('jobLog[updateGroupLog] ->' + errors)
baseModel.setWriteContentList(metaInfo.getvLogFilePath(), errors, 'a')
return "-99999"
def insertGroupLog():
try:
dbType, dbUserName, dbUserPassWord, dbName, dbPort, dbHost, charset = dbConfig.DBINFO[
0:]
sqlName = dbConfig.LOGSQLDICT['INSERTGROUPLOG'].format(
metaInfo.getvGroupId(), metaInfo.getvDate(),
metaInfo.getvSnapshot())
result = dbConnet.insertDeteleUpdate(dbType, dbUserName,
dbUserPassWord, dbName, dbPort,
dbHost, charset, sqlName)
if result == "0":
return "0"
else:
return "-99999"
except:
errors = traceback.format_exc()
print('jobLog[insertGroupLog] ->' + errors)
baseModel.setWriteContentList(metaInfo.getvLogFilePath(), errors, 'a')
return "-99999"
def updateJobLog(jobId, executeStep, vStatus, vErrorCode):
try:
dbType, dbUserName, dbUserPassWord, dbName, dbPort, dbHost, charset = dbConfig.DBINFO[
0:]
sqlName = dbConfig.LOGSQLDICT['UPDATELOG'].format(
vStatus, vErrorCode, metaInfo.getvSnapshot(),
metaInfo.getvGroupId(), jobId, metaInfo.getvDate(), executeStep)
#print(sqlName)
result = dbConnet.insertDeteleUpdate(dbType, dbUserName,
dbUserPassWord, dbName, dbPort,
dbHost, charset, sqlName)
if result == "0":
return "0"
else:
return "-99999"
except:
errors = traceback.format_exc()
#print('jobLog[updateJobLog] ->' + errors)
baseModel.setWriteContentList(metaInfo.getvLogFilePath(), errors, 'a')
return "-99999"
def getOraleInfo():
try:
global STARTJOBRELATIONDICT, JOBRELATIONDICT, JOBINFODICT, GROUPINFO, PUSHMAIL
dbType, dbUserName, dbUserPassWord, dbName, dbPort, dbHost, charset = dbConfig.DBINFO[
0:]
for key, value in dbConfig.SQLDICT.items():
if key == "JOBINFO":
sqlName = value.format(metaInfo.getvSnapshot(),
metaInfo.getvGroupId(),
metaInfo.getvDate())
#print(sqlName)
sqlResult = dbConnet.select(dbType, dbUserName, dbUserPassWord,
dbName, dbPort, dbHost, charset,
sqlName)
for i in sqlResult:
jobId, jobName, jobPath, executeTime, executeDay, retryCount, ruleName, mailList, statusId = i
if jobId not in JOBINFODICT.keys():
JOBINFODICT[jobId] = [
jobName, jobPath, executeTime, executeDay,
retryCount, ruleName, mailList, statusId
]
else:
print("调度有重复----->" + str(jobId))
elif key == "STARTJOBRELATION":
sqlName = value.format(metaInfo.getvGroupId())
#print(sqlName)
sqlResult = dbConnet.select(dbType, dbUserName, dbUserPassWord,
dbName, dbPort, dbHost, charset,
sqlName)
for i in sqlResult:
startJobId, jobList = i
if startJobId not in STARTJOBRELATIONDICT.keys():
STARTJOBRELATIONDICT[startJobId] = []
if jobList is not None:
for tmpI in jobList.split(','):
STARTJOBRELATIONDICT[startJobId].append(
int(tmpI))
else:
print("STARTJOBRELATIONDICT[依赖关系有重复]---->" +
str(startJobId))
#print(STARTJOBRELATIONDICT)
elif key == 'pushMailInfo':
sqlName = value
sqlResult = dbConnet.select(dbType, dbUserName, dbUserPassWord,
dbName, dbPort, dbHost, charset,
sqlName)
for i in sqlResult:
mailHost, mailUser, mailUserHead, mailUserPassword, mailPort, mailSubject = i
pc = passwordAes.prpcrypt('f$Jun%big@Dog!fisher.jie')
mailUserPassword = pc.decrypt(mailUserPassword)
PUSHMAIL = [
mailHost, mailUser, mailUserHead, mailUserPassword,
mailPort, mailSubject
] #passwordAes.aes_decrypt('f$Jun%big@Dog!fisher.jie',mailUserPassword), mailPort, mailSubject]
elif key == "JOBRELATION":
sqlName = value.format(metaInfo.getvGroupId())
#print(sqlName)
sqlResult = dbConnet.select(dbType, dbUserName, dbUserPassWord,
dbName, dbPort, dbHost, charset,
sqlName)
for i in sqlResult:
startJobId, jobList = i
if startJobId not in JOBRELATIONDICT.keys():
JOBRELATIONDICT[startJobId] = []
for tmpI in jobList.split(','):
JOBRELATIONDICT[startJobId].append(int(tmpI))
else:
print("JOBRELATION[依赖关系有重复]---->" + str(startJobId))
#print(JOBRELATIONDICT)
elif key == "GROUPINFO":
sqlName = value.format(metaInfo.getvGroupId())
#print(sqlName)
sqlResult = dbConnet.select(dbType, dbUserName, dbUserPassWord,
dbName, dbPort, dbHost, charset,
sqlName)
for i in sqlResult:
groupName, parallelNums, retryCount, mainList = i
GROUPINFO = [groupName, parallelNums, retryCount, mainList]
#print(GROUPINFO)
if metaInfo.getvJobId() is not None:
tmpJobId = int(metaInfo.getvJobId())
for key, value in JOBINFODICT.items():
JOBINFODICT.pop(key)
jobName, jobPath, executeTime, executeDay, retryCount, ruleName, mailList, statusId = value
JOBINFODICT[key] = [
jobName, jobPath, executeTime, executeDay, retryCount,
ruleName, mailList, 4
]
STARTJOBRELATIONDICT = {}
STARTJOBRELATIONDICT[tmpJobId] = []
getDeleteStartJobRela(metaInfo.getvGroupId(), metaInfo.getvDate(),
metaInfo.getvSnapshot(), tmpJobId)
print(STARTJOBRELATIONDICT)
print(JOBRELATIONDICT)
except:
errors = traceback.format_exc()
print(errors)