def execute_layer_command(self, cmdsList, layerCmdsDic):
"""
执行层命令
:param cmdsList: 任务列表，list
:param layerCmdsDic: 依赖任务列表，dic
:return:
"""
threadList = []
isFinalSucc = True
for cmd in cmdsList:
relies = layerCmdsDic[cmd]
is_ready = True
for rely in relies:
if rely not in self.complete:
logger.warning('[****命令【%s】依赖命令【%s】，但是依赖命令没有完成****]' % (cmd, rely))
is_ready = False
if not is_ready:
logger.warning('[****暂时跳过【%s】命令! ****]' % cmd)
continue
logger.info('[****即将开始执行【%s】命令!****]' % cmd)
thr = CmdThread(self.index, cmd)
self.index = self.index + 1
threadList.append(thr)
for thr in threadList:
thr.start()
for thr in threadList:
thr.join()
if thr.isSuccess:
self.complete.append(thr.cmd)
else:
isFinalSucc = False
return isFinalSucc
def build(self, tableFile):
with open(tableFile) as ts:
for t in ts.readlines():
t = t.replace('\n', '')
if (t.strip(' ') == ''):
continue
elif t.startswith('#'):
continue
sql = self._buildSql(t, self.database)
lines = self.db.read(sql)
jBase = {}
jField = {
"uuid": "uuid",
"sfzh": "",
"user_name": "",
"email": "",
"phoneno": "",
"password": "",
"explode_time": "explode_time",
"confidence": "confidence",
"source_table": "",
"source": ""
}
jRule = {"confidence": "confidence"}
for l in lines:
if l['column_comment'] in self.fieldTransMapping:
jField[self.fieldTransMapping[
l['column_comment']]] = l['column_name']
jRule[l['column_name']] = 'not_null'
if l['sjly'] in self.sourceTransMapping:
l['sjly'] = self.sourceTransMapping[l['sjly']]
jField['source'] = l['sjly']
jField['source_table'] = l['table_name']
self.table = l['table_name'] + '.json'
jBase['source'] = l['sjly']
jBase['table'] = l['table_name']
jBase['database'] = l['table_schema']
jBase['fieldMapping'] = jField
jBase['rule'] = jRule
if not bool(jBase):
logger.warning(
f'【{t}】表没有找到需要构建的mapping，请检查table.txt文件和数据库')
continue
logger.info(jBase['database'] + '.' + jBase['table'])
loads = json.dumps(jBase, ensure_ascii=False, indent=4)
if not os.path.isdir(self.filePath):
os.makedirs(self.filePath)
file = os.path.join(self.filePath, self.table)
with open(file, mode="w+", encoding="utf-8") as fd:
fd.write(loads)
fd.flush()
fd.close()
ts.close()
def wrapper(*args, **kargs):
sta, res = subprocess.getstatusoutput('ps -ef|grep scheJobHip.py')
# print(res)
numrows = 0
for line in res.split('\n'):
if line.find('Python scheJobHip.py') != -1:
numrows = numrows + 1
if numrows > 1:
logger.info('当前程序正在运行，不予执行!')
else:
return func(**kargs)
def checkJson(self, json):
"""
对文件或者json字符串进行模板匹配
:param json:
:return:
"""
is_json = False
if isinstance(json, str) and os.path.isfile(json):
if self._checkJsonFile(json):
logger.info(f'Json文件【{json}】验证通过！')
is_json = True
else:
logger.error(f"Json文件【{json}】错误，需要对应模板格式！")
else:
if self._checkJsonStr(json):
logger.info(f'Json格式数据验证通过！')
is_json = True
else:
logger.error("Json格式数据错误，需要对应模板格式！请检查上送数据")
return is_json
def execute_cmds(self):
"""
批量执行任务，任务列表见config.py
:return:
"""
cmdLayers = self.cfg.keys()
is_success = True
for cmdLayer in cmdLayers:
layerCmds = self.cfg[cmdLayer]
logger.info('[****即将开始执行第【%d】层****]' % cmdLayer)
cmds = layerCmds.keys()
isSucc = self.execute_layer_command(cmds, layerCmds)
if not isSucc:
logger.error('[****任务失败在第【%d】层 ****]' % cmdLayer)
is_success = False
break
if is_success:
logger.info('[****任务执行成功!****]')
else:
logger.error('[****任务执行失败!****]')
return is_success, self.complete
def run(self):
isComeon = True
if isComeon:
status = 0
result = '假装我就是执行结果'
# status, result = subprocess.getstatusoutput(self.cmd)
if status == 0:
logger.info('[****命令【%s】执行成功，退出进程!****]' % self.cmd)
logger.info('[EXCUTE_DONE]%s' % self.cmd)
logger.info('[****执行结果【%s】****]' % result)
self.isSuccess = True
else:
logger.error(
'[****命令【%s】执行失败! status=【%d】 result=【%s】进程退出!****]' %
(self.cmd, status, result))
logger.error('[EXCUTE_DONE]%s' % self.cmd)
def tick():
mark = str(time.time())
logger.info('%s' % mark)
# 1.先建mapping
expMapping = ExploreMappingRealtion('dbm_service')
expMapping.build('../script/table.txt')
# 2.根据mapping构建sql
for root, dirs, files in os.walk('../mappings'):
for file in files:
if file.endswith('json'):
file_ = os.path.join(root, file)
with open(file_, 'r') as f:
expSql = ExploteSql(f.name)
logger.info(f'正在处理【{f.name}】文件')
if not os.path.isdir('../sql'):
os.makedirs('../sql')
file = os.path.join('../sql', expSql.table + '.sql')
lis = [
'20', '01', '02', '03', '04', '05', '06', '07', '08',
'09', '10'
]
with open(file, mode="w+", encoding="utf-8") as fd:
for file_ in lis:
fd.write(expSql.getSql(file_))
fd.write('\n\n')
threadList = []
cnt = 1
for root, dirs, files in os.walk('../sql'):
for file in files:
if file.endswith('sql'):
file_ = os.path.join(root, file)
with open(file_, 'r') as f:
logger.info('正在处理【%s】文件' % f.name)
thr = CmdThread(cnt, f'hive -f "{f.name}"')
threadList.append(thr)
for t in threadList:
t.start()
for t in threadList:
t.join()
if t.isSuccess:
pass
splitlog('../logs/hipdataload.log', mark)
"email": "tag3",
"phoneno": "tag4",
"password": "******",
"explode_time": "explode_time",
"confidence": "confidence"
},
"rule": {
"sfzh": "not_null",
"user_name": ["not_null", "convert_empty"],
"email": "convert_empty",
"phoneno": "not_null",
"confidence": "confidence"
},
"test": "test"
}
logger.info(f'succData：{ck.checkJson(succData)}')
errData = {
"err": "12306",
"database": "sgk_source",
"table": "ssd_12306account_result",
"fieldMapping": {
"uuid": "uuid",
"sfzh": "tag1",
"user_name": "tag2",
"email": "tag3",
"phoneno": "tag4",
"password": "******",
"explode_time": "explode_time",
"confidence": "confidence"
},
if (k in ['source', 'source_table']):
v_ = f"'{v_}'"
col.append(v_)
c2_ = ','.join(col)
return self._buildSql(c2_, targetType)
if __name__ == '__main__':
LoggerPro().config()
for dirPath, dirNames, fileNames in os.walk('../mappings'):
for fileName in fileNames:
if fileName.endswith('json'):
i = os.path.join(dirPath, fileName)
with open(i, 'r') as f:
c = ExploteSql(f.name)
logger.info(f'正在处理【{f.name}】文件')
if not os.path.isdir('../sql'): # 无文件夹时创建
os.makedirs('../sql')
file = os.path.join('../sql', c.table + '.sql')
lis = [
'20', '01', '02', '03', '04', '05', '06', '07', '08',
'09', '10'
]
with open(file, mode="w+", encoding="utf-8") as fd:
for i in lis:
fd.write(c.getSql(i))
fd.write('\n\n')
fd.flush()
fd.close()