def execute_layer_command(self, cmdsList, layerCmdsDic): """ 执行层命令 :param cmdsList: 任务列表,list :param layerCmdsDic: 依赖任务列表,dic :return: """ threadList = [] isFinalSucc = True for cmd in cmdsList: relies = layerCmdsDic[cmd] is_ready = True for rely in relies: if rely not in self.complete: logger.warning('[****命令【%s】依赖命令【%s】,但是依赖命令没有完成****]' % (cmd, rely)) is_ready = False if not is_ready: logger.warning('[****暂时跳过【%s】命令! ****]' % cmd) continue logger.info('[****即将开始执行【%s】命令!****]' % cmd) thr = CmdThread(self.index, cmd) self.index = self.index + 1 threadList.append(thr) for thr in threadList: thr.start() for thr in threadList: thr.join() if thr.isSuccess: self.complete.append(thr.cmd) else: isFinalSucc = False return isFinalSucc
def build(self, tableFile): with open(tableFile) as ts: for t in ts.readlines(): t = t.replace('\n', '') if (t.strip(' ') == ''): continue elif t.startswith('#'): continue sql = self._buildSql(t, self.database) lines = self.db.read(sql) jBase = {} jField = { "uuid": "uuid", "sfzh": "", "user_name": "", "email": "", "phoneno": "", "password": "", "explode_time": "explode_time", "confidence": "confidence", "source_table": "", "source": "" } jRule = {"confidence": "confidence"} for l in lines: if l['column_comment'] in self.fieldTransMapping: jField[self.fieldTransMapping[ l['column_comment']]] = l['column_name'] jRule[l['column_name']] = 'not_null' if l['sjly'] in self.sourceTransMapping: l['sjly'] = self.sourceTransMapping[l['sjly']] jField['source'] = l['sjly'] jField['source_table'] = l['table_name'] self.table = l['table_name'] + '.json' jBase['source'] = l['sjly'] jBase['table'] = l['table_name'] jBase['database'] = l['table_schema'] jBase['fieldMapping'] = jField jBase['rule'] = jRule if not bool(jBase): logger.warning( f'【{t}】表没有找到需要构建的mapping,请检查table.txt文件和数据库') continue logger.info(jBase['database'] + '.' + jBase['table']) loads = json.dumps(jBase, ensure_ascii=False, indent=4) if not os.path.isdir(self.filePath): os.makedirs(self.filePath) file = os.path.join(self.filePath, self.table) with open(file, mode="w+", encoding="utf-8") as fd: fd.write(loads) fd.flush() fd.close() ts.close()
def wrapper(*args, **kargs): sta, res = subprocess.getstatusoutput('ps -ef|grep scheJobHip.py') # print(res) numrows = 0 for line in res.split('\n'): if line.find('Python scheJobHip.py') != -1: numrows = numrows + 1 if numrows > 1: logger.info('当前程序正在运行,不予执行!') else: return func(**kargs)
def checkJson(self, json): """ 对文件或者json字符串进行模板匹配 :param json: :return: """ is_json = False if isinstance(json, str) and os.path.isfile(json): if self._checkJsonFile(json): logger.info(f'Json文件【{json}】验证通过!') is_json = True else: logger.error(f"Json文件【{json}】错误,需要对应模板格式!") else: if self._checkJsonStr(json): logger.info(f'Json格式数据验证通过!') is_json = True else: logger.error("Json格式数据错误,需要对应模板格式!请检查上送数据") return is_json
def execute_cmds(self): """ 批量执行任务,任务列表见config.py :return: """ cmdLayers = self.cfg.keys() is_success = True for cmdLayer in cmdLayers: layerCmds = self.cfg[cmdLayer] logger.info('[****即将开始执行第【%d】层****]' % cmdLayer) cmds = layerCmds.keys() isSucc = self.execute_layer_command(cmds, layerCmds) if not isSucc: logger.error('[****任务失败在第【%d】层 ****]' % cmdLayer) is_success = False break if is_success: logger.info('[****任务执行成功!****]') else: logger.error('[****任务执行失败!****]') return is_success, self.complete
def run(self): isComeon = True if isComeon: status = 0 result = '假装我就是执行结果' # status, result = subprocess.getstatusoutput(self.cmd) if status == 0: logger.info('[****命令【%s】执行成功,退出进程!****]' % self.cmd) logger.info('[EXCUTE_DONE]%s' % self.cmd) logger.info('[****执行结果【%s】****]' % result) self.isSuccess = True else: logger.error( '[****命令【%s】执行失败! status=【%d】 result=【%s】进程退出!****]' % (self.cmd, status, result)) logger.error('[EXCUTE_DONE]%s' % self.cmd)
def tick(): mark = str(time.time()) logger.info('%s' % mark) # 1.先建mapping expMapping = ExploreMappingRealtion('dbm_service') expMapping.build('../script/table.txt') # 2.根据mapping构建sql for root, dirs, files in os.walk('../mappings'): for file in files: if file.endswith('json'): file_ = os.path.join(root, file) with open(file_, 'r') as f: expSql = ExploteSql(f.name) logger.info(f'正在处理【{f.name}】文件') if not os.path.isdir('../sql'): os.makedirs('../sql') file = os.path.join('../sql', expSql.table + '.sql') lis = [ '20', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10' ] with open(file, mode="w+", encoding="utf-8") as fd: for file_ in lis: fd.write(expSql.getSql(file_)) fd.write('\n\n') threadList = [] cnt = 1 for root, dirs, files in os.walk('../sql'): for file in files: if file.endswith('sql'): file_ = os.path.join(root, file) with open(file_, 'r') as f: logger.info('正在处理【%s】文件' % f.name) thr = CmdThread(cnt, f'hive -f "{f.name}"') threadList.append(thr) for t in threadList: t.start() for t in threadList: t.join() if t.isSuccess: pass splitlog('../logs/hipdataload.log', mark)
"email": "tag3", "phoneno": "tag4", "password": "******", "explode_time": "explode_time", "confidence": "confidence" }, "rule": { "sfzh": "not_null", "user_name": ["not_null", "convert_empty"], "email": "convert_empty", "phoneno": "not_null", "confidence": "confidence" }, "test": "test" } logger.info(f'succData:{ck.checkJson(succData)}') errData = { "err": "12306", "database": "sgk_source", "table": "ssd_12306account_result", "fieldMapping": { "uuid": "uuid", "sfzh": "tag1", "user_name": "tag2", "email": "tag3", "phoneno": "tag4", "password": "******", "explode_time": "explode_time", "confidence": "confidence" },
if (k in ['source', 'source_table']): v_ = f"'{v_}'" col.append(v_) c2_ = ','.join(col) return self._buildSql(c2_, targetType) if __name__ == '__main__': LoggerPro().config() for dirPath, dirNames, fileNames in os.walk('../mappings'): for fileName in fileNames: if fileName.endswith('json'): i = os.path.join(dirPath, fileName) with open(i, 'r') as f: c = ExploteSql(f.name) logger.info(f'正在处理【{f.name}】文件') if not os.path.isdir('../sql'): # 无文件夹时创建 os.makedirs('../sql') file = os.path.join('../sql', c.table + '.sql') lis = [ '20', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10' ] with open(file, mode="w+", encoding="utf-8") as fd: for i in lis: fd.write(c.getSql(i)) fd.write('\n\n') fd.flush() fd.close()