def _checkJsonStr(self, jsonStr):
"""
校验核心方法
:param jsonStr: 待校验字符串
:return: True False
"""
try:
validate(instance=jsonStr, schema=self._schema)
except Exception as e:
logger.error(e)
return False
else:
return True
def run(self):
isComeon = True
if isComeon:
status = 0
result = '假装我就是执行结果'
# status, result = subprocess.getstatusoutput(self.cmd)
if status == 0:
logger.info('[****命令【%s】执行成功，退出进程!****]' % self.cmd)
logger.info('[EXCUTE_DONE]%s' % self.cmd)
logger.info('[****执行结果【%s】****]' % result)
self.isSuccess = True
else:
logger.error(
'[****命令【%s】执行失败! status=【%d】 result=【%s】进程退出!****]' %
(self.cmd, status, result))
logger.error('[EXCUTE_DONE]%s' % self.cmd)
def checkJson(self, json):
"""
对文件或者json字符串进行模板匹配
:param json:
:return:
"""
is_json = False
if isinstance(json, str) and os.path.isfile(json):
if self._checkJsonFile(json):
logger.info(f'Json文件【{json}】验证通过！')
is_json = True
else:
logger.error(f"Json文件【{json}】错误，需要对应模板格式！")
else:
if self._checkJsonStr(json):
logger.info(f'Json格式数据验证通过！')
is_json = True
else:
logger.error("Json格式数据错误，需要对应模板格式！请检查上送数据")
return is_json
def execute_cmds(self):
"""
批量执行任务，任务列表见config.py
:return:
"""
cmdLayers = self.cfg.keys()
is_success = True
for cmdLayer in cmdLayers:
layerCmds = self.cfg[cmdLayer]
logger.info('[****即将开始执行第【%d】层****]' % cmdLayer)
cmds = layerCmds.keys()
isSucc = self.execute_layer_command(cmds, layerCmds)
if not isSucc:
logger.error('[****任务失败在第【%d】层 ****]' % cmdLayer)
is_success = False
break
if is_success:
logger.info('[****任务执行成功!****]')
else:
logger.error('[****任务执行失败!****]')
return is_success, self.complete
def _buildSql(self, cols, targetType):
"""
拼接要插入目标表的sql
:param self:
:param cols:字段字符串，select 后面的具体字段
:param targetType: 插入目标表类型
宽表:20,
关系表:01:身份证手机号关系；
02:身份证邮箱关系；
03:身份证用户名关系；
04:身份证密码关系；
05:手机号用户名关系；
06:手机号邮箱关系；
07:手机号密码关系；
08:邮箱用户名关系；
09:邮箱密码关系；
10:用户名密码关系
:return: sql
"""
sfzh_ = "and length(trim(sfzh)) in (15,18)"
phoneno_ = "and length(trim(phoneno))=11 and substr(trim(phoneno),1,1)=1"
email_ = "and upper(trim(email)) like '%.COM%'"
user_name_ = "and user_name!=email and upper(trim(user_name))!='NULL' and trim(user_name)!='' and user_name is not null"
password_ = "and upper(trim(password))!='NULL' and trim(password)!='' and password is not null"
s1 = ''
s2 = ''
sql = ''
if (targetType == '20'):
sql = format(
"insert into table sgk.t_ml_sgk_small_merge_%s "
"(uuid,sfzh,user_name,email,phoneno,password,explode_time,confidence,source_table,source) "
"select %s from sgk_source.%s "
"where 1=1 ;" % (self.source, cols, self.table))
else:
if (targetType == '01'):
sql = "sfzh,phoneno,'01'"
s1 = sfzh_
s2 = phoneno_
elif (targetType == '02'):
sql = "sfzh,email,'02'"
s1 = sfzh_
s2 = email_
elif (targetType == '03'):
sql = "sfzh,user_name,'03'"
s1 = sfzh_
s2 = user_name_
elif (targetType == '04'):
sql = "sfzh,password,'04'"
s1 = sfzh_
s2 = password_
elif (targetType == '05'):
sql = "phoneno,user_name,'05'"
s1 = phoneno_
s2 = user_name_
elif (targetType == '06'):
sql = "phoneno,email,'06'"
s1 = phoneno_
s2 = email_
elif (targetType == '07'):
sql = "phoneno,password,'07'"
s1 = phoneno_
s2 = password_
elif (targetType == '08'):
sql = "email,user_name,'08'"
s1 = email_
s2 = user_name_
elif (targetType == '09'):
sql = "email,password,'09'"
s1 = email_
s2 = password_
elif (targetType == '10'):
sql = "user_name,password,'10'"
s1 = user_name_
s2 = password_
else:
logger.error('参数错误，请检查！')
sql = format("-- insert into table sgk.t_ml_sgk_relation "
"select uuid,%s,'%s',confidence "
"from sgk.t_ml_sgk_small_merge_%s "
"where 1=1 %s %s ;" %
(sql, self.table, self.source, s1, s2))
return sql