def failed_procedure(self, execute_failed_programs):
for program in execute_failed_programs:
program_id = program['id']
sid = program['sid']
version = program['version']
subversion = program['subversion']
extra_info = program['extra_info']
sql = mysql.Mysql()
df = sql.mysql_read(get_program_base_info.format(sid=sid))
if len(df) == 0:
print("LoopCheckProgramResult.failed_procedure Err: sid({sid}) 不存在!".format(sid=sid))
continue
program_info = df.loc[0, ['sid', 'path', 'run_type', 'deploy_server', 'port', 'times']].to_dict()
program_info['version'] = version
program_info['id'] = program_id
slave_ip = program_info['deploy_server']
slave_port = program_info['port']
times = program_info['times']
if subversion:
pass
"""重跑流程"""
else:
if program_info['times'] == 0:
re_execute = 0
else:
if sid in self.program_rerun_dict:
if version in self.program_rerun_dict[sid]:
if self.program_rerun_dict[sid][version] != 0:
re_execute = 1
else:
# 计数器为0 删除计数器
del(self.program_rerun_dict[sid][version])
if len(self.program_rerun_dict[sid]) == 0:
del (self.program_rerun_dict[sid])
re_execute = 0
else:
self.program_rerun_dict[sid][version] = times
re_execute = 1
else:
self.program_rerun_dict[sid] = {}
self.program_rerun_dict[sid][version] = times
re_execute = 1
if re_execute == 1:
send_program = SendProgram(slave_ip, slave_port, program_info)
send_program.send()
self.program_rerun_dict[sid][version] -= 1
else:
SetProgramsCalculateInfo(sid, version).program_running_failed()
# 发送邮件
subversion_str = "main"
sql = mysql.Mysql()
df = sql.mysql_read(get_owner_info_from_sid.format(sid=sid))
program_api, owner_name, mail = (df.loc[0, 'name'], df.loc[0, 'owner'], df.loc[0, 'mail'])
msg = "{program_api}[{version}-{subversion}]:{info}". \
format(program_api=program_api, version=version, info=extra_info, subversion=subversion_str)
create_mail(msg, str(mail), to=owner_name,
subject='程序{program_api}执行错误！【错误】信息'.format(program_api=program_api))
def _update_version(sid, new_version):
sql = mysql.Mysql()
df = sql.mysql_read(get_old_version.format(sid=sid))
if len(df) == 0:
print("_update_version ERR: {sid} can't find!".format(sid=sid))
return
old_version = df.loc[0, 'version']
if new_version > old_version or old_version is None:
sql = mysql.Mysql()
sql.mysql_write(update_version.format(version=str(new_version), sid=sid))
def _get_max_flag():
sql1 = mysql.Mysql()
df1 = sql1.mysql_read(get_start_flag)
sql2 = mysql.Mysql()
df2 = sql2.mysql_read(get_end_flag)
try:
start_id = int(df1.loc[0, 'id'])
except Exception as e:
start_id = 0
finally:
try:
end_id = int(df2.loc[0, 'id'])
except Exception as e:
end_id = 0
finally:
return (start_id, end_id)
def _get_all_slaves():
sql = mysql.Mysql()
online_slaves = sql.mysql_read(get_all_slaves)
slaves_info = list()
for idx in online_slaves.index:
slaves_info.append(
online_slaves.loc[idx, ['id', 'ip', 'port']].to_dict())
return slaves_info
def _get_online_slaves():
sql = mysql.Mysql()
online_slaves = sql.mysql_read(sql_codes.get_online_slaves)
# print("online_slaves: ", online_slaves)
slaves_info_online = list()
for idx in online_slaves.index:
slaves_info_online.append(
online_slaves.loc[idx, ['ip', 'port']].to_dict())
return slaves_info_online
def run(self):
programs_info = None
sql = mysql.Mysql()
df = sql.mysql_read(self.sql_code)
if len(df) != 0:
if self.func == 'crawler':
programs_info = self._get_crawler_command(df, programs_info)
elif self.func == 'calculate':
programs_info = self._get_calculate_command(df, programs_info)
return programs_info
def load_slave_status():
global _slave_status
sql = mysql.Mysql()
online_slaves = sql.mysql_read(get_online_slaves)
slaves_info_online = list()
if len(online_slaves) != 0:
for idx in online_slaves.index:
slaves_info_online.append(online_slaves.loc[idx, ['ip', 'port']].to_dict())
if len(slaves_info_online) != 0:
for slave_info in slaves_info_online:
_slave_status[slave_info['ip']] = {"port": slave_info['port'], "count": 0}
def _get_execute_failed_programs(start_id, end_id):
execute_failed_programs = list()
if start_id == end_id:
pass
else:
sql = mysql.Mysql()
df = sql.mysql_read(get_execute_failed_programs.format(start_id=start_id, end_id=end_id))
if len(df) == 0:
pass
else:
for idx in df.index:
info = df.loc[idx, ['id', 'sid', 'version', 'subversion', 'extra_info']].to_dict()
execute_failed_programs.append(info)
return execute_failed_programs
def _get_execute_calculate_programs(start_id, end_id):
execute_calculate_programs = list()
if start_id == end_id:
pass
else:
sql = mysql.Mysql()
df = sql.mysql_read(get_execute_calculate_programs_result.format(start_id=start_id, end_id=end_id))
if len(df) == 0:
pass
else:
for idx in df.index:
info = df.loc[idx, ['id', 'sid', 'version', 'subversion', 'extra_info', 'event_type', 'program_type'
'name', 'owner','mail']].to_dict()
execute_calculate_programs.append(info)
return execute_calculate_programs
def success_procedure(self, execute_success_programs):
for program in execute_success_programs:
sid = program['sid']
version = program['version']
subversion = program['subversion']
extra_info = program['extra_info']
send_mail = False
if extra_info:
extra_info_dict = json.loads(extra_info)
for key in extra_info_dict:
if len(extra_info_dict[key]) != 0:
send_mail = True
break
if send_mail:
if not subversion:
subversion_str = "main"
else:
subversion_str = subversion
sql = mysql.Mysql()
df = sql.mysql_read(get_owner_info_from_sid.format(sid=sid))
if len(df) != 0:
program_api, owner_name, mail = (df.loc[0, 'name'], df.loc[0, 'owner'], df.loc[0, 'mail'])
msg = "{program_api}[{version}-{subversion}]:{info}".\
format(program_api=program_api, version=version, info=extra_info, subversion=subversion_str)
create_mail(msg, str(mail), to=owner_name,
subject='程序{program_api}执行成功【告警】信息'.format(program_api=program_api))
else:
print("找不到邮件发送人！")
"""质控流程
# quality_result = ControlQualityResult(sid, version, subversion)
# quality_result.run()
"""
if subversion:
pass
"""重跑流程"""
else:
SetProgramsCalculateInfo(sid, version).program_running_success()
day = timedelta(days=1)
new_version = (version + day)
self._update_version(sid, new_version)
if sid in self.program_rerun_dict:
if version in self.program_rerun_dict[sid]:
del(self.program_rerun_dict[sid][version])
def _program_start(self):
try:
sid = int(self.maps['data']["sid"])
event_time = self.maps['data']["time"]
name = self.maps['data']["program"]
version = self.maps['data']['version']
subversion = self.maps['data']["subversion"]
event_type = 100
sql_str = program_result_log.format(version=version, event_time=event_time, sid=sid,
event_type=event_type, extra_info="", subversion=subversion)
print(sql_str)
sql = mysql.Mysql()
sql.mysql_write(sql_str)
log_string = '程序编号: %s 程序名称: %s 程序执行开始时间: %s' % (str(sid), str(name), str(event_time))
write_log(log_string)
response = "ok"
self.request.sendall(response.encode('utf-8'))
except Exception as e:
print(str(e))
def loop_check_program_result(self):
start_id, end_id = self._get_max_flag()
if start_id != end_id and start_id != 0 and end_id != 0:
execute_calucate_programs = self._get_execute_calculate_programs(start_id, end_id)
execute_crawling_programs = self._get_execute_crawling_programs(start_id, end_id)
# print("execute_success_programs:", execute_success_programs)
# print("execute_failed_programs:", execute_failed_programs)
threads = []
if len(execute_crawling_programs) != 0:
threads.append(threading.Thread(target=self.crawling_procedure, args=(execute_crawling_programs, )))
if len(execute_calucate_programs) != 0:
threads.append(threading.Thread(target=self.failed_procedure, args=(execute_calucate_programs,)))
if threads:
for thread in threads:
thread.start()
for thread in threads:
thread.join()
try:
if self.program_rerun_dict:
rerun_dict = {}
v_dict = {}
print("self.program_rerun_dict:", self.program_rerun_dict)
for k, v in self.program_rerun_dict.items():
if len(v) != 0:
for k1, v1 in v.items():
v_dict[str(k1)] = v1
rerun_dict[str(k)] = v_dict
rerun_dict = json.dumps(rerun_dict)
print("rerun_dict:", rerun_dict)
r.set('crawling_program_rerun_dict', rerun_dict)
except Exception as e:
print("loop_check_program_result.loop_check_program_result Err: "
"[program_rerun_dict] {err}".format(err=str(e)))
finally:
sql = mysql.Mysql()
sql.mysql_write(update_flag.format(id=end_id))
def sql_write(sql_str):
sql = mysql.Mysql(mysql_host, mysql_user, mysql_pwd, db_name)
sql.mysql_write(sql_str)