def model_train_done_rnn(model_id, lag_dates, pcas, file_path_list,
sub_model_names, outputs_list, events_set, model_dir):
"""
更新t_event_model_file、t_event_model_detail、t_event_model_tran表的数据。
:param model_id: string. 模型编号
:param file_path_list: array. 模型地址
"""
db = DatabaseWrapper(dbname=event_dbname)
try:
db.begin_transaction()
sql = "INSERT INTO t_event_model_file(file_id, file_url, model_id) VALUES (%s, %s, %s) "
params = []
for model_fp in file_path_list:
param = (UuidHelper.guid(), model_fp, model_id)
params.append(param)
db.executemany(sql, params)
# 子模型信息入库
detail_ids = []
sql = "INSERT INTO t_event_model_detail(detail_id, model_name, status, model_id, lag_date, pca, create_date, " \
"create_time) values (%s, %s, %s, %s, %s, %s, %s, %s)"
params = []
for sub_model_name, lag_date, pca in zip(sub_model_names, lag_dates,
pcas):
detail_id = UuidHelper.guid()
detail_ids.append(detail_id)
params.append(
(detail_id, sub_model_name, DataStatus.SUCCESS.value, model_id,
int(lag_date), int(pca), sys_date(sys_date_formatter),
sys_time(sys_time_formatter)))
db.executemany(sql, params)
# 分事件模型信息入库
sql = "INSERT INTO t_event_model_tran(tran_id, event_name, num, detail_id, status, create_date, create_time) " \
"values (%s, %s, %s, %s, %s, %s, %s)"
params = []
for detail_id, outputs in zip(detail_ids, outputs_list):
events_num = pp.get_event_num(outputs, events_set)
for e in events_set:
tran_id = UuidHelper.guid()
event_num = events_num[e]
param = (tran_id, e, event_num,
detail_id, DataStatus.SUCCESS.value,
sys_date(sys_date_formatter),
sys_time(sys_time_formatter))
params.append(param)
db.executemany(sql, params)
sql = "UPDATE t_event_model SET model_dir = %s WHERE model_id = %s"
db.execute(sql, (model_dir, model_id))
db.commit()
return detail_ids
except Exception as e:
db.rollback()
raise RuntimeError(e)
finally:
db.close()
def insert_pred_result(probs, probs_all_days, dates, dates_pred_all,
dates_data, detail_ids, events_set, task_id):
"""向数据库插入预测结果
Args:
probs: 预测结果
probs_all_days: 多天预测结果
dates:
dates_pred_all: 多天预测日期
dates_data: 数据日期
detail_ids:
events_set:
task_id:
"""
sql_task_rs = "insert into t_event_task_rs(rs_id, event_name, probability, forecast_date, status, " \
"detail_id, task_id, create_date, create_time, predict_end_date) " \
"values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
sql_task_rs_his = "insert into t_event_task_rs_his(rs_id, event_name, probability, forecast_date," \
"detail_id, task_id, create_date, create_time, predict_end_date) values " \
"(%s, %s, %s, %s, %s, %s, %s, %s, %s)"
sql_task_rs_params = []
sql_task_rs_his_params_hist = []
db = DatabaseWrapper(dbname=event_dbname)
try:
db.begin_transaction()
for p, pa, ds, da, did, dd in zip(probs, probs_all_days, dates,
dates_pred_all, detail_ids,
dates_data):
for i, e in enumerate(events_set):
for j, d in enumerate(ds):
param = (UuidHelper.guid(), str(e), f'{p[j][i]:.4f}',
str(d), DataStatus.SUCCESS.value, did, task_id,
sys_date(sys_date_formatter),
sys_time(sys_time_formatter), str(dd[j]))
sql_task_rs_params.append(param)
for j, d in enumerate(da):
pd = pa[j]
for k, d_ in enumerate(d):
pd_ = pd[k]
param_hist = (UuidHelper.guid(), str(e),
f'{pd_[i]:.4f}', str(d_), did, task_id,
sys_date(sys_date_formatter),
sys_time(sys_time_formatter), str(dd[j]))
sql_task_rs_his_params_hist.append(param_hist)
db.executemany(sql_task_rs, sql_task_rs_params)
sql_task_rs_params = []
db.executemany(sql_task_rs_his, sql_task_rs_his_params_hist)
sql_task_rs_his_params_hist = []
db.commit() # 配置文件中设置不自动提交，所以手动提交
except Exception as e:
db.rollback()
raise RuntimeError(e)
finally:
db.close()
def insert_model_test(event, event_num, false_rate, recall_rate,
false_alarm_rate, tier_precision, tier_recall, bleu,
detail_id):
"""
记录模型评估信息
Args:
event: 事件名称
event_num: 事件在测试(评估)数据集中出现的次数
false_rate: 误报率
recall_rate: 召回率
false_alarm_rate: 虚警率
tier_precision: 头部精确率, 即将预测值降序排序后, 头部(前n个)预测值中预测正确的结果数所占的比例
tier_recall: 头部召回率, 即将预测值降序排序后, 头部(前n个)预测值中预测正确的结果数与真实正例个数的比值
bleu: bleu指标
detail_id: 子模型对应的id
"""
test_id = UuidHelper.guid()
sql = "insert into t_event_model_test(test_id, event_name, num, false_rate, recall_rate, false_alarm_rate, " \
"tier_precision, tier_recall, bleu, status, detail_id, create_date, create_time) " \
"values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
param = (test_id, event, event_num, false_rate, recall_rate,
false_alarm_rate, tier_precision, tier_recall, bleu,
DataStatus.SUCCESS.value, detail_id, sys_date(sys_date_formatter),
sys_time(sys_time_formatter))
__modify(sql, param, '插入子模型评估结果失败')
def model_eval_done(top_scores, events_num):
"""
更新t_event_model表的模型状态、训练开始日期、训练结束日期。
:param top_scores:
:param events_num:
"""
db = DatabaseWrapper(dbname=event_dbname)
try:
sql = "insert into t_event_model_tot(tot_id, num, false_rate, recall_rate, false_alarm_rate, " \
"tier_precision, tier_recall, bleu, score, status, detail_id, create_date, create_time) values " \
"(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
params = []
for score, bleu_summary, tier_precision_summary, tier_recall_summary, fr_summary, rc_summary, \
fa_summary, detail_id in top_scores:
num_events = np.sum(
[v for k, v in events_num.items() if str(k) != '0'])
param = (UuidHelper.guid(), str(num_events), str(fr_summary),
str(rc_summary), str(fa_summary),
str(tier_precision_summary), str(tier_recall_summary),
str(bleu_summary), str(score), DataStatus.SUCCESS.value,
detail_id, sys_date(sys_date_formatter),
sys_time(sys_time_formatter))
params.append(param)
db.executemany(sql, params)
db.commit()
except Exception as e:
raise RuntimeError(e)
finally:
db.close()
def model_train_finish(model_id, status):
"""
更新t_event_model表的模型状态。
:param model_id: string. 模型编号
:param status: string. 模型状态
"""
sql = "UPDATE t_event_model SET status = %s, tran_finish_date = %s, tran_finish_time = %s WHERE model_id = %s"
__modify(sql, (status, sys_date(sys_date_formatter),
sys_time(sys_time_formatter), model_id))
def predict_task_finish(task_id, date_data_pred, status):
"""
修改t_event_task表的运行状态、任务结束日期、预测结束日期、任务结束事件。
:param task_id: string. 任务编号
:param date_data_pred: string. 预测结束日期
:param status: string. 任务运行状态
"""
sql = "UPDATE t_event_task SET status = %s, task_finish_date = %s, predict_end_date = %s, task_finish_time = %s " \
"WHERE task_id = %s"
param = (status, sys_date(sys_date_formatter), str(date_data_pred),
sys_time(sys_time_formatter), task_id)
__modify(sql, param)
def insert_into_model_detail(sub_model_names, model_id):
"""
将子模型信息入库
Args:
sub_model_names: 子模型名称, 形如 model_name-input_len-output_len-n_pca
model_id: 模型编号
Returns:
array，detail_id
"""
detail_ids = []
sql = "INSERT INTO t_event_model_detail(detail_id, model_name, status, model_id, create_date, create_time) " \
"values (%s, %s, %s, %s, %s, %s)"
params = []
for sub_model_name in sub_model_names:
detail_id = UuidHelper.guid()
detail_ids.append(detail_id)
params.append(
(detail_id, sub_model_name, DataStatus.SUCCESS.value, model_id,
sys_date(sys_date_formatter), sys_time(sys_time_formatter)))
__modify_many(sql, params)
return detail_ids
def model_train_done_cnn(model_id, kernel_size_array, pool_size_array,
lag_date_array, file_path_array, sub_model_name_array,
output_array, event_set_array, model_dir):
"""
更新t_event_model_file、t_event_model_detail、t_event_model_tran表的数据。
:param model_id: string. 模型编号
:param kernel_size_array: array. 卷积核列表
:param pool_size_array: array. 过滤器列表
:param lag_date_array: array. 滞后期列表
:param file_path_array: array. 模型文件地址列表
:param sub_model_name_array: array. 模型文件名列表
:param output_array: array.事件样本列表（事件表数据）
:param event_set_array: array.事件类别列表
:param model_dir: str.模型存放地址
:return array, 子模型编号列表
"""
db = DatabaseWrapper(dbname=event_dbname)
try:
db.begin_transaction()
sql = "INSERT INTO t_event_model_file(file_id, file_url, model_id) VALUES (%s, %s, %s) "
params = []
for model_fp in file_path_array:
param = (UuidHelper.guid(), model_fp, model_id)
params.append(param)
db.executemany(sql, params)
# 子模型信息入库
detail_ids = []
sql = "INSERT INTO t_event_model_detail(detail_id, model_name, status, model_id, lag_date, kernel_size, " \
"pool_size, create_date, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s)"
params = []
for sub_model_name, lag_date, kernel_size, pool_size in zip(
sub_model_name_array, lag_date_array, kernel_size_array,
pool_size_array):
detail_id = UuidHelper.guid()
detail_ids.append(detail_id)
params.append(
(detail_id, sub_model_name, DataStatus.SUCCESS.value, model_id,
int(lag_date), int(kernel_size), int(pool_size),
sys_date(sys_date_formatter), sys_time(sys_time_formatter)))
db.executemany(sql, params)
# 分事件模型信息入库
sql = "INSERT INTO t_event_model_tran(tran_id, event_name, num, detail_id, status, create_date, create_time) " \
"values (%s, %s, %s, %s, %s, %s, %s)"
params = []
for detail_id, outputs in zip(detail_ids, output_array):
events_num = pp.get_event_num(outputs, event_set_array)
for event in event_set_array:
tran_id = UuidHelper.guid()
event_num = events_num[event]
param = (tran_id, event, event_num,
detail_id, DataStatus.SUCCESS.value,
sys_date(sys_date_formatter),
sys_time(sys_time_formatter))
params.append(param)
db.executemany(sql, params)
sql = "UPDATE t_event_model SET model_dir = %s WHERE model_id = %s"
db.execute(sql, (model_dir, model_id))
db.commit()
return detail_ids
except Exception as e:
db.rollback()
raise RuntimeError(e)
finally:
db.close()