Exemplo n.º 1
0
def model_train_done_rnn(model_id, lag_dates, pcas, file_path_list,
                         sub_model_names, outputs_list, events_set, model_dir):
    """
    更新t_event_model_file、t_event_model_detail、t_event_model_tran表的数据。

    :param model_id: string. 模型编号
    :param file_path_list: array. 模型地址
    """
    db = DatabaseWrapper(dbname=event_dbname)
    try:
        db.begin_transaction()

        sql = "INSERT INTO t_event_model_file(file_id, file_url, model_id) VALUES (%s, %s, %s) "
        params = []
        for model_fp in file_path_list:
            param = (UuidHelper.guid(), model_fp, model_id)
            params.append(param)
        db.executemany(sql, params)

        # 子模型信息入库
        detail_ids = []
        sql = "INSERT INTO t_event_model_detail(detail_id, model_name, status, model_id, lag_date, pca, create_date, " \
              "create_time) values (%s, %s, %s, %s, %s, %s, %s, %s)"
        params = []
        for sub_model_name, lag_date, pca in zip(sub_model_names, lag_dates,
                                                 pcas):
            detail_id = UuidHelper.guid()
            detail_ids.append(detail_id)
            params.append(
                (detail_id, sub_model_name, DataStatus.SUCCESS.value, model_id,
                 int(lag_date), int(pca), sys_date(sys_date_formatter),
                 sys_time(sys_time_formatter)))
        db.executemany(sql, params)

        # 分事件模型信息入库
        sql = "INSERT INTO t_event_model_tran(tran_id, event_name, num, detail_id, status, create_date, create_time) " \
              "values (%s, %s, %s, %s, %s, %s, %s)"
        params = []
        for detail_id, outputs in zip(detail_ids, outputs_list):
            events_num = pp.get_event_num(outputs, events_set)
            for e in events_set:
                tran_id = UuidHelper.guid()
                event_num = events_num[e]
                param = (tran_id, e, event_num,
                         detail_id, DataStatus.SUCCESS.value,
                         sys_date(sys_date_formatter),
                         sys_time(sys_time_formatter))
                params.append(param)
        db.executemany(sql, params)

        sql = "UPDATE t_event_model SET model_dir = %s WHERE model_id = %s"
        db.execute(sql, (model_dir, model_id))

        db.commit()
        return detail_ids
    except Exception as e:
        db.rollback()
        raise RuntimeError(e)
    finally:
        db.close()
Exemplo n.º 2
0
def insert_pred_result(probs, probs_all_days, dates, dates_pred_all,
                       dates_data, detail_ids, events_set, task_id):
    """向数据库插入预测结果
    Args:
      probs: 预测结果
      probs_all_days: 多天预测结果
      dates:
      dates_pred_all: 多天预测日期
      dates_data: 数据日期
      detail_ids:
      events_set:
      task_id:
    """
    sql_task_rs = "insert into t_event_task_rs(rs_id, event_name, probability, forecast_date, status, " \
                  "detail_id, task_id, create_date, create_time, predict_end_date) " \
                  "values(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
    sql_task_rs_his = "insert into t_event_task_rs_his(rs_id, event_name, probability, forecast_date," \
                      "detail_id, task_id, create_date, create_time, predict_end_date) values " \
                      "(%s, %s, %s, %s, %s, %s, %s, %s, %s)"
    sql_task_rs_params = []
    sql_task_rs_his_params_hist = []
    db = DatabaseWrapper(dbname=event_dbname)
    try:
        db.begin_transaction()
        for p, pa, ds, da, did, dd in zip(probs, probs_all_days, dates,
                                          dates_pred_all, detail_ids,
                                          dates_data):
            for i, e in enumerate(events_set):
                for j, d in enumerate(ds):
                    param = (UuidHelper.guid(), str(e), f'{p[j][i]:.4f}',
                             str(d), DataStatus.SUCCESS.value, did, task_id,
                             sys_date(sys_date_formatter),
                             sys_time(sys_time_formatter), str(dd[j]))
                    sql_task_rs_params.append(param)
                for j, d in enumerate(da):
                    pd = pa[j]
                    for k, d_ in enumerate(d):
                        pd_ = pd[k]
                        param_hist = (UuidHelper.guid(), str(e),
                                      f'{pd_[i]:.4f}', str(d_), did, task_id,
                                      sys_date(sys_date_formatter),
                                      sys_time(sys_time_formatter), str(dd[j]))
                        sql_task_rs_his_params_hist.append(param_hist)
            db.executemany(sql_task_rs, sql_task_rs_params)
            sql_task_rs_params = []
            db.executemany(sql_task_rs_his, sql_task_rs_his_params_hist)
            sql_task_rs_his_params_hist = []
        db.commit()  # 配置文件中设置不自动提交,所以手动提交
    except Exception as e:
        db.rollback()
        raise RuntimeError(e)
    finally:
        db.close()
Exemplo n.º 3
0
def insert_model_test(event, event_num, false_rate, recall_rate,
                      false_alarm_rate, tier_precision, tier_recall, bleu,
                      detail_id):
    """
    记录模型评估信息
    Args:
        event: 事件名称
        event_num: 事件在测试(评估)数据集中出现的次数
        false_rate: 误报率
        recall_rate: 召回率
        false_alarm_rate: 虚警率
        tier_precision: 头部精确率, 即将预测值降序排序后, 头部(前n个)预测值中预测正确的结果数所占的比例
        tier_recall: 头部召回率, 即将预测值降序排序后, 头部(前n个)预测值中预测正确的结果数与真实正例个数的比值
        bleu: bleu指标
        detail_id: 子模型对应的id
    """
    test_id = UuidHelper.guid()
    sql = "insert into t_event_model_test(test_id, event_name, num, false_rate, recall_rate, false_alarm_rate, " \
          "tier_precision, tier_recall, bleu, status, detail_id, create_date, create_time) " \
          "values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
    param = (test_id, event, event_num, false_rate, recall_rate,
             false_alarm_rate, tier_precision, tier_recall, bleu,
             DataStatus.SUCCESS.value, detail_id, sys_date(sys_date_formatter),
             sys_time(sys_time_formatter))

    __modify(sql, param, '插入子模型评估结果失败')
Exemplo n.º 4
0
def model_eval_done(top_scores, events_num):
    """
    更新t_event_model表的模型状态、训练开始日期、训练结束日期。

    :param top_scores:
    :param events_num:
    """
    db = DatabaseWrapper(dbname=event_dbname)
    try:
        sql = "insert into t_event_model_tot(tot_id, num, false_rate, recall_rate, false_alarm_rate, " \
              "tier_precision, tier_recall, bleu, score, status, detail_id, create_date, create_time) values " \
              "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
        params = []
        for score, bleu_summary, tier_precision_summary, tier_recall_summary, fr_summary, rc_summary, \
                fa_summary, detail_id in top_scores:
            num_events = np.sum(
                [v for k, v in events_num.items() if str(k) != '0'])
            param = (UuidHelper.guid(), str(num_events), str(fr_summary),
                     str(rc_summary), str(fa_summary),
                     str(tier_precision_summary), str(tier_recall_summary),
                     str(bleu_summary), str(score), DataStatus.SUCCESS.value,
                     detail_id, sys_date(sys_date_formatter),
                     sys_time(sys_time_formatter))
            params.append(param)
        db.executemany(sql, params)
        db.commit()
    except Exception as e:
        raise RuntimeError(e)
    finally:
        db.close()
Exemplo n.º 5
0
def model_train_finish(model_id, status):
    """
    更新t_event_model表的模型状态。

    :param model_id: string. 模型编号
    :param status: string. 模型状态
    """
    sql = "UPDATE t_event_model SET status = %s, tran_finish_date = %s, tran_finish_time = %s WHERE model_id = %s"

    __modify(sql, (status, sys_date(sys_date_formatter),
                   sys_time(sys_time_formatter), model_id))
Exemplo n.º 6
0
def predict_task_finish(task_id, date_data_pred, status):
    """
    修改t_event_task表的运行状态、任务结束日期、预测结束日期、任务结束事件。
    :param task_id: string. 任务编号
    :param date_data_pred: string. 预测结束日期
    :param status: string. 任务运行状态
    """
    sql = "UPDATE t_event_task SET status = %s, task_finish_date = %s, predict_end_date = %s, task_finish_time = %s " \
          "WHERE task_id = %s"
    param = (status, sys_date(sys_date_formatter), str(date_data_pred),
             sys_time(sys_time_formatter), task_id)
    __modify(sql, param)
Exemplo n.º 7
0
def insert_into_model_detail(sub_model_names, model_id):
    """
    将子模型信息入库
    Args:
      sub_model_names: 子模型名称, 形如 model_name-input_len-output_len-n_pca
      model_id: 模型编号

    Returns:
        array,detail_id
    """
    detail_ids = []
    sql = "INSERT INTO t_event_model_detail(detail_id, model_name, status, model_id, create_date, create_time) " \
          "values (%s, %s, %s, %s, %s, %s)"
    params = []
    for sub_model_name in sub_model_names:
        detail_id = UuidHelper.guid()
        detail_ids.append(detail_id)
        params.append(
            (detail_id, sub_model_name, DataStatus.SUCCESS.value, model_id,
             sys_date(sys_date_formatter), sys_time(sys_time_formatter)))

    __modify_many(sql, params)

    return detail_ids
Exemplo n.º 8
0
def model_train_done_cnn(model_id, kernel_size_array, pool_size_array,
                         lag_date_array, file_path_array, sub_model_name_array,
                         output_array, event_set_array, model_dir):
    """
    更新t_event_model_file、t_event_model_detail、t_event_model_tran表的数据。

    :param model_id: string. 模型编号
    :param kernel_size_array: array. 卷积核列表
    :param pool_size_array: array. 过滤器列表
    :param lag_date_array: array. 滞后期列表
    :param file_path_array: array. 模型文件地址列表
    :param sub_model_name_array: array. 模型文件名列表
    :param output_array: array.事件样本列表(事件表数据)
    :param event_set_array: array.事件类别列表
    :param model_dir: str.模型存放地址
    :return array, 子模型编号列表
    """
    db = DatabaseWrapper(dbname=event_dbname)
    try:
        db.begin_transaction()

        sql = "INSERT INTO t_event_model_file(file_id, file_url, model_id) VALUES (%s, %s, %s) "
        params = []
        for model_fp in file_path_array:
            param = (UuidHelper.guid(), model_fp, model_id)
            params.append(param)
        db.executemany(sql, params)

        # 子模型信息入库
        detail_ids = []
        sql = "INSERT INTO t_event_model_detail(detail_id, model_name, status, model_id, lag_date, kernel_size, " \
              "pool_size, create_date, create_time) values (%s, %s, %s, %s, %s, %s, %s, %s, %s)"
        params = []
        for sub_model_name, lag_date, kernel_size, pool_size in zip(
                sub_model_name_array, lag_date_array, kernel_size_array,
                pool_size_array):
            detail_id = UuidHelper.guid()
            detail_ids.append(detail_id)
            params.append(
                (detail_id, sub_model_name, DataStatus.SUCCESS.value, model_id,
                 int(lag_date), int(kernel_size), int(pool_size),
                 sys_date(sys_date_formatter), sys_time(sys_time_formatter)))
        db.executemany(sql, params)

        # 分事件模型信息入库
        sql = "INSERT INTO t_event_model_tran(tran_id, event_name, num, detail_id, status, create_date, create_time) " \
              "values (%s, %s, %s, %s, %s, %s, %s)"
        params = []
        for detail_id, outputs in zip(detail_ids, output_array):
            events_num = pp.get_event_num(outputs, event_set_array)
            for event in event_set_array:
                tran_id = UuidHelper.guid()
                event_num = events_num[event]
                param = (tran_id, event, event_num,
                         detail_id, DataStatus.SUCCESS.value,
                         sys_date(sys_date_formatter),
                         sys_time(sys_time_formatter))
                params.append(param)
        db.executemany(sql, params)

        sql = "UPDATE t_event_model SET model_dir = %s WHERE model_id = %s"
        db.execute(sql, (model_dir, model_id))

        db.commit()
        return detail_ids
    except Exception as e:
        db.rollback()
        raise RuntimeError(e)
    finally:
        db.close()