Пример #1
0
def eval_f(_net, test_data, ctx=mx.cpu()):
    from sklearn.metrics import precision_recall_fscore_support, accuracy_score

    ground_truth = []
    prediction = []

    def evaluation_function(y_true, y_pred):
        evaluation_result = {}
        precsion, recall, f1, _ = precision_recall_fscore_support(
            y_true, y_pred)
        evaluation_result.update(
            {"precision_%d" % i: precsion[i]
             for i in range(len(precsion))})
        evaluation_result.update(
            {"recall_%d" % i: recall[i]
             for i in range(len(recall))})
        evaluation_result.update({"f1_%d" % i: f1[i] for i in range(len(f1))})
        evaluation_result.update({"Accuracy": accuracy_score(y_true, y_pred)})
        return evaluation_result

    for batch_data in tqdm(test_data, "evaluating"):
        ctx_data = split_and_load(ctx, *batch_data, even_split=False)
        for (word, word_radical, char, char_radical, word_mask, char_mask,
             label) in ctx_data:
            output = _net(word, word_radical, char, char_radical, word_mask,
                          char_mask)
            pred = mx.nd.argmax(output, axis=1)
            ground_truth.extend(label.asnumpy().tolist())
            prediction.extend(pred.asnumpy().tolist())

    return evaluation_function(ground_truth, prediction)
Пример #2
0
def eval_f(_net, test_data, ctx=mx.cpu()):
    ground_truth = []
    prediction = []
    pred_labels = []

    for batch_data in tqdm(test_data, "evaluating"):
        ctx_data = split_and_load(ctx, *batch_data, even_split=False)
        for (keys, values, data_mask, label, label_mask) in ctx_data:
            output, _ = _net(keys, values, mask=data_mask)
            pred = output.asnumpy().tolist()
            label = label.asnumpy().tolist()
            for i, length in enumerate(label_mask.asnumpy().tolist()):
                length = int(length)
                ground_truth.extend(label[i][:length])
                prediction.extend(pred[i][:length])
                pred_labels.extend(
                    [0 if p < 0.5 else 1 for p in pred[i][:length]])

    auc = roc_auc_score(ground_truth, prediction)
    precision, recall, f1, _ = precision_recall_fscore_support(
        ground_truth, pred_labels)

    evaluation_result = {}

    evaluation_result.update(
        {"precision_%d" % i: precision[i]
         for i in range(len(precision))})
    evaluation_result.update(
        {"recall_%d" % i: recall[i]
         for i in range(len(recall))})
    evaluation_result.update({"f1_%d" % i: f1[i] for i in range(len(f1))})

    evaluation_result.update({"auc": auc})
    return evaluation_result
Пример #3
0
def eval_f(_net, test_data, ctx=mx.cpu()):
    k = test_data[1]["k"]
    k = as_list(k) if k is not None else []
    max_k = max(k) if k else None
    top_k_ground_truth = []
    top_k_prediction = []
    ground_truth = []
    prediction = []

    for batch_data in tqdm(test_data[0], "evaluating"):
        ctx_data = split_and_load(ctx, *batch_data, even_split=False)
        for (user, item, label) in ctx_data:
            output = _net(user, item)
            pred = output
            label = label.asnumpy().astype("int")
            pred = pred.asnumpy()
            ground_truth.append(label.tolist())
            prediction.append(pred.tolist())
            if max_k:
                top_k_indices = np.argsort(pred)[::-1]
                _top_k_indices = top_k_indices[:max_k]
                padding = [0] * (max_k - len(_top_k_indices)) if len(
                    _top_k_indices) < max_k else []
                top_k_prediction.append(pred[_top_k_indices].tolist() +
                                        padding)
                top_k_ground_truth.append(label[_top_k_indices].tolist() +
                                          padding)

    chained_ground_truth = list(chain(*ground_truth))
    chained_prediction = list(chain(*prediction))
    metrics = {
        "rmse": mean_squared_error(chained_ground_truth, chained_prediction),
        "mae": median_absolute_error(chained_ground_truth, chained_prediction),
    }

    metrics.update(
        classification_report(
            chained_ground_truth,
            [0 if v < 0.5 else 1
             for v in chained_prediction], chained_prediction))

    if k:
        metrics_k = {"ndcg": {}, "HR": {}}
        for _k in k:
            metrics_k["ndcg"][_k] = ndcg_score(top_k_ground_truth,
                                               top_k_prediction,
                                               k=_k)
            metrics_k["HR"][_k] = _hit_rate(top_k_ground_truth, k=_k)
        metrics.update(metrics_k)
    return metrics
Пример #4
0
def eval_f(_net, test_data, ctx=mx.cpu()):
    ground_truth = []
    prediction = []

    def evaluation_function(y_true, y_pred):
        return {"evaluation_name": 0}

    for batch_data in tqdm(test_data, "evaluating"):
        ctx_data = split_and_load(ctx, *batch_data, even_split=False)
        for (data, label) in ctx_data:
            output = _net(data)
            pred = output
            ground_truth.extend(label.asnumpy().tolist())
            prediction.extend(pred.asnumpy().tolist())

    return evaluation_function(ground_truth, prediction)
Пример #5
0
def fit_f(net,
          batch_size,
          batch_data,
          trainer,
          loss_function,
          loss_monitor=None,
          ctx=mx.cpu(),
          fit_step_func=_fit_f):
    """
    Defined how each step of batch train goes

    Parameters
    ----------
    net: HybridBlock
        The network which has been initialized
        or loaded from the existed model
    batch_size: int
            The size of each batch
    batch_data: Iterable
        The batch data for train
    trainer:
        The trainer used to update the parameters of the net
    loss_function: dict of function
        The functions to compute the loss for the procession
        of back propagation
    loss_monitor: LossMonitor
        Default to ``None``
    ctx: Context or list of Context
        Defaults to ``mx.cpu()``.
    fit_step_func: function

    Returns
    -------

    """
    ctx_data = split_and_load(ctx, *batch_data, even_split=False)

    with autograd.record():
        for _data in ctx_data:
            bp_loss = fit_step_func(net, _data, loss_function, loss_monitor)
            bp_loss.backward()

    # todo: confirm whether the train step is equal to batch_size
    if batch_size is not None:
        trainer.step(batch_size)
    else:
        trainer.step(len(batch_data[0]))
Пример #6
0
def fit_f(net,
          batch_size,
          batch_data,
          trainer,
          bp_loss_f,
          loss_function,
          loss_monitor=None,
          ctx=mx.cpu()):
    """
    Defined how each step of batch train goes

    Parameters
    ----------
    net: HybridBlock
        The network which has been initialized
        or loaded from the existed model
    batch_size: int
            The size of each batch
    batch_data: Iterable
        The batch data for train
    trainer:
        The trainer used to update the parameters of the net
    bp_loss_f: dict with only one value and one key
        The function to compute the loss for the procession
        of back propagation
    loss_function: dict of function
        Some other measurement in addition to bp_loss_f
    loss_monitor: LossMonitor
        Default to ``None``
    ctx: Context or list of Context
        Defaults to ``mx.cpu()``.

    Returns
    -------

    """
    # 此函数定义训练过程
    ctx_data = split_and_load(ctx, *batch_data, even_split=False)

    with autograd.record():
        for _data in ctx_data:
            bp_loss = _fit_f(net, _data, bp_loss_f, loss_function,
                             loss_monitor)
            assert bp_loss is not None
            bp_loss.backward()
    trainer.step(1)
Пример #7
0
def eval_f(_net, test_data, ctx=mx.cpu()):
    ground_truth = []
    prediction = []

    for batch_data in tqdm(test_data, "evaluating"):
        ctx_data = split_and_load(
            ctx, *batch_data,
            even_split=False
        )
        for (user, item, score) in ctx_data:
            output = _net(user, item)
            pred = output
            ground_truth.extend(score.asnumpy().tolist())
            prediction.extend(pred.asnumpy().tolist())

    return classification_report(
        ground_truth,
        y_pred=[0 if p < 0.5 else 1 for p in prediction],
        y_score=prediction
    )
Пример #8
0
def eval_f(_net, test_data, ctx=mx.cpu()):
    ground_truth = []
    prediction = []
    pred_labels = []

    for batch_data in tqdm(test_data, "evaluating"):
        ctx_data = split_and_load(ctx, *batch_data, even_split=False)
        for (data, data_mask, label, pick_index, label_mask) in ctx_data:
            output, _ = _net(data, data_mask)
            output = mx.nd.slice(output, (None, None), (None, -1))
            output = mx.nd.pick(output, pick_index)
            pred = output.asnumpy().tolist()
            label = label.asnumpy().tolist()
            for i, length in enumerate(label_mask.asnumpy().tolist()):
                length = int(length)
                ground_truth.extend(label[i][:length])
                prediction.extend(pred[i][:length])
                pred_labels.extend(
                    [0 if p < 0.5 else 1 for p in pred[i][:length]])

    return classification_report(ground_truth,
                                 y_pred=pred_labels,
                                 y_score=prediction)
Пример #9
0
 def split_and_load(ctx, *args, **kwargs):
     return split_and_load(ctx, *args, **kwargs)