def eval_f(_net, test_data, ctx=mx.cpu()): from sklearn.metrics import precision_recall_fscore_support, accuracy_score ground_truth = [] prediction = [] def evaluation_function(y_true, y_pred): evaluation_result = {} precsion, recall, f1, _ = precision_recall_fscore_support( y_true, y_pred) evaluation_result.update( {"precision_%d" % i: precsion[i] for i in range(len(precsion))}) evaluation_result.update( {"recall_%d" % i: recall[i] for i in range(len(recall))}) evaluation_result.update({"f1_%d" % i: f1[i] for i in range(len(f1))}) evaluation_result.update({"Accuracy": accuracy_score(y_true, y_pred)}) return evaluation_result for batch_data in tqdm(test_data, "evaluating"): ctx_data = split_and_load(ctx, *batch_data, even_split=False) for (word, word_radical, char, char_radical, word_mask, char_mask, label) in ctx_data: output = _net(word, word_radical, char, char_radical, word_mask, char_mask) pred = mx.nd.argmax(output, axis=1) ground_truth.extend(label.asnumpy().tolist()) prediction.extend(pred.asnumpy().tolist()) return evaluation_function(ground_truth, prediction)
def eval_f(_net, test_data, ctx=mx.cpu()): ground_truth = [] prediction = [] pred_labels = [] for batch_data in tqdm(test_data, "evaluating"): ctx_data = split_and_load(ctx, *batch_data, even_split=False) for (keys, values, data_mask, label, label_mask) in ctx_data: output, _ = _net(keys, values, mask=data_mask) pred = output.asnumpy().tolist() label = label.asnumpy().tolist() for i, length in enumerate(label_mask.asnumpy().tolist()): length = int(length) ground_truth.extend(label[i][:length]) prediction.extend(pred[i][:length]) pred_labels.extend( [0 if p < 0.5 else 1 for p in pred[i][:length]]) auc = roc_auc_score(ground_truth, prediction) precision, recall, f1, _ = precision_recall_fscore_support( ground_truth, pred_labels) evaluation_result = {} evaluation_result.update( {"precision_%d" % i: precision[i] for i in range(len(precision))}) evaluation_result.update( {"recall_%d" % i: recall[i] for i in range(len(recall))}) evaluation_result.update({"f1_%d" % i: f1[i] for i in range(len(f1))}) evaluation_result.update({"auc": auc}) return evaluation_result
def eval_f(_net, test_data, ctx=mx.cpu()): k = test_data[1]["k"] k = as_list(k) if k is not None else [] max_k = max(k) if k else None top_k_ground_truth = [] top_k_prediction = [] ground_truth = [] prediction = [] for batch_data in tqdm(test_data[0], "evaluating"): ctx_data = split_and_load(ctx, *batch_data, even_split=False) for (user, item, label) in ctx_data: output = _net(user, item) pred = output label = label.asnumpy().astype("int") pred = pred.asnumpy() ground_truth.append(label.tolist()) prediction.append(pred.tolist()) if max_k: top_k_indices = np.argsort(pred)[::-1] _top_k_indices = top_k_indices[:max_k] padding = [0] * (max_k - len(_top_k_indices)) if len( _top_k_indices) < max_k else [] top_k_prediction.append(pred[_top_k_indices].tolist() + padding) top_k_ground_truth.append(label[_top_k_indices].tolist() + padding) chained_ground_truth = list(chain(*ground_truth)) chained_prediction = list(chain(*prediction)) metrics = { "rmse": mean_squared_error(chained_ground_truth, chained_prediction), "mae": median_absolute_error(chained_ground_truth, chained_prediction), } metrics.update( classification_report( chained_ground_truth, [0 if v < 0.5 else 1 for v in chained_prediction], chained_prediction)) if k: metrics_k = {"ndcg": {}, "HR": {}} for _k in k: metrics_k["ndcg"][_k] = ndcg_score(top_k_ground_truth, top_k_prediction, k=_k) metrics_k["HR"][_k] = _hit_rate(top_k_ground_truth, k=_k) metrics.update(metrics_k) return metrics
def eval_f(_net, test_data, ctx=mx.cpu()): ground_truth = [] prediction = [] def evaluation_function(y_true, y_pred): return {"evaluation_name": 0} for batch_data in tqdm(test_data, "evaluating"): ctx_data = split_and_load(ctx, *batch_data, even_split=False) for (data, label) in ctx_data: output = _net(data) pred = output ground_truth.extend(label.asnumpy().tolist()) prediction.extend(pred.asnumpy().tolist()) return evaluation_function(ground_truth, prediction)
def fit_f(net, batch_size, batch_data, trainer, loss_function, loss_monitor=None, ctx=mx.cpu(), fit_step_func=_fit_f): """ Defined how each step of batch train goes Parameters ---------- net: HybridBlock The network which has been initialized or loaded from the existed model batch_size: int The size of each batch batch_data: Iterable The batch data for train trainer: The trainer used to update the parameters of the net loss_function: dict of function The functions to compute the loss for the procession of back propagation loss_monitor: LossMonitor Default to ``None`` ctx: Context or list of Context Defaults to ``mx.cpu()``. fit_step_func: function Returns ------- """ ctx_data = split_and_load(ctx, *batch_data, even_split=False) with autograd.record(): for _data in ctx_data: bp_loss = fit_step_func(net, _data, loss_function, loss_monitor) bp_loss.backward() # todo: confirm whether the train step is equal to batch_size if batch_size is not None: trainer.step(batch_size) else: trainer.step(len(batch_data[0]))
def fit_f(net, batch_size, batch_data, trainer, bp_loss_f, loss_function, loss_monitor=None, ctx=mx.cpu()): """ Defined how each step of batch train goes Parameters ---------- net: HybridBlock The network which has been initialized or loaded from the existed model batch_size: int The size of each batch batch_data: Iterable The batch data for train trainer: The trainer used to update the parameters of the net bp_loss_f: dict with only one value and one key The function to compute the loss for the procession of back propagation loss_function: dict of function Some other measurement in addition to bp_loss_f loss_monitor: LossMonitor Default to ``None`` ctx: Context or list of Context Defaults to ``mx.cpu()``. Returns ------- """ # 此函数定义训练过程 ctx_data = split_and_load(ctx, *batch_data, even_split=False) with autograd.record(): for _data in ctx_data: bp_loss = _fit_f(net, _data, bp_loss_f, loss_function, loss_monitor) assert bp_loss is not None bp_loss.backward() trainer.step(1)
def eval_f(_net, test_data, ctx=mx.cpu()): ground_truth = [] prediction = [] for batch_data in tqdm(test_data, "evaluating"): ctx_data = split_and_load( ctx, *batch_data, even_split=False ) for (user, item, score) in ctx_data: output = _net(user, item) pred = output ground_truth.extend(score.asnumpy().tolist()) prediction.extend(pred.asnumpy().tolist()) return classification_report( ground_truth, y_pred=[0 if p < 0.5 else 1 for p in prediction], y_score=prediction )
def eval_f(_net, test_data, ctx=mx.cpu()): ground_truth = [] prediction = [] pred_labels = [] for batch_data in tqdm(test_data, "evaluating"): ctx_data = split_and_load(ctx, *batch_data, even_split=False) for (data, data_mask, label, pick_index, label_mask) in ctx_data: output, _ = _net(data, data_mask) output = mx.nd.slice(output, (None, None), (None, -1)) output = mx.nd.pick(output, pick_index) pred = output.asnumpy().tolist() label = label.asnumpy().tolist() for i, length in enumerate(label_mask.asnumpy().tolist()): length = int(length) ground_truth.extend(label[i][:length]) prediction.extend(pred[i][:length]) pred_labels.extend( [0 if p < 0.5 else 1 for p in pred[i][:length]]) return classification_report(ground_truth, y_pred=pred_labels, y_score=prediction)
def split_and_load(ctx, *args, **kwargs): return split_and_load(ctx, *args, **kwargs)