def eval_f(_net, test_data, ctx=mx.cpu()):
from sklearn.metrics import precision_recall_fscore_support, accuracy_score
ground_truth = []
prediction = []
def evaluation_function(y_true, y_pred):
evaluation_result = {}
precsion, recall, f1, _ = precision_recall_fscore_support(
y_true, y_pred)
evaluation_result.update(
{"precision_%d" % i: precsion[i]
for i in range(len(precsion))})
evaluation_result.update(
{"recall_%d" % i: recall[i]
for i in range(len(recall))})
evaluation_result.update({"f1_%d" % i: f1[i] for i in range(len(f1))})
evaluation_result.update({"Accuracy": accuracy_score(y_true, y_pred)})
return evaluation_result
for batch_data in tqdm(test_data, "evaluating"):
ctx_data = split_and_load(ctx, *batch_data, even_split=False)
for (word, word_radical, char, char_radical, word_mask, char_mask,
label) in ctx_data:
output = _net(word, word_radical, char, char_radical, word_mask,
char_mask)
pred = mx.nd.argmax(output, axis=1)
ground_truth.extend(label.asnumpy().tolist())
prediction.extend(pred.asnumpy().tolist())
return evaluation_function(ground_truth, prediction)
def eval_f(_net, test_data, ctx=mx.cpu()):
ground_truth = []
prediction = []
pred_labels = []
for batch_data in tqdm(test_data, "evaluating"):
ctx_data = split_and_load(ctx, *batch_data, even_split=False)
for (keys, values, data_mask, label, label_mask) in ctx_data:
output, _ = _net(keys, values, mask=data_mask)
pred = output.asnumpy().tolist()
label = label.asnumpy().tolist()
for i, length in enumerate(label_mask.asnumpy().tolist()):
length = int(length)
ground_truth.extend(label[i][:length])
prediction.extend(pred[i][:length])
pred_labels.extend(
[0 if p < 0.5 else 1 for p in pred[i][:length]])
auc = roc_auc_score(ground_truth, prediction)
precision, recall, f1, _ = precision_recall_fscore_support(
ground_truth, pred_labels)
evaluation_result = {}
evaluation_result.update(
{"precision_%d" % i: precision[i]
for i in range(len(precision))})
evaluation_result.update(
{"recall_%d" % i: recall[i]
for i in range(len(recall))})
evaluation_result.update({"f1_%d" % i: f1[i] for i in range(len(f1))})
evaluation_result.update({"auc": auc})
return evaluation_result
def eval_f(_net, test_data, ctx=mx.cpu()):
k = test_data[1]["k"]
k = as_list(k) if k is not None else []
max_k = max(k) if k else None
top_k_ground_truth = []
top_k_prediction = []
ground_truth = []
prediction = []
for batch_data in tqdm(test_data[0], "evaluating"):
ctx_data = split_and_load(ctx, *batch_data, even_split=False)
for (user, item, label) in ctx_data:
output = _net(user, item)
pred = output
label = label.asnumpy().astype("int")
pred = pred.asnumpy()
ground_truth.append(label.tolist())
prediction.append(pred.tolist())
if max_k:
top_k_indices = np.argsort(pred)[::-1]
_top_k_indices = top_k_indices[:max_k]
padding = [0] * (max_k - len(_top_k_indices)) if len(
_top_k_indices) < max_k else []
top_k_prediction.append(pred[_top_k_indices].tolist() +
padding)
top_k_ground_truth.append(label[_top_k_indices].tolist() +
padding)
chained_ground_truth = list(chain(*ground_truth))
chained_prediction = list(chain(*prediction))
metrics = {
"rmse": mean_squared_error(chained_ground_truth, chained_prediction),
"mae": median_absolute_error(chained_ground_truth, chained_prediction),
}
metrics.update(
classification_report(
chained_ground_truth,
[0 if v < 0.5 else 1
for v in chained_prediction], chained_prediction))
if k:
metrics_k = {"ndcg": {}, "HR": {}}
for _k in k:
metrics_k["ndcg"][_k] = ndcg_score(top_k_ground_truth,
top_k_prediction,
k=_k)
metrics_k["HR"][_k] = _hit_rate(top_k_ground_truth, k=_k)
metrics.update(metrics_k)
return metrics
def eval_f(_net, test_data, ctx=mx.cpu()):
ground_truth = []
prediction = []
def evaluation_function(y_true, y_pred):
return {"evaluation_name": 0}
for batch_data in tqdm(test_data, "evaluating"):
ctx_data = split_and_load(ctx, *batch_data, even_split=False)
for (data, label) in ctx_data:
output = _net(data)
pred = output
ground_truth.extend(label.asnumpy().tolist())
prediction.extend(pred.asnumpy().tolist())
return evaluation_function(ground_truth, prediction)
def fit_f(net,
batch_size,
batch_data,
trainer,
loss_function,
loss_monitor=None,
ctx=mx.cpu(),
fit_step_func=_fit_f):
"""
Defined how each step of batch train goes
Parameters
----------
net: HybridBlock
The network which has been initialized
or loaded from the existed model
batch_size: int
The size of each batch
batch_data: Iterable
The batch data for train
trainer:
The trainer used to update the parameters of the net
loss_function: dict of function
The functions to compute the loss for the procession
of back propagation
loss_monitor: LossMonitor
Default to ``None``
ctx: Context or list of Context
Defaults to ``mx.cpu()``.
fit_step_func: function
Returns
-------
"""
ctx_data = split_and_load(ctx, *batch_data, even_split=False)
with autograd.record():
for _data in ctx_data:
bp_loss = fit_step_func(net, _data, loss_function, loss_monitor)
bp_loss.backward()
# todo: confirm whether the train step is equal to batch_size
if batch_size is not None:
trainer.step(batch_size)
else:
trainer.step(len(batch_data[0]))
def fit_f(net,
batch_size,
batch_data,
trainer,
bp_loss_f,
loss_function,
loss_monitor=None,
ctx=mx.cpu()):
"""
Defined how each step of batch train goes
Parameters
----------
net: HybridBlock
The network which has been initialized
or loaded from the existed model
batch_size: int
The size of each batch
batch_data: Iterable
The batch data for train
trainer:
The trainer used to update the parameters of the net
bp_loss_f: dict with only one value and one key
The function to compute the loss for the procession
of back propagation
loss_function: dict of function
Some other measurement in addition to bp_loss_f
loss_monitor: LossMonitor
Default to ``None``
ctx: Context or list of Context
Defaults to ``mx.cpu()``.
Returns
-------
"""
# 此函数定义训练过程
ctx_data = split_and_load(ctx, *batch_data, even_split=False)
with autograd.record():
for _data in ctx_data:
bp_loss = _fit_f(net, _data, bp_loss_f, loss_function,
loss_monitor)
assert bp_loss is not None
bp_loss.backward()
trainer.step(1)
def eval_f(_net, test_data, ctx=mx.cpu()):
ground_truth = []
prediction = []
for batch_data in tqdm(test_data, "evaluating"):
ctx_data = split_and_load(
ctx, *batch_data,
even_split=False
)
for (user, item, score) in ctx_data:
output = _net(user, item)
pred = output
ground_truth.extend(score.asnumpy().tolist())
prediction.extend(pred.asnumpy().tolist())
return classification_report(
ground_truth,
y_pred=[0 if p < 0.5 else 1 for p in prediction],
y_score=prediction
)
def eval_f(_net, test_data, ctx=mx.cpu()):
ground_truth = []
prediction = []
pred_labels = []
for batch_data in tqdm(test_data, "evaluating"):
ctx_data = split_and_load(ctx, *batch_data, even_split=False)
for (data, data_mask, label, pick_index, label_mask) in ctx_data:
output, _ = _net(data, data_mask)
output = mx.nd.slice(output, (None, None), (None, -1))
output = mx.nd.pick(output, pick_index)
pred = output.asnumpy().tolist()
label = label.asnumpy().tolist()
for i, length in enumerate(label_mask.asnumpy().tolist()):
length = int(length)
ground_truth.extend(label[i][:length])
prediction.extend(pred[i][:length])
pred_labels.extend(
[0 if p < 0.5 else 1 for p in pred[i][:length]])
return classification_report(ground_truth,
y_pred=pred_labels,
y_score=prediction)
def split_and_load(ctx, *args, **kwargs):
return split_and_load(ctx, *args, **kwargs)