def eval_model(model,
data,
metric_meta,
use_cuda=True,
with_label=True,
label_mapper=None,
task_type=TaskType.Classification):
if use_cuda:
model.cuda()
predictions = []
golds = []
scores = []
ids = []
metrics = {}
for idx, (batch_info, batch_data) in enumerate(data):
# if idx % 100 == 0:
# print("predicting {}".format(idx))
batch_info, batch_data = Collater.patch_data(use_cuda, batch_info,
batch_data)
score, pred, gold = model.predict(batch_info, batch_data)
predictions.extend(pred)
golds.extend(gold)
scores.extend(score)
ids.extend(batch_info['uids'])
if task_type == TaskType.Span:
from experiments.squad import squad_utils
golds = squad_utils.merge_answers(ids, golds)
predictions, scores = squad_utils.select_answers(
ids, predictions, scores)
if with_label:
metrics = calc_metrics(metric_meta, golds, predictions, scores,
label_mapper)
return metrics, predictions, scores, golds, ids
def eval_model(model,
data,
metric_meta,
device,
with_label=True,
label_mapper=None,
task_type=TaskType.Classification):
predictions = []
golds = []
scores = []
ids = []
metrics = {}
for (batch_info, batch_data) in data:
batch_info, batch_data = Collater.patch_data(device, batch_info,
batch_data)
score, pred, gold = model.predict(batch_info, batch_data)
predictions.extend(pred)
golds.extend(gold)
scores.extend(score)
ids.extend(batch_info['uids'])
if task_type == TaskType.Span:
from experiments.squad import squad_utils
golds = squad_utils.merge_answers(ids, golds)
predictions, scores = squad_utils.select_answers(
ids, predictions, scores)
if with_label:
metrics = calc_metrics(metric_meta, golds, predictions, scores,
label_mapper)
return metrics, predictions, scores, golds, ids
def eval_model(model,
data,
metric_meta,
device,
with_label=True,
label_mapper=None,
task_type=TaskType.Classification):
predictions = []
golds = []
scores = []
ids = []
metrics = {}
print("****device={}".format(device))
for (batch_info, batch_data) in data:
batch_info, batch_data = Collater.patch_data(device, batch_info,
batch_data)
score, pred, gold = model.predict(batch_info, batch_data)
predictions.extend(pred)
golds.extend(gold)
scores.extend(score)
ids.extend(batch_info['uids'])
if task_type == TaskType.Span:
from experiments.squad import squad_utils
golds = squad_utils.merge_answers(ids, golds)
predictions, scores = squad_utils.select_answers(
ids, predictions, scores)
if with_label:
metrics = calc_metrics(metric_meta, golds, predictions, scores,
label_mapper)
for i in range(min(len(ids), 10)):
print("{}\t{}\t{}\t{}\n".format(ids[i], predictions[i], scores[2 * i],
scores[2 * i + 1]))
#print("score heads={}".format(scores[:10]))
return metrics, predictions, scores, golds, ids