span_score = span_scorer(span_emb)
all_scores.extend(span_score.squeeze(1))
all_scores = torch.stack(all_scores)
all_labels = torch.stack(all_labels)
strict_preds = (all_scores > 0).to(torch.int)
eval = Evaluation(strict_preds, all_labels)
logger.info('Recall: {}, Precision: {}, F1: {}'.format(
eval.get_recall(), eval.get_precision(), eval.get_f1()))
if config.exact:
if eval.get_f1() > max_dev[0]:
max_dev = (eval.get_f1(), epoch)
torch.save(span_repr.state_dict(), span_repr_path)
torch.save(span_scorer.state_dict(), span_scorer_path)
else:
eval_range = [0.2, 0.25, 0.3
] if config['mention_type'] == 'events' else [
0.2, 0.25, 0.3, 0.4, 0.45
]
for k in eval_range:
s, i = torch.topk(all_scores,
int(k * dev_num_of_tokens),
sorted=False)
rank_preds = torch.zeros(len(all_scores), device=device)
rank_preds[i] = 1
eval = Evaluation(rank_preds, all_labels)
recall = eval.get_recall()
if recall > max_dev[0]:
[topic_spans.continuous_embeddings[k] for k in first_idx],
topic_spans.width[first_idx])
g2 = span_repr(topic_spans.start_end_embeddings[second_idx],
[topic_spans.continuous_embeddings[k] for k in second_idx],
topic_spans.width[second_idx])
scores = pairwise_model(g1, g2)
if config['training_method'] in ('continue', 'e2e') and not config['use_gold_mentions']:
g1_score = span_scorer(g1)
g2_score = span_scorer(g2)
scores += g1_score + g2_score
all_scores.extend(scores.squeeze(1))
all_labels.extend(batch_labels.to(torch.int))
all_labels = torch.stack(all_labels)
all_scores = torch.stack(all_scores)
strict_preds = (all_scores > 0).to(torch.int)
eval = Evaluation(strict_preds, all_labels.to(device))
logger.info('Number of predictions: {}/{}'.format(strict_preds.sum(), len(strict_preds)))
logger.info('Number of positive pairs: {}/{}'.format(len((all_labels == 1).nonzero()),
len(all_labels)))
logger.info('Strict - Recall: {}, Precision: {}, F1: {}'.format(eval.get_recall(),
eval.get_precision(), eval.get_f1()))
f1.append(eval.get_f1())
torch.save(span_repr.state_dict(), os.path.join(config['model_path'], 'span_repr_{}'.format(epoch)))
torch.save(span_scorer.state_dict(), os.path.join(config['model_path'], 'span_scorer_{}'.format(epoch)))
torch.save(pairwise_model.state_dict(), os.path.join(config['model_path'], 'pairwise_scorer_{}'.format(epoch)))
'e2e') and not config['use_gold_mentions']:
g1_score = span_scorer(g1)
g2_score = span_scorer(g2)
scores += g1_score + g2_score
all_scores.extend(scores.squeeze(1))
all_labels.extend(batch_labels.to(torch.int))
all_labels = torch.stack(all_labels)
all_scores = torch.stack(all_scores)
strict_preds = (all_scores > 0).to(torch.int)
eval = Evaluation(strict_preds, all_labels.to(device))
logger.info('Number of predictions: {}/{}'.format(
strict_preds.sum(), len(strict_preds)))
logger.info('Number of positive pairs: {}/{}'.format(
len(torch.nonzero(all_labels == 1)), len(all_labels)))
logger.info('Strict - Recall: {}, Precision: {}, F1: {}'.format(
eval.get_recall(), eval.get_precision(), eval.get_f1()))
f1.append(eval.get_f1())
torch.save(
span_repr.state_dict(),
os.path.join(config['model_path'], 'span_repr_{}'.format(epoch)))
torch.save(
span_scorer.state_dict(),
os.path.join(config['model_path'], 'span_scorer_{}'.format(epoch)))
torch.save(
pairwise_model.state_dict(),
os.path.join(config['model_path'],
'pairwise_scorer_{}'.format(epoch)))