def _evaluate_text_classifier(model: flair.nn.Model,
sentences: List[Sentence],
eval_mini_batch_size: int = 32,
embeddings_in_memory: bool = False,
out_path: Path = None) -> (dict, float):
with torch.no_grad():
eval_loss = 0
batches = [
sentences[x:x + eval_mini_batch_size]
for x in range(0, len(sentences), eval_mini_batch_size)
]
metric = Metric('Evaluation')
lines: List[str] = []
for batch in batches:
labels, loss = model.forward_labels_and_loss(batch)
clear_embeddings(
batch, also_clear_word_embeddings=not embeddings_in_memory)
eval_loss += loss
sentences_for_batch = [
sent.to_plain_string() for sent in batch
]
confidences_for_batch = [[
label.score for label in sent_labels
] for sent_labels in labels]
predictions_for_batch = [[
label.value for label in sent_labels
] for sent_labels in labels]
true_values_for_batch = [
sentence.get_label_names() for sentence in batch
]
available_labels = model.label_dictionary.get_items()
for sentence, confidence, prediction, true_value in zip(
sentences_for_batch, confidences_for_batch,
predictions_for_batch, true_values_for_batch):
eval_line = '{}\t{}\t{}\t{}\n'.format(
sentence, true_value, prediction, confidence)
lines.append(eval_line)
for predictions_for_sentence, true_values_for_sentence in zip(
predictions_for_batch, true_values_for_batch):
ModelTrainer._evaluate_sentence_for_text_classification(
metric, available_labels, predictions_for_sentence,
true_values_for_sentence)
eval_loss /= len(sentences)
if out_path is not None:
with open(out_path, "w", encoding='utf-8') as outfile:
outfile.write(''.join(lines))
return metric, eval_loss
def _evaluate_text_classifier(model: flair.nn.Model,
sentences: List[Sentence],
eval_mini_batch_size: int = 32,
embeddings_in_memory: bool = False) -> (dict, float):
with torch.no_grad():
eval_loss = 0
batches = [sentences[x:x + eval_mini_batch_size] for x in
range(0, len(sentences), eval_mini_batch_size)]
metric = Metric('Evaluation')
for batch in batches:
labels, loss = model.forward_labels_and_loss(batch)
clear_embeddings(batch, also_clear_word_embeddings=not embeddings_in_memory)
eval_loss += loss
predictions_for_batch = [[label.value for label in sent_labels] for sent_labels in labels]
true_values_for_batch = [sentence.get_label_names() for sentence in batch]
available_labels = model.label_dictionary.get_items()
for predictions_for_sentence, true_values_for_sentence in zip(predictions_for_batch, true_values_for_batch):
ModelTrainer._evaluate_sentence_for_text_classification(metric,
available_labels,
predictions_for_sentence,
true_values_for_sentence)
eval_loss /= len(sentences)
return metric, eval_loss
def _evaluate_sequence_tagger(model,
sentences: List[Sentence],
eval_mini_batch_size: int = 32,
embeddings_in_memory: bool = True,
out_path: Path = None) -> (dict, float):
with torch.no_grad():
eval_loss = 0
batch_no: int = 0
batches = [sentences[x:x + eval_mini_batch_size] for x in range(0, len(sentences), eval_mini_batch_size)]
metric = Metric('Evaluation')
lines: List[str] = []
for batch in batches:
batch_no += 1
tags, loss = model.forward_labels_and_loss(batch)
eval_loss += loss
for (sentence, sent_tags) in zip(batch, tags):
for (token, tag) in zip(sentence.tokens, sent_tags):
token: Token = token
token.add_tag_label('predicted', tag)
# append both to file for evaluation
eval_line = '{} {} {} {}\n'.format(token.text,
token.get_tag(model.tag_type).value, tag.value, tag.score)
lines.append(eval_line)
lines.append('\n')
for sentence in batch:
# make list of gold tags
gold_tags = [(tag.tag, str(tag)) for tag in sentence.get_spans(model.tag_type)]
# make list of predicted tags
predicted_tags = [(tag.tag, str(tag)) for tag in sentence.get_spans('predicted')]
# check for true positives, false positives and false negatives
for tag, prediction in predicted_tags:
if (tag, prediction) in gold_tags:
metric.add_tp(tag)
else:
metric.add_fp(tag)
for tag, gold in gold_tags:
if (tag, gold) not in predicted_tags:
metric.add_fn(tag)
else:
metric.add_tn(tag)
clear_embeddings(batch, also_clear_word_embeddings=not embeddings_in_memory)
eval_loss /= len(sentences)
if out_path is not None:
with open(out_path, "w", encoding='utf-8') as outfile:
outfile.write(''.join(lines))
return metric, eval_loss
def evaluate(self,
sentences: List[Sentence],
eval_class_metrics: bool = False,
mini_batch_size: int = 32,
embeddings_in_memory: bool = False,
metric_name: str = 'MICRO_AVG') -> (dict, float):
"""
Evaluates the model with the given list of sentences.
:param sentences: the list of sentences
:param eval_class_metrics: boolean indicating whether to print class metrics or not
:param mini_batch_size: the mini batch size to use
:param embeddings_in_memory: boolean value indicating, if embeddings should be kept in memory or not
:param metric_name: the name of the metrics to compute
:return: list of metrics, and the loss
"""
with torch.no_grad():
eval_loss = 0
batches = [
sentences[x:x + mini_batch_size]
for x in range(0, len(sentences), mini_batch_size)
]
metric = Metric(metric_name)
for batch in batches:
scores = self.model.forward(batch)
labels = self.model.obtain_labels(scores)
loss = self.model.calculate_loss(scores, batch)
clear_embeddings(
batch, also_clear_word_embeddings=not embeddings_in_memory)
eval_loss += loss
for predictions, true_values in zip(
[[label.value for label in sent_labels]
for sent_labels in labels],
[sentence.get_label_names() for sentence in batch]):
for prediction in predictions:
if prediction in true_values:
metric.tp()
if eval_class_metrics: metric.tp(prediction)
else:
metric.fp()
if eval_class_metrics: metric.fp(prediction)
for true_value in true_values:
if true_value not in predictions:
metric.fn()
if eval_class_metrics: metric.fn(true_value)
else:
metric.tn()
if eval_class_metrics: metric.tn(true_value)
eval_loss /= len(sentences)
return metric, eval_loss
def evaluate(
self,
data_loader: DataLoader,
out_path: Path = None,
embeddings_storage_mode: str = "cpu",
) -> (Result, float):
with torch.no_grad():
eval_loss = 0
batch_no: int = 0
metric = Metric("Evaluation")
lines: List[str] = []
for batch in data_loader:
batch_no += 1
with torch.no_grad():
features = self.forward(batch)
loss = self.calculate_loss(features, batch)
tags, _ = self.obtain_labels(features, batch)
eval_loss += loss
metric = self.obtain_performance_metric(
batch, tags, lines, metric)
store_embeddings(batch, embeddings_storage_mode)
eval_loss /= batch_no
if out_path is not None:
with open(out_path, "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))
detailed_result = (
f"\nMICRO_AVG: acc {metric.micro_avg_accuracy()} - f1-score {metric.micro_avg_f_score()}"
f"\nMACRO_AVG: acc {metric.macro_avg_accuracy()} - f1-score {metric.macro_avg_f_score()}"
)
for class_name in metric.get_classes():
detailed_result += (
f"\n{class_name:<10} tp: {metric.get_tp(class_name)} - fp: {metric.get_fp(class_name)} - "
f"fn: {metric.get_fn(class_name)} - tn: {metric.get_tn(class_name)} - precision: "
f"{metric.precision(class_name):.4f} - recall: {metric.recall(class_name):.4f} - "
f"accuracy: {metric.accuracy(class_name):.4f} - f1-score: "
f"{metric.f_score(class_name):.4f}")
result = Result(
main_score=metric.micro_avg_f_score(),
log_line=
f"{metric.precision()}\t{metric.recall()}\t{metric.micro_avg_f_score()}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=detailed_result,
)
return result, eval_loss
def test_metric_with_classes():
metric = Metric("Test")
metric.add_tp("class-1")
metric.add_tn("class-1")
metric.add_tn("class-1")
metric.add_fp("class-1")
metric.add_tp("class-2")
metric.add_tn("class-2")
metric.add_tn("class-2")
metric.add_fp("class-2")
for i in range(0, 10):
metric.add_tp("class-3")
for i in range(0, 90):
metric.add_fp("class-3")
metric.add_tp("class-4")
metric.add_tn("class-4")
metric.add_tn("class-4")
metric.add_fp("class-4")
print(metric)
assert metric.precision("class-1") == 0.5
assert metric.precision("class-2") == 0.5
assert metric.precision("class-3") == 0.1
assert metric.precision("class-4") == 0.5
assert metric.recall("class-1") == 1
assert metric.recall("class-2") == 1
assert metric.recall("class-3") == 1
assert metric.recall("class-4") == 1
assert metric.accuracy() == metric.micro_avg_accuracy()
assert metric.f_score() == metric.micro_avg_f_score()
assert metric.f_score("class-1") == 0.6666666666666666
assert metric.f_score("class-2") == 0.6666666666666666
assert metric.f_score("class-3") == 0.18181818181818182
assert metric.f_score("class-4") == 0.6666666666666666
assert metric.accuracy("class-1") == 0.75
assert metric.accuracy("class-2") == 0.75
assert metric.accuracy("class-3") == 0.1
assert metric.accuracy("class-4") == 0.75
assert metric.micro_avg_f_score() == 0.21848739495798317
assert metric.macro_avg_f_score() == 0.5454545454545454
assert metric.micro_avg_accuracy() == 0.16964285714285715
assert metric.macro_avg_accuracy() == 0.5875
assert metric.precision() == 0.12264150943396226
assert metric.recall() == 1
def test_metric_get_classes():
metric = Metric(u'Test')
metric.add_fn(u'class-1')
metric.add_fn(u'class-3')
metric.add_tn(u'class-1')
metric.add_tp(u'class-2')
assert (3 == len(metric.get_classes()))
assert (u'class-1' in metric.get_classes())
assert (u'class-2' in metric.get_classes())
assert (u'class-3' in metric.get_classes())
def test_metric_with_classes():
metric = Metric("Test")
metric.add_tp("class-1")
metric.add_tn("class-1")
metric.add_tn("class-1")
metric.add_fp("class-1")
metric.add_tp("class-2")
metric.add_tn("class-2")
metric.add_tn("class-2")
metric.add_fp("class-2")
for i in range(0, 10):
metric.add_tp("class-3")
for i in range(0, 90):
metric.add_fp("class-3")
metric.add_tp("class-4")
metric.add_tn("class-4")
metric.add_tn("class-4")
metric.add_fp("class-4")
assert metric.precision("class-1") == 0.5
assert metric.precision("class-2") == 0.5
assert metric.precision("class-3") == 0.1
assert metric.precision("class-4") == 0.5
assert metric.recall("class-1") == 1
assert metric.recall("class-2") == 1
assert metric.recall("class-3") == 1
assert metric.recall("class-4") == 1
assert metric.accuracy() == metric.micro_avg_accuracy()
assert metric.f_score() == metric.micro_avg_f_score()
assert metric.f_score("class-1") == 0.6667
assert metric.f_score("class-2") == 0.6667
assert metric.f_score("class-3") == 0.1818
assert metric.f_score("class-4") == 0.6667
assert metric.accuracy("class-1") == 0.5
assert metric.accuracy("class-2") == 0.5
assert metric.accuracy("class-3") == 0.1
assert metric.accuracy("class-4") == 0.5
assert metric.micro_avg_f_score() == 0.2184
assert metric.macro_avg_f_score() == 0.5454749999999999
assert metric.micro_avg_accuracy() == 0.1226
assert metric.macro_avg_accuracy() == 0.4
assert metric.precision() == 0.1226
assert metric.recall() == 1
def test_metric_with_classes():
metric = Metric('Test')
metric.add_tp('class-1')
metric.add_tn('class-1')
metric.add_tn('class-1')
metric.add_fp('class-1')
metric.add_tp('class-2')
metric.add_tn('class-2')
metric.add_tn('class-2')
metric.add_fp('class-2')
for i in range(0, 10):
metric.add_tp('class-3')
for i in range(0, 90):
metric.add_fp('class-3')
metric.add_tp('class-4')
metric.add_tn('class-4')
metric.add_tn('class-4')
metric.add_fp('class-4')
assert(metric.precision('class-1') == 0.5)
assert(metric.precision('class-2') == 0.5)
assert(metric.precision('class-3') == 0.1)
assert(metric.precision('class-4') == 0.5)
assert(metric.recall('class-1') == 1)
assert(metric.recall('class-2') == 1)
assert(metric.recall('class-3') == 1)
assert(metric.recall('class-4') == 1)
assert(metric.accuracy() == metric.micro_avg_accuracy())
assert(metric.f_score() == metric.micro_avg_f_score())
assert(metric.f_score('class-1') == 0.6667)
assert(metric.f_score('class-2') == 0.6667)
assert(metric.f_score('class-3') == 0.1818)
assert(metric.f_score('class-4') == 0.6667)
assert(metric.accuracy('class-1') == 0.75)
assert(metric.accuracy('class-2') == 0.75)
assert(metric.accuracy('class-3') == 0.1)
assert(metric.accuracy('class-4') == 0.75)
assert(metric.micro_avg_f_score() == 0.2184)
assert(metric.macro_avg_f_score() == 0.4)
assert(metric.micro_avg_accuracy() == 0.1696)
assert(metric.macro_avg_accuracy() == 0.5875)
assert(metric.precision() == 0.1226)
assert(metric.recall() == 1)
def test_metric_get_classes():
metric = Metric("Test")
metric.add_fn("class-1")
metric.add_fn("class-3")
metric.add_tn("class-1")
metric.add_tp("class-2")
assert 3 == len(metric.get_classes())
assert "class-1" in metric.get_classes()
assert "class-2" in metric.get_classes()
assert "class-3" in metric.get_classes()
def obtain_performance_metric(
self,
batch,
tags,
lines=None,
metric=Metric("Perf_Metric"),
) -> Metric:
for (sentence, sent_tags) in zip(batch, tags):
for (token, tag) in zip(sentence.tokens, sent_tags):
token: Token = token
token.add_tag_label("predicted", tag)
# append both to file for evaluation
eval_line = "{} {} {} {}\n".format(
token.text,
token.get_tag(self.tag_type).value,
tag.value,
tag.score,
)
# if provided None, dont append to the lines
if isinstance(lines, List):
lines.append(eval_line)
if isinstance(lines, List):
lines.append("\n")
for sentence in batch:
# make list of gold tags
gold_tags = [(tag.tag, str(tag))
for tag in sentence.get_spans(self.tag_type)]
# make list of predicted tags
predicted_tags = [(tag.tag, str(tag))
for tag in sentence.get_spans("predicted")]
# check for true positives, false positives and false negatives
for tag, prediction in predicted_tags:
if (tag, prediction) in gold_tags:
metric.add_tp(tag)
else:
metric.add_fp(tag)
for tag, gold in gold_tags:
if (tag, gold) not in predicted_tags:
metric.add_fn(tag)
else:
metric.add_tn(tag)
return metric
def test_multiclass_metrics():
metric = Metric('Test')
available_labels = ['A', 'B', 'C']
predictions = ['A', 'B']
true_values = ['A']
ModelTrainer._evaluate_sentence_for_text_classification(
metric, available_labels, predictions, true_values)
predictions = ['C', 'B']
true_values = ['A', 'B']
ModelTrainer._evaluate_sentence_for_text_classification(
metric, available_labels, predictions, true_values)
print(metric)
def test_multiclass_metrics():
metric = Metric("Test")
available_labels = ["A", "B", "C"]
predictions = ["A", "B"]
true_values = ["A"]
ModelTrainer._evaluate_sentence_for_text_classification(
metric, available_labels, predictions, true_values
)
predictions = ["C", "B"]
true_values = ["A", "B"]
ModelTrainer._evaluate_sentence_for_text_classification(
metric, available_labels, predictions, true_values
)
print(metric)
def _evaluate_text_classifier(
model: flair.nn.Model,
sentences: List[Sentence],
eval_mini_batch_size: int = 32,
embeddings_in_memory: bool = False) -> (dict, float):
with torch.no_grad():
eval_loss = 0
batches = [
sentences[x:x + eval_mini_batch_size]
for x in range(0, len(sentences), eval_mini_batch_size)
]
metric = Metric('Evaluation')
for batch in batches:
labels, loss = model.forward_labels_and_loss(batch)
clear_embeddings(
batch, also_clear_word_embeddings=not embeddings_in_memory)
eval_loss += loss
for predictions, true_values in zip(
[[label.value for label in sent_labels]
for sent_labels in labels],
[sentence.get_label_names() for sentence in batch]):
for prediction in predictions:
if prediction in true_values:
metric.add_tp(prediction)
else:
metric.add_fp(prediction)
for true_value in true_values:
if true_value not in predictions:
metric.add_fn(true_value)
else:
metric.add_tn(true_value)
eval_loss /= len(sentences)
return metric, eval_loss
def obtain_performance_metric(
self,
batch,
labels,
lines,
metric=Metric("Perf_Metric"),
) -> Metric:
sentences_for_batch = [sent.to_plain_string() for sent in batch]
confidences_for_batch = [[label.score for label in sent_labels]
for sent_labels in labels]
predictions_for_batch = [[label.value for label in sent_labels]
for sent_labels in labels]
true_values_for_batch = [
sentence.get_label_names() for sentence in batch
]
available_labels = self.label_dictionary.get_items()
for sentence, confidence, prediction, true_value in zip(
sentences_for_batch,
confidences_for_batch,
predictions_for_batch,
true_values_for_batch,
):
eval_line = "{}\t{}\t{}\t{}\n".format(sentence, true_value,
prediction, confidence)
lines.append(eval_line)
for predictions_for_sentence, true_values_for_sentence in zip(
predictions_for_batch, true_values_for_batch):
for label in available_labels:
if (label in predictions_for_sentence
and label in true_values_for_sentence):
metric.add_tp(label)
elif (label in predictions_for_sentence
and label not in true_values_for_sentence):
metric.add_fp(label)
elif (label not in predictions_for_sentence
and label in true_values_for_sentence):
metric.add_fn(label)
elif (label not in predictions_for_sentence
and label not in true_values_for_sentence):
metric.add_tn(label)
return metric
def evaluate(gold_file: Path, pred_file: Path, match_func: Callable[[Tuple, List], Tuple]) -> Metric:
gold_annotations = read_annotations(gold_file)
pred_annotations = read_annotations(pred_file)
metric = Metric("Evaluation", beta=1)
copy_gold = copy_dict(gold_annotations)
for document_id, annotations in pred_annotations.items():
for pred_entry in annotations:
# Documents may not contain any gold entity!
if document_id in copy_gold:
matched_gold = match_func(pred_entry, copy_gold[document_id])
else:
matched_gold = None
if matched_gold:
# Assert same document and same entity type!
assert matched_gold[0] == pred_entry[0] and matched_gold[3] == pred_entry[3]
copy_gold[document_id].remove(matched_gold)
metric.add_tp(pred_entry[3])
else:
metric.add_fp(pred_entry[3])
copy_pred = copy_dict(pred_annotations)
for document_id, annotations in gold_annotations.items():
for gold_entry in annotations:
if document_id in copy_pred:
matched_pred = match_func(gold_entry, copy_pred[document_id])
else:
matched_pred = None
if not matched_pred:
metric.add_fn(gold_entry[3])
else:
# Assert same document and same entity type!
assert matched_pred[0] == gold_entry[0] and matched_pred[3] == gold_entry[3]
copy_pred[document_id].remove(matched_pred)
return metric
def evaluate(self,
evaluation: List[Sentence],
out_path=None,
evaluation_method: str = 'F1',
eval_batch_size: int = 32,
embeddings_in_memory: bool = True):
batch_no: int = 0
batches = [
evaluation[x:x + eval_batch_size]
for x in range(0, len(evaluation), eval_batch_size)
]
metric = Metric('')
lines: List[str] = []
for batch in batches:
batch_no += 1
scores, tag_seq = self.model._predict_scores_batch(batch)
predicted_ids = tag_seq
all_tokens = []
for sentence in batch:
all_tokens.extend(sentence.tokens)
for (token, score, predicted_id) in zip(all_tokens, scores,
predicted_ids):
token: Token = token
# get the predicted tag
predicted_value = self.model.tag_dictionary.get_item_for_index(
predicted_id)
token.add_tag('predicted', predicted_value, score)
for sentence in batch:
# add predicted tags
for token in sentence.tokens:
predicted_tag: Label = token.get_tag('predicted')
# append both to file for evaluation
eval_line = '{} {} {}\n'.format(
token.text,
token.get_tag(self.model.tag_type).value,
predicted_tag.value)
lines.append(eval_line)
lines.append('\n')
# make list of gold tags
gold_tags = [
str(tag) for tag in sentence.get_spans(self.model.tag_type)
]
# make list of predicted tags
predicted_tags = [
str(tag) for tag in sentence.get_spans('predicted')
]
# check for true positives, false positives and false negatives
for prediction in predicted_tags:
if prediction in gold_tags:
metric.tp()
else:
metric.fp()
for gold in gold_tags:
if gold not in predicted_tags:
metric.fn()
if not embeddings_in_memory:
self.clear_embeddings_in_batch(batch)
if out_path is not None:
test_tsv = os.path.join(out_path, "test.tsv")
with open(test_tsv, "w", encoding='utf-8') as outfile:
outfile.write(''.join(lines))
if evaluation_method == 'accuracy':
score = metric.accuracy()
return score, metric
if evaluation_method == 'F1':
score = metric.f_score()
return score, metric
def evaluate(
self,
sentences: Dataset,
eval_mini_batch_size: int = 32,
embeddings_in_memory: bool = True,
out_path: Path = None,
) -> (Result, float):
with torch.no_grad():
eval_loss = 0
batch_no: int = 0
batch_loader = torch.utils.data.DataLoader(
sentences,
batch_size=eval_mini_batch_size,
shuffle=False,
num_workers=4,
collate_fn=list,
)
metric = Metric("Evaluation")
lines: List[str] = []
for batch in batch_loader:
batch_no += 1
with torch.no_grad():
features = self.forward(batch)
loss = self._calculate_loss(features, batch)
tags = self._obtain_labels(features, batch)
eval_loss += loss
for (sentence, sent_tags) in zip(batch, tags):
for (token, tag) in zip(sentence.tokens, sent_tags):
token: Token = token
token.add_tag_label("predicted", tag)
# append both to file for evaluation
eval_line = "{} {} {} {}\n".format(
token.text,
token.get_tag(self.tag_type).value,
tag.value,
tag.score,
)
lines.append(eval_line)
lines.append("\n")
for sentence in batch:
# make list of gold tags
gold_tags = [
(tag.tag, str(tag)) for tag in sentence.get_spans(self.tag_type)
]
# make list of predicted tags
predicted_tags = [
(tag.tag, str(tag)) for tag in sentence.get_spans("predicted")
]
# check for true positives, false positives and false negatives
for tag, prediction in predicted_tags:
if (tag, prediction) in gold_tags:
metric.add_tp(tag)
else:
metric.add_fp(tag)
for tag, gold in gold_tags:
if (tag, gold) not in predicted_tags:
metric.add_fn(tag)
else:
metric.add_tn(tag)
clear_embeddings(
batch, also_clear_word_embeddings=not embeddings_in_memory
)
eval_loss /= len(sentences)
if out_path is not None:
with open(out_path, "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))
detailed_result = (
f"\nMICRO_AVG: acc {metric.micro_avg_accuracy()} - f1-score {metric.micro_avg_f_score()}"
f"\nMACRO_AVG: acc {metric.macro_avg_accuracy()} - f1-score {metric.macro_avg_f_score()}"
)
for class_name in metric.get_classes():
detailed_result += (
f"\n{class_name:<10} tp: {metric.get_tp(class_name)} - fp: {metric.get_fp(class_name)} - "
f"fn: {metric.get_fn(class_name)} - tn: {metric.get_tn(class_name)} - precision: "
f"{metric.precision(class_name):.4f} - recall: {metric.recall(class_name):.4f} - "
f"accuracy: {metric.accuracy(class_name):.4f} - f1-score: "
f"{metric.f_score(class_name):.4f}"
)
result = Result(
main_score=metric.micro_avg_f_score(),
log_line=f"{metric.precision()}\t{metric.recall()}\t{metric.micro_avg_f_score()}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=detailed_result,
)
return result, eval_loss
def evaluate(
self,
data_loader: DataLoader,
out_path: Path = None,
embedding_storage_mode: str = "none",
) -> (Result, float):
if type(out_path) == str:
out_path = Path(out_path)
with torch.no_grad():
eval_loss = 0
batch_no: int = 0
metric = Metric("Evaluation", beta=self.beta)
lines: List[str] = []
if self.use_crf:
transitions = self.transitions.detach().cpu().numpy()
else:
transitions = None
for batch in data_loader:
batch_no += 1
with torch.no_grad():
features = self.forward(batch)
loss = self._calculate_loss(features, batch)
tags, _ = self._obtain_labels(
feature=features,
batch_sentences=batch,
transitions=transitions,
get_all_tags=False,
)
eval_loss += loss
for (sentence, sent_tags) in zip(batch, tags):
for (token, tag) in zip(sentence.tokens, sent_tags):
token: Token = token
token.add_tag("predicted", tag.value, tag.score)
# append both to file for evaluation
eval_line = "{} {} {} {}\n".format(
token.text,
token.get_tag(self.tag_type).value,
tag.value,
tag.score,
)
lines.append(eval_line)
lines.append("\n")
for sentence in batch:
# make list of gold tags
gold_tags = [
(tag.tag, tag.text) for tag in sentence.get_spans(self.tag_type)
]
# make list of predicted tags
predicted_tags = [
(tag.tag, tag.text) for tag in sentence.get_spans("predicted")
]
# check for true positives, false positives and false negatives
for tag, prediction in predicted_tags:
if (tag, prediction) in gold_tags:
metric.add_tp(tag)
else:
metric.add_fp(tag)
for tag, gold in gold_tags:
if (tag, gold) not in predicted_tags:
metric.add_fn(tag)
else:
metric.add_tn(tag)
store_embeddings(batch, embedding_storage_mode)
eval_loss /= batch_no
if out_path is not None:
with open(out_path, "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))
detailed_result = (
f"\nMICRO_AVG: acc {metric.micro_avg_accuracy():.4f} - f1-score {metric.micro_avg_f_score():.4f}"
f"\nMACRO_AVG: acc {metric.macro_avg_accuracy():.4f} - f1-score {metric.macro_avg_f_score():.4f}"
)
for class_name in metric.get_classes():
detailed_result += (
f"\n{class_name:<10} tp: {metric.get_tp(class_name)} - fp: {metric.get_fp(class_name)} - "
f"fn: {metric.get_fn(class_name)} - tn: {metric.get_tn(class_name)} - precision: "
f"{metric.precision(class_name):.4f} - recall: {metric.recall(class_name):.4f} - "
f"accuracy: {metric.accuracy(class_name):.4f} - f1-score: "
f"{metric.f_score(class_name):.4f}"
)
result = Result(
main_score=metric.micro_avg_f_score(),
log_line=f"{metric.precision():.4f}\t{metric.recall():.4f}\t{metric.micro_avg_f_score():.4f}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=detailed_result,
)
return result, eval_loss
def train_epoch(
self,
epoch,
batch_loader,
optimizer,
weight_extractor,
embeddings_storage_mode: str = "cpu",
param_selection_mode: bool = False,
):
metric = Metric("Training")
train_loss: float = 0
seen_batches = 0
total_number_of_batches = len(batch_loader)
modulo = max(1, int(total_number_of_batches / 10))
# process mini-batches
for batch_no, batch in enumerate(batch_loader):
optimizer.zero_grad()
features = self.model.forward(batch)
loss = self.model.calculate_loss(features, batch)
tags, _ = self.model.obtain_labels(features, batch)
# TODO: fix this for text regression model
metric = self.model.obtain_performance_metric(batch,
tags,
metric=metric)
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), 5.0)
optimizer.step()
seen_batches += 1
train_loss += loss.item()
# depending on memory mode, embeddings are moved to CPU, GPU or deleted
store_embeddings(batch, embeddings_storage_mode)
if batch_no % modulo == 0:
log.info(
f"epoch {epoch + 1} - iter {batch_no}/{total_number_of_batches} - loss "
f"{train_loss / seen_batches:.6f} - running_score "
f"{metric.micro_avg_f_score():.6f}")
iteration = epoch * total_number_of_batches + batch_no
if not param_selection_mode:
weight_extractor.extract_weights(self.model.state_dict(),
iteration)
train_loss /= seen_batches
detailed_result = (
f"\nMICRO_AVG: acc {metric.micro_avg_accuracy()} - f1-score {metric.micro_avg_f_score()}"
f"\nMACRO_AVG: acc {metric.macro_avg_accuracy()} - f1-score {metric.macro_avg_f_score()}"
)
for class_name in metric.get_classes():
detailed_result += (
f"\n{class_name:<10} tp: {metric.get_tp(class_name)} - fp: {metric.get_fp(class_name)} - "
f"fn: {metric.get_fn(class_name)} - tn: {metric.get_tn(class_name)} - precision: "
f"{metric.precision(class_name):.4f} - recall: {metric.recall(class_name):.4f} - "
f"accuracy: {metric.accuracy(class_name):.4f} - f1-score: "
f"{metric.f_score(class_name):.4f}")
result = Result(
main_score=metric.micro_avg_f_score(),
log_line=
f"{metric.precision()}\t{metric.recall()}\t{metric.micro_avg_f_score()}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=detailed_result,
)
return result, train_loss
def evaluate(
self,
data_loader: DataLoader,
out_path: Path = None,
embeddings_storage_mode: str = "cpu",
) -> (Result, float):
with torch.no_grad():
eval_loss = 0
metric = Metric("Evaluation")
lines: List[str] = []
batch_count: int = 0
for batch in data_loader:
batch_count += 1
labels, loss = self.forward_labels_and_loss(batch)
eval_loss += loss
sentences_for_batch = [
sent.to_plain_string() for sent in batch
]
confidences_for_batch = [[
label.score for label in sent_labels
] for sent_labels in labels]
predictions_for_batch = [[
label.value for label in sent_labels
] for sent_labels in labels]
true_values_for_batch = [
sentence.get_label_names() for sentence in batch
]
available_labels = self.label_dictionary.get_items()
for sentence, confidence, prediction, true_value in zip(
sentences_for_batch,
confidences_for_batch,
predictions_for_batch,
true_values_for_batch,
):
eval_line = "{}\t{}\t{}\t{}\n".format(
sentence, true_value, prediction, confidence)
lines.append(eval_line)
for predictions_for_sentence, true_values_for_sentence in zip(
predictions_for_batch, true_values_for_batch):
for label in available_labels:
if (label in predictions_for_sentence
and label in true_values_for_sentence):
metric.add_tp(label)
elif (label in predictions_for_sentence
and label not in true_values_for_sentence):
metric.add_fp(label)
elif (label not in predictions_for_sentence
and label in true_values_for_sentence):
metric.add_fn(label)
elif (label not in predictions_for_sentence
and label not in true_values_for_sentence):
metric.add_tn(label)
store_embeddings(batch, embeddings_storage_mode)
eval_loss /= batch_count
detailed_result = (
f"\nMICRO_AVG: acc {metric.micro_avg_accuracy()} - f1-score {metric.micro_avg_f_score()}"
f"\nMACRO_AVG: acc {metric.macro_avg_accuracy()} - f1-score {metric.macro_avg_f_score()}"
)
for class_name in metric.get_classes():
detailed_result += (
f"\n{class_name:<10} tp: {metric.get_tp(class_name)} - fp: {metric.get_fp(class_name)} - "
f"fn: {metric.get_fn(class_name)} - tn: {metric.get_tn(class_name)} - precision: "
f"{metric.precision(class_name):.4f} - recall: {metric.recall(class_name):.4f} - "
f"accuracy: {metric.accuracy(class_name):.4f} - f1-score: "
f"{metric.f_score(class_name):.4f}")
result = Result(
main_score=metric.micro_avg_f_score(),
log_line=
f"{metric.precision()}\t{metric.recall()}\t{metric.micro_avg_f_score()}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=detailed_result,
)
if out_path is not None:
with open(out_path, "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))
return result, eval_loss
def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: str = 'F1',
embeddings_in_memory: bool = True):
tp: int = 0
fp: int = 0
batch_no: int = 0
mini_batch_size = 32
batches = [evaluation[x:x + mini_batch_size] for x in
range(0, len(evaluation), mini_batch_size)]
metric = Metric('')
lines: List[str] = []
for batch in batches:
batch_no += 1
self.model.embeddings.embed(batch)
for sentence in batch:
sentence: Sentence = sentence
# Step 3. Run our forward pass.
score, tag_seq = self.model.predict_scores(sentence)
# Step 5. Compute predictions
predicted_id = tag_seq
for (token, pred_id) in zip(sentence.tokens, predicted_id):
token: Token = token
# get the predicted tag
predicted_tag = self.model.tag_dictionary.get_item_for_index(pred_id)
token.add_tag('predicted', predicted_tag)
# get the gold tag
gold_tag = token.get_tag(self.model.tag_type)
# append both to file for evaluation
eval_line = token.text + ' ' + gold_tag + ' ' + predicted_tag + "\n"
# positives
if predicted_tag != '':
# true positives
if predicted_tag == gold_tag:
metric.tp()
# false positive
if predicted_tag != gold_tag:
metric.fp()
# negatives
if predicted_tag == '':
# true negative
if predicted_tag == gold_tag:
metric.tn()
# false negative
if predicted_tag != gold_tag:
metric.fn()
lines.append(eval_line)
lines.append('\n')
if not embeddings_in_memory:
self.clear_embeddings_in_batch(batch)
if out_path is not None:
test_tsv = os.path.join(out_path, "test.tsv")
with open(test_tsv, "w", encoding='utf-8') as outfile:
outfile.write(''.join(lines))
if evaluation_method == 'span-F1':
# get the eval script
eval_script = cached_path('https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/scripts/conll03_eval_script.pl', cache_dir='scripts')
os.chmod(eval_script, 0o777)
eval_data = ''.join(lines)
p = run(eval_script, stdout=PIPE, input=eval_data, encoding='utf-8')
main_result = p.stdout
print(main_result)
main_result = main_result.split('\n')[1]
# parse the result file
main_result = re.sub(';', ' ', main_result)
main_result = re.sub('precision', 'p', main_result)
main_result = re.sub('recall', 'r', main_result)
main_result = re.sub('accuracy', 'acc', main_result)
f_score = float(re.findall(r'\d+\.\d+$', main_result)[0])
return f_score, metric._fp, main_result
if evaluation_method == 'accuracy':
score = metric.accuracy()
return score, metric._fp, str(score)
if evaluation_method == 'F1':
score = metric.f_score()
return score, metric._fp, str(metric)
def evaluate(
self,
sentences: List[Sentence],
eval_mini_batch_size: int = 32,
embeddings_in_memory: bool = False,
out_path: Path = None,
) -> (Result, float):
with torch.no_grad():
eval_loss = 0
batches = [
sentences[x:x + eval_mini_batch_size]
for x in range(0, len(sentences), eval_mini_batch_size)
]
metric = Metric("Evaluation")
lines: List[str] = []
for batch in batches:
labels, loss = self.forward_labels_and_loss(batch)
clear_embeddings(
batch, also_clear_word_embeddings=not embeddings_in_memory)
eval_loss += loss
sentences_for_batch = [
sent.to_plain_string() for sent in batch
]
confidences_for_batch = [[
label.score for label in sent_labels
] for sent_labels in labels]
predictions_for_batch = [[
label.value for label in sent_labels
] for sent_labels in labels]
true_values_for_batch = [
sentence.get_label_names() for sentence in batch
]
available_labels = self.label_dictionary.get_items()
for sentence, confidence, prediction, true_value in zip(
sentences_for_batch,
confidences_for_batch,
predictions_for_batch,
true_values_for_batch,
):
eval_line = "{}\t{}\t{}\t{}\n".format(
sentence, true_value, prediction, confidence)
lines.append(eval_line)
for predictions_for_sentence, true_values_for_sentence in zip(
predictions_for_batch, true_values_for_batch):
for label in available_labels:
if (label in predictions_for_sentence
and label in true_values_for_sentence):
metric.add_tp(label)
elif (label in predictions_for_sentence
and label not in true_values_for_sentence):
metric.add_fp(label)
elif (label not in predictions_for_sentence
and label in true_values_for_sentence):
metric.add_fn(label)
elif (label not in predictions_for_sentence
and label not in true_values_for_sentence):
metric.add_tn(label)
eval_loss /= len(sentences)
detailed_result = (
f"\nMICRO_AVG: acc {metric.micro_avg_accuracy()} - f1-score {metric.micro_avg_f_score()}"
f"\nMACRO_AVG: acc {metric.macro_avg_accuracy()} - f1-score {metric.macro_avg_f_score()}"
)
for class_name in metric.get_classes():
detailed_result += (
f"\n{class_name:<10} tp: {metric.get_tp(class_name)} - fp: {metric.get_fp(class_name)} - "
f"fn: {metric.get_fn(class_name)} - tn: {metric.get_tn(class_name)} - precision: "
f"{metric.precision(class_name):.4f} - recall: {metric.recall(class_name):.4f} - "
f"accuracy: {metric.accuracy(class_name):.4f} - f1-score: "
f"{metric.f_score(class_name):.4f}")
result = Result(
main_score=metric.micro_avg_f_score(),
log_line=
f"{metric.precision()}\t{metric.recall()}\t{metric.micro_avg_f_score()}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=detailed_result,
)
if out_path is not None:
with open(out_path, "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))
return result, eval_loss
def evaluate(
self,
sentences: Union[List[Sentence], Dataset],
out_path: Union[str, Path] = None,
embedding_storage_mode: str = "none",
mini_batch_size: int = 32,
num_workers: int = 8,
wsd_evaluation: bool = False,
**kwargs,
) -> (Result, float):
# read Dataset into data loader (if list of sentences passed, make Dataset first)
if not isinstance(sentences, Dataset):
sentences = SentenceDataset(sentences)
data_loader = DataLoader(sentences,
batch_size=mini_batch_size,
num_workers=num_workers)
eval_loss = 0
eval_count = 0
batch_no: int = 0
metric = Metric("Evaluation", beta=self.beta)
lines: List[str] = []
y_true = []
y_pred = []
for batch in data_loader:
# predict for batch
loss_and_count = self.predict(
batch,
embedding_storage_mode=embedding_storage_mode,
mini_batch_size=mini_batch_size,
label_name='predicted',
return_loss=True)
eval_loss += loss_and_count[0]
eval_count += loss_and_count[1]
batch_no += 1
for sentence in batch:
# make list of gold tags
gold_spans = sentence.get_spans(self.get_current_tag_type())
gold_tags = [(span.tag, repr(span)) for span in gold_spans]
# make list of predicted tags
predicted_spans = sentence.get_spans("predicted")
predicted_tags = [(span.tag, repr(span))
for span in predicted_spans]
# check for true positives, false positives and false negatives
for tag, prediction in predicted_tags:
if (tag, prediction) in gold_tags:
metric.add_tp(tag)
else:
metric.add_fp(tag)
for tag, gold in gold_tags:
if (tag, gold) not in predicted_tags:
metric.add_fn(tag)
tags_gold = []
tags_pred = []
# also write to file in BIO format to use old conlleval script
if out_path:
for token in sentence:
# check if in gold spans
gold_tag = 'O'
for span in gold_spans:
if token in span:
gold_tag = 'B-' + span.tag if token == span[
0] else 'I-' + span.tag
tags_gold.append(gold_tag)
predicted_tag = 'O'
# check if in predicted spans
for span in predicted_spans:
if token in span:
predicted_tag = 'B-' + span.tag if token == span[
0] else 'I-' + span.tag
tags_pred.append(predicted_tag)
lines.append(
f'{token.text} {gold_tag} {predicted_tag}\n')
lines.append('\n')
y_true.append(tags_gold)
y_pred.append(tags_pred)
if out_path:
with open(Path(out_path), "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))
detailed_result = (
"\nResults:"
f"\n- F1-score (micro) {metric.micro_avg_f_score():.4f}"
f"\n- F1-score (macro) {metric.macro_avg_f_score():.4f}"
'\n\nBy class:')
for class_name in metric.get_classes():
detailed_result += (
f"\n{class_name:<10} tp: {metric.get_tp(class_name)} - fp: {metric.get_fp(class_name)} - "
f"fn: {metric.get_fn(class_name)} - precision: "
f"{metric.precision(class_name):.4f} - recall: {metric.recall(class_name):.4f} - "
f"f1-score: "
f"{metric.f_score(class_name):.4f}")
result = Result(
main_score=metric.micro_avg_f_score(),
log_line=
f"{metric.precision():.4f}\t{metric.recall():.4f}\t{metric.micro_avg_f_score():.4f}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=detailed_result,
)
return result, eval_loss / eval_count
from __future__ import absolute_import
def eval_flair_spans(data, predicted_list, batch_size, out_path=None):
metric = Metric('Evaluation')
mini_batch_size = batch_size
batches = [
data[x:x + mini_batch_size]
for x in range(0, len(data), mini_batch_size)
]
lines: List[str] = []
word_counter = 0
for batch in batches:
for sentence in batch:
for token in sentence.tokens:
tag = Label(predicted_list[word_counter])
word_counter += 1
token.add_tag_label('predicted', tag)
# append both to file for evaluation
eval_line = '{} {} {} {}\n'.format(token.text,
token.get_tag('ner').value,
tag.value, tag.score)
lines.append(eval_line)
lines.append('\n')
for sentence in batch:
# make list of gold tags
gold_tags = [(tag.tag, str(tag))
for tag in sentence.get_spans('ner')]
# make list of predicted tags
predicted_tags = [(tag.tag, str(tag))
for tag in sentence.get_spans('predicted')]
# check for true positives, false positives and false negatives
for tag, prediction in predicted_tags:
if (tag, prediction) in gold_tags:
metric.add_tp(tag)
else:
metric.add_fp(tag)
for tag, gold in gold_tags:
if (tag, gold) not in predicted_tags:
metric.add_fn(tag)
else:
metric.add_tn(tag)
# add metrics scores at the beginning of the file
lines.insert(0, str(metric) + "\n\n")
if out_path is not None:
# create folder for json and corresponding output
if not os.path.exists(os.path.dirname(out_path)):
try:
os.makedirs(os.path.dirname(out_path))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
with open(out_path, "w", encoding='utf-8') as outfile:
outfile.write(''.join(lines))
#
# esnWrapper.model.output_activation = output_activation_training
return metric
def evaluate(
self,
data_loader: DataLoader,
out_path: Path = None,
embeddings_storage_mode: str = "cpu",
prediction_mode: bool = False,
) -> (Result, float):
eval_loss = 0
batch_no = 0
data_loader.assign_embeddings()
if out_path is not None:
outfile = open(out_path, "w", encoding="utf-8")
if not self.binary:
metric = Metric("Evaluation")
with torch.no_grad():
for batch in data_loader:
batch_no += 1
scores = self.forward(batch, prediction_mode=prediction_mode)
loss = self._calculate_loss(scores, batch, self.mask)
eval_loss += loss
if self.binary:
pdb.set_trace()
result = Result(
main_score=LF1,
log_line=f"\nUF1: {UF1} - LF1 {LF1}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=f"\nUF1: {UF1} - LF1 {LF1}",
)
else:
# if prediction_mode:
# eval_loss, metric=self.dependency_evaluate(data_loader,out_path=out_path,prediction_mode=prediction_mode)
# return eval_loss, metric
# else:
tags, _ = self._obtain_labels(scores, batch)
for (sentence, sent_tags) in zip(batch, tags):
for (token, tag) in zip(sentence.tokens, sent_tags):
token: Token = token
token.add_tag_label("predicted", tag)
# append both to file for evaluation
eval_line = "{} {} {} {}\n".format(
token.text,
token.get_tag(self.tag_type).value,
tag.value,
tag.score,
)
# lines.append(eval_line)
if out_path is not None:
outfile.write(eval_line)
# lines.append("\n")
if out_path is not None:
outfile.write("\n")
for sentence in batch:
# make list of gold tags
gold_tags = [
(tag.tag, str(tag))
for tag in sentence.get_spans(self.tag_type)
]
# make list of predicted tags
predicted_tags = [
(tag.tag, str(tag))
for tag in sentence.get_spans("predicted")
]
# check for true positives, false positives and false negatives
for tag, prediction in predicted_tags:
if (tag, prediction) in gold_tags:
metric.add_tp(tag)
else:
metric.add_fp(tag)
for tag, gold in gold_tags:
if (tag, gold) not in predicted_tags:
metric.add_fn(tag)
else:
metric.add_tn(tag)
eval_loss /= batch_no
if out_path is not None:
outfile.close()
detailed_result = (
f"\nMICRO_AVG: acc {metric.micro_avg_accuracy()} - f1-score {metric.micro_avg_f_score()}"
f"\nMACRO_AVG: acc {metric.macro_avg_accuracy()} - f1-score {metric.macro_avg_f_score()}"
)
for class_name in metric.get_classes():
detailed_result += (
f"\n{class_name:<10} tp: {metric.get_tp(class_name)} - fp: {metric.get_fp(class_name)} - "
f"fn: {metric.get_fn(class_name)} - tn: {metric.get_tn(class_name)} - precision: "
f"{metric.precision(class_name):.4f} - recall: {metric.recall(class_name):.4f} - "
f"accuracy: {metric.accuracy(class_name):.4f} - f1-score: "
f"{metric.f_score(class_name):.4f}")
result = Result(
main_score=metric.micro_avg_f_score(),
log_line=
f"{metric.precision()}\t{metric.recall()}\t{metric.micro_avg_f_score()}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=detailed_result,
)
return result, eval_loss
def dependency_evaluate(self,
loader,
out_path=None,
prediction_mode=False):
# self.model.eval()
loss, metric = 0, Metric()
# total_start_time=time.time()
# forward_time=0
# loss_time=0
# decode_time=0
# punct_time=0
lines = []
for batch in loader:
forward_start = time.time()
arc_scores, rel_scores = self.forward(batch)
# forward_end=time.time()
mask = self.mask
if not prediction_mode:
loss += self._calculate_loss(arc_scores, rel_scores, batch,
mask)
# loss_end=time.time()
# forward_time+=forward_end-forward_start
# loss_time+=loss_end-forward_end
mask = mask.bool()
# decode_start=time.time()
arc_preds, rel_preds = self.decode(arc_scores, rel_scores, mask)
# decode_end=time.time()
# decode_time+=decode_end-decode_start
# ignore all punctuation if not specified
# if out_path is not None:
# pdb.set_trace()
if not self.punct:
for sent_id, sentence in enumerate(batch):
for token_id, token in enumerate(sentence):
upos = token.get_tag('upos').value
xpos = token.get_tag('pos').value
word = token.text
if is_punctuation(word, upos,
self.punct_list) or is_punctuation(
word, upos, self.punct_list):
mask[sent_id][token_id] = 0
# mask &= words.unsqueeze(-1).ne(self.puncts).all(-1)
if out_path is not None:
for (sent_idx, sentence) in enumerate(batch):
for token_idx, token in enumerate(sentence):
if token_idx == 0:
continue
# append both to file for evaluation
eval_line = "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n".format(
token_idx,
token.text,
'X',
'X',
'X',
'X',
arc_preds[sent_idx, token_idx],
self.tag_dictionary.get_item_for_index(
rel_preds[sent_idx, token_idx]),
'X',
'X',
)
lines.append(eval_line)
lines.append("\n")
if not prediction_mode:
# punct_end=time.time()
# punct_time+=punct_end-decode_end
metric(arc_preds, rel_preds, self.arcs, self.rels, mask)
# if out_path is not None:
# with open(out_path, "w", encoding="utf-8") as outfile:
# outfile.write("".join(lines))
if prediction_mode:
return None, None
# total_end_time=time.time()
# print(total_start_time-total_end_time)
# print(forward_time)
# print(punct_time)
# print(decode_time)
loss /= len(loader)
return loss, metric
def _evaluate_with_span_F1(self, data_loader, embedding_storage_mode, mini_batch_size, out_path):
eval_loss = 0
batch_no: int = 0
metric = Metric("Evaluation", beta=self.beta)
lines: List[str] = []
y_true = []
y_pred = []
for batch in data_loader:
# predict for batch
loss = self.predict(batch,
embedding_storage_mode=embedding_storage_mode,
mini_batch_size=mini_batch_size,
label_name='predicted',
return_loss=True)
eval_loss += loss
batch_no += 1
for sentence in batch:
# make list of gold tags
gold_spans = sentence.get_spans(self.tag_type)
gold_tags = [(span.tag, repr(span)) for span in gold_spans]
# make list of predicted tags
predicted_spans = sentence.get_spans("predicted")
predicted_tags = [(span.tag, repr(span)) for span in predicted_spans]
# check for true positives, false positives and false negatives
for tag, prediction in predicted_tags:
if (tag, prediction) in gold_tags:
metric.add_tp(tag)
else:
metric.add_fp(tag)
for tag, gold in gold_tags:
if (tag, gold) not in predicted_tags:
metric.add_fn(tag)
tags_gold = []
tags_pred = []
# also write to file in BIO format to use old conlleval script
if out_path:
for token in sentence:
# check if in gold spans
gold_tag = 'O'
for span in gold_spans:
if token in span:
gold_tag = 'B-' + span.tag if token == span[0] else 'I-' + span.tag
tags_gold.append(gold_tag)
predicted_tag = 'O'
# check if in predicted spans
for span in predicted_spans:
if token in span:
predicted_tag = 'B-' + span.tag if token == span[0] else 'I-' + span.tag
tags_pred.append(predicted_tag)
lines.append(f'{token.text} {gold_tag} {predicted_tag}\n')
lines.append('\n')
y_true.append(tags_gold)
y_pred.append(tags_pred)
if out_path:
with open(Path(out_path), "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))
eval_loss /= batch_no
detailed_result = (
"\nResults:"
f"\n- F1-score (micro) {metric.micro_avg_f_score():.4f}"
f"\n- F1-score (macro) {metric.macro_avg_f_score():.4f}"
'\n\nBy class:'
)
for class_name in metric.get_classes():
detailed_result += (
f"\n{class_name:<10} tp: {metric.get_tp(class_name)} - fp: {metric.get_fp(class_name)} - "
f"fn: {metric.get_fn(class_name)} - precision: "
f"{metric.precision(class_name):.4f} - recall: {metric.recall(class_name):.4f} - "
f"f1-score: "
f"{metric.f_score(class_name):.4f}"
)
result = Result(
main_score=metric.micro_avg_f_score(),
log_line=f"{metric.precision():.4f}\t{metric.recall():.4f}\t{metric.micro_avg_f_score():.4f}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=detailed_result,
)
return result, eval_loss
def evaluate(
self,
data_loader: DataLoader,
out_path: Path = None,
embeddings_storage_mode: str = "none",
eval_mode: EvalMode = EvalMode.Standard,
misspell_mode: MisspellingMode = MisspellingMode.Random,
misspelling_rate: float = 0.0,
char_vocab: set = {},
lut: dict = {},
cmx: np.array = None,
typos: dict = {},
correction_mode: CorrectionMode = CorrectionMode.NotSpecified,
eval_dict_name=None,
evaluation_metric: EvaluationMetric = EvaluationMetric.MICRO_F1_SCORE,
) -> (Result, float):
if type(out_path) == str:
out_path = Path(out_path)
from robust_ner.spellcheck import load_correction_dict, get_lang_from_corpus_name
if correction_mode == CorrectionMode.NotSpecified:
eval_dict = None
else:
eval_dict = load_correction_dict(eval_dict_name, log)
# note: use 'save_correction_dict' to re-generate a dictionary
lang = get_lang_from_corpus_name(eval_dict_name)
eval_params = {}
eval_params["eval_mode"] = eval_mode
eval_params["misspelling_rate"] = misspelling_rate
eval_params["misspell_mode"] = misspell_mode
eval_params["char_vocab"] = char_vocab
eval_params["lut"] = lut
eval_params["cmx"] = cmx
eval_params["typos"] = typos
eval_params["correction_mode"] = correction_mode
eval_params["lang"] = lang
eval_params["dictionary"] = eval_dict
with torch.no_grad():
eval_loss = 0
batch_no: int = 0
metric = Metric("Evaluation")
lines: List[str] = []
if self.use_crf:
transitions = self.transitions.detach().cpu().numpy()
else:
transitions = None
for batch in data_loader:
batch_no += 1
with torch.no_grad():
features = self.forward(batch, eval_params)
loss = self._calculate_loss(features, batch)
tags, _ = self._obtain_labels(
feature=features,
batch_sentences=batch,
transitions=transitions,
get_all_tags=False,
)
eval_loss += loss
for (sentence, sent_tags) in zip(batch, tags):
for (token, tag) in zip(sentence.tokens, sent_tags):
token: Token = token
token.add_tag("predicted", tag.value, tag.score)
# append both to file for evaluation
eval_line = "{} {} {} {}\n".format(
token.text,
token.get_tag(self.tag_type).value,
tag.value,
tag.score,
)
lines.append(eval_line)
lines.append("\n")
for sentence in batch:
# make list of gold tags
gold_tags = [(tag.tag, tag.text)
for tag in sentence.get_spans(self.tag_type)]
# make list of predicted tags
predicted_tags = [
(tag.tag, tag.text)
for tag in sentence.get_spans("predicted")
]
# check for true positives, false positives and false negatives
for tag, prediction in predicted_tags:
if (tag, prediction) in gold_tags:
metric.add_tp(tag)
else:
metric.add_fp(tag)
for tag, gold in gold_tags:
if (tag, gold) not in predicted_tags:
metric.add_fn(tag)
else:
metric.add_tn(tag)
store_embeddings(batch, embeddings_storage_mode)
eval_loss /= batch_no
if out_path is not None:
with open(out_path, "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))
detailed_result = (
f"\nMICRO_AVG: acc {metric.micro_avg_accuracy():.4f} - f1-score {metric.micro_avg_f_score():.4f}"
f"\nMACRO_AVG: acc {metric.macro_avg_accuracy():.4f} - f1-score {metric.macro_avg_f_score():.4f}"
)
for class_name in metric.get_classes():
detailed_result += (
f"\n{class_name:<10} tp: {metric.get_tp(class_name)} - fp: {metric.get_fp(class_name)} - "
f"fn: {metric.get_fn(class_name)} - tn: {metric.get_tn(class_name)} - precision: "
f"{metric.precision(class_name):.4f} - recall: {metric.recall(class_name):.4f} - "
f"accuracy: {metric.accuracy(class_name):.4f} - f1-score: "
f"{metric.f_score(class_name):.4f}")
if evaluation_metric == EvaluationMetric.MICRO_F1_SCORE:
main_score = metric.micro_avg_f_score()
elif evaluation_metric == EvaluationMetric.MACRO_F1_SCORE:
main_score = metric.macro_avg_f_score()
elif evaluation_metric == EvaluationMetric.MICRO_ACCURACY:
main_score = metric.micro_avg_accuracy()
elif evaluation_metric == EvaluationMetric.MACRO_ACCURACY:
main_score = metric.macro_avg_accuracy()
elif evaluation_metric == EvaluationMetric.MEAN_SQUARED_ERROR:
main_score = metric.mean_squared_error()
else:
log.error(f"unknown evaluation metric: {evaluation_metric}")
result = Result(
main_score=main_score,
log_line=
f"{metric.precision():.4f}\t{metric.recall():.4f}\t{main_score:.4f}",
log_header="PRECISION\tRECALL\tF1",
detailed_results=detailed_result,
)
return result, eval_loss