def _evaluate_text_classifier(model: flair.nn.Model,
sentences: List[Sentence],
eval_mini_batch_size: int = 32,
embeddings_in_memory: bool = False,
out_path: Path = None) -> (dict, float):
with torch.no_grad():
eval_loss = 0
batches = [
sentences[x:x + eval_mini_batch_size]
for x in range(0, len(sentences), eval_mini_batch_size)
]
metric = Metric('Evaluation')
lines: List[str] = []
for batch in batches:
labels, loss = model.forward_labels_and_loss(batch)
clear_embeddings(
batch, also_clear_word_embeddings=not embeddings_in_memory)
eval_loss += loss
sentences_for_batch = [
sent.to_plain_string() for sent in batch
]
confidences_for_batch = [[
label.score for label in sent_labels
] for sent_labels in labels]
predictions_for_batch = [[
label.value for label in sent_labels
] for sent_labels in labels]
true_values_for_batch = [
sentence.get_label_names() for sentence in batch
]
available_labels = model.label_dictionary.get_items()
for sentence, confidence, prediction, true_value in zip(
sentences_for_batch, confidences_for_batch,
predictions_for_batch, true_values_for_batch):
eval_line = '{}\t{}\t{}\t{}\n'.format(
sentence, true_value, prediction, confidence)
lines.append(eval_line)
for predictions_for_sentence, true_values_for_sentence in zip(
predictions_for_batch, true_values_for_batch):
ModelTrainer._evaluate_sentence_for_text_classification(
metric, available_labels, predictions_for_sentence,
true_values_for_sentence)
eval_loss /= len(sentences)
if out_path is not None:
with open(out_path, "w", encoding='utf-8') as outfile:
outfile.write(''.join(lines))
return metric, eval_loss
def _evaluate_text_classifier(model: flair.nn.Model,
sentences: List[Sentence],
eval_mini_batch_size: int = 32,
embeddings_in_memory: bool = False) -> (dict, float):
with torch.no_grad():
eval_loss = 0
batches = [sentences[x:x + eval_mini_batch_size] for x in
range(0, len(sentences), eval_mini_batch_size)]
metric = Metric('Evaluation')
for batch in batches:
labels, loss = model.forward_labels_and_loss(batch)
clear_embeddings(batch, also_clear_word_embeddings=not embeddings_in_memory)
eval_loss += loss
predictions_for_batch = [[label.value for label in sent_labels] for sent_labels in labels]
true_values_for_batch = [sentence.get_label_names() for sentence in batch]
available_labels = model.label_dictionary.get_items()
for predictions_for_sentence, true_values_for_sentence in zip(predictions_for_batch, true_values_for_batch):
ModelTrainer._evaluate_sentence_for_text_classification(metric,
available_labels,
predictions_for_sentence,
true_values_for_sentence)
eval_loss /= len(sentences)
return metric, eval_loss
def _evaluate_text_regressor(model: flair.nn.Model,
sentences: List[Sentence],
eval_mini_batch_size: int = 32,
embeddings_in_memory: bool = False,
out_path: Path = None) -> (dict, float):
with torch.no_grad():
eval_loss = 0
batches = [
sentences[x:x + eval_mini_batch_size]
for x in range(0, len(sentences), eval_mini_batch_size)
]
metric = MetricRegression('Evaluation')
lines: List[str] = []
for batch in batches:
scores, loss = model.forward_labels_and_loss(batch)
true_values = []
for sentence in batch:
for label in sentence.labels:
true_values.append(float(label.value))
results = []
for score in scores:
if type(score[0]) is Label:
results.append(float(score[0].score))
else:
results.append(float(score[0]))
clear_embeddings(
batch, also_clear_word_embeddings=not embeddings_in_memory)
eval_loss += loss
metric.true.extend(true_values)
metric.pred.extend(results)
eval_loss /= len(sentences)
##TODO: not saving lines yet
if out_path is not None:
with open(out_path, "w", encoding='utf-8') as outfile:
outfile.write(''.join(lines))
return metric, eval_loss
def _evaluate_text_classifier(
model: flair.nn.Model,
sentences: List[Sentence],
eval_mini_batch_size: int = 32,
embeddings_in_memory: bool = False) -> (dict, float):
with torch.no_grad():
eval_loss = 0
batches = [
sentences[x:x + eval_mini_batch_size]
for x in range(0, len(sentences), eval_mini_batch_size)
]
metric = Metric('Evaluation')
for batch in batches:
labels, loss = model.forward_labels_and_loss(batch)
clear_embeddings(
batch, also_clear_word_embeddings=not embeddings_in_memory)
eval_loss += loss
for predictions, true_values in zip(
[[label.value for label in sent_labels]
for sent_labels in labels],
[sentence.get_label_names() for sentence in batch]):
for prediction in predictions:
if prediction in true_values:
metric.add_tp(prediction)
else:
metric.add_fp(prediction)
for true_value in true_values:
if true_value not in predictions:
metric.add_fn(true_value)
else:
metric.add_tn(true_value)
eval_loss /= len(sentences)
return metric, eval_loss