def test_paired_euclidean_distances():
# Check the paired Euclidean distances computation
X = [[0], [0]]
Y = [[1], [2]]
D = paired_euclidean_distances(X, Y)
assert_array_almost_equal(D, [1., 2.])
def __call__(self, model, output_path: str = None, epoch: int = -1, steps: int = -1) -> float:
model.eval()
if epoch != -1:
if steps == -1:
out_txt = " after epoch {}:".format(epoch)
else:
out_txt = " in epoch {} after {} steps:".format(epoch, steps)
else:
out_txt = ":"
logging.info("Evaluation the model on "+self.name+" dataset"+out_txt)
num_triplets = 0
num_correct_cos_triplets, num_correct_manhatten_triplets, num_correct_euclidean_triplets = 0, 0, 0
self.dataloader.collate_fn = model.smart_batching_collate
for step, batch in enumerate(tqdm(self.dataloader, desc="Evaluating")):
features, label_ids = batch_to_device(batch, self.device)
with torch.no_grad():
emb1, emb2, emb3 = [model(sent_features)['sentence_embedding'].to("cpu").numpy() for sent_features in features]
#Cosine distance
pos_cos_distance = paired_cosine_distances(emb1, emb2)
neg_cos_distances = paired_cosine_distances(emb1, emb3)
# Manhatten
pos_manhatten_distance = paired_manhattan_distances(emb1, emb2)
neg_manhatten_distances = paired_manhattan_distances(emb1, emb3)
# Euclidean
pos_euclidean_distance = paired_euclidean_distances(emb1, emb2)
neg_euclidean_distances = paired_euclidean_distances(emb1, emb3)
for idx in range(len(pos_cos_distance)):
num_triplets += 1
if pos_cos_distance[idx] < neg_cos_distances[idx]:
num_correct_cos_triplets += 1
if pos_manhatten_distance[idx] < neg_manhatten_distances[idx]:
num_correct_manhatten_triplets += 1
if pos_euclidean_distance[idx] < neg_euclidean_distances[idx]:
num_correct_euclidean_triplets += 1
accuracy_cos = num_correct_cos_triplets / num_triplets
accuracy_manhatten = num_correct_manhatten_triplets / num_triplets
accuracy_euclidean = num_correct_euclidean_triplets / num_triplets
logging.info("Accuracy Cosine Distance:\t{:.4f}".format(accuracy_cos))
logging.info("Accuracy Manhatten Distance:\t{:.4f}".format(accuracy_manhatten))
logging.info("Accuracy Euclidean Distance:\t{:.4f}\n".format(accuracy_euclidean))
if output_path is not None:
csv_path = os.path.join(output_path, self.csv_file)
if not os.path.isfile(csv_path):
with open(csv_path, mode="w", encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow(self.csv_headers)
writer.writerow([epoch, steps, accuracy_cos, accuracy_manhatten, accuracy_euclidean])
else:
store = []
with open(csv_path, mode="r", encoding="utf-8") as f:
read = csv.reader(f)
for obj in read:
if obj!=[]:
store.append(obj)
with open(csv_path, mode="w", encoding="utf-8") as f:
writer = csv.writer(f)
for row in store:
writer.writerow(row)
writer.writerow([epoch, steps, accuracy_cos, accuracy_manhatten, accuracy_euclidean])
if self.main_distance_function == SimilarityFunction.COSINE:
return accuracy_cos
if self.main_distance_function == SimilarityFunction.MANHATTAN:
return accuracy_manhatten
if self.main_distance_function == SimilarityFunction.EUCLIDEAN:
return accuracy_euclidean
return max(accuracy_cos, accuracy_manhatten, accuracy_euclidean)
def __call__(self,
model,
output_path: str = None,
epoch: int = -1,
steps: int = -1) -> float:
model.eval()
embeddings1 = []
embeddings2 = []
labels = []
if epoch != -1:
if steps == -1:
out_txt = f" after epoch {epoch}:"
else:
out_txt = f" in epoch {epoch} after {steps} steps:"
else:
out_txt = ":"
logging.info("Evaluation the model on " + self.name + " dataset" +
out_txt)
self.dataloader.collate_fn = model.smart_batching_collate
for step, batch in enumerate(tqdm(self.dataloader, desc="Evaluating")):
features, label_ids = batch_to_device(batch, self.device)
with torch.no_grad():
emb1, emb2 = [
model(sent_features)['sentence_embedding'].to(
"cpu").numpy() for sent_features in features
]
labels.extend(label_ids.to("cpu").numpy())
embeddings1.extend(emb1)
embeddings2.extend(emb2)
cosine_scores = 1 - paired_cosine_distances(embeddings1, embeddings2)
manhattan_distances = paired_manhattan_distances(
embeddings1, embeddings2)
euclidean_distances = paired_euclidean_distances(
embeddings1, embeddings2)
# Ensure labels are just 0 or 1
for label in labels:
assert (label == 0 or label == 1)
labels = np.asarray(labels)
cosine_acc, cosine_threshold = self.find_best_acc_and_threshold(
cosine_scores, labels, True)
manhattan_acc, manhatten_threshold = self.find_best_acc_and_threshold(
manhattan_distances, labels, False)
euclidean_acc, euclidean_threshold = self.find_best_acc_and_threshold(
euclidean_distances, labels, False)
logging.info(
"Accuracy with Cosine-Similarity:\t{:.2f}\t(Threshold: {:.4f})".
format(cosine_acc * 100, cosine_threshold))
logging.info(
"Accuracy with Manhattan-Distance:\t{:.2f}\t(Threshold: {:.4f})".
format(manhattan_acc * 100, manhatten_threshold))
logging.info(
"Accuracy with Euclidean-Distance:\t{:.2f}\t(Threshold: {:.4f})\n".
format(euclidean_acc * 100, euclidean_threshold))
if output_path is not None:
csv_path = os.path.join(output_path, self.csv_file)
if not os.path.isfile(csv_path):
with open(csv_path, mode="w", encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow(self.csv_headers)
writer.writerow([
epoch, steps, cosine_acc, euclidean_acc, manhattan_acc
])
else:
with open(csv_path, mode="a", encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow([
epoch, steps, cosine_acc, euclidean_acc, manhattan_acc
])
if self.main_similarity == SimilarityFunction.COSINE:
return cosine_acc
elif self.main_similarity == SimilarityFunction.EUCLIDEAN:
return euclidean_acc
elif self.main_similarity == SimilarityFunction.MANHATTAN:
return manhattan_acc
else:
raise ValueError("Unknown main_similarity value")
def test_paired_euclidean_distances():
# Check the paired Euclidean distances computation
X = [[0], [0]]
Y = [[1], [2]]
D = paired_euclidean_distances(X, Y)
assert_array_almost_equal(D, [1., 2.])
def __call__(self, model: 'SequentialSentenceEmbedder', output_path: str = None, epoch: int = -1, steps: int = -1) -> float:
model.eval()
embeddings1 = []
embeddings2 = []
labels = []
if epoch != -1:
if steps == -1:
out_txt = " after epoch {}:".format(epoch)
else:
out_txt = " in epoch {} after {} steps:".format(epoch, steps)
else:
out_txt = ":"
logging.info("Evaluation the model on "+self.name+" dataset"+out_txt)
self.dataloader.collate_fn = model.smart_batching_collate
iterator = self.dataloader
if self.show_progress_bar:
iterator = tqdm(iterator, desc="Convert Evaluating")
for step, batch in enumerate(iterator):
features, label_ids = batch_to_device(batch, self.device)
with torch.no_grad():
emb1, emb2 = [model(sent_features)['sentence_embedding'].to("cpu").numpy() for sent_features in features]
labels.extend(label_ids.to("cpu").numpy())
embeddings1.extend(emb1)
embeddings2.extend(emb2)
try:
cosine_scores = 1 - (paired_cosine_distances(embeddings1, embeddings2))
except Exception as e:
print(embeddings1)
print(embeddings2)
raise(e)
manhattan_distances = -paired_manhattan_distances(embeddings1, embeddings2)
euclidean_distances = -paired_euclidean_distances(embeddings1, embeddings2)
dot_products = [np.dot(emb1, emb2) for emb1, emb2 in zip(embeddings1, embeddings2)]
eval_pearson_cosine, _ = pearsonr(labels, cosine_scores)
eval_spearman_cosine, _ = spearmanr(labels, cosine_scores)
eval_pearson_manhattan, _ = pearsonr(labels, manhattan_distances)
eval_spearman_manhattan, _ = spearmanr(labels, manhattan_distances)
eval_pearson_euclidean, _ = pearsonr(labels, euclidean_distances)
eval_spearman_euclidean, _ = spearmanr(labels, euclidean_distances)
eval_pearson_dot, _ = pearsonr(labels, dot_products)
eval_spearman_dot, _ = spearmanr(labels, dot_products)
logging.info("Cosine-Similarity :\tPearson: {:.4f}\tSpearman: {:.4f}".format(
eval_pearson_cosine, eval_spearman_cosine))
logging.info("Manhattan-Distance:\tPearson: {:.4f}\tSpearman: {:.4f}".format(
eval_pearson_manhattan, eval_spearman_manhattan))
logging.info("Euclidean-Distance:\tPearson: {:.4f}\tSpearman: {:.4f}".format(
eval_pearson_euclidean, eval_spearman_euclidean))
logging.info("Dot-Product-Similarity:\tPearson: {:.4f}\tSpearman: {:.4f}".format(
eval_pearson_dot, eval_spearman_dot))
if output_path is not None:
csv_path = os.path.join(output_path, self.csv_file)
output_file_exists = os.path.isfile(csv_path)
with open(csv_path, mode="a" if output_file_exists else 'w', encoding="utf-8") as f:
writer = csv.writer(f)
if not output_file_exists:
writer.writerow(self.csv_headers)
writer.writerow([epoch, steps, eval_pearson_cosine, eval_spearman_cosine, eval_pearson_euclidean,
eval_spearman_euclidean, eval_pearson_manhattan, eval_spearman_manhattan, eval_pearson_dot, eval_spearman_dot])
if self.main_similarity == SimilarityFunction.COSINE:
return eval_spearman_cosine
elif self.main_similarity == SimilarityFunction.EUCLIDEAN:
return eval_spearman_euclidean
elif self.main_similarity == SimilarityFunction.MANHATTAN:
return eval_spearman_manhattan
elif self.main_similarity == SimilarityFunction.DOT_PRODUCT:
return eval_spearman_dot
elif self.main_similarity is None:
return max(eval_spearman_cosine, eval_spearman_manhattan, eval_spearman_euclidean, eval_spearman_dot)
else:
raise ValueError("Unknown main_similarity value")
def __call__(self,
model,
output_path: str = None,
epoch: int = -1,
steps: int = -1) -> float:
if epoch != -1:
if steps == -1:
out_txt = f" after epoch {epoch}:"
else:
out_txt = f" in epoch {epoch} after {steps} steps:"
else:
out_txt = ":"
logger.info("Binary Accuracy Evaluation of the model on " + self.name +
" dataset" + out_txt)
embeddings1 = model.encode(self.sentences1,
batch_size=self.batch_size,
show_progress_bar=self.show_progress_bar,
convert_to_numpy=True)
embeddings2 = model.encode(self.sentences2,
batch_size=self.batch_size,
show_progress_bar=self.show_progress_bar,
convert_to_numpy=True)
cosine_scores = 1 - paired_cosine_distances(embeddings1, embeddings2)
manhattan_distances = paired_manhattan_distances(
embeddings1, embeddings2)
euclidean_distances = paired_euclidean_distances(
embeddings1, embeddings2)
labels = np.asarray(self.labels)
file_output_data = [epoch, steps]
main_score = None
for name, scores, reverse in [[
'Cosine-Similarity', cosine_scores, True
], ['Manhatten-Distance', manhattan_distances,
False], ['Euclidean-Distance', euclidean_distances, False]]:
acc, acc_threshold = self.find_best_acc_and_threshold(
scores, labels, reverse)
f1, precision, recall, f1_threshold = self.find_best_f1_and_threshold(
scores, labels, reverse)
ap = average_precision_score(labels,
scores * (1 if reverse else -1))
logger.info(
"Accuracy with {}: {:.2f}\t(Threshold: {:.4f})".
format(name, acc * 100, acc_threshold))
logger.info(
"F1 with {}: {:.2f}\t(Threshold: {:.4f})".
format(name, f1 * 100, f1_threshold))
logger.info("Precision with {}: {:.2f}".format(
name, precision * 100))
logger.info("Recall with {}: {:.2f}".format(
name, recall * 100))
logger.info("Average Precision with {}: {:.2f}\n".format(
name, ap * 100))
file_output_data.extend(
[acc, acc_threshold, f1, precision, recall, f1_threshold, ap])
if main_score is None: #Use AveragePrecision with Cosine-Similarity as main score
main_score = ap
if output_path is not None:
csv_path = os.path.join(output_path, self.csv_file)
if not os.path.isfile(csv_path):
with open(csv_path, mode="w", encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow(self.csv_headers)
writer.writerow(file_output_data)
else:
with open(csv_path, mode="a", encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow(file_output_data)
return main_score
