def get_word_translation_accuracy(lang1, word2id1, emb1, lang2, word2id2, emb2,
method, dico_eval):
"""
Given source and target word embeddings, and a dictionary,
evaluate the translation accuracy using the precision@k.
"""
if dico_eval == 'default':
path = os.path.join(DIC_EVAL_PATH,
'%s-%s.5000-6500.txt' % (lang1, lang2))
else:
path = dico_eval
dico = load_dictionary(path, word2id1, word2id2)
dico = dico.cuda() if emb1.is_cuda else dico
assert dico[:, 0].max() < emb1.size(0)
assert dico[:, 1].max() < emb2.size(0)
# normalize word embeddings
emb1 = emb1 / emb1.norm(2, 1, keepdim=True).expand_as(emb1)
emb2 = emb2 / emb2.norm(2, 1, keepdim=True).expand_as(emb2)
# nearest neighbors
if method == 'nn':
query = emb1[dico[:, 0]]
scores = query.mm(emb2.transpose(0, 1))
# inverted softmax
elif method.startswith('invsm_beta_'):
beta = float(method[len('invsm_beta_'):])
bs = 128
word_scores = []
for i in range(0, emb2.size(0), bs):
scores = emb1.mm(emb2[i:i + bs].transpose(0, 1))
scores.mul_(beta).exp_()
scores.div_(scores.sum(0, keepdim=True).expand_as(scores))
word_scores.append(scores.index_select(0, dico[:, 0]))
scores = torch.cat(word_scores, 1)
# contextual dissimilarity measure
elif method.startswith('csls_knn_'):
# average distances to k nearest neighbors
knn = method[len('csls_knn_'):]
assert knn.isdigit()
knn = int(knn)
average_dist1 = get_nn_avg_dist(emb2, emb1, knn)
average_dist2 = get_nn_avg_dist(emb1, emb2, knn)
average_dist1 = torch.from_numpy(average_dist1).type_as(emb1)
average_dist2 = torch.from_numpy(average_dist2).type_as(emb2)
# queries / scores
query = emb1[dico[:, 0]]
scores = query.mm(emb2.transpose(0, 1))
scores.mul_(2)
scores.sub_(average_dist1[dico[:, 0]][:, None])
scores.sub_(average_dist2[None, :])
else:
raise Exception('Unknown method: "%s"' % method)
results = []
top_matches = scores.topk(10, 1, True)[1]
for k in [1, 5, 10]:
top_k_matches = top_matches[:, :k]
_matching = (
top_k_matches == dico[:, 1][:,
None].expand_as(top_k_matches)).sum(1)
# allow for multiple possible translations
matching = {}
for i, src_id in enumerate(dico[:, 0].cpu().numpy()):
matching[src_id] = min(matching.get(src_id, 0) + _matching[i], 1)
# evaluate precision@k
precision_at_k = 100 * np.mean(list(matching.values()))
logger.info("%i source words - %s - Precision at k = %i: %f" %
(len(matching), method, k, precision_at_k))
results.append(('precision_at_%i' % k, precision_at_k))
return results
def get_candidates(emb1, emb2, params):
"""
Get best translation pairs candidates.
"""
bs = 128
all_scores = []
all_targets = []
# number of source words to consider
n_src = emb1.size(0)
if params['dico_max_rank'] > 0 and not params['dico_method'].startswith(
'invsm_beta_'):
n_src = params['dico_max_rank']
# nearest neighbors
if params['dico_method'] == 'nn':
# for every source word
for i in range(0, n_src, bs):
# compute target words scores
scores = emb2.mm(emb1[i:min(n_src, i + bs)].transpose(
0, 1)).transpose(0, 1)
best_scores, best_targets = scores.topk(2,
dim=1,
largest=True,
sorted=True)
# update scores / potential targets
all_scores.append(best_scores.cpu())
all_targets.append(best_targets.cpu())
all_scores = torch.cat(all_scores, 0)
all_targets = torch.cat(all_targets, 0)
# inverted softmax
elif params['dico_method'].startswith('invsm_beta_'):
beta = float(params['dico_method'][len('invsm_beta_'):])
# for every target word
for i in range(0, emb2.size(0), bs):
# compute source words scores
scores = emb1.mm(emb2[i:i + bs].transpose(0, 1))
scores.mul_(beta).exp_()
scores.div_(scores.sum(0, keepdim=True).expand_as(scores))
best_scores, best_targets = scores.topk(2,
dim=1,
largest=True,
sorted=True)
# update scores / potential targets
all_scores.append(best_scores.cpu())
all_targets.append((best_targets + i).cpu())
all_scores = torch.cat(all_scores, 1)
all_targets = torch.cat(all_targets, 1)
all_scores, best_targets = all_scores.topk(2,
dim=1,
largest=True,
sorted=True)
all_targets = all_targets.gather(1, best_targets)
# contextual dissimilarity measure
elif params['dico_method'].startswith('csls_knn_'):
knn = params['dico_method'][len('csls_knn_'):]
assert knn.isdigit()
knn = int(knn)
# average distances to k nearest neighbors
average_dist1 = torch.from_numpy(get_nn_avg_dist(emb2, emb1, knn))
average_dist2 = torch.from_numpy(get_nn_avg_dist(emb1, emb2, knn))
average_dist1 = average_dist1.type_as(emb1)
average_dist2 = average_dist2.type_as(emb2)
# for every source word
for i in range(0, n_src, bs):
# compute target words scores
scores = emb2.mm(emb1[i:min(n_src, i + bs)].transpose(
0, 1)).transpose(0, 1)
scores.mul_(2)
scores.sub_(average_dist1[i:min(n_src, i + bs)][:, None] +
average_dist2[None, :])
best_scores, best_targets = scores.topk(2,
dim=1,
largest=True,
sorted=True)
# update scores / potential targets
all_scores.append(best_scores.cpu())
all_targets.append(best_targets.cpu())
all_scores = torch.cat(all_scores, 0)
all_targets = torch.cat(all_targets, 0)
all_pairs = torch.cat([
torch.arange(0, all_targets.size(0)).long().unsqueeze(1),
all_targets[:, 0].unsqueeze(1)
], 1)
# sanity check
assert all_scores.size() == all_pairs.size() == (n_src, 2)
# sort pairs by score confidence
diff = all_scores[:, 0] - all_scores[:, 1]
reordered = diff.sort(0, descending=True)[1]
all_scores = all_scores[reordered]
all_pairs = all_pairs[reordered]
# max dico words rank
if params['dico_max_rank'] > 0:
selected = all_pairs.max(1)[0] <= params['dico_max_rank']
mask = selected.unsqueeze(1).expand_as(all_scores).clone()
all_scores = all_scores.masked_select(mask).view(-1, 2)
all_pairs = all_pairs.masked_select(mask).view(-1, 2)
# max dico size
if params['dico_max_size'] > 0:
all_scores = all_scores[:params['dico_max_size']]
all_pairs = all_pairs[:params['dico_max_size']]
# min dico size
diff = all_scores[:, 0] - all_scores[:, 1]
if params['dico_min_size'] > 0:
diff[:params['dico_min_size']] = 1e9
# confidence threshold
if params['dico_threshold'] > 0:
mask = diff > params['dico_threshold']
print("Selected %i / %i pairs above the confidence threshold." %
(mask.sum(), diff.size(0)))
mask = mask.unsqueeze(1).expand_as(all_pairs).clone()
all_pairs = all_pairs.masked_select(mask).view(-1, 2)
return all_pairs
def get_sent_translation_accuracy(data, lg1, word2id1, emb1, lg2, word2id2, emb2,
n_keys, n_queries, method, idf):
"""
Given parallel sentences from Europarl, evaluate the
sentence translation accuracy using the precision@k.
"""
# get word vectors dictionaries
emb1 = emb1.cpu().numpy()
emb2 = emb2.cpu().numpy()
word_vec1 = dict([(w, emb1[word2id1[w]]) for w in word2id1])
word_vec2 = dict([(w, emb2[word2id2[w]]) for w in word2id2])
word_vect = {lg1: word_vec1, lg2: word_vec2}
lg_keys = lg2
lg_query = lg1
# get n_keys pairs of sentences
keys = data[lg_keys][:n_keys]
keys = bow_idf(keys, word_vect[lg_keys], idf_dict=idf[lg_keys])
# get n_queries query pairs from these n_keys pairs
rng = np.random.RandomState(1234)
idx_query = rng.choice(range(n_keys), size=n_queries, replace=False)
queries = data[lg_query][idx_query]
queries = bow_idf(queries, word_vect[lg_query], idf_dict=idf[lg_query])
# normalize embeddings
queries = torch.from_numpy(queries).float()
queries = queries / queries.norm(2, 1, keepdim=True).expand_as(queries)
keys = torch.from_numpy(keys).float()
keys = keys / keys.norm(2, 1, keepdim=True).expand_as(keys)
# nearest neighbors
if method == 'nn':
scores = keys.mm(queries.transpose(0, 1)).transpose(0, 1)
scores = scores.cpu()
# inverted softmax
elif method.startswith('invsm_beta_'):
beta = float(method[len('invsm_beta_'):])
scores = keys.mm(queries.transpose(0, 1)).transpose(0, 1)
scores.mul_(beta).exp_()
scores.div_(scores.sum(0, keepdim=True).expand_as(scores))
scores = scores.cpu()
# contextual dissimilarity measure
elif method.startswith('csls_knn_'):
knn = method[len('csls_knn_'):]
assert knn.isdigit()
knn = int(knn)
# average distances to k nearest neighbors
knn = method[len('csls_knn_'):]
assert knn.isdigit()
knn = int(knn)
average_dist_keys = torch.from_numpy(get_nn_avg_dist(queries, keys, knn))
average_dist_queries = torch.from_numpy(get_nn_avg_dist(keys, queries, knn))
# scores
scores = keys.mm(queries.transpose(0, 1)).transpose(0, 1)
scores.mul_(2)
scores.sub_(average_dist_queries[:, None].float() + average_dist_keys[None, :].float())
scores = scores.cpu()
results = []
top_matches = scores.topk(10, 1, True)[1]
for k in [1, 5, 10]:
top_k_matches = (top_matches[:, :k] == torch.from_numpy(idx_query)[:, None]).sum(1)
precision_at_k = 100 * top_k_matches.float().numpy().mean()
logger.info("%i queries (%s) - %s - Precision at k = %i: %f" %
(len(top_k_matches), lg_query.upper(), method, k, precision_at_k))
results.append(('sent-precision_at_%i' % k, precision_at_k))
return results