def get_word_translation_accuracy(lang1, word2id1, emb1, lang2, word2id2, emb2, method, dico_eval): """ Given source and target word embeddings, and a dictionary, evaluate the translation accuracy using the precision@k. """ if dico_eval == 'default': path = os.path.join(DIC_EVAL_PATH, '%s-%s.5000-6500.txt' % (lang1, lang2)) else: path = dico_eval dico = load_dictionary(path, word2id1, word2id2) dico = dico.cuda() if emb1.is_cuda else dico assert dico[:, 0].max() < emb1.size(0) assert dico[:, 1].max() < emb2.size(0) # normalize word embeddings emb1 = emb1 / emb1.norm(2, 1, keepdim=True).expand_as(emb1) emb2 = emb2 / emb2.norm(2, 1, keepdim=True).expand_as(emb2) # nearest neighbors if method == 'nn': query = emb1[dico[:, 0]] scores = query.mm(emb2.transpose(0, 1)) # inverted softmax elif method.startswith('invsm_beta_'): beta = float(method[len('invsm_beta_'):]) bs = 128 word_scores = [] for i in range(0, emb2.size(0), bs): scores = emb1.mm(emb2[i:i + bs].transpose(0, 1)) scores.mul_(beta).exp_() scores.div_(scores.sum(0, keepdim=True).expand_as(scores)) word_scores.append(scores.index_select(0, dico[:, 0])) scores = torch.cat(word_scores, 1) # contextual dissimilarity measure elif method.startswith('csls_knn_'): # average distances to k nearest neighbors knn = method[len('csls_knn_'):] assert knn.isdigit() knn = int(knn) average_dist1 = get_nn_avg_dist(emb2, emb1, knn) average_dist2 = get_nn_avg_dist(emb1, emb2, knn) average_dist1 = torch.from_numpy(average_dist1).type_as(emb1) average_dist2 = torch.from_numpy(average_dist2).type_as(emb2) # queries / scores query = emb1[dico[:, 0]] scores = query.mm(emb2.transpose(0, 1)) scores.mul_(2) scores.sub_(average_dist1[dico[:, 0]][:, None]) scores.sub_(average_dist2[None, :]) else: raise Exception('Unknown method: "%s"' % method) results = [] top_matches = scores.topk(10, 1, True)[1] for k in [1, 5, 10]: top_k_matches = top_matches[:, :k] _matching = ( top_k_matches == dico[:, 1][:, None].expand_as(top_k_matches)).sum(1) # allow for multiple possible translations matching = {} for i, src_id in enumerate(dico[:, 0].cpu().numpy()): matching[src_id] = min(matching.get(src_id, 0) + _matching[i], 1) # evaluate precision@k precision_at_k = 100 * np.mean(list(matching.values())) logger.info("%i source words - %s - Precision at k = %i: %f" % (len(matching), method, k, precision_at_k)) results.append(('precision_at_%i' % k, precision_at_k)) return results
def get_candidates(emb1, emb2, params): """ Get best translation pairs candidates. """ bs = 128 all_scores = [] all_targets = [] # number of source words to consider n_src = emb1.size(0) if params['dico_max_rank'] > 0 and not params['dico_method'].startswith( 'invsm_beta_'): n_src = params['dico_max_rank'] # nearest neighbors if params['dico_method'] == 'nn': # for every source word for i in range(0, n_src, bs): # compute target words scores scores = emb2.mm(emb1[i:min(n_src, i + bs)].transpose( 0, 1)).transpose(0, 1) best_scores, best_targets = scores.topk(2, dim=1, largest=True, sorted=True) # update scores / potential targets all_scores.append(best_scores.cpu()) all_targets.append(best_targets.cpu()) all_scores = torch.cat(all_scores, 0) all_targets = torch.cat(all_targets, 0) # inverted softmax elif params['dico_method'].startswith('invsm_beta_'): beta = float(params['dico_method'][len('invsm_beta_'):]) # for every target word for i in range(0, emb2.size(0), bs): # compute source words scores scores = emb1.mm(emb2[i:i + bs].transpose(0, 1)) scores.mul_(beta).exp_() scores.div_(scores.sum(0, keepdim=True).expand_as(scores)) best_scores, best_targets = scores.topk(2, dim=1, largest=True, sorted=True) # update scores / potential targets all_scores.append(best_scores.cpu()) all_targets.append((best_targets + i).cpu()) all_scores = torch.cat(all_scores, 1) all_targets = torch.cat(all_targets, 1) all_scores, best_targets = all_scores.topk(2, dim=1, largest=True, sorted=True) all_targets = all_targets.gather(1, best_targets) # contextual dissimilarity measure elif params['dico_method'].startswith('csls_knn_'): knn = params['dico_method'][len('csls_knn_'):] assert knn.isdigit() knn = int(knn) # average distances to k nearest neighbors average_dist1 = torch.from_numpy(get_nn_avg_dist(emb2, emb1, knn)) average_dist2 = torch.from_numpy(get_nn_avg_dist(emb1, emb2, knn)) average_dist1 = average_dist1.type_as(emb1) average_dist2 = average_dist2.type_as(emb2) # for every source word for i in range(0, n_src, bs): # compute target words scores scores = emb2.mm(emb1[i:min(n_src, i + bs)].transpose( 0, 1)).transpose(0, 1) scores.mul_(2) scores.sub_(average_dist1[i:min(n_src, i + bs)][:, None] + average_dist2[None, :]) best_scores, best_targets = scores.topk(2, dim=1, largest=True, sorted=True) # update scores / potential targets all_scores.append(best_scores.cpu()) all_targets.append(best_targets.cpu()) all_scores = torch.cat(all_scores, 0) all_targets = torch.cat(all_targets, 0) all_pairs = torch.cat([ torch.arange(0, all_targets.size(0)).long().unsqueeze(1), all_targets[:, 0].unsqueeze(1) ], 1) # sanity check assert all_scores.size() == all_pairs.size() == (n_src, 2) # sort pairs by score confidence diff = all_scores[:, 0] - all_scores[:, 1] reordered = diff.sort(0, descending=True)[1] all_scores = all_scores[reordered] all_pairs = all_pairs[reordered] # max dico words rank if params['dico_max_rank'] > 0: selected = all_pairs.max(1)[0] <= params['dico_max_rank'] mask = selected.unsqueeze(1).expand_as(all_scores).clone() all_scores = all_scores.masked_select(mask).view(-1, 2) all_pairs = all_pairs.masked_select(mask).view(-1, 2) # max dico size if params['dico_max_size'] > 0: all_scores = all_scores[:params['dico_max_size']] all_pairs = all_pairs[:params['dico_max_size']] # min dico size diff = all_scores[:, 0] - all_scores[:, 1] if params['dico_min_size'] > 0: diff[:params['dico_min_size']] = 1e9 # confidence threshold if params['dico_threshold'] > 0: mask = diff > params['dico_threshold'] print("Selected %i / %i pairs above the confidence threshold." % (mask.sum(), diff.size(0))) mask = mask.unsqueeze(1).expand_as(all_pairs).clone() all_pairs = all_pairs.masked_select(mask).view(-1, 2) return all_pairs
def get_sent_translation_accuracy(data, lg1, word2id1, emb1, lg2, word2id2, emb2, n_keys, n_queries, method, idf): """ Given parallel sentences from Europarl, evaluate the sentence translation accuracy using the precision@k. """ # get word vectors dictionaries emb1 = emb1.cpu().numpy() emb2 = emb2.cpu().numpy() word_vec1 = dict([(w, emb1[word2id1[w]]) for w in word2id1]) word_vec2 = dict([(w, emb2[word2id2[w]]) for w in word2id2]) word_vect = {lg1: word_vec1, lg2: word_vec2} lg_keys = lg2 lg_query = lg1 # get n_keys pairs of sentences keys = data[lg_keys][:n_keys] keys = bow_idf(keys, word_vect[lg_keys], idf_dict=idf[lg_keys]) # get n_queries query pairs from these n_keys pairs rng = np.random.RandomState(1234) idx_query = rng.choice(range(n_keys), size=n_queries, replace=False) queries = data[lg_query][idx_query] queries = bow_idf(queries, word_vect[lg_query], idf_dict=idf[lg_query]) # normalize embeddings queries = torch.from_numpy(queries).float() queries = queries / queries.norm(2, 1, keepdim=True).expand_as(queries) keys = torch.from_numpy(keys).float() keys = keys / keys.norm(2, 1, keepdim=True).expand_as(keys) # nearest neighbors if method == 'nn': scores = keys.mm(queries.transpose(0, 1)).transpose(0, 1) scores = scores.cpu() # inverted softmax elif method.startswith('invsm_beta_'): beta = float(method[len('invsm_beta_'):]) scores = keys.mm(queries.transpose(0, 1)).transpose(0, 1) scores.mul_(beta).exp_() scores.div_(scores.sum(0, keepdim=True).expand_as(scores)) scores = scores.cpu() # contextual dissimilarity measure elif method.startswith('csls_knn_'): knn = method[len('csls_knn_'):] assert knn.isdigit() knn = int(knn) # average distances to k nearest neighbors knn = method[len('csls_knn_'):] assert knn.isdigit() knn = int(knn) average_dist_keys = torch.from_numpy(get_nn_avg_dist(queries, keys, knn)) average_dist_queries = torch.from_numpy(get_nn_avg_dist(keys, queries, knn)) # scores scores = keys.mm(queries.transpose(0, 1)).transpose(0, 1) scores.mul_(2) scores.sub_(average_dist_queries[:, None].float() + average_dist_keys[None, :].float()) scores = scores.cpu() results = [] top_matches = scores.topk(10, 1, True)[1] for k in [1, 5, 10]: top_k_matches = (top_matches[:, :k] == torch.from_numpy(idx_query)[:, None]).sum(1) precision_at_k = 100 * top_k_matches.float().numpy().mean() logger.info("%i queries (%s) - %s - Precision at k = %i: %f" % (len(top_k_matches), lg_query.upper(), method, k, precision_at_k)) results.append(('sent-precision_at_%i' % k, precision_at_k)) return results