def load_data(src_lang, tgt_lang, n_eval_ex=20000): src_id2word, src_word2id, src_embeddings = utils.read_txt_embeddings('data/wiki.%s.vec' % src_lang, n_eval_ex, False) tgt_id2word, tgt_word2id, tgt_embeddings = utils.read_txt_embeddings('data/wiki.%s.vec' % tgt_lang, n_eval_ex, False) s2t_dict = utils.load_dictionary('data/%s-%s.5000-6500.txt' % (src_lang, tgt_lang), src_word2id, tgt_word2id) t2s_dict = utils.load_dictionary('data/%s-%s.5000-6500.txt' % (tgt_lang, src_lang), tgt_word2id, src_word2id) return src_embeddings, tgt_embeddings, s2t_dict, t2s_dict
import numpy as np import utils import params src_id2word, src_word2id, src_embeddings = utils.read_txt_embeddings('data/wiki.%s.vec' % params.src_lang, params.n_eval_ex, False) tgt_id2word, tgt_word2id, tgt_embeddings = utils.read_txt_embeddings('data/wiki.%s.vec' % params.tgt_lang, params.n_eval_ex, False) print("%s_%s" % (params.src_lang, params.tgt_lang)) TranslatedX = np.load("output/TranslatedX-relu.npy") cross_dict = utils.load_dictionary('../data-test/medical/test-dict-only-words.txt', src_word2id, tgt_word2id) utils.get_word_translation_accuracy(params.src_lang, src_word2id, TranslatedX, params.tgt_lang, tgt_word2id, tgt_embeddings, params.method, cross_dict, src_id2word)
# Copyright (c) Facebook, Inc. and its affiliates. # All rights reserved. # # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. # import numpy as np import utils import params src_id2word, src_word2id, src_embeddings = utils.read_txt_embeddings( 'data/wiki.%s.vec' % params.src_lang, params.n_init_ex, False) # = 5000, only read en 500 words np.save('data/%s_%d' % (params.src_lang, params.n_init_ex), src_embeddings) tgt_id2word, tgt_word2id, tgt_embeddings = utils.read_txt_embeddings( 'data/wiki.%s.vec' % params.tgt_lang, params.n_init_ex, False) # = 5000, only read es 500 words np.save('data/%s_%d' % (params.tgt_lang, params.n_init_ex), tgt_embeddings) src_id2word, src_word2id, src_embeddings = utils.read_txt_embeddings( 'data/wiki.%s.vec' % params.src_lang, params.n_ft_ex, False) # = 7500, only read en 7500 words np.save('data/%s_%d' % (params.src_lang, params.n_ft_ex), src_embeddings) tgt_id2word, tgt_word2id, tgt_embeddings = utils.read_txt_embeddings( 'data/wiki.%s.vec' % params.tgt_lang, params.n_ft_ex, False) # = 7500, only read es 7500 words np.save('data/%s_%d' % (params.tgt_lang, params.n_ft_ex), tgt_embeddings) src_id2word, src_word2id, src_embeddings = utils.read_txt_embeddings( 'data/wiki.%s.vec' % params.src_lang, 10000,
import numpy as np import utils import params src_id2word, src_word2id, src_embeddings = utils.read_txt_embeddings( 'data/full-new-test-dict-only-words-lower-%s-emb.vec' % params.src_lang, params.n_eval_ex, full_vocab=False) tgt_id2word, tgt_word2id, tgt_embeddings = utils.read_txt_embeddings( 'data/full-new-test-dict-only-words-lower-%s-emb.vec' % params.tgt_lang, params.n_eval_ex, full_vocab=False) to_translate_src_id2word, to_translate_src_word2id, to_translate_src_embeddings = utils.read_txt_embeddings( 'data/intersection-1-parsed-%s-emb.vec' % params.src_lang, params.n_eval_ex, False) translated_tgt_id2word, translated_tgt_word2id, translated_tgt_embeddings = utils.read_txt_embeddings( 'data/intersection-1-parsed-%s-emb.vec' % params.tgt_lang, params.n_eval_ex, False) print("%s_%s" % (params.src_lang, params.tgt_lang)) TranslatedX = translated_tgt_embeddings #uncomment for comparing with the global transformation # T = np.load("../Non-adversarialTranslation/%s/%s_%s_T.npy" % (params.cp_dir, params.src_lang, params.tgt_lang)) # TranslatedX = to_translate_src_embeddings.dot(np.transpose(T)) cross_dict, existing_emb_tgt_translated, existing_emb_tgt_real, existing_src_id2word = utils.load_dictionary( 'data/new-test-dict-only-words-no-parsed.txt',
def main(): parser = argparse.ArgumentParser() parser.add_argument('--cluster_num', type=int, default=2, help='The number of cluster to be divided') args = parser.parse_args() n_clusters = args.cluster_num src_id2word, src_word2id, src_embeddings = utils.read_txt_embeddings( 'data/wiki.%s.vec' % params.src_lang, params.n_eval_ex, False) #n_eval_ex = 200000 tgt_id2word, tgt_word2id, tgt_embeddings = utils.read_txt_embeddings( 'data/wiki.%s.vec' % params.tgt_lang, params.n_eval_ex, False) src_normed = normalize(src_embeddings) tgt_normed = normalize(tgt_embeddings) cross_dict_src2tgt = utils.load_dictionary( 'data/%s-%s.5000-6500.txt' % (params.src_lang, params.tgt_lang), src_word2id, tgt_word2id) cross_dict_tgt2src = utils.load_dictionary( 'data/%s-%s.5000-6500.txt' % (params.tgt_lang, params.src_lang), tgt_word2id, src_word2id) T = np.load("%s/%s_%s_T.npy" % (params.cp_dir, params.src_lang, params.tgt_lang)) T2 = np.load("%s/%s_%s_T.npy" % (params.cp_dir, params.tgt_lang, params.src_lang)) TranslatedX = src_embeddings.dot(np.transpose(T)) src_full = np.load( "data/%s_%d.npy" % (params.src_lang, params.n_init_ex)) # 10000, 10000 english src_trans = src_full.dot(np.transpose(T)) tgt_full = np.load("data/%s_%d.npy" % (params.tgt_lang, params.n_init_ex)) # 300, 10000 es src_trans_normed = normalize(src_trans) tgt_full_normed = normalize(tgt_full) src_clt = sklearn.cluster.KMeans(n_clusters=n_clusters, n_init=40, random_state=200) src_y = src_clt.fit_predict(src_trans_normed) tgt_y = src_clt.predict(tgt_full_normed) src_centers, tgt_centers, src_dic, tgt_dic = find_centers( src_full, tgt_full, src_y, tgt_y, n_clts=n_clusters) Translated_centers = src_centers.dot(np.transpose(T)) trans_c = normalize(Translated_centers) tgt_c = normalize(tgt_centers) cross_dict_src2tgt = utils.load_dictionary( 'data/%s-%s.5000-6500.txt' % (params.src_lang, params.tgt_lang), src_word2id, tgt_word2id) cross_dict_tgt2src = utils.load_dictionary( 'data/%s-%s.5000-6500.txt' % (params.tgt_lang, params.src_lang), tgt_word2id, src_word2id) src_classes = find_clts(data=src_embeddings, centers=src_centers, dico=cross_dict_src2tgt) tgt_classes = find_clts(data=tgt_embeddings, centers=tgt_centers, dico=cross_dict_tgt2src) src_classes_trans = find_clts(data=src_embeddings, centers=src_centers, dico=cross_dict_src2tgt, trans=True) src_correct = np.where(src_classes == src_classes_trans) src_acc = src_correct[0].shape[0] / 1500 tgt_classes_trans = find_clts(data=tgt_embeddings, centers=tgt_centers, dico=cross_dict_tgt2src, trans=True) tgt_correct = np.where(tgt_classes == tgt_classes_trans) tgt_acc = tgt_correct[0].shape[0] / 1500 print(src_acc, tgt_acc) TX, TY = utils.multi_ICP(src_full, tgt_full, src_y, tgt_y, src_dic, tgt_dic, n_clusters, time_run_icp=100) result_src = separate_eva(src_embeddings, tgt_embeddings, T, TX, src_classes, dico=cross_dict_src2tgt) result_tgt = separate_eva(tgt_embeddings, src_embeddings, T2, TY, tgt_classes, dico=cross_dict_tgt2src)
import numpy as np import utils import params import json def write_to_json_file(data, file_path): with open(file_path, 'w') as outfile: json.dump(data, outfile) src_id2word, src_word2id, src_embeddings = utils.read_txt_embeddings( params.init_emb_path_src, params.n_init_ex, False) np.save('data/%s_init' % (params.src_lang), src_embeddings) write_to_json_file(src_id2word, params.init_src_id2word_path) write_to_json_file(src_word2id, params.init_src_word2id_path) tgt_id2word, tgt_word2id, tgt_embeddings = utils.read_txt_embeddings( params.init_emb_path_tgt, params.n_init_ex, False) np.save('data/%s_init' % (params.tgt_lang), tgt_embeddings) write_to_json_file(tgt_id2word, params.init_tgt_id2word_path) write_to_json_file(tgt_word2id, params.init_tgt_word2id_path) src_id2word, src_word2id, src_embeddings = utils.read_txt_embeddings( params.training_emb_path_src, params.n_ft_ex, False) np.save('data/%s_training' % (params.src_lang), src_embeddings) write_to_json_file(src_id2word, params.training_src_id2word_path) write_to_json_file(src_word2id, params.training_src_word2id_path) tgt_id2word, tgt_word2id, tgt_embeddings = utils.read_txt_embeddings( params.training_emb_path_tgt, params.n_ft_ex, False)