def get_source_data(language): text_sentences = load_and_save.read_sentences_from_file('ud2_data/pos/' + language + '.pos') sentences, full_words, freq_words, sentences_pos, pos = load_and_save.integer_sentences_yuan( text_sentences, pos=pos_tags, max_words=1000, unk_word=unk_word) word_vec = get_word_vec(language, full_words) return full_words, text_sentences, sentences, sentences_pos, word_vec
def get_source_data(language): text_sentences = load_and_save.read_sentences_from_file('ud2_data/pos/'+language+'.pos') sentences, full_words, freq_words, sentences_pos, pos = load_and_save.integer_sentences_yuan(text_sentences, pos=pos_tags, max_words=1000, unk_word=unk_word) word_vec = get_word_vec(language, full_words) return full_words, text_sentences, sentences, sentences_pos, word_vec
print 'regularization coeff', base_reg_coeff print 'shared_reg_coeff', shared_reg_coeff print 'source language', source print 'target language', target print 'pair_str', pair_str print 'use_indicator', use_indicator print 'use_misc', use_misc word_vec_dim = 20 pos_tags = universal_pos_tags unk_word = '<UNK>' start_token = pos_tags.index('START') language = target text_sentences = load_and_save.read_sentences_from_file('ud2_data/pos/'+language+'.pos') sentences, full_words, freq_words, sentences_pos, pos = load_and_save.integer_sentences_yuan(text_sentences, pos=universal_pos_tags, max_words=1000, unk_word=unk_word) unk_id = full_words.index(unk_word) if unk_word in full_words else -1 target_word_vec = get_word_vec(language, full_words) test_text_sentences, test_sentences, test_sentences_pos = load_and_save.read_test_sentences_from_file('ud2_data/pos/' + language + '.test.pos', full_words, unk_id, pos_tags) if use_misc: misc_feature_map = get_emission_features(full_words) n_misc_feat = misc_feature_map.shape[1] n_pos = len(pos_tags) n_words = len(full_words) trans_feat_start = 0 trans_feat_end = n_pos * n_pos embedding_feat_start = trans_feat_end embedding_feat_end = trans_feat_end + n_pos * word_vec_dim projection_matrix_start = embedding_feat_end
print '%s words have no vector' % num_no_vect return word_vect_matrix source_language = 'spanish' target_language = 'english07' pair_filename = 'word_pairs/es-en.pair' reverse_pair = False target_vectors, target_vect_size = word_vect_loader.load('pos_data/'+target_language+'.train.sent.vec') source_vectors, source_vect_size = word_vect_loader.load('pos_data/'+source_language+'.train.sent.vec') assert source_vect_size == target_vect_size vect_size = source_vect_size print 'loading' source_text_sentences = load_and_save.read_sentences_from_file('pos_data/conll-'+source_language+'.pos') source_sentences, source_words, source_sentences_pos, _ = load_and_save.integer_sentences(source_text_sentences, pos=universal_pos_tags, max_words=10000) source_test_sentences = load_and_save.read_sentences_from_file('pos_data/conll-'+source_language+'-test.pos') source_test_sentences, _, source_test_sentences_pos, _ = load_and_save.integer_sentences(source_test_sentences, pos=universal_pos_tags, words=source_words) target_text_sentences = load_and_save.read_sentences_from_file('pos_data/conll-'+target_language+'-test.pos') target_sentences, target_words, target_sentences_pos, _ = load_and_save.integer_sentences(target_text_sentences, pos=universal_pos_tags, max_words=10000) source_vector_matrix = make_vector_matrix(source_words, source_vectors, vect_size) target_vector_matrix = make_vector_matrix(target_words, target_vectors, vect_size) print 'finding rotation' translation_pairs = [] pair_file = open(pair_filename) for line in pair_file: split = line.split()
print 'regularization coeff', base_reg_coeff print 'shared_reg_coeff', shared_reg_coeff print 'source language', source print 'target language', target print 'pair_str', pair_str print 'use_indicator', use_indicator print 'use_misc', use_misc word_vec_dim = 20 pos_tags = universal_pos_tags unk_word = '<UNK>' start_token = pos_tags.index('START') language = target text_sentences = load_and_save.read_sentences_from_file('ud2_data/pos/' + language + '.pos') sentences, full_words, freq_words, sentences_pos, pos = load_and_save.integer_sentences_yuan( text_sentences, pos=universal_pos_tags, max_words=1000, unk_word=unk_word) unk_id = full_words.index(unk_word) if unk_word in full_words else -1 target_word_vec = get_word_vec(language, full_words) test_text_sentences, test_sentences, test_sentences_pos = load_and_save.read_test_sentences_from_file( 'ud2_data/pos/' + language + '.test.pos', full_words, unk_id, pos_tags) if use_misc: misc_feature_map = get_emission_features(full_words) n_misc_feat = misc_feature_map.shape[1] n_pos = len(pos_tags) n_words = len(full_words) trans_feat_start = 0 trans_feat_end = n_pos * n_pos embedding_feat_start = trans_feat_end