def __init__(self, genre): self.genre = genre self.train_data = pickle_load( format_filename(PROCESSED_DATA_DIR, TRAIN_DATA_TEMPLATE, genre)) self.dev_data = pickle_load( format_filename(PROCESSED_DATA_DIR, DEV_DATA_TEMPLATE, genre)) self.test_data = pickle_load( format_filename(PROCESSED_DATA_DIR, TEST_DATA_TEMPLATE, genre)) if not os.path.exists(FEATURE_DIR): os.makedirs(FEATURE_DIR)
def load_single_ngram_data(variation, vectorizer_type, level, ngram_range, data_type): if data_type == 'train': filename = format_filename(PROCESSED_DATA_DIR, TRAIN_NGRAM_DATA_TEMPLATE, variation=variation, type=vectorizer_type, level=level, ngram_range=ngram_range) elif data_type == 'valid' or data_type == 'dev': filename = format_filename(PROCESSED_DATA_DIR, DEV_NGRAM_DATA_TEMPLATE, variation=variation, type=vectorizer_type, level=level, ngram_range=ngram_range) elif data_type == 'test': filename = format_filename(PROCESSED_DATA_DIR, TEST_NGRAM_DATA_TEMPLATE, variation=variation, type=vectorizer_type, level=level, ngram_range=ngram_range) else: raise ValueError('Data Type Not Understood: {}'.format(data_type)) if os.path.exists(filename): return pickle_load(filename) else: return None
def load_data(data_type): if data_type == 'train': data = pickle_load( format_filename(PROCESSED_DATA_DIR, TRAIN_DATA_FILENAME)) elif data_type == 'dev': data = pickle_load( format_filename(PROCESSED_DATA_DIR, DEV_DATA_FILENAME)) elif data_type == 'test': data = pickle_load( format_filename(PROCESSED_DATA_DIR, TEST_DATA_FILENAME)) elif data_type == 'test_final': data = pickle_load( format_filename(PROCESSED_DATA_DIR, TEST_FINAL_DATA_FILENAME)) else: raise ValueError('data tye not understood: {}'.format(data_type)) return data
def add_sent_fred_feature(self, data_type): """ Idea from https://www.kaggle.com/jturkewitz/magic-features-0-03-gain (Quora Question Pairs Competition) Magic features based on question frequency. The idea.md behind is a question that is asked often has more chances to be duplicated. """ feat_file = self.format_feature_file(data_type, 'sent_fred') if os.path.exists(feat_file): features = pickle_load(feat_file) else: sents_dict, p_vc, h_vc = self.get_sent_freq() data = pd.DataFrame(self.get_data(data_type)) data['p_hash'] = data['premise'].map(sents_dict) data['h_hash'] = data['hypotheis'].map(sents_dict) data['p_freq'] = data['p_hash'].map( lambda x: p_vc.get(x, 0) + h_vc.get(x, 0)) data['h_freq'] = data['h_hash'].map( lambda x: p_vc.get(x, 0) + h_vc.get(x, 0)) data['freq_mean'] = (data['p_freq'] + data['h_freq']) / 2 data['freq_cross'] = data['p_freq'] * data['h_freq'] data['p_freq_sq'] = data['p_freq'] * data['p_freq'] data['h_freq_sq'] = data['h_freq'] * data['h_freq'] features = data[[ 'p_freq', 'h_freq', 'freq_mean', 'freq_cross', 'p_freq_sq', 'h_freq_sq' ]].values pickle_dump(feat_file, features) return features
def tfidf_model(self): print('Logging Info - Get Tf-idf model...') tfidf_model_path = os.path.join(FEATURE_DIR, '{}_tfidf.model').format(self.genre) dict_path = os.path.join(FEATURE_DIR, '{}_tfidf.dict').format(self.genre) if os.path.exists(tfidf_model_path): dictionary = pickle_load(dict_path) tfidf_model = TfidfModel.load(tfidf_model_path) else: corpus = [ text.split() for text in self.train_data['premise'] + self.train_data['hypothesis'] + self.dev_data['premise'] + self.dev_data['hypothesis'] + self.test_data['premise'] + self.test_data['hypothesis'] ] dictionary = corpora.Dictionary(corpus) corpus = [dictionary.doc2bow(text) for text in corpus] tfidf_model = TfidfModel(corpus) del corpus tfidf_model.save(tfidf_model_path) pickle_dump(dict_path, dictionary) return dictionary, tfidf_model
def add_tfidf_feature(self, data_type): feat_file = self.format_feature_file(data_type, 'tfidf') if os.path.exists(feat_file): features = pickle_load(feat_file) else: dictionary, tfidf_model = self.tfidf_model() features = list() for premise, hypothesis in zip( self.get_data(data_type)['premise'], self.get_data(data_type)['hypothesis']): premise = premise.split() hypothesis = hypothesis.split() p_tfidf = dict(tfidf_model[dictionary.doc2bow(premise)]) h_tfidf = dict(tfidf_model[dictionary.doc2bow(hypothesis)]) features.append([ np.sum(list(p_tfidf.values())), np.sum(list(h_tfidf.values())), np.mean(list(p_tfidf.values())), np.mean(list(h_tfidf.values())) ]) features = np.array(features) pickle_dump(feat_file, features) print('Logging Info - {} : w_ngram_ol_tfidf feature shape : {}'.format( data_type, features.shape)) return features
def load_processed_data(genre, level, data_type): if data_type == 'train': filename = format_filename(PROCESSED_DATA_DIR, TRAIN_IDS_MATRIX_TEMPLATE, genre, level) elif data_type == 'valid' or data_type == 'dev': filename = format_filename(PROCESSED_DATA_DIR, DEV_IDS_MATRIX_TEMPLATE, genre, level) elif data_type == 'test': filename = format_filename(PROCESSED_DATA_DIR, TEST_IDS_MATRIX_TEMPLATE, genre, level) else: raise ValueError('Data Type Not Understood: {}'.format(data_type)) return pickle_load(filename)
def load_features(genre, data_type, scale_features): feat_type = 'all_scaled' if scale_features else 'all' if data_type == 'train': filename = format_filename(FEATURE_DIR, TRAIN_FEATURES_TEMPLATE, genre, feat_type) elif data_type == 'valid' or data_type == 'dev': filename = format_filename(FEATURE_DIR, DEV_FEATURES_TEMPLATE, genre, feat_type) elif data_type == 'test': filename = format_filename(FEATURE_DIR, TEST_FEATURES_TEMPLATE, genre, feat_type) else: raise ValueError('Data Type Not Understood: {}'.format(data_type)) return pickle_load(filename)
def get_sent_freq(self): print('Logging Info - Get sentence frequency...') sents_dict_path = os.path.join(FEATURE_DIR, '{}_sent_dict.pkl'.format(self.genre)) p_vc_path = os.path.join(FEATURE_DIR, '{}_premise_vc.pkl'.format(self.genre)) h_vc_path = os.path.join(FEATURE_DIR, '{}_hypothesis_vc.pkl'.format(self.genre)) if os.path.exists(p_vc_path): sents_dict = pickle_load(sents_dict_path) p_vc = pickle_load(p_vc_path) h_vc = pickle_load(h_vc_path) else: train_data = pd.DataFrame(self.train_data) dev_data = pd.DataFrame(self.dev_data) test_data = pd.DataFrame(self.test_data) all_data = pd.concat([train_data, dev_data, test_data]) df1 = all_data[['premise']] df2 = all_data[['hypothesis']] df2.rename(columns={'hypothesis': 'premise'}, inplace=True) train_sents = pd.concat([df1, df2]) train_sents.drop_duplicates(subset=['premise'], inplace=True) train_sents.reset_index(inplace=True, drop=True) sents_dict = pd.Series(train_sents.index.values, index=train_sents.premise.values).to_dict() all_data['p_hash'] = all_data['premise'].map(sents_dict) all_data['h_hash'] = all_data['hypothesis'].map(sents_dict) p_vc = all_data.p_hash.value_counts().to_dict() h_vc = all_data.h_hash.value_counts().to_dict() pickle_dump(sents_dict_path, sents_dict) pickle_dump(p_vc_path, p_vc) pickle_dump(h_vc_path, h_vc) del train_data, dev_data, test_data, all_data return sents_dict, p_vc, h_vc
def generate_graph(self): print('Logging Info - Get graph...') sent2id_path = os.path.join(FEATURE_DIR, '{}_graph_sent2id.pkl'.format(self.genre)) graph_path = os.path.join(FEATURE_DIR, '{}_graph.pkl'.format(self.genre)) if os.path.exists(graph_path): sent2id = pickle_load(sent2id_path) graph = pickle_load(graph_path) else: sent2id = {} # sentence to id graph = nx.Graph() for data_type in ['train', 'dev', 'test']: for premise, hypothesis in zip( self.get_data(data_type)['premise'], self.get_data(data_type)['hypothesis']): if premise not in sent2id: sent2id[premise] = len(sent2id) if hypothesis not in sent2id: sent2id[hypothesis] = len(sent2id) p_id = sent2id[premise] h_id = sent2id[hypothesis] match = 0.0 premise = premise.split() hypothesis = hypothesis.split() for w1 in premise: if w1 in hypothesis: match += 1 if len(premise) + len(hypothesis) == 0: weight = 0.0 else: weight = 2.0 * (match / (len(premise) + len(hypothesis))) graph.add_edge(p_id, h_id, weight=weight) pickle_dump(sent2id_path, sent2id) pickle_dump(graph_path, graph) return sent2id, graph
def plot_data(dir_data='/tmp', fn_data='data.pkl'): data = pickle_load(os.path.join(dir_data, fn_data)) #print data[0].shape shape = data[0].shape[0:2] print shape data = np.array([255-to_gray(A).flatten() for A in data]) plot_images(data, 10, 10, shape, border=2, reshape=True, figsize=None, colorbar=False, idx_highlight=None, vmin=0, vmax=255) plt.show()
def add_similarity_feature(self, data_type, feat_type, sim_func): feat_file = self.format_feature_file(data_type, feat_type) if os.path.exists(feat_file): features = pickle_load(feat_file) else: len_dist_feat = np.array([ sim_func(p, h) for p, h in zip( self.get_data(data_type)['premise'], self.get_data(data_type)['hypothesis']) ]) features = self.check_and_expand_shape(len_dist_feat) pickle_dump(feat_file, features) print('Logging Info - {} : {} feature shape : {}'.format( data_type, feat_type, features.shape)) return features
def gen_all_features(self, data_type, scaled=False): if scaled: feat_file = self.format_feature_file(data_type, 'all_scaled') else: feat_file = self.format_feature_file(data_type, 'all') if os.path.exists(feat_file): features = pickle_load(feat_file) else: features = list() feat_types = [('len_dis', length_distance), ('lcs_seq', lcs_seq_norm), ('lcs_str', lcs_str_1_norm), ('edit_dist', edit_distance), ('jaro', jaro_distance), ('jaro_winkler', jaro_winkler_dist), ('fuzz', fuzzy), ('simhash', simhash), ('w_share', word_share), ('w_ngram_dist', word_ngram_distance), ('c_ngram_ol', char_ngram_overlap), ('w_ngram_ol', word_ngram_overlap)] for feat_type, sim_func in feat_types: features.append( self.add_similarity_feature(data_type, feat_type, sim_func)) features.append( self.add_weighted_word_ngram_overlap_feature(data_type)) features.append(self.add_tfidf_feature(data_type)) features.append(self.add_word_power_feature(data_type)) features.append(self.add_graph_feature(data_type)) features = np.concatenate(features, axis=-1) if scaled: scaler = StandardScaler() features = scaler.fit_transform(features) joblib.dump( scaler, os.path.join(FEATURE_DIR, '{}_scaler.model'.format(self.genre))) pickle_dump(feat_file, features) print('Logging Info - {} : all feature shape : {}'.format( data_type, features.shape))
def plot_data(dir_data='/tmp', fn_data='data.pkl'): data = pickle_load(os.path.join(dir_data, fn_data)) #print data[0].shape shape = data[0].shape[0:2] print shape data = np.array([255 - to_gray(A).flatten() for A in data]) plot_images(data, 10, 10, shape, border=2, reshape=True, figsize=None, colorbar=False, idx_highlight=None, vmin=0, vmax=255) plt.show()
def add_word_power_feature(self, data_type): feat_file = self.format_feature_file(data_type, 'word_power') if os.path.exists(feat_file): features = pickle_load(feat_file) else: power_word = self.get_power_word() num_least = 100 features = list() for premise, hypothesis in zip( self.get_data(data_type)['premise'], self.get_data(data_type)['hypothesis']): premise = premise.split() hypothesis = hypothesis.split() rate = [1.0, 1.0] share_words = list(set(premise).intersection(set(hypothesis))) for word in share_words: if word not in power_word: continue if power_word[word][0] * power_word[word][ 5] < num_least: # 共享词出现在双侧语句对数量要大于num_least continue rate[0] *= (1.0 - power_word[word][6] ) # 共享词但是语句对不是正确的(label!=2) p_diff = list(set(premise).difference(set(hypothesis))) h_diff = list(set(premise).difference(set(hypothesis))) all_diff = set(p_diff + h_diff) for word in all_diff: if word not in power_word: continue if power_word[word][0] * power_word[word][ 3] < num_least: # 共享词只出现在单侧语句数量要大于num_least continue rate[1] *= (1.0 - power_word[word][4] ) # 非共享词但是语句对是正确的(label=2) rate = [1 - num for num in rate] features.append(rate) features = np.array(features) pickle_dump(feat_file, features) print('Logging Info - {} : word_power feature shape : {}'.format( data_type, features.shape)) return features
def load_processed_text_data(variation, data_type): if data_type == 'train': filename = format_filename(PROCESSED_DATA_DIR, TRAIN_DATA_TEMPLATE, variation=variation) elif data_type == 'valid' or data_type == 'dev': filename = format_filename(PROCESSED_DATA_DIR, DEV_DATA_TEMPLATE, variation=variation) elif data_type == 'test': filename = format_filename(PROCESSED_DATA_DIR, TEST_DATA_TEMPLATE, variation=variation) else: raise ValueError('Data Type Not Understood: {}'.format(data_type)) if os.path.exists(filename): return pickle_load(filename) else: return None
def add_weighted_word_ngram_overlap_feature(self, data_type): feat_file = self.format_feature_file(data_type, 'w_ngram_ol_tfidf') if os.path.exists(feat_file): features = pickle_load(feat_file) else: dictionary, tfidf_model = self.tfidf_model() idf_model = tfidf_model.idfs features = list() for premise, hypothesis in zip( self.get_data(data_type)['premise'], self.get_data(data_type)['hypothesis']): premise = premise.split() p_tfidf = dict(tfidf_model[dictionary.doc2bow(premise)]) input_premise = [ (word, idf_model.get(dictionary.token2id.get(word, 0), 0.0), p_tfidf.get(dictionary.token2id.get(word, 0), 0.0)) for word in premise ] hypothesis = hypothesis.split() h_tfidf = dict(tfidf_model[dictionary.doc2bow(hypothesis)]) input_hypothesis = [ (word, idf_model.get(dictionary.token2id.get(word, 0), 0.0), h_tfidf.get(dictionary.token2id.get(word, 0), 0.0)) for word in hypothesis ] features.append( weighted_word_ngram_overlap(input_premise, input_hypothesis)) features = np.array(features) pickle_dump(feat_file, features) print('Logging Info - {} : w_ngram_ol_tfidf feature shape : {}'.format( data_type, features.shape)) return features
def recognition(model_name, predict_log, label_schema='BIOES', batch_size=32, n_epoch=50, learning_rate=0.001, optimizer_type='adam', use_char_input=True, embed_type=None, embed_trainable=True, use_bert_input=False, bert_type='bert', bert_trainable=True, bert_layer_num=1, use_bichar_input=False, bichar_embed_type=None, bichar_embed_trainable=True, use_word_input=False, word_embed_type=None, word_embed_trainable=True, use_charpos_input=False, charpos_embed_type=None, charpos_embed_trainable=True, use_softword_input=False, use_dictfeat_input=False, use_maxmatch_input=False, callbacks_to_add=None, swa_type=None, predict_on_dev=True, predict_on_final_test=True, **kwargs): config = ModelConfig() config.model_name = model_name config.label_schema = label_schema config.batch_size = batch_size config.n_epoch = n_epoch config.learning_rate = learning_rate config.optimizer = get_optimizer(optimizer_type, learning_rate) config.embed_type = embed_type config.use_char_input = use_char_input if embed_type: config.embeddings = np.load( format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, type=embed_type)) config.embed_trainable = embed_trainable config.embed_dim = config.embeddings.shape[1] else: config.embeddings = None config.embed_trainable = True config.callbacks_to_add = callbacks_to_add or [ 'modelcheckpoint', 'earlystopping' ] config.vocab = pickle_load( format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, level='char')) config.vocab_size = len(config.vocab) + 2 config.mention_to_entity = pickle_load( format_filename(PROCESSED_DATA_DIR, MENTION_TO_ENTITY_FILENAME)) if config.use_char_input: config.exp_name = '{}_{}_{}_{}_{}_{}_{}'.format( model_name, config.embed_type if config.embed_type else 'random', 'tune' if config.embed_trainable else 'fix', batch_size, optimizer_type, learning_rate, label_schema) else: config.exp_name = '{}_{}_{}_{}_{}'.format(model_name, batch_size, optimizer_type, learning_rate, label_schema) if kwargs: config.exp_name += '_' + '_'.join( [str(k) + '_' + str(v) for k, v in kwargs.items()]) callback_str = '_' + '_'.join(config.callbacks_to_add) callback_str = callback_str.replace('_modelcheckpoint', '').replace('_earlystopping', '') config.exp_name += callback_str config.use_bert_input = use_bert_input config.bert_type = bert_type config.bert_trainable = bert_trainable config.bert_layer_num = bert_layer_num assert config.use_char_input or config.use_bert_input if config.use_bert_input: config.exp_name += '_{}_layer_{}_{}'.format( bert_type, bert_layer_num, 'tune' if config.bert_trainable else 'fix') config.use_bichar_input = use_bichar_input if config.use_bichar_input: config.bichar_vocab = pickle_load( format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, level='bichar')) config.bichar_vocab_size = len(config.bichar_vocab) + 2 if bichar_embed_type: config.bichar_embeddings = np.load( format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, type=bichar_embed_type)) config.bichar_embed_trainable = bichar_embed_trainable config.bichar_embed_dim = config.bichar_embeddings.shape[1] else: config.bichar_embeddings = None config.bichar_embed_trainable = True config.exp_name += '_bichar_{}_{}'.format( bichar_embed_type if bichar_embed_type else 'random', 'tune' if config.bichar_embed_trainable else 'fix') config.use_word_input = use_word_input if config.use_word_input: config.word_vocab = pickle_load( format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, level='word')) config.word_vocab_size = len(config.word_vocab) + 2 if word_embed_type: config.word_embeddings = np.load( format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, type=word_embed_type)) config.word_embed_trainable = word_embed_trainable config.word_embed_dim = config.word_embeddings.shape[1] else: config.word_embeddings = None config.word_embed_trainable = True config.exp_name += '_word_{}_{}'.format( word_embed_type if word_embed_type else 'random', 'tune' if config.word_embed_trainable else 'fix') config.use_charpos_input = use_charpos_input if config.use_charpos_input: config.charpos_vocab = pickle_load( format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, level='charpos')) config.charpos_vocab_size = len(config.charpos_vocab) + 2 if charpos_embed_type: config.charpos_embeddings = np.load( format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, type=charpos_embed_type)) config.charpos_embed_trainable = charpos_embed_trainable config.charpos_embed_dim = config.charpos_embeddings.shape[1] else: config.charpos_embeddings = None config.charpos_embed_trainable = True config.exp_name += '_charpos_{}_{}'.format( charpos_embed_type if charpos_embed_type else 'random', 'tune' if config.charpos_embed_trainable else 'fix') config.use_softword_input = use_softword_input if config.use_softword_input: config.exp_name += '_softword' config.use_dictfeat_input = use_dictfeat_input if config.use_dictfeat_input: config.exp_name += '_dictfeat' config.use_maxmatch_input = use_maxmatch_input if config.use_maxmatch_input: config.exp_name += '_maxmatch' # logger to log output of training process predict_log.update({ 'er_exp_name': config.exp_name, 'er_batch_size': batch_size, 'er_optimizer': optimizer_type, 'er_epoch': n_epoch, 'er_learning_rate': learning_rate, 'er_other_params': kwargs }) print('Logging Info - Experiment: %s' % config.exp_name) model = RecognitionModel(config, **kwargs) dev_data_type = 'dev' if predict_on_final_test: test_data_type = 'test_final' else: test_data_type = 'test' valid_generator = RecognitionDataGenerator( dev_data_type, config.batch_size, config.label_schema, config.label_to_one_hot[config.label_schema], config.vocab if config.use_char_input else None, config.bert_vocab_file(config.bert_type) if config.use_bert_input else None, config.bert_seq_len, config.bichar_vocab, config.word_vocab, config.use_word_input, config.charpos_vocab, config.use_softword_input, config.use_dictfeat_input, config.use_maxmatch_input) test_generator = RecognitionDataGenerator( test_data_type, config.batch_size, config.label_schema, config.label_to_one_hot[config.label_schema], config.vocab if config.use_char_input else None, config.bert_vocab_file(config.bert_type) if config.use_bert_input else None, config.bert_seq_len, config.bichar_vocab, config.word_vocab, config.use_word_input, config.charpos_vocab, config.use_softword_input, config.use_dictfeat_input, config.use_maxmatch_input) model_save_path = os.path.join(config.checkpoint_dir, '{}.hdf5'.format(config.exp_name)) if not os.path.exists(model_save_path): raise FileNotFoundError( 'Recognition model not exist: {}'.format(model_save_path)) if swa_type is None: model.load_best_model() elif 'swa' in callbacks_to_add: model.load_swa_model(swa_type) predict_log['er_exp_name'] += '_{}'.format(swa_type) if predict_on_dev: print('Logging Info - Generate submission for valid data:') dev_pred_mentions = model.predict(valid_generator) else: dev_pred_mentions = None print('Logging Info - Generate submission for test data:') test_pred_mentions = model.predict(test_generator) return dev_pred_mentions, test_pred_mentions
def get_power_word(self): """ 计算数据中词语的影响力,格式如下: 词语 --> [0. 出现语句对数量,1. 出现语句对比例,2. 正确语句对比例,3. 单侧语句对比例,4. 单侧语句对正确比例, 5. 双侧语句对比例,6. 双侧语句对正确比例] """ print('Logging Info - Get power word...') words_power_path = os.path.join(FEATURE_DIR, '{}_power_word.pkl'.format(self.genre)) if os.path.exists(words_power_path): words_power = pickle_load(words_power_path) else: words_power = {} x_a = [ text.split() for text in self.train_data['premise'] + self.dev_data['premise'] + self.test_data['premise'] ] x_b = [ text.split() for text in self.train_data['hypothesis'] + self.dev_data['hypothesis'] + self.test_data['hypothesis'] ] y = self.train_data['label'] + self.dev_data[ 'label'] + self.test_data['label'] for i in range(len(x_a)): label = y[i] q1_words = x_a[i] q2_words = x_b[i] all_words = set(q1_words + q2_words) q1_words = set(q1_words) q2_words = set(q2_words) for word in all_words: if word not in words_power: words_power[word] = [0. for _ in range(7)] words_power[word][0] += 1. # 计算出现语句对的数量 words_power[word][1] += 1. # 计算出现语句对比例 if ((word in q1_words) and (word not in q2_words)) or ((word not in q1_words) and (word in q2_words)): words_power[word][3] += 1. # 计算单侧语句对比例 if 0 == label: words_power[word][2] += 1. # 计算正确语句对比例 words_power[word][4] += 1. # 计算单侧语句正确比例 if (word in q1_words) and (word in q2_words): words_power[word][5] += 1. # 计算双侧语句数量 if 2 == label: words_power[word][2] += 1. # 计算正确语句对比例 words_power[word][6] += 1. # 计算双侧语句正确比例 for word in words_power: words_power[word][1] /= len(x_a) # 计算出现语句对比例=出现语句对数量/总的语句对数量 words_power[word][2] /= words_power[word][ 0] # 计算正确语句对比例=正确语句对数量/出现语句对数量 if words_power[word][3] > 1e-6: words_power[word][4] /= words_power[word][ 3] # 计算单侧语句正确比例=单侧语句正确数量/出现单侧语句数量 words_power[word][3] /= words_power[word][ 0] # 计算出现单侧语句对比例=出现单侧语句数量/出现语句对数量 if words_power[word][5] > 1e-6: words_power[word][6] /= words_power[word][ 5] # 计算双侧语句正确比例=双侧语句正确数量/出现双侧语句数量 words_power[word][5] /= words_power[word][ 0] # 计算出现双侧语句对比例=出现双侧语句数量/出现语句数量 del x_a, x_b, y pickle_dump(words_power_path, words_power) return words_power
def train_link(model_name, batch_size=32, n_epoch=50, learning_rate=0.001, optimizer_type='adam', embed_type=None, embed_trainable=True, callbacks_to_add=None, use_relative_pos=False, n_neg=1, omit_one_cand=True, overwrite=False, swa_start=5, early_stopping_patience=3, **kwargs): config = ModelConfig() config.model_name = model_name config.batch_size = batch_size config.n_epoch = n_epoch config.learning_rate = learning_rate config.optimizer = get_optimizer(optimizer_type, learning_rate) config.embed_type = embed_type if embed_type: config.embeddings = np.load( format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, type=embed_type)) config.embed_trainable = embed_trainable else: config.embeddings = None config.embed_trainable = True config.callbacks_to_add = callbacks_to_add or [ 'modelcheckpoint', 'earlystopping' ] if 'swa' in config.callbacks_to_add: config.swa_start = swa_start config.early_stopping_patience = early_stopping_patience config.vocab = pickle_load( format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, level='char')) config.vocab_size = len(config.vocab) + 2 config.mention_to_entity = pickle_load( format_filename(PROCESSED_DATA_DIR, MENTION_TO_ENTITY_FILENAME)) config.entity_desc = pickle_load( format_filename(PROCESSED_DATA_DIR, ENTITY_DESC_FILENAME)) config.exp_name = '{}_{}_{}_{}_{}_{}'.format( model_name, embed_type if embed_type else 'random', 'tune' if config.embed_trainable else 'fix', batch_size, optimizer_type, learning_rate) config.use_relative_pos = use_relative_pos if config.use_relative_pos: config.exp_name += '_rel' config.n_neg = n_neg if config.n_neg > 1: config.exp_name += '_neg_{}'.format(config.n_neg) config.omit_one_cand = omit_one_cand if not config.omit_one_cand: config.exp_name += '_not_omit' if kwargs: config.exp_name += '_' + '_'.join( [str(k) + '_' + str(v) for k, v in kwargs.items()]) callback_str = '_' + '_'.join(config.callbacks_to_add) callback_str = callback_str.replace('_modelcheckpoint', '').replace('_earlystopping', '') config.exp_name += callback_str # logger to log output of training process train_log = { 'exp_name': config.exp_name, 'batch_size': batch_size, 'optimizer': optimizer_type, 'epoch': n_epoch, 'learning_rate': learning_rate, 'other_params': kwargs } print('Logging Info - Experiment: %s' % config.exp_name) model_save_path = os.path.join(config.checkpoint_dir, '{}.hdf5'.format(config.exp_name)) model = LinkModel(config, **kwargs) train_data_type, dev_data_type = 'train', 'dev' train_generator = LinkDataGenerator( train_data_type, config.vocab, config.mention_to_entity, config.entity_desc, config.batch_size, config.max_desc_len, config.max_erl_len, config.use_relative_pos, config.n_neg, config.omit_one_cand) dev_data = load_data(dev_data_type) if not os.path.exists(model_save_path) or overwrite: start_time = time.time() model.train(train_generator, dev_data) elapsed_time = time.time() - start_time print('Logging Info - Training time: %s' % time.strftime("%H:%M:%S", time.gmtime(elapsed_time))) train_log['train_time'] = time.strftime("%H:%M:%S", time.gmtime(elapsed_time)) model.load_best_model() dev_text_data, dev_pred_mentions, dev_gold_mention_entities = [], [], [] for data in dev_data: dev_text_data.append(data['text']) dev_pred_mentions.append(data['mention_data']) dev_gold_mention_entities.append(data['mention_data']) print('Logging Info - Evaluate over valid data:') r, p, f1 = model.evaluate(dev_text_data, dev_pred_mentions, dev_gold_mention_entities) train_log['dev_performance'] = (r, p, f1) swa_type = None if 'swa' in config.callbacks_to_add: swa_type = 'swa' elif 'swa_clr' in config.callbacks_to_add: swa_type = 'swa_clr' if swa_type: model.load_swa_model(swa_type) print('Logging Info - Evaluate over valid data based on swa model:') r, p, f1 = model.evaluate(dev_text_data, dev_pred_mentions, dev_gold_mention_entities) train_log['swa_dev_performance'] = (r, p, f1) train_log['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) write_log(format_filename(LOG_DIR, PERFORMANCE_LOG, model_type='2step_el'), log=train_log, mode='a') del model gc.collect() K.clear_session()
def __init__(self, data_type, batch_size, label_schema, label_to_onehot, char_vocab=None, bert_vocab=None, bert_seq_len=None, bichar_vocab=None, word_vocab=None, use_word_input=False, charpos_vocab=None, use_softword_input=False, use_dictfeat_input=False, use_maxmatch_input=False, shuffle=True): self.data_type = data_type self.data = load_data(data_type) self.data_size = len(self.data) self.batch_size = batch_size self.indices = np.arange(self.data_size) self.steps = int(np.ceil(self.data_size / self.batch_size)) assert label_schema in ['BIO', 'BIOES'] self.label_schema = label_schema self.label_to_onehot = label_to_onehot # main input self.char_vocab = char_vocab self.use_char_input = False if self.char_vocab is None else True # additional feature input self.bert_vocab = bert_vocab self.use_bert_input = False if self.bert_vocab is None else True self.bert_seq_len = bert_seq_len if self.use_bert_input else None assert self.use_char_input or self.use_bert_input if self.use_bert_input: self.token_dict = {} with codecs.open(self.bert_vocab, 'r', 'utf8') as reader: for line in reader: token = line.strip() self.token_dict[token] = len(self.token_dict) self.bert_tokenizer = Tokenizer(self.token_dict) self.bichar_vocab = bichar_vocab self.use_bichar_input = False if self.bichar_vocab is None else True self.word_vocab = word_vocab self.use_word_input = use_word_input assert not (self.use_word_input and self.word_vocab is None) self.charpos_vocab = charpos_vocab self.use_charpos_input = False if self.charpos_vocab is None else True self.use_softword_input = use_softword_input self.use_dictfeat_input = use_dictfeat_input self.use_maxmatch_input = use_maxmatch_input self.mention_to_entity = None if self.use_word_input or self.use_charpos_input or self.use_softword_input: self.mention_to_entity = pickle_load( format_filename(PROCESSED_DATA_DIR, MENTION_TO_ENTITY_FILENAME)) for mention in self.mention_to_entity.keys(): jieba.add_word(mention, freq=1000000) if (self.use_dictfeat_input or self.use_maxmatch_input) and self.mention_to_entity is None: self.mention_to_entity = pickle_load( format_filename(PROCESSED_DATA_DIR, MENTION_TO_ENTITY_FILENAME)) self.shuffle = shuffle
def add_graph_feature(self, data_type): feat_file = self.format_feature_file(data_type, 'word_power') if os.path.exists(feat_file): graph_features = pickle_load(feat_file) else: sent2id, graph = self.generate_graph() n2clique = {} cliques = [] for clique in nx.find_cliques(graph): for n in clique: if n not in n2clique: n2clique[n] = [] n2clique[n].append(len(cliques)) cliques.append(clique) n2cc = {} ccs = [] for cc in nx.connected_components(graph): for n in cc: n2cc[n] = len(ccs) ccs.append(cc) pagerank = nx.pagerank(graph, alpha=0.9, max_iter=100) hits_h, hits_a = nx.hits(graph, max_iter=100) indegree_features = list() clique_features = list() cc_features = list() pagerank_features = list() hits_features = list() shortestpath_features = list() # neighbor_features = list() for premise, hypothesis in zip( self.get_data(data_type)['premise'], self.get_data(data_type)['hypothesis']): p_id = sent2id[premise] h_id = sent2id[hypothesis] # graph in-degree fetures indegree_features.append( [graph.degree[p_id], graph.degree[h_id]]) # clique features edge_max_clique_size = 0 num_clique = 0 for clique_id in n2clique[p_id]: if h_id in cliques[clique_id]: edge_max_clique_size = max(edge_max_clique_size, len(cliques[clique_id])) num_clique += 1 clique_features.append([edge_max_clique_size, num_clique]) lnode_max_clique_size = 0 rnode_max_clique_size = 0 for clique_id in n2clique[p_id]: lnode_max_clique_size = max(lnode_max_clique_size, len(cliques[clique_id])) for clique_id in n2clique[h_id]: rnode_max_clique_size = max(rnode_max_clique_size, len(cliques[clique_id])) clique_features[-1] += [ lnode_max_clique_size, rnode_max_clique_size, max(lnode_max_clique_size, rnode_max_clique_size), min(lnode_max_clique_size, rnode_max_clique_size) ] # connected components features cc_features.append([len(ccs[n2cc[p_id]])]) # page rank features pr1 = pagerank[p_id] * 1e6 pr2 = pagerank[h_id] * 1e6 pagerank_features.append( [pr1, pr2, max(pr1, pr2), min(pr1, pr2), (pr1 + pr2) / 2.]) # graph hits features h1 = hits_h[p_id] * 1e6 h2 = hits_h[h_id] * 1e6 a1 = hits_a[p_id] * 1e6 a2 = hits_a[h_id] * 1e6 hits_features.append([ h1, h2, a1, a2, max(h1, h2), max(a1, a2), min(h1, h2), min(a1, a2), (h1 + h2) / 2., (a1 + a2) / 2. ]) # graph shortest path features shortest_path = -1 weight = graph[p_id][h_id]['weight'] graph.remove_edge(p_id, h_id) if nx.has_path(graph, p_id, h_id): shortest_path = nx.dijkstra_path_length(graph, p_id, h_id) graph.add_edge(p_id, h_id, weight=weight) shortestpath_features.append([shortest_path]) # graph neighbour features # l = [] # r = [] # l_nb = graph.neighbors(p_id) # r_nb = graph.neighbors(h_id) # for n in l_nb: # if (n != h_id) and (n != p_id): # l.append(graph[p_id][n]['weight']) # for n in r_nb: # if (n != h_id) and (n != p_id): # r.append(graph[h_id][n]['weight']) # if len(l) == 0 or len(r) == 0: # neighbor_features.append([0.0] * 11) # else: # neighbor_features.append(l + r + # [len(list((set(l_nb).union(set(r_nb))) ^ (set(l_nb) ^ set(r_nb))))]) graph_features = np.concatenate( (np.array(indegree_features), np.array(clique_features), np.array(cc_features), np.array(pagerank_features), np.array(hits_features), np.array(shortestpath_features)), axis=-1) pickle_dump(feat_file, graph_features) print('Logging Info - {} : graph feature shape : {}'.format( data_type, graph_features.shape)) return graph_features
def link(model_name, dev_pred_mentions, test_pred_mentions, predict_log, batch_size=32, n_epoch=50, learning_rate=0.001, optimizer_type='adam', embed_type=None, embed_trainable=True, use_relative_pos=False, n_neg=1, omit_one_cand=True, callbacks_to_add=None, swa_type=None, predict_on_final_test=True, **kwargs): config = ModelConfig() config.model_name = model_name config.batch_size = batch_size config.n_epoch = n_epoch config.learning_rate = learning_rate config.optimizer = get_optimizer(optimizer_type, learning_rate) config.embed_type = embed_type if embed_type: config.embeddings = np.load( format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, type=embed_type)) config.embed_trainable = embed_trainable else: config.embeddings = None config.embed_trainable = True config.callbacks_to_add = callbacks_to_add or [ 'modelcheckpoint', 'earlystopping' ] config.vocab = pickle_load( format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, level='char')) config.vocab_size = len(config.vocab) + 2 config.mention_to_entity = pickle_load( format_filename(PROCESSED_DATA_DIR, MENTION_TO_ENTITY_FILENAME)) config.entity_desc = pickle_load( format_filename(PROCESSED_DATA_DIR, ENTITY_DESC_FILENAME)) config.exp_name = '{}_{}_{}_{}_{}_{}'.format( model_name, embed_type if embed_type else 'random', 'tune' if embed_trainable else 'fix', batch_size, optimizer_type, learning_rate) config.use_relative_pos = use_relative_pos if config.use_relative_pos: config.exp_name += '_rel' config.n_neg = n_neg if config.n_neg > 1: config.exp_name += '_neg_{}'.format(config.n_neg) config.omit_one_cand = omit_one_cand if not config.omit_one_cand: config.exp_name += '_not_omit' if kwargs: config.exp_name += '_' + '_'.join( [str(k) + '_' + str(v) for k, v in kwargs.items()]) callback_str = '_' + '_'.join(config.callbacks_to_add) callback_str = callback_str.replace('_modelcheckpoint', '').replace('_earlystopping', '') config.exp_name += callback_str # logger to log output of training process predict_log.update({ 'el_exp_name': config.exp_name, 'el_batch_size': batch_size, 'el_optimizer': optimizer_type, 'el_epoch': n_epoch, 'el_learning_rate': learning_rate, 'el_other_params': kwargs }) print('Logging Info - Experiment: %s' % config.exp_name) model = LinkModel(config, **kwargs) model_save_path = os.path.join(config.checkpoint_dir, '{}.hdf5'.format(config.exp_name)) if not os.path.exists(model_save_path): raise FileNotFoundError( 'Recognition model not exist: {}'.format(model_save_path)) if swa_type is None: model.load_best_model() elif 'swa' in callbacks_to_add: model.load_swa_model(swa_type) predict_log['er_exp_name'] += '_{}'.format(swa_type) dev_data_type = 'dev' dev_data = load_data(dev_data_type) dev_text_data, dev_gold_mention_entities = [], [] for data in dev_data: dev_text_data.append(data['text']) dev_gold_mention_entities.append(data['mention_data']) if predict_on_final_test: test_data_type = 'test_final' else: test_data_type = 'test' test_data = load_data(test_data_type) test_text_data = [data['text'] for data in test_data] if dev_pred_mentions is not None: print( 'Logging Info - Evaluate over valid data based on predicted mention:' ) r, p, f1 = model.evaluate(dev_text_data, dev_pred_mentions, dev_gold_mention_entities) dev_performance = 'dev_performance' if swa_type is None else '%s_dev_performance' % swa_type predict_log[dev_performance] = (r, p, f1) print('Logging Info - Generate submission for test data:') test_pred_mention_entities = model.predict(test_text_data, test_pred_mentions) test_submit_file = predict_log[ 'er_exp_name'] + '_' + config.exp_name + '_%s%ssubmit.json' % ( swa_type + '_' if swa_type else '', 'final_' if predict_on_final_test else '') submit_result(test_submit_file, test_data, test_pred_mention_entities) predict_log['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) write_log(format_filename(LOG_DIR, PERFORMANCE_LOG, model_type='2step'), log=predict_log, mode='a') return predict_log
def train_recognition(model_name, label_schema='BIOES', batch_size=32, n_epoch=50, learning_rate=0.001, optimizer_type='adam', use_char_input=True, embed_type=None, embed_trainable=True, use_bert_input=False, bert_type='bert', bert_trainable=True, bert_layer_num=1, use_bichar_input=False, bichar_embed_type=None, bichar_embed_trainable=True, use_word_input=False, word_embed_type=None, word_embed_trainable=True, use_charpos_input=False, charpos_embed_type=None, charpos_embed_trainable=True, use_softword_input=False, use_dictfeat_input=False, use_maxmatch_input=False, callbacks_to_add=None, overwrite=False, swa_start=3, early_stopping_patience=3, **kwargs): config = ModelConfig() config.model_name = model_name config.label_schema = label_schema config.batch_size = batch_size config.n_epoch = n_epoch config.learning_rate = learning_rate config.optimizer = get_optimizer(optimizer_type, learning_rate) config.embed_type = embed_type config.use_char_input = use_char_input if embed_type: config.embeddings = np.load(format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, type=embed_type)) config.embed_trainable = embed_trainable config.embed_dim = config.embeddings.shape[1] else: config.embeddings = None config.embed_trainable = True config.callbacks_to_add = callbacks_to_add or ['modelcheckpoint', 'earlystopping'] if 'swa' in config.callbacks_to_add: config.swa_start = swa_start config.early_stopping_patience = early_stopping_patience config.vocab = pickle_load(format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, level='char')) config.vocab_size = len(config.vocab) + 2 config.mention_to_entity = pickle_load(format_filename(PROCESSED_DATA_DIR, MENTION_TO_ENTITY_FILENAME)) if config.use_char_input: config.exp_name = '{}_{}_{}_{}_{}_{}_{}'.format(model_name, config.embed_type if config.embed_type else 'random', 'tune' if config.embed_trainable else 'fix', batch_size, optimizer_type, learning_rate, label_schema) else: config.exp_name = '{}_{}_{}_{}_{}'.format(model_name, batch_size, optimizer_type, learning_rate, label_schema) if config.n_epoch != 50: config.exp_name += '_{}'.format(config.n_epoch) if kwargs: config.exp_name += '_' + '_'.join([str(k) + '_' + str(v) for k, v in kwargs.items()]) callback_str = '_' + '_'.join(config.callbacks_to_add) callback_str = callback_str.replace('_modelcheckpoint', '').replace('_earlystopping', '') config.exp_name += callback_str config.use_bert_input = use_bert_input config.bert_type = bert_type config.bert_trainable = bert_trainable config.bert_layer_num = bert_layer_num assert config.use_char_input or config.use_bert_input if config.use_bert_input: config.exp_name += '_{}_layer_{}_{}'.format(bert_type, bert_layer_num, 'tune' if config.bert_trainable else 'fix') config.use_bichar_input = use_bichar_input if config.use_bichar_input: config.bichar_vocab = pickle_load(format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, level='bichar')) config.bichar_vocab_size = len(config.bichar_vocab) + 2 if bichar_embed_type: config.bichar_embeddings = np.load(format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, type=bichar_embed_type)) config.bichar_embed_trainable = bichar_embed_trainable config.bichar_embed_dim = config.bichar_embeddings.shape[1] else: config.bichar_embeddings = None config.bichar_embed_trainable = True config.exp_name += '_bichar_{}_{}'.format(bichar_embed_type if bichar_embed_type else 'random', 'tune' if config.bichar_embed_trainable else 'fix') config.use_word_input = use_word_input if config.use_word_input: config.word_vocab = pickle_load(format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, level='word')) config.word_vocab_size = len(config.word_vocab) + 2 if word_embed_type: config.word_embeddings = np.load(format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, type=word_embed_type)) config.word_embed_trainable = word_embed_trainable config.word_embed_dim = config.word_embeddings.shape[1] else: config.word_embeddings = None config.word_embed_trainable = True config.exp_name += '_word_{}_{}'.format(word_embed_type if word_embed_type else 'random', 'tune' if config.word_embed_trainable else 'fix') config.use_charpos_input = use_charpos_input if config.use_charpos_input: config.charpos_vocab = pickle_load(format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, level='charpos')) config.charpos_vocab_size = len(config.charpos_vocab) + 2 if charpos_embed_type: config.charpos_embeddings = np.load(format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, type=charpos_embed_type)) config.charpos_embed_trainable = charpos_embed_trainable config.charpos_embed_dim = config.charpos_embeddings.shape[1] else: config.charpos_embeddings = None config.charpos_embed_trainable = True config.exp_name += '_charpos_{}_{}'.format(charpos_embed_type if charpos_embed_type else 'random', 'tune' if config.charpos_embed_trainable else 'fix') config.use_softword_input = use_softword_input if config.use_softword_input: config.exp_name += '_softword' config.use_dictfeat_input = use_dictfeat_input if config.use_dictfeat_input: config.exp_name += '_dictfeat' config.use_maxmatch_input = use_maxmatch_input if config.use_maxmatch_input: config.exp_name += '_maxmatch' # logger to log output of training process train_log = {'exp_name': config.exp_name, 'batch_size': batch_size, 'optimizer': optimizer_type, 'epoch': n_epoch, 'learning_rate': learning_rate, 'other_params': kwargs} print('Logging Info - Experiment: %s' % config.exp_name) model_save_path = os.path.join(config.checkpoint_dir, '{}.hdf5'.format(config.exp_name)) model = RecognitionModel(config, **kwargs) train_data_type, dev_data_type = 'train', 'dev' train_generator = RecognitionDataGenerator(train_data_type, config.batch_size, config.label_schema, config.label_to_one_hot[config.label_schema], config.vocab if config.use_char_input else None, config.bert_vocab_file(config.bert_type) if config.use_bert_input else None, config.bert_seq_len, config.bichar_vocab, config.word_vocab, config.use_word_input, config.charpos_vocab, config.use_softword_input, config.use_dictfeat_input, config.use_maxmatch_input) valid_generator = RecognitionDataGenerator(dev_data_type, config.batch_size, config.label_schema, config.label_to_one_hot[config.label_schema], config.vocab if config.use_char_input else None, config.bert_vocab_file(config.bert_type) if config.use_bert_input else None, config.bert_seq_len, config.bichar_vocab, config.word_vocab, config.use_word_input, config.charpos_vocab, config.use_softword_input, config.use_dictfeat_input, config.use_maxmatch_input) if not os.path.exists(model_save_path) or overwrite: start_time = time.time() model.train(train_generator, valid_generator) elapsed_time = time.time() - start_time print('Logging Info - Training time: %s' % time.strftime("%H:%M:%S", time.gmtime(elapsed_time))) train_log['train_time'] = time.strftime("%H:%M:%S", time.gmtime(elapsed_time)) model.load_best_model() print('Logging Info - Evaluate over valid data:') r, p, f1 = model.evaluate(valid_generator) train_log['dev_performance'] = (r, p, f1) swa_type = None if 'swa' in config.callbacks_to_add: swa_type = 'swa' elif 'swa_clr' in config.callbacks_to_add: swa_type = 'swa_clr' if swa_type: model.load_swa_model(swa_type) print('Logging Info - Evaluate over valid data based on swa model:') r, p, f1 = model.evaluate(valid_generator) train_log['swa_dev_performance'] = (r, p, f1) train_log['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) write_log(format_filename(LOG_DIR, PERFORMANCE_LOG, model_type='2step_er'), log=train_log, mode='a') del model gc.collect() K.clear_session()
def train_model(genre, input_level, word_embed_type, word_embed_trainable, batch_size, learning_rate, optimizer_type, model_name, n_epoch=50, add_features=False, scale_features=False, overwrite=False, lr_range_test=False, callbacks_to_add=None, eval_on_train=False, **kwargs): config = ModelConfig() config.genre = genre config.input_level = input_level config.max_len = config.word_max_len[genre] if input_level == 'word' else config.char_max_len[genre] config.word_embed_type = word_embed_type config.word_embed_trainable = word_embed_trainable config.callbacks_to_add = callbacks_to_add or [] config.add_features = add_features config.batch_size = batch_size config.learning_rate = learning_rate config.optimizer = get_optimizer(optimizer_type, learning_rate) config.n_epoch = n_epoch config.word_embeddings = np.load(format_filename(PROCESSED_DATA_DIR, EMBEDDING_MATRIX_TEMPLATE, genre, word_embed_type)) vocab = pickle_load(format_filename(PROCESSED_DATA_DIR, VOCABULARY_TEMPLATE, genre, input_level)) config.idx2token = dict((idx, token) for token, idx in vocab.items()) # experiment name configuration config.exp_name = '{}_{}_{}_{}_{}_{}_{}_{}'.format(genre, model_name, input_level, word_embed_type, 'tune' if word_embed_trainable else 'fix', batch_size, '_'.join([str(k) + '_' + str(v) for k, v in kwargs.items()]), optimizer_type) if config.add_features: config.exp_name = config.exp_name + '_feature_scaled' if scale_features else config.exp_name + '_featured' if len(config.callbacks_to_add) > 0: callback_str = '_' + '_'.join(config.callbacks_to_add) callback_str = callback_str.replace('_modelcheckpoint', '').replace('_earlystopping', '') config.exp_name += callback_str input_config = kwargs['input_config'] if 'input_config' in kwargs else 'token' # input default is word embedding if input_config in ['cache_elmo', 'token_combine_cache_elmo']: # get elmo embedding based on cache, we first get a ELMoCache instance if 'elmo_model_type' in kwargs: elmo_model_type = kwargs['elmo_model_type'] kwargs.pop('elmo_model_type') # we don't need it in kwargs any more else: elmo_model_type = 'allennlp' if 'elmo_output_mode' in kwargs: elmo_output_mode = kwargs['elmo_output_mode'] kwargs.pop('elmo_output_mode') # we don't need it in kwargs any more else: elmo_output_mode ='elmo' elmo_cache = ELMoCache(options_file=config.elmo_options_file, weight_file=config.elmo_weight_file, cache_dir=config.cache_dir, idx2token=config.idx2token, max_sentence_length=config.max_len, elmo_model_type=elmo_model_type, elmo_output_mode=elmo_output_mode) elif input_config in ['elmo_id', 'elmo_s', 'token_combine_elmo_id', 'token_combine_elmo_s']: # get elmo embedding using tensorflow_hub, we must provide a tfhub_url kwargs['elmo_model_url'] = config.elmo_model_url # logger to log output of training process train_log = {'exp_name': config.exp_name, 'batch_size': batch_size, 'optimizer': optimizer_type, 'epoch': n_epoch, 'learning_rate': learning_rate, 'other_params': kwargs} print('Logging Info - Experiment: %s' % config.exp_name) if model_name == 'KerasInfersent': model = KerasInfersentModel(config, **kwargs) elif model_name == 'KerasEsim': model = KerasEsimModel(config, **kwargs) elif model_name == 'KerasDecomposable': model = KerasDecomposableAttentionModel(config, **kwargs) elif model_name == 'KerasSiameseBiLSTM': model = KerasSimaeseBiLSTMModel(config, **kwargs) elif model_name == 'KerasSiameseCNN': model = KerasSiameseCNNModel(config, **kwargs) elif model_name == 'KerasIACNN': model = KerasIACNNModel(config, **kwargs) elif model_name == 'KerasSiameseLSTMCNNModel': model = KerasSiameseLSTMCNNModel(config, **kwargs) elif model_name == 'KerasRefinedSSAModel': model = KerasRefinedSSAModel(config, **kwargs) else: raise ValueError('Model Name Not Understood : {}'.format(model_name)) # model.summary() train_input, dev_input, test_input = None, None, None if lr_range_test: # conduct lr range test to find optimal learning rate (not train model) train_input = load_input_data(genre, input_level, 'train', input_config, config.add_features, scale_features) dev_input = load_input_data(genre, input_level, 'dev', input_config, config.add_features, scale_features) model.lr_range_test(x_train=train_input['x'], y_train=train_input['y'], x_valid=dev_input['x'], y_valid=dev_input['y']) return model_save_path = os.path.join(config.checkpoint_dir, '{}.hdf5'.format(config.exp_name)) if not os.path.exists(model_save_path) or overwrite: start_time = time.time() if input_config in ['cache_elmo', 'token_combine_cache_elmo']: train_input = ELMoGenerator(genre, input_level, 'train', config.batch_size, elmo_cache, return_data=(input_config == 'token_combine_cache_elmo'), return_features=config.add_features) dev_input = ELMoGenerator(genre, input_level, 'dev', config.batch_size, elmo_cache, return_data=(input_config == 'token_combine_cache_elmo'), return_features=config.add_features) model.train_with_generator(train_input, dev_input) else: train_input = load_input_data(genre, input_level, 'train', input_config, config.add_features, scale_features) dev_input = load_input_data(genre, input_level, 'dev', input_config, config.add_features, scale_features) model.train(x_train=train_input['x'], y_train=train_input['y'], x_valid=dev_input['x'], y_valid=dev_input['y']) elapsed_time = time.time() - start_time print('Logging Info - Training time: %s' % time.strftime("%H:%M:%S", time.gmtime(elapsed_time))) train_log['train_time'] = time.strftime("%H:%M:%S", time.gmtime(elapsed_time)) def eval_on_data(eval_with_generator, input_data, data_type): model.load_best_model() if eval_with_generator: acc = model.evaluate_with_generator(generator=input_data, y=input_data.input_label) else: acc = model.evaluate(x=input_data['x'], y=input_data['y']) train_log['%s_acc' % data_type] = acc swa_type = None if 'swa' in config.callbacks_to_add: swa_type = 'swa' elif 'swa_clr' in config.callbacks_to_add: swa_type = 'swa_clr' if swa_type: print('Logging Info - %s Model' % swa_type) model.load_swa_model(swa_type=swa_type) swa_acc = model.evaluate(x=input_data['x'], y=input_data['y']) train_log['%s_%s_acc' % (swa_type, data_type)] = swa_acc ensemble_type = None if 'sse' in config.callbacks_to_add: ensemble_type = 'sse' elif 'fge' in config.callbacks_to_add: ensemble_type = 'fge' if ensemble_type: print('Logging Info - %s Ensemble Model' % ensemble_type) ensemble_predict = {} for model_file in os.listdir(config.checkpoint_dir): if model_file.startswith(config.exp_name+'_%s' % ensemble_type): match = re.match(r'(%s_%s_)([\d+])(.hdf5)' % (config.exp_name, ensemble_type), model_file) model_id = int(match.group(2)) model_path = os.path.join(config.checkpoint_dir, model_file) print('Logging Info: Loading {} ensemble model checkpoint: {}'.format(ensemble_type, model_file)) model.load_model(model_path) ensemble_predict[model_id] = model.predict(x=input_data['x']) ''' we expect the models saved towards the end of run may have better performance than models saved earlier in the run, we sort the models so that the older models ('s id) are first. ''' sorted_ensemble_predict = sorted(ensemble_predict.items(), key=lambda x: x[0], reverse=True) model_predicts = [] for model_id, model_predict in sorted_ensemble_predict: single_acc = eval_acc(model_predict, input_data['y']) print('Logging Info - %s_single_%d_%s Acc : %f' % (ensemble_type, model_id, data_type, single_acc)) train_log['%s_single_%d_%s_acc' % (ensemble_type, model_id, data_type)] = single_acc model_predicts.append(model_predict) ensemble_acc = eval_acc(np.mean(np.array(model_predicts), axis=0), input_data['y']) print('Logging Info - %s_ensemble_%d_%s Acc : %f' % (ensemble_type, model_id, data_type, ensemble_acc)) train_log['%s_ensemble_%d_%s_acc' % (ensemble_type, model_id, data_type)] = ensemble_acc if eval_on_train: # might take a long time print('Logging Info - Evaluate over train data:') if input_config in ['cache_elmo', 'token_combine_cache_elmo']: train_input = ELMoGenerator(genre, input_level, 'train', config.batch_size, elmo_cache, return_data=(input_config == 'token_combine_cache_elmo'), return_features=config.add_features, return_label=False) eval_on_data(eval_with_generator=True, input_data=train_input, data_type='train') else: train_input = load_input_data(genre, input_level, 'train', input_config, config.add_features, scale_features) eval_on_data(eval_with_generator=False, input_data=train_input, data_type='train') print('Logging Info - Evaluate over valid data:') if input_config in ['cache_elmo', 'token_combine_cache_elmo']: dev_input = ELMoGenerator(genre, input_level, 'dev', config.batch_size, elmo_cache, return_data=(input_config == 'token_combine_cache_elmo'), return_features=config.add_features, return_label=False) eval_on_data(eval_with_generator=True, input_data=dev_input, data_type='dev') else: if dev_input is None: dev_input = load_input_data(genre, input_level, 'dev', input_config, config.add_features, scale_features) eval_on_data(eval_with_generator=False, input_data=dev_input, data_type='dev') print('Logging Info - Evaluate over test data:') if input_config in ['cache_elmo', 'token_combine_cache_elmo']: test_input = ELMoGenerator(genre, input_level, 'test', config.batch_size, elmo_cache, return_data=(input_config == 'token_combine_cache_elmo'), return_features=config.add_features, return_label=False) eval_on_data(eval_with_generator=True, input_data=test_input, data_type='test') else: if test_input is None: test_input = load_input_data(genre, input_level, 'test', input_config, config.add_features, scale_features) eval_on_data(eval_with_generator=False, input_data=test_input, data_type='test') train_log['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) write_log(format_filename(LOG_DIR, PERFORMANCE_LOG, genre), log=train_log, mode='a') return train_log
if __name__ == '__main__': if not os.path.exists(PREDICT_DIR): os.makedirs(PREDICT_DIR) config = ModelConfig() raw_data = dict() raw_data['simplified'] = read_raw_test_data(SIMP_TEST_FILENAME) raw_data['traditional'] = read_raw_test_data(TRAD_TEST_FILENAME) for variation in raw_data.keys(): test_data = raw_data[variation] # prepare word embedding input word_tokenizer = pickle_load( format_filename(PROCESSED_DATA_DIR, TOKENIZER_TEMPLATE, variation=variation, level='word')) word_ids_test = create_token_ids_matrix(word_tokenizer, raw_data[variation], config.word_max_len) # prepare n-gram input vectorizer = pickle_load( format_filename(PROCESSED_DATA_DIR, VECTORIZER_TEMPLATE, variation=variation, type='binary', level='char', ngram_range=(2, 3))) n_gram_test = vectorizer.transform(raw_data[variation])