def __init__(self, mode): Directory.__init__(self, mode) self.data_path += '/cnf' ''' DATASET ''' self.raw_data_filename = self.data_path + self.raw_data_filename self.data_filename = self.data_path + self.data_filename self.gold_label_filename = self.data_path + self.gold_label_filename self.weak_label_filename = self.data_path + self.weak_label_filename
def __init__(self, mode): Directory.__init__(self, mode) self.data_path += '/tar' ''' DATASET ''' self.raw_data_filename = self.data_path + self.raw_data_filename self.data_filename = self.data_path + self.data_filename self.gold_label_filename = self.data_path + self.gold_label_filename # For valid gold label is used as weak label self.weak_label_filename = self.data_path + self.weak_label_filename # For train, only weak label
def main(domain): data_dir = Directory('TR').data_path source = data_dir + '/%s/pre_processed_restaurant/train.txt' % (domain) model_file = data_dir + '/%s/pre_processed_restaurant/w2v_embedding_new' % ( domain) sentences = MySentences(source) model = gensim.models.Word2Vec(sentences, size=200, window=5, min_count=10, workers=4) model.save(model_file)
def init_test(self, dict_obj): mode_train, mode_test = 'TR', 'TE' params_train = ParamsClass(mode=mode_train) dir_train = Directory(mode_train) # test object params_test = ParamsClass(mode=mode_test) dir_test = Directory(mode_test) params_test.num_instances, params_test.indices = self.get_length(dir_test.data_filename) # params_test.batch_size = 1 params_train.num_classes = params_test.num_classes = len(dict_obj.label_dict) min_loss = sys.float_info.max word_emb_path = dir_train.word_embedding word_emb_matrix = np.float32(np.genfromtxt(word_emb_path, delimiter=' ')) params_train.vocab_size = params_test.vocab_size = len(word_emb_matrix) print('***** INITIALIZING TF GRAPH *****') session = tf.Session() # train_writer = tf.summary.FileWriter(dir_train.log_path + '/train', session.graph) # test_writer = tf.summary.FileWriter(dir_train.log_path + '/test') # random_uniform_initializer = tf.random_uniform_initializer(-params_train.init_scale, params_train.init_scale) # xavier_initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32) with tf.variable_scope("classifier", reuse=None): test_obj = SMN(params_test, dir_test) model_saver = tf.train.Saver() print('Loading model ...') model_saver.restore(session, Directory('TE').test_model) print('**** MODEL LOADED ****\n') return session, test_obj
def run_test(self): global test_writer mode_test = 'TE' # test object params_test = ParamsClass(mode=mode_test) dir_test = Directory(mode_test) test_reader = Reader(params_test) test_instances = test_reader.read_image_data(dir_test.data_filename) random.seed(4321) global_min_loss = sys.float_info.max print('***** INITIALIZING TF GRAPH *****') with tf.Graph().as_default(), tf.Session() as session: with tf.variable_scope("model"): test_obj = Autoencoder(params_test, dir_test) model_saver = tf.train.Saver() model_saver.restore(session, test_obj.dir_obj.test_model) if params_test.log: test_writer = tf.summary.FileWriter(dir_test.log_path + '/test') print('**** TF GRAPH INITIALIZED ****') start_time = time.time() test_loss, _, = self.run_epoch(session, global_min_loss, test_obj, test_reader, test_instances, test_writer) print("Epoch: %d Test loss: %.4f" % (1, test_loss)) curr_time = time.time() print('1 epoch run takes ' + str((curr_time - start_time) / 60) + ' minutes.') if params_test.log: test_writer.close()
def train_util(): """ Utility function to execute the training pipeline :return: None """ # To use training set only # build_sampled_training_file.util() # build_word_vocab.util() # To use complete glove file + training set dir_obj = Directory('TR') vocab_builder = VocabBuilder() vocab_builder.create_vocab_dict(dir_obj.data_filename) vocab_builder.extract_glove_vectors(dir_obj.word_vocab_dict, dir_obj.glove_path) dict_obj = load_dictionary() call_train(dict_obj) return None
def run_epoch(self, session, eval_op, model_obj, dict_obj, verbose=False): global summary, iter_train, iter_valid epoch_combined_loss = 0.0 total_correct = 0.0 total_instances = 0.0 print('\nrun epoch') output_file = open(Directory('TE').test_cost_path, 'w') params = model_obj.params dir_obj = model_obj.dir_obj data_filename = dir_obj.data_filename label_filename = dir_obj.label_filename for step, (ctx_arr, ctx_len_arr, num_ctx_arr, resp_arr, resp_len_arr, label_arr) \ in enumerate(DataReader(params).data_iterator(data_filename, label_filename, model_obj.params.indices, dict_obj)): feed_dict = {model_obj.ctx: ctx_arr, model_obj.ctx_len_placeholders: ctx_len_arr, model_obj.num_ctx_placeholders: num_ctx_arr, model_obj.resp: resp_arr, model_obj.resp_len_placeholders: resp_len_arr, model_obj.label: label_arr } loss, prediction, probabilities, accuracy, _ = session.run([model_obj.loss, model_obj.prediction, model_obj.probabilities, model_obj.accuracy, eval_op], feed_dict=feed_dict) total_correct += np.sum(prediction == label_arr) total_instances += params.batch_size epoch_combined_loss += loss for idx, each_pred in enumerate(probabilities): # output_file.write(str(each_pred) + '\t' + str(probabilities[idx]) + '\n') output_file.write(str(each_pred[1]) + '\n') print 'CE loss: %.4f, Accuracy: %.4f' % (epoch_combined_loss, (total_correct / total_instances) * 100) return epoch_combined_loss
def init_test(self): mode_train, mode_valid, mode_test = 'TR', 'VA', 'TE' # test object params_test = Params(mode=mode_test) dir_test = Directory(mode_test) params_test.num_instances, params_test.indices = self.getLength( dir_test.data_filename) min_loss = sys.float_info.max word_emb_path = dir_test.word_embedding word_emb_matrix = np.float32( np.genfromtxt(word_emb_path, delimiter=' ')) params_test.VOCAB_SIZE = len(word_emb_matrix) params_test.batch_size = 10 print('***** INITIALIZING TF GRAPH *****') session = tf.Session() # with tf.name_scope('train'): with tf.variable_scope("model", reuse=None): test_obj = Model(params_test, dir_test) model_saver = tf.train.Saver() print('Loading model ...') model_saver.restore(session, dir_test.test_model) # builder = saved_model_builder.SavedModelBuilder(set_dir.Directory('TE').test_model) # builder.add_meta_graph_and_variables(session, ['serve'], signature_def_map=None) # builder.save() # return print('**** MODEL LOADED ****\n') return session, test_obj
def create_vocab_dict(self, input_filename): word_dict = {} word_count = {} word_counter = 2 max_sequence_length = 0 input_file = open(input_filename, 'r').readlines() for each_line in input_file: if self.params.all_lowercase: each_line = each_line.lower() tokenized_string = each_line.strip().split() for each_token in tokenized_string: if each_token != 'unk': if each_token not in word_count: word_count[each_token] = 1 else: word_count[each_token] = 1 + word_count[each_token] line_len = len(tokenized_string) if (line_len > max_sequence_length): max_sequence_length = line_len for each_token in word_count: if word_count[each_token] >= self.params.sampling_threshold: word_dict[each_token] = word_counter word_counter += 1 word_vocab = open(Directory('TR').word_vocab_dict, 'wb') pickle.dump(word_dict, word_vocab, protocol=cPickle.HIGHEST_PROTOCOL) word_vocab.close() print('Reading Completed \n ========================== ' '\n Unique tokens: excluding padding and unkown words %d ' '\n Max. sequence length: %d' '\n ==========================\n' % (word_counter - 2, max_sequence_length))
def run_train(self, dict_obj): mode_train, mode_valid, mode_test = 'TR', 'VA', 'TE' # train object params_train = Params(mode=mode_train) dir_train = Directory(mode_train) params_train.num_instances, params_train.indices = self.getLength( dir_train.data_filename) # valid object params_valid = Params(mode=mode_valid) dir_valid = Directory(mode_valid) params_valid.num_instances, params_valid.indices = self.getLength( dir_valid.data_filename) # test object # # params_test = Params(mode=mode_test) # dir_test = Directory(mode_test) # params_test.num_instances, params_test.indices = self.getLength(dir_test.data_filename) random.seed(1234) if (params_train.enable_shuffle): random.shuffle(params_train.indices) random.shuffle(params_valid.indices) min_loss = sys.float_info.max word_emb_path = dir_train.word_embedding word_emb_matrix = np.float32( np.genfromtxt(word_emb_path, delimiter=' ')) params_train.VOCAB_SIZE = params_valid.VOCAB_SIZE = len( word_emb_matrix) logger.info('***** INITIALIZING TF GRAPH *****') with tf.Graph().as_default(), tf.Session() as session: # train_writer = tf.summary.FileWriter(dir_train.log_path + '/train', session.graph) # test_writer = tf.summary.FileWriter(dir_train.log_path + '/test') # random_normal_initializer = tf.random_normal_initializer() # random_uniform_initializer = tf.random_uniform_initializer(-params_train.init_scale, params_train.init_scale) xavier_initializer = tf.contrib.layers.xavier_initializer( uniform=True, seed=None, dtype=tf.float32) # with tf.name_scope('train'): with tf.variable_scope("model", reuse=None, initializer=xavier_initializer): train_obj = Model(params_train, dir_train) # with tf.name_scope('valid'): with tf.variable_scope("model", reuse=True, initializer=xavier_initializer): valid_obj = Model(params_valid, dir_valid) # with tf.variable_scope("model", reuse=True, initializer=xavier_initializer): # test_obj = Model(params_test, dir_test) if not params_train.enable_checkpoint: session.run(tf.global_variables_initializer()) if params_train.enable_checkpoint: ckpt = tf.train.get_checkpoint_state(dir_train.model_path) if ckpt and ckpt.model_checkpoint_path: print("Loading model from: %s" % ckpt.model_checkpoint_path) tf.train.Saver().restore(session, ckpt.model_checkpoint_path) elif (not params_train.use_random_initializer): self.assign_word_emb_matrix(session, train_obj, word_emb_matrix) self.assign_aspect_emb_matrix(session, params_train, word_emb_matrix, train_obj) model_saver = tf.train.Saver() model_saver.save(session, save_path=dir_train.model_path + dir_train.model_name, latest_filename=dir_train.latest_checkpoint) print('==== Model saved! ====') logger.info('**** TF GRAPH INITIALIZED ****') # train_writer.add_graph(tf.get_default_graph()) start_time = time.time() reader = DataReader() for i in range(params_train.max_max_epoch): lr_decay = params_train.lr_decay**max( i - params_train.max_epoch, 0.0) # train_obj.assign_lr(session, params_train.learning_rate * lr_decay) # print(params_train.learning_rate * lr_decay) self.get_aspect_words(train_obj, dict_obj.word_dict) print('\n++++++++=========+++++++\n') lr = params_train.learning_rate * lr_decay logger.info("Epoch: %d Learning rate: %.5f" % (i + 1, lr)) train_loss, _, summary = self.run_epoch(session, min_loss, train_obj, dict_obj, i, reader, lr, verbose=True) print("Epoch: %d Train loss: %.4f" % (i + 1, train_loss)) valid_loss, curr_loss, summary = self.run_epoch( session, min_loss, valid_obj, dict_obj, i, reader) if (curr_loss < min_loss): min_loss = curr_loss print("Epoch: %d Valid loss: %.4f" % (i + 1, valid_loss)) # test_loss, _, _ = self.run_epoch(session, min_loss, test_obj, dict_obj, i, reader) # print("Epoch: %d Test loss: %.4f" % (i + 1, test_loss)) curr_time = time.time() print('1 epoch run takes ' + str( ((curr_time - start_time) / (i + 1)) / 60) + ' minutes.') # train_writer.close() # test_writer.close() self.get_aspect_words(train_obj, dict_obj.word_dict)
def run_train(self, dict_obj): mode_train, mode_valid, mode_all = 'TR', 'VA', 'ALL' # train object params_train = ParamsClass(mode=mode_train) dir_train = Directory(mode_train) params_train.num_instances, params_train.indices = self.get_length( dir_train.data_filename) # valid object params_valid = ParamsClass(mode=mode_valid) dir_valid = Directory(mode_valid) params_valid.num_instances, params_valid.indices = self.get_length( dir_valid.data_filename) params_train.num_classes = params_valid.num_classes = len( dict_obj.label_dict) if params_train.enable_shuffle: random.shuffle(params_train.indices) random.shuffle(params_valid.indices) min_loss = sys.float_info.max word_emb_path = dir_train.word_embedding word_emb_matrix = np.float32( np.genfromtxt(word_emb_path, delimiter=' ')) params_train.vocab_size = params_valid.vocab_size = len( word_emb_matrix) print('***** INITIALIZING TF GRAPH *****') timestamp = str(int(time.time())) train_out_dir = os.path.abspath( os.path.join(dir_train.log_path, "train", timestamp)) valid_out_dir = os.path.abspath( os.path.join(dir_train.log_path, "valid", timestamp)) print("Writing to {}\n".format(train_out_dir)) with tf.Graph().as_default(), tf.Session() as session: # random_normal_initializer = tf.random_normal_initializer() # random_uniform_initializer = tf.random_uniform_initializer(-params_train.init_scale, params_train.init_scale) xavier_initializer = tf.contrib.layers.xavier_initializer( uniform=True, seed=None, dtype=tf.float32) with tf.variable_scope("classifier", reuse=None, initializer=xavier_initializer): train_obj = SMN(params_train, dir_train) train_writer = tf.summary.FileWriter(train_out_dir, session.graph) valid_writer = tf.summary.FileWriter(valid_out_dir) if not params_train.enable_checkpoint: session.run(tf.global_variables_initializer()) if params_train.enable_checkpoint: ckpt = tf.train.get_checkpoint_state(dir_train.model_path) if ckpt and ckpt.model_checkpoint_path: print("Loading model from: %s" % ckpt.model_checkpoint_path) tf.train.Saver().restore(session, ckpt.model_checkpoint_path) elif not params_train.use_random_initializer: session.run( tf.assign(train_obj.word_emb_matrix, word_emb_matrix, name="word_embedding_matrix")) with tf.variable_scope("classifier", reuse=True, initializer=xavier_initializer): valid_obj = SMN(params_valid, dir_valid) print('**** TF GRAPH INITIALIZED ****') start_time = time.time() for i in range(params_train.max_max_epoch): lr_decay = params_train.lr_decay**max( i - params_train.max_epoch, 0.0) train_obj.assign_lr(session, params_train.learning_rate * lr_decay) # print(params_train.learning_rate * lr_decay) print('\n++++++++=========+++++++\n') print("Epoch: %d Learning rate: %.5f" % (i + 1, session.run(train_obj.lr))) train_loss, _ = self.run_epoch(session, train_writer, train_obj.train_op, min_loss, train_obj, dict_obj, i, verbose=True) print("Epoch: %d Train loss: %.3f" % (i + 1, train_loss)) valid_loss, curr_loss = self.run_epoch(session, valid_writer, tf.no_op(), min_loss, valid_obj, dict_obj, i) if curr_loss < min_loss: min_loss = curr_loss print("Epoch: %d Valid loss: %.3f" % (i + 1, valid_loss)) curr_time = time.time() print('1 epoch run takes ' + str( ((curr_time - start_time) / (i + 1)) / 60) + ' minutes.') train_writer.close() valid_writer.close()
def run_train(self, dict_obj): tf.set_random_seed(1234) mode_train, mode_valid, mode_test = 'TR', 'VA', 'TE' # global params and dir global_params = GlobalParams() global_dir = Directory('TR') # cnf train object cnf_params_train = ConfidenceNetworkParams(mode=mode_train) cnf_dir_train = ConfidenceNetworkDirectory(mode_train) cnf_params_train.num_instances, cnf_params_train.indices = self.get_length( cnf_dir_train.data_filename) # tar train object tar_params_train = TargetNetworkParams(mode=mode_train) tar_dir_train = TargetNetworkDirectory(mode_train) tar_params_train.num_instances, tar_params_train.indices = self.get_length( tar_dir_train.data_filename) # cnf valid object cnf_params_valid = ConfidenceNetworkParams(mode=mode_valid) cnf_dir_valid = ConfidenceNetworkDirectory(mode_valid) cnf_params_valid.num_instances, cnf_params_valid.indices = self.get_length( cnf_dir_valid.data_filename) cnf_params_valid.batch_size = 64 # tar valid object tar_params_valid = TargetNetworkParams(mode=mode_valid) tar_dir_valid = TargetNetworkDirectory(mode_valid) tar_params_valid.num_instances, tar_params_valid.indices = self.get_length( tar_dir_valid.data_filename) tar_params_valid.batch_size = 64 # params_train.num_classes = params_valid.num_classes = len(dict_obj.label_dict) if global_params.enable_shuffle: random.shuffle(cnf_params_train.indices) random.shuffle(cnf_params_valid.indices) random.shuffle(tar_params_train.indices) random.shuffle(tar_params_valid.indices) cnf_min_loss = tar_min_loss = sys.float_info.max word_emb_path = global_dir.word_embedding word_emb_matrix = np.float32( np.genfromtxt(word_emb_path, delimiter=' ')) global_params.vocab_size = len(word_emb_matrix) print('***** INITIALIZING TF GRAPH *****') timestamp = str(int(time.time())) # cnf_train_out_dir = os.path.abspath(os.path.join(dir_train.log_path, "train", timestamp)) # valid_out_dir = os.path.abspath(os.path.join(dir_train.log_path, "valid", timestamp)) # print("Writing to {}\n".format(cnf_train_out_dir)) cnf_train_out_dir = os.path.abspath( os.path.join(global_dir.cnf_log_path, "train", timestamp)) cnf_valid_out_dir = os.path.abspath( os.path.join(global_dir.cnf_log_path, "valid", timestamp)) tar_train_out_dir = os.path.abspath( os.path.join(global_dir.tar_log_path, "train", timestamp)) tar_valid_out_dir = os.path.abspath( os.path.join(global_dir.tar_log_path, "valid", timestamp)) with tf.Graph().as_default(), tf.Session() as session: # random_normal_initializer = tf.random_normal_initializer() random_uniform_initializer = tf.random_uniform_initializer( -global_params.init_scale, global_params.init_scale) # xavier_initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32) with tf.variable_scope("classifier", reuse=None, initializer=random_uniform_initializer): train_obj = L2LWS(global_params, cnf_params_train, tar_params_train, global_dir, cnf_dir_train, tar_dir_train) cnf_train_writer = tf.summary.FileWriter(cnf_train_out_dir, session.graph) cnf_valid_writer = tf.summary.FileWriter(cnf_valid_out_dir) tar_train_writer = tf.summary.FileWriter(tar_train_out_dir, session.graph) tar_valid_writer = tf.summary.FileWriter(tar_valid_out_dir) if not global_params.enable_checkpoint: session.run(tf.global_variables_initializer()) if global_params.enable_checkpoint: ckpt = tf.train.get_checkpoint_state(global_dir.model_path) if ckpt and ckpt.model_checkpoint_path: print("Loading model from: %s" % ckpt.model_checkpoint_path) tf.train.Saver().restore(session, ckpt.model_checkpoint_path) elif not global_params.use_random_initializer: session.run( tf.assign(train_obj.word_emb_matrix, word_emb_matrix, name="word_embedding_matrix")) with tf.variable_scope("classifier", reuse=True, initializer=random_uniform_initializer): valid_obj = L2LWS(global_params, cnf_params_valid, tar_params_valid, global_dir, cnf_dir_valid, tar_dir_valid) print('**** TF GRAPH INITIALIZED ****') start_time = time.time() for i in range(global_params.max_max_epoch): print('\n++++++++=========+++++++\n') if i >= global_params.pre_train_epoch: ckpt = tf.train.get_checkpoint_state(global_dir.model_path) if ckpt and ckpt.model_checkpoint_path: print("Loading model from: %s" % ckpt.model_checkpoint_path) tf.train.Saver().restore(session, ckpt.model_checkpoint_path) lr_decay = cnf_params_train.lr_decay**max( i - global_params.max_epoch, 0.0) train_obj.cnf_network.assign_lr( session, cnf_params_train.learning_rate * lr_decay) train_obj.tar_network.assign_lr( session, cnf_params_train.learning_rate * lr_decay) print("Epoch: %d Learning rate: %.5f" % (i + 1, session.run(train_obj.cnf_network.lr))) train_loss, _ = self.run_cnf_net_epoch(cnf_train_writer, session, cnf_min_loss, train_obj, dict_obj, i) print("CONFIDENCE NETWORK: Epoch: %d Train loss: %.3f\n" % (i + 1, train_loss)) valid_obj.cnf_network.assign_lr( session, cnf_params_train.learning_rate * lr_decay) valid_obj.tar_network.assign_lr( session, cnf_params_train.learning_rate * lr_decay) valid_loss, curr_loss = self.run_cnf_net_epoch( cnf_valid_writer, session, cnf_min_loss, valid_obj, dict_obj, i) if curr_loss < cnf_min_loss: cnf_min_loss = curr_loss print("CONFIDENCE NETWORK: Epoch: %d Valid loss: %.3f" % (i + 1, valid_loss)) curr_time = time.time() print('1 epoch run takes ' + str( ((curr_time - start_time) / (i + 1)) / 60) + ' minutes.') print('\n++++++++=========+++++++\n') ''' TARGET NETWORK ''' if i >= global_params.pre_train_epoch: if ckpt and ckpt.model_checkpoint_path: print("Loading model from: %s" % ckpt.model_checkpoint_path) tf.train.Saver().restore(session, ckpt.model_checkpoint_path) lr_decay = tar_params_train.lr_decay**max( i - global_params.max_epoch, 0.0) train_obj.cnf_network.assign_lr( session, cnf_params_train.learning_rate * lr_decay) train_obj.tar_network.assign_lr( session, cnf_params_train.learning_rate * lr_decay) print("Epoch: %d Learning rate: %.5f" % (i + 1, session.run(train_obj.tar_network.lr))) train_loss, _ = self.run_tar_net_epoch( tar_train_writer, session, tar_min_loss, train_obj, dict_obj, i) print("TARGET NETWORK: Epoch: %d Train loss: %.3f\n" % (i + 1, train_loss)) valid_obj.cnf_network.assign_lr( session, cnf_params_train.learning_rate * lr_decay) valid_obj.tar_network.assign_lr( session, cnf_params_train.learning_rate * lr_decay) valid_loss, curr_loss = self.run_tar_net_epoch( tar_valid_writer, session, tar_min_loss, valid_obj, dict_obj, i) if curr_loss < tar_min_loss: tar_min_loss = curr_loss print("TARGET NETWORK: Epoch: %d Valid loss: %.3f" % (i + 1, valid_loss)) curr_time = time.time() print('1 epoch run takes ' + str( ((curr_time - start_time) / (i + 1)) / 60) + ' minutes.') print('\n++++++++=========+++++++\n') cnf_train_writer.close() cnf_valid_writer.close() tar_train_writer.close() tar_valid_writer.close()
import gensim import pickle, cPickle import collections import numpy as np from global_module.settings_module import Directory def convert(emb_file): # model = gensim.models.KeyedVectors.load_word2vec_format(emb_file, binary=True, unicode_errors='ignore') model = gensim.models.Word2Vec.load(emb_file) output_emb_file = open(emb_file + '_dict.pkl', 'wb') vocab_dict = collections.OrderedDict() for word in model.wv.index2word: word_vec = model[word] # print word, model.wv.most_similar(positive=word) word_vec = np.array2string(word_vec) word_vec = word_vec.replace('[', '') word_vec = word_vec.replace(']', '') vocab_dict[word] = (' '.join(word_vec.split())).strip() pickle.dump(vocab_dict, output_emb_file, protocol=cPickle.HIGHEST_PROTOCOL) base_dir = Directory('TR').data_path convert(base_dir + '/restaurant/pre_processed_restaurant/w2v_embedding')
def main(): SMN(ParamsClass(), Directory())
def extract_glove_vectors(self, word_vocab_file, glove_file): glove_vocab_dict = cPickle.load(open(glove_file, 'rb')) word_vocab_dict = cPickle.load(open(word_vocab_file, 'rb')) length_word_vector = self.params.EMB_DIM glove_present_training_word_vocab_dict = {} glove_present_training_word_counter = 1 glove_present_word_vector_dict = {} freq_thres = len(glove_vocab_dict) if 'unk' not in glove_vocab_dict and 'UNK' not in glove_vocab_dict: glove_present_training_word_vocab_dict['unk'] = 1 vec_str = '' for i in range(length_word_vector): vec_str += str(round(random.uniform(-0.1, 0.1), 6)) + ' ' glove_present_word_vector_dict[1] = vec_str.strip() glove_present_training_word_counter = 2 for key, value in glove_vocab_dict.items()[:freq_thres]: # if key == 'UNK': # key = key.lower() vec = glove_vocab_dict.get(key) if self.params.all_lowercase: key = key.lower() if key in glove_vocab_dict: vec = glove_vocab_dict.get(key) if key not in glove_present_training_word_vocab_dict: glove_present_training_word_vocab_dict[ key] = glove_present_training_word_counter glove_present_word_vector_dict[ glove_present_training_word_counter] = vec glove_present_training_word_counter += 1 #to use training words not present in top frequent words of glove for key, value in word_vocab_dict.items(): if key not in glove_present_training_word_vocab_dict and key in glove_vocab_dict: glove_present_training_word_vocab_dict[ key] = glove_present_training_word_counter glove_present_word_vector_dict[ glove_present_training_word_counter] = glove_vocab_dict[ key] glove_present_training_word_counter += 1 # to use unknown words elif self.params.use_unknown_word: if self.params.all_lowercase: key = key.lower() glove_present_training_word_vocab_dict[ key] = glove_present_training_word_counter vec_str = '' for i in range(length_word_vector): vec_str += str(round(random.uniform(-0.1, 0.1), 6)) + ' ' glove_present_word_vector_dict[ glove_present_training_word_counter] = vec_str.strip() glove_present_training_word_counter += 1 word_vector_file = open(Directory('TR').word_embedding, 'w') writer = csv.writer(word_vector_file) string = '' for i in range(length_word_vector): string += '0.000001 ' word_vector_file.write(string.rstrip(' ') + '\n') # word_vector_file.write(string.rstrip(' ') + '\n') # zeros vector (id 1) for key, value in glove_present_word_vector_dict.items(): writer.writerow([value]) # op_file = open('abc.txt', 'w') # for each_word in glove_present_training_word_vocab_dict: # op_file.write(each_word + ' ' + str(glove_present_training_word_vocab_dict[each_word]) + '\n') # op_file.close() glove_present_training_word_vocab = open( Directory('TR').glove_present_training_word_vocab, 'wb') pickle.dump(glove_present_training_word_vocab_dict, glove_present_training_word_vocab, protocol=cPickle.HIGHEST_PROTOCOL) print(glove_present_training_word_vocab_dict) print( 'Glove_present_unique_training_tokens, Total unique tokens, Glove token size' ) print(len(glove_present_training_word_vocab_dict), len(word_vocab_dict), len(glove_vocab_dict)) word_vector_file.close() print('\nVocab Size:') # print(len(glove_present_word_vector_dict)+2) print(len(glove_present_word_vector_dict) + 1) glove_present_training_word_vocab.close() # return(len(glove_present_word_vector_dict)+2) return (len(glove_present_word_vector_dict) + 1)
import os import tensorflow as tf from tensorflow.contrib.tensorboard.plugins import projector from global_module.settings_module import Directory import numpy as np LOG_DIR = Directory('TE').log_emb_path metadata = LOG_DIR + '/latent_metadata.csv' latent_rep = LOG_DIR + '/latent_representation.csv' emb = tf.Variable(np.genfromtxt(latent_rep), name='latent_rep') with tf.Session() as sess: saver = tf.train.Saver([emb]) sess.run(emb.initializer) saver.save(sess, os.path.join(LOG_DIR, 'emb.ckpt')) config = projector.ProjectorConfig() # One can add multiple embeddings. embedding = config.embeddings.add() embedding.tensor_name = emb.name # Link this tensor to its metadata file (e.g. labels). embedding.metadata_path = metadata # Saves a config file that TensorBoard will read during startup. projector.visualize_embeddings(tf.summary.FileWriter(LOG_DIR), config)
def main(): model_obj = Model(Params('TR'), Directory('TR'))
def run_train(self): global train_writer, valid_writer mode_train, mode_valid = 'TR', 'VA' # train object params_train = ParamsClass(mode=mode_train) dir_train = Directory(mode_train) train_reader = Reader(params_train) train_instances = train_reader.read_image_data(dir_train.data_filename) # valid object params_valid = ParamsClass(mode=mode_valid) dir_valid = Directory(mode_valid) valid_reader = Reader(params_valid) if dir_valid.data_filename is None: all_instances = train_instances train_instances = all_instances[:int(0.8 * len(all_instances))] valid_instances = all_instances[int(0.8 * len(all_instances)):] else: valid_instances = valid_reader.read_image_data( dir_valid.data_filename) random.seed(4321) if (params_train.enable_shuffle): random.shuffle(train_instances) random.shuffle(valid_instances) global_min_loss = sys.float_info.max print('***** INITIALIZING TF GRAPH *****') with tf.Graph().as_default(), tf.Session() as session: # random_normal_initializer = tf.random_normal_initializer() # random_uniform_initializer = tf.random_uniform_initializer(-params_train.init_scale, params_train.init_scale) xavier_initializer = tf.contrib.layers.xavier_initializer( uniform=True, seed=None, dtype=tf.float32) # with tf.name_scope('train'): with tf.variable_scope("model", reuse=None, initializer=xavier_initializer): train_obj = Autoencoder(params_train, dir_train) # with tf.name_scope('valid'): with tf.variable_scope("model", reuse=True, initializer=xavier_initializer): valid_obj = Autoencoder(params_valid, dir_valid) if not params_train.enable_checkpoint: session.run(tf.global_variables_initializer()) if params_train.enable_checkpoint: ckpt = tf.train.get_checkpoint_state(dir_train.model_path) if ckpt and ckpt.model_checkpoint_path: print("Loading model from: %s" % ckpt.model_checkpoint_path) tf.train.Saver().restore(session, ckpt.model_checkpoint_path) print('**** TF GRAPH INITIALIZED ****') if params_train.log: train_writer = tf.summary.FileWriter( dir_train.log_path + '/train', session.graph) valid_writer = tf.summary.FileWriter(dir_train.log_path + '/valid') # train_writer.add_graph(tf.get_default_graph()) start_time = time.time() for i in range(params_train.max_max_epoch): lr_decay = params_train.lr_decay**max( i - params_train.max_epoch, 0.0) # train_obj.assign_lr(session, params_train.learning_rate * lr_decay) # print(params_train.learning_rate * lr_decay) print('\n++++++++=========+++++++\n') lr = params_train.learning_rate * lr_decay print("Epoch: %d Learning rate: %.5f" % (i + 1, lr)) train_loss, _, = self.run_epoch(session, global_min_loss, train_obj, train_reader, train_instances, train_writer) print("Epoch: %d Train loss: %.4f" % (i + 1, train_loss)) valid_loss, curr_min_loss = self.run_epoch( session, global_min_loss, valid_obj, valid_reader, valid_instances, valid_writer) if (curr_min_loss < global_min_loss): global_min_loss = curr_min_loss print("Epoch: %d Valid loss: %.4f" % (i + 1, valid_loss)) curr_time = time.time() print('1 epoch run takes ' + str(((curr_time - start_time) / (i + 1)) / 60) + ' minutes.') if params_train.log: train_writer.close() valid_writer.close()
def run_test(self, dict_obj): tf.set_random_seed(42) mode_train, mode_valid, mode_test = 'TR', 'VA', 'TE' # global params and dir global_params = GlobalParams() global_dir = Directory('TE') # Get the data files we are working with, Indices also # For now, just using all the indices i.e. the entire file # cnf test object cnf_params_test = ConfidenceNetworkParams(mode=mode_test) cnf_dir_test = ConfidenceNetworkDirectory(mode_test) cnf_params_test.num_instances, cnf_params_test.indices = self.get_length( cnf_dir_test.data_filename) # tar test object tar_params_test = TargetNetworkParams(mode=mode_test) tar_dir_test = TargetNetworkDirectory(mode_test) tar_params_test.num_instances, tar_params_test.indices = self.get_length( tar_dir_test.data_filename) if global_params.enable_shuffle: random.shuffle(cnf_params_test.indices) random.shuffle(tar_params_test.indices) cnf_min_loss = tar_min_loss = sys.float_info.max print('INITIALIZING TF GRAPH \n') timestamp = str(int(time.time())) # cnf_train_out_dir = os.path.abspath(os.path.join(dir_train.log_path, "train", timestamp)) # valid_out_dir = os.path.abspath(os.path.join(dir_train.log_path, "valid", timestamp)) # print("Writing to {}\n".format(cnf_train_out_dir)) cnf_test_out_dir = os.path.abspath( os.path.join(global_dir.cnf_log_path, "test", timestamp)) tar_test_out_dir = os.path.abspath( os.path.join(global_dir.tar_log_path, "test", timestamp)) with tf.Graph().as_default(), tf.Session() as session: # random_normal_initializer = tf.random_normal_initializer() random_uniform_initializer = tf.random_uniform_initializer( -global_params.init_scale, global_params.init_scale) # xavier_initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32) with tf.variable_scope("classifier", reuse=None, initializer=random_uniform_initializer): test_obj = L2LWS(global_params, cnf_params_test, tar_params_test, global_dir, cnf_dir_test, tar_dir_test) cnf_test_writer = tf.summary.FileWriter(cnf_test_out_dir, session.graph) tar_test_writer = tf.summary.FileWriter(tar_test_out_dir, session.graph) if not global_params.enable_checkpoint: session.run(tf.global_variables_initializer()) if global_params.enable_checkpoint: ckpt = tf.train.get_checkpoint_state(global_dir.model_path) if ckpt and ckpt.model_checkpoint_path: print( "Checkpoint while initializing the graph, Loading model from: %s" % ckpt.model_checkpoint_path) tf.train.Saver().restore(session, ckpt.model_checkpoint_path) elif not global_params.use_random_initializer: session.run( tf.assign(test_obj.word_emb_matrix, word_emb_matrix, name="word_embedding_matrix")) print('TF GRAPH INITIALIZED') start_time = time.time() epochs = 1 print('STARTING THE TEST EPOCHS: \n', epochs) for i in range(epochs): print('\n Starting CNF Epoch: \n', i + 1) if i >= 0: ckpt = tf.train.get_checkpoint_state(global_dir.model_path) if ckpt and ckpt.model_checkpoint_path: print( "Checkpoint during the Epoch: Loading model from: %s" % ckpt.model_checkpoint_path) tf.train.Saver().restore(session, ckpt.model_checkpoint_path) # lr_decay = cnf_params_train.lr_decay # train_obj.cnf_network.assign_lr(session, cnf_params_train.learning_rate * lr_decay) # train_obj.tar_network.assign_lr(session, cnf_params_train.learning_rate * lr_decay) # print("Epoch: %d Learning rate: %.5f \n" % (i + 1, session.run(train_obj.cnf_network.lr))) test_loss, _ = self.run_cnf_net_epoch(cnf_test_writer, session, cnf_min_loss, test_obj, dict_obj, i) print("CNF NETWORK: Epoch: %d Test loss: %.3f" % (i + 1, test_loss)) curr_time = time.time() print('1 CNF epoch run takes ' + str(((curr_time - start_time) / (i + 1)) / 60) + ' minutes.') print('\n Finished CNF epoch: \n', i + 1) ''' TARGET NETWORK ''' print('\nStarting TGT epoch: \n', i + 1) if i > 0: if ckpt and ckpt.model_checkpoint_path: print("Loading model from: %s" % ckpt.model_checkpoint_path) tf.train.Saver().restore(session, ckpt.model_checkpoint_path) # lr_decay = tar_params_train.lr_decay ** max(i - global_params.max_epoch, 0.0) # train_obj.cnf_network.assign_lr(session, cnf_params_train.learning_rate * lr_decay) # train_obj.tar_network.assign_lr(session, cnf_params_train.learning_rate * lr_decay) # print("Epoch: %d Learning rate: %.5f" % (i + 1, session.run(train_obj.tar_network.lr))) test_loss, _ = self.run_tar_net_epoch(tar_test_writer, session, tar_min_loss, test_obj, dict_obj, i) print("TGT NETWORK: Epoch: %d Test loss: %.3f\n" % (i + 1, test_loss)) curr_time = time.time() print('1 TGT epoch run takes ' + str(((curr_time - start_time) / (i + 1)) / 60) + ' minutes.') print('\nFinishing TGT epoch: \n', i + 1) cnf_test_writer.close() tar_test_writer.close()