def __init__(self, mode):
Directory.__init__(self, mode)
self.data_path += '/cnf'
''' DATASET '''
self.raw_data_filename = self.data_path + self.raw_data_filename
self.data_filename = self.data_path + self.data_filename
self.gold_label_filename = self.data_path + self.gold_label_filename
self.weak_label_filename = self.data_path + self.weak_label_filename
def __init__(self, mode):
Directory.__init__(self, mode)
self.data_path += '/tar'
''' DATASET '''
self.raw_data_filename = self.data_path + self.raw_data_filename
self.data_filename = self.data_path + self.data_filename
self.gold_label_filename = self.data_path + self.gold_label_filename # For valid gold label is used as weak label
self.weak_label_filename = self.data_path + self.weak_label_filename # For train, only weak label
def main(domain):
data_dir = Directory('TR').data_path
source = data_dir + '/%s/pre_processed_restaurant/train.txt' % (domain)
model_file = data_dir + '/%s/pre_processed_restaurant/w2v_embedding_new' % (
domain)
sentences = MySentences(source)
model = gensim.models.Word2Vec(sentences,
size=200,
window=5,
min_count=10,
workers=4)
model.save(model_file)
def init_test(self, dict_obj):
mode_train, mode_test = 'TR', 'TE'
params_train = ParamsClass(mode=mode_train)
dir_train = Directory(mode_train)
# test object
params_test = ParamsClass(mode=mode_test)
dir_test = Directory(mode_test)
params_test.num_instances, params_test.indices = self.get_length(dir_test.data_filename)
# params_test.batch_size = 1
params_train.num_classes = params_test.num_classes = len(dict_obj.label_dict)
min_loss = sys.float_info.max
word_emb_path = dir_train.word_embedding
word_emb_matrix = np.float32(np.genfromtxt(word_emb_path, delimiter=' '))
params_train.vocab_size = params_test.vocab_size = len(word_emb_matrix)
print('***** INITIALIZING TF GRAPH *****')
session = tf.Session()
# train_writer = tf.summary.FileWriter(dir_train.log_path + '/train', session.graph)
# test_writer = tf.summary.FileWriter(dir_train.log_path + '/test')
# random_uniform_initializer = tf.random_uniform_initializer(-params_train.init_scale, params_train.init_scale)
# xavier_initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32)
with tf.variable_scope("classifier", reuse=None):
test_obj = SMN(params_test, dir_test)
model_saver = tf.train.Saver()
print('Loading model ...')
model_saver.restore(session, Directory('TE').test_model)
print('**** MODEL LOADED ****\n')
return session, test_obj
def run_test(self):
global test_writer
mode_test = 'TE'
# test object
params_test = ParamsClass(mode=mode_test)
dir_test = Directory(mode_test)
test_reader = Reader(params_test)
test_instances = test_reader.read_image_data(dir_test.data_filename)
random.seed(4321)
global_min_loss = sys.float_info.max
print('***** INITIALIZING TF GRAPH *****')
with tf.Graph().as_default(), tf.Session() as session:
with tf.variable_scope("model"):
test_obj = Autoencoder(params_test, dir_test)
model_saver = tf.train.Saver()
model_saver.restore(session, test_obj.dir_obj.test_model)
if params_test.log:
test_writer = tf.summary.FileWriter(dir_test.log_path +
'/test')
print('**** TF GRAPH INITIALIZED ****')
start_time = time.time()
test_loss, _, = self.run_epoch(session, global_min_loss, test_obj,
test_reader, test_instances,
test_writer)
print("Epoch: %d Test loss: %.4f" % (1, test_loss))
curr_time = time.time()
print('1 epoch run takes ' + str((curr_time - start_time) / 60) +
' minutes.')
if params_test.log:
test_writer.close()
def train_util():
"""
Utility function to execute the training pipeline
:return: None
"""
# To use training set only
# build_sampled_training_file.util()
# build_word_vocab.util()
# To use complete glove file + training set
dir_obj = Directory('TR')
vocab_builder = VocabBuilder()
vocab_builder.create_vocab_dict(dir_obj.data_filename)
vocab_builder.extract_glove_vectors(dir_obj.word_vocab_dict, dir_obj.glove_path)
dict_obj = load_dictionary()
call_train(dict_obj)
return None
def run_epoch(self, session, eval_op, model_obj, dict_obj, verbose=False):
global summary, iter_train, iter_valid
epoch_combined_loss = 0.0
total_correct = 0.0
total_instances = 0.0
print('\nrun epoch')
output_file = open(Directory('TE').test_cost_path, 'w')
params = model_obj.params
dir_obj = model_obj.dir_obj
data_filename = dir_obj.data_filename
label_filename = dir_obj.label_filename
for step, (ctx_arr, ctx_len_arr, num_ctx_arr, resp_arr, resp_len_arr, label_arr) \
in enumerate(DataReader(params).data_iterator(data_filename, label_filename, model_obj.params.indices, dict_obj)):
feed_dict = {model_obj.ctx: ctx_arr,
model_obj.ctx_len_placeholders: ctx_len_arr,
model_obj.num_ctx_placeholders: num_ctx_arr,
model_obj.resp: resp_arr,
model_obj.resp_len_placeholders: resp_len_arr,
model_obj.label: label_arr
}
loss, prediction, probabilities, accuracy, _ = session.run([model_obj.loss,
model_obj.prediction,
model_obj.probabilities,
model_obj.accuracy,
eval_op],
feed_dict=feed_dict)
total_correct += np.sum(prediction == label_arr)
total_instances += params.batch_size
epoch_combined_loss += loss
for idx, each_pred in enumerate(probabilities):
# output_file.write(str(each_pred) + '\t' + str(probabilities[idx]) + '\n')
output_file.write(str(each_pred[1]) + '\n')
print 'CE loss: %.4f, Accuracy: %.4f' % (epoch_combined_loss, (total_correct / total_instances) * 100)
return epoch_combined_loss
def init_test(self):
mode_train, mode_valid, mode_test = 'TR', 'VA', 'TE'
# test object
params_test = Params(mode=mode_test)
dir_test = Directory(mode_test)
params_test.num_instances, params_test.indices = self.getLength(
dir_test.data_filename)
min_loss = sys.float_info.max
word_emb_path = dir_test.word_embedding
word_emb_matrix = np.float32(
np.genfromtxt(word_emb_path, delimiter=' '))
params_test.VOCAB_SIZE = len(word_emb_matrix)
params_test.batch_size = 10
print('***** INITIALIZING TF GRAPH *****')
session = tf.Session()
# with tf.name_scope('train'):
with tf.variable_scope("model", reuse=None):
test_obj = Model(params_test, dir_test)
model_saver = tf.train.Saver()
print('Loading model ...')
model_saver.restore(session, dir_test.test_model)
# builder = saved_model_builder.SavedModelBuilder(set_dir.Directory('TE').test_model)
# builder.add_meta_graph_and_variables(session, ['serve'], signature_def_map=None)
# builder.save()
# return
print('**** MODEL LOADED ****\n')
return session, test_obj
def create_vocab_dict(self, input_filename):
word_dict = {}
word_count = {}
word_counter = 2
max_sequence_length = 0
input_file = open(input_filename, 'r').readlines()
for each_line in input_file:
if self.params.all_lowercase:
each_line = each_line.lower()
tokenized_string = each_line.strip().split()
for each_token in tokenized_string:
if each_token != 'unk':
if each_token not in word_count:
word_count[each_token] = 1
else:
word_count[each_token] = 1 + word_count[each_token]
line_len = len(tokenized_string)
if (line_len > max_sequence_length):
max_sequence_length = line_len
for each_token in word_count:
if word_count[each_token] >= self.params.sampling_threshold:
word_dict[each_token] = word_counter
word_counter += 1
word_vocab = open(Directory('TR').word_vocab_dict, 'wb')
pickle.dump(word_dict, word_vocab, protocol=cPickle.HIGHEST_PROTOCOL)
word_vocab.close()
print('Reading Completed \n ========================== '
'\n Unique tokens: excluding padding and unkown words %d '
'\n Max. sequence length: %d'
'\n ==========================\n' %
(word_counter - 2, max_sequence_length))
def run_train(self, dict_obj):
mode_train, mode_valid, mode_test = 'TR', 'VA', 'TE'
# train object
params_train = Params(mode=mode_train)
dir_train = Directory(mode_train)
params_train.num_instances, params_train.indices = self.getLength(
dir_train.data_filename)
# valid object
params_valid = Params(mode=mode_valid)
dir_valid = Directory(mode_valid)
params_valid.num_instances, params_valid.indices = self.getLength(
dir_valid.data_filename)
# test object
#
# params_test = Params(mode=mode_test)
# dir_test = Directory(mode_test)
# params_test.num_instances, params_test.indices = self.getLength(dir_test.data_filename)
random.seed(1234)
if (params_train.enable_shuffle):
random.shuffle(params_train.indices)
random.shuffle(params_valid.indices)
min_loss = sys.float_info.max
word_emb_path = dir_train.word_embedding
word_emb_matrix = np.float32(
np.genfromtxt(word_emb_path, delimiter=' '))
params_train.VOCAB_SIZE = params_valid.VOCAB_SIZE = len(
word_emb_matrix)
logger.info('***** INITIALIZING TF GRAPH *****')
with tf.Graph().as_default(), tf.Session() as session:
# train_writer = tf.summary.FileWriter(dir_train.log_path + '/train', session.graph)
# test_writer = tf.summary.FileWriter(dir_train.log_path + '/test')
# random_normal_initializer = tf.random_normal_initializer()
# random_uniform_initializer = tf.random_uniform_initializer(-params_train.init_scale, params_train.init_scale)
xavier_initializer = tf.contrib.layers.xavier_initializer(
uniform=True, seed=None, dtype=tf.float32)
# with tf.name_scope('train'):
with tf.variable_scope("model",
reuse=None,
initializer=xavier_initializer):
train_obj = Model(params_train, dir_train)
# with tf.name_scope('valid'):
with tf.variable_scope("model",
reuse=True,
initializer=xavier_initializer):
valid_obj = Model(params_valid, dir_valid)
# with tf.variable_scope("model", reuse=True, initializer=xavier_initializer):
# test_obj = Model(params_test, dir_test)
if not params_train.enable_checkpoint:
session.run(tf.global_variables_initializer())
if params_train.enable_checkpoint:
ckpt = tf.train.get_checkpoint_state(dir_train.model_path)
if ckpt and ckpt.model_checkpoint_path:
print("Loading model from: %s" %
ckpt.model_checkpoint_path)
tf.train.Saver().restore(session,
ckpt.model_checkpoint_path)
elif (not params_train.use_random_initializer):
self.assign_word_emb_matrix(session, train_obj,
word_emb_matrix)
self.assign_aspect_emb_matrix(session, params_train,
word_emb_matrix, train_obj)
model_saver = tf.train.Saver()
model_saver.save(session,
save_path=dir_train.model_path +
dir_train.model_name,
latest_filename=dir_train.latest_checkpoint)
print('==== Model saved! ====')
logger.info('**** TF GRAPH INITIALIZED ****')
# train_writer.add_graph(tf.get_default_graph())
start_time = time.time()
reader = DataReader()
for i in range(params_train.max_max_epoch):
lr_decay = params_train.lr_decay**max(
i - params_train.max_epoch, 0.0)
# train_obj.assign_lr(session, params_train.learning_rate * lr_decay)
# print(params_train.learning_rate * lr_decay)
self.get_aspect_words(train_obj, dict_obj.word_dict)
print('\n++++++++=========+++++++\n')
lr = params_train.learning_rate * lr_decay
logger.info("Epoch: %d Learning rate: %.5f" % (i + 1, lr))
train_loss, _, summary = self.run_epoch(session,
min_loss,
train_obj,
dict_obj,
i,
reader,
lr,
verbose=True)
print("Epoch: %d Train loss: %.4f" % (i + 1, train_loss))
valid_loss, curr_loss, summary = self.run_epoch(
session, min_loss, valid_obj, dict_obj, i, reader)
if (curr_loss < min_loss):
min_loss = curr_loss
print("Epoch: %d Valid loss: %.4f" % (i + 1, valid_loss))
# test_loss, _, _ = self.run_epoch(session, min_loss, test_obj, dict_obj, i, reader)
# print("Epoch: %d Test loss: %.4f" % (i + 1, test_loss))
curr_time = time.time()
print('1 epoch run takes ' + str(
((curr_time - start_time) / (i + 1)) / 60) + ' minutes.')
# train_writer.close()
# test_writer.close()
self.get_aspect_words(train_obj, dict_obj.word_dict)
def run_train(self, dict_obj):
mode_train, mode_valid, mode_all = 'TR', 'VA', 'ALL'
# train object
params_train = ParamsClass(mode=mode_train)
dir_train = Directory(mode_train)
params_train.num_instances, params_train.indices = self.get_length(
dir_train.data_filename)
# valid object
params_valid = ParamsClass(mode=mode_valid)
dir_valid = Directory(mode_valid)
params_valid.num_instances, params_valid.indices = self.get_length(
dir_valid.data_filename)
params_train.num_classes = params_valid.num_classes = len(
dict_obj.label_dict)
if params_train.enable_shuffle:
random.shuffle(params_train.indices)
random.shuffle(params_valid.indices)
min_loss = sys.float_info.max
word_emb_path = dir_train.word_embedding
word_emb_matrix = np.float32(
np.genfromtxt(word_emb_path, delimiter=' '))
params_train.vocab_size = params_valid.vocab_size = len(
word_emb_matrix)
print('***** INITIALIZING TF GRAPH *****')
timestamp = str(int(time.time()))
train_out_dir = os.path.abspath(
os.path.join(dir_train.log_path, "train", timestamp))
valid_out_dir = os.path.abspath(
os.path.join(dir_train.log_path, "valid", timestamp))
print("Writing to {}\n".format(train_out_dir))
with tf.Graph().as_default(), tf.Session() as session:
# random_normal_initializer = tf.random_normal_initializer()
# random_uniform_initializer = tf.random_uniform_initializer(-params_train.init_scale, params_train.init_scale)
xavier_initializer = tf.contrib.layers.xavier_initializer(
uniform=True, seed=None, dtype=tf.float32)
with tf.variable_scope("classifier",
reuse=None,
initializer=xavier_initializer):
train_obj = SMN(params_train, dir_train)
train_writer = tf.summary.FileWriter(train_out_dir, session.graph)
valid_writer = tf.summary.FileWriter(valid_out_dir)
if not params_train.enable_checkpoint:
session.run(tf.global_variables_initializer())
if params_train.enable_checkpoint:
ckpt = tf.train.get_checkpoint_state(dir_train.model_path)
if ckpt and ckpt.model_checkpoint_path:
print("Loading model from: %s" %
ckpt.model_checkpoint_path)
tf.train.Saver().restore(session,
ckpt.model_checkpoint_path)
elif not params_train.use_random_initializer:
session.run(
tf.assign(train_obj.word_emb_matrix,
word_emb_matrix,
name="word_embedding_matrix"))
with tf.variable_scope("classifier",
reuse=True,
initializer=xavier_initializer):
valid_obj = SMN(params_valid, dir_valid)
print('**** TF GRAPH INITIALIZED ****')
start_time = time.time()
for i in range(params_train.max_max_epoch):
lr_decay = params_train.lr_decay**max(
i - params_train.max_epoch, 0.0)
train_obj.assign_lr(session,
params_train.learning_rate * lr_decay)
# print(params_train.learning_rate * lr_decay)
print('\n++++++++=========+++++++\n')
print("Epoch: %d Learning rate: %.5f" %
(i + 1, session.run(train_obj.lr)))
train_loss, _ = self.run_epoch(session,
train_writer,
train_obj.train_op,
min_loss,
train_obj,
dict_obj,
i,
verbose=True)
print("Epoch: %d Train loss: %.3f" % (i + 1, train_loss))
valid_loss, curr_loss = self.run_epoch(session, valid_writer,
tf.no_op(), min_loss,
valid_obj, dict_obj, i)
if curr_loss < min_loss:
min_loss = curr_loss
print("Epoch: %d Valid loss: %.3f" % (i + 1, valid_loss))
curr_time = time.time()
print('1 epoch run takes ' + str(
((curr_time - start_time) / (i + 1)) / 60) + ' minutes.')
train_writer.close()
valid_writer.close()
def run_train(self, dict_obj):
tf.set_random_seed(1234)
mode_train, mode_valid, mode_test = 'TR', 'VA', 'TE'
# global params and dir
global_params = GlobalParams()
global_dir = Directory('TR')
# cnf train object
cnf_params_train = ConfidenceNetworkParams(mode=mode_train)
cnf_dir_train = ConfidenceNetworkDirectory(mode_train)
cnf_params_train.num_instances, cnf_params_train.indices = self.get_length(
cnf_dir_train.data_filename)
# tar train object
tar_params_train = TargetNetworkParams(mode=mode_train)
tar_dir_train = TargetNetworkDirectory(mode_train)
tar_params_train.num_instances, tar_params_train.indices = self.get_length(
tar_dir_train.data_filename)
# cnf valid object
cnf_params_valid = ConfidenceNetworkParams(mode=mode_valid)
cnf_dir_valid = ConfidenceNetworkDirectory(mode_valid)
cnf_params_valid.num_instances, cnf_params_valid.indices = self.get_length(
cnf_dir_valid.data_filename)
cnf_params_valid.batch_size = 64
# tar valid object
tar_params_valid = TargetNetworkParams(mode=mode_valid)
tar_dir_valid = TargetNetworkDirectory(mode_valid)
tar_params_valid.num_instances, tar_params_valid.indices = self.get_length(
tar_dir_valid.data_filename)
tar_params_valid.batch_size = 64
# params_train.num_classes = params_valid.num_classes = len(dict_obj.label_dict)
if global_params.enable_shuffle:
random.shuffle(cnf_params_train.indices)
random.shuffle(cnf_params_valid.indices)
random.shuffle(tar_params_train.indices)
random.shuffle(tar_params_valid.indices)
cnf_min_loss = tar_min_loss = sys.float_info.max
word_emb_path = global_dir.word_embedding
word_emb_matrix = np.float32(
np.genfromtxt(word_emb_path, delimiter=' '))
global_params.vocab_size = len(word_emb_matrix)
print('***** INITIALIZING TF GRAPH *****')
timestamp = str(int(time.time()))
# cnf_train_out_dir = os.path.abspath(os.path.join(dir_train.log_path, "train", timestamp))
# valid_out_dir = os.path.abspath(os.path.join(dir_train.log_path, "valid", timestamp))
# print("Writing to {}\n".format(cnf_train_out_dir))
cnf_train_out_dir = os.path.abspath(
os.path.join(global_dir.cnf_log_path, "train", timestamp))
cnf_valid_out_dir = os.path.abspath(
os.path.join(global_dir.cnf_log_path, "valid", timestamp))
tar_train_out_dir = os.path.abspath(
os.path.join(global_dir.tar_log_path, "train", timestamp))
tar_valid_out_dir = os.path.abspath(
os.path.join(global_dir.tar_log_path, "valid", timestamp))
with tf.Graph().as_default(), tf.Session() as session:
# random_normal_initializer = tf.random_normal_initializer()
random_uniform_initializer = tf.random_uniform_initializer(
-global_params.init_scale, global_params.init_scale)
# xavier_initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32)
with tf.variable_scope("classifier",
reuse=None,
initializer=random_uniform_initializer):
train_obj = L2LWS(global_params, cnf_params_train,
tar_params_train, global_dir, cnf_dir_train,
tar_dir_train)
cnf_train_writer = tf.summary.FileWriter(cnf_train_out_dir,
session.graph)
cnf_valid_writer = tf.summary.FileWriter(cnf_valid_out_dir)
tar_train_writer = tf.summary.FileWriter(tar_train_out_dir,
session.graph)
tar_valid_writer = tf.summary.FileWriter(tar_valid_out_dir)
if not global_params.enable_checkpoint:
session.run(tf.global_variables_initializer())
if global_params.enable_checkpoint:
ckpt = tf.train.get_checkpoint_state(global_dir.model_path)
if ckpt and ckpt.model_checkpoint_path:
print("Loading model from: %s" %
ckpt.model_checkpoint_path)
tf.train.Saver().restore(session,
ckpt.model_checkpoint_path)
elif not global_params.use_random_initializer:
session.run(
tf.assign(train_obj.word_emb_matrix,
word_emb_matrix,
name="word_embedding_matrix"))
with tf.variable_scope("classifier",
reuse=True,
initializer=random_uniform_initializer):
valid_obj = L2LWS(global_params, cnf_params_valid,
tar_params_valid, global_dir, cnf_dir_valid,
tar_dir_valid)
print('**** TF GRAPH INITIALIZED ****')
start_time = time.time()
for i in range(global_params.max_max_epoch):
print('\n++++++++=========+++++++\n')
if i >= global_params.pre_train_epoch:
ckpt = tf.train.get_checkpoint_state(global_dir.model_path)
if ckpt and ckpt.model_checkpoint_path:
print("Loading model from: %s" %
ckpt.model_checkpoint_path)
tf.train.Saver().restore(session,
ckpt.model_checkpoint_path)
lr_decay = cnf_params_train.lr_decay**max(
i - global_params.max_epoch, 0.0)
train_obj.cnf_network.assign_lr(
session, cnf_params_train.learning_rate * lr_decay)
train_obj.tar_network.assign_lr(
session, cnf_params_train.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.5f" %
(i + 1, session.run(train_obj.cnf_network.lr)))
train_loss, _ = self.run_cnf_net_epoch(cnf_train_writer,
session, cnf_min_loss,
train_obj, dict_obj, i)
print("CONFIDENCE NETWORK: Epoch: %d Train loss: %.3f\n" %
(i + 1, train_loss))
valid_obj.cnf_network.assign_lr(
session, cnf_params_train.learning_rate * lr_decay)
valid_obj.tar_network.assign_lr(
session, cnf_params_train.learning_rate * lr_decay)
valid_loss, curr_loss = self.run_cnf_net_epoch(
cnf_valid_writer, session, cnf_min_loss, valid_obj,
dict_obj, i)
if curr_loss < cnf_min_loss:
cnf_min_loss = curr_loss
print("CONFIDENCE NETWORK: Epoch: %d Valid loss: %.3f" %
(i + 1, valid_loss))
curr_time = time.time()
print('1 epoch run takes ' + str(
((curr_time - start_time) / (i + 1)) / 60) + ' minutes.')
print('\n++++++++=========+++++++\n')
''' TARGET NETWORK '''
if i >= global_params.pre_train_epoch:
if ckpt and ckpt.model_checkpoint_path:
print("Loading model from: %s" %
ckpt.model_checkpoint_path)
tf.train.Saver().restore(session,
ckpt.model_checkpoint_path)
lr_decay = tar_params_train.lr_decay**max(
i - global_params.max_epoch, 0.0)
train_obj.cnf_network.assign_lr(
session, cnf_params_train.learning_rate * lr_decay)
train_obj.tar_network.assign_lr(
session, cnf_params_train.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.5f" %
(i + 1, session.run(train_obj.tar_network.lr)))
train_loss, _ = self.run_tar_net_epoch(
tar_train_writer, session, tar_min_loss, train_obj,
dict_obj, i)
print("TARGET NETWORK: Epoch: %d Train loss: %.3f\n" %
(i + 1, train_loss))
valid_obj.cnf_network.assign_lr(
session, cnf_params_train.learning_rate * lr_decay)
valid_obj.tar_network.assign_lr(
session, cnf_params_train.learning_rate * lr_decay)
valid_loss, curr_loss = self.run_tar_net_epoch(
tar_valid_writer, session, tar_min_loss, valid_obj,
dict_obj, i)
if curr_loss < tar_min_loss:
tar_min_loss = curr_loss
print("TARGET NETWORK: Epoch: %d Valid loss: %.3f" %
(i + 1, valid_loss))
curr_time = time.time()
print('1 epoch run takes ' + str(
((curr_time - start_time) / (i + 1)) / 60) +
' minutes.')
print('\n++++++++=========+++++++\n')
cnf_train_writer.close()
cnf_valid_writer.close()
tar_train_writer.close()
tar_valid_writer.close()
import gensim
import pickle, cPickle
import collections
import numpy as np
from global_module.settings_module import Directory
def convert(emb_file):
# model = gensim.models.KeyedVectors.load_word2vec_format(emb_file, binary=True, unicode_errors='ignore')
model = gensim.models.Word2Vec.load(emb_file)
output_emb_file = open(emb_file + '_dict.pkl', 'wb')
vocab_dict = collections.OrderedDict()
for word in model.wv.index2word:
word_vec = model[word]
# print word, model.wv.most_similar(positive=word)
word_vec = np.array2string(word_vec)
word_vec = word_vec.replace('[', '')
word_vec = word_vec.replace(']', '')
vocab_dict[word] = (' '.join(word_vec.split())).strip()
pickle.dump(vocab_dict, output_emb_file, protocol=cPickle.HIGHEST_PROTOCOL)
base_dir = Directory('TR').data_path
convert(base_dir + '/restaurant/pre_processed_restaurant/w2v_embedding')
def main():
SMN(ParamsClass(), Directory())
def extract_glove_vectors(self, word_vocab_file, glove_file):
glove_vocab_dict = cPickle.load(open(glove_file, 'rb'))
word_vocab_dict = cPickle.load(open(word_vocab_file, 'rb'))
length_word_vector = self.params.EMB_DIM
glove_present_training_word_vocab_dict = {}
glove_present_training_word_counter = 1
glove_present_word_vector_dict = {}
freq_thres = len(glove_vocab_dict)
if 'unk' not in glove_vocab_dict and 'UNK' not in glove_vocab_dict:
glove_present_training_word_vocab_dict['unk'] = 1
vec_str = ''
for i in range(length_word_vector):
vec_str += str(round(random.uniform(-0.1, 0.1), 6)) + ' '
glove_present_word_vector_dict[1] = vec_str.strip()
glove_present_training_word_counter = 2
for key, value in glove_vocab_dict.items()[:freq_thres]:
# if key == 'UNK':
# key = key.lower()
vec = glove_vocab_dict.get(key)
if self.params.all_lowercase:
key = key.lower()
if key in glove_vocab_dict:
vec = glove_vocab_dict.get(key)
if key not in glove_present_training_word_vocab_dict:
glove_present_training_word_vocab_dict[
key] = glove_present_training_word_counter
glove_present_word_vector_dict[
glove_present_training_word_counter] = vec
glove_present_training_word_counter += 1
#to use training words not present in top frequent words of glove
for key, value in word_vocab_dict.items():
if key not in glove_present_training_word_vocab_dict and key in glove_vocab_dict:
glove_present_training_word_vocab_dict[
key] = glove_present_training_word_counter
glove_present_word_vector_dict[
glove_present_training_word_counter] = glove_vocab_dict[
key]
glove_present_training_word_counter += 1
# to use unknown words
elif self.params.use_unknown_word:
if self.params.all_lowercase:
key = key.lower()
glove_present_training_word_vocab_dict[
key] = glove_present_training_word_counter
vec_str = ''
for i in range(length_word_vector):
vec_str += str(round(random.uniform(-0.1, 0.1), 6)) + ' '
glove_present_word_vector_dict[
glove_present_training_word_counter] = vec_str.strip()
glove_present_training_word_counter += 1
word_vector_file = open(Directory('TR').word_embedding, 'w')
writer = csv.writer(word_vector_file)
string = ''
for i in range(length_word_vector):
string += '0.000001 '
word_vector_file.write(string.rstrip(' ') + '\n')
# word_vector_file.write(string.rstrip(' ') + '\n') # zeros vector (id 1)
for key, value in glove_present_word_vector_dict.items():
writer.writerow([value])
# op_file = open('abc.txt', 'w')
# for each_word in glove_present_training_word_vocab_dict:
# op_file.write(each_word + ' ' + str(glove_present_training_word_vocab_dict[each_word]) + '\n')
# op_file.close()
glove_present_training_word_vocab = open(
Directory('TR').glove_present_training_word_vocab, 'wb')
pickle.dump(glove_present_training_word_vocab_dict,
glove_present_training_word_vocab,
protocol=cPickle.HIGHEST_PROTOCOL)
print(glove_present_training_word_vocab_dict)
print(
'Glove_present_unique_training_tokens, Total unique tokens, Glove token size'
)
print(len(glove_present_training_word_vocab_dict),
len(word_vocab_dict), len(glove_vocab_dict))
word_vector_file.close()
print('\nVocab Size:')
# print(len(glove_present_word_vector_dict)+2)
print(len(glove_present_word_vector_dict) + 1)
glove_present_training_word_vocab.close()
# return(len(glove_present_word_vector_dict)+2)
return (len(glove_present_word_vector_dict) + 1)
import os
import tensorflow as tf
from tensorflow.contrib.tensorboard.plugins import projector
from global_module.settings_module import Directory
import numpy as np
LOG_DIR = Directory('TE').log_emb_path
metadata = LOG_DIR + '/latent_metadata.csv'
latent_rep = LOG_DIR + '/latent_representation.csv'
emb = tf.Variable(np.genfromtxt(latent_rep), name='latent_rep')
with tf.Session() as sess:
saver = tf.train.Saver([emb])
sess.run(emb.initializer)
saver.save(sess, os.path.join(LOG_DIR, 'emb.ckpt'))
config = projector.ProjectorConfig()
# One can add multiple embeddings.
embedding = config.embeddings.add()
embedding.tensor_name = emb.name
# Link this tensor to its metadata file (e.g. labels).
embedding.metadata_path = metadata
# Saves a config file that TensorBoard will read during startup.
projector.visualize_embeddings(tf.summary.FileWriter(LOG_DIR), config)
def main():
model_obj = Model(Params('TR'), Directory('TR'))
def run_train(self):
global train_writer, valid_writer
mode_train, mode_valid = 'TR', 'VA'
# train object
params_train = ParamsClass(mode=mode_train)
dir_train = Directory(mode_train)
train_reader = Reader(params_train)
train_instances = train_reader.read_image_data(dir_train.data_filename)
# valid object
params_valid = ParamsClass(mode=mode_valid)
dir_valid = Directory(mode_valid)
valid_reader = Reader(params_valid)
if dir_valid.data_filename is None:
all_instances = train_instances
train_instances = all_instances[:int(0.8 * len(all_instances))]
valid_instances = all_instances[int(0.8 * len(all_instances)):]
else:
valid_instances = valid_reader.read_image_data(
dir_valid.data_filename)
random.seed(4321)
if (params_train.enable_shuffle):
random.shuffle(train_instances)
random.shuffle(valid_instances)
global_min_loss = sys.float_info.max
print('***** INITIALIZING TF GRAPH *****')
with tf.Graph().as_default(), tf.Session() as session:
# random_normal_initializer = tf.random_normal_initializer()
# random_uniform_initializer = tf.random_uniform_initializer(-params_train.init_scale, params_train.init_scale)
xavier_initializer = tf.contrib.layers.xavier_initializer(
uniform=True, seed=None, dtype=tf.float32)
# with tf.name_scope('train'):
with tf.variable_scope("model",
reuse=None,
initializer=xavier_initializer):
train_obj = Autoencoder(params_train, dir_train)
# with tf.name_scope('valid'):
with tf.variable_scope("model",
reuse=True,
initializer=xavier_initializer):
valid_obj = Autoencoder(params_valid, dir_valid)
if not params_train.enable_checkpoint:
session.run(tf.global_variables_initializer())
if params_train.enable_checkpoint:
ckpt = tf.train.get_checkpoint_state(dir_train.model_path)
if ckpt and ckpt.model_checkpoint_path:
print("Loading model from: %s" %
ckpt.model_checkpoint_path)
tf.train.Saver().restore(session,
ckpt.model_checkpoint_path)
print('**** TF GRAPH INITIALIZED ****')
if params_train.log:
train_writer = tf.summary.FileWriter(
dir_train.log_path + '/train', session.graph)
valid_writer = tf.summary.FileWriter(dir_train.log_path +
'/valid')
# train_writer.add_graph(tf.get_default_graph())
start_time = time.time()
for i in range(params_train.max_max_epoch):
lr_decay = params_train.lr_decay**max(
i - params_train.max_epoch, 0.0)
# train_obj.assign_lr(session, params_train.learning_rate * lr_decay)
# print(params_train.learning_rate * lr_decay)
print('\n++++++++=========+++++++\n')
lr = params_train.learning_rate * lr_decay
print("Epoch: %d Learning rate: %.5f" % (i + 1, lr))
train_loss, _, = self.run_epoch(session, global_min_loss,
train_obj, train_reader,
train_instances, train_writer)
print("Epoch: %d Train loss: %.4f" % (i + 1, train_loss))
valid_loss, curr_min_loss = self.run_epoch(
session, global_min_loss, valid_obj, valid_reader,
valid_instances, valid_writer)
if (curr_min_loss < global_min_loss):
global_min_loss = curr_min_loss
print("Epoch: %d Valid loss: %.4f" % (i + 1, valid_loss))
curr_time = time.time()
print('1 epoch run takes ' + str(((curr_time - start_time) /
(i + 1)) / 60) + ' minutes.')
if params_train.log:
train_writer.close()
valid_writer.close()
def run_test(self, dict_obj):
tf.set_random_seed(42)
mode_train, mode_valid, mode_test = 'TR', 'VA', 'TE'
# global params and dir
global_params = GlobalParams()
global_dir = Directory('TE')
# Get the data files we are working with, Indices also
# For now, just using all the indices i.e. the entire file
# cnf test object
cnf_params_test = ConfidenceNetworkParams(mode=mode_test)
cnf_dir_test = ConfidenceNetworkDirectory(mode_test)
cnf_params_test.num_instances, cnf_params_test.indices = self.get_length(
cnf_dir_test.data_filename)
# tar test object
tar_params_test = TargetNetworkParams(mode=mode_test)
tar_dir_test = TargetNetworkDirectory(mode_test)
tar_params_test.num_instances, tar_params_test.indices = self.get_length(
tar_dir_test.data_filename)
if global_params.enable_shuffle:
random.shuffle(cnf_params_test.indices)
random.shuffle(tar_params_test.indices)
cnf_min_loss = tar_min_loss = sys.float_info.max
print('INITIALIZING TF GRAPH \n')
timestamp = str(int(time.time()))
# cnf_train_out_dir = os.path.abspath(os.path.join(dir_train.log_path, "train", timestamp))
# valid_out_dir = os.path.abspath(os.path.join(dir_train.log_path, "valid", timestamp))
# print("Writing to {}\n".format(cnf_train_out_dir))
cnf_test_out_dir = os.path.abspath(
os.path.join(global_dir.cnf_log_path, "test", timestamp))
tar_test_out_dir = os.path.abspath(
os.path.join(global_dir.tar_log_path, "test", timestamp))
with tf.Graph().as_default(), tf.Session() as session:
# random_normal_initializer = tf.random_normal_initializer()
random_uniform_initializer = tf.random_uniform_initializer(
-global_params.init_scale, global_params.init_scale)
# xavier_initializer = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32)
with tf.variable_scope("classifier",
reuse=None,
initializer=random_uniform_initializer):
test_obj = L2LWS(global_params, cnf_params_test,
tar_params_test, global_dir, cnf_dir_test,
tar_dir_test)
cnf_test_writer = tf.summary.FileWriter(cnf_test_out_dir,
session.graph)
tar_test_writer = tf.summary.FileWriter(tar_test_out_dir,
session.graph)
if not global_params.enable_checkpoint:
session.run(tf.global_variables_initializer())
if global_params.enable_checkpoint:
ckpt = tf.train.get_checkpoint_state(global_dir.model_path)
if ckpt and ckpt.model_checkpoint_path:
print(
"Checkpoint while initializing the graph, Loading model from: %s"
% ckpt.model_checkpoint_path)
tf.train.Saver().restore(session,
ckpt.model_checkpoint_path)
elif not global_params.use_random_initializer:
session.run(
tf.assign(test_obj.word_emb_matrix,
word_emb_matrix,
name="word_embedding_matrix"))
print('TF GRAPH INITIALIZED')
start_time = time.time()
epochs = 1
print('STARTING THE TEST EPOCHS: \n', epochs)
for i in range(epochs):
print('\n Starting CNF Epoch: \n', i + 1)
if i >= 0:
ckpt = tf.train.get_checkpoint_state(global_dir.model_path)
if ckpt and ckpt.model_checkpoint_path:
print(
"Checkpoint during the Epoch: Loading model from: %s"
% ckpt.model_checkpoint_path)
tf.train.Saver().restore(session,
ckpt.model_checkpoint_path)
# lr_decay = cnf_params_train.lr_decay
# train_obj.cnf_network.assign_lr(session, cnf_params_train.learning_rate * lr_decay)
# train_obj.tar_network.assign_lr(session, cnf_params_train.learning_rate * lr_decay)
# print("Epoch: %d Learning rate: %.5f \n" % (i + 1, session.run(train_obj.cnf_network.lr)))
test_loss, _ = self.run_cnf_net_epoch(cnf_test_writer, session,
cnf_min_loss, test_obj,
dict_obj, i)
print("CNF NETWORK: Epoch: %d Test loss: %.3f" %
(i + 1, test_loss))
curr_time = time.time()
print('1 CNF epoch run takes ' +
str(((curr_time - start_time) /
(i + 1)) / 60) + ' minutes.')
print('\n Finished CNF epoch: \n', i + 1)
''' TARGET NETWORK '''
print('\nStarting TGT epoch: \n', i + 1)
if i > 0:
if ckpt and ckpt.model_checkpoint_path:
print("Loading model from: %s" %
ckpt.model_checkpoint_path)
tf.train.Saver().restore(session,
ckpt.model_checkpoint_path)
# lr_decay = tar_params_train.lr_decay ** max(i - global_params.max_epoch, 0.0)
# train_obj.cnf_network.assign_lr(session, cnf_params_train.learning_rate * lr_decay)
# train_obj.tar_network.assign_lr(session, cnf_params_train.learning_rate * lr_decay)
# print("Epoch: %d Learning rate: %.5f" % (i + 1, session.run(train_obj.tar_network.lr)))
test_loss, _ = self.run_tar_net_epoch(tar_test_writer, session,
tar_min_loss, test_obj,
dict_obj, i)
print("TGT NETWORK: Epoch: %d Test loss: %.3f\n" %
(i + 1, test_loss))
curr_time = time.time()
print('1 TGT epoch run takes ' +
str(((curr_time - start_time) /
(i + 1)) / 60) + ' minutes.')
print('\nFinishing TGT epoch: \n', i + 1)
cnf_test_writer.close()
tar_test_writer.close()
