def __init__(self, w2vPath, numHidden):
self.numHidden = numHidden
self.w2v = load_w2v(w2vPath, FLAGS.embedding_word_size)
self.words = tf.Variable(self.w2v, name="words")
with tf.variable_scope('CNN_Layer') as scope:
self.char_filter = tf.get_variable(
"char_filter",
shape=[
FLAGS.char_window_size, FLAGS.embedding_char_size, 1,
FLAGS.embedding_char_size
],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
with tf.variable_scope('Clfier_output') as scope:
self.clfier_softmax_W = tf.get_variable(
"clfier_W",
shape=[numHidden * 2, FLAGS.num_classes],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.clfier_softmax_b = tf.get_variable(
"clfier_softmax_b",
shape=[FLAGS.num_classes],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.inp_w = tf.placeholder(tf.int32,
shape=[None, FLAGS.max_sentence_len],
name="input_words")
def run_test(training_filename):
##Test for Pairwise_Tokenize
# data = pd.read_csv(training_filename)
# w2v_file = 'GoogleNews-vectors-negative300.bin.gz'
# w2v_url = 'https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM'
# w2v = utils.load_w2v(w2v_file,w2v_url)
# #Comparison
# with open('tokenize_check.txt','wb') as output_file:
# for i in xrange(10):
# original = data['Headline'].iloc[i]
# p_words = pairwise_tokenize(original,True,w2v)
# pair_sent = "|".join(words)
# output_file.write(original + "\n")
# output_file.write("111: " + pair_sent +"\n")
# output_file.write("\n")
#Run Augmented headlines
w2v_file = 'GoogleNews-vectors-negative300.bin.gz'
w2v_url = 'https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM'
print "loading w2v"
w2v = utils.load_w2v(w2v_file, w2v_url)
data = pd.read_csv(training_filename)
print "generating headlines"
augmented_temp = augment_headlines(data, w2v, "augmented_headlines.txt")
new_data = pd.read_csv("augmented_headlines.txt", sep="|")
def __init__(self, c2vPath, numHidden):
self.numHidden = numHidden
self.c2v = load_w2v(c2vPath, FLAGS.embedding_char_size)
self.chars = tf.Variable(self.c2v, name="chars")
self.common_id_embedding = tf.Variable(tf.random_uniform(
[len(ENTITY_TYPES), FLAGS.embedding_char_size], -1.0, 1.0),
name="common_id_embedding")
self.chars_emb = tf.concat([self.chars, self.common_id_embedding],
0,
name='concat')
with tf.variable_scope('Clfier_output') as scope:
self.clfier_softmax_W = tf.get_variable(
"clfier_W",
shape=[numHidden * 2, FLAGS.num_classes],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.clfier_softmax_b = tf.get_variable(
"clfier_softmax_b",
shape=[FLAGS.num_classes],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.inp_c = tf.placeholder(tf.int32,
shape=[None, FLAGS.max_sentence_len],
name="input_words")
def __init__(self,
embedding_dim=100,
batch_size=64,
n_hidden=100,
learning_rate=0.01,
n_class=3,
max_sentence_len=50,
l2_reg=0.,
display_step=4,
n_iter=100,
type_=''):
self.embedding_dim = embedding_dim
self.batch_size = batch_size
self.n_hidden = n_hidden
self.learning_rate = learning_rate
self.n_class = n_class
self.max_sentence_len = max_sentence_len
self.l2_reg = l2_reg
self.display_step = display_step
self.n_iter = n_iter
self.type_ = type_
self.word_id_mapping, self.w2v = load_w2v(FLAGS.embedding_file_path,
self.embedding_dim)
self.word_embedding = tf.constant(self.w2v, name='word_embedding')
# self.word_embedding = tf.Variable(self.w2v, name='word_embedding')
# self.word_id_mapping = load_word_id_mapping(FLAGS.word_id_file_path)
# self.word_embedding = tf.Variable(
# tf.random_uniform([len(self.word_id_mapping), self.embedding_dim], -0.1, 0.1), name='word_embedding')
self.dropout_keep_prob = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
self.x = tf.placeholder(tf.int32, [None, self.max_sentence_len])
self.y = tf.placeholder(tf.int32, [None, self.n_class])
self.sen_len = tf.placeholder(tf.int32, None)
self.x_bw = tf.placeholder(tf.int32, [None, self.max_sentence_len])
self.y_bw = tf.placeholder(tf.int32, [None, self.n_class])
self.sen_len_bw = tf.placeholder(tf.int32, [None])
self.target_words = tf.placeholder(tf.int32, [None, 1])
with tf.name_scope('weights'):
self.weights = {
'softmax_bi_lstm':
tf.get_variable(
name='bi_lstm_w',
shape=[2 * self.n_hidden, self.n_class],
initializer=tf.random_uniform_initializer(-0.003, 0.003),
regularizer=tf.contrib.layers.l2_regularizer(self.l2_reg))
}
with tf.name_scope('biases'):
self.biases = {
'softmax_bi_lstm':
tf.get_variable(
name='bi_lstm_b',
shape=[self.n_class],
initializer=tf.random_uniform_initializer(-0.003, 0.003),
regularizer=tf.contrib.layers.l2_regularizer(self.l2_reg))
}
def __init__(self, c2vPath, numHidden):
self.numHidden = numHidden
self.c2v = load_w2v(c2vPath, FLAGS.embedding_char_size)
self.chars = tf.Variable(self.c2v, name="chars")
self.common_id_embedding = tf.Variable(
tf.random_uniform([len(ENTITY_TYPES), FLAGS.embedding_char_size],
-1.0, 1.0),
name="common_id_embedding")
self.chars_emb = tf.concat(
[self.chars, self.common_id_embedding], 0, name='concat')
self.filter_sizes = list(map(int, FLAGS.filter_sizes.split(',')))
self.clfier_filters = [None] * len(self.filter_sizes)
self.clfier_bs = [None] * len(self.filter_sizes)
for i, filter_size in enumerate(self.filter_sizes):
with tf.variable_scope('Clfier_conv_maxpool') as scope:
self.clfier_filters[i] = tf.get_variable(
"clfier_filter_%d" % i,
shape=[
filter_size, FLAGS.embedding_char_size, 1,
FLAGS.num_filters
],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.clfier_bs[i] = tf.get_variable(
"clfier_b_%d" % i,
shape=[FLAGS.num_filters],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
with tf.variable_scope('Clfier_output') as scope:
self.clfier_softmax_W = tf.get_variable(
"clfier_W",
shape=[
FLAGS.num_filters * len(self.filter_sizes),
FLAGS.num_classes
],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.clfier_softmax_b = tf.get_variable(
"clfier_softmax_b",
shape=[FLAGS.num_classes],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.inp_c = tf.placeholder(
tf.int32, shape=[None, FLAGS.max_sentence_len], name="input_words")
def create_dataset(headlines_train, bodies_train, stance_train, headlines_test,
bodies_test, stance_test):
"""Given w2v model, train and test headlines, bodies and stance, return Dataset
either by reading in already made csv's or calling data_splitting functions"""
w2v_file = 'GoogleNews-vectors-negative300.bin.gz'
w2v_url = 'https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM'
w2v = utils.load_w2v(w2v_file, w2v_url)
if w2v == False:
print 'failed to load the file'
return
try:
tmp = pd.read_csv('training-all.csv')
except IOError:
print("File not found, try running cleanup.py")
"""
for (idx, data) in enumerate(even_split(tmp)):
try:
data = pd.read_csv(path +'_split.csv')
except IOError:
print("File not found, try running cleanup.py")
bodies = np.array(data['articleBody'])
headlines = np.array(data['Headline'])
stance = np.array(data['Stance'])
#if(idx==0):
X_train, y_train = prep_stanford(headlines, bodies, stance, w2v, concat)
print(X_train.shape)
print(y_train.shape)
#else:
X_test, y_test = prep_stanford(headlines, bodies, stance, w2v, concat)
print(X_test.shape)
print(y_test.shape)
"""
X_train, y_train = prep_data2(headlines_train,
bodies_train,
stance_train,
w2v,
concat,
flag=1)
X_test, y_test = prep_data2(headlines_test,
bodies_test,
stance_test,
w2v,
concat,
flag=0)
return X_train, y_train, X_test, y_test
def __init__(self, embedding_dim=100, batch_size=64, n_hidden=100, learning_rate=0.01,
n_class=3, max_sentence_len=50, l2_reg=0., display_step=4, n_iter=100, type_=''):
self.embedding_dim = embedding_dim
self.batch_size = batch_size
self.n_hidden = n_hidden
self.learning_rate = learning_rate
self.n_class = n_class
self.max_sentence_len = max_sentence_len
self.l2_reg = l2_reg
self.display_step = display_step
self.n_iter = n_iter
self.type_ = type_
self.word_id_mapping, self.w2v = load_w2v(FLAGS.embedding_file_path, self.embedding_dim)
self.word_embedding = tf.constant(self.w2v, name='word_embedding')
# self.word_embedding = tf.Variable(self.w2v, name='word_embedding')
# self.word_id_mapping = load_word_id_mapping(FLAGS.word_id_file_path)
# self.word_embedding = tf.Variable(
# tf.random_uniform([len(self.word_id_mapping), self.embedding_dim], -0.1, 0.1), name='word_embedding')
self.dropout_keep_prob = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
self.x = tf.placeholder(tf.int32, [None, self.max_sentence_len])
self.y = tf.placeholder(tf.int32, [None, self.n_class])
self.sen_len = tf.placeholder(tf.int32, None)
self.x_bw = tf.placeholder(tf.int32, [None, self.max_sentence_len])
self.y_bw = tf.placeholder(tf.int32, [None, self.n_class])
self.sen_len_bw = tf.placeholder(tf.int32, [None])
self.target_words = tf.placeholder(tf.int32, [None, 1])
with tf.name_scope('weights'):
self.weights = {
'softmax_bi_lstm': tf.get_variable(
name='bi_lstm_w',
shape=[2 * self.n_hidden, self.n_class],
initializer=tf.random_uniform_initializer(-0.003, 0.003),
regularizer=tf.contrib.layers.l2_regularizer(self.l2_reg)
)
}
with tf.name_scope('biases'):
self.biases = {
'softmax_bi_lstm': tf.get_variable(
name='bi_lstm_b',
shape=[self.n_class],
initializer=tf.random_uniform_initializer(-0.003, 0.003),
regularizer=tf.contrib.layers.l2_regularizer(self.l2_reg)
)
}
def w2v_augment():
w2v_file = 'googlenews-vectors-negative300.bin.gz'
w2v_url = 'https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM'
w2v = utils.load_w2v(w2v_file,w2v_url)
if w2v==False:
print 'failed to load the file'
return
sentence = 'this is a test sentence'
split_sentence=sentence.split()
print split_sentence
new_sentence = replace_w2v(split_sentence,w2v)
print new_sentence
def __init__(self, distinctTagNum, w2vPath, numHidden):
self.distinctTagNum = distinctTagNum
self.numHidden = numHidden
self.w2v = load_w2v(w2vPath, FLAGS.ner_embedding_word_size)
self.words = tf.Variable(self.w2v, name="words")
with tf.variable_scope('Ner_output') as scope:
self.W = tf.get_variable(
shape=[numHidden * 2, distinctTagNum],
initializer=tf.truncated_normal_initializer(stddev=0.01),
name="weights",
regularizer=tf.contrib.layers.l2_regularizer(0.001))
self.b = tf.Variable(tf.zeros([distinctTagNum], name="bias"))
self.inp_w = tf.placeholder(tf.int32,
shape=[None, FLAGS.ner_max_sentence_len],
name="input_words")
def main():
try:
data = pd.read_csv("../data/training-all.csv")
except IOError:
print("File not found, try running cleanup.py")
bodies = np.array(data['articleBody'])
headlines = np.array(data['Headline'])
stance = np.array(data['Stance'])
w2v_file = 'GoogleNews-vectors-negative300.bin.gz'
w2v_url = 'https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM'
w2v = utils.load_w2v(w2v_file, w2v_url)
if w2v == False:
print 'failed to load the file'
return
run_test(headlines, bodies, stance, w2v)
def preprocess_all(bodies, stances, save_dir='./', augment_syn=True, window=True, split_per=0.6, val_per=0.2, diff=0.03, max_tries=10):
w2v_file = 'GoogleNews-vectors-negative300.bin.gz'
w2v_url = 'https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM'
w2v = utils.load_w2v(w2v_file,w2v_url)
df_all = import_data(bodies, stances, w2v, window=window)
#df_tr, df_te, df_val = even_split(df_all, split_per, val_per, max_tries, diff)
df_tr, df_te = even_split(df_all, split_per, val_per, max_tries, diff)
if window:
print 'test'
df_te = explode_data(df_te)
#print 'val'
#df_val = explode_data(df_val)
print 'train'
df_tr = explode_data(df_tr)
print 'len train', len(df_tr)
print 'len test', len(df_te)
#print 'len val', len(df_val)
augment_synonym.augment_headlines(df_tr, w2v, os.path.join(save_dir,'train.csv'), nrows=None, augment=augment_syn)
print 'done train'
#augment_synonym.augment_headlines(df_val, w2v, os.path.join(save_dir,'val.csv'), nrows=None, augment=False)
augment_synonym.augment_headlines(df_te, w2v, os.path.join(save_dir,'test.csv'), nrows=None, augment=False)
def create_domain_weigths(config_pair):
"""
:param model_path:
:param tkn_path:
:param opt:
:return:
"""
vec_path, tkn_path, opt = config_pair
print('Working on pre-trained embedding: '+str(vec_path))
# load tokenizer
with open(tkn_path, 'rb') as tkn_file:
tkn = pickle.load(tkn_file)
# loop through each domain
for domain in tkn:
print('Working on domain: '+str(domain))
# get the vector generator
vec_generator = utils.load_w2v(vec_path)
tmp_w, tmp_v = next(vec_generator)
print('Embedding size: ' + str(len(tmp_v)))
embed_len = len(tkn[domain].word_index)
if embed_len > tkn[domain].num_words:
embed_len = tkn[domain].num_words
embedding_matrix = np.zeros((embed_len + 1, len(tmp_v)))
# add the word if the word in the tokenizer
# if tmp_w in tkn[domain].word_index:
# if tkn[domain].word_index[tmp_w] < tkn[domain].num_words:
# embedding_matrix[tkn[domain].word_index[tmp_w]] = tmp_v
# loop through each word vectors
for word, vectors in vec_generator:
if word in tkn[domain].word_index:
if tkn[domain].word_index[word] < tkn[domain].num_words:
embedding_matrix[tkn[domain].word_index[word]] = vectors
# save the matrix to the dir
np.save(opt+'weights#'+str(domain)+'.npy', embedding_matrix)
def __init__(self, params,sess):
print 'init LSTM'
self.params = params
#Model Params
self.batch_size = params['batch_size']
self.learning_rate = params['learning_rate']
self.l2 = params['l2']
self.lstm_units = 100
self.hidden_units = 512
#DataPrams
self.body_truncate_len = 150
self.headline_truncate_len = 40
self.input_truncate_len = self.body_truncate_len + self.headline_truncate_len
self.data_dim = params['data_dim']
#W2V
w2v_file = 'GoogleNews-vectors-negative300.bin.gz'
w2v_url = 'https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM'
self.w2v = utils.load_w2v(w2v_file,w2v_url)
#Build the model
self.output = self.add_model()
self.loss = self.calculate_loss(self.output)
self.train_step = self.add_training()
self.eval_correct = self.evaluate(self.output)
self.iterations = 0 #counts the iterations we've run
#Summary writers
self.merged = tf.summary.merge_all()
self.train_writer = tf.summary.FileWriter('visual_logs_lstm' + '/train',
sess.graph)
self.test_writer = tf.summary.FileWriter('visual_logs_lstm'+ '/test')
#Saver
self.saver = tf.train.Saver()
def __init__(self, distinctTagNum, c2vPath, numHidden):
self.distinctTagNum = distinctTagNum
self.numHidden = numHidden
self.c2v = load_w2v(c2vPath, FLAGS.embedding_char_size)
self.chars = tf.Variable(self.c2v, name="chars")
self.common_id_embedding_pad = tf.constant(
0.0, shape=[1, numHidden * 2], name="common_id_embedding_pad")
self.common_id_embedding = tf.Variable(tf.random_uniform(
[len(ENTITY_TYPES), numHidden * 2], -1.0, 1.0),
name="common_id_embedding")
self.common_embedding = tf.concat(
[self.common_id_embedding_pad, self.common_id_embedding],
0,
name='common_embedding')
with tf.variable_scope('Ner_output') as scope:
self.W = tf.get_variable(
shape=[numHidden * 2, distinctTagNum],
initializer=tf.truncated_normal_initializer(stddev=0.01),
name="weights",
regularizer=tf.contrib.layers.l2_regularizer(0.001))
self.b = tf.Variable(tf.zeros([distinctTagNum], name="bias"))
with tf.variable_scope('Attention') as scope:
self.attend_W = tf.get_variable(
"attend_W",
shape=[1, 1, self.numHidden * 2, self.numHidden * 2],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.attend_V = tf.get_variable(
"attend_V",
shape=[self.numHidden * 2],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
with tf.variable_scope('Clfier_output') as scope:
self.clfier_softmax_W = tf.get_variable(
"clfier_W",
shape=[numHidden * 2, FLAGS.num_classes],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.clfier_softmax_b = tf.get_variable(
"clfier_softmax_b",
shape=[FLAGS.num_classes],
regularizer=tf.contrib.layers.l2_regularizer(0.0001),
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32)
self.inp_c = tf.placeholder(tf.int32,
shape=[None, FLAGS.max_sentence_len],
name="input_words")
self.entity_info = tf.placeholder(tf.int32,
shape=[None, MAX_COMMON_LEN],
name="entity_info")
def main(train_path, test_path, accuracyOnt, test_size, remaining_size, learning_rate=FLAGS.learning_rate, keep_prob = FLAGS.keep_prob1):
print_config()
with tf.device('/gpu:1'):
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path, FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
# word_embedding = tf.Variable(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32, name="keep_prob1")
keep_prob2 = tf.placeholder(tf.float32, name="keep_prob2")
with tf.name_scope('inputs'):
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len], name="x")
y = tf.placeholder(tf.float32, [None, FLAGS.n_class], name="y")
sen_len = tf.placeholder(tf.int32, None, name="sen_len")
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len], name="x_bw")
sen_len_bw = tf.placeholder(tf.int32, [None], name="sen_len_bw")
target_words = tf.placeholder(tf.int32, [None, FLAGS.max_target_len], name="target_words")
tar_len = tf.placeholder(tf.int32, [None], name="tar_len")
sent_short = tf.placeholder(tf.int32, [None, None], name="sent_short")
sent = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len], name="sent")
mult_mask = tf.placeholder(tf.float32, [None, FLAGS.max_sentence_len], name="mult_mask")
inputs_fw = tf.nn.embedding_lookup(word_embedding, x) # batch x N x d
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw) # batch x N x d
sent_full = tf.nn.embedding_lookup(word_embedding, sent) # batch x N x d
#compute average sentence representation and target
sentence_mask = tf.cast(tf.sequence_mask(sen_len_bw + sen_len - tar_len, FLAGS.max_sentence_len), tf.int32)
sentence = tf.nn.embedding_lookup(word_embedding, tf.multiply(sent_short,sentence_mask))
ave_sent = tf.divide(tf.reduce_sum(sentence, 1),tf.reshape(tf.tile(tf.cast(sen_len_bw + sen_len - tar_len, tf.float32),[FLAGS.embedding_dim]),[-1,FLAGS.embedding_dim]))
target_mask = tf.cast(tf.sequence_mask(tar_len, FLAGS.max_target_len), tf.int32)
target = tf.nn.embedding_lookup(word_embedding, tf.multiply(target_words,target_mask))
target = tf.divide(tf.reduce_sum(target, 1),tf.reshape(tf.tile(tf.cast(tar_len, tf.float32), [FLAGS.embedding_dim]),[-1,FLAGS.embedding_dim]))
# target = reduce_mean_with_len(target, tar_len)
# for MLP & DOT
# target = tf.expand_dims(target, 1)
# batch_size = tf.shape(inputs_bw)[0]
# target = tf.zeros([batch_size, FLAGS.max_sentence_len, FLAGS.embedding_dim]) + target
# for BL
# target = tf.squeeze(target)
alpha_fw, alpha_bw = None, None
prob, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = cabasc(inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len, ave_sent, mult_mask, sent_full, keep_prob1, keep_prob2, 'all')
loss = loss_func(y, prob)
acc_num, acc_prob = acc_func(y, prob)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(loss, global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
true_y = tf.argmax(y, 1)
pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
keep_prob,
FLAGS.keep_prob2,
FLAGS.batch_size,
learning_rate,
FLAGS.l2_reg,
FLAGS.max_sentence_len,
FLAGS.embedding_dim,
FLAGS.n_hidden,
FLAGS.n_class
)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
import time
timestamp = str(int(time.time()))
_dir = 'summary/' + str(timestamp) + '_' + title
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
train_summary_op, test_summary_op, validate_summary_op, train_summary_writer, test_summary_writer, \
validate_summary_writer = summary_func(loss, acc_prob, test_loss, test_acc, _dir, title, sess)
save_dir = 'temp_model/' + str(timestamp) + '_' + title + '/'
# saver = saver_func(save_dir)
sess.run(tf.global_variables_initializer())
# saver.restore(sess, '/-')
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, tr_sen_short, tr_sent, tr_mult_mask = load_inputs_cabasc(
train_path,
word_id_mapping,
FLAGS.max_sentence_len,
'TC',
is_r,
FLAGS.max_target_len
)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, te_sen_short, te_sent, te_mult_mask = load_inputs_cabasc(
test_path,
word_id_mapping,
FLAGS.max_sentence_len,
'TC',
is_r,
FLAGS.max_target_len
)
def get_batch_data(x_f, sen_len_f, x_b, sen_len_b, yi, target, tl, batch_size, kp1, kp2, senshort, sen, multMask, is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
sent_short: sen[index],
sent: sen[index],
mult_mask: multMask[index]
}
yield feed_dict, len(index)
max_acc = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
for i in range(FLAGS.n_iter):
trainacc, traincnt = 0., 0
for train, numtrain in get_batch_data(tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len,
FLAGS.batch_size, keep_prob, keep_prob, tr_sen_short, tr_sent, tr_mult_mask):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, step, summary, _trainacc = sess.run([optimizer, global_step, train_summary_op, acc_num], feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc += _trainacc # saver.save(sess, save_dir, global_step=step)
traincnt += numtrain
print('finished train')
acc, cost, cnt = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y,
te_target_word, te_tar_len, 2000, 1.0, 1.0, te_sen_short, te_sent, te_mult_mask, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_loss, _acc, _fw, _bw, _tl, _tr, _ty, _py, _p = sess.run(
[loss, acc_num, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r, true_y, pred_y, prob], feed_dict=test)
fw += list(_fw)
bw += list(_bw)
tl += list(_tl)
tr += list(_tr)
else:
_loss, _acc, _ty, _py, _p, _fw = sess.run([loss, acc_num, true_y, pred_y, prob, alpha_fw], feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
fw = np.asarray(_fw)
p += list(_p)
acc += _acc
cost += _loss * num
cnt += num
print('all samples={}, correct prediction={}'.format(cnt, acc))
trainacc = trainacc / traincnt
acc = acc / cnt
totalacc = ((acc * remaining_size) + (accuracyOnt * (test_size - remaining_size))) / test_size
cost = cost / cnt
print('Iter {}: mini-batch loss={:.6f}, train acc={:.6f}, test acc={:.6f}, combined acc={:.6f}'.format(i, cost,trainacc, acc, totalacc))
summary = sess.run(test_summary_op, feed_dict={test_loss: cost, test_acc: acc})
test_summary_writer.add_summary(summary, step)
if acc > max_acc:
max_acc = acc
max_fw = fw
max_bw = bw
max_tl = tl
max_tr = tr
max_ty = ty
max_py = py
max_prob = p
P = precision_score(max_ty, max_py, average=None)
R = recall_score(max_ty, max_py, average=None)
F1 = f1_score(max_ty, max_py, average=None)
print('P:', P, 'avg=', sum(P) / FLAGS.n_class)
print('R:', R, 'avg=', sum(R) / FLAGS.n_class)
print('F1:', F1, 'avg=', sum(F1) / FLAGS.n_class)
fp = open(FLAGS.prob_file, 'w')
for item in max_prob:
fp.write(' '.join([str(it) for it in item]) + '\n')
fp = open(FLAGS.prob_file + '_fw', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_fw):
fp.write(str(y1) + ' ' + str(y2) + ' ' + ' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_bw', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_bw):
fp.write(str(y1) + ' ' + str(y2) + ' ' + ' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_tl', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_tl):
fp.write(str(y1) + ' ' + str(y2) + ' ' + ' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_tr', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_tr):
fp.write(str(y1) + ' ' + str(y2) + ' ' + ' '.join([str(w) for w in ws[0]]) + '\n')
print('Optimization Finished! Max acc={}'.format(max_acc))
print('Learning_rate={}, iter_num={}, batch_size={}, hidden_num={}, l2={}'.format(
FLAGS.learning_rate,
FLAGS.n_iter,
FLAGS.batch_size,
FLAGS.n_hidden,
FLAGS.l2_reg
))
return max_acc, np.where(np.subtract(max_py, max_ty) == 0, 0, 1), max_fw
def main(train_path,
eval_path,
test_path,
complete_path,
accuracyOnt,
test_size,
remaining_size,
l2=0.0001): #learning_rate=0.02, keep_prob=0.7,
#momentum=0.95, l2=0.0001):
#print_config()
with tf.device('/gpu:1'):
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path,
FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
learning_rate = tf.placeholder(tf.float32)
momentum = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
y = tf.placeholder(tf.float32, [None, FLAGS.n_class])
sen_len = tf.placeholder(tf.int32, None)
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
sen_len_bw = tf.placeholder(tf.int32, [None])
target_words = tf.placeholder(tf.int32,
[None, FLAGS.max_target_len])
tar_len = tf.placeholder(tf.int32, [None])
inputs_fw = tf.nn.embedding_lookup(word_embedding, x)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
alpha_fw, alpha_bw = None, None
prob, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = lcr_rot(
inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len,
keep_prob1, keep_prob2, l2, 'all')
loss = loss_func(y, prob)
acc_num, acc_prob = acc_func(y, prob)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=momentum).minimize(
loss,
global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
true_y = tf.argmax(y, 1)
pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
FLAGS.learning_rate, FLAGS.l2_reg, FLAGS.max_sentence_len,
FLAGS.embedding_dim, FLAGS.n_hidden, FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
import time
timestamp = str(int(time.time()))
_dir = 'summary/' + str(timestamp) + '_' + title
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
train_summary_op, test_summary_op, validate_summary_op, train_summary_writer, test_summary_writer, \
validate_summary_writer = summary_func(loss, acc_prob, test_loss, test_acc, _dir, title, sess)
#save_dir = 'temp_model/' + str(timestamp) + '_' + title + '/'
# saver = saver_func(save_dir)
#save_pth = 'savedModel' + str(FLAGS.year)
#if save_pth is not None:
# save_path = save_pth + '/'
# saver = saver_func(save_path)
sess.run(tf.global_variables_initializer())
# saver.restore(sess, '/-')
# restore_pth = 'savedModel' + str(FLAGS.year)
# meta = '-2350'
#if restore_pth is not None and meta is not None:
# restore_path = restore_pth + '/'
# restore_meta_path = restore_pth + '/' + meta + '.meta'
# restore = tf.train.import_meta_graph(restore_meta_path)
#restore.restore(sess, tf.train.latest_checkpoint(restore_path))
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, _, _, _ = load_inputs_twitter(
train_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
ev_x, ev_sen_len, ev_x_bw, ev_sen_len_bw, ev_y, ev_target_word, ev_tar_len, _, _, _ = load_inputs_twitter(
eval_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, _, _, _ = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
co_x, co_sen_len, co_x_bw, co_sen_len_bw, co_y, co_target_word, co_tar_len, _, _, _ = load_inputs_twitter(
complete_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
def train_get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
learning,
moment,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
learning_rate: learning,
momentum: moment,
}
yield feed_dict, len(index)
max_acc = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
train_time = 0
max_time = 0
print("number of training instances: {}, number of test instances: {}".
format(len(tr_y), len(te_y)))
cost_func_test = []
cost_func_train = []
cost_func_eval = []
acc_func_train = []
acc_func_test = []
acc_func_eval = []
i = 0
converged = False
all_evalloss = []
all_evalacc = []
max_evalloss = 100
lr = 0.02
keep_prob = 0.7
mom = 0.95
while i < FLAGS.n_iter and converged == False:
trainacc, trainloss, traincnt = 0., 0., 0
start_time = time.time()
for train, numtrain in train_get_batch_data(
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len, FLAGS.batch_size, keep_prob,
keep_prob, lr, mom):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, _trainloss, step, summary, _trainacc = sess.run(
[optimizer, loss, global_step, train_summary_op, acc_num],
feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc += _trainacc # saver.save(sess, save_dir, global_step=step)
traincnt += numtrain
trainloss += _trainloss * numtrain
#if save_pth is not None:
# saver.save(sess, save_path, global_step=step)
elapsed_time = time.time() - start_time
train_time += elapsed_time
evalacc, evalcost, evalcnt = 0., 0., 0
for eva, evalnum in get_batch_data(ev_x, ev_sen_len, ev_x_bw,
ev_sen_len_bw, ev_y,
ev_target_word, ev_tar_len,
2000, 1.0, 1.0, False):
_evalloss, _evalacc = sess.run([loss, acc_num], feed_dict=eva)
evalacc += _evalacc
evalcost += _evalloss * evalnum
evalcnt += evalnum
acc, cost, cnt = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_y,
te_target_word, te_tar_len, 2000,
1.0, 1.0, False):
_loss, _acc, _ty, _py, _p, _fw, _bw, _tl, _tr = sess.run(
[
loss, acc_num, true_y, pred_y, prob, alpha_fw,
alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
p = np.asarray(_p)
fw = np.asarray(_fw)
bw = np.asarray(_bw)
tl = np.asarray(_tl)
tr = np.asarray(_tr)
acc += _acc
cost += _loss * num
cnt += num
comacc, comcnt = 0., 0
for com, comnum in get_batch_data(co_x, co_sen_len, co_x_bw,
co_sen_len_bw, co_y,
co_target_word, co_tar_len,
FLAGS.batch_size, 1.0, 1.0,
False):
_comloss, _comacc, _cty, _cpy, _cp, _cfw, _cbw, _ctl, _ctr = sess.run(
[
loss, acc_num, true_y, pred_y, prob, alpha_fw,
alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=com)
comacc += _comacc
comcnt += comnum
print(
'all samples={}, correct prediction={}, training time={}, training time so far={}'
.format(cnt, acc, elapsed_time, train_time))
trainacc = trainacc / traincnt
acc = acc / cnt
evalacc = evalacc / evalcnt
comacc = comacc / comcnt
#totalacc = ((acc * remaining_size) + (accuracyOnt * (test_size - remaining_size))) / test_size
cost = cost / cnt
trainloss = trainloss / traincnt
evalcost = evalcost / evalcnt
cost_func_test.append(cost)
cost_func_train.append(trainloss)
cost_func_eval.append(evalcost)
acc_func_eval.append(evalacc)
acc_func_test.append(acc)
acc_func_train.append(trainacc)
print(
'Iter {}: mini-batch loss validation set={:.6f}, train loss={:.6f}, train acc={:.6f}, validation acc={:6f} test acc={:.6f}, total training acc={:6f}'
.format(i, evalcost, trainloss, trainacc, evalacc, acc,
comacc))
summary = sess.run(test_summary_op,
feed_dict={
test_loss: cost,
test_acc: acc
})
test_summary_writer.add_summary(summary, step)
all_evalloss.append(evalcost)
all_evalacc.append(evalacc)
if i > 2: # want to compare current train accuracy with train acc previous iterations
if (all_evalacc[i] - all_evalacc[i - 1] < 0.001) and (
all_evalacc[i - 1] - all_evalacc[i - 2] < 0.001) \
and (all_evalacc[i - 2] - all_evalacc[i - 3] < 0.001):
converged = True
i += 1
#if acc > max_acc:
# max_acc = acc
# max_fw = fw
# max_bw = bw
# max_tl = tl
# max_tr = tr
# max_ty = ty
# max_py = py
# max_prob = p
#P = precision_score(max_ty, max_py, average=None)
#R = recall_score(max_ty, max_py, average=None)
#F1 = f1_score(max_ty, max_py, average=None)
#print('P:', P, 'avg=', sum(P) / FLAGS.n_class)
#print('R:', R, 'avg=', sum(R) / FLAGS.n_class)
#print('F1:', F1, 'avg=', sum(F1) / FLAGS.n_class)
print("total train acc = ")
# Plotting chart of training and testing loss as a function of iterations
iterations = list(range(i))
plt.plot(iterations, cost_func_train, label='Cost func train')
plt.plot(iterations, cost_func_test, label='Cost func test')
plt.plot(iterations, cost_func_eval, label='Cost func validation')
plt.title('Model loss k=1')
plt.ylabel('Loss')
plt.xlabel('Iterations')
plt.legend(['train', 'test', 'eval'], loc='upper left')
plt.show()
# Plotting chart of training and testing accuracies as a function of iterations
iterations = list(range(i))
plt.plot(iterations, acc_func_train, label='Acc func train')
plt.plot(iterations, acc_func_test, label='Acc func test')
plt.plot(iterations, acc_func_eval, label='Acc func validation')
plt.title('Model accuracy k=1')
plt.ylabel('Loss')
plt.xlabel('Iterations')
plt.legend(['train', 'test', 'eval'], loc='upper left')
plt.show()
print(
'Optimization Finished! Iteration:{}: Validation loss={}, validation accuracy={}, test accuracy={}'
.format(i, evalcost, evalacc, acc))
print(acc_func_train)
print(acc_func_test)
return acc
#!/usr/bin/python
# -*- coding: UTF-8 -*-
#Author zhang
import utils
from tensorflow.keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, Flatten, Dense, GlobalAveragePooling1D
sequences, data = utils.read_corpus("data\data.tsv")
w2v_model = utils.load_w2v("data\word2vec.model")
word2id = utils.word2id(w2v_model)
X_data = utils.get_sequences(word2id, sequences)
# 截长补短
maxlen = 20
X_pad = pad_sequences(X_data, maxlen=maxlen)
# 取得标签
Y = data.sentiment.values
# 划分数据集
X_train, X_test, Y_train, Y_test = train_test_split(
X_pad,
Y,
test_size=0.2,
random_state=42)
"""
def __init__(self, config):
self.embedding_dim = config.embedding_dim
self.batch_size = config.batch_size
self.n_hidden = config.n_hidden
self.learning_rate = config.learning_rate
self.n_class = config.n_class
self.max_len = config.max_len
self.l2_reg = config.l2_reg
self.display_step = config.display_step
self.n_iter = config.n_iter
self.embedding_file = config.embedding_file_path
self.word2id_file = config.word_id_file_path
self.aspect_id_file = config.aspect_id_file_path
self.train_file = config.train_file_path
self.test_file = config.test_file_path
self.val_file = config.validate_file_path
self.word2id, self.w2v = load_w2v(self.embedding_file,
self.embedding_dim)
self.word_embedding = tf.constant(self.w2v, name='word_embedding')
self.aspect2id, self.a2v = load_aspect2id(self.aspect_id_file,
self.word2id, self.w2v,
self.embedding_dim)
self.aspect_embedding = tf.constant(self.a2v, name='aspect_embedding')
with tf.name_scope('inputs'):
self.x = tf.placeholder(tf.int32, [None, self.max_len], name='x')
self.y = tf.placeholder(tf.float32, [None, self.n_class], name='y')
self.sen_len = tf.placeholder(tf.int32, None, name='sen_len')
self.aspect_id = tf.placeholder(tf.int32, None, name='aspect_id')
self.position = tf.placeholder(tf.int32, [None, self.max_len],
name='position')
with tf.name_scope('GRU'):
self.w_r = tf.get_variable(
name='W_r',
shape=[2 * self.n_hidden + 1, self.n_hidden],
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(uniform=True))
self.u_r = tf.get_variable(
name='U_r',
shape=[self.n_hidden, self.n_hidden],
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(uniform=True))
self.w_z = tf.get_variable(
name='W_z',
shape=[2 * self.n_hidden + 1, self.n_hidden],
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(uniform=True))
self.u_z = tf.get_variable(
name='U_z',
shape=[self.n_hidden, self.n_hidden],
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(uniform=True))
self.w_x = tf.get_variable(
name='W_x',
shape=[self.n_hidden, self.n_hidden],
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(uniform=True))
self.w_g = tf.get_variable(
name='W_g',
shape=[2 * self.n_hidden + 1, self.n_hidden],
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(uniform=True))
def main(train_path,
test_path,
accuracyOnt,
test_size,
remaining_size,
learning_rate=0.09,
keep_prob=0.3,
momentum=0.85,
l2=0.00001):
print_config()
with tf.device('/gpu:1'):
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path,
FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
y = tf.placeholder(tf.float32, [None, FLAGS.n_class])
sen_len = tf.placeholder(tf.int32, None)
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
sen_len_bw = tf.placeholder(tf.int32, [None])
target_words = tf.placeholder(tf.int32,
[None, FLAGS.max_target_len])
tar_len = tf.placeholder(tf.int32, [None])
inputs_fw = tf.nn.embedding_lookup(word_embedding, x)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
alpha_fw, alpha_bw = None, None
prob, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = lcr_rot(
inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len,
keep_prob1, keep_prob2, l2, 'all')
loss = loss_func(y, prob)
acc_num, acc_prob = acc_func(y, prob)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=momentum).minimize(
loss,
global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
true_y = tf.argmax(y, 1)
pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
FLAGS.learning_rate, FLAGS.l2_reg, FLAGS.max_sentence_len,
FLAGS.embedding_dim, FLAGS.n_hidden, FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
import time
timestamp = str(int(time.time()))
_dir = 'summary/' + str(timestamp) + '_' + title
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
train_summary_op, test_summary_op, validate_summary_op, train_summary_writer, test_summary_writer, \
validate_summary_writer = summary_func(loss, acc_prob, test_loss, test_acc, _dir, title, sess)
sess.run(tf.global_variables_initializer())
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, _, _, _ = load_inputs_twitter(
train_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, _, _, _ = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
max_acc = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
Results_File = np.zeros((3, 1))
for i in range(FLAGS.n_iter):
trainacc, traincnt = 0., 0
for train, numtrain in get_batch_data(tr_x, tr_sen_len, tr_x_bw,
tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len,
FLAGS.batch_size, keep_prob,
keep_prob):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, step, summary, _trainacc = sess.run(
[optimizer, global_step, train_summary_op, acc_num],
feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc += _trainacc # saver.save(sess, save_dir, global_step=step)
traincnt += numtrain
acc, cost, cnt = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_y,
te_target_word, te_tar_len, 2000,
1.0, 1.0, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_loss, _acc, _fw, _bw, _tl, _tr, _ty, _py, _p = sess.run(
[
loss, acc_num, alpha_fw, alpha_bw, alpha_t_l,
alpha_t_r, true_y, pred_y, prob
],
feed_dict=test)
fw += list(_fw)
bw += list(_bw)
tl += list(_tl)
tr += list(_tr)
else:
_loss, _acc, _ty, _py, _p, _fw, _bw, _tl, _tr = sess.run(
[
loss, acc_num, true_y, pred_y, prob, alpha_fw,
alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
p = np.asarray(_p)
fw = np.asarray(_fw)
bw = np.asarray(_bw)
tl = np.asarray(_tl)
tr = np.asarray(_tr)
acc += _acc
cost += _loss * num
cnt += num
print('all samples={}, correct prediction={}'.format(cnt, acc))
trainacc = trainacc / traincnt
acc = acc / cnt
totalacc = ((acc * remaining_size) +
(accuracyOnt *
(test_size - remaining_size))) / test_size
cost = cost / cnt
print(
'Iter {}: mini-batch loss={:.6f}, train acc={:.6f}, test acc={:.6f}, combined acc={:.6f}'
.format(i, cost, trainacc, acc, totalacc))
summary = sess.run(test_summary_op,
feed_dict={
test_loss: cost,
test_acc: acc
})
test_summary_writer.add_summary(summary, step)
if acc > max_acc:
max_acc = acc
max_fw = fw
max_bw = bw
max_tl = tl
max_tr = tr
max_ty = ty
max_py = py
max_prob = p
Added = [[i + 1], [trainacc], [acc]]
Results_File = np.concatenate((Results_File, Added), 1)
# Saving training information as csv file
from datetime import datetime
dateTimeObj = datetime.now()
save_dir = '/Users/ronhochstenbach/Desktop/Ectrie Thesis/Venv_Thesis/Results_Run_Adversarial/Run_' + str(
dateTimeObj) + '_lcrrot_' + str(FLAGS.year) + '.csv'
np.savetxt(save_dir, Results_File, delimiter=",")
P = precision_score(max_ty, max_py, average=None)
R = recall_score(max_ty, max_py, average=None)
F1 = f1_score(max_ty, max_py, average=None)
print('P:', P, 'avg=', sum(P) / FLAGS.n_class)
print('R:', R, 'avg=', sum(R) / FLAGS.n_class)
print('F1:', F1, 'avg=', sum(F1) / FLAGS.n_class)
fp = open(FLAGS.prob_file, 'w')
for item in max_prob:
fp.write(' '.join([str(it) for it in item]) + '\n')
fp = open(FLAGS.prob_file + '_fw', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_fw):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_bw', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_bw):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_tl', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_tl):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_tr', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_tr):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
print('Optimization Finished! Max acc={}'.format(max_acc))
print(
'Learning_rate={}, iter_num={}, batch_size={}, hidden_num={}, l2={}'
.format(FLAGS.learning_rate, FLAGS.n_iter, FLAGS.batch_size,
FLAGS.n_hidden, FLAGS.l2_reg))
return max_acc, np.where(np.subtract(max_py, max_ty) == 0, 0,
1), max_fw.tolist(), max_bw.tolist(
), max_tl.tolist(), max_tr.tolist()
def main(train_path,
test_path,
accuracyOnt,
test_size,
remaining_size,
augment_data,
augmentation_file_path,
ct,
learning_rate=0.09,
keep_prob=0.3,
momentum=0.85,
l2=0.00001):
print_config()
augmenter = Augmentation(FLAGS.EDA_type, need_mixup=True)
with tf.device('/gpu:1'):
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path,
FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
y = tf.placeholder(tf.float32, [None, FLAGS.n_class])
sen_len = tf.placeholder(tf.int32, None)
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
sen_len_bw = tf.placeholder(tf.int32, [None])
target_words = tf.placeholder(tf.int32,
[None, FLAGS.max_target_len])
tar_len = tf.placeholder(tf.int32, [None])
inputs_fw = tf.nn.embedding_lookup(word_embedding, x)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
alpha_fw, alpha_bw = None, None
prob, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = lcr_rot(
inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len,
keep_prob1, keep_prob2, l2, 'all')
loss = loss_func(y, prob)
acc_num, acc_prob = acc_func(y, prob)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=momentum).minimize(
loss,
global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
true_y = tf.argmax(y, 1)
pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
FLAGS.learning_rate, FLAGS.l2_reg, FLAGS.max_sentence_len,
FLAGS.embedding_dim, FLAGS.n_hidden, FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
import time
timestamp = str(int(time.time()))
_dir = 'summary/' + str(timestamp) + '_' + title
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
train_summary_op, test_summary_op, validate_summary_op, train_summary_writer, test_summary_writer, \
validate_summary_writer = summary_func(loss, acc_prob, test_loss, test_acc, _dir, title, sess)
save_dir = 'temp_model/' + str(timestamp) + '_' + title + '/'
# saver = saver_func(save_dir)
sess.run(tf.global_variables_initializer())
# saver.restore(sess, '/-')
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
len_non_augmented, tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, _, _, _ = load_inputs_twitter(
train_path,
word_id_mapping,
FLAGS.max_sentence_len,
'TC',
is_r,
FLAGS.max_target_len,
augment_data=augment_data,
augmentation_file_path=augmentation_file_path)
_, te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, _, _, _ = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
max_records_mixup = len(
tr_x) if FLAGS.mixup_on_augmentations > 0 else len_non_augmented
if augment_data and FLAGS.use_word_mixup > 0:
print("The amount of records on which mixup is applied: {}".format(
max_records_mixup))
rand_mixup = np.array(range(max_records_mixup - 1))
print("applying mixup...")
for _ in range(FLAGS.use_word_mixup):
random.shuffle(rand_mixup)
for i, j in tqdm(zip(*[iter(rand_mixup)] * 2)):
first = (tr_x[i], tr_sen_len[i], tr_x_bw[i],
tr_sen_len_bw[i], tr_y[i], tr_target_word[i],
tr_tar_len[i])
second = (tr_x[j], tr_sen_len[j], tr_x_bw[j],
tr_sen_len_bw[j], tr_y[j], tr_target_word[j],
tr_tar_len[j])
augmenter.word_mixup(first, second)
print("Word mixup embeddings: {}".format(augmenter.counter))
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
max_acc = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
for i in range(FLAGS.n_iter):
trainacc, traincnt = 0., 0
for train, numtrain in get_batch_data(tr_x, tr_sen_len, tr_x_bw,
tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len,
FLAGS.batch_size, keep_prob,
keep_prob):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, step, summary, _trainacc = sess.run(
[optimizer, global_step, train_summary_op, acc_num],
feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc += _trainacc # saver.save(sess, save_dir, global_step=step)
traincnt += numtrain
acc, cost, cnt = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_y,
te_target_word, te_tar_len, 2000,
1.0, 1.0, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_loss, _acc, _fw, _bw, _tl, _tr, _ty, _py, _p = sess.run(
[
loss, acc_num, alpha_fw, alpha_bw, alpha_t_l,
alpha_t_r, true_y, pred_y, prob
],
feed_dict=test)
fw += list(_fw)
bw += list(_bw)
tl += list(_tl)
tr += list(_tr)
else:
_loss, _acc, _ty, _py, _p, _fw, _bw, _tl, _tr = sess.run(
[
loss, acc_num, true_y, pred_y, prob, alpha_fw,
alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
p = np.asarray(_p)
fw = np.asarray(_fw)
bw = np.asarray(_bw)
tl = np.asarray(_tl)
tr = np.asarray(_tr)
acc += _acc
cost += _loss * num
cnt += num
print('all samples={}, correct prediction={}'.format(cnt, acc))
trainacc = trainacc / traincnt
acc = acc / cnt
totalacc = ((acc * remaining_size) +
(accuracyOnt *
(test_size - remaining_size))) / test_size
cost = cost / cnt
print(
'Iter {}: mini-batch loss={:.6f}, train acc={:.6f}, test acc={:.6f}, combined acc={:.6f}'
.format(i, cost, trainacc, acc, totalacc))
summary = sess.run(test_summary_op,
feed_dict={
test_loss: cost,
test_acc: acc
})
test_summary_writer.add_summary(summary, step)
if acc > max_acc:
max_trainacc = trainacc
max_totalacc = totalacc
iteration = i
max_acc = acc
max_fw = fw
max_bw = bw
max_tl = tl
max_tr = tr
max_ty = ty
max_py = py
max_prob = p
P = precision_score(max_ty, max_py, average=None)
R = recall_score(max_ty, max_py, average=None)
F1 = f1_score(max_ty, max_py, average=None)
print('P:', P, 'avg=', sum(P) / FLAGS.n_class)
print('R:', R, 'avg=', sum(R) / FLAGS.n_class)
print('F1:', F1, 'avg=', sum(F1) / FLAGS.n_class)
keys_to_save = 'year EDA_type EDA_deletion EDA_replacement original_multiplier EDA_insertion EDA_swap EDA_pct backtranslation_langs use_word_mixup mixup_beta mixup_on_augmentations'.split(
' ')
try:
df = pd.read_json(FLAGS.results_file)
print('adding outcome to {}'.format(FLAGS.results_file))
except ValueError:
print(
'did not find an existing result file, creating a new one...')
df = pd.DataFrame([])
new_experiment = {}
for k, v in sorted(FLAGS.flag_values_dict().items()):
if k in keys_to_save:
new_experiment[k] = v
new_experiment['in_sample'] = max_trainacc
new_experiment['out_of_sample'] = max_acc
new_experiment['ontology_acc'] = accuracyOnt
new_experiment['total_acc'] = max_totalacc
new_experiment['at_iteration'] = iteration
new_experiment['#of_test'] = cnt
new_experiment['#of_train'] = len(tr_x)
new_experiment['pre_embed_aug'] = ct
new_experiment['post_embed_aug'] = augmenter.counter
df = df.append(new_experiment, ignore_index=True)
df.to_json(FLAGS.results_file)
print('Optimization Finished! Max acc={}'.format(max_acc))
print(
'Learning_rate={}, iter_num={}, batch_size={}, hidden_num={}, l2={}'
.format(FLAGS.learning_rate, FLAGS.n_iter, FLAGS.batch_size,
FLAGS.n_hidden, FLAGS.l2_reg))
return max_acc, np.where(np.subtract(max_py, max_ty) == 0, 0,
1), max_fw.tolist(), max_bw.tolist(
), max_tl.tolist(), max_tr.tolist()
def main(train_path,
test_path,
learning_rate=FLAGS.learning_rate,
keep_prob=FLAGS.keep_prob1,
l2=FLAGS.l2_reg,
beta=0.9,
number_epochs=100):
print_config()
with tf.device('/gpu:1'):
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path,
FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
y = tf.placeholder(tf.float32, [None, FLAGS.n_class])
sen_len = tf.placeholder(tf.int32, None)
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
sen_len_bw = tf.placeholder(tf.int32, [None])
target_words = tf.placeholder(tf.int32,
[None, FLAGS.max_target_len])
tar_len = tf.placeholder(tf.int32, [None])
inputs_fw = tf.nn.embedding_lookup(word_embedding, x)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
alpha_fw, alpha_bw = None, None
prob, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = lcr_rot(
inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len,
keep_prob1, keep_prob2, l2, 'all')
loss = loss_func(y, prob)
acc_num, acc_prob = acc_func(y, prob)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=beta).minimize(
loss,
global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
true_y = tf.argmax(y, 1)
pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
learning_rate, l2, FLAGS.max_sentence_len, FLAGS.embedding_dim,
FLAGS.n_hidden, FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
batch_size = FLAGS.batch_size
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
import time
timestamp = str(int(time.time()))
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
sess.run(tf.global_variables_initializer())
# saver.restore(sess, '/-')
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, _, _, _ = load_inputs_twitter(
train_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, _, _, _ = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
max_acc = 0.
all_training_losses, all_training_accuracies = [], []
all_test_losses, all_test_accuracies = [], []
for i in range(number_epochs):
learning_rate = (0.99) * learning_rate
number_of_training_examples_correct, number_of_training_examples, training_loss = 0., 0, 0.
for train, numtrain in get_batch_data(tr_x, tr_sen_len, tr_x_bw,
tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len,
FLAGS.batch_size, keep_prob,
keep_prob):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, step, _trainacc, _training_loss = sess.run(
[optimizer, global_step, acc_num, loss], feed_dict=train)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
number_of_training_examples_correct += _trainacc
number_of_training_examples += numtrain
training_loss += _training_loss * numtrain
number_of_test_examples_correct, test_loss, number_of_test_examples = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_y,
te_target_word, te_tar_len, 2000,
1.0, 1.0, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_loss, _acc, _fw, _bw, _tl, _tr, _ty, _py, _p = sess.run(
[
loss, acc_num, alpha_fw, alpha_bw, alpha_t_l,
alpha_t_r, true_y, pred_y, prob
],
feed_dict=test)
fw += list(_fw)
bw += list(_bw)
tl += list(_tl)
tr += list(_tr)
else:
_loss, _acc, _ty, _py, _p, _fw, _bw, _tl, _tr = sess.run(
[
loss, acc_num, true_y, pred_y, prob, alpha_fw,
alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
p = np.asarray(_p)
fw = np.asarray(_fw)
bw = np.asarray(_bw)
tl = np.asarray(_tl)
tr = np.asarray(_tr)
number_of_test_examples_correct += _acc
test_loss += _loss * num
number_of_test_examples += num
print(
'number of training examples={}, correct training examples={}, number of test examples={}, correct test examples={}'
.format(number_of_training_examples,
number_of_training_examples_correct,
number_of_test_examples,
number_of_test_examples_correct))
training_accuracy = number_of_training_examples_correct / number_of_training_examples
test_accuracy = number_of_test_examples_correct / number_of_test_examples
average_test_loss = test_loss / number_of_test_examples
average_training_loss = training_loss / number_of_training_examples
all_training_losses.append(average_training_loss)
all_training_accuracies.append(training_accuracy)
all_test_losses.append(average_test_loss)
all_test_accuracies.append(test_accuracy)
print(
'Epoch {}: average training loss={:.6f}, train acc={:.6f}, average test loss={:.6f}, test acc={:.6f}'
.format(i, average_training_loss, training_accuracy,
average_test_loss, test_accuracy))
min_training_loss = min(all_training_losses)
max_training_accuracy = max(all_training_accuracies)
min_test_loss = min(all_test_losses)
max_test_accuracy = max(all_test_accuracies)
print('Optimization Finished! Max acc={}'.format(max_acc))
print(
'Learning_rate={}, iter_num={}, batch_size={}, hidden_num={}, l2={}'
.format(learning_rate, number_epochs, FLAGS.batch_size,
FLAGS.n_hidden, l2))
return min_training_loss, max_training_accuracy, min_test_loss, max_test_accuracy, all_training_losses, all_training_accuracies, all_test_losses, all_test_accuracies
def main(train_path,
test_path,
accuracyOnt,
test_size,
remaining_size,
sort,
num_buckets,
l2=0.0001): # learning_rate=0.07,
# keep_prob=0.4,
# momentum=0.9):
# print_config()
with tf.device('/gpu:1'):
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path,
FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
learning_rate = tf.placeholder(tf.float32)
momentum = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
y = tf.placeholder(tf.float32, [None, FLAGS.n_class])
sen_len = tf.placeholder(tf.int32, None)
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
sen_len_bw = tf.placeholder(tf.int32, [None])
target_words = tf.placeholder(tf.int32,
[None, FLAGS.max_target_len])
tar_len = tf.placeholder(tf.int32, [None])
inputs_fw = tf.nn.embedding_lookup(word_embedding, x)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
alpha_fw, alpha_bw = None, None
prob, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = lcr_rot(
inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len,
keep_prob1, keep_prob2, l2, 'all')
loss = loss_func(y, prob)
acc_num, acc_prob = acc_func(y, prob)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=momentum).minimize(
loss,
global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
true_y = tf.argmax(y, 1)
pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
FLAGS.learning_rate, FLAGS.l2_reg, FLAGS.max_sentence_len,
FLAGS.embedding_dim, FLAGS.n_hidden, FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
import time
timestamp = str(int(time.time()))
_dir = 'summary/' + str(timestamp) + '_' + title
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
train_summary_op, test_summary_op, validate_summary_op, train_summary_writer, test_summary_writer, \
validate_summary_writer = summary_func(loss, acc_prob, test_loss, test_acc, _dir, title, sess)
# save_dir = 'temp_model/babysteps2buckets'
# saver = saver_func(save_dir)
sess.run(tf.global_variables_initializer())
# saver.restore(sess, '/-')
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, _, _, _ = load_inputs_twitter(
train_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, _, _, _ = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
def curr_get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
learning,
moment,
bucket,
is_shuffle=True):
for index in curr_batch_index(bucket, batch_size, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
learning_rate: learning,
momentum: moment,
}
yield feed_dict, len(index)
def eval_get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
bucket,
is_shuffle=True):
for index in curr_batch_index(bucket, batch_size, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
max_acc = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
train_time = 0
max_time = 0
total_iter = 0
cost_func_test = []
cost_func_train = []
cost_func_eval = []
acc_func_train = []
acc_func_test = []
acc_func_eval = []
alltrainacc = []
# split data in num_buckets
buckets = np.array_split(sort, num_buckets)
bucket_number = 1
lowest_val = 100
best_train = 0.
best_test = 0.
best_iteration = 0
for bucket in buckets: # for every bucket of data
print("bucket number:{}".format(bucket_number))
np.random.shuffle(bucket)
tmp = int(round(0.8 * len(bucket)))
traindata = bucket[:tmp]
evaldata = bucket[tmp:]
# update the hyperparameters for every bucket, depending on num_buckets
if bucket_number == 1:
lr = 0.01
keep_prob = 0.7
mom = 0.85
if bucket_number == 2:
lr = 0.01
keep_prob = 0.7
mom = 0.85
if bucket_number == 3:
lr = 0.02
keep_prob = 0.6
mom = 0.95
if bucket_number == 4:
lr = 0.08
keep_prob = 0.3
mom = 0.9
if bucket_number == 5:
lr = 0.07
keep_prob = 0.4
mom = 0.99
if bucket_number == 6:
lr = 0.02
keep_prob = 0.6
mom = 0.9
if bucket_number == 7:
lr = 0.05
keep_prob = 0.4
mom = 0.9
if bucket_number == 8:
lr = 0.05
keep_prob = 0.4
mom = 0.95
if bucket_number == 9:
lr = 0.01
keep_prob = 0.5
mom = 0.99
if bucket_number == 10: # all the instances
lr = 0.01
keep_prob = 0.6
mom = 0.9
print("Training instances: {}, validation instances: {}".format(
len(traindata), len(evaldata)))
i = 0
converged = False
all_evalloss = []
all_evalacc = []
while i < FLAGS.n_iter and converged == False: # until convergence or until certain amount of iterations
trainacc, trainloss, traincnt = 0., 0., 0
start_time = time.time()
for train, numtrain in curr_get_batch_data(
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len, FLAGS.batch_size,
keep_prob, keep_prob, lr, mom, traindata):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, _trainloss, step, summary, _trainacc = sess.run(
[
optimizer, loss, global_step, train_summary_op,
acc_num
],
feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc += _trainacc # saver.save(sess, save_dir, global_step=step)
traincnt += numtrain
trainloss += _trainloss * numtrain
elapsed_time = time.time() - start_time
train_time += elapsed_time
evalacc, evalcost, evalcnt = 0., 0., 0
for eva, evalnum in eval_get_batch_data(
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len, 2000, 1.0, 1.0, evaldata,
False):
_evalloss, _evalacc = sess.run([loss, acc_num],
feed_dict=eva)
evalacc += _evalacc
evalcost += _evalloss * evalnum
evalcnt += evalnum
acc, cost, cnt = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_y,
te_target_word, te_tar_len,
2000, 1.0, 1.0, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_loss, _acc, _fw, _bw, _tl, _tr, _ty, _py, _p = sess.run(
[
loss, acc_num, alpha_fw, alpha_bw, alpha_t_l,
alpha_t_r, true_y, pred_y, prob
],
feed_dict=test)
fw += list(_fw)
bw += list(_bw)
tl += list(_tl)
tr += list(_tr)
else:
_loss, _acc, _ty, _py, _p, _fw, _bw, _tl, _tr = sess.run(
[
loss, acc_num, true_y, pred_y, prob, alpha_fw,
alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
p = np.asarray(_p)
fw = np.asarray(_fw)
bw = np.asarray(_bw)
tl = np.asarray(_tl)
tr = np.asarray(_tr)
acc += _acc
cost += _loss * num
cnt += num
comacc, comcnt = 0., 0
for com, comnum in get_batch_data(tr_x, tr_sen_len, tr_x_bw,
tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len,
FLAGS.batch_size, 1.0, 1.0,
False):
_comloss, _comacc, _cty, _cpy, _cp, _cfw, _cbw, _ctl, _ctr = sess.run(
[
loss, acc_num, true_y, pred_y, prob, alpha_fw,
alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=com)
comacc += _comacc
comcnt += comnum
print(
'all samples={}, correct prediction={}, training time={}, training time so far={}'
.format(cnt, acc, elapsed_time, train_time))
trainacc = trainacc / traincnt
acc = acc / cnt
evalacc = evalacc / evalcnt
comacc = comacc / comcnt
alltrainacc.append(comacc)
totalacc = ((acc * remaining_size) +
(accuracyOnt *
(test_size - remaining_size))) / test_size
cost = cost / cnt
trainloss = trainloss / traincnt
evalcost = evalcost / evalcnt
cost_func_test.append(cost)
cost_func_train.append(trainloss)
cost_func_eval.append(evalcost)
acc_func_test.append(acc)
acc_func_train.append(trainacc)
acc_func_eval.append(evalacc)
print(
'Iter {}: mini-batch loss validation set={:.6f}, train loss={:.6f}, train acc={:.6f}, '
'validation acc={:6f} test acc={:.6f}, total train acc={:6f}'
.format(i, evalcost, trainloss, trainacc, evalacc, acc,
comacc))
summary = sess.run(test_summary_op,
feed_dict={
test_loss: cost,
test_acc: acc
})
test_summary_writer.add_summary(summary, step)
all_evalloss.append(evalcost)
all_evalacc.append(evalacc)
if i > 1: # want to compare current validation accuracy with val acc previous iterations
if (all_evalacc[i] - all_evalacc[i - 1] < 0.001) and (
all_evalacc[i - 1] - all_evalacc[i - 2] < 0.001) \
and (all_evalacc[i - 2] - all_evalacc[i - 3] < 0.001):
converged = True
# if (all_evalloss[i] - all_evalloss[i-1] > 0.00001) and (all_evalloss[i-1] - all_evalloss[i-2] > 0.00001) \
# and (all_evalloss[i-2] - all_evalloss[i-3] > 0.00001):
# converged = True
if bucket_number == num_buckets:
if evalcost < lowest_val:
lowest_val = evalcost
best_test = acc
best_train = comacc
best_iteration = i
i += 1
total_iter += 1
bucket_number += 1
# Plotting chart of training and testing losses as a function of iterations
iterations = list(range(total_iter))
plt.plot(iterations, cost_func_train, label='Cost func train')
plt.plot(iterations, cost_func_test, label='Cost func test')
plt.plot(iterations, cost_func_eval, label='Cost func validation')
plt.title('Model loss k={}'.format(num_buckets))
plt.ylabel('Loss')
plt.xlabel('Iterations')
plt.legend(['train', 'test', 'eval'], loc='upper left')
plt.show()
# Plotting chart of training and testing accuracies as a function of iterations
iterations = list(range(total_iter))
plt.plot(iterations, acc_func_train, label='Acc func train')
plt.plot(iterations, acc_func_test, label='Cost func test')
plt.plot(iterations, acc_func_eval, label='Acc func validation')
plt.title('Model accuracy k={}'.format(num_buckets))
plt.ylabel('Accuracy')
plt.xlabel('Iterations')
plt.legend(['train', 'test', 'eval'], loc='upper left')
plt.show()
print(
'Optimization Finished! Iteration:{}: Validation loss={}, validation accuracy={}, test accuracy={}, in-sample acc={}'
.format(total_iter, evalcost, evalacc, acc, comacc))
print(
"Lowest validation loss:{}, at iteration:{}, with out-of-sample acc:{} and in-sample acc:{}"
.format(lowest_val, best_iteration, best_test, best_train))
print('iter_num={}, batch_size={}, hidden_num={}, l2={}'.format(
total_iter, FLAGS.batch_size, FLAGS.n_hidden, l2))
print(acc_func_train)
print(acc_func_test)
print(acc_func_eval)
print(cost_func_eval)
print(alltrainacc)
return acc
def main(train_path,
test_path,
accuracyOnt,
test_size,
remaining_size,
learning_rate=0.09,
keep_prob=0.3,
momentum=0.85,
l2=0.0001):
# print_config()
with tf.device('/gpu:1'):
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path,
FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
y = tf.placeholder(tf.float32, [None, FLAGS.n_class])
sen_len = tf.placeholder(tf.int32, None)
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
sen_len_bw = tf.placeholder(tf.int32, [None])
target_words = tf.placeholder(tf.int32,
[None, FLAGS.max_target_len])
tar_len = tf.placeholder(tf.int32, [None])
inputs_fw = tf.nn.embedding_lookup(word_embedding, x)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
alpha_fw, alpha_bw = None, None
prob, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = lcr_rot(
inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len,
keep_prob1, keep_prob2, l2, 'all')
loss = loss_func(y, prob)
acc_num, acc_prob = acc_func(y, prob)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=momentum).minimize(
loss,
global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
true_y = tf.argmax(y, 1)
pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
FLAGS.learning_rate, FLAGS.l2_reg, FLAGS.max_sentence_len,
FLAGS.embedding_dim, FLAGS.n_hidden, FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
import time
timestamp = str(int(time.time()))
_dir = 'summary/' + str(timestamp) + '_' + title
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
train_summary_op, test_summary_op, validate_summary_op, train_summary_writer, test_summary_writer, \
validate_summary_writer = summary_func(loss, acc_prob, test_loss, test_acc, _dir, title, sess)
save_dir = 'temp_model/' + str(timestamp) + '_' + title + '/'
# saver = saver_func(save_dir)
sess.run(tf.global_variables_initializer())
# saver.restore(sess, '/-')
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, _, _, _ = load_inputs_twitter(
train_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, _, _, _ = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
max_acc = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
train_time = 0
max_time = 0
cost_func_test = []
cost_func_train = []
acc_func_train = []
acc_func_test = []
i = 0
converged = False
all_testloss = []
all_evalacc = []
max_testloss = 100
while i < FLAGS.n_iter and converged == False:
trainacc, trainloss, traincnt = 0., 0., 0
start_time = time.time()
for train, numtrain in get_batch_data(tr_x, tr_sen_len, tr_x_bw,
tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len,
FLAGS.batch_size, keep_prob,
keep_prob):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, _trainloss, step, summary, _trainacc = sess.run(
[optimizer, loss, global_step, train_summary_op, acc_num],
feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc += _trainacc # saver.save(sess, save_dir, global_step=step)
traincnt += numtrain
trainloss += _trainloss * numtrain
elapsed_time = time.time() - start_time
train_time += elapsed_time
acc, cost, cnt = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_y,
te_target_word, te_tar_len, 2000,
1.0, 1.0, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_loss, _acc, _fw, _bw, _tl, _tr, _ty, _py, _p = sess.run(
[
loss, acc_num, alpha_fw, alpha_bw, alpha_t_l,
alpha_t_r, true_y, pred_y, prob
],
feed_dict=test)
fw += list(_fw)
bw += list(_bw)
tl += list(_tl)
tr += list(_tr)
else:
_loss, _acc, _ty, _py, _p, _fw, _bw, _tl, _tr = sess.run(
[
loss, acc_num, true_y, pred_y, prob, alpha_fw,
alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
p = np.asarray(_p)
fw = np.asarray(_fw)
bw = np.asarray(_bw)
tl = np.asarray(_tl)
tr = np.asarray(_tr)
acc += _acc
cost += _loss * num
cnt += num
print('training samples= {}'.format(traincnt))
print(
'all samples={}, correct prediction={}, training time={}, training time so far={}'
.format(cnt, acc, elapsed_time, train_time))
trainacc = trainacc / traincnt
acc = acc / cnt
totalacc = ((acc * remaining_size) +
(accuracyOnt *
(test_size - remaining_size))) / test_size
cost = cost / cnt
trainloss = trainloss / traincnt
cost_func_test.append(cost)
cost_func_train.append(trainloss)
acc_func_test.append(acc)
acc_func_train.append(trainacc)
print(
'Iter {}: mini-batch loss={:.6f}, train loss={:.6f}, train acc={:.6f}, test acc={:.6f}'
.format(i, cost, trainloss, trainacc, acc))
summary = sess.run(test_summary_op,
feed_dict={
test_loss: cost,
test_acc: acc
})
test_summary_writer.add_summary(summary, step)
all_testloss.append(cost)
all_evalacc.append(acc)
if i > 2:
if (all_testloss[i] - all_testloss[i - 1] > 0.00001) and (
all_testloss[i - 1] - all_testloss[i - 2] > 0.00001) \
and (all_testloss[i - 2] - all_testloss[i - 3] > 0.00001):
converged = True
if np.isnan(cost):
acc = 0
converged = True
#if i > 2: # want to compare current validation accuracy with val acc previous iterations
# if (all_evalacc[i] - all_evalacc[i - 1] < 0.001) and (
# all_evalacc[i - 1] - all_evalacc[i - 2] < 0.001) \
# and (all_evalacc[i - 2] - all_evalacc[i - 3] < 0.001):
#converged = True
#if np.isnan(cost):
# converged = True
if cost < max_testloss:
max_testloss = cost
max_testacc = acc
max_iter = i
i += 1
if i == FLAGS.n_iter: # want niet geconvergeerd in n iteraties
acc = 0
# if acc > max_acc:
# max_acc = acc
# max_fw = fw
# max_bw = bw
# max_tl = tl
# max_tr = tr
# max_ty = ty
# max_py = py
# max_prob = p
# P = precision_score(max_ty, max_py, average=None)
# R = recall_score(max_ty, max_py, average=None)
# F1 = f1_score(max_ty, max_py, average=None)
# print('P:', P, 'avg=', sum(P) / FLAGS.n_class)
# print('R:', R, 'avg=', sum(R) / FLAGS.n_class)
# print('F1:', F1, 'avg=', sum(F1) / FLAGS.n_class)
# Plotting chart of training and testing loss as a function of iterations
# iterations = list(range(i))
# plt.plot(iterations, cost_func_train, label='Cost func train')
# plt.plot(iterations, cost_func_test, label='Cost func test')
# plt.title('Model loss k=1')
# plt.ylabel('Loss')
# plt.xlabel('Iterations')
# plt.legend(['train', 'test'], loc='upper left')
# plt.show()
# Plotting chart of training and testing accuracies as a function of iterations
# iterations = list(range(i))
# plt.plot(iterations, acc_func_train, label='Acc func train')
# plt.plot(iterations, acc_func_test, label='Cost func test')
# plt.title('Model accuracy k=1')
# plt.ylabel('Loss')
# plt.xlabel('Iterations')
# plt.legend(['train', 'test'], loc='upper left')
# plt.show()
print(
'Optimization Finished! Iteration:{}:Minimal test loss={}, test accuracy={}'
.format(max_iter, max_testloss, max_testacc))
# nu alleen kijken naar laatste iteratie! Laatste testacc wil je zo laag mogelijk uiteindelijk (als je hyperoptimized)
return acc
def main(train_path, test_path, accuracyOnt, test_size, remaining_size,
learning_rate_dis, learning_rate_gen, keep_prob, momentum_dis,
momentum_gen, l2, k, WriteFile):
print_config()
with tf.device('/gpu:1'):
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path,
FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
x_real = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
y_real = tf.placeholder(tf.float32, [None, FLAGS.n_class])
sen_len = tf.placeholder(tf.int32, None)
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
sen_len_bw = tf.placeholder(tf.int32, [None])
target_words = tf.placeholder(tf.int32,
[None, FLAGS.max_target_len])
tar_len = tf.placeholder(tf.int32, [None])
inputs_fw = tf.nn.embedding_lookup(word_embedding, x_real)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
l, r, t_l, t_r, l2, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = lcr_rot(
inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len,
keep_prob1, keep_prob2, l2, 'all')
gen_l, gen_r, gen_t_l, gen_t_r = generator(l2)
with tf.variable_scope(
"var_D", reuse=tf.AUTO_REUSE
) as scope: #re-using the discriminator parameters since it is called twice per iter
#Calculating prob for real data
prob_real = discriminator(l, r, t_l, t_r, keep_prob2, l2)
#Calculating prob for generated data
prob_generated = discriminator(gen_l, gen_r, gen_t_l, gen_t_r,
keep_prob2, l2)
loss = loss_func_adversarial(prob_real, prob_generated, y_real)
acc_num_real, acc_prob_real, acc_num_gen, acc_prob_gen = acc_func_adversarial(
prob_real, prob_generated, y_real)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
#set variable lists
var_list_D = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='var_D')
var_list_G = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='var_G')
#As we solve a min max problem, we optimize twice with respect to different variable sets , var_list = var_D , var_list = var_G
opti_min = tf.train.MomentumOptimizer(learning_rate=learning_rate_dis,
momentum=momentum_dis).minimize(
loss,
var_list=var_list_D,
global_step=global_step)
opti_max = tf.train.MomentumOptimizer(learning_rate=learning_rate_gen,
momentum=momentum_gen).minimize(
-loss, var_list=var_list_G)
true_y = tf.argmax(y_real, 1)
pred_y = tf.argmax(prob_real, 1)
title = '-d1-{}d2-{}b-{}rd-{}rg-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
learning_rate_dis, learning_rate_gen, FLAGS.l2_reg,
FLAGS.max_sentence_len, FLAGS.embedding_dim, FLAGS.n_hidden,
FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
import time
timestamp = str(int(time.time()))
_dir = 'summary/' + str(timestamp) + '_' + title
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
train_summary_op, test_summary_op, validate_summary_op, train_summary_writer, test_summary_writer, \
validate_summary_writer = summary_func_adversarial(loss, acc_prob_real, acc_prob_gen, test_loss, test_acc, _dir, title, sess)
save_dir = 'temp_model/' + str(timestamp) + '_' + title + '/'
# saver = saver_func(save_dir)
# saver.restore(sess, '/-')
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, _, _, _ = load_inputs_twitter(
train_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, _, _, _ = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x_real: x_f[index],
x_bw: x_b[index],
y_real: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
max_acc = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
Results_File = np.zeros(
(5, 1)
) #6 = number of rows / values to store:['Iteration','loss','trainacc_real','test_acc','avg prob assigned to correct generated']
for i in range(1, FLAGS.n_iter + 1):
avg_p_real = None
avg_p_gen = None
#update D more often than G
if k >= 1:
if i % k == 0:
print('In iter ' + str(i) + ' we update both G and D.')
trainacc_real, trainacc_gen, traincnt = 0., 0., 0
for train, numtrain in get_batch_data(
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len, FLAGS.batch_size,
keep_prob, keep_prob):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, _, step, summary, _trainacc_real, _trainacc_gen = sess.run(
[
opti_max, opti_min, global_step,
train_summary_op, acc_num_real, acc_num_gen
],
feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc_real += _trainacc_real # saver.save(sess, save_dir, global_step=step)
trainacc_gen += _trainacc_gen
traincnt += numtrain
else:
print('In iter ' + str(i) + ' we update only D.')
trainacc_real, trainacc_gen, traincnt = 0., 0., 0
for train, numtrain in get_batch_data(
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len, FLAGS.batch_size,
keep_prob, keep_prob):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, step, summary, _trainacc_real, _trainacc_gen = sess.run(
[
opti_min, global_step, train_summary_op,
acc_num_real, acc_num_gen
],
feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc_real += _trainacc_real # saver.save(sess, save_dir, global_step=step)
trainacc_gen += _trainacc_gen
traincnt += numtrain
#Update G more often than D
else:
k_inv = 1 / k
if i % k_inv == 0:
print('In iter ' + str(i) + ' we update both G and D.')
trainacc_real, trainacc_gen, traincnt = 0., 0., 0
for train, numtrain in get_batch_data(
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len, FLAGS.batch_size,
keep_prob, keep_prob):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, _, step, summary, _trainacc_real, _trainacc_gen = sess.run(
[
opti_max, opti_min, global_step,
train_summary_op, acc_num_real, acc_num_gen
],
feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc_real += _trainacc_real # saver.save(sess, save_dir, global_step=step)
trainacc_gen += _trainacc_gen
traincnt += numtrain
else:
print('In iter ' + str(i) + ' we update only G.')
trainacc_real, trainacc_gen, traincnt = 0., 0., 0
for train, numtrain in get_batch_data(
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len, FLAGS.batch_size,
keep_prob, keep_prob):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, step, summary, _trainacc_real, _trainacc_gen = sess.run(
[
opti_max, global_step, train_summary_op,
acc_num_real, acc_num_gen
],
feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc_real += _trainacc_real # saver.save(sess, save_dir, global_step=step)
trainacc_gen += _trainacc_gen
traincnt += numtrain
#Testing occurs in every iteration, regardless of what networks have been updated.
acc, cost, cnt = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_y,
te_target_word, te_tar_len, 2000,
1.0, 1.0, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_loss, _acc, _fw, _bw, _tl, _tr, _ty, _py, _p = sess.run(
[
loss, acc_num, alpha_fw, alpha_bw, alpha_t_l,
alpha_t_r, true_y, pred_y, prob_real
],
feed_dict=test)
fw += list(_fw)
bw += list(_bw)
tl += list(_tl)
tr += list(_tr)
else:
_loss, _acc, _ty, _py, _p, _fw, _bw, _tl, _tr, _p_g, _y_real, _prob_real = sess.run(
[
loss, acc_num_real, true_y, pred_y, prob_real,
alpha_fw, alpha_bw, alpha_t_l, alpha_t_r,
prob_generated, y_real, prob_real
],
feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
p = np.asarray(_p)
fw = np.asarray(_fw)
bw = np.asarray(_bw)
tl = np.asarray(_tl)
tr = np.asarray(_tr)
yr = np.asarray(y_real)
acc += _acc
cost += _loss * num
cnt += num
p_g = np.asarray(_p_g)
print('all samples={}, correct prediction={}'.format(cnt, acc))
trainacc_real = trainacc_real / traincnt
trainacc_gen = trainacc_gen / traincnt
acc = acc / cnt
totalacc = ((acc * remaining_size) +
(accuracyOnt *
(test_size - remaining_size))) / test_size
cost = cost / cnt
print(
'Iter {}: mini-batch loss={:.6f}, train acc real ={:.6f}, test acc={:.6f}, combined acc={:.6f}'
.format(i, cost, trainacc_real, acc, totalacc))
summary = sess.run(test_summary_op,
feed_dict={
test_loss: cost,
test_acc: acc
})
test_summary_writer.add_summary(summary, step)
if acc > max_acc:
max_acc = acc
max_fw = fw
max_bw = bw
max_tl = tl
max_tr = tr
max_ty = ty
max_py = py
max_prob = p
#Writing File
if WriteFile:
avg_p_real = np.mean(
np.multiply(_prob_real, _y_real)
) #average probability assigned to the correct class for real data
avg_p_gen = np.mean(
p_g, axis=0
)[3] #average probability assigned to the correct class for generated data
Added = [[i], [cost], [trainacc_real], [acc], [avg_p_gen]]
Results_File = np.concatenate((Results_File, Added), 1)
if np.isnan(_loss):
print('Ohw shit we obtained an NaN bro!!')
max_acc = max_acc * (
(i / 200)**2
) #Uncomment this line for hyperpar optim, to penalize
break
P = precision_score(max_ty, max_py, average=None)
R = recall_score(max_ty, max_py, average=None)
F1 = f1_score(max_ty, max_py, average=None)
print('P:', P, 'avg=', sum(P) / FLAGS.n_class)
print('R:', R, 'avg=', sum(R) / FLAGS.n_class)
print('F1:', F1, 'avg=', sum(F1) / FLAGS.n_class)
fp = open(FLAGS.prob_file, 'w')
for item in max_prob:
fp.write(' '.join([str(it) for it in item]) + '\n')
fp = open(FLAGS.prob_file + '_fw', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_fw):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_bw', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_bw):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_tl', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_tl):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_tr', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_tr):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
print('Optimization Finished! Max acc={}'.format(max_acc))
print(
'Learning_rate_dis={},Learning_rate_gen={}, momentum_dis={},momentum_gen={}, iter_num={}, batch_size={}, hidden_num={}, l2={},k={}'
.format(learning_rate_dis, learning_rate_gen, momentum_dis,
momentum_gen, FLAGS.n_iter, FLAGS.batch_size,
FLAGS.n_hidden, FLAGS.l2_reg, k))
if WriteFile:
#Saving training information as csv file
dateTimeObj = datetime.now()
save_dir = '/Results_Run_Adversarial/Run_' + str(
dateTimeObj) + '_lr' + str(learning_rate_dis) + '_lrg' + str(
learning_rate_gen) + '_kp' + str(
keep_prob) + '_mom_d' + str(
momentum_dis) + '_mom_g' + str(
momentum_gen) + '_k' + str(k) + '.csv'
np.savetxt(save_dir, Results_File, delimiter=",")
return max_acc, np.where(np.subtract(max_py, max_ty) == 0, 0,
1), max_fw.tolist(), max_bw.tolist(
), max_tl.tolist(), max_tr.tolist()
if __name__ == '__main__':
tf.app.run()
def main(train_path,
test_path,
accuracyOnt,
trainAccuracyOnt,
test_size,
remaining_size,
learning_rate=FLAGS.learning_rate,
keep_prob=FLAGS.keep_prob1,
momentum=0.9,
l2=FLAGS.l2_reg,
number_of_heads=FLAGS.heads):
print_config()
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path, FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
y = tf.placeholder(tf.float32, [None, FLAGS.n_class])
sen_len = tf.placeholder(tf.int32, None)
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
sen_len_bw = tf.placeholder(tf.int32, [None])
target_words = tf.placeholder(tf.int32, [None, FLAGS.max_target_len])
tar_len = tf.placeholder(tf.int32, [None])
inputs_fw = tf.nn.embedding_lookup(word_embedding, x)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
alpha_fw, alpha_bw = None, None
prob, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = lcr_rot(
inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len, keep_prob1,
keep_prob2, l2, 'all', number_of_heads)
loss = loss_func(y, prob)
acc_num, acc_prob = acc_func(y, prob)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
optimizer = tf.train.MomentumOptimizer(learning_rate=FLAGS.learning_rate,
momentum=momentum).minimize(
loss, global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
true_y = tf.argmax(y, 1)
pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
FLAGS.learning_rate, FLAGS.l2_reg, FLAGS.max_sentence_len,
FLAGS.embedding_dim, FLAGS.n_hidden, FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
sess.run(tf.global_variables_initializer())
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, _, _, _ = load_inputs_twitter(
train_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, _, _, _ = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x_ont, te_sen_len_ont, te_x_bw_ont, te_sen_len_bw_ont, te_y_ont, te_target_word_ont, te_tar_len_ont, _, _, _ = load_inputs_twitter(
FLAGS.remaining_test_path, word_id_mapping, FLAGS.max_sentence_len,
'TC', is_r, FLAGS.max_target_len)
tr_x_ont, tr_sen_len_ont, tr_x_bw_ont, tr_sen_len_bw_ont, tr_y_ont, tr_target_word_ont, tr_tar_len_ont, _, _, _ = load_inputs_twitter(
FLAGS.remaining_train_path, word_id_mapping,
FLAGS.max_sentence_len, 'TC', is_r, FLAGS.max_target_len)
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
max_acc = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
all_training_losses, all_training_accuracies = [], []
all_test_losses, all_test_accuracies = [], []
for i in range(FLAGS.n_iter):
learning_rate = (0.99) * learning_rate
number_of_training_examples_correct, number_of_training_examples, training_loss = 0., 0, 0.
for train, numtrain in get_batch_data(tr_x, tr_sen_len, tr_x_bw,
tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len,
FLAGS.batch_size, keep_prob,
keep_prob):
_, step, _trainacc, _training_loss = sess.run(
[optimizer, global_step, acc_num, loss], feed_dict=train)
number_of_training_examples_correct += _trainacc
number_of_training_examples += numtrain
training_loss += _training_loss * numtrain
number_of_test_examples_correct, test_loss, number_of_test_examples = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_y,
te_target_word, te_tar_len, 2000,
1.0, 1.0, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_loss, _acc, _fw, _bw, _tl, _tr, _ty, _py, _p = sess.run(
[
loss, acc_num, alpha_fw, alpha_bw, alpha_t_l,
alpha_t_r, true_y, pred_y, prob
],
feed_dict=test)
# fw += list(_fw)
# bw += list(_bw)
# tl += list(_tl)
# tr += list(_tr)
else:
_loss, _acc, _ty, _py, _p, _fw, _bw, _tl, _tr = sess.run(
[
loss, acc_num, true_y, pred_y, prob, alpha_fw,
alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
p = np.asarray(_p)
fw = np.asarray(_fw)
bw = np.asarray(_bw)
tl = np.asarray(_tl)
tr = np.asarray(_tr)
number_of_test_examples_correct += _acc
test_loss += _loss * num
number_of_test_examples += num
number_of_test_examples_correct_ont, number_of_test_examples_ont = 0., 0
for test_ont, num_ont in get_batch_data(
te_x_ont, te_sen_len_ont, te_x_bw_ont, te_sen_len_bw_ont,
te_y_ont, te_target_word_ont, te_tar_len_ont, 2000, 1.0,
1.0, False):
_acc_ont = sess.run(acc_num, feed_dict=test_ont)
number_of_test_examples_correct_ont += _acc_ont
number_of_test_examples_ont += num_ont
number_of_train_examples_correct_ont, number_of_train_examples_ont = 0., 0
for train_ont, num_train_ont in get_batch_data(
tr_x_ont, tr_sen_len_ont, tr_x_bw_ont, tr_sen_len_bw_ont,
tr_y_ont, tr_target_word_ont, tr_tar_len_ont, 2000, 1.0,
1.0, False):
_acc_ont_train = sess.run(acc_num, feed_dict=train_ont)
number_of_train_examples_correct_ont += _acc_ont_train
number_of_train_examples_ont += num_train_ont
print(
'number of training examples={}, correct training examples={}, number of test examples={}, correct test examples={}, number of examples without onto = {}'
.format(number_of_training_examples,
number_of_training_examples_correct,
number_of_test_examples,
number_of_test_examples_correct,
number_of_test_examples_ont))
training_accuracy = number_of_training_examples_correct / number_of_training_examples
test_accuracy = number_of_test_examples_correct / number_of_test_examples
test_accuracy_ont = number_of_test_examples_correct_ont / number_of_test_examples_ont
train_accuracy_ont = number_of_train_examples_correct_ont / number_of_train_examples_ont
totalacc_train = (
(train_accuracy_ont * number_of_train_examples_ont) +
(trainAccuracyOnt *
(number_of_training_examples - number_of_train_examples_ont))
) / number_of_training_examples
totalacc = (
(test_accuracy_ont * number_of_test_examples_ont) +
(accuracyOnt *
(number_of_test_examples - number_of_test_examples_ont))
) / number_of_test_examples
average_test_loss = test_loss / number_of_test_examples
average_training_loss = training_loss / number_of_training_examples
print(
'Epoch {}: average training loss={:.6f}, train acc={:.6f}, average test loss={:.6f}, test acc={:.6f}, combined acc={:.6f}, accuracy without onto={:.6f}, in-sample with onto = {}'
.format(i, average_training_loss, training_accuracy,
average_test_loss, test_accuracy, totalacc,
test_accuracy_ont, totalacc_train))
# max_acc = test_accuracy
# max_fw = np.average(np.abs(fw), axis=2)
# max_bw = np.average(np.abs(bw), axis=2)
# max_tl = np.average(np.abs(tl), axis=2)
# max_tr = np.average(np.abs(tr), axis=2)
# max_ty = ty
# max_py = py
# max_prob = p
max_acc = test_accuracy
max_fw = np.squeeze(fw)
max_bw = np.squeeze(bw)
max_tl = np.squeeze(tl)
max_tr = np.squeeze(tr)
max_ty = ty
max_py = py
max_prob = p
# print(max_fw)
# print(np.shape(max_fw))
# w1 = tf.get_variable("head_w_hiddenstl0")
# print(w1.eval(session=sess))
# fp = open(FLAGS.prob_file + '_multihead' + str(FLAGS.year) + '.txt', 'w')
# for y1, y2, item in zip(max_ty, max_py, max_prob):
# fp.write(str(y1) + ' ' + str(y2) + ' ' + ' '.join([str(it) for it in item]) + '\n')
#
# with open(FLAGS.prob_file + '_fw_multihead' + str(FLAGS.year) + '.txt', 'w') as outfile:
# np.savetxt(outfile, max_fw)
#
# with open(FLAGS.prob_file + '_bw_multihead' + str(FLAGS.year) + '.txt', 'w') as outfile:
# np.savetxt(outfile, max_bw)
#
# with open(FLAGS.prob_file + '_tl_multihead' + str(FLAGS.year) + '.txt', 'w') as outfile:
# np.savetxt(outfile, max_tl)
#
# with open(FLAGS.prob_file + '_tr_multihead' + str(FLAGS.year) + '.txt', 'w') as outfile:
# np.savetxt(outfile, max_tr)
print('Optimization Finished! Max acc={}'.format(max_acc))
print(
'Learning_rate={}, iter_num={}, batch_size={}, hidden_num={}, l2={}'
.format(FLAGS.learning_rate, FLAGS.n_iter, FLAGS.batch_size,
FLAGS.n_hidden, FLAGS.l2_reg))
return training_accuracy, max_acc, totalacc_train, totalacc, test_accuracy_ont, np.where(
np.subtract(max_py, max_ty) == 0, 0, 1), max_fw.tolist(
), max_bw.tolist(), max_tl.tolist(), max_tr.tolist()
def main(train_path,
test_path,
accuracyOnt,
test_size,
remaining_size,
learning_rate=0.09,
keep_prob=0.3,
momentum=0.85,
l2=0.00001):
#print_config()
with tf.device('/gpu:1'):
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path,
FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
y = tf.placeholder(tf.float32, [None, FLAGS.n_class])
sen_len = tf.placeholder(tf.int32, None)
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
sen_len_bw = tf.placeholder(tf.int32, [None])
target_words = tf.placeholder(tf.int32,
[None, FLAGS.max_target_len])
tar_len = tf.placeholder(tf.int32, [None])
inputs_fw = tf.nn.embedding_lookup(word_embedding, x)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
alpha_fw, alpha_bw = None, None
prob, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = lcr_rot(
inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len,
keep_prob1, keep_prob2, l2, 'all')
loss = loss_func(y, prob)
acc_num, acc_prob = acc_func(y, prob)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=momentum).minimize(
loss,
global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
true_y = tf.argmax(y, 1)
pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
FLAGS.learning_rate, FLAGS.l2_reg, FLAGS.max_sentence_len,
FLAGS.embedding_dim, FLAGS.n_hidden, FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
import time
timestamp = str(int(time.time()))
_dir = 'summary/' + str(timestamp) + '_' + title
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
train_summary_op, test_summary_op, validate_summary_op, train_summary_writer, test_summary_writer, \
validate_summary_writer = summary_func(loss, acc_prob, test_loss, test_acc, _dir, title, sess)
# save_dir = 'temp_model/' + str(timestamp) + '_' + title + '/'
# saver = saver_func(save_dir)
sess.run(tf.global_variables_initializer())
# saver.restore(sess, '/-')
restore_pth = 'savedModel' + str(FLAGS.year)
meta = '-2444'
if restore_pth is not None and meta is not None:
restore_path = restore_pth + '/'
restore_meta_path = restore_pth + '/' + meta + '.meta'
restore = tf.train.import_meta_graph(restore_meta_path)
restore.restore(sess, tf.train.latest_checkpoint(restore_path))
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, _, _, _ = load_inputs_twitter(
train_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, _, _, _ = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
max_acc = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
train_time = 0
max_time = 0
print("number of training instances: {}, number of test instances: {}".
format(len(tr_y), len(te_y)))
cost_func_test = []
cost_func_train = []
acc_func_train = []
acc_func_test = []
i = 0
converged = False
all_trainacc = []
while i < FLAGS.n_iter and converged == False:
trainacc, trainloss, traincnt = 0., 0., 1
start_time = time.time()
#for train, numtrain in get_batch_data(tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word,
# tr_tar_len,
# FLAGS.batch_size, keep_prob, keep_prob):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
# _, _trainloss, step, summary, _trainacc = sess.run([optimizer, loss, global_step, train_summary_op, acc_num],
# feed_dict=train)
#train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
#trainacc += _trainacc # saver.save(sess, save_dir, global_step=step)
#traincnt += numtrain
#trainloss += _trainloss * numtrain
elapsed_time = time.time() - start_time
train_time += elapsed_time
acc, cost, cnt = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_y,
te_target_word, te_tar_len, 2000,
1.0, 1.0, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_loss, _acc, _fw, _bw, _tl, _tr, _ty, _py, _p = sess.run(
[
loss, acc_num, alpha_fw, alpha_bw, alpha_t_l,
alpha_t_r, true_y, pred_y, prob
],
feed_dict=test)
fw += list(_fw)
bw += list(_bw)
tl += list(_tl)
tr += list(_tr)
else:
_loss, _acc, _ty, _py, _p, _fw, _bw, _tl, _tr = sess.run(
[
loss, acc_num, true_y, pred_y, prob, alpha_fw,
alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
p = np.asarray(_p)
fw = np.asarray(_fw)
bw = np.asarray(_bw)
tl = np.asarray(_tl)
tr = np.asarray(_tr)
acc += _acc
cost += _loss * num
cnt += num
print(
'all samples={}, correct prediction={}, training time={}, training time so far={}'
.format(cnt, acc, elapsed_time, train_time))
trainacc = trainacc / traincnt
acc = acc / cnt
#totalacc = ((acc * remaining_size) + (accuracyOnt * (test_size - remaining_size))) / test_size
cost = cost / cnt
trainloss = trainloss / traincnt
cost_func_test.append(cost)
cost_func_train.append(trainloss)
acc_func_test.append(acc)
acc_func_train.append(trainacc)
print(
'Iter {}: mini-batch loss={:.6f}, train loss={:.6f}, train acc={:.6f}, test acc={:.6f}'
.format(i, cost, trainloss, trainacc, acc))
summary = sess.run(test_summary_op,
feed_dict={
test_loss: cost,
test_acc: acc
})
test_summary_writer.add_summary(summary, step)
all_trainacc.append(trainacc)
if i > 2: # want to compare current train accuracy with train acc previous iterations
if (all_trainacc[i] - all_trainacc[i - 1] < 0.001) and (all_trainacc[i - 1] - all_trainacc[i - 2] < 0.001)\
and (all_trainacc[i-2] - all_trainacc[i-3] < 0.001):
converged = True
i += 1
if acc > max_acc:
max_acc = acc
max_fw = fw
max_bw = bw
max_tl = tl
max_tr = tr
max_ty = ty
max_py = py
max_prob = p
P = precision_score(max_ty, max_py, average=None)
R = recall_score(max_ty, max_py, average=None)
F1 = f1_score(max_ty, max_py, average=None)
print('P:', P, 'avg=', sum(P) / FLAGS.n_class)
print('R:', R, 'avg=', sum(R) / FLAGS.n_class)
print('F1:', F1, 'avg=', sum(F1) / FLAGS.n_class)
# Plotting chart of training and testing loss as a function of iterations
iterations = list(range(i))
plt.plot(iterations, cost_func_train, label='Cost func train')
plt.plot(iterations, cost_func_test, label='Cost func test')
plt.title('Model loss k=1')
plt.ylabel('Loss')
plt.xlabel('Iterations')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
# Plotting chart of training and testing accuracies as a function of iterations
iterations = list(range(i))
plt.plot(iterations, acc_func_train, label='Acc func train')
plt.plot(iterations, acc_func_test, label='Cost func test')
plt.title('Model accuracy k=1')
plt.ylabel('Loss')
plt.xlabel('Iterations')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
fp = open(FLAGS.prob_file, 'w')
for item in max_prob:
fp.write(' '.join([str(it) for it in item]) + '\n')
fp = open(FLAGS.prob_file + '_fw', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_fw):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_bw', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_bw):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_tl', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_tl):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
fp = open(FLAGS.prob_file + '_tr', 'w')
for y1, y2, ws in zip(max_ty, max_py, max_tr):
fp.write(
str(y1) + ' ' + str(y2) + ' ' +
' '.join([str(w) for w in ws[0]]) + '\n')
print('Optimization Finished! Final acc={}'.format(acc))
print(
'Learning_rate={}, iter_num={}, batch_size={}, hidden_num={}, l2={}'
.format(FLAGS.learning_rate, FLAGS.n_iter, FLAGS.batch_size,
FLAGS.n_hidden, FLAGS.l2_reg))
return acc, np.where(np.subtract(max_py, max_ty) == 0, 0, 1), max_fw.tolist(), max_bw.tolist(), \
max_tl.tolist(), max_tr.tolist()
def main(train_path,
test_path,
accuracyOnt,
test_size,
remaining_size,
learning_rate=0.09,
keep_prob=0.5,
momentum=0.9,
l2=0.0001):
print_config()
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path, FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.placeholder(tf.float32)
keep_prob2 = tf.placeholder(tf.float32)
with tf.name_scope('inputs'):
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
y = tf.placeholder(tf.float32, [None, FLAGS.n_class])
sen_len = tf.placeholder(tf.int32, None)
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len])
sen_len_bw = tf.placeholder(tf.int32, [None])
target_words = tf.placeholder(tf.int32, [None, FLAGS.max_target_len])
tar_len = tf.placeholder(tf.int32, [None])
inputs_fw = tf.nn.embedding_lookup(word_embedding, x)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
alpha_fw, alpha_bw = None, None
attention, attention_masked = lcr_rot(inputs_fw, inputs_bw, sen_len,
sen_len_bw, target, tar_len,
keep_prob1, keep_prob2, l2, 'all')
# loss = loss_func(y, prob)
# acc_num, acc_prob = acc_func(y, prob)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
# optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate, momentum=momentum).minimize(loss,
# global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
true_y = tf.argmax(y, 1)
# pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
FLAGS.learning_rate, FLAGS.l2_reg, FLAGS.max_sentence_len,
FLAGS.embedding_dim, FLAGS.n_hidden, FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
sess.run(tf.global_variables_initializer())
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_target_word, tr_tar_len, _, _, _ = load_inputs_twitter(
train_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_target_word, te_tar_len, _, _, _ = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
te_x_ont, te_sen_len_ont, te_x_bw_ont, te_sen_len_bw_ont, te_y_ont, te_target_word_ont, te_tar_len_ont, _, _, _ = load_inputs_twitter(
FLAGS.remaining_test_path, word_id_mapping, FLAGS.max_sentence_len,
'TC', is_r, FLAGS.max_target_len)
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
yi,
target,
tl,
batch_size,
kp1,
kp2,
is_shuffle=True):
for index in batch_index(len(yi), batch_size, 1, is_shuffle):
feed_dict = {
x: x_f[index],
x_bw: x_b[index],
y: yi[index],
sen_len: sen_len_f[index],
sen_len_bw: sen_len_b[index],
target_words: target[index],
tar_len: tl[index],
keep_prob1: kp1,
keep_prob2: kp2,
}
yield feed_dict, len(index)
max_acc = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
all_training_losses, all_training_accuracies = [], []
all_test_losses, all_test_accuracies = [], []
for i in range(FLAGS.n_iter):
learning_rate = (0.99) * learning_rate
number_of_training_examples_correct, number_of_training_examples, training_loss = 0., 0, 0.
for train, numtrain in get_batch_data(tr_x, tr_sen_len, tr_x_bw,
tr_sen_len_bw, tr_y,
tr_target_word, tr_tar_len,
FLAGS.batch_size, keep_prob,
keep_prob):
step = sess.run([global_step], feed_dict=train)
# number_of_training_examples_correct += _trainacc
# number_of_training_examples += numtrain
# training_loss += _training_loss * numtrain
number_of_test_examples_correct, test_loss, number_of_test_examples = 0., 0., 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
m = 1
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_y,
te_target_word, te_tar_len, 3, 1.0,
1.0, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_output_old, _output = sess.run([output_old, output],
feed_dict=test)
else:
if m == 1:
_attention, _attention_masked = sess.run(
[attention, attention_masked], feed_dict=test)
m += 1
print(_attention)
print(np.shape(_attention))
print(_attention_masked)
print(np.shape(_attention_masked))
def main(train_path,
test_path,
accuracyOnt,
test_size,
remaining_size,
momentum=0.85):
# print_config()
l2 = FLAGS.l2_reg
learning_rate = FLAGS.learning_rate
with tf.device('/gpu:1'):
word_id_mapping, w2v = load_w2v(FLAGS.embedding_path,
FLAGS.embedding_dim)
word_embedding = tf.constant(w2v, name='word_embedding')
keep_prob1 = tf.constant(FLAGS.keep_prob1, tf.float32)
keep_prob2 = tf.constant(FLAGS.keep_prob2, tf.float32)
lambda_0 = tf.constant(FLAGS.lambda_0, tf.float32)
lambda_1 = tf.constant(1 - FLAGS.lambda_0, tf.float32)
with tf.name_scope('inputs'):
y_sen = tf.placeholder(tf.float32, [None, FLAGS.n_class],
name='y_sentence_level')
n_asp = tf.placeholder(tf.int32, [None], name='n_asp')
x = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len],
name='x')
y = tf.placeholder(tf.float32, [None, FLAGS.n_class], name='y')
sen_len = tf.placeholder(tf.int32, None, name='sentence_length')
x_bw = tf.placeholder(tf.int32, [None, FLAGS.max_sentence_len],
name='x_backwards')
sen_len_bw = tf.placeholder(tf.int32, [None],
name='sentence_length_backwards')
target_words = tf.placeholder(tf.int32,
[None, FLAGS.max_target_len],
name='target_words')
tar_len = tf.placeholder(tf.int32, [None], name='target_length')
inputs_fw = tf.nn.embedding_lookup(word_embedding, x)
inputs_bw = tf.nn.embedding_lookup(word_embedding, x_bw)
target = tf.nn.embedding_lookup(word_embedding, target_words)
alpha_fw, alpha_bw = None, None
prob, prob_sen, _, alpha_fw, alpha_bw, alpha_t_l, alpha_t_r = lcr_rot(
n_asp, inputs_fw, inputs_bw, sen_len, sen_len_bw, target, tar_len,
keep_prob1, keep_prob2, l2, 'all')
loss_asp = loss_func(y, prob)
loss_sen = loss_func(y_sen, prob_sen)
loss = lambda_1 * loss_asp + lambda_0 * loss_sen
acc_num, acc_prob, f1_micro, f1_macro, f1_weighted = acc_func(
y, prob, y_sen, prob_sen, thre=FLAGS.threshold)
global_step = tf.Variable(0, name='tr_global_step', trainable=False)
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=momentum).minimize(
loss,
global_step=global_step)
# optimizer = train_func(loss, FLAGS.learning_rate, global_step)
# true_y = tf.argmax(y, 1)
true_y = y_sen
pred_y = tf.cast(tf.math.greater_equal(prob_sen, [FLAGS.threshold]),
tf.int32)
# pred_y = tf.argmax(prob, 1)
title = '-d1-{}d2-{}b-{}r-{}l2-{}sen-{}dim-{}h-{}c-{}'.format(
FLAGS.keep_prob1, FLAGS.keep_prob2, FLAGS.batch_size,
FLAGS.learning_rate, FLAGS.l2_reg, FLAGS.max_sentence_len,
FLAGS.embedding_dim, FLAGS.n_hidden, FLAGS.n_class)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
import datetime
# timestamp = str(int(time.time()))
timestamp = datetime.datetime.now().isoformat()
_dir = str(timestamp) + '_' + title
test_loss = tf.placeholder(tf.float32)
test_acc = tf.placeholder(tf.float32)
test_f1_micro = tf.placeholder(tf.float32)
train_summary_op, test_summary_op, validate_summary_op, train_summary_writer, test_summary_writer, \
validate_summary_writer = summary_func(loss, acc_prob, f1_micro, test_loss, test_acc, test_f1_micro, _dir, title, sess)
# validate_summary_writer = summary_func(loss, acc_prob, test_loss, test_acc, _dir, title, sess)
save_dir = 'temp_model/' + str(timestamp) + '_' + title + '/'
# saver = saver_func(save_dir)
sess.run(tf.global_variables_initializer())
# saver.restore(sess, '/-')
if FLAGS.is_r == '1':
is_r = True
else:
is_r = False
tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_y, tr_y_sen, tr_target_word, tr_tar_len, _, _, _, tr_n_asp = load_inputs_twitter(
train_path,
word_id_mapping,
FLAGS.max_sentence_len,
'TC',
is_r, # reverse
FLAGS.max_target_len)
te_x, te_sen_len, te_x_bw, te_sen_len_bw, te_y, te_y_sen, te_target_word, te_tar_len, _, _, _, te_n_asp = load_inputs_twitter(
test_path, word_id_mapping, FLAGS.max_sentence_len, 'TC', is_r,
FLAGS.max_target_len)
def get_batch_data(x_f,
sen_len_f,
x_b,
sen_len_b,
n_asp_b,
yi,
y_sen_i,
target,
tl,
batch_size,
is_shuffle=True):
# for index in batch_index(len(yi), batch_size, 1, is_shuffle):
for index in batch_index(len(n_asp_b), batch_size, 1, is_shuffle):
selected_rows = itemgetter(*index)(list(n_asp_b.values()))
r_index = []
for idxs in selected_rows:
if idxs != []:
r_index.extend(idxs)
_n_asp = np.asarray(
[len(tup) for tup in list(selected_rows) if len(tup) != 0])
# print(f"length of _n_asp: {_n_asp.shape[0]}")
feed_dict = {
x: x_f[r_index],
x_bw: x_b[r_index],
y: yi[r_index],
y_sen: y_sen_i[index],
n_asp: _n_asp,
sen_len: sen_len_f[r_index],
sen_len_bw: sen_len_b[r_index],
target_words: target[r_index],
tar_len: tl[r_index]
}
yield feed_dict, len(r_index)
max_acc = 0.
max_f1 = 0.
max_fw, max_bw = None, None
max_tl, max_tr = None, None
max_ty, max_py = None, None
max_prob = None
step = None
for i in range(FLAGS.n_iter):
trainacc, trainf1, traincnt, train_batchcnt = 0., 0., 0, 0
for train, numtrain in get_batch_data(tr_x, tr_sen_len, tr_x_bw, tr_sen_len_bw, tr_n_asp, tr_y, \
tr_y_sen, tr_target_word, tr_tar_len, FLAGS.batch_size):
# _, step = sess.run([optimizer, global_step], feed_dict=train)
_, step, summary, _trainacc, _trainf1 = sess.run(
[
optimizer, global_step, train_summary_op, acc_num,
f1_micro
],
feed_dict=train)
train_summary_writer.add_summary(summary, step)
# embed_update = tf.assign(word_embedding, tf.concat(0, [tf.zeros([1, FLAGS.embedding_dim]), word_embedding[1:]]))
# sess.run(embed_update)
trainacc += _trainacc # saver.save(sess, save_dir, global_step=step)
trainf1 += _trainf1
traincnt += numtrain
train_batchcnt += 1
acc, f1, cost, cnt, test_batchcnt = 0., 0., 0., 0, 0
fw, bw, tl, tr, ty, py = [], [], [], [], [], []
p = []
for test, num in get_batch_data(te_x, te_sen_len, te_x_bw,
te_sen_len_bw, te_n_asp, te_y,
te_y_sen, te_target_word,
te_tar_len, 2000, False):
if FLAGS.method == 'TD-ATT' or FLAGS.method == 'IAN':
_loss, _acc, _fw, _bw, _tl, _tr, _ty, _py, _p = sess.run(
[
loss, acc_num, alpha_fw, alpha_bw, alpha_t_l,
alpha_t_r, true_y, pred_y, prob
],
feed_dict=test)
fw += list(_fw)
bw += list(_bw)
tl += list(_tl)
tr += list(_tr)
else:
_loss, _acc, _f1, _ty, _py, _p, _fw, _bw, _tl, _tr = sess.run(
[
loss, acc_num, f1_micro, true_y, pred_y, prob,
alpha_fw, alpha_bw, alpha_t_l, alpha_t_r
],
feed_dict=test)
ty = np.asarray(_ty)
py = np.asarray(_py)
p = np.asarray(_p)
fw = np.asarray(_fw)
bw = np.asarray(_bw)
tl = np.asarray(_tl)
tr = np.asarray(_tr)
acc += _acc
f1 += _f1
# cost += _loss * num
cost += _loss
cnt += num
test_batchcnt += 1
print('all samples={}, correct prediction={}'.format(cnt, acc))
trainacc = trainacc / traincnt
trainf1 = trainf1 / train_batchcnt
acc = acc / cnt
f1 = f1 / test_batchcnt
totalacc = ((acc * remaining_size) +
(accuracyOnt *
(test_size - remaining_size))) / test_size
# cost = cost / cnt
cost = cost / test_batchcnt
print(
'Iter {}: mini-batch loss={:.6f}, train acc={:.6f}, train_f1_micro={:.6f}, test acc={:.6f}, \
test_f1_micro={:.6f}, combined acc={:.6f}'.format(
i, cost, trainacc, trainf1, acc, f1, totalacc))
summary = sess.run(test_summary_op,
feed_dict={
test_loss: cost,
test_acc: acc,
test_f1_micro: f1
})
test_summary_writer.add_summary(summary, step)
# if acc > max_acc:
if f1 > max_f1:
max_acc = acc
max_f1 = f1
max_fw = fw
max_bw = bw
max_tl = tl
max_tr = tr
max_ty = ty
max_py = py
max_prob = p
# encode training data
train_feed_dict = {
x: tr_x,
x_bw: tr_x_bw,
y: tr_y,
y_sen: tr_y_sen,
n_asp: tr_n_asp,
sen_len: tr_sen_len,
sen_len_bw: tr_sen_len_bw,
target_words: tr_target_word,
tar_len: tr_target_len
}
test_feed_dict = {
x: te_x,
x_bw: te_x_bw,
y: te_y,
y_sen: te_y_sen,
n_asp: te_n_asp,
sen_len: te_sen_len,
sen_len_bw: te_sen_len_bw,
target_words: te_target_word,
tar_len: te_target_len
}
tr_outputs = sess.run([outputs], feed_dict=train_feed_dict)
te_outputs = sess.run([outputs], feed_dict=test_feed_dict)
with open("results/embeddings/train_emb.npy", 'wb') as f:
np.save(f, tr_outputs)
with open("results/embeddings/test_emb.npy", 'wb') as f:
np.save(f, te_outputs)
P = precision_score(max_ty, max_py, average='micro')
R = recall_score(max_ty, max_py, average='micro')
F1 = f1_score(max_ty, max_py, average='micro')
print('(Individual aspect) P:', P, 'avg=', sum(P) / FLAGS.n_class)
print('(Individual aspect) R:', R, 'avg=', sum(R) / FLAGS.n_class)
print('(Individual aspect) F1:', F1, 'avg=', sum(F1) / FLAGS.n_class)
prob_data = {
'forward_att': max_fw,
'backward_att': max_bw,
'target_left_att': max_tl,
'target_right_att': max_tr,
'true': max_ty,
'predict': max_py
}
fp = open(FLAGS.prob_file, 'w')
pickle.dump(prob_data, fp)
# for item in max_prob:
# fp.write(' '.join([str(it) for it in item]) + '\n')
# fp = open(FLAGS.prob_file + '_fw', 'w')
# for y1, y2, ws in zip(max_ty, max_py, max_fw):
# fp.write(str(y1) + ' ' + str(y2) + ' ' + ' '.join([str(w) for w in ws[0]]) + '\n')
# fp = open(FLAGS.prob_file + '_bw', 'w')
# for y1, y2, ws in zip(max_ty, max_py, max_bw):
# fp.write(str(y1) + ' ' + str(y2) + ' ' + ' '.join([str(w) for w in ws[0]]) + '\n')
# fp = open(FLAGS.prob_file + '_tl', 'w')
# for y1, y2, ws in zip(max_ty, max_py, max_tl):
# fp.write(str(y1) + ' ' + str(y2) + ' ' + ' '.join([str(w) for w in ws[0]]) + '\n')
# fp = open(FLAGS.prob_file + '_tr', 'w')
# for y1, y2, ws in zip(max_ty, max_py, max_tr):
# fp.write(str(y1) + ' ' + str(y2) + ' ' + ' '.join([str(w) for w in ws[0]]) + '\n')
print('Optimization Finished! Max acc={}, Max micro f1={}'.format(
max_acc, max_f1))
print(
'Learning_rate={}, iter_num={}, batch_size={}, hidden_num={}, l2={}'
.format(FLAGS.learning_rate, FLAGS.n_iter, FLAGS.batch_size,
FLAGS.n_hidden, FLAGS.l2_reg))
return max_acc, np.where(np.subtract(max_py, max_ty) == 0, 0,
1), max_fw.tolist(), max_bw.tolist(
), max_tl.tolist(), max_tr.tolist()
