def train(self, epochs, exp_name, lr=1e-4, save_model=False):
# inputs & outputs format
x = tf.placeholder(tf.float32, [None, 25, 30, 188], name='x')
y = tf.placeholder('float', [None, self.output_dim], name='y')
# construct computation graph
logits = self.multi_layer_softmax_classifier(
tf.reshape(x, [-1, 25 * 30 * 188]))
loss = self.compute_loss(logits, y)
accuracy = self.compute_accuracy(logits, y)
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(
loss, name='train_op')
with tf.Session() as sess:
# initialization
init = tf.global_variables_initializer()
sess.run(init)
log_saver = logger.LogSaver(exp_name)
log_saver.set_log_cate(self.task_num)
# train
for epoch in range(epochs):
batch_x, batch_y = self.train_set.next_batch(100)
sess.run(train_op, feed_dict={x: batch_x, y: batch_y})
# validating
if epoch % 10 == 0:
train_loss = loss.eval(feed_dict={x: batch_x, y: batch_y})
val_x, val_y = self.val_set.next_batch(100)
val_acc = accuracy.eval(feed_dict={x: val_x, y: val_y})
print(
'Training {0} epoch, validation accuracy is {1}, training loss is {2}'
.format(epoch, val_acc, train_loss))
# save train process
log_saver.train_process_saver([epoch, train_loss, val_acc])
# evaluate
for index, test_set in enumerate(self.test_sets):
if index > 0:
test_x, test_y = test_set.next_batch(100)
test_acc = sess.run(accuracy,
feed_dict={
x: test_x,
y: test_y
})
print('test accuracy on test set {0} is {1}'.format(
index, test_acc))
# save training log
log_saver.test_result_saver([test_acc], index)
# Model save
if save_model:
log_saver.model_saver(sess)
def train(self, train_epoch, exp_name, lr):
# inputs & outputs format
x = tf.placeholder(tf.float32, [None, 160, 180])
y = tf.placeholder('float', [None, 8])
# x_ = tf.unstack(x, 160, 1)
# construct computation graph
pred = self.lstm_predictor(x)
loss = self.compute_loss(pred, y)
accuracy = self.compute_accuracy(pred, y)
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss)
with tf.Session() as sess:
# initialization
init = tf.global_variables_initializer()
sess.run(init)
dstc_data = data_provider.DataProvider(self.path, data_form=2)
log_saver = logger.LogSaver(exp_name)
# train
for epoch in range(train_epoch):
batch_x, batch_y = dstc_data.train.task1.next_batch(100)
sess.run(train_op, feed_dict={x: batch_x, y: batch_y})
# validating
if epoch % 10 == 0 and epoch != 0:
train_loss = loss.eval(feed_dict={x: batch_x, y: batch_y})
val_x, val_y = dstc_data.test1.task1.next_batch(100)
val_acc = accuracy.eval(feed_dict={x: val_x, y: val_y})
print(
'Training {0} epoch, validation accuracy is {1}, training loss is {2}'
.format(epoch, val_acc, train_loss))
log_saver.train_process_saver([epoch, train_loss, val_acc])
# self.save_log([epoch, train_loss, val_acc], exp_name)
# evaluate
test_sets = [
None, dstc_data.test1, dstc_data.test2, dstc_data.test3,
dstc_data.test4
]
for index, test_set in enumerate(test_sets):
test_x, test_y = test_set.task1.next_batch(100)
test_acc = sess.run(accuracy, feed_dict={x: test_x, y: test_y})
print('test accuracy on test set {0} is {1}'.format(
index, test_acc))
# save training log
log_saver.test_result_saver([test_acc], index)
def train(self, epochs, exp_name, lr, save_model=False):
print('Save model status: ', save_model)
# inputs & outputs format
x = tf.placeholder(tf.int32, [None, 180])
y = tf.placeholder('float', [None, self.output_dim])
embed_x = self.embedding_layer(x)
# construct computation graph
if self.bidirection:
pred = self.bd_lstm_predictor(embed_x)
else:
pred = self.lstm_predictor(embed_x)
loss = self.compute_loss(pred, y)
accuracy = self.compute_accuracy(pred, y)
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss)
with tf.Session() as sess:
# initialization
init = tf.global_variables_initializer()
sess.run(init)
# ini logger
log_saver = logger.LogSaver(exp_name)
log_saver.set_log_cate(self.task_num)
# train
for epoch in range(epochs):
for i in range(int(8000/100)):
batch_x, batch_y = self.train_set.next_batch(100)
sess.run(train_op, feed_dict={x: batch_x, y: batch_y})
# print validation information every 40 iteration
if i % 40 == 0 and i != 0:
train_loss = loss.eval(feed_dict={x: batch_x, y: batch_y})
train_acc = accuracy.eval(feed_dict={x: batch_x, y: batch_y})
val_x, val_y = self.val_set.next_batch(1000)
val_acc = accuracy.eval(feed_dict={
x: val_x,
y: val_y})
print('Epoch, {0}, Train loss,{1:2f}, Train acc, {2:3f}, Val_acc,{3:3f}'.format(epoch,
train_loss,
train_acc,
val_acc))
log_saver.train_process_saver([epoch, train_loss, train_acc, val_acc])
# save evaluation result per epoch
# train_loss = loss.eval(feed_dict={x: batch_x, y: batch_y})
# train_acc = accuracy.eval(feed_dict={x: batch_x, y: batch_y})
#
# val_x, val_y = self.val_set.next_batch(1000)
# val_acc = accuracy.eval(feed_dict={
# x: val_x,
# y: val_y})
#
# log_saver.train_process_saver([epoch, train_loss, train_acc, val_acc])
# evaluate on test set per epoch
for index, test_set in enumerate(self.test_sets):
if index > 0:
test_x, test_y = test_set.next_batch(1000)
test_acc = sess.run(
accuracy, feed_dict={
x: test_x,
y: test_y})
print('test accuracy on test set {0} is {1}'.format(index, test_acc))
# save training log
log_saver.test_result_saver([test_acc], index)
# Model save
if save_model:
log_saver.model_saver(sess)
def train(self,
epochs,
exp_name,
lr=1e-4,
keep_prob=0.5,
normal_gain=0.8,
save_model=False,
batch_size=10,
save_epoch=[]):
# inputs & outputs format
# -------------------------------------------------------------------- #
# x1 = tf.placeholder(tf.float32, [None, self.max_seq_len, self.vocab_size], name='x1')
# x2 = tf.placeholder(tf.float32, [None, self.max_seq_len, self.vocab_size], name='x2')
# y = tf.placeholder('float', [None, self.output_dim], name='y')
# -------------------------------------------------------------------- #
x1 = tf.placeholder(tf.int32, [None, self.max_seq_len], name='x1')
x1_one_hot = tf.one_hot(x1, self.vocab_size)
x2 = tf.placeholder(tf.int32, [None, self.max_seq_len], name='x2')
x2_one_hot = tf.one_hot(x2, self.vocab_size)
y = tf.placeholder(tf.int32, [None], name='y')
y_one_hot = tf.one_hot(y, 2)
prob = tf.placeholder('float', name='keep_prob')
norm_gain = tf.placeholder('float', name='norm_gain')
# construct computation graph
logits = self.representation_extractor(x1_one_hot, x2_one_hot, prob,
normal_gain)
loss = self.compute_loss(logits, y_one_hot)
pred = tf.argmax(tf.nn.softmax(logits), 1, name='prediction')
label = tf.argmax(y_one_hot, 1)
accuracy = self.compute_accuracy(logits, y_one_hot)
TP, TN, FP, FN = self.compute_f1_score(logits, y_one_hot)
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(
loss, name='train_op')
with tf.Session() as sess:
# initialization
init = tf.global_variables_initializer()
sess.run(init)
data_provider = DataProvider()
log_saver = logger.LogSaver(exp_name)
# train
for epoch in range(epochs):
# for iteration in range(1):
for iteration in range(int(112701 / batch_size)):
# time_start = time.time()
batch_x_1, batch_x_2, batch_y = data_provider.train.next_batch(
batch_size)
sess.run(train_op,
feed_dict={
x1: batch_x_1,
x2: batch_x_2,
y: batch_y,
prob: keep_prob,
norm_gain: normal_gain
})
# validating
if iteration % 500 == 0 and iteration != 0:
print('running validation ...')
train_loss = loss.eval(
feed_dict={
x1: batch_x_1,
x2: batch_x_2,
y: batch_y,
prob: keep_prob,
norm_gain: normal_gain
})
#val_x_1, val_x_2, val_y = data_provider.val.next_batch(1000)
#val_acc = accuracy.eval(feed_dict={
# x1: val_x_1,
# x2: val_x_2,
# y: val_y})
# Incremental Validation
mean_val_acc = 0
for i in tqdm(range(100)):
# time_start = time.time()
val_x_1, val_x_2, val_y = data_provider.val.next_batch(
100)
val_acc = accuracy.eval(
feed_dict={
x1: val_x_1,
x2: val_x_2,
y: val_y,
prob: 1.0,
norm_gain: 1.0
})
mean_val_acc = mean_val_acc + (
val_acc - mean_val_acc) / (i + 1)
# time_end = time.time()
# print('evaluate model with batch size=100 takes {0}s'.format(time_end - time_start))
# TP, TN, FP, FN
TP_score = TP.eval(
feed_dict={
x1: val_x_1,
x2: val_x_2,
y: val_y,
prob: 1.0,
norm_gain: 1.0
})
TN_score = TN.eval(
feed_dict={
x1: val_x_1,
x2: val_x_2,
y: val_y,
prob: 1.0,
norm_gain: 1.0
})
FP_score = FP.eval(
feed_dict={
x1: val_x_1,
x2: val_x_2,
y: val_y,
prob: 1.0,
norm_gain: 1.0
})
FN_score = FP.eval(
feed_dict={
x1: val_x_1,
x2: val_x_2,
y: val_y,
prob: 1.0,
norm_gain: 1.0
})
precision = TP_score / (TP_score + FP_score)
recall = TP_score / (TP_score + FN_score)
f1 = 2 * precision * recall / (precision + recall)
print(
'TP={0}, TN={1}, FP={2}, FN={3}, f1 score = {4}'
.format(TP_score, TN_score, FP_score, FN_score,
f1))
#prediction = pred.eval(feed_dict={
# x1: val_x_1,
# x2: val_x_2,
# y: val_y,
# prob: 1.0,
# norm_gain: 1.0}
# )
# labels = label.eval(feed_dict={
# x1: val_x_1,
# x2: val_x_2,
# y: val_y,
# prob: 1.0,
# norm_gain: 1.0}
# )
# print('\nprediction:', prediction, '\nlabel:', labels)
# print('f1-score={0}'.format(f1))
val_acc = mean_val_acc
print(
'Training {0} epoch, validation accuracy is {1}, training loss is {2}'
.format(epoch, val_acc, train_loss))
# save train process
log_saver.train_process_saver(
[epoch, train_loss, val_acc])
# Model save
if save_model and epoch + 1 in save_epoch:
log_saver.model_saver(sess, epoch)
# evaluate
mean_test_acc = 0
for i in tqdm(range(100)):
test_x_1, test_x_2, test_y = data_provider.test.next_batch(100)
test_acc = accuracy.eval(
feed_dict={
x1: test_x_1,
x2: test_x_2,
y: test_y,
prob: 1.0,
norm_gain: 1.0
})
mean_test_acc = mean_test_acc + (test_acc -
mean_test_acc) / (i + 1)
test_acc = mean_test_acc
# test_x_1, test_x_2, test_y = data_provider.test.next_batch(100)
# test_acc = sess.run(accuracy, feed_dict={x1: test_x_1, x2: test_x_2, y: test_y, prob: 1.0})
print('test accuracy is {0}'.format(test_acc))
# save training log
log_saver.test_result_saver([test_acc])
def train(self,
epochs,
exp_name,
lr=1e-3,
keep_prob=0.8,
save_model=False):
# inputs & outputs format
x = tf.placeholder(tf.int32, [None, 25, 30], name='x')
y = tf.placeholder('float', [None, self.output_dim], name='y')
dropout_rate = tf.placeholder('float', [])
# construct computation graph
embed_x = self.embedding_layer(x) # shape [Batch_size, 180, 300]
# pe_x = self.apply_positional_encoding(embed_x)
logits = self.hierarchical_attention_network(embed_x,
keep_prob=dropout_rate)
loss = self.compute_loss(logits, y)
accuracy = self.compute_accuracy(logits, y)
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(
loss, name='train_op')
with tf.Session() as sess:
# initialization
init = tf.global_variables_initializer()
sess.run(init)
log_saver = logger.LogSaver(exp_name)
log_saver.set_log_cate(self.task_num)
# train
# train
for epoch in range(epochs):
for i in range(int(8000 / 1000)):
batch_x, batch_y = self.train_set.next_batch(1000)
sess.run(train_op,
feed_dict={
x: batch_x,
y: batch_y,
dropout_rate: keep_prob
})
# print validation information every 40 iteration (half epoch)
if i % 4 == 0 and i != 0:
train_loss = loss.eval(feed_dict={
x: batch_x,
y: batch_y,
dropout_rate: keep_prob
})
train_acc = accuracy.eval(feed_dict={
x: batch_x,
y: batch_y,
dropout_rate: keep_prob
})
val_x, val_y = self.val_set.next_batch(1000)
val_acc = accuracy.eval(feed_dict={
x: val_x,
y: val_y,
dropout_rate: 1.0
})
print(
'Epoch, {0}, Train loss,{1:2f}, Train acc, {2:3f}, Val_acc,{3:3f}'
.format(epoch, train_loss, train_acc, val_acc))
log_saver.train_process_saver(
[epoch, train_loss, train_acc, val_acc])
# save evaluation result per epoch
# train_loss = loss.eval(feed_dict={x: batch_x, y: batch_y})
# train_acc = accuracy.eval(feed_dict={x: batch_x, y: batch_y})
#
# val_x, val_y = self.val_set.next_batch(1000)
# val_acc = accuracy.eval(feed_dict={
# x: val_x,
# y: val_y})
#
# log_saver.train_process_saver([epoch, train_loss, train_acc, val_acc])
# evaluate on test set per epoch
for index, test_set in enumerate(self.test_sets):
if index > 0:
test_x, test_y = test_set.next_batch(1000)
test_acc = sess.run(accuracy,
feed_dict={
x: test_x,
y: test_y,
dropout_rate: 1.0
})
print('test accuracy on test set {0} is {1}'.format(
index, test_acc))
# save training log
log_saver.test_result_saver([test_acc], index)
# Model save
if save_model:
log_saver.model_saver(sess)
def train(self, epochs, exp_name, lr=1e-4, save_model=False):
# inputs & outputs format
x1 = tf.placeholder(tf.float32, [None, 45, 8150], name='x1')
x2 = tf.placeholder(tf.float32, [None, 45, 8150], name='x2')
y = tf.placeholder('float', [None, self.output_dim], name='y')
# construct computation graph
logits = self.representation_extractor(x1, x2)
loss = self.compute_loss(logits, y)
accuracy = self.compute_accuracy(logits, y)
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(
loss, name='train_op')
with tf.Session() as sess:
# initialization
init = tf.global_variables_initializer()
sess.run(init)
data_provider = DataProvider(
'/Users/shyietliu/python/ATEC/project/NLP/dataset/atec_nlp_sim_train.csv'
)
log_saver = logger.LogSaver(exp_name)
# train
for epoch in range(epochs):
batch_x_1, batch_x_2, batch_y = data_provider.train.next_batch(
10)
sess.run(train_op,
feed_dict={
x1: batch_x_1,
x2: batch_x_2,
y: batch_y
})
# validating
if epoch % 10 == 0 and epoch != 0:
print('running validation ...')
train_loss = loss.eval(feed_dict={
x1: batch_x_1,
x2: batch_x_2,
y: batch_y
})
mean_val_acc = 0
for i in tqdm(range(800)):
val_x_1, val_x_2, val_y = data_provider.val.next_batch(
10)
val_acc = accuracy.eval(feed_dict={
x1: val_x_1,
x2: val_x_2,
y: val_y
})
mean_val_acc = mean_val_acc + (val_acc -
mean_val_acc) / (i + 1)
val_acc = mean_val_acc
print(
'Training {0} epoch, validation accuracy is {1}, training loss is {2}'
.format(epoch, val_acc, train_loss))
# save train process
log_saver.train_process_saver([epoch, train_loss, val_acc])
# evaluate
test_x_1, test_x_2, test_y = data_provider.test.next_batch(10)
test_acc = sess.run(accuracy,
feed_dict={
x1: test_x_1,
x2: test_x_2,
y: test_y
})
print('test accuracy is {0}'.format(test_acc))
# save training log
log_saver.test_result_saver([test_acc])
# Model save
if save_model:
log_saver.model_saver(sess)