def train(self, sess):
print('Start training')
out = open(self.outFile, 'w', 1)
out.write(self.outFile + '\n')
utils.writeInfo(out, self.args)
current_val_loss = np.inf
for e in range(self.args.epochs):
# training
trainBatches = self.textData.get_batches(tag='train')
totalTrainLoss = 0.0
# cnt of batches
cnt = 0
total_steps = 0
for nextBatch in tqdm(trainBatches):
cnt += 1
self.globalStep += 1
for sample in nextBatch.samples:
total_steps += sample.length
ops, feed_dict = self.model.step(nextBatch, test=False)
_, loss, trainPerplexity = sess.run(ops, feed_dict)
totalTrainLoss += loss
# average across samples in this step
trainPerplexity = np.mean(trainPerplexity)
self.summaryWriter.add_summary(utils.makeSummary({"trainLoss": loss}), self.globalStep)
self.summaryWriter.add_summary(utils.makeSummary({"trainPerplexity": trainPerplexity}), self.globalStep)
# compute perplexity over all samples in an epoch
trainPerplexity = np.exp(totalTrainLoss/total_steps)
print('\nepoch = {}, Train, loss = {}, perplexity = {}'.
format(e, totalTrainLoss, trainPerplexity))
out.write('\nepoch = {}, loss = {}, perplexity = {}\n'.
format(e, totalTrainLoss, trainPerplexity))
out.flush()
valLoss, val_num = self.test(sess, tag='val')
testLoss, test_num = self.test(sess, tag='test')
valPerplexity = np.exp(valLoss/val_num)
testPerplexity = np.exp(testLoss/test_num)
print('Val, loss = {}, perplexity = {}'.
format(valLoss, valPerplexity))
out.write('Val, loss = {}, perplexity = {}\n'.
format(valLoss, valPerplexity))
print('Test, loss = {}, perplexity = {}'.
format(testLoss, testPerplexity))
out.write('Test, loss = {}, perplexity = {}\n'.
format(testLoss, testPerplexity))
# we do not use cross val currently, just train, then evaluate
#if True:
if valLoss < current_val_loss:
current_val_loss = valLoss
print('New val loss {} at epoch {}'.format(valLoss, e))
out.write('New val loss {} at epoch {}\n'.format(valLoss, e))
save_path = self.saver.save(sess, save_path=self.model_name)
print('model saved at {}'.format(save_path))
out.write('model saved at {}\n'.format(save_path))
out.flush()
out.close()
def train(self, sess):
print('Start training')
out = open(self.outFile, 'w', 1)
out.write(self.outFile + '\n')
utils.writeInfo(out, self.args)
current_valAcc = 0.0
for e in range(self.args.epochs):
# training
#trainBatches = self.textData.train_batches
trainBatches = self.textData.get_batches(tag='train')
totalTrainLoss = 0.0
# cnt of batches
cnt = 0
total_samples = 0
total_corrects = 0
for nextBatch in tqdm(trainBatches):
cnt += 1
self.globalStep += 1
total_samples += nextBatch.batch_size
ops, feed_dict = self.model.step(nextBatch, test=False)
_, loss, predictions, corrects = sess.run(ops, feed_dict)
total_corrects += corrects
totalTrainLoss += loss
# average across samples in this step
self.summaryWriter.add_summary(utils.makeSummary({"trainLoss": loss}), self.globalStep)
# compute perplexity over all samples in an epoch
trainAcc = total_corrects*1.0/total_samples
print('\nepoch = {}, Train, loss = {}, trainAcc = {}'.
format(e, totalTrainLoss, trainAcc))
out.write('\nepoch = {}, loss = {}, trainAcc = {}\n'.
format(e, totalTrainLoss, trainAcc))
out.flush()
valAcc, valLoss = self.test(sess, tag='val')
print('Val, loss = {}, valAcc = {}'.
format(valLoss, valAcc))
out.write('Val, loss = {}, valAcc = {}\n'.
format(valLoss, valAcc))
testAcc, testLoss = self.test(sess, tag='test')
print('Test, loss = {}, testAcc = {}'.
format(testLoss, testAcc))
out.write('Test, loss = {}, testAcc = {}\n'.
format(testLoss, testAcc))
out.flush()
# we do not use cross val currently, just train, then evaluate
if valAcc >= current_valAcc:
current_valAcc = valAcc
print('New valAcc {} at epoch {}'.format(valAcc, e))
out.write('New valAcc {} at epoch {}\n'.format(valAcc, e))
save_path = self.saver.save(sess, save_path=self.model_name)
print('model saved at {}'.format(save_path))
out.write('model saved at {}\n'.format(save_path))
out.flush()
out.close()
def train(self, sess):
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
print('Start training')
out = open(self.outFile, 'w', 1)
out.write(self.outFile + '\n')
utils.writeInfo(out, self.args)
current_valAcc = 0.0
for e in range(self.args.epochs):
# training
trainBatches = self.textData.train_batches
#trainBatches = self.textData.get_batches(tag='train')
totalTrainLoss = 0.0
# cnt of batches
cnt = 0
total_samples = 0
total_corrects = 0
all_skip_rate = []
for idx, nextBatch in enumerate(tqdm(trainBatches)):
cnt += 1
#nextBatch = trainBatches[227]
self.globalStep += 1
total_samples += nextBatch.batch_size
ops, feed_dict, length = self.model.step(nextBatch, test=False)
# skip_rate: batch_size * n_samples
_, loss, predictions, corrects, skip_rate, skip_flag = sess.run(ops, feed_dict)
all_skip_rate.extend(skip_rate.tolist())
#print(loss, idx)
total_corrects += corrects
totalTrainLoss += loss
self.summaryWriter.add_summary(utils.makeSummary({"train_loss": loss}), self.globalStep)
break
trainAcc = total_corrects * 1.0 / (total_samples*self.args.nSamples)
train_skip_rate = np.average(all_skip_rate)
print('\nepoch = {}, Train, loss = {}, trainAcc = {}, train_skip_rate = {}'.
format(e, totalTrainLoss, trainAcc, train_skip_rate))
out.write('\nepoch = {}, loss = {}, trainAcc = {}, train_skip_rate = {}\n'.
format(e, totalTrainLoss, trainAcc, train_skip_rate))
out.flush()
continue
valAcc, valLoss, val_skip_rate = self.test(sess, tag='val')
testAcc, testLoss, test_skip_rate = self.test(sess, tag='test')
print('\tVal, loss = {}, valAcc = {}, val_skip_rate = {}'.
format(valLoss, valAcc, val_skip_rate))
out.write('\tVal, loss = {}, valAcc = {}, val_skip_rate = {}\n'.
format(valLoss, valAcc, val_skip_rate))
print('\tTest, loss = {}, testAcc = {}, test_skip_rate = {}'.
format(testLoss, testAcc, test_skip_rate))
out.write('\tTest, loss = {}, testAcc = {}, test_skip_rate = {}\n'.
format(testLoss, testAcc, test_skip_rate))
self.summaryWriter.add_summary(utils.makeSummary({"train_acc": trainAcc}), e)
self.summaryWriter.add_summary(utils.makeSummary({"val_acc": valAcc}), e)
self.summaryWriter.add_summary(utils.makeSummary({"test_acc": testAcc}), e)
self.summaryWriter.add_summary(utils.makeSummary({"train_skip_rate": train_skip_rate}), e)
self.summaryWriter.add_summary(utils.makeSummary({"val_skip_rate": val_skip_rate}), e)
self.summaryWriter.add_summary(utils.makeSummary({"test_skip_rate": test_skip_rate}), e)
# we do not use cross val currently, just train, then evaluate
if valAcc >= current_valAcc:
# with open('skip_train.pkl', 'wb') as f:
# p.dump(all_skip_rate, f)
current_valAcc = valAcc
print('New valAcc {} at epoch {}'.format(valAcc, e))
out.write('New valAcc {} at epoch {}\n'.format(valAcc, e))
save_path = self.saver.save(sess, save_path=self.model_name)
print('model saved at {}'.format(save_path))
out.write('model saved at {}\n'.format(save_path))
out.flush()
out.close()
def train(self, sess):
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
print('Start training')
out = open(self.outFile, 'w', 1)
out.write(self.outFile + '\n')
utils.writeInfo(out, self.args)
current_val_f1_micro_unweighted = 0.0
for e in range(self.args.epochs):
# training
trainBatches = self.textData.get_batches(tag='train',
augment=self.args.augment)
#trainBatches = self.textData.train_batches
totalTrainLoss = 0.0
# cnt of batches
cnt = 0
total_samples = 0
total_corrects = 0
all_predictions = []
all_labels = []
all_sample_weights = []
for idx, nextBatch in enumerate(tqdm(trainBatches)):
cnt += 1
self.globalStep += 1
total_samples += nextBatch.batch_size
# print(idx)
ops, feed_dict, labels, sample_weights = self.model.step(
nextBatch, test=False)
_, loss, predictions, corrects = sess.run(ops, feed_dict)
all_predictions.extend(predictions)
all_labels.extend(labels)
all_sample_weights.extend(sample_weights)
total_corrects += corrects
totalTrainLoss += loss
self.summaryWriter.add_summary(
utils.makeSummary({"train_loss": loss}), self.globalStep)
#break
trainAcc = total_corrects * 1.0 / total_samples
# calculate f1 score for train (weighted/unweighted)
train_f1_micro, train_f1_macro, train_p_micro, train_r_micro, train_p_macro, train_r_macro\
= self.cal_F1(y_pred=all_predictions, y_true=all_labels)
train_f1_micro_w, train_f1_macro_w, train_p_micro_w, train_r_micro_w, train_p_macro_w, train_r_macro_w\
= self.cal_F1(y_pred=all_predictions, y_true=all_labels,
sample_weight=all_sample_weights)
print(
'\nepoch = {}, Train, loss = {}, trainAcc = {}, train_f1_micro = {}, train_f1_macro = {},'
' train_f1_micro_w = {}, train_f1_macro_w = {}'.format(
e, totalTrainLoss, trainAcc, train_f1_micro,
train_f1_macro, train_f1_micro_w, train_f1_macro_w))
print(
'\ttrain_p_micro = {}, train_r_micro = {}, train_p_macro = {}, train_r_macro = {}'
.format(train_p_micro, train_r_micro, train_p_macro,
train_r_macro))
print(
'\ttrain_p_micro_w = {}, train_r_micro_w = {}, train_p_macro_w = {}, train_r_macro_w = {}'
.format(train_p_micro_w, train_r_micro_w, train_p_macro_w,
train_r_macro_w))
out.write(
'\nepoch = {}, loss = {}, trainAcc = {}, train_f1_micro = {}, train_f1_macro = {},'
' train_f1_micro_w = {}, train_f1_macro_w = {}\n'.format(
e, totalTrainLoss, trainAcc, train_f1_micro,
train_f1_macro, train_f1_micro_w, train_f1_macro_w))
out.write(
'\ttrain_p_micro = {}, train_r_micro = {}, train_p_macro = {}, train_r_macro = {}\n'
.format(train_p_micro, train_r_micro, train_p_macro,
train_r_macro))
out.write(
'\ttrain_p_micro_w = {}, train_r_micro_w = {}, train_p_macro_w = {}, train_r_macro_w = {}\n'
.format(train_p_micro_w, train_r_micro_w, train_p_macro_w,
train_r_macro_w))
#continue
out.flush()
# calculate f1 score for val (weighted/unweighted)
valAcc, valLoss, val_f1_micro, val_f1_macro, val_f1_micro_w, val_f1_macro_w,\
val_p_micro, val_r_micro, val_p_macro, val_r_macro, val_p_micro_w, val_r_micro_w, val_p_macro_w, val_r_macro_w\
= self.test(sess, tag='val')
print(
'\n\tVal, loss = {}, valAcc = {}, val_f1_micro = {}, val_f1_macro = {}, val_f1_micro_w = {}, val_f1_macro_w = {}'
.format(valLoss, valAcc, val_f1_micro, val_f1_macro,
val_f1_micro_w, val_f1_macro_w))
print(
'\t\t val_p_micro = {}, val_r_micro = {}, val_p_macro = {}, val_r_macro = {}'
.format(val_p_micro, val_r_micro, val_p_macro, val_r_macro))
print(
'\t\t val_p_micro_w = {}, val_r_micro_w = {}, val_p_macro_w = {}, val_r_macro_w = {}'
.format(val_p_micro_w, val_r_micro_w, val_p_macro_w,
val_r_macro_w))
out.write(
'\n\tVal, loss = {}, valAcc = {}, val_f1_micro = {}, val_f1_macro = {}, val_f1_micro_w = {}, val_f1_macro_w = {}\n'
.format(valLoss, valAcc, val_f1_micro, val_f1_macro,
val_f1_micro_w, val_f1_macro_w))
out.write(
'\t\t val_p_micro = {}, val_r_micro = {}, val_p_macro = {}, val_r_macro = {}\n'
.format(val_p_micro, val_r_micro, val_p_macro, val_r_macro))
out.write(
'\t\t val_p_micro_w = {}, val_r_micro_w = {}, val_p_macro_w = {}, val_r_macro_w = {}\n'
.format(val_p_micro_w, val_r_micro_w, val_p_macro_w,
val_r_macro_w))
# calculate f1 score for test (weighted/unweighted)
testAcc, testLoss, test_f1_micro, test_f1_macro, test_f1_micro_w, test_f1_macro_w,\
test_p_micro, test_r_micro, test_p_macro, test_r_macro, test_p_micro_w, test_r_micro_w, test_p_macro_w, test_r_macro_w\
= self.test(sess, tag='test')
print(
'\n\ttest, loss = {}, testAcc = {}, test_f1_micro = {}, test_f1_macro = {}, test_f1_micro_w = {}, test_f1_macro_w = {}'
.format(testLoss, testAcc, test_f1_micro, test_f1_macro,
test_f1_micro_w, test_f1_macro_w))
print(
'\t\t test_p_micro = {}, test_r_micro = {}, test_p_macro = {}, test_r_macro = {}'
.format(test_p_micro, test_r_micro, test_p_macro,
test_r_macro))
print(
'\t\t test_p_micro_w = {}, test_r_micro_w = {}, test_p_macro_w = {}, test_r_macro_w = {}'
.format(test_p_micro_w, test_r_micro_w, test_p_macro_w,
test_r_macro_w))
out.write(
'\n\ttest, loss = {}, testAcc = {}, test_f1_micro = {}, test_f1_macro = {}, test_f1_micro_w = {}, test_f1_macro_w = {}\n'
.format(testLoss, testAcc, test_f1_micro, test_f1_macro,
test_f1_micro_w, test_f1_macro_w))
out.write(
'\t\t test_p_micro = {}, test_r_micro = {}, test_p_macro = {}, test_r_macro = {}\n'
.format(test_p_micro, test_r_micro, test_p_macro,
test_r_macro))
out.write(
'\t\t test_p_micro_w = {}, test_r_micro_w = {}, test_p_macro_w = {}, test_r_macro_w = {}\n'
.format(test_p_micro_w, test_r_micro_w, test_p_macro_w,
test_r_macro_w))
out.flush()
self.summaryWriter.add_summary(
utils.makeSummary({"train_acc": trainAcc}), e)
self.summaryWriter.add_summary(
utils.makeSummary({"val_acc": valAcc}), e)
# use val_f1_micro (unweighted) as metric
if val_f1_micro >= current_val_f1_micro_unweighted:
current_val_f1_micro_unweighted = val_f1_micro
print('New val f1_micro {} at epoch {}'.format(
val_f1_micro, e))
out.write('New val f1_micro {} at epoch {}\n'.format(
val_f1_micro, e))
save_path = self.saver.save(sess, save_path=self.model_name)
print('model saved at {}'.format(save_path))
out.write('model saved at {}\n'.format(save_path))
test_predictions = self.test(sess, tag='test2')
print('Writing predictions at epoch {}'.format(e))
out.write('Writing predictions at epoch {}\n'.format(e))
test_file = self.write_predictions(test_predictions,
tag='unweighted')
print('Writing predictions to {}'.format(test_file))
out.write('Writing predictions to {}\n'.format(test_file))
out.flush()
out.close()