def run(self):
# load data
args_dict = self._default_configs()
args = dotdict(args_dict)
batchedData, maxTimeSteps, totalN = self.load_data(args,
mode=mode,
type=level)
model = model_fn(args, maxTimeSteps)
# count the num of params
num_params = count_params(model, mode='trainable')
all_num_params = count_params(model, mode='all')
model.config['trainable params'] = num_params
model.config['all params'] = all_num_params
print(model.config)
with tf.Session(graph=model.graph) as sess:
# restore from stored model
if keep == True:
ckpt = tf.train.get_checkpoint_state(savedir)
if ckpt and ckpt.model_checkpoint_path:
model.saver.restore(sess, ckpt.model_checkpoint_path)
print('Model restored from:' + savedir)
else:
print('Initializing')
sess.run(model.initial_op)
for epoch in range(num_epochs):
## training
start = time.time()
if mode == 'train':
print('Epoch', epoch + 1, '...')
batchErrors = np.zeros(len(batchedData))
batchRandIxs = np.random.permutation(len(batchedData))
for batch, batchOrigI in enumerate(batchRandIxs):
batchInputs, batchTargetSparse, batchSeqLengths = batchedData[
batchOrigI]
batchTargetIxs, batchTargetVals, batchTargetShape = batchTargetSparse
feedDict = {
model.inputX: batchInputs,
model.targetIxs: batchTargetIxs,
model.targetVals: batchTargetVals,
model.targetShape: batchTargetShape,
model.seqLengths: batchSeqLengths
}
if level == 'cha':
if mode == 'train':
_, l, pre, y, er = sess.run([
model.optimizer, model.loss, model.predictions,
model.targetY, model.errorRate
],
feed_dict=feedDict)
batchErrors[batch] = er
print(
'\n{} mode, total:{},batch:{}/{},epoch:{}/{},train loss={:.3f},mean train CER={:.3f}\n'
.format(level, totalN, batch + 1,
len(batchRandIxs), epoch + 1,
num_epochs, l, er / batch_size))
elif mode == 'test':
l, pre, y, er = sess.run([
model.loss, model.predictions, model.targetY,
model.errorRate
],
feed_dict=feedDict)
batchErrors[batch] = er
print(
'\n{} mode, total:{},batch:{}/{},test loss={:.3f},mean test CER={:.3f}\n'
.format(level, totalN, batch + 1,
len(batchRandIxs), l, er / batch_size))
elif level == 'phn':
if mode == 'train':
_, l, pre, y = sess.run([
model.optimizer, model.loss, model.predictions,
model.targetY
],
feed_dict=feedDict)
er = get_edit_distance([pre.values], [y.values],
True, level)
print(
'\n{} mode, total:{},batch:{}/{},epoch:{}/{},train loss={:.3f},mean train PER={:.3f}\n'
.format(level, totalN, batch + 1,
len(batchRandIxs), epoch + 1,
num_epochs, l, er))
batchErrors[batch] = er * len(batchSeqLengths)
elif mode == 'test':
l, pre, y = sess.run(
[model.loss, model.predictions, model.targetY],
feed_dict=feedDict)
er = get_edit_distance([pre.values], [y.values],
True, level)
print(
'\n{} mode, total:{},batch:{}/{},test loss={:.3f},mean test PER={:.3f}\n'
.format(level, totalN, batch + 1,
len(batchRandIxs), l, er))
batchErrors[batch] = er * len(batchSeqLengths)
# NOTE:
if er / batch_size == 1.0:
break
if batch % 30 == 0:
print('Truth:\n' + output_to_sequence(y, type=level))
print('Output:\n' +
output_to_sequence(pre, type=level))
if mode == 'train' and (
(epoch * len(batchRandIxs) + batch + 1) % 20 == 0 or
(epoch == num_epochs - 1
and batch == len(batchRandIxs) - 1)):
checkpoint_path = os.path.join(savedir, 'model.ckpt')
model.saver.save(sess,
checkpoint_path,
global_step=epoch)
print('Model has been saved in {}'.format(savedir))
end = time.time()
delta_time = end - start
print('Epoch ' + str(epoch + 1) + ' needs time:' +
str(delta_time) + ' s')
if mode == 'train':
if (epoch + 1) % 1 == 0:
checkpoint_path = os.path.join(savedir, 'model.ckpt')
model.saver.save(sess,
checkpoint_path,
global_step=epoch)
print('Model has been saved in {}'.format(savedir))
epochER = batchErrors.sum() / totalN
print('Epoch', epoch + 1, 'mean train error rate:',
epochER)
logging(model,
logfile,
epochER,
epoch,
delta_time,
mode='config')
logging(model,
logfile,
epochER,
epoch,
delta_time,
mode=mode)
if mode == 'test':
with open(os.path.join(resultdir, level + '_result.txt'),
'a') as result:
result.write(output_to_sequence(y, type=level) + '\n')
result.write(
output_to_sequence(pre, type=level) + '\n')
result.write('\n')
epochER = batchErrors.sum() / totalN
print(' test error rate:', epochER)
logging(model, logfile, epochER, mode=mode)
def run(self):
# load data
args_dict = self._default_configs()
args = dotdict(args_dict)
feature_dirs, label_dirs = get_data(datadir, level, train_dataset,
mode)
FL_pair = list(zip(feature_dirs, label_dirs))
feature_dirs, label_dirs = zip(*FL_pair)
for feature_dir, label_dir in zip(feature_dirs, label_dirs):
batchedData, maxTimeSteps, totalN = self.load_data(
feature_dir, label_dir, mode, level)
model = model_fn(args, maxTimeSteps)
num_params = count_params(model, mode='trainable')
all_num_params = count_params(model, mode='all')
model.config['trainable params'] = num_params
model.config['all params'] = all_num_params
with tf.Session(graph=model.graph) as sess:
print('Initializing')
sess.run(model.initial_op)
for epoch in range(num_epochs):
## training
start = time.time()
if mode == 'train':
print('Epoch {} ...'.format(epoch + 1))
avgloss = 0
batchErrors = np.zeros(len(batchedData))
batchRandIxs = np.arange(len(batchedData))
for batch, batchOrigI in enumerate(batchRandIxs):
batchInputs, batchTargetSparse, batchSeqLengths = batchedData[
batchOrigI]
batchTargetIxs, batchTargetVals, batchTargetShape = batchTargetSparse
feedDict = {
model.inputX: batchInputs,
model.targetIxs: batchTargetIxs,
model.targetVals: batchTargetVals,
model.targetShape: batchTargetShape,
model.seqLengths: batchSeqLengths
}
if level == 'cha':
if mode == 'train':
_, l, pre, y, er = sess.run([
model.optimizer, model.loss,
model.predictions, model.targetY,
model.errorRate
],
feed_dict=feedDict)
batchErrors[batch] = er
avgloss = avgloss + l
print(
'\n{} mode, total:{},batch:{}/{},epoch:{}/{},train loss={:.4f},mean train CER={:.4f}\n'
.format(level, totalN, batch + 1,
len(batchRandIxs), epoch + 1,
num_epochs, l, er / batch_size))
end = time.time()
delta_time = end - start
print('Epoch ' + str(epoch + 1) + ' needs time:' +
str(delta_time) + ' s')
if mode == 'train':
ctcloss = avgloss / (8440 / batch_size)
print('avarage CTC loss = ', ctcloss)
if (epoch + 1) % 1 == 0:
checkpoint_path = os.path.join(
savedir, 'model.ckpt')
model.saver.save(sess,
checkpoint_path,
global_step=epoch + 1)
print('Model has been saved in {}'.format(savedir))
epochER = batchErrors.sum() / totalN
print('Epoch', epoch + 1, 'mean train error rate:',
epochER)
if epoch < 1:
logging(model,
logfile,
epochER,
ctcloss,
epoch,
delta_time,
mode='config')
logging(model,
logfile,
epochER,
ctcloss,
epoch,
delta_time,
mode=mode)
if epochER <= 0.003:
break
def run(self):
# load data
args_dict = self._default_configs()
args = dotdict(args_dict)
feature_dirs, label_dirs = get_data(datadir, level, train_dataset, dev_dataset, test_dataset, mode)
batchedData, maxTimeSteps, totalN = self.load_data(feature_dirs[0], label_dirs[0], mode, level)
model = model_fn(args, maxTimeSteps)
## shuffle feature_dir and label_dir by same order
FL_pair = list(zip(feature_dirs, label_dirs))
random.shuffle(FL_pair)
feature_dirs, label_dirs = zip(*FL_pair)
for feature_dir, label_dir in zip(feature_dirs, label_dirs):
id_dir = feature_dirs.index(feature_dir)
print('dir id:{}'.format(id_dir))
batchedData, maxTimeSteps, totalN = self.load_data(feature_dir, label_dir, mode, level)
model = model_fn(args, maxTimeSteps)
num_params = count_params(model, mode='trainable')
all_num_params = count_params(model, mode='all')
model.config['trainable params'] = num_params
model.config['all params'] = all_num_params
print(model.config)
with tf.Session(graph=model.graph) as sess:
# restore from stored model
if keep == True:
ckpt = tf.train.get_checkpoint_state(savedir)
if ckpt and ckpt.model_checkpoint_path:
model.saver.restore(sess, ckpt.model_checkpoint_path)
print('Model restored from:' + savedir)
else:
print('Initializing')
sess.run(model.initial_op)
for epoch in range(num_epochs):
## training
start = time.time()
if mode == 'train':
print('Epoch {} ...'.format(epoch + 1))
batchErrors = np.zeros(len(batchedData))
batchRandIxs = np.random.permutation(len(batchedData))
for batch, batchOrigI in enumerate(batchRandIxs):
batchInputs, batchTargetSparse, batchSeqLengths = batchedData[batchOrigI]
batchTargetIxs, batchTargetVals, batchTargetShape = batchTargetSparse
feedDict = {model.inputX: batchInputs, model.targetIxs: batchTargetIxs,
model.targetVals: batchTargetVals, model.targetShape: batchTargetShape,
model.seqLengths: batchSeqLengths}
if level == 'cha':
if mode == 'train':
_, l, pre, y, er = sess.run([model.optimizer, model.loss,
model.predictions, model.targetY, model.errorRate],
feed_dict=feedDict)
batchErrors[batch] = er
print('\n{} mode, total:{},subdir:{}/{},batch:{}/{},epoch:{}/{},train loss={:.3f},mean train CER={:.3f}\n'.format(
level, totalN, id_dir+1, len(feature_dirs), batch+1, len(batchRandIxs), epoch+1, num_epochs, l, er/batch_size))
elif mode == 'dev':
l, pre, y, er = sess.run([model.loss, model.predictions,
model.targetY, model.errorRate], feed_dict=feedDict)
batchErrors[batch] = er
print('\n{} mode, total:{},subdir:{}/{},batch:{}/{},dev loss={:.3f},mean dev CER={:.3f}\n'.format(
level, totalN, id_dir+1, len(feature_dirs), batch+1, len(batchRandIxs), l, er/batch_size))
elif mode == 'test':
l, pre, y, er = sess.run([model.loss, model.predictions,
model.targetY, model.errorRate], feed_dict=feedDict)
batchErrors[batch] = er
print('\n{} mode, total:{},subdir:{}/{},batch:{}/{},test loss={:.3f},mean test CER={:.3f}\n'.format(
level, totalN, id_dir+1, len(feature_dirs), batch+1, len(batchRandIxs), l, er/batch_size))
elif level=='seq2seq':
raise ValueError('level %s is not supported now'%str(level))
# NOTE:
if er / batch_size == 1.0:
break
if batch % 20 == 0:
print('Truth:\n' + output_to_sequence(y, type=level))
print('Output:\n' + output_to_sequence(pre, type=level))
if mode=='train' and ((epoch * len(batchRandIxs) + batch + 1) % 20 == 0 or (
epoch == num_epochs - 1 and batch == len(batchRandIxs) - 1)):
checkpoint_path = os.path.join(savedir, 'model.ckpt')
model.saver.save(sess, checkpoint_path, global_step=epoch)
print('Model has been saved in {}'.format(savedir))
end = time.time()
delta_time = end - start
print('Epoch ' + str(epoch + 1) + ' needs time:' + str(delta_time) + ' s')
if mode=='train':
if (epoch + 1) % 1 == 0:
checkpoint_path = os.path.join(savedir, 'model.ckpt')
model.saver.save(sess, checkpoint_path, global_step=epoch)
print('Model has been saved in {}'.format(savedir))
epochER = batchErrors.sum() / totalN
print('Epoch', epoch + 1, 'mean train error rate:', epochER)
logging(model, logfile, epochER, epoch, delta_time, mode='config')
logging(model, logfile, epochER, epoch, delta_time, mode=mode)
if mode=='test' or mode=='dev':
with open(os.path.join(resultdir, level + '_result.txt'), 'a') as result:
result.write(output_to_sequence(y, type=level) + '\n')
result.write(output_to_sequence(pre, type=level) + '\n')
result.write('\n')
epochER = batchErrors.sum() / totalN
print(' test error rate:', epochER)
logging(model, logfile, epochER, mode=mode)
def test(self):
# load data
args = self.args
batchedData, maxTimeSteps, totalN = self.load_data(args,
mode='test',
type=args.level)
if args.model == 'ResNet':
model = ResNet(args, maxTimeSteps)
elif args.model == 'BiRNN':
model = BiRNN(args, maxTimeSteps)
elif args.model == 'DBiRNN':
model = DBiRNN(args, maxTimeSteps)
num_params = count_params(model, mode='trainable')
all_num_params = count_params(model, mode='all')
model.config['trainable params'] = num_params
model.config['all params'] = all_num_params
with tf.Session(graph=model.graph) as sess:
ckpt = tf.train.get_checkpoint_state(args.save_dir)
if ckpt and ckpt.model_checkpoint_path:
model.saver.restore(sess, ckpt.model_checkpoint_path)
print('Model restored from:' + args.save_dir)
batchErrors = np.zeros(len(batchedData))
batchRandIxs = np.random.permutation(len(batchedData))
for batch, batchOrigI in enumerate(batchRandIxs):
batchInputs, batchTargetSparse, batchSeqLengths = batchedData[
batchOrigI]
batchTargetIxs, batchTargetVals, batchTargetShape = batchTargetSparse
feedDict = {
model.inputX: batchInputs,
model.targetIxs: batchTargetIxs,
model.targetVals: batchTargetVals,
model.targetShape: batchTargetShape,
model.seqLengths: batchSeqLengths
}
if args.level == 'cha':
l, pre, y, er = sess.run([
model.loss, model.predictions, model.targetY,
model.errorRate
],
feed_dict=feedDict)
batchErrors[batch] = er
print(
'\ntotal:{},batch:{}/{},loss={:.3f},mean CER={:.3f}\n'.
format(totalN, batch + 1, len(batchRandIxs), l,
er / args.batch_size))
elif args.level == 'phn':
l, pre, y = sess.run(
[model.loss, model.predictions, model.targetY],
feed_dict=feedDict)
er = get_edit_distance([pre.values], [y.values], True,
'test', args.level)
print(
'\ntotal:{},batch:{}/{},loss={:.3f},mean PER={:.3f}\n'.
format(totalN, batch + 1, len(batchRandIxs), l,
er / args.batch_size))
batchErrors[batch] = er * len(batchSeqLengths)
print('Truth:\n' + output_to_sequence(y, type=args.level))
print('Output:\n' + output_to_sequence(pre, type=args.level))
'''
l, pre, y = sess.run([ model.loss,
model.predictions,
model.targetY],
feed_dict=feedDict)
er = get_edit_distance([pre.values], [y.values], True, 'test', args.level)
print(output_to_sequence(y,type=args.level))
print(output_to_sequence(pre,type=args.level))
'''
with open(args.task + '_result.txt', 'a') as result:
result.write(output_to_sequence(y, type=args.level) + '\n')
result.write(
output_to_sequence(pre, type=args.level) + '\n')
result.write('\n')
epochER = batchErrors.sum() / totalN
print(args.task + ' test error rate:', epochER)
logging(model, self.logfile, epochER, mode='test')
def run(self):
# load data
args_dict = self._default_configs()
args = dotdict(args_dict)
X, labels = get_data(level, train_dataset, test_dataset, mode)
totalN = len(X)
print("X :", len(X))
num_batches = len(X) / batch_size
maxLength = 0
for x in X:
maxLength = max(maxLength, x.shape[1])
if (maxLength % 5000 != 0):
maxLength = maxLength + 5000 - maxLength % 5000
#batchedData, maxTimeSteps, totalN = self.load_data(X,labels,batch_size,mode,level)
maxTimeSteps = maxLength
model = model_fn(args, maxTimeSteps)
model.build_graph(args, maxTimeSteps)
#print("hello")
#num_params = count_params(model, mode='trainable')
#all_num_params = count_params(model, mode='all')
#model.config['trainable params'] = num_params
#model.config['all params'] = all_num_params
print(model.config)
with tf.Session(graph=model.graph) as sess:
# restore from stored model
if keep == True:
ckpt = tf.train.get_checkpoint_state(savedir)
if ckpt and ckpt.model_checkpoint_path:
model.saver.restore(sess, ckpt.model_checkpoint_path)
print('Model restored from:' + savedir)
else:
sess.run(model.initial_op)
else:
print('Initializing')
sess.run(model.initial_op)
if (mode == 'train'):
writer = tf.summary.FileWriter("loggingdir", graph=model.graph)
for epoch in range(num_epochs):
# training
start = time.time()
print('Epoch {} ...'.format(epoch + 1))
batchErrors = np.zeros(num_batches)
batchRandIxs = np.random.permutation(num_batches)
for batch, batchOrigI in enumerate(batchRandIxs):
batchInputs, batchTargetSparse, batchSeqLengths = next(
load_batched_data(X, labels, batch_size, mode,
level))
batchTargetIxs, batchTargetVals, batchTargetShape = batchTargetSparse
feedDict = {
model.inputX: batchInputs,
model.targetIxs: batchTargetIxs,
model.targetVals: batchTargetVals,
model.targetShape: batchTargetShape,
model.seqLengths: batchSeqLengths
}
if level == 'cha':
_, l, pre, y, er, summary = sess.run(
[
model.optimizer, model.loss,
model.predictions, model.targetY,
model.errorRate, model.summary_op
],
feed_dict=feedDict)
writer.add_summary(summary,
epoch * num_batches + batch)
batchErrors[batch] = er
print(
'\n{} mode, batch:{}/{},epoch:{}/{},train loss={:.3f},mean train CER={:.3f}\n'
.format(level, batch + 1, len(batchRandIxs),
epoch + 1, num_epochs, l,
er / batch_size))
if (batch + 1) % 10 == 0:
print('Truth:\n' +
output_to_sequence(y, type='phn'))
print('Output:\n' +
output_to_sequence(pre, type='phn'))
if ((epoch * len(batchRandIxs) + batch + 1) % 20 == 0
or (epoch == num_epochs - 1
and batch == len(batchRandIxs) - 1)):
checkpoint_path = os.path.join(
savedir, 'model.ckpt')
model.saver.save(sess,
checkpoint_path,
global_step=epoch)
print('Model has been saved in {}'.format(savedir))
end = time.time()
delta_time = end - start
print('Epoch ' + str(epoch + 1) + ' needs time:' +
str(delta_time) + ' s')
if (epoch + 1) % 1 == 0:
checkpoint_path = os.path.join(savedir, 'model.ckpt')
model.saver.save(sess,
checkpoint_path,
global_step=epoch)
print('Model has been saved in {}'.format(savedir))
epochER = batchErrors.sum() / totalN
print('Epoch', epoch + 1, 'mean train error rate:',
epochER)
logging(model,
logfile,
epochER,
epoch,
delta_time,
mode='config')
logging(model,
logfile,
epochER,
epoch,
delta_time,
mode=mode)
elif (mode == 'test'):
for data in load_batched_data(X, labels, batch_size, mode,
level):
batchInputs, batchTargetSparse, batchSeqLengths = data
batchTargetIxs, batchTargetVals, batchTargetShape = batchTargetSparse
feedDict = {
model.inputX: batchInputs,
model.targetIxs: batchTargetIxs,
model.targetVals: batchTargetVals,
model.targetShape: batchTargetShape,
model.seqLengths: batchSeqLengths
}
_, l, pre, y, er = sess.run([
model.optimizer, model.loss, model.predictions,
model.targetY, model.errorRate
],
feed_dict=feedDict)
with open(os.path.join(resultdir, level + '_result.txt'),
'a') as result:
result.write(output_to_sequence(y, type='phn') + '\n')
result.write(
output_to_sequence(pre, type='phn') + '\n')
result.write('\n')
#epochER = batchErrors.sum() / totalN
print(' test error rate:', epochER)
logging(model, logfile, epochER, mode=mode)
def train(self, args):
''' import data, train model, save model
'''
args.data_dir = args.data_dir + args.style + '/'
args.save_dir = args.save_dir + args.style + '/'
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
print(args)
if args.attention is True:
print('attention mode')
text_parser = TextParser(args)
args.vocab_size = text_parser.vocab_size
self.word_embedding_file = os.path.join(args.data_dir,
"word_embedding.pkl")
if args.pretrained is True:
raise ValueError(
'pretrained has bug now, so don"t set it to be True now!!!')
if args.keep is False:
raise ValueError(
'when pre-trained is True, keep must be true!')
print("pretrained and keep mode...")
print("restoring pretrained model file")
ckpt = tf.train.get_checkpoint_state(
"/home/pony/github/jaylyrics_generation_tensorflow/data/pre-trained/"
)
if os.path.exists(os.path.join("./data/pre-trained/",'config.pkl')) and \
os.path.exists(os.path.join("./data/pre-trained/",'words_vocab.pkl')) and \
ckpt and ckpt.model_checkpoint_path:
with open(os.path.join("./data/pre-trained/", 'config.pkl'),
'rb') as f:
saved_model_args = cPickle.load(f)
with open(
os.path.join("./data/pre-trained/", 'words_vocab.pkl'),
'rb') as f:
saved_words, saved_vocab = cPickle.load(f)
else:
raise ValueError('configuration doesn"t exist!')
else:
ckpt = tf.train.get_checkpoint_state(args.save_dir)
if args.keep is True and args.pretrained is False:
# check if all necessary files exist
if os.path.exists(os.path.join(args.save_dir,'config.pkl')) and \
os.path.exists(os.path.join(args.save_dir,'words_vocab.pkl')) and \
ckpt and ckpt.model_checkpoint_path:
with open(os.path.join(args.save_dir, 'config.pkl'),
'rb') as f:
saved_model_args = cPickle.load(f)
with open(os.path.join(args.save_dir, 'words_vocab.pkl'),
'rb') as f:
saved_words, saved_vocab = cPickle.load(f)
else:
raise ValueError('configuration doesn"t exist!')
if args.model == 'seq2seq_rnn':
model = Model_rnn(args)
else:
pass
trainable_num_params = count_params(model, mode='trainable')
all_num_params = count_params(model, mode='all')
args.num_trainable_params = trainable_num_params
args.num_all_params = all_num_params
with open(os.path.join(args.save_dir, 'config.pkl'), 'wb') as f:
cPickle.dump(args, f)
with open(os.path.join(args.save_dir, 'words_vocab.pkl'), 'wb') as f:
cPickle.dump((text_parser.vocab_dict, text_parser.vocab_list), f)
with tf.Session() as sess:
if args.keep is True:
print('Restoring')
model.saver.restore(sess, ckpt.model_checkpoint_path)
else:
print('Initializing')
sess.run(model.initial_op)
sess.run(tf.assign(model.lr, args.learning_rate))
for e in range(args.num_epochs):
start = time.time()
model.initial_state = tf.convert_to_tensor(model.initial_state)
state = model.initial_state.eval()
total_loss = []
for b in range(text_parser.num_batches):
x, y = text_parser.next_batch()
if args.attention is True:
attention_states = sess.run(
tf.truncated_normal([
args.batch_size, model.attn_length,
model.attn_size
],
stddev=0.1,
dtype=tf.float32))
feed = {
model.input_data: x,
model.targets: y,
model.initial_state: state,
model.attention_states: attention_states
}
else:
feed = {
model.input_data: x,
model.targets: y,
model.initial_state: state
}
train_loss, state, _, word_embedding = sess.run([
model.cost, model.final_state, model.train_op,
model.word_embedding
], feed)
total_loss.append(train_loss)
print("{}/{} (epoch {}), train_loss = {:.3f}" \
.format(e * text_parser.num_batches + b, \
args.num_epochs * text_parser.num_batches, \
e, train_loss))
if (e * text_parser.num_batches +
b) % args.save_every == 0:
checkpoint_path = os.path.join(args.save_dir,
'model.ckpt')
model.saver.save(sess, checkpoint_path, global_step=e)
print("model has been saved in:" +
str(checkpoint_path))
np.save(self.word_embedding_file, word_embedding)
print("word embedding matrix has been saved in:" +
str(self.word_embedding_file))
end = time.time()
delta_time = end - start
ave_loss = np.array(total_loss).mean()
logging(model, ave_loss, e, delta_time, mode='train')
if ave_loss < 0.1:
checkpoint_path = os.path.join(args.save_dir, 'model.ckpt')
model.saver.save(sess, checkpoint_path, global_step=e)
print("model has been saved in:" + str(checkpoint_path))
np.save(self.word_embedding_file, word_embedding)
print("word embedding matrix has been saved in:" +
str(self.word_embedding_file))
break