def evaluate(x_text, y, x_word_ids):
import numpy as np
import csv
with tf.Session() as sess:
# Load the saved meta graph
checkpoint_file = tf.train.latest_checkpoint(Config.checkpoint_dir)
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
# Restore variables
saver.restore(sess, checkpoint_file)
graph = tf.get_default_graph()
input_x = graph.get_operation_by_name("input_x").outputs[0]
predictions = graph.get_operation_by_name("predictions").outputs[0]
batches = batch_iter(list(x_word_ids), batch_size=128, num_epochs=1, shuffle=False)
all_predictions = []
for x_test_batch in batches:
batch_predictions = sess.run(predictions, {input_x: x_test_batch})
all_predictions = np.concatenate([all_predictions, batch_predictions])
correct_predictions = sum(all_predictions == np.argmax(y, axis=1))
print("Total number of test examples: {}".format(len(y)))
print("Accuracy: {:g}".format(float(correct_predictions) / float(len(y))))
# Save the evaluation to a csv
predictions_human_readable = np.column_stack((np.array(x_text), all_predictions))
out_path = os.path.join("./prediction.csv")
print("Saving evaluation to {0}".format(out_path))
with open(out_path, 'w') as f:
csv.writer(f).writerows(predictions_human_readable)
def test(self, sess, x, y):
global test_loss, test_accuracy
batch_test = batch_iter(x, y, batch_size=pm.batch_size)
for x_batch, y_batch in batch_test:
feed_dict = self.feed_data(x_batch, y_batch, 1.0)
test_loss, test_accuracy = sess.run([self.loss, self.accuracy],
feed_dict=feed_dict)
return test_loss, test_accuracy
def train():
tensorboard_dir = './tensorboard/Lstm_CNN'
save_dir = './checkpoints/Lstm_CNN'
if not os.path.exists(tensorboard_dir):
os.makedirs(tensorboard_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_path = os.path.join(save_dir, 'best_validation')
tf.summary.scalar('loss', model.loss)
tf.summary.scalar('accuracy', model.accuracy)
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter(tensorboard_dir)
saver = tf.train.Saver()
session = tf.Session()
session.run(tf.global_variables_initializer())
writer.add_graph(session.graph)
print("Preparing the training data....")
x_train, y_train = process(pm.train_filename,
word_ids,
cat_to_id,
max_length=300)
print("Preparing the testing data....")
x_test, y_test = process(pm.test_filename,
word_ids,
cat_to_id,
max_length=300)
for epoch in range(pm.num_epochs):
print('Epoch:', epoch + 1)
num_batchs = int((len(x_train) - 1) / pm.batch_size) + 1
batch_train = batch_iter(x_train, y_train, batch_size=pm.batch_size)
for x_batch, y_batch in batch_train:
real_seq_len = seq_length(x_batch)
feed_dict = model.feed_data(x_batch, y_batch, real_seq_len,
pm.keep_prob)
_, global_step, _summary, train_loss, train_accuracy = session.run(
[
model.optimizer, model.global_step, merged_summary,
model.loss, model.accuracy
],
feed_dict=feed_dict)
if global_step % 100 == 0:
test_loss, test_accuracy = model.test(session, x_test, y_test)
print('global_step:', global_step, 'train_loss:', train_loss,
'train_accuracy:', train_accuracy, 'test_loss:',
test_loss, 'test_accuracy:', test_accuracy)
if global_step % num_batchs == 0:
print('Saving Model...')
saver.save(session, save_path, global_step=global_step)
def eval_model(model, x_valid, y_valid):
valid_data = data_process.batch_iter(x_valid,
y_valid,
opt.batch_size,
num_epochs=1)
total_epoch_loss, total_epoch_acc, total_epoch_pre, total_epoch_recall, total_epoch_f1 = 0, 0, 0, 0, 0
model.eval()
for bat_num, valid_batch in enumerate(valid_data):
text_lengths_val = [len(x) for x in valid_batch[0]]
max_len_val = max(text_lengths_val)
padded_text_val, text_lengths_val = data_process.pad_sequences(
valid_batch[0])
padded_tags_val, tag_lengths_val = data_process.pad_sequences(
valid_batch[1])
padded_text_val = torch.from_numpy(padded_text_val).long()
target_val = torch.from_numpy(padded_tags_val).long()
target_val = torch.autograd.Variable(target_val).long()
if torch.cuda.is_available():
padded_text_val = padded_text_val.cuda()
target_val = target_val.cuda()
prediction_val = model(padded_text_val, text_lengths_val)
loss_val = model.loss(padded_text_val, text_lengths_val, target_val)
""" evaluation :acc,precision,recall,f1"""
num_corrects_val = (prediction_val.view(1, -1).data == target_val.view(
1, -1).data).float().sum()
acc_val = num_corrects_val / prediction_val.view(1, -1).size()[1]
recall_val = data_process.get_score(prediction_val, target_val, 'r')
pre_val = data_process.get_score(prediction_val, target_val, 'p')
f1_val = data_process.get_score(prediction_val, target_val)
# out_val = F.softmax(prediction_val, 1)
""" 样本数据属于正例概率 """
total_epoch_loss += loss_val.item()
total_epoch_recall += recall_val
total_epoch_acc += acc_val.item()
total_epoch_pre += pre_val
total_epoch_f1 += f1_val
print(f"validation in batch:{bat_num+1}\n")
model.train()
return total_epoch_loss / (bat_num + 1), total_epoch_acc / (
bat_num + 1), total_epoch_f1 / (bat_num + 1), total_epoch_pre / (
bat_num + 1), total_epoch_recall / (bat_num + 1)
def predict():
pre_labels = []
labels = []
session = tf.Session()
save_path = tf.train.latest_checkpoint('./checkpoints/Lstm_CNN')
saver = tf.train.Saver()
saver.restore(sess=session, save_path=save_path)
val_x, val_y = process(pm.val_filename,
word_to_ids,
cat_to_id,
max_length=pm.seq_length)
batch_val = batch_iter(val_x, val_y, batch_size=64)
for x_batch, y_batch in batch_val:
real_seq_len = seq_length(x_batch)
feed_dict = model.feed_data(x_batch, y_batch, real_seq_len, 1.0)
predict_label = session.run(model.predict, feed_dict=feed_dict)
pre_labels.extend(predict_label)
labels.extend(y_batch)
return pre_labels, labels
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
# input_y = graph.get_operation_by_name("input_y").outputs[0]
dropout_keep_prob = graph.get_operation_by_name(
"dropout_keep_prob").outputs[0]
# Tensors we want to evaluate
predictions = graph.get_operation_by_name(
"output/predictions").outputs[0]
# Generate batches for one epoch
batches = data_process.batch_iter(list(x_test),
FLAGS.batch_size,
1,
shuffle=False)
# Collect the predictions here
all_predictions = []
for x_test_batch in batches:
batch_predictions = sess.run(predictions, {
input_x: x_test_batch,
dropout_keep_prob: 1.0
})
all_predictions = np.concatenate(
[all_predictions, batch_predictions])
# Print accuracy if y_test is defined
if y_test is not None:
with open('config.txt', 'w') as f:
f.write(str(vocab_size) + '\n')
f.write(str(max_length))
# import model
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
model = CNN(sess=sess,
vocab_size=vocab_size,
sequence_length=max_length,
trainable=True)
model.embedding_assign(embedding)
batches = batch_iter(list(zip(x_input, y_input)),
batch_size=64,
num_epochs=5)
saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=0.5)
# train model
print('모델 훈련을 시작합니다.')
avgLoss = []
for step, batch in enumerate(batches):
x_train, y_train = zip(*batch)
x_train = sentence_to_index_morphs(x_train, vocab, max_length)
l, _ = model.train(x_train, y_train)
avgLoss.append(l)
if step % 500 == 0:
print('batch:', '%04d' % step, 'loss:', '%05f' % np.mean(avgLoss))
saver.save(sess, os.path.join(DIR, "model"), global_step=step)
avgLoss = []
def train():
input_x = tf.placeholder(tf.float32, [None, x_train.shape[1], x_train.shape[2]], name='input_x')
input_y = tf.placeholder(tf.float32, [None, y_train.shape[1]], name='input_y')
regularizer =tf.contrib.layers.l2_regularizer(FLAGS.L2_reg_lambda)
cnn = TextCnn(input_x=input_x,sequence_length=x_train.shape[1],
num_classes=y_train.shape[1],
embedding_size = x_train.shape[2],
filter_sizes=list(map(int,FLAGS.filter_size.split(','))),
num_filters =FLAGS.number_filter,
regularizer=regularizer)
y= cnn.socres
global_step = tf.Variable(0,trainable=False)
variable_averages= tf.train.ExponentialMovingAverage(FLAGS.moving_averages_op,global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
# with tf.name_scope('loss'):
loss = tf.nn.softmax_cross_entropy_with_logits(logits=y ,labels=input_y)
losses = tf.reduce_mean(loss) +tf.add_n(tf.get_collection('losses'))
learning_rate=tf.train.exponential_decay(
FLAGS.learning_rate,
global_step,
len(y_train)/FLAGS.batch_size,
FLAGS.learning_rate_decay,
staircase=True
)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(losses,global_step=global_step)
# grads_and_vars = optimizer.compute_gradients(losses)
# train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
with tf.name_scope('accuracy'):
correct_predictions = tf.equal(cnn.predictions ,tf.argmax(input_y ,1))
accuracy1 = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
with tf.control_dependencies([train_step, variables_averages_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver(max_to_keep=FLAGS.num_checkpoints)
with tf.Session() as sess:
tf.global_variables_initializer().run()
batchs = batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.epochs)
def val_test_step(x_batch, y_batch,model):
feed_dict = {
input_x: x_batch,
input_y: y_batch,
cnn.dropout_keep_prob: 1.0
}
step, loss, accuracy = sess.run(
[global_step, losses,accuracy1], feed_dict=feed_dict
)
time_str = datetime.datetime.now().isoformat()
if model == 'test':
print('test {}: step:{} , loss:{} , acc:{}'.format(time_str, step, loss, accuracy))
else:
print('val {}: step:{} , loss:{} , acc:{}'.format(time_str, step, loss, accuracy))
def train_st(x_batch, y_batch):
feed_dict = {
input_x: x_batch,
input_y: y_batch,
cnn.dropout_keep_prob: FLAGS.dropout
}
_, step, loss_value, accuracy = sess.run(
[train_op, global_step, losses,accuracy1], feed_dict=feed_dict
)
time_str = datetime.datetime.now().isoformat()
print('{}:step:{} , loss:{} , acc:{}'.format(time_str, step, loss_value, accuracy))
if step % 1000 == 0:
print("After %d training step(s), loss on training batch is %g." % (step, loss_value))
print('\n evaluate_every')
val_test_step(x_test, y_test, 'test')
val_test_step(x_val, y_val, 'val')
saver.save(sess, os.path.join(FLAGS.model_save_path, 'model.ckpt'), global_step=step)
for batch in batchs:
x_batch, y_batch = zip(*batch)
train_st(x_batch,y_batch)
f.write(str(vocab_size) + '\n')
f.write(str(max_length))
# open session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# make model instance
model = CNN(sess=sess, vocab_size=vocab_size, sequence_length=max_length, trainable=True)
# assign pretrained embedding vectors
model.embedding_assign(embedding)
# make train batches
batches = batch_iter(list(zip(x_input, y_input)), batch_size=64, num_epochs=5)
# model saver
saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=0.5)
# train model
print('모델 훈련을 시작합니다.')
avgLoss = []
for step, batch in enumerate(batches):
x_train, y_train = zip(*batch)
x_train = sentence_to_index_morphs(x_train, vocab, max_length)
l, _ = model.train(x_train, y_train)
avgLoss.append(l)
if step % 500 == 0:
print('batch:', '%04d' % step, 'loss:', '%05f' % np.mean(avgLoss))
saver.save(sess, os.path.join(DIR, "model"), global_step=step)
def train(self):
"""
training
:return:
"""
print(
'****************************starting training***********************************'
)
data_train, data_valid = self.load_data()
test_x_batch, test_y_batch = zip(*data_valid)
# Training
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=self.train_args['allow_soft_placement'],
log_device_placement=self.train_args['log_device_placement'],
)
self.sess = tf.Session(config=session_conf)
with self.sess.as_default():
# 模型
self.init_model_class()
self.global_step = tf.Variable(0,
name="global_step",
trainable=False)
lr = tf.train.exponential_decay(
learning_rate=0.005,
global_step=self.global_step,
decay_steps=self.train_args['lr_decay_steps'],
decay_rate=0.96,
staircase=True,
name='learn_rate')
optimizer = tf.train.AdamOptimizer(lr)
grads_and_vars = optimizer.compute_gradients(self.cnn.loss)
self.train_op = optimizer.apply_gradients(
grads_and_vars, global_step=self.global_step)
# Output directory for models and summaries
train_summary_op, train_summary_writer, test_summary_op, test_summary_writer, valid_summary_op, \
valid_summary_writer = self.save_summary(grads_and_vars=grads_and_vars)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = self.cnn.embedding_W.name
# 将元数据路径指定为刚才保存的文件
embedding.metadata_path = os.path.join('metadata.tsv')
summary_writer = tf.summary.FileWriter(self.checkpoint_dir)
# 保存embedding
projector.visualize_embeddings(summary_writer, config)
checkpoint_prefix = os.path.join(self.checkpoint_dir, "model")
saver = tf.train.Saver(tf.global_variables(),
save_relative_paths=True,
max_to_keep=10)
self.sess.run(tf.global_variables_initializer())
# Generate batches
batches = data_process.batch_iter(
data_train, self.train_args['batch_size'],
self.train_args['num_epochs'])
# Training loop. For each batch...
for batch in batches:
x_batch, y_batch = zip(*batch)
current_step = self.train_step(
x_batch,
y_batch,
summary_op=train_summary_op,
summary_writer=train_summary_writer)
if current_step % self.train_args[
'evaluate_every'] == 0 and current_step != 0:
self.dev_step(test_x_batch,
test_y_batch,
current_step,
test_summary_writer,
test_summary_op,
op_type='test')
path = saver.save(self.sess,
checkpoint_prefix,
global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))
train_summary_writer.close()
test_summary_writer.close()
valid_summary_writer.close()
print(
'****************************finish training***********************************'
)
def predict(filename, input_file, max_document_length):
# Load data
if FLAGS.eval_train:
x_raw, y_test = data_process.load_data_and_labels(
input_file, FLAGS.input_label_file, num_labels)
else:
x_raw = [
"a masterpiece four years in the making", "everything is off."
]
y_test = [1, 0]
# Get Embedding vector x_test
if len(x_raw) == 0:
return
sentences, max_document_length = data_process.padding_sentences(
x_raw, '补', padding_sentence_length=max_document_length)
#此处有问题，data_process.padding_sentences返回后一些句子长度会增加1，这些句子都是过长进行裁剪的，原因不清楚，暂时通过二次裁剪修正
sentences = [sentence[:max_document_length] for sentence in sentences]
x_test = np.array(
word2vec_helpers.embedding_sentences(
sentences, file_to_load=trained_word2vec_model_file))
print(len(x_test))
print("x_test.shape = {}".format(x_test.shape))
# Evaluation
# ==================================================
print("\nEvaluating...\n")
checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
# input_y = graph.get_operation_by_name("input_y").outputs[0]
dropout_keep_prob = graph.get_operation_by_name(
"dropout_keep_prob").outputs[0]
# Tensors we want to evaluate
predictions = graph.get_operation_by_name(
"output/predictions").outputs[0]
# Generate batches for one epoch
batches = data_process.batch_iter(list(x_test),
FLAGS.batch_size,
1,
shuffle=False)
# Collect the predictions here
all_predictions = []
for x_test_batch in batches:
batch_predictions = sess.run(predictions, {
input_x: x_test_batch,
dropout_keep_prob: 1.0
})
all_predictions = np.concatenate(
[all_predictions, batch_predictions])
# Print accuracy if y_test is defined
if y_test is not None:
correct_predictions = float(sum(all_predictions == y_test))
print("Total number of test examples: {}".format(len(y_test)))
print("Accuracy: {:g}".format(correct_predictions /
float(len(y_test))))
# Save the evaluation to a csv
# predictions_human_readable = np.column_stack((np.array([text.encode('utf-8') for text in x_raw]), all_predictions))
predictions_human_readable = np.column_stack(
(np.array(x_raw), all_predictions))
out_path = os.path.join(FLAGS.checkpoint_dir, "..", filename)
print("Saving evaluation to {0}".format(out_path))
with open(out_path, 'w') as f:
csv.writer(f).writerows(predictions_human_readable)
}
step, summaries, loss, accuracy = sess.run(
[global_step, dev_summary_op, cnn.loss, cnn.accuracy],
feed_dict)
lis_loss.append(loss)
lis_accu.append(accuracy)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, acc {:g}".format(
time_str, step, loss, accuracy))
print("test_loss and test_acc" + "\t\t" +
str(sum(lis_loss) / num) + "\t\t" + str(sum(lis_accu) / num))
if writer:
writer.add_summary(summaries, step)
# Generate batches（生成器），得到一个generator，每一次返回一个batch，没有构成list[batch1,batch2,batch3,...]
batches = data_process.batch_iter(list(zip(x_train, y_train)),
FLAGS.batch_size, FLAGS.num_epochs)
# zip将样本与label配对，
# Training loop. For each batch...
for batch in batches:
x_batch, y_batch = zip(*batch) # unzip，将配对的样本，分离出来data和label
train_step(x_batch, y_batch) # 训练，输入batch样本，更新模型
current_step = tf.train.global_step(sess, global_step)
if current_step % FLAGS.evaluate_every == 0: # 每多少步，算一下验证集效果
print("\nEvaluation:")
dev_step(x_dev, y_dev, writer=dev_summary_writer
) # 喂的数据为验证集，此时大小不止一个batchsize1的大小
print("")
if current_step % FLAGS.checkpoint_every == 0: # 每多少步，保存模型
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
def train_and_evaluate(x_train, y_train, x_test, y_test):
text_cnn = TextCNN(Config.sequence_length, Config.num_classes, Config.vocab_size, Config.embedding_dim,
Config.filter_sizes, Config.num_filters)
x = tf.placeholder(tf.int32, [None, Config.sequence_length], name='input_x')
y_ = tf.placeholder(tf.float32, [None, Config.num_classes], name='input_y')
output = text_cnn(x)
# 交叉熵损失
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=output))
predictions = tf.argmax(output, axis=1, name='predictions')
correction_predictions = tf.equal(predictions, tf.argmax(y_, axis=1))
accuracy = tf.reduce_mean(tf.cast(correction_predictions, dtype=tf.float32))
# 选择梯度优化算法
global_step = tf.Variable(0, trainable=False)
train_op = tf.train.AdamOptimizer().minimize(loss, global_step)
with tf.Session() as sess:
loss_summary = tf.summary.scalar('loss', loss)
accuracy_summary = tf.summary.scalar('accuracy', accuracy)
summary_op = tf.summary.merge([loss_summary, accuracy_summary])
train_summary_writer = tf.summary.FileWriter(Config.train_summary_dir, sess.graph)
test_summary_writer = tf.summary.FileWriter(Config.test_summary_dir, sess.graph)
saver = tf.train.Saver(max_to_keep=Config.keep_checkpoints_max)
sess.run(tf.global_variables_initializer())
all_variables = tf.get_default_graph().get_collection('variables')
print('all variables:\n', all_variables)
trainable_variables = tf.get_default_graph().get_collection('trainable_variables')
print('trainable variables:\n', trainable_variables)
if not os.path.exists(Config.checkpoint_dir):
os.makedirs(Config.checkpoint_dir)
else:
# 加载检查点文件
checkpoint_file = tf.train.latest_checkpoint(Config.checkpoint_dir)
saver.restore(sess, checkpoint_file)
tf.logging.info("<<<< restore model from:", checkpoint_file)
train_dataset = list(zip(x_train, y_train))
train_batch_iter = batch_iter(train_dataset, Config.batch_size, num_epochs=100, shuffle=False)
for batch in train_batch_iter:
batch_xs, batch_ys = zip(*batch)
_, _train_loss, _train_acc, _train_summary = sess.run([train_op, loss, accuracy, summary_op],
feed_dict={
x: batch_xs,
y_: batch_ys
})
step = tf.train.global_step(sess, global_step)
train_summary_writer.add_summary(_train_summary, step)
# print("step {}, loss {:g}, acc {:g}".format(step, _train_loss, _train_acc))
# 在测试集上计算模型的准确度
if step % Config.eval_per_steps == 0:
_test_acc, _test_loss, _test_summary = sess.run([accuracy, loss, summary_op],
feed_dict={
x: x_test,
y_: y_test
})
print('step {} - loss: {:.4f}, aac: {:.4f}, test loss: {:.4f}, test acc: {:.4f}'.format(
step, _train_loss, _train_acc, _test_loss, _test_acc))
test_summary_writer.add_summary(_train_summary, step)
# 保存模型的权重
if step % Config.save_checkpoints_steps == 0:
checkpoint_prefix = os.path.join(Config.checkpoint_dir, 'model')
# global_step 拼接到 checkpoint_prefix 后面
# checkpoint_prefix-global_step.(index|meta|data-00000-of-00001)
save_path = saver.save(sess, checkpoint_prefix, global_step=step)
tf.logging.info(">>>> save model in file: {}".format(save_path))
sent = tokens_to_ids(sent, vocab)
sentence.append(sent)
tags = [[tag2label[tag] for tag in x[1]] for x in data]
opt.parse({'vocab_size': len(words), 'embedding_length': 300})
model = Bilstm_crf(opt, tag2label)
optim = torch.optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()))
if torch.cuda.is_available():
model = model.cuda()
x_train, x_valid, y_train, y_valid = train_test_split(sentence,
tags,
test_size=0.2)
train_data = data_process.batch_iter(x_train,
y_train,
opt.batch_size,
num_epochs=opt.num_epochs)
steps = 0
min_delta = opt.min_delta
best_loss = None
best_acc, best_pre, best_f1 = 0, 0, 0
epoch_length = len(x_train) // opt.batch_size
model.train()
for idx, batch in enumerate(train_data):
text = batch[0]
text_lengths = [len(x) for x in text]
max_len = max(text_lengths)
pad_token = words_dict['<pad>']
# 将每个batch的text pad到这个batch里最长的长度
padded_text = np.ones((opt.batch_size, max_len)) * pad_token
for i, x_len in enumerate(text_lengths):
vocab_path = os.path.join(Config.out_dir, "vocab")
vocab_processor = learn.preprocessing.VocabularyProcessor.restore(vocab_path)
x_word_ids = np.array(list(vocab_processor.transform(x_text)))
with tf.Session() as sess:
# Load the saved meta graph
saver = tf.train.import_meta_graph("{}.meta".format(Config.checkpoint_dir))
# Restore variables
saver.restore(sess, Config.checkpoint_dir)
graph = tf.get_default_graph()
input_x = graph.get_operation_by_name("input_x").outputs[0]
predictions = graph.get_operation_by_name("predictions").outputs[0]
batches = batch_iter(list(x_word_ids), batch_size=128, num_epochs=1, shuffle=False)
all_predictions = []
for x_test_batch in batches:
batch_predictions = sess.run(predictions, {input_x: x_test_batch})
all_predictions = np.concatenate([all_predictions, batch_predictions])
correct_predictions = sum(all_predictions == np.argmax(y, axis=1))
print("Total number of test examples: {}".format(len(y)))
print("Accuracy: {:g}".format(float(correct_predictions) / float(len(y))))
# Save the evaluation to a csv
predictions_human_readable = np.column_stack((np.array(x_text), all_predictions))
out_path = os.path.join("./prediction.csv")
print("Saving evaluation to {0}".format(out_path))
with open(out_path, 'w') as f:
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: 1.0
}
step, summeries, loss, accuracy = sess.run(
[global_step, train_summary_op, cnn.losses, cnn.accuracy],
feed_dict=feed_dict)
time_str = datetime.datetime.now().isoformat()
if model == 'test':
print('test {}: step:{} , loss:{} , acc:{}'.format(
time_str, step, loss, accuracy))
else:
print('val {}: step:{} , loss:{} , acc:{}'.format(
time_str, step, loss, accuracy))
batchs = batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size,
FLAGS.epochs)
for batch in batchs:
x_batch, y_batch = zip(*batch)
train_step(x_batch, y_batch)
current_step = tf.train.global_step(sess, global_step)
if current_step % FLAGS.evaluate_every == 0:
print('\n evaluate_every')
val_test_step(x_test, y_test, 'test')
val_test_step(x_val, y_val, 'val')
if current_step % FLAGS.checkpoint_every == 0:
path = saver.save(sess, './model/', global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))
json.dump(vocab, fp)
# open session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# make model instance
model = seq2seq(sess,
encoder_vocab_size=vocab_size,
decoder_vocab_size=vocab_size)
# make train batches
input, target = make_dataset(data)
batches = batch_iter(list(zip(input, target)),
batch_size=64,
num_epochs=1001)
# model saver
saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=0.5)
# train model
print('모델 훈련을 시작합니다.')
avgLoss = []
for step, batch in enumerate(batches):
x_train, y_train = zip(*batch)
x_train = sentence_to_char_index(x_train, vocab, is_target=False)
y_train = sentence_to_char_index(y_train, vocab, is_target=True)
l, _ = model.train(x_train, y_train)
avgLoss.append(l)
if step % 100 == 0:
def train_and_evaluate(self, x_train, y_train, x_test, y_test):
global_step = tf.Variable(0, trainable=False)
train_op = tf.train.AdamOptimizer().minimize(self.loss, global_step)
# summaries
loss_summary = tf.summary.scalar('loss', self.loss)
accuracy_summary = tf.summary.scalar('accuracy', self.accuracy)
summary_op = tf.summary.merge([loss_summary, accuracy_summary])
with tf.Session() as sess:
train_summary_writer = tf.summary.FileWriter(
Config.train_summary_dir, sess.graph)
test_summary_writer = tf.summary.FileWriter(
Config.test_summary_dir, sess.graph)
saver = tf.train.Saver(max_to_keep=Config.keep_checkpoints_max)
# 初始化所有变量
sess.run(tf.global_variables_initializer())
if not os.path.exists(Config.checkpoint_dir):
os.makedirs(Config.checkpoint_dir)
else:
checkpoint_file = tf.train.latest_checkpoint(
Config.checkpoint_dir)
if checkpoint_file is not None:
# 加载检查点文件
saver.restore(sess, checkpoint_file)
tf.logging.info(
"<<<< restore model from: {}".format(checkpoint_file))
train_dataset = list(zip(x_train, y_train))
train_batch_iter = batch_iter(train_dataset,
Config.batch_size,
num_epochs=Config.num_epochs,
shuffle=False)
for batch in train_batch_iter:
batch_xs, batch_ys = zip(*batch)
_, train_loss, train_acc, train_summary = sess.run(
[train_op, self.loss, self.accuracy, summary_op],
feed_dict={
self.x: batch_xs,
self.y_: batch_ys
})
step = tf.train.global_step(sess, global_step)
train_summary_writer.add_summary(train_summary, step)
# print("step {}, loss {:g}, acc {:g}".format(step, _train_loss, _train_acc))
# 保存模型检查点
if step % Config.save_checkpoints_steps == 0:
checkpoint_prefix = os.path.join(Config.checkpoint_dir,
'model')
save_path = saver.save(sess,
checkpoint_prefix,
global_step=step)
tf.logging.info(
">>>> save model in file: {}".format(save_path))
# 在测试集上计算模型的准确度
if step % Config.eval_per_steps == 0:
test_acc, test_loss, test_summary = sess.run(
[self.accuracy, self.loss, summary_op],
feed_dict={
self.x: x_test,
self.y_: y_test
})
print(
'step {} - loss: {:.4f}, aac: {:.4f}, test loss: {:.4f}, test acc: {:.4f}'
.format(step, train_loss, train_acc, test_loss,
test_acc))
test_summary_writer.add_summary(test_summary, step)
if not os.path.exists(tensorboard_dir):
os.makedirs(tensorboard_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_path = os.path.join(save_dir, 'best_validation')
tf.summary.scalar("loss", loss)
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter(tensorboard_dir)
saver = tf.train.Saver()
writer.add_graph(session.graph)
for epoch in range(pm.num_epochs):
print('Epoch:', epoch + 1)
num_batchs = int((len(label) - 1) / pm.batch_size) + 1
batch_train = batch_iter(input_id, input_segment_, mask_, label,
pm.batch_size)
for x_id, x_segment, x_mask, y_label in batch_train:
n += 1
sequ_length = sequence(x_id)
feed_dict = feed_data(x_id, x_mask, x_segment, y_label, sequ_length,
pm.keep_prob)
_, train_summary, train_loss = session.run(
[train_op, merged_summary, loss], feed_dict=feed_dict)
if n % 100 == 0:
print('step:', n, 'loss:', train_loss)
# P = evaluate(session, test_id, test_segment, test_mask, test_label)
# print('测试集准确率:', P)
# if P > best:
# best = P
if (epoch + 1) % 5 == 0:
def train():
print("Loading data...")
train_data, val_data, test_data = load_data(FLAGS.training,
FLAGS.validation,
FLAGS.testing)
print("Train {}".format(np.array(train_data).shape))
print("Initializing...")
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
cnn = Trash_CNN(
num_classes=FLAGS.num_classes,
input_shape=(FLAGS.input_size, FLAGS.input_size, 3),
filters=list(map(int, FLAGS.filter_sizes.split(","))),
input_channel=FLAGS.num_filters)
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.AdamOptimizer(1e-3)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars)
timestamp = str(int(time.time()))
outdir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
print("Writing to {}\n".format(outdir))
loss_summary = tf.summary.scalar("loss", cnn.loss)
acc_summary = tf.summary.scalar("acc", cnn.accuracy)
# Train Summaries
train_summary_op = tf.summary.merge([loss_summary, acc_summary])
train_summary_dir = os.path.join(outdir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Dev summaries
dev_summary_op = tf.summary.merge([loss_summary, acc_summary])
dev_summary_dir = os.path.join(outdir, "summaries", "dev")
dev_summary_writer = tf.summary.FileWriter(dev_summary_dir,
sess.graph)
# Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it
checkpoint_dir = os.path.abspath(
os.path.join(outdir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
sess.run(tf.global_variables_initializer())
def train_step(batch_x, batch_y):
'''
One single training step
'''
feed_dict = {cnn.input_x: batch_x, cnn.input_y: batch_y}
_, step, summaries, loss, accuracy = sess.run(
[
train_op, global_step, train_summary_op, cnn.loss,
cnn.accuracy
],
feed_dict=feed_dict)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, acc: {:g}".format(
time_str, step, loss, accuracy))
train_summary_writer.add_summary(summaries, step)
def dev_step(batch_x, batch_y, writer=None):
'''
Evaluate model
'''
feed_dict = {cnn.input_x: batch_x, cnn.input_y: batch_y}
step, summaries, loss, accuracy = sess.run(
[global_step, train_summary_op, cnn.loss, cnn.accuracy],
feed_dict=feed_dict)
time_str = datetime.datetime.now().isoformat()
print("-------------- Step summary ---------------")
print("{}: step {}, loss {:g}, acc: {:g}".format(
time_str, step, loss, accuracy))
if writer:
writer.add_summary(summaries, step)
batches = batch_iter(list(zip(train_data[0], train_data[1])),
FLAGS.batch_size, FLAGS.num_epochs)
for batch in batches:
print("Batch {}".format(batch.shape))
x_batch, y_batch = zip(*batch)
train_step(x_batch, y_batch)
current_step = tf.train.global_step(sess, global_step)
if current_step % FLAGS.evaluate_every == 0:
print(
"\n======================= Evaluation: ======================="
)
dev_step(val_data[0],
val_data[1],
writer=dev_summary_writer)
print("")
if current_step % FLAGS.checkpoint_every == 0:
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))
vocab, reverse_vocab, vocab_size = build_vocab(data)
# save vocab
with open('vocab.json', 'w') as fp:
json.dump(vocab, fp)
# open session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# make model instance
model = reRNN(sess=sess, vocab_size=vocab_size, lr=1e-1)
# make train batches
batches = batch_iter(data, batch_size=64, num_epochs=1001)
# model saver
saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=0.5)
# train model
print('모델 훈련을 시작합니다.')
avgLoss = []
for step, batch in enumerate(batches):
x_train, y_train = sentenceToIndex(batch, vocab)
l, _ = model.train(x_train, y_train)
avgLoss.append(l)
if step % 100 == 0:
print('batch:', '%04d' % step, 'loss:', '%.5f' % np.mean(avgLoss))
saver.save(sess,
os.path.join(DIR, 'my-model.ckpt'),
