def evaluate(x_text, y, x_word_ids): import numpy as np import csv with tf.Session() as sess: # Load the saved meta graph checkpoint_file = tf.train.latest_checkpoint(Config.checkpoint_dir) saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file)) # Restore variables saver.restore(sess, checkpoint_file) graph = tf.get_default_graph() input_x = graph.get_operation_by_name("input_x").outputs[0] predictions = graph.get_operation_by_name("predictions").outputs[0] batches = batch_iter(list(x_word_ids), batch_size=128, num_epochs=1, shuffle=False) all_predictions = [] for x_test_batch in batches: batch_predictions = sess.run(predictions, {input_x: x_test_batch}) all_predictions = np.concatenate([all_predictions, batch_predictions]) correct_predictions = sum(all_predictions == np.argmax(y, axis=1)) print("Total number of test examples: {}".format(len(y))) print("Accuracy: {:g}".format(float(correct_predictions) / float(len(y)))) # Save the evaluation to a csv predictions_human_readable = np.column_stack((np.array(x_text), all_predictions)) out_path = os.path.join("./prediction.csv") print("Saving evaluation to {0}".format(out_path)) with open(out_path, 'w') as f: csv.writer(f).writerows(predictions_human_readable)
def test(self, sess, x, y): global test_loss, test_accuracy batch_test = batch_iter(x, y, batch_size=pm.batch_size) for x_batch, y_batch in batch_test: feed_dict = self.feed_data(x_batch, y_batch, 1.0) test_loss, test_accuracy = sess.run([self.loss, self.accuracy], feed_dict=feed_dict) return test_loss, test_accuracy
def train(): tensorboard_dir = './tensorboard/Lstm_CNN' save_dir = './checkpoints/Lstm_CNN' if not os.path.exists(tensorboard_dir): os.makedirs(tensorboard_dir) if not os.path.exists(save_dir): os.makedirs(save_dir) save_path = os.path.join(save_dir, 'best_validation') tf.summary.scalar('loss', model.loss) tf.summary.scalar('accuracy', model.accuracy) merged_summary = tf.summary.merge_all() writer = tf.summary.FileWriter(tensorboard_dir) saver = tf.train.Saver() session = tf.Session() session.run(tf.global_variables_initializer()) writer.add_graph(session.graph) print("Preparing the training data....") x_train, y_train = process(pm.train_filename, word_ids, cat_to_id, max_length=300) print("Preparing the testing data....") x_test, y_test = process(pm.test_filename, word_ids, cat_to_id, max_length=300) for epoch in range(pm.num_epochs): print('Epoch:', epoch + 1) num_batchs = int((len(x_train) - 1) / pm.batch_size) + 1 batch_train = batch_iter(x_train, y_train, batch_size=pm.batch_size) for x_batch, y_batch in batch_train: real_seq_len = seq_length(x_batch) feed_dict = model.feed_data(x_batch, y_batch, real_seq_len, pm.keep_prob) _, global_step, _summary, train_loss, train_accuracy = session.run( [ model.optimizer, model.global_step, merged_summary, model.loss, model.accuracy ], feed_dict=feed_dict) if global_step % 100 == 0: test_loss, test_accuracy = model.test(session, x_test, y_test) print('global_step:', global_step, 'train_loss:', train_loss, 'train_accuracy:', train_accuracy, 'test_loss:', test_loss, 'test_accuracy:', test_accuracy) if global_step % num_batchs == 0: print('Saving Model...') saver.save(session, save_path, global_step=global_step)
def eval_model(model, x_valid, y_valid): valid_data = data_process.batch_iter(x_valid, y_valid, opt.batch_size, num_epochs=1) total_epoch_loss, total_epoch_acc, total_epoch_pre, total_epoch_recall, total_epoch_f1 = 0, 0, 0, 0, 0 model.eval() for bat_num, valid_batch in enumerate(valid_data): text_lengths_val = [len(x) for x in valid_batch[0]] max_len_val = max(text_lengths_val) padded_text_val, text_lengths_val = data_process.pad_sequences( valid_batch[0]) padded_tags_val, tag_lengths_val = data_process.pad_sequences( valid_batch[1]) padded_text_val = torch.from_numpy(padded_text_val).long() target_val = torch.from_numpy(padded_tags_val).long() target_val = torch.autograd.Variable(target_val).long() if torch.cuda.is_available(): padded_text_val = padded_text_val.cuda() target_val = target_val.cuda() prediction_val = model(padded_text_val, text_lengths_val) loss_val = model.loss(padded_text_val, text_lengths_val, target_val) """ evaluation :acc,precision,recall,f1""" num_corrects_val = (prediction_val.view(1, -1).data == target_val.view( 1, -1).data).float().sum() acc_val = num_corrects_val / prediction_val.view(1, -1).size()[1] recall_val = data_process.get_score(prediction_val, target_val, 'r') pre_val = data_process.get_score(prediction_val, target_val, 'p') f1_val = data_process.get_score(prediction_val, target_val) # out_val = F.softmax(prediction_val, 1) """ 样本数据属于正例概率 """ total_epoch_loss += loss_val.item() total_epoch_recall += recall_val total_epoch_acc += acc_val.item() total_epoch_pre += pre_val total_epoch_f1 += f1_val print(f"validation in batch:{bat_num+1}\n") model.train() return total_epoch_loss / (bat_num + 1), total_epoch_acc / ( bat_num + 1), total_epoch_f1 / (bat_num + 1), total_epoch_pre / ( bat_num + 1), total_epoch_recall / (bat_num + 1)
def predict(): pre_labels = [] labels = [] session = tf.Session() save_path = tf.train.latest_checkpoint('./checkpoints/Lstm_CNN') saver = tf.train.Saver() saver.restore(sess=session, save_path=save_path) val_x, val_y = process(pm.val_filename, word_to_ids, cat_to_id, max_length=pm.seq_length) batch_val = batch_iter(val_x, val_y, batch_size=64) for x_batch, y_batch in batch_val: real_seq_len = seq_length(x_batch) feed_dict = model.feed_data(x_batch, y_batch, real_seq_len, 1.0) predict_label = session.run(model.predict, feed_dict=feed_dict) pre_labels.extend(predict_label) labels.extend(y_batch) return pre_labels, labels
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) # Get the placeholders from the graph by name input_x = graph.get_operation_by_name("input_x").outputs[0] # input_y = graph.get_operation_by_name("input_y").outputs[0] dropout_keep_prob = graph.get_operation_by_name( "dropout_keep_prob").outputs[0] # Tensors we want to evaluate predictions = graph.get_operation_by_name( "output/predictions").outputs[0] # Generate batches for one epoch batches = data_process.batch_iter(list(x_test), FLAGS.batch_size, 1, shuffle=False) # Collect the predictions here all_predictions = [] for x_test_batch in batches: batch_predictions = sess.run(predictions, { input_x: x_test_batch, dropout_keep_prob: 1.0 }) all_predictions = np.concatenate( [all_predictions, batch_predictions]) # Print accuracy if y_test is defined if y_test is not None:
with open('config.txt', 'w') as f: f.write(str(vocab_size) + '\n') f.write(str(max_length)) # import model config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) model = CNN(sess=sess, vocab_size=vocab_size, sequence_length=max_length, trainable=True) model.embedding_assign(embedding) batches = batch_iter(list(zip(x_input, y_input)), batch_size=64, num_epochs=5) saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=0.5) # train model print('모델 훈련을 시작합니다.') avgLoss = [] for step, batch in enumerate(batches): x_train, y_train = zip(*batch) x_train = sentence_to_index_morphs(x_train, vocab, max_length) l, _ = model.train(x_train, y_train) avgLoss.append(l) if step % 500 == 0: print('batch:', '%04d' % step, 'loss:', '%05f' % np.mean(avgLoss)) saver.save(sess, os.path.join(DIR, "model"), global_step=step) avgLoss = []
def train(): input_x = tf.placeholder(tf.float32, [None, x_train.shape[1], x_train.shape[2]], name='input_x') input_y = tf.placeholder(tf.float32, [None, y_train.shape[1]], name='input_y') regularizer =tf.contrib.layers.l2_regularizer(FLAGS.L2_reg_lambda) cnn = TextCnn(input_x=input_x,sequence_length=x_train.shape[1], num_classes=y_train.shape[1], embedding_size = x_train.shape[2], filter_sizes=list(map(int,FLAGS.filter_size.split(','))), num_filters =FLAGS.number_filter, regularizer=regularizer) y= cnn.socres global_step = tf.Variable(0,trainable=False) variable_averages= tf.train.ExponentialMovingAverage(FLAGS.moving_averages_op,global_step) variables_averages_op = variable_averages.apply(tf.trainable_variables()) # with tf.name_scope('loss'): loss = tf.nn.softmax_cross_entropy_with_logits(logits=y ,labels=input_y) losses = tf.reduce_mean(loss) +tf.add_n(tf.get_collection('losses')) learning_rate=tf.train.exponential_decay( FLAGS.learning_rate, global_step, len(y_train)/FLAGS.batch_size, FLAGS.learning_rate_decay, staircase=True ) train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(losses,global_step=global_step) # grads_and_vars = optimizer.compute_gradients(losses) # train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) with tf.name_scope('accuracy'): correct_predictions = tf.equal(cnn.predictions ,tf.argmax(input_y ,1)) accuracy1 = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy") with tf.control_dependencies([train_step, variables_averages_op]): train_op = tf.no_op(name='train') saver = tf.train.Saver(max_to_keep=FLAGS.num_checkpoints) with tf.Session() as sess: tf.global_variables_initializer().run() batchs = batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.epochs) def val_test_step(x_batch, y_batch,model): feed_dict = { input_x: x_batch, input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, loss, accuracy = sess.run( [global_step, losses,accuracy1], feed_dict=feed_dict ) time_str = datetime.datetime.now().isoformat() if model == 'test': print('test {}: step:{} , loss:{} , acc:{}'.format(time_str, step, loss, accuracy)) else: print('val {}: step:{} , loss:{} , acc:{}'.format(time_str, step, loss, accuracy)) def train_st(x_batch, y_batch): feed_dict = { input_x: x_batch, input_y: y_batch, cnn.dropout_keep_prob: FLAGS.dropout } _, step, loss_value, accuracy = sess.run( [train_op, global_step, losses,accuracy1], feed_dict=feed_dict ) time_str = datetime.datetime.now().isoformat() print('{}:step:{} , loss:{} , acc:{}'.format(time_str, step, loss_value, accuracy)) if step % 1000 == 0: print("After %d training step(s), loss on training batch is %g." % (step, loss_value)) print('\n evaluate_every') val_test_step(x_test, y_test, 'test') val_test_step(x_val, y_val, 'val') saver.save(sess, os.path.join(FLAGS.model_save_path, 'model.ckpt'), global_step=step) for batch in batchs: x_batch, y_batch = zip(*batch) train_st(x_batch,y_batch)
f.write(str(vocab_size) + '\n') f.write(str(max_length)) # open session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # make model instance model = CNN(sess=sess, vocab_size=vocab_size, sequence_length=max_length, trainable=True) # assign pretrained embedding vectors model.embedding_assign(embedding) # make train batches batches = batch_iter(list(zip(x_input, y_input)), batch_size=64, num_epochs=5) # model saver saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=0.5) # train model print('모델 훈련을 시작합니다.') avgLoss = [] for step, batch in enumerate(batches): x_train, y_train = zip(*batch) x_train = sentence_to_index_morphs(x_train, vocab, max_length) l, _ = model.train(x_train, y_train) avgLoss.append(l) if step % 500 == 0: print('batch:', '%04d' % step, 'loss:', '%05f' % np.mean(avgLoss)) saver.save(sess, os.path.join(DIR, "model"), global_step=step)
def train(self): """ training :return: """ print( '****************************starting training***********************************' ) data_train, data_valid = self.load_data() test_x_batch, test_y_batch = zip(*data_valid) # Training with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=self.train_args['allow_soft_placement'], log_device_placement=self.train_args['log_device_placement'], ) self.sess = tf.Session(config=session_conf) with self.sess.as_default(): # 模型 self.init_model_class() self.global_step = tf.Variable(0, name="global_step", trainable=False) lr = tf.train.exponential_decay( learning_rate=0.005, global_step=self.global_step, decay_steps=self.train_args['lr_decay_steps'], decay_rate=0.96, staircase=True, name='learn_rate') optimizer = tf.train.AdamOptimizer(lr) grads_and_vars = optimizer.compute_gradients(self.cnn.loss) self.train_op = optimizer.apply_gradients( grads_and_vars, global_step=self.global_step) # Output directory for models and summaries train_summary_op, train_summary_writer, test_summary_op, test_summary_writer, valid_summary_op, \ valid_summary_writer = self.save_summary(grads_and_vars=grads_and_vars) config = projector.ProjectorConfig() embedding = config.embeddings.add() embedding.tensor_name = self.cnn.embedding_W.name # 将元数据路径指定为刚才保存的文件 embedding.metadata_path = os.path.join('metadata.tsv') summary_writer = tf.summary.FileWriter(self.checkpoint_dir) # 保存embedding projector.visualize_embeddings(summary_writer, config) checkpoint_prefix = os.path.join(self.checkpoint_dir, "model") saver = tf.train.Saver(tf.global_variables(), save_relative_paths=True, max_to_keep=10) self.sess.run(tf.global_variables_initializer()) # Generate batches batches = data_process.batch_iter( data_train, self.train_args['batch_size'], self.train_args['num_epochs']) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) current_step = self.train_step( x_batch, y_batch, summary_op=train_summary_op, summary_writer=train_summary_writer) if current_step % self.train_args[ 'evaluate_every'] == 0 and current_step != 0: self.dev_step(test_x_batch, test_y_batch, current_step, test_summary_writer, test_summary_op, op_type='test') path = saver.save(self.sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path)) train_summary_writer.close() test_summary_writer.close() valid_summary_writer.close() print( '****************************finish training***********************************' )
def predict(filename, input_file, max_document_length): # Load data if FLAGS.eval_train: x_raw, y_test = data_process.load_data_and_labels( input_file, FLAGS.input_label_file, num_labels) else: x_raw = [ "a masterpiece four years in the making", "everything is off." ] y_test = [1, 0] # Get Embedding vector x_test if len(x_raw) == 0: return sentences, max_document_length = data_process.padding_sentences( x_raw, '补', padding_sentence_length=max_document_length) #此处有问题,data_process.padding_sentences返回后一些句子长度会增加1,这些句子都是过长进行裁剪的,原因不清楚,暂时通过二次裁剪修正 sentences = [sentence[:max_document_length] for sentence in sentences] x_test = np.array( word2vec_helpers.embedding_sentences( sentences, file_to_load=trained_word2vec_model_file)) print(len(x_test)) print("x_test.shape = {}".format(x_test.shape)) # Evaluation # ================================================== print("\nEvaluating...\n") checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) graph = tf.Graph() with graph.as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph( "{}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) # Get the placeholders from the graph by name input_x = graph.get_operation_by_name("input_x").outputs[0] # input_y = graph.get_operation_by_name("input_y").outputs[0] dropout_keep_prob = graph.get_operation_by_name( "dropout_keep_prob").outputs[0] # Tensors we want to evaluate predictions = graph.get_operation_by_name( "output/predictions").outputs[0] # Generate batches for one epoch batches = data_process.batch_iter(list(x_test), FLAGS.batch_size, 1, shuffle=False) # Collect the predictions here all_predictions = [] for x_test_batch in batches: batch_predictions = sess.run(predictions, { input_x: x_test_batch, dropout_keep_prob: 1.0 }) all_predictions = np.concatenate( [all_predictions, batch_predictions]) # Print accuracy if y_test is defined if y_test is not None: correct_predictions = float(sum(all_predictions == y_test)) print("Total number of test examples: {}".format(len(y_test))) print("Accuracy: {:g}".format(correct_predictions / float(len(y_test)))) # Save the evaluation to a csv # predictions_human_readable = np.column_stack((np.array([text.encode('utf-8') for text in x_raw]), all_predictions)) predictions_human_readable = np.column_stack( (np.array(x_raw), all_predictions)) out_path = os.path.join(FLAGS.checkpoint_dir, "..", filename) print("Saving evaluation to {0}".format(out_path)) with open(out_path, 'w') as f: csv.writer(f).writerows(predictions_human_readable)
} step, summaries, loss, accuracy = sess.run( [global_step, dev_summary_op, cnn.loss, cnn.accuracy], feed_dict) lis_loss.append(loss) lis_accu.append(accuracy) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) print("test_loss and test_acc" + "\t\t" + str(sum(lis_loss) / num) + "\t\t" + str(sum(lis_accu) / num)) if writer: writer.add_summary(summaries, step) # Generate batches(生成器),得到一个generator,每一次返回一个batch,没有构成list[batch1,batch2,batch3,...] batches = data_process.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs) # zip将样本与label配对, # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) # unzip,将配对的样本,分离出来data和label train_step(x_batch, y_batch) # 训练,输入batch样本,更新模型 current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: # 每多少步,算一下验证集效果 print("\nEvaluation:") dev_step(x_dev, y_dev, writer=dev_summary_writer ) # 喂的数据为验证集,此时大小不止一个batchsize1的大小 print("") if current_step % FLAGS.checkpoint_every == 0: # 每多少步,保存模型 path = saver.save(sess, checkpoint_prefix, global_step=current_step)
def train_and_evaluate(x_train, y_train, x_test, y_test): text_cnn = TextCNN(Config.sequence_length, Config.num_classes, Config.vocab_size, Config.embedding_dim, Config.filter_sizes, Config.num_filters) x = tf.placeholder(tf.int32, [None, Config.sequence_length], name='input_x') y_ = tf.placeholder(tf.float32, [None, Config.num_classes], name='input_y') output = text_cnn(x) # 交叉熵损失 loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=output)) predictions = tf.argmax(output, axis=1, name='predictions') correction_predictions = tf.equal(predictions, tf.argmax(y_, axis=1)) accuracy = tf.reduce_mean(tf.cast(correction_predictions, dtype=tf.float32)) # 选择梯度优化算法 global_step = tf.Variable(0, trainable=False) train_op = tf.train.AdamOptimizer().minimize(loss, global_step) with tf.Session() as sess: loss_summary = tf.summary.scalar('loss', loss) accuracy_summary = tf.summary.scalar('accuracy', accuracy) summary_op = tf.summary.merge([loss_summary, accuracy_summary]) train_summary_writer = tf.summary.FileWriter(Config.train_summary_dir, sess.graph) test_summary_writer = tf.summary.FileWriter(Config.test_summary_dir, sess.graph) saver = tf.train.Saver(max_to_keep=Config.keep_checkpoints_max) sess.run(tf.global_variables_initializer()) all_variables = tf.get_default_graph().get_collection('variables') print('all variables:\n', all_variables) trainable_variables = tf.get_default_graph().get_collection('trainable_variables') print('trainable variables:\n', trainable_variables) if not os.path.exists(Config.checkpoint_dir): os.makedirs(Config.checkpoint_dir) else: # 加载检查点文件 checkpoint_file = tf.train.latest_checkpoint(Config.checkpoint_dir) saver.restore(sess, checkpoint_file) tf.logging.info("<<<< restore model from:", checkpoint_file) train_dataset = list(zip(x_train, y_train)) train_batch_iter = batch_iter(train_dataset, Config.batch_size, num_epochs=100, shuffle=False) for batch in train_batch_iter: batch_xs, batch_ys = zip(*batch) _, _train_loss, _train_acc, _train_summary = sess.run([train_op, loss, accuracy, summary_op], feed_dict={ x: batch_xs, y_: batch_ys }) step = tf.train.global_step(sess, global_step) train_summary_writer.add_summary(_train_summary, step) # print("step {}, loss {:g}, acc {:g}".format(step, _train_loss, _train_acc)) # 在测试集上计算模型的准确度 if step % Config.eval_per_steps == 0: _test_acc, _test_loss, _test_summary = sess.run([accuracy, loss, summary_op], feed_dict={ x: x_test, y_: y_test }) print('step {} - loss: {:.4f}, aac: {:.4f}, test loss: {:.4f}, test acc: {:.4f}'.format( step, _train_loss, _train_acc, _test_loss, _test_acc)) test_summary_writer.add_summary(_train_summary, step) # 保存模型的权重 if step % Config.save_checkpoints_steps == 0: checkpoint_prefix = os.path.join(Config.checkpoint_dir, 'model') # global_step 拼接到 checkpoint_prefix 后面 # checkpoint_prefix-global_step.(index|meta|data-00000-of-00001) save_path = saver.save(sess, checkpoint_prefix, global_step=step) tf.logging.info(">>>> save model in file: {}".format(save_path))
sent = tokens_to_ids(sent, vocab) sentence.append(sent) tags = [[tag2label[tag] for tag in x[1]] for x in data] opt.parse({'vocab_size': len(words), 'embedding_length': 300}) model = Bilstm_crf(opt, tag2label) optim = torch.optim.Adam( filter(lambda p: p.requires_grad, model.parameters())) if torch.cuda.is_available(): model = model.cuda() x_train, x_valid, y_train, y_valid = train_test_split(sentence, tags, test_size=0.2) train_data = data_process.batch_iter(x_train, y_train, opt.batch_size, num_epochs=opt.num_epochs) steps = 0 min_delta = opt.min_delta best_loss = None best_acc, best_pre, best_f1 = 0, 0, 0 epoch_length = len(x_train) // opt.batch_size model.train() for idx, batch in enumerate(train_data): text = batch[0] text_lengths = [len(x) for x in text] max_len = max(text_lengths) pad_token = words_dict['<pad>'] # 将每个batch的text pad到这个batch里最长的长度 padded_text = np.ones((opt.batch_size, max_len)) * pad_token for i, x_len in enumerate(text_lengths):
vocab_path = os.path.join(Config.out_dir, "vocab") vocab_processor = learn.preprocessing.VocabularyProcessor.restore(vocab_path) x_word_ids = np.array(list(vocab_processor.transform(x_text))) with tf.Session() as sess: # Load the saved meta graph saver = tf.train.import_meta_graph("{}.meta".format(Config.checkpoint_dir)) # Restore variables saver.restore(sess, Config.checkpoint_dir) graph = tf.get_default_graph() input_x = graph.get_operation_by_name("input_x").outputs[0] predictions = graph.get_operation_by_name("predictions").outputs[0] batches = batch_iter(list(x_word_ids), batch_size=128, num_epochs=1, shuffle=False) all_predictions = [] for x_test_batch in batches: batch_predictions = sess.run(predictions, {input_x: x_test_batch}) all_predictions = np.concatenate([all_predictions, batch_predictions]) correct_predictions = sum(all_predictions == np.argmax(y, axis=1)) print("Total number of test examples: {}".format(len(y))) print("Accuracy: {:g}".format(float(correct_predictions) / float(len(y)))) # Save the evaluation to a csv predictions_human_readable = np.column_stack((np.array(x_text), all_predictions)) out_path = os.path.join("./prediction.csv") print("Saving evaluation to {0}".format(out_path)) with open(out_path, 'w') as f:
feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summeries, loss, accuracy = sess.run( [global_step, train_summary_op, cnn.losses, cnn.accuracy], feed_dict=feed_dict) time_str = datetime.datetime.now().isoformat() if model == 'test': print('test {}: step:{} , loss:{} , acc:{}'.format( time_str, step, loss, accuracy)) else: print('val {}: step:{} , loss:{} , acc:{}'.format( time_str, step, loss, accuracy)) batchs = batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.epochs) for batch in batchs: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: print('\n evaluate_every') val_test_step(x_test, y_test, 'test') val_test_step(x_val, y_val, 'val') if current_step % FLAGS.checkpoint_every == 0: path = saver.save(sess, './model/', global_step=current_step) print("Saved model checkpoint to {}\n".format(path))
json.dump(vocab, fp) # open session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # make model instance model = seq2seq(sess, encoder_vocab_size=vocab_size, decoder_vocab_size=vocab_size) # make train batches input, target = make_dataset(data) batches = batch_iter(list(zip(input, target)), batch_size=64, num_epochs=1001) # model saver saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=0.5) # train model print('모델 훈련을 시작합니다.') avgLoss = [] for step, batch in enumerate(batches): x_train, y_train = zip(*batch) x_train = sentence_to_char_index(x_train, vocab, is_target=False) y_train = sentence_to_char_index(y_train, vocab, is_target=True) l, _ = model.train(x_train, y_train) avgLoss.append(l) if step % 100 == 0:
def train_and_evaluate(self, x_train, y_train, x_test, y_test): global_step = tf.Variable(0, trainable=False) train_op = tf.train.AdamOptimizer().minimize(self.loss, global_step) # summaries loss_summary = tf.summary.scalar('loss', self.loss) accuracy_summary = tf.summary.scalar('accuracy', self.accuracy) summary_op = tf.summary.merge([loss_summary, accuracy_summary]) with tf.Session() as sess: train_summary_writer = tf.summary.FileWriter( Config.train_summary_dir, sess.graph) test_summary_writer = tf.summary.FileWriter( Config.test_summary_dir, sess.graph) saver = tf.train.Saver(max_to_keep=Config.keep_checkpoints_max) # 初始化所有变量 sess.run(tf.global_variables_initializer()) if not os.path.exists(Config.checkpoint_dir): os.makedirs(Config.checkpoint_dir) else: checkpoint_file = tf.train.latest_checkpoint( Config.checkpoint_dir) if checkpoint_file is not None: # 加载检查点文件 saver.restore(sess, checkpoint_file) tf.logging.info( "<<<< restore model from: {}".format(checkpoint_file)) train_dataset = list(zip(x_train, y_train)) train_batch_iter = batch_iter(train_dataset, Config.batch_size, num_epochs=Config.num_epochs, shuffle=False) for batch in train_batch_iter: batch_xs, batch_ys = zip(*batch) _, train_loss, train_acc, train_summary = sess.run( [train_op, self.loss, self.accuracy, summary_op], feed_dict={ self.x: batch_xs, self.y_: batch_ys }) step = tf.train.global_step(sess, global_step) train_summary_writer.add_summary(train_summary, step) # print("step {}, loss {:g}, acc {:g}".format(step, _train_loss, _train_acc)) # 保存模型检查点 if step % Config.save_checkpoints_steps == 0: checkpoint_prefix = os.path.join(Config.checkpoint_dir, 'model') save_path = saver.save(sess, checkpoint_prefix, global_step=step) tf.logging.info( ">>>> save model in file: {}".format(save_path)) # 在测试集上计算模型的准确度 if step % Config.eval_per_steps == 0: test_acc, test_loss, test_summary = sess.run( [self.accuracy, self.loss, summary_op], feed_dict={ self.x: x_test, self.y_: y_test }) print( 'step {} - loss: {:.4f}, aac: {:.4f}, test loss: {:.4f}, test acc: {:.4f}' .format(step, train_loss, train_acc, test_loss, test_acc)) test_summary_writer.add_summary(test_summary, step)
if not os.path.exists(tensorboard_dir): os.makedirs(tensorboard_dir) if not os.path.exists(save_dir): os.makedirs(save_dir) save_path = os.path.join(save_dir, 'best_validation') tf.summary.scalar("loss", loss) merged_summary = tf.summary.merge_all() writer = tf.summary.FileWriter(tensorboard_dir) saver = tf.train.Saver() writer.add_graph(session.graph) for epoch in range(pm.num_epochs): print('Epoch:', epoch + 1) num_batchs = int((len(label) - 1) / pm.batch_size) + 1 batch_train = batch_iter(input_id, input_segment_, mask_, label, pm.batch_size) for x_id, x_segment, x_mask, y_label in batch_train: n += 1 sequ_length = sequence(x_id) feed_dict = feed_data(x_id, x_mask, x_segment, y_label, sequ_length, pm.keep_prob) _, train_summary, train_loss = session.run( [train_op, merged_summary, loss], feed_dict=feed_dict) if n % 100 == 0: print('step:', n, 'loss:', train_loss) # P = evaluate(session, test_id, test_segment, test_mask, test_label) # print('测试集准确率:', P) # if P > best: # best = P if (epoch + 1) % 5 == 0:
def train(): print("Loading data...") train_data, val_data, test_data = load_data(FLAGS.training, FLAGS.validation, FLAGS.testing) print("Train {}".format(np.array(train_data).shape)) print("Initializing...") with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) sess = tf.Session(config=session_conf) with sess.as_default(): cnn = Trash_CNN( num_classes=FLAGS.num_classes, input_shape=(FLAGS.input_size, FLAGS.input_size, 3), filters=list(map(int, FLAGS.filter_sizes.split(","))), input_channel=FLAGS.num_filters) global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars) timestamp = str(int(time.time())) outdir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) print("Writing to {}\n".format(outdir)) loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("acc", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge([loss_summary, acc_summary]) train_summary_dir = os.path.join(outdir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(outdir, "summaries", "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(outdir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) sess.run(tf.global_variables_initializer()) def train_step(batch_x, batch_y): ''' One single training step ''' feed_dict = {cnn.input_x: batch_x, cnn.input_y: batch_y} _, step, summaries, loss, accuracy = sess.run( [ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict=feed_dict) time_str = datetime.datetime.now().isoformat() print("{}: step {}, loss {:g}, acc: {:g}".format( time_str, step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(batch_x, batch_y, writer=None): ''' Evaluate model ''' feed_dict = {cnn.input_x: batch_x, cnn.input_y: batch_y} step, summaries, loss, accuracy = sess.run( [global_step, train_summary_op, cnn.loss, cnn.accuracy], feed_dict=feed_dict) time_str = datetime.datetime.now().isoformat() print("-------------- Step summary ---------------") print("{}: step {}, loss {:g}, acc: {:g}".format( time_str, step, loss, accuracy)) if writer: writer.add_summary(summaries, step) batches = batch_iter(list(zip(train_data[0], train_data[1])), FLAGS.batch_size, FLAGS.num_epochs) for batch in batches: print("Batch {}".format(batch.shape)) x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % FLAGS.evaluate_every == 0: print( "\n======================= Evaluation: =======================" ) dev_step(val_data[0], val_data[1], writer=dev_summary_writer) print("") if current_step % FLAGS.checkpoint_every == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) print("Saved model checkpoint to {}\n".format(path))
vocab, reverse_vocab, vocab_size = build_vocab(data) # save vocab with open('vocab.json', 'w') as fp: json.dump(vocab, fp) # open session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # make model instance model = reRNN(sess=sess, vocab_size=vocab_size, lr=1e-1) # make train batches batches = batch_iter(data, batch_size=64, num_epochs=1001) # model saver saver = tf.train.Saver(max_to_keep=3, keep_checkpoint_every_n_hours=0.5) # train model print('모델 훈련을 시작합니다.') avgLoss = [] for step, batch in enumerate(batches): x_train, y_train = sentenceToIndex(batch, vocab) l, _ = model.train(x_train, y_train) avgLoss.append(l) if step % 100 == 0: print('batch:', '%04d' % step, 'loss:', '%.5f' % np.mean(avgLoss)) saver.save(sess, os.path.join(DIR, 'my-model.ckpt'),