def train(epoch, dataset, config, log_dir): """Train model for one epoch.""" model_config = config['model'] train_config = config['train'] sess_config = config['session'] with tf.Graph().as_default(): model = GRA(model_config) model.build_inference() model.build_loss(train_config['reg_coeff'], train_config['shu_coeff']) model.build_train(train_config['learning_rate']) with tf.Session(config=sess_config) as sess: sum_dir = os.path.join(log_dir, 'summary') # create event file for graph if not os.path.exists(sum_dir): summary_writer = tf.summary.FileWriter(sum_dir, sess.graph) summary_writer.close() summary_writer = tf.summary.FileWriter(sum_dir) ckpt_dir = os.path.join(log_dir, 'checkpoint') ckpt_path = tf.train.latest_checkpoint(ckpt_dir) saver = tf.train.Saver() if ckpt_path: print('load checkpoint {}.'.format(ckpt_path)) lajidaima = int(ckpt_path.split('-')[-1]) - epoch + 1 saver.restore(sess, ckpt_path) else: print('no checkpoint.') if not os.path.exists(ckpt_dir): os.makedirs(ckpt_dir) sess.run(tf.global_variables_initializer()) epoch += lajidaima stats_dir = os.path.join(log_dir, 'stats') stats_path = os.path.join(stats_dir, 'train.json') if os.path.exists(stats_path): print('load stats file {}.'.format(stats_path)) stats = pd.read_json(stats_path, 'records') else: print('no stats file.') if not os.path.exists(stats_dir): os.makedirs(stats_dir) stats = pd.DataFrame(columns=['epoch', 'loss', 'acc']) # train iterate over batch batch_idx = 0 total_loss = 0 total_acc = 0 batch_total = np.sum(dataset.train_batch_total) while dataset.has_train_batch: vgg, c3d, question, answer = dataset.get_train_batch() vgg = np.zeros((len(vgg), len(vgg[0]), len(vgg[0][0]))) feed_dict = { model.appear: vgg, model.motion: c3d, model.question_encode: question, model.answer_encode: answer } _, loss, prediction = sess.run( [model.train, model.loss, model.prediction], feed_dict) # cal acc correct = 0 for i, row in enumerate(prediction[1]): #print(row) for index in row: if answer[i][index] == 1: correct += 1 break acc = correct / len(answer) total_loss += loss total_acc += acc if batch_idx % 10 == 0: print( '[TRAIN] epoch {}, batch {}/{}, loss {:.5f}, acc {:.5f}.' .format(epoch, batch_idx, batch_total, loss, acc)) batch_idx += 1 loss = total_loss / batch_total acc = total_acc / batch_total print('\n[TRAIN] epoch {}, loss {:.5f}, acc {:.5f}.\n'.format( epoch, loss, acc)) summary = tf.Summary() summary.value.add(tag='train/loss', simple_value=float(loss)) summary.value.add(tag='train/acc', simple_value=float(acc)) summary_writer.add_summary(summary, epoch) record = Series([epoch, loss, acc], ['epoch', 'loss', 'acc']) stats = stats.append(record, ignore_index=True) saver.save(sess, os.path.join(ckpt_dir, 'model.ckpt'), epoch) stats.to_json(stats_path, 'records') dataset.reset_train() return loss, acc
def val(epoch, dataset, config, log_dir): """Validate model.""" model_config = config['model'] sess_config = config['session'] train_config = config['train'] answerset = pd.read_csv( os.path.join(config['preprocess_dir'], 'answer_set.txt'), header=None)[0] example_id = 0 with tf.Graph().as_default(): model = GRA(model_config) model.build_inference() model.build_loss(train_config['reg_coeff'], train_config['shu_coeff']) result = DataFrame(columns=['id', 'answer']) with tf.Session(config=sess_config) as sess: sum_dir = os.path.join(log_dir, 'summary') summary_writer = tf.summary.FileWriter(sum_dir) ckpt_dir = os.path.join(log_dir, 'checkpoint') save_path = tf.train.latest_checkpoint(ckpt_dir) saver = tf.train.Saver() last_epoch = 0 if save_path: print('load checkpoint {}.'.format(save_path)) last_epoch = int(save_path.split('-')[-1]) - epoch saver.restore(sess, save_path) else: print('no checkpoint.') exit() stats_dir = os.path.join(log_dir, 'stats') stats_path = os.path.join(stats_dir, 'val.json') if os.path.exists(stats_path): print('load stats file {}.'.format(stats_path)) stats = pd.read_json(stats_path, 'records') else: print('no stats file.') if not os.path.exists(stats_dir): os.makedirs(stats_dir) stats = pd.DataFrame(columns=['epoch', 'acc']) # val iterate over examples correct = 0 loss_total = 0 while dataset.has_val_example: vgg, c3d, question, answer = dataset.get_val_example() feed_dict = { model.appear: [vgg], model.question_encode: [question], model.answer_encode: [answer] } loss, prediction = sess.run([model.loss, model.prediction], feed_dict=feed_dict) loss_total += loss prediction = prediction[1] for i, row in enumerate(prediction): for index in row: if answer[index] == 1: correct += 1 break result = result.append({'id': example_id, 'answer': prediction}, ignore_index=True) example_id += 1 acc = correct / dataset.val_example_total loss = loss_total / dataset.val_example_total result.to_json(os.path.join( log_dir, 'validation_' + str(int(acc * 100)) + '_' + str(epoch + last_epoch) + '.json'), 'records') print('\n[VAL] epoch {}, acc {:.5f}.\n'.format(epoch + last_epoch, acc)) summary = tf.Summary() summary.value.add(tag='val/acc', simple_value=float(acc)) summary.value.add(tag='val/loss', simple_value=float(loss)) summary_writer.add_summary(summary, epoch + last_epoch) record = Series([epoch + last_epoch, acc, loss], ['epoch', 'acc', 'loss']) stats = stats.append(record, ignore_index=True) stats.to_json(stats_path, 'records') dataset.reset_val() return acc