def evaluate(step, sess, model, summary_writer):
batch_gen = BatchGenerator(VALIDATION_DATA, FLAGS, isTest=True)
eval_step = 0
summary = None
while True:
batch = batch_gen.get_batch()
if batch is None:
break
recalls, loss, summary = model.validate(batch)
if eval_step != 0 and eval_step % FLAGS.save_summary_every == 0:
print("Loss:{0:.5f} Recall@1 = {1:.5f} Recall@2 = {2:.5f} "
"Recall@3 = {3:.5f} Recall@5 = {4:.5f}".format(
loss, recalls[0], recalls[1], recalls[2], recalls[3]))
eval_step += 1
summary_writer.add_summary(summary, step)
def train_loop(sess, model, summary_writer, model_saver):
batch_gen = BatchGenerator(TRAIN_DATA, FLAGS)
epoch = 0
while True:
batch = batch_gen.get_batch()
if batch is None:
tf.logging.info("Epoch {0} is over".format(epoch))
epoch += 1
del batch_gen
batch_gen = BatchGenerator(
TRAIN_DATA, FLAGS) # create batch generator again and proceed
continue
loss, step, summary = model.batch_fit(batch)
if step % FLAGS.save_summary_every == 0:
summary_writer.add_summary(summary, step)
tf.logging.info("Step: {0} Loss: {1}".format(step, loss))
if step != 0 and step % FLAGS.eval_every == 0:
save_signal = evaluate(step, sess, model, summary_writer)
if save_signal:
model.save_model(model_saver, MODEL_DIR, step)