time.time() - timeStart, currLearningRate[0],
currLearningRate[1]))
if (i + 1) % 5000 == 0:
# update image summaries
if imageSummaries is not None:
summary = sess.run(imageSummaries, feed_dict=trainBatch)
tfSummaryWriter.add_summary(summary, i + 1)
# evaluate on validation and test sets
validAccuracy = data.evaluate(validData, imageRawBatch, labelBatch,
prediction, sess, params)
validError = (1 - validAccuracy) * 100
print('Iter {} Accuracy: {}'.format(i, validAccuracy))
if validAccuracy > best_validation_accuracy:
best_validation_accuracy = validAccuracy
else:
params.baseLR = params.baseLR / 10
if params.baseLR <= 0.0001:
params.baseLR = 0.0001
summary = sess.run(validSummary,
feed_dict={validErrorPH: validError})
tfSummaryWriter.add_summary(summary, i + 1)
# save model
savePath = tfSaver.save(
sess,
"models_{2}/{0}_it{1}k.ckpt".format(saveFname, (i + 1) // 1000,
suffix))
print("model saved: {0}".format(savePath))
if (i + 1) % 10000 == 0:
# save intermediate model
tfSaverInterm.save(
sess, "models_{2}/interm/{0}_it{1}k.ckpt".format(
