epoch_num, val_metric_per_epoch[-1], val_loss_avg),
flush=True)
writer.add_scalar('loss/validation', val_loss_avg, epoch_num)
writer.add_scalar('accuracy/validation', val_metric_per_epoch[-1],
epoch_num)
global_val_loss.append(val_loss_avg)
global_val_acc.append(val_metric_per_epoch[-1])
if int(np.argmax(val_metric_per_epoch)) < (len(val_metric_per_epoch) - 1 -
params['trainer']['patience']):
print("Stopping at epoch {:2d}".format(epoch_num))
break
if scheduler:
save_checkpoint(model,
optimizer,
args.folder,
epoch_num,
val_metric_per_epoch,
is_best=int(np.argmax(val_metric_per_epoch)) == (
len(val_metric_per_epoch) - 1),
learning_rate_scheduler=scheduler)
else:
save_checkpoint(model,
optimizer,
args.folder,
epoch_num,
val_metric_per_epoch,
is_best=int(np.argmax(val_metric_per_epoch)) == (
len(val_metric_per_epoch) - 1))
# if epoch_num == 1:
# break
writer.close()
float(np.mean(val_labels == val_probs.argmax(1))))
if scheduler:
scheduler.step(val_metric_per_epoch[-1], epoch_num)
print("Val epoch {} has acc {:.3f} and loss {:.3f}".format(
epoch_num, val_metric_per_epoch[-1], val_loss_avg),
flush=True)
if int(np.argmax(val_metric_per_epoch)) < (len(val_metric_per_epoch) - 1 -
params['trainer']['patience']):
print("Stopping at epoch {:2d}".format(epoch_num))
break
save_checkpoint(
model,
optimizer,
args.folder,
epoch_num,
val_metric_per_epoch,
is_best=int(
np.argmax(val_metric_per_epoch)) == (len(val_metric_per_epoch) -
1))
print("STOPPING. now running the best model on the validation set", flush=True)
# Load best
restore_best_checkpoint(model, args.folder)
model.eval()
val_probs = []
val_labels = []
for b, (time_per_batch, batch) in enumerate(time_batch(val_loader)):
with torch.no_grad():
batch = _to_gpu(batch)
output_dict = model(**batch)
epoch_num, val_metric_per_epoch[-1], val_loss_avg),
flush=True)
logger.write("Val epoch {} has acc {:.4f} and loss {:.4f}".format(
epoch_num, val_metric_per_epoch[-1], val_loss_avg))
if int(np.argmax(val_metric_per_epoch)) < (len(val_metric_per_epoch) - 1 -
params['trainer']['patience']):
print("Stopping at epoch {:2d}".format(epoch_num))
logger.write("Stopping at epoch {:2d}".format(epoch_num))
break
save_checkpoint(
model,
optimizer,
folder,
epoch_num,
val_metric_per_epoch,
is_best=int(
np.argmax(val_metric_per_epoch)) == (len(val_metric_per_epoch) -
1),
q_att1=q_att1,
a_att1=a_att1,
q_att2=q_att2,
a_att2=a_att2)
writer.add_scalar('val_loss', val_loss_avg, global_step=epoch_num)
writer.add_scalar('val_accuracy',
val_metric_per_epoch[-1],
global_step=epoch_num)
print("STOPPING. now running the best model on the validation set", flush=True)
logger.write("STOPPING. now running the best model on the validation set")
# Load best
restore_best_checkpoint(model, folder)