lr_scheduler.step(validation_logs['loss'])
if epoch > 0:
ckpter.last_delete_and_save(epoch=epoch,
monitor='acc',
loss_acc=validation_logs)
ckpter_lr.last_delete_and_save(epoch=epoch,
monitor='acc',
loss_acc=validation_logs)
ckpter_auc.last_delete_and_save(epoch=epoch,
monitor='auc',
loss_acc=validation_logs)
ckpter_auc_lr.last_delete_and_save(epoch=epoch,
monitor='auc',
loss_acc=validation_logs)
ckpter.check_on(epoch=epoch, monitor='acc', loss_acc=validation_logs)
ckpter_lr.check_on(epoch=epoch,
monitor='acc',
loss_acc=validation_logs)
ckpter_auc.check_on(epoch=epoch,
monitor='auc',
loss_acc=validation_logs)
ckpter_auc_lr.check_on(epoch=epoch,
monitor='auc',
loss_acc=validation_logs)
dill.dump(train_hist,
file=open(
ROOT_DIR + "/ckpt/" + exp_name + train_hist.name +
".pickle", "wb"))
dill.dump(validation_hist,
file=open(
train_hist.add(logs=train_logs, epoch=epoch + 1)
epoch_time_end = time.time()
# --------------------------------------------------------------------------------------
# Save last model parameters and check if it is the best
# --------------------------------------------------------------------------------------
if epoch > 0:
ckpter.last_delete_and_save(epoch=epoch,
monitor='acc',
loss_acc=validation_logs)
ckpter_v2.last_delete_and_save(epoch=epoch,
monitor='acc001',
loss_acc=validation_logs)
if num_triplets:
ckpter.check_on(epoch=epoch,
monitor='acc',
loss_acc=validation_logs)
ckpter_v2.check_on(epoch=epoch,
monitor='acc001',
loss_acc=validation_logs)
print(
'Epoch {}:\tAverage Triplet Loss: {:.3f}\tEpoch Time: {:.3f} hours\tNumber of valid training triplets in epoch: {}'
.format(epoch + 1, avg_triplet_loss,
(epoch_time_end - epoch_time_start) / 3600, num_triplets))
dill.dump(train_hist,
file=open(
"ckpt/" + triplet_method + train_hist.name + ".pickle",
"wb"))
dill.dump(validation_hist,
file=open(
"ckpt/" + triplet_method + validation_hist.name +