if args.val_ratio > 0:
val_loss, val_metrics = validate(epoch, model, val_loader)
loss_str = "Validation"
loss_value_str = f"{val_loss:.5f}"
val_str = f"{loss_str} loss: {loss_value_str} "
improved = val_loss < best_loss
else:
loss_str = "Training"
loss_value_str = f"{train_loss:.5f}"
val_str = ""
improved = train_loss < best_loss
if improved:
# If model improved, save the model
model.save(model_save_name)
print(
f"{loss_str} loss improved from {best_loss:.5f} to {loss_value_str}. Saved model to: {model_save_name}"
)
best_loss = val_loss if args.val_ratio > 0 else train_loss
patience = args.patience
with torch.no_grad():
rho = mutation_effect_prediction(model,
args.data,
args.query_protein,
args.data_sheet,
args.metric_column,
device,
100,
args.results_dir,
print_seqs_iteration_time = time.time()
print_seqs_count = 0
if seqs_processed >= train_seqs_per_epoch:
epoch += 1
train_loss = acc_train_loss / train_loss_count
seqs_processed = 0
acc_train_loss = 0
train_loss_count = 0
val_loss, _ = validate(epoch, model, val_loader)
# If save the latest model
if args.multi_gpu:
model.module.save(model_save_name_latest)
else:
model.save(model_save_name_latest)
improved = val_loss < best_val_loss
if improved:
# If model improved, save the model
if args.multi_gpu:
model.module.save(model_save_name)
else:
model.save(model_save_name)
print(f"Validation loss improved from {best_val_loss:.5f} to {val_loss:.5f}. Saved model to: {model_save_name}")
best_val_loss = val_loss
patience = args.patience
elif args.patience:
# If save path and patience was specified, and model has not improved, decrease patience and possibly stop