elif args.train_mode == Args.TrainMode.continuation_from_autosaved2_best_model:
logger.info("Using the (best) AUTOSAVED2 model to continue the training.")
load_model_path = t2s_model.autosaved2_best_model_path
else:
raise ValueError(f"{args.train_mode=}")
elif (
t2s_model.model_path.exists()
or t2s_model.autosaved_model_path.exists()
or (
t2s_model.autosaved2_best_model_path is not None
and t2s_model.autosaved2_best_model_path.exists()
)
):
logger.error(
f"The model seems to already exist but this is not a continuation. Please, make sure the arguments are correct.")
raise ValueError(
f"{args.train_mode=} ==> {args.train_mode.is_continuation=} {t2s_model.name=}")
elif args.train_mode == Args.TrainMode.from_scratch:
logger.info(f"A new model will be instantiated!")
else:
raise NotImplementedError(f"{args.train_mode=}")
with gpu_strategy.scope():
if args.train_mode.is_continuation:
assert load_model_path.exists(
), f"Inconsistent arguments {args.train_mode.is_continuation=} {load_model_path=} {load_model_path.exists()=}."
def notify_error():
logger.error("Sending notification error during the training!")
notify("A problem occurred during the training.")