elif args.train_mode == Args.TrainMode.continuation_from_autosaved2_best_model: logger.info("Using the (best) AUTOSAVED2 model to continue the training.") load_model_path = t2s_model.autosaved2_best_model_path else: raise ValueError(f"{args.train_mode=}") elif ( t2s_model.model_path.exists() or t2s_model.autosaved_model_path.exists() or ( t2s_model.autosaved2_best_model_path is not None and t2s_model.autosaved2_best_model_path.exists() ) ): logger.error( f"The model seems to already exist but this is not a continuation. Please, make sure the arguments are correct.") raise ValueError( f"{args.train_mode=} ==> {args.train_mode.is_continuation=} {t2s_model.name=}") elif args.train_mode == Args.TrainMode.from_scratch: logger.info(f"A new model will be instantiated!") else: raise NotImplementedError(f"{args.train_mode=}") with gpu_strategy.scope(): if args.train_mode.is_continuation: assert load_model_path.exists( ), f"Inconsistent arguments {args.train_mode.is_continuation=} {load_model_path=} {load_model_path.exists()=}."
def notify_error(): logger.error("Sending notification error during the training!") notify("A problem occurred during the training.")