model_dir = os.path.join(args.finetune_model, "nnet") # Set the random seed. The random operations may appear in data input, batch forming, etc. tf.set_random_seed(params.seed) random.seed(params.seed) np.random.seed(params.seed) dim = FeatureReader(args.train_dir).get_dim() if "selected_dim" in params.dict: dim = params.selected_dim with open(os.path.join(model_dir, "feature_dim"), "w") as f: f.write("%d\n" % dim) num_total_train_speakers = KaldiDataRandomQueue(args.train_dir, args.train_spklist).num_total_speakers tf.logging.info("There are %d speakers in the training set and the dim is %d" % (num_total_train_speakers, dim)) min_valid_loss = ValidLoss() # The trainer is used to control the training process trainer = Trainer(params, args.finetune_model, dim, num_total_train_speakers) trainer.build("train") trainer.build("valid") # Load the pre-trained model and transfer to current model trainer.get_finetune_model(params.noload_var_list) trainer.train_tune_lr(args.train_dir, args.train_spklist, args.tune_period) trainer.close() tf.logging.info("Finish tuning.")
# the code doesn't know the previous learning rate if it is tuned using the validation set. # To solve that, just save the learning rate to an individual file. if os.path.isfile(os.path.join(model_dir, "learning_rate")): learning_rate_array = load_lr(os.path.join(model_dir, "learning_rate")) assert len(learning_rate_array) == start_epoch + 1, "Not enough learning rates in the learning_rate file." else: learning_rate_array = [float(learning_rate)] * (start_epoch + 1) dim = FeatureReader(args.train_dir).get_dim() with open(os.path.join(model_dir, "feature_dim"), "w") as f: f.write("%d\n" % dim) num_total_train_speakers = KaldiDataRandomQueue(args.train_dir, args.train_spklist).num_total_speakers tf.logging.info("There are %d speakers in the training set and the dim is %d" % (num_total_train_speakers, dim)) # Load the history valid loss min_valid_loss = ValidLoss() if os.path.isfile(os.path.join(model_dir, "valid_loss")): min_valid_loss = load_valid_loss(os.path.join(model_dir, "valid_loss")) # The trainer is used to control the training process trainer = Trainer(params, args.model) trainer.build("train", dim=dim, loss_type=params.loss_func, num_speakers=num_total_train_speakers) trainer.build("valid", dim=dim, loss_type=params.loss_func, num_speakers=num_total_train_speakers) if "early_stop_epochs" not in params.dict: