model_dir = os.path.join(args.finetune_model, "nnet")
# Set the random seed. The random operations may appear in data input, batch forming, etc.
tf.set_random_seed(params.seed)
random.seed(params.seed)
np.random.seed(params.seed)
dim = FeatureReader(args.train_dir).get_dim()
if "selected_dim" in params.dict:
dim = params.selected_dim
with open(os.path.join(model_dir, "feature_dim"), "w") as f:
f.write("%d\n" % dim)
num_total_train_speakers = KaldiDataRandomQueue(args.train_dir, args.train_spklist).num_total_speakers
tf.logging.info("There are %d speakers in the training set and the dim is %d" % (num_total_train_speakers, dim))
min_valid_loss = ValidLoss()
# The trainer is used to control the training process
trainer = Trainer(params, args.finetune_model, dim, num_total_train_speakers)
trainer.build("train")
trainer.build("valid")
# Load the pre-trained model and transfer to current model
trainer.get_finetune_model(params.noload_var_list)
trainer.train_tune_lr(args.train_dir, args.train_spklist, args.tune_period)
trainer.close()
tf.logging.info("Finish tuning.")
# the code doesn't know the previous learning rate if it is tuned using the validation set.
# To solve that, just save the learning rate to an individual file.
if os.path.isfile(os.path.join(model_dir, "learning_rate")):
learning_rate_array = load_lr(os.path.join(model_dir, "learning_rate"))
assert len(learning_rate_array) == start_epoch + 1, "Not enough learning rates in the learning_rate file."
else:
learning_rate_array = [float(learning_rate)] * (start_epoch + 1)
dim = FeatureReader(args.train_dir).get_dim()
with open(os.path.join(model_dir, "feature_dim"), "w") as f:
f.write("%d\n" % dim)
num_total_train_speakers = KaldiDataRandomQueue(args.train_dir, args.train_spklist).num_total_speakers
tf.logging.info("There are %d speakers in the training set and the dim is %d" % (num_total_train_speakers, dim))
# Load the history valid loss
min_valid_loss = ValidLoss()
if os.path.isfile(os.path.join(model_dir, "valid_loss")):
min_valid_loss = load_valid_loss(os.path.join(model_dir, "valid_loss"))
# The trainer is used to control the training process
trainer = Trainer(params, args.model)
trainer.build("train",
dim=dim,
loss_type=params.loss_func,
num_speakers=num_total_train_speakers)
trainer.build("valid",
dim=dim,
loss_type=params.loss_func,
num_speakers=num_total_train_speakers)
if "early_stop_epochs" not in params.dict: