def compute_dataset_stats(self):
initial_index = 0
attempts = 10
current_attempt = 0
total_length = len(self._training_loader)
train_mfccs = list()
while current_attempt < attempts:
try:
i = initial_index
train_bar = tqdm(self._training_loader, initial=initial_index)
for data in train_bar:
input_features = data['input_features']
train_mfccs.append(input_features.detach().view(input_features.size(1), input_features.size(2)).numpy())
i += 1
if i == total_length:
train_bar.update(total_length)
break
train_bar.close()
break
except KeyboardInterrupt:
train_bar.close()
ConsoleLogger.warn('Keyboard interrupt detected. Leaving the function...')
return
except:
error_message = 'An error occured in the data loader at {}. Current attempt: {}/{}'.format(i, current_attempt+1, attempts)
self._logger.exception(error_message)
ConsoleLogger.error(error_message)
initial_index = i
current_attempt += 1
continue
ConsoleLogger.status('Compute mean of mfccs training set...')
train_mean = np.concatenate(train_mfccs).mean(axis=0)
ConsoleLogger.status('Compute std of mfccs training set...')
train_std = np.concatenate(train_mfccs).std(axis=0)
stats = {
'train_mean': train_mean,
'train_std': train_std
}
ConsoleLogger.status('Writing stats in file...')
with open(self._normalizer_path, 'wb') as file: # TODO: do not use hardcoded path
pickle.dump(stats, file)
train_mfccs_norm = (train_mfccs[0] - train_mean) / train_std
ConsoleLogger.status('Computing example plot...')
_, axs = plt.subplots(2, sharex=True)
axs[0].imshow(train_mfccs[0].T, aspect='auto', origin='lower')
axs[0].set_ylabel('Unnormalized')
axs[1].imshow(train_mfccs_norm.T, aspect='auto', origin='lower')
axs[1].set_ylabel('Normalized')
plt.savefig('mfcc_normalization_comparaison.png') # TODO: do not use hardcoded path
def plot_training_losses(self, experiments, experiments_path):
all_train_losses = list()
all_train_perplexities = list()
all_results_paths = list()
all_experiments_names = list()
all_latest_epochs = list()
for experiment in experiments:
try:
train_res_losses, train_res_perplexities, latest_epoch = \
CheckpointUtils.retreive_losses_values(experiments_path, experiment)
all_train_losses.append(train_res_losses)
all_train_perplexities.append(train_res_perplexities)
all_results_paths.append(experiment.results_path)
all_experiments_names.append(experiment.name)
all_latest_epochs.append(latest_epoch)
except:
ConsoleLogger.error(
"Failed to retreive losses of experiment '{}'".format(
experiment.name))
n_final_losses_colors = len(all_train_losses)
final_losses_colors = self._get_colors_from_cmap(
self._colormap_name, n_final_losses_colors)
# for each experiment: final loss + perplexity
self._plot_loss_and_perplexity_figures(
all_results_paths, all_experiments_names, all_train_losses,
all_train_perplexities, all_latest_epochs, n_final_losses_colors,
final_losses_colors)
# merged experiment: merged final losses + merged perplexities
self._plot_merged_losses_and_perplexities_figure(
all_results_paths, all_experiments_names, all_train_losses,
all_train_perplexities, all_latest_epochs, n_final_losses_colors,
final_losses_colors)
# for each experiment: all possible losses
self._plot_merged_all_losses_figures(all_results_paths,
all_experiments_names,
all_train_losses,
all_train_perplexities,
all_latest_epochs)
# merged losses of a single type in all experiments
self._plot_merged_all_losses_type(all_results_paths,
all_experiments_names,
all_train_losses,
all_train_perplexities,
all_latest_epochs)
def load(experiments_path,
experiment_name,
results_path,
data_path='../data'):
error_caught = False
try:
configuration_file, checkpoint_files = PipelineFactory.load_configuration_and_checkpoints(
experiments_path, experiment_name)
except:
ConsoleLogger.error(
'Failed to load existing configuration. Building a new model...'
)
error_caught = True
# Load the configuration file
ConsoleLogger.status('Loading the configuration file')
configuration = None
with open(experiments_path + os.sep + configuration_file, 'r') as file:
configuration = yaml.load(file, Loader=yaml.FullLoader)
device_configuration = DeviceConfiguration.load_from_configuration(
configuration)
if error_caught or len(checkpoint_files) == 0:
trainer, evaluator = PipelineFactory.build(configuration,
device_configuration,
experiments_path,
experiment_name,
results_path)
else:
latest_checkpoint_file, latest_epoch = CheckpointUtils.search_latest_checkpoint_file(
checkpoint_files)
# Update the epoch number to begin with for the future training
configuration['start_epoch'] = latest_epoch
configuration['num_epochs'] = 60
#latest_checkpoint_file = 'baseline_15_checkpoint.pth'
#print(latest_checkpoint_file)
# Load the checkpoint file
checkpoint_path = experiments_path + os.sep + latest_checkpoint_file
ConsoleLogger.status(
"Loading the checkpoint file '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path,
map_location=device_configuration.device)
# Load the data stream
ConsoleLogger.status('Loading the data stream')
data_stream = VCTKFeaturesStream('/atlas/u/xuyilun/vctk',
configuration,
device_configuration.gpu_ids,
device_configuration.use_cuda)
def load_state_dicts(model, checkpoint, model_name,
optimizer_name):
# Load the state dict from the checkpoint to the model
model.load_state_dict(checkpoint[model_name])
# Create an Adam optimizer using the model parameters
optimizer = optim.Adam(model.parameters())
# Load the state dict from the checkpoint to the optimizer
optimizer.load_state_dict(checkpoint[optimizer_name])
# Map the optimizer memory into the specified device
for state in optimizer.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
state[k] = v.to(device_configuration.device)
return model, optimizer
# If the decoder type is a deconvolutional
if configuration['decoder_type'] == 'deconvolutional':
# Create the model and map it to the specified device
vqvae_model = ConvolutionalVQVAE(
configuration, device_configuration.device).to(
device_configuration.device)
evaluator = Evaluator(device_configuration.device, vqvae_model,
data_stream, configuration, results_path,
experiment_name)
# Load the model and optimizer state dicts
vqvae_model, vqvae_optimizer = load_state_dicts(
vqvae_model, checkpoint, 'model', 'optimizer')
elif configuration['decoder_type'] == 'wavenet':
vqvae_model = WaveNetVQVAE(configuration,
data_stream.speaker_dic,
device_configuration.device).to(
device_configuration.device)
evaluator = Evaluator(device_configuration.device, vqvae_model,
data_stream, configuration, results_path,
experiment_name)
# Load the model and optimizer state dicts
vqvae_model, vqvae_optimizer = load_state_dicts(
vqvae_model, checkpoint, 'model', 'optimizer')
else:
raise NotImplementedError(
"Decoder type '{}' isn't implemented for now".format(
configuration['decoder_type']))
# Temporary backward compatibility
if 'trainer_type' not in configuration:
ConsoleLogger.error(
"trainer_type was not found in configuration file. Use 'convolutional' by default."
)
configuration['trainer_type'] = 'convolutional'
if configuration['trainer_type'] == 'convolutional':
trainer = ConvolutionalTrainer(
device_configuration.device, data_stream, configuration,
experiments_path, experiment_name, **{
'model': vqvae_model,
'optimizer': vqvae_optimizer
})
else:
raise NotImplementedError(
"Trainer type '{}' isn't implemented for now".format(
configuration['trainer_type']))
# Use data parallelization if needed and available
vqvae_model = vqvae_model
return trainer, evaluator, configuration, device_configuration
def export_to_features(self, vctk_path, configuration):
if not os.path.isdir(vctk_path):
raise ValueError(
"VCTK dataset not found at path '{}'".format(vctk_path))
# Create the features path directory if it doesn't exist
features_path = vctk_path + os.sep + configuration['features_path']
if not os.path.isdir(features_path):
ConsoleLogger.status(
'Creating features directory at path: {}'.format(
features_path))
os.mkdir(features_path)
else:
ConsoleLogger.status(
'Features directory already created at path: {}'.format(
features_path))
# Create the features path directory if it doesn't exist
train_features_path = features_path + os.sep + 'train'
if not os.path.isdir(train_features_path):
ConsoleLogger.status(
'Creating train features directory at path: {}'.format(
train_features_path))
os.mkdir(train_features_path)
else:
ConsoleLogger.status(
'Train features directory already created at path: {}'.format(
train_features_path))
# Create the features path directory if it doesn't exist
val_features_path = features_path + os.sep + 'val'
if not os.path.isdir(val_features_path):
ConsoleLogger.status(
'Creating val features directory at path: {}'.format(
val_features_path))
os.mkdir(val_features_path)
else:
ConsoleLogger.status(
'Val features directory already created at path: {}'.format(
val_features_path))
def process(loader, output_dir, input_features_name,
output_features_name, rate, input_filters_number,
output_filters_number, input_target_shape,
augment_output_features, export_one_hot_features):
initial_index = 0
attempts = 10
current_attempt = 0
total_length = len(loader)
while current_attempt < attempts:
try:
i = initial_index
bar = tqdm(loader, initial=initial_index)
for data in bar:
(preprocessed_audio, one_hot, speaker_id, quantized,
wav_filename, sampling_rate, shifting_time,
random_starting_index, preprocessed_length,
top_db) = data
output_path = output_dir + os.sep + str(i) + '.pickle'
if os.path.isfile(output_path):
if os.path.getsize(output_path) == 0:
bar.set_description(
'{} already exists but is empty. Computing it again...'
.format(output_path))
os.remove(output_path)
else:
bar.set_description(
'{} already exists'.format(output_path))
i += 1
continue
input_features = SpeechFeatures.features_from_name(
name=input_features_name,
signal=preprocessed_audio,
rate=rate,
filters_number=input_filters_number)
if input_features.shape[0] != input_target_shape[
0] or input_features.shape[
1] != input_target_shape[1]:
ConsoleLogger.warn(
"Raw features number {} with invalid dimension {} will not be saved. Target shape: {}"
.format(i, input_features.shape,
input_target_shape))
i += 1
continue
output_features = SpeechFeatures.features_from_name(
name=output_features_name,
signal=preprocessed_audio,
rate=rate,
filters_number=output_filters_number,
augmented=augment_output_features)
# TODO: add an option in configuration to save quantized/one_hot or not
output = {
'preprocessed_audio':
preprocessed_audio,
'wav_filename':
wav_filename,
'input_features':
input_features,
'one_hot':
one_hot
if export_one_hot_features else np.array([]),
'quantized':
np.array([]),
'speaker_id':
speaker_id,
'output_features':
output_features,
'shifting_time':
shifting_time,
'random_starting_index':
random_starting_index,
'preprocessed_length':
preprocessed_length,
'sampling_rate':
sampling_rate,
'top_db':
top_db
}
with open(output_path, 'wb') as file:
pickle.dump(output, file)
bar.set_description('{} saved'.format(output_path))
i += 1
if i == total_length:
bar.update(total_length)
break
bar.close()
break
except KeyboardInterrupt:
bar.close()
ConsoleLogger.warn(
'Keyboard interrupt detected. Leaving the function...')
return
except:
error_message = 'An error occured in the data loader at {}/{}. Current attempt: {}/{}'.format(
output_dir, i, current_attempt + 1, attempts)
self._logger.exception(error_message)
ConsoleLogger.error(error_message)
initial_index = i
current_attempt += 1
continue
try:
ConsoleLogger.status('Processing training part')
process(
loader=self._training_loader,
output_dir=train_features_path,
input_features_name=configuration['input_features_type'],
output_features_name=configuration['output_features_type'],
rate=configuration['sampling_rate'],
input_filters_number=configuration['input_features_filters'],
output_filters_number=configuration['output_features_filters'],
input_target_shape=(configuration['input_features_dim'],
configuration['input_features_filters'] *
3),
augment_output_features=configuration[
'augment_output_features'],
export_one_hot_features=configuration[
'export_one_hot_features'])
ConsoleLogger.success('Training part processed')
except:
ConsoleLogger.error(
'An error occured during training features generation')
try:
ConsoleLogger.status('Processing validation part')
process(
loader=self._validation_loader,
output_dir=val_features_path,
input_features_name=configuration['input_features_type'],
output_features_name=configuration['output_features_type'],
rate=configuration['sampling_rate'],
input_filters_number=configuration['input_features_filters'],
output_filters_number=configuration['output_features_filters'],
input_target_shape=(configuration['input_features_dim'],
configuration['input_features_filters'] *
3),
augment_output_features=configuration[
'augment_output_features'],
export_one_hot_features=configuration[
'export_one_hot_features'])
ConsoleLogger.success('Validation part processed')
except:
ConsoleLogger.error(
'An error occured during validation features generation')
def process(loader, output_dir, input_features_name,
output_features_name, rate, input_filters_number,
output_filters_number, input_target_shape,
augment_output_features, export_one_hot_features):
initial_index = 0
attempts = 10
current_attempt = 0
total_length = len(loader)
while current_attempt < attempts:
try:
i = initial_index
bar = tqdm(loader, initial=initial_index)
for data in bar:
(preprocessed_audio, one_hot, speaker_id, quantized,
wav_filename, sampling_rate, shifting_time,
random_starting_index, preprocessed_length,
top_db) = data
output_path = output_dir + os.sep + str(i) + '.pickle'
if os.path.isfile(output_path):
if os.path.getsize(output_path) == 0:
bar.set_description(
'{} already exists but is empty. Computing it again...'
.format(output_path))
os.remove(output_path)
else:
bar.set_description(
'{} already exists'.format(output_path))
i += 1
continue
input_features = SpeechFeatures.features_from_name(
name=input_features_name,
signal=preprocessed_audio,
rate=rate,
filters_number=input_filters_number)
if input_features.shape[0] != input_target_shape[
0] or input_features.shape[
1] != input_target_shape[1]:
ConsoleLogger.warn(
"Raw features number {} with invalid dimension {} will not be saved. Target shape: {}"
.format(i, input_features.shape,
input_target_shape))
i += 1
continue
output_features = SpeechFeatures.features_from_name(
name=output_features_name,
signal=preprocessed_audio,
rate=rate,
filters_number=output_filters_number,
augmented=augment_output_features)
# TODO: add an option in configuration to save quantized/one_hot or not
output = {
'preprocessed_audio':
preprocessed_audio,
'wav_filename':
wav_filename,
'input_features':
input_features,
'one_hot':
one_hot
if export_one_hot_features else np.array([]),
'quantized':
np.array([]),
'speaker_id':
speaker_id,
'output_features':
output_features,
'shifting_time':
shifting_time,
'random_starting_index':
random_starting_index,
'preprocessed_length':
preprocessed_length,
'sampling_rate':
sampling_rate,
'top_db':
top_db
}
with open(output_path, 'wb') as file:
pickle.dump(output, file)
bar.set_description('{} saved'.format(output_path))
i += 1
if i == total_length:
bar.update(total_length)
break
bar.close()
break
except KeyboardInterrupt:
bar.close()
ConsoleLogger.warn(
'Keyboard interrupt detected. Leaving the function...')
return
except:
error_message = 'An error occured in the data loader at {}/{}. Current attempt: {}/{}'.format(
output_dir, i, current_attempt + 1, attempts)
self._logger.exception(error_message)
ConsoleLogger.error(error_message)
initial_index = i
current_attempt += 1
continue
def compute_groundtruth_alignments(self):
ConsoleLogger.status(
'Computing groundtruth alignments of VCTK val dataset...')
desired_time_interval = 0.02
extended_alignment_dataset = list()
possible_phonemes = set()
phonemes_counter = dict()
total_phonemes_apparations = 0
data_length = self._configuration['length'] / self._configuration[
'sampling_rate']
with tqdm(self._data_stream.validation_loader) as bar:
for data in bar:
speaker_ids = data['speaker_id'].to(self._device)
wav_filenames = data['wav_filename']
shifting_times = data['shifting_time'].to(self._device)
loader_indices = data['index'].to(self._device)
speaker_id = wav_filenames[0][0].split('/')[-2]
if speaker_id not in os.listdir(self._vctk.raw_folder +
os.sep + 'VCTK-Corpus' +
os.sep + 'phonemes'):
# TODO: log the missing folders
continue
for i in range(len(shifting_times)):
wav_filename = wav_filenames[0][i]
utterence_key = wav_filename.split('/')[-1].replace(
'.wav', '')
phonemes_alignment_path = os.sep.join(wav_filename.split('/')[:-3]) + os.sep + 'phonemes' + os.sep + utterence_key.split('_')[0] + os.sep \
+ utterence_key + '.TextGrid'
if not os.path.isfile(phonemes_alignment_path):
# TODO: log this warn instead of print it
#ConsoleLogger.warn('File {} not found'.format(phonemes_alignment_path))
break
shifting_time = shifting_times[0].detach().cpu().item()
target_time_scale = np.arange(
(data_length / desired_time_interval) +
1) * desired_time_interval + shifting_time
shifted_indices = np.where(
target_time_scale >= shifting_time)
tg = textgrid.TextGrid()
tg.read(phonemes_alignment_path)
"""if target_time_scale[-1] > tg.tiers[1][-1].maxTime:
ConsoleLogger.error('Shifting time error at {}.pickle: shifting_time:{}' \
' target_time_scale[-1]:{} > tg.tiers[1][-1].maxTime:{}'.format(
loader_indices[i].detach().cpu().item(),
shifting_time,
target_time_scale[-1],
tg.tiers[1][-1].maxTime))
continue"""
phonemes = list()
current_target_time_index = 0
for interval in tg.tiers[1]:
if interval.mark in ['', '-', "'"]:
if interval == tg.tiers[1][-1] and len(
phonemes) != int(
data_length / desired_time_interval):
previous_interval = tg.tiers[1][-2]
ConsoleLogger.warn(
"{}/{} phonemes aligned. Add the last valid phoneme '{}' in the list to have the correct number.\n"
"Sanity checks to find the possible cause:\n"
"current_target_time_index < (data_length / desired_time_interval): {}\n"
"target_time_scale[current_target_time_index] >= interval.minTime: {}\n"
"target_time_scale[current_target_time_index] <= interval.maxTime: {}"
.format(
len(phonemes),
int(data_length /
desired_time_interval),
previous_interval.mark,
current_target_time_index <
(data_length / desired_time_interval),
target_time_scale[
current_target_time_index] >=
previous_interval.minTime,
target_time_scale[
current_target_time_index] <=
previous_interval.maxTime))
phonemes.append(previous_interval.mark)
continue
interval.minTime = float(interval.minTime)
interval.maxTime = float(interval.maxTime)
if interval.maxTime < shifting_time:
continue
interval.mark = interval.mark[:-1] if interval.mark[
-1].isdigit() else interval.mark
possible_phonemes.add(interval.mark)
if interval.mark not in phonemes_counter:
phonemes_counter[interval.mark] = 0
phonemes_counter[interval.mark] += 1
total_phonemes_apparations += 1
while current_target_time_index < (data_length / desired_time_interval) and \
target_time_scale[current_target_time_index] >= interval.minTime and \
target_time_scale[current_target_time_index] <= interval.maxTime:
phonemes.append(interval.mark)
current_target_time_index += 1
if len(phonemes) == int(data_length /
desired_time_interval):
break
if len(phonemes) != int(
data_length / desired_time_interval):
intervals = [
'min:{} max:{} mark:{}'.format(
interval.minTime, interval.maxTime,
interval.mark) for interval in tg.tiers[1]
]
ConsoleLogger.error(
'Error - min:{} max:{} shifting:{} target_time_scale: {} intervals: {}'
.format(interval.minTime, interval.maxTime,
shifting_time, target_time_scale,
intervals))
ConsoleLogger.error('#phonemes:{} phonemes:{}'.format(
len(phonemes), phonemes))
else:
extended_alignment_dataset.append(
(utterence_key, phonemes))
with open(
self._results_path + os.sep +
'vctk_groundtruth_alignments.pickle', 'wb') as f:
pickle.dump(
{
'desired_time_interval': desired_time_interval,
'extended_alignment_dataset': extended_alignment_dataset,
'possible_phonemes': list(possible_phonemes),
'phonemes_counter': phonemes_counter,
'total_phonemes_apparations': total_phonemes_apparations
}, f)
def compute_clustering_metrics(self):
groundtruth_alignments_dic = None
with open(
self._results_path + os.sep +
'vctk_groundtruth_alignments.pickle', 'rb') as f:
groundtruth_alignments_dic = pickle.load(f)
empirical_alignments_dic = None
with open(
self._results_path + os.sep + self._experiment_name +
'_vctk_empirical_alignments.pickle', 'rb') as f:
empirical_alignments_dic = pickle.load(f)
groundtruth_alignments = np.array(
groundtruth_alignments_dic['extended_alignment_dataset'])
possible_phonemes = list(
groundtruth_alignments_dic['possible_phonemes'])
empirical_alignments = np.array(
empirical_alignments_dic['all_alignments'])
phonemes_indices = {
possible_phonemes[i]: i
for i in range(len(possible_phonemes))
}
ConsoleLogger.status('#{} possible phonemes: {}'.format(
len(possible_phonemes), possible_phonemes))
ConsoleLogger.status('# of raw groundtruth alignments: {}'.format(
len(groundtruth_alignments)))
ConsoleLogger.status('# of raw empirical alignments: {}'.format(
len(empirical_alignments)))
groundtruth_utterance_keys = set()
final_groundtruth_alignments = list()
final_empirical_alignments = list()
alignment_length = ((self._configuration['length'] /
self._configuration['sampling_rate']) * 100) / 2
for (utterence_key, alignment) in groundtruth_alignments:
if len(alignment) != alignment_length: # FIXME
ConsoleLogger.error(
'len(alignment) != alignment_length: {}'.format(
len(alignment)))
continue
groundtruth_utterance_keys.add(utterence_key)
final_groundtruth_alignments.append([
phonemes_indices[alignment[i]] for i in range(len(alignment))
])
for (utterence_key, alignment) in empirical_alignments:
if utterence_key in groundtruth_utterance_keys:
final_empirical_alignments.append(alignment)
final_groundtruth_alignments = np.asarray(final_groundtruth_alignments)
final_empirical_alignments = np.asarray(final_empirical_alignments)
ConsoleLogger.status('Groundtruth alignments shape: {}'.format(
final_groundtruth_alignments.shape))
ConsoleLogger.status('Empirical alignments shape: {}'.format(
final_empirical_alignments.shape))
ConsoleLogger.status('Groundtruth alignments samples: {}'.format(
[final_groundtruth_alignments[i] for i in range(2)]))
ConsoleLogger.status('Empirical alignments samples: {}'.format(
[final_empirical_alignments[i] for i in range(2)]))
concatenated_groundtruth_alignments = np.concatenate(
final_groundtruth_alignments)
concatenated_empirical_alignments = np.concatenate(
final_empirical_alignments)
ConsoleLogger.status(
'Concatenated groundtruth alignments shape: {}'.format(
concatenated_groundtruth_alignments.shape))
ConsoleLogger.status(
'Concatenated empirical alignments shape: {}'.format(
concatenated_empirical_alignments.shape))
adjusted_rand_score = sklearn.metrics.adjusted_rand_score(
concatenated_groundtruth_alignments,
concatenated_empirical_alignments)
adjusted_mutual_info_score = sklearn.metrics.adjusted_mutual_info_score(
concatenated_groundtruth_alignments,
concatenated_empirical_alignments)
normalized_mutual_info_score = sklearn.metrics.normalized_mutual_info_score(
concatenated_groundtruth_alignments,
concatenated_empirical_alignments)
ConsoleLogger.success(
'Adjusted rand score: {}'.format(adjusted_rand_score))
ConsoleLogger.success('Adjusted mututal info score: {}'.format(
adjusted_mutual_info_score))
ConsoleLogger.success(
'Normalized adjusted mututal info score: {}'.format(
normalized_mutual_info_score))
with open(
self._results_path + os.sep + self._experiment_name +
'_adjusted_rand_score.npy', 'wb') as f:
np.save(f, adjusted_rand_score)
with open(
self._results_path + os.sep + self._experiment_name +
'_adjusted_mutual_info_score.npy', 'wb') as f:
np.save(f, adjusted_mutual_info_score)
with open(
self._results_path + os.sep + self._experiment_name +
'_normalized_mutual_info_score.npy', 'wb') as f:
np.save(f, normalized_mutual_info_score)
ConsoleLogger.success(
'All scores from cluestering metrics were successfully saved')