def main():
# Load input SNR
all_snr_db = read_dataset(processed_data_dir, dataset_type, 'snr_db')
all_snr_db = np.array(all_snr_db)
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
# Fuse both list
args = [[file_path, snr_db]
for file_path, snr_db in zip(file_paths, all_snr_db)]
t1 = time.perf_counter()
with concurrent.futures.ProcessPoolExecutor(max_workers=None) as executor:
all_metrics = executor.map(compute_metrics_utt, args)
t2 = time.perf_counter()
print(f'Finished in {t2 - t1} seconds')
# Transform generator to list
all_metrics = list(all_metrics)
metrics_keys = ['SI-SDR', 'SI-SIR', 'SI-SAR', 'STOI', 'PESQ']
# Compute & save stats
compute_stats(metrics_keys=metrics_keys,
all_metrics=all_metrics,
all_snr_db=all_snr_db,
model_data_dir=model_data_dir,
confidence=confidence)
def main():
file = open('output.log','w')
print('Torch version: {}'.format(torch.__version__))
# Start context for GPU multiprocessing
ctx = multiprocessing.get_context('spawn')
print('Load models')
if classif_type == 'dnn':
classifier = Classifier([x_dim, h_dim_cl, y_dim])
classifier.load_state_dict(torch.load(classif_path, map_location="cpu"))
if classif_type in ['oracle', 'timo']:
classifier = None
model = DeepGenerativeModel([x_dim, y_dim, z_dim, h_dim], None)
model.load_state_dict(torch.load(model_path, map_location="cpu"))
print('- Number of learnable parameters: {}'.format(count_parameters(model)))
mcem = MCEM_M2(niter=niter,
nsamples_E_step=nsamples_E_step,
burnin_E_step=burnin_E_step, nsamples_WF=nsamples_WF,
burnin_WF=burnin_WF, var_RW=var_RW)
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir, dataset_type=dataset_type)
print('- Number of test samples: {}'.format(len(file_paths)))
# Split list in nb_devices * nb_processes_per_device
b = np.array_split(file_paths, nb_devices*nb_process_per_device)
# Assign each list to a process
b = [(i%nb_devices, sublist, mcem, model, classifier) for i, sublist in enumerate(b)]
print('Start evaluation')
# start = time.time()
t1 = time.perf_counter()
with ctx.Pool(processes=nb_process_per_device*nb_devices) as multi_pool:
multi_pool.starmap(process_sublist, b)
# # Test script on 1 sublist
# process_sublist(*b[0])
t2 = time.perf_counter()
print(f'Finished in {t2 - t1} seconds')
def main():
file = open('output.log', 'w')
print('Torch version: {}'.format(torch.__version__))
# print('Device: %s' % (device))
# if torch.cuda.device_count() >= 1: print("Number GPUs: ", torch.cuda.device_count())
ctx = multiprocessing.get_context('spawn')
# Load model
model = VariationalAutoencoder([x_dim, z_dim, h_dim])
model.load_state_dict(torch.load(model_path, map_location="cpu"))
# Create MCEM object
mcem = MCEM_M1(niter=niter,
nsamples_E_step=nsamples_E_step,
burnin_E_step=burnin_E_step,
nsamples_WF=nsamples_WF,
burnin_WF=burnin_WF,
var_RW=var_RW)
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
# Split list in nb_devices * nb_processes_per_device
b = np.array_split(file_paths, nb_devices * nb_process_per_device)
# Assign each list to a process
b = [(i % nb_devices, sublist, mcem, model) for i, sublist in enumerate(b)]
# print('Start evaluation')
# start = time.time()
# elapsed = []
print('Start evaluation')
t1 = time.perf_counter()
with ctx.Pool(processes=nb_process_per_device * nb_devices) as multi_pool:
multi_pool.starmap(process_sublist, b)
# Test script on 1 sublist
# process_sublist(*b[0])
t2 = time.perf_counter()
print(f'Finished in {t2 - t1} seconds')
def main_polqa():
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
# Fuse both list
v_reference_paths = [
processed_data_dir + os.path.splitext(file_path)[0] + '_s.wav'
for file_path in file_paths
]
v_processed_paths = [
model_data_dir + os.path.splitext(file_path)[0] + '_s_est.wav'
for file_path in file_paths
]
# POLQA on short audio files
extended_v_reference_paths = []
extended_v_processed_paths = []
for i, (file_path, v_reference_path, v_processed_path) in enumerate(
zip(file_paths, v_reference_paths, v_processed_paths)):
# Read files
s_t, fs_s = sf.read(v_reference_path) # clean speech
s_hat_t, fs_s_hat = sf.read(v_processed_path) # est. speech
# if smaller, then convert to numpy array and pad, and remove from list
if len(s_t) < 3 * fs:
s_t = np.pad(s_t, (0, 3 * fs - len(s_t)))
s_hat_t = np.pad(s_hat_t, (0, 3 * fs - len(s_hat_t)))
# Remove from path list
v_reference_paths.remove(v_reference_path)
v_processed_paths.remove(v_processed_path)
# Save as new files
extended_v_reference_path = processed_data_dir + os.path.splitext(
file_path)[0] + '_s_3sec.wav'
extended_v_processed_path = model_data_dir + os.path.splitext(
file_path)[0] + '_s_est_3sec.wav'
sf.write(extended_v_reference_path, s_t, fs)
sf.write(extended_v_processed_path, s_hat_t, fs)
# Append to extended path list
extended_v_reference_paths.append(extended_v_reference_path)
extended_v_processed_paths.append(extended_v_processed_path)
# Remove 3rd and last indices from extended list
del extended_v_reference_paths[3]
del extended_v_reference_paths[-1]
del extended_v_processed_paths[3]
del extended_v_processed_paths[-1]
t1 = time.perf_counter()
# path_all_polqa = polqa(v_reference=v_reference_paths[:1], v_processed=v_processed_paths[:1])
# extended_all_polqa = polqa(v_reference=extended_v_reference_paths[3:4], v_processed=extended_v_processed_paths[3:4])
path_all_polqa = polqa(v_reference=v_reference_paths,
v_processed=v_processed_paths)
# extended_all_polqa = polqa(v_reference=extended_v_reference_paths, v_processed=extended_v_processed_paths)
t2 = time.perf_counter()
print(f'Finished in {t2 - t1} seconds')
# Transform generator to list
path_all_polqa = list(path_all_polqa)
# extended_all_polqa = list(extended_all_polqa)
with open(model_data_dir + 'path_all_polqa.txt', 'w') as f:
for item in path_all_polqa:
f.write("%s\n" % item)
# with open(model_data_dir + 'extended_all_polqa.txt', 'w') as f:
# for item in extended_all_polqa:
# f.write("%s\n" % item)
# Merge lists
# all_polqa = path_all_polqa + extended_all_polqa
all_polqa = path_all_polqa
# all_polqa = extended_all_polqa
metrics_keys = ['POLQA']
# Compute & save stats
compute_stats_noisnr(metrics_keys=metrics_keys,
all_metrics=all_polqa,
model_data_dir=model_data_dir,
confidence=confidence)
def main():
if not os.path.exists(os.path.dirname(output_dataset_file)):
os.makedirs(os.path.dirname(output_dataset_file))
with h5.File(output_dataset_file,
'a',
rdcc_nbytes=rdcc_nbytes,
rdcc_nslots=rdcc_nslots) as f:
# STFT attributes
f.attrs['fs'] = fs
f.attrs['wlen_sec'] = wlen_sec
f.attrs['hop_percent'] = hop_percent
f.attrs['win'] = win
f.attrs['dtype'] = dtype
# label attributes
f.attrs['quantile_fraction'] = quantile_fraction
f.attrs['quantile_weight'] = quantile_weight
# HDF5 attributes
f.attrs['X_chunks'] = X_chunks
f.attrs['Y_chunks'] = Y_chunks
f.attrs['compression'] = compression
for dataset_type in dataset_types:
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
if dataset_type == 'train':
noise_types = [
'domestic', 'nature', 'office', 'transportation'
]
if dataset_type == 'validation':
noise_types = ['nature', 'office', 'public', 'transportation']
# Create SNR list
np.random.seed(0)
noise_index = np.random.randint(len(noise_types),
size=len(file_paths))
snrs = [-5, -2.5, 0, 2.5, 5.0]
snrs_index = np.random.randint(len(snrs), size=len(file_paths))
# Create noise_audios from processed noise files
preprocessed_noise_paths = noise_list_preprocessed(
preprocessed_noise_dir=output_noise_dir,
dataset_type=dataset_type)
noise_audios = {}
# Load the noise files
for noise_type, preprocessed_noise_path in preprocessed_noise_paths.items(
):
#if noise already preprocessed, read files directly
if os.path.exists(preprocessed_noise_path):
noise_audio, fs_noise = sf.read(preprocessed_noise_path)
if fs != fs_noise:
raise ValueError(
'Unexpected sampling rate. Did you preprocess the 16kHz version of the DEMAND database?'
)
noise_audios[noise_type] = noise_audio
# Init list of SNR
all_snr_dB = []
# Delete datasets if already exists
if 'X_' + dataset_type in f:
del f['X_' + dataset_type]
del f['Y_' + dataset_type]
# Exact shape of dataset is unknown in advance unfortunately
# Faster writing if you know the shape in advance
# Size of chunks corresponds to one spectrogram frame
f.create_dataset('X_' + dataset_type,
shape=X_shape,
dtype='float32',
maxshape=X_maxshape,
chunks=X_chunks,
compression=compression,
shuffle=shuffle)
f.create_dataset('Y_' + dataset_type,
shape=Y_shape,
dtype='float32',
maxshape=Y_maxshape,
chunks=Y_chunks,
compression=compression,
shuffle=shuffle)
# Store dataset in variables for faster I/O
fx = f['X_' + dataset_type]
fy = f['Y_' + dataset_type]
# Compute mean, std of the train set
if dataset_type == 'train':
# VAR = E[X**2] - E[X]**2
channels_sum, channels_squared_sum = 0., 0.
# Loop over the speech files
for i, file_path in tqdm(enumerate(file_paths)):
speech, fs_speech = sf.read(input_speech_dir + file_path,
samplerate=None)
# Cut burst at begining of file
speech = speech[int(0.1 * fs):]
# Normalize audio
speech = speech / (np.max(np.abs(speech)))
if fs != fs_speech:
raise ValueError('Unexpected sampling rate')
# Select noise_type
noise_type = noise_types[noise_index[i]]
# Extract noise segment
noise = noise_segment(noise_audios, noise_type, speech)
# Select SNR
snr_dB = snrs[snrs_index[i]]
all_snr_dB.append(snr_dB)
# Compute noise gain
speech_power = np.sum(np.power(speech, 2))
noise_power = np.sum(np.power(noise, 2))
noise_power_target = speech_power * np.power(10, -snr_dB / 10)
k = noise_power_target / noise_power
noise = noise * np.sqrt(k)
mixture = speech + noise
# # Normalize by max of speech, noise, speech+noise
# norm = np.max(abs(np.concatenate([speech, noise, speech+noise])))
# mixture = (speech+noise) / norm
# speech /= norm
# noise /= norm
if dataset_size == 'subset':
# Save .wav files, just to check if it working
output_path = output_wav_dir + file_path
output_path = os.path.splitext(output_path)[0]
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
sf.write(output_path + '_s.wav', speech, fs)
sf.write(output_path + '_n.wav', noise, fs)
sf.write(output_path + '_x.wav', mixture, fs)
# TF reprepsentation
mixture_tf = stft(mixture,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype)
noisy_spectrogram = np.power(abs(mixture_tf), 2)
# TF reprepsentation
speech_tf = stft(speech,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype)
if labels == 'noisy_wiener_labels':
# TF reprepsentation
noise_tf = stft(noise,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype)
# wiener mask
speech_wiener_mask = ideal_wiener_mask(
speech_tf, noise_tf, eps)
label = speech_wiener_mask
if labels == 'noisy_labels':
# binary mask
speech_ibm = clean_speech_IBM(
speech_tf,
quantile_fraction=quantile_fraction,
quantile_weight=quantile_weight)
label = speech_ibm
if labels == 'noisy_vad_labels':
# binary mask
speech_vad = clean_speech_VAD(
speech_tf,
quantile_fraction=quantile_fraction,
quantile_weight=quantile_weight)
label = speech_vad
# Compute mean, std
if dataset_type == 'train':
# VAR = E[X**2] - E[X]**2
channels_sum += np.sum(noisy_spectrogram, axis=-1)
channels_squared_sum += np.sum(noisy_spectrogram**2,
axis=-1)
# Store spectrogram in dataset
fx.resize((fx.shape[1] + noisy_spectrogram.shape[1]), axis=1)
fx[:, -noisy_spectrogram.shape[1]:] = noisy_spectrogram
# Store spectrogram in label
fy.resize((fy.shape[1] + label.shape[1]), axis=1)
fy[:, -label.shape[1]:] = label
# Compute and save mean, std
if dataset_type == 'train':
print('Compute mean and std')
#NB: compute the empirical std (!= regular std)
n_samples = fx.shape[1]
mean = channels_sum / n_samples
std = np.sqrt((1 / (n_samples - 1)) *
(channels_squared_sum - n_samples * mean**2))
# Delete datasets if already exists
if 'X_' + dataset_type + '_mean' in f:
del f['X_' + dataset_type + '_mean']
del f['X_' + dataset_type + '_std']
f.create_dataset('X_' + dataset_type + '_mean',
shape=X_chunks,
dtype='float32',
maxshape=X_chunks,
chunks=None,
compression=compression,
shuffle=shuffle)
f.create_dataset('X_' + dataset_type + '_std',
shape=X_chunks,
dtype='float32',
maxshape=X_chunks,
chunks=None,
compression=compression,
shuffle=shuffle)
f['X_' + dataset_type +
'_mean'][:] = mean[..., None] # Add axis to fit chunks shape
f['X_' + dataset_type +
'_std'][:] = std[..., None] # Add axis to fit chunks shape
print('Mean and std saved in HDF5.')
# TODO: save SNR, level_s, level_n in 1 big csv
write_dataset(all_snr_dB, output_wav_dir, dataset_type, 'snr_db')
def main():
# Load input SNR
all_snr_db = read_dataset(processed_data_dir, dataset_type, 'snr_db')
all_snr_db = np.array(all_snr_db)
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
# 1 list per metric
all_stoi = []
all_pesq = []
all_polqa = []
all_f1score = []
for i, file_path in tqdm(enumerate(file_paths)):
# Read files
s_t, fs_s = sf.read(processed_data_dir +
os.path.splitext(file_path)[0] +
'_s.wav') # clean speech
n_t, fs_n = sf.read(processed_data_dir +
os.path.splitext(file_path)[0] + '_n.wav') # noise
x_t, fs_x = sf.read(processed_data_dir +
os.path.splitext(file_path)[0] +
'_x.wav') # mixture
# compute metrics
## STOI (or ESTOI?)
stoi_s_hat = stoi(s_t, x_t, fs, extended=True)
all_stoi.append(stoi_s_hat)
## PESQ
pesq_s_hat = pesq(fs, s_t, x_t, 'wb') # wb = wideband
all_pesq.append(pesq_s_hat)
## POLQA
# polqa_s_hat = polqa(s, s_t, fs)
# all_polqa.append(polqa_s_hat)
# TF representation
n_tf = stft(n_t,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype) # shape = (freq_bins, frames)
s_tf = stft(s_t,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype) # shape = (freq_bins, frames)
# plots of target / estimation
# TF representation
x_tf = stft(x_t,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype) # shape = (freq_bins, frames)
# ## mixture signal (wav + spectro)
# ## target signal (wav + spectro + mask)
# ## estimated signal (wav + spectro + mask)
# signal_list = [
# [x_t, x_tf, None], # mixture: (waveform, tf_signal, no mask)
# [s_t, s_tf, None], # clean speech
# [n_t, n_tf, None]
# ]
# fig = display_multiple_signals(signal_list,
# fs=fs, vmin=vmin, vmax=vmax,
# wlen_sec=wlen_sec, hop_percent=hop_percent,
# xticks_sec=xticks_sec, fontsize=fontsize)
# # put all metrics in the title of the figure
# title = "Input SNR = {:.1f} dB \n" \
# "STOI = {:.2f}, " \
# "PESQ = {:.2f} \n" \
# "".format(all_snr_db[i], stoi_s_hat, pesq_s_hat)
# fig.suptitle(title, fontsize=40)
# # Save figure
# fig.savefig(processed_data_dir + os.path.splitext(file_path)[0] + '_fig.png')
# # Clear figure
# plt.close()
# Confidence interval
metrics = {'SNR': all_snr_db, 'STOI': all_stoi, 'PESQ': all_pesq}
stats = {}
# Print the names of the columns.
print("{:<10} {:<10} {:<10}".format('METRIC', 'AVERAGE', 'CONF. INT.'))
for key, metric in metrics.items():
m, h = mean_confidence_interval(metric, confidence=confidence)
stats[key] = {'avg': m, '+/-': h}
print("{:<10} {:<10} {:<10}".format(key, m, h))
print('\n')
# Save stats (si_sdr, si_sar, etc. )
with open(
processed_data_dir + os.path.dirname(os.path.dirname(file_path)) +
'stats.json', 'w') as f:
json.dump(stats, f)
# Metrics by input SNR
for snr_db in np.unique(all_snr_db):
stats = {}
print('Input SNR = {:.2f}'.format(snr_db))
# Print the names of the columns.
print("{:<10} {:<10} {:<10}".format('METRIC', 'AVERAGE', 'CONF. INT.'))
for key, metric in metrics.items():
subset_metric = np.array(metric)[np.where(all_snr_db == snr_db)]
m, h = mean_confidence_interval(subset_metric,
confidence=confidence)
stats[key] = {'avg': m, '+/-': h}
print("{:<10} {:<10} {:<10}".format(key, m, h))
print('\n')
# Save stats (si_sdr, si_sar, etc. )
with open(
processed_data_dir +
os.path.dirname(os.path.dirname(file_path)) +
'stats_{:g}.json'.format(snr_db), 'w') as f:
json.dump(stats, f)
def main():
# Load input SNR
all_snr_db = read_dataset(processed_data_dir, dataset_type, 'snr_db')
all_snr_db = np.array(all_snr_db)
device = torch.device("cuda" if cuda else "cpu")
file = open('output.log','w')
print('Torch version: {}'.format(torch.__version__))
print('Device: %s' % (device))
if torch.cuda.device_count() >= 1: print("Number GPUs: ", torch.cuda.device_count())
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
model = VariationalAutoencoder([x_dim, z_dim, h_dim])
model.load_state_dict(torch.load(model_data_path))
if cuda: model = model.cuda()
model.eval()
for param in model.parameters():
param.requires_grad = False
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
for i, file_path in tqdm(enumerate(file_paths)):
# Read files
s_t, fs_s = sf.read(processed_data_dir + os.path.splitext(file_path)[0] + '_s.wav') # clean speech
x_t, fs_x = sf.read(processed_data_dir + os.path.splitext(file_path)[0] + '_x.wav') # mixture
# x = x/np.max(x)
T_orig = len(x_t)
# TF representation
# Input should be (frames, freq_bibs)
x_tf = stft(x_t,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype)
# Transpose to match PyTorch
x_tf = x_tf.T # (frames, freq_bins)
# Power spectrogram (transpose)
x = torch.tensor(np.power(np.abs(x_tf), 2)).to(device)
# Encode-decode
reconstruction, _, _ = model(x)
reconstruction = reconstruction.cpu().numpy()
# plots of target / estimation
s_tf = stft(s_t,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype) # shape = (freq_bins, frames)
# Transpose to match librosa.display
reconstruction = reconstruction.T
# Transform to dB
x_psd = x.cpu().numpy().T
x_psd = librosa.core.power_to_db(x_psd)
s_psd = np.power(abs(s_tf),2)
s_psd = librosa.core.power_to_db(s_psd)
reconstruction = librosa.core.power_to_db(reconstruction)
## mixture signal (wav + spectro)
## target signal (wav + spectro + mask)
## estimated signal (wav + spectro + mask)
signal_list = [
[x_t, x_psd], # mixture: (waveform, tf_signal, no mask)
[s_t, s_psd], # clean speech
[None, reconstruction]
]
#TODO: modify
fig = display_multiple_spectro(signal_list,
fs=fs, vmin=vmin, vmax=vmax,
wlen_sec=wlen_sec, hop_percent=hop_percent,
xticks_sec=xticks_sec, fontsize=fontsize)
# put all metrics in the title of the figure
title = "Input SNR = {:.1f} dB \n" \
"".format(all_snr_db[i])
fig.suptitle(title, fontsize=40)
# Save figure
fig.savefig(output_data_dir + os.path.splitext(file_path)[0] + '_recon.png')
def main():
# Load input SNR
all_snr_db = read_dataset(processed_data_dir, dataset_type, 'snr_db')
all_snr_db = np.array(all_snr_db)
device = torch.device("cuda" if cuda else "cpu")
file = open('output.log', 'w')
print('Torch version: {}'.format(torch.__version__))
print('Device: %s' % (device))
if torch.cuda.device_count() >= 1:
print("Number GPUs: ", torch.cuda.device_count())
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
classifier = Classifier([x_dim, h_dim_cl, y_dim], batch_norm=batch_norm)
classifier.load_state_dict(
torch.load(classif_dir, map_location=cuda_device))
if cuda: classifier = classifier.cuda()
classifier.eval()
for param in classifier.parameters():
param.requires_grad = False
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
for i, file_path in tqdm(enumerate(file_paths)):
# Read files
s_t, fs_s = sf.read(processed_data_dir +
os.path.splitext(file_path)[0] +
'_s.wav') # clean speech
x_t, fs_x = sf.read(processed_data_dir +
os.path.splitext(file_path)[0] +
'_x.wav') # mixture
# x = x/np.max(x)
T_orig = len(x_t)
# TF representation
# Input should be (frames, freq_bibs)
x_tf = stft(x_t,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype)
# Transpose to match PyTorch
x_tf = x_tf.T # (frames, freq_bins)
# Power spectrogram (transpose)
x = torch.tensor(np.power(np.abs(x_tf), 2)).to(device)
# Normalize power spectrogram
if std_norm:
x -= mean.T
x /= (std + eps).T
# Classify
y_hat_soft = classifier(x)
y_hat_hard = (y_hat_soft > 0.5).int()
y_hat_hard = y_hat_hard.cpu().numpy()
y_hat_hard = y_hat_hard.T # Transpose to match librosa.display
# plots of target / estimation
s_tf = stft(s_t,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype) # shape = (freq_bins, frames)
if labels == 'labels':
# binary mask
target = clean_speech_IBM(s_tf,
quantile_fraction=quantile_fraction,
quantile_weight=quantile_weight)
if labels == 'vad_labels':
# vad
target = clean_speech_VAD(s_tf,
quantile_fraction=quantile_fraction,
quantile_weight=quantile_weight)
# Transpose to match librosa.display
x_tf = x_tf.T
# F1-score
f1score_s_hat = f1_score(target.flatten(),
y_hat_hard.flatten(),
average="binary")
## mixture signal (wav + spectro)
## target signal (wav + spectro + mask)
## estimated signal (wav + spectro + mask)
signal_list = [
[x_t, x_tf, None], # mixture: (waveform, tf_signal, no mask)
[s_t, s_tf, target], # clean speech
[None, None, y_hat_hard]
#[None, None, y_hat_soft]
]
#TODO: modify
fig = display_multiple_signals(signal_list,
fs=fs,
vmin=vmin,
vmax=vmax,
wlen_sec=wlen_sec,
hop_percent=hop_percent,
xticks_sec=xticks_sec,
fontsize=fontsize)
# put all metrics in the title of the figure
title = "Input SNR = {:.1f} dB \n" \
"F1-score = {:.3f} \n".format(all_snr_db[i], f1score_s_hat)
fig.suptitle(title, fontsize=40)
# Save figure
output_path = classif_data_dir + file_path
output_path = os.path.splitext(output_path)[0]
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
fig.savefig(output_path + '_hard_mask.png')
def test_write_read_labels(dataset_type):
# Parameters
## Dataset
input_speech_dir = 'data/subset/raw/'
output_speech_dir = 'data/subset/processed/'
output_data_dir = 'data/subset/pickle/'
fs = int(16e3) # Sampling rate
## STFT
wlen_sec = 64e-3 # window length in seconds
hop_percent = 0.25 # hop size as a percentage of the window length
win = 'hann' # type of window
dtype = 'complex64'
## Ideal binary mask
quantile_fraction = 0.98
quantile_weight = 0.999
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
labels = []
for path in file_paths:
x, fs_x = sf.read(input_speech_dir + path, samplerate=None)
# Cut burst at begining of file
x[:int(0.1 * fs)] = x[int(0.1 * fs):int(0.2 * fs)]
# Normalize audio
x = x / (np.max(np.abs(x)))
#x = x/(np.max(np.abs(x)) + 2)
#x = x/np.linalg.norm(x)
if fs != fs_x:
raise ValueError('Unexpected sampling rate')
# TF reprepsentation
x_tf = stft(x,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype)
# binary mask
x_ibm = clean_speech_IBM(x_tf,
quantile_fraction=quantile_fraction,
quantile_weight=quantile_weight)
labels.append(x_ibm)
labels = np.concatenate(labels, axis=1)
#labels = labels[1]
# write spectrograms
write_dataset(labels,
output_data_dir=output_data_dir,
dataset_type=dataset_type,
suffix='labels')
# Read pickle
pickle_labels = read_dataset(data_dir=output_data_dir,
dataset_type=dataset_type,
suffix='labels')
# Assert stored data is same as spectrograms
assert_array_equal(labels, pickle_labels)
def test_write_read_frames(dataset_type):
# Parameters
## Dataset
input_speech_dir = 'data/subset/raw/'
output_speech_dir = 'data/subset/processed/'
output_data_dir = 'data/subset/pickle/'
fs = int(16e3) # Sampling rate
## STFT
wlen_sec = 64e-3 # window length in seconds
hop_percent = 0.25 # hop size as a percentage of the window length
win = 'hann' # type of window
dtype = 'complex64'
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
spectrograms = []
for path in file_paths:
x, fs_x = sf.read(input_speech_dir + path, samplerate=None)
# Cut burst at begining of file
#x[:int(0.1*fs)] = x[int(0.1*fs):int(0.2*fs)]
x = x[int(0.1 * fs):]
# Normalize audio
x = x / (np.max(np.abs(x)))
#x = x/(np.max(np.abs(x)) + 2)
#x = x/np.linalg.norm(x)
if not os.path.exists(os.path.dirname(output_speech_dir + path)):
os.makedirs(os.path.dirname(output_speech_dir + path))
sf.write(output_speech_dir + path, x, fs_x)
if fs != fs_x:
raise ValueError('Unexpected sampling rate')
# TF reprepsentation
x_tf = stft(x,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype)
spectrograms.append(np.power(abs(x_tf), 2))
spectrograms = np.concatenate(spectrograms, axis=1)
#spectrograms = spectrograms[1]
# write spectrograms
write_dataset(spectrograms,
output_data_dir=output_data_dir,
dataset_type=dataset_type,
suffix='frames')
# Read pickle
pickle_spectrograms = read_dataset(data_dir=output_data_dir,
dataset_type=dataset_type,
suffix='frames')
# Assert stored data is same as spectrograms
assert_array_equal(spectrograms, pickle_spectrograms)
def main():
# Load input SNR
all_snr_db = read_dataset(processed_data_dir, dataset_type, 'snr_db')
all_snr_db = np.array(all_snr_db)
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
for i, file_path in tqdm(enumerate(file_paths)):
# Read files
s_t, fs_s = sf.read(processed_data_dir +
os.path.splitext(file_path)[0] +
'_s.wav') # clean speech
x_t, fs_x = sf.read(processed_data_dir +
os.path.splitext(file_path)[0] +
'_x.wav') # mixture
# x = x/np.max(x)
T_orig = len(x_t)
# TF representation
# Input should be (frames, freq_bibs)
x_tf = stft(x_t,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype)
# Power spectrogram (transpose)
x = np.power(np.abs(x_tf), 2)
# Estimate mask
y_hat_soft = timo_mask_estimation(x)
y_hat_hard = (y_hat_soft > 0.5).astype(int)
# plots of target / estimation
s_tf = stft(s_t,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype) # shape = (freq_bins, frames)
# binary mask
s_ibm = clean_speech_IBM(s_tf,
quantile_fraction=quantile_fraction,
quantile_weight=quantile_weight)
# F1-score
f1score_s_hat = f1_score(s_ibm.flatten(),
y_hat_hard.flatten(),
average="binary")
## mixture signal (wav + spectro)
## target signal (wav + spectro + mask)
## estimated signal (wav + spectro + mask)
signal_list = [
[x_t, x_tf, None], # mixture: (waveform, tf_signal, no mask)
[s_t, s_tf, s_ibm], # clean speech
#[None, None, y_hat_hard]
[None, None, y_hat_soft]
]
#TODO: modify
fig = display_multiple_signals(signal_list,
fs=fs,
vmin=vmin,
vmax=vmax,
wlen_sec=wlen_sec,
hop_percent=hop_percent,
xticks_sec=xticks_sec,
fontsize=fontsize)
# put all metrics in the title of the figure
title = "Input SNR = {:.1f} dB \n" \
"".format(all_snr_db[i])
fig.suptitle(title, fontsize=40)
# Save figure
output_path = model_data_dir + file_path
output_path = os.path.splitext(output_path)[0]
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
fig.savefig(output_path + '_soft_mask.png')
## mixture signal (wav + spectro)
## target signal (wav + spectro + mask)
## estimated signal (wav + spectro + mask)
signal_list = [
[x_t, x_tf, None], # mixture: (waveform, tf_signal, no mask)
[s_t, s_tf, s_ibm], # clean speech
#[None, None, y_hat_hard]
[None, None, y_hat_hard]
]
#TODO: modify
fig = display_multiple_signals(signal_list,
fs=fs,
vmin=vmin,
vmax=vmax,
wlen_sec=wlen_sec,
hop_percent=hop_percent,
xticks_sec=xticks_sec,
fontsize=fontsize)
# put all metrics in the title of the figure
title = "Input SNR = {:.1f} dB \n" \
"F1-score = {:.3f} \n".format(all_snr_db[i], f1score_s_hat)
fig.suptitle(title, fontsize=40)
# Save figure
output_path = model_data_dir + file_path
output_path = os.path.splitext(output_path)[0]
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
fig.savefig(output_path + '_hard_mask.png')
def main():
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
for path in file_paths:
x, fs_x = sf.read(input_speech_dir + path, samplerate=None)
# Cut burst at begining of file
x = x[int(0.1 * fs):]
# Normalize audio
x = x / np.max(np.abs(x))
if fs != fs_x:
raise ValueError('Unexpected sampling rate')
# TF reprepsentation
x_tf = stft(x,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent) # shape = (freq_bins, frames)
# # binary mask
# x_ibm = clean_speech_IBM(x_tf,
# quantile_fraction=quantile_fraction,
# quantile_weight=quantile_weight)
# compute only VAD
x_vad = clean_speech_VAD(x_tf,
quantile_fraction=quantile_fraction,
quantile_weight=quantile_weight)
x_vad = x_vad[0] # shape = (frames)
# # Plot waveplot + spectrogram + binary mask
# fig = display_wav_spectro_mask(x, x_tf, x_ibm,
# fs=fs, vmin=vmin, vmax=vmax,
# wlen_sec=wlen_sec, hop_percent=hop_percent,
# xticks_sec=xticks_sec, fontsize=fontsize)
# Plot waveplot + spectrogram + vad
fig = display_wav_spectro_mask(x,
x_tf,
x_vad,
fs=fs,
vmin=vmin,
vmax=vmax,
wlen_sec=wlen_sec,
hop_percent=hop_percent,
xticks_sec=xticks_sec,
fontsize=fontsize)
# signal_list = [
# [x, x_tf, x_ibm], # mixture: (waveform, tf_signal, no mask)
# [x, x_tf, x_ibm], # clean speech
# [x, x_tf, x_ibm]
# ]
# fig = display_multiple_signals(signal_list,
# fs=fs, vmin=vmin, vmax=vmax,
# wlen_sec=wlen_sec, hop_percent=hop_percent,
# xticks_sec=xticks_sec, fontsize=fontsize)
title = "quantile_fraction = {:.4f} \n" \
"quantile_weight = {:.4f} \n".format(quantile_fraction, quantile_weight)
fig.suptitle(title, fontsize=40)
# Save figure
# output_path = output_data_dir + os.path.splitext(path)[0] + '_fig.png'
output_path = output_data_dir + os.path.splitext(
path)[0] + '_fig_vad.png'
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
fig.savefig(output_path)
print("data is stored in " + output_data_dir)
def main():
if not os.path.exists(os.path.dirname(output_dataset_file)):
os.makedirs(os.path.dirname(output_dataset_file))
with h5.File(output_dataset_file,
'a',
rdcc_nbytes=rdcc_nbytes,
rdcc_nslots=rdcc_nslots) as f:
for dataset_type in dataset_types:
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
# Delete datasets if already exists
if 'X_' + dataset_type in f:
del f['X_' + dataset_type]
del f['Y_' + dataset_type]
# Exact shape of dataset is unknown in advance unfortunately
# Faster writing if you know the shape in advance
# Size of chunks corresponds to one spectrogram frame
f.create_dataset('X_' + dataset_type,
shape=X_shape,
dtype='float32',
maxshape=X_maxshape,
chunks=X_chunks,
compression=compression,
shuffle=shuffle)
f.create_dataset('Y_' + dataset_type,
shape=Y_shape,
dtype='float32',
maxshape=Y_maxshape,
chunks=Y_chunks,
compression=compression,
shuffle=shuffle)
# STFT attributes
f.attrs['fs'] = fs
f.attrs['wlen_sec'] = wlen_sec
f.attrs['hop_percent'] = hop_percent
f.attrs['win'] = win
f.attrs['dtype'] = dtype
# label attributes
f.attrs['quantile_fraction'] = quantile_fraction
f.attrs['quantile_weight'] = quantile_weight
# HDF5 attributes
f.attrs['X_chunks'] = X_chunks
f.attrs['Y_chunks'] = Y_chunks
f.attrs['compression'] = compression
# Store dataset in variables for faster I/O
fx = f['X_' + dataset_type]
fy = f['Y_' + dataset_type]
for i, file_path in tqdm(enumerate(file_paths)):
speech, fs_speech = sf.read(input_speech_dir + file_path,
samplerate=None)
# Cut burst at begining of file
speech = speech[int(0.1 * fs):]
# Normalize audio
speech = speech / (np.max(np.abs(speech)))
if fs != fs_speech:
raise ValueError('Unexpected sampling rate')
# TF reprepsentation
speech_tf = stft(speech,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype)
spectrogram = np.power(abs(speech_tf), 2)
if labels == 'vad_labels':
# vad
speech_vad = clean_speech_VAD(
speech_tf,
quantile_fraction=quantile_fraction,
quantile_weight=quantile_weight)
label = speech_vad
if labels == 'labels':
# binary mask
speech_ibm = clean_speech_IBM(
speech_tf,
quantile_fraction=quantile_fraction,
quantile_weight=quantile_weight)
label = speech_ibm
# Store spectrogram in dataset
fx.resize((fx.shape[1] + spectrogram.shape[1]), axis=1)
fx[:, -spectrogram.shape[1]:] = spectrogram
# Store spectrogram in label
fy.resize((fy.shape[1] + label.shape[1]), axis=1)
fy[:, -label.shape[1]:] = label
def test_write_read_frames(dataset_type):
# Parameters
## Dataset
input_speech_dir = 'data/subset/raw/'
output_speech_dir = 'data/subset/processed/'
output_data_dir = 'data/subset/h5/'
data_dir = 'CSR-1-WSJ-0'
suffix = 'lzf'
output_h5_dir = output_data_dir + data_dir + '_' + suffix + '.h5'
# Create file list
file_paths = speech_list(input_speech_dir=input_speech_dir,
dataset_type=dataset_type)
# Open hdf5 file
#We are using 400Mb of chunk_cache_mem here ("rdcc_nbytes" and "rdcc_nslots")
with h5.File(output_h5_dir, 'r', rdcc_nbytes=1024**2*400, rdcc_nslots=10e6) as f:
dx = f['X_' + dataset_type]
dy = f['Y_' + dataset_type]
## STFT
fs = f.attrs['fs'] # Sampling rate
wlen_sec = f.attrs['wlen_sec'] # window length in seconds
hop_percent = f.attrs['hop_percent'] # hop size as a percentage of the window length
win = f.attrs['win'] # type of window
dtype = f.attrs['dtype']
## Ideal binary mask
quantile_fraction = f.attrs['quantile_fraction']
quantile_weight = f.attrs['quantile_weight']
frame_begin = 0
frame_end = 0
for path in file_paths:
x, fs_x = sf.read(input_speech_dir + path, samplerate=None)
# Cut burst at begining of file
#x[:int(0.1*fs)] = x[int(0.1*fs):int(0.2*fs)]
x = x[int(0.1*fs):]
# Normalize audio
x = x/(np.max(np.abs(x)))
if fs != fs_x:
raise ValueError('Unexpected sampling rate')
# TF reprepsentation
x_tf = stft(x,
fs=fs,
wlen_sec=wlen_sec,
win=win,
hop_percent=hop_percent,
dtype=dtype)
spectrogram = np.power(abs(x_tf), 2)
# binary mask
label = clean_speech_IBM(x_tf,
quantile_fraction=quantile_fraction,
quantile_weight=quantile_weight)
# Read h5 spectrogram
frame_end += spectrogram.shape[1]
h5_spectrogram = dx[:,frame_begin:frame_end]
h5_label = dy[:,frame_begin:frame_end]
# Assert stored data is same as spectrograms
assert_array_equal(spectrogram, h5_spectrogram)
assert_array_equal(label, h5_label)
# Next iteration
frame_begin += spectrogram.shape[1]