def step(self, data):
x = cuda_variable(data['input'])
target = cuda_variable(data['target']).long()
recon_batch, mu, logvar = self(x)
batch_dim = x.shape[0]
recon_batch_flat = recon_batch.view(batch_dim, -1)
target_flat = target.view(batch_dim, -1)
loss = self.loss_function(recon_batch_flat, target_flat, mu, logvar,
self.beta)
return loss
def plot_tsne_latent(model, dataloader, savepath):
# Forward pass
model.eval()
zs = []
labels = []
for batch_counter, data in enumerate(dataloader):
x_cuda = cuda_variable(data['input'])
# Get z
mu, logvar = model.encode(x_cuda)
z = model.reparameterize(mu, logvar)
zs += list(z.cpu().detach().numpy())
labels += data['label']
if batch_counter > 20:
break
z_embedded = TSNE(n_components=3).fit_transform(zs)
label_to_points = {}
for index, label in enumerate(labels):
if label not in label_to_points:
label_to_points[label] = []
label_to_points[label].append(z_embedded[index])
# Plot
plt.clf()
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(111, projection='3d')
colormap = mpl.cm.Set1.colors
for colorind, (label, points) in enumerate(label_to_points.items()):
points_np = np.asarray(points)
color = colormap[colorind]
xs = points_np[:, 0]
ys = points_np[:, 1]
zs = points_np[:, 2]
ax.scatter(xs, ys, zs, color=color, marker='o', label=label)
ax.legend()
plt.show()
plt.savefig(f'{savepath}/tsne.pdf')
plt.close('all')
return label_to_points
def plot_interpolations(model, hparams, dataloader, savepath,
num_interpolated_points, method, custom_data):
# Forward pass
model.eval()
if custom_data is None:
for _, data in enumerate(dataloader):
x_cuda = cuda_variable(data['input'])
# Get z
mu, logvar = model.encode(x_cuda)
z = model.reparameterize(mu, logvar)
# Arbitrarily choose start and end points as batch_ind and batch_ind + 1
start_z = z[:-1]
end_z = z[1:]
batch_dim, rgb_dim, h_dim, w_dim = x_cuda.shape
num_examples = batch_dim - 1
break
else:
x_cuda = cuda_variable(torch.tensor(custom_data['start_data']))
# Get z
mu, logvar = model.encode(x_cuda)
start_z = model.reparameterize(mu, logvar)
x_cuda = cuda_variable(torch.tensor(custom_data['end_data']))
# Get z
mu, logvar = model.encode(x_cuda)
end_z = model.reparameterize(mu, logvar)
batch_dim, rgb_dim, h_dim, w_dim = x_cuda.shape
num_examples = batch_dim
x_interpolation = np.zeros(
(num_examples, rgb_dim, h_dim, w_dim, num_interpolated_points))
ind_interp = 0
for t in np.linspace(start=0, stop=1, num=num_interpolated_points):
# Perform interp
if method == 'linear':
this_z = start_z * (1 - t) + end_z * t
elif method == 'constant_radius':
this_z = constant_radius_interpolation(start_z, end_z, t)
else:
raise NotImplementedError
# Decode z
x_recon = model.decode(this_z).cpu().detach().numpy()
x_interpolation[:, :, :, :, ind_interp] = x_recon
ind_interp = ind_interp + 1
# Plot
dims = h_dim, w_dim
plt.clf()
fig, axes = plt.subplots(nrows=num_examples, ncols=num_interpolated_points)
for ind_example in range(num_examples):
for ind_interp in range(num_interpolated_points):
# show the image
axes[ind_example,
ind_interp].matshow(x_interpolation[ind_example, :, :, :,
ind_interp].reshape(dims),
origin="lower")
for ax in fig.get_axes():
ax.set_xticks([])
ax.set_yticks([])
plt.savefig(f'{savepath}/spectro.pdf')
plt.close('all')
# audio
audios = None
if hparams is not None:
mel_basis = build_mel_basis(hparams, hparams.sr, hparams.sr)
mel_inversion_basis = build_mel_inversion_basis(mel_basis)
for ind_example in range(num_examples):
for ind_interp in range(num_interpolated_points):
audio = inv_spectrogram_sp(
x_interpolation[ind_example, 0, :, :, ind_interp],
n_fft=hparams.n_fft,
win_length=hparams.win_length_samples,
hop_length=hparams.hop_length_samples,
ref_level_db=hparams.ref_level_db,
power=hparams.power,
mel_inversion_basis=mel_inversion_basis)
if audios is None:
audios = np.zeros(
(num_examples, num_interpolated_points, len(audio)))
audios[ind_example, ind_interp] = audio
sf.write(f'{savepath}/{ind_example}_{ind_interp}.wav',
audio,
samplerate=hparams.sr)
return {'audios': audios, 'spectros': x_interpolation, 'dims': dims}
def generate(self, batch_dim):
# Prior is a gaussian w/ mean 0 and variance 1
z = cuda_variable(torch.randn(batch_dim, self.n_z))
x = self.decode(z)
return x
def main(config_path, loading_epoch, source_path, contamination_path,
contamination_parameters, method):
# load model
model, _, _, _, hparams, _, _, config_path = get_model_and_dataset(
config=config_path, loading_epoch=loading_epoch)
# set savepath
savepath = f'{model.model_dir}/plots/contaminations'
if not os.path.isdir(savepath):
os.makedirs(savepath)
# load files
source = {}
source['path'] = source_path
waveform, chunks, start_samples, end_samples = get_chunks(
path=source['path'], hparams=hparams)
source['waveform'] = waveform
source['chunks'] = chunks
source['start_samples'] = start_samples
source['end_samples'] = end_samples
contamination = {}
contamination['path'] = contamination_path
waveform, chunks, start_samples, end_samples = get_chunks(
path=contamination['path'], hparams=hparams)
contamination['waveform'] = waveform
contamination['chunks'] = chunks
contamination['start_samples'] = start_samples
contamination['end_samples'] = end_samples
# Choose which samples to contaminate and by which degree
contamination_indices = []
contamination_degrees = []
xs = []
ys = []
p_contamination = contamination_parameters['p_contamination']
for index, chunk in enumerate(source['chunks']):
if random.random() < p_contamination:
contamination_indices.append(index)
contamination_degrees.append(random.random())
xs.append(chunk)
# choose (randomly?) a contaminating syllable
ys.append(random.choice(contamination['chunks']))
xs_cuda = cuda_variable(torch.tensor(np.stack(xs)))
ys_cuda = cuda_variable(torch.tensor(np.stack(ys)))
# Encode
mu, logvar = model.encode(xs_cuda)
x_z = model.reparameterize(mu, logvar)
mu, logvar = model.encode(ys_cuda)
y_z = model.reparameterize(mu, logvar)
z_out = torch.zeros_like(x_z)
# Contaminate
for batch_ind, t in enumerate(contamination_degrees):
if method == 'linear':
z_out[batch_ind] = x_z[batch_ind] * (1 - t) + y_z[batch_ind] * t
elif method == 'constant_radius':
z_out[batch_ind] = constant_radius_interpolation(
x_z[batch_ind], y_z[batch_ind], t)
# Decode z
x_recon = model.decode(z_out).cpu().detach().numpy()
# Replace contamined samples in original wave
out_wave = source['waveform']
mel_basis = build_mel_basis(hparams, hparams.sr, hparams.sr)
mel_inversion_basis = build_mel_inversion_basis(mel_basis)
for batch_index, contamination_index in enumerate(contamination_indices):
new_chunk = x_recon[batch_index, 0]
new_audio = inv_spectrogram_sp(new_chunk,
n_fft=hparams.n_fft,
win_length=hparams.win_length_samples,
hop_length=hparams.hop_length_samples,
ref_level_db=hparams.ref_level_db,
power=hparams.power,
mel_inversion_basis=mel_inversion_basis)
start_sample = source['start_samples'][contamination_index]
end_sample = source['end_samples'][contamination_index]
length_sample = end_sample - start_sample
# FAIRE UN FADE ICI
out_wave[start_sample:end_sample] = new_audio[:length_sample]
sf.write(f'{savepath}/contamination.wav', out_wave, samplerate=hparams.sr)
def plot_reconstruction(model, hparams, dataloader, savepath, custom_data):
# Forward pass
model.eval()
if custom_data is None:
for _, data in enumerate(dataloader):
x_orig = data['input'].numpy()
x_cuda = cuda_variable(data['input'])
x_recon = model.reconstruct(x_cuda).cpu().detach().numpy()
break
else:
x_orig = custom_data['all_data']
x_cuda = cuda_variable(torch.tensor(custom_data['all_data']))
x_recon = model.reconstruct(x_cuda).cpu().detach().numpy()
# Plot
dims = x_recon.shape[2:]
num_examples = x_recon.shape[0]
plt.clf()
fig, axes = plt.subplots(nrows=2, ncols=num_examples)
for i in range(num_examples):
# show the image
axes[0, i].matshow(x_orig[i].reshape(dims), origin="lower")
axes[1, i].matshow(x_recon[i].reshape(dims), origin="lower")
for ax in fig.get_axes():
ax.set_xticks([])
ax.set_yticks([])
plt.savefig(f'{savepath}/spectro.pdf')
plt.close('all')
# audio
original_audios = []
reconstruction_audios = []
if hparams is not None:
mel_basis = build_mel_basis(hparams, hparams.sr, hparams.sr)
mel_inversion_basis = build_mel_inversion_basis(mel_basis)
for i in range(num_examples):
original_audio = inv_spectrogram_sp(
x_orig[i, 0],
n_fft=hparams.n_fft,
win_length=hparams.win_length_samples,
hop_length=hparams.hop_length_samples,
ref_level_db=hparams.ref_level_db,
power=hparams.power,
mel_inversion_basis=mel_inversion_basis)
recon_audio = inv_spectrogram_sp(
x_recon[i, 0],
n_fft=hparams.n_fft,
win_length=hparams.win_length_samples,
hop_length=hparams.hop_length_samples,
ref_level_db=hparams.ref_level_db,
power=hparams.power,
mel_inversion_basis=mel_inversion_basis)
sf.write(f'{savepath}/{i}_original.wav',
original_audio,
samplerate=hparams.sr)
sf.write(f'{savepath}/{i}_recon.wav',
recon_audio,
samplerate=hparams.sr)
original_audios.append(original_audio)
reconstruction_audios.append(recon_audio)
return {
'original_audios': original_audios,
'reconstruction_audios': reconstruction_audios,
'original_spectros': x_orig,
'reconstruction_spectros': x_recon
}