def plot_histograms(model, x, y, epoch, out_dir):
"""
Plots some histograms
:param model: a C-GAE instance
:param batch: the batch to use for computing histograms
:param epoch: epoch nr
:param out_dir: directory where to save the histograms
"""
plot_hist(to_numpy(x), f"data_ep{epoch}",
os.path.join(out_dir, f"hist_data_ep{epoch}.png"))
mapping = to_numpy(model.mapping_(x, y))
plot_hist(mapping, f"mapping_ep{epoch}",
os.path.join(out_dir, f"hist_map_ep{epoch}.png"))
output = to_numpy(model(model.mapping_(x, y), y))
plot_hist(output, f"recon_ep{epoch}",
os.path.join(out_dir, f"hist_recon_ep{epoch}.png"))
weight_x = to_numpy(model.conv_x.weight)
plot_hist(weight_x, f"weightx_ep{epoch}",
os.path.join(out_dir, f"hist_weightx_ep{epoch}.png"))
weight_y = to_numpy(model.conv_y.weight)
plot_hist(weight_y, f"weighty_ep{epoch}",
os.path.join(out_dir, f"hist_weighty_ep{epoch}.png"))
def plot_kernels(model, epoch, out_dir):
"""
Plots the input weights of the C-GAE
:param model: a C-GAE instance
:param epoch: epoch nr (int)
:param out_dir: directory where to save the plot
"""
filter_x = to_numpy(model.conv_x.weight)
make_tiles(filter_x, os.path.join(out_dir, f"filtersx_ep{epoch}.png"))
filter_y = to_numpy(model.conv_y.weight)
make_tiles(filter_y, os.path.join(out_dir, f"filtersy_ep{epoch}.png"))
def plot_data(data, epoch, out_dir):
"""
Plots input data
:param data: a data batch to plot
:param epoch: epoch nr
:param out_dir: directory where to save the plot
"""
make_tiles(to_numpy(data), os.path.join(out_dir, f"data_ep{epoch}.png"))
def plot_recon(model, x, y, epoch, out_dir):
"""
Plots the reconstruction of an input batch
:param model: a C-GAE instance
:param batch: the batch to reconstruct
:param epoch: epoch nr
:param out_dir: directory where to save the plot
"""
output = to_numpy(model(model.mapping_(x, y), y))
make_tiles(output, os.path.join(out_dir, f"recon_ep{epoch}.png"))
def plot_mapping(model, x, y, epoch, out_dir):
"""
Plots the top most mapping layer given an input batch
:param model: a C-GAE instance
:param batch: the batch to reconstruct
:param epoch: epoch nr
:param out_dir: directory where to save the plot
"""
# x = cuda_variable(batch[0][None,:,:,:])
# y = cuda_variable(batch[1][None,:,:,:])
output = np.transpose(to_numpy(model.mapping_(x, y)), axes=(0, 3, 2, 1))
make_tiles(output, os.path.join(out_dir, f"mapping_ep{epoch}.png"))
def create_ss_matrix(ampls, mode='cosine'):
matrix = squareform(pdist(np.vstack(to_numpy(ampls)), metric=mode))
return matrix
def plot_train_state_2d(loss_curve_eval, loss_curve_train, model, x, y,
epoch, out_dir, length_ngram):
model.eval()
x = cuda_variable(x)
y = cuda_variable(y)
# calculate mapping of untransposed data
amp_x, phase_x = model(x)
amp_y, phase_y = model(y)
recon_y = model.backward(amp_x, phase_y)
recon_x = model.backward(amp_y, phase_x)
make_tiles(to_numpy(recon_y).reshape(recon_y.shape[0], 1, -1,
length_ngram),
os.path.join(out_dir, f"recon_y{epoch}.png"))
make_tiles(to_numpy(recon_x).reshape(recon_x.shape[0], 1, -1,
length_ngram),
os.path.join(out_dir, f"recon_x{epoch}.png"))
plot_hist(to_numpy(recon_x), f"recon_x_hist_ep{epoch}",
os.path.join(out_dir, f"recon_x_hist_ep{epoch}.png"))
plot_hist(to_numpy(recon_y), f"recon_y_hist_ep{epoch}",
os.path.join(out_dir, f"recon_y_hist_ep{epoch}.png"))
half_weight = model.layer.weight.shape[0] // 2
make_tiles(to_numpy(model.layer.weight[:half_weight]).reshape(len(
model.layer.weight[:half_weight]),
1, -1,
length_ngram),
os.path.join(out_dir, f"filters_x{epoch}.png"))
make_tiles(to_numpy(model.layer.weight[half_weight:]).reshape(len(
model.layer.weight[half_weight:]),
1, -1,
length_ngram),
os.path.join(out_dir, f"filters_y{epoch}.png"))
plot_hist(to_numpy(model.layer.weight[:half_weight]).reshape(len(
model.layer.weight[:half_weight]),
1, -1,
length_ngram),
f"filters_x_hist_ep{epoch}",
os.path.join(out_dir, f"filters_x_hist_ep{epoch}.png"))
plot_hist(to_numpy(model.layer.weight[half_weight:]).reshape(len(
model.layer.weight[half_weight:]),
1, -1,
length_ngram),
f"filters_y_hist_ep{epoch}",
os.path.join(out_dir, f"filters_y_hist_ep{epoch}.png"))
# make_tiles(to_numpy(model.layer.weight).reshape(len(
# model.layer.weight),
# 1, -1,
# length_ngram),
# os.path.join(out_dir, f"filters_x{epoch}.png"))
make_tiles(to_numpy(amp_x)[:, None, None, :], os.path.join(out_dir,
f"ampx_{epoch}.png"))
plot_hist(to_numpy(amp_x), f"ampx_hist_ep{epoch}",
os.path.join(out_dir, f"ampx_hist_ep{epoch}.png"))
make_tiles(to_numpy(amp_y)[:, None, None, :], os.path.join(out_dir,
f"ampy_{epoch}.png"))
plot_hist(to_numpy(amp_y), f"ampy_hist_ep{epoch}",
os.path.join(out_dir, f"ampy_hist_ep{epoch}.png"))
make_tiles(to_numpy(phase_x)[:, None, None, :], os.path.join(out_dir,
f"phasex_{epoch}.png"))
plot_hist(to_numpy(phase_x), f"phasex_hist_ep{epoch}",
os.path.join(out_dir, f"phasex_hist_ep{epoch}.png"))
make_tiles(to_numpy(phase_y)[:, None, None, :], os.path.join(out_dir,
f"phasey_{epoch}.png"))
plot_hist(to_numpy(phase_y), f"phasey_hist_ep{epoch}",
os.path.join(out_dir, f"phasey_hist_ep{epoch}.png"))
plot_hist(to_numpy(x.data), f"input_hist_ep{epoch}",
os.path.join(out_dir, f"input_hist_ep{epoch}.png"))
plot_hist(to_numpy(y.data), f"target_hist_ep{epoch}",
os.path.join(out_dir, f"target_hist_ep{epoch}.png"))
input_np = to_numpy(x.data)
make_tiles(input_np.reshape(x.shape[0], 1, -1, length_ngram),
os.path.join(out_dir, f"input_{epoch}.png"))
target_np = to_numpy(y.data)
make_tiles(target_np.reshape(y.shape[0], 1, -1, length_ngram),
os.path.join(out_dir, f"target_{epoch}.png"))
plt.clf()
plt.plot(loss_curve_train)
plt.plot(loss_curve_eval)
plt.savefig(
os.path.join(out_dir, f"loss_curve_{epoch}.png"))
def plot_train_state_1d(loss_curve_eval, loss_curve_train, model, x, y,
epoch, out_dir, length_ngram):
model.eval()
x = cuda_variable(x)
y = cuda_variable(y)
# calculate mapping of untransposed data
amp_x, phase_x = model(x)
amp_y, phase_y = model(y)
recon_y = model.backward(amp_x, phase_y)
recon_x = model.backward(amp_y, phase_x)
plot_audiobatch(recon_y[:20, None, :].detach().cpu(),
os.path.join(out_dir, f"recon_y{epoch}.png"))
plot_audiobatch(recon_x[:20, None, :].detach().cpu(),
os.path.join(out_dir, f"recon_x{epoch}.png"))
plot_audiobatch(y[:20, None, :].detach().cpu(),
os.path.join(out_dir, f"input_y_sig{epoch}.png"))
plot_audiobatch(x[:20, None, :].detach().cpu(),
os.path.join(out_dir, f"input_x_sig{epoch}.png"))
plot_hist(to_numpy(recon_x), f"recon_x_hist_ep{epoch}",
os.path.join(out_dir, f"recon_x_hist_ep{epoch}.png"))
plot_hist(to_numpy(recon_y), f"recon_y_hist_ep{epoch}",
os.path.join(out_dir, f"recon_y_hist_ep{epoch}.png"))
make_tiles(to_numpy(model.layer.weight).reshape(len(
model.layer.weight),
1, -1,
length_ngram),
os.path.join(out_dir, f"filters_x{epoch}.png"))
make_tiles(to_numpy(model.layer.weight).reshape(len(
model.layer.weight),
1, -1,
length_ngram),
os.path.join(out_dir, f"filters_y{epoch}.png"))
plot_audiobatch(model.layer.weight[:20, None, :].detach().cpu(),
os.path.join(out_dir, f"filters_sig_real{epoch}.png"))
half = model.layer.weight.shape[0] // 2
plot_audiobatch(model.layer.weight[half:half+20, None, :].detach().cpu(),
os.path.join(out_dir, f"filters_sig_compl{epoch}.png"))
make_tiles(to_numpy(amp_x)[:, None, None, :], os.path.join(out_dir,
f"ampx_{epoch}.png"))
plot_hist(to_numpy(amp_x), f"ampx_hist_ep{epoch}",
os.path.join(out_dir, f"ampx_hist_ep{epoch}.png"))
make_tiles(to_numpy(amp_y)[:, None, None, :], os.path.join(out_dir,
f"ampy_{epoch}.png"))
plot_hist(to_numpy(amp_y), f"ampy_hist_ep{epoch}",
os.path.join(out_dir, f"ampy_hist_ep{epoch}.png"))
make_tiles(to_numpy(phase_x)[:, None, None, :], os.path.join(out_dir,
f"phasex_{epoch}.png"))
plot_hist(to_numpy(phase_x), f"phasex_hist_ep{epoch}",
os.path.join(out_dir, f"phasex_hist_ep{epoch}.png"))
make_tiles(to_numpy(phase_y)[:, None, None, :], os.path.join(out_dir,
f"phasey_{epoch}.png"))
plot_hist(to_numpy(phase_y), f"phasey_hist_ep{epoch}",
os.path.join(out_dir, f"phasey_hist_ep{epoch}.png"))
plot_hist(to_numpy(x.data), f"input_hist_ep{epoch}",
os.path.join(out_dir, f"input_hist_ep{epoch}.png"))
plot_hist(to_numpy(y.data), f"target_hist_ep{epoch}",
os.path.join(out_dir, f"target_hist_ep{epoch}.png"))
input_np = to_numpy(x.data)
make_tiles(input_np.reshape(x.shape[0], 1, -1, length_ngram),
os.path.join(out_dir, f"input_{epoch}.png"))
target_np = to_numpy(y.data)
make_tiles(target_np.reshape(y.shape[0], 1, -1, length_ngram),
os.path.join(out_dir, f"target_{epoch}.png"))
plt.clf()
plt.plot(loss_curve_train)
plt.plot(loss_curve_eval)
plt.savefig(
os.path.join(out_dir, f"loss_curve_{epoch}.png"))
def trans_pitch_shift(self, ngram, shifty):
return to_numpy(self.transp0(torch.FloatTensor(ngram), shifty))
def __getitem__(self, index):
# Transform: pitch_shift (0), time shift (1), tempo-change (2)
if self.transform is None:
# random transform
transform = np.random.randint(0, 3)
else:
transform = np.random.choice(self.transform)
if self.random_pairs:
# song_id, start, end = self.get_random_ngram()
# ngram = self.data_x[song_id][:, start:end].copy()
# song_id, start, end = self.get_random_ngram()
# ngram_trans = self.data_x[song_id][:, start:end].copy()
if np.random.randint(2) == 0:
[ngram, ngram_trans], song_id = self.get_pairs_same_song()
label = -1
transform = -1 # skips transformation codes
else:
song_id, start, end = self.get_ngram_by_idx(index)
ngram = self.data_x[song_id][:, start:end].copy()
elif self.shuffle:
song_id, start, end = self.get_random_ngram()
ngram = self.data_x[song_id][:, start:end].copy()
else:
song_id, start, end = self.get_ngram_by_idx(index)
ngram = self.data_x[song_id][:, start:end].copy()
# Normalization needed for PIL image processing (scale)
ngram -= ngram.min()
if ngram.max() > 1e-6:
ngram /= ngram.max()
assert ngram.shape[1] != 0, f"{start}, {end}," \
f"{self.data_x[song_id].shape[1]}, " \
f"{self.max_x}"
if transform == 1:
if self.max_x == 0:
shiftx = 0
else:
shiftx = np.random.randint(-self.max_x, self.max_x)
ngram_trans = self.trans_time_shift(end, song_id, start, shiftx)
label = "shiftx" + str(shiftx)
if transform == 0:
if self.max_y == 0:
shifty = 0
else:
shifty = np.random.randint(-self.max_y, self.max_y)
ngram_trans = self.trans_pitch_shift(ngram, shifty)
label = "shifty" + str(shifty)
if transform == 2:
scale_x = 1 + self.scale_x * np.random.rand()
ngram, ngram_trans, minus = self.trans_speed_change(ngram, scale_x)
label = scale_x if not minus else -scale_x
label = "scale" + str(label)
ngram = to_numpy(ngram)
ngram_trans = to_numpy(ngram_trans)
ngram_onset = np.diff(np.concatenate((ngram[:, 0:1], ngram), axis=1),
axis=1)
ngram_trans_onset = np.diff(np.concatenate(
(ngram_trans[:, 0:1], ngram_trans), axis=1),
axis=1)
ngram_onset[ngram_onset < 0] = 0
ngram_trans_onset[ngram_trans_onset < 0] = 0
ngram = ngram + ngram_onset * self.emph_onset
ngram_trans = ngram_trans + ngram_trans_onset * self.emph_onset
if self.standard:
ngram = self.standardize(ngram)
ngram_trans = self.standardize(ngram_trans)
ngram = torch.FloatTensor(ngram).view(-1)
ngram_trans = torch.FloatTensor(ngram_trans).view(-1)
return ngram + 1e-8, ngram_trans + 1e-8, transform, song_id, label