def audio_to_normal_dist(
self, *, spec: torch.Tensor,
audio: torch.Tensor) -> (torch.Tensor, list, list):
audio = split_view(audio, self.n_group, 1).permute(0, 2, 1)
output_audio = []
log_s_list = []
log_det_W_list = []
for k in range(self.n_flows):
if k % self.n_early_every == 0 and k > 0:
output_audio.append(audio[:, :self.n_early_size, :])
audio = audio[:, self.n_early_size:, :]
audio, log_det_W = self.convinv[k](audio)
log_det_W_list.append(log_det_W)
n_half = int(audio.size(1) / 2)
audio_0 = audio[:, :n_half, :]
audio_1 = audio[:, n_half:, :]
output = self.wavenet[k]((audio_0, spec))
log_s = output[:, n_half:, :]
b = output[:, :n_half, :]
audio_1 = torch.exp(log_s) * audio_1 + b
log_s_list.append(log_s)
audio = torch.cat([audio_0, audio_1], 1)
output_audio.append(audio)
return torch.cat(output_audio, 1), log_s_list, log_det_W_list
def audio_to_normal_dist(self, *, spec: torch.Tensor,
audio: torch.Tensor) -> (torch.Tensor, float):
logdet = 0
spec = spec[:, :, :-1]
audio = split_view(audio, self.n_group, 1).permute(0, 2, 1)
if spec.size(2) != audio.size(2):
spec = F.interpolate(spec, size=audio.size(2))
for _ in range(self.n_flows):
audio, log_det_W = self.conv(audio)
logdet += log_det_W
n_half = int(audio.size(1) / 2)
audio_0 = audio[:, :n_half, :]
audio_1 = audio[:, n_half:, :]
output = self.wn((audio_0, spec))
log_s = output[:, n_half:, :]
b = output[:, :n_half, :]
audio_1 = torch.exp(log_s) * audio_1 + b
logdet += torch.sum(log_s)
audio = torch.cat([audio_0, audio_1], 1)
return audio, logdet
def audio_to_normal_dist(
self, *, spec: torch.Tensor,
audio: torch.Tensor) -> (torch.Tensor, list, list):
# Upsample spectrogram to size of audio
spec = self.upsample(spec)
assert spec.size(2) >= audio.size(1)
if spec.size(2) > audio.size(1):
spec = spec[:, :, :audio.size(1)]
# logging.debug(f"spec: {spec.shape}. n_group: {self.n_group}")
spec = split_view(spec, self.n_group, 2).permute(0, 2, 1, 3)
spec = spec.contiguous().view(spec.size(0), spec.size(1), -1)
spec = spec.permute(0, 2, 1)
audio = split_view(audio, self.n_group, 1).permute(0, 2, 1)
output_audio = []
log_s_list = []
log_det_W_list = []
for k in range(self.n_flows):
if k % self.n_early_every == 0 and k > 0:
output_audio.append(audio[:, :self.n_early_size, :])
audio = audio[:, self.n_early_size:, :]
audio, log_det_W = self.convinv[k](audio)
log_det_W_list.append(log_det_W)
n_half = int(audio.size(1) / 2)
audio_0 = audio[:, :n_half, :]
audio_1 = audio[:, n_half:, :]
output = self.wavenet[k]((audio_0, spec))
log_s = output[:, n_half:, :]
b = output[:, :n_half, :]
audio_1 = torch.exp(log_s) * audio_1 + b
log_s_list.append(log_s)
audio = torch.cat([audio_0, audio_1], 1)
output_audio.append(audio)
return torch.cat(output_audio, 1), log_s_list, log_det_W_list
def norm_dist_to_audio(self, *, spec, z=None, sigma: float = 1.0):
if self.converted_to_2D:
spec = torch.unsqueeze(spec, 3)
spec = self.upsample(spec)
spec = spec.contiguous().view(spec.size(0), spec.size(1), -1)
# trim conv artifacts. maybe pad spec to kernel multiple
if self.time_cutoff != 0:
spec = spec[:, :, :self.time_cutoff]
spec = split_view(spec, self.n_group, 2).permute(0, 2, 1, 3)
spec = spec.contiguous().view(spec.size(0), spec.size(1), -1)
spec = spec.permute(0, 2, 1)
z_size = torch.Size([spec.size(0), self.n_group, spec.size(2)])
if z is None:
z = sigma * torch.randn(z_size, device=spec.device).to(spec.dtype)
if self.converted_to_2D:
z = torch.unsqueeze(z, 3)
spec = torch.unsqueeze(spec, 3)
audio, z = torch.split(
z,
[self.n_remaining_channels,
z.size(1) - self.n_remaining_channels], 1)
for k in reversed(range(self.n_flows)):
n_half = self.n_halves[k]
audio_0, audio_1 = torch.split(
audio, [n_half, audio.size(1) - n_half], 1)
output = self.wavenet[k]((audio_0, spec))
b, s = torch.split(output, [n_half, output.size(1) - n_half], 1)
audio_1 = audio_1 - b
audio_1 = audio_1 / torch.exp(s)
audio = torch.cat((audio_0, audio_1), 1)
audio = self.convinv[k](audio, reverse=True)
if k % self.n_early_every == 0 and k > 0:
z1, z = torch.split(
z, [self.n_early_size,
z.size(1) - self.n_early_size], 1)
audio = torch.cat((z1, audio), 1)
if self.converted_to_2D:
audio = audio.view(audio.size(0), audio.size(1), -1)
return audio.permute(0, 2, 1).contiguous().view(audio.size(0), -1)
def get_upsample_factor(self) -> int:
"""
As the MelSpectrogram upsampling is done using interpolation, the upsampling factor is determined
by the ratio of the MelSpectrogram length and the waveform length
Returns:
An integer representing the upsampling factor
"""
audio = torch.ones(1, self._cfg.train_ds.dataset.n_segments)
spec, spec_len = self.audio_to_melspec_precessor(
audio, torch.FloatTensor([len(audio)]))
spec = spec[:, :, :-1]
audio = split_view(audio, self._cfg.uniglow.n_group,
1).permute(0, 2, 1)
upsample_factor = audio.shape[2] // spec.shape[2]
return upsample_factor