def generate_spectrogram(self, tokens: torch.Tensor,
**kwargs) -> torch.Tensor:
assert hasattr(self, '_durs_model') and hasattr(self, '_pitch_model')
if self.blanking:
tokens = [
AudioToCharWithDursF0Dataset.interleave(
x=torch.empty(len(t) + 1,
dtype=torch.long,
device=t.device).fill_(self.vocab.blank),
y=t,
) for t in tokens
]
tokens = AudioToCharWithDursF0Dataset.merge(tokens,
value=self.vocab.pad,
dtype=torch.long)
text_len = torch.tensor(tokens.shape[-1],
dtype=torch.long).unsqueeze(0)
durs = self._durs_model(tokens, text_len)
durs = durs.exp() - 1
durs[durs < 0.0] = 0.0
durs = durs.round().long()
# Pitch
f0_sil, f0_body = self._pitch_model(tokens, text_len, durs)
sil_mask = f0_sil.sigmoid() > 0.5
f0 = f0_body * self._pitch_model.f0_std + self._pitch_model.f0_mean
f0 = (~sil_mask * f0).float()
# Spect
mel = self(tokens, text_len, durs, f0)
return mel
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
if isinstance(cfg, dict):
cfg = OmegaConf.create(cfg)
self.learn_alignment = False
if "learn_alignment" in cfg:
self.learn_alignment = cfg.learn_alignment
self._parser = None
self._tb_logger = None
super().__init__(cfg=cfg, trainer=trainer)
schema = OmegaConf.structured(FastPitchConfig)
# ModelPT ensures that cfg is a DictConfig, but do this second check in case ModelPT changes
if isinstance(cfg, dict):
cfg = OmegaConf.create(cfg)
elif not isinstance(cfg, DictConfig):
raise ValueError(
f"cfg was type: {type(cfg)}. Expected either a dict or a DictConfig"
)
# Ensure passed cfg is compliant with schema
OmegaConf.merge(cfg, schema)
self.bin_loss_warmup_epochs = 100
self.aligner = None
self.log_train_images = False
self.mel_loss = MelLoss()
loss_scale = 0.1 if self.learn_alignment else 1.0
self.pitch_loss = PitchLoss(loss_scale=loss_scale)
self.duration_loss = DurationLoss(loss_scale=loss_scale)
input_fft_kwargs = {}
if self.learn_alignment:
self.aligner = instantiate(self._cfg.alignment_module)
self.forward_sum_loss = ForwardSumLoss()
self.bin_loss = BinLoss()
self.vocab = AudioToCharWithDursF0Dataset.make_vocab(
**self._cfg.train_ds.dataset.vocab)
input_fft_kwargs["n_embed"] = len(self.vocab.labels)
input_fft_kwargs["padding_idx"] = self.vocab.pad
self.preprocessor = instantiate(self._cfg.preprocessor)
input_fft = instantiate(self._cfg.input_fft, **input_fft_kwargs)
output_fft = instantiate(self._cfg.output_fft)
duration_predictor = instantiate(self._cfg.duration_predictor)
pitch_predictor = instantiate(self._cfg.pitch_predictor)
self.fastpitch = FastPitchModule(
input_fft,
output_fft,
duration_predictor,
pitch_predictor,
self.aligner,
cfg.n_speakers,
cfg.symbols_embedding_dim,
cfg.pitch_embedding_kernel_size,
cfg.n_mel_channels,
)
def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None):
super().__init__(cfg=cfg, trainer=trainer)
cfg = self._cfg
self.vocab = AudioToCharWithDursF0Dataset.make_vocab(
**cfg.train_ds.dataset.vocab)
self.embed = nn.Embedding(len(self.vocab.labels), cfg.d_char)
self.encoder = instantiate(cfg.encoder)
d_out = cfg.encoder.jasper[-1].filters
self.proj = nn.Conv1d(d_out, 1, kernel_size=1)
def __init__(self, cfg: DictConfig):
super().__init__(cfg=cfg)
typecheck.set_typecheck_enabled(enabled=False)
cfg = self._cfg
self.vocab = AudioToCharWithDursF0Dataset.make_vocab(**cfg.train_ds.dataset.vocab)
self.embed = nn.Embedding(len(self.vocab.labels), cfg.d_char)
self.model = instantiate(cfg.model)
d_out = cfg.model.jasper[-1].filters
self.proj = nn.Conv1d(d_out, 1, kernel_size=1)
def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None):
super().__init__(cfg=cfg, trainer=trainer)
cfg = self._cfg
self.vocab = AudioToCharWithDursF0Dataset.make_vocab(
**cfg.train_ds.dataset.vocab)
self.embed = GaussianEmbedding(self.vocab, cfg.d_char)
self.encoder = instantiate(cfg.encoder)
d_out = cfg.encoder.jasper[-1].filters
self.sil_proj = nn.Conv1d(d_out, 1, kernel_size=1)
self.body_proj = nn.Conv1d(d_out, 1, kernel_size=1)
self.f0_mean, self.f0_std = cfg.f0_mean, cfg.f0_std
def __init__(self, cfg: DictConfig):
super().__init__(cfg=cfg)
typecheck.set_typecheck_enabled(enabled=False)
cfg = self._cfg
self.vocab = AudioToCharWithDursF0Dataset.make_vocab(**cfg.train_ds.dataset.vocab)
self.preprocessor = instantiate(cfg.preprocessor)
self.embed = GaussianEmbedding(self.vocab, cfg.d_char)
self.norm_f0 = MaskedInstanceNorm1d(1)
self.res_f0 = StyleResidual(cfg.d_char, 1, kernel_size=3)
self.model = instantiate(cfg.model)
d_out = cfg.model.jasper[-1].filters
self.proj = nn.Conv1d(d_out, cfg.n_mels, kernel_size=1)
def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None):
super().__init__(cfg=cfg, trainer=trainer)
cfg = self._cfg
self.vocab = AudioToCharWithDursF0Dataset.make_vocab(
**cfg.train_ds.dataset.vocab)
self.blanking = cfg.train_ds.dataset.blanking
self.preprocessor = instantiate(cfg.preprocessor)
self.embed = GaussianEmbedding(self.vocab, cfg.d_char)
self.norm_f0 = MaskedInstanceNorm1d(1)
self.res_f0 = StyleResidual(cfg.d_char, 1, kernel_size=3)
self.encoder = instantiate(cfg.encoder)
d_out = cfg.encoder.jasper[-1].filters
self.proj = nn.Conv1d(d_out, cfg.n_mels, kernel_size=1)
def __init__(self, cfg: DictConfig, trainer: 'Trainer' = None):
super().__init__(cfg=cfg, trainer=trainer)
typecheck.set_typecheck_enabled(enabled=False)
cfg = self._cfg
self.vocab = AudioToCharWithDursF0Dataset.make_vocab(**cfg.train_ds.dataset.vocab)
self.embed = nn.Embedding(len(self.vocab.labels), cfg.d_char)
self.preprocessor = instantiate(cfg.preprocessor)
self.alignment_encoder = instantiate(cfg.alignment_encoder)
self.forward_sum_loss = ForwardSumLoss()
self.bin_loss = BinLoss()
self.bin_start_ratio = cfg.bin_start_ratio
self.add_bin_loss = False
def parser(self):
if self._parser is not None:
return self._parser
if self.learn_alignment:
vocab = AudioToCharWithDursF0Dataset.make_vocab(
**self._cfg.train_ds.dataset.vocab)
self._parser = vocab.encode
else:
self._parser = parsers.make_parser(
labels=self._cfg.labels,
name='en',
unk_id=-1,
blank_id=-1,
do_normalize=True,
abbreviation_version="fastpitch",
make_table=False,
)
return self._parser
def forward(self, text, durs):
"""See base class."""
# Fake padding
text = F.pad(text, [0, 2, 0, 0], value=self.pad)
durs = F.pad(durs, [0, 2, 0, 0], value=0)
repeats = AudioToCharWithDursF0Dataset.repeat_merge(text, durs, self.pad)
total_time = repeats.shape[-1]
# Centroids: [B,T,N]
c = (durs / 2.0) + F.pad(torch.cumsum(durs, dim=-1)[:, :-1], [1, 0, 0, 0], value=0)
c = c.unsqueeze(1).repeat(1, total_time, 1)
# Sigmas: [B,T,N]
sigmas = durs
sigmas = sigmas.float() / self.sigma_c
sigmas = sigmas.unsqueeze(1).repeat(1, total_time, 1) + self.EPS
assert c.shape == sigmas.shape
# Times at indexes
t = torch.arange(total_time, device=c.device).view(1, -1, 1).repeat(durs.shape[0], 1, durs.shape[-1]).float()
t = t + 0.5
ns = slice(None)
if self.merge_blanks:
ns = slice(1, None, 2)
# Weights: [B,T,N]
d = torch.distributions.normal.Normal(c, sigmas)
w = d.log_prob(t).exp()[:, :, ns] # [B,T,N]
pad_mask = (text == self.pad)[:, ns].unsqueeze(1).repeat(1, total_time, 1)
w.masked_fill_(pad_mask, 0.0) # noqa
w = w / (w.sum(-1, keepdim=True) + self.EPS)
pad_mask = (repeats == self.pad).unsqueeze(-1).repeat(1, 1, text[:, ns].size(1)) # noqa
w.masked_fill_(pad_mask, 0.0) # noqa
pad_mask[:, :, :-1] = False
w.masked_fill_(pad_mask, 1.0) # noqa
# Embeds
u = torch.bmm(w, self.embed(text)[:, ns, :]) # [B,T,E]
return u
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
# Convert to Hydra 1.0 compatible DictConfig
cfg = model_utils.convert_model_config_to_dict_config(cfg)
cfg = model_utils.maybe_update_config_version(cfg)
# Setup normalizer
self.normalizer = None
self.text_normalizer_call = None
self.text_normalizer_call_kwargs = {}
self._setup_normalizer(cfg)
self.learn_alignment = cfg.get("learn_alignment", False)
# Setup vocabulary (=tokenizer) and input_fft_kwargs (supported only with self.learn_alignment=True)
input_fft_kwargs = {}
if self.learn_alignment:
self.vocab = None
self.ds_class_name = cfg.train_ds.dataset._target_.split(".")[-1]
if self.ds_class_name == "AudioToCharWithPriorAndPitchDataset":
self.vocab = AudioToCharWithDursF0Dataset.make_vocab(
**cfg.train_ds.dataset.vocab)
input_fft_kwargs["n_embed"] = len(self.vocab.labels)
input_fft_kwargs["padding_idx"] = self.vocab.pad
elif self.ds_class_name == "TTSDataset":
self._setup_tokenizer(cfg)
assert self.vocab is not None
input_fft_kwargs["n_embed"] = len(self.vocab.tokens)
input_fft_kwargs["padding_idx"] = self.vocab.pad
else:
raise ValueError(
f"Unknown dataset class: {self.ds_class_name}")
self._parser = None
self._tb_logger = None
super().__init__(cfg=cfg, trainer=trainer)
self.bin_loss_warmup_epochs = cfg.get("bin_loss_warmup_epochs", 100)
self.log_train_images = False
loss_scale = 0.1 if self.learn_alignment else 1.0
dur_loss_scale = loss_scale
pitch_loss_scale = loss_scale
if "dur_loss_scale" in cfg:
dur_loss_scale = cfg.dur_loss_scale
if "pitch_loss_scale" in cfg:
pitch_loss_scale = cfg.pitch_loss_scale
self.mel_loss = MelLoss()
self.pitch_loss = PitchLoss(loss_scale=pitch_loss_scale)
self.duration_loss = DurationLoss(loss_scale=dur_loss_scale)
self.aligner = None
if self.learn_alignment:
self.aligner = instantiate(self._cfg.alignment_module)
self.forward_sum_loss = ForwardSumLoss()
self.bin_loss = BinLoss()
self.preprocessor = instantiate(self._cfg.preprocessor)
input_fft = instantiate(self._cfg.input_fft, **input_fft_kwargs)
output_fft = instantiate(self._cfg.output_fft)
duration_predictor = instantiate(self._cfg.duration_predictor)
pitch_predictor = instantiate(self._cfg.pitch_predictor)
self.fastpitch = FastPitchModule(
input_fft,
output_fft,
duration_predictor,
pitch_predictor,
self.aligner,
cfg.n_speakers,
cfg.symbols_embedding_dim,
cfg.pitch_embedding_kernel_size,
cfg.n_mel_channels,
)
self._input_types = self._output_types = None