class TextMelLoader(torch.utils.data.Dataset): """ 1) loads audio,text pairs 2) normalizes text and converts them to sequences of one-hot vectors 3) computes mel-spectrograms from audio files. """ def __init__(self, args, files): self.audiopaths_and_text = load_filepaths_and_text( args.dataset_path, files) self.text_cleaners = args.text_cleaners self.max_wav_value = args.max_wav_value self.sampling_rate = args.sampling_rate # self.load_mel_from_disk = args.load_mel_from_disk self.stft = TacotronSTFT(args.filter_length, args.hop_length, args.win_length, args.n_mel_channels, args.sampling_rate, args.mel_fmin, args.mel_fmax) random.seed(1234) random.shuffle(self.audiopaths_and_text) def get_mel_text_pair(self, audiopath_and_text): # separate filename and text audiopath, text = audiopath_and_text[0], audiopath_and_text[1] text_len = len(text) text = self.get_text(text) mel = self.get_mel(audiopath) return (text, mel, text_len) def get_mel(self, filename): if False: #not self.load_mel_from_disk: audio, sampling_rate = load_wav_to_torch(filename) if sampling_rate != self.stft.sampling_rate: raise ValueError("{} {} SR doesn't match target {} SR".format( sampling_rate, self.stft.sampling_rate)) audio_norm = audio / self.max_wav_value audio_norm = audio_norm.unsqueeze(0) melspec = self.stft.mel_spectrogram(audio_norm) melspec = torch.squeeze(melspec, 0) else: melspec = torch.from_numpy(np.load(filename).T) assert melspec.size(0) == self.stft.n_mel_channels, ( 'Mel dimension mismatch: given {}, expected {}'.format( melspec.size(0), self.stft.n_mel_channels)) return melspec def get_text(self, text): return text_to_sequence(text, self.text_cleaners) def __getitem__(self, index): return self.get_mel_text_pair(self.audiopaths_and_text[index]) def __len__(self): return len(self.audiopaths_and_text)
def load_mel(path): stft = TacotronSTFT() audio, sampling_rate = load_wav_to_torch(path) if sampling_rate != 16000: raise ValueError("{} SR doesn't match target {} SR".format( sampling_rate, stft.sampling_rate)) audio_norm = audio / 32768.0 # hparams.max_wav_value audio_norm = audio_norm.unsqueeze(0) audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False) melspec = stft.mel_spectrogram(audio_norm) #melspec = melspec.cuda() melspec = torch.squeeze(melspec, 0) return melspec
def get_mel_for_test(file_path): """ Library: from common.layers import TacotronSTFT """ audio, sr = load_wav_to_torch(file_path) audio_norm = audio / MAX_WAV_VALUE audio_norm = audio_norm.unsqueeze(0) audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False) stft = TacotronSTFT(filter_length=1024, hop_length=160, win_length=1024, sampling_rate=sr, mel_fmin=0.0, mel_fmax=8000.0) melspec = stft.mel_spectrogram(audio_norm) return melspec
class Mel2Samp(torch.utils.data.Dataset): """ This is the main class that calculates the spectrogram and returns the spectrogram, audio pair. """ def __init__(self, segment_length, filter_length, hop_length, win_length, sampling_rate, mel_fmin, mel_fmax, h5_melfile): self.stft = TacotronSTFT(filter_length=filter_length, hop_length=hop_length, win_length=win_length, sampling_rate=sampling_rate, mel_fmin=mel_fmin, mel_fmax=mel_fmax) self.segment_framelength = math.ceil(segment_length / hop_length) self.segment_length = self.segment_framelength * hop_length self.hop_length = hop_length self.sampling_rate = sampling_rate self.h5_melfile = h5_melfile self.h5_mel = None def get_mel(self, audio): audio_norm = audio / MAX_WAV_VALUE audio_norm = audio_norm.unsqueeze(0) audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False) melspec = self.stft.mel_spectrogram(audio_norm) melspec = torch.squeeze(melspec, 0) return melspec def __getitem__(self, index): if self.h5_mel is None: self.h5_mel = h5py.File(self.h5_melfile, "r") audio_gp = self.h5_mel[str(index)]["24k"] audio_start = random.randint(0, audio_gp.shape[0] - self.segment_length) audio = torch.FloatTensor(audio_gp[audio_start:audio_start + self.segment_length]) mel = self.get_mel(audio) audio = audio / MAX_WAV_VALUE return (mel, audio) def __len__(self): return len(self.audio_files)
def main(): """ Launches text to speech (inference). Inference is executed on a single GPU. """ parser = argparse.ArgumentParser( description='PyTorch Tacotron 2 Inference') parser = parse_training_args(parser) args, _ = parser.parse_known_args() LOGGER.set_model_name("Tacotron2_PyT") LOGGER.set_backends([ dllg.StdOutBackend(log_file=None, logging_scope=dllg.TRAIN_ITER_SCOPE, iteration_interval=1), dllg.JsonBackend(log_file=args.log_file, logging_scope=dllg.TRAIN_ITER_SCOPE, iteration_interval=1) ]) LOGGER.register_metric("tacotron2_frames_per_sec", metric_scope=dllg.TRAIN_ITER_SCOPE) LOGGER.register_metric("tacotron2_latency", metric_scope=dllg.TRAIN_ITER_SCOPE) LOGGER.register_metric("latency", metric_scope=dllg.TRAIN_ITER_SCOPE) model, args = load_and_setup_model(parser, args) log_hardware() log_args(args) os.makedirs(args.output_dir, exist_ok=True) LOGGER.iteration_start() measurements = {} anchor_dirs = [ os.path.join(args.dataset_path, anchor) for anchor in args.training_anchor_dirs ] metadatas = [load_metadata(anchor) for anchor in anchor_dirs] stft = TacotronSTFT(args.filter_length, args.hop_length, args.win_length, args.n_mel_channels, args.sampling_rate, args.mel_fmin, args.mel_fmax) with torch.no_grad(), MeasureTime(measurements, "tacotron2_time"): for speaker_id in range(len(anchor_dirs)): metadata = metadatas[speaker_id] for npy_path, text in tqdm(metadata): seq = text_to_sequence(text, speaker_id, ['basic_cleaners']) seqs = torch.from_numpy(np.stack(seq)).unsqueeze(0) seq_lens = torch.IntTensor([len(text)]) wav = load_wav_to_torch(npy_path) mel = stft.mel_spectrogram(wav.unsqueeze(0)) mel = mel.squeeze() max_target_len = mel.size(1) - 1 max_target_len += args.n_frames_per_step - max_target_len % args.n_frames_per_step padded_mel = np.pad(mel, [(0, 0), (0, max_target_len - mel.size(1))], mode='constant', constant_values=args.mel_pad_val) target = padded_mel[:, ::args.n_frames_per_step] targets = torch.from_numpy(np.stack(target)).unsqueeze(0) target_lengths = torch.IntTensor([target.shape[1]]) outputs = model.infer( to_gpu(seqs).long(), to_gpu(seq_lens).int(), to_gpu(targets).half(), to_gpu(target_lengths).int()) _, mel_out, _, _ = [ output.cpu() for output in outputs if output is not None ] mel_out = mel_out.squeeze()[:, :mel.size(-1) - 1] assert (mel_out.shape[-1] == wav.shape[-1] // args.hop_length) fname = os.path.basename(npy_path) np.save(os.path.join(args.output_dir, fname), mel_out, allow_pickle=False) # GTA synthesis # magnitudes = stft.inv_mel_spectrogram(mel_out.squeeze()) # wav = griffin_lim(magnitudes, stft.stft_fn, 60) # save_wav(wav, os.path.join(args.output_dir, 'eval.wav')) LOGGER.log(key="tacotron2_latency", value=measurements['tacotron2_time']) LOGGER.log(key="latency", value=(measurements['tacotron2_time'])) LOGGER.iteration_stop() LOGGER.finish()
class TextMelDataset(torch.utils.data.Dataset): """ 1) loads audio,text pairs 2) normalizes text and converts them to sequences of one-hot vectors 3) computes mel-spectrograms from audio files. """ def __init__(self, args, anchor_dirs): self.speaker_num = len(anchor_dirs) self.meta_dirs = [ os.path.join(args.dataset_path, anchor_dirs[i]) for i in range(self.speaker_num) ] self.metadatas = [ load_metadata(meta_dir) for meta_dir in self.meta_dirs ] self.offsets = [0] * self.speaker_num self.text_cleaners = args.text_cleaners self.sampling_rate = args.sampling_rate self.load_mel_from_disk = args.load_mel_from_disk self.stft = TacotronSTFT(args.filter_length, args.hop_length, args.win_length, args.n_mel_channels, args.sampling_rate, args.mel_fmin, args.mel_fmax) random.seed(1234) for i in range(self.speaker_num): random.shuffle(self.metadatas[i]) def get_mel_text_pair(self, speaker_id, metadata): mel_path, text = metadata seq_len = len(text) seq = self.get_sequence(text, speaker_id) mel = self.get_mel(mel_path) return (seq, mel, seq_len) def get_mel(self, filename): if not self.load_mel_from_disk: audio = load_wav_to_torch(filename) melspec = self.stft.mel_spectrogram(audio.unsqueeze(0)) melspec = torch.squeeze(melspec, 0) else: melspec = torch.from_numpy(np.load(filename)) assert melspec.size(0) == self.stft.n_mel_channels, ( 'Mel dimension mismatch: given {}, expected {}'.format( melspec.size(0), self.stft.n_mel_channels)) return melspec def get_sequence(self, text, speaker_id): return text_to_sequence(text, speaker_id, self.text_cleaners) def __getitem__(self, index): group = [ self.get_mel_text_pair(i, self.metadatas[i][self.offsets[i]]) for i in range(self.speaker_num) ] self.offsets = [(self.offsets[i] + 1) % len(self.metadatas[i]) for i in range(self.speaker_num)] return group def __len__(self): return sum([len(m) for m in self.metadatas]) // self.speaker_num