def plot_mels(base_dir: str,
ds_name: str,
wav_name: str,
custom_hparams: Optional[Dict[str, str]] = None):
print("Plotting wav mel spectograms...")
ds_dir = get_ds_dir(base_dir, ds_name)
plots_dir = get_plots_dir(ds_dir, wav_name)
if os.path.isdir(plots_dir):
print("Already exists.")
else:
wav_dir = get_wav_dir(ds_dir, wav_name)
assert os.path.isdir(wav_dir)
data = load_wav_csv(wav_dir)
ds_data = load_ds_csv(ds_dir)
assert len(data) > 0
save_callback = partial(save_plot,
dest_dir=plots_dir,
data_len=len(data))
all_paths = process(data, ds_data, custom_hparams, save_callback)
# all_paths = get_all_paths(plots_dir)
batches = make_batches_h_v(all_paths, VERTICAL_COUNT, HORIZONTAL_COUNT)
plot_batches_h_v(batches, plots_dir)
def preprocess_wavs(base_dir: str, ds_name: str, wav_name: str):
print("Preprocessing wavs...")
ds_dir = get_ds_dir(base_dir, ds_name)
wav_dir = get_wav_dir(ds_dir, wav_name)
if os.path.isdir(wav_dir):
print("Already exists.")
else:
data = load_ds_csv(ds_dir)
wav_data = preprocess(data)
os.makedirs(wav_dir)
save_wav_csv(wav_dir, wav_data)
def _wav_op(base_dir: str, ds_name: str, origin_wav_name: str,
destination_wav_name: str, op: Any):
ds_dir = get_ds_dir(base_dir, ds_name)
dest_wav_dir = get_wav_dir(ds_dir, destination_wav_name)
if os.path.isdir(dest_wav_dir):
print("Already exists.")
else:
orig_wav_dir = get_wav_dir(ds_dir, origin_wav_name)
assert os.path.isdir(orig_wav_dir)
data = load_wav_csv(orig_wav_dir)
os.makedirs(dest_wav_dir)
wav_data = op(data, dest_wav_dir)
save_wav_csv(dest_wav_dir, wav_data)
def preprocess_mels(base_dir: str, ds_name: str, wav_name: str, custom_hparams: Optional[Dict[str, str]] = None):
print("Preprocessing mels...")
ds_dir = get_ds_dir(base_dir, ds_name)
mel_dir = get_mel_dir(ds_dir, wav_name)
if os.path.isdir(mel_dir):
print("Already exists.")
else:
wav_dir = get_wav_dir(ds_dir, wav_name)
assert os.path.isdir(wav_dir)
data = load_wav_csv(wav_dir)
assert len(data) > 0
save_callback = partial(save_mel, dest_dir=mel_dir, data_len=len(data))
mel_data = process(data, custom_hparams, save_callback)
save_mel_csv(mel_dir, mel_data)
def preprocess_text(base_dir: str, ds_name: str, text_name: str):
print("Preprocessing text...")
ds_dir = get_ds_dir(base_dir, ds_name)
text_dir = get_text_dir(ds_dir, text_name)
if os.path.isdir(text_dir):
print("Already exists.")
else:
data = load_ds_csv(ds_dir)
symbol_ids = load_symbols_json(ds_dir)
text_data, conv, all_symbols = text_preprocess_core(data, symbol_ids)
os.makedirs(text_dir)
save_text_csv(text_dir, text_data)
save_text_symbol_converter(text_dir, conv)
save_text_symbols_json(text_dir, all_symbols)
def _text_op(base_dir: str, ds_name: str, orig_text_name: str,
dest_text_name: str, operation):
ds_dir = get_ds_dir(base_dir, ds_name)
orig_text_dir = get_text_dir(ds_dir, orig_text_name)
assert os.path.isdir(orig_text_dir)
dest_text_dir = get_text_dir(ds_dir, dest_text_name)
if os.path.isdir(dest_text_dir):
print("Already exists.")
else:
print("Reading data...")
data = load_text_csv(orig_text_dir)
orig_conv = load_text_symbol_converter(orig_text_dir)
text_data, conv, all_symbols = operation(data, orig_conv)
os.makedirs(dest_text_dir)
save_text_csv(dest_text_dir, text_data)
save_text_symbol_converter(dest_text_dir, conv)
save_text_symbols_json(dest_text_dir, all_symbols)
print("Dataset processed.")
def remove_silence_plot(base_dir: str,
ds_name: str,
wav_name: str,
chunk_size: int,
threshold_start: float,
threshold_end: float,
buffer_start_ms: float,
buffer_end_ms: float,
entry_id: Optional[int] = None):
ds_dir = get_ds_dir(base_dir, ds_name)
wav_dir = get_wav_dir(ds_dir, wav_name)
assert os.path.isdir(wav_dir)
data = load_wav_csv(wav_dir)
if entry_id is None:
entry = data.get_random_entry()
else:
entry = data.get_entry(entry_id)
dest_dir = _get_trim_dir(wav_dir, entry)
os.makedirs(dest_dir, exist_ok=True)
dest_name = f"cs={chunk_size},ts={threshold_start}dBFS,bs={buffer_start_ms}ms,te={threshold_end}dBFS,be={buffer_end_ms}ms"
wav_trimmed = os.path.join(dest_dir, f"{dest_name}.wav")
mel_orig, mel_trimmed = remove_silence_plot_core(
wav_path=entry.wav,
out_path=wav_trimmed,
chunk_size=chunk_size,
threshold_start=threshold_start,
threshold_end=threshold_end,
buffer_start_ms=buffer_start_ms,
buffer_end_ms=buffer_end_ms)
_save_orig_wav_if_not_exists(dest_dir, entry.wav)
orig = _save_orig_plot_if_not_exists(dest_dir, mel_orig)
trimmed = _save_trimmed_plot_temp(mel_trimmed)
resulting_path = _save_comparison(dest_dir, dest_name, [orig, trimmed])
os.remove(trimmed)
print(f"Saved result to: {resulting_path}")
def prepare_ds(base_dir: str, prep_name: str, ds_speakers: List[Tuple[str,
str]],
ds_text_audio: List[Tuple[str, str, str]]):
print(f"Preparing dataset: {prep_name}...")
prep_dir = get_prepared_dir(base_dir, prep_name)
if os.path.isdir(prep_dir):
print("Already created.")
else:
datasets = DsDatasetList()
for ds_name, text_name, audio_name in ds_text_audio:
# multiple uses of one ds are not valid
ds_dir = get_ds_dir(base_dir, ds_name)
text_dir = get_text_dir(ds_dir, text_name)
wav_dir = get_wav_dir(ds_dir, audio_name)
mel_dir = get_mel_dir(ds_dir, audio_name)
ds_dataset = DsDataset(
name=ds_name,
data=load_ds_csv(ds_dir),
texts=load_text_csv(text_dir),
wavs=load_wav_csv(wav_dir),
mels=load_mel_csv(mel_dir),
speakers=load_speaker_json(ds_dir),
symbol_ids=load_text_symbol_converter(text_dir),
accent_ids=load_accents_json(ds_dir))
datasets.append(ds_dataset)
merged_data = preprocess(datasets=datasets, ds_speakers=ds_speakers)
prep_data = PreparedDataList.init_from_merged_ds(merged_data.data)
os.makedirs(prep_dir)
save_filelist(prep_dir, prep_data)
save_prep_symbol_converter(prep_dir, merged_data.symbol_ids)
save_prep_accents_ids(prep_dir, merged_data.accent_ids)
save_prep_speakers_json(prep_dir, merged_data.speaker_ids)