def prepare_vctk(
corpus_dir: Pathlike,
output_dir: Optional[Pathlike] = None,
) -> Dict[str, Union[RecordingSet, SupervisionSet]]:
"""
Prepares and returns the L2 Arctic manifests which consist of Recordings and Supervisions.
:param corpus_dir: Pathlike, the path of the data dir.
:param output_dir: Pathlike, the path where to write the manifests.
:return: a dict with keys "read" and "spontaneous".
Each hold another dict of {'recordings': ..., 'supervisions': ...}
"""
corpus_dir = Path(corpus_dir)
assert corpus_dir.is_dir(), f"No such directory: {corpus_dir}"
speaker_meta = _parse_speaker_description(corpus_dir)
recordings = RecordingSet.from_recordings(
Recording.from_file(wav)
for wav in (corpus_dir / "wav48").rglob("*.wav"))
supervisions = []
for path in (corpus_dir / "txt").rglob("*.txt"):
# One utterance (line) per file
text = path.read_text().strip()
speaker = path.name.split("_")[0] # p226_001.txt -> p226
seg_id = path.stem
meta = speaker_meta.get(speaker, defaultdict(lambda: None))
if meta is None:
logging.warning(f"Cannot find metadata for speaker {speaker}.")
supervisions.append(
SupervisionSegment(
id=seg_id,
recording_id=seg_id,
start=0,
duration=recordings[seg_id].duration,
text=text,
language="English",
speaker=speaker,
gender=meta["gender"],
custom={
"accent": meta["accent"],
"age": meta["age"],
"region": meta["region"],
},
))
supervisions = SupervisionSet.from_segments(supervisions)
# note(pzelasko): There were 172 recordings without supervisions when I ran it.
# I am just removing them.
recordings, supervisions = remove_missing_recordings_and_supervisions(
recordings, supervisions)
validate_recordings_and_supervisions(recordings, supervisions)
if output_dir is not None:
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
recordings.to_json(output_dir / "recordings.json")
supervisions.to_json(output_dir / "supervisions.json")
return {"recordings": recordings, "supervisions": supervisions}
def test_remove_missing_recordings_and_supervisions():
recordings = DummyManifest(RecordingSet, begin_id=0, end_id=100)
supervisions = DummyManifest(SupervisionSet, begin_id=50, end_id=150)
fix_recs, fix_sups = remove_missing_recordings_and_supervisions(
recordings, supervisions)
expected_ids = [f'dummy-recording-{idx:04d}' for idx in range(50, 100)]
assert [r.id for r in fix_recs] == expected_ids
assert [s.recording_id for s in fix_sups] == expected_ids
def prepare_cmu_arctic(
corpus_dir: Pathlike,
output_dir: Optional[Pathlike] = None,
) -> Dict[str, Union[RecordingSet, SupervisionSet]]:
"""
Prepares and returns the CMU Arctic manifests,
which consist of Recordings and Supervisions.
:param corpus_dir: Pathlike, the path of the data dir.
:param output_dir: Pathlike, the path where to write the manifests.
:return: a dict of {'recordings': ..., 'supervisions': ...}
"""
corpus_dir = Path(corpus_dir)
assert corpus_dir.is_dir(), f"No such directory: {corpus_dir}"
recordings = RecordingSet.from_recordings(
# Example ID: cmu_us_sup_arctic-arctic_a0001
Recording.from_file(
wav, recording_id=f"{_get_speaker(wav.parent.parent.name)}-{wav.stem}"
)
for wav in corpus_dir.rglob("*.wav")
)
supervisions = []
for path in corpus_dir.rglob("txt.done.data"):
lines = path.read_text().splitlines()
speaker = _get_speaker(path.parent.parent.name)
for l in lines:
l = l[2:-2] # get rid of parentheses and whitespaces on the edges
seg_id, text = l.split(maxsplit=1)
seg_id = f"{speaker}-{seg_id}"
supervisions.append(
SupervisionSegment(
id=seg_id,
recording_id=seg_id,
start=0,
duration=recordings[seg_id].duration,
text=text.replace('"', ""), # get rid of quotation marks,
language="English",
speaker=speaker,
gender=GENDER_MAP.get(speaker),
custom={"accent": ACCENT_MAP.get(speaker)},
)
)
supervisions = SupervisionSet.from_segments(supervisions)
# There seem to be 20 recordings missing; remove the before validation
recordings, supervisions = remove_missing_recordings_and_supervisions(
recordings, supervisions
)
validate_recordings_and_supervisions(recordings, supervisions)
if output_dir is not None:
output_dir = Path(output_dir)
recordings.to_json(output_dir / "cmu_arctic_recordings.json")
supervisions.to_json(output_dir / "cmu_arctic_supervisions.json")
return {"recordings": recordings, "supervisions": supervisions}
def prepare_callhome_english(
audio_dir: Pathlike,
rttm_dir: Optional[Pathlike] = None,
output_dir: Optional[Pathlike] = None,
sph2pipe_path: Optional[Pathlike] = None,
) -> Dict[str, Union[RecordingSet, SupervisionSet]]:
"""
Prepare manifests for the Switchboard corpus.
We create two manifests: one with recordings, and the other one with text supervisions.
When ``sentiment_dir`` is provided, we create another supervision manifest with sentiment annotations.
:param audio_dir: Path to ``LDC2001S97`` package.
:param rttm_dir: Path to the transcripts directory (typically named "swb_ms98_transcriptions").
If not provided, the transcripts will be downloaded.
:param output_dir: Directory where the manifests should be written. Can be omitted to avoid writing.
:param sph2pipe_path: When provided, we will "hard-wire" the sph2pipe path into the recording manifests.
:return: A dict with manifests. The keys are: ``{'recordings', 'supervisions'}``.
"""
if rttm_dir is None:
rttm_dir = download_callhome_metadata()
rttm_path = rttm_dir / 'fullref.rttm'
supervisions = read_rttm(rttm_path)
audio_paths = check_and_rglob(audio_dir, '*.sph')
recordings = RecordingSet.from_recordings(
make_recording_callhome(p, sph2pipe_path=sph2pipe_path) for p in tqdm(audio_paths)
)
recordings, supervisions = remove_missing_recordings_and_supervisions(recordings, supervisions)
supervisions = trim_supervisions_to_recordings(recordings, supervisions)
validate_recordings_and_supervisions(recordings, supervisions)
if output_dir is not None:
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
recordings.to_json(output_dir / 'recordings.json')
supervisions.to_json(output_dir / 'supervisions.json')
return {
'recordings': recordings,
'supervisions': supervisions
}
def prepare_cmu_indic(
corpus_dir: Pathlike,
output_dir: Optional[Pathlike] = None,
) -> Dict[str, Union[RecordingSet, SupervisionSet]]:
"""
Prepares and returns the CMU Indic manifests,
which consist of Recordings and Supervisions.
:param corpus_dir: Pathlike, the path of the data dir.
:param output_dir: Pathlike, the path where to write the manifests.
:return: a dict of {'recordings': ..., 'supervisions': ...}
"""
corpus_dir = Path(corpus_dir)
assert corpus_dir.is_dir(), f"No such directory: {corpus_dir}"
recordings = RecordingSet.from_recordings(
# Example ID: cmu_indic_ben_rm_bn_00001
Recording.from_file(
wav, recording_id=f"{_get_speaker(wav.parent.parent.name)}-{wav.stem}"
)
for wav in corpus_dir.rglob("*.wav")
)
supervisions = []
for path in corpus_dir.rglob("txt.done.data"):
lines = path.read_text().splitlines()
speaker = _get_speaker(path.parent.parent.name)
lang_code = speaker.split("_")[0] # example: 'ben_rm' -> 'ben' (Bengali)
try:
# Example contents of voice.feats file:
# variant guj
# age 28
# gender male
# description Built with build_cg_rfs_voice, 3 rf and 3 dur
# gujarati_data h2r_prompts
# prompt_dur 59.27min
age = int(
(path.parent / "voice.feats")
.read_text()
.splitlines()[1]
.replace("age ", "")
.strip()
)
except:
age = None
for l in lines:
l = l[2:-2] # get rid of parentheses and whitespaces on the edges
seg_id, text = l.split(maxsplit=1)
seg_id = f"{speaker}-{seg_id}"
language = LANGUAGE_MAP[lang_code]
is_english = "arctic" in seg_id
# Determine available custom meta-data to attach.
custom = None
if is_english or age is not None:
custom = {}
if is_english:
custom["accent"] = language
if age is not None:
custom["age"] = age
supervisions.append(
SupervisionSegment(
id=seg_id,
recording_id=seg_id,
start=0,
duration=recordings[seg_id].duration,
text=text.replace('"', ""), # get rid of quotation marks,
language="English" if is_english else language,
speaker=speaker,
gender=GENDER_MAP.get(speaker),
custom=custom,
)
)
supervisions = SupervisionSet.from_segments(supervisions)
# There seem to be 20 recordings missing; remove the before validation
recordings, supervisions = remove_missing_recordings_and_supervisions(
recordings, supervisions
)
validate_recordings_and_supervisions(recordings, supervisions)
if output_dir is not None:
output_dir = Path(output_dir)
recordings.to_file(output_dir / "cmu-indic_recordings_all.jsonl.gz")
supervisions.to_file(output_dir / "cmu-indic_supervisions_all.jsonl.gz")
return {"recordings": recordings, "supervisions": supervisions}
def prepare_single_babel_language(
corpus_dir: Pathlike,
output_dir: Optional[Pathlike] = None,
no_eval_ok: bool = False,
) -> Dict[str, Dict[str, Union[RecordingSet, SupervisionSet]]]:
"""
Prepares manifests using a single BABEL LDC package.
This function works like the following:
- first, it will scan `corpus_dir` for a directory named `conversational`;
if there is more than once, it picks the first one (and emits a warning)
- then, it will try to find `dev`, `eval`, and `training` splits inside
(if any of them is not present, it will skip it with a warning)
- finally, it scans the selected location for SPHERE audio files and transcripts.
:param corpus_dir: Path to the root of the LDC package with a BABEL language.
:param output_dir: Path where the manifests are stored.json
:param no_eval_ok: When set to True, this function won't emit a warning
that the eval set was not found.
:return:
"""
manifests = defaultdict(dict)
# Auto-detect the location of the "conversational" directory
orig_corpus_dir = corpus_dir
corpus_dir = Path(corpus_dir)
corpus_dir = [d for d in corpus_dir.rglob("conversational") if d.is_dir()]
if not corpus_dir:
raise ValueError(
f"Could not find 'conversational' directory anywhere inside '{orig_corpus_dir}' "
f"- please check your path.")
if len(corpus_dir) > 1:
# People have very messy data distributions, the best we can do is warn them.
logging.warning(
f"It seems there are multiple 'conversational' directories in '{orig_corpus_dir}' - "
f"we are selecting the first one only ({corpus_dir[0]}). Please ensure that you provided "
f"the path to a single language's dir, and the root dir for all BABEL languages."
)
corpus_dir = corpus_dir[0].parent
for split in ("dev", "eval", "training"):
audio_dir = corpus_dir / f"conversational/{split}/audio"
sph_recordings = RecordingSet.from_recordings(
Recording.from_file(p) for p in audio_dir.glob("*.sph"))
wav_recordings = RecordingSet.from_recordings(
Recording.from_file(p) for p in audio_dir.glob("*.wav"))
recordings = combine(sph_recordings, wav_recordings)
if len(recordings) == 0:
if split == "eval" and no_eval_ok:
continue
logging.warning(f"No SPHERE or WAV files found in {audio_dir}")
supervisions = []
text_dir = corpus_dir / f"conversational/{split}/transcription"
for p in tqdm.tqdm(text_dir.glob("*")):
# p.stem -> BABEL_BP_101_10033_20111024_205740_inLine
# parts:
# 0 -> BABEL
# 1 -> BP
# 2 -> <language-code> (101)
# 3 -> <speaker-id> (10033)
# 4 -> <date> (20111024)
# 5 -> <hour> (205740)
# 6 -> channel (inLine) ; inLine <=> A ; outLine <=> B ; "scripted" <=> A
p0, p1, lang_code, speaker, date, hour, channel, *_ = p.stem.split(
"_")
channel = {"inLine": "A", "outLine": "B"}.get(channel, "A")
# Fix problematic segments that have two consecutive timestamp lines with no transcript in between
lines = p.read_text().splitlines() + [""]
lines = [
prev_l for prev_l, l in sliding_window(2, lines)
if not (prev_l.startswith("[") and l.startswith("["))
]
# Add a None at the end so that the last timestamp is only used as "next_timestamp"
# and ends the iretation (otherwise we'd lose the last segment).
lines += [None]
for (timestamp,
text), (next_timestamp,
_) in sliding_window(2, zip(lines[::2], lines[1::2])):
try:
start = float(timestamp[1:-1])
end = float(next_timestamp[1:-1])
# Create supervision
supervisions.append(
SupervisionSegment(
id=
f"{lang_code}_{speaker}_{channel}_{date}_{hour}_{int(100 * start):06}",
recording_id=p.stem,
start=start,
duration=round(end - start, ndigits=8),
channel=0,
text=normalize_text(text),
language=BABELCODE2LANG[lang_code],
speaker=f"{lang_code}_{speaker}_{channel}",
))
except Exception as e:
logging.warning(
f"Error while parsing segment. Message: {str(e)}")
raise ValueError(
f"Too many errors while parsing segments (file: '{p}'). "
f"Please check your data or increase the threshold.")
supervisions = deduplicate_supervisions(supervisions)
if len(supervisions) == 0:
logging.warning(f"No supervisions found in {text_dir}")
supervisions = SupervisionSet.from_segments(supervisions)
# Fixing and validation of manifests
if split == "eval" and len(supervisions) == 0:
# We won't remove missing recordings for the "eval" split in cases where
# the user does not have its corresponding transcripts (very likely).
pass
else:
recordings, supervisions = remove_missing_recordings_and_supervisions(
recordings, supervisions)
supervisions = trim_supervisions_to_recordings(
recordings, supervisions)
validate_recordings_and_supervisions(recordings, supervisions)
manifests[split] = {
"recordings": recordings,
"supervisions": supervisions
}
if output_dir is not None:
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
language = BABELCODE2LANG[lang_code]
save_split = "train" if split == "training" else split
recordings.to_file(output_dir /
f"recordings_{language}_{save_split}.json")
supervisions.to_file(output_dir /
f"supervisions_{language}_{save_split}.json")
return dict(manifests)
def prepare_single_mtedx_language(
corpus_dir: Pathlike,
output_dir: Optional[Pathlike] = None,
language: str = "language",
num_jobs: int = 1,
) -> Dict[str, Dict[str, Union[RecordingSet, SupervisionSet]]]:
"""
Prepares manifests using a single MTEDx language.
This function works as follows:
- First it looks for the audio directory in the data/wav where the .flac
files are stored.
- Then, it looks for the vtt directory in data/{train,dev,test}/vtt
which contains the segmentation and transcripts for the audio.
- The transcripts undergo some basic text normalization
:param corpus_dir: Path to the root of the MTEDx download
:param output_dir: Path where the manifests are stored as .json files
:param language: The two-letter language code.
:param num_jobs: Number of threads to use when preparing data.
:return:
"""
if isinstance(corpus_dir, str):
corpus_dir = Path(corpus_dir)
manifests = defaultdict(dict)
with ThreadPoolExecutor(num_jobs) as ex:
for split in ("train", "valid", "test"):
audio_dir = corpus_dir / f"data/{split}/wav"
recordings = RecordingSet.from_recordings(
Recording.from_file(p) for p in audio_dir.glob("*.flac")
)
if len(recordings) == 0:
logging.warning(f"No .flac files found in {audio_dir}")
supervisions = []
text_dir = corpus_dir / f"data/{split}/vtt"
futures = []
for p in text_dir.glob("*"):
futures.append(ex.submit(_filename_to_supervisions, p, language))
for future in tqdm(futures, desc="Processing", leave=False):
result = future.result()
if result is None:
continue
for sup in result:
supervisions.append(sup)
if len(supervisions) == 0:
logging.warning(f"No supervisions found in {text_dir}")
supervisions = SupervisionSet.from_segments(supervisions)
recordings, supervisions = remove_missing_recordings_and_supervisions(
recordings, supervisions
)
supervisions = trim_supervisions_to_recordings(recordings, supervisions)
validate_recordings_and_supervisions(recordings, supervisions)
manifests[split] = {
"recordings": recordings,
"supervisions": supervisions,
}
if output_dir is not None:
if isinstance(output_dir, str):
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
save_split = "dev" if split == "valid" else split
recordings.to_file(output_dir / f"recordings_{language}_{split}.json")
supervisions.to_file(
output_dir / f"supervisions_{language}_{split}.json"
)
return dict(manifests)
def prepare_callhome_egyptian(
audio_dir: Pathlike,
transcript_dir: Pathlike,
output_dir: Optional[Pathlike] = None,
sph2pipe_path: Optional[Pathlike] = None,
) -> Dict[str, Union[RecordingSet, SupervisionSet]]:
"""
Prepare manifests for the Switchboard corpus.
We create two manifests: one with recordings, and the other one with text supervisions.
When ``sentiment_dir`` is provided, we create another supervision manifest with sentiment annotations.
:param audio_dir: Path to ``LDC2001S97`` package.
:param rttm_dir: Path to the transcripts directory (typically named "swb_ms98_transcriptions").
If not provided, the transcripts will be downloaded.
:param output_dir: Directory where the manifests should be written. Can be omitted to avoid writing.
:param sph2pipe_path: When provided, we will "hard-wire" the sph2pipe path into the recording manifests.
:return: A dict with manifests. The keys are: ``{'recordings', 'supervisions'}``.
"""
audio_dir = Path(audio_dir)
transcript_dir = Path(transcript_dir)
manifests = {}
for split in ['train', 'devtest', 'evaltest']:
audio_paths = check_and_rglob(
# The LDC distribution has a typo.
audio_dir / 'callhome/arabic' /
split.replace('evaltest', 'evltest'),
'*.sph')
recordings = RecordingSet.from_recordings(
make_recording_callhome(p, sph2pipe_path=sph2pipe_path)
for p in tqdm(audio_paths))
transcript_paths = check_and_rglob(
transcript_dir /
f'callhome_arabic_trans_970711/transcrp/{split}/roman', '*.txt')
# TODO: Add text normalization like in Kaldi recipe.
# Not doing this right now as it's not needed for VAD/diarization...
supervisions = []
for p in transcript_paths:
idx = 0
for line in p.read_text().splitlines():
line = line.strip()
if not line:
continue
recording_id = p.stem
# example line:
# 19.33 21.18 B: %ah Tayyib
start, end, spk, text = line.split(maxsplit=3)
spk = spk.replace(":", "")
duration = float(Decimal(end) - Decimal(start))
if duration <= 0:
continue
start = float(start)
supervisions.append(
SupervisionSegment(id=f'{recording_id}_{idx}',
recording_id=recording_id,
start=start,
duration=duration,
speaker=f'{recording_id}_{spk}',
text=text))
idx += 1
supervisions = SupervisionSet.from_segments(supervisions)
recordings, supervisions = remove_missing_recordings_and_supervisions(
recordings, supervisions)
supervisions = trim_supervisions_to_recordings(recordings,
supervisions)
validate_recordings_and_supervisions(recordings, supervisions)
if output_dir is not None:
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
recordings.to_json(output_dir / f'recordings_{split}.json')
supervisions.to_json(output_dir / f'supervisions_{split}.json')
manifests[split] = {
'recordings': recordings,
'supervisions': supervisions
}
return manifests