def one_sample(sample):
mp3_filename = sample[0]
# Storing wav files next to the mp3 ones - just with a different suffix
wav_filename = path.splitext(mp3_filename)[0] + ".wav"
_maybe_convert_wav(mp3_filename, wav_filename)
frames = int(
subprocess.check_output(["soxi", "-s", wav_filename],
stderr=subprocess.STDOUT))
file_size = -1
if os.path.exists(wav_filename):
file_size = path.getsize(wav_filename)
frames = int(
subprocess.check_output(["soxi", "-s", wav_filename],
stderr=subprocess.STDOUT))
label = validate_label(sample[1])
rows = []
counter = get_counter()
if file_size == -1:
# Excluding samples that failed upon conversion
counter["failed"] += 1
elif label is None:
# Excluding samples that failed on label validation
counter["invalid_label"] += 1
elif int(frames / SAMPLE_RATE * 1000 / 10 / 2) < len(str(label)):
# Excluding samples that are too short to fit the transcript
counter["too_short"] += 1
elif frames / SAMPLE_RATE > MAX_SECS:
# Excluding very long samples to keep a reasonable batch-size
counter["too_long"] += 1
else:
# This one is good - keep it for the target CSV
rows.append((wav_filename, file_size, label))
counter["all"] += 1
counter["total_time"] += frames
return (counter, rows)
def one_sample(sample):
""" Take a audio file, and optionally convert it to 16kHz WAV """
wav_filename = sample[0]
file_size = -1
frames = 0
if os.path.exists(wav_filename):
file_size = os.path.getsize(wav_filename)
frames = int(
subprocess.check_output(["soxi", "-s", wav_filename],
stderr=subprocess.STDOUT))
label = label_filter(sample[1])
counter = get_counter()
rows = []
if file_size == -1:
# Excluding samples that failed upon conversion
counter["failed"] += 1
elif label is None:
# Excluding samples that failed on label validation
counter["invalid_label"] += 1
elif int(frames / SAMPLE_RATE * 1000 / 15 / 2) < len(str(label)):
# Excluding samples that are too short to fit the transcript
counter["too_short"] += 1
elif frames / SAMPLE_RATE > MAX_SECS:
# Excluding very long samples to keep a reasonable batch-size
counter["too_long"] += 1
else:
# This one is good - keep it for the target CSV
rows.append((wav_filename, file_size, label))
counter["all"] += 1
counter["total_time"] += frames
return (counter, rows)
def one_sample(sample):
file_size = -1
frames = 0
audio_source = sample[0]
target_dir = sample[1]
dataset_basename = sample[2]
start_time = sample[3]
duration = sample[4]
label = label_filter_fun(sample[5])
sample_id = sample[6]
_wav_filename = os.path.basename(
audio_source.replace(".wav", "_{:06}.wav".format(sample_id)))
wav_fullname = os.path.join(target_dir, dataset_basename, _wav_filename)
if not os.path.exists(wav_fullname):
subprocess.check_output([
"ffmpeg", "-i", audio_source, "-ss",
str(start_time), "-t",
str(duration), "-c", "copy", wav_fullname
],
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
file_size = os.path.getsize(wav_fullname)
frames = int(
subprocess.check_output(["soxi", "-s", wav_fullname],
stderr=subprocess.STDOUT))
_counter = get_counter()
_rows = []
if file_size == -1:
# Excluding samples that failed upon conversion
_counter["failed"] += 1
elif label is None:
# Excluding samples that failed on label validation
_counter["invalid_label"] += 1
elif int(frames / SAMPLE_RATE * 1000 / 10 / 2) < len(str(label)):
# Excluding samples that are too short to fit the transcript
_counter["too_short"] += 1
elif frames / SAMPLE_RATE < MIN_SECS:
# Excluding samples that are too short
_counter["too_short"] += 1
elif frames / SAMPLE_RATE > MAX_SECS:
# Excluding very long samples to keep a reasonable batch-size
_counter["too_long"] += 1
else:
# This one is good - keep it for the target CSV
_rows.append((os.path.join(dataset_basename,
_wav_filename), file_size, label))
_counter["imported_time"] += frames
_counter["all"] += 1
_counter["total_time"] += frames
return (_counter, _rows)
def _maybe_convert_set(input_tsv, audio_dir, space_after_every_character=None):
output_csv = os.path.join(
audio_dir,
os.path.split(input_tsv)[-1].replace("tsv", "csv"))
print("Saving new DeepSpeech-formatted CSV file to: ", output_csv)
# Get audiofile path and transcript for each sentence in tsv
samples = []
with open(input_tsv, encoding="utf-8") as input_tsv_file:
reader = csv.DictReader(input_tsv_file, delimiter="\t")
for row in reader:
samples.append((os.path.join(audio_dir,
row["path"]), row["sentence"]))
counter = get_counter()
num_samples = len(samples)
rows = []
print("Importing mp3 files...")
pool = Pool()
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
for i, processed in enumerate(pool.imap_unordered(one_sample, samples),
start=1):
counter += processed[0]
rows += processed[1]
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
with open(output_csv, "w", encoding="utf-8") as output_csv_file:
print("Writing CSV file for DeepSpeech.py as: ", output_csv)
writer = csv.DictWriter(output_csv_file, fieldnames=FIELDNAMES)
writer.writeheader()
bar = progressbar.ProgressBar(max_value=len(rows), widgets=SIMPLE_BAR)
for filename, file_size, transcript in bar(rows):
if space_after_every_character:
writer.writerow({
"wav_filename": filename,
"wav_filesize": file_size,
"transcript": " ".join(transcript),
})
else:
writer.writerow({
"wav_filename": filename,
"wav_filesize": file_size,
"transcript": transcript,
})
imported_samples = get_imported_samples(counter)
assert counter["all"] == num_samples
assert len(rows) == imported_samples
print_import_report(counter, SAMPLE_RATE, MAX_SECS)
def _maybe_convert_set(extracted_dir, source_csv, target_csv):
print()
if os.path.exists(target_csv):
print('Found CSV file "%s" - not importing "%s".' %
(target_csv, source_csv))
return
print('No CSV file "%s" - importing "%s"...' % (target_csv, source_csv))
samples = []
with open(source_csv) as source_csv_file:
reader = csv.DictReader(source_csv_file)
for row in reader:
samples.append((os.path.join(extracted_dir,
row["filename"]), row["text"]))
# Mutable counters for the concurrent embedded routine
counter = get_counter()
num_samples = len(samples)
rows = []
print("Importing mp3 files...")
pool = Pool()
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
for i, processed in enumerate(pool.imap_unordered(one_sample, samples),
start=1):
counter += processed[0]
rows += processed[1]
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
print('Writing "%s"...' % target_csv)
with open(target_csv, "w", encoding="utf-8",
newline="") as target_csv_file:
writer = csv.DictWriter(target_csv_file, fieldnames=FIELDNAMES)
writer.writeheader()
bar = progressbar.ProgressBar(max_value=len(rows), widgets=SIMPLE_BAR)
for filename, file_size, transcript in bar(rows):
writer.writerow({
"wav_filename": filename,
"wav_filesize": file_size,
"transcript": transcript,
})
imported_samples = get_imported_samples(counter)
assert counter["all"] == num_samples
assert len(rows) == imported_samples
print_import_report(counter, SAMPLE_RATE, MAX_SECS)
def one_sample(sample):
""" Take an audio file, and optionally convert it to 16kHz WAV """
global AUDIO_DIR
source_filename = sample[0]
if not os.path.splitext(source_filename.lower())[1] == ".wav":
source_filename += ".wav"
# Storing wav files next to the mp3 ones - just with a different suffix
output_filename = f"{sample[2]}.wav"
output_filepath = os.path.join(AUDIO_DIR, output_filename)
_maybe_convert_wav(source_filename, output_filepath)
file_size = -1
frames = 0
if os.path.exists(output_filepath):
file_size = os.path.getsize(output_filepath)
if file_size == 0:
frames = 0
else:
frames = int(
subprocess.check_output(["soxi", "-s", output_filepath],
stderr=subprocess.STDOUT))
label = FILTER_OBJ.filter(sample[1])
rows = []
counter = get_counter()
if file_size == -1:
# Excluding samples that failed upon conversion
counter["failed"] += 1
elif label is None:
# Excluding samples that failed on label validation
counter["invalid_label"] += 1
# + 1 added for filtering surname dataset with too short audio files
elif int(frames / SAMPLE_RATE * 1000 / 10 / 2) < len(str(label)) + 1:
# Excluding samples that are too short to fit the transcript
counter["too_short"] += 1
elif frames / SAMPLE_RATE > MAX_SECS:
# Excluding very long samples to keep a reasonable batch-size
counter["too_long"] += 1
else:
# This one is good - keep it for the target CSV
rows.append(
(os.path.split(output_filename)[-1], file_size, label, sample[2]))
counter["imported_time"] += frames
counter["all"] += 1
counter["total_time"] += frames
return (counter, rows)
def one_sample(args):
""" Take an audio file, and optionally convert it to 16kHz WAV """
sample, filter_obj = args
mp3_filename = sample[0]
if not os.path.splitext(mp3_filename.lower())[1] == ".mp3":
mp3_filename += ".mp3"
# Storing wav files next to the mp3 ones - just with a different suffix
wav_filename = os.path.splitext(mp3_filename)[0] + ".wav"
_maybe_convert_wav(mp3_filename, wav_filename)
file_size = -1
frames = 0
if os.path.exists(wav_filename):
file_size = os.path.getsize(wav_filename)
frames = int(
subprocess.check_output(["soxi", "-s", wav_filename],
stderr=subprocess.STDOUT))
label = filter_obj.filter(sample[1])
rows = []
counter = get_counter()
if file_size == -1:
# Excluding samples that failed upon conversion
counter["failed"] += 1
elif label is None:
# Excluding samples that failed on label validation
counter["invalid_label"] += 1
elif int(frames / SAMPLE_RATE * 1000 / 10 / 2) < len(str(label)):
# Excluding samples that are too short to fit the transcript
counter["too_short"] += 1
elif frames / SAMPLE_RATE > MAX_SECS:
# Excluding very long samples to keep a reasonable batch-size
counter["too_long"] += 1
else:
# This one is good - keep it for the target CSV
rows.append(
(os.path.split(wav_filename)[-1], file_size, label, sample[2]))
counter["imported_time"] += frames
counter["all"] += 1
counter["total_time"] += frames
return (counter, rows)
def one_sample(sample):
""" Take a audio file, and optionally convert it to 16kHz WAV """
orig_filename = sample["path"]
# Storing wav files next to the wav ones - just with a different suffix
wav_filename = os.path.splitext(orig_filename)[0] + ".converted.wav"
_maybe_convert_wav(orig_filename, wav_filename)
file_size = -1
frames = 0
if os.path.exists(wav_filename):
file_size = os.path.getsize(wav_filename)
frames = int(
subprocess.check_output(
["soxi", "-s", wav_filename], stderr=subprocess.STDOUT
)
)
label = sample["text"]
rows = []
# Keep track of how many samples are good vs. problematic
counter = get_counter()
if file_size == -1:
# Excluding samples that failed upon conversion
counter["failed"] += 1
elif label is None:
# Excluding samples that failed on label validation
counter["invalid_label"] += 1
elif int(frames / SAMPLE_RATE * 1000 / 10 / 2) < len(str(label)):
# Excluding samples that are too short to fit the transcript
counter["too_short"] += 1
elif frames / SAMPLE_RATE > MAX_SECS:
# Excluding very long samples to keep a reasonable batch-size
counter["too_long"] += 1
else:
# This one is good - keep it for the target CSV
rows.append((wav_filename, file_size, label))
counter["all"] += 1
counter["total_time"] += frames
return (counter, rows)
def _maybe_convert_sets(target_dir, extracted_data):
extracted_dir = os.path.join(target_dir, extracted_data)
# override existing CSV with normalized one
target_csv_template = os.path.join(
target_dir, ARCHIVE_DIR_NAME,
ARCHIVE_NAME.replace(".tar.gz", "_{}.csv"))
if os.path.isfile(target_csv_template):
return
wav_root_dir = os.path.join(extracted_dir)
all_files = [
"transcripts/train/yaounde/fn_text.txt",
"transcripts/train/ca16_conv/transcripts.txt",
"transcripts/train/ca16_read/conditioned.txt",
"transcripts/dev/niger_west_african_fr/transcripts.txt",
"speech/dev/niger_west_african_fr/niger_wav_file_name_transcript.tsv",
"transcripts/devtest/ca16_read/conditioned.txt",
"transcripts/test/ca16/prompts.txt",
]
transcripts = {}
for tr in all_files:
with open(os.path.join(target_dir, ARCHIVE_DIR_NAME, tr),
"r") as tr_source:
for line in tr_source.readlines():
line = line.strip()
if ".tsv" in tr:
sep = " "
else:
sep = " "
audio = os.path.basename(line.split(sep)[0])
if not (".wav" in audio):
if ".tdf" in audio:
audio = audio.replace(".tdf", ".wav")
else:
audio += ".wav"
transcript = " ".join(line.split(sep)[1:])
transcripts[audio] = transcript
# Get audiofile path and transcript for each sentence in tsv
samples = []
glob_dir = os.path.join(wav_root_dir, "**/*.wav")
for record in glob(glob_dir, recursive=True):
record_file = os.path.basename(record)
if record_file in transcripts:
samples.append((record, transcripts[record_file]))
# Keep track of how many samples are good vs. problematic
counter = get_counter()
num_samples = len(samples)
rows = []
print("Importing WAV files...")
pool = Pool()
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
for i, processed in enumerate(pool.imap_unordered(one_sample, samples),
start=1):
counter += processed[0]
rows += processed[1]
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
with open(target_csv_template.format("train"),
"w") as train_csv_file: # 80%
with open(target_csv_template.format("dev"),
"w") as dev_csv_file: # 10%
with open(target_csv_template.format("test"),
"w") as test_csv_file: # 10%
train_writer = csv.DictWriter(train_csv_file,
fieldnames=FIELDNAMES)
train_writer.writeheader()
dev_writer = csv.DictWriter(dev_csv_file,
fieldnames=FIELDNAMES)
dev_writer.writeheader()
test_writer = csv.DictWriter(test_csv_file,
fieldnames=FIELDNAMES)
test_writer.writeheader()
for i, item in enumerate(rows):
transcript = validate_label(item[2])
if not transcript:
continue
wav_filename = item[0]
i_mod = i % 10
if i_mod == 0:
writer = test_writer
elif i_mod == 1:
writer = dev_writer
else:
writer = train_writer
writer.writerow(
dict(
wav_filename=wav_filename,
wav_filesize=os.path.getsize(wav_filename),
transcript=transcript,
))
imported_samples = get_imported_samples(counter)
assert counter["all"] == num_samples
assert len(rows) == imported_samples
print_import_report(counter, SAMPLE_RATE, MAX_SECS)
def _maybe_convert_set(dataset,
tsv_dir,
audio_dir,
filter_obj,
space_after_every_character=None,
rows=None,
exclude=None):
exclude_transcripts = set()
exclude_speakers = set()
if exclude is not None:
for sample in exclude:
exclude_transcripts.add(sample[2])
exclude_speakers.add(sample[3])
if rows is None:
rows = []
input_tsv = os.path.join(os.path.abspath(tsv_dir), dataset + ".tsv")
if not os.path.isfile(input_tsv):
return rows
print("Loading TSV file: ", input_tsv)
# Get audiofile path and transcript for each sentence in tsv
samples = []
with open(input_tsv, encoding="utf-8") as input_tsv_file:
reader = csv.DictReader(input_tsv_file, delimiter="\t")
for row in reader:
samples.append((os.path.join(audio_dir, row["path"]),
row["sentence"], row["client_id"]))
counter = get_counter()
num_samples = len(samples)
print("Importing mp3 files...")
pool = Pool()
bar = progressbar.ProgressBar(max_value=num_samples,
widgets=SIMPLE_BAR)
samples_with_context = itertools.zip_longest(samples, [],
fillvalue=filter_obj)
for i, processed in enumerate(pool.imap_unordered(
one_sample, samples_with_context),
start=1):
counter += processed[0]
rows += processed[1]
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
imported_samples = get_imported_samples(counter)
assert counter["all"] == num_samples
assert len(rows) == imported_samples
print_import_report(counter, SAMPLE_RATE, MAX_SECS)
output_csv = os.path.join(os.path.abspath(audio_dir), dataset + ".csv")
print("Saving new DeepSpeech-formatted CSV file to: ", output_csv)
with open(output_csv, "w", encoding="utf-8") as output_csv_file:
print("Writing CSV file for DeepSpeech.py as: ", output_csv)
writer = csv.DictWriter(output_csv_file, fieldnames=FIELDNAMES)
writer.writeheader()
bar = progressbar.ProgressBar(max_value=len(rows), widgets=SIMPLE_BAR)
for filename, file_size, transcript, speaker in bar(rows):
if transcript in exclude_transcripts or speaker in exclude_speakers:
continue
if space_after_every_character:
writer.writerow({
"wav_filename": filename,
"wav_filesize": file_size,
"transcript": " ".join(transcript),
})
else:
writer.writerow({
"wav_filename": filename,
"wav_filesize": file_size,
"transcript": transcript,
})
return rows
def _maybe_convert_sets(target_dir, extracted_data):
extracted_dir = os.path.join(target_dir, extracted_data)
# override existing CSV with normalized one
target_csv_template = os.path.join(
target_dir,
ARCHIVE_DIR_NAME + "_" + ARCHIVE_NAME.replace(".zip", "_{}.csv"))
if os.path.isfile(target_csv_template):
return
ogg_root_dir = os.path.join(extracted_dir,
ARCHIVE_NAME.replace(".zip", ""))
# Get audiofile path and transcript for each sentence in tsv
samples = []
glob_dir = os.path.join(ogg_root_dir, "**/*.ogg")
for record in glob(glob_dir, recursive=True):
record_file = record.replace(ogg_root_dir + os.path.sep, "")
if record_filter(record_file):
samples.append((
os.path.join(ogg_root_dir, record_file),
os.path.splitext(os.path.basename(record_file))[0],
))
counter = get_counter()
num_samples = len(samples)
rows = []
print("Importing ogg files...")
pool = Pool()
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
for i, processed in enumerate(pool.imap_unordered(one_sample, samples),
start=1):
counter += processed[0]
rows += processed[1]
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
with open(target_csv_template.format("train"),
"w") as train_csv_file: # 80%
with open(target_csv_template.format("dev"),
"w") as dev_csv_file: # 10%
with open(target_csv_template.format("test"),
"w") as test_csv_file: # 10%
train_writer = csv.DictWriter(train_csv_file,
fieldnames=FIELDNAMES)
train_writer.writeheader()
dev_writer = csv.DictWriter(dev_csv_file,
fieldnames=FIELDNAMES)
dev_writer.writeheader()
test_writer = csv.DictWriter(test_csv_file,
fieldnames=FIELDNAMES)
test_writer.writeheader()
for i, item in enumerate(rows):
transcript = validate_label(item[2])
if not transcript:
continue
wav_filename = os.path.join(
ogg_root_dir, item[0].replace(".ogg", ".wav"))
i_mod = i % 10
if i_mod == 0:
writer = test_writer
elif i_mod == 1:
writer = dev_writer
else:
writer = train_writer
writer.writerow(
dict(
wav_filename=wav_filename,
wav_filesize=os.path.getsize(wav_filename),
transcript=transcript,
))
imported_samples = get_imported_samples(counter)
assert counter["all"] == num_samples
assert len(rows) == imported_samples
print_import_report(counter, SAMPLE_RATE, MAX_SECS)
def _maybe_import_data(xml_file, audio_source, target_dir, rel_tol=1e-1):
dataset_basename = os.path.splitext(os.path.split(xml_file)[1])[0]
wav_root = os.path.join(target_dir, dataset_basename)
if not os.path.exists(wav_root):
os.makedirs(wav_root)
source_frames = int(
subprocess.check_output(["soxi", "-s", audio_source],
stderr=subprocess.STDOUT))
print("Source audio length: %s" %
secs_to_hours(source_frames / SAMPLE_RATE))
# Get audiofile path and transcript for each sentence in tsv
samples = []
tree = ET.parse(xml_file)
root = tree.getroot()
seq_id = 0
this_time = 0.0
this_duration = 0.0
prev_time = 0.0
prev_duration = 0.0
this_text = ""
for child in root:
if child.tag == "row":
cur_time = float(child.attrib["timestamp"])
cur_duration = float(child.attrib["timedur"])
cur_text = child.text
if this_time == 0.0:
this_time = cur_time
delta = cur_time - (prev_time + prev_duration)
# rel_tol value is made from trial/error to try and compromise between:
# - cutting enough to skip missing words
# - not too short, not too long sentences
is_close = math.isclose(cur_time,
this_time + this_duration,
rel_tol=rel_tol)
is_short = ((this_duration + cur_duration + delta) < MAX_SECS)
# when the previous element is close enough **and** this does not
# go over MAX_SECS, we append content
if (is_close and is_short):
this_duration += cur_duration + delta
this_text += cur_text
else:
samples.append((audio_source, target_dir, dataset_basename,
this_time, this_duration, this_text, seq_id))
this_time = cur_time
this_duration = cur_duration
this_text = cur_text
seq_id += 1
prev_time = cur_time
prev_duration = cur_duration
# Keep track of how many samples are good vs. problematic
_counter = get_counter()
num_samples = len(samples)
_rows = []
print("Processing XML data: {}".format(xml_file))
pool = Pool()
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
for i, processed in enumerate(pool.imap_unordered(one_sample, samples),
start=1):
_counter += processed[0]
_rows += processed[1]
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
imported_samples = get_imported_samples(_counter)
assert _counter["all"] == num_samples
assert len(_rows) == imported_samples
print_import_report(_counter, SAMPLE_RATE, MAX_SECS)
print("Import efficiency: %.1f%%" %
((_counter["total_time"] / source_frames) * 100))
print("")
return _counter, _rows
label = unicodedata.normalize("NFKD", label.strip()) \
.encode("ascii", "ignore") \
.decode("ascii", "ignore")
label = maybe_normalize(label)
label = validate_label(label)
if ALPHABET and label:
try:
ALPHABET.encode(label)
except KeyError:
label = None
return label
dataset_sources = _download_and_preprocess_data(
csv_url=DATASET_RELEASE_CSV, target_dir=PARAMS.target_dir)
sources_root_dir = os.path.dirname(dataset_sources)
all_counter = get_counter()
all_rows = []
with open(dataset_sources, "r") as sources:
for line in sources.readlines():
d = line.split(",")
this_xml = os.path.join(sources_root_dir, d[0])
this_mp3 = os.path.join(sources_root_dir, d[1])
this_rel = float(d[2])
wav_filename = os.path.join(
sources_root_dir,
os.path.splitext(os.path.basename(this_mp3))[0] + ".wav")
_maybe_convert_wav(this_mp3, wav_filename)
counter, rows = _maybe_import_data(this_xml, wav_filename,
sources_root_dir, this_rel)
def _maybe_convert_sets(target_dir, extracted_data, english_compatible=False):
extracted_dir = os.path.join(target_dir, extracted_data)
# override existing CSV with normalized one
target_csv_template = os.path.join(target_dir, "ts_" + ARCHIVE_NAME + "_{}.csv")
if os.path.isfile(target_csv_template):
return
path_to_original_csv = os.path.join(extracted_dir, "data.csv")
with open(path_to_original_csv) as csv_f:
data = [
d
for d in csv.DictReader(csv_f, delimiter=",")
if float(d["duration"]) <= MAX_SECS
]
for line in data:
line["path"] = os.path.join(extracted_dir, line["path"])
num_samples = len(data)
rows = []
counter = get_counter()
print("Importing {} wav files...".format(num_samples))
pool = Pool()
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
for i, processed in enumerate(pool.imap_unordered(one_sample, data), start=1):
counter += processed[0]
rows += processed[1]
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
with open(target_csv_template.format("train"), "w") as train_csv_file: # 80%
with open(target_csv_template.format("dev"), "w") as dev_csv_file: # 10%
with open(target_csv_template.format("test"), "w") as test_csv_file: # 10%
train_writer = csv.DictWriter(train_csv_file, fieldnames=FIELDNAMES)
train_writer.writeheader()
dev_writer = csv.DictWriter(dev_csv_file, fieldnames=FIELDNAMES)
dev_writer.writeheader()
test_writer = csv.DictWriter(test_csv_file, fieldnames=FIELDNAMES)
test_writer.writeheader()
for i, item in enumerate(rows):
transcript = validate_label(
cleanup_transcript(
item[2], english_compatible=english_compatible
)
)
if not transcript:
continue
wav_filename = os.path.join(target_dir, extracted_data, item[0])
i_mod = i % 10
if i_mod == 0:
writer = test_writer
elif i_mod == 1:
writer = dev_writer
else:
writer = train_writer
writer.writerow(
dict(
wav_filename=wav_filename,
wav_filesize=os.path.getsize(wav_filename),
transcript=transcript,
)
)
imported_samples = get_imported_samples(counter)
assert counter["all"] == num_samples
assert len(rows) == imported_samples
print_import_report(counter, SAMPLE_RATE, MAX_SECS)
def _maybe_convert_sets(target_dir, extracted_data):
extracted_dir = os.path.join(target_dir, extracted_data)
# override existing CSV with normalized one
target_csv_template = os.path.join(target_dir, ARCHIVE_DIR_NAME,
ARCHIVE_NAME.replace(".tgz", "_{}.csv"))
if os.path.isfile(target_csv_template):
return
wav_root_dir = os.path.join(extracted_dir)
# Get audiofile path and transcript for each sentence in tsv
samples = []
glob_dir = os.path.join(wav_root_dir, "**/metadata.csv")
for record in glob(glob_dir, recursive=True):
if any(map(lambda sk: sk in record, SKIP_LIST)): # pylint: disable=cell-var-from-loop
continue
with open(record, "r") as rec:
for re in rec.readlines():
re = re.strip().split("|")
audio = os.path.join(os.path.dirname(record), "wavs",
re[0] + ".wav")
transcript = re[2]
samples.append((audio, transcript))
counter = get_counter()
num_samples = len(samples)
rows = []
print("Importing WAV files...")
pool = Pool()
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
for i, processed in enumerate(pool.imap_unordered(one_sample, samples),
start=1):
counter += processed[0]
rows += processed[1]
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
with open(target_csv_template.format("train"),
"w",
encoding="utf-8",
newline="") as train_csv_file: # 80%
with open(target_csv_template.format("dev"),
"w",
encoding="utf-8",
newline="") as dev_csv_file: # 10%
with open(target_csv_template.format("test"),
"w",
encoding="utf-8",
newline="") as test_csv_file: # 10%
train_writer = csv.DictWriter(train_csv_file,
fieldnames=FIELDNAMES)
train_writer.writeheader()
dev_writer = csv.DictWriter(dev_csv_file,
fieldnames=FIELDNAMES)
dev_writer.writeheader()
test_writer = csv.DictWriter(test_csv_file,
fieldnames=FIELDNAMES)
test_writer.writeheader()
for i, item in enumerate(rows):
transcript = validate_label(item[2])
if not transcript:
continue
wav_filename = item[0]
i_mod = i % 10
if i_mod == 0:
writer = test_writer
elif i_mod == 1:
writer = dev_writer
else:
writer = train_writer
writer.writerow(
dict(
wav_filename=os.path.relpath(
wav_filename, extracted_dir),
wav_filesize=os.path.getsize(wav_filename),
transcript=transcript,
))
imported_samples = get_imported_samples(counter)
assert counter["all"] == num_samples
assert len(rows) == imported_samples
print_import_report(counter, SAMPLE_RATE, MAX_SECS)
def _maybe_convert_set(dataset_dir,
audio_dir,
filter_obj,
space_after_every_character=None,
rows=None):
# iterate over all data lists and write converted version near them
speaker_iterator = 1
samples = []
total_file_dict = dict()
for subdir, dirs, files in os.walk(dataset_dir):
for file in files:
# Get audiofile path and transcript for each sentence in tsv
if file.endswith(".data"):
file_path = os.path.join(subdir, file)
file = open(file_path, mode="r")
data = []
file_folder = os.path.join(os.path.dirname(subdir), "wav")
file_dict = dict()
for row in file.readlines():
if row.isspace():
continue
splitted_row = row.replace("\n", "").replace(
" wav ", ".wav ").split(" ", 1)
if len(splitted_row) != 2:
continue
file_name, transcript = splitted_row
if file_name.endswith(".wav"):
pass
elif file_name.endswith(".mp3"):
pass
elif file_name.find(".") == -1:
file_name += ".wav"
if file_name.startswith("/"):
file_name = file_name[1::]
file_name = os.path.join(dataset_dir, file_name)
file_dict[file_name] = convert_transcript(transcript)
file.close()
for wav_subdir, wav_dirs, wav_files in os.walk(file_folder):
for wav_file in wav_files:
wav_file_path = os.path.join(wav_subdir, wav_file)
if file_dict.get(wav_file_path) is not None:
total_file_dict[wav_file_path] = file_dict[
wav_file_path]
for key in total_file_dict.keys():
samples.append((key, total_file_dict[key], speaker_iterator))
speaker_iterator += 1
del (total_file_dict)
if rows is None:
rows = []
counter = get_counter()
num_samples = len(samples)
print("Importing dataset files...")
pool = Pool(initializer=init_worker, initargs=(PARAMS, ))
bar = progressbar.ProgressBar(max_value=num_samples,
widgets=SIMPLE_BAR)
for i, processed in enumerate(pool.imap_unordered(one_sample, samples),
start=1):
counter += processed[0]
rows += processed[1]
bar.update(i)
bar.update(num_samples)
pool.close()
pool.join()
imported_samples = get_imported_samples(counter)
assert counter["all"] == num_samples
assert len(rows) == imported_samples
print_import_report(counter, SAMPLE_RATE, MAX_SECS)
output_csv = os.path.join(os.path.abspath(audio_dir), "train.csv")
print("Saving new DeepSpeech-formatted CSV file to: ", output_csv)
with open(output_csv, "w", encoding="utf-8",
newline="") as output_csv_file:
print("Writing CSV file for DeepSpeech.py as: ", output_csv)
writer = csv.DictWriter(output_csv_file, fieldnames=FIELDNAMES)
writer.writeheader()
bar = progressbar.ProgressBar(max_value=len(rows), widgets=SIMPLE_BAR)
for filename, file_size, transcript, speaker in bar(rows):
if space_after_every_character:
writer.writerow({
"wav_filename": filename,
"wav_filesize": file_size,
"transcript": " ".join(transcript),
})
else:
writer.writerow({
"wav_filename": filename,
"wav_filesize": file_size,
"transcript": transcript,
})
return rows
