def transcribe(self, on_complete: Callable = None):
self.status = "transcribing"
infer_path = self.model.path.joinpath('espnet-asr1', 'data', 'infer')
exp_path = self.model.path.joinpath('espnet-asr1', 'exp')
os.makedirs(f"{infer_path}", exist_ok=True)
dir_util.copy_tree(f'{self.path}', f"{infer_path}")
file_util.copy_file(
f'{self.audio_file_path}',
f"{self.model.path.joinpath('espnet-asr1', 'audio.wav')}")
local_espnet_path = Path(
self.model.path
) / "espnet-asr1" # TODO This is now not a single point of control. Make this dir an attribute of the model.
prepare_log_path = Path(self.model.path) / "prepare_transcribe_log.txt"
transcribe_log_path = Path(self.model.path) / "transcribe_log.txt"
from elpis.engines.common.objects.command import run
run(f"cd {local_espnet_path}; ./decode.sh --nj 1 --stage 0 --stop_stage 2 --recog_set infer &> {prepare_log_path}"
)
run(f"cd {local_espnet_path}; ./decode.sh --nj 1 --stage 5 --recog_set infer &> {transcribe_log_path}"
)
result_paths = list(exp_path.glob("train_nodev*/decode_infer*"))
assert len(
result_paths
) == 1, f"Incorrect number of result files ({len(result_paths)})"
result_path = result_paths[0] / "data.json"
file_util.copy_file(result_path, f'{self.path}/results.txt')
self.convert_to_text()
# TODO Need to produce an output eaf.
self.status = "transcribed"
if on_complete is not None:
on_complete()
def train():
local_espnet_path = Path(self.path).joinpath("espnet-asr1")
run_log_path = Path(self.path).joinpath('train.log')
print(f"SELF PATH {self.path}")
if os.path.isfile(run_log_path):
os.remove(run_log_path)
p = run(
f"cd {local_espnet_path}; ./run.sh --nj 1 &> {run_log_path}")
print(p.stdout)
print('train double done.')
def status(self):
# read the log here and pass it back to the api
run_log_path = Path(self.path).joinpath('train.log')
if not Path(run_log_path).is_file():
run(f"touch {run_log_path};")
with open(run_log_path) as log_file:
log_text = log_file.read()
self.stage_status = ("train", 'in-progress', '', log_text)
# Stage 4 train log is not written to the main train log, so read it from the exp dir
is_stage_4 = re.search('stage 4: Network Training\n\Z',
log_text,
flags=re.MULTILINE)
if is_stage_4:
# Train stage 4 log is in the exp dir... something like exp/train_nodev_pytorch_train_mtlalpha1.0
path_gen = Path(self.path).glob("espnet-asr1/exp/train*/train.log")
# Assumes just one decode_test* directory, which is true in the current
# implementation (transcription will use decode_infer*...)
stage_4_log_path = next(path_gen, None)
with open(stage_4_log_path) as stage_4_log_file:
stage_4_text = stage_4_log_file.read()
self.stage_status = ("train", "in-progress", "",
f"{log_text}\n{stage_4_text}")
return self.config['status']
def train():
local_kaldi_path = self.path.joinpath('kaldi')
# Setup for Training complete
######################################################################
# task _test-train
run_log_path = self.path.joinpath('train.log')
if os.path.isfile(run_log_path):
os.remove(run_log_path)
stages = os.listdir(local_kaldi_path.joinpath('stages'))
run(f"touch {run_log_path};")
for stage in sorted(stages):
print(f"======== STAGE {stage} STARTING ========")
self.stage_status = (stage, 'in-progress', '')
with open(
local_kaldi_path.joinpath('stages').joinpath(stage),
'r') as file:
filedata = file.read()
filedata = filedata.replace('lm_order=1',
f'lm_order={self.ngram}')
with open(
local_kaldi_path.joinpath('stages').joinpath(stage),
'w') as file:
file.write(filedata)
p = run(
f"cd {local_kaldi_path}; stages/{stage} >> {run_log_path}")
print(p.stdout)
print(f"======== STAGE {stage} COMPLETE ========")
self.stage_status = (stage, 'complete', '')
self.log = run(f"cd {local_kaldi_path}; cat {run_log_path}").stdout
print('train double done.')
def train():
local_espnet_path = Path(self.path).joinpath("espnet-asr1")
run_log_path = Path(self.path).joinpath('train.log')
print(f"SELF PATH {self.path}")
if os.path.isfile(run_log_path):
os.remove(run_log_path)
try:
p = run(
f"cd {local_espnet_path}; ./run.sh --ngpu {self.config['gpus']} --nj 1 &> {run_log_path}"
)
print('done')
self.status = 'trained'
except subprocess.CalledProcessError as e:
with open(run_log_path, 'a+') as file:
print('stderr', e.stderr, file=file)
print('failed', file=file)
print('failed')
self.status = f'failed with code {e.returncode}'
def prepare_for_training():
print("prepare_for_training")
# task make-kaldi-subfolders
kaldi_structure = PathStructure(self.path)
local_kaldi_path = self.path.joinpath('kaldi')
local_kaldi_path.mkdir(parents=True, exist_ok=True)
kaldi_data_local_dict = local_kaldi_path.joinpath(
'data', 'local', 'dict')
kaldi_data_local_dict.mkdir(parents=True, exist_ok=True)
kaldi_data_local = local_kaldi_path.joinpath('data', 'local')
kaldi_data_local.mkdir(parents=True, exist_ok=True)
kaldi_data_test = local_kaldi_path.joinpath('data', 'test')
kaldi_data_test.mkdir(parents=True, exist_ok=True)
kaldi_data_train = local_kaldi_path.joinpath('data', 'train')
kaldi_data_train.mkdir(parents=True, exist_ok=True)
kaldi_conf = local_kaldi_path.joinpath('conf')
kaldi_conf.mkdir(parents=True, exist_ok=True)
kaldi_local = local_kaldi_path.joinpath('local')
kaldi_local.mkdir(parents=True, exist_ok=True)
# copy the pron dict
shutil.copy(f"{self.pron_dict.lexicon_txt_path}",
f"{kaldi_data_local_dict.joinpath('lexicon.txt')}")
# task generate-kaldi-configs
path_file_path = kaldi_structure.path.joinpath('path.sh')
mfcc_file_path = kaldi_structure.conf.joinpath('mfcc.conf')
decode_config_file_path = kaldi_structure.conf.joinpath(
'decode.config')
template_path = Path('/elpis/elpis/engines/kaldi/templates')
path_resource = template_path.joinpath('path.sh')
mfcc_resource = template_path.joinpath('mfcc.conf')
decode_config_resource = template_path.joinpath('decode.config')
# task make-nonsil-phones > {{ .KALDI_OUTPUT_PATH }}/tmp/nonsilence_phones.txt
nonsilence_phones_path = kaldi_data_local_dict.joinpath(
'nonsilence_phones.txt')
# build a unnique non-sorted list of the phone symbols
# can't use sorting, because the rules may have order significance
# ignore comment lines that begin with #
seen = OrderedDict()
for line in open(self.pron_dict.l2s_path, "r"):
if line[0] == "#":
pass
else:
line = line.split()[1:]
if len(line) > 0:
line = line[0]
seen[line] = seen.get(line, 0) + 1
with nonsilence_phones_path.open(mode='w') as fout:
for (item, i) in seen.items():
fout.write("%s\n" % item)
with path_file_path.open(mode='w') as fout:
with path_resource.open() as fin:
content = Template(fin.read()).render({
'KALDI_ROOT':
'/kaldi',
'HELPERS_PATH':
'/kaldi-helpers',
'CORPUS_PATH':
f'..{self.dataset.pathto.original}'
})
fout.write(content)
with mfcc_file_path.open(mode='w') as fout:
with mfcc_resource.open() as fin:
content = Template(fin.read()).render({
'MFCC_SAMPLE_FREQUENCY':
'44100',
'MFCC_FRAME_LENGTH':
'25',
'MFCC_LOW_FREQ':
'20',
'MFCC_HIGH_FREQ':
'22050',
'MFCC_NUM_CEPS':
'7',
})
fout.write(content)
with decode_config_file_path.open(mode='w') as fout:
with decode_config_resource.open() as fin:
content = Template(fin.read()).render({
'DECODE_BEAM':
'11.0',
'DECODE_FIRST_BEAM':
'8.0'
})
fout.write(content)
try:
# task copy-generated-files
output_path = self.path.joinpath('output')
output_path.mkdir(parents=True, exist_ok=True)
# - cp {{ .KALDI_OUTPUT_PATH }}/tmp/json_splitted/training/corpus.txt {{ .KALDI_OUTPUT_PATH }}/kaldi/data/local/
shutil.move(
f"{output_path.joinpath('training', 'corpus.txt')}",
f"{kaldi_data_local}")
shutil.move(f"{output_path.joinpath('testing', 'segments')}",
f"{kaldi_data_test.joinpath('segments')}")
shutil.move(f"{output_path.joinpath('testing', 'text')}",
f"{kaldi_data_test.joinpath('text')}")
shutil.move(f"{output_path.joinpath('testing', 'utt2spk')}",
f"{kaldi_data_test.joinpath('utt2spk')}")
shutil.move(f"{output_path.joinpath('testing', 'wav.scp')}",
f"{kaldi_data_test.joinpath('wav.scp')}")
shutil.move(f"{output_path.joinpath('training', 'segments')}",
f"{kaldi_data_train.joinpath('segments')}")
shutil.move(f"{output_path.joinpath('training', 'text')}",
f"{kaldi_data_train.joinpath('text')}")
shutil.move(f"{output_path.joinpath('training', 'utt2spk')}",
f"{kaldi_data_train.joinpath('utt2spk')}")
shutil.move(f"{output_path.joinpath('training', 'wav.scp')}",
f"{kaldi_data_train.joinpath('wav.scp')}")
# task copy-phones-configs
optional_silence_file_path = kaldi_data_local_dict.joinpath(
'optional_silence.txt')
silence_phones_file_path = kaldi_data_local_dict.joinpath(
'silence_phones.txt')
with optional_silence_file_path.open(mode='w') as fout:
fout.write('SIL\n')
with silence_phones_file_path.open(mode='w') as fout:
fout.write('SIL\nsil\nspn\n')
shutil.copy(f"{template_path.joinpath('cmd.sh')}",
f"{local_kaldi_path}")
shutil.copytree(f"{template_path.joinpath('stages')}",
local_kaldi_path.joinpath('stages'))
for file in os.listdir(local_kaldi_path.joinpath('stages')):
os.chmod(
local_kaldi_path.joinpath('stages').joinpath(file),
0o774)
shutil.copy(f"{template_path.joinpath('score.sh')}",
f"{kaldi_local}")
run(f"cp -L -r /kaldi/egs/wsj/s5/steps {local_kaldi_path}/steps"
)
run(f"cp -L -r /kaldi/egs/wsj/s5/utils {local_kaldi_path}/utils"
)
# modified extract-wavs
for audio_file in os.listdir(self.dataset.pathto.resampled):
src = f'{self.dataset.pathto.resampled.joinpath(audio_file)}'
dst = f'{local_kaldi_path}'
shutil.copy(src, dst)
print('kaldi dirs preparation done.')
except OSError as error:
print('couldnt prepare kaldi dirs: ', error)
def train():
local_kaldi_path = self.path.joinpath('kaldi')
# Prepare (dump, recreate) main train log file
run_log_path = self.path.joinpath('train.log')
if os.path.isfile(run_log_path):
os.remove(run_log_path)
run(f"touch {run_log_path};")
# Organise stage logs in a dir
train_log_dir = self.path.joinpath('train-logs')
if os.path.exists(train_log_dir):
shutil.rmtree(train_log_dir)
os.mkdir(train_log_dir)
stage_count = 0
stages = os.listdir(local_kaldi_path.joinpath('stages'))
for stage in sorted(stages):
print(f"Stage {stage} starting")
self.stage_status = (stage, 'in-progress', '')
# Create log file
stage_log_path = self.path.joinpath(
os.path.join(train_log_dir, f'stage_{stage_count}.log'))
with open(stage_log_path, 'w+'):
pass
# Manipulate stage templates with user-defined settings
# TODO replace with jinja templates or something similar
with open(
local_kaldi_path.joinpath('stages').joinpath(stage),
'r') as file:
filedata = file.read()
# Add settings to replace here
filedata = filedata.replace('lm_order=1',
f'lm_order={self.ngram}')
with open(
local_kaldi_path.joinpath('stages').joinpath(stage),
'w') as file:
file.write(filedata)
# Run the command, log output. Also redirect Kaldi sterr output to log. These are often not errors :-(
try:
stage_process = run(
f"cd {local_kaldi_path}; stages/{stage} >> {stage_log_path}"
)
with open(stage_log_path, 'a+') as file:
print('stdout', stage_process.stdout, file=file)
print('stderr', stage_process.stderr, file=file)
print('done', file=file)
print(f"Stage {stage} complete")
self.stage_status = (stage, 'complete', '')
stage_count = stage_count + 1
except CalledProcessError as error:
with open(stage_log_path, 'a+') as file:
print('stderr', error.stderr, file=file)
print('failed', file=file)
print(f"Stage {stage} failed")
self.stage_status = (stage, 'failed', '')
break
self.log = ''
# Concat all the files in the train-log dir
log_filenames = os.listdir(train_log_dir)
log_filenames.sort()
with open(run_log_path, 'w') as outfile:
for log_file in log_filenames:
with open(os.path.join(train_log_dir, log_file)) as infile:
log_contents = infile.read()
outfile.write(log_contents)
outfile.write("\n")
self.log += log_contents
def transcribe(self, on_complete: Callable = None):
self.status = "transcribing"
local_kaldi_path = self.model.path.joinpath('kaldi')
kaldi_infer_path = self.model.path.joinpath('kaldi', 'data', 'infer')
# TODO move this to templates/infer/stages/ and move the train templates into templates/train/stages
gmm_decode_path = Path(
'/elpis/elpis/engines/kaldi/inference/gmm-decode')
# Prepare (dump, recreate) main transcription log file
run_log_path = self.path.joinpath('transcription.log')
if os.path.isfile(run_log_path):
os.remove(run_log_path)
run(f"touch {run_log_path};")
# Organise stage logs in a dir
transcription_log_dir = self.path.joinpath('transcription-logs')
if os.path.exists(transcription_log_dir):
shutil.rmtree(transcription_log_dir)
os.mkdir(transcription_log_dir)
stage_count = 0
# Build stage scripts
os.makedirs(f"{kaldi_infer_path}", exist_ok=True)
dir_util.copy_tree(f'{self.path}', f"{kaldi_infer_path}")
file_util.copy_file(
f'{self.audio_file_path}',
f"{self.model.path.joinpath('kaldi', 'audio.wav')}")
# Copy parts of transcription process and chmod
dir_util.copy_tree(f'{gmm_decode_path}',
f"{kaldi_infer_path.joinpath('gmm-decode')}")
stages = os.listdir(kaldi_infer_path.joinpath('gmm-decode'))
for file in stages:
os.chmod(
kaldi_infer_path.joinpath('gmm-decode').joinpath(file), 0o774)
print('*** kaldi_infer_path', kaldi_infer_path)
for stage in sorted(stages):
print(f"Stage {stage} starting")
self.stage_status = (stage, 'in-progress', '')
# Create log file
stage_log_path = self.path.joinpath(
os.path.join(transcription_log_dir,
f'stage_{stage_count}.log'))
with open(stage_log_path, 'w+'):
pass
# Run the command, log output. Also redirect Kaldi sterr output to log. These are often not errors :-(
# These scripts must run from the kaldi dir (so set cwd)
try:
script_path = kaldi_infer_path.joinpath("gmm-decode", stage)
stage_process = run(f"sh {script_path} >> {stage_log_path}",
cwd=f"{local_kaldi_path}")
with open(stage_log_path, 'a+') as file:
print('stdout', stage_process.stdout, file=file)
print('stderr', stage_process.stderr, file=file)
print('done', file=file)
print(f"Stage {stage} complete")
self.stage_status = (stage, 'complete', '')
stage_count = stage_count + 1
except CalledProcessError as error:
with open(stage_log_path, 'a+') as file:
print('stderr', error.stderr, file=file)
print('failed', file=file)
print(f"Stage {stage} failed")
self.stage_status = (stage, 'failed', '')
break
# Concat all the files in the transcription-log dir
log_filenames = os.listdir(transcription_log_dir)
log_filenames.sort()
with open(run_log_path, 'w') as outfile:
for log_file in log_filenames:
with open(os.path.join(transcription_log_dir,
log_file)) as infile:
outfile.write(infile.read())
outfile.write("\n")
file_util.copy_file(
f"{kaldi_infer_path.joinpath('one-best-hypothesis.txt')}",
f'{self.path}/one-best-hypothesis.txt')
file_util.copy_file(f"{kaldi_infer_path.joinpath('utterance-0.eaf')}",
f'{self.path}/{self.hash}.eaf')
self.status = "transcribed"
if on_complete is not None:
on_complete()