def transcribe(self, on_complete: Callable = None): self.status = "transcribing" infer_path = self.model.path.joinpath('espnet-asr1', 'data', 'infer') exp_path = self.model.path.joinpath('espnet-asr1', 'exp') os.makedirs(f"{infer_path}", exist_ok=True) dir_util.copy_tree(f'{self.path}', f"{infer_path}") file_util.copy_file( f'{self.audio_file_path}', f"{self.model.path.joinpath('espnet-asr1', 'audio.wav')}") local_espnet_path = Path( self.model.path ) / "espnet-asr1" # TODO This is now not a single point of control. Make this dir an attribute of the model. prepare_log_path = Path(self.model.path) / "prepare_transcribe_log.txt" transcribe_log_path = Path(self.model.path) / "transcribe_log.txt" from elpis.engines.common.objects.command import run run(f"cd {local_espnet_path}; ./decode.sh --nj 1 --stage 0 --stop_stage 2 --recog_set infer &> {prepare_log_path}" ) run(f"cd {local_espnet_path}; ./decode.sh --nj 1 --stage 5 --recog_set infer &> {transcribe_log_path}" ) result_paths = list(exp_path.glob("train_nodev*/decode_infer*")) assert len( result_paths ) == 1, f"Incorrect number of result files ({len(result_paths)})" result_path = result_paths[0] / "data.json" file_util.copy_file(result_path, f'{self.path}/results.txt') self.convert_to_text() # TODO Need to produce an output eaf. self.status = "transcribed" if on_complete is not None: on_complete()
def train(): local_espnet_path = Path(self.path).joinpath("espnet-asr1") run_log_path = Path(self.path).joinpath('train.log') print(f"SELF PATH {self.path}") if os.path.isfile(run_log_path): os.remove(run_log_path) p = run( f"cd {local_espnet_path}; ./run.sh --nj 1 &> {run_log_path}") print(p.stdout) print('train double done.')
def status(self): # read the log here and pass it back to the api run_log_path = Path(self.path).joinpath('train.log') if not Path(run_log_path).is_file(): run(f"touch {run_log_path};") with open(run_log_path) as log_file: log_text = log_file.read() self.stage_status = ("train", 'in-progress', '', log_text) # Stage 4 train log is not written to the main train log, so read it from the exp dir is_stage_4 = re.search('stage 4: Network Training\n\Z', log_text, flags=re.MULTILINE) if is_stage_4: # Train stage 4 log is in the exp dir... something like exp/train_nodev_pytorch_train_mtlalpha1.0 path_gen = Path(self.path).glob("espnet-asr1/exp/train*/train.log") # Assumes just one decode_test* directory, which is true in the current # implementation (transcription will use decode_infer*...) stage_4_log_path = next(path_gen, None) with open(stage_4_log_path) as stage_4_log_file: stage_4_text = stage_4_log_file.read() self.stage_status = ("train", "in-progress", "", f"{log_text}\n{stage_4_text}") return self.config['status']
def train(): local_kaldi_path = self.path.joinpath('kaldi') # Setup for Training complete ###################################################################### # task _test-train run_log_path = self.path.joinpath('train.log') if os.path.isfile(run_log_path): os.remove(run_log_path) stages = os.listdir(local_kaldi_path.joinpath('stages')) run(f"touch {run_log_path};") for stage in sorted(stages): print(f"======== STAGE {stage} STARTING ========") self.stage_status = (stage, 'in-progress', '') with open( local_kaldi_path.joinpath('stages').joinpath(stage), 'r') as file: filedata = file.read() filedata = filedata.replace('lm_order=1', f'lm_order={self.ngram}') with open( local_kaldi_path.joinpath('stages').joinpath(stage), 'w') as file: file.write(filedata) p = run( f"cd {local_kaldi_path}; stages/{stage} >> {run_log_path}") print(p.stdout) print(f"======== STAGE {stage} COMPLETE ========") self.stage_status = (stage, 'complete', '') self.log = run(f"cd {local_kaldi_path}; cat {run_log_path}").stdout print('train double done.')
def train(): local_espnet_path = Path(self.path).joinpath("espnet-asr1") run_log_path = Path(self.path).joinpath('train.log') print(f"SELF PATH {self.path}") if os.path.isfile(run_log_path): os.remove(run_log_path) try: p = run( f"cd {local_espnet_path}; ./run.sh --ngpu {self.config['gpus']} --nj 1 &> {run_log_path}" ) print('done') self.status = 'trained' except subprocess.CalledProcessError as e: with open(run_log_path, 'a+') as file: print('stderr', e.stderr, file=file) print('failed', file=file) print('failed') self.status = f'failed with code {e.returncode}'
def prepare_for_training(): print("prepare_for_training") # task make-kaldi-subfolders kaldi_structure = PathStructure(self.path) local_kaldi_path = self.path.joinpath('kaldi') local_kaldi_path.mkdir(parents=True, exist_ok=True) kaldi_data_local_dict = local_kaldi_path.joinpath( 'data', 'local', 'dict') kaldi_data_local_dict.mkdir(parents=True, exist_ok=True) kaldi_data_local = local_kaldi_path.joinpath('data', 'local') kaldi_data_local.mkdir(parents=True, exist_ok=True) kaldi_data_test = local_kaldi_path.joinpath('data', 'test') kaldi_data_test.mkdir(parents=True, exist_ok=True) kaldi_data_train = local_kaldi_path.joinpath('data', 'train') kaldi_data_train.mkdir(parents=True, exist_ok=True) kaldi_conf = local_kaldi_path.joinpath('conf') kaldi_conf.mkdir(parents=True, exist_ok=True) kaldi_local = local_kaldi_path.joinpath('local') kaldi_local.mkdir(parents=True, exist_ok=True) # copy the pron dict shutil.copy(f"{self.pron_dict.lexicon_txt_path}", f"{kaldi_data_local_dict.joinpath('lexicon.txt')}") # task generate-kaldi-configs path_file_path = kaldi_structure.path.joinpath('path.sh') mfcc_file_path = kaldi_structure.conf.joinpath('mfcc.conf') decode_config_file_path = kaldi_structure.conf.joinpath( 'decode.config') template_path = Path('/elpis/elpis/engines/kaldi/templates') path_resource = template_path.joinpath('path.sh') mfcc_resource = template_path.joinpath('mfcc.conf') decode_config_resource = template_path.joinpath('decode.config') # task make-nonsil-phones > {{ .KALDI_OUTPUT_PATH }}/tmp/nonsilence_phones.txt nonsilence_phones_path = kaldi_data_local_dict.joinpath( 'nonsilence_phones.txt') # build a unnique non-sorted list of the phone symbols # can't use sorting, because the rules may have order significance # ignore comment lines that begin with # seen = OrderedDict() for line in open(self.pron_dict.l2s_path, "r"): if line[0] == "#": pass else: line = line.split()[1:] if len(line) > 0: line = line[0] seen[line] = seen.get(line, 0) + 1 with nonsilence_phones_path.open(mode='w') as fout: for (item, i) in seen.items(): fout.write("%s\n" % item) with path_file_path.open(mode='w') as fout: with path_resource.open() as fin: content = Template(fin.read()).render({ 'KALDI_ROOT': '/kaldi', 'HELPERS_PATH': '/kaldi-helpers', 'CORPUS_PATH': f'..{self.dataset.pathto.original}' }) fout.write(content) with mfcc_file_path.open(mode='w') as fout: with mfcc_resource.open() as fin: content = Template(fin.read()).render({ 'MFCC_SAMPLE_FREQUENCY': '44100', 'MFCC_FRAME_LENGTH': '25', 'MFCC_LOW_FREQ': '20', 'MFCC_HIGH_FREQ': '22050', 'MFCC_NUM_CEPS': '7', }) fout.write(content) with decode_config_file_path.open(mode='w') as fout: with decode_config_resource.open() as fin: content = Template(fin.read()).render({ 'DECODE_BEAM': '11.0', 'DECODE_FIRST_BEAM': '8.0' }) fout.write(content) try: # task copy-generated-files output_path = self.path.joinpath('output') output_path.mkdir(parents=True, exist_ok=True) # - cp {{ .KALDI_OUTPUT_PATH }}/tmp/json_splitted/training/corpus.txt {{ .KALDI_OUTPUT_PATH }}/kaldi/data/local/ shutil.move( f"{output_path.joinpath('training', 'corpus.txt')}", f"{kaldi_data_local}") shutil.move(f"{output_path.joinpath('testing', 'segments')}", f"{kaldi_data_test.joinpath('segments')}") shutil.move(f"{output_path.joinpath('testing', 'text')}", f"{kaldi_data_test.joinpath('text')}") shutil.move(f"{output_path.joinpath('testing', 'utt2spk')}", f"{kaldi_data_test.joinpath('utt2spk')}") shutil.move(f"{output_path.joinpath('testing', 'wav.scp')}", f"{kaldi_data_test.joinpath('wav.scp')}") shutil.move(f"{output_path.joinpath('training', 'segments')}", f"{kaldi_data_train.joinpath('segments')}") shutil.move(f"{output_path.joinpath('training', 'text')}", f"{kaldi_data_train.joinpath('text')}") shutil.move(f"{output_path.joinpath('training', 'utt2spk')}", f"{kaldi_data_train.joinpath('utt2spk')}") shutil.move(f"{output_path.joinpath('training', 'wav.scp')}", f"{kaldi_data_train.joinpath('wav.scp')}") # task copy-phones-configs optional_silence_file_path = kaldi_data_local_dict.joinpath( 'optional_silence.txt') silence_phones_file_path = kaldi_data_local_dict.joinpath( 'silence_phones.txt') with optional_silence_file_path.open(mode='w') as fout: fout.write('SIL\n') with silence_phones_file_path.open(mode='w') as fout: fout.write('SIL\nsil\nspn\n') shutil.copy(f"{template_path.joinpath('cmd.sh')}", f"{local_kaldi_path}") shutil.copytree(f"{template_path.joinpath('stages')}", local_kaldi_path.joinpath('stages')) for file in os.listdir(local_kaldi_path.joinpath('stages')): os.chmod( local_kaldi_path.joinpath('stages').joinpath(file), 0o774) shutil.copy(f"{template_path.joinpath('score.sh')}", f"{kaldi_local}") run(f"cp -L -r /kaldi/egs/wsj/s5/steps {local_kaldi_path}/steps" ) run(f"cp -L -r /kaldi/egs/wsj/s5/utils {local_kaldi_path}/utils" ) # modified extract-wavs for audio_file in os.listdir(self.dataset.pathto.resampled): src = f'{self.dataset.pathto.resampled.joinpath(audio_file)}' dst = f'{local_kaldi_path}' shutil.copy(src, dst) print('kaldi dirs preparation done.') except OSError as error: print('couldnt prepare kaldi dirs: ', error)
def train(): local_kaldi_path = self.path.joinpath('kaldi') # Prepare (dump, recreate) main train log file run_log_path = self.path.joinpath('train.log') if os.path.isfile(run_log_path): os.remove(run_log_path) run(f"touch {run_log_path};") # Organise stage logs in a dir train_log_dir = self.path.joinpath('train-logs') if os.path.exists(train_log_dir): shutil.rmtree(train_log_dir) os.mkdir(train_log_dir) stage_count = 0 stages = os.listdir(local_kaldi_path.joinpath('stages')) for stage in sorted(stages): print(f"Stage {stage} starting") self.stage_status = (stage, 'in-progress', '') # Create log file stage_log_path = self.path.joinpath( os.path.join(train_log_dir, f'stage_{stage_count}.log')) with open(stage_log_path, 'w+'): pass # Manipulate stage templates with user-defined settings # TODO replace with jinja templates or something similar with open( local_kaldi_path.joinpath('stages').joinpath(stage), 'r') as file: filedata = file.read() # Add settings to replace here filedata = filedata.replace('lm_order=1', f'lm_order={self.ngram}') with open( local_kaldi_path.joinpath('stages').joinpath(stage), 'w') as file: file.write(filedata) # Run the command, log output. Also redirect Kaldi sterr output to log. These are often not errors :-( try: stage_process = run( f"cd {local_kaldi_path}; stages/{stage} >> {stage_log_path}" ) with open(stage_log_path, 'a+') as file: print('stdout', stage_process.stdout, file=file) print('stderr', stage_process.stderr, file=file) print('done', file=file) print(f"Stage {stage} complete") self.stage_status = (stage, 'complete', '') stage_count = stage_count + 1 except CalledProcessError as error: with open(stage_log_path, 'a+') as file: print('stderr', error.stderr, file=file) print('failed', file=file) print(f"Stage {stage} failed") self.stage_status = (stage, 'failed', '') break self.log = '' # Concat all the files in the train-log dir log_filenames = os.listdir(train_log_dir) log_filenames.sort() with open(run_log_path, 'w') as outfile: for log_file in log_filenames: with open(os.path.join(train_log_dir, log_file)) as infile: log_contents = infile.read() outfile.write(log_contents) outfile.write("\n") self.log += log_contents
def transcribe(self, on_complete: Callable = None): self.status = "transcribing" local_kaldi_path = self.model.path.joinpath('kaldi') kaldi_infer_path = self.model.path.joinpath('kaldi', 'data', 'infer') # TODO move this to templates/infer/stages/ and move the train templates into templates/train/stages gmm_decode_path = Path( '/elpis/elpis/engines/kaldi/inference/gmm-decode') # Prepare (dump, recreate) main transcription log file run_log_path = self.path.joinpath('transcription.log') if os.path.isfile(run_log_path): os.remove(run_log_path) run(f"touch {run_log_path};") # Organise stage logs in a dir transcription_log_dir = self.path.joinpath('transcription-logs') if os.path.exists(transcription_log_dir): shutil.rmtree(transcription_log_dir) os.mkdir(transcription_log_dir) stage_count = 0 # Build stage scripts os.makedirs(f"{kaldi_infer_path}", exist_ok=True) dir_util.copy_tree(f'{self.path}', f"{kaldi_infer_path}") file_util.copy_file( f'{self.audio_file_path}', f"{self.model.path.joinpath('kaldi', 'audio.wav')}") # Copy parts of transcription process and chmod dir_util.copy_tree(f'{gmm_decode_path}', f"{kaldi_infer_path.joinpath('gmm-decode')}") stages = os.listdir(kaldi_infer_path.joinpath('gmm-decode')) for file in stages: os.chmod( kaldi_infer_path.joinpath('gmm-decode').joinpath(file), 0o774) print('*** kaldi_infer_path', kaldi_infer_path) for stage in sorted(stages): print(f"Stage {stage} starting") self.stage_status = (stage, 'in-progress', '') # Create log file stage_log_path = self.path.joinpath( os.path.join(transcription_log_dir, f'stage_{stage_count}.log')) with open(stage_log_path, 'w+'): pass # Run the command, log output. Also redirect Kaldi sterr output to log. These are often not errors :-( # These scripts must run from the kaldi dir (so set cwd) try: script_path = kaldi_infer_path.joinpath("gmm-decode", stage) stage_process = run(f"sh {script_path} >> {stage_log_path}", cwd=f"{local_kaldi_path}") with open(stage_log_path, 'a+') as file: print('stdout', stage_process.stdout, file=file) print('stderr', stage_process.stderr, file=file) print('done', file=file) print(f"Stage {stage} complete") self.stage_status = (stage, 'complete', '') stage_count = stage_count + 1 except CalledProcessError as error: with open(stage_log_path, 'a+') as file: print('stderr', error.stderr, file=file) print('failed', file=file) print(f"Stage {stage} failed") self.stage_status = (stage, 'failed', '') break # Concat all the files in the transcription-log dir log_filenames = os.listdir(transcription_log_dir) log_filenames.sort() with open(run_log_path, 'w') as outfile: for log_file in log_filenames: with open(os.path.join(transcription_log_dir, log_file)) as infile: outfile.write(infile.read()) outfile.write("\n") file_util.copy_file( f"{kaldi_infer_path.joinpath('one-best-hypothesis.txt')}", f'{self.path}/one-best-hypothesis.txt') file_util.copy_file(f"{kaldi_infer_path.joinpath('utterance-0.eaf')}", f'{self.path}/{self.hash}.eaf') self.status = "transcribed" if on_complete is not None: on_complete()