def prepare_data():
dataset_names = [
'dev-clean', 'dev-other', 'test-clean', 'test-other',
'train-clean-100', 'train-clean-360'
]
bliss_flac_corpus_dict = {}
zip_flac_corpus_dict = {}
for dataset_name in dataset_names:
dataset_path = Path("../data/dataset-raw/LibriSpeech/%s/" %
dataset_name)
ls_to_bliss_job = LibriSpeechToBliss(corpus_path=dataset_path,
name=dataset_name)
ls_to_bliss_job.add_alias("data/LibriSpeechToBliss/%s" % dataset_name)
bliss_flac_corpus_dict[dataset_name] = ls_to_bliss_job.out
tk.register_output("data/bliss/%s.xml.gz" % dataset_name,
ls_to_bliss_job.out)
bliss_to_zip_job = BlissToZipDataset(name=dataset_name,
corpus_file=ls_to_bliss_job.out,
use_full_seq_name=False)
bliss_to_zip_job.add_alias("data/BlissToZipDataset/%s" % dataset_name)
zip_flac_corpus_dict[dataset_name] = bliss_to_zip_job.out
tk.register_output("data/asr_zip/%s.zip" % dataset_name,
bliss_to_zip_job.out)
return bliss_flac_corpus_dict, zip_flac_corpus_dict
def prepare_data_librispeech():
"""
This function creates the LibriSpeech data in Bliss format and zip format.
For the evaluation sets, the text is extracted in dictionary form for WER scoring
:return:
"""
# all datasets that are used in the experiments for LibriSpeech
dataset_names = [
'dev-clean', 'dev-other', 'test-clean', 'test-other',
'train-clean-100', 'train-clean-360'
]
evaluation_names = ['dev-clean', 'dev-other', 'test-clean', 'test-other']
bliss_flac_corpus_dict = {}
zip_flac_corpus_dict = {}
transcription_corpus_dict = {}
for dataset_name in dataset_names:
dataset_path = Path("../data/dataset-raw/LibriSpeech/%s/" %
dataset_name)
# open the raw LibriSpeech data and create bliss corpus
ls_to_bliss_job = LibriSpeechToBliss(corpus_path=dataset_path,
name=dataset_name)
ls_to_bliss_job.add_alias("data/LibriSpeechToBliss/%s" % dataset_name)
bliss_flac_corpus_dict[dataset_name] = ls_to_bliss_job.out
tk.register_output("data/bliss/%s.xml.gz" % dataset_name,
ls_to_bliss_job.out)
# create a unified zip corpus file from the bliss corpus
bliss_to_zip_job = BlissToZipDataset(name=dataset_name,
corpus_file=ls_to_bliss_job.out,
use_full_seq_name=False)
bliss_to_zip_job.add_alias("data/BlissToZipDataset/%s" % dataset_name)
zip_flac_corpus_dict[dataset_name] = bliss_to_zip_job.out
tk.register_output("data/asr_zip/%s.zip" % dataset_name,
bliss_to_zip_job.out)
for dataset_name in evaluation_names:
# create the dictionary format transcription files
bliss_to_text_dict_job = BlissExtractTextDictionary(
bliss_flac_corpus_dict[dataset_name], segment_key_only=True)
bliss_to_text_dict_job.add_alias("data/BlissExtractTextDictionary/%s" %
dataset_name)
transcription_corpus_dict[dataset_name] = bliss_to_text_dict_job.out
return bliss_flac_corpus_dict, zip_flac_corpus_dict, transcription_corpus_dict
def prepare_tts_data(bliss_dict):
"""
:param dict bliss_dict:
:return:
"""
from recipe.returnn.vocabulary import BuildCharacterVocabulary
build_char_vocab_job = BuildCharacterVocabulary(uppercase=True)
char_vocab = build_char_vocab_job.out
processed_corpora = {}
processed_zip_corpora = {}
for name, corpus in bliss_dict.items():
tts_name = "tts-" + name
processed_corpus = process_corpus(bliss_corpus=corpus,
char_vocab=char_vocab,
silence_duration=0.1,
name=tts_name)
processed_corpora[tts_name] = processed_corpus
tk.register_output("data/bliss/%s.processed.xml.gz" % name,
processed_corpus)
processed_zip_corpora[tts_name] = BlissToZipDataset(
tts_name, processed_corpus).out
return processed_corpora, processed_zip_corpora, char_vocab
def evaluate_tts(ttf_config_file, ttf_model_dir, ttf_epoch, f2l_config_file,
f2l_model_dir, f2l_epoch, train_zip_corpus, test_zip_corpus,
test_bliss_corpus, test_text, default_parameter_dict, name):
embedding_hdf = generate_speaker_embeddings(
ttf_config_file,
ttf_model_dir,
ttf_epoch,
train_zip_corpus,
name=name,
default_parameter_dict=default_parameter_dict)
from recipe.tts.corpus import DistributeSpeakerEmbeddings
dist_speaker_embeds_job = DistributeSpeakerEmbeddings(
test_bliss_corpus,
embedding_hdf,
use_full_seq_name=False,
options=None)
dist_embeddings_hdf = dist_speaker_embeds_job.out
unstacked_feature_hdf, _, _ = decode_with_speaker_embeddings(
ttf_config_file, ttf_model_dir, ttf_epoch, test_zip_corpus,
dist_embeddings_hdf, name, default_parameter_dict)
linear_features, _ = convert_with_f2l(f2l_config_file, name, f2l_model_dir,
f2l_epoch, unstacked_feature_hdf)
generated_audio_bliss, _ = griffin_lim_ogg(linear_features, name)
asr_bliss = BlissAddTextFromBliss(generated_audio_bliss,
test_bliss_corpus).out
from recipe.corpus import BlissToZipDataset
asr_zip = BlissToZipDataset("test", asr_bliss, use_full_seq_name=False).out
from config.asr import decode_and_evaluate_asr_config
# ASR EVALUATION
trafo_specaug = Path(
"/u/rossenbach/experiments/switchboard_test/config/alberts_configs/trafo.specaug4.12l.ffdim4.pretrain3.natctc_recognize_pretrained.config"
)
decode_and_evaluate_asr_config("test-%s" % name, trafo_specaug, None, 0,
asr_zip, test_text, {})
def main():
# prepare the datasets in bliss and zip format
with tk.block("data_preparation"):
bliss_dict, zip_dict, transcription_text_dict = prepare_data_librispeech(
)
# compute the subword codes and the ASR vocabulary
bpe_codes, bpe_vocab, num_classes = build_subwords(
[bliss_dict['train-clean-100']],
num_segments=10000,
name="librispeech-100")
mean, stddev = get_asr_dataset_stats(zip_dict['train-clean-100'])
asr_global_parameter_dict = {
'ext_norm_mean': mean,
'ext_norm_std_dev': stddev,
'ext_bpe_file': bpe_codes,
'ext_vocab_file': bpe_vocab,
'ext_num_classes': num_classes
}
initial_checkpoint_training_params = {
'ext_partition_epoch': 20,
'ext_training_zips': [zip_dict['train-clean-100']],
'ext_dev_zips': [zip_dict['dev-clean'], zip_dict['dev-other']],
'ext_num_epochs': 80
}
models = {}
with tk.block("baseline_training"):
# training of the initial model
initial_checkpoint_training_params.update(asr_global_parameter_dict)
asr_training_config = Path(
"returnn_configs/asr/train-clean-100.exp3.ctc.ogg.lrwarmupextra10.config"
)
initial_training_job = train_asr_config(
asr_training_config, "librispeech-100-initial-training",
initial_checkpoint_training_params)
# training and decoding of the baseline model
baseline_training_params = copy.deepcopy(
initial_checkpoint_training_params)
baseline_training_params['ext_num_epochs'] = 250
baseline_training_params['ext_partition_epoch'] = 5
#baseline_training_params['import_model_train_epoch1'] = initial_training_job.models[80].model
baseline_training_params['load'] = initial_training_job.models[
80].model
baseline_training_params.update(asr_global_parameter_dict)
continued_training_job = train_asr_config(
asr_training_config, "librispeech-100-baseline-training",
baseline_training_params)
from recipe.returnn.search import GetBestEpoch
best_epoch = GetBestEpoch(continued_training_job.model_dir,
continued_training_job.learning_rates,
key="dev_score_output/output_prob").out_var
models['baseline'] = (continued_training_job, best_epoch)
with tk.block("specaug_training"):
asr_specaug_config = Path(
"returnn_configs/asr/train-clean-100.exp3.ctc.ogg.lrwarmupextra10.specaug.config"
)
# training and decoding of the baseline model
continued_training_job = train_asr_config(
asr_specaug_config, "librispeech-100-specaug-training",
baseline_training_params)
from recipe.returnn.search import GetBestEpoch
best_epoch = GetBestEpoch(continued_training_job.model_dir,
continued_training_job.learning_rates,
key="dev_score_output/output_prob").out_var
models['specaug'] = (continued_training_job, best_epoch)
##########################3
# TTS
tts_bliss_dict = {
k: v
for k, v in bliss_dict.items()
if k in ['dev-clean', 'train-clean-100', 'train-clean-360']
}
# this will run the preprocessing and add a "tts-" prefix to the corpus names
tts_bliss_corpora, tts_zip_corpora, char_vocab = prepare_ttf_data(
tts_bliss_dict)
mean, stddev = get_ttf_dataset_stats(
tts_zip_corpora['tts-train-clean-100'])
tts_global_parameter_dict = {
'ext_norm_mean_value': mean,
'ext_norm_std_dev_value': stddev,
'ext_char_vocab': char_vocab,
'ext_training_zips': [tts_zip_corpora['tts-train-clean-100']],
'ext_dev_zips': [tts_zip_corpora['tts-dev-clean']],
'ext_num_epochs': 200,
'ext_partition_epoch': 5,
}
tts_training_config = Path(
"returnn_configs/tts/tts-clean-100.dec640.enc256.enclstm512.config",
hash_overwrite="TTS_DEC640_ENC256_ENCLSTM512_v1")
tts_training_job = train_ttf_config(
tts_training_config,
name="tts-baseline-training",
parameter_dict=tts_global_parameter_dict)
f2l_global_parameter_dict = copy.deepcopy(tts_global_parameter_dict)
f2l_global_parameter_dict['ext_num_epochs'] = 100
f2l_global_parameter_dict['ext_partition_epoch'] = 1
f2l_global_parameter_dict.pop('ext_char_vocab')
f2l_training_config = Path(
"returnn_configs/f2l/f2l.2layer.blstm.residual.config",
hash_overwrite="F2L_2LAYER_ENC256_ENCLSTM512_v1")
f2l_training_job = train_ttf_config(
f2l_training_config,
name="f2l-baseline-training",
parameter_dict=f2l_global_parameter_dict)
embeddings = generate_speaker_embeddings(
config_file=tts_training_config,
model_dir=tts_training_job.model_dir,
epoch=200,
zip_corpus=tts_zip_corpora['tts-train-clean-100'],
name="tts-baseline",
default_parameter_dict=tts_global_parameter_dict)
from recipe.tts.corpus import DistributeSpeakerEmbeddings
dist_speaker_embeds_job = DistributeSpeakerEmbeddings(
tts_bliss_dict['train-clean-360'],
embeddings,
use_full_seq_name=False,
options=None)
tk.register_output("embed_dist.hdf", dist_speaker_embeds_job.out)
from config.f2l import convert_with_f2l
from recipe.corpus import SegmentCorpus, MergeCorpora, BlissAddTextFromBliss
from recipe.tts.corpus import VerifyCorpus
from recipe.util import AutoCleanup
TTS_GENERATION_SPLITS = 10
segment_job = SegmentCorpus(tts_bliss_corpora['tts-train-clean-360'],
TTS_GENERATION_SPLITS)
segments = segment_job.segment_files
verification_result = None
corpora = []
for i in range(TTS_GENERATION_SPLITS):
unstacked_features, decode_job, convert_job = decode_with_speaker_embeddings(
config_file=tts_training_config,
model_dir=tts_training_job.model_dir,
epoch=200,
zip_corpus=tts_zip_corpora['tts-train-clean-360'],
speaker_hdf=dist_speaker_embeds_job.out,
segment_file=segments[i],
name="tts-baseline_decode_%i" % i,
default_parameter_dict=tts_global_parameter_dict)
if verification_result:
decode_job.add_input(verification_result)
linear_features, f2l_job = convert_with_f2l(
f2l_training_config,
name="tts-baseline_forward_%i" % i,
features=unstacked_features,
model_dir=f2l_training_job.model_dir,
epoch=100)
generated_audio_bliss, gl_job = griffin_lim_ogg(
linear_features, name="tts-baseline_gl_%i" % i)
verification_result = VerifyCorpus(generated_audio_bliss).out
# remove this for debugging purposes
# WARNING: high HDD consumption
cleanup_success = AutoCleanup([decode_job, convert_job, f2l_job],
verification_result).out
tk.register_output("cleanup_result/cleanup_%i" % i, cleanup_success)
corpora.append(generated_audio_bliss)
# merge the splitted audio corpora back to one corpus and add the original text from Librispeech-360h
merge_job = MergeCorpora(corpora, "synthetic-ls-360", subcorpora=False)
# confirm that the last corpus was correct
merge_job.add_input(verification_result)
synthetic_audio_corpus = merge_job.merged_corpus
synthetic_corpus = BlissAddTextFromBliss(synthetic_audio_corpus,
bliss_dict['train-clean-360']).out
from recipe.corpus import BlissToZipDataset
synthetic_zip_corpus = BlissToZipDataset("synthetic-ls-360",
synthetic_corpus,
use_full_seq_name=False).out
tk.register_output("synthetic_data/synthetic_librispeech_360h.zip",
synthetic_zip_corpus)
# training with synthetic data
with tk.block("baseline_with_synthetic"):
synthetic_training_params = copy.deepcopy(
initial_checkpoint_training_params)
synthetic_training_params['ext_num_epochs'] = 250
synthetic_training_params['load'] = initial_training_job.models[
80].model
synthetic_training_params.update(asr_global_parameter_dict)
synthetic_training_params['ext_training_zips'] = [
zip_dict['train-clean-100'], synthetic_zip_corpus
]
synthetic_training_job = train_asr_config(
asr_training_config, "librispeech-100-synthetic-training",
synthetic_training_params)
from recipe.returnn.search import GetBestEpoch
best_epoch = GetBestEpoch(continued_training_job.model_dir,
continued_training_job.learning_rates,
key="dev_score_output/output_prob").out_var
models['baseline+synthetic'] = (synthetic_training_job, best_epoch)
# training with synthetic data
with tk.block("specaug_with_synthetic"):
synthetic_training_params = copy.deepcopy(
initial_checkpoint_training_params)
synthetic_training_params['ext_num_epochs'] = 250
synthetic_training_params['load'] = initial_training_job.models[
80].model
synthetic_training_params.update(asr_global_parameter_dict)
synthetic_training_params['ext_training_zips'] = [
zip_dict['train-clean-100'], synthetic_zip_corpus
]
synthetic_training_job = train_asr_config(
asr_specaug_config, "librispeech-100-specaug-synthetic-training",
synthetic_training_params)
from recipe.returnn.search import GetBestEpoch
best_epoch = GetBestEpoch(continued_training_job.model_dir,
continued_training_job.learning_rates,
key="dev_score_output/output_prob").out_var
models['specaug+synthetic'] = (synthetic_training_job, best_epoch)
# evaluate the experiments
for experiment_name, (training_job, best_epoch) in models.items():
results = {}
template = ""
with tk.block("%s_decoding" % experiment_name):
for key in transcription_text_dict:
wer = decode_and_evaluate_asr_config(
key,
asr_training_config,
training_job.model_dir,
epoch=best_epoch,
zip_corpus=zip_dict[key],
text=transcription_text_dict[key],
parameter_dict=asr_global_parameter_dict,
training_name=experiment_name)
results[key] = wer
template += "%s: {%s}" % (key, key)
tk.register_report("%s/results" % experiment_name,
results,
template=template)
def main():
# prepare the datasets in bliss and zip format
with tk.block("data_preparation"):
bliss_dict, zip_dict, transcription_text_dict = prepare_data_librispeech(
)
# compute the subword codes and the ASR vocabulary
bpe_codes, bpe_vocab, num_classes = build_subwords(
[bliss_dict['train-clean-100']],
num_segments=10000,
name="librispeech-100")
# extract the mean and the deviation based on the statistics from the baseline training data
mean, stddev = get_asr_dataset_stats(zip_dict['train-clean-100'])
# set the data-derived parameters for all ASR systems
asr_global_parameter_dict = {
'ext_norm_mean': mean,
'ext_norm_std_dev': stddev,
'ext_bpe_file': bpe_codes,
'ext_vocab_file': bpe_vocab,
'ext_num_classes': num_classes
}
# define the parameters for the initial convergence training
initial_checkpoint_training_params = {
'ext_partition_epoch': 20,
'ext_training_zips': [zip_dict['train-clean-100']],
'ext_dev_zips': [zip_dict['dev-clean'], zip_dict['dev-other']],
'ext_num_epochs': 80
}
# this dict contains all models that will be evaluated
models = {}
with tk.block("baseline_training"):
# training of the initial model
initial_checkpoint_training_params.update(asr_global_parameter_dict)
asr_training_config = Path(
"returnn_configs/asr/train-clean-100.exp3.ctc.ogg.lrwarmupextra10.config"
)
initial_training_job = train_asr_config(
asr_training_config, "librispeech-100-initial-training",
initial_checkpoint_training_params)
# training and decoding of the baseline model
baseline_training_params = copy.deepcopy(
initial_checkpoint_training_params)
baseline_training_params['ext_num_epochs'] = 250
baseline_training_params['ext_partition_epoch'] = 5
#baseline_training_params['import_model_train_epoch1'] = initial_training_job.models[80].model
baseline_training_params['load'] = initial_training_job.models[
80].model
baseline_training_params.update(asr_global_parameter_dict)
continued_training_job = train_asr_config(
asr_training_config, "librispeech-100-baseline-training",
baseline_training_params)
from recipe.returnn.search import GetBestEpoch
best_epoch = GetBestEpoch(continued_training_job.model_dir,
continued_training_job.learning_rates,
key="dev_score_output/output_prob").out_var
models['baseline'] = (continued_training_job, best_epoch)
with tk.block("specaug_training"):
asr_specaug_config = Path(
"returnn_configs/asr/train-clean-100.exp3.ctc.ogg.lrwarmupextra10.specaug.config"
)
# training and decoding of the specaug model
continued_training_job = train_asr_config(
asr_specaug_config, "librispeech-100-specaug-training",
baseline_training_params)
from recipe.returnn.search import GetBestEpoch
best_epoch = GetBestEpoch(continued_training_job.model_dir,
continued_training_job.learning_rates,
key="dev_score_output/output_prob").out_var
models['specaug'] = (continued_training_job, best_epoch)
###########################
# TTS
###########################
tts_bliss_dict = {
k: v
for k, v in bliss_dict.items()
if k in ['dev-clean', 'train-clean-100', 'train-clean-360']
}
# this will run the preprocessing and add a "tts-" prefix to the corpus names
tts_bliss_corpora, tts_zip_corpora, char_vocab = prepare_ttf_data(
tts_bliss_dict)
mean, stddev = get_ttf_dataset_stats(
tts_zip_corpora['tts-train-clean-100'])
# set the data-derived parameters for the TTS system
# also add the global epoch and partitioning settings
tts_global_parameter_dict = {
'ext_norm_mean_value': mean,
'ext_norm_std_dev_value': stddev,
'ext_char_vocab': char_vocab,
'ext_training_zips': [tts_zip_corpora['tts-train-clean-100']],
'ext_dev_zips': [tts_zip_corpora['tts-dev-clean']],
'ext_num_epochs': 200,
'ext_partition_epoch': 3,
}
# define the tts training config and run the training
tts_training_config = Path(
"returnn_configs/tts/tts-clean-100.dec640.enc256.enclstm512.config",
hash_overwrite="TTS_DEC640_ENC256_ENCLSTM512_v1")
tts_training_job = train_ttf_config(
tts_training_config,
name="tts-baseline-training",
parameter_dict=tts_global_parameter_dict)
# copy the dataset-derived parameters for the f2l training
f2l_global_parameter_dict = copy.deepcopy(tts_global_parameter_dict)
f2l_global_parameter_dict['ext_num_epochs'] = 100
f2l_global_parameter_dict['ext_partition_epoch'] = 1
f2l_global_parameter_dict.pop('ext_char_vocab')
# define the config for the mel-to-linear feature conversion model and run the training
f2l_training_config = Path(
"returnn_configs/f2l/f2l.2layer.blstm.residual.config",
hash_overwrite="F2L_2LAYER_ENC256_ENCLSTM512_v1")
f2l_training_job = train_ttf_config(
f2l_training_config,
name="f2l-baseline-training",
parameter_dict=f2l_global_parameter_dict)
# generate the speaker embeddings using the GST network of the TTS model for all utterances in the training corpus
embeddings = generate_speaker_embeddings(
config_file=tts_training_config,
model_dir=tts_training_job.model_dir,
epoch=200,
zip_corpus=tts_zip_corpora['tts-train-clean-100'],
name="tts-baseline",
default_parameter_dict=tts_global_parameter_dict)
from recipe.tts.corpus import DistributeSpeakerEmbeddings
# randomly distribute the speaker embeddings for the sentences in the "text-only" data
dist_speaker_embeds_job = DistributeSpeakerEmbeddings(
tts_bliss_dict['train-clean-360'],
embeddings,
use_full_seq_name=False,
options=None)
tk.register_output("embed_dist.hdf", dist_speaker_embeds_job.out)
# the generation will be split in to N successive runs with cleaning in between to reduce the file space consumtion
TTS_GENERATION_SPLITS = 10
segment_job = SegmentCorpus(tts_bliss_corpora['tts-train-clean-360'],
TTS_GENERATION_SPLITS)
segments = segment_job.segment_files
verification_result = None
corpora = []
for i in range(TTS_GENERATION_SPLITS):
# run the tts decoding on the segmented part of the "text-only" data
unstacked_features, decode_job, convert_job = decode_with_speaker_embeddings(
config_file=tts_training_config,
model_dir=tts_training_job.model_dir,
epoch=200,
zip_corpus=tts_zip_corpora['tts-train-clean-360'],
speaker_hdf=dist_speaker_embeds_job.out,
segment_file=segments[i],
name="tts-baseline_decode_%i" % i,
default_parameter_dict=tts_global_parameter_dict)
# we add the verification result of the previous decoding to ensure sequential execution of the jobs
if verification_result:
decode_job.add_input(verification_result)
# run the f2l system to convert the log-mel features to linear spectograms
linear_features, f2l_job = convert_with_f2l(
f2l_training_config,
name="tts-baseline_forward_%i" % i,
features=unstacked_features,
model_dir=f2l_training_job.model_dir,
epoch=100)
# run the Griffin & Lim synthesis and store the ogg files
generated_audio_bliss, gl_job = griffin_lim_ogg(
linear_features, name="tts-baseline_gl_%i" % i)
# verify that all data is complete and no file was corrupted
verification_result = VerifyCorpus(generated_audio_bliss).out
# automatic cleanup of the jobs that are not needed
# remove this for debugging purposes
# WARNING: high HDD consumption
cleanup_success = AutoCleanup([decode_job, convert_job, f2l_job],
verification_result).out
tk.register_output("cleanup_result/cleanup_%i" % i, cleanup_success)
corpora.append(generated_audio_bliss)
# merge the splitted audio corpora back to one corpus and add the original text from Librispeech-360h
merge_job = MergeCorpora(corpora, "synthetic-ls-360", subcorpora=False)
# confirm that the last corpus was correct before running the merge
merge_job.add_input(verification_result)
# add the original text in LibriSpeech format to the synthesized audio
synthetic_audio_corpus = merge_job.merged_corpus
synthetic_corpus = BlissAddTextFromBliss(synthetic_audio_corpus,
bliss_dict['train-clean-360']).out
# pack the corpus into the Zip format for RETURNN ASR training
synthetic_zip_corpus = BlissToZipDataset("synthetic-ls-360",
synthetic_corpus,
use_full_seq_name=False).out
tk.register_output("synthetic_data/synthetic_librispeech_360h.zip",
synthetic_zip_corpus)
# training with synthetic data
with tk.block("baseline_with_synthetic"):
synthetic_training_params = copy.deepcopy(
initial_checkpoint_training_params)
synthetic_training_params['ext_num_epochs'] = 250
synthetic_training_params['load'] = initial_training_job.models[
80].model
synthetic_training_params.update(asr_global_parameter_dict)
synthetic_training_params['ext_training_zips'] = [
zip_dict['train-clean-100'], synthetic_zip_corpus
]
synthetic_training_job = train_asr_config(
asr_training_config, "librispeech-100-synthetic-training",
synthetic_training_params)
from recipe.returnn.search import GetBestEpoch
best_epoch = GetBestEpoch(continued_training_job.model_dir,
continued_training_job.learning_rates,
key="dev_score_output/output_prob").out_var
models['baseline+synthetic'] = (synthetic_training_job, best_epoch)
# training with synthetic data and specaugment
with tk.block("specaug_with_synthetic"):
synthetic_training_params = copy.deepcopy(
initial_checkpoint_training_params)
synthetic_training_params['ext_num_epochs'] = 250
synthetic_training_params['load'] = initial_training_job.models[
80].model
synthetic_training_params.update(asr_global_parameter_dict)
synthetic_training_params['ext_training_zips'] = [
zip_dict['train-clean-100'], synthetic_zip_corpus
]
synthetic_training_job = train_asr_config(
asr_specaug_config, "librispeech-100-specaug-synthetic-training",
synthetic_training_params)
from recipe.returnn.search import GetBestEpoch
best_epoch = GetBestEpoch(continued_training_job.model_dir,
continued_training_job.learning_rates,
key="dev_score_output/output_prob").out_var
models['specaug+synthetic'] = (synthetic_training_job, best_epoch)
############################
# Evaluation
############################
for experiment_name, (training_job, best_epoch) in models.items():
with tk.block("%s_decoding" % experiment_name):
for key in transcription_text_dict:
wer = decode_and_evaluate_asr_config(
key,
asr_training_config,
training_job.model_dir,
epoch=best_epoch,
zip_corpus=zip_dict[key],
text=transcription_text_dict[key],
parameter_dict=asr_global_parameter_dict,
training_name=experiment_name)
tk.register_output(
"results/%s_%s.wer" % (experiment_name, key), wer)