def generate_chain_egs(dir, data, lat_dir, egs_dir, left_context, right_context, run_opts, stage=0, left_tolerance=None, right_tolerance=None, left_context_initial=-1, right_context_final=-1, frame_subsampling_factor=3, alignment_subsampling_factor=3, feat_type='raw', online_ivector_dir=None, frames_per_iter=20000, frames_per_eg_str="20", srand=0, egs_opts=None, cmvn_opts=None, transform_dir=None): """Wrapper for steps/nnet3/chain/get_egs.sh See options in that script. """ common_lib.run_job( """steps/nnet3/chain/get_egs.sh {egs_opts} \ --cmd "{command}" \ --cmvn-opts "{cmvn_opts}" \ --feat-type {feat_type} \ --transform-dir "{transform_dir}" \ --online-ivector-dir "{ivector_dir}" \ --left-context {left_context} \ --right-context {right_context} \ --left-context-initial {left_context_initial} \ --right-context-final {right_context_final} \ --left-tolerance '{left_tolerance}' \ --right-tolerance '{right_tolerance}' \ --frame-subsampling-factor {frame_subsampling_factor} \ --alignment-subsampling-factor {alignment_subsampling_factor} \ --stage {stage} \ --frames-per-iter {frames_per_iter} \ --frames-per-eg {frames_per_eg_str} \ --srand {srand} \ {data} {dir} {lat_dir} {egs_dir}""".format( command=run_opts.command, cmvn_opts=cmvn_opts if cmvn_opts is not None else '', feat_type=feat_type, transform_dir=(transform_dir if transform_dir is not None else ''), ivector_dir=(online_ivector_dir if online_ivector_dir is not None else ''), left_context=left_context, right_context=right_context, left_context_initial=left_context_initial, right_context_final=right_context_final, left_tolerance=(left_tolerance if left_tolerance is not None else ''), right_tolerance=(right_tolerance if right_tolerance is not None else ''), frame_subsampling_factor=frame_subsampling_factor, alignment_subsampling_factor=alignment_subsampling_factor, stage=stage, frames_per_iter=frames_per_iter, frames_per_eg_str=frames_per_eg_str, srand=srand, data=data, lat_dir=lat_dir, dir=dir, egs_dir=egs_dir, egs_opts=egs_opts if egs_opts is not None else ''))
def create_phone_lm(dir, tree_dir, run_opts, lm_opts=None): """Create a phone LM for chain training This method trains a phone LM for chain training using the alignments in "tree_dir" """ try: f = open(tree_dir + "/num_jobs", 'r') num_ali_jobs = int(f.readline()) assert num_ali_jobs > 0 except: raise Exception("""There was an error getting the number of alignment jobs from {0}/num_jobs""".format(tree_dir)) alignments=' '.join(['{0}/ali.{1}.gz'.format(tree_dir, job) for job in range(1, num_ali_jobs + 1)]) common_lib.run_job( """{command} {dir}/log/make_phone_lm.log \ gunzip -c {alignments} \| \ ali-to-phones {tree_dir}/final.mdl ark:- ark:- \| \ chain-est-phone-lm {lm_opts} ark:- {dir}/phone_lm.fst""".format( command=run_opts.command, dir=dir, alignments=alignments, lm_opts=lm_opts if lm_opts is not None else '', tree_dir=tree_dir))
def align(dir, data, lang, run_opts, iter=None, transform_dir=None, online_ivector_dir=None): alidir = '{dir}/ali{ali_suffix}'.format( dir=dir, ali_suffix="_iter_{0}".format(iter) if iter is not None else "") logger.info("Aligning the data{gpu}with {num_jobs} jobs.".format( gpu=" using gpu " if run_opts.realign_use_gpu else " ", num_jobs=run_opts.realign_num_jobs)) common_lib.run_job( """steps/nnet3/align.sh --nj {num_jobs_align} \ --cmd "{align_cmd} {align_queue_opt}" \ --use-gpu {align_use_gpu} \ --transform-dir "{transform_dir}" \ --online-ivector-dir "{online_ivector_dir}" \ --iter "{iter}" {data} {lang} {dir} {alidir}""".format( dir=dir, align_use_gpu=("yes" if run_opts.realign_use_gpu else "no"), align_cmd=run_opts.realign_command, align_queue_opt=run_opts.realign_queue_opt, num_jobs_align=run_opts.realign_num_jobs, transform_dir=(transform_dir if transform_dir is not None else ""), online_ivector_dir=(online_ivector_dir if online_ivector_dir is not None else ""), iter=iter if iter is not None else "", alidir=alidir, lang=lang, data=data)) return alidir
def compute_presoftmax_prior_scale(dir, alidir, num_jobs, run_opts, presoftmax_prior_scale_power=-0.25): # getting the raw pdf count common_lib.run_job( """{command} JOB=1:{num_jobs} {dir}/log/acc_pdf.JOB.log \ ali-to-post "ark:gunzip -c {alidir}/ali.JOB.gz|" ark:- \| \ post-to-tacc --per-pdf=true {alidir}/final.mdl ark:- \ {dir}/pdf_counts.JOB""".format(command=run_opts.command, num_jobs=num_jobs, dir=dir, alidir=alidir)) common_lib.run_job( """{command} {dir}/log/sum_pdf_counts.log \ vector-sum --binary=false {dir}/pdf_counts.* {dir}/pdf_counts \ """.format(command=run_opts.command, dir=dir)) for file in glob.glob('{0}/pdf_counts.*'.format(dir)): os.remove(file) pdf_counts = common_lib.read_kaldi_matrix('{0}/pdf_counts'.format(dir))[0] scaled_counts = smooth_presoftmax_prior_scale_vector( pdf_counts, presoftmax_prior_scale_power=presoftmax_prior_scale_power, smooth=0.01) output_file = "{0}/presoftmax_prior_scale.vec".format(dir) common_lib.write_kaldi_matrix(output_file, [scaled_counts]) common_lib.force_symlink("../presoftmax_prior_scale.vec", "{0}/configs/presoftmax_prior_scale.vec".format( dir))
def compute_progress(dir, iter, egs_dir, left_context, right_context, run_opts, background_process_handler=None, wait=False, get_raw_nnet_from_am=True): if get_raw_nnet_from_am: prev_model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format( dir, iter - 1) model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format(dir, iter) else: prev_model = '{0}/{1}.raw'.format(dir, iter - 1) model = '{0}/{1}.raw'.format(dir, iter) context_opts = "--left-context={lc} --right-context={rc}".format( lc=left_context, rc=right_context) common_lib.run_job("""{command} {dir}/log/progress.{iter}.log \ nnet3-info "{model}" '&&' \ nnet3-show-progress --use-gpu=no "{prev_model}" "{model}" \ "ark,bg:nnet3-copy-egs {context_opts} \ ark:{egs_dir}/train_diagnostic.egs ark:- | \ nnet3-merge-egs --minibatch-size=1:64 ark:- \ ark:- |" """.format(command=run_opts.command, dir=dir, iter=iter, model=model, context_opts=context_opts, prev_model=prev_model, egs_dir=egs_dir), wait=wait, background_process_handler=background_process_handler)
def get_best_nnet_model(dir, iter, best_model_index, run_opts, get_raw_nnet_from_am=True, shrink=None): scale = 1.0 if shrink is not None: scale = shrink best_model = "{dir}/{next_iter}.{best_model_index}.raw".format( dir=dir, next_iter=iter + 1, best_model_index=best_model_index) if get_raw_nnet_from_am: out_model = ("""- \| nnet3-am-copy --set-raw-nnet=- \ {dir}/{iter}.mdl {dir}/{next_iter}.mdl""".format( dir=dir, iter=iter, next_iter=iter + 1)) else: out_model = "{dir}/{next_iter}.raw".format(dir=dir, next_iter=iter + 1) common_lib.run_job("""{command} {dir}/log/select.{iter}.log \ nnet3-copy --scale={scale} {best_model} \ {out_model}""".format(command=run_opts.command, dir=dir, iter=iter, best_model=best_model, out_model=out_model, scale=scale))
def get_best_nnet_model(dir, iter, best_model_index, run_opts, get_raw_nnet_from_am=True, shrink=None): scale = 1.0 if shrink is not None: scale = shrink best_model = "{dir}/{next_iter}.{best_model_index}.raw".format( dir=dir, next_iter=iter + 1, best_model_index=best_model_index) if get_raw_nnet_from_am: out_model = ("""- \| nnet3-am-copy --set-raw-nnet=- \ {dir}/{iter}.mdl {dir}/{next_iter}.mdl""".format( dir=dir, iter=iter, next_iter=iter + 1)) else: out_model = "{dir}/{next_iter}.raw".format(dir=dir, next_iter=iter + 1) common_lib.run_job( """{command} {dir}/log/select.{iter}.log \ nnet3-copy --scale={scale} {best_model} \ {out_model}""".format(command=run_opts.command, dir=dir, iter=iter, best_model=best_model, out_model=out_model, scale=scale))
def realign(dir, iter, feat_dir, lang, prev_egs_dir, cur_egs_dir, prior_subset_size, num_archives, left_context, right_context, run_opts, transform_dir=None, online_ivector_dir=None): raise Exception("Realignment stage has not been implemented in nnet3") logger.info("Getting average posterior for purposes of adjusting " "the priors.") # Note: this just uses CPUs, using a smallish subset of data. # always use the first egs archive, which makes the script simpler; # we're using different random subsets of it. avg_post_vec_file = compute_average_posterior( dir=dir, iter=iter, egs_dir=prev_egs_dir, num_archives=num_archives, prior_subset_size=prior_subset_size, left_context=left_context, right_context=right_context, run_opts=run_opts) avg_post_vec_file = "{dir}/post.{iter}.vec".format(dir=dir, iter=iter) logger.info("Re-adjusting priors based on computed posteriors") model = '{0}/{1}.mdl'.format(dir, iter) adjust_am_priors(dir, model, avg_post_vec_file, model, run_opts) alidir = align(dir, feat_dir, lang, run_opts, iter, transform_dir, online_ivector_dir) common_lib.run_job( """steps/nnet3/relabel_egs.sh --cmd "{command}" --iter {iter} \ {alidir} {prev_egs_dir} {cur_egs_dir}""".format( command=run_opts.command, iter=iter, dir=dir, alidir=alidir, prev_egs_dir=prev_egs_dir, cur_egs_dir=cur_egs_dir))
def compute_progress(dir, iter, egs_dir, left_context, right_context, run_opts, mb_size=256, background_process_handler=None, wait=False, get_raw_nnet_from_am=True): if get_raw_nnet_from_am: prev_model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format( dir, iter - 1) model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format(dir, iter) else: prev_model = '{0}/{1}.raw'.format(dir, iter - 1) model = '{0}/{1}.raw'.format(dir, iter) context_opts = "--left-context={lc} --right-context={rc}".format( lc=left_context, rc=right_context) common_lib.run_job( """{command} {dir}/log/progress.{iter}.log \ nnet3-info "{model}" '&&' \ nnet3-show-progress --use-gpu=no "{prev_model}" "{model}" \ "ark,bg:nnet3-copy-egs {context_opts} \ ark:{egs_dir}/train_diagnostic.egs ark:- | \ nnet3-merge-egs --minibatch-size={mb_size} ark:- \ ark:- |" """.format(command=run_opts.command, dir=dir, iter=iter, model=model, context_opts=context_opts, mb_size=mb_size, prev_model=prev_model, egs_dir=egs_dir), wait=wait, background_process_handler=background_process_handler)
def create_phone_lm(dir, tree_dir, run_opts, lm_opts=None): """Create a phone LM for chain training This method trains a phone LM for chain training using the alignments in "tree_dir" """ try: f = open(tree_dir + "/num_jobs", 'r') num_ali_jobs = int(f.readline()) assert num_ali_jobs > 0 except: raise Exception("""There was an error getting the number of alignment jobs from {0}/num_jobs""".format(tree_dir)) alignments = ' '.join([ '{0}/ali.{1}.gz'.format(tree_dir, job) for job in range(1, num_ali_jobs + 1) ]) common_lib.run_job("""{command} {dir}/log/make_phone_lm.log \ gunzip -c {alignments} \| \ ali-to-phones {tree_dir}/final.mdl ark:- ark:- \| \ chain-est-phone-lm {lm_opts} ark:- {dir}/phone_lm.fst""".format( command=run_opts.command, dir=dir, alignments=alignments, lm_opts=lm_opts if lm_opts is not None else '', tree_dir=tree_dir))
def compute_presoftmax_prior_scale(dir, alidir, num_jobs, run_opts, presoftmax_prior_scale_power=-0.25): # getting the raw pdf count common_lib.run_job( """{command} JOB=1:{num_jobs} {dir}/log/acc_pdf.JOB.log \ ali-to-post "ark:gunzip -c {alidir}/ali.JOB.gz|" ark:- \| \ post-to-tacc --per-pdf=true {alidir}/final.mdl ark:- \ {dir}/pdf_counts.JOB""".format(command=run_opts.command, num_jobs=num_jobs, dir=dir, alidir=alidir)) common_lib.run_job("""{command} {dir}/log/sum_pdf_counts.log \ vector-sum --binary=false {dir}/pdf_counts.* {dir}/pdf_counts \ """.format(command=run_opts.command, dir=dir)) for file in glob.glob('{0}/pdf_counts.*'.format(dir)): os.remove(file) pdf_counts = common_lib.read_kaldi_matrix('{0}/pdf_counts'.format(dir))[0] scaled_counts = smooth_presoftmax_prior_scale_vector( pdf_counts, presoftmax_prior_scale_power=presoftmax_prior_scale_power, smooth=0.01) output_file = "{0}/presoftmax_prior_scale.vec".format(dir) common_lib.write_kaldi_matrix(output_file, [scaled_counts]) common_lib.force_symlink( "../presoftmax_prior_scale.vec", "{0}/configs/presoftmax_prior_scale.vec".format(dir))
def get_average_nnet_model(dir, iter, nnets_list, run_opts, get_raw_nnet_from_am=True, shrink=None): scale = 1.0 if shrink is not None: scale = shrink next_iter = iter + 1 if get_raw_nnet_from_am: out_model = ("""- \| nnet3-am-copy --set-raw-nnet=- --scale={scale} \ {dir}/{iter}.mdl {dir}/{next_iter}.mdl""".format( dir=dir, iter=iter, next_iter=next_iter, scale=scale)) else: if shrink is not None: out_model = """- \| nnet3-copy --scale={scale} \ - {dir}/{next_iter}.raw""".format( dir=dir, next_iter=next_iter, scale=scale) else: out_model = "{dir}/{next_iter}.raw".format(dir=dir, next_iter=next_iter) common_lib.run_job( """{command} {dir}/log/average.{iter}.log \ nnet3-average {nnets_list} \ {out_model}""".format(command=run_opts.command, dir=dir, iter=iter, nnets_list=nnets_list, out_model=out_model))
def prepare_initial_network(dir, run_opts, srand=-3): common_lib.run_job( """{command} {dir}/log/add_first_layer.log \ nnet3-init --srand={srand} {dir}/init.raw \ {dir}/configs/layer1.config {dir}/0.raw""".format( command=run_opts.command, srand=srand, dir=dir))
def align(dir, data, lang, run_opts, iter=None, transform_dir=None, online_ivector_dir=None): alidir = '{dir}/ali{ali_suffix}'.format( dir=dir, ali_suffix="_iter_{0}".format(iter) if iter is not None else "") logger.info("Aligning the data{gpu}with {num_jobs} jobs.".format( gpu=" using gpu " if run_opts.realign_use_gpu else " ", num_jobs=run_opts.realign_num_jobs)) common_lib.run_job("""steps/nnet3/align.sh --nj {num_jobs_align} \ --cmd "{align_cmd} {align_queue_opt}" \ --use-gpu {align_use_gpu} \ --transform-dir "{transform_dir}" \ --online-ivector-dir "{online_ivector_dir}" \ --iter "{iter}" {data} {lang} {dir} {alidir}""".format( dir=dir, align_use_gpu=("yes" if run_opts.realign_use_gpu else "no"), align_cmd=run_opts.realign_command, align_queue_opt=run_opts.realign_queue_opt, num_jobs_align=run_opts.realign_num_jobs, transform_dir=(transform_dir if transform_dir is not None else ""), online_ivector_dir=(online_ivector_dir if online_ivector_dir is not None else ""), iter=iter if iter is not None else "", alidir=alidir, lang=lang, data=data)) return alidir
def get_average_nnet_model(dir, iter, nnets_list, run_opts, get_raw_nnet_from_am=True, shrink=None): scale = 1.0 if shrink is not None: scale = shrink next_iter = iter + 1 if get_raw_nnet_from_am: out_model = ("""- \| nnet3-am-copy --set-raw-nnet=- --scale={scale} \ {dir}/{iter}.mdl {dir}/{next_iter}.mdl""".format( dir=dir, iter=iter, next_iter=next_iter, scale=scale)) else: if shrink is not None: out_model = """- \| nnet3-copy --scale={scale} \ - {dir}/{next_iter}.raw""".format( dir=dir, next_iter=next_iter, scale=scale) else: out_model = "{dir}/{next_iter}.raw".format(dir=dir, next_iter=next_iter) common_lib.run_job("""{command} {dir}/log/average.{iter}.log \ nnet3-average {nnets_list} \ {out_model}""".format(command=run_opts.command, dir=dir, iter=iter, nnets_list=nnets_list, out_model=out_model))
def create_denominator_fst(dir, tree_dir, run_opts): common_lib.run_job("""copy-transition-model {tree_dir}/final.mdl \ {dir}/0.trans_mdl""".format(dir=dir, tree_dir=tree_dir)) common_lib.run_job("""{command} {dir}/log/make_den_fst.log \ chain-make-den-fst {dir}/tree {dir}/0.trans_mdl \ {dir}/phone_lm.fst \ {dir}/den.fst {dir}/normalization.fst""".format( dir=dir, command=run_opts.command))
def compute_average_posterior(dir, iter, egs_dir, num_archives, prior_subset_size, left_context, right_context, run_opts, get_raw_nnet_from_am=True): """ Computes the average posterior of the network Note: this just uses CPUs, using a smallish subset of data. """ for file in glob.glob('{0}/post.{1}.*.vec'.format(dir, iter)): os.remove(file) if run_opts.num_jobs_compute_prior > num_archives: egs_part = 1 else: egs_part = 'JOB' if get_raw_nnet_from_am: model = "nnet3-am-copy --raw=true {0}/combined.mdl -|".format(dir) else: model = "{dir}/final.raw".format(dir=dir) context_opts = "--left-context={lc} --right-context={rc}".format( lc=left_context, rc=right_context) common_lib.run_job( """{command} JOB=1:{num_jobs_compute_prior} {prior_queue_opt} \ {dir}/log/get_post.{iter}.JOB.log \ nnet3-copy-egs {context_opts} \ ark:{egs_dir}/egs.{egs_part}.ark ark:- \| \ nnet3-subset-egs --srand=JOB --n={prior_subset_size} \ ark:- ark:- \| \ nnet3-merge-egs --measure-output-frames=true \ --minibatch-size=128 ark:- ark:- \| \ nnet3-compute-from-egs {prior_gpu_opt} --apply-exp=true \ "{model}" ark:- ark:- \| \ matrix-sum-rows ark:- ark:- \| vector-sum ark:- \ {dir}/post.{iter}.JOB.vec""".format( command=run_opts.command, dir=dir, model=model, num_jobs_compute_prior=run_opts.num_jobs_compute_prior, prior_queue_opt=run_opts.prior_queue_opt, iter=iter, prior_subset_size=prior_subset_size, egs_dir=egs_dir, egs_part=egs_part, context_opts=context_opts, prior_gpu_opt=run_opts.prior_gpu_opt)) # make sure there is time for $dir/post.{iter}.*.vec to appear. time.sleep(5) avg_post_vec_file = "{dir}/post.{iter}.vec".format(dir=dir, iter=iter) common_lib.run_job( """{command} {dir}/log/vector_sum.{iter}.log \ vector-sum {dir}/post.{iter}.*.vec {output_file} """.format(command=run_opts.command, dir=dir, iter=iter, output_file=avg_post_vec_file)) for file in glob.glob('{0}/post.{1}.*.vec'.format(dir, iter)): os.remove(file) return avg_post_vec_file
def generate_egs(data, alidir, egs_dir, left_context, right_context, valid_left_context, valid_right_context, run_opts, stage=0, feat_type='raw', online_ivector_dir=None, samples_per_iter=20000, frames_per_eg=20, srand=0, egs_opts=None, cmvn_opts=None, transform_dir=None): """ Wrapper for calling steps/nnet3/get_egs.sh Generates targets from alignment directory 'alidir', which contains the model final.mdl and alignments. """ common_lib.run_job("""steps/nnet3/get_egs.sh {egs_opts} \ --cmd "{command}" \ --cmvn-opts "{cmvn_opts}" \ --feat-type {feat_type} \ --transform-dir "{transform_dir}" \ --online-ivector-dir "{ivector_dir}" \ --left-context {left_context} --right-context {right_context} \ --valid-left-context {valid_left_context} \ --valid-right-context {valid_right_context} \ --stage {stage} \ --samples-per-iter {samples_per_iter} \ --frames-per-eg {frames_per_eg} \ --srand {srand} \ {data} {alidir} {egs_dir} """.format( command=run_opts.command, cmvn_opts=cmvn_opts if cmvn_opts is not None else '', feat_type=feat_type, transform_dir=(transform_dir if transform_dir is not None else ''), ivector_dir=(online_ivector_dir if online_ivector_dir is not None else ''), left_context=left_context, right_context=right_context, valid_left_context=valid_left_context, valid_right_context=valid_right_context, stage=stage, samples_per_iter=samples_per_iter, frames_per_eg=frames_per_eg, srand=srand, data=data, alidir=alidir, egs_dir=egs_dir, egs_opts=egs_opts if egs_opts is not None else ''))
def combine_models(dir, num_iters, models_to_combine, num_chunk_per_minibatch, egs_dir, left_context, right_context, leaky_hmm_coefficient, l2_regularize, xent_regularize, run_opts, background_process_handler=None): """ Function to do model combination In the nnet3 setup, the logic for doing averaging of subsets of the models in the case where there are too many models to reliably esetimate interpolation factors (max_models_combine) is moved into the nnet3-combine. """ raw_model_strings = [] logger.info("Combining {0} models.".format(models_to_combine)) models_to_combine.add(num_iters) for iter in sorted(models_to_combine): model_file = '{0}/{1}.mdl'.format(dir, iter) if os.path.exists(model_file): raw_model_strings.append( '"nnet3-am-copy --raw=true {0} -|"'.format(model_file)) else: print("{0}: warning: model file {1} does not exist " "(final combination)".format(sys.argv[0], model_file)) common_lib.run_job( """{command} {combine_queue_opt} {dir}/log/combine.log \ nnet3-chain-combine --num-iters=40 \ --l2-regularize={l2} --leaky-hmm-coefficient={leaky} \ --enforce-sum-to-one=true --enforce-positive-weights=true \ --verbose=3 {dir}/den.fst {raw_models} \ "ark,bg:nnet3-chain-copy-egs --left-context={lc} \ --right-context={rc} ark:{egs_dir}/combine.cegs ark:- | \ nnet3-chain-merge-egs --minibatch-size={num_chunk_per_mb} \ ark:- ark:- |" - \| \ nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl \ {dir}/final.mdl""".format( command=run_opts.command, combine_queue_opt=run_opts.combine_queue_opt, lc=left_context, rc=right_context, l2=l2_regularize, leaky=leaky_hmm_coefficient, dir=dir, raw_models=" ".join(raw_model_strings), num_chunk_per_mb=num_chunk_per_minibatch, num_iters=num_iters, egs_dir=egs_dir)) # Compute the probability of the final, combined model with # the same subset we used for the previous compute_probs, as the # different subsets will lead to different probs. compute_train_cv_probabilities( dir=dir, iter='final', egs_dir=egs_dir, left_context=left_context, right_context=right_context, l2_regularize=l2_regularize, xent_regularize=xent_regularize, leaky_hmm_coefficient=leaky_hmm_coefficient, run_opts=run_opts, wait=False, background_process_handler=background_process_handler)
def adjust_am_priors(dir, input_model, avg_posterior_vector, output_model, run_opts): common_lib.run_job( """{command} {dir}/log/adjust_priors.final.log \ nnet3-am-adjust-priors "{input_model}" {avg_posterior_vector} \ "{output_model}" """.format( command=run_opts.command, dir=dir, input_model=input_model, avg_posterior_vector=avg_posterior_vector, output_model=output_model))
def create_denominator_fst(dir, tree_dir, run_opts): common_lib.run_job( """copy-transition-model {tree_dir}/final.mdl \ {dir}/0.trans_mdl""".format(dir=dir, tree_dir=tree_dir)) common_lib.run_job( """{command} {dir}/log/make_den_fst.log \ chain-make-den-fst {dir}/tree {dir}/0.trans_mdl \ {dir}/phone_lm.fst \ {dir}/den.fst {dir}/normalization.fst""".format( dir=dir, command=run_opts.command))
def adjust_am_priors(dir, input_model, avg_posterior_vector, output_model, run_opts): common_lib.run_job("""{command} {dir}/log/adjust_priors.final.log \ nnet3-am-adjust-priors "{input_model}" {avg_posterior_vector} \ "{output_model}" """.format( command=run_opts.command, dir=dir, input_model=input_model, avg_posterior_vector=avg_posterior_vector, output_model=output_model))
def compute_train_cv_probabilities(dir, iter, egs_dir, left_context, right_context, l2_regularize, xent_regularize, leaky_hmm_coefficient, run_opts, wait=False, background_process_handler=None): model = '{0}/{1}.mdl'.format(dir, iter) common_lib.run_job("""{command} {dir}/log/compute_prob_valid.{iter}.log \ nnet3-chain-compute-prob --l2-regularize={l2} \ --leaky-hmm-coefficient={leaky} --xent-regularize={xent_reg} \ "nnet3-am-copy --raw=true {model} - |" {dir}/den.fst \ "ark,bg:nnet3-chain-copy-egs --left-context={lc} \ --right-context={rc} ark:{egs_dir}/valid_diagnostic.cegs \ ark:- | nnet3-chain-merge-egs --minibatch-size=1:64 ark:- ark:- |" \ """.format(command=run_opts.command, dir=dir, iter=iter, model=model, lc=left_context, rc=right_context, l2=l2_regularize, leaky=leaky_hmm_coefficient, xent_reg=xent_regularize, egs_dir=egs_dir), wait=wait, background_process_handler=background_process_handler) common_lib.run_job("""{command} {dir}/log/compute_prob_train.{iter}.log \ nnet3-chain-compute-prob --l2-regularize={l2} \ --leaky-hmm-coefficient={leaky} --xent-regularize={xent_reg} \ "nnet3-am-copy --raw=true {model} - |" {dir}/den.fst \ "ark,bg:nnet3-chain-copy-egs --left-context={lc} \ --right-context={rc} ark:{egs_dir}/train_diagnostic.cegs \ ark:- | nnet3-chain-merge-egs --minibatch-size=1:64 ark:- ark:- |" \ """.format(command=run_opts.command, dir=dir, iter=iter, model=model, lc=left_context, rc=right_context, l2=l2_regularize, leaky=leaky_hmm_coefficient, xent_reg=xent_regularize, egs_dir=egs_dir), wait=wait, background_process_handler=background_process_handler)
def prepare_initial_acoustic_model(dir, alidir, run_opts, srand=-3): """ Adds the first layer; this will also add in the lda.mat and presoftmax_prior_scale.vec. It will also prepare the acoustic model with the transition model.""" common_train_lib.prepare_initial_network(dir, run_opts, srand=srand) # Convert to .mdl, train the transitions, set the priors. common_lib.run_job("""{command} {dir}/log/init_mdl.log \ nnet3-am-init {alidir}/final.mdl {dir}/0.raw - \| \ nnet3-am-train-transitions - \ "ark:gunzip -c {alidir}/ali.*.gz|" {dir}/0.mdl """.format(command=run_opts.command, dir=dir, alidir=alidir))
def compute_progress(dir, iter, egs_dir, left_context, right_context, run_opts, background_process_handler=None, wait=False, get_raw_nnet_from_am=True, use_multitask_egs=False): if get_raw_nnet_from_am: prev_model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format( dir, iter - 1) model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format(dir, iter) else: prev_model = '{0}/{1}.raw'.format(dir, iter - 1) model = '{0}/{1}.raw'.format(dir, iter) context_opts = "--left-context={lc} --right-context={rc}".format( lc=left_context, rc=right_context) scp_or_ark = "scp" if use_multitask_egs else "ark" egs_suffix = ".scp" if use_multitask_egs else ".egs" egs_rspecifier = "{0}:{1}/train_diagnostic{2}".format( scp_or_ark, egs_dir, egs_suffix) multitask_egs_opts = common_train_lib.get_multitask_egs_opts( egs_dir, egs_prefix="train_diagnostic.", use_multitask_egs=use_multitask_egs) common_lib.run_job("""{command} {dir}/log/progress.{iter}.log \ nnet3-info "{model}" '&&' \ nnet3-show-progress --use-gpu=no "{prev_model}" "{model}" \ "ark,bg:nnet3-copy-egs {context_opts} {multitask_egs_opts} \ {egs_rspecifier} ark:- | \ nnet3-merge-egs --minibatch-size=1:64 ark:- \ ark:- |" """.format( command=run_opts.command, dir=dir, iter=iter, egs_rspecifier=egs_rspecifier, model=model, context_opts=context_opts, prev_model=prev_model, egs_dir=egs_dir, multitask_egs_opts=multitask_egs_opts), wait=wait, background_process_handler=background_process_handler)
def create_phone_lm(dir, tree_dir, run_opts, lm_opts=None): """Create a phone LM for chain training This method trains a phone LM for chain training using the alignments in "tree_dir" """ common_lib.run_job( """{command} {dir}/log/make_phone_lm.log \ chain-est-phone-lm {lm_opts} \ "ark:gunzip -c {tree_dir}/ali.*.gz | \ ali-to-phones {tree_dir}/final.mdl ark:- ark:- |" \ {dir}/phone_lm.fst""".format( command=run_opts.command, dir=dir, lm_opts=lm_opts if lm_opts is not None else '', tree_dir=tree_dir))
def compute_train_cv_probabilities(dir, iter, egs_dir, left_context, right_context, run_opts, mb_size=256, wait=False, background_process_handler=None, get_raw_nnet_from_am=True): if get_raw_nnet_from_am: model = "nnet3-am-copy --raw=true {dir}/{iter}.mdl - |".format( dir=dir, iter=iter) else: model = "{dir}/{iter}.raw".format(dir=dir, iter=iter) context_opts = "--left-context={lc} --right-context={rc}".format( lc=left_context, rc=right_context) common_lib.run_job(""" {command} {dir}/log/compute_prob_valid.{iter}.log \ nnet3-compute-prob "{model}" \ "ark,bg:nnet3-copy-egs {context_opts} \ ark:{egs_dir}/valid_diagnostic.egs ark:- | \ nnet3-merge-egs --minibatch-size={mb_size} ark:- \ ark:- |" """.format(command=run_opts.command, dir=dir, iter=iter, context_opts=context_opts, mb_size=mb_size, model=model, egs_dir=egs_dir), wait=wait, background_process_handler=background_process_handler) common_lib.run_job("""{command} {dir}/log/compute_prob_train.{iter}.log \ nnet3-compute-prob "{model}" \ "ark,bg:nnet3-copy-egs {context_opts} \ ark:{egs_dir}/train_diagnostic.egs ark:- | \ nnet3-merge-egs --minibatch-size={mb_size} ark:- \ ark:- |" """.format(command=run_opts.command, dir=dir, iter=iter, context_opts=context_opts, mb_size=mb_size, model=model, egs_dir=egs_dir), wait=wait, background_process_handler=background_process_handler)
def prepare_initial_acoustic_model(dir, run_opts, srand=-1): """ Adds the first layer; this will also add in the lda.mat and presoftmax_prior_scale.vec. It will also prepare the acoustic model with the transition model.""" common_train_lib.prepare_initial_network(dir, run_opts, srand=srand) # The model-format for a 'chain' acoustic model is just the transition # model and then the raw nnet, so we can use 'cat' to create this, as # long as they have the same mode (binary or not binary). # We ensure that they have the same mode (even if someone changed the # script to make one or both of them text mode) by copying them both # before concatenating them. common_lib.run_job("""{command} {dir}/log/init_mdl.log \ nnet3-am-init {dir}/0.trans_mdl {dir}/0.raw \ {dir}/0.mdl""".format(command=run_opts.command, dir=dir))
def prepare_initial_acoustic_model(dir, alidir, run_opts, srand=-3): """ Adds the first layer; this will also add in the lda.mat and presoftmax_prior_scale.vec. It will also prepare the acoustic model with the transition model.""" common_train_lib.prepare_initial_network(dir, run_opts, srand=srand) # Convert to .mdl, train the transitions, set the priors. common_lib.run_job( """{command} {dir}/log/init_mdl.log \ nnet3-am-init {alidir}/final.mdl {dir}/0.raw - \| \ nnet3-am-train-transitions - \ "ark:gunzip -c {alidir}/ali.*.gz|" {dir}/0.mdl """.format( command=run_opts.command, dir=dir, alidir=alidir ) )
def generate_egs(data, alidir, egs_dir, left_context, right_context, valid_left_context, valid_right_context, run_opts, stage=0, feat_type='raw', online_ivector_dir=None, samples_per_iter=20000, frames_per_eg=20, srand=0, egs_opts=None, cmvn_opts=None, transform_dir=None): """ Wrapper for calling steps/nnet3/get_egs.sh Generates targets from alignment directory 'alidir', which contains the model final.mdl and alignments. """ common_lib.run_job( """steps/nnet3/get_egs.sh {egs_opts} \ --cmd "{command}" \ --cmvn-opts "{cmvn_opts}" \ --feat-type {feat_type} \ --transform-dir "{transform_dir}" \ --online-ivector-dir "{ivector_dir}" \ --left-context {left_context} --right-context {right_context} \ --valid-left-context {valid_left_context} \ --valid-right-context {valid_right_context} \ --stage {stage} \ --samples-per-iter {samples_per_iter} \ --frames-per-eg {frames_per_eg} \ --srand {srand} \ {data} {alidir} {egs_dir} """.format(command=run_opts.command, cmvn_opts=cmvn_opts if cmvn_opts is not None else '', feat_type=feat_type, transform_dir=(transform_dir if transform_dir is not None else ''), ivector_dir=(online_ivector_dir if online_ivector_dir is not None else ''), left_context=left_context, right_context=right_context, valid_left_context=valid_left_context, valid_right_context=valid_right_context, stage=stage, samples_per_iter=samples_per_iter, frames_per_eg=frames_per_eg, srand=srand, data=data, alidir=alidir, egs_dir=egs_dir, egs_opts=egs_opts if egs_opts is not None else ''))
def compute_progress(dir, iter, run_opts, wait=False, background_process_handler=None): prev_model = '{0}/{1}.mdl'.format(dir, iter - 1) model = '{0}/{1}.mdl'.format(dir, iter) common_lib.run_job( """{command} {dir}/log/progress.{iter}.log \ nnet3-am-info {model} '&&' \ nnet3-show-progress --use-gpu=no \ "nnet3-am-copy --raw=true {prev_model} - |" \ "nnet3-am-copy --raw=true {model} - |" """.format(command=run_opts.command, dir=dir, iter=iter, model=model, prev_model=prev_model), wait=wait, background_process_handler=background_process_handler)
def prepare_initial_acoustic_model(dir, run_opts, srand=-1): """ Adds the first layer; this will also add in the lda.mat and presoftmax_prior_scale.vec. It will also prepare the acoustic model with the transition model.""" common_train_lib.prepare_initial_network(dir, run_opts, srand=srand) # The model-format for a 'chain' acoustic model is just the transition # model and then the raw nnet, so we can use 'cat' to create this, as # long as they have the same mode (binary or not binary). # We ensure that they have the same mode (even if someone changed the # script to make one or both of them text mode) by copying them both # before concatenating them. common_lib.run_job( """{command} {dir}/log/init_mdl.log \ nnet3-am-init {dir}/0.trans_mdl {dir}/0.raw \ {dir}/0.mdl""".format(command=run_opts.command, dir=dir))
def compute_train_cv_probabilities(dir, iter, egs_dir, left_context, right_context, run_opts, mb_size=256, wait=False, background_process_handler=None, get_raw_nnet_from_am=True): if get_raw_nnet_from_am: model = "nnet3-am-copy --raw=true {dir}/{iter}.mdl - |".format( dir=dir, iter=iter) else: model = "{dir}/{iter}.raw".format(dir=dir, iter=iter) context_opts = "--left-context={lc} --right-context={rc}".format( lc=left_context, rc=right_context) common_lib.run_job( """ {command} {dir}/log/compute_prob_valid.{iter}.log \ nnet3-compute-prob "{model}" \ "ark,bg:nnet3-copy-egs {context_opts} \ ark:{egs_dir}/valid_diagnostic.egs ark:- | \ nnet3-merge-egs --minibatch-size={mb_size} ark:- \ ark:- |" """.format(command=run_opts.command, dir=dir, iter=iter, context_opts=context_opts, mb_size=mb_size, model=model, egs_dir=egs_dir), wait=wait, background_process_handler=background_process_handler) common_lib.run_job( """{command} {dir}/log/compute_prob_train.{iter}.log \ nnet3-compute-prob "{model}" \ "ark,bg:nnet3-copy-egs {context_opts} \ ark:{egs_dir}/train_diagnostic.egs ark:- | \ nnet3-merge-egs --minibatch-size={mb_size} ark:- \ ark:- |" """.format(command=run_opts.command, dir=dir, iter=iter, context_opts=context_opts, mb_size=mb_size, model=model, egs_dir=egs_dir), wait=wait, background_process_handler=background_process_handler)
def compute_train_cv_probabilities(dir, iter, egs_dir, left_context, right_context, l2_regularize, xent_regularize, leaky_hmm_coefficient, run_opts, wait=False, background_process_handler=None): model = '{0}/{1}.mdl'.format(dir, iter) common_lib.run_job( """{command} {dir}/log/compute_prob_valid.{iter}.log \ nnet3-chain-compute-prob --l2-regularize={l2} \ --leaky-hmm-coefficient={leaky} --xent-regularize={xent_reg} \ "nnet3-am-copy --raw=true {model} - |" {dir}/den.fst \ "ark,bg:nnet3-chain-copy-egs --left-context={lc} \ --right-context={rc} ark:{egs_dir}/valid_diagnostic.cegs \ ark:- | nnet3-chain-merge-egs ark:- ark:- |" \ """.format(command=run_opts.command, dir=dir, iter=iter, model=model, lc=left_context, rc=right_context, l2=l2_regularize, leaky=leaky_hmm_coefficient, xent_reg=xent_regularize, egs_dir=egs_dir), wait=wait, background_process_handler=background_process_handler) common_lib.run_job( """{command} {dir}/log/compute_prob_train.{iter}.log \ nnet3-chain-compute-prob --l2-regularize={l2} \ --leaky-hmm-coefficient={leaky} --xent-regularize={xent_reg} \ "nnet3-am-copy --raw=true {model} - |" {dir}/den.fst \ "ark,bg:nnet3-chain-copy-egs --left-context={lc} \ --right-context={rc} ark:{egs_dir}/train_diagnostic.cegs \ ark:- | nnet3-chain-merge-egs ark:- ark:- |" \ """.format(command=run_opts.command, dir=dir, iter=iter, model=model, lc=left_context, rc=right_context, l2=l2_regularize, leaky=leaky_hmm_coefficient, xent_reg=xent_regularize, egs_dir=egs_dir), wait=wait, background_process_handler=background_process_handler)
def compute_preconditioning_matrix(dir, egs_dir, num_lda_jobs, run_opts, max_lda_jobs=None, rand_prune=4.0, lda_opts=None): """ Function to estimate and write LDA matrix from cegs This function is exactly similar to the version in module libs.nnet3.train.frame_level_objf.common except this uses cegs instead of egs files. """ if max_lda_jobs is not None: if num_lda_jobs > max_lda_jobs: num_lda_jobs = max_lda_jobs # Write stats with the same format as stats for LDA. common_lib.run_job( """{command} JOB=1:{num_lda_jobs} {dir}/log/get_lda_stats.JOB.log \ nnet3-chain-acc-lda-stats --rand-prune={rand_prune} \ {dir}/init.raw "ark:{egs_dir}/cegs.JOB.ark" \ {dir}/JOB.lda_stats""".format(command=run_opts.command, num_lda_jobs=num_lda_jobs, dir=dir, egs_dir=egs_dir, rand_prune=rand_prune)) # the above command would have generated dir/{1..num_lda_jobs}.lda_stats lda_stat_files = map(lambda x: '{0}/{1}.lda_stats'.format(dir, x), range(1, num_lda_jobs + 1)) common_lib.run_job("""{command} {dir}/log/sum_transform_stats.log \ sum-lda-accs {dir}/lda_stats {lda_stat_files}""".format( command=run_opts.command, dir=dir, lda_stat_files=" ".join(lda_stat_files))) for file in lda_stat_files: try: os.remove(file) except OSError: raise Exception("There was error while trying to remove " "lda stat files.") # this computes a fixed affine transform computed in the way we described # in Appendix C.6 of http://arxiv.org/pdf/1410.7455v6.pdf; it's a scaled # variant of an LDA transform but without dimensionality reduction. common_lib.run_job("""{command} {dir}/log/get_transform.log \ nnet-get-feature-transform {lda_opts} {dir}/lda.mat \ {dir}/lda_stats""".format( command=run_opts.command, dir=dir, lda_opts=lda_opts if lda_opts is not None else "")) common_lib.force_symlink("../lda.mat", "{0}/configs/lda.mat".format(dir))
def compute_preconditioning_matrix(dir, egs_dir, num_lda_jobs, run_opts, max_lda_jobs=None, rand_prune=4.0, lda_opts=None): """ Function to estimate and write LDA matrix from cegs This function is exactly similar to the version in module libs.nnet3.train.frame_level_objf.common except this uses cegs instead of egs files. """ if max_lda_jobs is not None: if num_lda_jobs > max_lda_jobs: num_lda_jobs = max_lda_jobs # Write stats with the same format as stats for LDA. common_lib.run_job( """{command} JOB=1:{num_lda_jobs} {dir}/log/get_lda_stats.JOB.log \ nnet3-chain-acc-lda-stats --rand-prune={rand_prune} \ {dir}/init.raw "ark:{egs_dir}/cegs.JOB.ark" \ {dir}/JOB.lda_stats""".format( command=run_opts.command, num_lda_jobs=num_lda_jobs, dir=dir, egs_dir=egs_dir, rand_prune=rand_prune)) # the above command would have generated dir/{1..num_lda_jobs}.lda_stats lda_stat_files = map(lambda x: '{0}/{1}.lda_stats'.format(dir, x), range(1, num_lda_jobs + 1)) common_lib.run_job( """{command} {dir}/log/sum_transform_stats.log \ sum-lda-accs {dir}/lda_stats {lda_stat_files}""".format( command=run_opts.command, dir=dir, lda_stat_files=" ".join(lda_stat_files))) for file in lda_stat_files: try: os.remove(file) except OSError: raise Exception("There was error while trying to remove " "lda stat files.") # this computes a fixed affine transform computed in the way we described # in Appendix C.6 of http://arxiv.org/pdf/1410.7455v6.pdf; it's a scaled # variant of an LDA transform but without dimensionality reduction. common_lib.run_job( """{command} {dir}/log/get_transform.log \ nnet-get-feature-transform {lda_opts} {dir}/lda.mat \ {dir}/lda_stats""".format( command=run_opts.command, dir=dir, lda_opts=lda_opts if lda_opts is not None else "")) common_lib.force_symlink("../lda.mat", "{0}/configs/lda.mat".format(dir))
def combine_models(dir, num_iters, models_to_combine, egs_dir, left_context, right_context, minibatch_size_str, run_opts, background_process_handler=None, chunk_width=None, get_raw_nnet_from_am=True, sum_to_one_penalty=0.0, use_multitask_egs=False): """ Function to do model combination In the nnet3 setup, the logic for doing averaging of subsets of the models in the case where there are too many models to reliably esetimate interpolation factors (max_models_combine) is moved into the nnet3-combine. """ raw_model_strings = [] logger.info("Combining {0} models.".format(models_to_combine)) models_to_combine.add(num_iters) for iter in sorted(models_to_combine): if get_raw_nnet_from_am: model_file = '{0}/{1}.mdl'.format(dir, iter) if not os.path.exists(model_file): raise Exception('Model file {0} missing'.format(model_file)) raw_model_strings.append( '"nnet3-am-copy --raw=true {0} -|"'.format(model_file)) else: model_file = '{0}/{1}.raw'.format(dir, iter) if not os.path.exists(model_file): raise Exception('Model file {0} missing'.format(model_file)) raw_model_strings.append(model_file) if get_raw_nnet_from_am: out_model = ("| nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl " "{dir}/combined.mdl".format(dir=dir, num_iters=num_iters)) else: out_model = '{dir}/final.raw'.format(dir=dir) context_opts = "--left-context={lc} --right-context={rc}".format( lc=left_context, rc=right_context) scp_or_ark = "scp" if use_multitask_egs else "ark" egs_suffix = ".scp" if use_multitask_egs else ".egs" egs_rspecifier = "{0}:{1}/combine{2}".format(scp_or_ark, egs_dir, egs_suffix) multitask_egs_opts = common_train_lib.get_multitask_egs_opts( egs_dir, egs_prefix="combine.", use_multitask_egs=use_multitask_egs) common_lib.run_job("""{command} {combine_queue_opt} {dir}/log/combine.log \ nnet3-combine --num-iters=80 \ --enforce-sum-to-one={hard_enforce} \ --sum-to-one-penalty={penalty} \ --enforce-positive-weights=true \ --verbose=3 {raw_models} \ "ark,bg:nnet3-copy-egs {context_opts} {multitask_egs_opts} \ {egs_rspecifier} ark:- | \ nnet3-merge-egs --minibatch-size={mbsize} ark:- ark:- |" \ "{out_model}" """.format(command=run_opts.command, combine_queue_opt=run_opts.combine_queue_opt, dir=dir, raw_models=" ".join(raw_model_strings), egs_rspecifier=egs_rspecifier, hard_enforce=(sum_to_one_penalty <= 0), penalty=sum_to_one_penalty, context_opts=context_opts, mbsize=minibatch_size_str, out_model=out_model, egs_dir=egs_dir, multitask_egs_opts=multitask_egs_opts)) # Compute the probability of the final, combined model with # the same subset we used for the previous compute_probs, as the # different subsets will lead to different probs. if get_raw_nnet_from_am: compute_train_cv_probabilities( dir=dir, iter='combined', egs_dir=egs_dir, left_context=left_context, right_context=right_context, run_opts=run_opts, wait=False, background_process_handler=background_process_handler, use_multitask_egs=use_multitask_egs) else: compute_train_cv_probabilities( dir=dir, iter='final', egs_dir=egs_dir, left_context=left_context, right_context=right_context, run_opts=run_opts, wait=False, background_process_handler=background_process_handler, get_raw_nnet_from_am=False, use_multitask_egs=use_multitask_egs)
def generate_egs_using_targets(data, targets_scp, egs_dir, left_context, right_context, run_opts, stage=0, left_context_initial=-1, right_context_final=-1, feat_type='raw', online_ivector_dir=None, target_type='dense', num_targets=-1, samples_per_iter=20000, frames_per_eg_str="20", srand=0, egs_opts=None, cmvn_opts=None, transform_dir=None): """ Wrapper for calling steps/nnet3/get_egs_targets.sh This method generates egs directly from an scp file of targets, instead of getting them from the alignments (as with the method generate_egs() in module nnet3.train.frame_level_objf.acoustic_model). Args: target_type: "dense" if the targets are in matrix format "sparse" if the targets are in posterior format num_targets: must be explicitly specified for "sparse" targets. For "dense" targets, this option is ignored and the target dim is computed from the target matrix dimension For other options, see the file steps/nnet3/get_egs_targets.sh """ if target_type == 'dense': num_targets = common_lib.get_feat_dim_from_scp(targets_scp) else: if num_targets == -1: raise Exception("--num-targets is required if " "target-type is sparse") common_lib.run_job( """steps/nnet3/get_egs_targets.sh {egs_opts} \ --cmd "{command}" \ --cmvn-opts "{cmvn_opts}" \ --feat-type {feat_type} \ --transform-dir "{transform_dir}" \ --online-ivector-dir "{ivector_dir}" \ --left-context {left_context} \ --right-context {right_context} \ --left-context-initial {left_context_initial} \ --right-context-final {right_context_final} \ --stage {stage} \ --samples-per-iter {samples_per_iter} \ --frames-per-eg {frames_per_eg_str} \ --srand {srand} \ --target-type {target_type} \ --num-targets {num_targets} \ {data} {targets_scp} {egs_dir} """.format(command=run_opts.egs_command, cmvn_opts=cmvn_opts if cmvn_opts is not None else '', feat_type=feat_type, transform_dir=(transform_dir if transform_dir is not None else ''), ivector_dir=(online_ivector_dir if online_ivector_dir is not None else ''), left_context=left_context, right_context=right_context, left_context_initial=left_context_initial, right_context_final=right_context_final, stage=stage, samples_per_iter=samples_per_iter, frames_per_eg_str=frames_per_eg_str, srand=srand, num_targets=num_targets, data=data, targets_scp=targets_scp, target_type=target_type, egs_dir=egs_dir, egs_opts=egs_opts if egs_opts is not None else ''))
def train(args, run_opts, background_process_handler): """ The main function for training. Args: args: a Namespace object with the required parameters obtained from the function process_args() run_opts: RunOpts object obtained from the process_args() """ arg_string = pprint.pformat(vars(args)) logger.info("Arguments for the experiment\n{0}".format(arg_string)) # Set some variables. num_jobs = common_lib.get_number_of_jobs(args.ali_dir) feat_dim = common_lib.get_feat_dim(args.feat_dir) ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir) # split the training data into parts for individual jobs # we will use the same number of jobs as that used for alignment common_lib.split_data(args.feat_dir, num_jobs) shutil.copy('{0}/tree'.format(args.ali_dir), args.dir) with open('{0}/num_jobs'.format(args.dir), 'w') as f: f.write(str(num_jobs)) config_dir = '{0}/configs'.format(args.dir) var_file = '{0}/vars'.format(config_dir) variables = common_train_lib.parse_generic_config_vars_file(var_file) # Set some variables. try: model_left_context = variables['model_left_context'] model_right_context = variables['model_right_context'] # this is really the number of times we add layers to the network for # discriminative pretraining num_hidden_layers = variables['num_hidden_layers'] except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined in " "{1}".format(str(e), '{0}/configs'.format(args.dir))) left_context = args.chunk_left_context + model_left_context right_context = args.chunk_right_context + model_right_context # Initialize as "raw" nnet, prior to training the LDA-like preconditioning # matrix. This first config just does any initial splicing that we do; # we do this as it's a convenient way to get the stats for the 'lda-like' # transform. if (args.stage <= -5): logger.info("Initializing a basic network for estimating " "preconditioning matrix") common_lib.run_job( """{command} {dir}/log/nnet_init.log \ nnet3-init --srand=-2 {dir}/configs/init.config \ {dir}/init.raw""".format(command=run_opts.command, dir=args.dir)) default_egs_dir = '{0}/egs'.format(args.dir) if (args.stage <= -4) and args.egs_dir is None: logger.info("Generating egs") train_lib.acoustic_model.generate_egs( data=args.feat_dir, alidir=args.ali_dir, egs_dir=default_egs_dir, left_context=left_context, right_context=right_context, valid_left_context=left_context + args.chunk_width, valid_right_context=right_context + args.chunk_width, run_opts=run_opts, frames_per_eg=args.chunk_width, srand=args.srand, egs_opts=args.egs_opts, cmvn_opts=args.cmvn_opts, online_ivector_dir=args.online_ivector_dir, samples_per_iter=args.samples_per_iter, transform_dir=args.transform_dir, stage=args.egs_stage) if args.egs_dir is None: egs_dir = default_egs_dir else: egs_dir = args.egs_dir [egs_left_context, egs_right_context, frames_per_eg, num_archives] = ( common_train_lib.verify_egs_dir(egs_dir, feat_dim, ivector_dim, left_context, right_context)) assert(args.chunk_width == frames_per_eg) if (args.num_jobs_final > num_archives): raise Exception('num_jobs_final cannot exceed the number of archives ' 'in the egs directory') # copy the properties of the egs to dir for # use during decoding common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir) if (args.stage <= -3): logger.info('Computing the preconditioning matrix for input features') train_lib.common.compute_preconditioning_matrix( args.dir, egs_dir, num_archives, run_opts, max_lda_jobs=args.max_lda_jobs, rand_prune=args.rand_prune) if (args.stage <= -2): logger.info("Computing initial vector for FixedScaleComponent before" " softmax, using priors^{prior_scale} and rescaling to" " average 1".format( prior_scale=args.presoftmax_prior_scale_power)) common_train_lib.compute_presoftmax_prior_scale( args.dir, args.ali_dir, num_jobs, run_opts, presoftmax_prior_scale_power=args.presoftmax_prior_scale_power) if (args.stage <= -1): logger.info("Preparing the initial acoustic model.") train_lib.acoustic_model.prepare_initial_acoustic_model( args.dir, args.ali_dir, run_opts) # set num_iters so that as close as possible, we process the data # $num_epochs times, i.e. $num_iters*$avg_num_jobs) == # $num_epochs*$num_archives, where # avg_num_jobs=(num_jobs_initial+num_jobs_final)/2. num_archives_to_process = args.num_epochs * num_archives num_archives_processed = 0 num_iters = ((num_archives_to_process * 2) / (args.num_jobs_initial + args.num_jobs_final)) models_to_combine = common_train_lib.verify_iterations( num_iters, args.num_epochs, num_hidden_layers, num_archives, args.max_models_combine, args.add_layers_period, args.num_jobs_final) def learning_rate(iter, current_num_jobs, num_archives_processed): return common_train_lib.get_learning_rate(iter, current_num_jobs, num_iters, num_archives_processed, num_archives_to_process, args.initial_effective_lrate, args.final_effective_lrate) min_deriv_time = None max_deriv_time = None if args.deriv_truncate_margin is not None: min_deriv_time = -args.deriv_truncate_margin - model_left_context max_deriv_time = (args.chunk_width - 1 + args.deriv_truncate_margin + model_right_context) logger.info("Training will run for {0} epochs = " "{1} iterations".format(args.num_epochs, num_iters)) for iter in range(num_iters): if (args.exit_stage is not None) and (iter == args.exit_stage): logger.info("Exiting early due to --exit-stage {0}".format(iter)) return current_num_jobs = int(0.5 + args.num_jobs_initial + (args.num_jobs_final - args.num_jobs_initial) * float(iter) / num_iters) if args.stage <= iter: model_file = "{dir}/{iter}.mdl".format(dir=args.dir, iter=iter) shrinkage_value = 1.0 if args.shrink_value != 1.0: shrinkage_value = (args.shrink_value if common_train_lib.do_shrinkage( iter, model_file, args.shrink_saturation_threshold) else 1 ) logger.info("On iteration {0}, learning rate is {1} and " "shrink value is {2}.".format( iter, learning_rate(iter, current_num_jobs, num_archives_processed), shrinkage_value)) train_lib.common.train_one_iteration( dir=args.dir, iter=iter, srand=args.srand, egs_dir=egs_dir, num_jobs=current_num_jobs, num_archives_processed=num_archives_processed, num_archives=num_archives, learning_rate=learning_rate(iter, current_num_jobs, num_archives_processed), shrinkage_value=shrinkage_value, minibatch_size=args.num_chunk_per_minibatch, num_hidden_layers=num_hidden_layers, add_layers_period=args.add_layers_period, left_context=left_context, right_context=right_context, min_deriv_time=min_deriv_time, max_deriv_time=max_deriv_time, momentum=args.momentum, max_param_change=args.max_param_change, shuffle_buffer_size=args.shuffle_buffer_size, cv_minibatch_size=args.cv_minibatch_size, run_opts=run_opts, background_process_handler=background_process_handler) if args.cleanup: # do a clean up everythin but the last 2 models, under certain # conditions common_train_lib.remove_model( args.dir, iter-2, num_iters, models_to_combine, args.preserve_model_interval) if args.email is not None: reporting_iter_interval = num_iters * args.reporting_interval if iter % reporting_iter_interval == 0: # lets do some reporting [report, times, data] = ( nnet3_log_parse.generate_accuracy_report(args.dir)) message = report subject = ("Update : Expt {dir} : " "Iter {iter}".format(dir=args.dir, iter=iter)) common_lib.send_mail(message, subject, args.email) num_archives_processed = num_archives_processed + current_num_jobs if args.stage <= num_iters: logger.info("Doing final combination to produce final.mdl") train_lib.common.combine_models( dir=args.dir, num_iters=num_iters, models_to_combine=models_to_combine, egs_dir=egs_dir, run_opts=run_opts, left_context=left_context, right_context=right_context, background_process_handler=background_process_handler, chunk_width=args.chunk_width) if args.stage <= num_iters + 1: logger.info("Getting average posterior for purposes of " "adjusting the priors.") avg_post_vec_file = train_lib.common.compute_average_posterior( dir=args.dir, iter='combined', egs_dir=egs_dir, num_archives=num_archives, left_context=left_context, right_context=right_context, prior_subset_size=args.prior_subset_size, run_opts=run_opts) logger.info("Re-adjusting priors based on computed posteriors") combined_model = "{dir}/combined.mdl".format(dir=args.dir) final_model = "{dir}/final.mdl".format(dir=args.dir) train_lib.common.adjust_am_priors(args.dir, combined_model, avg_post_vec_file, final_model, run_opts) if args.cleanup: logger.info("Cleaning up the experiment directory " "{0}".format(args.dir)) remove_egs = args.remove_egs if args.egs_dir is not None: # this egs_dir was not created by this experiment so we will not # delete it remove_egs = False common_train_lib.clean_nnet_dir( nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir, preserve_model_interval=args.preserve_model_interval, remove_egs=remove_egs) # do some reporting [report, times, data] = nnet3_log_parse.generate_accuracy_report(args.dir) if args.email is not None: common_lib.send_mail(report, "Update : Expt {0} : " "complete".format(args.dir), args.email) with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f: f.write(report) common_lib.run_job("steps/info/nnet3_dir_info.pl " "{0}".format(args.dir))
def train(args, run_opts, background_process_handler): """ The main function for training. Args: args: a Namespace object with the required parameters obtained from the function process_args() run_opts: RunOpts object obtained from the process_args() """ arg_string = pprint.pformat(vars(args)) logger.info("Arguments for the experiment\n{0}".format(arg_string)) # Set some variables. # num_leaves = common_lib.get_number_of_leaves_from_tree(args.ali_dir) num_jobs = common_lib.get_number_of_jobs(args.ali_dir) feat_dim = common_lib.get_feat_dim(args.feat_dir) ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir) # split the training data into parts for individual jobs # we will use the same number of jobs as that used for alignment common_lib.split_data(args.feat_dir, num_jobs) shutil.copy('{0}/tree'.format(args.ali_dir), args.dir) with open('{0}/num_jobs'.format(args.dir), 'w') as f: f.write(str(num_jobs)) config_dir = '{0}/configs'.format(args.dir) var_file = '{0}/vars'.format(config_dir) variables = common_train_lib.parse_generic_config_vars_file(var_file) # Set some variables. try: model_left_context = variables['model_left_context'] model_right_context = variables['model_right_context'] # this is really the number of times we add layers to the network for # discriminative pretraining num_hidden_layers = variables['num_hidden_layers'] except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined in " "{1}".format(str(e), '{0}/configs'.format(args.dir))) left_context = args.chunk_left_context + model_left_context right_context = args.chunk_right_context + model_right_context # Initialize as "raw" nnet, prior to training the LDA-like preconditioning # matrix. This first config just does any initial splicing that we do; # we do this as it's a convenient way to get the stats for the 'lda-like' # transform. if (args.stage <= -5): logger.info("Initializing a basic network for estimating " "preconditioning matrix") common_lib.run_job( """{command} {dir}/log/nnet_init.log \ nnet3-init --srand=-2 {dir}/configs/init.config \ {dir}/init.raw""".format(command=run_opts.command, dir=args.dir)) default_egs_dir = '{0}/egs'.format(args.dir) if (args.stage <= -4) and args.egs_dir is None: logger.info("Generating egs") train_lib.acoustic_model.generate_egs( data=args.feat_dir, alidir=args.ali_dir, egs_dir=default_egs_dir, left_context=left_context, right_context=right_context, run_opts=run_opts, frames_per_eg=args.frames_per_eg, srand=args.srand, egs_opts=args.egs_opts, cmvn_opts=args.cmvn_opts, online_ivector_dir=args.online_ivector_dir, samples_per_iter=args.samples_per_iter, transform_dir=args.transform_dir, stage=args.egs_stage) if args.egs_dir is None: egs_dir = default_egs_dir else: egs_dir = args.egs_dir [egs_left_context, egs_right_context, frames_per_eg, num_archives] = ( common_train_lib.verify_egs_dir(egs_dir, feat_dim, ivector_dim, left_context, right_context)) assert(args.frames_per_eg == frames_per_eg) if (args.num_jobs_final > num_archives): raise Exception('num_jobs_final cannot exceed the number of archives ' 'in the egs directory') # copy the properties of the egs to dir for # use during decoding common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir) if (args.stage <= -3): logger.info('Computing the preconditioning matrix for input features') train_lib.common.compute_preconditioning_matrix( args.dir, egs_dir, num_archives, run_opts, max_lda_jobs=args.max_lda_jobs, rand_prune=args.rand_prune) if (args.stage <= -2): logger.info("Computing initial vector for FixedScaleComponent before" " softmax, using priors^{prior_scale} and rescaling to" " average 1".format( prior_scale=args.presoftmax_prior_scale_power)) common_train_lib.compute_presoftmax_prior_scale( args.dir, args.ali_dir, num_jobs, run_opts, presoftmax_prior_scale_power=args.presoftmax_prior_scale_power) if (args.stage <= -1): logger.info("Preparing the initial acoustic model.") train_lib.acoustic_model.prepare_initial_acoustic_model( args.dir, args.ali_dir, run_opts) # set num_iters so that as close as possible, we process the data # $num_epochs times, i.e. $num_iters*$avg_num_jobs) == # $num_epochs*$num_archives, where # avg_num_jobs=(num_jobs_initial+num_jobs_final)/2. num_archives_expanded = num_archives * args.frames_per_eg num_archives_to_process = args.num_epochs * num_archives_expanded num_archives_processed = 0 num_iters = ((num_archives_to_process * 2) / (args.num_jobs_initial + args.num_jobs_final)) models_to_combine = common_train_lib.verify_iterations( num_iters, args.num_epochs, num_hidden_layers, num_archives_expanded, args.max_models_combine, args.add_layers_period, args.num_jobs_final) def learning_rate(iter, current_num_jobs, num_archives_processed): return common_train_lib.get_learning_rate(iter, current_num_jobs, num_iters, num_archives_processed, num_archives_to_process, args.initial_effective_lrate, args.final_effective_lrate) logger.info("Training will run for {0} epochs = " "{1} iterations".format(args.num_epochs, num_iters)) for iter in range(num_iters): if (args.exit_stage is not None) and (iter == args.exit_stage): logger.info("Exiting early due to --exit-stage {0}".format(iter)) return current_num_jobs = int(0.5 + args.num_jobs_initial + (args.num_jobs_final - args.num_jobs_initial) * float(iter) / num_iters) if args.stage <= iter: logger.info("On iteration {0}, learning rate is {1}.".format( iter, learning_rate(iter, current_num_jobs, num_archives_processed))) train_lib.common.train_one_iteration( dir=args.dir, iter=iter, srand=args.srand, egs_dir=egs_dir, num_jobs=current_num_jobs, num_archives_processed=num_archives_processed, num_archives=num_archives, learning_rate=learning_rate(iter, current_num_jobs, num_archives_processed), minibatch_size=args.minibatch_size, frames_per_eg=args.frames_per_eg, num_hidden_layers=num_hidden_layers, add_layers_period=args.add_layers_period, left_context=left_context, right_context=right_context, momentum=args.momentum, max_param_change=args.max_param_change, shuffle_buffer_size=args.shuffle_buffer_size, run_opts=run_opts, background_process_handler=background_process_handler) if args.cleanup: # do a clean up everythin but the last 2 models, under certain # conditions common_train_lib.remove_model( args.dir, iter-2, num_iters, models_to_combine, args.preserve_model_interval) if args.email is not None: reporting_iter_interval = num_iters * args.reporting_interval if iter % reporting_iter_interval == 0: # lets do some reporting [report, times, data] = ( nnet3_log_parse.generate_accuracy_report(args.dir)) message = report subject = ("Update : Expt {dir} : " "Iter {iter}".format(dir=args.dir, iter=iter)) common_lib.send_mail(message, subject, args.email) num_archives_processed = num_archives_processed + current_num_jobs if args.stage <= num_iters: logger.info("Doing final combination to produce final.mdl") train_lib.common.combine_models( dir=args.dir, num_iters=num_iters, models_to_combine=models_to_combine, egs_dir=egs_dir, left_context=left_context, right_context=right_context, run_opts=run_opts, background_process_handler=background_process_handler) if args.stage <= num_iters + 1: logger.info("Getting average posterior for purposes of " "adjusting the priors.") avg_post_vec_file = train_lib.common.compute_average_posterior( dir=args.dir, iter='combined', egs_dir=egs_dir, num_archives=num_archives, left_context=left_context, right_context=right_context, prior_subset_size=args.prior_subset_size, run_opts=run_opts) logger.info("Re-adjusting priors based on computed posteriors") combined_model = "{dir}/combined.mdl".format(dir=args.dir) final_model = "{dir}/final.mdl".format(dir=args.dir) train_lib.common.adjust_am_priors(args.dir, combined_model, avg_post_vec_file, final_model, run_opts) if args.cleanup: logger.info("Cleaning up the experiment directory " "{0}".format(args.dir)) remove_egs = args.remove_egs if args.egs_dir is not None: # this egs_dir was not created by this experiment so we will not # delete it remove_egs = False common_train_lib.clean_nnet_dir( nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir, preserve_model_interval=args.preserve_model_interval, remove_egs=remove_egs) # do some reporting [report, times, data] = nnet3_log_parse.generate_accuracy_report(args.dir) if args.email is not None: common_lib.send_mail(report, "Update : Expt {0} : " "complete".format(args.dir), args.email) with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f: f.write(report) common_lib.run_job("steps/info/nnet3_dir_info.pl " "{0}".format(args.dir))
def generate_chain_egs(dir, data, lat_dir, egs_dir, left_context, right_context, run_opts, stage=0, valid_left_context=None, valid_right_context=None, left_tolerance=None, right_tolerance=None, frame_subsampling_factor=3, alignment_subsampling_factor=3, feat_type='raw', online_ivector_dir=None, frames_per_iter=20000, frames_per_eg=20, srand=0, egs_opts=None, cmvn_opts=None, transform_dir=None): """Wrapper for steps/nnet3/chain/get_egs.sh See options in that script. """ common_lib.run_job( """steps/nnet3/chain/get_egs.sh {egs_opts} \ --cmd "{command}" \ --cmvn-opts "{cmvn_opts}" \ --feat-type {feat_type} \ --transform-dir "{transform_dir}" \ --online-ivector-dir "{ivector_dir}" \ --left-context {left_context} --right-context {right_context} \ --valid-left-context '{valid_left_context}' \ --valid-right-context '{valid_right_context}' \ --left-tolerance '{left_tolerance}' \ --right-tolerance '{right_tolerance}' \ --frame-subsampling-factor {frame_subsampling_factor} \ --alignment-subsampling-factor {alignment_subsampling_factor} \ --stage {stage} \ --frames-per-iter {frames_per_iter} \ --frames-per-eg {frames_per_eg} \ --srand {srand} \ {data} {dir} {lat_dir} {egs_dir}""".format( command=run_opts.command, cmvn_opts=cmvn_opts if cmvn_opts is not None else '', feat_type=feat_type, transform_dir=(transform_dir if transform_dir is not None else ''), ivector_dir=(online_ivector_dir if online_ivector_dir is not None else ''), left_context=left_context, right_context=right_context, valid_left_context=(valid_left_context if valid_left_context is not None else ''), valid_right_context=(valid_right_context if valid_right_context is not None else ''), left_tolerance=(left_tolerance if left_tolerance is not None else ''), right_tolerance=(right_tolerance if right_tolerance is not None else ''), frame_subsampling_factor=frame_subsampling_factor, alignment_subsampling_factor=alignment_subsampling_factor, stage=stage, frames_per_iter=frames_per_iter, frames_per_eg=frames_per_eg, srand=srand, data=data, lat_dir=lat_dir, dir=dir, egs_dir=egs_dir, egs_opts=egs_opts if egs_opts is not None else ''))
def combine_models(dir, num_iters, models_to_combine, egs_dir, left_context, right_context, run_opts, background_process_handler=None, chunk_width=None, get_raw_nnet_from_am=True): """ Function to do model combination In the nnet3 setup, the logic for doing averaging of subsets of the models in the case where there are too many models to reliably esetimate interpolation factors (max_models_combine) is moved into the nnet3-combine. """ raw_model_strings = [] logger.info("Combining {0} models.".format(models_to_combine)) models_to_combine.add(num_iters) for iter in models_to_combine: if get_raw_nnet_from_am: model_file = '{0}/{1}.mdl'.format(dir, iter) if not os.path.exists(model_file): raise Exception('Model file {0} missing'.format(model_file)) raw_model_strings.append( '"nnet3-am-copy --raw=true {0} -|"'.format(model_file)) else: model_file = '{0}/{1}.raw'.format(dir, iter) if not os.path.exists(model_file): raise Exception('Model file {0} missing'.format(model_file)) raw_model_strings.append(model_file) if chunk_width is not None: # this is an RNN model mbsize = int(1024.0/(chunk_width)) else: mbsize = 1024 if get_raw_nnet_from_am: out_model = ("| nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl " "{dir}/combined.mdl".format(dir=dir, num_iters=num_iters)) else: out_model = '{dir}/final.raw'.format(dir=dir) context_opts = "--left-context={lc} --right-context={rc}".format( lc=left_context, rc=right_context) common_lib.run_job( """{command} {combine_queue_opt} {dir}/log/combine.log \ nnet3-combine --num-iters=40 \ --enforce-sum-to-one=true --enforce-positive-weights=true \ --verbose=3 {raw_models} \ "ark,bg:nnet3-copy-egs {context_opts} \ ark:{egs_dir}/combine.egs ark:- | \ nnet3-merge-egs --measure-output-frames=false \ --minibatch-size={mbsize} ark:- ark:- |" \ "{out_model}" """.format(command=run_opts.command, combine_queue_opt=run_opts.combine_queue_opt, dir=dir, raw_models=" ".join(raw_model_strings), context_opts=context_opts, mbsize=mbsize, out_model=out_model, egs_dir=egs_dir)) # Compute the probability of the final, combined model with # the same subset we used for the previous compute_probs, as the # different subsets will lead to different probs. if get_raw_nnet_from_am: compute_train_cv_probabilities( dir=dir, iter='combined', egs_dir=egs_dir, left_context=left_context, right_context=right_context, run_opts=run_opts, wait=False, background_process_handler=background_process_handler) else: compute_train_cv_probabilities( dir=dir, iter='final', egs_dir=egs_dir, left_context=left_context, right_context=right_context, run_opts=run_opts, wait=False, background_process_handler=background_process_handler, get_raw_nnet_from_am=False)
def remove_nnet_egs(egs_dir): common_lib.run_job( "steps/nnet2/remove_egs.sh {egs_dir}".format(egs_dir=egs_dir))
def train(args, run_opts, background_process_handler): """ The main function for training. Args: args: a Namespace object with the required parameters obtained from the function process_args() run_opts: RunOpts object obtained from the process_args() """ arg_string = pprint.pformat(vars(args)) logger.info("Arguments for the experiment\n{0}".format(arg_string)) # Set some variables. feat_dim = common_lib.get_feat_dim(args.feat_dir) ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir) ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir) config_dir = '{0}/configs'.format(args.dir) var_file = '{0}/vars'.format(config_dir) variables = common_train_lib.parse_generic_config_vars_file(var_file) # Set some variables. try: model_left_context = variables['model_left_context'] model_right_context = variables['model_right_context'] # this is really the number of times we add layers to the network for # discriminative pretraining num_hidden_layers = variables['num_hidden_layers'] add_lda = common_lib.str_to_bool(variables['add_lda']) include_log_softmax = common_lib.str_to_bool( variables['include_log_softmax']) except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined in " "{1}".format(str(e), '{0}/configs'.format(args.dir))) left_context = args.chunk_left_context + model_left_context right_context = args.chunk_right_context + model_right_context left_context_initial = (args.chunk_left_context_initial + model_left_context if args.chunk_left_context_initial >= 0 else -1) right_context_final = (args.chunk_right_context_final + model_right_context if args.chunk_right_context_final >= 0 else -1) # Initialize as "raw" nnet, prior to training the LDA-like preconditioning # matrix. This first config just does any initial splicing that we do; # we do this as it's a convenient way to get the stats for the 'lda-like' # transform. if (args.stage <= -4): logger.info("Initializing a basic network") common_lib.run_job( """{command} {dir}/log/nnet_init.log \ nnet3-init --srand=-2 {dir}/configs/init.config \ {dir}/init.raw""".format(command=run_opts.command, dir=args.dir)) default_egs_dir = '{0}/egs'.format(args.dir) if (args.stage <= -3) and args.egs_dir is None: logger.info("Generating egs") if args.use_dense_targets: target_type = "dense" try: num_targets = int(variables['num_targets']) if (common_lib.get_feat_dim_from_scp(args.targets_scp) != num_targets): raise Exception("Mismatch between num-targets provided to " "script vs configs") except KeyError as e: num_targets = -1 else: target_type = "sparse" try: num_targets = int(variables['num_targets']) except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined " "in {1}".format( str(e), '{0}/configs'.format(args.dir))) train_lib.raw_model.generate_egs_using_targets( data=args.feat_dir, targets_scp=args.targets_scp, egs_dir=default_egs_dir, left_context=left_context, right_context=right_context, left_context_initial=left_context_initial, right_context_final=right_context_final, run_opts=run_opts, frames_per_eg_str=args.chunk_width, srand=args.srand, egs_opts=args.egs_opts, cmvn_opts=args.cmvn_opts, online_ivector_dir=args.online_ivector_dir, samples_per_iter=args.samples_per_iter, transform_dir=args.transform_dir, stage=args.egs_stage, target_type=target_type, num_targets=num_targets) if args.egs_dir is None: egs_dir = default_egs_dir else: egs_dir = args.egs_dir [egs_left_context, egs_right_context, frames_per_eg_str, num_archives] = ( common_train_lib.verify_egs_dir(egs_dir, feat_dim, ivector_dim, ivector_id, left_context, right_context)) if args.chunk_width != frames_per_eg_str: raise Exception("mismatch between --egs.chunk-width and the frames_per_eg " "in the egs dir {0} vs {1}".format(args.chunk_width, frames_per_eg_str)) if (args.num_jobs_final > num_archives): raise Exception('num_jobs_final cannot exceed the number of archives ' 'in the egs directory') # copy the properties of the egs to dir for # use during decoding common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir) if (add_lda and args.stage <= -2): logger.info('Computing the preconditioning matrix for input features') train_lib.common.compute_preconditioning_matrix( args.dir, egs_dir, num_archives, run_opts, max_lda_jobs=args.max_lda_jobs, rand_prune=args.rand_prune) if (args.stage <= -1): logger.info("Preparing the initial network.") common_train_lib.prepare_initial_network(args.dir, run_opts) # set num_iters so that as close as possible, we process the data # $num_epochs times, i.e. $num_iters*$avg_num_jobs) == # $num_epochs*$num_archives, where # avg_num_jobs=(num_jobs_initial+num_jobs_final)/2. num_archives_to_process = int(args.num_epochs * num_archives) num_archives_processed = 0 num_iters = ((num_archives_to_process * 2) / (args.num_jobs_initial + args.num_jobs_final)) models_to_combine = common_train_lib.verify_iterations( num_iters, args.num_epochs, num_hidden_layers, num_archives, args.max_models_combine, args.add_layers_period, args.num_jobs_final) def learning_rate(iter, current_num_jobs, num_archives_processed): return common_train_lib.get_learning_rate(iter, current_num_jobs, num_iters, num_archives_processed, num_archives_to_process, args.initial_effective_lrate, args.final_effective_lrate) min_deriv_time = None max_deriv_time_relative = None if args.deriv_truncate_margin is not None: min_deriv_time = -args.deriv_truncate_margin - model_left_context max_deriv_time_relative = \ args.deriv_truncate_margin + model_right_context logger.info("Training will run for {0} epochs = " "{1} iterations".format(args.num_epochs, num_iters)) for iter in range(num_iters): if (args.exit_stage is not None) and (iter == args.exit_stage): logger.info("Exiting early due to --exit-stage {0}".format(iter)) return current_num_jobs = int(0.5 + args.num_jobs_initial + (args.num_jobs_final - args.num_jobs_initial) * float(iter) / num_iters) if args.stage <= iter: model_file = "{dir}/{iter}.raw".format(dir=args.dir, iter=iter) shrinkage_value = 1.0 if args.shrink_value != 1.0: shrinkage_value = (args.shrink_value if common_train_lib.do_shrinkage( iter, model_file, args.shrink_saturation_threshold, get_raw_nnet_from_am=False) else 1 ) train_lib.common.train_one_iteration( dir=args.dir, iter=iter, srand=args.srand, egs_dir=egs_dir, num_jobs=current_num_jobs, num_archives_processed=num_archives_processed, num_archives=num_archives, learning_rate=learning_rate(iter, current_num_jobs, num_archives_processed), dropout_edit_string=common_train_lib.get_dropout_edit_string( args.dropout_schedule, float(num_archives_processed) / num_archives_to_process, iter), shrinkage_value=shrinkage_value, minibatch_size_str=args.num_chunk_per_minibatch, num_hidden_layers=num_hidden_layers, add_layers_period=args.add_layers_period, left_context=left_context, right_context=right_context, min_deriv_time=min_deriv_time, max_deriv_time_relative=max_deriv_time_relative, momentum=args.momentum, max_param_change=args.max_param_change, shuffle_buffer_size=args.shuffle_buffer_size, run_opts=run_opts, get_raw_nnet_from_am=False, background_process_handler=background_process_handler) if args.cleanup: # do a clean up everythin but the last 2 models, under certain # conditions common_train_lib.remove_model( args.dir, iter-2, num_iters, models_to_combine, args.preserve_model_interval, get_raw_nnet_from_am=False) if args.email is not None: reporting_iter_interval = num_iters * args.reporting_interval if iter % reporting_iter_interval == 0: # lets do some reporting [report, times, data] = ( nnet3_log_parse.generate_acc_logprob_report(args.dir)) message = report subject = ("Update : Expt {dir} : " "Iter {iter}".format(dir=args.dir, iter=iter)) common_lib.send_mail(message, subject, args.email) num_archives_processed = num_archives_processed + current_num_jobs if args.stage <= num_iters: logger.info("Doing final combination to produce final.raw") train_lib.common.combine_models( dir=args.dir, num_iters=num_iters, models_to_combine=models_to_combine, egs_dir=egs_dir, left_context=left_context, right_context=right_context, minibatch_size_str=args.num_chunk_per_minibatch, run_opts=run_opts, chunk_width=args.chunk_width, background_process_handler=background_process_handler, get_raw_nnet_from_am=False, sum_to_one_penalty=args.combine_sum_to_one_penalty) if include_log_softmax and args.stage <= num_iters + 1: logger.info("Getting average posterior for purposes of " "adjusting the priors.") train_lib.common.compute_average_posterior( dir=args.dir, iter='final', egs_dir=egs_dir, num_archives=num_archives, left_context=left_context, right_context=right_context, prior_subset_size=args.prior_subset_size, run_opts=run_opts, get_raw_nnet_from_am=False) if args.cleanup: logger.info("Cleaning up the experiment directory " "{0}".format(args.dir)) remove_egs = args.remove_egs if args.egs_dir is not None: # this egs_dir was not created by this experiment so we will not # delete it remove_egs = False common_train_lib.clean_nnet_dir( nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir, preserve_model_interval=args.preserve_model_interval, remove_egs=remove_egs, get_raw_nnet_from_am=False) # do some reporting [report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir) if args.email is not None: common_lib.send_mail(report, "Update : Expt {0} : " "complete".format(args.dir), args.email) with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f: f.write(report) common_lib.run_job("steps/info/nnet3_dir_info.pl " "{0}".format(args.dir))
def train_new_models(dir, iter, srand, num_jobs, num_archives_processed, num_archives, raw_model_string, egs_dir, left_context, right_context, momentum, max_param_change, shuffle_buffer_size, minibatch_size, cache_read_opt, run_opts, frames_per_eg=-1, min_deriv_time=None, max_deriv_time=None): """ Called from train_one_iteration(), this model does one iteration of training with 'num_jobs' jobs, and writes files like exp/tdnn_a/24.{1,2,3,..<num_jobs>}.raw We cannot easily use a single parallel SGE job to do the main training, because the computation of which archive and which --frame option to use for each job is a little complex, so we spawn each one separately. this is no longer true for RNNs as we use do not use the --frame option but we use the same script for consistency with FF-DNN code Args: frames_per_eg: The default value -1 implies chunk_level_training, which is particularly applicable to RNN training. If it is > 0, then it implies frame-level training, which is applicable for DNN training. If it is > 0, then each parallel SGE job created, a different frame numbered 0..frames_per_eg-1 is used. min_deriv_time: Applicable for RNN training. A default value of None implies a min_deriv_time of 0 is used. During RNN training, its value is set to chunk_width - num_bptt_steps in the training script. """ chunk_level_training = False if frames_per_eg > 0 else True deriv_time_opts = [] if min_deriv_time is not None: deriv_time_opts.append( "--optimization.min-deriv-time={0}".format(min_deriv_time)) if max_deriv_time is not None: deriv_time_opts.append( "--optimization.max-deriv-time={0}".format(max_deriv_time)) context_opts = "--left-context={0} --right-context={1}".format( left_context, right_context) processes = [] for job in range(1, num_jobs + 1): # k is a zero-based index that we will derive the other indexes from. k = num_archives_processed + job - 1 # work out the 1-based archive index. archive_index = (k % num_archives) + 1 if not chunk_level_training: frame = (k / num_archives) % frames_per_eg cache_write_opt = "" if job == 1: # an option for writing cache (storing pairs of nnet-computations # and computation-requests) during training. cache_write_opt = "--write-cache={dir}/cache.{iter}".format( dir=dir, iter=iter + 1) process_handle = common_lib.run_job( """{command} {train_queue_opt} {dir}/log/train.{iter}.{job}.log \ nnet3-train {parallel_train_opts} {cache_read_opt} \ {cache_write_opt} --print-interval=10 \ --momentum={momentum} \ --max-param-change={max_param_change} \ {deriv_time_opts} "{raw_model}" \ "ark,bg:nnet3-copy-egs {frame_opts} {context_opts} """ """ark:{egs_dir}/egs.{archive_index}.ark ark:- |""" """nnet3-shuffle-egs --buffer-size={shuffle_buffer_size} """ """--srand={srand} ark:- ark:- | """ """nnet3-merge-egs --minibatch-size={minibatch_size} """ """--measure-output-frames=false """ """--discard-partial-minibatches=true ark:- ark:- |" \ {dir}/{next_iter}.{job}.raw""".format( command=run_opts.command, train_queue_opt=run_opts.train_queue_opt, dir=dir, iter=iter, srand=iter + srand, next_iter=iter + 1, job=job, parallel_train_opts=run_opts.parallel_train_opts, cache_read_opt=cache_read_opt, cache_write_opt=cache_write_opt, frame_opts=("" if chunk_level_training else "--frame={0}".format(frame)), momentum=momentum, max_param_change=max_param_change, deriv_time_opts=" ".join(deriv_time_opts), raw_model=raw_model_string, context_opts=context_opts, egs_dir=egs_dir, archive_index=archive_index, shuffle_buffer_size=shuffle_buffer_size, minibatch_size=minibatch_size), wait=False) processes.append(process_handle) all_success = True for process in processes: process.wait() process.communicate() if process.returncode != 0: all_success = False if not all_success: open('{0}/.error'.format(dir), 'w').close() raise Exception("There was error during training " "iteration {0}".format(iter))
def remove_nnet_egs(egs_dir): common_lib.run_job("steps/nnet2/remove_egs.sh {egs_dir}".format( egs_dir=egs_dir))
def train_new_models(dir, iter, srand, num_jobs, num_archives_processed, num_archives, raw_model_string, egs_dir, left_context, right_context, momentum, max_param_change, shuffle_buffer_size, minibatch_size, cache_read_opt, run_opts, frames_per_eg=-1, min_deriv_time=None, max_deriv_time=None): """ Called from train_one_iteration(), this model does one iteration of training with 'num_jobs' jobs, and writes files like exp/tdnn_a/24.{1,2,3,..<num_jobs>}.raw We cannot easily use a single parallel SGE job to do the main training, because the computation of which archive and which --frame option to use for each job is a little complex, so we spawn each one separately. this is no longer true for RNNs as we use do not use the --frame option but we use the same script for consistency with FF-DNN code Args: frames_per_eg: The default value -1 implies chunk_level_training, which is particularly applicable to RNN training. If it is > 0, then it implies frame-level training, which is applicable for DNN training. If it is > 0, then each parallel SGE job created, a different frame numbered 0..frames_per_eg-1 is used. min_deriv_time: Applicable for RNN training. A default value of None implies a min_deriv_time of 0 is used. During RNN training, its value is set to chunk_width - num_bptt_steps in the training script. """ chunk_level_training = False if frames_per_eg > 0 else True deriv_time_opts = [] if min_deriv_time is not None: deriv_time_opts.append("--optimization.min-deriv-time={0}".format( min_deriv_time)) if max_deriv_time is not None: deriv_time_opts.append("--optimization.max-deriv-time={0}".format( max_deriv_time)) context_opts = "--left-context={0} --right-context={1}".format( left_context, right_context) processes = [] for job in range(1, num_jobs+1): # k is a zero-based index that we will derive the other indexes from. k = num_archives_processed + job - 1 # work out the 1-based archive index. archive_index = (k % num_archives) + 1 if not chunk_level_training: frame = (k / num_archives) % frames_per_eg cache_write_opt = "" if job == 1: # an option for writing cache (storing pairs of nnet-computations # and computation-requests) during training. cache_write_opt = "--write-cache={dir}/cache.{iter}".format( dir=dir, iter=iter+1) process_handle = common_lib.run_job( """{command} {train_queue_opt} {dir}/log/train.{iter}.{job}.log \ nnet3-train {parallel_train_opts} {cache_read_opt} \ {cache_write_opt} --print-interval=10 \ --momentum={momentum} \ --max-param-change={max_param_change} \ {deriv_time_opts} "{raw_model}" \ "ark,bg:nnet3-copy-egs {frame_opts} {context_opts} """ """ark:{egs_dir}/egs.{archive_index}.ark ark:- |""" """nnet3-shuffle-egs --buffer-size={shuffle_buffer_size} """ """--srand={srand} ark:- ark:- | """ """nnet3-merge-egs --minibatch-size={minibatch_size} """ """--measure-output-frames=false """ """--discard-partial-minibatches=true ark:- ark:- |" \ {dir}/{next_iter}.{job}.raw""".format( command=run_opts.command, train_queue_opt=run_opts.train_queue_opt, dir=dir, iter=iter, srand=iter + srand, next_iter=iter + 1, job=job, parallel_train_opts=run_opts.parallel_train_opts, cache_read_opt=cache_read_opt, cache_write_opt=cache_write_opt, frame_opts=("" if chunk_level_training else "--frame={0}".format(frame)), momentum=momentum, max_param_change=max_param_change, deriv_time_opts=" ".join(deriv_time_opts), raw_model=raw_model_string, context_opts=context_opts, egs_dir=egs_dir, archive_index=archive_index, shuffle_buffer_size=shuffle_buffer_size, minibatch_size=minibatch_size), wait=False) processes.append(process_handle) all_success = True for process in processes: process.wait() process.communicate() if process.returncode != 0: all_success = False if not all_success: open('{0}/.error'.format(dir), 'w').close() raise Exception("There was error during training " "iteration {0}".format(iter))
def train_new_models(dir, iter, srand, num_jobs, num_archives_processed, num_archives, raw_model_string, egs_dir, left_context, right_context, apply_deriv_weights, min_deriv_time, max_deriv_time, l2_regularize, xent_regularize, leaky_hmm_coefficient, momentum, max_param_change, shuffle_buffer_size, num_chunk_per_minibatch, frame_subsampling_factor, truncate_deriv_weights, cache_io_opts, run_opts): """ Called from train_one_iteration(), this method trains new models with 'num_jobs' jobs, and writes files like exp/tdnn_a/24.{1,2,3,..<num_jobs>}.raw We cannot easily use a single parallel SGE job to do the main training, because the computation of which archive and which --frame option to use for each job is a little complex, so we spawn each one separately. this is no longer true for RNNs as we use do not use the --frame option but we use the same script for consistency with FF-DNN code """ deriv_time_opts = [] if min_deriv_time is not None: deriv_time_opts.append("--optimization.min-deriv-time={0}".format( min_deriv_time)) if max_deriv_time is not None: deriv_time_opts.append("--optimization.max-deriv-time={0}".format( int(max_deriv_time))) processes = [] for job in range(1, num_jobs+1): # k is a zero-based index that we will derive the other indexes from. k = num_archives_processed + job - 1 # work out the 1-based archive index. archive_index = (k % num_archives) + 1 # previous : frame_shift = (k/num_archives) % frame_subsampling_factor frame_shift = ((archive_index + k/num_archives) % frame_subsampling_factor) if job == 1: cur_cache_io_opts = "{0} --write-cache={1}/cache.{2}".format( cache_io_opts, dir, iter + 1) else: cur_cache_io_opts = cache_io_opts process_handle = common_lib.run_job( """{command} {train_queue_opt} {dir}/log/train.{iter}.{job}.log \ nnet3-chain-train {parallel_train_opts} \ --apply-deriv-weights={app_deriv_wts} \ --l2-regularize={l2} --leaky-hmm-coefficient={leaky} \ {cache_io_opts} --xent-regularize={xent_reg} \ {deriv_time_opts} \ --print-interval=10 --momentum={momentum} \ --max-param-change={max_param_change} \ "{raw_model}" {dir}/den.fst \ "ark,bg:nnet3-chain-copy-egs \ --left-context={lc} --right-context={rc} \ --truncate-deriv-weights={trunc_deriv} \ --frame-shift={fr_shft} \ ark:{egs_dir}/cegs.{archive_index}.ark ark:- | \ nnet3-chain-shuffle-egs --buffer-size={buf_size} \ --srand={srand} ark:- ark:- | nnet3-chain-merge-egs \ --minibatch-size={num_chunk_per_mb} ark:- ark:- |" \ {dir}/{next_iter}.{job}.raw""".format( command=run_opts.command, train_queue_opt=run_opts.train_queue_opt, dir=dir, iter=iter, srand=iter + srand, next_iter=iter + 1, job=job, deriv_time_opts=" ".join(deriv_time_opts), lc=left_context, rc=right_context, trunc_deriv=truncate_deriv_weights, app_deriv_wts=apply_deriv_weights, fr_shft=frame_shift, l2=l2_regularize, xent_reg=xent_regularize, leaky=leaky_hmm_coefficient, parallel_train_opts=run_opts.parallel_train_opts, momentum=momentum, max_param_change=max_param_change, raw_model=raw_model_string, egs_dir=egs_dir, archive_index=archive_index, buf_size=shuffle_buffer_size, cache_io_opts=cur_cache_io_opts, num_chunk_per_mb=num_chunk_per_minibatch), wait=False) processes.append(process_handle) all_success = True for process in processes: process.wait() process.communicate() if process.returncode != 0: all_success = False if not all_success: open('{0}/.error'.format(dir), 'w').close() raise Exception("There was error during training " "iteration {0}".format(iter))
def combine_models(dir, num_iters, models_to_combine, egs_dir, left_context, right_context, run_opts, background_process_handler=None, chunk_width=None, get_raw_nnet_from_am=True): """ Function to do model combination In the nnet3 setup, the logic for doing averaging of subsets of the models in the case where there are too many models to reliably esetimate interpolation factors (max_models_combine) is moved into the nnet3-combine. """ raw_model_strings = [] logger.info("Combining {0} models.".format(models_to_combine)) models_to_combine.add(num_iters) for iter in models_to_combine: if get_raw_nnet_from_am: model_file = '{0}/{1}.mdl'.format(dir, iter) if not os.path.exists(model_file): raise Exception('Model file {0} missing'.format(model_file)) raw_model_strings.append( '"nnet3-am-copy --raw=true {0} -|"'.format(model_file)) else: model_file = '{0}/{1}.raw'.format(dir, iter) if not os.path.exists(model_file): raise Exception('Model file {0} missing'.format(model_file)) raw_model_strings.append(model_file) if chunk_width is not None: # this is an RNN model mbsize = int(1024.0 / (chunk_width)) else: mbsize = 1024 if get_raw_nnet_from_am: out_model = ("| nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl " "{dir}/combined.mdl".format(dir=dir, num_iters=num_iters)) else: out_model = '{dir}/final.raw'.format(dir=dir) context_opts = "--left-context={lc} --right-context={rc}".format( lc=left_context, rc=right_context) common_lib.run_job("""{command} {combine_queue_opt} {dir}/log/combine.log \ nnet3-combine --num-iters=40 \ --enforce-sum-to-one=true --enforce-positive-weights=true \ --verbose=3 {raw_models} \ "ark,bg:nnet3-copy-egs {context_opts} \ ark:{egs_dir}/combine.egs ark:- | \ nnet3-merge-egs --measure-output-frames=false \ --minibatch-size={mbsize} ark:- ark:- |" \ "{out_model}" """.format(command=run_opts.command, combine_queue_opt=run_opts.combine_queue_opt, dir=dir, raw_models=" ".join(raw_model_strings), context_opts=context_opts, mbsize=mbsize, out_model=out_model, egs_dir=egs_dir)) # Compute the probability of the final, combined model with # the same subset we used for the previous compute_probs, as the # different subsets will lead to different probs. if get_raw_nnet_from_am: compute_train_cv_probabilities( dir=dir, iter='combined', egs_dir=egs_dir, left_context=left_context, right_context=right_context, run_opts=run_opts, wait=False, background_process_handler=background_process_handler) else: compute_train_cv_probabilities( dir=dir, iter='final', egs_dir=egs_dir, left_context=left_context, right_context=right_context, run_opts=run_opts, wait=False, background_process_handler=background_process_handler, get_raw_nnet_from_am=False)
def generate_egs_using_targets(data, targets_scp, egs_dir, left_context, right_context, valid_left_context, valid_right_context, run_opts, stage=0, feat_type='raw', online_ivector_dir=None, target_type='dense', num_targets=-1, samples_per_iter=20000, frames_per_eg=20, srand=0, egs_opts=None, cmvn_opts=None, transform_dir=None): """ Wrapper for calling steps/nnet3/get_egs_targets.sh This method generates egs directly from an scp file of targets, instead of getting them from the alignments (as with the method generate_egs() in module nnet3.train.frame_level_objf.acoustic_model). Args: target_type: "dense" if the targets are in matrix format "sparse" if the targets are in posterior format num_targets: must be explicitly specified for "sparse" targets. For "dense" targets, this option is ignored and the target dim is computed from the target matrix dimension For other options, see the file steps/nnet3/get_egs_targets.sh """ if target_type == 'dense': num_targets = common_lib.get_feat_dim_from_scp(targets_scp) else: if num_targets == -1: raise Exception("--num-targets is required if " "target-type is sparse") common_lib.run_job("""steps/nnet3/get_egs_targets.sh {egs_opts} \ --cmd "{command}" \ --cmvn-opts "{cmvn_opts}" \ --feat-type {feat_type} \ --transform-dir "{transform_dir}" \ --online-ivector-dir "{ivector_dir}" \ --left-context {left_context} --right-context {right_context} \ --valid-left-context {valid_left_context} \ --valid-right-context {valid_right_context} \ --stage {stage} \ --samples-per-iter {samples_per_iter} \ --frames-per-eg {frames_per_eg} \ --srand {srand} \ --target-type {target_type} \ --num-targets {num_targets} \ {data} {targets_scp} {egs_dir} """.format( command=run_opts.egs_command, cmvn_opts=cmvn_opts if cmvn_opts is not None else '', feat_type=feat_type, transform_dir=(transform_dir if transform_dir is not None else ''), ivector_dir=(online_ivector_dir if online_ivector_dir is not None else ''), left_context=left_context, right_context=right_context, valid_left_context=valid_left_context, valid_right_context=valid_right_context, stage=stage, samples_per_iter=samples_per_iter, frames_per_eg=frames_per_eg, srand=srand, num_targets=num_targets, data=data, targets_scp=targets_scp, target_type=target_type, egs_dir=egs_dir, egs_opts=egs_opts if egs_opts is not None else ''))