def compute_presoftmax_prior_scale(dir, alidir, num_jobs, run_opts, presoftmax_prior_scale_power=-0.25): # getting the raw pdf count common_lib.run_job( """{command} JOB=1:{num_jobs} {dir}/log/acc_pdf.JOB.log \ ali-to-post "ark:gunzip -c {alidir}/ali.JOB.gz|" ark:- \| \ post-to-tacc --per-pdf=true {alidir}/final.mdl ark:- \ {dir}/pdf_counts.JOB""".format(command=run_opts.command, num_jobs=num_jobs, dir=dir, alidir=alidir)) common_lib.run_job( """{command} {dir}/log/sum_pdf_counts.log \ vector-sum --binary=false {dir}/pdf_counts.* {dir}/pdf_counts \ """.format(command=run_opts.command, dir=dir)) for file in glob.glob('{0}/pdf_counts.*'.format(dir)): os.remove(file) pdf_counts = common_lib.read_kaldi_matrix('{0}/pdf_counts'.format(dir))[0] scaled_counts = smooth_presoftmax_prior_scale_vector( pdf_counts, presoftmax_prior_scale_power=presoftmax_prior_scale_power, smooth=0.01) output_file = "{0}/presoftmax_prior_scale.vec".format(dir) common_lib.write_kaldi_matrix(output_file, [scaled_counts]) common_lib.force_symlink("../presoftmax_prior_scale.vec", "{0}/configs/presoftmax_prior_scale.vec".format( dir))
def compute_presoftmax_prior_scale(dir, alidir, num_jobs, run_opts, presoftmax_prior_scale_power=-0.25): # getting the raw pdf count common_lib.execute_command( """{command} JOB=1:{num_jobs} {dir}/log/acc_pdf.JOB.log \ ali-to-post "ark:gunzip -c {alidir}/ali.JOB.gz|" ark:- \| \ post-to-tacc --per-pdf=true {alidir}/final.mdl ark:- \ {dir}/pdf_counts.JOB""".format(command=run_opts.command, num_jobs=num_jobs, dir=dir, alidir=alidir)) common_lib.execute_command( """{command} {dir}/log/sum_pdf_counts.log \ vector-sum --binary=false {dir}/pdf_counts.* {dir}/pdf_counts \ """.format(command=run_opts.command, dir=dir)) for file in glob.glob('{0}/pdf_counts.*'.format(dir)): os.remove(file) pdf_counts = common_lib.read_kaldi_matrix('{0}/pdf_counts'.format(dir))[0] scaled_counts = smooth_presoftmax_prior_scale_vector( pdf_counts, presoftmax_prior_scale_power=presoftmax_prior_scale_power, smooth=0.01) output_file = "{0}/presoftmax_prior_scale.vec".format(dir) common_lib.write_kaldi_matrix(output_file, [scaled_counts]) common_lib.force_symlink("../presoftmax_prior_scale.vec", "{0}/configs/presoftmax_prior_scale.vec".format( dir))
def add_back_compatibility_info(config_dir): """This will be removed when python script refactoring is done.""" common_lib.run_kaldi_command("nnet3-init {0}/ref.config " "{0}/ref.raw".format(config_dir)) out, err = common_lib.run_kaldi_command("nnet3-info {0}/ref.raw | " "head -4".format(config_dir)) # out looks like this # left-context: 7 # right-context: 0 # num-parameters: 90543902 # modulus: 1 info = {} for line in out.split("\n"): parts = line.split(":") if len(parts) != 2: continue info[parts[0].strip()] = int(parts[1].strip()) # Writing the back-compatible vars file # model_left_context=0 # model_right_context=7 # num_hidden_layers=3 vf = open('{0}/vars'.format(config_dir), 'w') vf.write('model_left_context={0}\n'.format(info['left-context'])) vf.write('model_right_context={0}\n'.format(info['right-context'])) vf.write('num_hidden_layers=1\n') vf.close() common_lib.force_symlink("final.config".format(config_dir), "{0}/layer1.config".format(config_dir))
def compute_preconditioning_matrix(dir, egs_dir, num_lda_jobs, run_opts, max_lda_jobs=None, rand_prune=4.0, lda_opts=None, use_multitask_egs=False): if max_lda_jobs is not None: if num_lda_jobs > max_lda_jobs: num_lda_jobs = max_lda_jobs multitask_egs_opts = common_train_lib.get_multitask_egs_opts( egs_dir, egs_prefix="egs.", archive_index="JOB", use_multitask_egs=use_multitask_egs) scp_or_ark = "scp" if use_multitask_egs else "ark" egs_rspecifier = ( "ark:nnet3-copy-egs {multitask_egs_opts} " "{scp_or_ark}:{egs_dir}/egs.JOB.{scp_or_ark} ark:- |" "".format(egs_dir=egs_dir, scp_or_ark=scp_or_ark, multitask_egs_opts=multitask_egs_opts)) # Write stats with the same format as stats for LDA. common_lib.execute_command( """{command} JOB=1:{num_lda_jobs} {dir}/log/get_lda_stats.JOB.log \ nnet3-acc-lda-stats --rand-prune={rand_prune} \ {dir}/init.raw "{egs_rspecifier}" \ {dir}/JOB.lda_stats""".format( command=run_opts.command, num_lda_jobs=num_lda_jobs, dir=dir, egs_rspecifier=egs_rspecifier, rand_prune=rand_prune)) # the above command would have generated dir/{1..num_lda_jobs}.lda_stats lda_stat_files = list(map(lambda x: '{0}/{1}.lda_stats'.format(dir, x), range(1, num_lda_jobs + 1))) common_lib.execute_command( """{command} {dir}/log/sum_transform_stats.log \ sum-lda-accs {dir}/lda_stats {lda_stat_files}""".format( command=run_opts.command, dir=dir, lda_stat_files=" ".join(lda_stat_files))) for file in lda_stat_files: try: os.remove(file) except OSError: logger.error("There was error while trying to remove " "lda stat files.") raise # this computes a fixed affine transform computed in the way we described # in Appendix C.6 of http://arxiv.org/pdf/1410.7455v6.pdf; it's a scaled # variant of an LDA transform but without dimensionality reduction. common_lib.execute_command( """{command} {dir}/log/get_transform.log \ nnet-get-feature-transform {lda_opts} {dir}/lda.mat \ {dir}/lda_stats""".format( command=run_opts.command, dir=dir, lda_opts=lda_opts if lda_opts is not None else "")) common_lib.force_symlink("../lda.mat", "{0}/configs/lda.mat".format(dir))
def compute_preconditioning_matrix(dir, egs_dir, num_lda_jobs, run_opts, max_lda_jobs=None, rand_prune=4.0, lda_opts=None): """ Function to estimate and write LDA matrix from cegs This function is exactly similar to the version in module libs.nnet3.train.frame_level_objf.common except this uses cegs instead of egs files. """ if max_lda_jobs is not None: if num_lda_jobs > max_lda_jobs: num_lda_jobs = max_lda_jobs # Write stats with the same format as stats for LDA. common_lib.execute_command( """{command} JOB=1:{num_lda_jobs} {dir}/log/get_lda_stats.JOB.log \ nnet3-chain-acc-lda-stats --rand-prune={rand_prune} \ {dir}/init.raw "ark:{egs_dir}/cegs.JOB.ark" \ {dir}/JOB.lda_stats""".format(command=run_opts.command, num_lda_jobs=num_lda_jobs, dir=dir, egs_dir=egs_dir, rand_prune=rand_prune)) # the above command would have generated dir/{1..num_lda_jobs}.lda_stats lda_stat_files = list( map(lambda x: '{0}/{1}.lda_stats'.format(dir, x), range(1, num_lda_jobs + 1))) common_lib.execute_command("""{command} {dir}/log/sum_transform_stats.log \ sum-lda-accs {dir}/lda_stats {lda_stat_files}""".format( command=run_opts.command, dir=dir, lda_stat_files=" ".join(lda_stat_files))) for file in lda_stat_files: try: os.remove(file) except OSError: raise Exception("There was error while trying to remove " "lda stat files.") # this computes a fixed affine transform computed in the way we described # in Appendix C.6 of http://arxiv.org/pdf/1410.7455v6.pdf; it's a scaled # variant of an LDA transform but without dimensionality reduction. common_lib.execute_command("""{command} {dir}/log/get_transform.log \ nnet-get-feature-transform {lda_opts} {dir}/lda.mat \ {dir}/lda_stats""".format( command=run_opts.command, dir=dir, lda_opts=lda_opts if lda_opts is not None else "")) common_lib.force_symlink("../lda.mat", "{0}/configs/lda.mat".format(dir))
def compute_preconditioning_matrix(dir, egs_dir, num_lda_jobs, run_opts, max_lda_jobs=None, rand_prune=4.0, lda_opts=None): """ Function to estimate and write LDA matrix from cegs This function is exactly similar to the version in module libs.nnet3.train.frame_level_objf.common except this uses cegs instead of egs files. """ if max_lda_jobs is not None: if num_lda_jobs > max_lda_jobs: num_lda_jobs = max_lda_jobs # Write stats with the same format as stats for LDA. common_lib.run_job( """{command} JOB=1:{num_lda_jobs} {dir}/log/get_lda_stats.JOB.log \ nnet3-chain-acc-lda-stats --rand-prune={rand_prune} \ {dir}/init.raw "ark:{egs_dir}/cegs.JOB.ark" \ {dir}/JOB.lda_stats""".format( command=run_opts.command, num_lda_jobs=num_lda_jobs, dir=dir, egs_dir=egs_dir, rand_prune=rand_prune)) # the above command would have generated dir/{1..num_lda_jobs}.lda_stats lda_stat_files = map(lambda x: '{0}/{1}.lda_stats'.format(dir, x), range(1, num_lda_jobs + 1)) common_lib.run_job( """{command} {dir}/log/sum_transform_stats.log \ sum-lda-accs {dir}/lda_stats {lda_stat_files}""".format( command=run_opts.command, dir=dir, lda_stat_files=" ".join(lda_stat_files))) for file in lda_stat_files: try: os.remove(file) except OSError: raise Exception("There was error while trying to remove " "lda stat files.") # this computes a fixed affine transform computed in the way we described # in Appendix C.6 of http://arxiv.org/pdf/1410.7455v6.pdf; it's a scaled # variant of an LDA transform but without dimensionality reduction. common_lib.run_job( """{command} {dir}/log/get_transform.log \ nnet-get-feature-transform {lda_opts} {dir}/lda.mat \ {dir}/lda_stats""".format( command=run_opts.command, dir=dir, lda_opts=lda_opts if lda_opts is not None else "")) common_lib.force_symlink("../lda.mat", "{0}/configs/lda.mat".format(dir))
def compute_presoftmax_prior_scale_targets(dir, counts_path, presoftmax_prior_scale_power=-0.25): # total num of frames per target already prepared target_counts = common_lib.read_kaldi_matrix(counts_path)[0] scaled_counts = smooth_presoftmax_prior_scale_vector( target_counts, presoftmax_prior_scale_power=presoftmax_prior_scale_power, smooth=0.01) output_file = "{0}/presoftmax_prior_scale.vec".format(dir) common_lib.write_kaldi_matrix(output_file, [scaled_counts]) common_lib.force_symlink("../presoftmax_prior_scale.vec", "{0}/configs/presoftmax_prior_scale.vec".format( dir))
def train(args, run_opts): """ The main function for training. Args: args: a Namespace object with the required parameters obtained from the function process_args() run_opts: RunOpts object obtained from the process_args() """ arg_string = pprint.pformat(vars(args)) logger.info("Arguments for the experiment\n{0}".format(arg_string)) # Set some variables. config_dir = '{0}/configs'.format(args.dir) am_var_file = '{0}/vars_am'.format(config_dir) xvec_var_file = '{0}/vars_xvec'.format(config_dir) am_variables = common_train_lib.parse_generic_config_vars_file(am_var_file) xvec_variables = common_train_lib.parse_generic_config_vars_file(xvec_var_file) # Set some variables. try: am_model_left_context = am_variables['model_left_context'] am_model_right_context = am_variables['model_right_context'] xvec_model_left_context = xvec_variables['model_left_context'] xvec_model_right_context = xvec_variables['model_right_context'] except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined in " "{1}".format(str(e), '{0}/configs'.format(args.dir))) am_left_context = am_model_left_context am_right_context = am_model_right_context xvec_left_context = xvec_model_left_context xvec_right_context = xvec_model_right_context # Initialize as "raw" nnet, prior to training the LDA-like preconditioning # matrix. This first config just does any initial splicing that we do; # we do this as it's a convenient way to get the stats for the 'lda-like' # transform. if (args.stage <= -5) and os.path.exists(args.dir+"/configs/init.config"): logger.info("Initializing a basic network for estimating " "preconditioning matrix") common_lib.execute_command( """{command} {dir}/log/nnet_init.log \ nnet3-init --srand=-2 {dir}/configs/init.config \ {dir}/init.raw""".format(command=run_opts.command, dir=args.dir)) am_egs_dir = args.am_egs_dir xvec_egs_dir = args.xvec_egs_dir am_output_name = args.am_output_name xvec_output_name = args.xvec_output_name am_weight = args.am_weight xvec_weight = args.xvec_weight feat_dim = int(common_lib.get_command_stdout("cat {0}/info/feat_dim".format(am_egs_dir))) num_archives = int(common_lib.get_command_stdout("cat {0}/info/num_archives".format(am_egs_dir))) tmp_feat_dim = int(common_lib.get_command_stdout("cat {0}/info/feat_dim".format(xvec_egs_dir))) tmp_num_archives = int(common_lib.get_command_stdout("cat {0}/info/num_archives".format(xvec_egs_dir))) # frames_per_eg is no longer a parameter but load from am_egs/info/frames_per_eg am_frames_per_eg = int(common_lib.get_command_stdout("cat {0}/info/frames_per_eg".format(am_egs_dir))) if feat_dim != tmp_feat_dim or num_archives*am_frames_per_eg != tmp_num_archives: raise Exception('The am egs and xvec egs do not match') if args.num_jobs_final > num_archives: raise Exception('num_jobs_final cannot exceed the number of archives ' 'in the egs directory') # # No need to copy files for decoding # common_train_lib.copy_egs_properties_to_exp_dir(am_egs_dir, args.dir) if args.stage <= -3 and os.path.exists(args.dir+"/configs/init.config"): logger.info('Computing the preconditioning matrix for input features') train_lib.common.compute_preconditioning_matrix( args.dir, egs_dir, num_archives, run_opts, max_lda_jobs=args.max_lda_jobs, rand_prune=args.rand_prune) if args.stage <= -1: logger.info("Preparing the initial network.") common_train_lib.prepare_initial_network(args.dir, run_opts) # set num_iters so that as close as possible, we process the data # $num_epochs times, i.e. $num_iters*$avg_num_jobs) == # $num_epochs*$num_archives, where # avg_num_jobs=(num_jobs_initial+num_jobs_final)/2. num_archives_expanded = num_archives * am_frames_per_eg num_archives_to_process = int(args.num_epochs * num_archives_expanded) num_archives_processed = 0 num_iters = ((num_archives_to_process * 2) / (args.num_jobs_initial + args.num_jobs_final)) # If do_final_combination is True, compute the set of models_to_combine. # Otherwise, models_to_combine will be none. if args.do_final_combination: models_to_combine = common_train_lib.get_model_combine_iters( num_iters, args.num_epochs, num_archives_expanded, args.max_models_combine, args.num_jobs_final) else: models_to_combine = None logger.info("Training will run for {0} epochs = " "{1} iterations".format(args.num_epochs, num_iters)) for iter in range(num_iters): if (args.exit_stage is not None) and (iter == args.exit_stage): logger.info("Exiting early due to --exit-stage {0}".format(iter)) return current_num_jobs = int(0.5 + args.num_jobs_initial + (args.num_jobs_final - args.num_jobs_initial) * float(iter) / num_iters) if args.stage <= iter: lrate = common_train_lib.get_learning_rate(iter, current_num_jobs, num_iters, num_archives_processed, num_archives_to_process, args.initial_effective_lrate, args.final_effective_lrate) shrinkage_value = 1.0 - (args.proportional_shrink * lrate) if shrinkage_value <= 0.5: raise Exception("proportional-shrink={0} is too large, it gives " "shrink-value={1}".format(args.proportional_shrink, shrinkage_value)) percent = num_archives_processed * 100.0 / num_archives_to_process epoch = (num_archives_processed * args.num_epochs / num_archives_to_process) shrink_info_str = '' if shrinkage_value != 1.0: shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value) logger.info("Iter: {0}/{1} " "Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) " "lr: {5:0.6f} {6}".format(iter, num_iters - 1, epoch, args.num_epochs, percent, lrate, shrink_info_str)) train_lib.common.train_cvector_one_iteration( dir=args.dir, iter=iter, srand=args.srand, am_output_name=am_output_name, am_weight=am_weight, am_egs_dir=am_egs_dir, xvec_output_name=xvec_output_name, xvec_weight=xvec_weight, xvec_egs_dir=xvec_egs_dir, num_jobs=current_num_jobs, num_archives_processed=num_archives_processed, num_archives=num_archives, learning_rate=lrate, minibatch_size_str=args.minibatch_size, momentum=args.momentum, max_param_change=args.max_param_change, shuffle_buffer_size=args.shuffle_buffer_size, run_opts=run_opts, am_frames_per_eg=am_frames_per_eg, dropout_edit_string=common_train_lib.get_dropout_edit_string( args.dropout_schedule, float(num_archives_processed) / num_archives_to_process, iter), shrinkage_value=shrinkage_value, get_raw_nnet_from_am=False, backstitch_training_scale=args.backstitch_training_scale, backstitch_training_interval=args.backstitch_training_interval) if args.cleanup: # do a clean up everythin but the last 2 models, under certain # conditions common_train_lib.remove_model( args.dir, iter-2, num_iters, models_to_combine, args.preserve_model_interval, get_raw_nnet_from_am=False) if args.email is not None: reporting_iter_interval = num_iters * args.reporting_interval if iter % reporting_iter_interval == 0: # lets do some reporting [report, times, data] = ( nnet3_log_parse.generate_acc_logprob_report(args.dir)) message = report subject = ("Update : Expt {dir} : " "Iter {iter}".format(dir=args.dir, iter=iter)) common_lib.send_mail(message, subject, args.email) num_archives_processed = num_archives_processed + current_num_jobs # when we do final combination, just use the xvector egs if args.stage <= num_iters: if args.do_final_combination: logger.info("Doing final combination to produce final.mdl") train_lib.common.combine_models( dir=args.dir, num_iters=num_iters, models_to_combine=models_to_combine, egs_dir=xvec_egs_dir, minibatch_size_str="64", run_opts=run_opts, get_raw_nnet_from_am=False, max_objective_evaluations=args.max_objective_evaluations, use_egs=True) # sum_to_one_penalty=args.combine_sum_to_one_penalty, else: common_lib.force_symlink("{0}.raw".format(num_iters), "{0}/final.raw".format(args.dir)) if args.cleanup: logger.info("Cleaning up the experiment directory " "{0}".format(args.dir)) remove_egs = False common_train_lib.clean_nnet_dir( nnet_dir=args.dir, num_iters=num_iters, egs_dir=am_egs_dir, preserve_model_interval=args.preserve_model_interval, remove_egs=remove_egs, get_raw_nnet_from_am=False) # TODO: we may trace other output nodes expect for "output" # do some reporting outputs_list = common_train_lib.get_outputs_list("{0}/final.raw".format( args.dir), get_raw_nnet_from_am=False) if 'output' in outputs_list: [report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir) if args.email is not None: common_lib.send_mail(report, "Update : Expt {0} : " "complete".format(args.dir), args.email) with open("{dir}/accuracy.{output_name}.report".format(dir=args.dir, output_name="output"), "w") as f: f.write(report) common_lib.execute_command("subtools/kaldi/steps/info/nnet3_dir_info.pl " "{0}".format(args.dir))
def train(args, run_opts): """ The main function for training. Args: args: a Namespace object with the required parameters obtained from the function process_args() run_opts: RunOpts object obtained from the process_args() """ arg_string = pprint.pformat(vars(args)) logger.info("Arguments for the experiment\n{0}".format(arg_string)) # Set some variables. # note, feat_dim gets set to 0 if args.feat_dir is unset (None). feat_dim = common_lib.get_feat_dim(args.feat_dir) ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir) ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir) config_dir = '{0}/configs'.format(args.dir) var_file = '{0}/vars'.format(config_dir) variables = common_train_lib.parse_generic_config_vars_file(var_file) # Set some variables. try: model_left_context = variables['model_left_context'] model_right_context = variables['model_right_context'] except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined in " "{1}".format(str(e), '{0}/configs'.format(args.dir))) left_context = model_left_context right_context = model_right_context # Initialize as "raw" nnet, prior to training the LDA-like preconditioning # matrix. This first config just does any initial splicing that we do; # we do this as it's a convenient way to get the stats for the 'lda-like' # transform. if (args.stage <= -5) and os.path.exists(args.dir + "/configs/init.config"): logger.info("Initializing the network for computing the LDA stats") common_lib.execute_command("""{command} {dir}/log/nnet_init.log \ nnet3-init --srand=-2 {dir}/configs/init.config \ {dir}/init.raw""".format(command=run_opts.command, dir=args.dir)) default_egs_dir = '{0}/egs'.format(args.dir) if (args.stage <= -4) and args.egs_dir is None: if args.targets_scp is None or args.feat_dir is None: raise Exception( "If you don't supply the --egs-dir option, the " "--targets-scp and --feat-dir options are required.") logger.info("Generating egs") if args.use_dense_targets: target_type = "dense" try: num_targets = int(variables['num_targets']) if (common_lib.get_feat_dim_from_scp(args.targets_scp) != num_targets): raise Exception("Mismatch between num-targets provided to " "script vs configs") except KeyError as e: num_targets = -1 else: target_type = "sparse" try: num_targets = int(variables['num_targets']) except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined " "in {1}".format(str(e), '{0}/configs'.format( args.dir))) train_lib.raw_model.generate_egs_using_targets( data=args.feat_dir, targets_scp=args.targets_scp, egs_dir=default_egs_dir, left_context=left_context, right_context=right_context, run_opts=run_opts, frames_per_eg_str=str(args.frames_per_eg), srand=args.srand, egs_opts=args.egs_opts, cmvn_opts=args.cmvn_opts, online_ivector_dir=args.online_ivector_dir, samples_per_iter=args.samples_per_iter, transform_dir=args.transform_dir, stage=args.egs_stage, target_type=target_type, num_targets=num_targets) if args.egs_dir is None: egs_dir = default_egs_dir else: egs_dir = args.egs_dir [egs_left_context, egs_right_context, frames_per_eg_str, num_archives] = (common_train_lib.verify_egs_dir(egs_dir, feat_dim, ivector_dim, ivector_id, left_context, right_context)) assert str(args.frames_per_eg) == frames_per_eg_str if args.num_jobs_final > num_archives: raise Exception('num_jobs_final cannot exceed the number of archives ' 'in the egs directory') # copy the properties of the egs to dir for # use during decoding common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir) if args.stage <= -3 and os.path.exists(args.dir + "/configs/init.config"): logger.info('Computing the preconditioning matrix for input features') train_lib.common.compute_preconditioning_matrix( args.dir, egs_dir, num_archives, run_opts, max_lda_jobs=args.max_lda_jobs, rand_prune=args.rand_prune) if args.stage <= -1: logger.info("Preparing the initial network.") common_train_lib.prepare_initial_network(args.dir, run_opts) # set num_iters so that as close as possible, we process the data # $num_epochs times, i.e. $num_iters*$avg_num_jobs) == # $num_epochs*$num_archives, where # avg_num_jobs=(num_jobs_initial+num_jobs_final)/2. num_archives_expanded = num_archives * args.frames_per_eg num_archives_to_process = int(args.num_epochs * num_archives_expanded) num_archives_processed = 0 num_iters = ((num_archives_to_process * 2) / (args.num_jobs_initial + args.num_jobs_final)) # If do_final_combination is True, compute the set of models_to_combine. # Otherwise, models_to_combine will be none. if args.do_final_combination: models_to_combine = common_train_lib.get_model_combine_iters( num_iters, args.num_epochs, num_archives_expanded, args.max_models_combine, args.num_jobs_final) else: models_to_combine = None if os.path.exists('{0}/valid_diagnostic.scp'.format(egs_dir)): if os.path.exists('{0}/valid_diagnostic.egs'.format(egs_dir)): raise Exception('both {0}/valid_diagnostic.egs and ' '{0}/valid_diagnostic.scp exist.' 'This script expects only one of them to exist.' ''.format(egs_dir)) use_multitask_egs = True else: if not os.path.exists('{0}/valid_diagnostic.egs'.format(egs_dir)): raise Exception('neither {0}/valid_diagnostic.egs nor ' '{0}/valid_diagnostic.scp exist.' 'This script expects one of them.' ''.format(egs_dir)) use_multitask_egs = False logger.info("Training will run for {0} epochs = " "{1} iterations".format(args.num_epochs, num_iters)) for iter in range(num_iters): if (args.exit_stage is not None) and (iter == args.exit_stage): logger.info("Exiting early due to --exit-stage {0}".format(iter)) return current_num_jobs = int(0.5 + args.num_jobs_initial + (args.num_jobs_final - args.num_jobs_initial) * float(iter) / num_iters) if args.stage <= iter: lrate = common_train_lib.get_learning_rate( iter, current_num_jobs, num_iters, num_archives_processed, num_archives_to_process, args.initial_effective_lrate, args.final_effective_lrate) shrinkage_value = 1.0 - (args.proportional_shrink * lrate) if shrinkage_value <= 0.5: raise Exception( "proportional-shrink={0} is too large, it gives " "shrink-value={1}".format(args.proportional_shrink, shrinkage_value)) train_lib.common.train_one_iteration( dir=args.dir, iter=iter, srand=args.srand, egs_dir=egs_dir, num_jobs=current_num_jobs, num_archives_processed=num_archives_processed, num_archives=num_archives, learning_rate=lrate, dropout_edit_string=common_train_lib.get_dropout_edit_string( args.dropout_schedule, float(num_archives_processed) / num_archives_to_process, iter), minibatch_size_str=args.minibatch_size, frames_per_eg=args.frames_per_eg, momentum=args.momentum, max_param_change=args.max_param_change, shrinkage_value=shrinkage_value, shuffle_buffer_size=args.shuffle_buffer_size, run_opts=run_opts, get_raw_nnet_from_am=False, image_augmentation_opts=args.image_augmentation_opts, use_multitask_egs=use_multitask_egs, backstitch_training_scale=args.backstitch_training_scale, backstitch_training_interval=args.backstitch_training_interval) if args.cleanup: # do a clean up everything but the last 2 models, under certain # conditions common_train_lib.remove_model(args.dir, iter - 2, num_iters, models_to_combine, args.preserve_model_interval, get_raw_nnet_from_am=False) if args.email is not None: reporting_iter_interval = num_iters * args.reporting_interval if iter % reporting_iter_interval == 0: # lets do some reporting [report, times, data] = (nnet3_log_parse.generate_acc_logprob_report( args.dir)) message = report subject = ("Update : Expt {dir} : " "Iter {iter}".format(dir=args.dir, iter=iter)) common_lib.send_mail(message, subject, args.email) num_archives_processed = num_archives_processed + current_num_jobs if args.stage <= num_iters: if args.do_final_combination: logger.info("Doing final combination to produce final.raw") train_lib.common.combine_models( dir=args.dir, num_iters=num_iters, models_to_combine=models_to_combine, egs_dir=egs_dir, minibatch_size_str=args.minibatch_size, run_opts=run_opts, get_raw_nnet_from_am=False, max_objective_evaluations=args.max_objective_evaluations, use_multitask_egs=use_multitask_egs) else: common_lib.force_symlink("{0}.raw".format(num_iters), "{0}/final.raw".format(args.dir)) if args.compute_average_posteriors and args.stage <= num_iters + 1: logger.info("Getting average posterior for output-node 'output'.") train_lib.common.compute_average_posterior( dir=args.dir, iter='final', egs_dir=egs_dir, num_archives=num_archives, prior_subset_size=args.prior_subset_size, run_opts=run_opts, get_raw_nnet_from_am=False) if args.cleanup: logger.info("Cleaning up the experiment directory " "{0}".format(args.dir)) remove_egs = args.remove_egs if args.egs_dir is not None: # this egs_dir was not created by this experiment so we will not # delete it remove_egs = False common_train_lib.clean_nnet_dir( nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir, preserve_model_interval=args.preserve_model_interval, remove_egs=remove_egs, get_raw_nnet_from_am=False) # do some reporting outputs_list = common_train_lib.get_outputs_list( "{0}/final.raw".format(args.dir), get_raw_nnet_from_am=False) if 'output' in outputs_list: [report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir) if args.email is not None: common_lib.send_mail( report, "Update : Expt {0} : " "complete".format(args.dir), args.email) with open( "{dir}/accuracy.{output_name}.report".format( dir=args.dir, output_name="output"), "w") as f: f.write(report) common_lib.execute_command("steps/info/nnet3_dir_info.pl " "{0}".format(args.dir))
def train(args, run_opts): """ The main function for training. Args: args: a Namespace object with the required parameters obtained from the function process_args() run_opts: RunOpts object obtained from the process_args() """ arg_string = pprint.pformat(vars(args)) logger.info("Arguments for the experiment\n{0}".format(arg_string)) # Set some variables. feat_dim = common_lib.get_feat_dim(args.feat_dir) ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir) ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir) if args.input_model is None: config_dir = '{0}/configs'.format(args.dir) var_file = '{0}/vars'.format(config_dir) variables = common_train_lib.parse_generic_config_vars_file(var_file) else: # If args.input_model is specified, the model left and right contexts # are computed using input_model. variables = common_train_lib.get_input_model_info(args.input_model) # Set some variables. try: model_left_context = variables['model_left_context'] model_right_context = variables['model_right_context'] except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined in " "{1}".format(str(e), '{0}/configs'.format(args.dir))) left_context = args.chunk_left_context + model_left_context right_context = args.chunk_right_context + model_right_context left_context_initial = (args.chunk_left_context_initial + model_left_context if args.chunk_left_context_initial >= 0 else -1) right_context_final = (args.chunk_right_context_final + model_right_context if args.chunk_right_context_final >= 0 else -1) # Initialize as "raw" nnet, prior to training the LDA-like preconditioning # matrix. This first config just does any initial splicing that we do; # we do this as it's a convenient way to get the stats for the 'lda-like' # transform. if (args.stage <= -4) and os.path.exists(args.dir+"/configs/init.config") and \ (args.input_model is None): logger.info("Initializing the network for computing the LDA stats") common_lib.execute_command( """{command} {dir}/log/nnet_init.log \ nnet3-init --srand=-2 {dir}/configs/init.config \ {dir}/init.raw""".format(command=run_opts.command, dir=args.dir)) default_egs_dir = '{0}/egs'.format(args.dir) if (args.stage <= -3) and args.egs_dir is None: logger.info("Generating egs") if args.use_dense_targets: target_type = "dense" try: num_targets = int(variables['num_targets']) if (common_lib.get_feat_dim_from_scp(args.targets_scp) != num_targets): raise Exception("Mismatch between num-targets provided to " "script vs configs") except KeyError as e: num_targets = -1 else: target_type = "sparse" try: num_targets = int(variables['num_targets']) except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined " "in {1}".format( str(e), '{0}/configs'.format(args.dir))) train_lib.raw_model.generate_egs_using_targets( data=args.feat_dir, targets_scp=args.targets_scp, egs_dir=default_egs_dir, left_context=left_context, right_context=right_context, left_context_initial=left_context_initial, right_context_final=right_context_final, run_opts=run_opts, frames_per_eg_str=args.chunk_width, srand=args.srand, egs_opts=args.egs_opts, cmvn_opts=args.cmvn_opts, online_ivector_dir=args.online_ivector_dir, samples_per_iter=args.samples_per_iter, stage=args.egs_stage, target_type=target_type, num_targets=num_targets) if args.egs_dir is None: egs_dir = default_egs_dir else: egs_dir = args.egs_dir [egs_left_context, egs_right_context, frames_per_eg_str, num_archives] = ( common_train_lib.verify_egs_dir(egs_dir, feat_dim, ivector_dim, ivector_id, left_context, right_context, left_context_initial, right_context_final)) if args.chunk_width != frames_per_eg_str: raise Exception("mismatch between --egs.chunk-width and the frames_per_eg " "in the egs dir {0} vs {1}".format(args.chunk_width, frames_per_eg_str)) if args.num_jobs_final > num_archives: raise Exception('num_jobs_final cannot exceed the number of archives ' 'in the egs directory') # copy the properties of the egs to dir for # use during decoding common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir) if args.stage <= -2 and os.path.exists(args.dir+"/configs/init.config") and \ (args.input_model is None): logger.info('Computing the preconditioning matrix for input features') train_lib.common.compute_preconditioning_matrix( args.dir, egs_dir, num_archives, run_opts, max_lda_jobs=args.max_lda_jobs, rand_prune=args.rand_prune) if args.stage <= -1: logger.info("Preparing the initial network.") common_train_lib.prepare_initial_network(args.dir, run_opts, args.srand, args.input_model) # set num_iters so that as close as possible, we process the data # $num_epochs times, i.e. $num_iters*$avg_num_jobs) == # $num_epochs*$num_archives, where # avg_num_jobs=(num_jobs_initial+num_jobs_final)/2. num_archives_to_process = int(args.num_epochs * num_archives) num_archives_processed = 0 num_iters = int((num_archives_to_process * 2) / (args.num_jobs_initial + args.num_jobs_final)) # If do_final_combination is True, compute the set of models_to_combine. # Otherwise, models_to_combine will be none. if args.do_final_combination: models_to_combine = common_train_lib.get_model_combine_iters( num_iters, args.num_epochs, num_archives, args.max_models_combine, args.num_jobs_final) else: models_to_combine = None if (os.path.exists('{0}/valid_diagnostic.scp'.format(egs_dir))): if (os.path.exists('{0}/valid_diagnostic.egs'.format(egs_dir))): raise Exception('both {0}/valid_diagnostic.egs and ' '{0}/valid_diagnostic.scp exist.' 'This script expects only one of them to exist.' ''.format(egs_dir)) use_multitask_egs = True else: if (not os.path.exists('{0}/valid_diagnostic.egs' ''.format(egs_dir))): raise Exception('neither {0}/valid_diagnostic.egs nor ' '{0}/valid_diagnostic.scp exist.' 'This script expects one of them.' ''.format(egs_dir)) use_multitask_egs = False min_deriv_time = None max_deriv_time_relative = None if args.deriv_truncate_margin is not None: min_deriv_time = -args.deriv_truncate_margin - model_left_context max_deriv_time_relative = \ args.deriv_truncate_margin + model_right_context logger.info("Training will run for {0} epochs = " "{1} iterations".format(args.num_epochs, num_iters)) for iter in range(num_iters): if (args.exit_stage is not None) and (iter == args.exit_stage): logger.info("Exiting early due to --exit-stage {0}".format(iter)) return current_num_jobs = common_train_lib.get_current_num_jobs( iter, num_iters, args.num_jobs_initial, args.num_jobs_step, args.num_jobs_final) if args.stage <= iter: model_file = "{dir}/{iter}.raw".format(dir=args.dir, iter=iter) lrate = common_train_lib.get_learning_rate(iter, current_num_jobs, num_iters, num_archives_processed, num_archives_to_process, args.initial_effective_lrate, args.final_effective_lrate) # shrinkage_value is a scale on the parameters. shrinkage_value = 1.0 - (args.proportional_shrink * lrate) if shrinkage_value <= 0.5: raise Exception("proportional-shrink={0} is too large, it gives " "shrink-value={1}".format(args.proportional_shrink, shrinkage_value)) if args.shrink_value < shrinkage_value: shrinkage_value = (args.shrink_value if common_train_lib.should_do_shrinkage( iter, model_file, args.shrink_saturation_threshold, get_raw_nnet_from_am=False) else shrinkage_value) percent = num_archives_processed * 100.0 / num_archives_to_process epoch = (num_archives_processed * args.num_epochs / num_archives_to_process) shrink_info_str = '' if shrinkage_value != 1.0: shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value) logger.info("Iter: {0}/{1} Jobs: {2} " "Epoch: {3:0.2f}/{4:0.1f} ({5:0.1f}% complete) " "lr: {6:0.6f} {7}".format(iter, num_iters - 1, current_num_jobs, epoch, args.num_epochs, percent, lrate, shrink_info_str)) train_lib.common.train_one_iteration( dir=args.dir, iter=iter, srand=args.srand, egs_dir=egs_dir, num_jobs=current_num_jobs, num_archives_processed=num_archives_processed, num_archives=num_archives, learning_rate=lrate, dropout_edit_string=common_train_lib.get_dropout_edit_string( args.dropout_schedule, float(num_archives_processed) / num_archives_to_process, iter), train_opts=' '.join(args.train_opts), shrinkage_value=shrinkage_value, minibatch_size_str=args.num_chunk_per_minibatch, min_deriv_time=min_deriv_time, max_deriv_time_relative=max_deriv_time_relative, momentum=args.momentum, max_param_change=args.max_param_change, shuffle_buffer_size=args.shuffle_buffer_size, run_opts=run_opts, get_raw_nnet_from_am=False, use_multitask_egs=use_multitask_egs, compute_per_dim_accuracy=args.compute_per_dim_accuracy) if args.cleanup: # do a clean up everythin but the last 2 models, under certain # conditions common_train_lib.remove_model( args.dir, iter-2, num_iters, models_to_combine, args.preserve_model_interval, get_raw_nnet_from_am=False) if args.email is not None: reporting_iter_interval = num_iters * args.reporting_interval if iter % reporting_iter_interval == 0: # lets do some reporting [report, times, data] = ( nnet3_log_parse.generate_acc_logprob_report(args.dir)) message = report subject = ("Update : Expt {dir} : " "Iter {iter}".format(dir=args.dir, iter=iter)) common_lib.send_mail(message, subject, args.email) num_archives_processed = num_archives_processed + current_num_jobs if args.stage <= num_iters: if args.do_final_combination: logger.info("Doing final combination to produce final.raw") train_lib.common.combine_models( dir=args.dir, num_iters=num_iters, models_to_combine=models_to_combine, egs_dir=egs_dir, minibatch_size_str=args.num_chunk_per_minibatch, run_opts=run_opts, chunk_width=args.chunk_width, get_raw_nnet_from_am=False, compute_per_dim_accuracy=args.compute_per_dim_accuracy, max_objective_evaluations=args.max_objective_evaluations, use_multitask_egs=use_multitask_egs) else: common_lib.force_symlink("{0}.raw".format(num_iters), "{0}/final.raw".format(args.dir)) if args.compute_average_posteriors and args.stage <= num_iters + 1: logger.info("Getting average posterior for purposes of " "adjusting the priors.") train_lib.common.compute_average_posterior( dir=args.dir, iter='final', egs_dir=egs_dir, num_archives=num_archives, prior_subset_size=args.prior_subset_size, run_opts=run_opts, get_raw_nnet_from_am=False) if args.cleanup: logger.info("Cleaning up the experiment directory " "{0}".format(args.dir)) remove_egs = args.remove_egs if args.egs_dir is not None: # this egs_dir was not created by this experiment so we will not # delete it remove_egs = False common_train_lib.clean_nnet_dir( nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir, preserve_model_interval=args.preserve_model_interval, remove_egs=remove_egs, get_raw_nnet_from_am=False) # do some reporting [report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir) if args.email is not None: common_lib.send_mail(report, "Update : Expt {0} : " "complete".format(args.dir), args.email) with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f: f.write(report) common_lib.execute_command("subtools/kaldi/steps/info/nnet3_dir_info.pl " "{0}".format(args.dir))
def train(args, run_opts): """ The main function for training. Args: args: a Namespace object with the required parameters obtained from the function process_args() run_opts: RunOpts object obtained from the process_args() """ arg_string = pprint.pformat(vars(args)) logger.info("Arguments for the experiment\n{0}".format(arg_string)) # Set some variables. feat_dim = common_lib.get_feat_dim(args.feat_dir) ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir) ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir) config_dir = '{0}/configs'.format(args.dir) var_file = '{0}/vars'.format(config_dir) variables = common_train_lib.parse_generic_config_vars_file(var_file) # Set some variables. try: model_left_context = variables['model_left_context'] model_right_context = variables['model_right_context'] except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined in " "{1}".format(str(e), '{0}/configs'.format(args.dir))) left_context = args.chunk_left_context + model_left_context right_context = args.chunk_right_context + model_right_context left_context_initial = (args.chunk_left_context_initial + model_left_context if args.chunk_left_context_initial >= 0 else -1) right_context_final = (args.chunk_right_context_final + model_right_context if args.chunk_right_context_final >= 0 else -1) # Initialize as "raw" nnet, prior to training the LDA-like preconditioning # matrix. This first config just does any initial splicing that we do; # we do this as it's a convenient way to get the stats for the 'lda-like' # transform. if (args.stage <= -4) and os.path.exists(args.dir+"/configs/init.config"): logger.info("Initializing the network for computing the LDA stats") common_lib.execute_command( """{command} {dir}/log/nnet_init.log \ nnet3-init --srand=-2 {dir}/configs/init.config \ {dir}/init.raw""".format(command=run_opts.command, dir=args.dir)) default_egs_dir = '{0}/egs'.format(args.dir) if (args.stage <= -3) and args.egs_dir is None: logger.info("Generating egs") if args.use_dense_targets: target_type = "dense" try: num_targets = int(variables['num_targets']) if (common_lib.get_feat_dim_from_scp(args.targets_scp) != num_targets): raise Exception("Mismatch between num-targets provided to " "script vs configs") except KeyError as e: num_targets = -1 else: target_type = "sparse" try: num_targets = int(variables['num_targets']) except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined " "in {1}".format( str(e), '{0}/configs'.format(args.dir))) train_lib.raw_model.generate_egs_using_targets( data=args.feat_dir, targets_scp=args.targets_scp, egs_dir=default_egs_dir, left_context=left_context, right_context=right_context, left_context_initial=left_context_initial, right_context_final=right_context_final, run_opts=run_opts, frames_per_eg_str=args.chunk_width, srand=args.srand, egs_opts=args.egs_opts, cmvn_opts=args.cmvn_opts, online_ivector_dir=args.online_ivector_dir, samples_per_iter=args.samples_per_iter, stage=args.egs_stage, target_type=target_type, num_targets=num_targets) if args.egs_dir is None: egs_dir = default_egs_dir else: egs_dir = args.egs_dir [egs_left_context, egs_right_context, frames_per_eg_str, num_archives] = ( common_train_lib.verify_egs_dir(egs_dir, feat_dim, ivector_dim, ivector_id, left_context, right_context, left_context_initial, right_context_final)) if args.chunk_width != frames_per_eg_str: raise Exception("mismatch between --egs.chunk-width and the frames_per_eg " "in the egs dir {0} vs {1}".format(args.chunk_width, frames_per_eg_str)) if args.num_jobs_final > num_archives: raise Exception('num_jobs_final cannot exceed the number of archives ' 'in the egs directory') # copy the properties of the egs to dir for # use during decoding common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir) if args.stage <= -2 and os.path.exists(args.dir+"/configs/init.config"): logger.info('Computing the preconditioning matrix for input features') train_lib.common.compute_preconditioning_matrix( args.dir, egs_dir, num_archives, run_opts, max_lda_jobs=args.max_lda_jobs, rand_prune=args.rand_prune) if args.stage <= -1: logger.info("Preparing the initial network.") common_train_lib.prepare_initial_network(args.dir, run_opts) # set num_iters so that as close as possible, we process the data # $num_epochs times, i.e. $num_iters*$avg_num_jobs) == # $num_epochs*$num_archives, where # avg_num_jobs=(num_jobs_initial+num_jobs_final)/2. num_archives_to_process = int(args.num_epochs * num_archives) num_archives_processed = 0 num_iters = ((num_archives_to_process * 2) / (args.num_jobs_initial + args.num_jobs_final)) # If do_final_combination is True, compute the set of models_to_combine. # Otherwise, models_to_combine will be none. if args.do_final_combination: models_to_combine = common_train_lib.get_model_combine_iters( num_iters, args.num_epochs, num_archives, args.max_models_combine, args.num_jobs_final) else: models_to_combine = None if (os.path.exists('{0}/valid_diagnostic.scp'.format(egs_dir))): if (os.path.exists('{0}/valid_diagnostic.egs'.format(egs_dir))): raise Exception('both {0}/valid_diagnostic.egs and ' '{0}/valid_diagnostic.scp exist.' 'This script expects only one of them to exist.' ''.format(egs_dir)) use_multitask_egs = True else: if (not os.path.exists('{0}/valid_diagnostic.egs' ''.format(egs_dir))): raise Exception('neither {0}/valid_diagnostic.egs nor ' '{0}/valid_diagnostic.scp exist.' 'This script expects one of them.' ''.format(egs_dir)) use_multitask_egs = False min_deriv_time = None max_deriv_time_relative = None if args.deriv_truncate_margin is not None: min_deriv_time = -args.deriv_truncate_margin - model_left_context max_deriv_time_relative = \ args.deriv_truncate_margin + model_right_context logger.info("Training will run for {0} epochs = " "{1} iterations".format(args.num_epochs, num_iters)) for iter in range(num_iters): if (args.exit_stage is not None) and (iter == args.exit_stage): logger.info("Exiting early due to --exit-stage {0}".format(iter)) return current_num_jobs = int(0.5 + args.num_jobs_initial + (args.num_jobs_final - args.num_jobs_initial) * float(iter) / num_iters) if args.stage <= iter: model_file = "{dir}/{iter}.raw".format(dir=args.dir, iter=iter) lrate = common_train_lib.get_learning_rate(iter, current_num_jobs, num_iters, num_archives_processed, num_archives_to_process, args.initial_effective_lrate, args.final_effective_lrate) # shrinkage_value is a scale on the parameters. shrinkage_value = 1.0 - (args.proportional_shrink * lrate) if shrinkage_value <= 0.5: raise Exception("proportional-shrink={0} is too large, it gives " "shrink-value={1}".format(args.proportional_shrink, shrinkage_value)) if args.shrink_value < shrinkage_value: shrinkage_value = (args.shrink_value if common_train_lib.should_do_shrinkage( iter, model_file, args.shrink_saturation_threshold, get_raw_nnet_from_am=False) else shrinkage_value) percent = num_archives_processed * 100.0 / num_archives_to_process epoch = (num_archives_processed * args.num_epochs / num_archives_to_process) shrink_info_str = '' if shrinkage_value != 1.0: shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value) logger.info("Iter: {0}/{1} " "Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) " "lr: {5:0.6f} {6}".format(iter, num_iters - 1, epoch, args.num_epochs, percent, lrate, shrink_info_str)) train_lib.common.train_one_iteration( dir=args.dir, iter=iter, srand=args.srand, egs_dir=egs_dir, num_jobs=current_num_jobs, num_archives_processed=num_archives_processed, num_archives=num_archives, learning_rate=lrate, dropout_edit_string=common_train_lib.get_dropout_edit_string( args.dropout_schedule, float(num_archives_processed) / num_archives_to_process, iter), train_opts=' '.join(args.train_opts), shrinkage_value=shrinkage_value, minibatch_size_str=args.num_chunk_per_minibatch, min_deriv_time=min_deriv_time, max_deriv_time_relative=max_deriv_time_relative, momentum=args.momentum, max_param_change=args.max_param_change, shuffle_buffer_size=args.shuffle_buffer_size, run_opts=run_opts, get_raw_nnet_from_am=False, use_multitask_egs=use_multitask_egs, compute_per_dim_accuracy=args.compute_per_dim_accuracy) if args.cleanup: # do a clean up everythin but the last 2 models, under certain # conditions common_train_lib.remove_model( args.dir, iter-2, num_iters, models_to_combine, args.preserve_model_interval, get_raw_nnet_from_am=False) if args.email is not None: reporting_iter_interval = num_iters * args.reporting_interval if iter % reporting_iter_interval == 0: # lets do some reporting [report, times, data] = ( nnet3_log_parse.generate_acc_logprob_report(args.dir)) message = report subject = ("Update : Expt {dir} : " "Iter {iter}".format(dir=args.dir, iter=iter)) common_lib.send_mail(message, subject, args.email) num_archives_processed = num_archives_processed + current_num_jobs if args.stage <= num_iters: if args.do_final_combination: logger.info("Doing final combination to produce final.raw") train_lib.common.combine_models( dir=args.dir, num_iters=num_iters, models_to_combine=models_to_combine, egs_dir=egs_dir, minibatch_size_str=args.num_chunk_per_minibatch, run_opts=run_opts, chunk_width=args.chunk_width, get_raw_nnet_from_am=False, compute_per_dim_accuracy=args.compute_per_dim_accuracy, max_objective_evaluations=args.max_objective_evaluations) else: common_lib.force_symlink("{0}.raw".format(num_iters), "{0}/final.raw".format(args.dir)) if args.compute_average_posteriors and args.stage <= num_iters + 1: logger.info("Getting average posterior for purposes of " "adjusting the priors.") train_lib.common.compute_average_posterior( dir=args.dir, iter='final', egs_dir=egs_dir, num_archives=num_archives, prior_subset_size=args.prior_subset_size, run_opts=run_opts, get_raw_nnet_from_am=False) if args.cleanup: logger.info("Cleaning up the experiment directory " "{0}".format(args.dir)) remove_egs = args.remove_egs if args.egs_dir is not None: # this egs_dir was not created by this experiment so we will not # delete it remove_egs = False common_train_lib.clean_nnet_dir( nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir, preserve_model_interval=args.preserve_model_interval, remove_egs=remove_egs, get_raw_nnet_from_am=False) # do some reporting [report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir) if args.email is not None: common_lib.send_mail(report, "Update : Expt {0} : " "complete".format(args.dir), args.email) with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f: f.write(report) common_lib.execute_command("steps/info/nnet3_dir_info.pl " "{0}".format(args.dir))
def train(args, run_opts): """ The main function for training. Args: args: a Namespace object with the required parameters obtained from the function process_args() run_opts: RunOpts object obtained from the process_args() """ arg_string = pprint.pformat(vars(args)) logger.info("Arguments for the experiment\n{0}".format(arg_string)) # Check files chain_lib.check_for_required_files( args.feat_dir, args.tree_dir, args.lat_dir if args.egs_dir is None else None) # Copy phones.txt from tree-dir to dir. Later, steps/nnet3/decode.sh will # use it to check compatibility between training and decoding phone-sets. shutil.copy('{0}/phones.txt'.format(args.tree_dir), args.dir) # Set some variables. num_jobs = common_lib.get_number_of_jobs(args.tree_dir) feat_dim = common_lib.get_feat_dim(args.feat_dir) ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir) ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir) # split the training data into parts for individual jobs # we will use the same number of jobs as that used for alignment common_lib.execute_command("utils/split_data.sh {0} {1}" "".format(args.feat_dir, num_jobs)) with open('{0}/num_jobs'.format(args.dir), 'w') as f: f.write(str(num_jobs)) if args.input_model is None: config_dir = '{0}/configs'.format(args.dir) var_file = '{0}/vars'.format(config_dir) variables = common_train_lib.parse_generic_config_vars_file(var_file) else: # If args.input_model is specified, the model left and right contexts # are computed using input_model. variables = common_train_lib.get_input_model_info(args.input_model) # Set some variables. try: model_left_context = variables['model_left_context'] model_right_context = variables['model_right_context'] except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined in " "{1}".format(str(e), '{0}/configs'.format(args.dir))) left_context = args.chunk_left_context + model_left_context right_context = args.chunk_right_context + model_right_context left_context_initial = (args.chunk_left_context_initial + model_left_context if args.chunk_left_context_initial >= 0 else -1) right_context_final = (args.chunk_right_context_final + model_right_context if args.chunk_right_context_final >= 0 else -1) # Initialize as "raw" nnet, prior to training the LDA-like preconditioning # matrix. This first config just does any initial splicing that we do; # we do this as it's a convenient way to get the stats for the 'lda-like' # transform. if (args.stage <= -6): logger.info("Creating phone language-model") chain_lib.create_phone_lm(args.dir, args.tree_dir, run_opts, lm_opts=args.lm_opts) if (args.stage <= -5): logger.info("Creating denominator FST") shutil.copy('{0}/tree'.format(args.tree_dir), args.dir) chain_lib.create_denominator_fst(args.dir, args.tree_dir, run_opts) if ((args.stage <= -4) and os.path.exists("{0}/configs/init.config".format(args.dir)) and (args.input_model is None)): logger.info("Initializing a basic network for estimating " "preconditioning matrix") common_lib.execute_command("""{command} {dir}/log/nnet_init.log \ nnet3-init --srand=-2 {dir}/configs/init.config \ {dir}/init.raw""".format(command=run_opts.command, dir=args.dir)) egs_left_context = left_context + args.frame_subsampling_factor // 2 egs_right_context = right_context + args.frame_subsampling_factor // 2 # note: the '+ args.frame_subsampling_factor / 2' is to allow for the # fact that we'll be shifting the data slightly during training to give # variety to the training data. egs_left_context_initial = (left_context_initial + args.frame_subsampling_factor // 2 if left_context_initial >= 0 else -1) egs_right_context_final = (right_context_final + args.frame_subsampling_factor // 2 if right_context_final >= 0 else -1) default_egs_dir = '{0}/egs'.format(args.dir) if ((args.stage <= -3) and args.egs_dir is None): logger.info("Generating egs") if (not os.path.exists("{0}/den.fst".format(args.dir)) or not os.path.exists("{0}/normalization.fst".format(args.dir)) or not os.path.exists("{0}/tree".format(args.dir))): raise Exception("Chain egs generation expects {0}/den.fst, " "{0}/normalization.fst and {0}/tree " "to exist.".format(args.dir)) # this is where get_egs.sh is called. chain_lib.generate_chain_egs( dir=args.dir, data=args.feat_dir, lat_dir=args.lat_dir, egs_dir=default_egs_dir, left_context=egs_left_context, right_context=egs_right_context, left_context_initial=egs_left_context_initial, right_context_final=egs_right_context_final, run_opts=run_opts, left_tolerance=args.left_tolerance, right_tolerance=args.right_tolerance, frame_subsampling_factor=args.frame_subsampling_factor, alignment_subsampling_factor=args.alignment_subsampling_factor, frames_per_eg_str=args.chunk_width, srand=args.srand, egs_opts=args.egs_opts, cmvn_opts=args.cmvn_opts, online_ivector_dir=args.online_ivector_dir, frames_per_iter=args.frames_per_iter, stage=args.egs_stage) if args.egs_dir is None: egs_dir = default_egs_dir else: egs_dir = args.egs_dir [egs_left_context, egs_right_context, frames_per_eg_str, num_archives] = (common_train_lib.verify_egs_dir( egs_dir, feat_dim, ivector_dim, ivector_id, egs_left_context, egs_right_context, egs_left_context_initial, egs_right_context_final)) assert (args.chunk_width == frames_per_eg_str) num_archives_expanded = num_archives * args.frame_subsampling_factor if (args.num_jobs_final > num_archives_expanded): raise Exception('num_jobs_final cannot exceed the ' 'expanded number of archives') # copy the properties of the egs to dir for # use during decoding logger.info("Copying the properties from {0} to {1}".format( egs_dir, args.dir)) common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir) if not os.path.exists('{0}/valid_diagnostic.cegs'.format(egs_dir)): if (not os.path.exists('{0}/valid_diagnostic.scp'.format(egs_dir))): raise Exception('Neither {0}/valid_diagnostic.cegs nor ' '{0}/valid_diagnostic.scp exist.' 'This script expects one of them.'.format(egs_dir)) use_multitask_egs = True else: use_multitask_egs = False if ((args.stage <= -2) and (os.path.exists(args.dir + "/configs/init.config")) and (args.input_model is None)): logger.info('Computing the preconditioning matrix for input features') chain_lib.compute_preconditioning_matrix( args.dir, egs_dir, num_archives, run_opts, max_lda_jobs=args.max_lda_jobs, rand_prune=args.rand_prune, use_multitask_egs=use_multitask_egs) if (args.stage <= -1): logger.info("Preparing the initial acoustic model.") chain_lib.prepare_initial_acoustic_model(args.dir, run_opts, input_model=args.input_model) with open("{0}/frame_subsampling_factor".format(args.dir), "w") as f: f.write(str(args.frame_subsampling_factor)) # set num_iters so that as close as possible, we process the data # $num_epochs times, i.e. $num_iters*$avg_num_jobs) == # $num_epochs*$num_archives, where # avg_num_jobs=(num_jobs_initial+num_jobs_final)/2. num_archives_to_process = int(args.num_epochs * num_archives_expanded) num_archives_processed = 0 num_iters = ((num_archives_to_process * 2) // (args.num_jobs_initial + args.num_jobs_final)) # If do_final_combination is True, compute the set of models_to_combine. # Otherwise, models_to_combine will be none. if args.do_final_combination: models_to_combine = common_train_lib.get_model_combine_iters( num_iters, args.num_epochs, num_archives_expanded, args.max_models_combine, args.num_jobs_final) else: models_to_combine = None min_deriv_time = None max_deriv_time_relative = None if args.deriv_truncate_margin is not None: min_deriv_time = -args.deriv_truncate_margin - model_left_context max_deriv_time_relative = \ args.deriv_truncate_margin + model_right_context logger.info("Training will run for {0} epochs = " "{1} iterations".format(args.num_epochs, num_iters)) for iter in range(num_iters): if (args.exit_stage is not None) and (iter == args.exit_stage): logger.info("Exiting early due to --exit-stage {0}".format(iter)) return current_num_jobs = int(0.5 + args.num_jobs_initial + (args.num_jobs_final - args.num_jobs_initial) * float(iter) / num_iters) if args.stage <= iter: model_file = "{dir}/{iter}.mdl".format(dir=args.dir, iter=iter) lrate = common_train_lib.get_learning_rate( iter, current_num_jobs, num_iters, num_archives_processed, num_archives_to_process, args.initial_effective_lrate, args.final_effective_lrate) shrinkage_value = 1.0 - (args.proportional_shrink * lrate) if shrinkage_value <= 0.5: raise Exception( "proportional-shrink={0} is too large, it gives " "shrink-value={1}".format(args.proportional_shrink, shrinkage_value)) if args.shrink_value < shrinkage_value: shrinkage_value = ( args.shrink_value if common_train_lib.should_do_shrinkage( iter, model_file, args.shrink_saturation_threshold) else shrinkage_value) percent = num_archives_processed * 100.0 / num_archives_to_process epoch = (num_archives_processed * args.num_epochs / num_archives_to_process) shrink_info_str = '' if shrinkage_value != 1.0: shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value) logger.info("Iter: {0}/{1} " "Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) " "lr: {5:0.6f} {6}".format(iter, num_iters - 1, epoch, args.num_epochs, percent, lrate, shrink_info_str)) chain_lib.train_one_iteration( dir=args.dir, iter=iter, srand=args.srand, egs_dir=egs_dir, num_jobs=current_num_jobs, num_archives_processed=num_archives_processed, num_archives=num_archives, learning_rate=lrate, dropout_edit_string=common_train_lib.get_dropout_edit_string( args.dropout_schedule, float(num_archives_processed) / num_archives_to_process, iter), train_opts=' '.join(args.train_opts), shrinkage_value=shrinkage_value, num_chunk_per_minibatch_str=args.num_chunk_per_minibatch, apply_deriv_weights=args.apply_deriv_weights, min_deriv_time=min_deriv_time, max_deriv_time_relative=max_deriv_time_relative, l2_regularize=args.l2_regularize, xent_regularize=args.xent_regularize, leaky_hmm_coefficient=args.leaky_hmm_coefficient, momentum=args.momentum, max_param_change=args.max_param_change, shuffle_buffer_size=args.shuffle_buffer_size, frame_subsampling_factor=args.frame_subsampling_factor, run_opts=run_opts, backstitch_training_scale=args.backstitch_training_scale, backstitch_training_interval=args.backstitch_training_interval, use_multitask_egs=use_multitask_egs) if args.cleanup: # do a clean up everything but the last 2 models, under certain # conditions common_train_lib.remove_model(args.dir, iter - 2, num_iters, models_to_combine, args.preserve_model_interval) if args.email is not None: reporting_iter_interval = num_iters * args.reporting_interval if iter % reporting_iter_interval == 0: # lets do some reporting [report, times, data] = (nnet3_log_parse.generate_acc_logprob_report( args.dir, "log-probability")) message = report subject = ("Update : Expt {dir} : " "Iter {iter}".format(dir=args.dir, iter=iter)) common_lib.send_mail(message, subject, args.email) num_archives_processed = num_archives_processed + current_num_jobs if args.stage <= num_iters: if args.do_final_combination: logger.info("Doing final combination to produce final.mdl") chain_lib.combine_models( dir=args.dir, num_iters=num_iters, models_to_combine=models_to_combine, num_chunk_per_minibatch_str=args.num_chunk_per_minibatch, egs_dir=egs_dir, leaky_hmm_coefficient=args.leaky_hmm_coefficient, l2_regularize=args.l2_regularize, xent_regularize=args.xent_regularize, run_opts=run_opts, max_objective_evaluations=args.max_objective_evaluations, use_multitask_egs=use_multitask_egs) else: logger.info("Copying the last-numbered model to final.mdl") common_lib.force_symlink("{0}.mdl".format(num_iters), "{0}/final.mdl".format(args.dir)) chain_lib.compute_train_cv_probabilities( dir=args.dir, iter=num_iters, egs_dir=egs_dir, l2_regularize=args.l2_regularize, xent_regularize=args.xent_regularize, leaky_hmm_coefficient=args.leaky_hmm_coefficient, run_opts=run_opts, use_multitask_egs=use_multitask_egs) common_lib.force_symlink( "compute_prob_valid.{iter}.log" "".format(iter=num_iters), "{dir}/log/compute_prob_valid.final.log".format(dir=args.dir)) if args.cleanup: logger.info("Cleaning up the experiment directory " "{0}".format(args.dir)) remove_egs = args.remove_egs if args.egs_dir is not None: # this egs_dir was not created by this experiment so we will not # delete it remove_egs = False # leave the last-two-numbered models, for diagnostic reasons. common_train_lib.clean_nnet_dir( args.dir, num_iters - 1, egs_dir, preserve_model_interval=args.preserve_model_interval, remove_egs=remove_egs) # do some reporting [report, times, data ] = nnet3_log_parse.generate_acc_logprob_report(args.dir, "log-probability") if args.email is not None: common_lib.send_mail( report, "Update : Expt {0} : " "complete".format(args.dir), args.email) with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f: f.write(report) common_lib.execute_command("steps/info/chain_dir_info.pl " "{0}".format(args.dir))
def train(args, run_opts): """ The main function for training. Args: args: a Namespace object with the required parameters obtained from the function process_args() run_opts: RunOpts object obtained from the process_args() """ arg_string = pprint.pformat(vars(args)) logger.info("Arguments for the experiment\n{0}".format(arg_string)) # Check files chain_lib.check_for_required_files(args.feat_dir, args.tree_dir, args.lat_dir) # Copy phones.txt from tree-dir to dir. Later, steps/nnet3/decode.sh will # use it to check compatibility between training and decoding phone-sets. shutil.copy('{0}/phones.txt'.format(args.tree_dir), args.dir) # Set some variables. num_jobs = common_lib.get_number_of_jobs(args.tree_dir) feat_dim = common_lib.get_feat_dim(args.feat_dir) ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir) ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir) # split the training data into parts for individual jobs # we will use the same number of jobs as that used for alignment common_lib.execute_command("utils/split_data.sh {0} {1}" "".format(args.feat_dir, num_jobs)) with open('{0}/num_jobs'.format(args.dir), 'w') as f: f.write(str(num_jobs)) if args.input_model is None: config_dir = '{0}/configs'.format(args.dir) var_file = '{0}/vars'.format(config_dir) variables = common_train_lib.parse_generic_config_vars_file(var_file) else: # If args.input_model is specified, the model left and right contexts # are computed using input_model. variables = common_train_lib.get_input_model_info(args.input_model) # Set some variables. try: model_left_context = variables['model_left_context'] model_right_context = variables['model_right_context'] except KeyError as e: raise Exception("KeyError {0}: Variables need to be defined in " "{1}".format(str(e), '{0}/configs'.format(args.dir))) left_context = args.chunk_left_context + model_left_context right_context = args.chunk_right_context + model_right_context left_context_initial = (args.chunk_left_context_initial + model_left_context if args.chunk_left_context_initial >= 0 else -1) right_context_final = (args.chunk_right_context_final + model_right_context if args.chunk_right_context_final >= 0 else -1) # Initialize as "raw" nnet, prior to training the LDA-like preconditioning # matrix. This first config just does any initial splicing that we do; # we do this as it's a convenient way to get the stats for the 'lda-like' # transform. if (args.stage <= -6): logger.info("Creating phone language-model") chain_lib.create_phone_lm(args.dir, args.tree_dir, run_opts, lm_opts=args.lm_opts) if (args.stage <= -5): logger.info("Creating denominator FST") shutil.copy('{0}/tree'.format(args.tree_dir), args.dir) chain_lib.create_denominator_fst(args.dir, args.tree_dir, run_opts) if ((args.stage <= -4) and os.path.exists("{0}/configs/init.config".format(args.dir)) and (args.input_model is None)): logger.info("Initializing a basic network for estimating " "preconditioning matrix") common_lib.execute_command( """{command} {dir}/log/nnet_init.log \ nnet3-init --srand=-2 {dir}/configs/init.config \ {dir}/init.raw""".format(command=run_opts.command, dir=args.dir)) egs_left_context = left_context + args.frame_subsampling_factor / 2 egs_right_context = right_context + args.frame_subsampling_factor / 2 # note: the '+ args.frame_subsampling_factor / 2' is to allow for the # fact that we'll be shifting the data slightly during training to give # variety to the training data. egs_left_context_initial = (left_context_initial + args.frame_subsampling_factor / 2 if left_context_initial >= 0 else -1) egs_right_context_final = (right_context_final + args.frame_subsampling_factor / 2 if right_context_final >= 0 else -1) default_egs_dir = '{0}/egs'.format(args.dir) if ((args.stage <= -3) and args.egs_dir is None): logger.info("Generating egs") if (not os.path.exists("{0}/den.fst".format(args.dir)) or not os.path.exists("{0}/normalization.fst".format(args.dir)) or not os.path.exists("{0}/tree".format(args.dir))): raise Exception("Chain egs generation expects {0}/den.fst, " "{0}/normalization.fst and {0}/tree " "to exist.".format(args.dir)) # this is where get_egs.sh is called. chain_lib.generate_chain_egs( dir=args.dir, data=args.feat_dir, lat_dir=args.lat_dir, egs_dir=default_egs_dir, left_context=egs_left_context, right_context=egs_right_context, left_context_initial=egs_left_context_initial, right_context_final=egs_right_context_final, run_opts=run_opts, left_tolerance=args.left_tolerance, right_tolerance=args.right_tolerance, frame_subsampling_factor=args.frame_subsampling_factor, alignment_subsampling_factor=args.alignment_subsampling_factor, frames_per_eg_str=args.chunk_width, srand=args.srand, egs_opts=args.egs_opts, cmvn_opts=args.cmvn_opts, online_ivector_dir=args.online_ivector_dir, frames_per_iter=args.frames_per_iter, transform_dir=args.transform_dir, stage=args.egs_stage) if args.egs_dir is None: egs_dir = default_egs_dir else: egs_dir = args.egs_dir [egs_left_context, egs_right_context, frames_per_eg_str, num_archives] = ( common_train_lib.verify_egs_dir(egs_dir, feat_dim, ivector_dim, ivector_id, egs_left_context, egs_right_context, egs_left_context_initial, egs_right_context_final)) assert(args.chunk_width == frames_per_eg_str) num_archives_expanded = num_archives * args.frame_subsampling_factor if (args.num_jobs_final > num_archives_expanded): raise Exception('num_jobs_final cannot exceed the ' 'expanded number of archives') # copy the properties of the egs to dir for # use during decoding logger.info("Copying the properties from {0} to {1}".format(egs_dir, args.dir)) common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir) if ((args.stage <= -2) and (os.path.exists(args.dir+"/configs/init.config")) and (args.input_model is None)): logger.info('Computing the preconditioning matrix for input features') chain_lib.compute_preconditioning_matrix( args.dir, egs_dir, num_archives, run_opts, max_lda_jobs=args.max_lda_jobs, rand_prune=args.rand_prune) if (args.stage <= -1): logger.info("Preparing the initial acoustic model.") chain_lib.prepare_initial_acoustic_model(args.dir, run_opts, input_model=args.input_model) with open("{0}/frame_subsampling_factor".format(args.dir), "w") as f: f.write(str(args.frame_subsampling_factor)) # set num_iters so that as close as possible, we process the data # $num_epochs times, i.e. $num_iters*$avg_num_jobs) == # $num_epochs*$num_archives, where # avg_num_jobs=(num_jobs_initial+num_jobs_final)/2. num_archives_to_process = int(args.num_epochs * num_archives_expanded) num_archives_processed = 0 num_iters = ((num_archives_to_process * 2) / (args.num_jobs_initial + args.num_jobs_final)) # If do_final_combination is True, compute the set of models_to_combine. # Otherwise, models_to_combine will be none. if args.do_final_combination: models_to_combine = common_train_lib.get_model_combine_iters( num_iters, args.num_epochs, num_archives_expanded, args.max_models_combine, args.num_jobs_final) else: models_to_combine = None min_deriv_time = None max_deriv_time_relative = None if args.deriv_truncate_margin is not None: min_deriv_time = -args.deriv_truncate_margin - model_left_context max_deriv_time_relative = \ args.deriv_truncate_margin + model_right_context logger.info("Training will run for {0} epochs = " "{1} iterations".format(args.num_epochs, num_iters)) for iter in range(num_iters): if (args.exit_stage is not None) and (iter == args.exit_stage): logger.info("Exiting early due to --exit-stage {0}".format(iter)) return current_num_jobs = int(0.5 + args.num_jobs_initial + (args.num_jobs_final - args.num_jobs_initial) * float(iter) / num_iters) if args.stage <= iter: model_file = "{dir}/{iter}.mdl".format(dir=args.dir, iter=iter) lrate = common_train_lib.get_learning_rate(iter, current_num_jobs, num_iters, num_archives_processed, num_archives_to_process, args.initial_effective_lrate, args.final_effective_lrate) shrinkage_value = 1.0 - (args.proportional_shrink * lrate) if shrinkage_value <= 0.5: raise Exception("proportional-shrink={0} is too large, it gives " "shrink-value={1}".format(args.proportional_shrink, shrinkage_value)) if args.shrink_value < shrinkage_value: shrinkage_value = (args.shrink_value if common_train_lib.should_do_shrinkage( iter, model_file, args.shrink_saturation_threshold) else shrinkage_value) percent = num_archives_processed * 100.0 / num_archives_to_process epoch = (num_archives_processed * args.num_epochs / num_archives_to_process) shrink_info_str = '' if shrinkage_value != 1.0: shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value) logger.info("Iter: {0}/{1} " "Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) " "lr: {5:0.6f} {6}".format(iter, num_iters - 1, epoch, args.num_epochs, percent, lrate, shrink_info_str)) chain_lib.train_one_iteration( dir=args.dir, iter=iter, srand=args.srand, egs_dir=egs_dir, num_jobs=current_num_jobs, num_archives_processed=num_archives_processed, num_archives=num_archives, learning_rate=lrate, dropout_edit_string=common_train_lib.get_dropout_edit_string( args.dropout_schedule, float(num_archives_processed) / num_archives_to_process, iter), train_opts=' '.join(args.train_opts), shrinkage_value=shrinkage_value, num_chunk_per_minibatch_str=args.num_chunk_per_minibatch, apply_deriv_weights=args.apply_deriv_weights, min_deriv_time=min_deriv_time, max_deriv_time_relative=max_deriv_time_relative, l2_regularize=args.l2_regularize, xent_regularize=args.xent_regularize, leaky_hmm_coefficient=args.leaky_hmm_coefficient, momentum=args.momentum, max_param_change=args.max_param_change, shuffle_buffer_size=args.shuffle_buffer_size, frame_subsampling_factor=args.frame_subsampling_factor, run_opts=run_opts, backstitch_training_scale=args.backstitch_training_scale, backstitch_training_interval=args.backstitch_training_interval) if args.cleanup: # do a clean up everything but the last 2 models, under certain # conditions common_train_lib.remove_model( args.dir, iter-2, num_iters, models_to_combine, args.preserve_model_interval) if args.email is not None: reporting_iter_interval = num_iters * args.reporting_interval if iter % reporting_iter_interval == 0: # lets do some reporting [report, times, data] = ( nnet3_log_parse.generate_acc_logprob_report( args.dir, "log-probability")) message = report subject = ("Update : Expt {dir} : " "Iter {iter}".format(dir=args.dir, iter=iter)) common_lib.send_mail(message, subject, args.email) num_archives_processed = num_archives_processed + current_num_jobs if args.stage <= num_iters: if args.do_final_combination: logger.info("Doing final combination to produce final.mdl") chain_lib.combine_models( dir=args.dir, num_iters=num_iters, models_to_combine=models_to_combine, num_chunk_per_minibatch_str=args.num_chunk_per_minibatch, egs_dir=egs_dir, leaky_hmm_coefficient=args.leaky_hmm_coefficient, l2_regularize=args.l2_regularize, xent_regularize=args.xent_regularize, run_opts=run_opts, max_objective_evaluations=args.max_objective_evaluations) else: logger.info("Copying the last-numbered model to final.mdl") common_lib.force_symlink("{0}.mdl".format(num_iters), "{0}/final.mdl".format(args.dir)) common_lib.force_symlink("compute_prob_valid.{iter}.log" "".format(iter=num_iters-1), "{dir}/log/compute_prob_valid.final.log".format( dir=args.dir)) if args.cleanup: logger.info("Cleaning up the experiment directory " "{0}".format(args.dir)) remove_egs = args.remove_egs if args.egs_dir is not None: # this egs_dir was not created by this experiment so we will not # delete it remove_egs = False # leave the last-two-numbered models, for diagnostic reasons. common_train_lib.clean_nnet_dir( args.dir, num_iters - 1, egs_dir, preserve_model_interval=args.preserve_model_interval, remove_egs=remove_egs) # do some reporting [report, times, data] = nnet3_log_parse.generate_acc_logprob_report( args.dir, "log-probability") if args.email is not None: common_lib.send_mail(report, "Update : Expt {0} : " "complete".format(args.dir), args.email) with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f: f.write(report) common_lib.execute_command("steps/info/chain_dir_info.pl " "{0}".format(args.dir))