def Main():
args = GetArgs()
logger.info("Converting nnet2 model {0} to nnet3 model {1}"
.format(os.path.join(args.nnet2_dir, args.model),
os.path.join(args.nnet3_dir, args.model)))
global tmpdir
tmpdir = tempfile.mkdtemp(dir=args.tmpdir)
# Convert nnet2 model to text and remove preconditioning
common_lib.execute_command("nnet-am-copy "
"--remove-preconditioning=true --binary=false {0}/{1} {2}/{1}"
.format(args.nnet2_dir, args.model, tmpdir))
# Parse nnet2 and return nnet3 object
with open(os.path.join(tmpdir, args.model)) as f:
nnet3 = parse_nnet2_to_nnet3(f)
# Write model
nnet3.write_config(os.path.join(tmpdir, "config"))
nnet3.write_model(os.path.join(args.nnet3_dir, args.model),
binary=args.binary)
if not args.skip_cleanup:
shutil.rmtree(tmpdir)
else:
logger.info("Not removing temporary directory {0}".format(tmpdir))
logger.info("Wrote nnet3 model to {0}".format(os.path.join(args.nnet3_dir,
args.model)))
def add_nnet_context_info(config_dir, nnet_edits=None,
existing_model=None):
"""Create the 'vars' file that specifies model_left_context, etc."""
common_lib.execute_command("nnet3-init {0} {1}/ref.config "
"{1}/ref.raw"
"".format(existing_model if
existing_model is not None else "",
config_dir))
model = "{0}/ref.raw".format(config_dir)
if nnet_edits is not None:
model = "nnet3-copy --edits='{0}' {1} - |".format(nnet_edits,
model)
out = common_lib.get_command_stdout('nnet3-info "{0}" | head -n 4 '
.format(model))
# out looks like this
# left-context: 7
# right-context: 0
# num-parameters: 90543902
# modulus: 1
info = {}
for line in out.split("\n"):
parts = line.split(":")
if len(parts) != 2:
continue
info[parts[0].strip()] = int(parts[1].strip())
# Writing the 'vars' file:
# model_left_context=0
# model_right_context=7
vf = open('{0}/vars'.format(config_dir), 'w')
vf.write('model_left_context={0}\n'.format(info['left-context']))
vf.write('model_right_context={0}\n'.format(info['right-context']))
vf.close()
def realign(dir, iter, feat_dir, lang, prev_egs_dir, cur_egs_dir,
prior_subset_size, num_archives,
run_opts, transform_dir=None, online_ivector_dir=None):
raise Exception("Realignment stage has not been implemented in nnet3")
logger.info("Getting average posterior for purposes of adjusting "
"the priors.")
# Note: this just uses CPUs, using a smallish subset of data.
# always use the first egs archive, which makes the script simpler;
# we're using different random subsets of it.
avg_post_vec_file = compute_average_posterior(
dir=dir, iter=iter, egs_dir=prev_egs_dir,
num_archives=num_archives, prior_subset_size=prior_subset_size,
run_opts=run_opts)
avg_post_vec_file = "{dir}/post.{iter}.vec".format(dir=dir, iter=iter)
logger.info("Re-adjusting priors based on computed posteriors")
model = '{0}/{1}.mdl'.format(dir, iter)
adjust_am_priors(dir, model, avg_post_vec_file, model, run_opts)
alidir = align(dir, feat_dir, lang, run_opts, iter,
transform_dir, online_ivector_dir)
common_lib.execute_command(
"""steps/nnet3/relabel_egs.sh --cmd "{command}" --iter {iter} \
{alidir} {prev_egs_dir} {cur_egs_dir}""".format(
command=run_opts.command,
iter=iter,
dir=dir,
alidir=alidir,
prev_egs_dir=prev_egs_dir,
cur_egs_dir=cur_egs_dir))
def create_phone_lm(dir, tree_dir, run_opts, lm_opts=None):
"""Create a phone LM for chain training
This method trains a phone LM for chain training using the alignments
in "tree_dir"
"""
try:
f = open(tree_dir + "/num_jobs", 'r')
num_ali_jobs = int(f.readline())
assert num_ali_jobs > 0
except:
raise Exception("""There was an error getting the number of alignment
jobs from {0}/num_jobs""".format(tree_dir))
alignments=' '.join(['{0}/ali.{1}.gz'.format(tree_dir, job)
for job in range(1, num_ali_jobs + 1)])
common_lib.execute_command(
"""{command} {dir}/log/make_phone_lm.log \
gunzip -c {alignments} \| \
ali-to-phones {tree_dir}/final.mdl ark:- ark:- \| \
chain-est-phone-lm {lm_opts} ark:- {dir}/phone_lm.fst""".format(
command=run_opts.command, dir=dir,
alignments=alignments,
lm_opts=lm_opts if lm_opts is not None else '',
tree_dir=tree_dir))
def compute_presoftmax_prior_scale(dir, alidir, num_jobs, run_opts,
presoftmax_prior_scale_power=-0.25):
# getting the raw pdf count
common_lib.execute_command(
"""{command} JOB=1:{num_jobs} {dir}/log/acc_pdf.JOB.log \
ali-to-post "ark:gunzip -c {alidir}/ali.JOB.gz|" ark:- \| \
post-to-tacc --per-pdf=true {alidir}/final.mdl ark:- \
{dir}/pdf_counts.JOB""".format(command=run_opts.command,
num_jobs=num_jobs,
dir=dir,
alidir=alidir))
common_lib.execute_command(
"""{command} {dir}/log/sum_pdf_counts.log \
vector-sum --binary=false {dir}/pdf_counts.* {dir}/pdf_counts \
""".format(command=run_opts.command, dir=dir))
for file in glob.glob('{0}/pdf_counts.*'.format(dir)):
os.remove(file)
pdf_counts = common_lib.read_kaldi_matrix('{0}/pdf_counts'.format(dir))[0]
scaled_counts = smooth_presoftmax_prior_scale_vector(
pdf_counts,
presoftmax_prior_scale_power=presoftmax_prior_scale_power,
smooth=0.01)
output_file = "{0}/presoftmax_prior_scale.vec".format(dir)
common_lib.write_kaldi_matrix(output_file, [scaled_counts])
common_lib.force_symlink("../presoftmax_prior_scale.vec",
"{0}/configs/presoftmax_prior_scale.vec".format(
dir))
def align(dir, data, lang, run_opts, iter=None, transform_dir=None,
online_ivector_dir=None):
alidir = '{dir}/ali{ali_suffix}'.format(
dir=dir,
ali_suffix="_iter_{0}".format(iter) if iter is not None else "")
logger.info("Aligning the data{gpu}with {num_jobs} jobs.".format(
gpu=" using gpu " if run_opts.realign_use_gpu else " ",
num_jobs=run_opts.realign_num_jobs))
common_lib.execute_command(
"""steps/nnet3/align.sh --nj {num_jobs_align} \
--cmd "{align_cmd} {align_queue_opt}" \
--use-gpu {align_use_gpu} \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{online_ivector_dir}" \
--iter "{iter}" {data} {lang} {dir} {alidir}""".format(
dir=dir, align_use_gpu=("yes"
if run_opts.realign_use_gpu
else "no"),
align_cmd=run_opts.realign_command,
align_queue_opt=run_opts.realign_queue_opt,
num_jobs_align=run_opts.realign_num_jobs,
transform_dir=(transform_dir
if transform_dir is not None
else ""),
online_ivector_dir=(online_ivector_dir
if online_ivector_dir is not None
else ""),
iter=iter if iter is not None else "",
alidir=alidir,
lang=lang, data=data))
return alidir
def generate_chain_egs(dir, data, lat_dir, egs_dir,
left_context, right_context,
run_opts, stage=0,
left_tolerance=None, right_tolerance=None,
left_context_initial=-1, right_context_final=-1,
frame_subsampling_factor=3,
alignment_subsampling_factor=3,
online_ivector_dir=None,
frames_per_iter=20000, frames_per_eg_str="20", srand=0,
egs_opts=None, cmvn_opts=None, transform_dir=None):
"""Wrapper for steps/nnet3/chain/get_egs.sh
See options in that script.
"""
common_lib.execute_command(
"""steps/nnet3/chain/get_egs.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} \
--right-context {right_context} \
--left-context-initial {left_context_initial} \
--right-context-final {right_context_final} \
--left-tolerance '{left_tolerance}' \
--right-tolerance '{right_tolerance}' \
--frame-subsampling-factor {frame_subsampling_factor} \
--alignment-subsampling-factor {alignment_subsampling_factor} \
--stage {stage} \
--frames-per-iter {frames_per_iter} \
--frames-per-eg {frames_per_eg_str} \
--srand {srand} \
{data} {dir} {lat_dir} {egs_dir}""".format(
command=run_opts.egs_command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
transform_dir=(transform_dir
if transform_dir is not None
else ''),
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None
else ''),
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
left_tolerance=(left_tolerance
if left_tolerance is not None
else ''),
right_tolerance=(right_tolerance
if right_tolerance is not None
else ''),
frame_subsampling_factor=frame_subsampling_factor,
alignment_subsampling_factor=alignment_subsampling_factor,
stage=stage, frames_per_iter=frames_per_iter,
frames_per_eg_str=frames_per_eg_str, srand=srand,
data=data, lat_dir=lat_dir, dir=dir, egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
def adjust_am_priors(dir, input_model, avg_posterior_vector, output_model,
run_opts):
common_lib.execute_command(
"""{command} {dir}/log/adjust_priors.final.log \
nnet3-am-adjust-priors "{input_model}" {avg_posterior_vector} \
"{output_model}" """.format(
command=run_opts.command,
dir=dir, input_model=input_model,
avg_posterior_vector=avg_posterior_vector,
output_model=output_model))
def create_denominator_fst(dir, tree_dir, run_opts):
common_lib.execute_command(
"""copy-transition-model {tree_dir}/final.mdl \
{dir}/0.trans_mdl""".format(dir=dir, tree_dir=tree_dir))
common_lib.execute_command(
"""{command} {dir}/log/make_den_fst.log \
chain-make-den-fst {dir}/tree {dir}/0.trans_mdl \
{dir}/phone_lm.fst \
{dir}/den.fst {dir}/normalization.fst""".format(
dir=dir, command=run_opts.command))
def compute_average_posterior(dir, iter, egs_dir, num_archives,
prior_subset_size,
run_opts, get_raw_nnet_from_am=True):
""" Computes the average posterior of the network
"""
for file in glob.glob('{0}/post.{1}.*.vec'.format(dir, iter)):
os.remove(file)
if run_opts.num_jobs_compute_prior > num_archives:
egs_part = 1
else:
egs_part = 'JOB'
if get_raw_nnet_from_am:
model = "nnet3-am-copy --raw=true {0}/{1}.mdl -|".format(dir, iter)
else:
model = "{dir}/{iter}.raw".format(dir=dir, iter=iter)
common_lib.execute_command(
"""{command} JOB=1:{num_jobs_compute_prior} {prior_queue_opt} \
{dir}/log/get_post.{iter}.JOB.log \
nnet3-copy-egs \
ark:{egs_dir}/egs.{egs_part}.ark ark:- \| \
nnet3-subset-egs --srand=JOB --n={prior_subset_size} \
ark:- ark:- \| \
nnet3-merge-egs --minibatch-size=128 ark:- ark:- \| \
nnet3-compute-from-egs {prior_gpu_opt} --apply-exp=true \
"{model}" ark:- ark:- \| \
matrix-sum-rows ark:- ark:- \| vector-sum ark:- \
{dir}/post.{iter}.JOB.vec""".format(
command=run_opts.command,
dir=dir, model=model,
num_jobs_compute_prior=run_opts.num_jobs_compute_prior,
prior_queue_opt=run_opts.prior_queue_opt,
iter=iter, prior_subset_size=prior_subset_size,
egs_dir=egs_dir, egs_part=egs_part,
prior_gpu_opt=run_opts.prior_gpu_opt))
# make sure there is time for $dir/post.{iter}.*.vec to appear.
time.sleep(5)
avg_post_vec_file = "{dir}/post.{iter}.vec".format(dir=dir, iter=iter)
common_lib.execute_command(
"""{command} {dir}/log/vector_sum.{iter}.log \
vector-sum {dir}/post.{iter}.*.vec {output_file}
""".format(command=run_opts.command,
dir=dir, iter=iter, output_file=avg_post_vec_file))
for file in glob.glob('{0}/post.{1}.*.vec'.format(dir, iter)):
os.remove(file)
return avg_post_vec_file
def prepare_initial_network(dir, run_opts, srand=-3):
if os.path.exists(dir+"/configs/init.config"):
common_lib.execute_command(
"""{command} {dir}/log/add_first_layer.log \
nnet3-init --srand={srand} {dir}/init.raw \
{dir}/configs/final.config {dir}/0.raw""".format(
command=run_opts.command, srand=srand,
dir=dir))
else:
common_lib.execute_command(
"""{command} {dir}/log/init_model.log \
nnet3-init --srand={srand} {dir}/configs/final.config {dir}/0.raw""".format(
command=run_opts.command, srand=srand,
dir=dir))
def prepare_initial_acoustic_model(dir, alidir, run_opts,
srand=-3):
""" Adds the first layer; this will also add in the lda.mat and
presoftmax_prior_scale.vec. It will also prepare the acoustic model
with the transition model."""
common_train_lib.prepare_initial_network(dir, run_opts,
srand=srand)
# Convert to .mdl, train the transitions, set the priors.
common_lib.execute_command(
"""{command} {dir}/log/init_mdl.log \
nnet3-am-init {alidir}/final.mdl {dir}/0.raw - \| \
nnet3-am-train-transitions - \
"ark:gunzip -c {alidir}/ali.*.gz|" {dir}/0.mdl
""".format(command=run_opts.command,
dir=dir, alidir=alidir))
def check_model_contexts(config_dir, nnet_edits=None, existing_model=None):
contexts = {}
for file_name in ['init', 'ref']:
if os.path.exists('{0}/{1}.config'.format(config_dir, file_name)):
contexts[file_name] = {}
common_lib.execute_command("nnet3-init {0} {1}/{2}.config "
"{1}/{2}.raw"
"".format(existing_model if
existing_model is not
None else '',
config_dir, file_name))
model = "{0}/{1}.raw".format(config_dir, file_name)
if nnet_edits is not None:
model = "nnet3-copy --edits='{0}' {1} - |".format(nnet_edits,
model)
out = common_lib.get_command_stdout('nnet3-info "{0}" | head -n 4 '
.format(model))
# out looks like this
# left-context: 7
# right-context: 0
# num-parameters: 90543902
# modulus: 1
for line in out.split("\n"):
parts = line.split(":")
if len(parts) != 2:
continue
key = parts[0].strip()
value = int(parts[1].strip())
if key in ['left-context', 'right-context']:
contexts[file_name][key] = value
if 'init' in contexts:
assert('ref' in contexts)
if ('left-context' in contexts['init'] and
'left-context' in contexts['ref']):
if ((contexts['init']['left-context']
> contexts['ref']['left-context'])
or (contexts['init']['right-context']
> contexts['ref']['right-context'])):
raise Exception(
"Model specified in {0}/init.config requires greater"
" context than the model specified in {0}/ref.config."
" This might be due to use of label-delay at the output"
" in ref.config. Please use delay=$label_delay in the"
" initial fixed-affine-layer of the network, to avoid"
" this issue.")
def compile(self):
root, ext = os.path.splitext(self.pdf_file)
dir_name = os.path.dirname(self.pdf_file)
latex_file = root + ".tex"
lat_file = open(latex_file, "w")
lat_file.write("\n".join(self.document))
lat_file.close()
logger.info("Compiling the latex report.")
try:
common_lib.execute_command(
"pdflatex -interaction=batchmode "
"-output-directory={0} {1}".format(dir_name, latex_file))
except Exception as e:
logger.warning("There was an error compiling the latex file {0}, "
"please do it manually: {1}".format(latex_file, e))
return False
return True
def generate_egs(data, alidir, egs_dir,
left_context, right_context,
run_opts, stage=0,
left_context_initial=-1, right_context_final=-1,
online_ivector_dir=None,
samples_per_iter=20000, frames_per_eg_str="20", srand=0,
egs_opts=None, cmvn_opts=None, transform_dir=None):
""" Wrapper for calling steps/nnet3/get_egs.sh
Generates targets from alignment directory 'alidir', which contains
the model final.mdl and alignments.
"""
common_lib.execute_command(
"""steps/nnet3/get_egs.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} \
--right-context {right_context} \
--left-context-initial {left_context_initial} \
--right-context-final {right_context_final} \
--stage {stage} \
--samples-per-iter {samples_per_iter} \
--frames-per-eg {frames_per_eg_str} \
--srand {srand} \
{data} {alidir} {egs_dir}
""".format(command=run_opts.egs_command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
transform_dir=(transform_dir
if transform_dir is not None else
''),
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None
else ''),
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
stage=stage, samples_per_iter=samples_per_iter,
frames_per_eg_str=frames_per_eg_str, srand=srand, data=data,
alidir=alidir, egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
def prepare_initial_acoustic_model(dir, run_opts, srand=-1):
""" Adds the first layer; this will also add in the lda.mat and
presoftmax_prior_scale.vec. It will also prepare the acoustic model
with the transition model."""
common_train_lib.prepare_initial_network(dir, run_opts,
srand=srand)
# The model-format for a 'chain' acoustic model is just the transition
# model and then the raw nnet, so we can use 'cat' to create this, as
# long as they have the same mode (binary or not binary).
# We ensure that they have the same mode (even if someone changed the
# script to make one or both of them text mode) by copying them both
# before concatenating them.
common_lib.execute_command(
"""{command} {dir}/log/init_mdl.log \
nnet3-am-init {dir}/0.trans_mdl {dir}/0.raw \
{dir}/0.mdl""".format(command=run_opts.command, dir=dir))
def compile(self):
root, ext = os.path.splitext(self.pdf_file)
dir_name = os.path.dirname(self.pdf_file)
latex_file = root + ".tex"
lat_file = open(latex_file, "w")
lat_file.write("\n".join(self.document))
lat_file.close()
logger.info("Compiling the LaTeX report.")
try:
common_lib.execute_command(
"pdflatex -interaction=batchmode "
"-output-directory={0} {1}".format(dir_name, latex_file))
except Exception as e:
logger.warning("There was an error compiling LaTeX file %s. "
"Check report.log generated by pdflatex in the same directory. %s",
latex_file, e)
return False
return True
def get_average_nnet_model(dir, iter, nnets_list, run_opts,
get_raw_nnet_from_am=True):
next_iter = iter + 1
if get_raw_nnet_from_am:
out_model = ("""- \| nnet3-am-copy --set-raw-nnet=- \
{dir}/{iter}.mdl {dir}/{next_iter}.mdl""".format(
dir=dir, iter=iter,
next_iter=next_iter))
else:
out_model = "{dir}/{next_iter}.raw".format(
dir=dir, next_iter=next_iter)
common_lib.execute_command(
"""{command} {dir}/log/average.{iter}.log \
nnet3-average {nnets_list} \
{out_model}""".format(command=run_opts.command,
dir=dir,
iter=iter,
nnets_list=nnets_list,
out_model=out_model))
def compute_preconditioning_matrix(dir, egs_dir, num_lda_jobs, run_opts,
max_lda_jobs=None, rand_prune=4.0,
lda_opts=None, use_multitask_egs=False):
if max_lda_jobs is not None:
if num_lda_jobs > max_lda_jobs:
num_lda_jobs = max_lda_jobs
multitask_egs_opts = common_train_lib.get_multitask_egs_opts(
egs_dir,
egs_prefix="egs.",
archive_index="JOB",
use_multitask_egs=use_multitask_egs)
scp_or_ark = "scp" if use_multitask_egs else "ark"
egs_rspecifier = (
"ark:nnet3-copy-egs {multitask_egs_opts} "
"{scp_or_ark}:{egs_dir}/egs.JOB.{scp_or_ark} ark:- |"
"".format(egs_dir=egs_dir, scp_or_ark=scp_or_ark,
multitask_egs_opts=multitask_egs_opts))
# Write stats with the same format as stats for LDA.
common_lib.execute_command(
"""{command} JOB=1:{num_lda_jobs} {dir}/log/get_lda_stats.JOB.log \
nnet3-acc-lda-stats --rand-prune={rand_prune} \
{dir}/init.raw "{egs_rspecifier}" \
{dir}/JOB.lda_stats""".format(
command=run_opts.command,
num_lda_jobs=num_lda_jobs,
dir=dir,
egs_rspecifier=egs_rspecifier,
rand_prune=rand_prune))
# the above command would have generated dir/{1..num_lda_jobs}.lda_stats
lda_stat_files = list(map(lambda x: '{0}/{1}.lda_stats'.format(dir, x),
range(1, num_lda_jobs + 1)))
common_lib.execute_command(
"""{command} {dir}/log/sum_transform_stats.log \
sum-lda-accs {dir}/lda_stats {lda_stat_files}""".format(
command=run_opts.command,
dir=dir, lda_stat_files=" ".join(lda_stat_files)))
for file in lda_stat_files:
try:
os.remove(file)
except OSError:
logger.error("There was error while trying to remove "
"lda stat files.")
raise
# this computes a fixed affine transform computed in the way we described
# in Appendix C.6 of http://arxiv.org/pdf/1410.7455v6.pdf; it's a scaled
# variant of an LDA transform but without dimensionality reduction.
common_lib.execute_command(
"""{command} {dir}/log/get_transform.log \
nnet-get-feature-transform {lda_opts} {dir}/lda.mat \
{dir}/lda_stats""".format(
command=run_opts.command, dir=dir,
lda_opts=lda_opts if lda_opts is not None else ""))
common_lib.force_symlink("../lda.mat", "{0}/configs/lda.mat".format(dir))
def get_best_nnet_model(dir, iter, best_model_index, run_opts,
get_raw_nnet_from_am=True):
best_model = "{dir}/{next_iter}.{best_model_index}.raw".format(
dir=dir,
next_iter=iter + 1,
best_model_index=best_model_index)
if get_raw_nnet_from_am:
out_model = ("""- \| nnet3-am-copy --set-raw-nnet=- \
{dir}/{iter}.mdl {dir}/{next_iter}.mdl""".format(
dir=dir, iter=iter, next_iter=iter + 1))
else:
out_model = "{dir}/{next_iter}.raw".format(dir=dir,
next_iter=iter + 1)
common_lib.execute_command(
"""{command} {dir}/log/select.{iter}.log \
nnet3-copy {best_model} \
{out_model}""".format(command=run_opts.command,
dir=dir, iter=iter,
best_model=best_model,
out_model=out_model))
def prepare_initial_acoustic_model(dir, run_opts, srand=-1, input_model=None):
""" This function adds the first layer; It will also prepare the acoustic
model with the transition model.
If 'input_model' is specified, no initial network preparation(adding
the first layer) is done and this model is used as initial 'raw' model
instead of '0.raw' model to prepare '0.mdl' as acoustic model by adding the
transition model.
"""
if input_model is None:
common_train_lib.prepare_initial_network(dir, run_opts,
srand=srand)
# The model-format for a 'chain' acoustic model is just the transition
# model and then the raw nnet, so we can use 'cat' to create this, as
# long as they have the same mode (binary or not binary).
# We ensure that they have the same mode (even if someone changed the
# script to make one or both of them text mode) by copying them both
# before concatenating them.
common_lib.execute_command(
"""{command} {dir}/log/init_mdl.log \
nnet3-am-init {dir}/0.trans_mdl {raw_mdl} \
{dir}/0.mdl""".format(command=run_opts.command, dir=dir,
raw_mdl=(input_model if input_model is not None
else '{0}/0.raw'.format(dir))))
def compute_preconditioning_matrix(dir, egs_dir, num_lda_jobs, run_opts,
max_lda_jobs=None, rand_prune=4.0,
lda_opts=None):
""" Function to estimate and write LDA matrix from cegs
This function is exactly similar to the version in module
libs.nnet3.train.frame_level_objf.common except this uses cegs instead of
egs files.
"""
if max_lda_jobs is not None:
if num_lda_jobs > max_lda_jobs:
num_lda_jobs = max_lda_jobs
# Write stats with the same format as stats for LDA.
common_lib.execute_command(
"""{command} JOB=1:{num_lda_jobs} {dir}/log/get_lda_stats.JOB.log \
nnet3-chain-acc-lda-stats --rand-prune={rand_prune} \
{dir}/init.raw "ark:{egs_dir}/cegs.JOB.ark" \
{dir}/JOB.lda_stats""".format(
command=run_opts.command,
num_lda_jobs=num_lda_jobs,
dir=dir,
egs_dir=egs_dir,
rand_prune=rand_prune))
# the above command would have generated dir/{1..num_lda_jobs}.lda_stats
lda_stat_files = map(lambda x: '{0}/{1}.lda_stats'.format(dir, x),
range(1, num_lda_jobs + 1))
common_lib.execute_command(
"""{command} {dir}/log/sum_transform_stats.log \
sum-lda-accs {dir}/lda_stats {lda_stat_files}""".format(
command=run_opts.command,
dir=dir, lda_stat_files=" ".join(lda_stat_files)))
for file in lda_stat_files:
try:
os.remove(file)
except OSError:
raise Exception("There was error while trying to remove "
"lda stat files.")
# this computes a fixed affine transform computed in the way we described
# in Appendix C.6 of http://arxiv.org/pdf/1410.7455v6.pdf; it's a scaled
# variant of an LDA transform but without dimensionality reduction.
common_lib.execute_command(
"""{command} {dir}/log/get_transform.log \
nnet-get-feature-transform {lda_opts} {dir}/lda.mat \
{dir}/lda_stats""".format(
command=run_opts.command, dir=dir,
lda_opts=lda_opts if lda_opts is not None else ""))
common_lib.force_symlink("../lda.mat", "{0}/configs/lda.mat".format(dir))
def write_model(self, model, binary="true"):
if not os.path.exists(self.config):
raise IOError("Config file {0} does not exist.".format(self.config))
# write raw model
common_lib.execute_command("nnet3-init --binary=true {0} {1}"
.format(self.config, os.path.join(tmpdir, "nnet3.raw")))
# add transition model
common_lib.execute_command("nnet3-am-init --binary=true {0} {1} {2}"
.format(self.transition_model, os.path.join(tmpdir, "nnet3.raw"),
os.path.join(tmpdir, "nnet3_no_prior.mdl")))
# add priors
common_lib.execute_command("nnet3-am-adjust-priors "
"--binary={0} {1} {2} {3}"
.format(binary, os.path.join(tmpdir, "nnet3_no_prior.mdl"),
self.priors, model))
def write_model(self, model, binary="true"):
if not os.path.exists(self.config):
raise IOError("Config file {0} does not exist.".format(
self.config))
# write raw model
common_lib.execute_command("nnet3-init --binary=true {0} {1}".format(
self.config, os.path.join(tmpdir, "nnet3.raw")))
# add transition model
common_lib.execute_command(
"nnet3-am-init --binary=true {0} {1} {2}".format(
self.transition_model, os.path.join(tmpdir, "nnet3.raw"),
os.path.join(tmpdir, "nnet3_no_prior.mdl")))
# add priors
common_lib.execute_command("nnet3-am-adjust-priors "
"--binary={0} {1} {2} {3}".format(
binary,
os.path.join(tmpdir,
"nnet3_no_prior.mdl"),
self.priors, model))
def combine_models(dir, num_iters, models_to_combine, num_chunk_per_minibatch_str,
egs_dir, leaky_hmm_coefficient, l2_regularize,
xent_regularize, run_opts,
max_objective_evaluations=30,
use_multitask_egs=False):
""" Function to do model combination
In the nnet3 setup, the logic
for doing averaging of subsets of the models in the case where
there are too many models to reliably esetimate interpolation
factors (max_models_combine) is moved into the nnet3-combine.
"""
raw_model_strings = []
logger.info("Combining {0} models.".format(models_to_combine))
models_to_combine.add(num_iters)
for iter in sorted(models_to_combine):
model_file = '{0}/{1}.mdl'.format(dir, iter)
if os.path.exists(model_file):
# we used to copy them with nnet3-am-copy --raw=true, but now
# the raw-model-reading code discards the other stuff itself.
raw_model_strings.append(model_file)
else:
print("{0}: warning: model file {1} does not exist "
"(final combination)".format(sys.argv[0], model_file))
scp_or_ark = "scp" if use_multitask_egs else "ark"
egs_suffix = ".scp" if use_multitask_egs else ".cegs"
multitask_egs_opts = common_train_lib.get_multitask_egs_opts(
egs_dir,
egs_prefix="combine.",
use_multitask_egs=use_multitask_egs)
# We reverse the order of the raw model strings so that the freshest one
# goes first. This is important for systems that include batch
# normalization-- it means that the freshest batch-norm stats are used.
# Since the batch-norm stats are not technically parameters, they are not
# combined in the combination code, they are just obtained from the first
# model.
raw_model_strings = list(reversed(raw_model_strings))
common_lib.execute_command(
"""{command} {combine_queue_opt} {dir}/log/combine.log \
nnet3-chain-combine \
--max-objective-evaluations={max_objective_evaluations} \
--l2-regularize={l2} --leaky-hmm-coefficient={leaky} \
--verbose=3 {combine_gpu_opt} {dir}/den.fst {raw_models} \
"ark,bg:nnet3-chain-copy-egs {multitask_egs_opts} {scp_or_ark}:{egs_dir}/combine{egs_suffix} ark:- | \
nnet3-chain-merge-egs --minibatch-size={num_chunk_per_mb} \
ark:- ark:- |" - \| \
nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl \
{dir}/final.mdl""".format(
command=run_opts.command,
combine_queue_opt=run_opts.combine_queue_opt,
combine_gpu_opt=run_opts.combine_gpu_opt,
max_objective_evaluations=max_objective_evaluations,
l2=l2_regularize, leaky=leaky_hmm_coefficient,
dir=dir, raw_models=" ".join(raw_model_strings),
num_chunk_per_mb=num_chunk_per_minibatch_str,
num_iters=num_iters,
egs_dir=egs_dir,
multitask_egs_opts=multitask_egs_opts,
scp_or_ark=scp_or_ark, egs_suffix=egs_suffix))
# Compute the probability of the final, combined model with
# the same subset we used for the previous compute_probs, as the
# different subsets will lead to different probs.
compute_train_cv_probabilities(
dir=dir, iter='final', egs_dir=egs_dir,
l2_regularize=l2_regularize, xent_regularize=xent_regularize,
leaky_hmm_coefficient=leaky_hmm_coefficient,
run_opts=run_opts,
use_multitask_egs=use_multitask_egs)
def combine_models(dir,
num_iters,
models_to_combine,
egs_dir,
minibatch_size_str,
run_opts,
chunk_width=None,
get_raw_nnet_from_am=True,
sum_to_one_penalty=0.0):
""" Function to do model combination
In the nnet3 setup, the logic
for doing averaging of subsets of the models in the case where
there are too many models to reliably esetimate interpolation
factors (max_models_combine) is moved into the nnet3-combine.
"""
raw_model_strings = []
logger.info("Combining {0} models.".format(models_to_combine))
models_to_combine.add(num_iters)
for iter in sorted(models_to_combine):
if get_raw_nnet_from_am:
model_file = '{0}/{1}.mdl'.format(dir, iter)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(
'"nnet3-am-copy --raw=true {0} -|"'.format(model_file))
else:
model_file = '{0}/{1}.raw'.format(dir, iter)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(model_file)
if get_raw_nnet_from_am:
out_model = ("| nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl "
"{dir}/combined.mdl".format(dir=dir, num_iters=num_iters))
else:
out_model = '{dir}/final.raw'.format(dir=dir)
# We reverse the order of the raw model strings so that the freshest one
# goes first. This is important for systems that include batch
# normalization-- it means that the freshest batch-norm stats are used.
# Since the batch-norm stats are not technically parameters, they are not
# combined in the combination code, they are just obtained from the first
# model.
raw_model_strings = list(reversed(raw_model_strings))
common_lib.execute_command(
"""{command} {combine_queue_opt} {dir}/log/combine.log \
nnet3-combine --num-iters=80 \
--enforce-sum-to-one={hard_enforce} \
--sum-to-one-penalty={penalty} \
--enforce-positive-weights=true \
--verbose=3 {raw_models} \
"ark,bg:nnet3-copy-egs \
ark:{egs_dir}/combine.egs ark:- | \
nnet3-merge-egs --minibatch-size={mbsize} ark:- ark:- |" \
"{out_model}"
""".format(command=run_opts.command,
combine_queue_opt=run_opts.combine_queue_opt,
dir=dir,
raw_models=" ".join(raw_model_strings),
hard_enforce=(sum_to_one_penalty <= 0),
penalty=sum_to_one_penalty,
mbsize=minibatch_size_str,
out_model=out_model,
egs_dir=egs_dir))
# Compute the probability of the final, combined model with
# the same subset we used for the previous compute_probs, as the
# different subsets will lead to different probs.
if get_raw_nnet_from_am:
compute_train_cv_probabilities(dir=dir,
iter='combined',
egs_dir=egs_dir,
run_opts=run_opts)
else:
compute_train_cv_probabilities(dir=dir,
iter='final',
egs_dir=egs_dir,
run_opts=run_opts,
get_raw_nnet_from_am=False)
def egs_prepare(args, run_opts):
""" The main function for egs_generate.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Check files
chain_lib.check_for_required_files(args.feat_dir, args.tree_dir,
args.lat_dir)
# Set some variables.
num_jobs = common_lib.get_number_of_jobs(args.tree_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.execute_command("utils/split_data.sh {0} {1}"
"".format(args.feat_dir, num_jobs))
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
if args.input_model is None:
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
else:
# If args.input_model is specified, the model left and right contexts
# are computed using input_model.
variables = common_train_lib.get_input_model_info(args.input_model)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial + model_left_context if
args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context if
args.chunk_right_context_final >= 0 else -1)
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -6):
logger.info("Creating phone language-model")
chain_lib.create_phone_lm(args.dir, args.tree_dir, run_opts,
lm_opts=args.lm_opts)
if (args.stage <= -5):
logger.info("Creating denominator FST")
shutil.copy('{0}/tree'.format(args.tree_dir), args.dir)
chain_lib.create_denominator_fst(args.dir, args.tree_dir, run_opts)
if ((args.stage <= -4) and
os.path.exists("{0}/configs/init.config".format(args.dir))
and (args.input_model is None)):
logger.info("Initializing a basic network for estimating "
"preconditioning matrix")
common_lib.execute_command(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
egs_left_context = left_context + args.frame_subsampling_factor / 2
egs_right_context = right_context + args.frame_subsampling_factor / 2
# note: the '+ args.frame_subsampling_factor / 2' is to allow for the
# fact that we'll be shifting the data slightly during training to give
# variety to the training data.
egs_left_context_initial = (left_context_initial +
args.frame_subsampling_factor / 2 if
left_context_initial >= 0 else -1)
egs_right_context_final = (right_context_final +
args.frame_subsampling_factor / 2 if
right_context_final >= 0 else -1)
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Copy phones.txt from ali-dir to dir. Later, steps/nnet3/decode.sh will
# use it to check compatibility between training and decoding phone-sets.
shutil.copy('{0}/phones.txt'.format(args.ali_dir), args.dir)
# Set some variables.
# num_leaves = common_lib.get_number_of_leaves_from_tree(args.ali_dir)
num_jobs = common_lib.get_number_of_jobs(args.ali_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.execute_command("utils/split_data.sh {0} {1}".format(
args.feat_dir, num_jobs))
shutil.copy('{0}/tree'.format(args.ali_dir), args.dir)
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
if args.input_model is None:
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
else:
# If args.input_model is specified, the model left and right contexts
# are computed using input_model.
variables = common_train_lib.get_input_model_info(args.input_model)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = model_left_context
right_context = model_right_context
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -5) and os.path.exists(args.dir+"/configs/init.config") and \
(args.input_model is None):
logger.info("Initializing a basic network for estimating "
"preconditioning matrix")
common_lib.execute_command("""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
default_egs_dir = '{0}/egs'.format(args.dir)
if (args.stage <= -4) and args.egs_dir is None:
logger.info("Generating egs")
if args.feat_dir is None:
raise Exception(
"--feat-dir option is required if you don't supply --egs-dir")
train_lib.acoustic_model.generate_egs(
data=args.feat_dir,
alidir=args.ali_dir,
egs_dir=default_egs_dir,
left_context=left_context,
right_context=right_context,
run_opts=run_opts,
frames_per_eg_str=str(args.frames_per_eg),
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
samples_per_iter=args.samples_per_iter,
stage=args.egs_stage)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context, frames_per_eg_str,
num_archives] = (common_train_lib.verify_egs_dir(egs_dir, feat_dim,
ivector_dim, ivector_id,
left_context,
right_context))
assert str(args.frames_per_eg) == frames_per_eg_str
if args.num_jobs_final > num_archives:
raise Exception('num_jobs_final cannot exceed the number of archives '
'in the egs directory')
# copy the properties of the egs to dir for
# use during decoding
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if args.stage <= -3 and os.path.exists(args.dir +
"/configs/init.config") and (
args.input_model is None):
logger.info('Computing the preconditioning matrix for input features')
train_lib.common.compute_preconditioning_matrix(
args.dir,
egs_dir,
num_archives,
run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if args.stage <= -2 and (args.input_model is None):
logger.info(
"Computing initial vector for FixedScaleComponent before"
" softmax, using priors^{prior_scale} and rescaling to"
" average 1".format(prior_scale=args.presoftmax_prior_scale_power))
common_train_lib.compute_presoftmax_prior_scale(
args.dir,
args.ali_dir,
num_jobs,
run_opts,
presoftmax_prior_scale_power=args.presoftmax_prior_scale_power)
if args.stage <= -1:
logger.info("Preparing the initial acoustic model.")
train_lib.acoustic_model.prepare_initial_acoustic_model(
args.dir, args.ali_dir, run_opts, input_model=args.input_model)
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_expanded = num_archives * args.frames_per_eg
num_archives_to_process = int(args.num_epochs * num_archives_expanded)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2) /
(args.num_jobs_initial + args.num_jobs_final))
# If do_final_combination is True, compute the set of models_to_combine.
# Otherwise, models_to_combine will be none.
if args.do_final_combination:
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs, num_archives_expanded,
args.max_models_combine, args.num_jobs_final)
else:
models_to_combine = None
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial +
(args.num_jobs_final - args.num_jobs_initial) *
float(iter) / num_iters)
if args.stage <= iter:
lrate = common_train_lib.get_learning_rate(
iter, current_num_jobs, num_iters, num_archives_processed,
num_archives_to_process, args.initial_effective_lrate,
args.final_effective_lrate)
shrinkage_value = 1.0 - (args.proportional_shrink * lrate)
if shrinkage_value <= 0.5:
raise Exception(
"proportional-shrink={0} is too large, it gives "
"shrink-value={1}".format(args.proportional_shrink,
shrinkage_value))
percent = num_archives_processed * 100.0 / num_archives_to_process
epoch = (num_archives_processed * args.num_epochs /
num_archives_to_process)
shrink_info_str = ''
if shrinkage_value != 1.0:
shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value)
logger.info("Iter: {0}/{1} "
"Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) "
"lr: {5:0.6f} {6}".format(iter, num_iters - 1,
epoch, args.num_epochs,
percent, lrate,
shrink_info_str))
train_lib.common.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=lrate,
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
train_opts=' '.join(args.train_opts),
minibatch_size_str=args.minibatch_size,
frames_per_eg=args.frames_per_eg,
momentum=args.momentum,
max_param_change=args.max_param_change,
shrinkage_value=shrinkage_value,
shuffle_buffer_size=args.shuffle_buffer_size,
run_opts=run_opts)
if args.cleanup:
# do a clean up everythin but the last 2 models, under certain
# conditions
common_train_lib.remove_model(args.dir, iter - 2, num_iters,
models_to_combine,
args.preserve_model_interval)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times,
data] = (nnet3_log_parse.generate_acc_logprob_report(
args.dir))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
if args.do_final_combination:
logger.info("Doing final combination to produce final.mdl")
train_lib.common.combine_models(
dir=args.dir,
num_iters=num_iters,
models_to_combine=models_to_combine,
egs_dir=egs_dir,
minibatch_size_str=args.minibatch_size,
run_opts=run_opts,
max_objective_evaluations=args.max_objective_evaluations)
if args.stage <= num_iters + 1:
logger.info("Getting average posterior for purposes of "
"adjusting the priors.")
# If args.do_final_combination is true, we will use the combined model.
# Otherwise, we will use the last_numbered model.
real_iter = 'combined' if args.do_final_combination else num_iters
avg_post_vec_file = train_lib.common.compute_average_posterior(
dir=args.dir,
iter=real_iter,
egs_dir=egs_dir,
num_archives=num_archives,
prior_subset_size=args.prior_subset_size,
run_opts=run_opts)
logger.info("Re-adjusting priors based on computed posteriors")
combined_or_last_numbered_model = "{dir}/{iter}.mdl".format(
dir=args.dir, iter=real_iter)
final_model = "{dir}/final.mdl".format(dir=args.dir)
train_lib.common.adjust_am_priors(args.dir,
combined_or_last_numbered_model,
avg_post_vec_file, final_model,
run_opts)
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir,
num_iters=num_iters,
egs_dir=egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs)
# do some reporting
[report, times,
data] = nnet3_log_parse.generate_acc_logprob_report(args.dir)
if args.email is not None:
common_lib.send_mail(
report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.execute_command("steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Set some variables.
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial + model_left_context if
args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context if
args.chunk_right_context_final >= 0 else -1)
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -4) and os.path.exists(args.dir+"/configs/init.config"):
logger.info("Initializing the network for computing the LDA stats")
common_lib.execute_command(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
default_egs_dir = '{0}/egs'.format(args.dir)
if (args.stage <= -3) and args.egs_dir is None:
logger.info("Generating egs")
if args.use_dense_targets:
target_type = "dense"
try:
num_targets = int(variables['num_targets'])
if (common_lib.get_feat_dim_from_scp(args.targets_scp)
!= num_targets):
raise Exception("Mismatch between num-targets provided to "
"script vs configs")
except KeyError as e:
num_targets = -1
else:
target_type = "sparse"
try:
num_targets = int(variables['num_targets'])
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined "
"in {1}".format(
str(e), '{0}/configs'.format(args.dir)))
train_lib.raw_model.generate_egs_using_targets(
data=args.feat_dir, targets_scp=args.targets_scp,
egs_dir=default_egs_dir,
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
run_opts=run_opts,
frames_per_eg_str=args.chunk_width,
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
samples_per_iter=args.samples_per_iter,
stage=args.egs_stage,
target_type=target_type,
num_targets=num_targets)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context,
frames_per_eg_str, num_archives] = (
common_train_lib.verify_egs_dir(egs_dir, feat_dim,
ivector_dim, ivector_id,
left_context, right_context,
left_context_initial,
right_context_final))
if args.chunk_width != frames_per_eg_str:
raise Exception("mismatch between --egs.chunk-width and the frames_per_eg "
"in the egs dir {0} vs {1}".format(args.chunk_width,
frames_per_eg_str))
if args.num_jobs_final > num_archives:
raise Exception('num_jobs_final cannot exceed the number of archives '
'in the egs directory')
# copy the properties of the egs to dir for
# use during decoding
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if args.stage <= -2 and os.path.exists(args.dir+"/configs/init.config"):
logger.info('Computing the preconditioning matrix for input features')
train_lib.common.compute_preconditioning_matrix(
args.dir, egs_dir, num_archives, run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if args.stage <= -1:
logger.info("Preparing the initial network.")
common_train_lib.prepare_initial_network(args.dir, run_opts)
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = int(args.num_epochs * num_archives)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2)
/ (args.num_jobs_initial + args.num_jobs_final))
# If do_final_combination is True, compute the set of models_to_combine.
# Otherwise, models_to_combine will be none.
if args.do_final_combination:
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs,
num_archives, args.max_models_combine,
args.num_jobs_final)
else:
models_to_combine = None
if (os.path.exists('{0}/valid_diagnostic.scp'.format(egs_dir))):
if (os.path.exists('{0}/valid_diagnostic.egs'.format(egs_dir))):
raise Exception('both {0}/valid_diagnostic.egs and '
'{0}/valid_diagnostic.scp exist.'
'This script expects only one of them to exist.'
''.format(egs_dir))
use_multitask_egs = True
else:
if (not os.path.exists('{0}/valid_diagnostic.egs'
''.format(egs_dir))):
raise Exception('neither {0}/valid_diagnostic.egs nor '
'{0}/valid_diagnostic.scp exist.'
'This script expects one of them.'
''.format(egs_dir))
use_multitask_egs = False
min_deriv_time = None
max_deriv_time_relative = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time_relative = \
args.deriv_truncate_margin + model_right_context
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial
+ (args.num_jobs_final - args.num_jobs_initial)
* float(iter) / num_iters)
if args.stage <= iter:
model_file = "{dir}/{iter}.raw".format(dir=args.dir, iter=iter)
lrate = common_train_lib.get_learning_rate(iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
# shrinkage_value is a scale on the parameters.
shrinkage_value = 1.0 - (args.proportional_shrink * lrate)
if shrinkage_value <= 0.5:
raise Exception("proportional-shrink={0} is too large, it gives "
"shrink-value={1}".format(args.proportional_shrink,
shrinkage_value))
if args.shrink_value < shrinkage_value:
shrinkage_value = (args.shrink_value
if common_train_lib.should_do_shrinkage(
iter, model_file,
args.shrink_saturation_threshold,
get_raw_nnet_from_am=False)
else shrinkage_value)
percent = num_archives_processed * 100.0 / num_archives_to_process
epoch = (num_archives_processed * args.num_epochs
/ num_archives_to_process)
shrink_info_str = ''
if shrinkage_value != 1.0:
shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value)
logger.info("Iter: {0}/{1} "
"Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) "
"lr: {5:0.6f} {6}".format(iter, num_iters - 1,
epoch, args.num_epochs,
percent,
lrate, shrink_info_str))
train_lib.common.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=lrate,
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
train_opts=' '.join(args.train_opts),
shrinkage_value=shrinkage_value,
minibatch_size_str=args.num_chunk_per_minibatch,
min_deriv_time=min_deriv_time,
max_deriv_time_relative=max_deriv_time_relative,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
run_opts=run_opts,
get_raw_nnet_from_am=False,
use_multitask_egs=use_multitask_egs,
compute_per_dim_accuracy=args.compute_per_dim_accuracy)
if args.cleanup:
# do a clean up everythin but the last 2 models, under certain
# conditions
common_train_lib.remove_model(
args.dir, iter-2, num_iters, models_to_combine,
args.preserve_model_interval,
get_raw_nnet_from_am=False)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times, data] = (
nnet3_log_parse.generate_acc_logprob_report(args.dir))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
if args.do_final_combination:
logger.info("Doing final combination to produce final.raw")
train_lib.common.combine_models(
dir=args.dir, num_iters=num_iters,
models_to_combine=models_to_combine, egs_dir=egs_dir,
minibatch_size_str=args.num_chunk_per_minibatch,
run_opts=run_opts, chunk_width=args.chunk_width,
get_raw_nnet_from_am=False,
compute_per_dim_accuracy=args.compute_per_dim_accuracy,
max_objective_evaluations=args.max_objective_evaluations)
else:
common_lib.force_symlink("{0}.raw".format(num_iters),
"{0}/final.raw".format(args.dir))
if args.compute_average_posteriors and args.stage <= num_iters + 1:
logger.info("Getting average posterior for purposes of "
"adjusting the priors.")
train_lib.common.compute_average_posterior(
dir=args.dir, iter='final', egs_dir=egs_dir,
num_archives=num_archives,
prior_subset_size=args.prior_subset_size, run_opts=run_opts,
get_raw_nnet_from_am=False)
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs,
get_raw_nnet_from_am=False)
# do some reporting
[report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir)
if args.email is not None:
common_lib.send_mail(report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.execute_command("steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def combine_models(dir,
num_iters,
models_to_combine,
num_chunk_per_minibatch_str,
egs_dir,
leaky_hmm_coefficient,
l2_regularize,
xent_regularize,
run_opts,
sum_to_one_penalty=0.0):
""" Function to do model combination
In the nnet3 setup, the logic
for doing averaging of subsets of the models in the case where
there are too many models to reliably esetimate interpolation
factors (max_models_combine) is moved into the nnet3-combine.
"""
raw_model_strings = []
logger.info("Combining {0} models.".format(models_to_combine))
models_to_combine.add(num_iters)
# TODO: if it turns out the sum-to-one-penalty code is not useful,
# remove support for it.
for iter in sorted(models_to_combine):
model_file = '{0}/{1}.mdl'.format(dir, iter)
if os.path.exists(model_file):
# we used to copy them with nnet3-am-copy --raw=true, but now
# the raw-model-reading code discards the other stuff itself.
raw_model_strings.append(model_file)
else:
print("{0}: warning: model file {1} does not exist "
"(final combination)".format(sys.argv[0], model_file))
# We reverse the order of the raw model strings so that the freshest one
# goes first. This is important for systems that include batch
# normalization-- it means that the freshest batch-norm stats are used.
# Since the batch-norm stats are not technically parameters, they are not
# combined in the combination code, they are just obtained from the first
# model.
raw_model_strings = list(reversed(raw_model_strings))
common_lib.execute_command(
"""{command} {combine_queue_opt} {dir}/log/combine.log \
nnet3-chain-combine --num-iters={opt_iters} \
--l2-regularize={l2} --leaky-hmm-coefficient={leaky} \
--separate-weights-per-component={separate_weights} \
--enforce-sum-to-one={hard_enforce} \
--sum-to-one-penalty={penalty} \
--enforce-positive-weights=true \
--verbose=3 {dir}/den.fst {raw_models} \
"ark,bg:nnet3-chain-copy-egs ark:{egs_dir}/combine.cegs ark:- | \
nnet3-chain-merge-egs --minibatch-size={num_chunk_per_mb} \
ark:- ark:- |" - \| \
nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl \
{dir}/final.mdl""".format(
command=run_opts.command,
combine_queue_opt=run_opts.combine_queue_opt,
opt_iters=(20 if sum_to_one_penalty <= 0 else 80),
separate_weights=(sum_to_one_penalty > 0),
l2=l2_regularize,
leaky=leaky_hmm_coefficient,
dir=dir,
raw_models=" ".join(raw_model_strings),
hard_enforce=(sum_to_one_penalty <= 0),
penalty=sum_to_one_penalty,
num_chunk_per_mb=num_chunk_per_minibatch_str,
num_iters=num_iters,
egs_dir=egs_dir))
# Compute the probability of the final, combined model with
# the same subset we used for the previous compute_probs, as the
# different subsets will lead to different probs.
compute_train_cv_probabilities(dir=dir,
iter='final',
egs_dir=egs_dir,
l2_regularize=l2_regularize,
xent_regularize=xent_regularize,
leaky_hmm_coefficient=leaky_hmm_coefficient,
run_opts=run_opts)
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Check files
chain_lib.check_for_required_files(args.feat_dir, args.tree_dir,
args.lat_dir)
# Copy phones.txt from tree-dir to dir. Later, steps/nnet3/decode.sh will
# use it to check compatibility between training and decoding phone-sets.
shutil.copy('{0}/phones.txt'.format(args.tree_dir), args.dir)
# Set some variables.
num_jobs = common_lib.get_number_of_jobs(args.tree_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.execute_command("utils/split_data.sh {0} {1}"
"".format(args.feat_dir, num_jobs))
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
if args.input_model is None:
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
else:
# If args.input_model is specified, the model left and right contexts
# are computed using input_model.
variables = common_train_lib.get_input_model_info(args.input_model)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial + model_left_context if
args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context if
args.chunk_right_context_final >= 0 else -1)
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -6):
logger.info("Creating phone language-model")
chain_lib.create_phone_lm(args.dir, args.tree_dir, run_opts,
lm_opts=args.lm_opts)
if (args.stage <= -5):
logger.info("Creating denominator FST")
shutil.copy('{0}/tree'.format(args.tree_dir), args.dir)
chain_lib.create_denominator_fst(args.dir, args.tree_dir, run_opts)
if ((args.stage <= -4) and
os.path.exists("{0}/configs/init.config".format(args.dir))
and (args.input_model is None)):
logger.info("Initializing a basic network for estimating "
"preconditioning matrix")
common_lib.execute_command(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
egs_left_context = left_context + args.frame_subsampling_factor / 2
egs_right_context = right_context + args.frame_subsampling_factor / 2
# note: the '+ args.frame_subsampling_factor / 2' is to allow for the
# fact that we'll be shifting the data slightly during training to give
# variety to the training data.
egs_left_context_initial = (left_context_initial +
args.frame_subsampling_factor / 2 if
left_context_initial >= 0 else -1)
egs_right_context_final = (right_context_final +
args.frame_subsampling_factor / 2 if
right_context_final >= 0 else -1)
default_egs_dir = '{0}/egs'.format(args.dir)
if ((args.stage <= -3) and args.egs_dir is None):
logger.info("Generating egs")
if (not os.path.exists("{0}/den.fst".format(args.dir)) or
not os.path.exists("{0}/normalization.fst".format(args.dir)) or
not os.path.exists("{0}/tree".format(args.dir))):
raise Exception("Chain egs generation expects {0}/den.fst, "
"{0}/normalization.fst and {0}/tree "
"to exist.".format(args.dir))
# this is where get_egs.sh is called.
chain_lib.generate_chain_egs(
dir=args.dir, data=args.feat_dir,
lat_dir=args.lat_dir, egs_dir=default_egs_dir,
left_context=egs_left_context,
right_context=egs_right_context,
left_context_initial=egs_left_context_initial,
right_context_final=egs_right_context_final,
run_opts=run_opts,
left_tolerance=args.left_tolerance,
right_tolerance=args.right_tolerance,
frame_subsampling_factor=args.frame_subsampling_factor,
alignment_subsampling_factor=args.alignment_subsampling_factor,
frames_per_eg_str=args.chunk_width,
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
frames_per_iter=args.frames_per_iter,
transform_dir=args.transform_dir,
stage=args.egs_stage)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context,
frames_per_eg_str, num_archives] = (
common_train_lib.verify_egs_dir(egs_dir, feat_dim,
ivector_dim, ivector_id,
egs_left_context, egs_right_context,
egs_left_context_initial,
egs_right_context_final))
assert(args.chunk_width == frames_per_eg_str)
num_archives_expanded = num_archives * args.frame_subsampling_factor
if (args.num_jobs_final > num_archives_expanded):
raise Exception('num_jobs_final cannot exceed the '
'expanded number of archives')
# copy the properties of the egs to dir for
# use during decoding
logger.info("Copying the properties from {0} to {1}".format(egs_dir, args.dir))
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if ((args.stage <= -2) and (os.path.exists(args.dir+"/configs/init.config"))
and (args.input_model is None)):
logger.info('Computing the preconditioning matrix for input features')
chain_lib.compute_preconditioning_matrix(
args.dir, egs_dir, num_archives, run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if (args.stage <= -1):
logger.info("Preparing the initial acoustic model.")
chain_lib.prepare_initial_acoustic_model(args.dir, run_opts,
input_model=args.input_model)
with open("{0}/frame_subsampling_factor".format(args.dir), "w") as f:
f.write(str(args.frame_subsampling_factor))
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = int(args.num_epochs * num_archives_expanded)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2)
/ (args.num_jobs_initial + args.num_jobs_final))
# If do_final_combination is True, compute the set of models_to_combine.
# Otherwise, models_to_combine will be none.
if args.do_final_combination:
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs,
num_archives_expanded, args.max_models_combine,
args.num_jobs_final)
else:
models_to_combine = None
min_deriv_time = None
max_deriv_time_relative = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time_relative = \
args.deriv_truncate_margin + model_right_context
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial
+ (args.num_jobs_final - args.num_jobs_initial)
* float(iter) / num_iters)
if args.stage <= iter:
model_file = "{dir}/{iter}.mdl".format(dir=args.dir, iter=iter)
lrate = common_train_lib.get_learning_rate(iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
shrinkage_value = 1.0 - (args.proportional_shrink * lrate)
if shrinkage_value <= 0.5:
raise Exception("proportional-shrink={0} is too large, it gives "
"shrink-value={1}".format(args.proportional_shrink,
shrinkage_value))
if args.shrink_value < shrinkage_value:
shrinkage_value = (args.shrink_value
if common_train_lib.should_do_shrinkage(
iter, model_file,
args.shrink_saturation_threshold)
else shrinkage_value)
percent = num_archives_processed * 100.0 / num_archives_to_process
epoch = (num_archives_processed * args.num_epochs
/ num_archives_to_process)
shrink_info_str = ''
if shrinkage_value != 1.0:
shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value)
logger.info("Iter: {0}/{1} "
"Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) "
"lr: {5:0.6f} {6}".format(iter, num_iters - 1,
epoch, args.num_epochs,
percent,
lrate, shrink_info_str))
chain_lib.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=lrate,
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
train_opts=' '.join(args.train_opts),
shrinkage_value=shrinkage_value,
num_chunk_per_minibatch_str=args.num_chunk_per_minibatch,
apply_deriv_weights=args.apply_deriv_weights,
min_deriv_time=min_deriv_time,
max_deriv_time_relative=max_deriv_time_relative,
l2_regularize=args.l2_regularize,
xent_regularize=args.xent_regularize,
leaky_hmm_coefficient=args.leaky_hmm_coefficient,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
frame_subsampling_factor=args.frame_subsampling_factor,
run_opts=run_opts,
backstitch_training_scale=args.backstitch_training_scale,
backstitch_training_interval=args.backstitch_training_interval)
if args.cleanup:
# do a clean up everything but the last 2 models, under certain
# conditions
common_train_lib.remove_model(
args.dir, iter-2, num_iters, models_to_combine,
args.preserve_model_interval)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times, data] = (
nnet3_log_parse.generate_acc_logprob_report(
args.dir, "log-probability"))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
if args.do_final_combination:
logger.info("Doing final combination to produce final.mdl")
chain_lib.combine_models(
dir=args.dir, num_iters=num_iters,
models_to_combine=models_to_combine,
num_chunk_per_minibatch_str=args.num_chunk_per_minibatch,
egs_dir=egs_dir,
leaky_hmm_coefficient=args.leaky_hmm_coefficient,
l2_regularize=args.l2_regularize,
xent_regularize=args.xent_regularize,
run_opts=run_opts,
max_objective_evaluations=args.max_objective_evaluations)
else:
logger.info("Copying the last-numbered model to final.mdl")
common_lib.force_symlink("{0}.mdl".format(num_iters),
"{0}/final.mdl".format(args.dir))
common_lib.force_symlink("compute_prob_valid.{iter}.log"
"".format(iter=num_iters-1),
"{dir}/log/compute_prob_valid.final.log".format(
dir=args.dir))
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
# leave the last-two-numbered models, for diagnostic reasons.
common_train_lib.clean_nnet_dir(
args.dir, num_iters - 1, egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs)
# do some reporting
[report, times, data] = nnet3_log_parse.generate_acc_logprob_report(
args.dir, "log-probability")
if args.email is not None:
common_lib.send_mail(report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.execute_command("steps/info/chain_dir_info.pl "
"{0}".format(args.dir))
def combine_models(dir,
num_iters,
models_to_combine,
num_chunk_per_minibatch_str,
egs_dir,
leaky_hmm_coefficient,
l2_regularize,
xent_regularize,
run_opts,
max_objective_evaluations=30,
use_multitask_egs=False):
""" Function to do model combination
In the nnet3 setup, the logic
for doing averaging of subsets of the models in the case where
there are too many models to reliably esetimate interpolation
factors (max_models_combine) is moved into the nnet3-combine.
"""
raw_model_strings = []
logger.info("Combining {0} models.".format(models_to_combine))
models_to_combine.add(num_iters)
for iter in sorted(models_to_combine):
model_file = '{0}/{1}.mdl'.format(dir, iter)
if os.path.exists(model_file):
# we used to copy them with nnet3-am-copy --raw=true, but now
# the raw-model-reading code discards the other stuff itself.
raw_model_strings.append(model_file)
else:
print("{0}: warning: model file {1} does not exist "
"(final combination)".format(sys.argv[0], model_file))
scp_or_ark = "scp" if use_multitask_egs else "ark"
egs_suffix = ".scp" if use_multitask_egs else ".cegs"
multitask_egs_opts = common_train_lib.get_multitask_egs_opts(
egs_dir, egs_prefix="combine.", use_multitask_egs=use_multitask_egs)
# We reverse the order of the raw model strings so that the freshest one
# goes first. This is important for systems that include batch
# normalization-- it means that the freshest batch-norm stats are used.
# Since the batch-norm stats are not technically parameters, they are not
# combined in the combination code, they are just obtained from the first
# model.
raw_model_strings = list(reversed(raw_model_strings))
common_lib.execute_command(
"""{command} {combine_queue_opt} {dir}/log/combine.log \
nnet3-chain-combine \
--max-objective-evaluations={max_objective_evaluations} \
--l2-regularize={l2} --leaky-hmm-coefficient={leaky} \
--verbose=3 {combine_gpu_opt} {dir}/den.fst {raw_models} \
"ark,bg:nnet3-chain-copy-egs {multitask_egs_opts} {scp_or_ark}:{egs_dir}/combine{egs_suffix} ark:- | \
nnet3-chain-merge-egs --minibatch-size={num_chunk_per_mb} \
ark:- ark:- |" - \| \
nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl \
{dir}/final.mdl""".format(
command=run_opts.command,
combine_queue_opt=run_opts.combine_queue_opt,
combine_gpu_opt=run_opts.combine_gpu_opt,
max_objective_evaluations=max_objective_evaluations,
l2=l2_regularize,
leaky=leaky_hmm_coefficient,
dir=dir,
raw_models=" ".join(raw_model_strings),
num_chunk_per_mb=num_chunk_per_minibatch_str,
num_iters=num_iters,
egs_dir=egs_dir,
multitask_egs_opts=multitask_egs_opts,
scp_or_ark=scp_or_ark,
egs_suffix=egs_suffix))
# Compute the probability of the final, combined model with
# the same subset we used for the previous compute_probs, as the
# different subsets will lead to different probs.
compute_train_cv_probabilities(dir=dir,
iter='final',
egs_dir=egs_dir,
l2_regularize=l2_regularize,
xent_regularize=xent_regularize,
leaky_hmm_coefficient=leaky_hmm_coefficient,
run_opts=run_opts,
use_multitask_egs=use_multitask_egs)
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Set some variables.
# num_leaves = common_lib.get_number_of_leaves_from_tree(args.ali_dir)
num_jobs = common_lib.get_number_of_jobs(args.ali_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
# 将特征数据 按照job数量进行划分, 会使用相同数量的job 进行对齐.
# feat_dir -- data/train_sp_hires 划分为jos 个子集.
common_lib.execute_command("utils/split_data.sh {0} {1}".format(
args.feat_dir, num_jobs))
# copy tri5a_sp_ali/tree(最后训练好的模型tree) ---> exp/nnet3/tdnn_sp
shutil.copy('{0}/tree'.format(args.ali_dir), args.dir)
# 将ali_dir(exp/tri5a_sp_ali/num_jobs) 读取写入 目标目录 exp/nnet3/tdnn_sp/num_jos
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
# 目标目录中 存入 nnet3结构/ 以及nnet3的 left right context配置信息
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
# liujunnan@innovem:configs$ cat vars
# model_left_context=16
# model_right_context=12
# liujunnan@innovem:configs$
# Set some variables.
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
left_context = model_left_context
right_context = model_right_context
# 初始化 原始nnet结构, 在训练 LDA预处理矩阵之前
# 第一个配置 只做了一些初始化拼接操作
# 这样做,是因为这样很方变能够获得LDA变换矩阵的统计信息.??????
if (args.stage <= -5) and os.path.exists(args.dir+"/configs/init.config"):
logger.info("Initializing a basic network for estimating preconditioning matrix")
# nnet3-init 利用 exp/nnet3/tdnn_sp/configs/init.config ===> exp/nnet3/tdnn_sp/init.raw.
common_lib.execute_command(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Set some variables.
num_jobs = common_lib.get_number_of_jobs(args.ali_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.execute_command("utils/split_data.sh {0} {1}".format(
args.feat_dir, num_jobs))
shutil.copy('{0}/tree'.format(args.ali_dir), args.dir)
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial +
model_left_context
if args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context
if args.chunk_right_context_final >= 0 else -1)
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -5):
logger.info("Initializing a basic network for estimating "
"preconditioning matrix")
common_lib.execute_command("""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
default_egs_dir = '{0}/egs'.format(args.dir)
if (args.stage <= -4) and args.egs_dir is None:
logger.info("Generating egs")
if args.feat_dir is None:
raise Exception(
"--feat-dir option is required if you don't supply --egs-dir")
train_lib.acoustic_model.generate_egs(
data=args.feat_dir,
alidir=args.ali_dir,
egs_dir=default_egs_dir,
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
run_opts=run_opts,
frames_per_eg_str=args.chunk_width,
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
samples_per_iter=args.samples_per_iter,
transform_dir=args.transform_dir,
stage=args.egs_stage)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context, frames_per_eg_str, num_archives
] = (common_train_lib.verify_egs_dir(egs_dir, feat_dim, ivector_dim,
ivector_id, left_context,
right_context, left_context_initial,
right_context_final))
if args.chunk_width != frames_per_eg_str:
raise Exception(
"mismatch between --egs.chunk-width and the frames_per_eg "
"in the egs dir {0} vs {1}".format(args.chunk_width,
frames_per_eg_str))
if (args.num_jobs_final > num_archives):
raise Exception('num_jobs_final cannot exceed the number of archives '
'in the egs directory')
# copy the properties of the egs to dir for
# use during decoding
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if (args.stage <= -3):
logger.info('Computing the preconditioning matrix for input features')
train_lib.common.compute_preconditioning_matrix(
args.dir,
egs_dir,
num_archives,
run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if (args.stage <= -2):
logger.info(
"Computing initial vector for FixedScaleComponent before"
" softmax, using priors^{prior_scale} and rescaling to"
" average 1".format(prior_scale=args.presoftmax_prior_scale_power))
common_train_lib.compute_presoftmax_prior_scale(
args.dir,
args.ali_dir,
num_jobs,
run_opts,
presoftmax_prior_scale_power=args.presoftmax_prior_scale_power)
if (args.stage <= -1):
logger.info("Preparing the initial acoustic model.")
train_lib.acoustic_model.prepare_initial_acoustic_model(
args.dir, args.ali_dir, run_opts)
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = int(args.num_epochs * num_archives)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2) /
(args.num_jobs_initial + args.num_jobs_final))
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs, num_archives, args.max_models_combine,
args.num_jobs_final)
def learning_rate(iter, current_num_jobs, num_archives_processed):
return common_train_lib.get_learning_rate(iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
min_deriv_time = None
max_deriv_time_relative = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time_relative = \
args.deriv_truncate_margin + model_right_context
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial +
(args.num_jobs_final - args.num_jobs_initial) *
float(iter) / num_iters)
if args.stage <= iter:
model_file = "{dir}/{iter}.mdl".format(dir=args.dir, iter=iter)
shrinkage_value = 1.0
if args.shrink_value != 1.0:
shrinkage_value = (
args.shrink_value if common_train_lib.should_do_shrinkage(
iter, model_file,
args.shrink_saturation_threshold) else 1)
train_lib.common.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=learning_rate(iter, current_num_jobs,
num_archives_processed),
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
shrinkage_value=shrinkage_value,
minibatch_size_str=args.num_chunk_per_minibatch,
min_deriv_time=min_deriv_time,
max_deriv_time_relative=max_deriv_time_relative,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
run_opts=run_opts)
if args.cleanup:
# do a clean up everythin but the last 2 models, under certain
# conditions
common_train_lib.remove_model(args.dir, iter - 2, num_iters,
models_to_combine,
args.preserve_model_interval)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times,
data] = (nnet3_log_parse.generate_acc_logprob_report(
args.dir))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
logger.info("Doing final combination to produce final.mdl")
train_lib.common.combine_models(
dir=args.dir,
num_iters=num_iters,
models_to_combine=models_to_combine,
egs_dir=egs_dir,
run_opts=run_opts,
minibatch_size_str=args.num_chunk_per_minibatch,
chunk_width=args.chunk_width,
sum_to_one_penalty=args.combine_sum_to_one_penalty)
if args.stage <= num_iters + 1:
logger.info("Getting average posterior for purposes of "
"adjusting the priors.")
avg_post_vec_file = train_lib.common.compute_average_posterior(
dir=args.dir,
iter='combined',
egs_dir=egs_dir,
num_archives=num_archives,
prior_subset_size=args.prior_subset_size,
run_opts=run_opts)
logger.info("Re-adjusting priors based on computed posteriors")
combined_model = "{dir}/combined.mdl".format(dir=args.dir)
final_model = "{dir}/final.mdl".format(dir=args.dir)
train_lib.common.adjust_am_priors(args.dir, combined_model,
avg_post_vec_file, final_model,
run_opts)
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir,
num_iters=num_iters,
egs_dir=egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs)
# do some reporting
[report, times,
data] = nnet3_log_parse.generate_acc_logprob_report(args.dir)
if args.email is not None:
common_lib.send_mail(
report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.execute_command("steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def model_init(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Check files
chain_lib.check_for_required_files(args.feat_dir, args.tree_dir,
args.lat_dir)
# Set some variables.
num_jobs = common_lib.get_number_of_jobs(args.tree_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.execute_command("utils/split_data.sh {0} {1}"
"".format(args.feat_dir, num_jobs))
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
if args.input_model is None:
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
else:
# If args.input_model is specified, the model left and right contexts
# are computed using input_model.
variables = common_train_lib.get_input_model_info(args.input_model)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial + model_left_context if
args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context if
args.chunk_right_context_final >= 0 else -1)
egs_left_context = left_context + args.frame_subsampling_factor / 2
egs_right_context = right_context + args.frame_subsampling_factor / 2
# note: the '+ args.frame_subsampling_factor / 2' is to allow for the
# fact that we'll be shifting the data slightly during training to give
# variety to the training data.
egs_left_context_initial = (left_context_initial +
args.frame_subsampling_factor / 2 if
left_context_initial >= 0 else -1)
egs_right_context_final = (right_context_final +
args.frame_subsampling_factor / 2 if
right_context_final >= 0 else -1)
default_egs_dir = '{0}/egs'.format(args.dir)
num_jobs = common_lib.get_number_of_jobs(args.tree_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.execute_command("utils/split_data.sh {0} {1}"
"".format(args.feat_dir, num_jobs))
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
if args.input_model is None:
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
else:
# If args.input_model is specified, the model left and right contexts
# are computed using input_model.
variables = common_train_lib.get_input_model_info(args.input_model)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial + model_left_context if
args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context if
args.chunk_right_context_final >= 0 else -1)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context,
frames_per_eg_str, num_archives] = (
common_train_lib.verify_egs_dir(egs_dir, feat_dim,
ivector_dim, ivector_id,
egs_left_context, egs_right_context,
egs_left_context_initial,
egs_right_context_final))
assert(args.chunk_width == frames_per_eg_str)
num_archives_expanded = num_archives * args.frame_subsampling_factor
if (args.num_jobs_final > num_archives_expanded):
raise Exception('num_jobs_final cannot exceed the '
'expanded number of archives')
# copy the properties of the egs to dir for
# use during decoding
logger.info("Copying the properties from {0} to {1}".format(egs_dir, args.dir))
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if ((args.stage <= -2) and (os.path.exists(args.dir+"/configs/init.config"))
and (args.input_model is None)):
logger.info('Computing the preconditioning matrix for input features')
chain_lib.compute_preconditioning_matrix(
args.dir, egs_dir, num_archives, run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if (args.stage <= -1):
logger.info("Preparing the initial acoustic model.")
chain_lib.prepare_initial_acoustic_model(args.dir, run_opts,
input_model=args.input_model)
with open("{0}/frame_subsampling_factor".format(args.dir), "w") as f:
f.write(str(args.frame_subsampling_factor))
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = int(args.num_epochs * num_archives_expanded)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2)
/ (args.num_jobs_initial + args.num_jobs_final))
# If do_final_combination is True, compute the set of models_to_combine.
# Otherwise, models_to_combine will be none.
if args.do_final_combination:
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs,
num_archives_expanded, args.max_models_combine,
args.num_jobs_final)
else:
models_to_combine = None
min_deriv_time = None
max_deriv_time_relative = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time_relative = \
args.deriv_truncate_margin + model_right_context
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Set some variables.
# note, feat_dim gets set to 0 if args.feat_dir is unset (None).
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = model_left_context
right_context = model_right_context
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -5) and os.path.exists(args.dir +
"/configs/init.config"):
logger.info("Initializing the network for computing the LDA stats")
common_lib.execute_command("""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
default_egs_dir = '{0}/egs'.format(args.dir)
if (args.stage <= -4) and args.egs_dir is None:
if args.targets_scp is None or args.feat_dir is None:
raise Exception(
"If you don't supply the --egs-dir option, the "
"--targets-scp and --feat-dir options are required.")
logger.info("Generating egs")
if args.use_dense_targets:
target_type = "dense"
try:
num_targets = int(variables['num_targets'])
if (common_lib.get_feat_dim_from_scp(args.targets_scp) !=
num_targets):
raise Exception("Mismatch between num-targets provided to "
"script vs configs")
except KeyError as e:
num_targets = -1
else:
target_type = "sparse"
try:
num_targets = int(variables['num_targets'])
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined "
"in {1}".format(str(e), '{0}/configs'.format(
args.dir)))
train_lib.raw_model.generate_egs_using_targets(
data=args.feat_dir,
targets_scp=args.targets_scp,
egs_dir=default_egs_dir,
left_context=left_context,
right_context=right_context,
run_opts=run_opts,
frames_per_eg_str=str(args.frames_per_eg),
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
samples_per_iter=args.samples_per_iter,
transform_dir=args.transform_dir,
stage=args.egs_stage,
target_type=target_type,
num_targets=num_targets)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context, frames_per_eg_str,
num_archives] = (common_train_lib.verify_egs_dir(egs_dir, feat_dim,
ivector_dim, ivector_id,
left_context,
right_context))
assert str(args.frames_per_eg) == frames_per_eg_str
if args.num_jobs_final > num_archives:
raise Exception('num_jobs_final cannot exceed the number of archives '
'in the egs directory')
# copy the properties of the egs to dir for
# use during decoding
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if args.stage <= -3 and os.path.exists(args.dir + "/configs/init.config"):
logger.info('Computing the preconditioning matrix for input features')
train_lib.common.compute_preconditioning_matrix(
args.dir,
egs_dir,
num_archives,
run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if args.stage <= -1:
logger.info("Preparing the initial network.")
common_train_lib.prepare_initial_network(args.dir, run_opts)
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_expanded = num_archives * args.frames_per_eg
num_archives_to_process = int(args.num_epochs * num_archives_expanded)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2) /
(args.num_jobs_initial + args.num_jobs_final))
# If do_final_combination is True, compute the set of models_to_combine.
# Otherwise, models_to_combine will be none.
if args.do_final_combination:
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs, num_archives_expanded,
args.max_models_combine, args.num_jobs_final)
else:
models_to_combine = None
if os.path.exists('{0}/valid_diagnostic.scp'.format(egs_dir)):
if os.path.exists('{0}/valid_diagnostic.egs'.format(egs_dir)):
raise Exception('both {0}/valid_diagnostic.egs and '
'{0}/valid_diagnostic.scp exist.'
'This script expects only one of them to exist.'
''.format(egs_dir))
use_multitask_egs = True
else:
if not os.path.exists('{0}/valid_diagnostic.egs'.format(egs_dir)):
raise Exception('neither {0}/valid_diagnostic.egs nor '
'{0}/valid_diagnostic.scp exist.'
'This script expects one of them.'
''.format(egs_dir))
use_multitask_egs = False
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial +
(args.num_jobs_final - args.num_jobs_initial) *
float(iter) / num_iters)
if args.stage <= iter:
lrate = common_train_lib.get_learning_rate(
iter, current_num_jobs, num_iters, num_archives_processed,
num_archives_to_process, args.initial_effective_lrate,
args.final_effective_lrate)
shrinkage_value = 1.0 - (args.proportional_shrink * lrate)
if shrinkage_value <= 0.5:
raise Exception(
"proportional-shrink={0} is too large, it gives "
"shrink-value={1}".format(args.proportional_shrink,
shrinkage_value))
train_lib.common.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=lrate,
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
minibatch_size_str=args.minibatch_size,
frames_per_eg=args.frames_per_eg,
momentum=args.momentum,
max_param_change=args.max_param_change,
shrinkage_value=shrinkage_value,
shuffle_buffer_size=args.shuffle_buffer_size,
run_opts=run_opts,
get_raw_nnet_from_am=False,
image_augmentation_opts=args.image_augmentation_opts,
use_multitask_egs=use_multitask_egs,
backstitch_training_scale=args.backstitch_training_scale,
backstitch_training_interval=args.backstitch_training_interval)
if args.cleanup:
# do a clean up everything but the last 2 models, under certain
# conditions
common_train_lib.remove_model(args.dir,
iter - 2,
num_iters,
models_to_combine,
args.preserve_model_interval,
get_raw_nnet_from_am=False)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times,
data] = (nnet3_log_parse.generate_acc_logprob_report(
args.dir))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
if args.do_final_combination:
logger.info("Doing final combination to produce final.raw")
train_lib.common.combine_models(
dir=args.dir,
num_iters=num_iters,
models_to_combine=models_to_combine,
egs_dir=egs_dir,
minibatch_size_str=args.minibatch_size,
run_opts=run_opts,
get_raw_nnet_from_am=False,
max_objective_evaluations=args.max_objective_evaluations,
use_multitask_egs=use_multitask_egs)
else:
common_lib.force_symlink("{0}.raw".format(num_iters),
"{0}/final.raw".format(args.dir))
if args.compute_average_posteriors and args.stage <= num_iters + 1:
logger.info("Getting average posterior for output-node 'output'.")
train_lib.common.compute_average_posterior(
dir=args.dir,
iter='final',
egs_dir=egs_dir,
num_archives=num_archives,
prior_subset_size=args.prior_subset_size,
run_opts=run_opts,
get_raw_nnet_from_am=False)
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir,
num_iters=num_iters,
egs_dir=egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs,
get_raw_nnet_from_am=False)
# do some reporting
outputs_list = common_train_lib.get_outputs_list(
"{0}/final.raw".format(args.dir), get_raw_nnet_from_am=False)
if 'output' in outputs_list:
[report, times,
data] = nnet3_log_parse.generate_acc_logprob_report(args.dir)
if args.email is not None:
common_lib.send_mail(
report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open(
"{dir}/accuracy.{output_name}.report".format(
dir=args.dir, output_name="output"), "w") as f:
f.write(report)
common_lib.execute_command("steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Check files
files = ['{0}/feats.scp'.format(args.feat_dir), '{0}/fst.1.scp'.format(args.tree_dir),
'{0}/final.mdl'.format(args.tree_dir), '{0}/tree'.format(args.tree_dir),
'{0}/phone_lm.fst'.format(args.tree_dir),
'{0}/num_jobs'.format(args.tree_dir)]
for file in files:
if not os.path.isfile(file):
raise Exception('Expected {0} to exist.'.format(file))
# Set some variables.
num_jobs = common_lib.get_number_of_jobs(args.tree_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
logger.info("feat-dim: {}, ivector-dim: {}".format(feat_dim, ivector_dim))
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for compiling FSTs
common_lib.execute_command("utils/split_data.sh {0} {1}".format(
args.feat_dir, num_jobs))
shutil.copy('{0}/tree'.format(args.tree_dir), args.dir)
shutil.copy('{0}/phones.txt'.format(args.tree_dir), args.dir)
shutil.copy('{0}/phone_lm.fst'.format(args.tree_dir), args.dir)
shutil.copy('{0}/0.trans_mdl'.format(args.tree_dir), args.dir)
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial + model_left_context if
args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context if
args.chunk_right_context_final >= 0 else -1)
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -5):
logger.info("Creating denominator FST")
chain_lib.create_denominator_fst(args.dir, args.tree_dir, run_opts)
if (args.stage <= -4):
logger.info("Initializing a basic network...")
common_lib.execute_command(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/final.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
egs_left_context = left_context + args.frame_subsampling_factor / 2
egs_right_context = right_context + args.frame_subsampling_factor / 2
egs_left_context_initial = (left_context_initial + args.frame_subsampling_factor / 2 if
left_context_initial >= 0 else -1)
egs_right_context_final = (right_context_final + args.frame_subsampling_factor / 2 if
right_context_final >= 0 else -1)
default_egs_dir = '{0}/egs'.format(args.dir)
if (args.stage <= -3) and args.egs_dir is None:
logger.info("Generating end-to-end egs...")
common_lib.execute_command(
"""steps/nnet3/chain/e2e/get_egs_e2e.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} \
--right-context {right_context} \
--left-context-initial {left_context_initial} \
--right-context-final {right_context_final} \
--frame-subsampling-factor {frame_subsampling_factor} \
--stage {stage} \
--frames-per-iter {frames_per_iter} \
--srand {srand} \
{data} {dir} {fst_dir} {egs_dir}""".format(
command=run_opts.command,
cmvn_opts=args.cmvn_opts if args.cmvn_opts is not None else '',
ivector_dir=(args.online_ivector_dir
if args.online_ivector_dir is not None
else ''),
left_context=egs_left_context,
right_context=egs_right_context,
left_context_initial=egs_left_context_initial,
right_context_final=egs_right_context_final,
frame_subsampling_factor=args.frame_subsampling_factor,
stage=args.egs_stage, frames_per_iter=args.frames_per_iter,
srand=args.srand,
data=args.feat_dir, dir=args.dir, fst_dir=args.tree_dir,
egs_dir=default_egs_dir,
egs_opts=args.egs_opts if args.egs_opts is not None else ''))
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context,
frames_per_eg_str, num_archives] = (
common_train_lib.verify_egs_dir(egs_dir, feat_dim,
ivector_dim, ivector_id,
egs_left_context, egs_right_context,
egs_left_context_initial,
egs_right_context_final))
num_archives_expanded = num_archives * args.frame_subsampling_factor
if (args.num_jobs_final > num_archives_expanded):
raise Exception('num_jobs_final cannot exceed the '
'expanded number of archives')
# copy the properties of the egs to dir for
# use during decoding
logger.info("Copying the properties from {0} to {1}".format(egs_dir, args.dir))
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if (args.stage <= -1):
logger.info("Preparing the initial acoustic model.")
chain_lib.prepare_initial_acoustic_model(args.dir, run_opts)
with open("{0}/frame_subsampling_factor".format(args.dir), "w") as f:
f.write(str(args.frame_subsampling_factor))
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = int(args.num_epochs * num_archives_expanded)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2)
/ (args.num_jobs_initial + args.num_jobs_final))
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs,
num_archives_expanded, args.max_models_combine,
args.num_jobs_final)
min_deriv_time = None
max_deriv_time_relative = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time_relative = \
args.deriv_truncate_margin + model_right_context
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
percent = num_archives_processed * 100.0 / num_archives_to_process
epoch = (num_archives_processed * args.num_epochs
/ num_archives_to_process)
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial
+ (args.num_jobs_final - args.num_jobs_initial)
* float(iter) / num_iters)
if args.stage <= iter:
model_file = "{dir}/{iter}.mdl".format(dir=args.dir, iter=iter)
lrate = common_train_lib.get_learning_rate(iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
shrinkage_value = 1.0 - (args.proportional_shrink * lrate)
if shrinkage_value <= 0.5:
raise Exception("proportional-shrink={0} is too large, it gives "
"shrink-value={1}".format(args.proportional_shrink,
shrinkage_value))
if args.shrink_value < shrinkage_value:
shrinkage_value = (args.shrink_value
if common_train_lib.should_do_shrinkage(
iter, model_file,
args.shrink_saturation_threshold)
else shrinkage_value)
shrink_info_str = ''
if shrinkage_value != 1.0:
shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value)
logger.info("Iter: {0}/{1} "
"Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) "
"lr: {5:0.6f} {6}".format(iter, num_iters - 1,
epoch, args.num_epochs,
percent,
lrate, shrink_info_str))
chain_lib.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=lrate,
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
shrinkage_value=shrinkage_value,
num_chunk_per_minibatch_str=args.num_chunk_per_minibatch,
apply_deriv_weights=args.apply_deriv_weights,
min_deriv_time=min_deriv_time,
max_deriv_time_relative=max_deriv_time_relative,
l2_regularize=args.l2_regularize,
xent_regularize=args.xent_regularize,
leaky_hmm_coefficient=args.leaky_hmm_coefficient,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
frame_subsampling_factor=args.frame_subsampling_factor,
run_opts=run_opts)
if args.cleanup:
# do a clean up everything but the last 2 models, under certain
# conditions
common_train_lib.remove_model(
args.dir, iter-2, num_iters, models_to_combine,
args.preserve_model_interval)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times, data] = (
nnet3_log_parse.generate_acc_logprob_report(
args.dir, "log-probability"))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
logger.info("Doing final combination to produce final.mdl")
chain_lib.combine_models(
dir=args.dir, num_iters=num_iters,
models_to_combine=models_to_combine,
num_chunk_per_minibatch_str=args.num_chunk_per_minibatch,
egs_dir=egs_dir,
leaky_hmm_coefficient=args.leaky_hmm_coefficient,
l2_regularize=args.l2_regularize,
xent_regularize=args.xent_regularize,
run_opts=run_opts)
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
common_train_lib.clean_nnet_dir(
args.dir, num_iters, egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs)
# do some reporting
[report, times, data] = nnet3_log_parse.generate_acc_logprob_report(
args.dir, "log-probability")
if args.email is not None:
common_lib.send_mail(report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.execute_command("steps/info/chain_dir_info.pl "
"{0}".format(args.dir))
def generate_egs(data,
alidir,
egs_dir,
scale,
scale_factor,
scale_file,
left_context,
right_context,
run_opts,
stage=0,
left_context_initial=-1,
right_context_final=-1,
online_ivector_dir=None,
samples_per_iter=20000,
frames_per_eg_str="20",
srand=0,
egs_opts=None,
cmvn_opts=None,
transform_dir=None):
""" Wrapper for calling steps/nnet3/get_egs.sh
Generates targets from alignment directory 'alidir', which contains
the model final.mdl and alignments.
"""
scale = str(scale).lower()
common_lib.execute_command(
"""steps/nnet3/get_egs_added_scaling.sh {egs_opts} \
--cmd "{command}" \
--scale "{scale}" \
--scale-factor "{scale_factor}" \
--scale-file "{scale_file}" \
--cmvn-opts "{cmvn_opts}" \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} \
--right-context {right_context} \
--left-context-initial {left_context_initial} \
--right-context-final {right_context_final} \
--stage {stage} \
--samples-per-iter {samples_per_iter} \
--frames-per-eg {frames_per_eg_str} \
--srand {srand} \
{data} {alidir} {egs_dir}
""".format(
command=run_opts.egs_command,
scale=scale,
scale_factor=scale_factor,
scale_file=scale_file,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
transform_dir=(transform_dir if transform_dir is not None else ''),
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None else ''),
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
stage=stage,
samples_per_iter=samples_per_iter,
frames_per_eg_str=frames_per_eg_str,
srand=srand,
data=data,
alidir=alidir,
egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
def combine_models(dir,
num_iters,
models_to_combine,
egs_dir,
minibatch_size_str,
run_opts,
chunk_width=None,
get_raw_nnet_from_am=True,
max_objective_evaluations=30,
use_multitask_egs=False,
compute_per_dim_accuracy=False):
""" Function to do model combination
In the nnet3 setup, the logic
for doing averaging of subsets of the models in the case where
there are too many models to reliably esetimate interpolation
factors (max_models_combine) is moved into the nnet3-combine.
"""
raw_model_strings = []
logger.info("Combining {0} models.".format(models_to_combine))
models_to_combine.add(num_iters)
for iter in sorted(models_to_combine):
suffix = "mdl" if get_raw_nnet_from_am else "raw"
model_file = '{0}/{1}.{2}'.format(dir, iter, suffix)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(model_file)
if get_raw_nnet_from_am:
out_model = ("| nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl "
"{dir}/combined.mdl".format(dir=dir, num_iters=num_iters))
else:
out_model = '{dir}/final.raw'.format(dir=dir)
# We reverse the order of the raw model strings so that the freshest one
# goes first. This is important for systems that include batch
# normalization-- it means that the freshest batch-norm stats are used.
# Since the batch-norm stats are not technically parameters, they are not
# combined in the combination code, they are just obtained from the first
# model.
raw_model_strings = list(reversed(raw_model_strings))
scp_or_ark = "scp" if use_multitask_egs else "ark"
egs_suffix = ".scp" if use_multitask_egs else ".egs"
egs_rspecifier = "{0}:{1}/combine{2}".format(scp_or_ark, egs_dir,
egs_suffix)
multitask_egs_opts = common_train_lib.get_multitask_egs_opts(
egs_dir, egs_prefix="combine.", use_multitask_egs=use_multitask_egs)
common_lib.execute_command(
"""{command} {combine_queue_opt} {dir}/log/combine.log \
nnet3-combine \
--max-objective-evaluations={max_objective_evaluations} \
--verbose=3 {raw_models} \
"ark,bg:nnet3-copy-egs {multitask_egs_opts} \
{egs_rspecifier} ark:- | \
nnet3-merge-egs --minibatch-size=1:{mbsize} ark:- ark:- |" \
"{out_model}"
""".format(command=run_opts.command,
combine_queue_opt=run_opts.combine_queue_opt,
dir=dir,
raw_models=" ".join(raw_model_strings),
max_objective_evaluations=max_objective_evaluations,
egs_rspecifier=egs_rspecifier,
mbsize=minibatch_size_str,
out_model=out_model,
multitask_egs_opts=multitask_egs_opts))
# Compute the probability of the final, combined model with
# the same subset we used for the previous compute_probs, as the
# different subsets will lead to different probs.
if get_raw_nnet_from_am:
compute_train_cv_probabilities(
dir=dir,
iter='combined',
egs_dir=egs_dir,
run_opts=run_opts,
use_multitask_egs=use_multitask_egs,
compute_per_dim_accuracy=compute_per_dim_accuracy)
else:
compute_train_cv_probabilities(
dir=dir,
iter='final',
egs_dir=egs_dir,
run_opts=run_opts,
get_raw_nnet_from_am=False,
use_multitask_egs=use_multitask_egs,
compute_per_dim_accuracy=compute_per_dim_accuracy)
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Copy phones.txt from ali-dir to dir. Later, steps/nnet3/decode.sh will
# use it to check compatibility between training and decoding phone-sets.
shutil.copy('{0}/phones.txt'.format(args.ali_dir), args.dir)
# Set some variables.
num_jobs = common_lib.get_number_of_jobs(args.ali_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.execute_command("utils/split_data.sh {0} {1}".format(
args.feat_dir, num_jobs))
shutil.copy('{0}/tree'.format(args.ali_dir), args.dir)
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write('{}'.format(num_jobs))
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
if args.input_model is None:
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
else:
# If args.input_model is specified, the model left and right contexts
# are computed using input_model.
variables = common_train_lib.get_input_model_info(args.input_model)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial + model_left_context if
args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context if
args.chunk_right_context_final >= 0 else -1)
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -5) and (args.input_model is None):
logger.info("Initializing a basic network for estimating "
"preconditioning matrix")
common_lib.execute_command(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
default_egs_dir = '{0}/egs'.format(args.dir)
if args.stage <= -4 and args.egs_dir is None:
logger.info("Generating egs")
if args.feat_dir is None:
raise Exception("--feat-dir option is required if you don't supply --egs-dir")
train_lib.acoustic_model.generate_egs(
data=args.feat_dir, alidir=args.ali_dir,
egs_dir=default_egs_dir,
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
run_opts=run_opts,
frames_per_eg_str=args.chunk_width,
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
samples_per_iter=args.samples_per_iter,
stage=args.egs_stage)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context,
frames_per_eg_str, num_archives] = (
common_train_lib.verify_egs_dir(egs_dir, feat_dim,
ivector_dim, ivector_id,
left_context, right_context,
left_context_initial, right_context_final))
if args.chunk_width != frames_per_eg_str:
raise Exception("mismatch between --egs.chunk-width and the frames_per_eg "
"in the egs dir {0} vs {1}".format(args.chunk_width,
frames_per_eg_str))
if args.num_jobs_final > num_archives:
raise Exception('num_jobs_final cannot exceed the number of archives '
'in the egs directory')
# copy the properties of the egs to dir for
# use during decoding
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if args.stage <= -3 and (args.input_model is None):
logger.info('Computing the preconditioning matrix for input features')
train_lib.common.compute_preconditioning_matrix(
args.dir, egs_dir, num_archives, run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if args.stage <= -2 and (args.input_model is None):
logger.info("Computing initial vector for FixedScaleComponent before"
" softmax, using priors^{prior_scale} and rescaling to"
" average 1".format(
prior_scale=args.presoftmax_prior_scale_power))
common_train_lib.compute_presoftmax_prior_scale(
args.dir, args.ali_dir, num_jobs, run_opts,
presoftmax_prior_scale_power=args.presoftmax_prior_scale_power)
if args.stage <= -1:
logger.info("Preparing the initial acoustic model.")
train_lib.acoustic_model.prepare_initial_acoustic_model(
args.dir, args.ali_dir, run_opts,
input_model=args.input_model)
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = int(args.num_epochs * num_archives)
num_archives_processed = 0
num_iters = int((num_archives_to_process * 2) / (args.num_jobs_initial + args.num_jobs_final))
# If do_final_combination is True, compute the set of models_to_combine.
# Otherwise, models_to_combine will be none.
if args.do_final_combination:
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs,
num_archives, args.max_models_combine,
args.num_jobs_final)
else:
models_to_combine = None
min_deriv_time = None
max_deriv_time_relative = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time_relative = \
args.deriv_truncate_margin + model_right_context
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial
+ (args.num_jobs_final - args.num_jobs_initial)
* float(iter) / num_iters)
if args.stage <= iter:
model_file = "{dir}/{iter}.mdl".format(dir=args.dir, iter=iter)
lrate = common_train_lib.get_learning_rate(iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
shrinkage_value = 1.0 - (args.proportional_shrink * lrate)
if shrinkage_value <= 0.5:
raise Exception("proportional-shrink={0} is too large, it gives "
"shrink-value={1}".format(args.proportional_shrink,
shrinkage_value))
if args.shrink_value < shrinkage_value:
shrinkage_value = (args.shrink_value
if common_train_lib.should_do_shrinkage(
iter, model_file,
args.shrink_saturation_threshold) else 1.0)
percent = num_archives_processed * 100.0 / num_archives_to_process
epoch = (num_archives_processed * args.num_epochs
/ num_archives_to_process)
shrink_info_str = ''
if shrinkage_value != 1.0:
shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value)
logger.info("Iter: {0}/{1} "
"Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) "
"lr: {5:0.6f} {6}".format(iter, num_iters - 1,
epoch, args.num_epochs,
percent,
lrate, shrink_info_str))
train_lib.common.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=lrate,
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
train_opts=' '.join(args.train_opts),
shrinkage_value=shrinkage_value,
minibatch_size_str=args.num_chunk_per_minibatch,
min_deriv_time=min_deriv_time,
max_deriv_time_relative=max_deriv_time_relative,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
run_opts=run_opts,
backstitch_training_scale=args.backstitch_training_scale,
backstitch_training_interval=args.backstitch_training_interval,
compute_per_dim_accuracy=args.compute_per_dim_accuracy)
if args.cleanup:
# do a clean up everythin but the last 2 models, under certain
# conditions
common_train_lib.remove_model(
args.dir, iter-2, num_iters, models_to_combine,
args.preserve_model_interval)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times, data] = (
nnet3_log_parse.generate_acc_logprob_report(args.dir))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
if args.do_final_combination:
logger.info("Doing final combination to produce final.mdl")
train_lib.common.combine_models(
dir=args.dir, num_iters=num_iters,
models_to_combine=models_to_combine, egs_dir=egs_dir,
run_opts=run_opts,
minibatch_size_str=args.num_chunk_per_minibatch,
chunk_width=args.chunk_width,
max_objective_evaluations=args.max_objective_evaluations,
compute_per_dim_accuracy=args.compute_per_dim_accuracy)
if args.stage <= num_iters + 1:
logger.info("Getting average posterior for purposes of "
"adjusting the priors.")
# If args.do_final_combination is true, we will use the combined model.
# Otherwise, we will use the last_numbered model.
real_iter = 'combined' if args.do_final_combination else num_iters
avg_post_vec_file = train_lib.common.compute_average_posterior(
dir=args.dir, iter=real_iter, egs_dir=egs_dir,
num_archives=num_archives,
prior_subset_size=args.prior_subset_size, run_opts=run_opts)
logger.info("Re-adjusting priors based on computed posteriors")
combined_or_last_numbered_model = "{dir}/{iter}.mdl".format(dir=args.dir,
iter=real_iter)
final_model = "{dir}/final.mdl".format(dir=args.dir)
train_lib.common.adjust_am_priors(args.dir, combined_or_last_numbered_model,
avg_post_vec_file, final_model,
run_opts)
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs)
# do some reporting
[report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir)
if args.email is not None:
common_lib.send_mail(report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.execute_command("steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Check files
chain_lib.check_for_required_files(
args.feat_dir, args.tree_dir,
args.lat_dir if args.egs_dir is None else None)
# Copy phones.txt from tree-dir to dir. Later, steps/nnet3/decode.sh will
# use it to check compatibility between training and decoding phone-sets.
shutil.copy('{0}/phones.txt'.format(args.tree_dir), args.dir)
# Set some variables.
num_jobs = common_lib.get_number_of_jobs(args.tree_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.execute_command("utils/split_data.sh {0} {1}"
"".format(args.feat_dir, num_jobs))
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
if args.input_model is None:
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
else:
# If args.input_model is specified, the model left and right contexts
# are computed using input_model.
variables = common_train_lib.get_input_model_info(args.input_model)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial +
model_left_context
if args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context
if args.chunk_right_context_final >= 0 else -1)
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -6):
logger.info("Creating phone language-model")
chain_lib.create_phone_lm(args.dir,
args.tree_dir,
run_opts,
lm_opts=args.lm_opts)
if (args.stage <= -5):
logger.info("Creating denominator FST")
shutil.copy('{0}/tree'.format(args.tree_dir), args.dir)
chain_lib.create_denominator_fst(args.dir, args.tree_dir, run_opts)
if ((args.stage <= -4)
and os.path.exists("{0}/configs/init.config".format(args.dir))
and (args.input_model is None)):
logger.info("Initializing a basic network for estimating "
"preconditioning matrix")
common_lib.execute_command("""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command, dir=args.dir))
egs_left_context = left_context + args.frame_subsampling_factor // 2
egs_right_context = right_context + args.frame_subsampling_factor // 2
# note: the '+ args.frame_subsampling_factor / 2' is to allow for the
# fact that we'll be shifting the data slightly during training to give
# variety to the training data.
egs_left_context_initial = (left_context_initial +
args.frame_subsampling_factor // 2
if left_context_initial >= 0 else -1)
egs_right_context_final = (right_context_final +
args.frame_subsampling_factor // 2
if right_context_final >= 0 else -1)
default_egs_dir = '{0}/egs'.format(args.dir)
if ((args.stage <= -3) and args.egs_dir is None):
logger.info("Generating egs")
if (not os.path.exists("{0}/den.fst".format(args.dir))
or not os.path.exists("{0}/normalization.fst".format(args.dir))
or not os.path.exists("{0}/tree".format(args.dir))):
raise Exception("Chain egs generation expects {0}/den.fst, "
"{0}/normalization.fst and {0}/tree "
"to exist.".format(args.dir))
# this is where get_egs.sh is called.
chain_lib.generate_chain_egs(
dir=args.dir,
data=args.feat_dir,
lat_dir=args.lat_dir,
egs_dir=default_egs_dir,
left_context=egs_left_context,
right_context=egs_right_context,
left_context_initial=egs_left_context_initial,
right_context_final=egs_right_context_final,
run_opts=run_opts,
left_tolerance=args.left_tolerance,
right_tolerance=args.right_tolerance,
frame_subsampling_factor=args.frame_subsampling_factor,
alignment_subsampling_factor=args.alignment_subsampling_factor,
frames_per_eg_str=args.chunk_width,
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
frames_per_iter=args.frames_per_iter,
stage=args.egs_stage)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context, frames_per_eg_str,
num_archives] = (common_train_lib.verify_egs_dir(
egs_dir, feat_dim, ivector_dim, ivector_id, egs_left_context,
egs_right_context, egs_left_context_initial, egs_right_context_final))
assert (args.chunk_width == frames_per_eg_str)
num_archives_expanded = num_archives * args.frame_subsampling_factor
if (args.num_jobs_final > num_archives_expanded):
raise Exception('num_jobs_final cannot exceed the '
'expanded number of archives')
# copy the properties of the egs to dir for
# use during decoding
logger.info("Copying the properties from {0} to {1}".format(
egs_dir, args.dir))
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if not os.path.exists('{0}/valid_diagnostic.cegs'.format(egs_dir)):
if (not os.path.exists('{0}/valid_diagnostic.scp'.format(egs_dir))):
raise Exception('Neither {0}/valid_diagnostic.cegs nor '
'{0}/valid_diagnostic.scp exist.'
'This script expects one of them.'.format(egs_dir))
use_multitask_egs = True
else:
use_multitask_egs = False
if ((args.stage <= -2)
and (os.path.exists(args.dir + "/configs/init.config"))
and (args.input_model is None)):
logger.info('Computing the preconditioning matrix for input features')
chain_lib.compute_preconditioning_matrix(
args.dir,
egs_dir,
num_archives,
run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune,
use_multitask_egs=use_multitask_egs)
if (args.stage <= -1):
logger.info("Preparing the initial acoustic model.")
chain_lib.prepare_initial_acoustic_model(args.dir,
run_opts,
input_model=args.input_model)
with open("{0}/frame_subsampling_factor".format(args.dir), "w") as f:
f.write(str(args.frame_subsampling_factor))
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = int(args.num_epochs * num_archives_expanded)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2) //
(args.num_jobs_initial + args.num_jobs_final))
# If do_final_combination is True, compute the set of models_to_combine.
# Otherwise, models_to_combine will be none.
if args.do_final_combination:
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs, num_archives_expanded,
args.max_models_combine, args.num_jobs_final)
else:
models_to_combine = None
min_deriv_time = None
max_deriv_time_relative = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time_relative = \
args.deriv_truncate_margin + model_right_context
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial +
(args.num_jobs_final - args.num_jobs_initial) *
float(iter) / num_iters)
if args.stage <= iter:
model_file = "{dir}/{iter}.mdl".format(dir=args.dir, iter=iter)
lrate = common_train_lib.get_learning_rate(
iter, current_num_jobs, num_iters, num_archives_processed,
num_archives_to_process, args.initial_effective_lrate,
args.final_effective_lrate)
shrinkage_value = 1.0 - (args.proportional_shrink * lrate)
if shrinkage_value <= 0.5:
raise Exception(
"proportional-shrink={0} is too large, it gives "
"shrink-value={1}".format(args.proportional_shrink,
shrinkage_value))
if args.shrink_value < shrinkage_value:
shrinkage_value = (
args.shrink_value if common_train_lib.should_do_shrinkage(
iter, model_file,
args.shrink_saturation_threshold) else shrinkage_value)
percent = num_archives_processed * 100.0 / num_archives_to_process
epoch = (num_archives_processed * args.num_epochs /
num_archives_to_process)
shrink_info_str = ''
if shrinkage_value != 1.0:
shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value)
logger.info("Iter: {0}/{1} "
"Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) "
"lr: {5:0.6f} {6}".format(iter, num_iters - 1,
epoch, args.num_epochs,
percent, lrate,
shrink_info_str))
chain_lib.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=lrate,
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
train_opts=' '.join(args.train_opts),
shrinkage_value=shrinkage_value,
num_chunk_per_minibatch_str=args.num_chunk_per_minibatch,
apply_deriv_weights=args.apply_deriv_weights,
min_deriv_time=min_deriv_time,
max_deriv_time_relative=max_deriv_time_relative,
l2_regularize=args.l2_regularize,
xent_regularize=args.xent_regularize,
leaky_hmm_coefficient=args.leaky_hmm_coefficient,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
frame_subsampling_factor=args.frame_subsampling_factor,
run_opts=run_opts,
backstitch_training_scale=args.backstitch_training_scale,
backstitch_training_interval=args.backstitch_training_interval,
use_multitask_egs=use_multitask_egs)
if args.cleanup:
# do a clean up everything but the last 2 models, under certain
# conditions
common_train_lib.remove_model(args.dir, iter - 2, num_iters,
models_to_combine,
args.preserve_model_interval)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times,
data] = (nnet3_log_parse.generate_acc_logprob_report(
args.dir, "log-probability"))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
if args.do_final_combination:
logger.info("Doing final combination to produce final.mdl")
chain_lib.combine_models(
dir=args.dir,
num_iters=num_iters,
models_to_combine=models_to_combine,
num_chunk_per_minibatch_str=args.num_chunk_per_minibatch,
egs_dir=egs_dir,
leaky_hmm_coefficient=args.leaky_hmm_coefficient,
l2_regularize=args.l2_regularize,
xent_regularize=args.xent_regularize,
run_opts=run_opts,
max_objective_evaluations=args.max_objective_evaluations,
use_multitask_egs=use_multitask_egs)
else:
logger.info("Copying the last-numbered model to final.mdl")
common_lib.force_symlink("{0}.mdl".format(num_iters),
"{0}/final.mdl".format(args.dir))
chain_lib.compute_train_cv_probabilities(
dir=args.dir,
iter=num_iters,
egs_dir=egs_dir,
l2_regularize=args.l2_regularize,
xent_regularize=args.xent_regularize,
leaky_hmm_coefficient=args.leaky_hmm_coefficient,
run_opts=run_opts,
use_multitask_egs=use_multitask_egs)
common_lib.force_symlink(
"compute_prob_valid.{iter}.log"
"".format(iter=num_iters),
"{dir}/log/compute_prob_valid.final.log".format(dir=args.dir))
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
# leave the last-two-numbered models, for diagnostic reasons.
common_train_lib.clean_nnet_dir(
args.dir,
num_iters - 1,
egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs)
# do some reporting
[report, times, data
] = nnet3_log_parse.generate_acc_logprob_report(args.dir,
"log-probability")
if args.email is not None:
common_lib.send_mail(
report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.execute_command("steps/info/chain_dir_info.pl "
"{0}".format(args.dir))
def compute_preconditioning_matrix(dir,
egs_dir,
num_lda_jobs,
run_opts,
max_lda_jobs=None,
rand_prune=4.0,
lda_opts=None,
use_multitask_egs=False):
if max_lda_jobs is not None:
if num_lda_jobs > max_lda_jobs:
num_lda_jobs = max_lda_jobs
multitask_egs_opts = common_train_lib.get_multitask_egs_opts(
egs_dir,
egs_prefix="egs.",
archive_index="JOB",
use_multitask_egs=use_multitask_egs)
scp_or_ark = "scp" if use_multitask_egs else "ark"
egs_rspecifier = ("ark:nnet3-copy-egs {multitask_egs_opts} "
"{scp_or_ark}:{egs_dir}/egs.JOB.{scp_or_ark} ark:- |"
"".format(egs_dir=egs_dir,
scp_or_ark=scp_or_ark,
multitask_egs_opts=multitask_egs_opts))
# Write stats with the same format as stats for LDA.
common_lib.execute_command(
"""{command} JOB=1:{num_lda_jobs} {dir}/log/get_lda_stats.JOB.log \
nnet3-acc-lda-stats --rand-prune={rand_prune} \
{dir}/init.raw "{egs_rspecifier}" \
{dir}/JOB.lda_stats""".format(command=run_opts.command,
num_lda_jobs=num_lda_jobs,
dir=dir,
egs_rspecifier=egs_rspecifier,
rand_prune=rand_prune))
# the above command would have generated dir/{1..num_lda_jobs}.lda_stats
lda_stat_files = list(
map(lambda x: '{0}/{1}.lda_stats'.format(dir, x),
range(1, num_lda_jobs + 1)))
common_lib.execute_command("""{command} {dir}/log/sum_transform_stats.log \
sum-lda-accs {dir}/lda_stats {lda_stat_files}""".format(
command=run_opts.command,
dir=dir,
lda_stat_files=" ".join(lda_stat_files)))
for file in lda_stat_files:
try:
os.remove(file)
except OSError:
logger.error("There was error while trying to remove "
"lda stat files.")
raise
# this computes a fixed affine transform computed in the way we described
# in Appendix C.6 of http://arxiv.org/pdf/1410.7455v6.pdf; it's a scaled
# variant of an LDA transform but without dimensionality reduction.
common_lib.execute_command("""{command} {dir}/log/get_transform.log \
nnet-get-feature-transform {lda_opts} {dir}/lda.mat \
{dir}/lda_stats""".format(
command=run_opts.command,
dir=dir,
lda_opts=lda_opts if lda_opts is not None else ""))
common_lib.force_symlink("../lda.mat", "{0}/configs/lda.mat".format(dir))
def generate_chain_egs(dir,
data,
lat_dir,
egs_dir,
left_context,
right_context,
run_opts,
stage=0,
left_tolerance=None,
right_tolerance=None,
left_context_initial=-1,
right_context_final=-1,
frame_subsampling_factor=3,
alignment_subsampling_factor=3,
online_ivector_dir=None,
frames_per_iter=20000,
frames_per_eg_str="20",
srand=0,
egs_opts=None,
cmvn_opts=None,
transform_dir=None):
"""Wrapper for steps/nnet3/chain/get_egs.sh
See options in that script.
"""
common_lib.execute_command("""steps/nnet3/chain/get_egs.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} \
--right-context {right_context} \
--left-context-initial {left_context_initial} \
--right-context-final {right_context_final} \
--left-tolerance '{left_tolerance}' \
--right-tolerance '{right_tolerance}' \
--frame-subsampling-factor {frame_subsampling_factor} \
--alignment-subsampling-factor {alignment_subsampling_factor} \
--stage {stage} \
--frames-per-iter {frames_per_iter} \
--frames-per-eg {frames_per_eg_str} \
--srand {srand} \
{data} {dir} {lat_dir} {egs_dir}""".format(
command=run_opts.command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
transform_dir=(transform_dir if transform_dir is not None else ''),
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None else ''),
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
left_tolerance=(left_tolerance if left_tolerance is not None else ''),
right_tolerance=(right_tolerance
if right_tolerance is not None else ''),
frame_subsampling_factor=frame_subsampling_factor,
alignment_subsampling_factor=alignment_subsampling_factor,
stage=stage,
frames_per_iter=frames_per_iter,
frames_per_eg_str=frames_per_eg_str,
srand=srand,
data=data,
lat_dir=lat_dir,
dir=dir,
egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs)
# do some reporting
[report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir)
if args.email is not None:
common_lib.send_mail(report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.execute_command("steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def main():
[args, run_opts] = get_args()
try:
# 开始训练.
train(args, run_opts)
common_lib.wait_for_background_commands()
except BaseException as e:
# look for BaseException so we catch KeyboardInterrupt, which is
def one_iter(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Check files
chain_lib.check_for_required_files(args.feat_dir, args.tree_dir,
args.lat_dir)
# Set some variables.
num_jobs = common_lib.get_number_of_jobs(args.tree_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.execute_command("utils/split_data.sh {0} {1}"
"".format(args.feat_dir, num_jobs))
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
if args.input_model is None:
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
else:
# If args.input_model is specified, the model left and right contexts
# are computed using input_model.
variables = common_train_lib.get_input_model_info(args.input_model)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial +
model_left_context
if args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context
if args.chunk_right_context_final >= 0 else -1)
egs_left_context = left_context + args.frame_subsampling_factor / 2
egs_right_context = right_context + args.frame_subsampling_factor / 2
# note: the '+ args.frame_subsampling_factor / 2' is to allow for the
# fact that we'll be shifting the data slightly during training to give
# variety to the training data.
egs_left_context_initial = (left_context_initial +
args.frame_subsampling_factor / 2
if left_context_initial >= 0 else -1)
egs_right_context_final = (right_context_final +
args.frame_subsampling_factor / 2
if right_context_final >= 0 else -1)
default_egs_dir = '{0}/egs'.format(args.dir)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context, frames_per_eg_str,
num_archives] = (common_train_lib.verify_egs_dir(
egs_dir, feat_dim, ivector_dim, ivector_id, egs_left_context,
egs_right_context, egs_left_context_initial, egs_right_context_final))
assert (args.chunk_width == frames_per_eg_str)
num_archives_expanded = num_archives * args.frame_subsampling_factor
if (args.num_jobs_final > num_archives_expanded):
raise Exception('num_jobs_final cannot exceed the '
'expanded number of archives')
# copy the properties of the egs to dir for
# use during decoding
logger.info("Copying the properties from {0} to {1}".format(
egs_dir, args.dir))
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if ((args.stage <= -2)
and (os.path.exists(args.dir + "/configs/init.config"))
and (args.input_model is None)):
logger.info('Computing the preconditioning matrix for input features')
chain_lib.compute_preconditioning_matrix(
args.dir,
egs_dir,
num_archives,
run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if (args.stage <= -1):
logger.info("Preparing the initial acoustic model.")
chain_lib.prepare_initial_acoustic_model(args.dir,
run_opts,
input_model=args.input_model)
with open("{0}/frame_subsampling_factor".format(args.dir), "w") as f:
f.write(str(args.frame_subsampling_factor))
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = int(args.num_epochs * num_archives_expanded)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2) /
(args.num_jobs_initial + args.num_jobs_final))
# If do_final_combination is True, compute the set of models_to_combine.
# Otherwise, models_to_combine will be none.
if args.do_final_combination:
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs, num_archives_expanded,
args.max_models_combine, args.num_jobs_final)
else:
models_to_combine = None
min_deriv_time = None
max_deriv_time_relative = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time_relative = \
args.deriv_truncate_margin + model_right_context
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
if (args.exit_stage is not None) and (args.iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(args.iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial +
(args.num_jobs_final - args.num_jobs_initial) *
float(args.iter) / num_iters)
model_file = "{dir}/{iter}.mdl".format(dir=args.dir, iter=args.iter)
lrate = common_train_lib.get_learning_rate(args.iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
shrinkage_value = 1.0 - (args.proportional_shrink * lrate)
if shrinkage_value <= 0.5:
raise Exception("proportional-shrink={0} is too large, it gives "
"shrink-value={1}".format(args.proportional_shrink,
shrinkage_value))
if args.shrink_value < shrinkage_value:
shrinkage_value = (
args.shrink_value if common_train_lib.should_do_shrinkage(
args.iter, model_file,
args.shrink_saturation_threshold) else shrinkage_value)
percent = num_archives_processed * 100.0 / num_archives_to_process
epoch = (num_archives_processed * args.num_epochs /
num_archives_to_process)
shrink_info_str = ''
if shrinkage_value != 1.0:
shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value)
logger.info("Iter: {0}/{1} "
"Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) "
"lr: {5:0.6f} {6}".format(args.iter, num_iters - 1, epoch,
args.num_epochs, percent, lrate,
shrink_info_str))
chain_lib.train_one_iteration(
dir=args.dir,
iter=args.iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=lrate,
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
args.iter),
shrinkage_value=shrinkage_value,
num_chunk_per_minibatch_str=args.num_chunk_per_minibatch,
apply_deriv_weights=args.apply_deriv_weights,
min_deriv_time=min_deriv_time,
max_deriv_time_relative=max_deriv_time_relative,
l2_regularize=args.l2_regularize,
xent_regularize=args.xent_regularize,
leaky_hmm_coefficient=args.leaky_hmm_coefficient,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
frame_subsampling_factor=args.frame_subsampling_factor,
run_opts=run_opts,
backstitch_training_scale=args.backstitch_training_scale,
backstitch_training_interval=args.backstitch_training_interval)
num_archives_processed = num_archives_processed + current_num_jobs
# do some reporting
[report, times, data
] = nnet3_log_parse.generate_acc_logprob_report(args.dir,
"log-probability")
if args.email is not None:
common_lib.send_mail(
report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.execute_command("steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def generate_egs_using_targets(data, targets_scp, egs_dir,
left_context, right_context,
run_opts, stage=0,
left_context_initial=-1, right_context_final=-1,
online_ivector_dir=None,
target_type='dense', num_targets=-1,
samples_per_iter=20000, frames_per_eg_str="20",
srand=0, egs_opts=None, cmvn_opts=None):
""" Wrapper for calling steps/nnet3/get_egs_targets.sh
This method generates egs directly from an scp file of targets, instead of
getting them from the alignments (as with the method generate_egs() in
module nnet3.train.frame_level_objf.acoustic_model).
Args:
target_type: "dense" if the targets are in matrix format
"sparse" if the targets are in posterior format
num_targets: must be explicitly specified for "sparse" targets.
For "dense" targets, this option is ignored and the target dim
is computed from the target matrix dimension
For other options, see the file steps/nnet3/get_egs_targets.sh
"""
if target_type == 'dense':
num_targets = common_lib.get_feat_dim_from_scp(targets_scp)
else:
if num_targets == -1:
raise Exception("--num-targets is required if "
"target-type is sparse")
common_lib.execute_command(
"""steps/nnet3/get_egs_targets.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} \
--right-context {right_context} \
--left-context-initial {left_context_initial} \
--right-context-final {right_context_final} \
--stage {stage} \
--samples-per-iter {samples_per_iter} \
--frames-per-eg {frames_per_eg_str} \
--srand {srand} \
--target-type {target_type} \
--num-targets {num_targets} \
{data} {targets_scp} {egs_dir}
""".format(command=run_opts.egs_command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None
else ''),
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
stage=stage, samples_per_iter=samples_per_iter,
frames_per_eg_str=frames_per_eg_str, srand=srand,
num_targets=num_targets,
data=data,
targets_scp=targets_scp, target_type=target_type,
egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
def remove_nnet_egs(egs_dir):
common_lib.execute_command("steps/nnet2/remove_egs.sh {egs_dir}".format(
egs_dir=egs_dir))
def generate_egs_using_targets(data, targets_scp, egs_dir,
left_context, right_context,
run_opts, stage=0,
left_context_initial=-1, right_context_final=-1,
online_ivector_dir=None,
target_type='dense', num_targets=-1,
samples_per_iter=20000, frames_per_eg_str="20",
srand=0, egs_opts=None, cmvn_opts=None):
""" Wrapper for calling subtools/kaldi/steps/nnet3/get_egs_targets.sh
This method generates egs directly from an scp file of targets, instead of
getting them from the alignments (as with the method generate_egs() in
module nnet3.train.frame_level_objf.acoustic_model).
Args:
target_type: "dense" if the targets are in matrix format
"sparse" if the targets are in posterior format
num_targets: must be explicitly specified for "sparse" targets.
For "dense" targets, this option is ignored and the target dim
is computed from the target matrix dimension
For other options, see the file subtools/kaldi/steps/nnet3/get_egs_targets.sh
"""
if target_type == 'dense':
num_targets = common_lib.get_feat_dim_from_scp(targets_scp)
else:
if num_targets == -1:
raise Exception("--num-targets is required if "
"target-type is sparse")
common_lib.execute_command(
"""subtools/kaldi/steps/nnet3/get_egs_targets.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} \
--right-context {right_context} \
--left-context-initial {left_context_initial} \
--right-context-final {right_context_final} \
--stage {stage} \
--samples-per-iter {samples_per_iter} \
--frames-per-eg {frames_per_eg_str} \
--srand {srand} \
--target-type {target_type} \
--num-targets {num_targets} \
{data} {targets_scp} {egs_dir}
""".format(command=run_opts.egs_command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None
else ''),
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
stage=stage, samples_per_iter=samples_per_iter,
frames_per_eg_str=frames_per_eg_str, srand=srand,
num_targets=num_targets,
data=data,
targets_scp=targets_scp, target_type=target_type,
egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Set some variables.
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
if args.input_model is None:
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
else:
# If args.input_model is specified, the model left and right contexts
# are computed using input_model.
variables = common_train_lib.get_input_model_info(args.input_model)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial + model_left_context if
args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context if
args.chunk_right_context_final >= 0 else -1)
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -4) and os.path.exists(args.dir+"/configs/init.config") and \
(args.input_model is None):
logger.info("Initializing the network for computing the LDA stats")
common_lib.execute_command(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
default_egs_dir = '{0}/egs'.format(args.dir)
if (args.stage <= -3) and args.egs_dir is None:
logger.info("Generating egs")
if args.use_dense_targets:
target_type = "dense"
try:
num_targets = int(variables['num_targets'])
if (common_lib.get_feat_dim_from_scp(args.targets_scp)
!= num_targets):
raise Exception("Mismatch between num-targets provided to "
"script vs configs")
except KeyError as e:
num_targets = -1
else:
target_type = "sparse"
try:
num_targets = int(variables['num_targets'])
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined "
"in {1}".format(
str(e), '{0}/configs'.format(args.dir)))
train_lib.raw_model.generate_egs_using_targets(
data=args.feat_dir, targets_scp=args.targets_scp,
egs_dir=default_egs_dir,
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
run_opts=run_opts,
frames_per_eg_str=args.chunk_width,
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
samples_per_iter=args.samples_per_iter,
stage=args.egs_stage,
target_type=target_type,
num_targets=num_targets)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context,
frames_per_eg_str, num_archives] = (
common_train_lib.verify_egs_dir(egs_dir, feat_dim,
ivector_dim, ivector_id,
left_context, right_context,
left_context_initial,
right_context_final))
if args.chunk_width != frames_per_eg_str:
raise Exception("mismatch between --egs.chunk-width and the frames_per_eg "
"in the egs dir {0} vs {1}".format(args.chunk_width,
frames_per_eg_str))
if args.num_jobs_final > num_archives:
raise Exception('num_jobs_final cannot exceed the number of archives '
'in the egs directory')
# copy the properties of the egs to dir for
# use during decoding
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if args.stage <= -2 and os.path.exists(args.dir+"/configs/init.config") and \
(args.input_model is None):
logger.info('Computing the preconditioning matrix for input features')
train_lib.common.compute_preconditioning_matrix(
args.dir, egs_dir, num_archives, run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if args.stage <= -1:
logger.info("Preparing the initial network.")
common_train_lib.prepare_initial_network(args.dir, run_opts, args.srand, args.input_model)
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = int(args.num_epochs * num_archives)
num_archives_processed = 0
num_iters = int((num_archives_to_process * 2) / (args.num_jobs_initial + args.num_jobs_final))
# If do_final_combination is True, compute the set of models_to_combine.
# Otherwise, models_to_combine will be none.
if args.do_final_combination:
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs,
num_archives, args.max_models_combine,
args.num_jobs_final)
else:
models_to_combine = None
if (os.path.exists('{0}/valid_diagnostic.scp'.format(egs_dir))):
if (os.path.exists('{0}/valid_diagnostic.egs'.format(egs_dir))):
raise Exception('both {0}/valid_diagnostic.egs and '
'{0}/valid_diagnostic.scp exist.'
'This script expects only one of them to exist.'
''.format(egs_dir))
use_multitask_egs = True
else:
if (not os.path.exists('{0}/valid_diagnostic.egs'
''.format(egs_dir))):
raise Exception('neither {0}/valid_diagnostic.egs nor '
'{0}/valid_diagnostic.scp exist.'
'This script expects one of them.'
''.format(egs_dir))
use_multitask_egs = False
min_deriv_time = None
max_deriv_time_relative = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time_relative = \
args.deriv_truncate_margin + model_right_context
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = common_train_lib.get_current_num_jobs(
iter, num_iters,
args.num_jobs_initial, args.num_jobs_step, args.num_jobs_final)
if args.stage <= iter:
model_file = "{dir}/{iter}.raw".format(dir=args.dir, iter=iter)
lrate = common_train_lib.get_learning_rate(iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
# shrinkage_value is a scale on the parameters.
shrinkage_value = 1.0 - (args.proportional_shrink * lrate)
if shrinkage_value <= 0.5:
raise Exception("proportional-shrink={0} is too large, it gives "
"shrink-value={1}".format(args.proportional_shrink,
shrinkage_value))
if args.shrink_value < shrinkage_value:
shrinkage_value = (args.shrink_value
if common_train_lib.should_do_shrinkage(
iter, model_file,
args.shrink_saturation_threshold,
get_raw_nnet_from_am=False)
else shrinkage_value)
percent = num_archives_processed * 100.0 / num_archives_to_process
epoch = (num_archives_processed * args.num_epochs
/ num_archives_to_process)
shrink_info_str = ''
if shrinkage_value != 1.0:
shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value)
logger.info("Iter: {0}/{1} Jobs: {2} "
"Epoch: {3:0.2f}/{4:0.1f} ({5:0.1f}% complete) "
"lr: {6:0.6f} {7}".format(iter, num_iters - 1,
current_num_jobs,
epoch, args.num_epochs,
percent,
lrate, shrink_info_str))
train_lib.common.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=lrate,
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
train_opts=' '.join(args.train_opts),
shrinkage_value=shrinkage_value,
minibatch_size_str=args.num_chunk_per_minibatch,
min_deriv_time=min_deriv_time,
max_deriv_time_relative=max_deriv_time_relative,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
run_opts=run_opts,
get_raw_nnet_from_am=False,
use_multitask_egs=use_multitask_egs,
compute_per_dim_accuracy=args.compute_per_dim_accuracy)
if args.cleanup:
# do a clean up everythin but the last 2 models, under certain
# conditions
common_train_lib.remove_model(
args.dir, iter-2, num_iters, models_to_combine,
args.preserve_model_interval,
get_raw_nnet_from_am=False)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times, data] = (
nnet3_log_parse.generate_acc_logprob_report(args.dir))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
if args.do_final_combination:
logger.info("Doing final combination to produce final.raw")
train_lib.common.combine_models(
dir=args.dir, num_iters=num_iters,
models_to_combine=models_to_combine, egs_dir=egs_dir,
minibatch_size_str=args.num_chunk_per_minibatch,
run_opts=run_opts, chunk_width=args.chunk_width,
get_raw_nnet_from_am=False,
compute_per_dim_accuracy=args.compute_per_dim_accuracy,
max_objective_evaluations=args.max_objective_evaluations,
use_multitask_egs=use_multitask_egs)
else:
common_lib.force_symlink("{0}.raw".format(num_iters),
"{0}/final.raw".format(args.dir))
if args.compute_average_posteriors and args.stage <= num_iters + 1:
logger.info("Getting average posterior for purposes of "
"adjusting the priors.")
train_lib.common.compute_average_posterior(
dir=args.dir, iter='final', egs_dir=egs_dir,
num_archives=num_archives,
prior_subset_size=args.prior_subset_size, run_opts=run_opts,
get_raw_nnet_from_am=False)
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs,
get_raw_nnet_from_am=False)
# do some reporting
[report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir)
if args.email is not None:
common_lib.send_mail(report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.execute_command("subtools/kaldi/steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def combine_models(dir, num_iters, models_to_combine, egs_dir,
minibatch_size_str,
run_opts,
chunk_width=None, get_raw_nnet_from_am=True,
sum_to_one_penalty=0.0,
use_multitask_egs=False):
""" Function to do model combination
In the nnet3 setup, the logic
for doing averaging of subsets of the models in the case where
there are too many models to reliably esetimate interpolation
factors (max_models_combine) is moved into the nnet3-combine.
"""
raw_model_strings = []
logger.info("Combining {0} models.".format(models_to_combine))
models_to_combine.add(num_iters)
for iter in sorted(models_to_combine):
if get_raw_nnet_from_am:
model_file = '{0}/{1}.mdl'.format(dir, iter)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(
'"nnet3-am-copy --raw=true {0} -|"'.format(model_file))
else:
model_file = '{0}/{1}.raw'.format(dir, iter)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(model_file)
if get_raw_nnet_from_am:
out_model = ("| nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl "
"{dir}/combined.mdl".format(dir=dir, num_iters=num_iters))
else:
out_model = '{dir}/final.raw'.format(dir=dir)
# We reverse the order of the raw model strings so that the freshest one
# goes first. This is important for systems that include batch
# normalization-- it means that the freshest batch-norm stats are used.
# Since the batch-norm stats are not technically parameters, they are not
# combined in the combination code, they are just obtained from the first
# model.
raw_model_strings = list(reversed(raw_model_strings))
scp_or_ark = "scp" if use_multitask_egs else "ark"
egs_suffix = ".scp" if use_multitask_egs else ".egs"
egs_rspecifier = "{0}:{1}/combine{2}".format(scp_or_ark,
egs_dir, egs_suffix)
multitask_egs_opts = common_train_lib.get_multitask_egs_opts(
egs_dir,
egs_prefix="combine.",
use_multitask_egs=use_multitask_egs)
common_lib.execute_command(
"""{command} {combine_queue_opt} {dir}/log/combine.log \
nnet3-combine --num-iters=80 \
--enforce-sum-to-one={hard_enforce} \
--sum-to-one-penalty={penalty} \
--enforce-positive-weights=true \
--verbose=3 {raw_models} \
"ark,bg:nnet3-copy-egs {multitask_egs_opts} \
{egs_rspecifier} ark:- | \
nnet3-merge-egs --minibatch-size={mbsize} ark:- ark:- |" \
"{out_model}"
""".format(command=run_opts.command,
combine_queue_opt=run_opts.combine_queue_opt,
dir=dir, raw_models=" ".join(raw_model_strings),
egs_rspecifier=egs_rspecifier,
hard_enforce=(sum_to_one_penalty <= 0),
penalty=sum_to_one_penalty,
mbsize=minibatch_size_str,
out_model=out_model,
multitask_egs_opts=multitask_egs_opts))
# Compute the probability of the final, combined model with
# the same subset we used for the previous compute_probs, as the
# different subsets will lead to different probs.
if get_raw_nnet_from_am:
compute_train_cv_probabilities(
dir=dir, iter='combined', egs_dir=egs_dir,
run_opts=run_opts, use_multitask_egs=use_multitask_egs)
else:
compute_train_cv_probabilities(
dir=dir, iter='final', egs_dir=egs_dir,
run_opts=run_opts, get_raw_nnet_from_am=False,
use_multitask_egs=use_multitask_egs)
def train(args, run_opts):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Set some variables.
config_dir = '{0}/configs'.format(args.dir)
am_var_file = '{0}/vars_am'.format(config_dir)
xvec_var_file = '{0}/vars_xvec'.format(config_dir)
am_variables = common_train_lib.parse_generic_config_vars_file(am_var_file)
xvec_variables = common_train_lib.parse_generic_config_vars_file(xvec_var_file)
# Set some variables.
try:
am_model_left_context = am_variables['model_left_context']
am_model_right_context = am_variables['model_right_context']
xvec_model_left_context = xvec_variables['model_left_context']
xvec_model_right_context = xvec_variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
am_left_context = am_model_left_context
am_right_context = am_model_right_context
xvec_left_context = xvec_model_left_context
xvec_right_context = xvec_model_right_context
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -5) and os.path.exists(args.dir+"/configs/init.config"):
logger.info("Initializing a basic network for estimating "
"preconditioning matrix")
common_lib.execute_command(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
am_egs_dir = args.am_egs_dir
xvec_egs_dir = args.xvec_egs_dir
am_output_name = args.am_output_name
xvec_output_name = args.xvec_output_name
am_weight = args.am_weight
xvec_weight = args.xvec_weight
feat_dim = int(common_lib.get_command_stdout("cat {0}/info/feat_dim".format(am_egs_dir)))
num_archives = int(common_lib.get_command_stdout("cat {0}/info/num_archives".format(am_egs_dir)))
tmp_feat_dim = int(common_lib.get_command_stdout("cat {0}/info/feat_dim".format(xvec_egs_dir)))
tmp_num_archives = int(common_lib.get_command_stdout("cat {0}/info/num_archives".format(xvec_egs_dir)))
# frames_per_eg is no longer a parameter but load from am_egs/info/frames_per_eg
am_frames_per_eg = int(common_lib.get_command_stdout("cat {0}/info/frames_per_eg".format(am_egs_dir)))
if feat_dim != tmp_feat_dim or num_archives*am_frames_per_eg != tmp_num_archives:
raise Exception('The am egs and xvec egs do not match')
if args.num_jobs_final > num_archives:
raise Exception('num_jobs_final cannot exceed the number of archives '
'in the egs directory')
# # No need to copy files for decoding
# common_train_lib.copy_egs_properties_to_exp_dir(am_egs_dir, args.dir)
if args.stage <= -3 and os.path.exists(args.dir+"/configs/init.config"):
logger.info('Computing the preconditioning matrix for input features')
train_lib.common.compute_preconditioning_matrix(
args.dir, egs_dir, num_archives, run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if args.stage <= -1:
logger.info("Preparing the initial network.")
common_train_lib.prepare_initial_network(args.dir, run_opts)
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_expanded = num_archives * am_frames_per_eg
num_archives_to_process = int(args.num_epochs * num_archives_expanded)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2)
/ (args.num_jobs_initial + args.num_jobs_final))
# If do_final_combination is True, compute the set of models_to_combine.
# Otherwise, models_to_combine will be none.
if args.do_final_combination:
models_to_combine = common_train_lib.get_model_combine_iters(
num_iters, args.num_epochs,
num_archives_expanded, args.max_models_combine,
args.num_jobs_final)
else:
models_to_combine = None
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial
+ (args.num_jobs_final - args.num_jobs_initial)
* float(iter) / num_iters)
if args.stage <= iter:
lrate = common_train_lib.get_learning_rate(iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
shrinkage_value = 1.0 - (args.proportional_shrink * lrate)
if shrinkage_value <= 0.5:
raise Exception("proportional-shrink={0} is too large, it gives "
"shrink-value={1}".format(args.proportional_shrink,
shrinkage_value))
percent = num_archives_processed * 100.0 / num_archives_to_process
epoch = (num_archives_processed * args.num_epochs
/ num_archives_to_process)
shrink_info_str = ''
if shrinkage_value != 1.0:
shrink_info_str = 'shrink: {0:0.5f}'.format(shrinkage_value)
logger.info("Iter: {0}/{1} "
"Epoch: {2:0.2f}/{3:0.1f} ({4:0.1f}% complete) "
"lr: {5:0.6f} {6}".format(iter, num_iters - 1,
epoch, args.num_epochs,
percent,
lrate, shrink_info_str))
train_lib.common.train_cvector_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
am_output_name=am_output_name,
am_weight=am_weight,
am_egs_dir=am_egs_dir,
xvec_output_name=xvec_output_name,
xvec_weight=xvec_weight,
xvec_egs_dir=xvec_egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=lrate,
minibatch_size_str=args.minibatch_size,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
run_opts=run_opts,
am_frames_per_eg=am_frames_per_eg,
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
shrinkage_value=shrinkage_value,
get_raw_nnet_from_am=False,
backstitch_training_scale=args.backstitch_training_scale,
backstitch_training_interval=args.backstitch_training_interval)
if args.cleanup:
# do a clean up everythin but the last 2 models, under certain
# conditions
common_train_lib.remove_model(
args.dir, iter-2, num_iters, models_to_combine,
args.preserve_model_interval,
get_raw_nnet_from_am=False)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times, data] = (
nnet3_log_parse.generate_acc_logprob_report(args.dir))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
# when we do final combination, just use the xvector egs
if args.stage <= num_iters:
if args.do_final_combination:
logger.info("Doing final combination to produce final.mdl")
train_lib.common.combine_models(
dir=args.dir, num_iters=num_iters,
models_to_combine=models_to_combine,
egs_dir=xvec_egs_dir,
minibatch_size_str="64", run_opts=run_opts,
get_raw_nnet_from_am=False,
max_objective_evaluations=args.max_objective_evaluations,
use_egs=True)
# sum_to_one_penalty=args.combine_sum_to_one_penalty,
else:
common_lib.force_symlink("{0}.raw".format(num_iters),
"{0}/final.raw".format(args.dir))
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = False
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir, num_iters=num_iters, egs_dir=am_egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs,
get_raw_nnet_from_am=False)
# TODO: we may trace other output nodes expect for "output"
# do some reporting
outputs_list = common_train_lib.get_outputs_list("{0}/final.raw".format(
args.dir), get_raw_nnet_from_am=False)
if 'output' in outputs_list:
[report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir)
if args.email is not None:
common_lib.send_mail(report, "Update : Expt {0} : "
"complete".format(args.dir),
args.email)
with open("{dir}/accuracy.{output_name}.report".format(dir=args.dir,
output_name="output"),
"w") as f:
f.write(report)
common_lib.execute_command("subtools/kaldi/steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def egs_generate(args, run_opts):
""" The main function for egs_generate.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Check files
chain_lib.check_for_required_files(args.feat_dir, args.tree_dir,
args.lat_dir)
# Set some variables.
num_jobs = common_lib.get_number_of_jobs(args.tree_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.execute_command("utils/split_data.sh {0} {1}"
"".format(args.feat_dir, num_jobs))
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
if args.input_model is None:
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
else:
# If args.input_model is specified, the model left and right contexts
# are computed using input_model.
variables = common_train_lib.get_input_model_info(args.input_model)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial + model_left_context if
args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context if
args.chunk_right_context_final >= 0 else -1)
egs_left_context = left_context + args.frame_subsampling_factor / 2
egs_right_context = right_context + args.frame_subsampling_factor / 2
# note: the '+ args.frame_subsampling_factor / 2' is to allow for the
# fact that we'll be shifting the data slightly during training to give
# variety to the training data.
egs_left_context_initial = (left_context_initial +
args.frame_subsampling_factor / 2 if
left_context_initial >= 0 else -1)
egs_right_context_final = (right_context_final +
args.frame_subsampling_factor / 2 if
right_context_final >= 0 else -1)
default_egs_dir = '{0}/egs_{1}'.format(args.dir, args.suffix)
if ((args.stage <= -3) and args.egs_dir is None):
logger.info("Generating egs")
if (not os.path.exists("{0}/den.fst".format(args.dir)) or
not os.path.exists("{0}/normalization.fst".format(args.dir)) or
not os.path.exists("{0}/tree".format(args.dir))):
raise Exception("Chain egs generation expects {0}/den.fst, "
"{0}/normalization.fst and {0}/tree "
"to exist.".format(args.dir))
# this is where get_egs.sh is called.
chain_lib.generate_chain_egs(
dir=args.dir, data=args.feat_dir,
lat_dir=args.lat_dir, egs_dir=default_egs_dir,
left_context=egs_left_context,
right_context=egs_right_context,
left_context_initial=egs_left_context_initial,
right_context_final=egs_right_context_final,
run_opts=run_opts,
left_tolerance=args.left_tolerance,
right_tolerance=args.right_tolerance,
frame_subsampling_factor=args.frame_subsampling_factor,
alignment_subsampling_factor=args.alignment_subsampling_factor,
frames_per_eg_str=args.chunk_width,
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
frames_per_iter=args.frames_per_iter,
transform_dir=args.transform_dir,
stage=args.egs_stage)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context,
frames_per_eg_str, num_archives] = (
common_train_lib.verify_egs_dir(egs_dir, feat_dim,
ivector_dim, ivector_id,
egs_left_context, egs_right_context,
egs_left_context_initial,
egs_right_context_final))
assert(args.chunk_width == frames_per_eg_str)
num_archives_expanded = num_archives * args.frame_subsampling_factor
