def generate_chain_egs(dir, data, lat_dir, egs_dir,
left_context, right_context,
run_opts, stage=0,
left_tolerance=None, right_tolerance=None,
left_context_initial=-1, right_context_final=-1,
frame_subsampling_factor=3,
alignment_subsampling_factor=3,
feat_type='raw', online_ivector_dir=None,
frames_per_iter=20000, frames_per_eg_str="20", srand=0,
egs_opts=None, cmvn_opts=None, transform_dir=None):
"""Wrapper for steps/nnet3/chain/get_egs.sh
See options in that script.
"""
common_lib.run_job(
"""steps/nnet3/chain/get_egs.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--feat-type {feat_type} \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} \
--right-context {right_context} \
--left-context-initial {left_context_initial} \
--right-context-final {right_context_final} \
--left-tolerance '{left_tolerance}' \
--right-tolerance '{right_tolerance}' \
--frame-subsampling-factor {frame_subsampling_factor} \
--alignment-subsampling-factor {alignment_subsampling_factor} \
--stage {stage} \
--frames-per-iter {frames_per_iter} \
--frames-per-eg {frames_per_eg_str} \
--srand {srand} \
{data} {dir} {lat_dir} {egs_dir}""".format(
command=run_opts.command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
feat_type=feat_type,
transform_dir=(transform_dir
if transform_dir is not None
else ''),
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None
else ''),
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
left_tolerance=(left_tolerance
if left_tolerance is not None
else ''),
right_tolerance=(right_tolerance
if right_tolerance is not None
else ''),
frame_subsampling_factor=frame_subsampling_factor,
alignment_subsampling_factor=alignment_subsampling_factor,
stage=stage, frames_per_iter=frames_per_iter,
frames_per_eg_str=frames_per_eg_str, srand=srand,
data=data, lat_dir=lat_dir, dir=dir, egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
def create_phone_lm(dir, tree_dir, run_opts, lm_opts=None):
"""Create a phone LM for chain training
This method trains a phone LM for chain training using the alignments
in "tree_dir"
"""
try:
f = open(tree_dir + "/num_jobs", 'r')
num_ali_jobs = int(f.readline())
assert num_ali_jobs > 0
except:
raise Exception("""There was an error getting the number of alignment
jobs from {0}/num_jobs""".format(tree_dir))
alignments=' '.join(['{0}/ali.{1}.gz'.format(tree_dir, job)
for job in range(1, num_ali_jobs + 1)])
common_lib.run_job(
"""{command} {dir}/log/make_phone_lm.log \
gunzip -c {alignments} \| \
ali-to-phones {tree_dir}/final.mdl ark:- ark:- \| \
chain-est-phone-lm {lm_opts} ark:- {dir}/phone_lm.fst""".format(
command=run_opts.command, dir=dir,
alignments=alignments,
lm_opts=lm_opts if lm_opts is not None else '',
tree_dir=tree_dir))
def align(dir, data, lang, run_opts, iter=None, transform_dir=None,
online_ivector_dir=None):
alidir = '{dir}/ali{ali_suffix}'.format(
dir=dir,
ali_suffix="_iter_{0}".format(iter) if iter is not None else "")
logger.info("Aligning the data{gpu}with {num_jobs} jobs.".format(
gpu=" using gpu " if run_opts.realign_use_gpu else " ",
num_jobs=run_opts.realign_num_jobs))
common_lib.run_job(
"""steps/nnet3/align.sh --nj {num_jobs_align} \
--cmd "{align_cmd} {align_queue_opt}" \
--use-gpu {align_use_gpu} \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{online_ivector_dir}" \
--iter "{iter}" {data} {lang} {dir} {alidir}""".format(
dir=dir, align_use_gpu=("yes"
if run_opts.realign_use_gpu
else "no"),
align_cmd=run_opts.realign_command,
align_queue_opt=run_opts.realign_queue_opt,
num_jobs_align=run_opts.realign_num_jobs,
transform_dir=(transform_dir
if transform_dir is not None
else ""),
online_ivector_dir=(online_ivector_dir
if online_ivector_dir is not None
else ""),
iter=iter if iter is not None else "",
alidir=alidir,
lang=lang, data=data))
return alidir
def compute_presoftmax_prior_scale(dir, alidir, num_jobs, run_opts,
presoftmax_prior_scale_power=-0.25):
# getting the raw pdf count
common_lib.run_job(
"""{command} JOB=1:{num_jobs} {dir}/log/acc_pdf.JOB.log \
ali-to-post "ark:gunzip -c {alidir}/ali.JOB.gz|" ark:- \| \
post-to-tacc --per-pdf=true {alidir}/final.mdl ark:- \
{dir}/pdf_counts.JOB""".format(command=run_opts.command,
num_jobs=num_jobs,
dir=dir,
alidir=alidir))
common_lib.run_job(
"""{command} {dir}/log/sum_pdf_counts.log \
vector-sum --binary=false {dir}/pdf_counts.* {dir}/pdf_counts \
""".format(command=run_opts.command, dir=dir))
for file in glob.glob('{0}/pdf_counts.*'.format(dir)):
os.remove(file)
pdf_counts = common_lib.read_kaldi_matrix('{0}/pdf_counts'.format(dir))[0]
scaled_counts = smooth_presoftmax_prior_scale_vector(
pdf_counts,
presoftmax_prior_scale_power=presoftmax_prior_scale_power,
smooth=0.01)
output_file = "{0}/presoftmax_prior_scale.vec".format(dir)
common_lib.write_kaldi_matrix(output_file, [scaled_counts])
common_lib.force_symlink("../presoftmax_prior_scale.vec",
"{0}/configs/presoftmax_prior_scale.vec".format(
dir))
def compute_progress(dir,
iter,
egs_dir,
left_context,
right_context,
run_opts,
background_process_handler=None,
wait=False,
get_raw_nnet_from_am=True):
if get_raw_nnet_from_am:
prev_model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format(
dir, iter - 1)
model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format(dir, iter)
else:
prev_model = '{0}/{1}.raw'.format(dir, iter - 1)
model = '{0}/{1}.raw'.format(dir, iter)
context_opts = "--left-context={lc} --right-context={rc}".format(
lc=left_context, rc=right_context)
common_lib.run_job("""{command} {dir}/log/progress.{iter}.log \
nnet3-info "{model}" '&&' \
nnet3-show-progress --use-gpu=no "{prev_model}" "{model}" \
"ark,bg:nnet3-copy-egs {context_opts} \
ark:{egs_dir}/train_diagnostic.egs ark:- | \
nnet3-merge-egs --minibatch-size=1:64 ark:- \
ark:- |" """.format(command=run_opts.command,
dir=dir,
iter=iter,
model=model,
context_opts=context_opts,
prev_model=prev_model,
egs_dir=egs_dir),
wait=wait,
background_process_handler=background_process_handler)
def get_best_nnet_model(dir,
iter,
best_model_index,
run_opts,
get_raw_nnet_from_am=True,
shrink=None):
scale = 1.0
if shrink is not None:
scale = shrink
best_model = "{dir}/{next_iter}.{best_model_index}.raw".format(
dir=dir, next_iter=iter + 1, best_model_index=best_model_index)
if get_raw_nnet_from_am:
out_model = ("""- \| nnet3-am-copy --set-raw-nnet=- \
{dir}/{iter}.mdl {dir}/{next_iter}.mdl""".format(
dir=dir, iter=iter, next_iter=iter + 1))
else:
out_model = "{dir}/{next_iter}.raw".format(dir=dir, next_iter=iter + 1)
common_lib.run_job("""{command} {dir}/log/select.{iter}.log \
nnet3-copy --scale={scale} {best_model} \
{out_model}""".format(command=run_opts.command,
dir=dir,
iter=iter,
best_model=best_model,
out_model=out_model,
scale=scale))
def get_best_nnet_model(dir, iter, best_model_index, run_opts,
get_raw_nnet_from_am=True, shrink=None):
scale = 1.0
if shrink is not None:
scale = shrink
best_model = "{dir}/{next_iter}.{best_model_index}.raw".format(
dir=dir,
next_iter=iter + 1,
best_model_index=best_model_index)
if get_raw_nnet_from_am:
out_model = ("""- \| nnet3-am-copy --set-raw-nnet=- \
{dir}/{iter}.mdl {dir}/{next_iter}.mdl""".format(
dir=dir, iter=iter, next_iter=iter + 1))
else:
out_model = "{dir}/{next_iter}.raw".format(dir=dir,
next_iter=iter + 1)
common_lib.run_job(
"""{command} {dir}/log/select.{iter}.log \
nnet3-copy --scale={scale} {best_model} \
{out_model}""".format(command=run_opts.command,
dir=dir, iter=iter,
best_model=best_model,
out_model=out_model, scale=scale))
def realign(dir, iter, feat_dir, lang, prev_egs_dir, cur_egs_dir,
prior_subset_size, num_archives, left_context, right_context,
run_opts, transform_dir=None, online_ivector_dir=None):
raise Exception("Realignment stage has not been implemented in nnet3")
logger.info("Getting average posterior for purposes of adjusting "
"the priors.")
# Note: this just uses CPUs, using a smallish subset of data.
# always use the first egs archive, which makes the script simpler;
# we're using different random subsets of it.
avg_post_vec_file = compute_average_posterior(
dir=dir, iter=iter, egs_dir=prev_egs_dir,
num_archives=num_archives, prior_subset_size=prior_subset_size,
left_context=left_context, right_context=right_context,
run_opts=run_opts)
avg_post_vec_file = "{dir}/post.{iter}.vec".format(dir=dir, iter=iter)
logger.info("Re-adjusting priors based on computed posteriors")
model = '{0}/{1}.mdl'.format(dir, iter)
adjust_am_priors(dir, model, avg_post_vec_file, model, run_opts)
alidir = align(dir, feat_dir, lang, run_opts, iter,
transform_dir, online_ivector_dir)
common_lib.run_job(
"""steps/nnet3/relabel_egs.sh --cmd "{command}" --iter {iter} \
{alidir} {prev_egs_dir} {cur_egs_dir}""".format(
command=run_opts.command,
iter=iter,
dir=dir,
alidir=alidir,
prev_egs_dir=prev_egs_dir,
cur_egs_dir=cur_egs_dir))
def compute_progress(dir, iter, egs_dir, left_context, right_context,
run_opts, mb_size=256,
background_process_handler=None, wait=False,
get_raw_nnet_from_am=True):
if get_raw_nnet_from_am:
prev_model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format(
dir, iter - 1)
model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format(dir, iter)
else:
prev_model = '{0}/{1}.raw'.format(dir, iter - 1)
model = '{0}/{1}.raw'.format(dir, iter)
context_opts = "--left-context={lc} --right-context={rc}".format(
lc=left_context, rc=right_context)
common_lib.run_job(
"""{command} {dir}/log/progress.{iter}.log \
nnet3-info "{model}" '&&' \
nnet3-show-progress --use-gpu=no "{prev_model}" "{model}" \
"ark,bg:nnet3-copy-egs {context_opts} \
ark:{egs_dir}/train_diagnostic.egs ark:- | \
nnet3-merge-egs --minibatch-size={mb_size} ark:- \
ark:- |" """.format(command=run_opts.command,
dir=dir,
iter=iter,
model=model,
context_opts=context_opts,
mb_size=mb_size,
prev_model=prev_model,
egs_dir=egs_dir),
wait=wait, background_process_handler=background_process_handler)
def create_phone_lm(dir, tree_dir, run_opts, lm_opts=None):
"""Create a phone LM for chain training
This method trains a phone LM for chain training using the alignments
in "tree_dir"
"""
try:
f = open(tree_dir + "/num_jobs", 'r')
num_ali_jobs = int(f.readline())
assert num_ali_jobs > 0
except:
raise Exception("""There was an error getting the number of alignment
jobs from {0}/num_jobs""".format(tree_dir))
alignments = ' '.join([
'{0}/ali.{1}.gz'.format(tree_dir, job)
for job in range(1, num_ali_jobs + 1)
])
common_lib.run_job("""{command} {dir}/log/make_phone_lm.log \
gunzip -c {alignments} \| \
ali-to-phones {tree_dir}/final.mdl ark:- ark:- \| \
chain-est-phone-lm {lm_opts} ark:- {dir}/phone_lm.fst""".format(
command=run_opts.command,
dir=dir,
alignments=alignments,
lm_opts=lm_opts if lm_opts is not None else '',
tree_dir=tree_dir))
def compute_presoftmax_prior_scale(dir,
alidir,
num_jobs,
run_opts,
presoftmax_prior_scale_power=-0.25):
# getting the raw pdf count
common_lib.run_job(
"""{command} JOB=1:{num_jobs} {dir}/log/acc_pdf.JOB.log \
ali-to-post "ark:gunzip -c {alidir}/ali.JOB.gz|" ark:- \| \
post-to-tacc --per-pdf=true {alidir}/final.mdl ark:- \
{dir}/pdf_counts.JOB""".format(command=run_opts.command,
num_jobs=num_jobs,
dir=dir,
alidir=alidir))
common_lib.run_job("""{command} {dir}/log/sum_pdf_counts.log \
vector-sum --binary=false {dir}/pdf_counts.* {dir}/pdf_counts \
""".format(command=run_opts.command, dir=dir))
for file in glob.glob('{0}/pdf_counts.*'.format(dir)):
os.remove(file)
pdf_counts = common_lib.read_kaldi_matrix('{0}/pdf_counts'.format(dir))[0]
scaled_counts = smooth_presoftmax_prior_scale_vector(
pdf_counts,
presoftmax_prior_scale_power=presoftmax_prior_scale_power,
smooth=0.01)
output_file = "{0}/presoftmax_prior_scale.vec".format(dir)
common_lib.write_kaldi_matrix(output_file, [scaled_counts])
common_lib.force_symlink(
"../presoftmax_prior_scale.vec",
"{0}/configs/presoftmax_prior_scale.vec".format(dir))
def get_average_nnet_model(dir, iter, nnets_list, run_opts,
get_raw_nnet_from_am=True, shrink=None):
scale = 1.0
if shrink is not None:
scale = shrink
next_iter = iter + 1
if get_raw_nnet_from_am:
out_model = ("""- \| nnet3-am-copy --set-raw-nnet=- --scale={scale} \
{dir}/{iter}.mdl {dir}/{next_iter}.mdl""".format(
dir=dir, iter=iter,
next_iter=next_iter,
scale=scale))
else:
if shrink is not None:
out_model = """- \| nnet3-copy --scale={scale} \
- {dir}/{next_iter}.raw""".format(
dir=dir, next_iter=next_iter, scale=scale)
else:
out_model = "{dir}/{next_iter}.raw".format(dir=dir,
next_iter=next_iter)
common_lib.run_job(
"""{command} {dir}/log/average.{iter}.log \
nnet3-average {nnets_list} \
{out_model}""".format(command=run_opts.command,
dir=dir,
iter=iter,
nnets_list=nnets_list,
out_model=out_model))
def prepare_initial_network(dir, run_opts, srand=-3):
common_lib.run_job(
"""{command} {dir}/log/add_first_layer.log \
nnet3-init --srand={srand} {dir}/init.raw \
{dir}/configs/layer1.config {dir}/0.raw""".format(
command=run_opts.command, srand=srand,
dir=dir))
def align(dir,
data,
lang,
run_opts,
iter=None,
transform_dir=None,
online_ivector_dir=None):
alidir = '{dir}/ali{ali_suffix}'.format(
dir=dir,
ali_suffix="_iter_{0}".format(iter) if iter is not None else "")
logger.info("Aligning the data{gpu}with {num_jobs} jobs.".format(
gpu=" using gpu " if run_opts.realign_use_gpu else " ",
num_jobs=run_opts.realign_num_jobs))
common_lib.run_job("""steps/nnet3/align.sh --nj {num_jobs_align} \
--cmd "{align_cmd} {align_queue_opt}" \
--use-gpu {align_use_gpu} \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{online_ivector_dir}" \
--iter "{iter}" {data} {lang} {dir} {alidir}""".format(
dir=dir,
align_use_gpu=("yes" if run_opts.realign_use_gpu else "no"),
align_cmd=run_opts.realign_command,
align_queue_opt=run_opts.realign_queue_opt,
num_jobs_align=run_opts.realign_num_jobs,
transform_dir=(transform_dir if transform_dir is not None else ""),
online_ivector_dir=(online_ivector_dir
if online_ivector_dir is not None else ""),
iter=iter if iter is not None else "",
alidir=alidir,
lang=lang,
data=data))
return alidir
def prepare_initial_network(dir, run_opts, srand=-3):
common_lib.run_job(
"""{command} {dir}/log/add_first_layer.log \
nnet3-init --srand={srand} {dir}/init.raw \
{dir}/configs/layer1.config {dir}/0.raw""".format(
command=run_opts.command, srand=srand,
dir=dir))
def get_average_nnet_model(dir,
iter,
nnets_list,
run_opts,
get_raw_nnet_from_am=True,
shrink=None):
scale = 1.0
if shrink is not None:
scale = shrink
next_iter = iter + 1
if get_raw_nnet_from_am:
out_model = ("""- \| nnet3-am-copy --set-raw-nnet=- --scale={scale} \
{dir}/{iter}.mdl {dir}/{next_iter}.mdl""".format(
dir=dir, iter=iter, next_iter=next_iter, scale=scale))
else:
if shrink is not None:
out_model = """- \| nnet3-copy --scale={scale} \
- {dir}/{next_iter}.raw""".format(
dir=dir, next_iter=next_iter, scale=scale)
else:
out_model = "{dir}/{next_iter}.raw".format(dir=dir,
next_iter=next_iter)
common_lib.run_job("""{command} {dir}/log/average.{iter}.log \
nnet3-average {nnets_list} \
{out_model}""".format(command=run_opts.command,
dir=dir,
iter=iter,
nnets_list=nnets_list,
out_model=out_model))
def realign(dir, iter, feat_dir, lang, prev_egs_dir, cur_egs_dir,
prior_subset_size, num_archives, left_context, right_context,
run_opts, transform_dir=None, online_ivector_dir=None):
raise Exception("Realignment stage has not been implemented in nnet3")
logger.info("Getting average posterior for purposes of adjusting "
"the priors.")
# Note: this just uses CPUs, using a smallish subset of data.
# always use the first egs archive, which makes the script simpler;
# we're using different random subsets of it.
avg_post_vec_file = compute_average_posterior(
dir=dir, iter=iter, egs_dir=prev_egs_dir,
num_archives=num_archives, prior_subset_size=prior_subset_size,
left_context=left_context, right_context=right_context,
run_opts=run_opts)
avg_post_vec_file = "{dir}/post.{iter}.vec".format(dir=dir, iter=iter)
logger.info("Re-adjusting priors based on computed posteriors")
model = '{0}/{1}.mdl'.format(dir, iter)
adjust_am_priors(dir, model, avg_post_vec_file, model, run_opts)
alidir = align(dir, feat_dir, lang, run_opts, iter,
transform_dir, online_ivector_dir)
common_lib.run_job(
"""steps/nnet3/relabel_egs.sh --cmd "{command}" --iter {iter} \
{alidir} {prev_egs_dir} {cur_egs_dir}""".format(
command=run_opts.command,
iter=iter,
dir=dir,
alidir=alidir,
prev_egs_dir=prev_egs_dir,
cur_egs_dir=cur_egs_dir))
def create_denominator_fst(dir, tree_dir, run_opts):
common_lib.run_job("""copy-transition-model {tree_dir}/final.mdl \
{dir}/0.trans_mdl""".format(dir=dir, tree_dir=tree_dir))
common_lib.run_job("""{command} {dir}/log/make_den_fst.log \
chain-make-den-fst {dir}/tree {dir}/0.trans_mdl \
{dir}/phone_lm.fst \
{dir}/den.fst {dir}/normalization.fst""".format(
dir=dir, command=run_opts.command))
def compute_average_posterior(dir, iter, egs_dir, num_archives,
prior_subset_size, left_context, right_context,
run_opts, get_raw_nnet_from_am=True):
""" Computes the average posterior of the network
Note: this just uses CPUs, using a smallish subset of data.
"""
for file in glob.glob('{0}/post.{1}.*.vec'.format(dir, iter)):
os.remove(file)
if run_opts.num_jobs_compute_prior > num_archives:
egs_part = 1
else:
egs_part = 'JOB'
if get_raw_nnet_from_am:
model = "nnet3-am-copy --raw=true {0}/combined.mdl -|".format(dir)
else:
model = "{dir}/final.raw".format(dir=dir)
context_opts = "--left-context={lc} --right-context={rc}".format(
lc=left_context, rc=right_context)
common_lib.run_job(
"""{command} JOB=1:{num_jobs_compute_prior} {prior_queue_opt} \
{dir}/log/get_post.{iter}.JOB.log \
nnet3-copy-egs {context_opts} \
ark:{egs_dir}/egs.{egs_part}.ark ark:- \| \
nnet3-subset-egs --srand=JOB --n={prior_subset_size} \
ark:- ark:- \| \
nnet3-merge-egs --measure-output-frames=true \
--minibatch-size=128 ark:- ark:- \| \
nnet3-compute-from-egs {prior_gpu_opt} --apply-exp=true \
"{model}" ark:- ark:- \| \
matrix-sum-rows ark:- ark:- \| vector-sum ark:- \
{dir}/post.{iter}.JOB.vec""".format(
command=run_opts.command,
dir=dir, model=model,
num_jobs_compute_prior=run_opts.num_jobs_compute_prior,
prior_queue_opt=run_opts.prior_queue_opt,
iter=iter, prior_subset_size=prior_subset_size,
egs_dir=egs_dir, egs_part=egs_part,
context_opts=context_opts,
prior_gpu_opt=run_opts.prior_gpu_opt))
# make sure there is time for $dir/post.{iter}.*.vec to appear.
time.sleep(5)
avg_post_vec_file = "{dir}/post.{iter}.vec".format(dir=dir, iter=iter)
common_lib.run_job(
"""{command} {dir}/log/vector_sum.{iter}.log \
vector-sum {dir}/post.{iter}.*.vec {output_file}
""".format(command=run_opts.command,
dir=dir, iter=iter, output_file=avg_post_vec_file))
for file in glob.glob('{0}/post.{1}.*.vec'.format(dir, iter)):
os.remove(file)
return avg_post_vec_file
def generate_egs(data,
alidir,
egs_dir,
left_context,
right_context,
valid_left_context,
valid_right_context,
run_opts,
stage=0,
feat_type='raw',
online_ivector_dir=None,
samples_per_iter=20000,
frames_per_eg=20,
srand=0,
egs_opts=None,
cmvn_opts=None,
transform_dir=None):
""" Wrapper for calling steps/nnet3/get_egs.sh
Generates targets from alignment directory 'alidir', which contains
the model final.mdl and alignments.
"""
common_lib.run_job("""steps/nnet3/get_egs.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--feat-type {feat_type} \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} --right-context {right_context} \
--valid-left-context {valid_left_context} \
--valid-right-context {valid_right_context} \
--stage {stage} \
--samples-per-iter {samples_per_iter} \
--frames-per-eg {frames_per_eg} \
--srand {srand} \
{data} {alidir} {egs_dir}
""".format(
command=run_opts.command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
feat_type=feat_type,
transform_dir=(transform_dir if transform_dir is not None else ''),
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None else ''),
left_context=left_context,
right_context=right_context,
valid_left_context=valid_left_context,
valid_right_context=valid_right_context,
stage=stage,
samples_per_iter=samples_per_iter,
frames_per_eg=frames_per_eg,
srand=srand,
data=data,
alidir=alidir,
egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
def combine_models(dir, num_iters, models_to_combine, num_chunk_per_minibatch,
egs_dir, left_context, right_context,
leaky_hmm_coefficient, l2_regularize,
xent_regularize, run_opts, background_process_handler=None):
""" Function to do model combination
In the nnet3 setup, the logic
for doing averaging of subsets of the models in the case where
there are too many models to reliably esetimate interpolation
factors (max_models_combine) is moved into the nnet3-combine.
"""
raw_model_strings = []
logger.info("Combining {0} models.".format(models_to_combine))
models_to_combine.add(num_iters)
for iter in sorted(models_to_combine):
model_file = '{0}/{1}.mdl'.format(dir, iter)
if os.path.exists(model_file):
raw_model_strings.append(
'"nnet3-am-copy --raw=true {0} -|"'.format(model_file))
else:
print("{0}: warning: model file {1} does not exist "
"(final combination)".format(sys.argv[0], model_file))
common_lib.run_job(
"""{command} {combine_queue_opt} {dir}/log/combine.log \
nnet3-chain-combine --num-iters=40 \
--l2-regularize={l2} --leaky-hmm-coefficient={leaky} \
--enforce-sum-to-one=true --enforce-positive-weights=true \
--verbose=3 {dir}/den.fst {raw_models} \
"ark,bg:nnet3-chain-copy-egs --left-context={lc} \
--right-context={rc} ark:{egs_dir}/combine.cegs ark:- | \
nnet3-chain-merge-egs --minibatch-size={num_chunk_per_mb} \
ark:- ark:- |" - \| \
nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl \
{dir}/final.mdl""".format(
command=run_opts.command,
combine_queue_opt=run_opts.combine_queue_opt,
lc=left_context, rc=right_context,
l2=l2_regularize, leaky=leaky_hmm_coefficient,
dir=dir, raw_models=" ".join(raw_model_strings),
num_chunk_per_mb=num_chunk_per_minibatch,
num_iters=num_iters,
egs_dir=egs_dir))
# Compute the probability of the final, combined model with
# the same subset we used for the previous compute_probs, as the
# different subsets will lead to different probs.
compute_train_cv_probabilities(
dir=dir, iter='final', egs_dir=egs_dir,
left_context=left_context, right_context=right_context,
l2_regularize=l2_regularize, xent_regularize=xent_regularize,
leaky_hmm_coefficient=leaky_hmm_coefficient,
run_opts=run_opts, wait=False,
background_process_handler=background_process_handler)
def compute_average_posterior(dir, iter, egs_dir, num_archives,
prior_subset_size, left_context, right_context,
run_opts, get_raw_nnet_from_am=True):
""" Computes the average posterior of the network
Note: this just uses CPUs, using a smallish subset of data.
"""
for file in glob.glob('{0}/post.{1}.*.vec'.format(dir, iter)):
os.remove(file)
if run_opts.num_jobs_compute_prior > num_archives:
egs_part = 1
else:
egs_part = 'JOB'
if get_raw_nnet_from_am:
model = "nnet3-am-copy --raw=true {0}/combined.mdl -|".format(dir)
else:
model = "{dir}/final.raw".format(dir=dir)
context_opts = "--left-context={lc} --right-context={rc}".format(
lc=left_context, rc=right_context)
common_lib.run_job(
"""{command} JOB=1:{num_jobs_compute_prior} {prior_queue_opt} \
{dir}/log/get_post.{iter}.JOB.log \
nnet3-copy-egs {context_opts} \
ark:{egs_dir}/egs.{egs_part}.ark ark:- \| \
nnet3-subset-egs --srand=JOB --n={prior_subset_size} \
ark:- ark:- \| \
nnet3-merge-egs --measure-output-frames=true \
--minibatch-size=128 ark:- ark:- \| \
nnet3-compute-from-egs {prior_gpu_opt} --apply-exp=true \
"{model}" ark:- ark:- \| \
matrix-sum-rows ark:- ark:- \| vector-sum ark:- \
{dir}/post.{iter}.JOB.vec""".format(
command=run_opts.command,
dir=dir, model=model,
num_jobs_compute_prior=run_opts.num_jobs_compute_prior,
prior_queue_opt=run_opts.prior_queue_opt,
iter=iter, prior_subset_size=prior_subset_size,
egs_dir=egs_dir, egs_part=egs_part,
context_opts=context_opts,
prior_gpu_opt=run_opts.prior_gpu_opt))
# make sure there is time for $dir/post.{iter}.*.vec to appear.
time.sleep(5)
avg_post_vec_file = "{dir}/post.{iter}.vec".format(dir=dir, iter=iter)
common_lib.run_job(
"""{command} {dir}/log/vector_sum.{iter}.log \
vector-sum {dir}/post.{iter}.*.vec {output_file}
""".format(command=run_opts.command,
dir=dir, iter=iter, output_file=avg_post_vec_file))
for file in glob.glob('{0}/post.{1}.*.vec'.format(dir, iter)):
os.remove(file)
return avg_post_vec_file
def adjust_am_priors(dir, input_model, avg_posterior_vector, output_model,
run_opts):
common_lib.run_job(
"""{command} {dir}/log/adjust_priors.final.log \
nnet3-am-adjust-priors "{input_model}" {avg_posterior_vector} \
"{output_model}" """.format(
command=run_opts.command,
dir=dir, input_model=input_model,
avg_posterior_vector=avg_posterior_vector,
output_model=output_model))
def create_denominator_fst(dir, tree_dir, run_opts):
common_lib.run_job(
"""copy-transition-model {tree_dir}/final.mdl \
{dir}/0.trans_mdl""".format(dir=dir, tree_dir=tree_dir))
common_lib.run_job(
"""{command} {dir}/log/make_den_fst.log \
chain-make-den-fst {dir}/tree {dir}/0.trans_mdl \
{dir}/phone_lm.fst \
{dir}/den.fst {dir}/normalization.fst""".format(
dir=dir, command=run_opts.command))
def adjust_am_priors(dir, input_model, avg_posterior_vector, output_model,
run_opts):
common_lib.run_job("""{command} {dir}/log/adjust_priors.final.log \
nnet3-am-adjust-priors "{input_model}" {avg_posterior_vector} \
"{output_model}" """.format(
command=run_opts.command,
dir=dir,
input_model=input_model,
avg_posterior_vector=avg_posterior_vector,
output_model=output_model))
def compute_train_cv_probabilities(dir,
iter,
egs_dir,
left_context,
right_context,
l2_regularize,
xent_regularize,
leaky_hmm_coefficient,
run_opts,
wait=False,
background_process_handler=None):
model = '{0}/{1}.mdl'.format(dir, iter)
common_lib.run_job("""{command} {dir}/log/compute_prob_valid.{iter}.log \
nnet3-chain-compute-prob --l2-regularize={l2} \
--leaky-hmm-coefficient={leaky} --xent-regularize={xent_reg} \
"nnet3-am-copy --raw=true {model} - |" {dir}/den.fst \
"ark,bg:nnet3-chain-copy-egs --left-context={lc} \
--right-context={rc} ark:{egs_dir}/valid_diagnostic.cegs \
ark:- | nnet3-chain-merge-egs --minibatch-size=1:64 ark:- ark:- |" \
""".format(command=run_opts.command,
dir=dir,
iter=iter,
model=model,
lc=left_context,
rc=right_context,
l2=l2_regularize,
leaky=leaky_hmm_coefficient,
xent_reg=xent_regularize,
egs_dir=egs_dir),
wait=wait,
background_process_handler=background_process_handler)
common_lib.run_job("""{command} {dir}/log/compute_prob_train.{iter}.log \
nnet3-chain-compute-prob --l2-regularize={l2} \
--leaky-hmm-coefficient={leaky} --xent-regularize={xent_reg} \
"nnet3-am-copy --raw=true {model} - |" {dir}/den.fst \
"ark,bg:nnet3-chain-copy-egs --left-context={lc} \
--right-context={rc} ark:{egs_dir}/train_diagnostic.cegs \
ark:- | nnet3-chain-merge-egs --minibatch-size=1:64 ark:- ark:- |" \
""".format(command=run_opts.command,
dir=dir,
iter=iter,
model=model,
lc=left_context,
rc=right_context,
l2=l2_regularize,
leaky=leaky_hmm_coefficient,
xent_reg=xent_regularize,
egs_dir=egs_dir),
wait=wait,
background_process_handler=background_process_handler)
def prepare_initial_acoustic_model(dir, alidir, run_opts, srand=-3):
""" Adds the first layer; this will also add in the lda.mat and
presoftmax_prior_scale.vec. It will also prepare the acoustic model
with the transition model."""
common_train_lib.prepare_initial_network(dir, run_opts, srand=srand)
# Convert to .mdl, train the transitions, set the priors.
common_lib.run_job("""{command} {dir}/log/init_mdl.log \
nnet3-am-init {alidir}/final.mdl {dir}/0.raw - \| \
nnet3-am-train-transitions - \
"ark:gunzip -c {alidir}/ali.*.gz|" {dir}/0.mdl
""".format(command=run_opts.command, dir=dir, alidir=alidir))
def compute_progress(dir,
iter,
egs_dir,
left_context,
right_context,
run_opts,
background_process_handler=None,
wait=False,
get_raw_nnet_from_am=True,
use_multitask_egs=False):
if get_raw_nnet_from_am:
prev_model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format(
dir, iter - 1)
model = "nnet3-am-copy --raw=true {0}/{1}.mdl - |".format(dir, iter)
else:
prev_model = '{0}/{1}.raw'.format(dir, iter - 1)
model = '{0}/{1}.raw'.format(dir, iter)
context_opts = "--left-context={lc} --right-context={rc}".format(
lc=left_context, rc=right_context)
scp_or_ark = "scp" if use_multitask_egs else "ark"
egs_suffix = ".scp" if use_multitask_egs else ".egs"
egs_rspecifier = "{0}:{1}/train_diagnostic{2}".format(
scp_or_ark, egs_dir, egs_suffix)
multitask_egs_opts = common_train_lib.get_multitask_egs_opts(
egs_dir,
egs_prefix="train_diagnostic.",
use_multitask_egs=use_multitask_egs)
common_lib.run_job("""{command} {dir}/log/progress.{iter}.log \
nnet3-info "{model}" '&&' \
nnet3-show-progress --use-gpu=no "{prev_model}" "{model}" \
"ark,bg:nnet3-copy-egs {context_opts} {multitask_egs_opts} \
{egs_rspecifier} ark:- | \
nnet3-merge-egs --minibatch-size=1:64 ark:- \
ark:- |" """.format(
command=run_opts.command,
dir=dir,
iter=iter,
egs_rspecifier=egs_rspecifier,
model=model,
context_opts=context_opts,
prev_model=prev_model,
egs_dir=egs_dir,
multitask_egs_opts=multitask_egs_opts),
wait=wait,
background_process_handler=background_process_handler)
def create_phone_lm(dir, tree_dir, run_opts, lm_opts=None):
"""Create a phone LM for chain training
This method trains a phone LM for chain training using the alignments
in "tree_dir"
"""
common_lib.run_job(
"""{command} {dir}/log/make_phone_lm.log \
chain-est-phone-lm {lm_opts} \
"ark:gunzip -c {tree_dir}/ali.*.gz | \
ali-to-phones {tree_dir}/final.mdl ark:- ark:- |" \
{dir}/phone_lm.fst""".format(
command=run_opts.command, dir=dir,
lm_opts=lm_opts if lm_opts is not None else '',
tree_dir=tree_dir))
def compute_train_cv_probabilities(dir,
iter,
egs_dir,
left_context,
right_context,
run_opts,
mb_size=256,
wait=False,
background_process_handler=None,
get_raw_nnet_from_am=True):
if get_raw_nnet_from_am:
model = "nnet3-am-copy --raw=true {dir}/{iter}.mdl - |".format(
dir=dir, iter=iter)
else:
model = "{dir}/{iter}.raw".format(dir=dir, iter=iter)
context_opts = "--left-context={lc} --right-context={rc}".format(
lc=left_context, rc=right_context)
common_lib.run_job(""" {command} {dir}/log/compute_prob_valid.{iter}.log \
nnet3-compute-prob "{model}" \
"ark,bg:nnet3-copy-egs {context_opts} \
ark:{egs_dir}/valid_diagnostic.egs ark:- | \
nnet3-merge-egs --minibatch-size={mb_size} ark:- \
ark:- |" """.format(command=run_opts.command,
dir=dir,
iter=iter,
context_opts=context_opts,
mb_size=mb_size,
model=model,
egs_dir=egs_dir),
wait=wait,
background_process_handler=background_process_handler)
common_lib.run_job("""{command} {dir}/log/compute_prob_train.{iter}.log \
nnet3-compute-prob "{model}" \
"ark,bg:nnet3-copy-egs {context_opts} \
ark:{egs_dir}/train_diagnostic.egs ark:- | \
nnet3-merge-egs --minibatch-size={mb_size} ark:- \
ark:- |" """.format(command=run_opts.command,
dir=dir,
iter=iter,
context_opts=context_opts,
mb_size=mb_size,
model=model,
egs_dir=egs_dir),
wait=wait,
background_process_handler=background_process_handler)
def prepare_initial_acoustic_model(dir, run_opts, srand=-1):
""" Adds the first layer; this will also add in the lda.mat and
presoftmax_prior_scale.vec. It will also prepare the acoustic model
with the transition model."""
common_train_lib.prepare_initial_network(dir, run_opts, srand=srand)
# The model-format for a 'chain' acoustic model is just the transition
# model and then the raw nnet, so we can use 'cat' to create this, as
# long as they have the same mode (binary or not binary).
# We ensure that they have the same mode (even if someone changed the
# script to make one or both of them text mode) by copying them both
# before concatenating them.
common_lib.run_job("""{command} {dir}/log/init_mdl.log \
nnet3-am-init {dir}/0.trans_mdl {dir}/0.raw \
{dir}/0.mdl""".format(command=run_opts.command, dir=dir))
def prepare_initial_acoustic_model(dir, alidir, run_opts, srand=-3):
""" Adds the first layer; this will also add in the lda.mat and
presoftmax_prior_scale.vec. It will also prepare the acoustic model
with the transition model."""
common_train_lib.prepare_initial_network(dir, run_opts, srand=srand)
# Convert to .mdl, train the transitions, set the priors.
common_lib.run_job(
"""{command} {dir}/log/init_mdl.log \
nnet3-am-init {alidir}/final.mdl {dir}/0.raw - \| \
nnet3-am-train-transitions - \
"ark:gunzip -c {alidir}/ali.*.gz|" {dir}/0.mdl
""".format(
command=run_opts.command, dir=dir, alidir=alidir
)
)
def generate_egs(data, alidir, egs_dir,
left_context, right_context,
valid_left_context, valid_right_context,
run_opts, stage=0,
feat_type='raw', online_ivector_dir=None,
samples_per_iter=20000, frames_per_eg=20, srand=0,
egs_opts=None, cmvn_opts=None, transform_dir=None):
""" Wrapper for calling steps/nnet3/get_egs.sh
Generates targets from alignment directory 'alidir', which contains
the model final.mdl and alignments.
"""
common_lib.run_job(
"""steps/nnet3/get_egs.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--feat-type {feat_type} \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} --right-context {right_context} \
--valid-left-context {valid_left_context} \
--valid-right-context {valid_right_context} \
--stage {stage} \
--samples-per-iter {samples_per_iter} \
--frames-per-eg {frames_per_eg} \
--srand {srand} \
{data} {alidir} {egs_dir}
""".format(command=run_opts.command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
feat_type=feat_type,
transform_dir=(transform_dir
if transform_dir is not None else
''),
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None
else ''),
left_context=left_context, right_context=right_context,
valid_left_context=valid_left_context,
valid_right_context=valid_right_context,
stage=stage, samples_per_iter=samples_per_iter,
frames_per_eg=frames_per_eg, srand=srand, data=data,
alidir=alidir, egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
def compute_progress(dir, iter, run_opts, wait=False,
background_process_handler=None):
prev_model = '{0}/{1}.mdl'.format(dir, iter - 1)
model = '{0}/{1}.mdl'.format(dir, iter)
common_lib.run_job(
"""{command} {dir}/log/progress.{iter}.log \
nnet3-am-info {model} '&&' \
nnet3-show-progress --use-gpu=no \
"nnet3-am-copy --raw=true {prev_model} - |" \
"nnet3-am-copy --raw=true {model} - |"
""".format(command=run_opts.command,
dir=dir,
iter=iter,
model=model,
prev_model=prev_model), wait=wait,
background_process_handler=background_process_handler)
def prepare_initial_acoustic_model(dir, run_opts, srand=-1):
""" Adds the first layer; this will also add in the lda.mat and
presoftmax_prior_scale.vec. It will also prepare the acoustic model
with the transition model."""
common_train_lib.prepare_initial_network(dir, run_opts,
srand=srand)
# The model-format for a 'chain' acoustic model is just the transition
# model and then the raw nnet, so we can use 'cat' to create this, as
# long as they have the same mode (binary or not binary).
# We ensure that they have the same mode (even if someone changed the
# script to make one or both of them text mode) by copying them both
# before concatenating them.
common_lib.run_job(
"""{command} {dir}/log/init_mdl.log \
nnet3-am-init {dir}/0.trans_mdl {dir}/0.raw \
{dir}/0.mdl""".format(command=run_opts.command, dir=dir))
def compute_train_cv_probabilities(dir, iter, egs_dir, left_context,
right_context, run_opts, mb_size=256,
wait=False, background_process_handler=None,
get_raw_nnet_from_am=True):
if get_raw_nnet_from_am:
model = "nnet3-am-copy --raw=true {dir}/{iter}.mdl - |".format(
dir=dir, iter=iter)
else:
model = "{dir}/{iter}.raw".format(dir=dir, iter=iter)
context_opts = "--left-context={lc} --right-context={rc}".format(
lc=left_context, rc=right_context)
common_lib.run_job(
""" {command} {dir}/log/compute_prob_valid.{iter}.log \
nnet3-compute-prob "{model}" \
"ark,bg:nnet3-copy-egs {context_opts} \
ark:{egs_dir}/valid_diagnostic.egs ark:- | \
nnet3-merge-egs --minibatch-size={mb_size} ark:- \
ark:- |" """.format(command=run_opts.command,
dir=dir,
iter=iter,
context_opts=context_opts,
mb_size=mb_size,
model=model,
egs_dir=egs_dir),
wait=wait, background_process_handler=background_process_handler)
common_lib.run_job(
"""{command} {dir}/log/compute_prob_train.{iter}.log \
nnet3-compute-prob "{model}" \
"ark,bg:nnet3-copy-egs {context_opts} \
ark:{egs_dir}/train_diagnostic.egs ark:- | \
nnet3-merge-egs --minibatch-size={mb_size} ark:- \
ark:- |" """.format(command=run_opts.command,
dir=dir,
iter=iter,
context_opts=context_opts,
mb_size=mb_size,
model=model,
egs_dir=egs_dir),
wait=wait, background_process_handler=background_process_handler)
def compute_train_cv_probabilities(dir, iter, egs_dir, left_context,
right_context, l2_regularize,
xent_regularize, leaky_hmm_coefficient,
run_opts, wait=False,
background_process_handler=None):
model = '{0}/{1}.mdl'.format(dir, iter)
common_lib.run_job(
"""{command} {dir}/log/compute_prob_valid.{iter}.log \
nnet3-chain-compute-prob --l2-regularize={l2} \
--leaky-hmm-coefficient={leaky} --xent-regularize={xent_reg} \
"nnet3-am-copy --raw=true {model} - |" {dir}/den.fst \
"ark,bg:nnet3-chain-copy-egs --left-context={lc} \
--right-context={rc} ark:{egs_dir}/valid_diagnostic.cegs \
ark:- | nnet3-chain-merge-egs ark:- ark:- |" \
""".format(command=run_opts.command, dir=dir, iter=iter, model=model,
lc=left_context, rc=right_context,
l2=l2_regularize, leaky=leaky_hmm_coefficient,
xent_reg=xent_regularize,
egs_dir=egs_dir), wait=wait,
background_process_handler=background_process_handler)
common_lib.run_job(
"""{command} {dir}/log/compute_prob_train.{iter}.log \
nnet3-chain-compute-prob --l2-regularize={l2} \
--leaky-hmm-coefficient={leaky} --xent-regularize={xent_reg} \
"nnet3-am-copy --raw=true {model} - |" {dir}/den.fst \
"ark,bg:nnet3-chain-copy-egs --left-context={lc} \
--right-context={rc} ark:{egs_dir}/train_diagnostic.cegs \
ark:- | nnet3-chain-merge-egs ark:- ark:- |" \
""".format(command=run_opts.command, dir=dir, iter=iter, model=model,
lc=left_context, rc=right_context,
l2=l2_regularize, leaky=leaky_hmm_coefficient,
xent_reg=xent_regularize,
egs_dir=egs_dir), wait=wait,
background_process_handler=background_process_handler)
def compute_preconditioning_matrix(dir,
egs_dir,
num_lda_jobs,
run_opts,
max_lda_jobs=None,
rand_prune=4.0,
lda_opts=None):
""" Function to estimate and write LDA matrix from cegs
This function is exactly similar to the version in module
libs.nnet3.train.frame_level_objf.common except this uses cegs instead of
egs files.
"""
if max_lda_jobs is not None:
if num_lda_jobs > max_lda_jobs:
num_lda_jobs = max_lda_jobs
# Write stats with the same format as stats for LDA.
common_lib.run_job(
"""{command} JOB=1:{num_lda_jobs} {dir}/log/get_lda_stats.JOB.log \
nnet3-chain-acc-lda-stats --rand-prune={rand_prune} \
{dir}/init.raw "ark:{egs_dir}/cegs.JOB.ark" \
{dir}/JOB.lda_stats""".format(command=run_opts.command,
num_lda_jobs=num_lda_jobs,
dir=dir,
egs_dir=egs_dir,
rand_prune=rand_prune))
# the above command would have generated dir/{1..num_lda_jobs}.lda_stats
lda_stat_files = map(lambda x: '{0}/{1}.lda_stats'.format(dir, x),
range(1, num_lda_jobs + 1))
common_lib.run_job("""{command} {dir}/log/sum_transform_stats.log \
sum-lda-accs {dir}/lda_stats {lda_stat_files}""".format(
command=run_opts.command,
dir=dir,
lda_stat_files=" ".join(lda_stat_files)))
for file in lda_stat_files:
try:
os.remove(file)
except OSError:
raise Exception("There was error while trying to remove "
"lda stat files.")
# this computes a fixed affine transform computed in the way we described
# in Appendix C.6 of http://arxiv.org/pdf/1410.7455v6.pdf; it's a scaled
# variant of an LDA transform but without dimensionality reduction.
common_lib.run_job("""{command} {dir}/log/get_transform.log \
nnet-get-feature-transform {lda_opts} {dir}/lda.mat \
{dir}/lda_stats""".format(
command=run_opts.command,
dir=dir,
lda_opts=lda_opts if lda_opts is not None else ""))
common_lib.force_symlink("../lda.mat", "{0}/configs/lda.mat".format(dir))
def compute_preconditioning_matrix(dir, egs_dir, num_lda_jobs, run_opts,
max_lda_jobs=None, rand_prune=4.0,
lda_opts=None):
""" Function to estimate and write LDA matrix from cegs
This function is exactly similar to the version in module
libs.nnet3.train.frame_level_objf.common except this uses cegs instead of
egs files.
"""
if max_lda_jobs is not None:
if num_lda_jobs > max_lda_jobs:
num_lda_jobs = max_lda_jobs
# Write stats with the same format as stats for LDA.
common_lib.run_job(
"""{command} JOB=1:{num_lda_jobs} {dir}/log/get_lda_stats.JOB.log \
nnet3-chain-acc-lda-stats --rand-prune={rand_prune} \
{dir}/init.raw "ark:{egs_dir}/cegs.JOB.ark" \
{dir}/JOB.lda_stats""".format(
command=run_opts.command,
num_lda_jobs=num_lda_jobs,
dir=dir,
egs_dir=egs_dir,
rand_prune=rand_prune))
# the above command would have generated dir/{1..num_lda_jobs}.lda_stats
lda_stat_files = map(lambda x: '{0}/{1}.lda_stats'.format(dir, x),
range(1, num_lda_jobs + 1))
common_lib.run_job(
"""{command} {dir}/log/sum_transform_stats.log \
sum-lda-accs {dir}/lda_stats {lda_stat_files}""".format(
command=run_opts.command,
dir=dir, lda_stat_files=" ".join(lda_stat_files)))
for file in lda_stat_files:
try:
os.remove(file)
except OSError:
raise Exception("There was error while trying to remove "
"lda stat files.")
# this computes a fixed affine transform computed in the way we described
# in Appendix C.6 of http://arxiv.org/pdf/1410.7455v6.pdf; it's a scaled
# variant of an LDA transform but without dimensionality reduction.
common_lib.run_job(
"""{command} {dir}/log/get_transform.log \
nnet-get-feature-transform {lda_opts} {dir}/lda.mat \
{dir}/lda_stats""".format(
command=run_opts.command, dir=dir,
lda_opts=lda_opts if lda_opts is not None else ""))
common_lib.force_symlink("../lda.mat", "{0}/configs/lda.mat".format(dir))
def combine_models(dir,
num_iters,
models_to_combine,
egs_dir,
left_context,
right_context,
minibatch_size_str,
run_opts,
background_process_handler=None,
chunk_width=None,
get_raw_nnet_from_am=True,
sum_to_one_penalty=0.0,
use_multitask_egs=False):
""" Function to do model combination
In the nnet3 setup, the logic
for doing averaging of subsets of the models in the case where
there are too many models to reliably esetimate interpolation
factors (max_models_combine) is moved into the nnet3-combine.
"""
raw_model_strings = []
logger.info("Combining {0} models.".format(models_to_combine))
models_to_combine.add(num_iters)
for iter in sorted(models_to_combine):
if get_raw_nnet_from_am:
model_file = '{0}/{1}.mdl'.format(dir, iter)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(
'"nnet3-am-copy --raw=true {0} -|"'.format(model_file))
else:
model_file = '{0}/{1}.raw'.format(dir, iter)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(model_file)
if get_raw_nnet_from_am:
out_model = ("| nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl "
"{dir}/combined.mdl".format(dir=dir, num_iters=num_iters))
else:
out_model = '{dir}/final.raw'.format(dir=dir)
context_opts = "--left-context={lc} --right-context={rc}".format(
lc=left_context, rc=right_context)
scp_or_ark = "scp" if use_multitask_egs else "ark"
egs_suffix = ".scp" if use_multitask_egs else ".egs"
egs_rspecifier = "{0}:{1}/combine{2}".format(scp_or_ark, egs_dir,
egs_suffix)
multitask_egs_opts = common_train_lib.get_multitask_egs_opts(
egs_dir, egs_prefix="combine.", use_multitask_egs=use_multitask_egs)
common_lib.run_job("""{command} {combine_queue_opt} {dir}/log/combine.log \
nnet3-combine --num-iters=80 \
--enforce-sum-to-one={hard_enforce} \
--sum-to-one-penalty={penalty} \
--enforce-positive-weights=true \
--verbose=3 {raw_models} \
"ark,bg:nnet3-copy-egs {context_opts} {multitask_egs_opts} \
{egs_rspecifier} ark:- | \
nnet3-merge-egs --minibatch-size={mbsize} ark:- ark:- |" \
"{out_model}"
""".format(command=run_opts.command,
combine_queue_opt=run_opts.combine_queue_opt,
dir=dir,
raw_models=" ".join(raw_model_strings),
egs_rspecifier=egs_rspecifier,
hard_enforce=(sum_to_one_penalty <= 0),
penalty=sum_to_one_penalty,
context_opts=context_opts,
mbsize=minibatch_size_str,
out_model=out_model,
egs_dir=egs_dir,
multitask_egs_opts=multitask_egs_opts))
# Compute the probability of the final, combined model with
# the same subset we used for the previous compute_probs, as the
# different subsets will lead to different probs.
if get_raw_nnet_from_am:
compute_train_cv_probabilities(
dir=dir,
iter='combined',
egs_dir=egs_dir,
left_context=left_context,
right_context=right_context,
run_opts=run_opts,
wait=False,
background_process_handler=background_process_handler,
use_multitask_egs=use_multitask_egs)
else:
compute_train_cv_probabilities(
dir=dir,
iter='final',
egs_dir=egs_dir,
left_context=left_context,
right_context=right_context,
run_opts=run_opts,
wait=False,
background_process_handler=background_process_handler,
get_raw_nnet_from_am=False,
use_multitask_egs=use_multitask_egs)
def generate_egs_using_targets(data, targets_scp, egs_dir,
left_context, right_context,
run_opts, stage=0,
left_context_initial=-1, right_context_final=-1,
feat_type='raw', online_ivector_dir=None,
target_type='dense', num_targets=-1,
samples_per_iter=20000, frames_per_eg_str="20",
srand=0, egs_opts=None, cmvn_opts=None,
transform_dir=None):
""" Wrapper for calling steps/nnet3/get_egs_targets.sh
This method generates egs directly from an scp file of targets, instead of
getting them from the alignments (as with the method generate_egs() in
module nnet3.train.frame_level_objf.acoustic_model).
Args:
target_type: "dense" if the targets are in matrix format
"sparse" if the targets are in posterior format
num_targets: must be explicitly specified for "sparse" targets.
For "dense" targets, this option is ignored and the target dim
is computed from the target matrix dimension
For other options, see the file steps/nnet3/get_egs_targets.sh
"""
if target_type == 'dense':
num_targets = common_lib.get_feat_dim_from_scp(targets_scp)
else:
if num_targets == -1:
raise Exception("--num-targets is required if "
"target-type is sparse")
common_lib.run_job(
"""steps/nnet3/get_egs_targets.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--feat-type {feat_type} \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} \
--right-context {right_context} \
--left-context-initial {left_context_initial} \
--right-context-final {right_context_final} \
--stage {stage} \
--samples-per-iter {samples_per_iter} \
--frames-per-eg {frames_per_eg_str} \
--srand {srand} \
--target-type {target_type} \
--num-targets {num_targets} \
{data} {targets_scp} {egs_dir}
""".format(command=run_opts.egs_command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
feat_type=feat_type,
transform_dir=(transform_dir
if transform_dir is not None
else ''),
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None
else ''),
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
stage=stage, samples_per_iter=samples_per_iter,
frames_per_eg_str=frames_per_eg_str, srand=srand,
num_targets=num_targets,
data=data,
targets_scp=targets_scp, target_type=target_type,
egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
def train(args, run_opts, background_process_handler):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Set some variables.
num_jobs = common_lib.get_number_of_jobs(args.ali_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.split_data(args.feat_dir, num_jobs)
shutil.copy('{0}/tree'.format(args.ali_dir), args.dir)
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
# this is really the number of times we add layers to the network for
# discriminative pretraining
num_hidden_layers = variables['num_hidden_layers']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -5):
logger.info("Initializing a basic network for estimating "
"preconditioning matrix")
common_lib.run_job(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
default_egs_dir = '{0}/egs'.format(args.dir)
if (args.stage <= -4) and args.egs_dir is None:
logger.info("Generating egs")
train_lib.acoustic_model.generate_egs(
data=args.feat_dir, alidir=args.ali_dir, egs_dir=default_egs_dir,
left_context=left_context, right_context=right_context,
valid_left_context=left_context + args.chunk_width,
valid_right_context=right_context + args.chunk_width,
run_opts=run_opts,
frames_per_eg=args.chunk_width,
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
samples_per_iter=args.samples_per_iter,
transform_dir=args.transform_dir,
stage=args.egs_stage)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context,
frames_per_eg, num_archives] = (
common_train_lib.verify_egs_dir(egs_dir, feat_dim, ivector_dim,
left_context, right_context))
assert(args.chunk_width == frames_per_eg)
if (args.num_jobs_final > num_archives):
raise Exception('num_jobs_final cannot exceed the number of archives '
'in the egs directory')
# copy the properties of the egs to dir for
# use during decoding
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if (args.stage <= -3):
logger.info('Computing the preconditioning matrix for input features')
train_lib.common.compute_preconditioning_matrix(
args.dir, egs_dir, num_archives, run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if (args.stage <= -2):
logger.info("Computing initial vector for FixedScaleComponent before"
" softmax, using priors^{prior_scale} and rescaling to"
" average 1".format(
prior_scale=args.presoftmax_prior_scale_power))
common_train_lib.compute_presoftmax_prior_scale(
args.dir, args.ali_dir, num_jobs, run_opts,
presoftmax_prior_scale_power=args.presoftmax_prior_scale_power)
if (args.stage <= -1):
logger.info("Preparing the initial acoustic model.")
train_lib.acoustic_model.prepare_initial_acoustic_model(
args.dir, args.ali_dir, run_opts)
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = args.num_epochs * num_archives
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2)
/ (args.num_jobs_initial + args.num_jobs_final))
models_to_combine = common_train_lib.verify_iterations(
num_iters, args.num_epochs,
num_hidden_layers, num_archives,
args.max_models_combine, args.add_layers_period,
args.num_jobs_final)
def learning_rate(iter, current_num_jobs, num_archives_processed):
return common_train_lib.get_learning_rate(iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
min_deriv_time = None
max_deriv_time = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time = (args.chunk_width - 1 + args.deriv_truncate_margin
+ model_right_context)
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial
+ (args.num_jobs_final - args.num_jobs_initial)
* float(iter) / num_iters)
if args.stage <= iter:
model_file = "{dir}/{iter}.mdl".format(dir=args.dir, iter=iter)
shrinkage_value = 1.0
if args.shrink_value != 1.0:
shrinkage_value = (args.shrink_value
if common_train_lib.do_shrinkage(
iter, model_file,
args.shrink_saturation_threshold)
else 1
)
logger.info("On iteration {0}, learning rate is {1} and "
"shrink value is {2}.".format(
iter, learning_rate(iter, current_num_jobs,
num_archives_processed),
shrinkage_value))
train_lib.common.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=learning_rate(iter, current_num_jobs,
num_archives_processed),
shrinkage_value=shrinkage_value,
minibatch_size=args.num_chunk_per_minibatch,
num_hidden_layers=num_hidden_layers,
add_layers_period=args.add_layers_period,
left_context=left_context,
right_context=right_context,
min_deriv_time=min_deriv_time,
max_deriv_time=max_deriv_time,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
cv_minibatch_size=args.cv_minibatch_size,
run_opts=run_opts,
background_process_handler=background_process_handler)
if args.cleanup:
# do a clean up everythin but the last 2 models, under certain
# conditions
common_train_lib.remove_model(
args.dir, iter-2, num_iters, models_to_combine,
args.preserve_model_interval)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times, data] = (
nnet3_log_parse.generate_accuracy_report(args.dir))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
logger.info("Doing final combination to produce final.mdl")
train_lib.common.combine_models(
dir=args.dir, num_iters=num_iters,
models_to_combine=models_to_combine, egs_dir=egs_dir,
run_opts=run_opts,
left_context=left_context, right_context=right_context,
background_process_handler=background_process_handler,
chunk_width=args.chunk_width)
if args.stage <= num_iters + 1:
logger.info("Getting average posterior for purposes of "
"adjusting the priors.")
avg_post_vec_file = train_lib.common.compute_average_posterior(
dir=args.dir, iter='combined', egs_dir=egs_dir,
num_archives=num_archives,
left_context=left_context, right_context=right_context,
prior_subset_size=args.prior_subset_size, run_opts=run_opts)
logger.info("Re-adjusting priors based on computed posteriors")
combined_model = "{dir}/combined.mdl".format(dir=args.dir)
final_model = "{dir}/final.mdl".format(dir=args.dir)
train_lib.common.adjust_am_priors(args.dir, combined_model,
avg_post_vec_file, final_model,
run_opts)
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs)
# do some reporting
[report, times, data] = nnet3_log_parse.generate_accuracy_report(args.dir)
if args.email is not None:
common_lib.send_mail(report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.run_job("steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def train(args, run_opts, background_process_handler):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Set some variables.
# num_leaves = common_lib.get_number_of_leaves_from_tree(args.ali_dir)
num_jobs = common_lib.get_number_of_jobs(args.ali_dir)
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
# split the training data into parts for individual jobs
# we will use the same number of jobs as that used for alignment
common_lib.split_data(args.feat_dir, num_jobs)
shutil.copy('{0}/tree'.format(args.ali_dir), args.dir)
with open('{0}/num_jobs'.format(args.dir), 'w') as f:
f.write(str(num_jobs))
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
# this is really the number of times we add layers to the network for
# discriminative pretraining
num_hidden_layers = variables['num_hidden_layers']
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -5):
logger.info("Initializing a basic network for estimating "
"preconditioning matrix")
common_lib.run_job(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
default_egs_dir = '{0}/egs'.format(args.dir)
if (args.stage <= -4) and args.egs_dir is None:
logger.info("Generating egs")
train_lib.acoustic_model.generate_egs(
data=args.feat_dir, alidir=args.ali_dir, egs_dir=default_egs_dir,
left_context=left_context, right_context=right_context,
run_opts=run_opts,
frames_per_eg=args.frames_per_eg,
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
samples_per_iter=args.samples_per_iter,
transform_dir=args.transform_dir,
stage=args.egs_stage)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context,
frames_per_eg, num_archives] = (
common_train_lib.verify_egs_dir(egs_dir, feat_dim, ivector_dim,
left_context, right_context))
assert(args.frames_per_eg == frames_per_eg)
if (args.num_jobs_final > num_archives):
raise Exception('num_jobs_final cannot exceed the number of archives '
'in the egs directory')
# copy the properties of the egs to dir for
# use during decoding
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if (args.stage <= -3):
logger.info('Computing the preconditioning matrix for input features')
train_lib.common.compute_preconditioning_matrix(
args.dir, egs_dir, num_archives, run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if (args.stage <= -2):
logger.info("Computing initial vector for FixedScaleComponent before"
" softmax, using priors^{prior_scale} and rescaling to"
" average 1".format(
prior_scale=args.presoftmax_prior_scale_power))
common_train_lib.compute_presoftmax_prior_scale(
args.dir, args.ali_dir, num_jobs, run_opts,
presoftmax_prior_scale_power=args.presoftmax_prior_scale_power)
if (args.stage <= -1):
logger.info("Preparing the initial acoustic model.")
train_lib.acoustic_model.prepare_initial_acoustic_model(
args.dir, args.ali_dir, run_opts)
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_expanded = num_archives * args.frames_per_eg
num_archives_to_process = args.num_epochs * num_archives_expanded
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2)
/ (args.num_jobs_initial + args.num_jobs_final))
models_to_combine = common_train_lib.verify_iterations(
num_iters, args.num_epochs,
num_hidden_layers, num_archives_expanded,
args.max_models_combine, args.add_layers_period,
args.num_jobs_final)
def learning_rate(iter, current_num_jobs, num_archives_processed):
return common_train_lib.get_learning_rate(iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial
+ (args.num_jobs_final - args.num_jobs_initial)
* float(iter) / num_iters)
if args.stage <= iter:
logger.info("On iteration {0}, learning rate is {1}.".format(
iter, learning_rate(iter, current_num_jobs,
num_archives_processed)))
train_lib.common.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=learning_rate(iter, current_num_jobs,
num_archives_processed),
minibatch_size=args.minibatch_size,
frames_per_eg=args.frames_per_eg,
num_hidden_layers=num_hidden_layers,
add_layers_period=args.add_layers_period,
left_context=left_context,
right_context=right_context,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
run_opts=run_opts,
background_process_handler=background_process_handler)
if args.cleanup:
# do a clean up everythin but the last 2 models, under certain
# conditions
common_train_lib.remove_model(
args.dir, iter-2, num_iters, models_to_combine,
args.preserve_model_interval)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times, data] = (
nnet3_log_parse.generate_accuracy_report(args.dir))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
logger.info("Doing final combination to produce final.mdl")
train_lib.common.combine_models(
dir=args.dir, num_iters=num_iters,
models_to_combine=models_to_combine,
egs_dir=egs_dir,
left_context=left_context, right_context=right_context,
run_opts=run_opts,
background_process_handler=background_process_handler)
if args.stage <= num_iters + 1:
logger.info("Getting average posterior for purposes of "
"adjusting the priors.")
avg_post_vec_file = train_lib.common.compute_average_posterior(
dir=args.dir, iter='combined', egs_dir=egs_dir,
num_archives=num_archives,
left_context=left_context, right_context=right_context,
prior_subset_size=args.prior_subset_size, run_opts=run_opts)
logger.info("Re-adjusting priors based on computed posteriors")
combined_model = "{dir}/combined.mdl".format(dir=args.dir)
final_model = "{dir}/final.mdl".format(dir=args.dir)
train_lib.common.adjust_am_priors(args.dir, combined_model,
avg_post_vec_file, final_model,
run_opts)
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs)
# do some reporting
[report, times, data] = nnet3_log_parse.generate_accuracy_report(args.dir)
if args.email is not None:
common_lib.send_mail(report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.run_job("steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def generate_chain_egs(dir, data, lat_dir, egs_dir,
left_context, right_context,
run_opts, stage=0,
valid_left_context=None, valid_right_context=None,
left_tolerance=None, right_tolerance=None,
frame_subsampling_factor=3,
alignment_subsampling_factor=3,
feat_type='raw', online_ivector_dir=None,
frames_per_iter=20000, frames_per_eg=20, srand=0,
egs_opts=None, cmvn_opts=None, transform_dir=None):
"""Wrapper for steps/nnet3/chain/get_egs.sh
See options in that script.
"""
common_lib.run_job(
"""steps/nnet3/chain/get_egs.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--feat-type {feat_type} \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} --right-context {right_context} \
--valid-left-context '{valid_left_context}' \
--valid-right-context '{valid_right_context}' \
--left-tolerance '{left_tolerance}' \
--right-tolerance '{right_tolerance}' \
--frame-subsampling-factor {frame_subsampling_factor} \
--alignment-subsampling-factor {alignment_subsampling_factor} \
--stage {stage} \
--frames-per-iter {frames_per_iter} \
--frames-per-eg {frames_per_eg} \
--srand {srand} \
{data} {dir} {lat_dir} {egs_dir}""".format(
command=run_opts.command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
feat_type=feat_type,
transform_dir=(transform_dir
if transform_dir is not None
else ''),
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None
else ''),
left_context=left_context, right_context=right_context,
valid_left_context=(valid_left_context
if valid_left_context is not None
else ''),
valid_right_context=(valid_right_context
if valid_right_context is not None
else ''),
left_tolerance=(left_tolerance
if left_tolerance is not None
else ''),
right_tolerance=(right_tolerance
if right_tolerance is not None
else ''),
frame_subsampling_factor=frame_subsampling_factor,
alignment_subsampling_factor=alignment_subsampling_factor,
stage=stage, frames_per_iter=frames_per_iter,
frames_per_eg=frames_per_eg, srand=srand,
data=data, lat_dir=lat_dir, dir=dir, egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
def combine_models(dir, num_iters, models_to_combine, egs_dir,
left_context, right_context,
run_opts, background_process_handler=None,
chunk_width=None, get_raw_nnet_from_am=True):
""" Function to do model combination
In the nnet3 setup, the logic
for doing averaging of subsets of the models in the case where
there are too many models to reliably esetimate interpolation
factors (max_models_combine) is moved into the nnet3-combine.
"""
raw_model_strings = []
logger.info("Combining {0} models.".format(models_to_combine))
models_to_combine.add(num_iters)
for iter in models_to_combine:
if get_raw_nnet_from_am:
model_file = '{0}/{1}.mdl'.format(dir, iter)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(
'"nnet3-am-copy --raw=true {0} -|"'.format(model_file))
else:
model_file = '{0}/{1}.raw'.format(dir, iter)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(model_file)
if chunk_width is not None:
# this is an RNN model
mbsize = int(1024.0/(chunk_width))
else:
mbsize = 1024
if get_raw_nnet_from_am:
out_model = ("| nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl "
"{dir}/combined.mdl".format(dir=dir, num_iters=num_iters))
else:
out_model = '{dir}/final.raw'.format(dir=dir)
context_opts = "--left-context={lc} --right-context={rc}".format(
lc=left_context, rc=right_context)
common_lib.run_job(
"""{command} {combine_queue_opt} {dir}/log/combine.log \
nnet3-combine --num-iters=40 \
--enforce-sum-to-one=true --enforce-positive-weights=true \
--verbose=3 {raw_models} \
"ark,bg:nnet3-copy-egs {context_opts} \
ark:{egs_dir}/combine.egs ark:- | \
nnet3-merge-egs --measure-output-frames=false \
--minibatch-size={mbsize} ark:- ark:- |" \
"{out_model}"
""".format(command=run_opts.command,
combine_queue_opt=run_opts.combine_queue_opt,
dir=dir, raw_models=" ".join(raw_model_strings),
context_opts=context_opts,
mbsize=mbsize,
out_model=out_model,
egs_dir=egs_dir))
# Compute the probability of the final, combined model with
# the same subset we used for the previous compute_probs, as the
# different subsets will lead to different probs.
if get_raw_nnet_from_am:
compute_train_cv_probabilities(
dir=dir, iter='combined', egs_dir=egs_dir,
left_context=left_context, right_context=right_context,
run_opts=run_opts, wait=False,
background_process_handler=background_process_handler)
else:
compute_train_cv_probabilities(
dir=dir, iter='final', egs_dir=egs_dir,
left_context=left_context, right_context=right_context,
run_opts=run_opts, wait=False,
background_process_handler=background_process_handler,
get_raw_nnet_from_am=False)
def remove_nnet_egs(egs_dir):
common_lib.run_job(
"steps/nnet2/remove_egs.sh {egs_dir}".format(egs_dir=egs_dir))
def train(args, run_opts, background_process_handler):
""" The main function for training.
Args:
args: a Namespace object with the required parameters
obtained from the function process_args()
run_opts: RunOpts object obtained from the process_args()
"""
arg_string = pprint.pformat(vars(args))
logger.info("Arguments for the experiment\n{0}".format(arg_string))
# Set some variables.
feat_dim = common_lib.get_feat_dim(args.feat_dir)
ivector_dim = common_lib.get_ivector_dim(args.online_ivector_dir)
ivector_id = common_lib.get_ivector_extractor_id(args.online_ivector_dir)
config_dir = '{0}/configs'.format(args.dir)
var_file = '{0}/vars'.format(config_dir)
variables = common_train_lib.parse_generic_config_vars_file(var_file)
# Set some variables.
try:
model_left_context = variables['model_left_context']
model_right_context = variables['model_right_context']
# this is really the number of times we add layers to the network for
# discriminative pretraining
num_hidden_layers = variables['num_hidden_layers']
add_lda = common_lib.str_to_bool(variables['add_lda'])
include_log_softmax = common_lib.str_to_bool(
variables['include_log_softmax'])
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined in "
"{1}".format(str(e), '{0}/configs'.format(args.dir)))
left_context = args.chunk_left_context + model_left_context
right_context = args.chunk_right_context + model_right_context
left_context_initial = (args.chunk_left_context_initial + model_left_context if
args.chunk_left_context_initial >= 0 else -1)
right_context_final = (args.chunk_right_context_final + model_right_context if
args.chunk_right_context_final >= 0 else -1)
# Initialize as "raw" nnet, prior to training the LDA-like preconditioning
# matrix. This first config just does any initial splicing that we do;
# we do this as it's a convenient way to get the stats for the 'lda-like'
# transform.
if (args.stage <= -4):
logger.info("Initializing a basic network")
common_lib.run_job(
"""{command} {dir}/log/nnet_init.log \
nnet3-init --srand=-2 {dir}/configs/init.config \
{dir}/init.raw""".format(command=run_opts.command,
dir=args.dir))
default_egs_dir = '{0}/egs'.format(args.dir)
if (args.stage <= -3) and args.egs_dir is None:
logger.info("Generating egs")
if args.use_dense_targets:
target_type = "dense"
try:
num_targets = int(variables['num_targets'])
if (common_lib.get_feat_dim_from_scp(args.targets_scp)
!= num_targets):
raise Exception("Mismatch between num-targets provided to "
"script vs configs")
except KeyError as e:
num_targets = -1
else:
target_type = "sparse"
try:
num_targets = int(variables['num_targets'])
except KeyError as e:
raise Exception("KeyError {0}: Variables need to be defined "
"in {1}".format(
str(e), '{0}/configs'.format(args.dir)))
train_lib.raw_model.generate_egs_using_targets(
data=args.feat_dir, targets_scp=args.targets_scp,
egs_dir=default_egs_dir,
left_context=left_context,
right_context=right_context,
left_context_initial=left_context_initial,
right_context_final=right_context_final,
run_opts=run_opts,
frames_per_eg_str=args.chunk_width,
srand=args.srand,
egs_opts=args.egs_opts,
cmvn_opts=args.cmvn_opts,
online_ivector_dir=args.online_ivector_dir,
samples_per_iter=args.samples_per_iter,
transform_dir=args.transform_dir,
stage=args.egs_stage,
target_type=target_type,
num_targets=num_targets)
if args.egs_dir is None:
egs_dir = default_egs_dir
else:
egs_dir = args.egs_dir
[egs_left_context, egs_right_context,
frames_per_eg_str, num_archives] = (
common_train_lib.verify_egs_dir(egs_dir, feat_dim,
ivector_dim, ivector_id,
left_context, right_context))
if args.chunk_width != frames_per_eg_str:
raise Exception("mismatch between --egs.chunk-width and the frames_per_eg "
"in the egs dir {0} vs {1}".format(args.chunk_width,
frames_per_eg_str))
if (args.num_jobs_final > num_archives):
raise Exception('num_jobs_final cannot exceed the number of archives '
'in the egs directory')
# copy the properties of the egs to dir for
# use during decoding
common_train_lib.copy_egs_properties_to_exp_dir(egs_dir, args.dir)
if (add_lda and args.stage <= -2):
logger.info('Computing the preconditioning matrix for input features')
train_lib.common.compute_preconditioning_matrix(
args.dir, egs_dir, num_archives, run_opts,
max_lda_jobs=args.max_lda_jobs,
rand_prune=args.rand_prune)
if (args.stage <= -1):
logger.info("Preparing the initial network.")
common_train_lib.prepare_initial_network(args.dir, run_opts)
# set num_iters so that as close as possible, we process the data
# $num_epochs times, i.e. $num_iters*$avg_num_jobs) ==
# $num_epochs*$num_archives, where
# avg_num_jobs=(num_jobs_initial+num_jobs_final)/2.
num_archives_to_process = int(args.num_epochs * num_archives)
num_archives_processed = 0
num_iters = ((num_archives_to_process * 2)
/ (args.num_jobs_initial + args.num_jobs_final))
models_to_combine = common_train_lib.verify_iterations(
num_iters, args.num_epochs,
num_hidden_layers, num_archives,
args.max_models_combine, args.add_layers_period,
args.num_jobs_final)
def learning_rate(iter, current_num_jobs, num_archives_processed):
return common_train_lib.get_learning_rate(iter, current_num_jobs,
num_iters,
num_archives_processed,
num_archives_to_process,
args.initial_effective_lrate,
args.final_effective_lrate)
min_deriv_time = None
max_deriv_time_relative = None
if args.deriv_truncate_margin is not None:
min_deriv_time = -args.deriv_truncate_margin - model_left_context
max_deriv_time_relative = \
args.deriv_truncate_margin + model_right_context
logger.info("Training will run for {0} epochs = "
"{1} iterations".format(args.num_epochs, num_iters))
for iter in range(num_iters):
if (args.exit_stage is not None) and (iter == args.exit_stage):
logger.info("Exiting early due to --exit-stage {0}".format(iter))
return
current_num_jobs = int(0.5 + args.num_jobs_initial
+ (args.num_jobs_final - args.num_jobs_initial)
* float(iter) / num_iters)
if args.stage <= iter:
model_file = "{dir}/{iter}.raw".format(dir=args.dir, iter=iter)
shrinkage_value = 1.0
if args.shrink_value != 1.0:
shrinkage_value = (args.shrink_value
if common_train_lib.do_shrinkage(
iter, model_file,
args.shrink_saturation_threshold,
get_raw_nnet_from_am=False)
else 1
)
train_lib.common.train_one_iteration(
dir=args.dir,
iter=iter,
srand=args.srand,
egs_dir=egs_dir,
num_jobs=current_num_jobs,
num_archives_processed=num_archives_processed,
num_archives=num_archives,
learning_rate=learning_rate(iter, current_num_jobs,
num_archives_processed),
dropout_edit_string=common_train_lib.get_dropout_edit_string(
args.dropout_schedule,
float(num_archives_processed) / num_archives_to_process,
iter),
shrinkage_value=shrinkage_value,
minibatch_size_str=args.num_chunk_per_minibatch,
num_hidden_layers=num_hidden_layers,
add_layers_period=args.add_layers_period,
left_context=left_context,
right_context=right_context,
min_deriv_time=min_deriv_time,
max_deriv_time_relative=max_deriv_time_relative,
momentum=args.momentum,
max_param_change=args.max_param_change,
shuffle_buffer_size=args.shuffle_buffer_size,
run_opts=run_opts,
get_raw_nnet_from_am=False,
background_process_handler=background_process_handler)
if args.cleanup:
# do a clean up everythin but the last 2 models, under certain
# conditions
common_train_lib.remove_model(
args.dir, iter-2, num_iters, models_to_combine,
args.preserve_model_interval,
get_raw_nnet_from_am=False)
if args.email is not None:
reporting_iter_interval = num_iters * args.reporting_interval
if iter % reporting_iter_interval == 0:
# lets do some reporting
[report, times, data] = (
nnet3_log_parse.generate_acc_logprob_report(args.dir))
message = report
subject = ("Update : Expt {dir} : "
"Iter {iter}".format(dir=args.dir, iter=iter))
common_lib.send_mail(message, subject, args.email)
num_archives_processed = num_archives_processed + current_num_jobs
if args.stage <= num_iters:
logger.info("Doing final combination to produce final.raw")
train_lib.common.combine_models(
dir=args.dir, num_iters=num_iters,
models_to_combine=models_to_combine, egs_dir=egs_dir,
left_context=left_context, right_context=right_context,
minibatch_size_str=args.num_chunk_per_minibatch,
run_opts=run_opts, chunk_width=args.chunk_width,
background_process_handler=background_process_handler,
get_raw_nnet_from_am=False,
sum_to_one_penalty=args.combine_sum_to_one_penalty)
if include_log_softmax and args.stage <= num_iters + 1:
logger.info("Getting average posterior for purposes of "
"adjusting the priors.")
train_lib.common.compute_average_posterior(
dir=args.dir, iter='final', egs_dir=egs_dir,
num_archives=num_archives,
left_context=left_context, right_context=right_context,
prior_subset_size=args.prior_subset_size, run_opts=run_opts,
get_raw_nnet_from_am=False)
if args.cleanup:
logger.info("Cleaning up the experiment directory "
"{0}".format(args.dir))
remove_egs = args.remove_egs
if args.egs_dir is not None:
# this egs_dir was not created by this experiment so we will not
# delete it
remove_egs = False
common_train_lib.clean_nnet_dir(
nnet_dir=args.dir, num_iters=num_iters, egs_dir=egs_dir,
preserve_model_interval=args.preserve_model_interval,
remove_egs=remove_egs,
get_raw_nnet_from_am=False)
# do some reporting
[report, times, data] = nnet3_log_parse.generate_acc_logprob_report(args.dir)
if args.email is not None:
common_lib.send_mail(report, "Update : Expt {0} : "
"complete".format(args.dir), args.email)
with open("{dir}/accuracy.report".format(dir=args.dir), "w") as f:
f.write(report)
common_lib.run_job("steps/info/nnet3_dir_info.pl "
"{0}".format(args.dir))
def train_new_models(dir,
iter,
srand,
num_jobs,
num_archives_processed,
num_archives,
raw_model_string,
egs_dir,
left_context,
right_context,
momentum,
max_param_change,
shuffle_buffer_size,
minibatch_size,
cache_read_opt,
run_opts,
frames_per_eg=-1,
min_deriv_time=None,
max_deriv_time=None):
""" Called from train_one_iteration(), this model does one iteration of
training with 'num_jobs' jobs, and writes files like
exp/tdnn_a/24.{1,2,3,..<num_jobs>}.raw
We cannot easily use a single parallel SGE job to do the main training,
because the computation of which archive and which --frame option
to use for each job is a little complex, so we spawn each one separately.
this is no longer true for RNNs as we use do not use the --frame option
but we use the same script for consistency with FF-DNN code
Args:
frames_per_eg: The default value -1 implies chunk_level_training, which
is particularly applicable to RNN training. If it is > 0, then it
implies frame-level training, which is applicable for DNN training.
If it is > 0, then each parallel SGE job created, a different frame
numbered 0..frames_per_eg-1 is used.
min_deriv_time: Applicable for RNN training. A default value of None
implies a min_deriv_time of 0 is used. During RNN training, its
value is set to chunk_width - num_bptt_steps in the training
script.
"""
chunk_level_training = False if frames_per_eg > 0 else True
deriv_time_opts = []
if min_deriv_time is not None:
deriv_time_opts.append(
"--optimization.min-deriv-time={0}".format(min_deriv_time))
if max_deriv_time is not None:
deriv_time_opts.append(
"--optimization.max-deriv-time={0}".format(max_deriv_time))
context_opts = "--left-context={0} --right-context={1}".format(
left_context, right_context)
processes = []
for job in range(1, num_jobs + 1):
# k is a zero-based index that we will derive the other indexes from.
k = num_archives_processed + job - 1
# work out the 1-based archive index.
archive_index = (k % num_archives) + 1
if not chunk_level_training:
frame = (k / num_archives) % frames_per_eg
cache_write_opt = ""
if job == 1:
# an option for writing cache (storing pairs of nnet-computations
# and computation-requests) during training.
cache_write_opt = "--write-cache={dir}/cache.{iter}".format(
dir=dir, iter=iter + 1)
process_handle = common_lib.run_job(
"""{command} {train_queue_opt} {dir}/log/train.{iter}.{job}.log \
nnet3-train {parallel_train_opts} {cache_read_opt} \
{cache_write_opt} --print-interval=10 \
--momentum={momentum} \
--max-param-change={max_param_change} \
{deriv_time_opts} "{raw_model}" \
"ark,bg:nnet3-copy-egs {frame_opts} {context_opts} """
"""ark:{egs_dir}/egs.{archive_index}.ark ark:- |"""
"""nnet3-shuffle-egs --buffer-size={shuffle_buffer_size} """
"""--srand={srand} ark:- ark:- | """
"""nnet3-merge-egs --minibatch-size={minibatch_size} """
"""--measure-output-frames=false """
"""--discard-partial-minibatches=true ark:- ark:- |" \
{dir}/{next_iter}.{job}.raw""".format(
command=run_opts.command,
train_queue_opt=run_opts.train_queue_opt,
dir=dir,
iter=iter,
srand=iter + srand,
next_iter=iter + 1,
job=job,
parallel_train_opts=run_opts.parallel_train_opts,
cache_read_opt=cache_read_opt,
cache_write_opt=cache_write_opt,
frame_opts=("" if chunk_level_training else
"--frame={0}".format(frame)),
momentum=momentum,
max_param_change=max_param_change,
deriv_time_opts=" ".join(deriv_time_opts),
raw_model=raw_model_string,
context_opts=context_opts,
egs_dir=egs_dir,
archive_index=archive_index,
shuffle_buffer_size=shuffle_buffer_size,
minibatch_size=minibatch_size),
wait=False)
processes.append(process_handle)
all_success = True
for process in processes:
process.wait()
process.communicate()
if process.returncode != 0:
all_success = False
if not all_success:
open('{0}/.error'.format(dir), 'w').close()
raise Exception("There was error during training "
"iteration {0}".format(iter))
def remove_nnet_egs(egs_dir):
common_lib.run_job("steps/nnet2/remove_egs.sh {egs_dir}".format(
egs_dir=egs_dir))
def train_new_models(dir, iter, srand, num_jobs,
num_archives_processed, num_archives,
raw_model_string, egs_dir,
left_context, right_context,
momentum, max_param_change,
shuffle_buffer_size, minibatch_size,
cache_read_opt, run_opts,
frames_per_eg=-1,
min_deriv_time=None, max_deriv_time=None):
""" Called from train_one_iteration(), this model does one iteration of
training with 'num_jobs' jobs, and writes files like
exp/tdnn_a/24.{1,2,3,..<num_jobs>}.raw
We cannot easily use a single parallel SGE job to do the main training,
because the computation of which archive and which --frame option
to use for each job is a little complex, so we spawn each one separately.
this is no longer true for RNNs as we use do not use the --frame option
but we use the same script for consistency with FF-DNN code
Args:
frames_per_eg: The default value -1 implies chunk_level_training, which
is particularly applicable to RNN training. If it is > 0, then it
implies frame-level training, which is applicable for DNN training.
If it is > 0, then each parallel SGE job created, a different frame
numbered 0..frames_per_eg-1 is used.
min_deriv_time: Applicable for RNN training. A default value of None
implies a min_deriv_time of 0 is used. During RNN training, its
value is set to chunk_width - num_bptt_steps in the training
script.
"""
chunk_level_training = False if frames_per_eg > 0 else True
deriv_time_opts = []
if min_deriv_time is not None:
deriv_time_opts.append("--optimization.min-deriv-time={0}".format(
min_deriv_time))
if max_deriv_time is not None:
deriv_time_opts.append("--optimization.max-deriv-time={0}".format(
max_deriv_time))
context_opts = "--left-context={0} --right-context={1}".format(
left_context, right_context)
processes = []
for job in range(1, num_jobs+1):
# k is a zero-based index that we will derive the other indexes from.
k = num_archives_processed + job - 1
# work out the 1-based archive index.
archive_index = (k % num_archives) + 1
if not chunk_level_training:
frame = (k / num_archives) % frames_per_eg
cache_write_opt = ""
if job == 1:
# an option for writing cache (storing pairs of nnet-computations
# and computation-requests) during training.
cache_write_opt = "--write-cache={dir}/cache.{iter}".format(
dir=dir, iter=iter+1)
process_handle = common_lib.run_job(
"""{command} {train_queue_opt} {dir}/log/train.{iter}.{job}.log \
nnet3-train {parallel_train_opts} {cache_read_opt} \
{cache_write_opt} --print-interval=10 \
--momentum={momentum} \
--max-param-change={max_param_change} \
{deriv_time_opts} "{raw_model}" \
"ark,bg:nnet3-copy-egs {frame_opts} {context_opts} """
"""ark:{egs_dir}/egs.{archive_index}.ark ark:- |"""
"""nnet3-shuffle-egs --buffer-size={shuffle_buffer_size} """
"""--srand={srand} ark:- ark:- | """
"""nnet3-merge-egs --minibatch-size={minibatch_size} """
"""--measure-output-frames=false """
"""--discard-partial-minibatches=true ark:- ark:- |" \
{dir}/{next_iter}.{job}.raw""".format(
command=run_opts.command,
train_queue_opt=run_opts.train_queue_opt,
dir=dir, iter=iter, srand=iter + srand,
next_iter=iter + 1,
job=job,
parallel_train_opts=run_opts.parallel_train_opts,
cache_read_opt=cache_read_opt,
cache_write_opt=cache_write_opt,
frame_opts=(""
if chunk_level_training
else "--frame={0}".format(frame)),
momentum=momentum, max_param_change=max_param_change,
deriv_time_opts=" ".join(deriv_time_opts),
raw_model=raw_model_string, context_opts=context_opts,
egs_dir=egs_dir, archive_index=archive_index,
shuffle_buffer_size=shuffle_buffer_size,
minibatch_size=minibatch_size), wait=False)
processes.append(process_handle)
all_success = True
for process in processes:
process.wait()
process.communicate()
if process.returncode != 0:
all_success = False
if not all_success:
open('{0}/.error'.format(dir), 'w').close()
raise Exception("There was error during training "
"iteration {0}".format(iter))
def train_new_models(dir, iter, srand, num_jobs,
num_archives_processed, num_archives,
raw_model_string, egs_dir, left_context, right_context,
apply_deriv_weights,
min_deriv_time, max_deriv_time,
l2_regularize, xent_regularize, leaky_hmm_coefficient,
momentum, max_param_change,
shuffle_buffer_size, num_chunk_per_minibatch,
frame_subsampling_factor, truncate_deriv_weights,
cache_io_opts, run_opts):
"""
Called from train_one_iteration(), this method trains new models
with 'num_jobs' jobs, and
writes files like exp/tdnn_a/24.{1,2,3,..<num_jobs>}.raw
We cannot easily use a single parallel SGE job to do the main training,
because the computation of which archive and which --frame option
to use for each job is a little complex, so we spawn each one separately.
this is no longer true for RNNs as we use do not use the --frame option
but we use the same script for consistency with FF-DNN code
"""
deriv_time_opts = []
if min_deriv_time is not None:
deriv_time_opts.append("--optimization.min-deriv-time={0}".format(
min_deriv_time))
if max_deriv_time is not None:
deriv_time_opts.append("--optimization.max-deriv-time={0}".format(
int(max_deriv_time)))
processes = []
for job in range(1, num_jobs+1):
# k is a zero-based index that we will derive the other indexes from.
k = num_archives_processed + job - 1
# work out the 1-based archive index.
archive_index = (k % num_archives) + 1
# previous : frame_shift = (k/num_archives) % frame_subsampling_factor
frame_shift = ((archive_index + k/num_archives)
% frame_subsampling_factor)
if job == 1:
cur_cache_io_opts = "{0} --write-cache={1}/cache.{2}".format(
cache_io_opts, dir, iter + 1)
else:
cur_cache_io_opts = cache_io_opts
process_handle = common_lib.run_job(
"""{command} {train_queue_opt} {dir}/log/train.{iter}.{job}.log \
nnet3-chain-train {parallel_train_opts} \
--apply-deriv-weights={app_deriv_wts} \
--l2-regularize={l2} --leaky-hmm-coefficient={leaky} \
{cache_io_opts} --xent-regularize={xent_reg} \
{deriv_time_opts} \
--print-interval=10 --momentum={momentum} \
--max-param-change={max_param_change} \
"{raw_model}" {dir}/den.fst \
"ark,bg:nnet3-chain-copy-egs \
--left-context={lc} --right-context={rc} \
--truncate-deriv-weights={trunc_deriv} \
--frame-shift={fr_shft} \
ark:{egs_dir}/cegs.{archive_index}.ark ark:- | \
nnet3-chain-shuffle-egs --buffer-size={buf_size} \
--srand={srand} ark:- ark:- | nnet3-chain-merge-egs \
--minibatch-size={num_chunk_per_mb} ark:- ark:- |" \
{dir}/{next_iter}.{job}.raw""".format(
command=run_opts.command,
train_queue_opt=run_opts.train_queue_opt,
dir=dir, iter=iter, srand=iter + srand,
next_iter=iter + 1, job=job,
deriv_time_opts=" ".join(deriv_time_opts),
lc=left_context, rc=right_context,
trunc_deriv=truncate_deriv_weights,
app_deriv_wts=apply_deriv_weights,
fr_shft=frame_shift, l2=l2_regularize,
xent_reg=xent_regularize, leaky=leaky_hmm_coefficient,
parallel_train_opts=run_opts.parallel_train_opts,
momentum=momentum, max_param_change=max_param_change,
raw_model=raw_model_string,
egs_dir=egs_dir, archive_index=archive_index,
buf_size=shuffle_buffer_size,
cache_io_opts=cur_cache_io_opts,
num_chunk_per_mb=num_chunk_per_minibatch),
wait=False)
processes.append(process_handle)
all_success = True
for process in processes:
process.wait()
process.communicate()
if process.returncode != 0:
all_success = False
if not all_success:
open('{0}/.error'.format(dir), 'w').close()
raise Exception("There was error during training "
"iteration {0}".format(iter))
def combine_models(dir,
num_iters,
models_to_combine,
egs_dir,
left_context,
right_context,
run_opts,
background_process_handler=None,
chunk_width=None,
get_raw_nnet_from_am=True):
""" Function to do model combination
In the nnet3 setup, the logic
for doing averaging of subsets of the models in the case where
there are too many models to reliably esetimate interpolation
factors (max_models_combine) is moved into the nnet3-combine.
"""
raw_model_strings = []
logger.info("Combining {0} models.".format(models_to_combine))
models_to_combine.add(num_iters)
for iter in models_to_combine:
if get_raw_nnet_from_am:
model_file = '{0}/{1}.mdl'.format(dir, iter)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(
'"nnet3-am-copy --raw=true {0} -|"'.format(model_file))
else:
model_file = '{0}/{1}.raw'.format(dir, iter)
if not os.path.exists(model_file):
raise Exception('Model file {0} missing'.format(model_file))
raw_model_strings.append(model_file)
if chunk_width is not None:
# this is an RNN model
mbsize = int(1024.0 / (chunk_width))
else:
mbsize = 1024
if get_raw_nnet_from_am:
out_model = ("| nnet3-am-copy --set-raw-nnet=- {dir}/{num_iters}.mdl "
"{dir}/combined.mdl".format(dir=dir, num_iters=num_iters))
else:
out_model = '{dir}/final.raw'.format(dir=dir)
context_opts = "--left-context={lc} --right-context={rc}".format(
lc=left_context, rc=right_context)
common_lib.run_job("""{command} {combine_queue_opt} {dir}/log/combine.log \
nnet3-combine --num-iters=40 \
--enforce-sum-to-one=true --enforce-positive-weights=true \
--verbose=3 {raw_models} \
"ark,bg:nnet3-copy-egs {context_opts} \
ark:{egs_dir}/combine.egs ark:- | \
nnet3-merge-egs --measure-output-frames=false \
--minibatch-size={mbsize} ark:- ark:- |" \
"{out_model}"
""".format(command=run_opts.command,
combine_queue_opt=run_opts.combine_queue_opt,
dir=dir,
raw_models=" ".join(raw_model_strings),
context_opts=context_opts,
mbsize=mbsize,
out_model=out_model,
egs_dir=egs_dir))
# Compute the probability of the final, combined model with
# the same subset we used for the previous compute_probs, as the
# different subsets will lead to different probs.
if get_raw_nnet_from_am:
compute_train_cv_probabilities(
dir=dir,
iter='combined',
egs_dir=egs_dir,
left_context=left_context,
right_context=right_context,
run_opts=run_opts,
wait=False,
background_process_handler=background_process_handler)
else:
compute_train_cv_probabilities(
dir=dir,
iter='final',
egs_dir=egs_dir,
left_context=left_context,
right_context=right_context,
run_opts=run_opts,
wait=False,
background_process_handler=background_process_handler,
get_raw_nnet_from_am=False)
def generate_egs_using_targets(data,
targets_scp,
egs_dir,
left_context,
right_context,
valid_left_context,
valid_right_context,
run_opts,
stage=0,
feat_type='raw',
online_ivector_dir=None,
target_type='dense',
num_targets=-1,
samples_per_iter=20000,
frames_per_eg=20,
srand=0,
egs_opts=None,
cmvn_opts=None,
transform_dir=None):
""" Wrapper for calling steps/nnet3/get_egs_targets.sh
This method generates egs directly from an scp file of targets, instead of
getting them from the alignments (as with the method generate_egs() in
module nnet3.train.frame_level_objf.acoustic_model).
Args:
target_type: "dense" if the targets are in matrix format
"sparse" if the targets are in posterior format
num_targets: must be explicitly specified for "sparse" targets.
For "dense" targets, this option is ignored and the target dim
is computed from the target matrix dimension
For other options, see the file steps/nnet3/get_egs_targets.sh
"""
if target_type == 'dense':
num_targets = common_lib.get_feat_dim_from_scp(targets_scp)
else:
if num_targets == -1:
raise Exception("--num-targets is required if "
"target-type is sparse")
common_lib.run_job("""steps/nnet3/get_egs_targets.sh {egs_opts} \
--cmd "{command}" \
--cmvn-opts "{cmvn_opts}" \
--feat-type {feat_type} \
--transform-dir "{transform_dir}" \
--online-ivector-dir "{ivector_dir}" \
--left-context {left_context} --right-context {right_context} \
--valid-left-context {valid_left_context} \
--valid-right-context {valid_right_context} \
--stage {stage} \
--samples-per-iter {samples_per_iter} \
--frames-per-eg {frames_per_eg} \
--srand {srand} \
--target-type {target_type} \
--num-targets {num_targets} \
{data} {targets_scp} {egs_dir}
""".format(
command=run_opts.egs_command,
cmvn_opts=cmvn_opts if cmvn_opts is not None else '',
feat_type=feat_type,
transform_dir=(transform_dir if transform_dir is not None else ''),
ivector_dir=(online_ivector_dir
if online_ivector_dir is not None else ''),
left_context=left_context,
right_context=right_context,
valid_left_context=valid_left_context,
valid_right_context=valid_right_context,
stage=stage,
samples_per_iter=samples_per_iter,
frames_per_eg=frames_per_eg,
srand=srand,
num_targets=num_targets,
data=data,
targets_scp=targets_scp,
target_type=target_type,
egs_dir=egs_dir,
egs_opts=egs_opts if egs_opts is not None else ''))
