def evaulate_trf(model, vocab, read_model, tsize, fres):
res_name = '{}:'.format(int(tsize)) + os.path.split(read_model)[-1]
tskdir = '{}/'.format(tsize)
# rescore
config = ' -vocab {} '.format(vocab)
config += ' -read {}.model '.format(read_model)
config += ' -nbest {} '.format(data(tskdir)[3])
config += ' -lmscore {0}.lmscore'.format(read_model)
model.use(config)
# WER
[read_nbest, read_templ, read_acscore, read_lmscore] = data(tskdir)[3:7]
read_lmscore = read_model + '.lmscore'
[wer, lmscale, acscale] = wb.TuneWER(read_nbest, read_templ,
wb.LoadScore(read_lmscore),
wb.LoadScore(read_acscore),
np.linspace(0.1, 0.9, 9))
print('wer={:.4f} lmscale={:.2f} acscale={:.2f}'.format(
wer, lmscale, acscale))
# calculate the ppl on wsj test
templ_txt = model.workdir + os.path.split(read_templ)[-1] + '.rmlabel'
wb.file_rmlabel(read_templ, templ_txt)
PPL_templ = model.ppl(vocab, read_model + '.model', templ_txt)
LL_templ = -wb.PPL2LL(PPL_templ, templ_txt)
# output the result
fres.Add(res_name, ['LL-wsj', 'PPL-wsj'], [LL_templ, PPL_templ])
fres.AddWER(res_name, wer)
def main():
print(sys.argv)
if len(sys.argv) == 1:
print('\"python run_ngram.py -train\" train \n',
'\"python run_ngram.py -rescore\" rescore nbest\n',
'\"python run_ngram.py -wer\" compute WER')
bindir = '../../tools/srilm/'
fres = wb.FRes('result.txt') # the result file
order_reg = [5]
for tsize in [1, 2, 4]:
tskdir = '{}/'.format(tsize)
workdir = tskdir + 'ngramlm/'
model = ngram.model(bindir, workdir)
for order in order_reg:
write_model = workdir + '{}gram.lm'.format(order)
write_name = '{}:KN{}'.format(tsize, order)
print(write_model)
if '-train' in sys.argv or '-all' in sys.argv:
if order_reg.index(order) == 0:
model.prepare(
data(tskdir)[0],
data(tskdir)[1],
data(tskdir)[2])
model.train(order, write_model)
if '-test' in sys.argv or '-all' in sys.argv:
PPL = [0] * 3
PPL[0] = model.ppl(write_model, order, data(tskdir)[0])
PPL[1] = model.ppl(write_model, order, data(tskdir)[1])
PPL[2] = model.ppl(write_model, order, data(tskdir)[2])
fres.AddPPL(write_name, PPL, data(tskdir)[0:3])
if '-rescore' in sys.argv or '-all' in sys.argv:
model.rescore(write_model, order,
data(tskdir)[3], write_model[0:-3] + '.lmscore')
if '-wer' in sys.argv or '-all' in sys.argv:
[nbest, templ] = data(tskdir)[3:5]
lmscore = wb.LoadScore(write_model[0:-3] + '.lmscore')
acscore = wb.LoadScore(data(tskdir)[5])
[wer, lmscale, acscale] = wb.TuneWER(nbest, templ, lmscore,
acscore,
np.linspace(0.1, 0.9, 9))
print('wer={} lmscale={} acscale={}'.format(
wer, lmscale, acscale))
fres.AddWER(write_name, wer)
trans_txt = workdir + os.path.split(templ)[-1] + '.txt'
wb.file_rmlabel(templ, trans_txt)
PPL_temp = model.ppl(write_model, order, trans_txt)
LL_temp = -wb.PPL2LL(PPL_temp, trans_txt)
fres.Add(write_name, ['LL-wsj', 'PPL-wsj'],
[LL_temp, PPL_temp])
def wer_all(lmpaths, lmtypes, outlog):
fres = wb.FRes(outlog, True)
fres.Clean()
[read_nbest, read_trans, read_acscore] = data()[3:6]
lmscale_vec = np.linspace(0.1, 0.9, 9)
weight_vec = np.linspace(0.5, 0.5, 1)
for type in lmtypes:
exist_multiple_run = False
a = type.split('+')
for lm in a:
if lmpaths[lm].find('<run>') != -1:
exist_multiple_run = True
break
run_vec = [0]
run_name = type
if exist_multiple_run:
run_vec = range(0, 10)
run_name = type + ':<run>'
for run in run_vec:
run_str = 'run{}'.format(run)
name = run_name.replace('<run>', run_str)
opt_wer_vec = [100, 1.0, 1.0]
opt_weight = 1.0
if len(a) == 1:
lmscore = wb.LoadScore(lmpaths[a[0]].replace('<run>', run_str))
opt_wer_vec = wb.TuneWER(read_nbest, read_trans, lmscore,
read_acscore, lmscale_vec)
opt_weight = 1.0
else:
lmscore1 = np.array(
wb.LoadScore(lmpaths[a[0]].replace('<run>', run_str)))
lmscore2 = np.array(
wb.LoadScore(lmpaths[a[1]].replace('<run>', run_str)))
for w in weight_vec:
lmscore = w * lmscore1 + (1 - w) * lmscore2
[wer, lmscale,
acscale] = wb.TuneWER(read_nbest, read_trans, lmscore,
read_acscore, lmscale_vec)
if wer < opt_wer_vec[0]:
opt_wer_vec = [wer, lmscale, acscale]
opt_weight = w
fres.Add(name, ['wer', 'lmscale', 'acscale', 'weight'],
opt_wer_vec + [opt_weight])
def wer_all(workdir, nbestdir, lmpaths, lmtypes):
wb.mkdir(workdir)
# calculate the wer for each task, each lmscale, each combination
for tsk in ['nbestlist_{}_{}'.format(a, b) for a in ['dt05', 'et05'] for b in ['real', 'simu']]:
print(tsk)
wb.mkdir(workdir + tsk)
fwer = open(workdir + tsk + '/wer.txt', 'wt')
read_nbest_txt = nbestdir + tsk + '/words_text'
read_transcript = nbestdir + tsk + '/text'
read_acscore = nbestdir + tsk + '/acwt'
read_gfscore = nbestdir + tsk + '/lmwt.nolm'
# remove the <UNK> in nbest
read_nbest_rmunk = workdir + tsk + '/words_text_rmunk'
nbest_rmUNK(read_nbest_txt, read_nbest_rmunk)
# load score
acscore = np.array(wb.LoadScore(read_acscore))
gfscore = np.array(wb.LoadScore(read_gfscore))
# load label
score_label = wb.LoadLabel(read_acscore)
# lm config
for lmtype in lmtypes:
a = lmtype.split('+')
if len(a) == 1:
lmscore = np.array(wb.LoadScore(lmpaths[a[0]].replace('<tsk>', tsk)))
elif len(a) == 2:
s1 = wb.LoadScore(lmpaths[a[0]].replace('<tsk>', tsk))
s2 = wb.LoadScore(lmpaths[a[1]].replace('<tsk>', tsk))
lmscore = 0.5 * np.array(s1) + 0.5 * np.array(s2)
# write lmscore
wb.WriteScore(workdir + tsk + '/' + lmtype + '.lmscore', lmscore, score_label)
for lmscale in np.linspace(9, 15, 7):
write_best = workdir + tsk + '/{}_lmscale={}.best'.format(lmtype, lmscale)
wb.GetBest(read_nbest_rmunk, (acscore + lmscale * (lmscore + gfscore)).tolist(), write_best)
[err, num, wer] = wb.CmpWER(write_best, read_transcript)
os.remove(write_best)
s = '{} wer={:.2f} err={} num={} lmscale={}'.format(lmtype, wer, err, num, lmscale)
print(' ' + s)
fwer.write(s + '\n')
fwer.flush()
fwer.close()
def main():
print(sys.argv)
if len(sys.argv) == 1:
print('\"python run_ngram.py -train\" train \n',
'\"python run_ngram.py -rescore\" rescore nbest\n',
'\"python run_ngram.py -wer\" compute WER')
absdir = os.getcwd() + '/'
bindir = absdir + '../../tools/srilm/'
workdir = absdir + 'ngramlm/'
wb.mkdir(workdir)
datas = [absdir + i for i in data()]
result_file = absdir + 'models_ppl.txt' # the result file
model = ngram.model(bindir, workdir)
order_reg = [2, 3, 4, 5]
for order in order_reg:
write_model = workdir + '{}gram.lm'.format(order)
print(write_model)
if '-train' in sys.argv:
if order_reg.index(order) == 0:
model.prepare(datas[0], datas[1], datas[2])
model.train(order, write_model, absdir + 'models_ppl.txt')
if '-rescore' in sys.argv:
model.rescore(write_model, order, datas[3],
write_model[0:-3] + '.lmscore')
if '-wer' in sys.argv:
[nbest, templ] = datas[3:5]
lmscore = wb.LoadScore(write_model[0:-3] + '.lmscore')
acscore = wb.LoadScore(datas[5])
[wer, lmscale, acscale] = wb.TuneWER(nbest, templ, lmscore,
acscore,
np.linspace(0.1, 0.9, 9))
print('wer={} lmscale={} acscale={}'.format(wer, lmscale, acscale))
fres = wb.FRes(result_file)
fres.AddWER('KN{}'.format(order), wer)
trans_txt = workdir + os.path.split(templ)[-1] + '.txt'
wb.file_rmlabel(templ, trans_txt)
PPL_temp = model.ppl(write_model, order, trans_txt)
LL_temp = -wb.PPL2LL(PPL_temp, trans_txt)
fres.Add('KN{}'.format(order), ['LL-wsj', 'PPL-wsj'],
[LL_temp, PPL_temp])
def main():
if len(sys.argv) == 1:
print('\"python run.py -train\" train LSTM\n',
'\"python run.py -rescore\" rescore nbest\n',
'\"python run.py -wer\" compute WER'
)
run_times = range(0, 10) # for multiple run
bindir = '../../tools/trf/bin/'
workdir = 'trflm/'
fres = wb.FRes('models_ppl.txt')
model = trf.model(bindir, workdir)
class_num = 200
train = workdir + 'train.id'
valid = workdir + 'valid.id'
test = workdir + 'test.id'
vocab = workdir + 'vocab_c{}.list'.format(class_num)
order = 4
feat = 'g4_w_c_ws_cs_wsh_csh_tied.fs'
#feat = 'g4_w_c_ws_cs_cpw.fs'
maxlen = 0
tmax = 20000
t0 = 2000
minibatch = 100
gamma_lambda = '3000,0'
gamma_zeta = '0,0.6'
reg = 4e-5
thread = 8
if '-res' in sys.argv:
fres.Read()
for i in range(1,len(fres.head)):
value = []
for runnum in run_times:
write_name = 'trf_c{}_{}.run{}'.format(class_num, feat[0:-3], runnum)
line = fres.Get(write_name)
value.append(line[i])
fres.Add('trf_c{}_{}.runavg'.format(class_num, feat[0:-3]), [fres.head[i]],
['{:.2f}+{:.2f}'.format(np.mean(value), np.std(value))] )
for runnum in run_times:
write_model = workdir + 'trf_c{}_{}.run{}'.format(class_num, feat[0:-3], runnum)
if '-train' in sys.argv or '-all' in sys.argv:
config = '-vocab {} -train {} -valid {} -test {} '.format(vocab, train, valid, test)
config += ' -order {} -feat {} '.format(order, feat)
config += ' -len {} '.format(maxlen)
config += ' -write {0}.model -log {0}.log '.format(write_model)
config += ' -t0 {} -iter {}'.format(t0, tmax)
config += ' -gamma-lambda {} -gamma-zeta {}'.format(gamma_lambda, gamma_zeta)
config += ' -L2 {} '.format(reg)
config += ' -mini-batch {} '.format(minibatch)
config += ' -thread {} '.format(thread)
config += ' -print-per-iter 10 '
config += ' -write-at-iter [{}:1000:{}]'.format(tmax-5000, tmax) # output the intermediate models
model.prepare(data()[0], data()[1], data()[2], class_num)
model.train(config)
if '-plot' in sys.argv:
baseline = fres.Get('KN5')
trf.PlotLog([write_model], [baseline])
if '-rescore' in sys.argv or '-all' in sys.argv:
config = ' -vocab {} '.format(vocab)
config += ' -read {}.model '.format(write_model)
config += ' -nbest {} '.format(data()[3])
config += ' -lmscore {0}.lmscore -lmscore-test-id {0}.test-id '.format(write_model)
model.use(config)
if '-wer' in sys.argv or '-all' in sys.argv:
[read_nbest, read_templ, read_acscore, read_lmscore] = data()[3:7]
read_lmscore = write_model + '.lmscore'
[wer, lmscale, acscale] = wb.TuneWER(read_nbest, read_templ,
wb.LoadScore(read_lmscore),
wb.LoadScore(read_acscore), np.linspace(0.1,0.9,9))
print('wer={:.4f} lmscale={:.2f} acscale={:.2f}'.format(wer, lmscale, acscale))
# calculate the ppl on wsj test
write_templ_id = workdir + os.path.split(read_templ)[1] + '.id'
v = trf.ReadVocab(vocab)
trf.NbestToID(read_templ, write_templ_id, v)
config = ' -vocab {} '.format(vocab)
config += ' -read {}.model '.format(write_model)
config += ' -test {} '.format(write_templ_id)
LL_templ = model.use(config)
PPL_templ = wb.LL2PPL(-LL_templ, write_templ_id)
LL = model.get_last_value(write_model + '.log')
# output the result
name = os.path.split(write_model)[1]
fres.AddLL(name, LL, data()[0:3])
fres.Add(name, ['LL-wsj', 'PPL-wsj'], [LL_templ, PPL_templ])
fres.AddWER(name, wer)
if '-stat' in sys.argv:
# calculate the mean and var of wers of the intermediate models
inte_wer = []
inte_model = []
# find model
for file_name in os.listdir(os.path.split(write_model)[0]):
file_path = os.path.split(write_model)[0] + os.sep + file_name
if not os.path.isfile(file_path):
continue
if file_name.find(os.path.split(write_model)[1]) == 0 and \
file_path.split('.')[-1] == 'model' and \
file_path.split('.')[-2][0] == 'n':
inte_model.append(file_path)
# compute wer
flog = open(workdir + 'inte_model_wer.log', 'wt')
for file_path in sorted(inte_model):
print(file_path)
t = int(file_path.split('.')[-2][1:])
# lmscore
write_lmscore = os.path.splitext(file_path)[0] + '.lmscore'
config = ' -vocab {} '.format(vocab)
config += ' -read {} '.format(file_path)
config += ' -nbest {} '.format(data()[3])
config += ' -lmscore {0} '.format(write_lmscore)
model.use(config, False)
# wer
[wer, lmscale, acscale] = wb.TuneWER(data()[3], data()[4],
wb.LoadScore(write_lmscore),
wb.LoadScore(data()[5]), np.linspace(0.1, 0.9, 9))
print('t={} wer={}'.format(t, wer))
flog.write('{} \t wer={}\n'.format(file_path, wer))
inte_wer.append([t, wer])
flog.close()
# plot wer
inte_wer = sorted(inte_wer, key=lambda d: d[0])
t_list = [i[0] for i in inte_wer]
wer_list = [i[1] for i in inte_wer]
wer_mean = np.mean(wer_list[-20:])
wer_std = np.std(wer_list[-20:])
print('wer_mean={} wer_std={}'.format(wer_mean, wer_std))
plt.figure()
plt.plot(t_list, wer_list)
plt.xlabel('t')
plt.ylabel('wer')
plt.show()
if '-ais' in sys.argv:
[read_nbest, read_templ, read_acscore] = data()[3:6]
write_templ_id = workdir + os.path.split(read_templ)[1] + '.id'
v = trf.ReadVocab(vocab)
trf.NbestToID(read_templ, write_templ_id, v)
# run asi to calculate the normalization constnts of models
ais_chain = 10
ais_inter = 10000
ais_model = '{}.ais{}_{}.model'.format(write_model, ais_chain, ais_inter)
if not os.path.exists(ais_model):
config = ' -vocab {0} -read {1}.model -write {2}'.format(vocab, write_model, ais_model)
config += ' -norm-method AIS -AIS-chain {} -AIS-inter {} -thread {} '.format(ais_chain, ais_inter, thread)
config += ' -norm-len-max {} '.format(trf.FileMaxLen(read_nbest)-1) # just compute the needed length
model.use(config)
# rescore and compute wer
write_lmscore = os.path.splitext(ais_model)[0] + '.lmscore'
config = ' -vocab {} -read {}'.format(vocab, ais_model)
config += ' -nbest {} -test {} '.format(read_nbest, write_templ_id) # calculate the ppl of test set
config += ' -lmscore {} '.format(write_lmscore)
LL_templ = model.use(config, False)
PPL_templ = wb.LL2PPL(-LL_templ, write_templ_id)
[wer, lmscale, acscale] = wb.TuneWER(read_nbest, read_templ,
write_lmscore, read_acscore, np.linspace(0.1, 0.9, 9))
# calculate the LL of train/valid/test
LL = [0]*3
id_data = [train, valid, test] # are id files
for i in range(3):
config = ' -vocab {} -read {} -test {} '.format(vocab, ais_model, id_data[i])
LL[i] = model.use(config, False)
# write to res file
name = os.path.split(write_model)[1]+":AIS{}-{}".format(ais_chain, ais_inter)
fres.AddLL(name, LL, id_data)
fres.AddWER(name, wer)
fres.Add(name, ['LL-wsj', 'PPL-wsj'], [LL_templ, PPL_templ])
def main():
if len(sys.argv) == 1:
print('\"python run.py -train\" train LSTM\n',
'\"python run.py -rescore\" rescore nbest\n',
'\"python run.py -wer\" compute WER')
for tsize in [1]:
bindir = '../../tools/trf/bin/'
tskdir = '{}/'.format(tsize)
workdir = tskdir + 'trflm/'
fres = wb.FRes('result.txt')
model = trf.model(bindir, workdir)
class_num = 200
train = workdir + 'train.id'
valid = workdir + 'valid.id'
test = workdir + 'test.id'
vocab = workdir + 'vocab_c{}.list'.format(class_num)
order = 4
feat = 'g4_w_c_ws_cs_wsh_csh_tied.fs'
#feat = 'g4_w_c_ws_cs_cpw.fs'
maxlen = 100
tmax = 50000
t0 = 2000
minibatch = 100
gamma_lambda = '1000,0'
gamma_zeta = '0,0.6'
reg = 1e-6
thread = 8
write_model = workdir + 'trf_c{}_{}'.format(class_num, feat[0:-3])
write_name = '{}:{}'.format(tsize, os.path.split(write_model)[1])
if '-class' in sys.argv:
# just cluster for each tsks.
model.prepare(
data(tskdir)[0],
data(tskdir)[1],
data(tskdir)[2], class_num)
if '-train' in sys.argv or '-all' in sys.argv:
config = '-vocab {} -train {} -valid {} -test {} '.format(
vocab, train, valid, test)
config += ' -order {} -feat {} '.format(order, feat)
config += ' -len {} '.format(maxlen)
config += ' -write {0}.model -log {0}.log '.format(write_model)
config += ' -t0 {} -iter {}'.format(t0, tmax)
config += ' -gamma-lambda {} -gamma-zeta {}'.format(
gamma_lambda, gamma_zeta)
config += ' -L2 {} '.format(reg)
config += ' -mini-batch {} '.format(minibatch)
config += ' -thread {} '.format(thread)
config += ' -print-per-iter 10 -write-at-iter 10000:10000:{}'.format(
tmax)
model.prepare(
data(tskdir)[0],
data(tskdir)[1],
data(tskdir)[2], class_num)
model.train(config)
# output
LL = model.get_last_value(write_model + '.log')
fres.AddLL(write_name, LL, data(tskdir)[0:3])
if '-plot' in sys.argv:
baseline = fres.Get('{}:KN5'.format(tsize))
trf.PlotLog([write_model], [baseline])
if '-rescore' in sys.argv or '-all' in sys.argv:
config = ' -vocab {} '.format(vocab)
config += ' -read {}.model '.format(write_model)
config += ' -nbest {} '.format(data(tskdir)[3])
config += ' -lmscore {0}.lmscore -lmscore-test-id {0}.test-id '.format(
write_model)
model.use(config)
if '-wer' in sys.argv or '-all' in sys.argv:
[read_nbest, read_templ, read_acscore,
read_lmscore] = data(tskdir)[3:7]
read_lmscore = write_model + '.lmscore'
[wer, lmscale, acscale] = wb.TuneWER(read_nbest, read_templ,
wb.LoadScore(read_lmscore),
wb.LoadScore(read_acscore),
np.linspace(0.1, 0.9, 9))
print('wer={:.4f} lmscale={:.2f} acscale={:.2f}'.format(
wer, lmscale, acscale))
# calculate the ppl on wsj test
write_templ_id = workdir + os.path.split(read_templ)[1] + '.id'
v = trf.ReadVocab(vocab)
trf.NbestToID(read_templ, write_templ_id, v)
config = ' -vocab {} '.format(vocab)
config += ' -read {}.model '.format(write_model)
config += ' -test {} '.format(write_templ_id)
LL_templ = model.use(config)
PPL_templ = wb.LL2PPL(-LL_templ, write_templ_id)
# output the result
fres.Add(write_name, ['LL-wsj', 'PPL-wsj'], [LL_templ, PPL_templ])
fres.AddWER(write_name, wer)
