def main():
# ------------- Parse arguments from command line ----------------------
# 1. Add a discription of this program
args.discribe("This program is used to prepare TIMIT data.")
# 2. Add some options
args.add("--timitRoot", dtype=str, abbr="-t", default="/Corpus/TIMIT", discription="The root path of timit dataset.")
args.add("--expDir", dtype=str, abbr="-e", default="exp", discription="The output path to save generated data.")
# 3. Then start to parse arguments.
args.parse()
# 4. Take a backup of arguments
args.save( os.path.join(args.expDir,"conf","prepare_data.args") )
# ------------- Do some preparative work ----------------------
# 2. Ensure Kaldi has existed
declare.kaldi_existed()
# 3. sph2pipe tool will be used if the timit data is sph format.
sph2pipeTool = os.path.join(info.KALDI_ROOT,"tools","sph2pipe_v2.5","sph2pipe")
declare.is_file("sph2pipe tool",sph2pipeTool)
# ------------- Check TIMIT data format -------------
# 1. Get the directory name
declare.is_dir("TIMIT root directory", args.timitRoot)
dirNames = os.listdir(args.timitRoot)
if "TRAIN" in dirNames and "TEST" in dirNames:
uppercaseFlag = True
trainResourceDir = "TRAIN"
testResourceDir = "TEST"
testWavFile = os.path.join(args.timitRoot,"TRAIN","DR1","FCJF0","SA1.WAV") # used to test the file format
wavFileSuffix = "WAV"
txtFileSuffix = "PHN"
elif "train" in dirNames and "test" in dirNames:
uppercaseFlag = False
trainResourceDir = "train"
testResourceDir = "test"
testWavFile = os.path.join(args.timitRoot,"train","dr1","fcjf0","sa1.wav") # used to test the file format
wavFileSuffix = "wav"
txtFileSuffix = "phn"
else:
raise Exception(f"Wrong format of train or test data directories.")
# 2. check whether wave file is sph format.
formatCheckCmd = f"{sph2pipeTool} -f wav {testWavFile}"
out,err,cod = exkaldi.utils.run_shell_command(formatCheckCmd, stderr="PIPE")
if cod == 0:
sphFlag = True
else:
sphFlag = False
# --------- Generate phone-map dictionary --------
# 1. Generate 60-48 catagories and 48-39 catagories mapping dictionary
phoneMap_60_to_48 = exkaldi.ListTable(name="69-48")
phoneMap_48_to_39 = exkaldi.ListTable(name="48-39")
mapFile = os.path.join(info.KALDI_ROOT,"egs","timit","s5","conf","phones.60-48-39.map")
declare.is_file("60-48-39 phone map", mapFile) # Check whether or not it existed
with open(mapFile,"r",encoding="utf-8") as fr:
lines = fr.readlines()
for line in lines:
line = line.strip().split()
if len(line) < 3: #phone "q" will be omitted temporarily.
continue
phoneMap_60_to_48[line[0]] = line[1]
phoneMap_48_to_39[line[1]] = line[2]
# 2. Save 48-39 phone map for futher use.
phoneMap_48_to_39.save(os.path.join(args.expDir,"dict","phones.48_to_39.map"))
# --------- Generate train dataset --------
wavs = glob.glob(os.path.join(args.timitRoot,trainResourceDir,"*","*",f"*.{wavFileSuffix}"))
out = os.path.join(args.expDir,"data","train")
generate_data(wavs,out,sphFlag,sph2pipeTool,txtFileSuffix,phoneMap_60_to_48)
# --------- Generate dev and test data --------
for Name in ["dev", "test"]:
spkListFile = os.path.join( info.KALDI_ROOT,"egs","timit","s5","conf",f"{Name}_spk.list" )
declare.is_file(f"speakers list for {Name}", spkListFile) # Check whether or not it existed
with open(spkListFile,"r",encoding="utf-8") as fr:
spkList = fr.readlines()
wavs = []
for spk in spkList:
spk = spk.strip()
if len(spk) == 0:
continue
if uppercaseFlag:
spk = spk.upper()
wavs.extend(glob.glob(os.path.join(args.timitRoot,testResourceDir,"*",spk,f"*.{wavFileSuffix}")))
out = os.path.join(args.expDir,"data",Name)
generate_data(wavs,out,sphFlag,sph2pipeTool,txtFileSuffix,phoneMap_60_to_48)
def main():
# ------------- Parse arguments from command line ----------------------
# 1. Add a discription of this program
args.discribe(
"This program is used to make dictionaries and language model")
# 2. Add options
args.add(
"--expDir",
abbr="-e",
dtype=str,
default="exp",
discription="The data resources and output path of current experiment."
)
args.add("--order",
abbr="-o",
dtype=int,
default=6,
minV=1,
maxV=6,
discription="The maximum order of N-grams language model.")
# 3. Then start to parse arguments.
args.parse()
# 4. Take a backup of arguments
args.print_args() # print arguments to display
argsLogFile = os.path.join(args.expDir, "conf", "make_dict_and_LM.args")
args.save(argsLogFile)
# ------- Make the word-pronumciation lexicon file ------
textFile = os.path.join(args.expDir, "data", "train", "text")
trainTrans = exkaldi.load_transcription(
textFile) # "trans" is an exkaldi Transcription object
wordCount = trainTrans.count_word().sort(
) # accumulate all words and their frequency in the transcription
word2pron = exkaldi.ListTable(
dict((word, word)
for word in wordCount.keys())) # word to pronunciation
pronFile = os.path.join(args.expDir, "dict", "pronunciation.txt")
word2pron.save(pronFile) # save it to file
# ------- Make lexicons ------
# 1. Generate the LexiconBank object from word-pronumciation file.
# Depending on task, about 20 lexicons will be generated and managed by the LexiconBank.
lexicons = exkaldi.decode.graph.lexicon_bank(pronFile,
silWords={"sil": "sil"},
unkSymbol={"sil": "sil"},
optionalSilPhone="sil",
extraQuestions=[],
positionDependent=False,
shareSilPdf=False,
extraDisambigPhoneNumbers=1,
extraDisambigWords=[])
# 2. Add two extra questions.
lexicons.add_extra_question(lexicons("silence_phones"))
lexicons.add_extra_question(lexicons("nonsilence_phones"))
# 3. Save this lexicon bank for future use.
lexicons.save(os.path.join(args.expDir, "dict", "lexicons.lex"))
print(f"Generate lexicon bank done.")
# ------- Make Lexicon fst ------
# 1. Generate the Lexicon fst
exkaldi.decode.graph.make_L(lexicons,
outFile=os.path.join(args.expDir, "dict",
"L.fst"),
useSilprob=0.0,
useDisambigLexicon=False)
print(f"Generate lexicon fst done.")
# 1. Generate the disambig Lexicon fst
exkaldi.decode.graph.make_L(lexicons,
outFile=os.path.join(args.expDir, "dict",
"L_disambig.fst"),
useSilprob=0.0,
useDisambigLexicon=True)
print(f"Generate disambiguation lexicon fst done.")
# ------- Make GMM-HMM topological structure for GMM-HMM ------
exkaldi.hmm.make_topology(
lexicons,
outFile=os.path.join(args.expDir, "dict", "topo"),
numNonsilStates=3,
numSilStates=5,
)
print(f"Generate topo file done.")
# ------- Train N-Grams language model ------
# 1. Train a LM.
# We have trained 2,3,4 grams model with both srilm and kenlm and chose the best one, which is 3-grams model back kenlm.
# So we directly train this one.
exkaldi.lm.train_ngrams_kenlm(
lexicons,
order=args.order,
text=
trainTrans, # If "text" received an exkaldi Transcription object, the information of utterance IDs will be omitted automatically.
outFile=os.path.join(args.expDir, "lm", f"{args.order}grams.arpa"),
config={
"--discount_fallback": True,
"-S": "20%"
},
)
print(f"Generate ARPA language model done.")
# 2. Then test this model by compute the perplexity.
exkaldi.lm.arpa_to_binary(
arpaFile=os.path.join(args.expDir, "lm", f"{args.order}grams.arpa"),
outFile=os.path.join(args.expDir, "lm", f"{args.order}grams.binary"),
)
model = exkaldi.load_ngrams(
os.path.join(args.expDir, "lm", f"{args.order}grams.binary")
) # Actually, "load_ngrams" function also accepts ARPA format file.
# 3. Prepare test transcription
testTrans = exkaldi.load_transcription(
os.path.join(args.expDir, "data", "test", "text"))
# 4. score
perScore = model.perplexity(testTrans)
print(f"The weighted average perplexity of this model is: {perScore}.")
del model
del testTrans
# ------- Make Grammar fst ------
exkaldi.decode.graph.make_G(
lexicons,
arpaFile=os.path.join(args.expDir, "lm", f"{args.order}grams.arpa"),
outFile=os.path.join(args.expDir, "lm", f"G.{args.order}.fst"),
order=args.order)
print(f"Make Grammar fst done.")
# ------- Compose LG fst for futher use ------
exkaldi.decode.graph.compose_LG(
LFile=os.path.join(args.expDir, "dict", "L_disambig.fst"),
GFile=os.path.join(args.expDir, "lm", f"G.{args.order}.fst"),
outFile=os.path.join(args.expDir, "lm", f"LG.{args.order}.fst"),
)
print(f"Compose LG fst done.")
def main():
# ------------- Parse arguments from command line ----------------------
# 1. Add a discription of this program
args.discribe(
"This program is used to train triphone DNN acoustic model with Tensorflow"
)
# 2. Add options
args.add("--expDir",
abbr="-e",
dtype=str,
default="exp",
discription="The data and output path of current experiment.")
args.add(
"--LDAsplice",
dtype=int,
default=3,
discription="Splice how many frames to head and tail for LDA feature.")
args.add("--randomSeed",
dtype=int,
default=1234,
discription="Random seed.")
args.add("--batchSize",
abbr="-b",
dtype=int,
default=128,
discription="Mini batch size.")
args.add("--gpu",
abbr="-g",
dtype=str,
default="all",
choices=["all", "0", "1"],
discription="Use GPU.")
args.add("--epoch", dtype=int, default=30, discription="Epoches.")
args.add("--testStartEpoch",
dtype=int,
default=5,
discription="Start to evaluate test dataset.")
args.add("--dropout",
abbr="-d",
dtype=float,
default=0.2,
discription="Dropout.")
args.add("--useCMVN",
dtype=bool,
default=False,
discription="Wether apply CMVN to fmllr feature.")
args.add(
"--splice",
dtype=int,
default=10,
discription="Splice how many frames to head and tail for Fmllr feature."
)
args.add("--delta",
dtype=int,
default=2,
discription="Wether add delta to fmllr feature.")
args.add("--normalizeFeat",
dtype=bool,
default=True,
discription="Wether normalize the chunk dataset.")
args.add("--normalizeAMP",
dtype=bool,
default=False,
discription="Wether normalize the post-probability.")
args.add("--order",
abbr="-o",
dtype=int,
default=6,
discription="Language model order.")
args.add("--beam", dtype=int, default=13, discription="Decode beam size.")
args.add("--latBeam",
dtype=int,
default=6,
discription="Lattice beam size.")
args.add("--acwt",
dtype=float,
default=0.083333,
discription="Acoustic model weight.")
args.add("--predictModel",
abbr="-m",
dtype=str,
default="",
discription="If not void, skip training. Do decoding only.")
# 3. Then start to parse arguments.
args.parse()
# 4. Take a backup of arguments
argsLogFile = os.path.join(args.expDir, "conf", "train_dnn.args")
args.save(argsLogFile)
random.seed(args.randomSeed)
np.random.seed(args.randomSeed)
tf.random.set_seed(args.randomSeed)
# ------------- Prepare data for dnn training ----------------------
if not os.path.isfile(f"./{args.expDir}/train_dnn/data/dims"):
prepare_DNN_data()
# ------------- Prepare data for dnn training ----------------------
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
outDir = f"{args.expDir}/train_dnn/out_{stamp}"
exkaldi.utils.make_dependent_dirs(outDir, pathIsFile=False)
#------------------------ Training and Validation -----------------------------
dims = exkaldi.load_list_table(f"{args.expDir}/train_dnn/data/dims")
featDim = int(dims["fmllr"])
pdfDim = int(dims["pdfs"])
phoneDim = int(dims["phones"])
# Initialize model
if args.delta > 0:
featDim *= (args.delta + 1)
if args.splice > 0:
featDim *= (2 * args.splice + 1)
if len(args.predictModel.strip()) == 0:
print('Prepare Data Iterator...')
# Prepare fMLLR feature files
trainDataset = process_feat_ali(training=True)
traindataLen = len(trainDataset)
train_gen = tf.data.Dataset.from_generator(
lambda: make_generator(trainDataset), (tf.float32, {
"pdfID": tf.int32,
"phoneID": tf.int32
})).batch(args.batchSize).prefetch(3)
steps_per_epoch = traindataLen // args.batchSize
devDataset = process_feat_ali(training=False)
devdataLen = len(devDataset)
dev_gen = tf.data.Dataset.from_generator(
lambda: make_generator(devDataset), (tf.float32, {
"pdfID": tf.int32,
"phoneID": tf.int32
})).batch(args.batchSize).prefetch(3)
validation_steps = devdataLen // args.batchSize
print('Prepare test data')
testFeat, testBias, testTrans = prepare_test_data(postProbDim=pdfDim)
def train_step():
model = make_DNN_model(featDim, pdfDim, phoneDim)
model.summary()
model.compile(
loss={
"pdfID":
keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
"phoneID":
keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
},
loss_weights={
"pdfID": 1,
"phoneID": 1
},
metrics={
"pdfID": keras.metrics.SparseCategoricalAccuracy(),
"phoneID": keras.metrics.SparseCategoricalAccuracy(),
},
optimizer=keras.optimizers.SGD(0.08, momentum=0.0),
)
def lrScheduler(epoch):
if epoch > 25:
return 0.001
elif epoch > 22:
return 0.0025
elif epoch > 19:
return 0.005
elif epoch > 17:
return 0.01
elif epoch > 15:
return 0.02
elif epoch > 10:
return 0.04
else:
return 0.08
model.fit(
x=train_gen,
steps_per_epoch=steps_per_epoch,
epochs=args.epoch,
validation_data=dev_gen,
validation_steps=validation_steps,
verbose=1,
initial_epoch=0,
callbacks=[
keras.callbacks.EarlyStopping(patience=5, verbose=1),
keras.callbacks.TensorBoard(log_dir=outDir),
keras.callbacks.LearningRateScheduler(lrScheduler),
EvaluateWER(model, testFeat, testBias, testTrans, outDir),
ModelSaver(model, outDir),
],
)
print("Using GPU: ", args.gpu)
if args.gpu != "all":
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
train_step()
else:
my_strategy = tf.distribute.MirroredStrategy()
with my_strategy.scope():
train_step()
else:
declare.is_file(args.predictModel)
model = make_DNN_model(featDim, pdfDim, phoneDim)
model.summary()
model.load_weights(args.predictModel)
print('Prepare test data')
testFeat, testBias, testTrans = prepare_test_data(postProbDim=pdfDim)
scorer = EvaluateWER(model, testFeat, testBias, testTrans, outDir)
logs = {}
scorer.on_epoch_end(5, logs)
def main():
# ------------- Parse arguments from command line ----------------------
# 1. Add a discription of this program
args.describe(
"This program is used to compute MFCC feature and CMVN statistics")
# 2. Add options
args.add("--expDir",
abbr="-e",
dtype=str,
default="exp",
discription="The data and output path of current experiment.")
args.add("--useEnergy",
abbr="-u",
dtype=bool,
default=False,
discription="Whether add energy to MFCC feature.")
args.add(
"--parallel",
abbr="-p",
dtype=int,
default=4,
minV=1,
maxV=10,
discription=
"The number of parallel process to compute train feature of train dataset."
)
# 3. Then start to parse arguments.
args.parse()
# 4. Take a backup of arguments
args.print_args() # print arguments to display
argsLogFile = os.path.join(args.expDir, "conf", "compute_mfcc.args")
args.save(argsLogFile)
# ---------- Compute mfcc feature of train, dev and test dataset -----------
if args.useEnergy:
mfccConfig = {"--use-energy": "true"}
else:
mfccConfig = {"--use-energy": "false"}
for Name in ["train", "dev", "test"]:
print(f"Compute {Name} MFCC feature.")
# 1. compute feature
if Name == "train" and args.parallel > 1: # use mutiple processes
wavFiles = exkaldi.utils.split_txt_file(
os.path.join(args.expDir, "data", "train", "wav.scp"),
chunks=args.parallel,
)
feats = exkaldi.compute_mfcc(wavFiles,
config=mfccConfig,
outFile=os.path.join(
args.expDir, "mfcc", "train",
"raw_mfcc.ark"))
feat = exkaldi.merge_archives(feats)
else:
feat = exkaldi.compute_mfcc(
os.path.join(args.expDir, "data", Name, "wav.scp"),
config=mfccConfig,
)
feat.save(os.path.join(args.expDir, "mfcc", Name, "raw_mfcc.ark"))
print(f"Generate raw MFCC feature done.")
# Compute CMVN
cmvn = exkaldi.compute_cmvn_stats(
feat=feat,
spk2utt=os.path.join(args.expDir, "data", Name, "spk2utt"),
)
cmvn.save(os.path.join(args.expDir, "mfcc", Name, "cmvn.ark"))
print(f"Generate CMVN statistics done.")
# Apply CMVN
feat = exkaldi.use_cmvn(
feat=feat,
cmvn=cmvn,
utt2spk=os.path.join(args.expDir, "data", Name, "utt2spk"),
)
feat.save(os.path.join(args.expDir, "mfcc", Name, "mfcc_cmvn.ark"))
print(f"Generate MFCC feature (applied CMVN) done.")
print("Compute MFCC done.")
def main():
# ------------- Parse arguments from command line ----------------------
# 1. Add a discription of this program
args.discribe("This program is used to train triphone LSTM scoustic model with Tensorflow")
# 2. Add options
args.add("--expDir", abbr="-e", dtype=str, default="exp", discription="The data and output path of current experiment.")
args.add("--LDAsplice", dtype=int, default=3, discription="Splice how many frames to head and tail for LDA feature.")
args.add("--randomSeed", dtype=int, default=1234, discription="Random seed.")
args.add("--batchSize", abbr="-b", dtype=int, default=8, discription="Mini batch size.")
args.add("--gpu", abbr="-g", dtype=str, default="all", choices=["all","0","1"], discription="Use GPU.")
args.add("--epoch", dtype=int, default=30, discription="Epoches.")
args.add("--testStartEpoch", dtype=int, default=5, discription="Start to evaluate test dataset.")
args.add("--dropout", abbr="-d", dtype=float, default=0.2, discription="Dropout.")
args.add("--useCMVN", dtype=bool, default=False, discription="Wether apply CMVN to fmllr feature.")
args.add("--splice", dtype=int, default=0, discription="Splice how many frames to head and tail for Fmllr feature.")
args.add("--delta", dtype=int, default=2, discription="Wether add delta to fmllr feature.")
args.add("--normalizeFeat", dtype=bool, default=True, discription="Wether normalize the chunk dataset.")
args.add("--normalizeAMP", dtype=bool, default=False, discription="Wether normalize the post-probability.")
args.add("--order", abbr="-o", dtype=int, default=6, discription="Language model order.")
args.add("--beam", dtype=int, default=13, discription="Decode beam size.")
args.add("--latBeam", dtype=int, default=6, discription="Lattice beam size.")
args.add("--acwt", dtype=float, default=0.083333, discription="Acoustic model weight.")
args.add("--predictModel", abbr="-m", dtype=str, default="", discription="If not void, skip training. Do decoding only.")
# 3. Then start to parse arguments.
args.parse()
# 4. Take a backup of arguments
args.save( f"./{args.expDir}/conf/train_lstm.args" )
random.seed(args.randomSeed)
np.random.seed(args.randomSeed)
tf.random.set_seed(args.randomSeed)
# ------------- Prepare data for dnn training ----------------------
if not os.path.isfile(f"./{args.expDir}/train_lstm/data/dims"):
prepare_LSTM_data()
# ------------- Prepare data for lstm training ----------------------
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
outDir = f"{args.expDir}/train_lstm/out_{stamp}"
exkaldi.utils.make_dependent_dirs(outDir, pathIsFile=False)
#------------------------ Training and Validation -----------------------------
dims = exkaldi.load_list_table( f"{args.expDir}/train_lstm/data/dims" )
featDim = int(dims["fmllr"])
pdfDim = int(dims["pdfs"])
phoneDim = int(dims["phones"])
# Initialize model
if args.delta > 0:
featDim *= (args.delta+1)
if args.splice > 0:
featDim *= (2*args.splice+1)
if len(args.predictModel.strip()) == 0:
print('Prepare Data Iterator...')
# Prepare fMLLR feature files
trainIterator = DataIterator(batchSize=args.batchSize, training=True)
devIterator = DataIterator(batchSize=args.batchSize, training=False)
print('Prepare test data')
testFeat, testBias, testTrans = prepare_test_data(postProbDim=pdfDim)
metris = {
"train_loss":keras.metrics.Mean(name="train/loss", dtype=tf.float32),
"train_pdfID_accuracy":keras.metrics.Mean(name="train/pdfID_accuracy", dtype=tf.float32),
"train_phoneID_accuracy":keras.metrics.Mean(name="train/phoneID_accuracy", dtype=tf.float32),
"dev_loss":keras.metrics.Mean(name="eval/loss", dtype=tf.float32),
"dev_pdfID_accuracy":keras.metrics.Mean(name="eval/pdfID_accuracy", dtype=tf.float32),
"dev_phoneID_accuracy":keras.metrics.Mean(name="eval/phoneID_accuracy", dtype=tf.float32),
}
def train_step(model,optimizer,batch):
feat, pdfAli, phoneAli = batch
with tf.GradientTape() as tape:
pdfPred, phonePred = model(feat, training=True)
L1 = keras.losses.sparse_categorical_crossentropy(pdfAli, pdfPred, from_logits=True)
L2 = keras.losses.sparse_categorical_crossentropy(phoneAli, phonePred, from_logits=True)
loss = L1 + L2
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
metris["train_loss"](loss)
#pdfPred = tf.convert_to_tensor(pdfPred, np.float32)
A1 = keras.metrics.sparse_categorical_accuracy(pdfAli, pdfPred)
metris["train_pdfID_accuracy"](A1)
#phonePred = tf.convert_to_tensor(phonePred, np.float32)
A2 = keras.metrics.sparse_categorical_accuracy(phoneAli, phonePred)
metris["train_phoneID_accuracy"](A2)
return float(np.mean(L1.numpy())), float(np.mean(L2.numpy())),float(np.mean(A1.numpy())),float(np.mean(A2.numpy()))
def dev_step(model,batch):
feat, pdfAli, phoneAli = batch
pdfPred, phonePred = model(feat, training=False)
L1 = keras.losses.sparse_categorical_crossentropy(pdfAli, pdfPred, from_logits=True)
L2 = keras.losses.sparse_categorical_crossentropy(phoneAli, phonePred, from_logits=True)
loss = L1 + L2
metris["dev_loss"](loss)
#pdfPred = tf.convert_to_tensor(pdfPred, np.float32)
A1 = keras.metrics.sparse_categorical_accuracy(pdfAli, pdfPred)
metris["dev_pdfID_accuracy"](A1)
#phonePred = tf.convert_to_tensor(phonePred, np.float32)
A2 = keras.metrics.sparse_categorical_accuracy(phoneAli, phonePred)
metris["dev_phoneID_accuracy"](A2)
return float(np.mean(L1.numpy())), float(np.mean(L2.numpy())),float(np.mean(A1.numpy())),float(np.mean(A2.numpy()))
def main_loop():
model = make_LSTM_model(args, featDim, pdfDim, phoneDim)
model.summary()
optimizer = keras.optimizers.RMSprop(learning_rate=0.0004, rho=0.95, momentum=0.0, epsilon=1e-07)
scorer = EvaluateWER( model, testFeat, testBias, testTrans, outDir)
modelSaver = ModelSaver(model, outDir)
for e in range(args.epoch):
# Training
startTime = time.time()
for i in range(trainIterator.epochSize):
batch = trainIterator.next()
pdfLoss, phoneLoss, pdfAcc, phoneAcc = train_step(model, optimizer, batch)
tf.print(f"\rtraining: {i}/{trainIterator.epochSize} pdfID loss {pdfLoss:.3f} phoneID loss {phoneLoss:.3f} pdfID accuracy {pdfAcc:.3f} phoneID accuracy {phoneAcc:.3f}", end="\t")
tf.print()
# Evaluate
for i in range(devIterator.epochSize):
batch = devIterator.next()
pdfLoss, phoneLoss, pdfAcc, phoneAcc = dev_step(model, batch)
tf.print(f"\revaluate: {i}/{devIterator.epochSize} pdfID loss {pdfLoss:.3f} phoneID loss {phoneLoss:.3f} pdfID accuracy {pdfAcc:.3f} phoneID accuracy {phoneAcc:.3f}", end="\t")
tf.print()
# Test
tf.print("testing:", end=" ")
testWER = scorer.test(e)
tf.print()
endTime = time.time()
message = f"epoch {e} "
for Name in metris.keys():
message += f"{Name} {float(metris[Name].result().numpy()):.3f} "
metris[Name].reset_states()
message += f"test PER {testWER:.2f} time cost {int(endTime-startTime)}s"
tf.print(message)
modelSaver.save(e)
main_loop()
else:
declare.is_file(args.predictModel)
model = make_LSTM_model(featDim,pdfDim,phoneDim)
model.summary()
model.load_weights(args.predictModel)
print('Prepare test data')
testFeat, testBias, testTrans = prepare_test_data(postProbDim=pdfDim)
scorer = EvaluateWER(model,testFeat,testBias,testTrans,outDir)
scorer.test(0)
def main():
# ------------- Parse arguments from command line ----------------------
# 1. Add a discription of this program
args.discribe("This program is used to train triphone GMM-HMM model")
# 2. Add options
args.add("--expDir",
abbr="-e",
dtype=str,
default="exp",
discription="The data and output path of current experiment.")
args.add("--splice",
abbr="-c",
dtype=int,
default=3,
discription="How many left-right frames to splice.")
args.add("--numIters",
abbr="-n",
dtype=int,
default=35,
discription="How many iterations to train.")
args.add("--maxIterInc",
abbr="-m",
dtype=int,
default=25,
discription="The final iteration of increasing gaussians.")
args.add("--realignIter",
abbr="-r",
dtype=int,
default=[10, 20, 30],
discription="The iteration to realign feature.")
args.add("--fmllrIter",
abbr="-f",
dtype=int,
default=[2, 4, 6, 12],
discription="The iteration to estimate fmllr matrix.")
args.add("--order",
abbr="-o",
dtype=int,
default=6,
discription="Which N-grams model to use.")
args.add("--beam",
abbr="-b",
dtype=int,
default=13,
discription="Decode beam size.")
args.add("--latBeam",
abbr="-l",
dtype=int,
default=6,
discription="Lattice beam size.")
args.add("--acwt",
abbr="-a",
dtype=float,
default=0.083333,
discription="Acoustic model weight.")
args.add(
"--parallel",
abbr="-p",
dtype=int,
default=4,
minV=1,
maxV=10,
discription=
"The number of parallel process to compute feature of train dataset.")
args.add("--skipTrain",
abbr="-s",
dtype=bool,
default=False,
discription="If True, skip training. Do decoding only.")
# 3. Then start to parse arguments.
args.parse()
# 4. Take a backup of arguments
argsLogFile = os.path.join(args.expDir, "conf", "train_sat.args")
args.save(argsLogFile)
if not args.skipTrain:
# ------------- Prepare feature and previous alignment for training ----------------------
# 1. Load the feature for training
print(f"Load MFCC+CMVN feature.")
feat = exkaldi.load_index_table(
os.path.join(args.expDir, "mfcc", "train", "mfcc_cmvn.ark"))
print(f"Splice {args.splice} frames.")
originalFeat = exkaldi.splice_feature(feat,
left=args.splice,
right=args.splice,
outFile=os.path.join(
args.expDir, "train_delta",
"mfcc_cmvn_splice.ark"))
print(f"Transform LDA feature")
ldaFeat = exkaldi.transform_feat(
feat=originalFeat,
matFile=os.path.join(args.expDir, "train_lda_mllt", "trans.mat"),
outFile=os.path.join(args.expDir, "train_sat", "lda_feat.ark"),
)
del originalFeat
# 2. Load previous alignment and lexicons
ali = exkaldi.load_index_table(os.path.join(args.expDir,
"train_lda_mllt",
"*final.ali"),
useSuffix="ark")
lexicons = exkaldi.load_lex(
os.path.join(args.expDir, "dict", "lexicons.lex"))
# 3. Estimate the primary fMLLR transform matrix
print("Estiminate the primary fMLLR transform matrixs")
fmllrTransMat = exkaldi.hmm.estimate_fMLLR_matrix(
aliOrLat=ali,
lexicons=lexicons,
aliHmm=os.path.join(args.expDir, "train_lda_mllt", "final.mdl"),
feat=ldaFeat,
spk2utt=os.path.join(args.expDir, "data", "train", "spk2utt"),
outFile=os.path.join(args.expDir, "train_sat", "trans.ark"),
)
print("Transform feature")
fmllrFeat = exkaldi.use_fmllr(
ldaFeat,
fmllrTransMat,
utt2spk=os.path.join("exp", "data", "train", "utt2spk"),
outFile=os.path.join(args.expDir, "train_sat", "fmllr_feat.ark"),
)
# -------------- Build the decision tree ------------------------
print("Start build a tree")
tree = exkaldi.hmm.DecisionTree(lexicons=lexicons,
contextWidth=3,
centralPosition=1)
tree.train(
feat=fmllrFeat,
hmm=os.path.join(args.expDir, "train_lda_mllt", "final.mdl"),
ali=ali,
topoFile=os.path.join(args.expDir, "dict", "topo"),
numLeaves=2500,
tempDir=os.path.join(args.expDir, "train_sat"),
)
tree.save(os.path.join(args.expDir, "train_sat", "tree"))
print(f"Build tree done.")
del fmllrFeat
# ------------- Start training ----------------------
# 1. Initialize a monophone HMM object
print("Initialize a triphone HMM object")
model = exkaldi.hmm.TriphoneHMM(lexicons=lexicons)
model.initialize(
tree=tree,
topoFile=os.path.join(args.expDir, "dict", "topo"),
treeStatsFile=os.path.join(args.expDir, "train_sat",
"treeStats.acc"),
)
print(f"Initialized a monophone HMM-GMM model: {model.info}.")
# 2. convert the previous alignment
print(f"Transform the alignment")
newAli = exkaldi.hmm.convert_alignment(
ali=ali,
originHmm=os.path.join(args.expDir, "train_lda_mllt", "final.mdl"),
targetHmm=model,
tree=tree,
outFile=os.path.join(args.expDir, "train_sat", "initial.ali"),
)
# 2. Split data for parallel training
transcription = exkaldi.load_transcription(
os.path.join(args.expDir, "data", "train", "text"))
transcription = transcription.sort()
if args.parallel > 1:
# split feature
ldaFeat = ldaFeat.sort(by="utt").subset(chunks=args.parallel)
# split transcription depending on utterance IDs of each feat
tempTrans = []
tempAli = []
tempFmllrMat = []
for f in ldaFeat:
tempTrans.append(transcription.subset(keys=f.utts))
tempAli.append(newAli.subset(keys=f.utts))
spks = exkaldi.utt_to_spk(f.utts,
utt2spk=os.path.join(
args.expDir, "data", "train",
"utt2spk"))
tempFmllrMat.append(fmllrTransMat.subset(keys=spks))
transcription = tempTrans
newAli = tempAli
fmllrTransMat = tempFmllrMat
# 3. Train
print("Train the triphone model")
model.train(
ldaFeat,
transcription,
os.path.join(args.expDir, "dict", "L.fst"),
tree,
tempDir=os.path.join(args.expDir, "train_sat"),
initialAli=newAli,
fmllrTransMat=fmllrTransMat,
spk2utt=os.path.join(args.expDir, "data", "train", "spk2utt"),
utt2spk=os.path.join(args.expDir, "data", "train", "utt2spk"),
numIters=args.numIters,
maxIterInc=args.maxIterInc,
totgauss=15000,
realignIter=args.realignIter,
fmllrIter=args.fmllrIter,
boostSilence=1.0,
power=0.2,
fmllrSilWt=0.0,
)
print(model.info)
del ldaFeat
del fmllrTransMat
del newAli
else:
declare.is_file(os.path.join(args.expDir, "train_sat", "final.mdl"))
declare.is_file(os.path.join(args.expDir, "train_sat", "tree"))
model = exkaldi.load_hmm(
os.path.join(args.expDir, "train_sat", "final.mdl"))
tree = exkaldi.load_tree(os.path.join(args.expDir, "train_sat",
"tree"))
# ------------- Compile WFST training ----------------------
# Make a WFST decoding graph
make_WFST_graph(
outDir=os.path.join(args.expDir, "train_sat", "graph"),
hmm=model,
tree=tree,
)
# Decode test data
GMM_decode_fmllr_and_score(
outDir=os.path.join(args.expDir, "train_sat",
f"decode_{args.order}grams"),
hmm=model,
HCLGfile=os.path.join(args.expDir, "train_sat", "graph",
f"HCLG.{args.order}.fst"),
tansformMatFile=os.path.join(args.expDir, "train_lda_mllt",
"trans.mat"),
)
def train():
# ------------- Prepare data for dnn training ----------------------
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
outDir = f"dnn_exp/out_{stamp}"
exkaldi.utils.make_dependent_dirs(outDir, pathIsFile=False)
args.save( os.path.join(outDir,"conf") )
#------------------------ Training and Validation dataset-----------------------------
hmm = exkaldi.load_hmm(f"{args.root}/exp/tri3b_ali_train_clean_5/final.mdl")
pdfDim = hmm.info.pdfs
del hmm
print('Prepare Data Iterator...')
# Prepare fMLLR feature files
featDim, trainDataset = prepare_data()
traindataLen = len(trainDataset)
train_gen = tf.data.Dataset.from_generator(
lambda: make_generator(trainDataset),
(tf.float32, tf.int32),
(tf.TensorShape([featDim,]), tf.TensorShape([])),
).batch(args.batchSize).prefetch(3)
steps_per_epoch = traindataLen//args.batchSize
featDim, devDataset = prepare_data(training=False)
devdataLen = len(devDataset)
dev_gen = tf.data.Dataset.from_generator(
lambda: make_generator(devDataset),
(tf.float32, tf.int32),
(tf.TensorShape([featDim,]), tf.TensorShape([])),
).batch(args.batchSize).prefetch(3)
validation_steps = devdataLen//args.batchSize
#------------------------ Train Step -----------------------------
model = make_DNN_acoustic_model(featDim,pdfDim)
#model.summary()
model.compile(
loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics = keras.metrics.SparseCategoricalAccuracy(),
optimizer = keras.optimizers.SGD(0.08,momentum=0.0),
)
def lrScheduler(epoch):
if epoch > 25:
return 0.001
elif epoch > 22:
return 0.0025
elif epoch > 19:
return 0.005
elif epoch > 17:
return 0.01
elif epoch > 15:
return 0.02
elif epoch > 10:
return 0.04
else:
return 0.08
model.fit(
x = train_gen,
steps_per_epoch=steps_per_epoch,
epochs=args.epoch,
validation_data=dev_gen,
validation_steps=validation_steps,
verbose=1,
initial_epoch=0,
callbacks=[
keras.callbacks.EarlyStopping(patience=5, verbose=1),
keras.callbacks.TensorBoard(log_dir=outDir),
keras.callbacks.LearningRateScheduler(lrScheduler),
ModelSaver(model,outDir),
],
)
def main():
# ------------- Parse arguments from command line ----------------------
# 1. Add a discription of this program
args.describe("This program is used to train monophone GMM-HMM model")
# 2. Add options
args.add("--expDir",
abbr="-e",
dtype=str,
default="exp",
discription="The data and output path of current experiment.")
args.add("--delta",
abbr="-d",
dtype=int,
default=2,
discription="Add n-order to feature.")
args.add("--numIters",
abbr="-n",
dtype=int,
default=40,
discription="How many iterations to train.")
args.add("--maxIterInc",
abbr="-m",
dtype=int,
default=30,
discription="The final iteration of increasing gaussians.")
args.add("--realignIter",
abbr="-r",
dtype=int,
default=[
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 23, 26, 29,
32, 35, 38
],
discription="the iteration to realign feature.")
args.add("--order",
abbr="-o",
dtype=int,
default=6,
minV=1,
maxV=6,
discription="Which N-grams model to use.")
args.add("--beam",
abbr="-b",
dtype=int,
default=13,
discription="Decode beam size.")
args.add("--latBeam",
abbr="-l",
dtype=int,
default=6,
discription="Lattice beam size.")
args.add("--acwt",
abbr="-a",
dtype=float,
default=0.083333,
discription="Acoustic model weight.")
args.add(
"--parallel",
abbr="-p",
dtype=int,
default=4,
minV=1,
maxV=10,
discription=
"The number of parallel process to compute feature of train dataset.")
args.add("--skipTrain",
abbr="-s",
dtype=bool,
default=False,
discription="If True, skip training. Do decoding only.")
# 3. Then start to parse arguments.
args.parse()
# 4. Take a backup of arguments
args.print_args() # print arguments to display
argsLogFile = os.path.join(args.expDir, "conf", "train_mono.args")
args.save(argsLogFile)
if not args.skipTrain:
# ------------- Prepare feature for training ----------------------
# 1. Load the feature for training (We use the index table format)
feat = exkaldi.load_index_table(
os.path.join(args.expDir, "mfcc", "train", "mfcc_cmvn.ark"))
print(f"Load MFCC+CMVN feature.")
feat = exkaldi.add_delta(feat,
order=args.delta,
outFile=os.path.join(args.expDir,
"train_mono",
"mfcc_cmvn_delta.ark"))
print(f"Add {args.delta}-order deltas.")
# 2. Load lexicon bank
lexicons = exkaldi.load_lex(
os.path.join(args.expDir, "dict", "lexicons.lex"))
print(f"Restorage lexicon bank.")
# ------------- Start training ----------------------
# 1. Initialize a monophone HMM object
model = exkaldi.hmm.MonophoneHMM(lexicons=lexicons, name="mono")
model.initialize(feat=feat,
topoFile=os.path.join(args.expDir, "dict", "topo"))
print(f"Initialized a monophone HMM-GMM model: {model.info}.")
# 2. Split data for parallel training
transcription = exkaldi.load_transcription(
os.path.join(args.expDir, "data", "train", "text"))
transcription = transcription.sort()
if args.parallel > 1:
# split feature
feat = feat.sort(by="utt").subset(chunks=args.parallel)
# split transcription depending on utterance IDs of each feature
temp = []
for f in feat:
temp.append(transcription.subset(keys=f.utts))
transcription = temp
# 3. Train
model.train(
feat,
transcription,
LFile=os.path.join(args.expDir, "dict", "L.fst"),
tempDir=os.path.join(args.expDir, "train_mono"),
numIters=args.numIters,
maxIterInc=args.maxIterInc,
totgauss=1000,
realignIter=args.realignIter,
boostSilence=1.0,
)
print(model.info)
# Save the tree
model.tree.save(os.path.join(args.expDir, "train_mono", "tree"))
print(f"Tree has been saved.")
# 4. Realign with boostSilence 1.25
print("Realign the training feature (boost silence = 1.25)")
trainGraphFiles = exkaldi.utils.list_files(
os.path.join(args.expDir, "train_mono", "*train_graph"))
model.align(
feat,
trainGraphFile=
trainGraphFiles, # train graphs have been generated in the train step.
boostSilence=1.25, #1.5
outFile=os.path.join(args.expDir, "train_mono", "final.ali"))
del feat
print("Save the new alignment done.")
tree = model.tree
else:
declare.is_file(os.path.join(args.expDir, "train_mono", "final.mdl"))
declare.is_file(os.path.join(args.expDir, "train_mono", "tree"))
model = exkaldi.load_hmm(
os.path.join(args.expDir, "train_mono", "final.mdl"))
tree = exkaldi.load_tree(
os.path.join(args.expDir, "train_mono", "tree"))
# ------------- Compile WFST training ----------------------
# Make a WFST decoding graph
make_WFST_graph(
outDir=os.path.join(args.expDir, "train_mono", "graph"),
hmm=model,
tree=tree,
)
# Decode test data
GMM_decode_mfcc_and_score(
outDir=os.path.join(args.expDir, "train_mono",
f"decode_{args.order}grams"),
hmm=model,
HCLGfile=os.path.join(args.expDir, "train_mono", "graph",
f"HCLG.{args.order}.fst"),
)
def main():
# ------------- Parse arguments from command line ----------------------
# 1. Add a discription of this program
args.discribe("This program is used to train triphone GMM-HMM model")
# 2. Add options
args.add("--expDir",
abbr="-e",
dtype=str,
default="exp",
discription="The data and output path of current experiment.")
args.add("--delta",
abbr="-d",
dtype=int,
default=2,
discription="Add n-order to feature.")
args.add("--numIters",
abbr="-n",
dtype=int,
default=35,
discription="How many iterations to train.")
args.add("--maxIterInc",
abbr="-m",
dtype=int,
default=25,
discription="The final iteration of increasing gaussians.")
args.add("--realignIter",
abbr="-r",
dtype=int,
default=[10, 20, 30],
discription="the iteration to realign feature.")
args.add("--order",
abbr="-o",
dtype=int,
default=6,
discription="Which N-grams model to use.")
args.add("--beam",
abbr="-b",
dtype=int,
default=13,
discription="Decode beam size.")
args.add("--latBeam",
abbr="-l",
dtype=int,
default=6,
discription="Lattice beam size.")
args.add("--acwt",
abbr="-a",
dtype=float,
default=0.083333,
discription="Acoustic model weight.")
args.add(
"--parallel",
abbr="-p",
dtype=int,
default=4,
minV=1,
maxV=10,
discription=
"The number of parallel process to compute feature of train dataset.")
args.add("--skipTrain",
abbr="-s",
dtype=bool,
default=False,
discription="If True, skip training. Do decoding only.")
# 3. Then start to parse arguments.
args.parse()
# 4. Take a backup of arguments
argsLogFile = os.path.join(args.expDir, "conf", "train_delta.args")
args.save(argsLogFile)
if not args.skipTrain:
# ------------- Prepare feature and previous alignment for training ----------------------
# 1. Load the feature for training
feat = exkaldi.load_index_table(
os.path.join(args.expDir, "mfcc", "train", "mfcc_cmvn.ark"))
print(f"Load MFCC+CMVN feature.")
feat = exkaldi.add_delta(feat,
order=args.delta,
outFile=os.path.join(args.expDir,
"train_delta",
"mfcc_cmvn_delta.ark"))
print(f"Add {args.delta}-order deltas.")
# 2. Load lexicon bank
lexicons = exkaldi.load_lex(
os.path.join(args.expDir, "dict", "lexicons.lex"))
print(f"Restorage lexicon bank.")
# 3. Load previous alignment
ali = exkaldi.load_index_table(os.path.join(args.expDir, "train_mono",
"*final.ali"),
useSuffix="ark")
# -------------- Build the decision tree ------------------------
print("Start build a tree")
tree = exkaldi.hmm.DecisionTree(lexicons=lexicons,
contextWidth=3,
centralPosition=1)
tree.train(
feat=feat,
hmm=os.path.join(args.expDir, "train_mono", "final.mdl"),
ali=ali,
topoFile=os.path.join(args.expDir, "dict", "topo"),
numLeaves=2500,
tempDir=os.path.join(args.expDir, "train_delta"),
)
print(f"Build tree done.")
# ------------- Start training ----------------------
# 1. Initialize a monophone HMM object
model = exkaldi.hmm.TriphoneHMM(lexicons=lexicons, name="mono")
model.initialize(
tree=tree,
topoFile=os.path.join(args.expDir, "dict", "topo"),
treeStatsFile=os.path.join(args.expDir, "train_delta",
"treeStats.acc"),
)
print(f"Initialized a monophone HMM-GMM model: {model.info}.")
# 2. convert the previous alignment
print(f"Transform the alignment")
newAli = exkaldi.hmm.convert_alignment(
ali=ali,
originHmm=os.path.join("exp", "train_mono", "final.mdl"),
targetHmm=model,
tree=tree,
outFile=os.path.join(args.expDir, "train_delta", "initial.ali"),
)
# 2. Split data for parallel training
transcription = exkaldi.load_transcription(
os.path.join(args.expDir, "data", "train", "text"))
transcription = transcription.sort()
if args.parallel > 1:
# split feature
feat = feat.sort(by="utt").subset(chunks=args.parallel)
# split transcription depending on utterance IDs of each feat
tempTrans = []
tempAli = []
for f in feat:
tempTrans.append(transcription.subset(keys=f.utts))
tempAli.append(newAli.subset(keys=f.utts))
transcription = tempTrans
newAli = tempAli
# 3. Train
print("Train the triphone model")
model.train(
feat,
transcription,
os.path.join("exp", "dict", "L.fst"),
tree,
tempDir=os.path.join(args.expDir, "train_delta"),
initialAli=newAli,
numIters=args.numIters,
maxIterInc=args.maxIterInc,
totgauss=15000,
realignIter=args.realignIter,
boostSilence=1.0,
)
print(model.info)
# Save the tree
model.tree.save(os.path.join(args.expDir, "train_delta", "tree"))
print(f"Tree has been saved.")
del feat
else:
declare.is_file(os.path.join(args.expDir, "train_delta", "final.mdl"))
declare.is_file(os.path.join(args.expDir, "train_delta", "tree"))
model = exkaldi.load_hmm(
os.path.join(args.expDir, "train_delta", "final.mdl"))
tree = exkaldi.load_tree(
os.path.join(args.expDir, "train_delta", "tree"))
# ------------- Compile WFST training ----------------------
# Make a WFST decoding graph
make_WFST_graph(
outDir=os.path.join(args.expDir, "train_delta", "graph"),
hmm=model,
tree=tree,
)
# Decode test data
GMM_decode_mfcc_and_score(
outDir=os.path.join(args.expDir, "train_delta",
f"decode_{args.order}grams"),
hmm=model,
HCLGfile=os.path.join(args.expDir, "train_delta", "graph",
f"HCLG.{args.order}.fst"),
)