def loglikelihood(data, model):
lpr = log_multivariate_normal_density_diag(data['lmfcc'], model['means'],
model['covars'])
log_st_prob = np.log(model['startprob'])
log_transmat = np.log(model['transmat'])
alpha_matrix = forward(lpr, log_st_prob, log_transmat)
return lpr, logsumexp(alpha_matrix[-1])
def gaussian_emission_prob():
example_x = example['lmfcc']
wordHMMs = {}
isolated = get_isolated(prondict)
wordHMMs['o'] = concatHMMs(phoneHMMsAll, isolated['o'])
lpr = log_multivariate_normal_density_diag(example_x,
wordHMMs['o']['means'],
wordHMMs['o']['covars'])
test_o = data[0]
test_o_lmfcc = test_o['lmfcc']
lpr_test = log_multivariate_normal_density_diag(test_o_lmfcc,
wordHMMs['o']['means'],
wordHMMs['o']['covars'])
plot_p_color_mesh(lpr, 'example log likelihood')
plot_p_color_mesh(lpr_test, 'test log likelihood')
def forcedAlignment(lmfcc, phoneHMMs, phoneTrans):
""" forcedAlignmen: aligns a phonetic transcription at the state level
Args:
lmfcc: NxD array of MFCC feature vectors (N vectors of dimension D)
computed the same way as for the training of phoneHMMs
phoneHMMs: set of phonetic Gaussian HMM models
phoneTrans: list of phonetic symbols to be aligned including initial and
final silence
Returns:
list of strings in the form phoneme_index specifying, for each time step
the state from phoneHMMs corresponding to the viterbi path.
"""
# phone transcription => state transcription
phones = sorted(phoneHMMs.keys())
nstates = {phone: phoneHMMs[phone]['means'].shape[0] for phone in phones}
stateTrans = [p + '_' + str(i) for p in phoneTrans for i in range(nstates[p])]
# combined HMM for utterance
utteranceHMM = concatHMMs(phoneHMMs, phoneTrans)
# Viterbi decoder
obsloglik = log_multivariate_normal_density_diag(lmfcc, utteranceHMM['means'], utteranceHMM['covars'])
viterbiPath = viterbi(obsloglik, np.log(utteranceHMM['startprob']), np.log(utteranceHMM['transmat']))[1]
# time alignment (frame-by-frame state transcription)
viterbiStateTrans = [stateTrans[s] for s in viterbiPath]
return viterbiStateTrans
def forward_algorithm():
wordHMMs = {}
isolated = get_isolated(prondict)
plot_p_color_mesh(example['logalpha'], 'example alpha matrix')
# verify implementation
wordHMMs['o'] = concatHMMs(phoneHMMsAll, isolated['o'])
log_st_prob = np.log(wordHMMs['o']['startprob'])
log_transmat = np.log(wordHMMs['o']['transmat'])
alpha_matrix = forward(example['obsloglik'], log_st_prob, log_transmat)
plot_p_color_mesh(alpha_matrix, "hmms all output example alpha matrix")
# 44 data labels
keys_list = [x for x in isolated.keys()]
scores_models_all = np.zeros((len(data), len(isolated)))
scores_models_onespkr = np.zeros_like(scores_models_all)
for j in range(len(keys_list)):
key = keys_list[j]
hmms = concatHMMs(phoneHMMsAll, isolated[key])
log_st_prob = np.log(hmms['startprob'])
log_transmat = np.log(hmms['transmat'])
for i in range(len(data)):
lpr_test = log_multivariate_normal_density_diag(
data[i]['lmfcc'], hmms['means'], hmms['covars'])
alpha = forward(lpr_test, log_st_prob, log_transmat)
scores_models_all[i, j] = logsumexp(alpha[len(alpha) - 1])
hmms = concatHMMs(phoneHMMsOne, isolated[key])
log_st_prob = np.log(hmms['startprob'])
log_transmat = np.log(hmms['transmat'])
for i in range(len(data)):
lpr_test = log_multivariate_normal_density_diag(
data[i]['lmfcc'], hmms['means'], hmms['covars'])
alpha = forward(lpr_test, log_st_prob, log_transmat)
scores_models_onespkr[i, j] = logsumexp(alpha[len(alpha) - 1])
predict_all = np.argmax(scores_models_all, axis=1)
predict_one = np.argmax(scores_models_onespkr, axis=1)
label_all = [keys_list[x] for x in predict_all]
label_one = [keys_list[x] for x in predict_one]
true_label = [data[x]['digit'] for x in range(len(data))]
print(true_label)
print(label_all)
print(label_one)
def get_loglik(feature, hmm):
trans_mat = hmm['transmat'][:-1, :-1]
pi_vec = hmm['startprob'][:-1]
means = hmm['means']
covars = hmm['covars']
obsloglik = log_multivariate_normal_density_diag(feature, means, covars)
log_alpha = forward(obsloglik, np.log(pi_vec), np.log(trans_mat))
ret = logsumexp(log_alpha[-1])
return ret
def get_best_path_loglik(feature, hmm):
trans_mat = hmm['transmat'][:-1, :-1]
pi_vec = hmm['startprob'][:-1]
means = hmm['means']
covars = hmm['covars']
obsloglik = log_multivariate_normal_density_diag(feature, means, covars)
ret, _ = viterbi(obsloglik, np.log(pi_vec), np.log(trans_mat))
return ret
def retrain(feature, model, num_iters=20, threshold=1):
# extract params
means = model['means']
covars = model['covars']
transmat = model['transmat'][:-1, :-1]
startprob = model['startprob'][:-1]
log_pi = np.log(startprob)
log_trans = np.log(transmat)
# calculate the emission
obsloglik = log_multivariate_normal_density_diag(feature, means, covars)
# EM algorithm
loglik_old = -np.inf
for iter_ in range(num_iters):
# E-step
log_alpha = forward(obsloglik, log_pi, log_trans)
log_beta = backward(obsloglik, log_pi, log_trans)
log_gamma = statePosteriors(log_alpha, log_beta)
# M-step
means, covars = updateMeanAndVar(feature, log_gamma)
# update
obsloglik = log_multivariate_normal_density_diag(
feature, means, covars)
loglik = logsumexp(log_alpha[-1])
print("Iter {}: The log likelihood in EM:".format(iter_), loglik)
# check if terminate EM
if (loglik - loglik_old) < threshold or np.isnan(loglik):
print("Terminating the EM")
break
else:
loglik_old = loglik
def test_concatHMMs(self):
"""
test_concatHMMs: Required verification in section 5.1
"""
# obtain the concated HMM model
wordHMMs = {}
wordHMMs['o'] = concatHMMs(self.phoneHMMs, isolated['o'])
means = wordHMMs['o']['means']
covars = wordHMMs['o']['covars']
# use example observation to calculate the log likelihood of observation
obsloglik = log_multivariate_normal_density_diag(
self.example['lmfcc'], means, covars)
# check against with the example
assert_allclose(obsloglik, self.example['obsloglik'])
def forcedAlignment(lmfcc, phoneHMMs, phoneTrans):
""" forcedAlignmen: aligns a phonetic transcription at the state level
Args:
lmfcc: NxD array of MFCC feature vectors (N vectors of dimension D)
computed the same way as for the training of phoneHMMs
phoneHMMs: set of phonetic Gaussian HMM models
phoneTrans: list of phonetic symbols to be aligned including initial and
final silence
Returns:
list of strings in the form phoneme_index specifying, for each time step
the state from phoneHMMs corresponding to the viterbi path.
"""
utteranceHMM = concatHMMs(phoneHMMs, phoneTrans)
emmision = log_multivariate_normal_density_diag(lmfcc,
utteranceHMM['means'],
utteranceHMM['covars'])
return viterbi(emmision, np.log(utteranceHMM['startprob']),
np.log(utteranceHMM['transmat']))
def state_posterior_test():
posterior = statePosteriors(example['logalpha'], example['logbeta'])
plot_p_color_mesh(posterior, 'log state hmm posterior')
posterior = np.exp(posterior)
print('verification')
print(np.sum(posterior, axis=1))
print('6.1 summing over time steps')
print(np.sum(posterior, axis=0))
print('6.1 summing over time steps and states')
print(np.sum(posterior))
example_x = example['lmfcc']
wordHMMs = {}
isolated = get_isolated(prondict)
wordHMMs['o'] = concatHMMs(phoneHMMsAll, isolated['o'])
lpr = log_multivariate_normal_density_diag(example_x,
wordHMMs['o']['means'],
wordHMMs['o']['covars'])
plot_p_color_mesh(lpr, 'log state gmm posterior')
# transcription
wordTrans = list(path2info(filename)
[2]) # word transcription (contained in the filename)
phoneTrans = words2phones(
wordTrans, prondict) # word transcription => phone transcription
stateTrans = [
p + '_' + str(i) for p in phoneTrans for i in range(nstates[p])
] # phone transcription => state transcription
# combined HMM for utterance
utteranceHMM = concatHMMs(phoneHMMs, phoneTrans)
# Viterbi decoder
obsloglik = log_multivariate_normal_density_diag(lmfcc,
utteranceHMM['means'],
utteranceHMM['covars'])
viterbiLoglik, viterbiPath = viterbi(obsloglik,
np.log(utteranceHMM['startprob']),
np.log(utteranceHMM['transmat']))
# time alignment (frame-by-frame state transcription)
viterbiStateTrans = [stateTrans[s] for s in viterbiPath]
# save in standard format (to use it, put it in the same directory of .wav and open .wav with wavesurfer)
frames2trans(viterbiStateTrans, outfilename='data/transcriptions/z43a.lab')
# check results
plt.figure()
pcolormesh(lmfcc, 'MFCC - computed', ylabel='MFCC')
plt.figure()
phoneHMMs = np.load('data/lab2_models_onespkr.npz')['phoneHMMs'].item()
# Build hmm
wordHMMs = {}
for d in isolated.keys():
wordHMMs[d] = concatHMMs(phoneHMMs, isolated[d])
# get the observation sequence
feature = data[10]['lmfcc']
# First part
# calculate the emissions
digit = '4'
means = wordHMMs[digit]['means']
covars = wordHMMs[digit]['covars']
obsloglik = log_multivariate_normal_density_diag(feature, means, covars)
# calculate the log likelihood
trans_mat = wordHMMs[digit]['transmat'][:-1, :-1]
pi_vec = wordHMMs[digit]['startprob'][:-1]
# log space
log_pi = np.log(pi_vec)
log_trans = np.log(trans_mat)
log_alpha = forward(obsloglik, log_pi, log_trans)
log_seq_likelihood = logsumexp(log_alpha[-1])
print("The log likelihood of the digit {}:".format(digit),
log_seq_likelihood)
# =========================================================================
for d in wordHMMs.keys():
print("========================================================")
wordHMMs = {}
for digit in isolated.keys():
wordHMMs[digit] = concatHMMs(phoneHMMs_one, isolated[digit])
print(list(wordHMMs['o'].keys()))
print(list(wordHMMs.keys()))
wordHMMs_all = {}
for digit in isolated.keys():
wordHMMs_all[digit] = concatHMMs(phoneHMMs_all, isolated[digit])
print(list(wordHMMs_all.keys()))
#Testing log likelihood function
o_obsloglik = lab2_tools.log_multivariate_normal_density_diag(example['lmfcc'], wordHMMs['o']['means'],
wordHMMs['o']['covars'])
print("Testing if likelihood is correct: ")
np.testing.assert_almost_equal(o_obsloglik, example['obsloglik'], 6)
print("Likelihood is correct.")
# plotting likelihood functions
fig, axs = plt.subplots(2)
axs[0].set_title("Computed \"o\" obsloglik")
axs[0].pcolormesh(o_obsloglik.T)
axs[1].set_title("Example \"o\" obsloglik")
axs[1].pcolormesh(example['obsloglik'].T)
plt.show()
# The dark bars in the middle refer to 'ow', while the higher prob light bars
# refer the 'sil' on either side of the dark.
def viterbi_algorithm():
wordHMMs = {}
isolated = get_isolated(prondict)
# verify implementation
wordHMMs['o'] = concatHMMs(phoneHMMsAll, isolated['o'])
log_st_prob = np.log(wordHMMs['o']['startprob'])
log_transmat = np.log(wordHMMs['o']['transmat'])
vloglik, bestPath = viterbi(example['obsloglik'], log_st_prob,
log_transmat)
alpha_matrix = forward(example['obsloglik'], log_st_prob, log_transmat)
print('vloglik from viterbi():', vloglik)
print('vloglik from example:', example['vloglik'])
# plot
fig = plt.figure(figsize=(12, 6))
ax = plt.subplot(121)
ax.set_title('viterbi path from Viterbi()')
plt.pcolormesh(alpha_matrix)
plt.plot(bestPath, np.arange(len(bestPath)), color='red')
plt.colorbar()
plt.show()
fig = plt.figure(figsize=(12, 6))
ax = plt.subplot(121)
ax.set_title('viterbi path from example')
plt.pcolormesh(example['logalpha'])
plt.plot(example['vpath'], np.arange(len(bestPath)), color='red')
plt.colorbar()
plt.show()
# 44 data labels
keys_list = [x for x in isolated.keys()]
scores_models_all = np.zeros((len(data), len(isolated)))
scores_models_onespkr = np.zeros_like(scores_models_all)
for j in range(len(keys_list)):
key = keys_list[j]
hmms = concatHMMs(phoneHMMsAll, isolated[key])
log_st_prob = np.log(hmms['startprob'])
log_transmat = np.log(hmms['transmat'])
for i in range(len(data)):
lpr_test = log_multivariate_normal_density_diag(
data[i]['lmfcc'], hmms['means'], hmms['covars'])
loglik, path = viterbi(lpr_test, log_st_prob, log_transmat)
scores_models_all[i, j] = loglik
hmms = concatHMMs(phoneHMMsOne, isolated[key])
log_st_prob = np.log(hmms['startprob'])
log_transmat = np.log(hmms['transmat'])
for i in range(len(data)):
lpr_test = log_multivariate_normal_density_diag(
data[i]['lmfcc'], hmms['means'], hmms['covars'])
loglik, path = viterbi(lpr_test, log_st_prob, log_transmat)
scores_models_onespkr[i, j] = loglik
predict_all = np.argmax(scores_models_all, axis=1)
predict_one = np.argmax(scores_models_onespkr, axis=1)
label_all = [keys_list[x] for x in predict_all]
label_one = [keys_list[x] for x in predict_one]
true_label = [data[x]['digit'] for x in range(len(data))]
print(true_label)
print(label_all)
print(label_one)
from prondict import isolated
from lab2_proto import concatHMMs
if __name__ == '__main__':
# load data
data = np.load('data/lab2_data.npz')['data']
example = np.load('data/lab2_example.npz')['example'].item()
phoneHMMs = np.load('data/lab2_models_onespkr.npz')['phoneHMMs'].item()
# Build hmm
wordHMMs = {}
wordHMMs['o'] = concatHMMs(phoneHMMs, isolated['o'])
# caculate the obsloglik
obsloglik = log_multivariate_normal_density_diag(example['lmfcc'],
wordHMMs['o']['means'],
wordHMMs['o']['covars'])
# # Plot for o
# fig, ax = plt.subplots(figsize=(16, 4))
# im = ax.pcolormesh(obsloglik.T)
# fig.colorbar(im, ax=ax)
# fig.tight_layout()
# plt.savefig("./plots/obsloglik_o_woman.png",
# dpi=80, bbox_inches='tight')
# # caculate the obsloglik
# obsloglik = log_multivariate_normal_density_diag(
# data[22]['lmfcc'], wordHMMs['7']['means'], wordHMMs['7']['covars'])
# # Plot for o
# fig, ax = plt.subplots(figsize=(16, 4))
# im = ax.pcolormesh(obsloglik.T)