def loglikelihood(data, model): lpr = log_multivariate_normal_density_diag(data['lmfcc'], model['means'], model['covars']) log_st_prob = np.log(model['startprob']) log_transmat = np.log(model['transmat']) alpha_matrix = forward(lpr, log_st_prob, log_transmat) return lpr, logsumexp(alpha_matrix[-1])
def gaussian_emission_prob(): example_x = example['lmfcc'] wordHMMs = {} isolated = get_isolated(prondict) wordHMMs['o'] = concatHMMs(phoneHMMsAll, isolated['o']) lpr = log_multivariate_normal_density_diag(example_x, wordHMMs['o']['means'], wordHMMs['o']['covars']) test_o = data[0] test_o_lmfcc = test_o['lmfcc'] lpr_test = log_multivariate_normal_density_diag(test_o_lmfcc, wordHMMs['o']['means'], wordHMMs['o']['covars']) plot_p_color_mesh(lpr, 'example log likelihood') plot_p_color_mesh(lpr_test, 'test log likelihood')
def forcedAlignment(lmfcc, phoneHMMs, phoneTrans): """ forcedAlignmen: aligns a phonetic transcription at the state level Args: lmfcc: NxD array of MFCC feature vectors (N vectors of dimension D) computed the same way as for the training of phoneHMMs phoneHMMs: set of phonetic Gaussian HMM models phoneTrans: list of phonetic symbols to be aligned including initial and final silence Returns: list of strings in the form phoneme_index specifying, for each time step the state from phoneHMMs corresponding to the viterbi path. """ # phone transcription => state transcription phones = sorted(phoneHMMs.keys()) nstates = {phone: phoneHMMs[phone]['means'].shape[0] for phone in phones} stateTrans = [p + '_' + str(i) for p in phoneTrans for i in range(nstates[p])] # combined HMM for utterance utteranceHMM = concatHMMs(phoneHMMs, phoneTrans) # Viterbi decoder obsloglik = log_multivariate_normal_density_diag(lmfcc, utteranceHMM['means'], utteranceHMM['covars']) viterbiPath = viterbi(obsloglik, np.log(utteranceHMM['startprob']), np.log(utteranceHMM['transmat']))[1] # time alignment (frame-by-frame state transcription) viterbiStateTrans = [stateTrans[s] for s in viterbiPath] return viterbiStateTrans
def forward_algorithm(): wordHMMs = {} isolated = get_isolated(prondict) plot_p_color_mesh(example['logalpha'], 'example alpha matrix') # verify implementation wordHMMs['o'] = concatHMMs(phoneHMMsAll, isolated['o']) log_st_prob = np.log(wordHMMs['o']['startprob']) log_transmat = np.log(wordHMMs['o']['transmat']) alpha_matrix = forward(example['obsloglik'], log_st_prob, log_transmat) plot_p_color_mesh(alpha_matrix, "hmms all output example alpha matrix") # 44 data labels keys_list = [x for x in isolated.keys()] scores_models_all = np.zeros((len(data), len(isolated))) scores_models_onespkr = np.zeros_like(scores_models_all) for j in range(len(keys_list)): key = keys_list[j] hmms = concatHMMs(phoneHMMsAll, isolated[key]) log_st_prob = np.log(hmms['startprob']) log_transmat = np.log(hmms['transmat']) for i in range(len(data)): lpr_test = log_multivariate_normal_density_diag( data[i]['lmfcc'], hmms['means'], hmms['covars']) alpha = forward(lpr_test, log_st_prob, log_transmat) scores_models_all[i, j] = logsumexp(alpha[len(alpha) - 1]) hmms = concatHMMs(phoneHMMsOne, isolated[key]) log_st_prob = np.log(hmms['startprob']) log_transmat = np.log(hmms['transmat']) for i in range(len(data)): lpr_test = log_multivariate_normal_density_diag( data[i]['lmfcc'], hmms['means'], hmms['covars']) alpha = forward(lpr_test, log_st_prob, log_transmat) scores_models_onespkr[i, j] = logsumexp(alpha[len(alpha) - 1]) predict_all = np.argmax(scores_models_all, axis=1) predict_one = np.argmax(scores_models_onespkr, axis=1) label_all = [keys_list[x] for x in predict_all] label_one = [keys_list[x] for x in predict_one] true_label = [data[x]['digit'] for x in range(len(data))] print(true_label) print(label_all) print(label_one)
def get_loglik(feature, hmm): trans_mat = hmm['transmat'][:-1, :-1] pi_vec = hmm['startprob'][:-1] means = hmm['means'] covars = hmm['covars'] obsloglik = log_multivariate_normal_density_diag(feature, means, covars) log_alpha = forward(obsloglik, np.log(pi_vec), np.log(trans_mat)) ret = logsumexp(log_alpha[-1]) return ret
def get_best_path_loglik(feature, hmm): trans_mat = hmm['transmat'][:-1, :-1] pi_vec = hmm['startprob'][:-1] means = hmm['means'] covars = hmm['covars'] obsloglik = log_multivariate_normal_density_diag(feature, means, covars) ret, _ = viterbi(obsloglik, np.log(pi_vec), np.log(trans_mat)) return ret
def retrain(feature, model, num_iters=20, threshold=1): # extract params means = model['means'] covars = model['covars'] transmat = model['transmat'][:-1, :-1] startprob = model['startprob'][:-1] log_pi = np.log(startprob) log_trans = np.log(transmat) # calculate the emission obsloglik = log_multivariate_normal_density_diag(feature, means, covars) # EM algorithm loglik_old = -np.inf for iter_ in range(num_iters): # E-step log_alpha = forward(obsloglik, log_pi, log_trans) log_beta = backward(obsloglik, log_pi, log_trans) log_gamma = statePosteriors(log_alpha, log_beta) # M-step means, covars = updateMeanAndVar(feature, log_gamma) # update obsloglik = log_multivariate_normal_density_diag( feature, means, covars) loglik = logsumexp(log_alpha[-1]) print("Iter {}: The log likelihood in EM:".format(iter_), loglik) # check if terminate EM if (loglik - loglik_old) < threshold or np.isnan(loglik): print("Terminating the EM") break else: loglik_old = loglik
def test_concatHMMs(self): """ test_concatHMMs: Required verification in section 5.1 """ # obtain the concated HMM model wordHMMs = {} wordHMMs['o'] = concatHMMs(self.phoneHMMs, isolated['o']) means = wordHMMs['o']['means'] covars = wordHMMs['o']['covars'] # use example observation to calculate the log likelihood of observation obsloglik = log_multivariate_normal_density_diag( self.example['lmfcc'], means, covars) # check against with the example assert_allclose(obsloglik, self.example['obsloglik'])
def forcedAlignment(lmfcc, phoneHMMs, phoneTrans): """ forcedAlignmen: aligns a phonetic transcription at the state level Args: lmfcc: NxD array of MFCC feature vectors (N vectors of dimension D) computed the same way as for the training of phoneHMMs phoneHMMs: set of phonetic Gaussian HMM models phoneTrans: list of phonetic symbols to be aligned including initial and final silence Returns: list of strings in the form phoneme_index specifying, for each time step the state from phoneHMMs corresponding to the viterbi path. """ utteranceHMM = concatHMMs(phoneHMMs, phoneTrans) emmision = log_multivariate_normal_density_diag(lmfcc, utteranceHMM['means'], utteranceHMM['covars']) return viterbi(emmision, np.log(utteranceHMM['startprob']), np.log(utteranceHMM['transmat']))
def state_posterior_test(): posterior = statePosteriors(example['logalpha'], example['logbeta']) plot_p_color_mesh(posterior, 'log state hmm posterior') posterior = np.exp(posterior) print('verification') print(np.sum(posterior, axis=1)) print('6.1 summing over time steps') print(np.sum(posterior, axis=0)) print('6.1 summing over time steps and states') print(np.sum(posterior)) example_x = example['lmfcc'] wordHMMs = {} isolated = get_isolated(prondict) wordHMMs['o'] = concatHMMs(phoneHMMsAll, isolated['o']) lpr = log_multivariate_normal_density_diag(example_x, wordHMMs['o']['means'], wordHMMs['o']['covars']) plot_p_color_mesh(lpr, 'log state gmm posterior')
# transcription wordTrans = list(path2info(filename) [2]) # word transcription (contained in the filename) phoneTrans = words2phones( wordTrans, prondict) # word transcription => phone transcription stateTrans = [ p + '_' + str(i) for p in phoneTrans for i in range(nstates[p]) ] # phone transcription => state transcription # combined HMM for utterance utteranceHMM = concatHMMs(phoneHMMs, phoneTrans) # Viterbi decoder obsloglik = log_multivariate_normal_density_diag(lmfcc, utteranceHMM['means'], utteranceHMM['covars']) viterbiLoglik, viterbiPath = viterbi(obsloglik, np.log(utteranceHMM['startprob']), np.log(utteranceHMM['transmat'])) # time alignment (frame-by-frame state transcription) viterbiStateTrans = [stateTrans[s] for s in viterbiPath] # save in standard format (to use it, put it in the same directory of .wav and open .wav with wavesurfer) frames2trans(viterbiStateTrans, outfilename='data/transcriptions/z43a.lab') # check results plt.figure() pcolormesh(lmfcc, 'MFCC - computed', ylabel='MFCC') plt.figure()
phoneHMMs = np.load('data/lab2_models_onespkr.npz')['phoneHMMs'].item() # Build hmm wordHMMs = {} for d in isolated.keys(): wordHMMs[d] = concatHMMs(phoneHMMs, isolated[d]) # get the observation sequence feature = data[10]['lmfcc'] # First part # calculate the emissions digit = '4' means = wordHMMs[digit]['means'] covars = wordHMMs[digit]['covars'] obsloglik = log_multivariate_normal_density_diag(feature, means, covars) # calculate the log likelihood trans_mat = wordHMMs[digit]['transmat'][:-1, :-1] pi_vec = wordHMMs[digit]['startprob'][:-1] # log space log_pi = np.log(pi_vec) log_trans = np.log(trans_mat) log_alpha = forward(obsloglik, log_pi, log_trans) log_seq_likelihood = logsumexp(log_alpha[-1]) print("The log likelihood of the digit {}:".format(digit), log_seq_likelihood) # ========================================================================= for d in wordHMMs.keys(): print("========================================================")
wordHMMs = {} for digit in isolated.keys(): wordHMMs[digit] = concatHMMs(phoneHMMs_one, isolated[digit]) print(list(wordHMMs['o'].keys())) print(list(wordHMMs.keys())) wordHMMs_all = {} for digit in isolated.keys(): wordHMMs_all[digit] = concatHMMs(phoneHMMs_all, isolated[digit]) print(list(wordHMMs_all.keys())) #Testing log likelihood function o_obsloglik = lab2_tools.log_multivariate_normal_density_diag(example['lmfcc'], wordHMMs['o']['means'], wordHMMs['o']['covars']) print("Testing if likelihood is correct: ") np.testing.assert_almost_equal(o_obsloglik, example['obsloglik'], 6) print("Likelihood is correct.") # plotting likelihood functions fig, axs = plt.subplots(2) axs[0].set_title("Computed \"o\" obsloglik") axs[0].pcolormesh(o_obsloglik.T) axs[1].set_title("Example \"o\" obsloglik") axs[1].pcolormesh(example['obsloglik'].T) plt.show() # The dark bars in the middle refer to 'ow', while the higher prob light bars # refer the 'sil' on either side of the dark.
def viterbi_algorithm(): wordHMMs = {} isolated = get_isolated(prondict) # verify implementation wordHMMs['o'] = concatHMMs(phoneHMMsAll, isolated['o']) log_st_prob = np.log(wordHMMs['o']['startprob']) log_transmat = np.log(wordHMMs['o']['transmat']) vloglik, bestPath = viterbi(example['obsloglik'], log_st_prob, log_transmat) alpha_matrix = forward(example['obsloglik'], log_st_prob, log_transmat) print('vloglik from viterbi():', vloglik) print('vloglik from example:', example['vloglik']) # plot fig = plt.figure(figsize=(12, 6)) ax = plt.subplot(121) ax.set_title('viterbi path from Viterbi()') plt.pcolormesh(alpha_matrix) plt.plot(bestPath, np.arange(len(bestPath)), color='red') plt.colorbar() plt.show() fig = plt.figure(figsize=(12, 6)) ax = plt.subplot(121) ax.set_title('viterbi path from example') plt.pcolormesh(example['logalpha']) plt.plot(example['vpath'], np.arange(len(bestPath)), color='red') plt.colorbar() plt.show() # 44 data labels keys_list = [x for x in isolated.keys()] scores_models_all = np.zeros((len(data), len(isolated))) scores_models_onespkr = np.zeros_like(scores_models_all) for j in range(len(keys_list)): key = keys_list[j] hmms = concatHMMs(phoneHMMsAll, isolated[key]) log_st_prob = np.log(hmms['startprob']) log_transmat = np.log(hmms['transmat']) for i in range(len(data)): lpr_test = log_multivariate_normal_density_diag( data[i]['lmfcc'], hmms['means'], hmms['covars']) loglik, path = viterbi(lpr_test, log_st_prob, log_transmat) scores_models_all[i, j] = loglik hmms = concatHMMs(phoneHMMsOne, isolated[key]) log_st_prob = np.log(hmms['startprob']) log_transmat = np.log(hmms['transmat']) for i in range(len(data)): lpr_test = log_multivariate_normal_density_diag( data[i]['lmfcc'], hmms['means'], hmms['covars']) loglik, path = viterbi(lpr_test, log_st_prob, log_transmat) scores_models_onespkr[i, j] = loglik predict_all = np.argmax(scores_models_all, axis=1) predict_one = np.argmax(scores_models_onespkr, axis=1) label_all = [keys_list[x] for x in predict_all] label_one = [keys_list[x] for x in predict_one] true_label = [data[x]['digit'] for x in range(len(data))] print(true_label) print(label_all) print(label_one)
from prondict import isolated from lab2_proto import concatHMMs if __name__ == '__main__': # load data data = np.load('data/lab2_data.npz')['data'] example = np.load('data/lab2_example.npz')['example'].item() phoneHMMs = np.load('data/lab2_models_onespkr.npz')['phoneHMMs'].item() # Build hmm wordHMMs = {} wordHMMs['o'] = concatHMMs(phoneHMMs, isolated['o']) # caculate the obsloglik obsloglik = log_multivariate_normal_density_diag(example['lmfcc'], wordHMMs['o']['means'], wordHMMs['o']['covars']) # # Plot for o # fig, ax = plt.subplots(figsize=(16, 4)) # im = ax.pcolormesh(obsloglik.T) # fig.colorbar(im, ax=ax) # fig.tight_layout() # plt.savefig("./plots/obsloglik_o_woman.png", # dpi=80, bbox_inches='tight') # # caculate the obsloglik # obsloglik = log_multivariate_normal_density_diag( # data[22]['lmfcc'], wordHMMs['7']['means'], wordHMMs['7']['covars']) # # Plot for o # fig, ax = plt.subplots(figsize=(16, 4)) # im = ax.pcolormesh(obsloglik.T)