def evaluate_model(model, testdir, n_classes, totalsamples, model_name, dict_name, results_dir, batch_size=16, testing=False): print('...Evaluating...') if testing: totalsamples = 10 gen = generator_test_bufferedseq_wfname(train_dir=testdir, batch_size=batch_size, n_classes=n_classes) history = model.evaluate_generator(generator=gen, steps=totalsamples) loss, acc = history print(f'Model fit history:{history}') print(f'Trained on {totalsamples} files.') print(f'Results directory: {results_dir}') cdnn_records_add(loss=loss, accuracy=acc, model_name=model_name, nn_records_name=dict_name, results_address=results_dir) del gen return
def predict_model(model, testdir, n_classes, totalsamples, model_name, dict_name, results_dir, w2pdict, batch_size=16, testing=False): print('...Evaluating...') if testing: totalsamples = 10 gen = generator_test_bufferedseq_wfname(train_dir=testdir, batch_size=batch_size, n_classes=n_classes) for s in range(totalsamples): x, y, file = next(gen) print(file) fname = file.split('.')[0] potphones = w2pdict[fname] print(potphones) predictions = model.predict_proba(x=x) print(predictions) return
def main(testing=False): # Config Values[DNN params] Frame_length = 0.025 Frame_step = 0.01 Dbdir = './speakers_db_correct/' overwrite_MFCCs = False TrainAll = False testing = True FramelevelORword = False cwd = os.getcwd() SysPath = cwd.split('GOP-LSTM')[0] Wavdir = SysPath + 'corpus/dat/speakers/' Dbdir = SysPath +'GOP-LSTM/PhoneInfo/speakers_db_correct/' Holddir = SysPath + 'HoldDir/' Traindir = Holddir + 'Train/' Testdir = Holddir + 'Test/' PhoInfDir = SysPath + 'GOP-LSTM/PhoneInfo/' N_context = 2 N_ceps = 26 wordcount = 10 # Training & Test Data if FramelevelORword: speakers_trainNtest(db_corpus=Dbdir, wav_corpus=Wavdir, n_ceps=N_ceps, n_context=N_context, frame_length=Frame_length, frame_step=Frame_step, inmat=True, holddir=Holddir, overwrite=overwrite_MFCCs) cdnn_dict_name = 'crnn_gridsearch_records_ALL.pk' if not TrainAll: ByCount = 4000 Traindir = Holddir + 'Train_Correct/' Testdir = Holddir + 'Test_Correct/' selected_phones, totalcount = select_trainNtest(bycount=ByCount, holddir=Holddir, train_corpus=Traindir, test_corpus=Testdir, overwrite=False) Traindir = Holddir + f'Train_Select_{ByCount}/' Testdir = Holddir + f'Test_Select_{ByCount}/' N_classes = len(selected_phones) print(f'N selected classes: {N_classes}') cdnn_dict_name = f'crnn_gridsearch_records_{ByCount}.pk' else: selected_phones, totalcount, w2pdict = createNcount_trainNtest(frame_length=Frame_length, frame_step=Frame_step, n_ceps=N_ceps, n_context=N_context, dbdir=Dbdir, datdir=Wavdir, holddir=Holddir, wordcount=wordcount, phoinfdir=PhoInfDir) N_classes = len(selected_phones) Traindir = Holddir + f'FLP_Train_{wordcount}/' Testdir = Holddir + f'FLP_Test_{wordcount}/' cdnn_dict_name = f'ddcp_blstm_gridsearch_records_wl_{wordcount}.pk' print(f'Selected phones: {selected_phones}') print(f'Train count & test count: {totalcount}') if testing: cdnn_dict_name = f'testing_records.pk' cdnn_address = SysPath + 'GOP-LSTM/Results/CDNN_phones/' # Iterate over gridsearch N_epochs = 70 Input_tuple = (5, 26, 1) ConvLayerList = [[32 for _ in range(15)]] DropoutList =[0.8] # add one for sil N_classes += 1 selected_phones.append('_') seq_sizelist = [64] for seq_size in seq_sizelist: totaltrain = nseqsofnsize(Traindir,seq_size=seq_size) totaltest = nseqsofnsize(Testdir,seq_size=seq_size) for cl in ConvLayerList: for dl in DropoutList: # Compile Params #cname = '_'.join(str(x) for x in cl) cname = f'{cl[0]}_x{len(cl)}' Model_name = f'BLSTM_CP{cname}_FBN_SS{seq_size}_DL{dl}_V3' model = make_CNNLSTM_classifier(input_tuple=Input_tuple, conv_layers=cl, n_classes=N_classes, seq_size=seq_size, dropout_rate=dl, channel_order='channels_last') model, Model_name = train_model(n_epochs=N_epochs, model=model, traindir=Traindir, model_name=Model_name, n_classes=N_classes, totalsamples=totaltrain, dict_name=cdnn_dict_name, results_dir=cdnn_address, batch_size=seq_size, testing=testing) print('...Evaluating...') evaluate_model(model=model, testdir=Testdir, n_classes=N_classes, totalsamples=totaltest, model_name=Model_name, dict_name=cdnn_dict_name, results_dir=cdnn_address, batch_size=seq_size, testing=testing) # Forced Accuracy print('...Predicting...') if testing: totaltest = 30 gen = generator_test_bufferedseq_wfname(train_dir=Testdir, batch_size=seq_size, n_classes=N_classes, wfname=True) ''' Return Word Accuracy (by Softmax & ForcedMax), Max Seg Accuracy (from goldstandard-gst)''' that = True if that: selected_phones.append('_') diagnose = False s_correct = 0 f_correct = 0 total = 0 s_IDS = 0 f_IDS = 0 maxsegtotal = 0 s_seg = 0 f_seg = 0 print(f'Total Test size:{totaltest}\n') x, y, file = next(gen) cfile = file for _ in range(totaltest): # amount of words to be judged if diagnose: print(f'Current file:{file}') print(f'Word\'s phones{potphones}') print(file) segcount = 0 gwordphones = [] # gold standard word segments swordphones = [] # softmax word segments fwordphones = [] # forced word segments fname = file.split('.')[0] potphones = w2pdict[fname] potphones.append('_') pind = [selected_phones.index(sp) for sp in potphones] predictions = model.predict(x=x) segcount += 1 gstd = [selected_phones[sp] for sp in np.argmax(y, axis=2)[0]] softmax = [selected_phones[sp] for sp in np.argmax(predictions, axis=2)[0]] forceda = [selected_phones[pind[sp]] for sp in np.argmax(predictions[:, :, pind][0], axis=1)] gwordphones += gstd swordphones += softmax fwordphones += forceda x, y, file = next(gen) predictions = model.predict(x=x) if cfile == file: # break out of word while loop samefile = True cfile = file else: samefile = False cfile = file while samefile: # track error for same file segcount += 1 gstd = [selected_phones[sp] for sp in np.argmax(y, axis=2)[0]] softmax = [selected_phones[sp] for sp in np.argmax(predictions, axis=2)[0]] forceda = [selected_phones[pind[sp]] for sp in np.argmax(predictions[:, :, pind][0], axis=1)] gwordphones += gstd swordphones += softmax fwordphones += forceda x, y, file = next(gen) predictions = model.predict(x=x) if cfile != file: # break out of word while loop samefile = False cfile = file # got word segs, process them gseg = segmentphonelist(gwordphones) sseg = segmentphonelist(swordphones) fseg = segmentphonelist(fwordphones) sLD = uttLD(gseg, sseg) fLD = uttLD(gseg, fseg) s_IDS += sLD f_IDS += fLD if diagnose: print('\n') print(gseg) print(sseg) print(fseg) print('\n') print(sLD) print(fLD) print('\n') # accuracy startsil = gseg[-1][1] # Index of Silence g_len = len(gwordphones[:startsil]) s_correct += segCorrect(gwordphones[:startsil], swordphones[:startsil]) f_correct += segCorrect(gwordphones[:startsil], fwordphones[:startsil]) total += g_len # max-seg-score with known boundaries # per word, then test set score wordweight = 0 for seg in gseg[:-1]: # last phone is silence '_' maxsegtotal += 1 cphone = seg[0] sboundedlist = swordphones[seg[1]:seg[2]] fboundedlist = fwordphones[seg[1]:seg[2]] smaxphone = max(sboundedlist, key=sboundedlist.count) fmaxphone = max(fboundedlist, key=fboundedlist.count) if smaxphone == cphone: s_seg += 1 if fmaxphone == cphone: f_seg += 1 if diagnose: print(seg) print(smaxphone, fmaxphone, cphone) sLDpercent = sLD / total * 100 fLDpercent = fLD / total * 100 print(f'Insertions, Deletions, Substitions (SM):{sLD} out of {total}: {sLDpercent}%') print(f'Insertions, Deletions, Substitions (FM):{fLD} out of {total}: {fLDpercent}%') Spercent = s_correct / total * 100 Fpercent = f_correct / total * 100 print('\n') print(f'Softmax: {s_correct} out of {total}, {Spercent}%') print(f'Forced: {f_correct} out of {total}, {Fpercent}%') Spercent = s_seg / maxsegtotal * 100 Fpercent = f_seg / maxsegtotal * 100 print(f'Softmax (seg): {s_seg} out of {maxsegtotal}, {Spercent}%') print(f'Forced (seg): {f_seg} out of {maxsegtotal}, {Fpercent}%') cdnn_records_add(loss=Spercent, accuracy=Fpercent, model_name=Model_name, nn_records_name=cdnn_dict_name, results_address=cdnn_address) del gen del model k.clear_session()
def main(testing=False): # Config Values, Feature Assembly, focused on WordCount/FrameCount Frame_length = 0.025 Frame_step = 0.01 overwrite_MFCCs = False TrainAll = False #testing = True FramelevelORword = False cwd = os.getcwd() SysPath = cwd.split('GOP-LSTM')[0] Dbdir = SysPath + 'GOP-LSTM/PhoneInfo/speakers_db_correct/' '''Based on your computer setup. (I have both folders on same level as GOP-LSTM)''' Wavdir = SysPath + 'corpus/dat/speakers/' # On SysPath level Holddir = SysPath + 'HoldDir/' Traindir = Holddir + 'Train/' Testdir = Holddir + 'Test/' PhoInfDir = SysPath + 'GOP-LSTM/PhoneInfo/' N_context = 2 N_ceps = 26 wordcount = 30 # Training & Test Data if FramelevelORword: speakers_trainNtest(db_corpus=Dbdir, wav_corpus=Wavdir, n_ceps=N_ceps, n_context=N_context, frame_length=Frame_length, frame_step=Frame_step, inmat=True, holddir=Holddir, overwrite=overwrite_MFCCs) cdnn_dict_name = 'crnn_gridsearch_records_ALL.pk' if not TrainAll: ByCount = 4000 Traindir = Holddir + 'Train_Correct/' Testdir = Holddir + 'Test_Correct/' selected_phones, totalcount = select_trainNtest( bycount=ByCount, holddir=Holddir, train_corpus=Traindir, test_corpus=Testdir, overwrite=False) Traindir = Holddir + f'Train_Select_{ByCount}/' Testdir = Holddir + f'Test_Select_{ByCount}/' N_classes = len(selected_phones) print(f'N selected classes: {N_classes}') cdnn_dict_name = f'segmentation_records_{ByCount}.pk' else: selected_phones, totalcount, w2pdict = createNcount_trainNtest( frame_length=Frame_length, frame_step=Frame_step, n_ceps=N_ceps, n_context=N_context, dbdir=Dbdir, datdir=Wavdir, holddir=Holddir, phoinfdir=PhoInfDir, wordcount=wordcount) N_classes = len(selected_phones) Traindir = Holddir + f'FLP_Train_{wordcount}/' Testdir = Holddir + f'FLP_Test_{wordcount}/' cdnn_dict_name = f'segment_blstm_records_wl_{wordcount}.pk' print(f'Selected phones: {selected_phones}') print(f'Train count & test count: {totalcount}') if testing: cdnn_dict_name = f'testing_records.pk' print(f' Using:{cdnn_dict_name}') cdnn_address = SysPath + 'GOP-LSTM/Results/CDNN_segments/' # Iterate over gridsearch N_epochs = 70 Input_tuple = (5, 26, 1) # ,[32 for _ in range(10)] ConvLayerList = [[32 for _ in range(5)]] DropoutList = [0.8] # add one for sil N_classes += 1 seq_sizelist = [64] for seq_size in seq_sizelist: totaltrain = nseqsofnsize(Traindir, seq_size=seq_size) totaltest = nseqsofnsize(Testdir, seq_size=seq_size) for cl in ConvLayerList: for dl in DropoutList: # Compile Params #cname = '_'.join(str(x) for x in cl) cname = f'{cl[0]}_x{len(cl)}' Model_name = f'BLSTM_CP{cname}_FBN_SS{seq_size}_DL{dl}_V2' # check if model exist with said name Model_address = f'{Model_name}model.hdf5' # if manual entry Model_address = 'BLSTM_CP32_x5_FBN_SS64_DL0.8_V2model.hdf5' if os.path.exists(Model_address): print(f'loading model: {Model_address}') model = load_model(Model_address) print(model.inputs, model.outputs) firsttime = False print(f'Loaded') else: print(f'No such model as :{Model_address}') # Forced Accuracy print('...Predicting...') if testing: totaltest = 30 gen = generator_test_bufferedseq_wfname(train_dir=Testdir, batch_size=seq_size, n_classes=N_classes, wfname=True) ''' Need to add dictionary with similar sounding word however it seems there are too few words to do this with a small test set. also, might be having problems with switching to 25 wc from 30wc''' this = True if this: sameworddiffprondict = defaultdict(list) for keys, v in w2pdict.items(): wordnum = keys.split('_')[-1] #check if already in dict list if v not in sameworddiffprondict[wordnum]: sameworddiffprondict[wordnum].append(v) sameworddiffprondict.pop('17') sameworddiffprondict.pop('18') for key, v in sameworddiffprondict.items(): v[0] += '_' print(key, v) similardict = {'13': '48', '48': '13', '25': '5', '5': '25'} dolist = ['13', '48', '25', '5'] ''' Return Word Accuracy (by Softmax & ForcedMax), Max Seg Accuracy (from goldstandard-gst)''' that = True # import the whole list for testing each possible word, instead of softmax totaltest = 15 # for testing purpose shorten if that: selected_phones.append('_') print(f'Total Test size:{totaltest}\n') x, y, file = next(gen) cfile = file cfname = cfile.split('.')[0].split('_')[-1] n = 0 for _ in range(totaltest): # amount of words to be judged # Two versions, Right Word and Random Wrong Word print(f'starting file:{file}, {cfname}') cfile = file cfname = cfile.split('.')[0].split('_')[-1] if cfname not in dolist: notinlist = True while notinlist: n += 1 x, y, file = next(gen) cfile = file cfname = cfile.split('.')[0].split('_')[-1] if cfname in dolist: print(f'in') notinlist = False print(f'N:{n}') elif n > 100: notinlist = False print(f'N:{n}') predictions = [] samefile = True while samefile: tp = model.predict(x=x)[0].tolist() predictions += tp # get all values then roll through loops x, y, file = next(gen) if cfile != file: # break out of word while loop samefile = False print(f'Next file:{file}') print(f'Going in: {cfile},{cfname}') for turnN in range(2): sscores = [] fscores = [] fwordphones = [] if turnN == 1: fname = similardict[cfname] print(f'Wrong: {fname}') potphones = sameworddiffprondict[fname][0] else: print(f'Right: {cfname}') potphones = sameworddiffprondict[cfname][0] print(potphones) pind = [ selected_phones.index(sp) for sp in potphones ] print(f'Expected word: {potphones}') predictions = np.asarray(predictions) smargs = np.max(predictions, axis=1) fargs = np.max(predictions[:, pind], axis=1) fwordphones += [ selected_phones[pind[sp]] for sp in np.argmax(predictions[:, pind], axis=1) ] sscores += smargs.tolist() fscores += fargs.tolist() # got word segs, process them fseg = segmentphonelist(fwordphones) # use fseg to separate phones and then show their prob fscores = np.asarray(fscores) sscores = np.asarray(sscores) gop_per_frame = np.round( np.abs(np.log(fscores / sscores)), 4) gop_list = [] avg_phone_error = [] for seg in fseg[:-1]: #skip space phone_chuck = gop_per_frame[seg[1] - 1:seg[2] - 1] phonescore = np.round( np.sum(phone_chuck) / seg[3], 3) gop_list.append((seg[0], phonescore, seg[3])) avg_phone_error.append(phonescore) print(gop_list) print(np.round(np.average(avg_phone_error))) print('\n') # accuracy # GOP, create ratios between RNN-WF/RNN-O and RNN-WF/(0.5*WF + 0.5*O) # for correct words, for similar words, then for opposite words del gen del model k.clear_session()
def main(testing=False): # Config Values, Feature Assembly, focused on WordCount/FrameCount Frame_length = 0.025 Frame_step = 0.01 overwrite_MFCCs = False TrainAll = False #testing = True FramelevelORword = False cwd = os.getcwd() SysPath = cwd.split('GOP-LSTM')[0] Dbdir = SysPath + 'GOP-LSTM/PhoneInfo/speakers_db_correct/' '''Based on your computer setup. (I have both folders on same level as GOP-LSTM)''' Wavdir = SysPath + 'corpus/dat/speakers/' # On SysPath level Holddir = SysPath + 'HoldDir/' Traindir = Holddir + 'Train/' Testdir = Holddir + 'Test/' PhoInfDir = SysPath + 'GOP-LSTM/PhoneInfo/' N_context = 2 N_ceps = 26 wordcount = 10 # Training & Test Data if FramelevelORword: speakers_trainNtest(db_corpus=Dbdir, wav_corpus=Wavdir, n_ceps=N_ceps, n_context=N_context, frame_length=Frame_length, frame_step=Frame_step, inmat=True, holddir=Holddir, overwrite=overwrite_MFCCs) cdnn_dict_name = 'crnn_gridsearch_records_ALL.pk' if not TrainAll: ByCount = 4000 Traindir = Holddir + 'Train_Correct/' Testdir = Holddir + 'Test_Correct/' selected_phones, totalcount = select_trainNtest( bycount=ByCount, holddir=Holddir, train_corpus=Traindir, test_corpus=Testdir, overwrite=False) Traindir = Holddir + f'Train_Select_{ByCount}/' Testdir = Holddir + f'Test_Select_{ByCount}/' N_classes = len(selected_phones) print(f'N selected classes: {N_classes}') cdnn_dict_name = f'segmentation_records_{ByCount}.pk' else: selected_phones, totalcount, w2pdict = createNcount_trainNtest( frame_length=Frame_length, frame_step=Frame_step, n_ceps=N_ceps, n_context=N_context, dbdir=Dbdir, datdir=Wavdir, holddir=Holddir, phoinfdir=PhoInfDir, wordcount=wordcount) N_classes = len(selected_phones) Traindir = Holddir + f'FLP_Train_{wordcount}/' Testdir = Holddir + f'FLP_Test_{wordcount}/' cdnn_dict_name = f'segment_blstm_records_wl_{wordcount}.pk' print(f'Selected phones: {selected_phones}') print(f'Train count & test count: {totalcount}') if testing: cdnn_dict_name = f'testing_records.pk' print(f' Using:{cdnn_dict_name}') cdnn_address = SysPath + 'GOP-LSTM/Results/CDNN_segments/' # Iterate over gridsearch N_epochs = 70 Input_tuple = (5, 26, 1) # ,[32 for _ in range(10)] ConvLayerList = [[32 for _ in range(10)]] DropoutList = [0.8] # add one for sil N_classes += 1 seq_sizelist = [64] for seq_size in seq_sizelist: totaltrain = nseqsofnsize(Traindir, seq_size=seq_size) totaltest = nseqsofnsize(Testdir, seq_size=seq_size) for cl in ConvLayerList: for dl in DropoutList: # Compile Params #cname = '_'.join(str(x) for x in cl) cname = f'{cl[0]}_x{len(cl)}' Model_name = f'BLSTM_CP{cname}_FBN_SS{seq_size}_DL{dl}_V2' # check if model exist with said name Model_address = f'{Model_name}model.hdf5' if os.path.exists(Model_address): print(f'loading model: {Model_address}') model = load_model(Model_address) firsttime = False print(f'Loaded') else: print(f'No such model as :{Model_address}') # Forced Accuracy print('...Predicting...') if testing: totaltest = 30 gen = generator_test_bufferedseq_wfname(train_dir=Testdir, batch_size=seq_size, n_classes=N_classes, wfname=True) ''' Need to add dictionary with similar sounding word however it seems there are too few words to do this with a small test set. also, might be having problems with switching to 25 wc from 30wc''' this = True if this: sameworddiffprondict = defaultdict(list) for keys, v in w2pdict.items(): wordnum = keys.split('_')[-1] #check if already in dict list if v not in sameworddiffprondict[wordnum]: sameworddiffprondict[wordnum].append(v) for keys, v in sameworddiffprondict.items(): print(keys, v) ''' Return Word Accuracy (by Softmax & ForcedMax), Max Seg Accuracy (from goldstandard-gst)''' that = True # possibly create function to address, which words are possibly useful forced-word-max #sameworddiffname swdname = ['9', '44'] swdforms = [[['b', 'ɔ', 'l', '_'], ['b', 'a', 'l', '_'], ['b', 'ɔ', '_']], [['d', 'ɹ', 'ʌ', 'm', '_'], ['dʒ', 'ɹ', 'ʌ', 'm', '_']]] # import the whole list for testing each possible word, instead of softmax #totaltest = 3 # for testing purpose shorten if that: perphoneaccdict = defaultdict( list) # Word-key,list-accuracy, post-process selected_phones.append('_') diagnose = False s_correct = 0 f_correct = 0 fw_correct = 0 total = 0 s_IDS = 0 f_IDS = 0 maxsegtotal = 0 s_seg = 0 f_seg = 0 print(f'Total Test size:{totaltest}\n') x, y, file = next(gen) cfile = file for _ in range(totaltest): # amount of words to be judged if diagnose: print(f'Current file:{file}') print(f'Word\'s phones{potphones}') segcount = 0 gwordphones = [] # gold standard word segments swordphones = [] # softmax word segments fwordphones = [] # forced word segments fname = file.split('.')[0] potphones = w2pdict[fname] fnamelast = fname.split('_')[-1] potphones.append('_') pind = [selected_phones.index(sp) for sp in potphones] predictions = model.predict(x=x) segcount += 1 gstd = argmaxpredicts2phones(y[0], selected_phones) softmax = argmaxpredicts2phones( predictions[0], selected_phones) forceda = argmaxpredicts2forcedphones( predictions[0], selected_phones, pind) wordmaxforce = [] if fnamelast in swdname: wordmaxforce = swdforms[swdname.index(fnamelast)] swdlist = [] swdlistscore = [0 for _ in wordmaxforce] for index, wsp in enumerate(wordmaxforce): wspind = [ selected_phones.index(sp) for sp in wsp ] tv, tswd = argmaxpredicts2forcedphones( predictions[0], selected_phones, wspind, fwords=True) swdlist.append(tswd) swdlistscore[index] += tv gwordphones += gstd swordphones += softmax fwordphones += forceda x, y, file = next(gen) predictions = model.predict(x=x) if cfile == file: # break out of word while loop samefile = True cfile = file else: samefile = False cfile = file while samefile: # track error for same file segcount += 1 gstd = argmaxpredicts2phones(y[0], selected_phones) softmax = argmaxpredicts2phones( predictions[0], selected_phones) forceda = argmaxpredicts2forcedphones( predictions[0], selected_phones, pind) if wordmaxforce: for index, wsp in enumerate(wordmaxforce): wspind = [ selected_phones.index(sp) for sp in wsp ] tv, tswd = argmaxpredicts2forcedphones( predictions[0], selected_phones, wspind, fwords=True) swdlist[index] += tswd swdlistscore[index] += tv gwordphones += gstd swordphones += softmax fwordphones += forceda x, y, file = next(gen) predictions = model.predict(x=x) if cfile != file: # break out of word while loop samefile = False cfile = file # got word segs, process them gseg = segmentphonelist(gwordphones) sseg = segmentphonelist(swordphones) fseg = segmentphonelist(fwordphones) sLD = uttLD(gseg, sseg) fLD = uttLD(gseg, fseg) s_IDS += sLD f_IDS += fLD if diagnose: print('\n') print(gseg) print(sseg) print(fseg) print('\n') print(sLD) print(fLD) print('\n') # accuracy startsil = gseg[-1][1] #Index of Silence g_len = len(gwordphones[:startsil]) s_correct += segCorrect(gwordphones[:startsil], swordphones[:startsil]) f_correct += segCorrect(gwordphones[:startsil], fwordphones[:startsil]) total += g_len if wordmaxforce: print(f'actual:{potphones}') print(f'swd {wordmaxforce}' ) # list for individual comparision print(f'scores{swdlistscore}') for wlist in swdlist: print( segCorrect(gwordphones[:startsil], wlist[:startsil]) / g_len) print('\n') # max-seg-score with known boundaries # per word, then test set score wordweight = 0 for seg in gseg[:-1]: # last phone is silence '_' maxsegtotal += 1 cphone = seg[0] sboundedlist = swordphones[seg[1]:seg[2]] fboundedlist = fwordphones[seg[1]:seg[2]] smaxphone = max(sboundedlist, key=sboundedlist.count) fmaxphone = max(fboundedlist, key=fboundedlist.count) if smaxphone == cphone: s_seg += 1 if fmaxphone == cphone: f_seg += 1 #if cphone in perphoneaccdict.keys(): # perphoneaccdict[cphone].append() if diagnose: print(seg) print(smaxphone, fmaxphone, cphone) sLDpercent = sLD / total * 100 fLDpercent = fLD / total * 100 print( f'Insertions, Deletions, Substitions (SM):{sLD} out of {total}: {sLDpercent}%' ) print( f'Insertions, Deletions, Substitions (FM):{fLD} out of {total}: {fLDpercent}%' ) Spercent = s_correct / total * 100 Fpercent = f_correct / total * 100 print('\n') print(f'Softmax: {s_correct} out of {total}, {Spercent}%') print(f'Forced: {f_correct} out of {total}, {Fpercent}%') Spercent = s_seg / maxsegtotal * 100 Fpercent = f_seg / maxsegtotal * 100 print( f'Softmax (seg): {s_seg} out of {maxsegtotal}, {Spercent}%' ) print( f'Forced (seg): {f_seg} out of {maxsegtotal}, {Fpercent}%' ) del gen del model k.clear_session()
def main(testing=False): # Config Values, Feature Assembly, focused on WordCount/FrameCount Frame_length = 0.025 Frame_step = 0.01 overwrite_MFCCs = False TrainAll = False #testing = True FramelevelORword = False cwd = os.getcwd() SysPath = cwd.split('GOP-LSTM')[0] Dbdir = SysPath + 'GOP-LSTM/PhoneInfo/speakers_db_correct/' '''Based on your computer setup. (I have both folders on same level as GOP-LSTM)''' Wavdir = SysPath + 'corpus/dat/speakers/' # On SysPath level Holddir = SysPath + 'HoldDir/' Traindir = Holddir + 'Train/' Testdir = Holddir + 'Test/' PhoInfDir = SysPath + 'GOP-LSTM/PhoneInfo/' N_context = 2 N_ceps = 26 wordcount = 25 # Training & Test Data if FramelevelORword: speakers_trainNtest(db_corpus=Dbdir, wav_corpus=Wavdir, n_ceps=N_ceps, n_context=N_context, frame_length=Frame_length, frame_step=Frame_step, inmat=True, holddir=Holddir, overwrite=overwrite_MFCCs) cdnn_dict_name = 'crnn_gridsearch_records_ALL.pk' if not TrainAll: ByCount = 4000 Traindir = Holddir + 'Train_Correct/' Testdir = Holddir + 'Test_Correct/' selected_phones, totalcount = select_trainNtest( bycount=ByCount, holddir=Holddir, train_corpus=Traindir, test_corpus=Testdir, overwrite=False) Traindir = Holddir + f'Train_Select_{ByCount}/' Testdir = Holddir + f'Test_Select_{ByCount}/' N_classes = len(selected_phones) print(f'N selected classes: {N_classes}') cdnn_dict_name = f'segmentation_records_{ByCount}.pk' else: selected_phones, totalcount, w2pdict = createNcount_trainNtest( frame_length=Frame_length, frame_step=Frame_step, n_ceps=N_ceps, n_context=N_context, dbdir=Dbdir, datdir=Wavdir, holddir=Holddir, phoinfdir=PhoInfDir, wordcount=wordcount) N_classes = len(selected_phones) Traindir = Holddir + f'FLP_Train_{wordcount}/' Testdir = Holddir + f'FLP_Test_{wordcount}/' cdnn_dict_name = f'segment_blstm_records_wl_{wordcount}.pk' print(f'Selected phones: {selected_phones}') print(f'Train count & test count: {totalcount}') if testing: cdnn_dict_name = f'testing_records.pk' print(f' Using:{cdnn_dict_name}') cdnn_address = SysPath + 'GOP-LSTM/Results/CDNN_segments/' # Iterate over gridsearch N_epochs = 70 Input_tuple = (5, 26, 1) # ,[32 for _ in range(10)] ConvLayerList = [[32 for _ in range(10)]] DropoutList = [0.8] # add one for sil print(f'Number of phones:{N_classes}') N_classes += 1 seq_sizelist = [64] for seq_size in seq_sizelist: totaltrain = nseqsofnsize(Traindir, seq_size=seq_size) totaltest = nseqsofnsize(Testdir, seq_size=seq_size) for cl in ConvLayerList: for dl in DropoutList: # Compile Params #cname = '_'.join(str(x) for x in cl) cname = f'{cl[0]}_x{len(cl)}' Model_name = f'BLSTM_CP{cname}_FBN_SS{seq_size}_DL{dl}_V2' # check if model exist with said name Model_address = f'{Model_name}model.hdf5' if os.path.exists(Model_address): print(f'loading model: {Model_address}') model = load_model(Model_address) firsttime = False print(f'Loaded') else: print(f'No such model as :{Model_address}') # Forced Accuracy print('...Predicting...') if testing: totaltest = 30 gen = generator_test_bufferedseq_wfname(train_dir=Testdir, batch_size=seq_size, n_classes=N_classes, wfname=True) ''' Need to add dictionary with similar sounding word however it seems there are too few words to do this with a small test set. also, might be having problems with switching to 25 wc from 30wc''' this = True if this: swddict = defaultdict(list) for keys, v in w2pdict.items(): wordnum = keys.split('_')[-1] if v not in swddict[wordnum]: swddict[wordnum].append(v) for keys, v in swddict.items(): v[0].append('_') #print(keys,v) that = True if that: perwordaccdict = defaultdict( list) # Word-key,list-accuracy, post-process perwordmaxdict = defaultdict( list) # Word-key,list-accuracy, post-process skiplist = [17, 18] selected_phones.append('_') diagnose = False print(f'Total Test size:{totaltest}\n') x, y, file = next(gen) cfile = file for _ in range(totaltest): # amount of words to be judged if diagnose: print(f'Word\'s phones{potphones}') #print(f'Current file:{file}') trueword = file.split('.')[0].split('_')[1] #print(trueword) segcount = 0 gwordphones = [] # gold standard word segments # we need to do this for all words, not just expected predictions = model.predict(x=x) segcount += 1 gstd = argmaxpredicts2phones(y[0], selected_phones) twordlistdict = defaultdict(list) twordscoredict = defaultdict(list) twordaccdict = defaultdict(list) for index, plist in swddict.items(): twordlistdict[index] = [] twordscoredict[index] = 0 twordaccdict[index] = 0 if index not in skiplist: for iindex, wsp in enumerate( plist ): # should only be one, two later wspind = [ selected_phones.index(sp) for sp in wsp ] tv, tswd = argmaxpredicts2forcedphones( predictions[0], selected_phones, wspind, fwords=True) twordlistdict[index].append(tswd) twordscoredict[index] += tv gwordphones += gstd x, y, file = next(gen) predictions = model.predict(x=x) if cfile == file: # break out of word while loop samefile = True cfile = file else: samefile = False cfile = file while samefile: # track error for same file segcount += 1 gstd = argmaxpredicts2phones(y[0], selected_phones) for index, plist in swddict.items(): if index not in skiplist: for iindex, wsp in enumerate( plist ): # should only be one, two later wspind = [ selected_phones.index(sp) for sp in wsp ] tv, tswd = argmaxpredicts2forcedphones( predictions[0], selected_phones, wspind, fwords=True) twordlistdict[index] += (tswd) twordscoredict[index] += tv gwordphones += gstd x, y, file = next(gen) predictions = model.predict(x=x) if cfile != file: # break out of word while loop samefile = False cfile = file gseg = segmentphonelist(gwordphones) startsil = gseg[-1][1] #Index of Silence g_len = len(gwordphones[:startsil]) for index, _ in swddict.items(): if index not in skiplist: tacc = segCorrect( gwordphones[:startsil], twordlistdict[index][0][:startsil]) / g_len twordaccdict[index] = tacc sortedacc = sorted(twordaccdict.items(), key=operator.itemgetter(1), reverse=True) sortedmax = sorted(twordscoredict.items(), key=operator.itemgetter(1), reverse=True) for index, tuple in enumerate(sortedacc): if tuple[0] == trueword: perwordaccdict[trueword].append(index) for index, tuple in enumerate(sortedmax): if tuple[0] == trueword: perwordmaxdict[trueword].append(index) #print(sortedmax) #print(sortedacc) # Number of correct, binary score, then a relative score, the greater the worse averageaccuracy = 0 numberoftrials = 0 averagelengthaway = 0 rankingsacc = [] rankingsmax = [] for _, alist in perwordaccdict.items(): for score in alist: numberoftrials += 1 averagelengthaway += score if score == 0: averageaccuracy += 1 rankingsacc.append(score) print( f'Avg num of distance from 0, {averagelengthaway/numberoftrials} using max accuracy' ) print(f'Avg accuracy {averageaccuracy/numberoftrials}') averageaccuracy = 0 averagelengthaway = 0 for _, mlist in perwordmaxdict.items(): for score in mlist: numberoftrials += 1 averagelengthaway += score if score == 0: averageaccuracy += 1 rankingsmax.append(score) print( f'Avg num of distance from 0, {averagelengthaway/numberoftrials} using sum of max' ) print( f'Avg sum phone maxs {averageaccuracy/numberoftrials}') print(f'Out {len(perwordaccdict.keys())} words') plt.subplot(1, 2, 1) plt.title('Rankings by Accuracy') plt.hist(rankingsacc, bins=38) plt.xlabel('Occurrences') plt.xlabel('Distance') plt.subplot(1, 2, 2) plt.title('Rankings by Sum of Softmax') plt.hist(rankingsmax, bins=38) plt.xlabel('Distance') plt.tight_layout() plt.show() del gen del model k.clear_session()
def main(testing=False): # Config Values[DNN params] Frame_length = 0.025 Frame_step = 0.01 Dbdir = './speakers_db_correct/' overwrite_MFCCs = False TrainAll = False #testing = True FramelevelORword = False cwd = os.getcwd() SysPath = cwd.split('GOP-LSTM')[0] Wavdir = SysPath + 'corpus/dat/speakers/' Dbdir = SysPath + 'GOP-LSTM/PhoneInfo/speakers_db_correct/' Holddir = SysPath + 'HoldDir/' Traindir = Holddir + 'Train/' Testdir = Holddir + 'Test/' PhoInfDir = SysPath + 'GOP-LSTM/PhoneInfo/' N_context = 2 N_ceps = 26 wordcount = 10 # Training & Test Data if FramelevelORword: speakers_trainNtest(db_corpus=Dbdir, wav_corpus=Wavdir, n_ceps=N_ceps, n_context=N_context, frame_length=Frame_length, frame_step=Frame_step, inmat=True, holddir=Holddir, overwrite=overwrite_MFCCs) cdnn_dict_name = 'crnn_gridsearch_records_ALL.pk' if not TrainAll: ByCount = 4000 Traindir = Holddir + 'Train_Correct/' Testdir = Holddir + 'Test_Correct/' selected_phones, totalcount = select_trainNtest( bycount=ByCount, holddir=Holddir, train_corpus=Traindir, test_corpus=Testdir, overwrite=False) Traindir = Holddir + f'Train_Select_{ByCount}/' Testdir = Holddir + f'Test_Select_{ByCount}/' N_classes = len(selected_phones) print(f'N selected classes: {N_classes}') cdnn_dict_name = f'crnn_gridsearch_records_{ByCount}.pk' else: selected_phones, totalcount, w2pdict = createNcount_trainNtest( frame_length=Frame_length, frame_step=Frame_step, n_ceps=N_ceps, n_context=N_context, dbdir=Dbdir, datdir=Wavdir, holddir=Holddir, phoinfdir=PhoInfDir, wordcount=wordcount) N_classes = len(selected_phones) Traindir = Holddir + f'FLP_Train_{wordcount}/' Testdir = Holddir + f'FLP_Test_{wordcount}/' cdnn_dict_name = f'drescp_blstm_records_wl_{wordcount}.pk' print(f'Selected phones (amount: {selected_phones}') print(f'Train count & test count: {totalcount}') if testing: cdnn_dict_name = f'testing_records.pk' cdnn_address = SysPath + 'GOP-LSTM/Results/CDNN_phones/' # Iterate over gridsearch N_epochs = 80 Input_tuple = (5, 26, 1) ConvLayerList = [[32, 7, 1024]] DropoutList = [0.8] # add one for sil N_classes += 1 selected_phones.append('_') seq_sizelist = [64] for seq_size in seq_sizelist: totaltrain = nseqsofnsize(Traindir, seq_size=seq_size) totaltest = nseqsofnsize(Testdir, seq_size=seq_size) for cl in ConvLayerList: for dl in DropoutList: # Compile Params cname = '_'.join(str(x) for x in cl) Model_name = f'ResBLSTM_C{cname}_FBN_SS{seq_size}_DL{dl}_V3' Model_address = f'{Model_name}model.hdf5' if os.path.exists(Model_address): print(f'Loading Model:{Model_address}') model = load_model(Model_address) firsttime = False print('Loaded') else: model = make_CNNLSTM_classifier( input_tuple=Input_tuple, conv_layers=cl, n_classes=N_classes, seq_size=seq_size, dropout_rate=dl, channel_order='channels_last') model, Model_name2 = train_model(n_epochs=N_epochs, model=model, traindir=Traindir, model_name=Model_name, n_classes=N_classes, totalsamples=totaltrain, dict_name=cdnn_dict_name, results_dir=cdnn_address, batch_size=seq_size, testing=testing) print('...Evaluating...') evaluate_model(model=model, testdir=Testdir, n_classes=N_classes, totalsamples=totaltest, model_name=Model_name2, dict_name=cdnn_dict_name, results_dir=cdnn_address, batch_size=seq_size, testing=testing) firsttime = True if firsttime: model.save(Model_address) # Forced Accuracy print('...Predicting...') if testing: totaltest = 30 gen = generator_test_bufferedseq_wfname(train_dir=Testdir, batch_size=seq_size, n_classes=N_classes, wfname=True) that = True # possibly create function to address, which words are possibly useful forced-word-max #sameworddiffname swdname = ['9', '44'] swdforms = [[['b', 'ɔ', 'l', '_'], ['b', 'a', 'l', '_'], ['b', 'ɔ', '_']], [['d', 'ɹ', 'ʌ', 'm', '_'], ['dʒ', 'ɹ', 'ʌ', 'm', '_']]] # import the whole list for testing each possible word, instead of softmax #totaltest = 3 # for testing purpose shorten if that: perphoneaccdict = defaultdict(list) selected_phones.append('_') diagnose = False s_correct = 0 f_correct = 0 fw_correct = 0 total = 0 s_IDS = 0 f_IDS = 0 maxsegtotal = 0 s_seg = 0 f_seg = 0 print(f'Total Test size:{totaltest}\n') x, y, file = next(gen) cfile = file for _ in range(totaltest): # amount of words to be judged if diagnose: print(f'Current file:{file}') print(f'Word\'s phones{potphones}') segcount = 0 gwordphones = [] # gold standard word segments swordphones = [] # softmax word segments fwordphones = [] # forced word segments fname = file.split('.')[0] potphones = w2pdict[fname] fnamelast = fname.split('_')[-1] potphones.append('_') pind = [selected_phones.index(sp) for sp in potphones] predictions = model.predict(x=x) segcount += 1 gstd = argmaxpredicts2phones(y[0], selected_phones) softmax = argmaxpredicts2phones( predictions[0], selected_phones) forceda = argmaxpredicts2forcedphones( predictions[0], selected_phones, pind) wordmaxforce = [] if fnamelast in swdname: wordmaxforce = swdforms[swdname.index(fnamelast)] swdlist = [] swdlistscore = [0 for _ in wordmaxforce] for index, wsp in enumerate(wordmaxforce): wspind = [ selected_phones.index(sp) for sp in wsp ] tv, tswd = argmaxpredicts2forcedphones( predictions[0], selected_phones, wspind, fwords=True) swdlist.append(tswd) swdlistscore[index] += tv gwordphones += gstd swordphones += softmax fwordphones += forceda x, y, file = next(gen) predictions = model.predict(x=x) if cfile == file: # break out of word while loop samefile = True cfile = file else: samefile = False cfile = file while samefile: # track error for same file segcount += 1 gstd = argmaxpredicts2phones(y[0], selected_phones) softmax = argmaxpredicts2phones( predictions[0], selected_phones) forceda = argmaxpredicts2forcedphones( predictions[0], selected_phones, pind) if wordmaxforce: for index, wsp in enumerate(wordmaxforce): wspind = [ selected_phones.index(sp) for sp in wsp ] tv, tswd = argmaxpredicts2forcedphones( predictions[0], selected_phones, wspind, fwords=True) swdlist[index] += tswd swdlistscore[index] += tv gwordphones += gstd swordphones += softmax fwordphones += forceda x, y, file = next(gen) predictions = model.predict(x=x) if cfile != file: # break out of word while loop samefile = False cfile = file # got word segs, process them gseg = segmentphonelist(gwordphones) sseg = segmentphonelist(swordphones) fseg = segmentphonelist(fwordphones) sLD = uttLD(gseg, sseg) fLD = uttLD(gseg, fseg) s_IDS += sLD f_IDS += fLD if diagnose: print('\n') print(gseg) print(sseg) print(fseg) print('\n') print(sLD) print(fLD) print('\n') # accuracy startsil = gseg[-1][1] #Index of Silence g_len = len(gwordphones[:startsil]) s_correct += segCorrect(gwordphones[:startsil], swordphones[:startsil]) f_correct += segCorrect(gwordphones[:startsil], fwordphones[:startsil]) total += g_len if wordmaxforce: print(f'actual:{potphones}') print(f'swd {wordmaxforce}' ) # list for individual comparision print(f'scores{swdlistscore}') for wlist in swdlist: print( segCorrect(gwordphones[:startsil], wlist[:startsil]) / g_len) print('\n') # max-seg-score with known boundaries # per word, then test set score wordweight = 0 for seg in gseg[:-1]: # last phone is silence '_' maxsegtotal += 1 cphone = seg[0] sboundedlist = swordphones[seg[1]:seg[2]] fboundedlist = fwordphones[seg[1]:seg[2]] smaxphone = max(sboundedlist, key=sboundedlist.count) fmaxphone = max(fboundedlist, key=fboundedlist.count) if smaxphone == cphone: s_seg += 1 if fmaxphone == cphone: f_seg += 1 #if cphone in perphoneaccdict.keys(): # perphoneaccdict[cphone].append() if diagnose: print(seg) print(smaxphone, fmaxphone, cphone) sLDpercent = sLD / total * 100 fLDpercent = fLD / total * 100 print( f'Insertions, Deletions, Substitions (SM):{sLD} out of {total}: {sLDpercent}%' ) print( f'Insertions, Deletions, Substitions (FM):{fLD} out of {total}: {fLDpercent}%' ) Spercent = s_correct / total * 100 Fpercent = f_correct / total * 100 print('\n') print(f'Softmax: {s_correct} out of {total}, {Spercent}%') print(f'Forced: {f_correct} out of {total}, {Fpercent}%') Spercent = s_seg / maxsegtotal * 100 Fpercent = f_seg / maxsegtotal * 100 print( f'Softmax (seg): {s_seg} out of {maxsegtotal}, {Spercent}%' ) print( f'Forced (seg): {f_seg} out of {maxsegtotal}, {Fpercent}%' ) del gen del model k.clear_session()