def set_pre_suc(): tones = ['01234'] name_list_path = '/home/h1/decha/Dropbox/python_workspace/Inter_speech_2016/playground/list_file_for_preceeding_suceeding/list_gpr_file/' for t in tones: path = '/home/h1/decha/Dropbox/Inter_speech_2016/Syllable_object/mix_object/current_version/all_vowel_type/syllable_object_{}.pickle'.format( t) print path syl_management = Utility.load_obj(path) for syl in syl_management.syllables_list: if 'manual' in syl.name_index: continue name = syl.name_index.split('_') file_tar = '{}/{}/{}.lab'.format(name_list_path, name[2][0], name[2]) list_file = Utility.read_file_line_by_line(file_tar) for idx, l in enumerate(list_file): f = Utility.trim(l) if f == syl.name_index: # print '--------------------' preceeding = Utility.trim(list_file[idx - 1]) # print f succeeding = Utility.trim(list_file[idx + 1]) # print '--------------------' syl.set_preceeding_succeeding_name_index( preceeding, succeeding) # sys.exit() Utility.save_obj(syl_management, path)
def gen_dur_and_name_list_for_phone(syllable_label_path, name, phone_duration_path): pattern = re.compile(r""".+/A:.+/D:.+\-(?P<num_phone>.+)\+.+/E:.+""", re.VERBOSE) dur_list = [] for idx, line in enumerate( Utility.read_file_line_by_line(phone_duration_path)): spl = Utility.trim(line).split(' ') frame = int(spl[1]) - int(spl[0]) frame = frame / 50000 dur_list.append(frame) names = [] for idx, line in enumerate( Utility.read_file_line_by_line(syllable_label_path)): spl = Utility.trim(line).split(' ') match = re.match(pattern, line) if match: num_phone = match.group('num_phone') if num_phone == 'x': num_phone = 1 else: num_phone = int(num_phone) for phone_index in xrange(num_phone): names.append('{}_{}_{}'.format(name, (idx + 1), phone_index)) return (dur_list, names)
def fix_database(db_file, change_list_file, out_file): global db db = None db = Utility.load_obj(db_file) change_list = [] less_than = None for line in Utility.read_file_line_by_line(change_list_file): if 'tsc' in line: n = Utility.trim(line).replace(' ', '_') change_list.append(n) elif '<' in line: # print line less_than = line.split(' ')[1] pass # print change_list # print less_than if (len(change_list) == 0) | (less_than == None): raise 'Change list file false' new_list = change_stress(change_list, less_than) Utility.save_obj(new_list, out_file) pass
def add_stress_to_full(full_file, out_file, out_full_no_time_file, name): # print name pattern = re.compile(r"""(?P<start>.+)\s(?P<end>.+)\s.+ /A:.+\-(?P<cur_phone_position>.+)_.+\+.+ /B:.+ """,re.VERBOSE) count = 0 o = [] for line in Utility.read_file_line_by_line(full_file): match = re.match(pattern, line) if match: cur_phone_position = match.group('cur_phone_position') # print cur_phone_position if (cur_phone_position == '1') | (cur_phone_position == 'x'): count = count + 1 # syl_id = '{}_{}'.format(name, count) # print syl_id pre_syl = '{}_{}'.format(name, count-1) cur_syl = '{}_{}'.format(name, count) suc_syl = '{}_{}'.format(name, count+1) if pre_syl not in db: pre_stress = 'x' else: pre_stress = db[pre_syl]['stress'] if cur_phone_position == 'x': cur_stress = 'x' else : cur_stress = db[cur_syl]['stress'] if suc_syl not in db: suc_stress = 'x' else: suc_stress = db[suc_syl]['stress'] stress_context = '/I:{}-{}+{}'.format(pre_stress, cur_stress, suc_stress) # print stress_context context = '{}{}'.format(Utility.trim(line), stress_context) # print context o.append(context) o_no_time = get_remove_time(o) # print o_no_time if (not (len(Utility.read_file_line_by_line(full_file)) == len(o))) & (not (len(Utility.read_file_line_by_line(full_file)) == len(o_no_time))): print name Utility.write_to_file_line_by_line(out_file, o) Utility.write_to_file_line_by_line(out_full_no_time_file, o_no_time) pass
def read_rmse_file(rmse_file): param_old = [] rmse_old = 100000000.0 line_old = '' const = '' thefile = Utility.read_file_line_by_line(rmse_file) # print rmse_file for idx, line in enumerate(thefile): if 'const' in line: const = Utility.trim(line) l = thefile[idx + 1].split(' ') current_param = [l[0], l[1], l[2], l[3], l[4]] if 'Only' in line: spl = line.split(' ') rmse = float(spl[5]) if rmse < rmse_old: rmse_old = rmse param_old = current_param line_old = line return (rmse_old, param_old, line_old, const)
def load_fix_list(path): out = [] for f in Utility.list_file(path): if f.startswith('.'): continue for line in Utility.read_file_line_by_line('{}/{}'.format(path, f)): # print Utility.trim(line) out.append(Utility.trim(line)) return out pass
def find_config(name): for line in config_setting: spl = Utility.trim(line).split(' ') if name == spl[0]: print spl n_cluster = spl[1] n_neighbor = spl[2] return (n_cluster, n_neighbor)
def gen_dur_and_name_list(label_path, name): dur_list = [] names = [] for idx, line in enumerate(Utility.read_file_line_by_line(label_path)): spl = Utility.trim(line).split(' ') frame = int(spl[1]) - int(spl[0]) frame = frame / 50000 dur_list.append(frame) names.append('{}_{}'.format(name, (idx + 1))) return (dur_list, names)
def gen_all_method(base): print base mono = '{}/{}.lab'.format(mono_label, base) stress_tone = '{}/{}.npy'.format(stress_list, base) lf0_out = np.array([]) for line, (st, t) in zip(Utility.read_file_line_by_line(mono), np.load(stress_tone)): # print line, st, t l = Utility.trim(line) spl = l.split(' ') stress_type = 'unstress' if st == '1': stress_type = 'stress' if spl[2] in ['sil', 'pau']: target_file = original_file else: config = result_450['{}_{}'.format(stress_type, t)] if config['beta'] == '': target_file = original_file else: target_file = np.load( '{}/tone_{}/Alpha_1.0_Beta_{}_Gamma_{}/lf0/{}.npy'.format( config['basepath'], t, config['beta'], config['gamma'], base)) start = int(float(spl[0]) / 50000.0) end = int(float(spl[1]) / 50000.0) # print target_file.shape target_lf0 = target_file[start:end] lf0_out = np.append(lf0_out, target_lf0, axis=0) print lf0_out.shape np.save('{}/{}.npy'.format(outpath, base), lf0_out) # sys.exit() pass
def load_mono(mono_file): m = [] for line in Utility.read_file_line_by_line(mono_file): spl = line.split(' ') ph = Utility.trim(spl[2]) # print ph m.append(ph) # print m return m pass
def gen_json_data(): outpath = '/home/h1/decha/Dropbox/python_workspace/Inter_speech_2016/playground/generate_json/latent_data/' obj = Utility.load_obj( '/home/h1/decha/Dropbox/Inter_speech_2016/Syllable_object/mix_object/current_version/all_vowel_type/syllable_object_01234.pickle' ) start_set, end_set = 'a', 'j' base_path = '/home/h1/decha/Dropbox/python_workspace/Inter_speech_2016/playground/list_file_for_preceeding_suceeding/list_gpr_file/' for sett in Utility.char_range(start_set, end_set): set_path = '{}/{}/'.format(base_path, sett) for f in Utility.list_file(set_path): if f.startswith('.'): continue file_path = '{}/{}'.format(set_path, f) out_list = [] for line in Utility.read_file_line_by_line(file_path): name = Utility.trim(line) # "duration" "syllable_context" duration = '' syllable_context = '' d = dict() if name == 'sil': syllable_context = 'sil-sil-sil-x' duration = [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] elif name == 'pau': syllable_context = 'pau-pau-pau-x' duration = [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] else: syl = obj.get_syllable_by_name_index(name) syllable_context = '{}-{}-{}-{}'.format( syl.consonant, syl.vowel, syl.final_consonant, syl.tone) duration = syl.single_space_latent.tolist() d['duration'] = duration d['syllable_context'] = syllable_context out_list.append(d) outfile_path = '{}/tscsd{}.json'.format(outpath, f) Utility.save_json(outfile_path, out_list)
def plot_syllable(lab_path, lf0_path, out_set_path, name, plot_set_out): lines = Utility.read_file_line_by_line(lab_path) lf0 = Utility.read_lf0_into_ascii(lf0_path) print lf0 path = '{}/{}/'.format(out_set_path, name) Utility.make_directory(path) for idx, line in enumerate(lines): line = Utility.trim(line) spl = line.split(' ') # print spl start = float(spl[0]) / 50000 end = float(spl[1]) / 50000 syl = spl[2] # print start, end, syl if end > len(lf0): end = len(lf0) - 1 cur_lf0 = lf0[start:end] # print len(cur_lf0) o = '{}_{}'.format((idx + 1), syl) out_name = '{}/{}.lf0'.format(path, o) print out_name # Utility.write_to_file_line_by_line(out_name, cur_lf0) Utility.make_directory('{}/{}/'.format(plot_set_out, name)) plot_out_file_path = '{}/{}/{}.eps'.format(plot_set_out, name, o) plot(cur_lf0, plot_out_file_path) print len(lf0) pass
def run_gen_mono(utt_set): set_path = '{}/{}/'.format(utterance_path, utt_set) set_syllable_base_path = '{}/{}/'.format(syllable_base, utt_set) out_set_path = '{}/{}/'.format(output_path, utt_set) Utility.make_directory(out_set_path) for i in xrange(1, 51): utt_file = Utility.yaml_load('{}/tscsd{}{}.utt.yaml'.format( set_path, utt_set, Utility.fill_zero(i, 2))) # print utt_file out_file = '{}/tscsd{}{}.lab'.format(out_set_path, utt_set, Utility.fill_zero(i, 2)) stress_list = [] recursion(utt_file, stress_list) syllable_time_label = Utility.read_file_line_by_line( '{}/tscsd{}{}.lab'.format(set_syllable_base_path, utt_set, Utility.fill_zero(i, 2))) # print stress_list, len(stress_list) # print len(syllable_time_label) if len(syllable_time_label) != len(stress_list): print utt_set, i # print 'Error' # sys.exit() out = [] for idx, line in enumerate(syllable_time_label): # print line, stress_list[idx] o = '{}::{}'.format( Utility.trim(line).replace('-', '_').replace('+', '_'), stress_list[idx]) # print o out.append(o) Utility.write_to_file_line_by_line(out_file, out) # sys.exit() pass
def find_stress_info(stress_path, name): spl = name.split('_') sett = spl[0][5] syl_index = spl[1] filename = spl[0] # print sett, syl_index for s in Utility.list_file(stress_path): if s[0] == sett: file_path = '{}/{}/{}.lab'.format(stress_path, s, filename) # print file_path lines = Utility.read_file_line_by_line(file_path) ss = lines[int(syl_index) - 1].split(' ') return Utility.trim(ss[3]) break
import numpy import array if __name__ == '__main__': mono_path = '/work/w2/decha/Data/GPR_speccom_data/mono/tsc/sd/' mono_for_gpr_path = '/work/w2/decha/Data/GPR_speccom_data/mono_for_gpr_gen/tsc/sd/' for sett in Utility.char_range('a', 'z'): Utility.make_directory('{}/{}/'.format(mono_for_gpr_path, sett)) for i in range(1, 51): base = 'tscsd{}{}'.format(sett, Utility.fill_zero(i, 2)) mono = '{}/{}/{}.lab'.format(mono_path, sett, base) mono_for_gpr = '{}/{}/{}.npy'.format(mono_for_gpr_path, sett, base) out = [] for line in Utility.read_file_line_by_line(mono): l = Utility.trim(line).split(' ') out.append([(float(l[1]) - float(l[0])) / 10000000]) print out np.save(mono_for_gpr, np.array(out)) pass
target_file = 'tscsdj01' syllable_dct_dict = Utility.load_obj( '/work/w2/decha/Data/GPR_speccom_data/Interspeech2017/syllable_dct_with_delta_dictionary.pkl' ) label_file = '/work/w2/decha/Data/GPR_speccom_data/00_syllable_level_data/mono/tsc/sd/j/tscsdj01.lab' org = '/work/w21/decha/Interspeech_2017/GPR_data/450/param_align/lf0/param_mean/tscsdj01.npy' org = np.load(org) count = 1 for line in Utility.read_file_line_by_line(label_file): l = Utility.trim(line) spl = l.split(' ') dur = float(int(spl[1]) - int(spl[0])) / 50000 name = '{}_{}'.format(target_file, count) if name in syllable_dct_dict: dct = syllable_dct_dict[name][:, 0] print dct print dct[0:7] lf0 = idct(dct[1:15], norm='ortho') print lf0
# sys.path.append('/Users/dechamoungsri/Dropbox_dechamoungsri/Dropbox/python_workspace/Utility/') sys.path.append('/home/h1/decha/Dropbox/python_workspace/Utility/') from tool_box.util.utility import Utility log_file = '/home/h1/decha/Dropbox/python_workspace/Inter_speech_2016/playground/GP_LVM/vowel_type/Result_analysis/recogniton_result.txt' p = [] for line in Utility.read_file_line_by_line(log_file): if '01234' in line: continue if 'Label for scan' in line: p = [] p.append(Utility.trim(line)) elif 'Accuracy score :' in line: # print Utility.trim(line) if len(p) == 0: continue p.append(Utility.trim(line)) ppp = '' for pp in p: ppp += pp + ' | ' print ppp # /work/w13/decha/Inter_speech_2016_workplace/Tonal_projection/06_Tonal_part_projection_noise_reduction-250-iters-opt//vvvsg/input_dims_10/delta-True_delta-delta-False/BGP_LVM_Tone_2//LABEL_TYPE_STRESS.eps elif 'LABEL_TYPE_STRESS' in line:
for f in Utility.list_file(lab_path): if 'wav' in f: continue if f.startswith('.'): continue print f count = 0 for line in Utility.read_file_line_by_line('{}/{}'.format(lab_path, f)): count = count + 1 name = '{}_{}'.format(Utility.get_basefilename(f), count) spl = line.split(' ') stress = Utility.trim(spl[2].split('_')[5]) # print name, stress if stress not in ['0', '1', '2']: print name, 'Error' sys.exit() syl = dict() syl['stress'] = stress db_dict[name] = syl print len(db_dict) Utility.save_obj( db_dict,
def gen_mono(path, mono, mono_to_syl, mono_outfile, syl_outfile): dur = np.load(path) lab = Utility.read_file_line_by_line(mono) m_to_s = np.array(Utility.load_obj(mono_to_syl)) print dur.shape, m_to_s.shape # print m_to_s start = 0 end = 0 new_mono = [] new_syl = [] all_dur = 0.0 for d, line in zip(dur, lab): # print d, line l = Utility.trim(line) spl = l.split(' ') end = int(start + d * 10000000) o = '{} {} {}'.format(int(start), end, spl[2]) # print o new_mono.append(o) start = end all_dur = all_dur + d start_idx = 0 start = 0 end = 0 for syl in m_to_s: phs = len(syl) # print phs syl_dur = 0 for d in dur[start_idx:start_idx + phs]: syl_dur = syl_dur + d # print syl_dur end = int(start + syl_dur * 10000000) spl = Utility.trim(lab[start_idx]).split(' ') o_syl = '{} {} {}'.format(int(start), end, spl[2]) new_syl.append(o_syl) # print o_syl start = end start_idx = start_idx + phs print all_dur * 10000000 Utility.write_to_file_line_by_line(mono_outfile, new_mono) Utility.write_to_file_line_by_line(syl_outfile, new_syl) pass
'/work/w21/decha/Interspeech_2017/Product_of_Gaussian_2/29_run_keep_result/stress/log_Exp_2/log.0.450.23:23:28.235108.txt', '/work/w21/decha/Interspeech_2017/Product_of_Gaussian_2/29_run_keep_result/stress/log_Exp_2/log.1.450.00:00:14.972609.txt', '/work/w21/decha/Interspeech_2017/Product_of_Gaussian_2/29_run_keep_result/stress/log_Exp_2/log.2.450.00:38:23.486544.txt', '/work/w21/decha/Interspeech_2017/Product_of_Gaussian_2/29_run_keep_result/stress/log_Exp_2/log.3.450.01:15:50.907911.txt', '/work/w21/decha/Interspeech_2017/Product_of_Gaussian_2/29_run_keep_result/stress/log_Exp_2/log.4.450.02:10:26.865873.txt' ] for target_file in tars: opt = '' rmse = 10000000.0 cur_val = '' for line in Utility.read_file_line_by_line(target_file): if 'Beta' in line: cur_val = Utility.trim(line) if 'Only' in line: r = line.split(' ')[4] # print r if float(r) < float(rmse): # print rmse, r rmse = r opt = cur_val print opt print rmse pass
def match_process(target_file): dim = '' vvv = '' dur = '' name = '' data_detail = re.compile( r"""Vowel\s:\s(?P<vvv>.+),\sDelta\s:\s(?P<delta>.+),\sTone\s:\s(?P<tone>.+)""", re.VERBOSE) ann_out_path = re.compile( r"""ANN\soutpath\s.+input_dims_(?P<dims>.+)/delta-False_delta-delta-False/.+""", re.VERBOSE) number_of_training = re.compile( r"""Number\sof\straining\spatterns:\s\s(?P<num_trn>.+)""", re.VERBOSE) input_output_dims = re.compile( r"""Input\sand\soutput\sdimensions: \s(?P<input>.+)\s(?P<output>.+)""", re.VERBOSE) lines = Utility.read_file_line_by_line(target_file) for idx, line in enumerate(lines): if '------------Start-----------' in line: dim = '' vvv = '' dur = '' name = '' pass elif '------------End-----------' in line: pass elif re.match(data_detail, line): # print Utility.trim(line) match = re.match(data_detail, line) vvv = match.group('vvv') if 'No_duration' in vvv: dur = 'NoDur' vvv = vvv.split('No_duration_')[1] else: dur = 'Dur' pass elif re.match(ann_out_path, line): match = re.match(ann_out_path, line) dim = match.group('dims') # print dims name = '{}_{}_{}'.format(dim, vvv, dur) # print name pass elif re.match(number_of_training, line): match = re.match(number_of_training, line) num_trn = match.group('num_trn') o = 'Nunber of training data : {}'.format(num_trn) printing_info[name].append(o) elif re.match(input_output_dims, line): match = re.match(input_output_dims, line) inputs = match.group('input') outputs = match.group('output') # print 'Input dims : {}, Output dims : {}'.format(inputs, outputs) # printing_info[2] = 'Input dims : {}, Output dims : {}'.format(inputs, outputs) o = 'Input dims : {}, Output dims : {}'.format(inputs, outputs) printing_info[name].append(o) elif 'Plain' in line: for ii in range(idx + 1, len(lines)): if 'Accuracy : ' in lines[ii]: tar = lines[ii].split(' ') # plain_data.append( Utility.trim(tar[2]) ) real = 'real_{}_{}'.format(vvv, dur) out_dict[real].append(Utility.trim(tar[2])) break elif 'Latent' in line: for ii in range(idx + 1, len(lines)): if 'Accuracy : ' in lines[ii]: tar = lines[ii].split(' ') # latent_data.append( Utility.trim(tar[2]) ) out_dict[name].append(Utility.trim(tar[2])) break pass
print("%.2f" % (np.mean(out_result['real']) * 100)) for d in dims_run: print d for idx, d in enumerate(dims_run): # print 'N_Dim : {}, score : {}'.format(d, out_result[d]) # print out_result[d] # print out_result print("%.2f" % (out_result[d] * 100)) print '------------End-----------' out_result['real'] = [] for d in dims_run: out_result[d] = '' # break elif re.match(data_detail, line): # print Utility.trim(line) printing_info[0] = Utility.trim(line) pass elif re.match(ann_out_path, line): match = re.match(ann_out_path, line) dims = match.group('dims') vvv_type = match.group('vvv_type') # print dims, vvv_type if 'No_duration' in vvv_type: vvv_type = 'No' dims = '{}_{}'.format(vvv_type, dims) # print dims # print 'N dims : {}'.format(dims) # printing_info[0] = dims elif re.match(number_of_training, line): match = re.match(number_of_training, line) num_trn = match.group('num_trn')
def run_make_obj_for_an_utterance(full_path, dur_path, lf0_path): global syl_database pattern = re.compile( r"""(?P<start>.+)\s(?P<end>.+)\s(?P<consonant>.+)\-(?P<vowel>.+)\+(?P<finalconsonant>.+)/A:.+/B:.+\-(?P<tone>.+)\+.+/C:.+/E:.+""", re.VERBOSE) # print dur_path # print lf0_path dur_list = Utility.load_obj(dur_path) lf0_list = Utility.list_file(lf0_path) lf0_list = sort_dur_list_by_index(lf0_list) # print lf0_list for idx, line in enumerate(Utility.read_file_line_by_line(full_path)): c = idx + 1 # print line l = Utility.trim(line) match = re.match(pattern, line) if match: start = float(match.group('start')) end = float(match.group('end')) consonant = match.group('consonant') vowel = match.group('vowel') finalconsonant = match.group('finalconsonant') tone = match.group('tone') dur = dur_list[idx] lf0 = Utility.read_file_line_by_line('{}/{}'.format( lf0_path, lf0_list[idx])) lf0 = [float(i) for i in lf0] iden = '{}_{}'.format(Utility.get_basefilename(full_path), c) # print start, end, consonant, vowel, finalconsonant, tone, dur, lf0, iden syl = dict( ) #Syllable(iden, tone, consonant, vowel, finalconsonant, dur, lf0) syl['id'] = iden syl['tone'] = tone syl['consonant'] = consonant syl['vowel'] = vowel syl['finalconsonant'] = finalconsonant syl['dur'] = dur syl['raw_lf0'] = lf0 syl_database.append(syl) # sys.exit(0) else: raise TypeError("WTF") pass