def set_pre_suc():
tones = ['01234']
name_list_path = '/home/h1/decha/Dropbox/python_workspace/Inter_speech_2016/playground/list_file_for_preceeding_suceeding/list_gpr_file/'
for t in tones:
path = '/home/h1/decha/Dropbox/Inter_speech_2016/Syllable_object/mix_object/current_version/all_vowel_type/syllable_object_{}.pickle'.format(
t)
print path
syl_management = Utility.load_obj(path)
for syl in syl_management.syllables_list:
if 'manual' in syl.name_index: continue
name = syl.name_index.split('_')
file_tar = '{}/{}/{}.lab'.format(name_list_path, name[2][0],
name[2])
list_file = Utility.read_file_line_by_line(file_tar)
for idx, l in enumerate(list_file):
f = Utility.trim(l)
if f == syl.name_index:
# print '--------------------'
preceeding = Utility.trim(list_file[idx - 1])
# print f
succeeding = Utility.trim(list_file[idx + 1])
# print '--------------------'
syl.set_preceeding_succeeding_name_index(
preceeding, succeeding)
# sys.exit()
Utility.save_obj(syl_management, path)
def gen_dur_and_name_list_for_phone(syllable_label_path, name,
phone_duration_path):
pattern = re.compile(r""".+/A:.+/D:.+\-(?P<num_phone>.+)\+.+/E:.+""",
re.VERBOSE)
dur_list = []
for idx, line in enumerate(
Utility.read_file_line_by_line(phone_duration_path)):
spl = Utility.trim(line).split(' ')
frame = int(spl[1]) - int(spl[0])
frame = frame / 50000
dur_list.append(frame)
names = []
for idx, line in enumerate(
Utility.read_file_line_by_line(syllable_label_path)):
spl = Utility.trim(line).split(' ')
match = re.match(pattern, line)
if match:
num_phone = match.group('num_phone')
if num_phone == 'x':
num_phone = 1
else:
num_phone = int(num_phone)
for phone_index in xrange(num_phone):
names.append('{}_{}_{}'.format(name, (idx + 1), phone_index))
return (dur_list, names)
def fix_database(db_file, change_list_file, out_file):
global db
db = None
db = Utility.load_obj(db_file)
change_list = []
less_than = None
for line in Utility.read_file_line_by_line(change_list_file):
if 'tsc' in line:
n = Utility.trim(line).replace(' ', '_')
change_list.append(n)
elif '<' in line:
# print line
less_than = line.split(' ')[1]
pass
# print change_list
# print less_than
if (len(change_list) == 0) | (less_than == None):
raise 'Change list file false'
new_list = change_stress(change_list, less_than)
Utility.save_obj(new_list, out_file)
pass
def add_stress_to_full(full_file, out_file, out_full_no_time_file, name):
# print name
pattern = re.compile(r"""(?P<start>.+)\s(?P<end>.+)\s.+
/A:.+\-(?P<cur_phone_position>.+)_.+\+.+
/B:.+
""",re.VERBOSE)
count = 0
o = []
for line in Utility.read_file_line_by_line(full_file):
match = re.match(pattern, line)
if match:
cur_phone_position = match.group('cur_phone_position')
# print cur_phone_position
if (cur_phone_position == '1') | (cur_phone_position == 'x'):
count = count + 1
# syl_id = '{}_{}'.format(name, count)
# print syl_id
pre_syl = '{}_{}'.format(name, count-1)
cur_syl = '{}_{}'.format(name, count)
suc_syl = '{}_{}'.format(name, count+1)
if pre_syl not in db:
pre_stress = 'x'
else:
pre_stress = db[pre_syl]['stress']
if cur_phone_position == 'x':
cur_stress = 'x'
else :
cur_stress = db[cur_syl]['stress']
if suc_syl not in db:
suc_stress = 'x'
else:
suc_stress = db[suc_syl]['stress']
stress_context = '/I:{}-{}+{}'.format(pre_stress, cur_stress, suc_stress)
# print stress_context
context = '{}{}'.format(Utility.trim(line), stress_context)
# print context
o.append(context)
o_no_time = get_remove_time(o)
# print o_no_time
if (not (len(Utility.read_file_line_by_line(full_file)) == len(o))) & (not (len(Utility.read_file_line_by_line(full_file)) == len(o_no_time))):
print name
Utility.write_to_file_line_by_line(out_file, o)
Utility.write_to_file_line_by_line(out_full_no_time_file, o_no_time)
pass
def read_rmse_file(rmse_file):
param_old = []
rmse_old = 100000000.0
line_old = ''
const = ''
thefile = Utility.read_file_line_by_line(rmse_file)
# print rmse_file
for idx, line in enumerate(thefile):
if 'const' in line:
const = Utility.trim(line)
l = thefile[idx + 1].split(' ')
current_param = [l[0], l[1], l[2], l[3], l[4]]
if 'Only' in line:
spl = line.split(' ')
rmse = float(spl[5])
if rmse < rmse_old:
rmse_old = rmse
param_old = current_param
line_old = line
return (rmse_old, param_old, line_old, const)
def load_fix_list(path):
out = []
for f in Utility.list_file(path):
if f.startswith('.'): continue
for line in Utility.read_file_line_by_line('{}/{}'.format(path, f)):
# print Utility.trim(line)
out.append(Utility.trim(line))
return out
pass
def find_config(name):
for line in config_setting:
spl = Utility.trim(line).split(' ')
if name == spl[0]:
print spl
n_cluster = spl[1]
n_neighbor = spl[2]
return (n_cluster, n_neighbor)
def gen_dur_and_name_list(label_path, name):
dur_list = []
names = []
for idx, line in enumerate(Utility.read_file_line_by_line(label_path)):
spl = Utility.trim(line).split(' ')
frame = int(spl[1]) - int(spl[0])
frame = frame / 50000
dur_list.append(frame)
names.append('{}_{}'.format(name, (idx + 1)))
return (dur_list, names)
def gen_all_method(base):
print base
mono = '{}/{}.lab'.format(mono_label, base)
stress_tone = '{}/{}.npy'.format(stress_list, base)
lf0_out = np.array([])
for line, (st, t) in zip(Utility.read_file_line_by_line(mono),
np.load(stress_tone)):
# print line, st, t
l = Utility.trim(line)
spl = l.split(' ')
stress_type = 'unstress'
if st == '1':
stress_type = 'stress'
if spl[2] in ['sil', 'pau']:
target_file = original_file
else:
config = result_450['{}_{}'.format(stress_type, t)]
if config['beta'] == '':
target_file = original_file
else:
target_file = np.load(
'{}/tone_{}/Alpha_1.0_Beta_{}_Gamma_{}/lf0/{}.npy'.format(
config['basepath'], t, config['beta'], config['gamma'],
base))
start = int(float(spl[0]) / 50000.0)
end = int(float(spl[1]) / 50000.0)
# print target_file.shape
target_lf0 = target_file[start:end]
lf0_out = np.append(lf0_out, target_lf0, axis=0)
print lf0_out.shape
np.save('{}/{}.npy'.format(outpath, base), lf0_out)
# sys.exit()
pass
def load_mono(mono_file):
m = []
for line in Utility.read_file_line_by_line(mono_file):
spl = line.split(' ')
ph = Utility.trim(spl[2])
# print ph
m.append(ph)
# print m
return m
pass
def gen_json_data():
outpath = '/home/h1/decha/Dropbox/python_workspace/Inter_speech_2016/playground/generate_json/latent_data/'
obj = Utility.load_obj(
'/home/h1/decha/Dropbox/Inter_speech_2016/Syllable_object/mix_object/current_version/all_vowel_type/syllable_object_01234.pickle'
)
start_set, end_set = 'a', 'j'
base_path = '/home/h1/decha/Dropbox/python_workspace/Inter_speech_2016/playground/list_file_for_preceeding_suceeding/list_gpr_file/'
for sett in Utility.char_range(start_set, end_set):
set_path = '{}/{}/'.format(base_path, sett)
for f in Utility.list_file(set_path):
if f.startswith('.'): continue
file_path = '{}/{}'.format(set_path, f)
out_list = []
for line in Utility.read_file_line_by_line(file_path):
name = Utility.trim(line)
# "duration" "syllable_context"
duration = ''
syllable_context = ''
d = dict()
if name == 'sil':
syllable_context = 'sil-sil-sil-x'
duration = [
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
]
elif name == 'pau':
syllable_context = 'pau-pau-pau-x'
duration = [
0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
]
else:
syl = obj.get_syllable_by_name_index(name)
syllable_context = '{}-{}-{}-{}'.format(
syl.consonant, syl.vowel, syl.final_consonant,
syl.tone)
duration = syl.single_space_latent.tolist()
d['duration'] = duration
d['syllable_context'] = syllable_context
out_list.append(d)
outfile_path = '{}/tscsd{}.json'.format(outpath, f)
Utility.save_json(outfile_path, out_list)
def plot_syllable(lab_path, lf0_path, out_set_path, name, plot_set_out):
lines = Utility.read_file_line_by_line(lab_path)
lf0 = Utility.read_lf0_into_ascii(lf0_path)
print lf0
path = '{}/{}/'.format(out_set_path, name)
Utility.make_directory(path)
for idx, line in enumerate(lines):
line = Utility.trim(line)
spl = line.split(' ')
# print spl
start = float(spl[0]) / 50000
end = float(spl[1]) / 50000
syl = spl[2]
# print start, end, syl
if end > len(lf0):
end = len(lf0) - 1
cur_lf0 = lf0[start:end]
# print len(cur_lf0)
o = '{}_{}'.format((idx + 1), syl)
out_name = '{}/{}.lf0'.format(path, o)
print out_name
# Utility.write_to_file_line_by_line(out_name, cur_lf0)
Utility.make_directory('{}/{}/'.format(plot_set_out, name))
plot_out_file_path = '{}/{}/{}.eps'.format(plot_set_out, name, o)
plot(cur_lf0, plot_out_file_path)
print len(lf0)
pass
def run_gen_mono(utt_set):
set_path = '{}/{}/'.format(utterance_path, utt_set)
set_syllable_base_path = '{}/{}/'.format(syllable_base, utt_set)
out_set_path = '{}/{}/'.format(output_path, utt_set)
Utility.make_directory(out_set_path)
for i in xrange(1, 51):
utt_file = Utility.yaml_load('{}/tscsd{}{}.utt.yaml'.format(
set_path, utt_set, Utility.fill_zero(i, 2)))
# print utt_file
out_file = '{}/tscsd{}{}.lab'.format(out_set_path, utt_set,
Utility.fill_zero(i, 2))
stress_list = []
recursion(utt_file, stress_list)
syllable_time_label = Utility.read_file_line_by_line(
'{}/tscsd{}{}.lab'.format(set_syllable_base_path, utt_set,
Utility.fill_zero(i, 2)))
# print stress_list, len(stress_list)
# print len(syllable_time_label)
if len(syllable_time_label) != len(stress_list):
print utt_set, i
# print 'Error'
# sys.exit()
out = []
for idx, line in enumerate(syllable_time_label):
# print line, stress_list[idx]
o = '{}::{}'.format(
Utility.trim(line).replace('-', '_').replace('+', '_'),
stress_list[idx])
# print o
out.append(o)
Utility.write_to_file_line_by_line(out_file, out)
# sys.exit()
pass
def find_stress_info(stress_path, name):
spl = name.split('_')
sett = spl[0][5]
syl_index = spl[1]
filename = spl[0]
# print sett, syl_index
for s in Utility.list_file(stress_path):
if s[0] == sett:
file_path = '{}/{}/{}.lab'.format(stress_path, s, filename)
# print file_path
lines = Utility.read_file_line_by_line(file_path)
ss = lines[int(syl_index) - 1].split(' ')
return Utility.trim(ss[3])
break
import numpy
import array
if __name__ == '__main__':
mono_path = '/work/w2/decha/Data/GPR_speccom_data/mono/tsc/sd/'
mono_for_gpr_path = '/work/w2/decha/Data/GPR_speccom_data/mono_for_gpr_gen/tsc/sd/'
for sett in Utility.char_range('a', 'z'):
Utility.make_directory('{}/{}/'.format(mono_for_gpr_path, sett))
for i in range(1, 51):
base = 'tscsd{}{}'.format(sett, Utility.fill_zero(i, 2))
mono = '{}/{}/{}.lab'.format(mono_path, sett, base)
mono_for_gpr = '{}/{}/{}.npy'.format(mono_for_gpr_path, sett, base)
out = []
for line in Utility.read_file_line_by_line(mono):
l = Utility.trim(line).split(' ')
out.append([(float(l[1]) - float(l[0])) / 10000000])
print out
np.save(mono_for_gpr, np.array(out))
pass
target_file = 'tscsdj01'
syllable_dct_dict = Utility.load_obj(
'/work/w2/decha/Data/GPR_speccom_data/Interspeech2017/syllable_dct_with_delta_dictionary.pkl'
)
label_file = '/work/w2/decha/Data/GPR_speccom_data/00_syllable_level_data/mono/tsc/sd/j/tscsdj01.lab'
org = '/work/w21/decha/Interspeech_2017/GPR_data/450/param_align/lf0/param_mean/tscsdj01.npy'
org = np.load(org)
count = 1
for line in Utility.read_file_line_by_line(label_file):
l = Utility.trim(line)
spl = l.split(' ')
dur = float(int(spl[1]) - int(spl[0])) / 50000
name = '{}_{}'.format(target_file, count)
if name in syllable_dct_dict:
dct = syllable_dct_dict[name][:, 0]
print dct
print dct[0:7]
lf0 = idct(dct[1:15], norm='ortho')
print lf0
# sys.path.append('/Users/dechamoungsri/Dropbox_dechamoungsri/Dropbox/python_workspace/Utility/')
sys.path.append('/home/h1/decha/Dropbox/python_workspace/Utility/')
from tool_box.util.utility import Utility
log_file = '/home/h1/decha/Dropbox/python_workspace/Inter_speech_2016/playground/GP_LVM/vowel_type/Result_analysis/recogniton_result.txt'
p = []
for line in Utility.read_file_line_by_line(log_file):
if '01234' in line: continue
if 'Label for scan' in line:
p = []
p.append(Utility.trim(line))
elif 'Accuracy score :' in line:
# print Utility.trim(line)
if len(p) == 0:
continue
p.append(Utility.trim(line))
ppp = ''
for pp in p:
ppp += pp + ' | '
print ppp
# /work/w13/decha/Inter_speech_2016_workplace/Tonal_projection/06_Tonal_part_projection_noise_reduction-250-iters-opt//vvvsg/input_dims_10/delta-True_delta-delta-False/BGP_LVM_Tone_2//LABEL_TYPE_STRESS.eps
elif 'LABEL_TYPE_STRESS' in line:
for f in Utility.list_file(lab_path):
if 'wav' in f: continue
if f.startswith('.'): continue
print f
count = 0
for line in Utility.read_file_line_by_line('{}/{}'.format(lab_path,
f)):
count = count + 1
name = '{}_{}'.format(Utility.get_basefilename(f), count)
spl = line.split(' ')
stress = Utility.trim(spl[2].split('_')[5])
# print name, stress
if stress not in ['0', '1', '2']:
print name, 'Error'
sys.exit()
syl = dict()
syl['stress'] = stress
db_dict[name] = syl
print len(db_dict)
Utility.save_obj(
db_dict,
def gen_mono(path, mono, mono_to_syl, mono_outfile, syl_outfile):
dur = np.load(path)
lab = Utility.read_file_line_by_line(mono)
m_to_s = np.array(Utility.load_obj(mono_to_syl))
print dur.shape, m_to_s.shape
# print m_to_s
start = 0
end = 0
new_mono = []
new_syl = []
all_dur = 0.0
for d, line in zip(dur, lab):
# print d, line
l = Utility.trim(line)
spl = l.split(' ')
end = int(start + d * 10000000)
o = '{} {} {}'.format(int(start), end, spl[2])
# print o
new_mono.append(o)
start = end
all_dur = all_dur + d
start_idx = 0
start = 0
end = 0
for syl in m_to_s:
phs = len(syl)
# print phs
syl_dur = 0
for d in dur[start_idx:start_idx + phs]:
syl_dur = syl_dur + d
# print syl_dur
end = int(start + syl_dur * 10000000)
spl = Utility.trim(lab[start_idx]).split(' ')
o_syl = '{} {} {}'.format(int(start), end, spl[2])
new_syl.append(o_syl)
# print o_syl
start = end
start_idx = start_idx + phs
print all_dur * 10000000
Utility.write_to_file_line_by_line(mono_outfile, new_mono)
Utility.write_to_file_line_by_line(syl_outfile, new_syl)
pass
'/work/w21/decha/Interspeech_2017/Product_of_Gaussian_2/29_run_keep_result/stress/log_Exp_2/log.0.450.23:23:28.235108.txt',
'/work/w21/decha/Interspeech_2017/Product_of_Gaussian_2/29_run_keep_result/stress/log_Exp_2/log.1.450.00:00:14.972609.txt',
'/work/w21/decha/Interspeech_2017/Product_of_Gaussian_2/29_run_keep_result/stress/log_Exp_2/log.2.450.00:38:23.486544.txt',
'/work/w21/decha/Interspeech_2017/Product_of_Gaussian_2/29_run_keep_result/stress/log_Exp_2/log.3.450.01:15:50.907911.txt',
'/work/w21/decha/Interspeech_2017/Product_of_Gaussian_2/29_run_keep_result/stress/log_Exp_2/log.4.450.02:10:26.865873.txt'
]
for target_file in tars:
opt = ''
rmse = 10000000.0
cur_val = ''
for line in Utility.read_file_line_by_line(target_file):
if 'Beta' in line:
cur_val = Utility.trim(line)
if 'Only' in line:
r = line.split(' ')[4]
# print r
if float(r) < float(rmse):
# print rmse, r
rmse = r
opt = cur_val
print opt
print rmse
pass
def match_process(target_file):
dim = ''
vvv = ''
dur = ''
name = ''
data_detail = re.compile(
r"""Vowel\s:\s(?P<vvv>.+),\sDelta\s:\s(?P<delta>.+),\sTone\s:\s(?P<tone>.+)""",
re.VERBOSE)
ann_out_path = re.compile(
r"""ANN\soutpath\s.+input_dims_(?P<dims>.+)/delta-False_delta-delta-False/.+""",
re.VERBOSE)
number_of_training = re.compile(
r"""Number\sof\straining\spatterns:\s\s(?P<num_trn>.+)""", re.VERBOSE)
input_output_dims = re.compile(
r"""Input\sand\soutput\sdimensions: \s(?P<input>.+)\s(?P<output>.+)""",
re.VERBOSE)
lines = Utility.read_file_line_by_line(target_file)
for idx, line in enumerate(lines):
if '------------Start-----------' in line:
dim = ''
vvv = ''
dur = ''
name = ''
pass
elif '------------End-----------' in line:
pass
elif re.match(data_detail, line):
# print Utility.trim(line)
match = re.match(data_detail, line)
vvv = match.group('vvv')
if 'No_duration' in vvv:
dur = 'NoDur'
vvv = vvv.split('No_duration_')[1]
else:
dur = 'Dur'
pass
elif re.match(ann_out_path, line):
match = re.match(ann_out_path, line)
dim = match.group('dims')
# print dims
name = '{}_{}_{}'.format(dim, vvv, dur)
# print name
pass
elif re.match(number_of_training, line):
match = re.match(number_of_training, line)
num_trn = match.group('num_trn')
o = 'Nunber of training data : {}'.format(num_trn)
printing_info[name].append(o)
elif re.match(input_output_dims, line):
match = re.match(input_output_dims, line)
inputs = match.group('input')
outputs = match.group('output')
# print 'Input dims : {}, Output dims : {}'.format(inputs, outputs)
# printing_info[2] = 'Input dims : {}, Output dims : {}'.format(inputs, outputs)
o = 'Input dims : {}, Output dims : {}'.format(inputs, outputs)
printing_info[name].append(o)
elif 'Plain' in line:
for ii in range(idx + 1, len(lines)):
if 'Accuracy : ' in lines[ii]:
tar = lines[ii].split(' ')
# plain_data.append( Utility.trim(tar[2]) )
real = 'real_{}_{}'.format(vvv, dur)
out_dict[real].append(Utility.trim(tar[2]))
break
elif 'Latent' in line:
for ii in range(idx + 1, len(lines)):
if 'Accuracy : ' in lines[ii]:
tar = lines[ii].split(' ')
# latent_data.append( Utility.trim(tar[2]) )
out_dict[name].append(Utility.trim(tar[2]))
break
pass
print("%.2f" % (np.mean(out_result['real']) * 100))
for d in dims_run:
print d
for idx, d in enumerate(dims_run):
# print 'N_Dim : {}, score : {}'.format(d, out_result[d])
# print out_result[d]
# print out_result
print("%.2f" % (out_result[d] * 100))
print '------------End-----------'
out_result['real'] = []
for d in dims_run:
out_result[d] = ''
# break
elif re.match(data_detail, line):
# print Utility.trim(line)
printing_info[0] = Utility.trim(line)
pass
elif re.match(ann_out_path, line):
match = re.match(ann_out_path, line)
dims = match.group('dims')
vvv_type = match.group('vvv_type')
# print dims, vvv_type
if 'No_duration' in vvv_type:
vvv_type = 'No'
dims = '{}_{}'.format(vvv_type, dims)
# print dims
# print 'N dims : {}'.format(dims)
# printing_info[0] = dims
elif re.match(number_of_training, line):
match = re.match(number_of_training, line)
num_trn = match.group('num_trn')
def run_make_obj_for_an_utterance(full_path, dur_path, lf0_path):
global syl_database
pattern = re.compile(
r"""(?P<start>.+)\s(?P<end>.+)\s(?P<consonant>.+)\-(?P<vowel>.+)\+(?P<finalconsonant>.+)/A:.+/B:.+\-(?P<tone>.+)\+.+/C:.+/E:.+""",
re.VERBOSE)
# print dur_path
# print lf0_path
dur_list = Utility.load_obj(dur_path)
lf0_list = Utility.list_file(lf0_path)
lf0_list = sort_dur_list_by_index(lf0_list)
# print lf0_list
for idx, line in enumerate(Utility.read_file_line_by_line(full_path)):
c = idx + 1
# print line
l = Utility.trim(line)
match = re.match(pattern, line)
if match:
start = float(match.group('start'))
end = float(match.group('end'))
consonant = match.group('consonant')
vowel = match.group('vowel')
finalconsonant = match.group('finalconsonant')
tone = match.group('tone')
dur = dur_list[idx]
lf0 = Utility.read_file_line_by_line('{}/{}'.format(
lf0_path, lf0_list[idx]))
lf0 = [float(i) for i in lf0]
iden = '{}_{}'.format(Utility.get_basefilename(full_path), c)
# print start, end, consonant, vowel, finalconsonant, tone, dur, lf0, iden
syl = dict(
) #Syllable(iden, tone, consonant, vowel, finalconsonant, dur, lf0)
syl['id'] = iden
syl['tone'] = tone
syl['consonant'] = consonant
syl['vowel'] = vowel
syl['finalconsonant'] = finalconsonant
syl['dur'] = dur
syl['raw_lf0'] = lf0
syl_database.append(syl)
# sys.exit(0)
else:
raise TypeError("WTF")
pass
