def add_phonemes():
data_path = os.environ['FUEL_DATA_PATH']
data_path = os.path.join(data_path,'blizzard/')
save_name = "sp_blizzard_80h_phon.hdf5"
phon_file = "tbptt_blizzard_80h.hdf5"
data_file = "sp_blizzard_80h.hdf5"
save_path = os.path.join(data_path, save_name)
phon_path = os.path.join(data_path, phon_file)
data_path = os.path.join(data_path, data_file)
resulth5 = h5py.File(save_path, mode='w')
phonh5 = h5py.File(phon_path, mode = 'r')
datah5 = h5py.File(data_path, mode = 'r')
sp_h5 = resulth5.create_dataset(
'sp', (TOTAL_ROWS, 512, 257), dtype='float32')
f0_h5 = resulth5.create_dataset(
'f0', (TOTAL_ROWS, 512), dtype='float32')
phon_h5 = resulth5.create_dataset(
'phonemes', (TOTAL_ROWS, 512), dtype = 'int16')
f0_h5[:] = datah5['f0'][:]
phon_h5[:] = phonh5['phonemes'][:,::64]
n_times = 100
idx = chunkIt(range(TOTAL_ROWS), n_times)
for num_indx, indx in enumerate(idx):
print num_indx, 100
sp_h5[indx] = datah5['sp'][indx]
cont = TOTAL_ROWS
end_train = int(.9*cont)
end_valid = int(.95*cont)
end_test = cont
split_dict = {
'train': {'sp': (0, end_train),
'f0': (0, end_train),
'phonemes': (0, end_train)},
'valid': {'sp': (end_train, end_valid),
'f0': (end_train, end_valid),
'phonemes': (end_train, end_valid)},
'test': {'sp': (end_valid, end_test),
'f0': (end_valid, end_test),
'phonemes': (end_valid, end_test)}
}
resulth5.attrs['split'] = H5PYDataset.create_split_array(split_dict)
resulth5.flush()
resulth5.close()
phonh5.close()
datah5.close()
def add_phonemes():
data_path = os.environ["FUEL_DATA_PATH"]
data_path = os.path.join(data_path, "blizzard/")
save_name = "sp_blizzard_80h_phon.hdf5"
phon_file = "tbptt_blizzard_80h.hdf5"
data_file = "sp_blizzard_80h.hdf5"
save_path = os.path.join(data_path, save_name)
phon_path = os.path.join(data_path, phon_file)
data_path = os.path.join(data_path, data_file)
resulth5 = h5py.File(save_path, mode="w")
phonh5 = h5py.File(phon_path, mode="r")
datah5 = h5py.File(data_path, mode="r")
sp_h5 = resulth5.create_dataset("sp", (TOTAL_ROWS, 512, 257), dtype="float32")
f0_h5 = resulth5.create_dataset("f0", (TOTAL_ROWS, 512), dtype="float32")
phon_h5 = resulth5.create_dataset("phonemes", (TOTAL_ROWS, 512), dtype="int16")
f0_h5[:] = datah5["f0"][:]
phon_h5[:] = phonh5["phonemes"][:, ::64]
n_times = 100
idx = chunkIt(range(TOTAL_ROWS), n_times)
for num_indx, indx in enumerate(idx):
print num_indx, 100
sp_h5[indx] = datah5["sp"][indx]
cont = TOTAL_ROWS
end_train = int(0.9 * cont)
end_valid = int(0.95 * cont)
end_test = cont
split_dict = {
"train": {"sp": (0, end_train), "f0": (0, end_train), "phonemes": (0, end_train)},
"valid": {"sp": (end_train, end_valid), "f0": (end_train, end_valid), "phonemes": (end_train, end_valid)},
"test": {"sp": (end_valid, end_test), "f0": (end_valid, end_test), "phonemes": (end_valid, end_test)},
}
resulth5.attrs["split"] = H5PYDataset.create_split_array(split_dict)
resulth5.flush()
resulth5.close()
phonh5.close()
datah5.close()
def process_chunk(num_chunk):
# Total number of rows
TOTAL_ROWS = 105856
n_times = 50
n_process = 11 # 7 for briaree
files_per_batch = 25
num_files = n_process * n_times * files_per_batch
# total_chunks = numpy.ceil(TOTAL_ROWS/float(num_files))
const = num_files * num_chunk
indx_mp = range(num_files)
indx_mp = [x + const for x in indx_mp]
indx_mp = [x for x in indx_mp if x < TOTAL_ROWS]
num_files = len(indx_mp)
indx_mp = chunkIt(indx_mp, n_times)
indx_mp = [chunkIt(x, n_process) for x in indx_mp]
data_path = os.environ["FUEL_DATA_PATH"]
data_path = os.path.join(data_path, "blizzard/")
file_name = "tbptt_blizzard_80h.hdf5"
save_name = "chunk_{}.hdf5".format(num_chunk)
hdf5_path = os.path.join(data_path, file_name)
save_dir = os.environ["RESULTS_DIR"]
if "blizzard" not in save_dir:
save_dir = os.path.join(save_dir, "blizzard/")
save_path = os.path.join(save_dir, save_name)
resulth5 = h5py.File(save_path, mode="w")
h5file = h5py.File(hdf5_path, mode="r")
raw_data = h5file["features"]
# Prepare output file
# Hardcoded values
mgc_h5 = resulth5.create_dataset("mgc", (num_files, 512, 35), dtype="float32")
f0_h5 = resulth5.create_dataset("f0", (num_files, 512), dtype="float32")
def process_batch(q, x, i):
results = []
for n, f in enumerate(x):
if n % 10 == 0:
print ("Reading row %i of %i" % (n + 1, len(x)))
results.append(wav2mgcf0(f))
return q.put((i, results))
total_time = time.time()
cont = 0
for time_step in xrange(n_times):
print ("Time step %i" % (time_step))
q = Queue()
process_list = []
results_list = []
for i_process in xrange(n_process):
this_process = Process(target=process_batch, args=(q, raw_data[indx_mp[time_step][i_process]], i_process))
process_list.append(this_process)
process_list[i_process].start()
results_list = [q.get() for i in xrange(n_process)]
results_list = sorted(results_list, key=lambda x: x[0])
_, results_list = zip(*results_list)
results_list = [x for small_list in results_list for x in small_list]
# mgcc, f0 = zip(*results_list)
# Add to hdf5 file
for mgc, f0 in results_list:
mgc_h5[cont] = mgc
f0_h5[cont] = f0
cont += 1
print "total time: ", (time.time() - total_time) / 60.0
sys.stdout.flush()
resulth5.flush()
resulth5.close()
h5file.close()
def convert_to_spectrum():
# ipdb.set_trace()
data_path = os.environ["FUEL_DATA_PATH"]
data_path = os.path.join(data_path, "blizzard/")
data_name = "mgc_blizzard_80h.hdf5"
save_name = "sp_blizzard_80h.hdf5"
save_path = os.path.join(data_path, save_name)
resulth5 = h5py.File(save_path, mode="w")
sp_h5 = resulth5.create_dataset("sp", (TOTAL_ROWS, 512, 257), dtype="float32")
f0_h5 = resulth5.create_dataset("f0", (TOTAL_ROWS, 512), dtype="float32")
means = []
stds = []
cont = 0
hdf5_path = os.path.join(data_path, data_name)
h5file = h5py.File(hdf5_path, mode="r")
mgc = h5file["mgc"]
f0 = h5file["f0"]
# Check next line
f0_h5[:] = f0[:]
import time
alpha = 0.4
stage = 2
gamma = -1.0 / stage
frame_window = 512
# TOTAL_ROWS = 5000#138368
n_times = 100
n_process = 7 # marge
# files_per_batch = 100
# num_files = n_process*n_times*files_per_batch
# total_chunks = numpy.ceil(TOTAL_ROWS/float(num_files))
# const = num_files*num_chunk
indx_mp = range(TOTAL_ROWS)
indx_mp = chunkIt(indx_mp, n_times)
indx_mp = [chunkIt(x, n_process) for x in indx_mp]
def process_batch(q, x, i):
results = []
for n, f in enumerate(x):
if n % 10 == 0:
print ("Reading row %i of %i" % (n + 1, len(x)))
# Check if same results
f = f.astype("float64").copy(order="C")
results.append(numpy.apply_along_axis(SPTK.mgc2sp, 1, f, alpha, gamma, frame_window).real)
return q.put((i, results))
total_time = time.time()
cont = 0
for time_step in xrange(n_times):
print ("Time step %i" % (time_step))
q = Queue()
process_list = []
results_list = []
for i_process in xrange(n_process):
this_process = Process(target=process_batch, args=(q, mgc[indx_mp[time_step][i_process]], i_process))
process_list.append(this_process)
process_list[i_process].start()
results_list = [q.get() for i in xrange(n_process)]
results_list = sorted(results_list, key=lambda x: x[0])
_, results_list = zip(*results_list)
results_list = [x for small_list in results_list for x in small_list]
# Add to hdf5 file
for sp in results_list:
sp_h5[cont] = sp.astype("float32")
cont += 1
print "total time: ", (time.time() - total_time) / 60.0
sys.stdout.flush()
cont = TOTAL_ROWS
end_train = int(0.9 * cont)
end_valid = int(0.95 * cont)
end_test = cont
split_dict = {
"train": {"sp": (0, end_train), "f0": (0, end_train)},
"valid": {"sp": (end_train, end_valid), "f0": (end_train, end_valid)},
"test": {"sp": (end_valid, end_test), "f0": (end_valid, end_test)},
}
resulth5.attrs["split"] = H5PYDataset.create_split_array(split_dict)
resulth5.flush()
resulth5.close()
for n, f in enumerate(data_files):
if n % 10 == 0:
print("Reading file %i of %i" % (n + 1, len(data_files)))
try:
di = wavfile.read(f)[1]
if len(di.shape) > 1:
di = di[:, 0]
results.append(di)
except:
pass
return q.put((i, results))
n_times = 10
n_process = 8
indx_mp = chunkIt(file_list, n_times)
size_per_iteration = [len(x) for x in indx_mp]
indx_mp = [chunkIt(x, n_process) for x in indx_mp]
size_first_iteration = [len(x) for x in indx_mp[0]]
features = h5file.create_dataset(
'features', (len(file_list), ),
dtype=h5py.special_dtype(vlen=numpy.dtype('int16')))
phonemes = h5file.create_dataset(
'phonemes', (len(file_list), ),
dtype=h5py.special_dtype(vlen=numpy.dtype('int16')))
import ipdb
results = []
for n, f in enumerate(data_files):
if n % 10 == 0:
print("Reading file %i of %i" % (n+1, len(data_files)))
try:
di = wavfile.read(f)[1]
if len(di.shape) > 1:
di = di[:, 0]
results.append(di)
except:
pass
return q.put((i, results))
n_times = 10
n_process = 8
indx_mp = chunkIt(file_list, n_times)
size_per_iteration = [len(x) for x in indx_mp]
indx_mp = [chunkIt(x, n_process) for x in indx_mp]
size_first_iteration = [len(x) for x in indx_mp[0]]
features = h5file.create_dataset(
'features', (len(file_list),),
dtype=h5py.special_dtype(vlen=numpy.dtype('int16')))
phonemes = h5file.create_dataset(
'phonemes', (len(file_list),),
dtype=h5py.special_dtype(vlen=numpy.dtype('int16')))
def process_chunk(num_chunk):
#Total number of rows
TOTAL_ROWS = 105856
n_times = 50
n_process = 11 # 7 for briaree
files_per_batch = 25
num_files = n_process*n_times*files_per_batch
#total_chunks = numpy.ceil(TOTAL_ROWS/float(num_files))
const = num_files*num_chunk
indx_mp = range(num_files)
indx_mp = [x + const for x in indx_mp]
indx_mp = [x for x in indx_mp if x < TOTAL_ROWS]
num_files = len(indx_mp)
indx_mp = chunkIt(indx_mp, n_times)
indx_mp = [chunkIt(x, n_process) for x in indx_mp]
data_path = os.environ['FUEL_DATA_PATH']
data_path = os.path.join(data_path,'blizzard/')
file_name = "tbptt_blizzard_80h.hdf5"
save_name = "chunk_{}.hdf5".format(num_chunk)
hdf5_path = os.path.join(data_path, file_name)
save_dir = os.environ['RESULTS_DIR']
if 'blizzard' not in save_dir:
save_dir = os.path.join(save_dir,'blizzard/')
save_path = os.path.join(save_dir, save_name)
resulth5 = h5py.File(save_path, mode='w')
h5file = h5py.File(hdf5_path, mode='r')
raw_data = h5file['features']
# Prepare output file
#Hardcoded values
mgc_h5 = resulth5.create_dataset(
'mgc', (num_files, 512, 35), dtype='float32')
f0_h5 = resulth5.create_dataset(
'f0', (num_files, 512), dtype='float32')
def process_batch(q, x, i):
results = []
for n, f in enumerate(x):
if n % 10 == 0:
print("Reading row %i of %i" % (n+1, len(x)))
results.append(wav2mgcf0(f))
return q.put((i, results))
total_time = time.time()
cont = 0
for time_step in xrange(n_times):
print("Time step %i" % (time_step))
q = Queue()
process_list = []
results_list = []
for i_process in xrange(n_process):
this_process = Process(
target=process_batch,
args=(q, raw_data[indx_mp[time_step][i_process]], i_process))
process_list.append(this_process)
process_list[i_process].start()
results_list = [q.get() for i in xrange(n_process)]
results_list = sorted(results_list, key=lambda x: x[0])
_, results_list = zip(*results_list)
results_list = [x for small_list in results_list
for x in small_list]
#mgcc, f0 = zip(*results_list)
# Add to hdf5 file
for mgc, f0 in results_list:
mgc_h5[cont] = mgc
f0_h5[cont] = f0
cont += 1
print "total time: ", (time.time()-total_time)/60.
sys.stdout.flush()
resulth5.flush()
resulth5.close()
h5file.close()
def convert_to_spectrum():
#ipdb.set_trace()
data_path = os.environ['FUEL_DATA_PATH']
data_path = os.path.join(data_path,'blizzard/')
data_name = "mgc_blizzard_80h.hdf5"
save_name = "sp_blizzard_80h.hdf5"
save_path = os.path.join(data_path, save_name)
resulth5 = h5py.File(save_path, mode='w')
sp_h5 = resulth5.create_dataset(
'sp', (TOTAL_ROWS, 512, 257), dtype='float32')
f0_h5 = resulth5.create_dataset(
'f0', (TOTAL_ROWS, 512), dtype='float32')
means = []
stds = []
cont = 0
hdf5_path = os.path.join(data_path, data_name)
h5file = h5py.File(hdf5_path, mode='r')
mgc = h5file['mgc']
f0 = h5file['f0']
#Check next line
f0_h5[:] = f0[:]
import time
alpha = 0.4
stage = 2
gamma = -1.0 / stage
frame_window = 512
#TOTAL_ROWS = 5000#138368
n_times = 100
n_process = 7 # marge
#files_per_batch = 100
#num_files = n_process*n_times*files_per_batch
#total_chunks = numpy.ceil(TOTAL_ROWS/float(num_files))
#const = num_files*num_chunk
indx_mp = range(TOTAL_ROWS)
indx_mp = chunkIt(indx_mp, n_times)
indx_mp = [chunkIt(x, n_process) for x in indx_mp]
def process_batch(q, x, i):
results = []
for n, f in enumerate(x):
if n % 10 == 0:
print("Reading row %i of %i" % (n+1, len(x)))
# Check if same results
f = f.astype('float64').copy(order = 'C')
results.append(
numpy.apply_along_axis(SPTK.mgc2sp, 1, f, alpha, gamma, frame_window).real)
return q.put((i, results))
total_time = time.time()
cont = 0
for time_step in xrange(n_times):
print("Time step %i" % (time_step))
q = Queue()
process_list = []
results_list = []
for i_process in xrange(n_process):
this_process = Process(
target=process_batch,
args=(q, mgc[indx_mp[time_step][i_process]], i_process))
process_list.append(this_process)
process_list[i_process].start()
results_list = [q.get() for i in xrange(n_process)]
results_list = sorted(results_list, key=lambda x: x[0])
_, results_list = zip(*results_list)
results_list = [x for small_list in results_list
for x in small_list]
# Add to hdf5 file
for sp in results_list:
sp_h5[cont] = sp.astype('float32')
cont += 1
print "total time: ", (time.time()-total_time)/60.
sys.stdout.flush()
cont = TOTAL_ROWS
end_train = int(.9*cont)
end_valid = int(.95*cont)
end_test = cont
split_dict = {
'train': {'sp': (0, end_train),
'f0': (0, end_train)},
'valid': {'sp': (end_train, end_valid),
'f0': (end_train, end_valid)},
'test': {'sp': (end_valid, end_test),
'f0': (end_valid, end_test)}
}
resulth5.attrs['split'] = H5PYDataset.create_split_array(split_dict)
resulth5.flush()
resulth5.close()
def process_chunk(num_chunk):
#Total number of rows
TOTAL_ROWS = 138368
n_times = 800
n_process = 10 #14
files_per_batch = 20
num_files = n_process*n_times*files_per_batch
#total_chunks = numpy.ceil(TOTAL_ROWS/float(num_files))
const = num_files*num_chunk
indx_mp = range(num_files)
indx_mp = [x + const for x in indx_mp]
indx_mp = [x for x in indx_mp if x < TOTAL_ROWS]
num_files = len(indx_mp)
indx_mp = chunkIt(indx_mp, n_times)
indx_mp = [chunkIt(x, n_process) for x in indx_mp]
data_path = os.environ['FUEL_DATA_PATH']
data_path = os.path.join(data_path,'blizzard/')
file_name = "tbptt_blizzard.hdf5"
save_name = "chunk_{}.hdf5".format(num_chunk)
hdf5_path = os.path.join(data_path, file_name)
save_dir = os.environ['RESULTS_DIR']
if 'blizzard' not in save_dir:
save_dir = os.path.join(save_dir,'blizzard/')
save_path = os.path.join('/Tmp/sotelo', save_name)
resulth5 = h5py.File(save_path, mode='w')
h5file = h5py.File(hdf5_path, mode='r')
raw_data = h5file['features']
# Prepare output file
#Hardcoded values
amplitude = resulth5.create_dataset(
'amplitude', (num_files, 326, 401), dtype='float32')
phase = resulth5.create_dataset(
'phase', (num_files, 326, 401), dtype='float32')
def process_batch(q, x, i):
results = []
for n, f in enumerate(x):
if n % 1 == 0:
print("Reading row %i of %i" % (n+1, len(x)))
d = f.astype('float32') / (2 ** 15)
#Sample_rate = 16000
sample_rate = 16000
x_a = stft(d, sample_rate, 0.050, 0.025)[:,:401]
results.append([abs(x_a), numpy.angle(x_a)])
return q.put((i, results))
total_time = time.time()
cont = 0
for time_step in xrange(n_times):
print("Time step %i" % (time_step))
q = Queue()
process_list = []
results_list = []
for i_process in xrange(n_process):
this_process = Process(
target=process_batch,
args=(q, raw_data[indx_mp[time_step][i_process]], i_process))
process_list.append(this_process)
process_list[i_process].start()
print "Waiting for results..."
results_list = [q.get() for i in xrange(n_process)]
print "Results found!"
results_list = sorted(results_list, key=lambda x: x[0])
_, results_list = zip(*results_list)
results_list = [x for small_list in results_list
for x in small_list]
# Add to hdf5 file
# for spec in results_list:
# print "inputing data: ", cont
# amplitude[cont] = spec[0]
# phase[cont] = spec[1]
# cont += 1
len_res = len(results_list)
all_amplitude = numpy.array([x[0] for x in results_list])
all_phase = numpy.array([x[1] for x in results_list])
amplitude[cont:(cont+len_res)] = all_amplitude
phase[cont:(cont+len_res)] = all_phase
cont += len_res
print "total time: ", (time.time()-total_time)/60.
#sys.stdout.flush()
end_train = int(.9*cont)
end_valid = int(.95*cont)
end_test = cont
split_dict = {
'train': {'amplitude': (0, end_train),
'phase': (0, end_train)},
'valid': {'amplitude': (end_train, end_valid),
'phase': (end_train, end_valid)},
'test': {'amplitude': (end_valid, end_test),
'phase': (end_valid, end_test)}
}
resulth5.attrs['split'] = H5PYDataset.create_split_array(split_dict)
resulth5.flush()
resulth5.close()
def process_chunk(num_chunk):
#Total number of rows
TOTAL_ROWS = 36212
n_times = 30
n_process = 14 # 7 for briaree
files_per_batch = 10
num_files = n_process * n_times * files_per_batch
#total_chunks = numpy.ceil(TOTAL_ROWS/float(num_files))
const = num_files * num_chunk
indx_mp = range(num_files)
indx_mp = [x + const for x in indx_mp]
indx_mp = [x for x in indx_mp if x < TOTAL_ROWS]
all_indx = indx_mp
num_files = len(indx_mp)
indx_mp = chunkIt(indx_mp, n_times)
indx_mp = [chunkIt(x, n_process) for x in indx_mp]
data_path = os.environ['FUEL_DATA_PATH']
data_path = os.path.join(data_path, 'blizzard/')
file_name = "raw_blizzard_80h.hdf5"
save_name = "chunk_{}_sentence.hdf5".format(num_chunk)
hdf5_path = os.path.join(data_path, file_name)
save_dir = os.environ['RESULTS_DIR']
if 'blizzard' not in save_dir:
save_dir = os.path.join(save_dir, 'blizzard/')
save_path = os.path.join(save_dir, save_name)
resulth5 = h5py.File(save_path, mode='w')
h5file = h5py.File(hdf5_path, mode='r')
raw_data = h5file['features']
#Hardcoded values
mgc_h5 = resulth5.create_dataset(
'mgc', (num_files, ),
dtype=h5py.special_dtype(vlen=numpy.dtype('float32')))
mgc_shape_h5 = resulth5.create_dataset(
'mgc_shapes', (num_files, 2), dtype='int32')
mgc_h5.dims.create_scale(mgc_shape_h5, 'shapes')
mgc_h5.dims[0].attach_scale(mgc_shape_h5)
mgc_shape_labels = resulth5.create_dataset(
'mgc_shape_labels', (2, ), dtype='S7')
mgc_shape_labels[...] = [
'time_step'.encode('utf8'), 'mgc_frequency'.encode('utf8')
]
mgc_h5.dims.create_scale(mgc_shape_labels, 'shape_labels')
mgc_h5.dims[0].attach_scale(mgc_shape_labels)
sp_h5 = resulth5.create_dataset(
'spectrum', (num_files, ),
dtype=h5py.special_dtype(vlen=numpy.dtype('float32')))
sp_shape_h5 = resulth5.create_dataset(
'sp_shapes', (num_files, 2), dtype='int32')
sp_h5.dims.create_scale(sp_shape_h5, 'shapes')
sp_h5.dims[0].attach_scale(sp_shape_h5)
sp_shape_labels = resulth5.create_dataset(
'sp_shape_labels', (2, ), dtype='S7')
sp_shape_labels[...] = [
'time_step'.encode('utf8'), 'sp_frequency'.encode('utf8')
]
sp_h5.dims.create_scale(sp_shape_labels, 'shape_labels')
sp_h5.dims[0].attach_scale(sp_shape_labels)
vs_h5 = resulth5.create_dataset(
'voicing_str', (num_files, ),
dtype=h5py.special_dtype(vlen=numpy.dtype('float32')))
vs_shape_h5 = resulth5.create_dataset(
'vs_shapes', (num_files, 2), dtype='int32')
vs_h5.dims.create_scale(vs_shape_h5, 'shapes')
vs_h5.dims[0].attach_scale(vs_shape_h5)
vs_shape_labels = resulth5.create_dataset(
'vs_shape_labels', (2, ), dtype='S7')
vs_shape_labels[...] = [
'time_step'.encode('utf8'), 'vs_frequency'.encode('utf8')
]
vs_h5.dims.create_scale(vs_shape_labels, 'shape_labels')
vs_h5.dims[0].attach_scale(vs_shape_labels)
f0_h5 = resulth5.create_dataset(
'f0', (num_files, ),
dtype=h5py.special_dtype(vlen=numpy.dtype('float32')))
transcripts = resulth5.create_dataset(
'transcripts', (num_files, ),
dtype=h5py.special_dtype(vlen=numpy.dtype('int16')))
def process_batch(q, x, i):
results = []
for n, f in enumerate(x):
if n % 1 == 0:
print("Reading row %i of %i" % (n + 1, len(x)))
mgc, f0, vs = wav2mgcf0(f)
f = mgc.astype('float64').copy(order='C')
sp = numpy.apply_along_axis(SPTK.mgc2sp, 1, f, alpha, gamma,
frame_window).real
results.append((mgc, f0, vs, sp))
return q.put((i, results))
transcripts[...] = [h5file['transcripts'][indx] for indx in all_indx]
total_time = time.time()
cont = 0
for time_step in xrange(n_times):
print("Time step %i" % (time_step))
q = Queue()
process_list = []
results_list = []
for i_process in xrange(n_process):
this_process = Process(
target=process_batch,
args=(q, raw_data[indx_mp[time_step][i_process]], i_process))
process_list.append(this_process)
process_list[i_process].start()
results_list = [q.get() for i in xrange(n_process)]
results_list = sorted(results_list, key=lambda x: x[0])
_, results_list = zip(*results_list)
results_list = [x for small_list in results_list for x in small_list]
#mgcc, f0 = zip(*results_list)
# Add to hdf5 file
for mgc, f0, vs, sp in results_list:
mgc_h5[cont] = mgc.flatten()
sp_h5[cont] = sp.flatten()
vs_h5[cont] = vs.flatten()
mgc_shape_h5[cont] = numpy.array(mgc.shape)
sp_shape_h5[cont] = numpy.array(sp.shape)
vs_shape_h5[cont] = numpy.array(vs.shape)
f0_h5[cont] = f0
cont += 1
print "total time: ", (time.time() - total_time) / 60.
sys.stdout.flush()
split_dict = {
'all': {
'mgc': (0, num_files),
'f0': (0, num_files),
'transcripts': (0, num_files),
'spectrum': (0, num_files),
'voicing_str': (0, num_files)
}
}
resulth5.attrs['split'] = H5PYDataset.create_split_array(split_dict)
resulth5.flush()
resulth5.close()
h5file.close()
def process_chunk(num_chunk):
#Total number of rows
TOTAL_ROWS = 36212
n_times = 30
n_process = 14 # 7 for briaree
files_per_batch = 10
num_files = n_process*n_times*files_per_batch
#total_chunks = numpy.ceil(TOTAL_ROWS/float(num_files))
const = num_files*num_chunk
indx_mp = range(num_files)
indx_mp = [x + const for x in indx_mp]
indx_mp = [x for x in indx_mp if x < TOTAL_ROWS]
all_indx = indx_mp
num_files = len(indx_mp)
indx_mp = chunkIt(indx_mp, n_times)
indx_mp = [chunkIt(x, n_process) for x in indx_mp]
data_path = os.environ['FUEL_DATA_PATH']
data_path = os.path.join(data_path,'blizzard/')
file_name = "raw_blizzard_80h.hdf5"
save_name = "chunk_{}_sentence.hdf5".format(num_chunk)
hdf5_path = os.path.join(data_path, file_name)
save_dir = os.environ['RESULTS_DIR']
if 'blizzard' not in save_dir:
save_dir = os.path.join(save_dir,'blizzard/')
save_path = os.path.join(save_dir, save_name)
resulth5 = h5py.File(save_path, mode='w')
h5file = h5py.File(hdf5_path, mode='r')
raw_data = h5file['features']
#Hardcoded values
mgc_h5 = resulth5.create_dataset(
'mgc', (num_files,),
dtype=h5py.special_dtype(vlen=numpy.dtype('float32')))
mgc_shape_h5 = resulth5.create_dataset(
'mgc_shapes', (num_files,2), dtype = 'int32')
mgc_h5.dims.create_scale(mgc_shape_h5, 'shapes')
mgc_h5.dims[0].attach_scale(mgc_shape_h5)
mgc_shape_labels = resulth5.create_dataset(
'mgc_shape_labels', (2,), dtype='S7')
mgc_shape_labels[...] = [
'time_step'.encode('utf8'),
'mgc_frequency'.encode('utf8')]
mgc_h5.dims.create_scale(
mgc_shape_labels, 'shape_labels')
mgc_h5.dims[0].attach_scale(mgc_shape_labels)
sp_h5 = resulth5.create_dataset(
'spectrum', (num_files,),
dtype=h5py.special_dtype(vlen=numpy.dtype('float32')))
sp_shape_h5 = resulth5.create_dataset(
'sp_shapes', (num_files,2), dtype = 'int32')
sp_h5.dims.create_scale(sp_shape_h5, 'shapes')
sp_h5.dims[0].attach_scale(sp_shape_h5)
sp_shape_labels = resulth5.create_dataset(
'sp_shape_labels', (2,), dtype='S7')
sp_shape_labels[...] = [
'time_step'.encode('utf8'),
'sp_frequency'.encode('utf8')]
sp_h5.dims.create_scale(
sp_shape_labels, 'shape_labels')
sp_h5.dims[0].attach_scale(sp_shape_labels)
vs_h5 = resulth5.create_dataset(
'voicing_str', (num_files,),
dtype=h5py.special_dtype(vlen=numpy.dtype('float32')))
vs_shape_h5 = resulth5.create_dataset(
'vs_shapes', (num_files,2), dtype = 'int32')
vs_h5.dims.create_scale(vs_shape_h5, 'shapes')
vs_h5.dims[0].attach_scale(vs_shape_h5)
vs_shape_labels = resulth5.create_dataset(
'vs_shape_labels', (2,), dtype='S7')
vs_shape_labels[...] = [
'time_step'.encode('utf8'),
'vs_frequency'.encode('utf8')]
vs_h5.dims.create_scale(
vs_shape_labels, 'shape_labels')
vs_h5.dims[0].attach_scale(vs_shape_labels)
f0_h5 = resulth5.create_dataset(
'f0', (num_files, ),
dtype=h5py.special_dtype(vlen=numpy.dtype('float32')))
transcripts = resulth5.create_dataset(
'transcripts', (num_files,),
dtype=h5py.special_dtype(vlen=numpy.dtype('int16')))
def process_batch(q, x, i):
results = []
for n, f in enumerate(x):
if n % 1 == 0:
print("Reading row %i of %i" % (n+1, len(x)))
mgc, f0, vs = wav2mgcf0(f)
f = mgc.astype('float64').copy(order = 'C')
sp = numpy.apply_along_axis(SPTK.mgc2sp, 1, f, alpha, gamma, frame_window).real
results.append((mgc, f0, vs, sp))
return q.put((i, results))
transcripts[...] = [h5file['transcripts'][indx] for indx in all_indx]
total_time = time.time()
cont = 0
for time_step in xrange(n_times):
print("Time step %i" % (time_step))
q = Queue()
process_list = []
results_list = []
for i_process in xrange(n_process):
this_process = Process(
target=process_batch,
args=(q, raw_data[indx_mp[time_step][i_process]], i_process))
process_list.append(this_process)
process_list[i_process].start()
results_list = [q.get() for i in xrange(n_process)]
results_list = sorted(results_list, key=lambda x: x[0])
_, results_list = zip(*results_list)
results_list = [x for small_list in results_list
for x in small_list]
#mgcc, f0 = zip(*results_list)
# Add to hdf5 file
for mgc, f0, vs, sp in results_list:
mgc_h5[cont] = mgc.flatten()
sp_h5[cont] = sp.flatten()
vs_h5[cont] = vs.flatten()
mgc_shape_h5[cont] = numpy.array(mgc.shape)
sp_shape_h5[cont] = numpy.array(sp.shape)
vs_shape_h5[cont] = numpy.array(vs.shape)
f0_h5[cont] = f0
cont += 1
print "total time: ", (time.time()-total_time)/60.
sys.stdout.flush()
split_dict = {
'all': {'mgc': (0, num_files),
'f0': (0, num_files),
'transcripts': (0, num_files),
'spectrum': (0, num_files),
'voicing_str': (0, num_files)}
}
resulth5.attrs['split'] = H5PYDataset.create_split_array(split_dict)
resulth5.flush()
resulth5.close()
h5file.close()
def process_chunk(num_chunk):
#Total number of rows
TOTAL_ROWS = 138368
n_times = 800
n_process = 10 #14
files_per_batch = 20
num_files = n_process * n_times * files_per_batch
#total_chunks = numpy.ceil(TOTAL_ROWS/float(num_files))
const = num_files * num_chunk
indx_mp = range(num_files)
indx_mp = [x + const for x in indx_mp]
indx_mp = [x for x in indx_mp if x < TOTAL_ROWS]
num_files = len(indx_mp)
indx_mp = chunkIt(indx_mp, n_times)
indx_mp = [chunkIt(x, n_process) for x in indx_mp]
data_path = os.environ['FUEL_DATA_PATH']
data_path = os.path.join(data_path, 'blizzard/')
file_name = "tbptt_blizzard.hdf5"
save_name = "chunk_{}.hdf5".format(num_chunk)
hdf5_path = os.path.join(data_path, file_name)
save_dir = os.environ['RESULTS_DIR']
if 'blizzard' not in save_dir:
save_dir = os.path.join(save_dir, 'blizzard/')
save_path = os.path.join('/Tmp/sotelo', save_name)
resulth5 = h5py.File(save_path, mode='w')
h5file = h5py.File(hdf5_path, mode='r')
raw_data = h5file['features']
# Prepare output file
#Hardcoded values
amplitude = resulth5.create_dataset('amplitude', (num_files, 326, 401),
dtype='float32')
phase = resulth5.create_dataset('phase', (num_files, 326, 401),
dtype='float32')
def process_batch(q, x, i):
results = []
for n, f in enumerate(x):
if n % 1 == 0:
print("Reading row %i of %i" % (n + 1, len(x)))
d = f.astype('float32') / (2**15)
#Sample_rate = 16000
sample_rate = 16000
x_a = stft(d, sample_rate, 0.050, 0.025)[:, :401]
results.append([abs(x_a), numpy.angle(x_a)])
return q.put((i, results))
total_time = time.time()
cont = 0
for time_step in xrange(n_times):
print("Time step %i" % (time_step))
q = Queue()
process_list = []
results_list = []
for i_process in xrange(n_process):
this_process = Process(
target=process_batch,
args=(q, raw_data[indx_mp[time_step][i_process]], i_process))
process_list.append(this_process)
process_list[i_process].start()
print "Waiting for results..."
results_list = [q.get() for i in xrange(n_process)]
print "Results found!"
results_list = sorted(results_list, key=lambda x: x[0])
_, results_list = zip(*results_list)
results_list = [x for small_list in results_list for x in small_list]
# Add to hdf5 file
# for spec in results_list:
# print "inputing data: ", cont
# amplitude[cont] = spec[0]
# phase[cont] = spec[1]
# cont += 1
len_res = len(results_list)
all_amplitude = numpy.array([x[0] for x in results_list])
all_phase = numpy.array([x[1] for x in results_list])
amplitude[cont:(cont + len_res)] = all_amplitude
phase[cont:(cont + len_res)] = all_phase
cont += len_res
print "total time: ", (time.time() - total_time) / 60.
#sys.stdout.flush()
end_train = int(.9 * cont)
end_valid = int(.95 * cont)
end_test = cont
split_dict = {
'train': {
'amplitude': (0, end_train),
'phase': (0, end_train)
},
'valid': {
'amplitude': (end_train, end_valid),
'phase': (end_train, end_valid)
},
'test': {
'amplitude': (end_valid, end_test),
'phase': (end_valid, end_test)
}
}
resulth5.attrs['split'] = H5PYDataset.create_split_array(split_dict)
resulth5.flush()
resulth5.close()
