def main():
"""Main entry function
"""
global args
# Load configuration
config = load(args.configuration, Loader=Loader)
ignored = []
if "IgnoredID" in config:
ignored = config["IgnoredID"]
# Loading corpus
corpus = roots.Corpus(args.corpus)
# Convert duration to labels
q = JoinableQueue()
processes = []
for base in range(args.nb_proc):
t = UtteranceToLabel(corpus, args.output_dir, q, config)
t.start()
processes.append(t)
# Fill the queue for the workers
for i in range(0, corpus.count_utterances()):
if i not in ignored:
q.put(i)
# Fill the queue by adding a None to indicate the end
for i in range(len(processes)):
q.put(None)
# Wait the end of the processes
for t in processes:
t.join()
def main():
"""Main entry function
"""
global args
corpus = roots.Corpus(args.corpus)
# Convert duration to labels
q = queue.Queue()
threads = []
for base in range(args.nb_proc):
t = WavExtraction(q)
t.start()
threads.append(t)
for i in range(0, corpus.count_utterances()):
utt = [i, corpus, args.output_dir]
q.put(utt)
# block until all tasks are done
q.join()
# stop workers
for i in range(len(threads)):
q.put(None)
for t in threads:
t.join()
def main():
args = build_args()
datadir = args.datadir[0]
filelist = args.filelist[0]
corpusFileList = get_file_list(datadir, filelist)
for rootsFilename in corpusFileList:
corpus = roots.Corpus()
corpus.load(rootsFilename)
nutts = corpus.utterance_count()
utts = corpus.get_utterances(0, nutts)
def main():
args = build_args()
rootsFilename = args.corpus[0]
corpus = roots.Corpus()
corpus.load(rootsFilename)
w2vmodel = Word2Vec.load(args.w2v_model_name[0])
seq_name = args.seqname
if not os.path.exists(args.outdirname[0]):
os.mkdir(args.outdirname[0])
nbutts = corpus.count_utterances()
utts = corpus.get_utterances(0, nbutts)
for iutt, utt in enumerate(utts):
words = utt.get_sequence(seq_name).as_word_sequence().get_all_items()
words = [
word for word in words if len(word.get_related_items('Phone')) > 0
]
sig_item = utt.get_sequence('Signal').get_item(
0).as_acoustic_SignalSegment()
uttFilename = sig_item.get_file_name()
baseFilename = os.path.splitext(os.path.basename(uttFilename))[0]
speaker_id = baseFilename.split('-')[0]
if speaker_id in spkear_id_mapping.keys():
speaker_name = spkear_id_mapping[speaker_id]
uttFilename = baseFilename.replace(speaker_id + '-0001-',
speaker_name + '_arctic_a')
print(uttFilename)
words_list = []
prev_seg_end = 0
for word in words:
word_dict = OrderedDict()
get_embedding_wordvect(word, w2vmodel, word_dict)
if int(word_dict['begSeg']) != prev_seg_end:
silence_dict = OrderedDict()
add_silence(prev_seg_end, word_dict['begSeg'], silence_dict)
words_list.append(silence_dict)
print(word_dict['begSeg'], word_dict['endSeg'], word_dict['word'])
prev_seg_end = word_dict['endSeg']
words_list.append(word_dict)
end_utt = int(sig_item.get_segment_duration() / 1e-7)
if end_utt > prev_seg_end:
silence_dict = OrderedDict()
add_silence(prev_seg_end, end_utt, silence_dict)
words_list.append(silence_dict)
outFilename = os.path.join(args.outdirname[0], uttFilename)
pandas.DataFrame(words_list).to_csv(outFilename + '.csv')
def main():
# Build the feature extractor
args=build_arg_parser().parse_args()
rootsFilename=args.in_corpus[0]
print("Loading corpus {}...".format(args.in_corpus[0]))
corpus = roots.Corpus()
corpus.load(args.in_corpus[0])
nbutts = corpus.count_utterances()
utts = corpus.get_utterances(0, nbutts)
print("Generate data...")
out_dir_name=args.out_data[0]
seq_name_dict={
'disc':args.disc_seq,
'bgr':args.phr_seq,
'bgr_f0':args.phr_f0_seq,
'bgr_rate':args.phr_rate_seq,
'wrd':args.wrd_seq,
'pos':args.pos_seq,
'syl':args.syl_seq,
'pho':args.pho_seq,
'seg':args.seg_seq,
'pho_dur':args.pho_dur_seq,
'pho_energy':args.pho_energy_seq,
'f0_seq':args.f0_seq
}
baseFilename=os.path.basename(rootsFilename).split('.')[0]
outFilename=os.path.join(out_dir_name,'{}.csv'.format(baseFilename))
brg_data_list=[]
for iutt,utt in enumerate(utts):
if utt.is_valid_sequence(args.sign_seq):
# audioFn=utt.get_sequence(args.sign_seq).get_item(0).as_acoustic_SignalSegment().get_file_name()
# speaker_name=os.path.basename(audioFn).split('_')[0]
for bg in get_properties(utt,speaker_name,seq_name_dict):
brg_data_list.append(bg)
pd.DataFrame(brg_data_list).to_csv(outFilename)
def write_wav_text_data(dirname, roots_file, out_file):
fo = open(out_file, 'a')
#outputIpaAlphabet = roots.phonology_ipa_Ipa()
#basename = dirname + '/' + roots_file
basename = dirname + roots_file
corpus = roots.Corpus(basename)
nutts = corpus.count_utterances()
#basename, ext = os.path.splitext(roots_file)
basename_ = dirname + '/'
#dirname, filename = os.path.splitext(roots_file)
for utt_index in range(0, nutts): #niveau paragraphe ?
try:
utt = corpus.get_utterance(utt_index)
except:
continue
signal_sequence = utt.get_sequence('Signal').as_segment_sequence()
signal_item = signal_sequence.get_item(0).as_acoustic_SignalSegment()
try:
fs, signal_full = wavfile.read(basename_ +
signal_item.to_string(1))
except:
continue
#sentence_seq = utt.get_sequence('Character Label')
#if len(sentence_seq.to_string()) == 0:
#continue
#print(seq_sentence.to_string())
#word_JTrans_seq = utt.get_sequence('Word JTrans')
#print(word_JTrans_seq.to_string())
#word_item0 = word_JTrans_seq.get_item(0)
#start_time0, end_time0 = extract_start_end_time(word_item0, 'Time Segment JTrans')
#num_sentence = 0
#text1 = word_item0.to_string()wavs/
word_raw_seq = utt.get_sequence('Word Raw')
#print(word_raw_seq.to_string())
word_raw_nb = word_raw_seq.count()
text2 = []
raw = ''
for k in range(word_raw_nb):
word_raw_string = word_raw_seq.get_item(k).to_string()
#print(':::', word_raw_string)
#raw = raw + ' ' + word_raw_string
if word_raw_string == '':
continue
if k < word_raw_nb - 1:
if ((word_raw_string[-1] == '.') | ('.]' in word_raw_string) |
('],' in word_raw_string) | ('[[' in word_raw_string) |
('?' in word_raw_string) | (word_raw_string == '...') |
('...' in word_raw_string) | ('…' in word_raw_string) |
('.)' in word_raw_string) |
('!...' in word_raw_string)) & (
(word_raw_string != 'M.') &
(word_raw_string != '?...') &
('.]...' not in word_raw_string) &
(word_raw_string != '?…')):
#print("(word_raw_string)",word_raw_string)
raw = raw + ' ' + word_raw_string
text2.append(raw)
#print("# (text2):",text2)
raw = ''
elif (word_raw_string == '?...') | (word_raw_string == '?…'):
raw = raw + ' ?'
text2.append(raw)
raw = '...'
text2.append(raw)
raw = ''
elif ']...' in word_raw_string:
raw = raw + ']'
text2.append(raw)
raw = '...'
text2.append(raw)
raw = ''
elif word_raw_string == '[Note':
if raw != '': text2.append(raw)
raw = word_raw_string
elif ('note.]' in word_raw_string) | ('note]'
in word_raw_string):
raw = raw + ' ' + word_raw_string
text2.append(raw)
raw = ''
elif word_raw_string == 'fusil.Il':
raw = raw + ' fusil.'
text2.append(raw)
raw = 'Il'
elif word_raw_string == 'roi.Les':
raw = raw + ' roi.'
text2.append(raw)
raw = 'Les'
#elif word_raw_string == 'Introduction.)':
#raw = raw + ' ' + word_raw_string
#text2.append(raw)
#raw = ''
elif word_raw_string == "Provinces[Note":
raw = raw + ' Provinces'
text2.append(raw)
raw = '[Note'
elif word_raw_string == "four[Note":
raw = raw + ' four'
text2.append(raw)
raw = '[Note'
#elif (raw == " Je le conçois, les apparences ont pu") & (word_raw_string == 'te'):
#del(text2[-1])
#raw = '– Non... ' + raw
else:
raw = raw + ' ' + word_raw_string
else:
raw = raw + ' ' + word_raw_string
text2.append(raw)
print(word_raw_seq.to_string())
#print(text2)
#sys.exit()
word_seq = utt.get_sequence('Word JTrans')
word_item0 = word_seq.get_item(0)
phone_items0 = word_item0.get_related_items('Phone JTrans')
start_time0, end_time0 = extract_start_end_time(
word_item0, 'Time Segment JTrans')
num_sentence = 0
text1 = word_item0.to_string()
phone_seq_string = ' '.join(
[phone_item.to_string() for phone_item in phone_items0])
text3 = phone_seq_string
word_item_nb = word_seq.count()
for word_index in range(1, word_item_nb):
word_item = word_seq.get_item(word_index)
word_string = word_item.to_string()
phone_items = word_item.get_related_items('Phone JTrans')
phone_item = ' '.join([phon.to_string() for phon in phone_items])
#print(word_string, text1)
#text1 = text1 + ' ' + word_string
#if text1 == " la nuit était noire , le quartier .":
#continue
#if text1 == "ferme l' châssis mon enfant , dit la mère Morlaix , après un instant de .":
#continue
#if text1 == "le gardien était en effet pensif et .":
#continue
#if text1 == "Ézéchiel restait .":
#continue
#if text1 == " la porte se referma , et la maison redevint .":
#continue
#if text1 == "il y eut encore un .":
#continue
#if text1 == "demanda René de Kervoz après un long .":
#continue
#print('--', text1, ':', word_string)
if text1 + word_string == '.':
continue
if word_index < word_item_nb - 1:
#print(word_index, '/', word_item_nb)
word_next = word_seq.get_item(word_index + 1).to_string()
#print(word_next)
if ('silenc' in word_next):
continue
if (word_string in ['.', '?', '...', '?...']):
text1 = text1 + ' ' + word_string
text3 = text3 + '§' + phone_item
start_time1, end_time1 = extract_start_end_time(
word_item, 'Time Segment JTrans')
signal_sentence_values = signal_full[int(start_time0 *
fs):int(end_time1 *
fs)]
#print('1 (JTrans):',text1)
#print('2 (Raw):',text2[num_sentence])
wavname = 'SynPaFlex_' + roots_file[0:-16] + '_' + str(
utt_index) + '_' + str(num_sentence) + '.wav'
fo.write(wavname + '|' + text1 + '|' + text2[num_sentence] +
'|' + text3 + '\n')
wavfile.write(
"/lium/raid01_b/tgranjon/synpaflex/wavs/" + wavname, fs,
signal_sentence_values) # | ('Silenc' in word_next):
start_time0 = end_time1
num_sentence += 1
text1 = ''
text3 = ''
elif (word_string == 'Note') & (word_next == ':'):
start_time1, end_time1 = extract_start_end_time(
word_item, 'Time Segment JTrans')
signal_sentence_values = signal_full[int(start_time0 *
fs):int(end_time1 *
fs)]
#print('1:', text1)
#print('2:', text2[num_sentence])
wavname = 'SynPaFlex_' + roots_file[0:-16] + '_' + str(
utt_index) + '_' + str(num_sentence) + '.wav'
fo.write(wavname + '|' + text1 + '|' + text2[num_sentence] +
'|' + text3 + '\n')
wavfile.write(
"/lium/raid01_b/tgranjon/synpaflex/wavs/" + wavname, fs,
signal_sentence_values)
start_time0 = end_time1
num_sentence += 1
text1 = 'Note'
text3 = 'n O t @'
elif (text1 == ' fin de la') & ((word_string == 'note') |
(word_string == 'note.')):
text1 = text1 + ' ' + word_string
text3 = text3 + '§' + phone_item
start_time1, end_time1 = extract_start_end_time(
word_item, 'Time Segment JTrans')
signal_sentence_values = signal_full[int(start_time0 *
fs):int(end_time1 *
fs)]
#print('1:', text1)
#print('2:', text2[num_sentence])
wavname = 'SynPaFlex_' + roots_file[0:-16] + '_' + str(
utt_index) + '_' + str(num_sentence) + '.wav'
fo.write(wavname + '|' + text1 + '|' + text2[num_sentence] +
'|' + text3 + '\n')
wavfile.write(
"/lium/raid01_b/tgranjon/synpaflex/wavs/" + wavname, fs,
signal_sentence_values)
start_time0 = end_time1
num_sentence += 1
text1 = ''
text3 = ''
elif word_index == word_item_nb - 1:
start_time1, end_time1 = extract_start_end_time(
word_item, 'Time Segment JTrans')
signal_sentence_values = signal_full[int(start_time0 *
fs):int(end_time1 *
fs)]
#print('1:', text1)
#print('2:', text2[num_sentence])
wavname = 'SynPaFlex_' + roots_file[0:-16] + '_' + str(
utt_index) + '_' + str(num_sentence) + '.wav'
fo.write(wavname + '|' + text1 + '|' + text2[num_sentence] +
'|' + text3 + '\n')
wavfile.write(
"/lium/raid01_b/tgranjon/synpaflex/wavs/" + wavname, fs,
signal_sentence_values)
elif (
word_string == 'phrantsesos'
) & (text1 ==
" à Chalcis j' ai eu l' honneur d' être annoncé comme un milordos"
):
text1 = text1 + ' ' + word_string
text3 = text3 + '§' + phone_item
start_time1, end_time1 = extract_start_end_time(
word_item, 'Time Segment JTrans')
signal_sentence_values = signal_full[int(start_time0 *
fs):int(end_time1 *
fs)]
#print('1:', text1)
#print('2:', text2[num_sentence])
wavname = 'SynPaFlex_' + roots_file[0:-16] + '_' + str(
utt_index) + '_' + str(num_sentence) + '.wav'
fo.write(wavname + '|' + text1 + '|' + text2[num_sentence] +
'|' + text3 + '\n')
wavfile.write(
"/lium/raid01_b/tgranjon/synpaflex/wavs/" + wavname, fs,
signal_sentence_values)
start_time0 = end_time1
num_sentence += 1
text1 = ''
text3 = ''
elif (word_string
== 'camarades') & (text1 == "Note :") & (word_next == ','):
text1 = text1 + ' ' + word_string
text3 = text3 + '§' + phone_item
start_time1, end_time1 = extract_start_end_time(
word_item, 'Time Segment JTrans')
signal_sentence_values = signal_full[int(start_time0 *
fs):int(end_time1 *
fs)]
#print('1:', text1)
#print('2:', text2[num_sentence])
wavname = 'SynPaFlex_' + roots_file[0:-16] + '_' + str(
utt_index) + '_' + str(num_sentence) + '.wav'
fo.write(wavname + '|' + text1 + '|' + text2[num_sentence] +
'|' + text3 + '\n')
wavfile.write(
"/lium/raid01_b/tgranjon/synpaflex/wavs/" + wavname, fs,
signal_sentence_values)
start_time0 = end_time1
num_sentence += 1
text1 = ''
text3 = ''
else:
text1 = text1 + ' ' + word_string
text3 = text3 + '§' + phone_item
word_item.destroy()
#sys.exit()
#utt.destroy()
#print('---done corpus')
fo.close()