def phonemize(self,
text,
separator=default_separator,
strip=False,
njobs=1):
"""Returns the `text` phonemized for the given language"""
text, text_type, punctuation_marks = self._phonemize_preprocess(text)
if njobs == 1:
# phonemize the text forced as a string
text = self._phonemize_aux(list2str(text), separator, strip)
else:
# If using parallel jobs, disable the log as stderr is not
# picklable.
self.logger.info('running %s on %s jobs', self.name(), njobs)
log_storage = self.logger
self.logger = None
# we have here a list of phonemized chunks
text = joblib.Parallel(n_jobs=njobs)(
joblib.delayed(self._phonemize_aux)(t, separator, strip)
for t in chunks(text, njobs))
# flatten them in a single list
text = list(itertools.chain(*text))
# restore the log as it was before parallel processing
self.logger = log_storage
return self._phonemize_postprocess(text, text_type, punctuation_marks)
def _phonemize( # pylint: disable=too-many-arguments
backend, text, separator, strip, njobs, prepend_text):
"""Auxiliary function to phonemize()
Does the phonemization and returns the phonemized text. Raises a
RuntimeError on error.
"""
# remember the text type for output (either list or string), force the text
# as a list and ignore empty lines
text_type = type(text)
text = (line.strip(os.linesep) for line in str2list(text))
text = [line for line in text if line.strip()]
# phonemize the text
phonemized = backend.phonemize(text,
separator=separator,
strip=strip,
njobs=njobs)
# at that point, the phonemized text is a list of str. Format it as
# expected by the parameters
if prepend_text:
return list(zip(text, phonemized))
if text_type == str:
return list2str(phonemized)
return phonemized
def phonemize(self,
text,
separator=default_separator,
strip=False,
njobs=1):
"""Returns the `text` phonemized for the given language"""
# remember the text type for output (either list or string)
text_type = type(text)
# deals with punctuation: remove it and keep track of it for
# restoration at the end if asked for
punctuation_marks = []
if self.preserve_punctuation:
text, punctuation_marks = self._punctuator.preserve(text)
else:
text = self._punctuator.remove(text)
if njobs == 1:
# phonemize the text forced as a string
text = self._phonemize_aux(list2str(text), separator, strip)
else:
# If using parallel jobs, disable the log as stderr is not
# picklable.
self.logger.info('running %s on %s jobs', self.name(), njobs)
log_storage = self.logger
self.logger = None
# we have here a list of phonemized chunks
text = joblib.Parallel(n_jobs=njobs)(
joblib.delayed(self._phonemize_aux)(t, separator, strip)
for t in chunks(text, njobs))
# flatten them in a single list
text = list(itertools.chain(*text))
# restore the log as it was before parallel processing
self.logger = log_storage
# restore the punctuation is asked for
if self.preserve_punctuation:
text = self._punctuator.restore(text, punctuation_marks)
# output the result formatted as a string or a list of strings
# according to type(text)
return (list2str(text)
if text_type in six.string_types else str2list(text))
def _phonemize_postprocess(self, text, text_type, punctuation_marks):
# restore the punctuation is asked for
if self.preserve_punctuation:
text = self._punctuator.restore(text, punctuation_marks)
# output the result formatted as a string or a list of strings
# according to type(text)
return (list2str(text)
if text_type in six.string_types else str2list(text))
def phonemize(self,
text,
separator=default_separator,
strip=False,
njobs=1):
text, text_type, punctuation_marks = self._phonemize_preprocess(text)
lang_switches = []
if njobs == 1:
# phonemize the text forced as a string
text, lang_switches = self._phonemize_aux(list2str(text),
separator, strip)
else:
# If using parallel jobs, disable the log as stderr is not
# picklable.
self.logger.info('running %s on %s jobs', self.name(), njobs)
log_storage = self.logger
self.logger = None
# divide the input text in chunks, each chunk being processed in a
# separate job
text_chunks = chunks(text, njobs)
# offset used below to recover the line numbers in the input text
# wrt the chunks
offset = [0] + cumsum(
(c.count('\n') + 1 for c in text_chunks[:-1]))
# we have here a list of (phonemized chunk, lang_switches)
output = joblib.Parallel(n_jobs=njobs)(
joblib.delayed(self._phonemize_aux)(t, separator, strip)
for t in text_chunks)
# flatten both the phonemized chunks and language switches in a
# list. For language switches lines we need to add an offset to
# have the correct lines numbers wrt the input text.
text = list(itertools.chain(*(chunk[0] for chunk in output)))
lang_switches = [chunk[1] for chunk in output]
for i in range(len(lang_switches)):
for j in range(len(lang_switches[i])):
lang_switches[i][j] += offset[i]
lang_switches = list(itertools.chain(*lang_switches))
# restore the log as it was before parallel processing
self.logger = log_storage
# warn the user if language switches occured during phonemization
self._warn_on_lang_switch(lang_switches)
# finally restore the punctuation
return self._phonemize_postprocess(text, text_type, punctuation_marks)
def test_list2str():
assert list2str('') == ''
assert list2str([]) == ''
assert list2str(['']) == ''
assert list2str(['abc']) == 'abc'
assert list2str(['a', 'b', 'c']) == 'a\nb\nc'
def test_list2str():
assert list2str('') == ''
assert list2str([]) == ''
assert list2str(['']) == ''
assert list2str(['abc']) == 'abc'
assert list2str(['a', 'b', 'c']) == os.linesep.join('abc')