def test_espeak():
text = 'hello, world!'
expected1 = 'həloʊ wɜːld'
expected2 = 'həloʊ, wɜːld!'
expected3 = 'həloʊ wɜːld '
expected4 = 'həloʊ, wɜːld!'
out1 = EspeakBackend('en-us',
preserve_punctuation=False).phonemize([text],
strip=True)[0]
assert out1 == expected1
out2 = EspeakBackend('en-us',
preserve_punctuation=True).phonemize([text],
strip=True)[0]
assert out2 == expected2
out3 = EspeakBackend('en-us',
preserve_punctuation=False).phonemize([text],
strip=False)[0]
assert out3 == expected3
out4 = EspeakBackend('en-us',
preserve_punctuation=True).phonemize([text],
strip=False)[0]
assert out4 == expected4
def test_french():
backend = EspeakBackend('fr-fr')
text = u'bonjour le monde'
sep = separator.Separator(word=';eword ', syllable=None, phone=' ')
expected = [u'b ɔ̃ ʒ u ʁ ;eword l ə ;eword m ɔ̃ d ;eword ']
out = backend._phonemize_aux(text, sep, False)
assert out == expected
def test_french():
backend = EspeakBackend('fr-fr')
text = u'bonjour le monde'
sep = separator.Separator(word=';eword ', syllable=None, phone=' ')
expected = [u'b ɔ̃ ʒ u ʁ ;eword l ə- ;eword m ɔ̃ d ;eword ']
out = backend._phonemize_aux(text, sep, False)
assert out == expected
def version():
"""Return version information for front and backends"""
# version of the phonemizer
version = ('phonemizer-' +
pkg_resources.get_distribution('phonemizer').version)
# for each backend, check if it is available or not. If so get its version
available = []
unavailable = []
if EspeakBackend.is_available():
available.append('espeak-' +
('ng-' if EspeakBackend.is_espeak_ng() else '') +
EspeakBackend.version())
else: # pragma: nocover
unavailable.append('espeak')
if FestivalBackend.is_available():
available.append('festival-' + FestivalBackend.version())
else: # pragma: nocover
unavailable.append('festival')
if SegmentsBackend.is_available():
available.append('segments-' + SegmentsBackend.version())
else: # pragma: nocover
unavailable.append('segments')
# resumes the backends status in the final version string
if available:
version += '\navailable backends: ' + ', '.join(available)
if unavailable: # pragma: nocover
version += '\nuninstalled backends: ' + ', '.join(unavailable)
return version
def test_no_switch(policy, caplog):
text = ["j'aime l'anglais", "tu parles le français"]
backend = EspeakBackend('fr-fr', language_switch=policy)
out = backend.phonemize(text, separator=Separator(), strip=True)
assert out == ['ʒɛm lɑ̃ɡlɛ', 'ty paʁl lə fʁɑ̃sɛ']
messages = [msg[2] for msg in caplog.record_tuples]
assert not messages
def test_bad_text():
backend = EspeakBackend('en-us')
text = 'hello world'
with pytest.raises(RuntimeError) as err:
backend.phonemize(text, default_separator, True)
assert 'input text to phonemize() is str' in str(err)
assert backend.phonemize([text], default_separator,
True) == ['həloʊ wɜːld']
def version():
"""Return version information for front and backends"""
version = ('phonemizer-' +
pkg_resources.get_distribution('phonemizer').version)
return version + '\navailable backends: ' + ', '.join(
('festival-' + FestivalBackend.version(),
('espeak-' + ('ng-' if EspeakBackend.is_espeak_ng() else '') +
EspeakBackend.version()), 'segments-' + SegmentsBackend.version()))
def version():
"""Return version information for front and backends"""
version = ('phonemizer-'
+ pkg_resources.get_distribution('phonemizer').version)
return version + '\navailable backends: ' + ', '.join(
('festival-' + FestivalBackend.version(),
('espeak-' + ('ng-' if EspeakBackend.is_espeak_ng() else '')
+ EspeakBackend.version()),
'segments-' + SegmentsBackend.version()))
def test_tie_simple(caplog, tie, expected):
backend = EspeakBackend('en-us', tie=tie)
assert backend.phonemize(['Jackie Chan'],
separator=Separator(word=' ',
phone='_'))[0] == expected
if tie:
messages = [msg[2] for msg in caplog.record_tuples]
assert (
'cannot use ties AND phone separation, ignoring phone separator'
in messages)
def test_arabic():
backend = EspeakBackend('ar')
text = u'السلام عليكم'
sep = separator.Separator()
# Arabic seems to have changed starting at espeak-ng-1.49.3
if tuple(EspeakBackend.version().split('.')) >= ('1', '49', '3'):
expected = [u'ʔassalaːm ʕliːkm ']
else:
expected = [u'ʔassalaam ʕaliijkum ']
out = backend._phonemize_aux(text, sep, False)
assert out == expected
def test_arabic():
backend = EspeakBackend('ar')
text = u'السلام عليكم'
sep = separator.Separator()
# Arabic seems to have changed starting at espeak-ng-1.49.3
if tuple(EspeakBackend.version().split('.')) >= ('1', '49', '3'):
expected = [u'ʔassalaːm ʕliːkm ']
else:
expected = [u'ʔassalaam ʕaliijkum ']
out = backend._phonemize_aux(text, sep, False)
assert out == expected
def test_phone_separator_simple():
text = 'The lion and the tiger ran'
sep = separator.Separator(phone='_')
backend = EspeakBackend('en-us')
output = backend.phonemize(text, separator=sep, strip=True)
expected = 'ð_ə l_aɪə_n æ_n_d ð_ə t_aɪ_ɡ_ɚ ɹ_æ_n'
assert expected == output
output = backend.phonemize(text, separator=sep, strip=False)
expected = 'ð_ə_ l_aɪə_n_ æ_n_d_ ð_ə_ t_aɪ_ɡ_ɚ_ ɹ_æ_n_ '
assert expected == output
def test_arabic():
backend = EspeakBackend('ar')
text = ['السلام عليكم']
sep = Separator()
# Arabic seems to have changed starting at espeak-ng-1.49.3
if EspeakBackend.version() >= (1, 49, 3):
expected = ['ʔassalaːm ʕliːkm ']
else:
expected = ['ʔassalaam ʕaliijkum ']
out = backend.phonemize(text, sep, False)
assert out == expected
def test_tie_utf8():
# NOTE this is a bug in espeak to append ties on (en) language switch
# flags. For now phonemizer does not fix it.
backend = EspeakBackend('fr-fr', tie=True)
# used to be 'bɔ̃͡ʒuʁ '
assert backend.phonemize(['bonjour']) == ['bɔ̃ʒuʁ ']
# used to be 'ty ɛm lə (͡e͡n͡)fʊtbɔ͡ːl(͡f͡r͡)'
assert backend.phonemize(['tu aimes le football'
]) == ['ty ɛm lə (͡e͡n)fʊtbɔːl(͡f͡r) ']
assert backend.phonemize(['bonjour apple'
]) == ['bɔ̃ʒuʁ (͡e͡n)apə͡l(͡f͡r) ']
def test_language_switch_remove_utterance(caplog, langswitch_text, njobs):
backend = EspeakBackend('fr-fr', language_switch='remove-utterance')
out = backend.phonemize(langswitch_text,
separator=Separator(),
strip=True,
njobs=njobs)
assert out == ['ʒɛm lɑ̃ɡlɛ', '', '', '', '']
messages = [msg[2] for msg in caplog.record_tuples]
assert ('removed 4 utterances containing language switches '
'(applying "remove-utterance" policy)' in messages)
with pytest.raises(RuntimeError):
backend = EspeakBackend('fr-fr', language_switch='foo')
def test_language_switch_remove_flags(caplog, langswitch_text, njobs):
backend = EspeakBackend('fr-fr', language_switch='remove-flags')
out = backend.phonemize(langswitch_text,
separator=Separator(),
strip=True,
njobs=njobs)
assert out == [
'ʒɛm lɑ̃ɡlɛ', 'ʒɛm lə fʊtbɔːl', 'fʊtbɔːl', 'syʁtu lə ɹiəl madʁid',
'nytiliz pa ɡuːɡəl'
]
messages = [msg[2] for msg in caplog.record_tuples]
assert ('4 utterances containing language switches on lines 2, 3, 4, 5'
in messages)
assert ('language switch flags have been removed '
'(applying "remove-flags" policy)' in messages)
def test_path_venv():
try:
os.environ['PHONEMIZER_ESPEAK_LIBRARY'] = (shutil.which('python'))
with pytest.raises(RuntimeError):
EspeakBackend('en-us').phonemize(['hello'])
with pytest.raises(RuntimeError):
EspeakBackend.version()
os.environ['PHONEMIZER_ESPEAK_LIBRARY'] = __file__
with pytest.raises(RuntimeError):
EspeakBackend.version()
finally:
try:
del os.environ['PHONEMIZER_ESPEAK_LIBRARY']
except KeyError:
pass
def test_path_venv():
try:
os.environ['PHONEMIZER_ESPEAK_PATH'] = (
distutils.spawn.find_executable('python'))
with pytest.raises(RuntimeError):
EspeakBackend('en-us').phonemize('hello')
with pytest.raises(RuntimeError):
EspeakBackend.version()
os.environ['PHONEMIZER_ESPEAK_PATH'] = __file__
with pytest.raises(ValueError):
EspeakBackend.version()
finally:
try:
del os.environ['PHONEMIZER_ESPEAK_PATH']
except KeyError:
pass
def test_path_bad():
espeak = EspeakBackend.espeak_path()
try:
# corrupt the default espeak path, try to use python executable instead
binary = distutils.spawn.find_executable('python')
EspeakBackend.set_espeak_path(binary)
with pytest.raises(RuntimeError):
EspeakBackend('en-us').phonemize('hello')
with pytest.raises(RuntimeError):
EspeakBackend.version()
with pytest.raises(ValueError):
EspeakBackend.set_espeak_path(__file__)
# restore the espeak path to default
finally:
EspeakBackend.set_espeak_path(espeak)
def test_punctuation(text, strip, sep):
if sep == separator.Separator():
expected = 'ɐ kɑːmə ɐ pɔɪnt' if strip else 'ɐ kɑːmə ɐ pɔɪnt '
else:
expected = (
'ɐ_k ɑː m ə_ɐ_p ɔɪ n t' if strip else 'ɐ _k ɑː m ə _ɐ _p ɔɪ n t _')
output = EspeakBackend('en-us').phonemize(text, strip=strip, separator=sep)
assert expected == output
def test_path_bad():
try:
# corrupt the default espeak path, try to use python executable instead
binary = shutil.which('python')
EspeakBackend.set_library(binary)
with pytest.raises(RuntimeError):
EspeakBackend('en-us')
with pytest.raises(RuntimeError):
EspeakBackend.version()
EspeakBackend.set_library(__file__)
with pytest.raises(RuntimeError):
EspeakBackend('en-us')
# restore the espeak path to default
finally:
EspeakBackend.set_library(None)
def test_language_switch_default(caplog, langswitch_text, njobs):
# default behavior is to keep the flags
backend = EspeakBackend('fr-fr')
out = backend.phonemize(langswitch_text,
separator=Separator(),
strip=True,
njobs=njobs)
assert out == [
'ʒɛm lɑ̃ɡlɛ', 'ʒɛm lə (en)fʊtbɔːl(fr)', '(en)fʊtbɔːl(fr)',
'syʁtu lə (en)ɹiəl(fr) madʁid', 'nytiliz pa (en)ɡuːɡəl(fr)'
]
messages = [msg[2] for msg in caplog.record_tuples]
assert ('4 utterances containing language switches on lines 2, 3, 4, 5'
in messages)
assert (
'language switch flags have been kept (applying "keep-flags" policy)'
in messages)
def __init__(
self,
text_cleaners=["basic_cleaners"],
use_phonemes=True,
n_jobs=1,
with_stress=True,
language="en-us",
):
"""
Text sequencies preprocessor with G2P support.
:param text_cleaners: text cleaner type:
* `basic_cleaners`: basic pipeline that lowercases and collapses whitespace without transliteration.
* `transliteration_cleaners`: pipeline for non-English text that transliterates to ASCII.
* `english_cleaners`: pipeline for English text, including number and abbreviation expansion.
:param use_phonemes: file path with phonemes set separated by `|`
:param n_jobs: number of workers for phonemization
:param with_stress: set `True` to stress words during phonemization
"""
self.text_cleaners = text_cleaners
self.use_phonemes = use_phonemes
self.n_jobs = n_jobs
self.with_stress = with_stress
self.language = language
CHARS = _GRAPHEMES if not self.use_phonemes else _PHONEMES
self.SYMBOLS = ([_PAD, _EOS, _SPACE] + _PUNCTUATIONS + ["¡", "¿"] +
_NUMBERS + CHARS)
# Mappings from symbol to numeric ID and vice versa:
self._symbol_to_id = {s: i for i, s in enumerate(self.SYMBOLS)}
self._id_to_symbol = {i: s for i, s in enumerate(self.SYMBOLS)}
self._separator = Separator(word=_WORD_SEP,
syllable="",
phone=_PHONEME_SEP)
self.p = EspeakBackend(
self.language,
punctuation_marks="".join(_PUNCTUATIONS),
preserve_punctuation=True,
with_stress=self.with_stress,
)
def test_espeak(njobs):
text = ['one two', 'three', 'four five']
out = phonemize(text,
language='en-us',
backend='espeak',
strip=True,
njobs=njobs)
assert out == [u'wʌn tuː', u'θɹiː', u'foːɹ faɪv']
if EspeakBackend.is_espeak_ng():
out = phonemize(text,
language='en-us',
backend='espeak',
use_sampa=True,
strip=True,
njobs=njobs)
assert out == [u'wVn tu:', u'Tri:', u'fo@ faIv']
out = phonemize(text,
language='en-us',
backend='espeak',
strip=False,
njobs=njobs)
assert out == [u'wʌn tuː ', u'θɹiː ', u'foːɹ faɪv ']
out = phonemize(' '.join(text),
language='en-us',
backend='espeak',
strip=True,
njobs=njobs)
assert out == ' '.join([u'wʌn tuː', u'θɹiː', u'foːɹ faɪv'])
out = phonemize(' '.join(text),
language='en-us',
backend='espeak',
strip=False,
njobs=njobs)
assert out == ' '.join([u'wʌn tuː', u'θɹiː', u'foːɹ faɪv '])
out = phonemize('\n'.join(text),
language='en-us',
backend='espeak',
strip=True,
njobs=njobs)
assert out == '\n'.join([u'wʌn tuː', u'θɹiː', u'foːɹ faɪv'])
out = phonemize('\n'.join(text),
language='en-us',
backend='espeak',
strip=False,
njobs=njobs)
assert out == '\n'.join([u'wʌn tuː ', u'θɹiː ', u'foːɹ faɪv '])
def test_path_good():
espeak = EspeakBackend.espeak_path()
try:
EspeakBackend.set_espeak_path(None)
assert espeak == EspeakBackend.espeak_path()
binary = distutils.spawn.find_executable('espeak')
EspeakBackend.set_espeak_path(binary)
test_english()
# restore the espeak path to default
finally:
EspeakBackend.set_espeak_path(espeak)
def test_path_good():
espeak = EspeakBackend.library()
try:
EspeakBackend.set_library(None)
assert espeak == EspeakBackend.library()
library = EspeakWrapper().library_path
EspeakBackend.set_library(library)
test_english()
# restore the espeak path to default
finally:
EspeakBackend.set_library(None)
def test_stress():
backend = EspeakBackend('en-us', with_stress=False)
assert backend.phonemize(['hello world'], default_separator,
True) == ['həloʊ wɜːld']
backend = EspeakBackend('en-us', with_stress=True)
assert backend.phonemize(['hello world'], default_separator,
True) == ['həlˈoʊ wˈɜːld']
def test_stress():
backend = EspeakBackend('en-us', with_stress=False)
assert u'həloʊ wɜːld' == backend._phonemize_aux(
u'hello world', separator.default_separator, True)[0]
backend = EspeakBackend('en-us', with_stress=True)
assert u'həlˈoʊ wˈɜːld' == backend._phonemize_aux(
u'hello world', separator.default_separator, True)[0]
def parse_args():
"""Argument parser for the phonemization script"""
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description='''Multilingual text to phonemes converter
The 'phonemize' program allows simple phonemization of words and texts
in many language using three backends: espeak, festival and segments.
- espeak is a text-to-speech software supporting multiple languages
and IPA (Internatinal Phonetic Alphabet) output. See
http://espeak.sourceforge.net or
https://github.com/espeak-ng/espeak-ng
- festival is also a text-to-speech software. Currently only American
English is supported and festival uses a custom phoneset
(http://www.festvox.org/bsv/c4711.html), but festival is the only
backend supporting tokenization at the syllable
level. See http://www.cstr.ed.ac.uk/projects/festival
- segments is a Unicode tokenizer that build a phonemization from a
grapheme to phoneme mapping provided as a file by the user. See
https://github.com/cldf/segments.
See the '--language' option below for details on the languages
supported by each backend.
''',
epilog='''
Languages supported by the festival backend are:
{festival}
Languages supported by the segments backend are:
{segments}
Instead of a language you can also provide a file specifying a
grapheme to phoneme mapping (see the files above for exemples).
Languages supported by the espeak backend are:
{espeak}
Exemples:
* Phonemize a US English text with espeak
$ echo 'hello world' | phonemize -l en-us -b espeak
həloʊ wɜːld
* Phonemize a US English text with festival
$ echo 'hello world' | phonemize -l en-us -b festival
hhaxlow werld
* Phonemize a Japanese text with segments
$ echo 'konnichiwa tsekai' | phonemize -l japanese -b segments
konnitʃiwa t͡sekai
* Add a separator between phones
$ echo 'hello world' | phonemize -l en-us -b festival -p '-' --strip
hh-ax-l-ow w-er-l-d
* Phonemize some French text file using espeak
$ phonemize -l fr-fr -b espeak text.txt -o phones.txt
'''.format(festival='\n'.join(
'\t{}\t->\t{}'.format(k, v)
for k, v in sorted(FestivalBackend.supported_languages().items())),
segments='\n'.join(
'\t{}\t->\t{}'.format(k, v) for k, v in sorted(
SegmentsBackend.supported_languages().items())),
espeak='\n'.join(
'\t{}\t->\t{}'.format(k, v) for k, v in sorted(
EspeakBackend.supported_languages().items()))))
# general arguments
parser.add_argument('--version',
action='store_true',
help='show version information and exit.')
group = parser.add_mutually_exclusive_group()
group.add_argument('-v',
'--verbose',
action='store_true',
help='write all log messages to stderr '
'(displays only warnings by default).')
group.add_argument('-q',
'--quiet',
action='store_true',
help='do not display any log message, even warnings.')
parser.add_argument(
'-j',
'--njobs',
type=int,
metavar='<int>',
default=1,
help='number of parallel jobs, default is %(default)s.')
# input/output arguments
group = parser.add_argument_group('input/output')
group.add_argument(
'input',
default=sys.stdin,
nargs='?',
metavar='<file>',
help='input text file to phonemize, if not specified read from stdin.')
group.add_argument(
'-o',
'--output',
default=sys.stdout,
metavar='<file>',
help='output text file to write, if not specified write to stdout.')
group = parser.add_argument_group('separators')
group.add_argument('-p',
'--phone-separator',
metavar='<str>',
default=separator.default_separator.phone,
help='phone separator, default is "%(default)s".')
group.add_argument('-w',
'--word-separator',
metavar='<str>',
default=separator.default_separator.word,
help='word separator, default is "%(default)s".')
group.add_argument(
'-s',
'--syllable-separator',
metavar='<str>',
default=separator.default_separator.syllable,
help='''syllable separator, only valid for festival backend,
this option has no effect if espeak or segments is used.
Default is "%(default)s".''')
group.add_argument('--strip',
action='store_true',
help='removes the end separators in phonemized tokens.')
group = parser.add_argument_group('backends')
group.add_argument(
'-b',
'--backend',
metavar='<str>',
default='espeak',
choices=['espeak', 'festival', 'segments'],
help="""the phonemization backend, must be 'espeak', 'festival' or
'segments'. Default is %(default)s.""")
group = parser.add_argument_group('specific to espeak backend')
group.add_argument(
'--with-stress',
action='store_true',
help='''when the option is set, the stresses on phonemes are present
(stresses characters are ˈ'ˌ). By default stresses are removed.''')
group.add_argument(
'--sampa',
action='store_true',
help='''only valid for espeak-ng and NOT supported for espeak, use the
"sampa" (Speech Assessment Methods Phonetic Alphabet) alphabet instead
of "ipa" (International Phonetic Alphabet).''')
group.add_argument(
'--language-switch',
default='keep-flags',
choices=['keep-flags', 'remove-flags', 'remove-utterance'],
help="""espeak can pronounce some words in another language (typically
English) when phonemizing a text. This option setups the policy to use
when such a language switch occurs. Three values are available:
'keep-flags' (the default), 'remove-flags' or 'remove-utterance'. The
'keep-flags' policy keeps the language switching flags, for example
(en) or (jp), in the output. The 'remove-flags' policy removes them and
the 'remove-utterance' policy removes the whole line of text including
a language switch.""")
group = parser.add_argument_group('language')
group.add_argument(
'-l',
'--language',
metavar='<str|file>',
default='en-us',
help='''the language code of the input text, see below for a list of
supported languages. According to the language code you
specify, the appropriate backend (segments, espeak or
festival) will be called in background. Default is
%(default)s.''')
return parser.parse_args()
def parse_args():
"""Argument parser for the phonemization script"""
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description='''Multilingual text to phonemes converter
The 'phonemize' program allows simple phonemization of words and texts
in many language using three backends: espeak, festival and segments.
- espeak is a text-to-speech software supporting multiple languages
and IPA (Internatinal Phonetic Alphabet) output. See
http://espeak.sourceforge.net or
https://github.com/espeak-ng/espeak-ng
- festival is also a text-to-speech software. Currently only American
English is supported and festival uses a custom phoneset
(http://www.festvox.org/bsv/c4711.html), but festival is the only
backend supporting tokenization at the syllable
level. See http://www.cstr.ed.ac.uk/projects/festival
- segments is a Unicode tokenizer that build a phonemization from a
grapheme to phoneme mapping provided as a file by the user. See
https://github.com/cldf/segments.
See the '--language' option below for details on the languages
supported by each backend.
''',
epilog='''
Languages supported by the festival backend are:
{festival}
Languages supported by the segments backend are:
{segments}
Instead of a language you can also provide a file specifying a
grapheme to phoneme mapping (see the files above for exemples).
Languages supported by the espeak backend are:
{espeak}
Exemples:
* Phonemize a US English text with espeak
$ echo 'hello world' | phonemize -l en-us -b espeak
həloʊ wɜːld
* Phonemize a US English text with festival
$ echo 'hello world' | phonemize -l en-us -b festival
hhaxlow werld
* Phonemize a Japanese text with segments
$ echo 'konnichiwa tsekai' | phonemize -l japanese -b segments
konnitʃiwa t͡sekai
* Add a separator between phones
$ echo 'hello world' | phonemize -l en-us -b festival -p '-' --strip
hh-ax-l-ow w-er-l-d
* Phonemize some French text file using espeak
$ phonemize -l fr-fr -b espeak text.txt -o phones.txt
'''.format(
festival='\n'.join(
'\t{}\t->\t{}'.format(k, v) for k, v in
sorted(FestivalBackend.supported_languages().items())),
segments='\n'.join(
'\t{}\t->\t{}'.format(k, v) for k, v in
sorted(SegmentsBackend.supported_languages().items())),
espeak='\n'.join(
'\t{}\t->\t{}'.format(k, v) for k, v in
sorted(EspeakBackend.supported_languages().items()))))
# general arguments
parser.add_argument(
'--version', action='store_true',
help='show version information and exit')
parser.add_argument(
'-v', '--verbose', action='store_true',
help='write some log messages to stderr')
parser.add_argument(
'-j', '--njobs', type=int, metavar='<int>', default=1,
help='number of parallel jobs, default is %(default)s')
# input/output arguments
group = parser.add_argument_group('input/output')
group.add_argument(
'input', default=sys.stdin, nargs='?', metavar='<file>',
help='input text file to phonemize, if not specified read from stdin')
group.add_argument(
'-o', '--output', default=sys.stdout, metavar='<file>',
help='output text file to write, if not specified write to stdout')
group = parser.add_argument_group('separators')
group.add_argument(
'-p', '--phone-separator', metavar='<str>',
default=separator.default_separator.phone,
help='phone separator, default is "%(default)s"')
group.add_argument(
'-w', '--word-separator', metavar='<str>',
default=separator.default_separator.word,
help='word separator, default is "%(default)s"')
group.add_argument(
'-s', '--syllable-separator', metavar='<str>',
default=separator.default_separator.syllable,
help='''syllable separator is available only for the festival backend,
this option has no effect if espeak or segments is used.
Default is "%(default)s"''')
group.add_argument(
'--strip', action='store_true',
help='removes the end separators in phonemized tokens')
group = parser.add_argument_group('language')
group.add_argument(
'-b', '--backend', metavar='<str>', default='espeak',
choices=['espeak', 'festival', 'segments'],
help="""the phonemization backend, must be 'espeak', 'festival' or
'segments'. Default is %(default)s""")
group.add_argument(
'-l', '--language', metavar='<str|file>', default='en-us',
help='''the language code of the input text, see below for a list of
supported languages. According to the language code you
specify, the appropriate backend (segments, espeak or
festival) will be called in background. Default is
%(default)s''')
return parser.parse_args()
def phonemize(text,
language='en-us',
backend='festival',
separator=default_separator,
strip=False,
with_stress=False,
use_sampa=False,
language_switch='keep-flags',
njobs=1,
logger=get_logger()):
"""Multilingual text to phonemes converter
Return a phonemized version of an input `text`, given its
`language` and a phonemization `backend`.
Parameters
----------
text (str or list of str): The text to be phonemized. Any empty
line will be ignored. If `text` is an str, it can be multiline
(lines being separated by \n). If `text` is a list, each
element is considered as a separated line. Each line is
considered as a text utterance.
language (str): The language code of the input text, must be
supported by the backend. If `backend` is 'segments', the
language can be a file with a grapheme to phoneme mapping.
backend (str): The software backend to use for phonemization, must
be 'festival' (US English only is supported, coded 'en-us'),
'espeak' or 'segments'.
separator (Separator): string separators between phonemes,
syllables and words, default to separator.default_separator.
strip (bool): If True, don't output the last word and phone
separators of a token, default to False.
with_stress (bool): This option is only valid for the espeak/espeak-ng
backend. When True the stresses on phonemes are present (stresses
characters are ˈ'ˌ). When False stresses are removed. Default to False.
use_sampa (bool): Use the 'sampa' phonetic alphabet (Speech Assessment
Methods Phonetic Alphabet) instead of 'ipa' (International Phonetic
Alphabet). This option is only valid for the 'espeak-ng' backend. Default
to False.
language_switch (str) : espeak can pronounce some words in another language
(typically English) when phonemizing a text. This option setups the
policy to use when such a language switch occurs. Three values are
available: 'keep-flags' (the default), 'remove-flags' or
'remove-utterance'. The 'keep-flags' policy keeps the language switching
flags, for example (en) or (jp), in the output. The 'remove-flags' policy
removes them and the 'remove-utterance' policy removes the whole line of
text including a language switch.
njobs (int): The number of parallel jobs to launch. The input text
is split in `njobs` parts, phonemized on parallel instances of
the backend and the outputs are finally collapsed.
logger (logging.Logger): the logging instance where to send
messages. If not specified, use the default system logger.
Returns
-------
phonemized text (str or list of str) : The input `text` phonemized
for the given `language` and `backend`. The returned value has
the same type of the input text (either a list or a string).
Raises
------
RuntimeError
If the `backend` is not valid or is valid but not installed, if the
`language` is not supported by the `backend`, if `use_sampa`,
`with_stress` or `language_switch` are used but the backend is not
'espeak-ng'.
"""
# ensure the backend is either espeak, festival or segments
if backend not in ('espeak', 'festival', 'segments'):
raise RuntimeError(
'{} is not a supported backend, choose in {}.'.format(
backend, ', '.join(('espeak', 'festival', 'segments'))))
# ensure the phonetic alphabet is valid
if use_sampa is True:
if backend == 'espeak' and not EspeakBackend.is_espeak_ng():
raise RuntimeError( # pragma: nocover
'sampa alphabet is not supported by espeak, '
'please install espeak-ng')
if backend != 'espeak':
raise RuntimeError(
'sampa alphabet is only supported by espeak backend')
# with_stress option only valid for espeak
if with_stress and backend != 'espeak':
raise RuntimeError(
'the "with_stress" option is available for espeak backend only, '
'but you are using {} backend'.format(backend))
# language_switch option only valid for espeak
if language_switch != 'keep-flags' and backend != 'espeak':
raise RuntimeError(
'the "language_switch" option is available for espeak backend '
'only, but you are using {} backend'.format(backend))
# instanciate the requested backend for the given language (raises
# a RuntimeError if the language is not supported).
backends = {
b.name(): b
for b in (EspeakBackend, FestivalBackend, SegmentsBackend)
}
if backend == 'espeak':
phonemizer = backends[backend](language,
with_stress=with_stress,
use_sampa=use_sampa,
language_switch=language_switch,
logger=logger)
else:
phonemizer = backends[backend](language, logger=logger)
# phonemize the input text with the backend
return phonemizer.phonemize(text,
separator=separator,
strip=strip,
njobs=njobs)
backend = EspeakBackend('en-us')
text = u'hello world\ngoodbye\nthird line\nyet another'
out = '\n'.join(backend._phonemize_aux(
text, separator.default_separator, True))
assert out == u'həloʊ wɜːld\nɡʊdbaɪ\nθɜːd laɪn\njɛt ɐnʌðɚ'
def test_french():
backend = EspeakBackend('fr-fr')
text = u'bonjour le monde'
sep = separator.Separator(word=';eword ', syllable=None, phone=' ')
expected = [u'b ɔ̃ ʒ u ʁ ;eword l ə- ;eword m ɔ̃ d ;eword ']
out = backend._phonemize_aux(text, sep, False)
assert out == expected
@pytest.mark.skipif(
not EspeakBackend.is_espeak_ng(),
reason='Arabic is only supported by espeak-ng')
def test_arabic():
backend = EspeakBackend('ar')
text = u'السلام عليكم'
sep = separator.Separator()
# Arabic seems to have changed starting at espeak-ng-1.49.3
if tuple(EspeakBackend.version().split('.')) >= ('1', '49', '3'):
expected = [u'ʔassalaːm ʕliːkm ']
else:
expected = [u'ʔassalaam ʕaliijkum ']
out = backend._phonemize_aux(text, sep, False)
assert out == expected
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with phonemizer. If not, see <http://www.gnu.org/licenses/>.
"""Test of the punctuation processing"""
# pylint: disable=missing-docstring
import pytest
from phonemizer.backend import EspeakBackend, FestivalBackend, SegmentsBackend
from phonemizer.punctuation import Punctuation
from phonemizer.phonemize import phonemize
# True if we are using espeak>=1.50
ESPEAK_150 = (EspeakBackend.version() >= (1, 50))
# True if we are using espeak>=1.49.3
ESPEAK_143 = (EspeakBackend.version() >= (1, 49, 3))
# True if we are using festival>=2.5
FESTIVAL_25 = (FestivalBackend.version() >= (2, 5))
@pytest.mark.parametrize('inp, out', [('a, b,c.', 'a b c'),
('abc de', 'abc de'),
('!d.d. dd?? d!', 'd d dd d')])
def test_remove(inp, out):
assert Punctuation().remove(inp) == out
def test_english():
backend = EspeakBackend('en-us')
text = u'hello world\ngoodbye\nthird line\nyet another'
out = '\n'.join(backend._phonemize_aux(
text, separator.default_separator, True))
assert out == u'həloʊ wɜːld\nɡʊdbaɪ\nθɜːd laɪn\njɛt ɐnʌðɚ'
out3 = phonemize(text2,
language='en-us',
backend='espeak',
strip=True,
prepend_text=True)
text3 = [o[0] for o in out3]
phn3 = [o[1] for o in out3]
assert isinstance(phn1, list)
assert isinstance(phn2, str)
assert os.linesep.join(phn1) == phn2
assert os.linesep.join(phn3) == phn2
assert text3 == text1
@pytest.mark.skipif(not EspeakBackend.is_espeak_ng(),
reason='language switch only exists for espeak-ng')
def test_lang_switch():
text = ['bonjour apple', 'bonjour toi']
out = phonemize(text,
language='fr-fr',
backend='espeak',
prepend_text=True,
language_switch='remove-utterance')
assert out == [('bonjour apple', ''), ('bonjour toi', 'bɔ̃ʒuʁ twa ')]
@pytest.mark.parametrize('njobs', [2, 4])
def test_espeak(njobs):
text = ['one two', 'three', 'four five']
def main():
"""Phonemize a text from command-line arguments"""
args = parse_args()
# setup a custom path to espeak and festival if required (this must be done
# before generating the version message)
if args.espeak_path:
EspeakBackend.set_espeak_path(args.espeak_path)
if args.festival_path:
FestivalBackend.set_festival_path(args.festival_path)
# display version information and exit
if args.version:
print(version.version())
return
# list supported languages and exit
if args.list_languages:
backends = (['festival', 'segments', 'espeak', 'espeak-mbrola']
if not args.backend else [args.backend])
for backend in backends:
print(f'supported languages for {backend} are:\n' +
'\n'.join(f'\t{k}\t->\t{v}' for k, v in sorted(
BACKENDS_MAP[backend].supported_languages().items())))
return
# set default backend as espeak if not specified
args.backend = args.backend or 'espeak'
# configure logging according to --verbose/--quiet options
verbosity = 'normal'
if args.verbose:
verbosity = 'verbose'
elif args.quiet:
verbosity = 'quiet'
log = logger.get_logger(verbosity=verbosity)
# configure input as a readable stream
streamin = args.input
if isinstance(streamin, str):
streamin = codecs.open(streamin, 'r', encoding='utf8')
log.debug('reading from %s', streamin.name)
# configure output as a writable stream
streamout = args.output
if isinstance(streamout, str):
streamout = codecs.open(streamout, 'w', 'utf8')
log.debug('writing to %s', streamout.name)
# configure the separator for phonemes, syllables and words.
if args.backend == 'espeak-mbrola':
log.debug('using espeak-mbrola backend: ignoring word separator')
sep = separator.Separator(phone=args.phone_separator,
syllable=None,
word=None)
else:
sep = separator.Separator(phone=args.phone_separator,
syllable=args.syllable_separator,
word=args.word_separator)
log.debug('separator is %s', sep)
text = [line.strip() for line in streamin]
# phonemize the input text
out = phonemize(text,
language=args.language,
backend=args.backend,
separator=sep,
strip=args.strip,
preserve_punctuation=args.preserve_punctuation,
punctuation_marks=args.punctuation_marks,
with_stress=args.with_stress,
language_switch=args.language_switch,
njobs=args.njobs,
logger=log)
if out:
streamout.write('\n'.join(out) + '\n')
