def test_distance_errors(self):
src_mappings = [{"in": "ᐃ", "out": "i"}]
src_mapping = Mapping(src_mappings, in_lang="crj", out_lang="crj-ipa")
# Exercise looking up distances in the known list
with self.assertRaises(ValueError):
mapping = create_mapping(src_mapping,
self.target_mapping,
distance="not_a_distance")
with self.assertRaises(ValueError):
mapping = create_multi_mapping(
[(src_mapping, "out")],
[(self.target_mapping, "in")],
distance="not_a_distance",
)
# White box testing: monkey-patch an invalid distance to validate the
# second way we make sure distances are supported
DISTANCE_METRICS.append("not_a_real_distance")
with self.assertRaises(ValueError):
mapping = create_mapping(src_mapping,
self.target_mapping,
distance="not_a_real_distance")
with self.assertRaises(ValueError):
mapping = create_multi_mapping(
[(src_mapping, "out")],
[(self.target_mapping, "in")],
distance="not_a_real_distance",
)
DISTANCE_METRICS.pop()
def generate_mapping(in_lang, dummy, ipa, list_dummy, out_dir):
''' For specified IN_LANG, generate a mapping from IN_LANG-ipa to eng-ipa,
or from IN_LANG-ipa to a dummy minimalist phone inventory.
If you just modified or wrote the IN_LANG to IN_LANG-ipa mapping, don't forget
to call "g2p update" first so "g2p generate-mapping" sees the latest version.
Call "g2p update" again after calling "g2p generate-mapping" to make the new
IN_LANG-ipa to eng-ipa mapping available.
'''
if not ipa and not dummy and not list_dummy:
click.echo('You have to choose to generate either an IPA-based mapping or a dummy fallback mapping. Check the docs for more information.')
if out_dir and (os.path.exists(os.path.join(out_dir, 'config.yaml')) or os.path.exists(os.path.join(out_dir, 'config.yaml'))):
click.echo(
f'There is already a mapping config file in \'{out_dir}\' \nPlease choose another path.')
return
if list_dummy:
print("Dummy phone inventory: {}".format(DUMMY_INVENTORY))
if ipa:
check_ipa_known_segs([f'{in_lang}-ipa'])
eng_ipa = Mapping(in_lang='eng-ipa', out_lang='eng-arpabet')
new_mapping = Mapping(in_lang=in_lang, out_lang=f'{in_lang}-ipa')
click.echo(f"Writing English IPA mapping for {in_lang} to file")
create_mapping(new_mapping, eng_ipa,
write_to_file=True, out_dir=out_dir)
if dummy:
new_mapping = Mapping(in_lang=in_lang, out_lang=f'{in_lang}-ipa')
click.echo(f"Writing dummy fallback mapping for {in_lang} to file")
dummy_config, dummy_mapping = align_to_dummy_fallback(
new_mapping, write_to_file=True, out_dir=out_dir)
def generate_mapping(in_lang, dummy, ipa):
''' Generate English mapping
'''
if not ipa and not dummy:
click.echo('You have to choose to generate either an IPA-based mapping or a dummy fallback mapping. Check the docs for more information.')
if ipa:
eng_ipa = Mapping(in_lang='eng-ipa', out_lang='eng-arpabet')
new_mapping = Mapping(in_lang=in_lang, out_lang=f'{in_lang}-ipa')
click.echo(f"Writing English IPA mapping for {in_lang} to file")
create_mapping(new_mapping, eng_ipa, write_to_file=True)
if dummy:
new_mapping = Mapping(in_lang=in_lang, out_lang=f'{in_lang}-ipa')
click.echo(f"Writing dummy fallback mapping for {in_lang} to file")
dummy_config, dummy_mapping = align_to_dummy_fallback(new_mapping, write_to_file=True)
def test_distances(self):
# These mapppings are chosen to create different generated mappings
# from the various distances.
src_mappings = [
{
"in": "ᐧᐯ",
"out": "pʷeː"
},
{
"in": "ᒋ",
"out": "t͡ʃi"
},
{
"in": "ᕃ",
"out": "ʁaj"
},
]
src_mapping = Mapping(src_mappings, in_lang="crj", out_lang="crj-ipa")
mapping = create_mapping(src_mapping, self.target_mapping)
# print("mapping", mapping, list(mapping), "distance", "default")
self.assertTrue(isinstance(mapping, Mapping))
set_of_mappings = {tuple(m["out"] for m in mapping)}
for distance in DISTANCE_METRICS:
mapping = create_mapping(src_mapping,
self.target_mapping,
distance=distance)
# print("mapping", mapping, list(mapping), "distance", distance)
self.assertTrue(isinstance(mapping, Mapping))
set_of_mappings.add(tuple(m["out"] for m in mapping))
mapping = create_multi_mapping([(src_mapping, "out")],
[(self.target_mapping, "in")],
distance=distance)
self.assertTrue(isinstance(mapping, Mapping))
set_of_mappings.add(tuple(m["out"] for m in mapping))
self.assertGreater(len(set_of_mappings), 3)
def test_bigram_mappings(self):
src_mappings = [
{
"in": "ᐱ",
"out": "pi"
},
{
"in": "ᑎ",
"out": "ti"
},
{
"in": "ᑭ",
"out": "ki"
},
]
src_mapping = Mapping(src_mappings, in_lang='crj', out_lang='crj-ipa')
mapping = create_mapping(src_mapping, self.target_mapping)
transducer = Transducer(mapping)
self.assertEqual(transducer('pi'), 'pi')
self.assertEqual(transducer('ti'), 'ti')
self.assertEqual(transducer('ki'), 'ki')
def test_unigram_mappings(self):
src_mappings = [
{
"in": "ᐃ",
"out": "i"
},
{
"in": "ᐅ",
"out": "u"
},
{
"in": "ᐊ",
"out": "a"
},
]
src_mapping = Mapping(src_mappings, in_lang='crj', out_lang='crj-ipa')
mapping = create_mapping(src_mapping, self.target_mapping)
transducer = Transducer(mapping)
self.assertEqual(transducer('a'), 'ɑ')
self.assertEqual(transducer('i'), 'i')
self.assertEqual(transducer('u'), 'u')
def test_long_mappings(self):
src_mappings = [
{
"in": "ᐧᐯ",
"out": "pʷeː"
},
{
"in": "ᐧᑌ",
"out": "tʷeː"
},
{
"in": "ᐧᑫ",
"out": "kʷeː"
},
]
src_mapping = Mapping(src_mappings, in_lang='crj', out_lang='crj-ipa')
mapping = create_mapping(src_mapping, self.target_mapping)
transducer = Transducer(mapping)
self.assertEqual(transducer('pʷeː'), 'pweː')
self.assertEqual(transducer('tʷeː'), 'tweː')
self.assertEqual(transducer('kʷeː'), 'kweː')
def test_trigram_mappings(self):
src_mappings = [
{
"in": "ᒋ",
"out": "t͡ʃi"
},
{
"in": "ᒍ",
"out": "t͡ʃu"
},
{
"in": "ᒐ",
"out": "t͡ʃa"
},
]
src_mapping = Mapping(src_mappings, in_lang='crj', out_lang='crj-ipa')
mapping = create_mapping(src_mapping, self.target_mapping)
transducer = Transducer(mapping)
self.assertEqual(transducer('t͡ʃi'), 'tʃi')
self.assertEqual(transducer('t͡ʃu'), 'tʃu')
self.assertEqual(transducer('t͡ʃa'), 'tʃɑ')
def test_bigram_mappings(self):
src_mappings = [
{
"in": "ᐱ",
"out": "pi"
},
{
"in": "ᑎ",
"out": "ti"
},
{
"in": "ᑭ",
"out": "ki"
},
]
src_mapping = Mapping(src_mappings, in_lang="crj", out_lang="crj-ipa")
mapping = create_mapping(src_mapping, self.target_mapping)
transducer = Transducer(mapping)
self.assertEqual(transducer("pi").output_string, "pi")
self.assertEqual(transducer("ti").output_string, "ti")
self.assertEqual(transducer("ki").output_string, "ki")
def test_unigram_mappings(self):
src_mappings = [
{
"in": "ᐃ",
"out": "i"
},
{
"in": "ᐅ",
"out": "u"
},
{
"in": "ᐊ",
"out": "a"
},
]
src_mapping = Mapping(src_mappings, in_lang="crj", out_lang="crj-ipa")
mapping = create_mapping(src_mapping, self.target_mapping)
transducer = Transducer(mapping)
self.assertEqual(transducer("a").output_string, "ɑ")
self.assertEqual(transducer("i").output_string, "i")
self.assertEqual(transducer("u").output_string, "u")
def test_long_mappings(self):
src_mappings = [
{
"in": "ᐧᐯ",
"out": "pʷeː"
},
{
"in": "ᐧᑌ",
"out": "tʷeː"
},
{
"in": "ᐧᑫ",
"out": "kʷeː"
},
]
src_mapping = Mapping(src_mappings, in_lang="crj", out_lang="crj-ipa")
mapping = create_mapping(src_mapping, self.target_mapping)
transducer = Transducer(mapping)
self.assertEqual(transducer("pʷeː").output_string, "pweː")
self.assertEqual(transducer("tʷeː").output_string, "tweː")
self.assertEqual(transducer("kʷeː").output_string, "kweː")
def test_trigram_mappings(self):
src_mappings = [
{
"in": "ᒋ",
"out": "t͡ʃi"
},
{
"in": "ᒍ",
"out": "t͡ʃu"
},
{
"in": "ᒐ",
"out": "t͡ʃa"
},
]
src_mapping = Mapping(src_mappings, in_lang="crj", out_lang="crj-ipa")
mapping = create_mapping(src_mapping, self.target_mapping)
transducer = Transducer(mapping)
self.assertEqual(transducer("t͡ʃi").output_string, "tʃi")
self.assertEqual(transducer("t͡ʃu").output_string, "tʃu")
self.assertEqual(transducer("t͡ʃa").output_string, "tʃɑ")
def generate_mapping(
in_lang,
out_lang,
dummy,
ipa,
list_dummy,
out_dir,
merge,
from_langs,
to_langs,
distance,
):
""" Generate a new mapping from existing mappings in the g2p system.
This command has different modes of operation.
Standard mode:
g2p generate-mapping [--dummy|--ipa] IN_LANG [OUT_LANG]
For specified IN_LANG, generate a mapping from IN_LANG-ipa to eng-ipa,
or from IN_LANG-ipa to a dummy minimalist phone inventory. This assumes
the mapping IN_LANG -> IN_LANG-ipa exists and creates a mapping from its
output inventory.
To generate a mapping from IN_LANG-ipa to eng-ipa from a mapping
following a different patterns, e.g., from crl-equiv -> crl-ipa, specify
both IN_LANG (crl-equiv in this example) and OUT_LANG (crl-ipa in this
example).
\b
Sample usage:
Generate Algonquin IPA to English IPA from alq -> alq-ipa:
g2p generate-mapping --ipa alq
Generate Mohawk IPA to English IPA from moh-equiv -> moh-ipa:
g2p generate-mapping --ipa moh-equiv moh-ipa
Generate Michif IPA to English IPA from the union of crg-dv -> crg-ipa
and crg-tmd -> crg-ipa:
g2p generate-mapping --ipa --merge crg-dv:crg-tmd crg-ipa
List the dummy inventory used by --dummy:
g2p generate-mapping --list-dummy
From/to IPA mode:
\b
g2p generate-mapping --from FROM_L1 --to TO_L1
g2p generate-mapping --from FROM_L1:FROM_L2:... --to TO_L1:TO_L2:...
Generate an IPA mapping from the union of FROM_L1-ipa, FROM-L2-ipa, etc to
the union of TO_L1-ipa, TO-L2-ipa, etc. One or more from/to language
code(s) can be specified in colon- or comma-separated lists. Note, by default
we use Panphon's weighted_feature_edit_distance, but you can change this with
the --distance argument
\b
Sample usage:
Generate a mapping from kwk-ipa to moh-ipa based on all mappings into
kwk-ipa and moh-ipa:
g2p generate-mapping --from kwk --to moh
Generate a mapping from eng-ipa to crg-ipa based only on crg-dv -> crg-ipa:
g2p generate-mapping --from eng --to crg-dv_to_crg-ipa
Generate a mapping from kwk-ipa to moh-ipa+crg-ipa+eng-ipa based on
all mappings into kwk-ipa (from side) and the union of all mappings
into moh-ipa and crg-ipa plus eng-ipa_to_eng-arpabet (to side):
g2p generate-mapping --from kwk --to moh:crg:eng
Full syntax for specifying FROM_Ln and TO_Ln:
\b
lang (i.e., 3-letter code):
- If there is only one mapping into lang-ipa, "lang" refers to the
output of that mapping, e.g., "fra" means "fra_to_fra-ipa[out]".
- If there are several mappings into lang-ipa, "lang" refers to the
union of the outputs of those mappings, e.g., "moh" means the union
of "moh-equiv_to_moh-ipa[out]" and "moh-festival_to_moh-ipa[out]".
- It is an error if there are no mappings into lang-ipa.
- Only mappings from non-IPA to IPA are considered (i.e., IPA-to-IPA
mappings created by this command will not be included: use the
longer syntax below if you want to use them).
- Special case: "eng" refers to "eng-ipa_to_eng-arpabet[in]".
\b
in-lang_to_out-lang[[in]|[out]]:
- This expanded syntax is used to avoid the union when it is not
desired, e.g., "moh-equiv_to_moh-ipa" refers only to
"moh-equiv_to_moh-ipa,out" rather than the union "moh" represents.
- If out-lang is IPA, the output inventory is used; else if in-lang
is IPA, the input inventory is used; it is an error if neither
language is IPA.
- Specify "[in]" or "[out]" to override the above default.
- "_to_" is the joiner used to specify "the mapping from 'in-lang' to
'out-lang'" in the g2p network, regardless of the name of the file
it is stored in.
If you just modified or created the mappings from which the new mapping is
to be generated, don't forget to call "g2p update" first, so that "g2p
generate-mapping" can see the latest version.
Call "g2p update" again after calling "g2p generate-mapping" to compile
the newly generated mapping and make it available.
Note: exactly one of --ipa, --dummy, --from/--to, or --list-dummy is
required.
You can list available mappings with "g2p doctor --list-ipa", or by
visiting http://g2p-studio.herokuapp.com/api/v1/langs .
"""
# Make sure only one mode was specified on the command line
mode_count = ((1 if ipa else 0) + (1 if dummy else 0) +
(1 if list_dummy else 0) + (1 if
(from_langs or to_langs) else 0))
if mode_count == 0:
raise click.UsageError(
"Nothing to do! Please specify at least one of --ipa, --dummy, "
"--list-dummy, or --from/--to.")
if mode_count > 1:
raise click.UsageError(
"Multiple modes selected. Choose only one of --ipa, --dummy, "
"--list-dummy, or --from/--to.")
if list_dummy or from_langs is not None or to_langs is not None:
if in_lang is not None:
raise click.UsageError(
"IN_LANG is not allowed with --list-dummy or --from/--too", )
if from_langs is not None or to_langs is not None:
if from_langs is None or to_langs is None:
raise click.UsageError("--from and --to must be used together")
if merge:
if not ipa and not dummy:
raise click.UsageError(
"--merge is only compatible with --ipa and --dummy.")
if out_lang is None:
raise click.UsageError("OUT_LANG is required with --merge.")
if out_dir and not os.path.isdir(out_dir):
raise click.BadParameter(
f'Output directory "{out_dir}" does not exist. Cannot write mapping.',
param_hint="--out-dir",
)
if list_dummy:
# --list-dummy mode
print("Dummy phone inventory: {}".format(DUMMY_INVENTORY))
elif ipa or dummy:
# --ipa and --dummy modes
if in_lang is None:
raise click.UsageError("Missing argument 'IN_LANG'.")
if merge:
in_langs = in_lang.split(":")
else:
in_langs = [in_lang]
in_lang_choices = [
x for x in LANGS_NETWORK.nodes
if not is_ipa(x) and not is_xsampa(x)
]
for l in in_langs:
if l not in in_lang_choices:
raise click.UsageError(
f'Invalid value for IN_LANG: "{l}".\n'
"IN_LANG must be a non-IPA language code with an existing IPA mapping, "
f"i.e., one of:\n{', '.join(in_lang_choices)}.")
out_lang_choices = [x for x in LANGS_NETWORK.nodes if is_ipa(x)]
if out_lang is None:
out_lang = f"{in_lang}-ipa"
elif out_lang not in out_lang_choices:
raise click.UsageError(
f'Invalid value for OUT_LANG: "{out_lang}".\n'
"OUT_LANG must be an IPA language code with an existing mapping from IN_LANG, "
f"i.e., one of:\n{', '.join(out_lang_choices)}")
source_mappings = []
for l in in_langs:
try:
source_mapping = Mapping(in_lang=l, out_lang=out_lang)
except MappingMissing as e:
raise click.BadParameter(
f'Cannot find IPA mapping from "{l}" to "{out_lang}": {e}',
param_hint=["IN_LANG", "OUT_LANG"],
)
source_mappings.append(source_mapping)
if ipa:
check_ipa_known_segs([f"{in_lang}-ipa"])
eng_ipa = Mapping(in_lang="eng-ipa", out_lang="eng-arpabet")
click.echo(f"Writing English IPA mapping for {out_lang} to file")
new_mapping = create_mapping(source_mappings[0],
eng_ipa,
distance=distance)
for m in source_mappings[1:]:
new_mapping.extend(
create_mapping(m, eng_ipa, distance=distance))
else: # dummy
click.echo(
f"Writing dummy fallback mapping for {out_lang} to file")
new_mapping = align_to_dummy_fallback(source_mappings[0],
distance=distance)
for m in source_mappings[1:]:
new_mapping.extend(
align_to_dummy_fallback(m, distance=distance))
new_mapping.deduplicate()
if out_dir:
new_mapping.config_to_file(out_dir)
new_mapping.mapping_to_file(out_dir)
else:
new_mapping.config_to_file()
new_mapping.mapping_to_file()
elif from_langs is not None:
# --from/--to mode
assert to_langs is not None
from_mappings = []
for from_lang in re.split(r"[:,]", from_langs):
from_mappings.extend(parse_from_or_to_lang_spec(from_lang))
to_mappings = []
for to_lang in re.split(r"[:,]", to_langs):
to_mappings.extend(parse_from_or_to_lang_spec(to_lang))
if not from_mappings:
raise click.UsageError(
f'Invalid --from value "{from_langs}": no mappings found.')
if not to_mappings:
raise click.UsageError(
f'Invalid --to value "{to_langs}": no mappings found.')
for from_mapping, in_or_out in from_mappings:
LOGGER.info(
f'From mapping: {from_mapping.kwargs["in_lang"]}_to_{from_mapping.kwargs["out_lang"]}[{in_or_out}]'
)
for to_mapping, in_or_out in to_mappings:
LOGGER.info(
f'To mapping: {to_mapping.kwargs["in_lang"]}_to_{to_mapping.kwargs["out_lang"]}[{in_or_out}]'
)
new_mapping = create_multi_mapping(from_mappings,
to_mappings,
distance=distance)
if out_dir:
new_mapping.config_to_file(out_dir)
new_mapping.mapping_to_file(out_dir)
else:
new_mapping.config_to_file()
new_mapping.mapping_to_file()
