def jyutping2tipa(jp_str):
"""
Convert *jp_str* to a list of LaTeX TIPA strings.
"""
jp_parsed_list = parse_jyutping(jp_str)
tipa_list = []
for jp_parsed in jp_parsed_list:
onset = jp_parsed[0]
# TODO: Separate "final" as "nucleus" and "coda" instead?
final = jp_parsed[1] + jp_parsed[2]
tone = jp_parsed[3]
tipa = ONSETS_TIPA[onset] + FINALS_TIPA[final]
tipa = tipa.strip() + TONES_TIPA[tone]
tipa_list.append(tipa)
return tipa_list
def jyutping2tipa(jp_str):
"""
Convert *jp_str* to a list of LaTeX TIPA strings.
"""
jp_parsed_list = parse_jyutping(jp_str)
tipa_list = []
for jp_parsed in jp_parsed_list:
onset = jp_parsed[0]
# TODO: Separate "final" as "nucleus" and "coda" instead?
final = jp_parsed[1] + jp_parsed[2]
tone = jp_parsed[3]
tipa = ONSETS_TIPA[onset] + FINALS_TIPA[final]
tipa = tipa.strip() + TONES_TIPA[tone]
tipa_list.append(tipa)
return tipa_list
def jyutping2yale(jp_str, as_list=False):
"""
Convert *jp_str* to Yale.
:param as_list: If True (default: False), return a list of Yale strings
for individual syllables.
"""
if PY2 and isinstance(jp_str, str): # pragma: no cover
jp_str = unicode(jp_str) # noqa F821 ('unicode' undefined in py >= 3)
jp_parsed_list = parse_jyutping(jp_str)
yale_list = []
for jp_parsed in jp_parsed_list:
onset = ONSETS_YALE[jp_parsed[0]]
nucleus = NUCLEI_YALE[jp_parsed[1]]
coda = CODAS_YALE[jp_parsed[2]]
tone = jp_parsed[3] # still in parse_jyutping
# jyutping2yale system uses "h" to mark the three low tones
if tone in {"4", "5", "6"}:
low_tone_h = "h"
else:
low_tone_h = ""
# in jyutping2yale, long "aa" vowel with no coda is denoted by "a"
if nucleus == "aa" and coda == "":
nucleus = "a"
# when nucleus is "yu"...
# 1. disallow "yyu" (when onset is "y")
# 2. change nucleus "yu" into "u" -- this is a hack for adding tone
# diacritic, since we don't want "y" to bear the diacritic
if nucleus == "yu":
if onset == "y":
onset = ""
nucleus = "u"
# when nucleus is "ng"
# the tone diacritic has to be on "g" but not "n"
# now we pretend that the nucleus is "g", and will prepend the "n" back
# at the end
if nucleus == 'ng':
nucleus = 'g'
# add the jyutping2yale tone diacritic to the first nucleus letter
# parse_jyutping tone 1 --> add macron
# parse_jyutping tone 2 or 5 --> add acute
# parse_jyutping tone 4 --> add grave
# parse_jyutping tone 3 or 6 --> (no diacritic)
# If the accented letter doesn't exist in unicode, use the combining
# accent instead.
letter = nucleus[0] # nucleus 1st letter
unicode_letter_name = unicodedata.name(letter)
if tone == "1":
try:
letter_with_diacritic = unicodedata.lookup(
unicode_letter_name + " WITH MACRON")
except KeyError:
letter_with_diacritic = letter + "\u0304"
elif tone in {"2", "5"}:
try:
letter_with_diacritic = unicodedata.lookup(
unicode_letter_name + " WITH ACUTE")
except KeyError:
letter_with_diacritic = letter + "\u0301"
elif tone == "4":
try:
letter_with_diacritic = unicodedata.lookup(
unicode_letter_name + " WITH GRAVE")
except KeyError:
letter_with_diacritic = letter + "\u0300"
else:
# either tone 3 or tone 6
letter_with_diacritic = letter
nucleus = letter_with_diacritic + nucleus[1:]
# add back "y" if the nucleus is "yu"
# ("y" was taken away for convenience in adding tone diacritic)
if jp_parsed[1] == "yu":
nucleus = "y" + nucleus
# add back "n" if the nucleus is "ng"
# ('n' was taken away so that tone diacritic is on "g" but not "n")
if jp_parsed[1] == 'ng':
nucleus = 'n' + nucleus
# parse_jyutping final "eu" should be jyutping2yale "ew" (not "eu")
if coda == "u" and nucleus == "e":
coda = "w"
# save the resultant jyutping2yale
if coda in {"i", "u", "w"} and tone in {"4", "5", "6"}:
yale = onset + nucleus + coda + low_tone_h
else:
yale = onset + nucleus + low_tone_h + coda
yale_list.append(yale)
if as_list:
return yale_list
# Output yale_list as a string
# Check if there's potential ambiguity when Yale strings are concatenated
# Ambiguity case 1:
# 1st syllable coda is one of the "ambiguous_consonants"
# and 2nd syllable starts with a vowel *letter*
# Ambiguity case 2:
# 1st syllable has no coda and 2nd syllable starts with one of the
# "ambiguous_consonants"
# e.g., hei3hau6 'climate' --> heihauh
# (middle "h" for tone in 1st syllable or being onset of 2nd syllable?)
if len(yale_list) == 1:
return yale_list[0]
ambiguous_consonants = {'h', 'p', 't', 'k', 'm', 'n', 'ng'}
vowel_letters = {'a', 'e', 'i', 'o', 'u',
'á', 'é', 'í', 'ó', 'ú',
'à', 'è', 'ì', 'ò', 'ù',
'ā', 'ē', 'ī', 'ō', 'ū'}
output_str = ''
for i in range(len(yale_list) - 1):
yale1 = yale_list[i]
yale2 = yale_list[i+1]
ambiguous = False
# test case 1:
if endswithoneof(yale1, ambiguous_consonants) and \
startswithoneof(yale2, vowel_letters):
ambiguous = True
# test case 2:
if not ambiguous and \
not endswithoneof(yale1, ambiguous_consonants) and \
startswithoneof(yale2, ambiguous_consonants):
ambiguous = True
output_str += yale1
if ambiguous:
output_str += '\''
output_str += yale_list[-1]
return output_str
def perform_search(fn_to_tagged_sents,
onset=None,
nucleus=None,
coda=None,
tone=None,
initial=None,
final=None,
jyutping=None,
character=None,
pos=None,
word_range=(0, 0),
sent_range=(0, 0),
tagged=True,
sents=False):
"""
overall strategy: deal with jp (and all jp-related elements) first, and
then the character
1. jp
hierarchy of jp and associated search elements:
jp
/ | \
onset/initial final tone
/ \
nucleus coda
lower search elements cannot be used together with dominating higher
elements
"""
# ensure tuple type: word_range and sent_range
if not (type(word_range) == type(sent_range) == tuple):
raise ValueError('word_range and sent_range must be tuples')
words_left, words_right = word_range
sents_left, sents_right = sent_range
# ensure int type: words_left, words_right, sents_left, sents_right
if not (type(words_left) == type(words_right) == type(sents_left) ==
type(sents_right) == int):
raise ValueError('int required for {words, sents}_{left, right}')
if sents_left > 0 or sents_right > 0:
sents = True
# determine what kinds of search we are doing
character_search = False
jp_search = False
pos_search = False
if character:
character_search = True
if onset or nucleus or coda or tone or final or jyutping:
jp_search = True
if pos:
pos_search = True
if not (character_search or jp_search or pos_search):
raise ValueError('no search elements')
# check if jyutping search is valid
jp_search_tuple = (None, None, None, None)
if jp_search:
# ensure compatible jyutping search elements
if final and (nucleus or coda):
raise ValueError('final cannot be used together with '
'either nucleus or coda (or both)')
if jyutping and (onset or final or nucleus or coda or tone):
raise ValueError('jyutping cannot be used together with other '
'Jyutping elements')
if (onset != initial) and onset and initial:
raise ValueError('onset conflicts with initial')
# onset/initial
if initial:
onset = initial
# determine jp_search_tuple
if jyutping:
try:
jp_search_list = parse_jyutping(jyutping)
except ValueError:
raise ValueError('invalid jyutping -- %s' % (repr(jyutping)))
if len(jp_search_list) > 1:
raise ValueError('only jyutping for one character is allowed')
else:
jp_search_tuple = jp_search_list[0]
else:
if final:
nucleus, coda = parse_final(final)
jp_search_tuple = (onset, nucleus, coda, tone)
fn_to_results = {}
for fn, tagged_sents in fn_to_tagged_sents.items():
sent_word_index_pairs = []
for i_sent, tagged_sent in enumerate(tagged_sents):
for i_word, tagged_word in enumerate(tagged_sent):
c_characters, c_pos, c_mor, _ = tagged_word # c = current
c_jyutping = get_jyutping_from_mor(c_mor)
# determine character_search and pos_search
if character_search:
character_match = character in c_characters
else:
character_match = True
# if 'V' in c_pos.upper():
# import pdb; pdb.set_trace()
if pos_search:
pos_match = bool(re.search(pos, c_pos))
else:
pos_match = True
if not (character_match and pos_match):
continue
# determine if jyutping matches c_jyutping
jyutping_match = False
if not jp_search:
jyutping_match = True
elif not c_jyutping:
pass
else:
try:
c_parsed_jyutpings = parse_jyutping(c_jyutping)
except ValueError:
continue
for c_parsed_jyutping in c_parsed_jyutpings:
booleans = [
_jp_element_match(search_, current_)
for search_, current_ in zip(
jp_search_tuple, c_parsed_jyutping)
]
if all(booleans):
jyutping_match = True
break
if jyutping_match:
sent_word_index_pairs.append((i_sent, i_word))
results_list = []
for i_sent, i_word in sent_word_index_pairs:
if not sents:
tagged_sent = tagged_sents[i_sent]
i_word_start = i_word - words_left
i_word_end = i_word + words_right + 1
if i_word_start < 0:
i_word_start = 0
if i_word_end > len(tagged_sent):
i_word_end = len(tagged_sent)
words_wanted = tagged_sent[i_word_start:i_word_end]
if not tagged:
words_wanted = [x[0] for x in words_wanted]
if len(words_wanted) == 1:
words_wanted = words_wanted[0]
results_list.append(words_wanted)
else:
i_sent_start = i_sent - sents_left
i_sent_end = i_sent + sents_right + 1
if i_sent_start < 0:
i_sent_start = 0
if i_sent_end > len(tagged_sents):
i_sent_end = len(tagged_sents)
sents_wanted = tagged_sents[i_sent_start:i_sent_end]
if not tagged:
for i, sent in enumerate(sents_wanted[:]):
sents_wanted[i] = [x[0] for x in sent]
if len(sents_wanted) == 1:
sents_wanted = sents_wanted[0]
results_list.append(sents_wanted)
fn_to_results[fn] = results_list
return fn_to_results
def test_unicode_str_compatibility():
assert parse_jyutping('wui5') == [('w', 'u', 'i', '5')]
assert parse_jyutping(u'wui5') == [('w', 'u', 'i', '5')] # note prefix u
def test_no_noda():
assert parse_jyutping('gaa1') == [('g', 'aa', '', '1')]
def test_coda_ng():
assert parse_jyutping('hoeng1') == [('h', 'oe', 'ng', '1')]
def test_invalid_onset():
with pytest.raises(ValueError) as e:
parse_jyutping('shaa1')
assert 'onset error' in str(e.value)
def test_invalid_nucleus():
with pytest.raises(ValueError) as e:
parse_jyutping('sk3')
assert 'nucleus error' in str(e.value)
def test_invalid_coda():
with pytest.raises(ValueError) as e:
parse_jyutping('leil3')
assert 'coda error' in str(e.value)
def test_fewer_than_2_characters():
with pytest.raises(ValueError) as e:
parse_jyutping('3')
assert 'fewer than 2 characters' in str(e.value)
def test_no_tone():
with pytest.raises(ValueError) as e:
parse_jyutping('lei')
assert 'tone error' in str(e.value)
def test_syllabic_nasals():
# TODO assert parse_jyutping('hm4') == [('h', 'm', '', '4')]
assert parse_jyutping('ng5') == [('', 'ng', '', '5')]
assert parse_jyutping('m4') == [('', 'm', '', '4')]
assert parse_jyutping('n3') == [('', 'n', '', '3')]
def test_wrong_data_type():
with pytest.raises(ValueError):
parse_jyutping(123)
def test_basic_case_gwong2dung1waa2():
assert parse_jyutping('gwong2dung1waa2') == [
('gw', 'o', 'ng', '2'), ('d', 'u', 'ng', '1'), ('w', 'aa', '', '2')
]
def jyutping2yale(jp_str, as_list=False):
"""
Convert *jp_str* to Yale.
:param as_list: If True (default: False), return a list of Yale strings
for individual syllables.
"""
if PY2 and isinstance(jp_str, str): # pragma: no cover
jp_str = unicode(jp_str) # noqa F821 ('unicode' undefined in py >= 3)
jp_parsed_list = parse_jyutping(jp_str)
yale_list = []
for jp_parsed in jp_parsed_list:
onset = ONSETS_YALE[jp_parsed[0]]
nucleus = NUCLEI_YALE[jp_parsed[1]]
coda = CODAS_YALE[jp_parsed[2]]
tone = jp_parsed[3] # still in parse_jyutping
# jyutping2yale system uses "h" to mark the three low tones
if tone in {"4", "5", "6"}:
low_tone_h = "h"
else:
low_tone_h = ""
# in jyutping2yale, long "aa" vowel with no coda is denoted by "a"
if nucleus == "aa" and coda == "":
nucleus = "a"
# when nucleus is "yu"...
# 1. disallow "yyu" (when onset is "y")
# 2. change nucleus "yu" into "u" -- this is a hack for adding tone
# diacritic, since we don't want "y" to bear the diacritic
if nucleus == "yu":
if onset == "y":
onset = ""
nucleus = "u"
# when nucleus is "ng"
# the tone diacritic has to be on "g" but not "n"
# now we pretend that the nucleus is "g", and will prepend the "n" back
# at the end
if nucleus == 'ng':
nucleus = 'g'
# add the jyutping2yale tone diacritic to the first nucleus letter
# parse_jyutping tone 1 --> add macron
# parse_jyutping tone 2 or 5 --> add acute
# parse_jyutping tone 4 --> add grave
# parse_jyutping tone 3 or 6 --> (no diacritic)
# If the accented letter doesn't exist in unicode, use the combining
# accent instead.
letter = nucleus[0] # nucleus 1st letter
unicode_letter_name = unicodedata.name(letter)
if tone == "1":
try:
letter_with_diacritic = unicodedata.lookup(
unicode_letter_name + " WITH MACRON")
except KeyError:
letter_with_diacritic = letter + "\u0304"
elif tone in {"2", "5"}:
try:
letter_with_diacritic = unicodedata.lookup(
unicode_letter_name + " WITH ACUTE")
except KeyError:
letter_with_diacritic = letter + "\u0301"
elif tone == "4":
try:
letter_with_diacritic = unicodedata.lookup(
unicode_letter_name + " WITH GRAVE")
except KeyError:
letter_with_diacritic = letter + "\u0300"
else:
# either tone 3 or tone 6
letter_with_diacritic = letter
nucleus = letter_with_diacritic + nucleus[1:]
# add back "y" if the nucleus is "yu"
# ("y" was taken away for convenience in adding tone diacritic)
if jp_parsed[1] == "yu":
nucleus = "y" + nucleus
# add back "n" if the nucleus is "ng"
# ('n' was taken away so that tone diacritic is on "g" but not "n")
if jp_parsed[1] == 'ng':
nucleus = 'n' + nucleus
# parse_jyutping final "eu" should be jyutping2yale "ew" (not "eu")
if coda == "u" and nucleus == "e":
coda = "w"
# save the resultant jyutping2yale
if coda in {"i", "u", "w"} and tone in {"4", "5", "6"}:
yale = onset + nucleus + coda + low_tone_h
else:
yale = onset + nucleus + low_tone_h + coda
yale_list.append(yale)
if as_list:
return yale_list
# Output yale_list as a string
# Check if there's potential ambiguity when Yale strings are concatenated
# Ambiguity case 1:
# 1st syllable coda is one of the "ambiguous_consonants"
# and 2nd syllable starts with a vowel *letter*
# Ambiguity case 2:
# 1st syllable has no coda and 2nd syllable starts with one of the
# "ambiguous_consonants"
# e.g., hei3hau6 'climate' --> heihauh
# (middle "h" for tone in 1st syllable or being onset of 2nd syllable?)
if len(yale_list) == 1:
return yale_list[0]
ambiguous_consonants = {'h', 'p', 't', 'k', 'm', 'n', 'ng'}
vowel_letters = {'a', 'e', 'i', 'o', 'u',
'á', 'é', 'í', 'ó', 'ú',
'à', 'è', 'ì', 'ò', 'ù',
'ā', 'ē', 'ī', 'ō', 'ū'}
output_str = ''
for i in range(len(yale_list) - 1):
yale1 = yale_list[i]
yale2 = yale_list[i+1]
ambiguous = False
# test case 1:
if endswithoneof(yale1, ambiguous_consonants) and \
startswithoneof(yale2, vowel_letters):
ambiguous = True
# test case 2:
if not ambiguous and \
not endswithoneof(yale1, ambiguous_consonants) and \
startswithoneof(yale2, ambiguous_consonants):
ambiguous = True
output_str += yale1
if ambiguous:
output_str += '\''
output_str += yale_list[-1]
return output_str