def get_structure_profile(wordlist,
column='ipa',
text=False,
semi_diacritics='hsʃʂʒʐzθɕʑfvθð',
debug=False,
language=None):
profile = defaultdict(list)
modify = lambda x: x
if column == 'ipa':
modify = lambda x: x.replace(' ', '_')
for idx, lang, segments in lingpy.iter_rows(wordlist, 'doculect', column):
if debug: print(idx, lang, segments)
if not language or language == lang:
for structure, morpheme in get_structure(
modify(segments), zipped=True,
semi_diacritics=semi_diacritics):
im, nc, t = [[], []], [[], []], [[], []]
for pos, seg in structure:
if pos in 'i':
im[0] += [pos]
im[1] += [seg]
elif pos in 'mnc':
nc[0] += [pos]
nc[1] += [seg]
else:
t[0] += [pos]
t[1] += [seg]
if im[0]:
profile[' '.join(im[0]),
' '.join(im[1])] += [(lang, morpheme)]
if nc[0]:
profile[' '.join(nc[0]),
' '.join(nc[1])] += [(lang, morpheme)]
if t[0]:
profile[' '.join(t[0]),
' '.join(t[1])] += [(lang, morpheme)]
for (pos, seg), langs_ in sorted(profile.items(),
key=lambda x: (x[0][0], len(x[1])),
reverse=True):
langs = [x[0] for x in langs_]
examples = [''.join(x[1]) for x in langs_]
if not text:
yield (seg.replace(' ',
''), seg, seg, pos, codepoint(s), len(langs),
','.join(sorted(set(langs), key=lambda x: langs.count(x))),
', '.join(examples[:5]))
else:
yield '\t'.join([
strip_chars(' ∼', seg), seg, seg, pos,
codepoint(seg),
str(len(langs)),
','.join(sorted(set(langs), key=lambda x: langs.count(x))),
', '.join(examples[:5])
])
def test_codepoint(self):
assert codepoint('á') == 'U+00e1'
def simple_profile(wordlist,
ref='ipa',
semi_diacritics='hsʃ̢ɕʂʐʑʒw',
merge_vowels=False,
brackets=None,
splitters='/,;~',
merge_geminates=True,
bad_word="<???>",
bad_sound="<?>",
clts=None,
unknown_sound="!{0}"):
"""
Create an initial Orthography Profile using Lingpy's clean_string procedure.
Parameters
----------
wordlist : ~lingpy.basic.wordlist.Wordlist
A wordlist from which you want to derive an initial
orthography profile.
ref : str (default="ipa")
The name of the reference column in which the words are stored.
semi_diacritics : str
Indicate characters which can occur both as "diacritics" (second part
in a sound) or alone.
merge_vowels : bool (default=True)
Indicate whether consecutive vowels should be merged.
brackets : dict
A dictionary with opening brackets as key and closing brackets as
values. Defaults to a pre-defined set of frequently occurring brackets.
splitters : str
The characters which force the automatic splitting of an entry.
clts : dict (default=None)
A dictionary(like) object that converts a given source sound into a
potential target sound, using the get()-method of the dictionary.
Normally, we think of a CLTS instance here (that is: a cross-linguistic
transcription system as defined in the pyclts package).
bad_word : str (default="«???»")
Indicate how words that could not be parsed should be handled. Note
that both "bad_word" and "bad_sound" are format-strings, so you can add
formatting information here.
bad_sound : str (default="«?»")
Indicate how sounds that could not be converted to a sound class be
handled. Note that both "bad_word" and "bad_sound" are format-strings,
so you can add formatting information here.
unknown_sound : str (default="!{0}")
If with_clts is set to True, use this string to indicate that sounds
are classified as "unknown sound" in the CLTS framework.
Returns
-------
profile : generator
A generator of tuples (three items), indicating the segment, its frequency,
the conversion to sound classes in the Dolgopolsky sound-class model,
and the unicode-codepoints.
"""
clts = clts or {}
nulls = set()
bad_words = set()
brackets = brackets or "([{『（₍⁽«)]}）』⁾₎"
profile = defaultdict(int)
words = [wordlist[idx, ref] for idx in wordlist]
for word in pb(words, desc='iterating over words'):
if isinstance(word, list):
word = ' '.join(word)
cleaned_string = clean_string(word,
semi_diacritics=semi_diacritics,
merge_vowels=merge_vowels,
brackets=None,
ignore_brackets=False,
split_entries=False,
preparse=None,
rules=None,
merge_geminates=merge_geminates)[0]
# retain whole word if there are splitters in the word
if [x for x in cleaned_string if x in brackets + splitters]:
profile[word] += 1
bad_words.add(word)
else:
for segment in cleaned_string.split(' '):
profile[segment] += 1
for segment in [x for x in word if x not in cleaned_string]:
profile[segment] += 1
nulls.add(segment)
for s, f in pb(sorted(profile.items(), key=lambda x: x[1], reverse=True),
desc='preparing profile'):
sclass = token2class(s, 'dolgo')
if s in bad_words:
ipa = bad_word.format(s)
elif sclass == '0' and s not in nulls:
ipa = bad_sound.format(s)
elif s in nulls:
ipa = 'NULL'
elif clts:
sound = clts.get(s, False)
if not sound:
ipa = '!' + s
else:
ipa = text_type(sound)
else:
ipa = s
yield s, ipa, text_type(f), codepoint(s)
def context_profile(wordlist,
ref='ipa',
col="doculect",
semi_diacritics='hsʃ̢ɕʂʐʑʒw',
merge_vowels=False,
brackets=None,
splitters='/,;~',
merge_geminates=True,
clts=False,
bad_word="<???>",
bad_sound="<?>",
unknown_sound="!{0}",
examples=2,
max_entries=100):
"""
Create an advanced Orthography Profile with context and doculect information.
Parameters
----------
wordlist : ~lingpy.basic.wordlist.Wordlist
A wordlist from which you want to derive an initial
orthography profile.
ref : str (default="ipa")
The name of the reference column in which the words are stored.
col : str (default="doculect")
Indicate in which column the information on the language variety is
stored.
semi_diacritics : str
Indicate characters which can occur both as "diacritics" (second part
in a sound) or alone.
merge_vowels : bool (default=True)
Indicate whether consecutive vowels should be merged.
brackets : dict
A dictionary with opening brackets as key and closing brackets as
values. Defaults to a pre-defined set of frequently occurring brackets.
splitters : str
The characters which force the automatic splitting of an entry.
clts : dict (default=None)
A dictionary(like) object that converts a given source sound into a
potential target sound, using the get()-method of the dictionary.
Normally, we think of a CLTS instance here (that is: a cross-linguistic
transcription system as defined in the pyclts package).
bad_word : str (default="«???»")
Indicate how words that could not be parsed should be handled. Note
that both "bad_word" and "bad_sound" are format-strings, so you can add
formatting information here.
bad_sound : str (default="«?»")
Indicate how sounds that could not be converted to a sound class be
handled. Note that both "bad_word" and "bad_sound" are format-strings,
so you can add formatting information here.
unknown_sound : str (default="!{0}")
If with_clts is set to True, use this string to indicate that sounds
are classified as "unknown sound" in the CLTS framework.
examples : int(default=2)
Indicate the number of examples that should be printed out.
Returns
-------
profile : generator
A generator of tuples (three items), indicating the segment, its frequency,
the conversion to sound classes in the Dolgopolsky sound-class model,
and the unicode-codepoints.
"""
clts_ = clts or {}
nulls = set()
bad_words = set()
brackets = brackets or "([{『（₍⁽«)]}）』⁾₎"
profile = defaultdict(list)
errors = set()
for idx, word, language in pb(wordlist.iter_rows(ref, col),
desc='iter words',
total=len(wordlist)):
log.info('processing {0}-{1}'.format(idx, word))
if isinstance(word, list):
word = ' '.join(word)
if word.strip():
try:
cleaned_string = clean_string(
word,
semi_diacritics=semi_diacritics,
merge_vowels=merge_vowels,
brackets=None,
ignore_brackets=False,
split_entries=False,
preparse=None,
rules=None,
merge_geminates=merge_geminates)[0].split(' ')
# retain whole word if there are splitters in the word
if [x for x in cleaned_string if x in brackets + splitters]:
profile[word] += [(language, word)]
bad_words.add(word)
else:
context_pre = ['^'] + (len(cleaned_string) - 1) * ['']
context_post = (len(cleaned_string) - 1) * [''] + ['$']
for ctxA, ctxB, segment in zip(context_pre, context_post,
cleaned_string):
profile[ctxA + segment + ctxB] += [(language, word)]
for segment in [
x for x in word
if x not in ' '.join(cleaned_string)
]:
profile[segment] += [(language, word)]
nulls.add(segment)
except:
errors.add(idx)
log.warn('problem parsing {0}'.format(word))
for s in '^$':
yield s, 'NULL', '', '', '', ''
for idx, (s, entries) in pb(enumerate(
sorted(profile.items(), key=lambda x: len(x[1]), reverse=True)),
desc='yielding entries',
total=len(profile)):
sclass = token2class(s.strip('^$'), 'dolgo')
words, langs = [l[1] for l in entries
][:max_entries], [l[0] for l in entries][:max_entries]
languages = ', '.join(
sorted(set(langs), key=lambda x: langs.count(x), reverse=True))
frequency = str(len(langs))
codepoints = codepoint(s)
examples_ = ', '.join(
sorted(set(words), key=lambda x: words.count(x),
reverse=True)[:examples])
if s in bad_words:
ipa = bad_word.format(s)
elif sclass == '0':
ipa = bad_sound.format(s)
elif s in nulls:
ipa = 'NULL'
elif clts_:
sound = clts_.get(s.strip('^$'), False)
if not sound:
ipa = '!' + s.strip('^$')
else:
ipa = text_type(sound)
else:
ipa = s.strip('^$')
yield s, ipa, examples_, languages, frequency, codepoints
def test_codepoint(self):
assert codepoint('á') == 'U+00e1'
def test_codepoint():
from lingpy.sequence.sound_classes import codepoint
assert codepoint('á') == 'U+00e1'