def __init__(self): latin = deepcopy(LATIN) exceptions = self.get_exceptions() latin['exceptions'] = exceptions latin['diphthongs'] = ["ae", "au", "oe"] # Not: eu, ei latin['mute_consonants_and_f'].append('h') self.syllabifier = Syllabifier(latin)
def test_latin_syllabifier(self): """Test Latin syllabifier.""" word = 'sidere' syllabifier = Syllabifier() syllables = syllabifier.syllabify(word) target = ['si', 'de', 're'] self.assertEqual(syllables, target)
def test_latin_syllabifier(self): """Test Latin syllabifier.""" word = 'sidere' syllabifier = Syllabifier() syllables = syllabifier.syllabify(word) target = ['si', 'de', 're'] self.assertEqual(syllables, target)
def third_declension(self): from cltk.stem.latin.syllabifier import Syllabifier syllabifier = Syllabifier() vowels = ['a', 'e', 'i', 'o', 'u', 'ā', 'ē', 'ī', 'ō', 'ū'] base = DeclineNoun.id_declension(self)[1] forms = [self.nom, self.gen] endings = [ 'ī', 'em', 'e', 'blah', 'ēs', 'um', 'ibus', 'ēs', 'ibus', 'ēs' ] for i in endings: forms.append(base + i) forms[5] = self.nom nom_syllable = len(syllabifier.syllabify(self.nom)) gen_syllable = len(syllabifier.syllabify(self.gen)) i_stem = False if nom_syllable == gen_syllable: if self.nom[-2:] in ['is', 'es']: i_stem = True elif self.nom[-1] in ['x', 's']: if base[-1] not in vowels and base[-2] not in vowels: i_stem = True if i_stem == True: forms[7] = forms[7][:-2] + 'i' + forms[7][-2:] # if int(self.gender) == 3: if self.gender == 'neutrum': forms[5] = self.nom forms[3] = self.nom forms[6] = base + 'a' forms[9] = base + 'a' forms[11] = base + 'a' if self.nom[-1] == 'e' or self.nom[-2:] in ['al', 'ar']: forms[4] = base + 'ī' forms[6] = base + 'ia' forms[7] = base + 'ium' forms[9] = base + 'ia' forms[11] = base + 'ia' return forms
def test_latin_syllabifier(self): """Test Latin syllabifier.""" word = 'sidere' syllabifier = Syllabifier() syllables = syllabifier.syllabify(word) target = ['si', 'de', 're'] self.assertEqual(syllables, target) # tests for macronized words macronized_word = 'audītū' macronized_syllables = syllabifier.syllabify(macronized_word) macronized_target = ['au', 'dī', 'tū'] self.assertEqual(macronized_syllables, macronized_target) macronized_word2 = 'conjiciō' macronized_syllables2 = syllabifier.syllabify(macronized_word2) macronized_target2 = ['con', 'ji', 'ci', 'ō'] self.assertEqual(macronized_syllables2, macronized_target2) macronized_word3 = 'ā' macronized_syllables3 = syllabifier.syllabify(macronized_word3) macronized_target3 = ['ā'] self.assertEqual(macronized_syllables3, macronized_target3)
def test_latin_syllabifier(self): """Test Latin syllabifier.""" word = 'sidere' syllabifier = Syllabifier() syllables = syllabifier.syllabify(word) target = ['si', 'de', 're'] self.assertEqual(syllables, target) # tests for macronized words macronized_word = 'audītū' macronized_syllables = syllabifier.syllabify(macronized_word) macronized_target = ['au', 'dī', 'tū'] self.assertEqual(macronized_syllables, macronized_target) macronized_word2 = 'conjiciō' macronized_syllables2 = syllabifier.syllabify(macronized_word2) macronized_target2 = ['con', 'ji', 'ci', 'ō'] self.assertEqual(macronized_syllables2, macronized_target2) macronized_word3 = 'ā' macronized_syllables3 = syllabifier.syllabify(macronized_word3) macronized_target3 = ['ā'] self.assertEqual(macronized_syllables3, macronized_target3)
from cltk.stem.latin.syllabifier import Syllabifier from cltk.tokenize.word import WordTokenizer cato_agri_praef = "Est interdum praestare mercaturis rem quaerere, nisi tam periculosum sit, et item foenerari, si tam honestum. Maiores nostri sic habuerunt et ita in legibus posiverunt: furem dupli condemnari, foeneratorem quadrupli. Quanto peiorem civem existimarint foeneratorem quam furem, hinc licet existimare. Et virum bonum quom laudabant, ita laudabant: bonum agricolam bonumque colonum; amplissime laudari existimabatur qui ita laudabatur. Mercatorem autem strenuum studiosumque rei quaerendae existimo, verum, ut supra dixi, periculosum et calamitosum. At ex agricolis et viri fortissimi et milites strenuissimi gignuntur, maximeque pius quaestus stabilissimusque consequitur minimeque invidiosus, minimeque male cogitantes sunt qui in eo studio occupati sunt. Nunc, ut ad rem redeam, quod promisi institutum principium hoc erit." word_tokenizer = WordTokenizer('latin') cato_word_tokens = word_tokenizer.tokenize(cato_agri_praef) cato_word_tokens_no_punt = [ token for token in cato_word_tokens if token not in ['.', ',', ':', ';'] ] #print(cato_word_tokens_no_punt) syllabifier = Syllabifier() #for word in cato_word_tokens_no_punt: #syllables = syllabifier.syllabify(word) #print(word, syllables) ############################################################ #use the macronizer from cltk.prosody.latin.macronizer import Macronizer macronizer = Macronizer('tag_ngram_123_backoff') text = 'Quo usque tandem, O Catilina, abutere nostra patientia?' prose_text = macronizer.macronize_text(text) print(prose_text)
class ChantSyllabifier(metaclass=Singleton): def __init__(self): latin = deepcopy(LATIN) exceptions = self.get_exceptions() latin['exceptions'] = exceptions latin['diphthongs'] = ["ae", "au", "oe"] # Not: eu, ei latin['mute_consonants_and_f'].append('h') self.syllabifier = Syllabifier(latin) def get_exceptions(self): # See notebook "Identify syllabification errors" for background: # We checked the most frequent under/oversegmentations, and # manually corrected those undersegmented = { 'euouae': ['e', 'u', 'o', 'u', 'a', 'e'], # 29.43% 'quia': ['qui', 'a'], # 13.91% 'seuouae': ['se', 'u', 'o', 'u', 'a', 'e'], # 6.65% 'israel': ['is', 'ra', 'el'], # 2.64% 'cui': ['cu', 'i'], # 1.74% 'michael': ['mic', 'ha', 'el'], # 0.84% #'qui': ['qui'], # 0.50% 'requiem': ['re', 'qui', 'em'], # 0.41% 'huic': ['hu', 'ic'], # 0.41% #'jerusalem': ['je', 'ru', 'sa', 'lem'], # 0.38% # 'alleluia': ['al', 'le', 'lu', 'ia'], # 0.27% #'noe': ['noe'], # 0.22% 'requiescet': ['re', 'qui', 'es', 'cet'], # 0.21% 'exiit': ['ex', 'i', 'it'], # 0.17% 'exierunt': ['ex', 'i', 'e', 'runt'], # 0.13% 'eloquium': ['e', 'lo', 'qui', 'um'], # 0.12% 'exiet': ['ex', 'i', 'et'], # 0.12% # 'gelboe': ['gel', 'boe'], # 0.11% 'ierit': ['i', 'e', 'rit'], # 0.10% 'christi': ['chris', 'ti'], # 0.10% 'saul': ['sa', 'ul'], # 0.09% 'colloquiis': ['col', 'lo', 'qui', 'is'], # 0.09% 'israelita': ['is', 'ra', 'e', 'li', 'ta'], # 0.09% 'michaele': ['mic', 'ha', 'e', 'le'], # 0.08% 'requiescit': ['re', 'qui', 'es', 'cit'], # 0.08% 'obsequia': ['ob', 'se', 'qui', 'a'], # 0.07% # 'jesus': ['je', 'sus'], # 0.07% 'nicolaum': ['ni', 'co', 'laum'], # 0.06% 'requies': ['re', 'qui', 'es'], # 0.06% 'requiescunt': ['re', 'qui', 'es', 'c**t'], # 0.06% 'exierit': ['ex', 'i', 'e', 'rit'], # 0.06% 'michaelis': ['mic', 'ha', 'e', 'lis'], # 0.05% 'requiescent': ['re', 'qui', 'es', 'cent'], # 0.05% } # Recurring issues are "guen" and "quu" oversegmented = { 'sanguine': ['san', 'gui', 'ne'], # 1.45% 'sanguinem': ['san', 'gui', 'nem'], # 1.43% 'lingua': ['lin', 'gua'], # 1.11% 'alleluya': ['al', 'le', 'lu', 'ya'], # 0.88% 'sanguis': ['san', 'guis'], # 0.83% 'est*': ['est*'], # 0.64% #'eleison': ['e', 'le', 'i', 'son'], # 0.59% 'linguis': ['lin', 'guis'], # 0.59% 'linguae': ['lin', 'guae'], # 0.47% 'sequuntur': ['se', 'quun', 'tur'], # 0.42% 'sanguinis': ['san', 'gui', 'nis'], # 0.40% #'euge': ['e', 'u', 'ge'], # 0.29% 'eleemosynam': ['e', 'lee', 'mo', 'sy', 'nam'], # 0.27% 'iniquum': ['in', 'i', 'quum'], # 0.23% 'sunt*': ['sunt*'], # 0.23% 'unguenti': ['un', 'guen', 'ti'], # 0.21% 'persequuntur': ['per', 'se', 'quun', 'tur'], # 0.20% 'unguentum': ['un', 'guen', 'tum'], # 0.20% 'unguentorum': ['un', 'guen', 'to', 'rum'], # 0.16% 'urbs': ['urbs'], # 0.16% 'equuleo': ['e', 'quu', 'le', 'o'], # 0.15% #'perpetuum': ['per', 'pe', 'tu', 'um'], # 0.14% #'antiquus': ['an', 'ti', 'qu', 'us'], # 0.14% 'sanguinibus': ['san', 'gui', 'ni', 'bus'], # 0.13% 'eleemosyna': ['e', 'lee', 'mo', 'sy', 'na'], # 0.13% 'linguam': ['lin', 'guam'], # 0.13% 'stirps': ['stirps'], # 0.11% #'ait': ['a', 'it'], # 0.11% 'languores': ['lan', 'guo', 'res'], # 0.11% #'jerusalem': ['je', 'ru', 'sa', 'lem'], # 0.10% 'loquuntur': ['lo', 'quun', 'tur'], # 0.09% # 'tuum': ['tu', 'um'], # 0.09% # 'ideoque': ['i', 'de', 'o', 'que'], # 0.09% 'annuntiaverunt*': ['an', 'nun', 'ti', 'a', 've', 'runt*'], # 0.09% 'linguarum': ['lin', 'gua', 'rum'], # 0.09% 'in*': ['in*'], # 0.09% 'unguento': ['un', 'guen', 'to'], # 0.09% 'urguentes': ['ur', 'guen', 'tes'], # 0.09% 'langueo': ['lan', 'gue', 'o'], # 0.08% 'sanguinum': ['san', 'gui', 'num'], # 0.08% 'ihesum': ['ihe', 'sum'], # 0.08% 'languoribus': ['lan', 'guo', 'ri', 'bus'], # 0.07% 'probaverunt': ['pro', 'ba', 've', 'runt'], # 0.07% 'faciam': ['fa', 'ci', 'am'], # 0.07% #'equum': ['e', 'qu', 'um'], # 0.07% #'jerusalem*': ['je', 'ru', 'sa', 'lem*'], # 0.07% 'moyses': ['moy', 'ses'], # 0.07% 'pinguedine': ['pin', 'gue', 'di', 'ne'], # 0.07% 'linguas': ['lin', 'guas'], # 0.06% #'erue': ['e', 'ru', 'e'], # 0.06% 'galaaditim': ['ga', 'laa', 'di', 'tim'], # 0.06% 'languentium': ['lan', 'guen', 'ti', 'um'], # 0.05% 'mansuetudinem': ['man', 'sue', 'tu', 'di', 'nem'], # 0.05% #'iniquus': ['in', 'i', 'quus'], # 0.05% #'filiis': ['fi', 'li', 'is'], # 0.05% 'gloria*': ['glo', 'ri', 'a*'], # 0.05% 'leyson': ['ley', 'son'], # 0.05% 'moysi': ['moy', 'si'], # 0.05% #'suavitatis': ['su', 'a', 'vi', 'ta', 'tis'], # 0.05% 'accipite': ['ac', 'ci', 'pi', 'te'], # 0.05% 'exsurgens*': ['ex', 'sur', 'gens*'], # 0.05% } js_cantus_exceptions = { # Exceptions from the alignment algorithm used on the # Cantus website #'euouae': ['e', 'u', 'o', 'u', 'a', 'e'], #'seuouae': ['se', 'u', 'o', 'u', 'a', 'e'], #'alleluya': ['al', 'le', 'lu', 'ya'], 'hierusalem': ['hie', 'ru', 'sa', 'lem'], 'hiesum': ['hie', 'sum'], 'kyrieleison': ['ky', 'ri', 'e', 'lei', 'son'], 'xpisteleison': ['xpi', 'ste', 'lei', 'son'], 'eleison': ['e', 'lei', 'son'], } exceptions = dict(LATIN['exceptions'], **undersegmented, **oversegmented, **js_cantus_exceptions) return exceptions def syllabify(self, text): """ Syllabifies the (lowercased) text Lowercased since since otherwise CLTK doesn't work well """ return self.syllabifier.syllabify(text.lower())
def _recalculate(): from cltk.stem.latin.syllabifier import Syllabifier syllabifier = Syllabifier() corpus = lmloCorpus() # populate chant data frame translate_subcorpus = dict() translate_subcorpus['Feast'] = 'LF' translate_subcorpus['Saint'] = 'LS' translate_subcorpus['Humbert'] = 'H' translate_subcorpus['Humbert Sanct.'] = 'HS' translate_subcorpus['Humbert Temp.'] = 'HT' _data = defaultdict(list) for i, c in enumerate(corpus.chants): _data['chantID'].append(i) _data['corpus'].append('L') _subcorpus = c.office.split(']')[0][1:] # if _subcorpus == 'Saint': # _subcorpus = 'Sanctorale' # if _subcorpus == 'Humber': # _subcorpus = 'Humbert' # if _subcorpus == 'Feast': # _subcorpus = 'Feast' _data['subcorpus'].append(translate_subcorpus[_subcorpus]) _data['Modus'].append(c.mode) _data['modus'].append(c.mode.lower()) if c.mode[0] in ['1','2']: _data['maneria'].append('protus') elif c.mode[0] in ['3','4']: _data['maneria'].append('deuterus') elif c.mode[0] in ['5','6']: _data['maneria'].append('tritus') elif c.mode[0] in ['7','8']: _data['maneria'].append('tetrardus') else: _data['maneria'].append('unknown') if c.mode[1] == c.mode[1].upper(): _data['ambitus'].append('excessive') elif c.mode[0] in ['1','3','5','7']: _data['ambitus'].append('authentic') elif c.mode[0] in ['2','4','6','8']: _data['ambitus'].append('plagal') else: _data['ambitus'].append('unknown') _data['office'].append(' '.join(c.office.split()[1:])) # switching the names Service/service and Genre/genre from lmlo module # for consistency with Modus/modus: capital is the more granular grouping _data['Service'].append(c.service) _data['service'].append(c.Service) _data['ordinal'].append(c.index) _data['Genre'].append(c.genre) _data['genre'].append(c.Genre) _data['text'].append(c.fulltext) # _data['lmloHeader'].append(c.header) # _data['lmloEncoding'].append(c.lmloEncoding) _data['volpiano'].append(c.volpiano) chants = pd.DataFrame(_data) chants.to_pickle('chantData.zip', protocol=4) # populate note data frame # first some utils we'll use in the loop below def pindex(sd): return (int(sd[0])*7 + int(sd[1])) def intclass(interval): interval = abs(interval) if interval == 0: return 'rep' elif interval == 1: return 'step' elif interval == 2: return 'slip' else: return 'leap' _data = defaultdict(list) for i_c, c in enumerate(corpus.chants): i = 1 for i_w, w in enumerate(c.words): for i_s, s in enumerate(w.syllables): for i_n, n in enumerate(s.notes): # identify note's location in the corpus _data['chantID'].append(i_c) _data['word'].append(i_w) _data['syll'].append(i_s) _data['note'].append(i_n) # identify initial and final syllable (1) and word (2) boundaries initial = 0 if i_n == 0: initial += 1 if i_s == 0: initial *= 2 final = 0 if i_n == len(s.notes) - 1: final += 1 if i_s == len(w.syllables) - 1: final *= 2 _data['boundary_before'].append(initial) _data['boundary_after'].append(final) # extract pitch and register features _data['reg_abs'].append(n.letter[0]) _data['pc_abs'].append(n.letter[1]) _data['pitch_abs'].append( n.letter[0] + '.' + n.letter[1]) _data['reg_rel'].append(n.sd[0]) _data['pc_rel'].append(n.sd[1]) _data['pitch_rel'].append( n.sd[0] + '.' + n.sd[1]) # calculate intervallic context if i == 1: _data['lint'].append(99) _data['lint_class'].append('edge') _data['lint_dir'].append('edge') else: interval = int(pindex(c.flatSD[i]) - pindex(c.flatSD[i-1])) _data['lint'].append(interval) _data['lint_class'].append(intclass(interval)) if interval > 0: _data['lint_dir'].append('up') elif interval < 0: _data['lint_dir'].append('down') else: _data['lint_dir'].append('rep') if i == len(c.flatSD)-2: _data['rint_class'].append('edge') _data['rint_dir'].append('edge') _data['rint'].append(99) else: interval = int(pindex(c.flatSD[i+1]) - pindex(c.flatSD[i])) _data['rint'].append(interval) _data['rint_class'].append(intclass(interval)) if interval > 0: _data['rint_dir'].append('up') elif interval < 0: _data['rint_dir'].append('down') else: _data['rint_dir'].append('rep') i += 1 # add interval info notes = pd.DataFrame(_data) modekey = chants.merge(notes).query("word == 0 and syll == 0 and note == 0").set_index('chantID').modus.to_frame() notes = notes.join(modekey.modus, on='chantID', how='inner') notes.to_pickle('noteData.zip', protocol=4) syllables = defaultdict(list) override = dict() override['eius'] = ['e','ius'] override['dei'] = ['de','i'] override['deus'] = ['de','us'] override['quia'] = ['qui','a'] override['christi'] = ['chris','ti'] override['christe'] = ['chris','te'] override['eum'] = ['e','um'] override['deum'] = ['de','um'] override['meum'] = ['me','um'] override['meus'] = ['me','us'] override['christo'] = ['chris','to'] override['christus'] = ['chris','tus'] override['christum'] = ['chris','tum'] override['mei'] = ['me','i'] override['ei'] = ['e','i'] override['cui'] = ['cu','i'] override['israel'] = ['is','ra','el'] override['sanguine'] = ['san','gui','ne'] override['meis'] = ['me','is'] override['eis'] = ['e','is'] override['fidei'] = ['fi','de','i'] override['sanguinem'] = ['san','gui','nem'] override['lingua'] = ['lin','gua'] override['thronum'] = ['thro','num'] override['pulchra'] = ['pul','chra'] override['oleum'] = ['o','le','um'] override['adiutor'] = ['ad','iu','tor'] override['sanguis'] = ['san','guis'] override['sanguinis'] = ['san','gui','nis'] override['huic'] = ['hu','ic'] override['alleluia'] = ['al','le','lu','ia'] override['michael'] = ['mi','cha','el'] override['noe'] = ['no','e'] for i, c in chants.iterrows(): if c.modus not in basicModes+['6c']: continue # if i>200: break words = c.text.lower().split() vwords = c.volpiano[4:-3].split('--') if len(words) != len(vwords): # print(f'oops: {len(words)} {len(vwords)}') # print(words) # print(vwords) vwords[-2] = vwords[-2] + '-' + vwords[-1] vwords.pop(-1) # print('--fixing--') # print(words) # print(vwords) for j in range(len(words)): if words[j] in override: sylls = override[words[j]] else: sylls = syllabifier.syllabify(words[j].lower()) vsylls = vwords[j].split('-') if len(sylls) != len(vsylls): sylls = [f'[{words[j]}]'] * len(vsylls) for k in range(len(vsylls)): syllables['chantID'].append(c.chantID) syllables['syllable'].append(sylls[k]) syllables['last_syll'].append(k+1 == len(vsylls)) v = vsylls[k] syllables['n_notes'].append(len(v)) syllables['volpiano'].append(v) notes = '' for vchar in v: notes += f'{gamut_pitches[gamut_volpiano.index(vchar)]} ' syllables['notes'].append(notes) syllables['pitch_initial'].append(gamut_pitches[gamut_volpiano.index(v[0])]) syllables['pitch_final'].append(gamut_pitches[gamut_volpiano.index(v[-1])]) syllables['t_type'].append(v2r(v)) syllables['e_type'].append(v2r(v[0]+v[-1])) syllables['c_type'].append(v2c(v)) syllables = pd.DataFrame(syllables) syllables = syllables.join(modekey.modus, on='chantID', how='inner') syllables['extrema'] = syllables['pitch_initial'] + '-' + syllables['pitch_final'] syllables.to_pickle('syllableData.zip', protocol=4) ngrams = defaultdict(list) for i, c in chants.iterrows(): # if i>0: break v = c.volpiano.replace('-','') # V = v with duplicate pitches removed V = v[0] for k in range(1, len(v)): if v[k] != V[-1]: V += v[k] for n in range(1, n_limit+1): for k in range(1,len(V)-n): v = V[k:k+n] ngrams['chantID'].append(c.chantID) ngrams['pos'].append(k) ngrams['n_notes'].append(len(v)) ngrams['volpiano'].append(v) notes = '' for vchar in v: notes += f'{gamut_pitches[gamut_volpiano.index(vchar)]} ' ngrams['notes'].append(notes) ngrams['pitch_initial'].append(gamut_pitches[gamut_volpiano.index(v[0])]) ngrams['pitch_final'].append(gamut_pitches[gamut_volpiano.index(v[-1])]) ngrams['t_type'].append(v2r(v)) ngrams['e_type'].append(v2r(v[0]+v[-1])) ngrams['c_type'].append(v2c(v)) ngrams = pd.DataFrame(ngrams) ngrams = ngrams.join(modekey.modus, on='chantID', how='inner') ngrams['extrema'] = ngrams['pitch_initial'] + '-' + ngrams['pitch_final'] print('making pickles') ngrams.to_pickle('ngramData.zip', protocol=4)