def my_tokenizer(form, prf): value = form.strip() for form in split_text(value, separators='/,~', strip=True): value = form.strip() form = "^%s$" % form.replace(" ", "{} ") form = strip_brackets(form, brackets={'[': ']'}) i = 0 tokens = [] while True: added = False for length in range(len(form[i:]), 0, -1): needle = form[i:i + length] if needle in prf: tokens.append(prf[needle]) i += length added = True break if not added: if form[i] == ' ': tokens.append("#") else: tokens.append('<%s>' % form[i]) i += 1 if i == len(form): break # Remove NULLs tokens = [token for token in tokens if token != "NULL"] return ' '.join(tokens)
def cogids2cogid(wordlist, ref="cogids", cognates="cogid", morphemes="morphemes"): C, M = {}, {} current = 1 for concept in wordlist.rows: base = split_text(strip_brackets(concept))[0].upper().replace(" ", "_") idxs = wordlist.get_list(row=concept, flat=True) cogids = defaultdict(list) for idx in idxs: M[idx] = [c for c in wordlist[idx, ref]] for cogid in basictypes.ints(wordlist[idx, ref]): cogids[cogid] += [idx] for i, (cogid, idxs) in enumerate( sorted(cogids.items(), key=lambda x: len(x[1]), reverse=True)): for idx in idxs: if idx not in C: C[idx] = current M[idx][M[idx].index(cogid)] = base else: M[idx][M[idx].index(cogid)] = "_" + base.lower() current += 1 wordlist.add_entries(cognates, C, lambda x: x) if morphemes: wordlist.add_entries(morphemes, M, lambda x: x)
def wikibooks(): with open('wikibooks.txt') as f: data = f.readlines() out = [] gsr = defaultdict(dict) for i, line in enumerate(data): line = strip_brackets(line.strip().replace('\t', ' '), brackets={'(': ')'}) if line.startswith('*'): if not line[1] == ' ': line = line.replace('*', '* ') elms = line.split(' ') if elms and len(elms) > 1: kgsc = elms[1].split('/') if len(kgsc) == 1: schuessler = '' karlgren = kgsc[0] elif len(kgsc) == 2: karlgren = kgsc[1] schuessler = kgsc[0] else: print('[ERROR:schuessler/karlgren] {0}'.format(line)) try: char = elms[2].split('|')[-1][0] except IndexError: print('[ERROR:character] {0}'.format(line)) char = '' mch = [x[:-1] if x.endswith(',') else x for x in elms[3:]] if len(karlgren) not in [4, 5, 6]: print('[ERROR:karlgren] {0}'.format(line, karlgren)) elif not sinopy.is_chinese(char): print('[ERROR:char] {0}'.format(line)) elif char: pinyin = sinopy.pinyin(char) if '?' in pinyin or sinopy.is_chinese(pinyin): pinyin = '' out += [(char, pinyin, 'Old_Chinese', karlgren[:4], karlgren, '', 'Karlgren1954')] for reading in mch: out += [(char, pinyin, 'Middle_Chinese', '', karlgren, reading, 'Wikibooks2016a')] gsr[char][reading] = [pinyin, reading, karlgren] with open('karlgren.tsv', 'w') as f: f.write( 'ID\tCHARACTER\tPINYIN\tDOCULECT\tPHONETIC_CLASS\tKARLGREN_ID\tREADING\tSOURCE\n' ) for i, line in enumerate(out): f.write(str(i + 1) + '\t' + '\t'.join(line) + '\n') return gsr
def normalize_grapheme(text): """ Apply simple, non-CLTS, normalization. """ new_text = unicodedata.normalize("NFD", text) if new_text[0] == "(" and new_text[-1] == ")": new_text = new_text[1:-1] new_text = strip_brackets(new_text) if new_text: return new_text
def clean(self, form, item=None): """ Called when a row is added to a CLDF dataset. :param form: :return: None to skip the form, or the cleaned form as string. """ if form not in self.missing_data: if self.normalize_whitespace: form = re.sub(r'\s+', ' ', form.strip()) for source, target in self.replacements: form = form.replace(source, target) if self.strip_inside_brackets: return text.strip_brackets(form, brackets=self.brackets) return form
def clean_entry(string): """simple replacements to enhance segmentation""" st = [(' (GEM)', ''), (' (NEP)', ''), (' LO$', ''), (' L$', ''), (' C$', ''), (' ING$', ''), (' TIB', ''), (' NEP', ''), ('?', ''), (':', 'ː'), ('.', ''), ('=', ''), (' ', '_'), ('_LF', ''), ('*', ''), ('^', ''), ('$', '')] mapper = dict(zip('0123456789', '⁰¹²³⁴⁵⁶⁸⁹')) mapper.update(dict(st)) string = re.split(';|/|,|>', string)[0] string = '^' + string + '$' for s, t in st: string = string.replace(s, t) for s in [' ', '_', '-']: string = string.strip(s) string = ''.join([x for x in string if not is_chinese(x)]) return strip_brackets(''.join([mapper.get(s, s) for s in string.strip()]))
def clean_form(self, item, form): if form not in ["*", "---", ""]: return split_text(strip_brackets(form), ",;/")[0]
def clean_string( sequence, semi_diacritics='hsʃ̢ɕʂʐʑʒw', merge_vowels=False, segmentized=False, rules=None, ignore_brackets=True, brackets=None, split_entries=True, splitters='/,;~', preparse=None, merge_geminates=True, normalization_form="NFC"): """ Function exhaustively checks how well a sequence is understood by \ LingPy. Parameters ---------- semi_diacritics : str Indicate characters which can occur both as "diacritics" (second part in a sound) or alone. merge_vowels : bool (default=True) Indicate whether consecutive vowels should be merged. segmentized : False Indicate whether the input string is already segmentized or not. If set to True, items in brackets can no longer be ignored. rules : dict Replacement rules to be applied to a segmentized string. ignore_brackets : bool If set to True, ignore all content within a given bracket. brackets : dict A dictionary with opening brackets as key and closing brackets as values. Defaults to a pre-defined set of frequently occurring brackets. split_entries : bool (default=True) Indicate whether multiple entries (with a comma etc.) should be split into separate entries. splitters : str The characters which force the automatic splitting of an entry. preparse : list List of tuples, giving simple replacement patterns (source and target), which are applied before every processing starts. Returns ------- cleaned_strings : list A list of cleaned strings which are segmented by space characters. If splitters are encountered, indicating that the entry contains two variants, the list will contain one for each element in a separate entry. If there are no splitters, the list has only size one. """ sequence = unicodedata.normalize(normalization_form, sequence) rules = rules or {} preparse = preparse or [] # replace white space if not indicated otherwise if segmentized: segment_list = [sequence.split(' ') if not isinstance(sequence, (list, tuple)) else sequence] else: for s, t in preparse: sequence = sequence.replace(s, t) segment_list = [] if ignore_brackets: new_sequence = strip_brackets(sequence, brackets=brackets) else: new_sequence = sequence # splitting needs to be done afterwards if split_entries: new_sequences = split_text(new_sequence, splitters, brackets='' if not ignore_brackets else brackets) else: new_sequences = [new_sequence] for new_sequence in new_sequences: segments = ipa2tokens( re.sub(r'\s+', '_', new_sequence.strip()), semi_diacritics=semi_diacritics, merge_vowels=merge_vowels, merge_geminates=merge_geminates) segment_list += [segments] out = [] for segments in segment_list: segments = [rules.get(s, s) for s in segments] out += [' '.join(segments)] return out
def clean_string( sequence, semi_diacritics='hsʃ̢ɕʂʐʑʒw', merge_vowels=False, segmentized=False, rules=None, ignore_brackets=True, brackets=None, split_entries=True, splitters='/,;~', preparse=None, merge_geminates=True, normalization_form="NFC"): """ Function exhaustively checks how well a sequence is understood by \ LingPy. Parameters ---------- semi_diacritics : str Indicate characters which can occur both as "diacritics" (second part in a sound) or alone. merge_vowels : bool (default=True) Indicate whether consecutive vowels should be merged. segmentized : False Indicate whether the input string is already segmentized or not. If set to True, items in brackets can no longer be ignored. rules : dict Replacement rules to be applied to a segmentized string. ignore_brackets : bool If set to True, ignore all content within a given bracket. brackets : dict A dictionary with opening brackets as key and closing brackets as values. Defaults to a pre-defined set of frequently occurring brackets. split_entries : bool (default=True) Indicate whether multiple entries (with a comma etc.) should be split into separate entries. splitters : str The characters which force the automatic splitting of an entry. preparse : list List of tuples, giving simple replacement patterns (source and target), which are applied before every processing starts. Returns ------- cleaned_strings : list A list of cleaned strings which are segmented by space characters. If splitters are encountered, indicating that the entry contains two variants, the list will contain one for each element in a separate entry. If there are no splitters, the list has only size one. """ sequence = unicodedata.normalize(normalization_form, sequence) rules = rules or {} preparse = preparse or [] # replace white space if not indicated otherwise if segmentized: segment_list = [sequence.split(' ') if not isinstance(sequence, (list, tuple)) else sequence] else: for s, t in preparse: sequence = sequence.replace(s, t) segment_list = [] if ignore_brackets: new_sequence = strip_brackets(sequence, brackets=brackets) else: new_sequence = sequence # splitting needs to be done afterwards if split_entries: new_sequences = split_text(new_sequence, splitters, brackets='' if not ignore_brackets else brackets) else: new_sequences = [new_sequence] for new_sequence in new_sequences: segments = ipa2tokens( re.sub(r'\s+', '_', new_sequence.strip()), semi_diacritics=semi_diacritics, merge_vowels=merge_vowels, merge_geminates=merge_geminates) segment_list += [segments] out = [] for segments in segment_list: segments = [rules.get(s, s) for s in segments] out += [' '.join(segments)] return out
def test_strip_brackets(): strings = ['arm((h)an[d])', '(hand)arm', 'a(hand)r(hand)m(hand)', 'arm⁽hand⁾'] for string in strings: assert text.strip_brackets(string) == 'arm' assert text.strip_brackets('arm<hand>', brackets={"<": ">"}) == 'arm'
query = r""" select ?lexemeId ?lemma WHERE {{ ?lexemeId <http://purl.org/dc/terms/language> wd:Q1860; wikibase:lemma ?lemma. FILTER (regex(?lemma, '^{0}$')) }} """ findings = defaultdict(list) mappings, concepticon = get_mappings() visited = set() words = defaultdict(list) for w, vals in mappings['en'].items(): w2 = strip_brackets(w.lower().strip()) if w2 not in words: words[w2] = vals[0][0] with open('wikidata.tsv', 'r') as f: for line in f: if line.strip(): cid = line.split('\t')[0] visited.add(cid) print(len(words)) with open('fails.tsv', 'r') as f: for line in f: visited.add(line.strip()) current = ''
def name(self): m = re.search(r'tree\s+(?P<name>[^=]+)\s*=', strip_brackets(self, {'[': ']'})) if m: return m.group('name').strip()
def clean_form(self, item, form): if form not in ["*", "---", "-"]: form = strip_brackets(split_text(form, separators=";,/")[0]) return form.replace(" ", "_")