def test_add_dataset(): c = Cihai() c.add_dataset(SimplestDataset, namespace='simple') assert hasattr(c, 'simple') assert isinstance(c.simple, extend.Dataset) assert hasattr(c.simple, 'a_method') assert callable(c.simple.a_method) assert c.simple.a_method() == 'hi'
def __init__(self): super().__init__() self.c = Cihai() if not self.c.is_bootstrapped: # download and install Unihan to db bootstrap_unihan(self.c.metadata) self.c.reflect_db()
def run(): c = Cihai(unihan=False) c.add_dataset(MyDataset, namespace='moo') c.moo.bootstrap() print('Definitions exactly for 好', c.moo.givemedata('好')) print('Definitions matching with 你好:', ', '.join(c.moo.search('好'))) print('Reverse definition with Good:', ', '.join(c.moo.backwards('Good')))
def run(): c = Cihai(unihan=False) c.add_dataset(MyDataset, namespace='moo') c.moo.bootstrap() print('Definitions exactly for 好', c.moo.givemedata('好')) print('Definitions matching with 你好:', ', '.join(c.moo.search('好'))) print('Reverse definition with Good:', ', '.join(c.moo.backwards('Good')))
def test_reflect_db(tmpdb_file, unihan_options): c = Cihai({ 'database': { 'url': 'sqlite:///{tmpdb_file}'.format(tmpdb_file=tmpdb_file) } }) assert not c.is_bootstrapped bootstrap.bootstrap_unihan(c.metadata, unihan_options) assert not hasattr(c.base.classes, 'Unihan') c.reflect_db() assert hasattr(c.base.classes, 'Unihan')
def run(unihan_options={}): c = Cihai(unihan=False) c.add_dataset('cihai.data.unihan.dataset.Unihan', namespace='unihan') if not c.unihan.is_bootstrapped: # download and install Unihan to db c.unihan.bootstrap(unihan_options) query = c.unihan.lookup_char('好') glyph = query.first() print("lookup for 好: %s" % glyph.kDefinition) query = c.unihan.reverse_char('good') print('matches for "good": %s ' % ', '.join([glph.char for glph in query]))
def run(unihan_options={}): c = Cihai() if not c.unihan.is_bootstrapped: # download and install Unihan to db c.unihan.bootstrap(unihan_options) c.unihan.add_plugin('cihai.data.unihan.dataset.UnihanVariants', namespace='variants') print("This example prints some tricky cases of character-by-character " "Traditional-Simplified mapping.") print("https://www.unicode.org/reports/tr38/#N10211") print("3.7.1 bullet 4") for char in c.unihan.with_fields("kTraditionalVariant", "kSimplifiedVariant"): print("Character: {}".format(char.char)) trad = set(char.untagged_vars("kTraditionalVariant")) simp = set(char.untagged_vars("kSimplifiedVariant")) Unihan = c.sql.base.classes.Unihan if Unihan.char in trad and Unihan.char in simp: print("Case 1") else: print("Case 2 (non-idempotent)") for trad_var in trad: print("s2t: {}".format(trad_var)) for simp_var in simp: print("t2s: {}".format(simp_var))
def cli(ctx, config, log_level): """Retrieve CJK information via CLI. For help and example usage, see documentation: https://cihai-cli.git-pull.com and https://cihai.git-pull.com""" setup_logger(level=log_level.upper()) if config: c = Cihai.from_file(config) else: c = Cihai() if not c.unihan.is_bootstrapped: click.echo("Bootstrapping Unihan database") c.unihan.bootstrap(options=c.config.get('unihan_options', {})) ctx.obj['c'] = c # pass Cihai object down to other commands
def test_unihan_options(unihan_options, test_config_file): app = Cihai.from_file(test_config_file) bootstrap.bootstrap_unihan(app.metadata, unihan_options) assert 'Unihan' in app.metadata.tables assert app.metadata.tables['Unihan'].columns assert set(app.metadata.tables['Unihan'].columns.keys()) == \ set(bootstrap.UNIHAN_FIELDS + ['ucn', 'char']) assert bootstrap.is_bootstrapped(app.metadata)
def test_unihan_options(unihan_options, test_config_file): app = Cihai.from_file(test_config_file) bootstrap.bootstrap_unihan(app.sql.metadata, unihan_options) assert 'Unihan' in app.sql.metadata.tables assert app.sql.metadata.tables['Unihan'].columns assert set(app.sql.metadata.tables['Unihan'].columns.keys()) == set( bootstrap.UNIHAN_FIELDS + ['ucn', 'char'] ) assert bootstrap.is_bootstrapped(app.sql.metadata)
def test_config_dict_args(): """Accepts dict as config.""" expected = 'world' app = Cihai({'hello': expected}) result = app.config['hello'] assert result == expected
def test_add_dataset_unihan(unihan_options): c = Cihai() c.add_dataset(Unihan, namespace='unihan') assert hasattr(c, 'unihan') assert isinstance(c.unihan, extend.Dataset) c.unihan.sql c.unihan.bootstrap(options=unihan_options) U = c.sql.base.classes.Unihan first_glyph = (c.unihan.sql.session.query(U).filter( U.kDefinition.isnot(None)).first()) char = first_glyph.char assert (c.unihan.lookup_char( char=char).first().kDefinition == first_glyph.kDefinition) assert (c.unihan.reverse_char( hints=[first_glyph.kDefinition]).first().char == char ), 'works with list of column value matches' assert (c.unihan.reverse_char( hints=first_glyph.kDefinition).first().char == char ), 'works with strings' c.unihan.add_plugin(UnihanVariants, 'variants') assert hasattr(c.unihan, 'variants') def variant_list(field): for char in c.unihan.with_fields(field): variants = [] for var in char.untagged_vars(field): variants.append(var) yield (char, variants) result = {char: variants for (char, variants) in variant_list('kZVariant')} assert len(result.values()) > 0 assert len(result.keys()) > 0
def run(unihan_options={}): """Wrapped so we can test in tests/test_examples.py""" print("This example prints variant character data.") c = Cihai() if not c.unihan.is_bootstrapped: # download and install Unihan to db c.unihan.bootstrap(unihan_options) c.unihan.add_plugin('cihai.data.unihan.dataset.UnihanVariants', namespace='variants') print("## ZVariants") variant_list(c.unihan, "kZVariant") print("## kSemanticVariant") variant_list(c.unihan, "kSemanticVariant") print("## kSpecializedSemanticVariant") variant_list(c.unihan, "kSpecializedSemanticVariant")
#!/usr/bin/env python3 import sys from cihai.core import Cihai verbose = False debug = "" c_dict = Cihai() def codepoints2chars(codepoints): if codepoints is None: return None results = [] temp = codepoints.split() for each in temp: num = each[2:] #print("num={}".format(num)) # The [0] is to strip dictionary (e.g. Matthews) name if there is one results.append(f'\\u{num}'.encode().decode('unicode_escape')[0]) return results def slow_search(zi, shujuku): global verbose global debug global c_dict # See cihai.git-pull.com/api.html query = c_dict.unihan.lookup_char(zi) glyph = query.first()
from typing import List, Optional, Tuple, Iterator, Set from cihai.core import Cihai from django.http.request import HttpRequest from django.forms.models import model_to_dict from django.db import IntegrityError from django.db.models.query import QuerySet from .forms import SenseForm from .models import Headword, Sense, Phrase, Example logger = logging.getLogger(__name__) SEP_RE = re.compile(r'[;;,]') C = Cihai() if not C.unihan.is_bootstrapped: C.unihan.bootstrap() @dataclass class Entry: headword: str headword_sense_no: int char_strokes_first: str char_strokes_all: str only_letters: str root: str root_sense_no: int word_class: list focus: list
#!/usr/bin/env python # -*- coding: utf8 - *- from __future__ import unicode_literals, print_function from cihai.core import Cihai from cihai.bootstrap import bootstrap_unihan c = Cihai() if not c.is_bootstrapped: # download and install Unihan to db bootstrap_unihan(c.metadata) c.reflect_db() # automap new table created during bootstrap query = c.lookup_char('好') glyph = query.first() print("lookup for 好: %s" % glyph.kDefinition) query = c.reverse_char('good') print('matches for "good": %s ' % ', '.join([glph.char for glph in query]))
from sqlalchemy.sql import column from cihai.core import Cihai from cihai.bootstrap import bootstrap_unihan from utils import parse_vars print("References from https://www.unicode.org/reports/tr38/#N10211") cihan = Cihai() if not cihan.is_bootstrapped: bootstrap_unihan(cihan.metadata) cihan.reflect_db() double_var = ( cihan.session.query(cihan.base.classes.Unihan) .filter(column("kTraditionalVariant").isnot(None)) .filter(column("kSimplifiedVariant").isnot(None)) .all() ) print("## 3.7.1 bullet 4") for c in double_var: print("Character: {}".format(c.char)) trad = parse_vars(c.kTraditionalVariant) simp = parse_vars(c.kSimplifiedVariant) if c.char in trad and c.char in simp: print("Case 1") else:
import sys sys.__stdout__ = sys.stdout from cihai.core import Cihai from cihai.bootstrap import bootstrap_unihan c = Cihai() if not c.is_bootstrapped: # download and install Unihan to db bootstrap_unihan(c.metadata) c.reflect_db() def jp_reverse(word): query = c.reverse_char(word) return ', '.join([glyph.char for glyph in query])
class ExplainKanji(BaseFilter): def __init__(self): super().__init__() self.c = Cihai() if not self.c.is_bootstrapped: # download and install Unihan to db bootstrap_unihan(self.c.metadata) self.c.reflect_db() def __call__(self, chunk): from src.Sequencer import TextChunk, JingleChunk chunk = self._duplicate_chunk(chunk) result = [chunk] if not isinstance(chunk, TextChunk) or chunk.language != 'japanese': return result explanations = self._get_explanations(chunk.text) result.append( TextChunk(text='[', audible=False, printable=True, final=True)) for k, ons, kuns, explanation in explanations: result.append( TextChunk(text=k, language='japanese', audible=False, printable=True, final=True)) result.append( TextChunk(text='on', language='english', audible=True, printable=False, final=True)) result.append(JingleChunk(jingle='silence')) for on in ons: result.append( TextChunk(text=on, language='japanese', audible=True, printable=True, final=True)) result.append(JingleChunk(jingle='silence')) result.append( TextChunk(text='、', audible=False, printable=True, final=True)) result.append( TextChunk(text='koon', language='english', audible=True, printable=False, final=True)) result.append(JingleChunk(jingle='silence')) for kun in kuns: result.append( TextChunk(text=kun, language='japanese', audible=True, printable=True, final=True)) result.append(JingleChunk(jingle='silence')) result.append( TextChunk(text='、', audible=False, printable=True, final=True)) result.append(JingleChunk(jingle='definition')) result.append( TextChunk(text=explanation, language='english', audible=True, printable=True, final=True)) result.append( TextChunk(text=']', audible=False, printable=True, final=True)) result.append( JingleChunk(jingle='silence_long', audible=False, printable=True, final=True)) return result def _kanji_to_kana(self, char): glyph = self.c.lookup_char(char).first() if glyph is None: return None romaji_on = glyph.kJapaneseKun.lower() romaji_kun = glyph.kJapaneseOn.lower() jp_on = jaconv.alphabet2kana(romaji_on).split(' ') jp_kun = jaconv.hira2kata(jaconv.alphabet2kana(romaji_kun)).split(' ') return jp_on, jp_kun, glyph.kDefinition @staticmethod def is_kana(char): return ('\u30A0' <= char <= '\u30FF') or ( '\u3040' <= char <= '\u309F') # Katakana and Hiragana blocks @classmethod def is_kanji(cls, char): return not cls.is_kana(char) def _get_explanations(self, text): kanji = set(filter(self.is_kanji, text)) detail_list = [] for k in kanji: triplet = self._kanji_to_kana(k) if triplet is None: continue on, kun, definition = triplet # detail_list.append(f'{k}: {", ".join(on)}; {", ".join(kun)}; {definition}') detail_list.append((k, on, kun, definition)) return detail_list
def test_cihai_without_unihan(): app = Cihai(unihan=False) assert (UNIHAN_CONFIG.items() != app.config.items()), 'app can be initialized without unihan' assert not hasattr(app, 'unihan')
def test_yaml_config_and_override(test_config_file): app = Cihai.from_file(test_config_file) assert app.config['database']
def test_bootstraps_unihan_by_default(): app = Cihai() assert UNIHAN_CONFIG.items() == app.config.items() assert app.unihan, 'cihai bootstraps unihan by default'
def test_config_defaults(): """Test config defaults.""" app = Cihai() assert 'database' in app.config
from auto import * from pprint import pprint as pp from cihai.core import Cihai c=Cihai() # c.unihan.bootstrap({}) """ 造zào - create/make 话huà - speaking 辶 - radical - zǒu zhī páng (Chuò) - indicates motion 福fú - blessing 福fú - blessing 礻 - radical - shì - show reveal. 神shén - god. 一yī - one 口kǒu - mouth 田tián - field (garden) 造zào - create/make 話huà - speaking 辶 - radical 162 - zǒu zhī páng - walk/walking (indicates motion) 福fú - blessing 礻- radical 113 - shì (cult), represents 示Shì - reveal, manifest; demonstrate 神shén - god. 一yī - one 口kǒu - mouth 田tián - field (garden) 園yuán - garden 土tǔ - earth/clay 口kǒu - mouth 亻rén - radical (man)
def test_yaml_config_and_override(test_config_file): app = Cihai.from_file(test_config_file) assert app.config['database']
# -*- coding: utf-8 -*- import xml.etree.ElementTree as elemTree import os, uuid from operator import methodcaller, itemgetter, mul from cihai.core import Cihai c = Cihai() import hms.tex as tex import hms.html as html import multiprocessing as mp from multiprocessing import Process, Lock, Queue, Value def generateIdent(): return str(uuid.uuid4()) def textify(e, spell, ident, coder=tex): total = '' part = e.text beforehand = False for child in e: if part != None: part = part.replace('.', '。').replace(',', ',').replace( ' ', '').replace('\t', '').replace('\n', '').replace(' ', '') if part != '': total += part beforehand = False if child.tag == 'quote': temp = textify(child, spell, ident, coder)