def load_lexicon(filename=_jmdict_path): " Reloads the lexicon into the database." log.start('Rebuilding the lexicon', nSteps=5) if not Checksum.needs_update(_checksum_tag, _dependencies + [filename]): log.finish('Already up-to-date') return log.log('Loading probability distributions') models.initialise() log.start('Loading JMdict', nSteps=2) _clear_lexicon() log.log('Reading from %s' % path.basename(filename)) iStream = sopen(filename, 'r', 'byte') data = iStream.read() iStream.close() log.log('Parsing XML tree') tree = ElementTree.fromstring(data) del data log.finish() _store_lexemes(tree.getchildren()) log.log('Storing checksum') Checksum.store(_checksum_tag, _dependencies + [filename]) log.finish()
def add_all_syllabi(force=False): syllabi = bundle.list_names() _log.start('Adding all syllabi', nSteps=len(syllabi)) dependencies = [] for syllabus in syllabi: dependencies.extend(bundle.SyllabusBundle.get_dependencies(syllabus)) if not force and not Checksum.needs_update('syllabi', dependencies, ['lexicon']): _log.finish('Already up-to-date') return for syllabus_name in syllabi: add_syllabus(syllabus_name, force=force) Checksum.store('syllabi', dependencies) _log.finish()
def build(cls): """ Build the tables needed to generate search results at runtime. These tables describe readings and reading alternations for each kanji which might be searched for. """ kanji_set = set(row.kanji for row in Kanji.objects.all()) log.start('Building reading tables') if not Checksum.needs_update('reading_alt', _dependencies, ['lexicon']): log.finish('Already up-to-date') return alt_tree = cls._build_alternation_tree(kanji_set) cls._store_alternation_tree(alt_tree) log.log('Storing readings per kanji') cls._store_kanji_readings(alt_tree) Checksum.store('reading_alt', _dependencies) log.finish()