def load_lexicon(filename=_jmdict_path):
" Reloads the lexicon into the database."
log.start('Rebuilding the lexicon', nSteps=5)
if not Checksum.needs_update(_checksum_tag, _dependencies + [filename]):
log.finish('Already up-to-date')
return
log.log('Loading probability distributions')
models.initialise()
log.start('Loading JMdict', nSteps=2)
_clear_lexicon()
log.log('Reading from %s' % path.basename(filename))
iStream = sopen(filename, 'r', 'byte')
data = iStream.read()
iStream.close()
log.log('Parsing XML tree')
tree = ElementTree.fromstring(data)
del data
log.finish()
_store_lexemes(tree.getchildren())
log.log('Storing checksum')
Checksum.store(_checksum_tag, _dependencies + [filename])
log.finish()
def load_lexicon(filename=_jmdict_path):
" Reloads the lexicon into the database."
log.start('Rebuilding the lexicon', nSteps=5)
if not Checksum.needs_update(_checksum_tag, _dependencies + [filename]):
log.finish('Already up-to-date')
return
log.log('Loading probability distributions')
models.initialise()
log.start('Loading JMdict', nSteps=2)
_clear_lexicon()
log.log('Reading from %s' % path.basename(filename))
iStream = sopen(filename, 'r', 'byte')
data = iStream.read()
iStream.close()
log.log('Parsing XML tree')
tree = ElementTree.fromstring(data)
del data
log.finish()
_store_lexemes(tree.getchildren())
log.log('Storing checksum')
Checksum.store(_checksum_tag, _dependencies + [filename])
log.finish()
def add_all_syllabi(force=False):
syllabi = bundle.list_names()
_log.start('Adding all syllabi', nSteps=len(syllabi))
dependencies = []
for syllabus in syllabi:
dependencies.extend(bundle.SyllabusBundle.get_dependencies(syllabus))
if not force and not Checksum.needs_update('syllabi', dependencies,
['lexicon']):
_log.finish('Already up-to-date')
return
for syllabus_name in syllabi:
add_syllabus(syllabus_name, force=force)
Checksum.store('syllabi', dependencies)
_log.finish()
def build(cls):
"""
Build the tables needed to generate search results at runtime. These
tables describe readings and reading alternations for each kanji which
might be searched for.
"""
kanji_set = set(row.kanji for row in Kanji.objects.all())
log.start('Building reading tables')
if not Checksum.needs_update('reading_alt', _dependencies,
['lexicon']):
log.finish('Already up-to-date')
return
alt_tree = cls._build_alternation_tree(kanji_set)
cls._store_alternation_tree(alt_tree)
log.log('Storing readings per kanji')
cls._store_kanji_readings(alt_tree)
Checksum.store('reading_alt', _dependencies)
log.finish()
def build(cls):
"""
Build the tables needed to generate search results at runtime. These
tables describe readings and reading alternations for each kanji which
might be searched for.
"""
kanji_set = set(row.kanji for row in Kanji.objects.all())
log.start('Building reading tables')
if not Checksum.needs_update('reading_alt', _dependencies,
['lexicon']):
log.finish('Already up-to-date')
return
alt_tree = cls._build_alternation_tree(kanji_set)
cls._store_alternation_tree(alt_tree)
log.log('Storing readings per kanji')
cls._store_kanji_readings(alt_tree)
Checksum.store('reading_alt', _dependencies)
log.finish()
