def compile(self, id):
"""Compile the script of a phonology as a foma FST.
:URL: ``PUT /phonologies/compile/id``
:param str id: the ``id`` value of the phonology whose script will be compiled.
:returns: if the phonology exists and foma is installed, the phonology
model is returned; ``GET /phonologies/id`` must be polled to
determine when and how the compilation task has terminated.
.. note::
The script is compiled asynchronously in a worker thread. See
:mod:`onlinelinguisticdatabase.lib.foma_worker`.
"""
phonology = Session.query(Phonology).get(id)
if phonology:
if h.foma_installed():
foma_worker_q.put({
'id': h.generate_salt(),
'func': 'compile_phonology',
'args': {
'phonology_id': phonology.id,
'user_id': session['user'].id,
'timeout': h.phonology_compile_timeout
}
})
return phonology
else:
response.status_int = 400
return {'error': 'Foma and flookup are not installed.'}
else:
response.status_int = 404
return {'error': 'There is no phonology with id %s' % id}
def generate(self, id):
"""Generate the files that constitute the morpheme language model, crucially the file that holds the pickled LM trie.
:URL: ``PUT /morpheme_language_model/id/generate``
:param str id: the ``id`` value of the morpheme language model whose files will be generated.
:returns: the morpheme language model is returned; ``GET /morpheme_language_model/id`` must be polled to
determine when the generation task has terminated.
"""
lm = Session.query(MorphemeLanguageModel).get(id)
if not lm:
response.status_int = 404
return {
'error': 'There is no morpheme language model with id %s' % id
}
args = {
'morpheme_language_model_id': lm.id,
'user_id': session['user'].id,
'timeout': h.morpheme_language_model_generate_timeout
}
foma_worker_q.put({
'id': h.generate_salt(),
'func': 'generate_language_model',
'args': args
})
return lm
def generate_and_compile_morphological_parser(morphological_parser_id, compile_=True):
morphological_parser = Session.query(MorphologicalParser).get(morphological_parser_id)
if not morphological_parser:
response.status_int = 404
return {'error': 'There is no morphological parser with id %s' % id}
if compile_ and not h.foma_installed():
response.status_int = 400
return {'error': 'Foma and flookup are not installed.'}
foma_worker_q.put({
'id': h.generate_salt(),
'func': 'generate_and_compile_parser',
'args': {
'morphological_parser_id': morphological_parser.id,
'compile': compile_,
'user_id': session['user'].id,
'timeout': h.morphological_parser_compile_timeout
}
})
return morphological_parser
def generate_and_compile_morphological_parser(morphological_parser_id,
compile_=True):
morphological_parser = Session.query(MorphologicalParser).get(
morphological_parser_id)
if not morphological_parser:
response.status_int = 404
return {'error': 'There is no morphological parser with id %s' % id}
if compile_ and not h.foma_installed():
response.status_int = 400
return {'error': 'Foma and flookup are not installed.'}
foma_worker_q.put({
'id': h.generate_salt(),
'func': 'generate_and_compile_parser',
'args': {
'morphological_parser_id': morphological_parser.id,
'compile': compile_,
'user_id': session['user'].id,
'timeout': h.morphological_parser_compile_timeout
}
})
return morphological_parser
def compute_perplexity(self, id):
"""Compute the perplexity of the LM's corpus according to the LM.
Randomly divide the corpus into training and test sets multiple times and compute
the perplexity and return the average. See ``evaluate_morpheme_language_model`` in lib/foma_worker.py.
"""
lm = Session.query(MorphemeLanguageModel).get(id)
if not lm:
response.status_int = 404
return {'error': 'There is no morpheme language model with id %s' % id}
args = {
'morpheme_language_model_id': lm.id,
'user_id': session['user'].id,
'timeout': h.morpheme_language_model_generate_timeout
}
foma_worker_q.put({
'id': h.generate_salt(),
'func': 'compute_perplexity',
'args': args
})
return lm
def generate(self, id):
"""Generate the files that constitute the morpheme language model, crucially the file that holds the pickled LM trie.
:URL: ``PUT /morpheme_language_model/id/generate``
:param str id: the ``id`` value of the morpheme language model whose files will be generated.
:returns: the morpheme language model is returned; ``GET /morpheme_language_model/id`` must be polled to
determine when the generation task has terminated.
"""
lm = Session.query(MorphemeLanguageModel).get(id)
if not lm:
response.status_int = 404
return {'error': 'There is no morpheme language model with id %s' % id}
args = {
'morpheme_language_model_id': lm.id,
'user_id': session['user'].id,
'timeout': h.morpheme_language_model_generate_timeout
}
foma_worker_q.put({
'id': h.generate_salt(),
'func': 'generate_language_model',
'args': args
})
return lm
def compute_perplexity(self, id):
"""Compute the perplexity of the LM's corpus according to the LM.
Randomly divide the corpus into training and test sets multiple times and compute
the perplexity and return the average. See ``evaluate_morpheme_language_model`` in lib/foma_worker.py.
"""
lm = Session.query(MorphemeLanguageModel).get(id)
if not lm:
response.status_int = 404
return {
'error': 'There is no morpheme language model with id %s' % id
}
args = {
'morpheme_language_model_id': lm.id,
'user_id': session['user'].id,
'timeout': h.morpheme_language_model_generate_timeout
}
foma_worker_q.put({
'id': h.generate_salt(),
'func': 'compute_perplexity',
'args': args
})
return lm
