def compile(self, id): """Compile the script of a phonology as a foma FST. :URL: ``PUT /phonologies/compile/id`` :param str id: the ``id`` value of the phonology whose script will be compiled. :returns: if the phonology exists and foma is installed, the phonology model is returned; ``GET /phonologies/id`` must be polled to determine when and how the compilation task has terminated. .. note:: The script is compiled asynchronously in a worker thread. See :mod:`onlinelinguisticdatabase.lib.foma_worker`. """ phonology = Session.query(Phonology).get(id) if phonology: if h.foma_installed(): foma_worker_q.put({ 'id': h.generate_salt(), 'func': 'compile_phonology', 'args': { 'phonology_id': phonology.id, 'user_id': session['user'].id, 'timeout': h.phonology_compile_timeout } }) return phonology else: response.status_int = 400 return {'error': 'Foma and flookup are not installed.'} else: response.status_int = 404 return {'error': 'There is no phonology with id %s' % id}
def generate(self, id): """Generate the files that constitute the morpheme language model, crucially the file that holds the pickled LM trie. :URL: ``PUT /morpheme_language_model/id/generate`` :param str id: the ``id`` value of the morpheme language model whose files will be generated. :returns: the morpheme language model is returned; ``GET /morpheme_language_model/id`` must be polled to determine when the generation task has terminated. """ lm = Session.query(MorphemeLanguageModel).get(id) if not lm: response.status_int = 404 return { 'error': 'There is no morpheme language model with id %s' % id } args = { 'morpheme_language_model_id': lm.id, 'user_id': session['user'].id, 'timeout': h.morpheme_language_model_generate_timeout } foma_worker_q.put({ 'id': h.generate_salt(), 'func': 'generate_language_model', 'args': args }) return lm
def generate_and_compile_morphological_parser(morphological_parser_id, compile_=True): morphological_parser = Session.query(MorphologicalParser).get(morphological_parser_id) if not morphological_parser: response.status_int = 404 return {'error': 'There is no morphological parser with id %s' % id} if compile_ and not h.foma_installed(): response.status_int = 400 return {'error': 'Foma and flookup are not installed.'} foma_worker_q.put({ 'id': h.generate_salt(), 'func': 'generate_and_compile_parser', 'args': { 'morphological_parser_id': morphological_parser.id, 'compile': compile_, 'user_id': session['user'].id, 'timeout': h.morphological_parser_compile_timeout } }) return morphological_parser
def generate_and_compile_morphological_parser(morphological_parser_id, compile_=True): morphological_parser = Session.query(MorphologicalParser).get( morphological_parser_id) if not morphological_parser: response.status_int = 404 return {'error': 'There is no morphological parser with id %s' % id} if compile_ and not h.foma_installed(): response.status_int = 400 return {'error': 'Foma and flookup are not installed.'} foma_worker_q.put({ 'id': h.generate_salt(), 'func': 'generate_and_compile_parser', 'args': { 'morphological_parser_id': morphological_parser.id, 'compile': compile_, 'user_id': session['user'].id, 'timeout': h.morphological_parser_compile_timeout } }) return morphological_parser
def compute_perplexity(self, id): """Compute the perplexity of the LM's corpus according to the LM. Randomly divide the corpus into training and test sets multiple times and compute the perplexity and return the average. See ``evaluate_morpheme_language_model`` in lib/foma_worker.py. """ lm = Session.query(MorphemeLanguageModel).get(id) if not lm: response.status_int = 404 return {'error': 'There is no morpheme language model with id %s' % id} args = { 'morpheme_language_model_id': lm.id, 'user_id': session['user'].id, 'timeout': h.morpheme_language_model_generate_timeout } foma_worker_q.put({ 'id': h.generate_salt(), 'func': 'compute_perplexity', 'args': args }) return lm
def generate(self, id): """Generate the files that constitute the morpheme language model, crucially the file that holds the pickled LM trie. :URL: ``PUT /morpheme_language_model/id/generate`` :param str id: the ``id`` value of the morpheme language model whose files will be generated. :returns: the morpheme language model is returned; ``GET /morpheme_language_model/id`` must be polled to determine when the generation task has terminated. """ lm = Session.query(MorphemeLanguageModel).get(id) if not lm: response.status_int = 404 return {'error': 'There is no morpheme language model with id %s' % id} args = { 'morpheme_language_model_id': lm.id, 'user_id': session['user'].id, 'timeout': h.morpheme_language_model_generate_timeout } foma_worker_q.put({ 'id': h.generate_salt(), 'func': 'generate_language_model', 'args': args }) return lm
def compute_perplexity(self, id): """Compute the perplexity of the LM's corpus according to the LM. Randomly divide the corpus into training and test sets multiple times and compute the perplexity and return the average. See ``evaluate_morpheme_language_model`` in lib/foma_worker.py. """ lm = Session.query(MorphemeLanguageModel).get(id) if not lm: response.status_int = 404 return { 'error': 'There is no morpheme language model with id %s' % id } args = { 'morpheme_language_model_id': lm.id, 'user_id': session['user'].id, 'timeout': h.morpheme_language_model_generate_timeout } foma_worker_q.put({ 'id': h.generate_salt(), 'func': 'compute_perplexity', 'args': args }) return lm