class CorporaController(BaseController):
"""Generate responses to requests on corpus resources.
REST Controller styled on the Atom Publishing Protocol.
.. note::
The ``h.jsonify`` decorator converts the return value of the methods to
JSON.
"""
query_builder_for_ordering = SQLAQueryBuilder('Corpus', config=config)
query_builder = SQLAQueryBuilder('Form', config=config)
@h.jsonify
@h.restrict('SEARCH', 'POST')
@h.authenticate
def search_corpora(self):
"""Return the list of corpora that match the input JSON query.
:URL: ``SEARCH/POST /corpora/searchcorpora``
:request body: A JSON object of the form::
{"query": {"filter": [ ... ], "order_by": [ ... ]},
"paginator": { ... }}
where the ``order_by`` and ``paginator`` attributes are optional.
.. note::
This action *does* result in a search across corpora resources.
Contrast this with the `search` method below which allows one to
search across the forms in a specified corpus.
"""
try:
json_search_params = unicode(request.body, request.charset)
python_search_params = json.loads(json_search_params)
SQLAQuery = self.query_builder_for_ordering.get_SQLA_query(
python_search_params.get('query'))
return h.add_pagination(SQLAQuery,
python_search_params.get('paginator'))
except h.JSONDecodeError:
response.status_int = 400
return h.JSONDecodeErrorResponse
except (OLDSearchParseError, Invalid), e:
response.status_int = 400
return {'errors': e.unpack_errors()}
except:
class FormbackupsController(BaseController):
"""Generate responses to requests on form backup resources.
REST Controller styled on the Atom Publishing Protocol.
.. note::
The ``h.jsonify`` decorator converts the return value of the methods to
JSON.
.. note::
Form backups are created when updating and deleting forms; they cannot
be created directly and they should never be deleted. This controller
facilitates searching and getting of form backups only.
"""
query_builder = SQLAQueryBuilder('FormBackup', config=config)
@h.jsonify
@h.restrict('SEARCH', 'POST')
@h.authenticate
def search(self):
"""Return the list of form backup resources matching the input
JSON query.
:URL: ``SEARCH /formbackups`` (or ``POST /formbackups/search``)
:request body: A JSON object of the form::
{"query": {"filter": [ ... ], "order_by": [ ... ]},
"paginator": { ... }}
where the ``order_by`` and ``paginator`` attributes are optional.
"""
try:
json_search_params = unicode(request.body, request.charset)
python_search_params = json.loads(json_search_params)
SQLAQuery = self.query_builder.get_SQLA_query(
python_search_params.get('query'))
query = h.filter_restricted_models('FormBackup', SQLAQuery)
return h.add_pagination(query,
python_search_params.get('paginator'))
except h.JSONDecodeError:
response.status_int = 400
return h.JSONDecodeErrorResponse
except (OLDSearchParseError, Invalid), e:
response.status_int = 400
return {'errors': e.unpack_errors()}
# SQLAQueryBuilder should have captured these exceptions (and packed
# them into an OLDSearchParseError) or sidestepped them, but here we'll
# handle any that got past -- just in case.
except (OperationalError, AttributeError, InvalidRequestError,
RuntimeError):
response.status_int = 400
return {
'error':
u'The specified search parameters generated an invalid database query'
}
def test_new_search(self):
"""Tests that GET /languages/new_search returns the search parameters for searching the languages resource."""
query_builder = SQLAQueryBuilder('Language')
response = self.app.get(url('/languages/new_search'), headers=self.json_headers,
extra_environ=self.extra_environ_view)
resp = json.loads(response.body)
assert resp['search_parameters'] == h.get_search_parameters(query_builder)
class MorphologicalparsersController(BaseController):
"""Generate responses to requests on morphological parser resources.
A morphological parser, as here conceived, is a morphophonological FST that is both a recognizer
and a transducer, plus a morpheme-based language model that ranks candidate parses according to
their probabilities. The morphophonological FST component maps surface strings to sequences of
morphemes and delimiters -- it is the composition of a pre-existing morphology FST and a pre-existing
phonology FST. The language model component is generated from the corpus specified in the language_model
attribute.
REST Controller styled on the Atom Publishing Protocol.
.. note::
The ``h.jsonify`` decorator converts the return value of the methods to
JSON.
"""
query_builder = SQLAQueryBuilder('MorphologicalParser', config=config)
@h.jsonify
@h.restrict('SEARCH', 'POST')
@h.authenticate
def search(self):
"""Return the list of morphological parser resources matching the input
JSON query.
:URL: ``SEARCH /morphologicalparsers``
(or ``POST /morphologicalparsers/search``)
:request body: A JSON object of the form::
{"query": {"filter": [ ... ], "order_by": [ ... ]},
"paginator": { ... }}
where the ``order_by`` and ``paginator`` attributes are optional.
"""
try:
json_search_params = unicode(request.body, request.charset)
python_search_params = json.loads(json_search_params)
query = self.query_builder.get_SQLA_query(
python_search_params.get('query'))
return h.add_pagination(query,
python_search_params.get('paginator'))
except h.JSONDecodeError:
response.status_int = 400
return h.JSONDecodeErrorResponse
except (OLDSearchParseError, Invalid), e:
response.status_int = 400
return {'errors': e.unpack_errors()}
except:
class LanguagesController(BaseController):
"""Generate responses to requests on language resources.
REST Controller styled on the Atom Publishing Protocol.
.. note::
The ``h.jsonify`` decorator converts the return value of the methods to
JSON.
.. note::
The language table is populated from an ISO 639-3 file upon application
setup. The language resouces are read-only. This controller
facilitates searching and getting of languages resources.
"""
query_builder = SQLAQueryBuilder('Language', 'Id', config=config)
@h.jsonify
@h.restrict('SEARCH', 'POST')
@h.authenticate
def search(self):
"""Return the list of language resources matching the input JSON query.
:URL: ``SEARCH /languages`` (or ``POST /languages/search``)
:request body: A JSON object of the form::
{"query": {"filter": [ ... ], "order_by": [ ... ]},
"paginator": { ... }}
where the ``order_by`` and ``paginator`` attributes are optional.
"""
try:
json_search_params = unicode(request.body, request.charset)
python_search_params = json.loads(json_search_params)
SQLAQuery = self.query_builder.get_SQLA_query(
python_search_params.get('query'))
return h.add_pagination(SQLAQuery,
python_search_params.get('paginator'))
except h.JSONDecodeError:
response.status_int = 400
return h.JSONDecodeErrorResponse
except (OLDSearchParseError, Invalid), e:
response.status_int = 400
return {'errors': e.unpack_errors()}
except:
class FormsearchesController(BaseController):
"""Generate responses to requests on form search resources.
REST Controller styled on the Atom Publishing Protocol.
.. note::
The ``h.jsonify`` decorator converts the return value of the methods to
JSON.
"""
query_builder = SQLAQueryBuilder('FormSearch', config=config)
@h.jsonify
@h.restrict('SEARCH', 'POST')
@h.authenticate
def search(self):
"""Return the list of form search resources matching the input JSON query.
:URL: ``SEARCH /formsearches`` (or ``POST /formsearches/search``)
:request body: A JSON object of the form::
{"query": {"filter": [ ... ], "order_by": [ ... ]},
"paginator": { ... }}
where the ``order_by`` and ``paginator`` attributes are optional.
.. note::
Yes, that's right, you can search form searches. (No, you can't
search searches of form searches :)
"""
try:
json_search_params = unicode(request.body, request.charset)
python_search_params = json.loads(json_search_params)
query = h.eagerload_form_search(
self.query_builder.get_SQLA_query(
python_search_params.get('query')))
return h.add_pagination(query,
python_search_params.get('paginator'))
except h.JSONDecodeError:
response.status_int = 400
return h.JSONDecodeErrorResponse
except (OLDSearchParseError, Invalid), e:
response.status_int = 400
return {'errors': e.unpack_errors()}
except:
class MorphemelanguagemodelsController(BaseController):
"""Generate responses to requests on morpheme language model resources.
A morpheme language model is a function that computes probabilities for sequences of morphemes.
REST Controller styled on the Atom Publishing Protocol.
.. note::
The ``h.jsonify`` decorator converts the return value of the methods to
JSON.
"""
query_builder = SQLAQueryBuilder('MorphemeLanguageModel', config=config)
@h.jsonify
@h.restrict('SEARCH', 'POST')
@h.authenticate
def search(self):
"""Return the list of morpheme language model resources matching the
input JSON query.
:URL: ``SEARCH /morphemelanguagemodels`` (or ``POST
/morphemelanguagemodels/search``)
:request body: A JSON object of the form::
{"query": {"filter": [ ... ], "order_by": [ ... ]},
"paginator": { ... }}
where the ``order_by`` and ``paginator`` attributes are optional.
"""
try:
json_search_params = unicode(request.body, request.charset)
python_search_params = json.loads(json_search_params)
query = self.query_builder.get_SQLA_query(
python_search_params.get('query'))
return h.add_pagination(query,
python_search_params.get('paginator'))
except h.JSONDecodeError:
response.status_int = 400
return h.JSONDecodeErrorResponse
except (OLDSearchParseError, Invalid), e:
response.status_int = 400
return {'errors': e.unpack_errors()}
except:
class MorphemelanguagemodelbackupsController(BaseController):
"""Generate responses to requests on morpheme language model backup resources.
REST Controller styled on the Atom Publishing Protocol.
.. note::
The ``h.jsonify`` decorator converts the return value of the methods to
JSON.
.. note::
Morpheme language model backups are created when updating and deleting
morpheme language models; they cannot be created directly and they should
never be deleted. This controller facilitates retrieval of morpheme
language model backups only.
"""
query_builder = SQLAQueryBuilder('MorphemeLanguageModelBackup',
config=config)
@h.jsonify
@h.restrict('GET')
@h.authenticate
def index(self):
"""Get all morpheme language model backup resources.
:URL: ``GET /morphemelanguagemodelbackups``
:returns: a list of all morpheme language model backup resources.
"""
try:
query = Session.query(MorphemeLanguageModelBackup)
query = h.add_order_by(query, dict(request.GET),
self.query_builder)
return h.add_pagination(query, dict(request.GET))
except Invalid, e:
response.status_int = 400
return {'errors': e.unpack_errors()}
class UsersController(BaseController):
"""Generate responses to requests on user resources.
REST Controller styled on the Atom Publishing Protocol.
.. note::
The ``h.jsonify`` decorator converts the return value of the methods to
JSON.
"""
query_builder = SQLAQueryBuilder('User', config=config)
@h.jsonify
@h.restrict('GET')
@h.authenticate
def index(self):
"""Get all user resources.
:URL: ``GET /users`` with optional query string parameters for
ordering and pagination.
:returns: a list of all user resources.
.. note::
See :func:`utils.add_order_by` and :func:`utils.add_pagination` for the
query string parameters that effect ordering and pagination.
"""
try:
query = Session.query(User)
query = h.add_order_by(query, dict(request.GET),
self.query_builder)
return h.add_pagination(query, dict(request.GET))
except Invalid, e:
response.status_int = 400
return {'errors': e.unpack_errors()}
class OldcollectionsController(BaseController):
"""Generate responses to requests on collection resources.
REST Controller styled on the Atom Publishing Protocol.
The collections controller is one of the more complex ones. A great deal of
this complexity arised from the fact that collections can reference forms
and other collections in the value of their ``contents`` attribute. The
propagation of restricted tags and associated forms and the generation of
the html from these contents-with-references, necessitates some complex
logic for updates and deletions.
.. warning::
There is a potential issue with collection-collection reference. A
restricted user can restrict their own collection *A* and that
restriction would be propagated up the reference chain, possibly causing
another collection *B* (that was not created by the updater) to become
restricted. That is, collection-collection reference permits restricted
users to indirectly restrict collections they would otherwise not be
permitted to restrict. This will be bothersome to other restricted users
since they will no longer be able to access the newly restricted
collection *B*.
.. note::
The ``h.jsonify`` decorator converts the return value of the methods to
JSON.
"""
query_builder = SQLAQueryBuilder('Collection', config=config)
@h.jsonify
@h.restrict('SEARCH', 'POST')
@h.authenticate
def search(self):
"""Return the list of collection resources matching the input JSON query.
:URL: ``SEARCH /collections`` (or ``POST /collections/search``)
:request body: A JSON object of the form::
{"query": {"filter": [ ... ], "order_by": [ ... ]},
"paginator": { ... }}
where the ``order_by`` and ``paginator`` attributes are optional.
.. note::
Search does not return the forms of all collections that match the
search. For that, a second request is required, i.e., to
``GET /collections/id``.
"""
try:
json_search_params = unicode(request.body, request.charset)
python_search_params = json.loads(json_search_params)
SQLAQuery = h.eagerload_collection(
self.query_builder.get_SQLA_query(
python_search_params.get('query')))
query = h.filter_restricted_models('Collection', SQLAQuery)
return h.add_pagination(query,
python_search_params.get('paginator'))
except h.JSONDecodeError:
response.status_int = 400
return h.JSONDecodeErrorResponse
except (OLDSearchParseError, Invalid), e:
response.status_int = 400
return {'errors': e.unpack_errors()}
except:
class RememberedformsController(BaseController):
"""Generate responses to requests on remembered forms resources.
REST Controller styled on the Atom Publishing Protocol.
.. note::
Remembered forms is a pseudo-REST-ful resource. Remembered forms are
stored in the ``userform`` many-to-many table (cf. ``model/user.py``)
which defines the contents of a user's ``remembered_forms`` attribute
(as well as the contents of a form's ``memorizers`` attribute). A user's
remembered forms are not affected by requests to the user resource.
Instead, the remembered forms resource handles modification, retrieval
and search of a user's remembered forms.
Overview of the interface:
* ``GET /rememberedforms/id``
* ``UPDATE /rememberedforms/id``
* ``SEARCH /rememberedforms/id``
.. note::
The ``h.jsonify`` decorator converts the return value of the methods to
JSON.
"""
query_builder = SQLAQueryBuilder(config=config)
@h.jsonify
@h.authenticate
@h.restrict('GET')
def show(self, id):
"""Return a user's remembered forms.
:URL: ``GET /rememberedforms/id`` with optional query string parameters
for ordering and pagination.
:param str id: the ``id`` value of a user model.
:returns: a list form models.
.. note::
Any authenticated user is authorized to access this resource.
Restricted forms are filtered from the array on a per-user basis.
.. note::
See :func:`utils.add_order_by` and :func:`utils.add_pagination` for the
query string parameters that effect ordering and pagination.
"""
user = Session.query(User).get(id)
if user:
try:
query = h.eagerload_form(Session.query(Form))\
.filter(Form.memorizers.contains(user))
query = h.add_order_by(query, dict(request.GET),
self.query_builder)
query = h.filter_restricted_models('Form', query)
return h.add_pagination(query, dict(request.GET))
except Invalid, e:
response.status_int = 400
return {'errors': e.unpack_errors()}
else:
def test_search(self):
"""Tests that corpora search works correctly.
"""
# Create a corpus defined by ``content`` that contains all sentences
# with five or more words.
# Get ids of all sentences with more than 5 words.
long_sentences = Session.query(model.Form).\
filter(and_(
model.Form.syntactic_category.has(model.SyntacticCategory.name==u'S'),
model.Form.transcription.op('regexp')(u'^([^ ]+ ){5}[^ ]+'))).all()
long_sentence = long_sentences[0]
len_long_sentences = len(long_sentences)
long_sentence_ids = [f.id for f in long_sentences]
long_sentences = u','.join(map(str, long_sentence_ids))
# Restrict one of the forms that will be in the corpus.
restricted_tag = h.get_restricted_tag()
long_sentence.tags.append(restricted_tag)
Session.add(long_sentence)
Session.commit()
# Create the corpus
name = u'Sentences with 6 or more words.'
params = self.corpus_create_params.copy()
params.update({'name': name, 'content': long_sentences})
params = json.dumps(params)
original_corpus_count = Session.query(Corpus).count()
response = self.app.post(url('corpora'), params, self.json_headers,
self.extra_environ_admin)
resp = json.loads(response.body)
corpus_id = resp['id']
new_corpus_count = Session.query(Corpus).count()
corpus = Session.query(Corpus).get(corpus_id)
corpus_dir = os.path.join(self.corpora_path, 'corpus_%d' % corpus_id)
corpus_dir_contents = os.listdir(corpus_dir)
assert new_corpus_count == original_corpus_count + 1
assert resp['name'] == name
assert corpus_dir_contents == []
assert response.content_type == 'application/json'
assert resp['content'] == long_sentences
# The ``forms`` attribute is a collection, no repeats, that's why the following is true:
assert len(corpus.forms) == len_long_sentences
# Search the corpus for forms beginning in vowels.
query = json.dumps({
"query": {
"filter": ['Form', 'transcription', 'regex', '^[AEIOUaeiou]']
},
"paginator": {
'page': 1,
'items_per_page': 10
}
})
response = self.app.post(url('/corpora/%d/search' % corpus_id), query,
self.json_headers, self.extra_environ_admin)
resp = json.loads(response.body)
matches = resp['items']
assert not set([f['id'] for f in matches]) - set(long_sentence_ids)
assert not filter(
lambda f: f['transcription'][0].lower() not in
['a', 'e', 'i', 'o', 'u'], matches)
assert not filter(lambda f: len(f['transcription'].split(' ')) < 6,
matches)
# Vacuous search of the corpus returns everything.
query = json.dumps(
{"query": {
"filter": ['Form', 'transcription', 'like', '%']
}})
response = self.app.post(url('/corpora/%d/search' % corpus_id), query,
self.json_headers, self.extra_environ_admin)
resp = json.loads(response.body)
assert set([f['id'] for f in resp]) == set(long_sentence_ids)
# Vacuous search as the viewer returns everything that is not restricted.
query = json.dumps(
{"query": {
"filter": ['Form', 'transcription', 'like', '%']
}})
response = self.app.post(url('/corpora/%d/search' % corpus_id), query,
self.json_headers, self.extra_environ_view)
resp2 = json.loads(response.body)
# Viewer will get 1 or 2 forms fewer (2 are restricted, 1 assuredly a long sentence.)
assert len(resp) > len(resp2)
# Failed search with an invalid corpus id
query = json.dumps(
{"query": {
"filter": ['Form', 'transcription', 'like', '%']
}})
response = self.app.post(url('/corpora/123456789/search'),
query,
self.json_headers,
self.extra_environ_admin,
status=404)
resp = json.loads(response.body)
assert resp['error'] == u'There is no corpus with id 123456789'
# Failed search with an invalid query
query = json.dumps(
{"query": {
"filter": ['Form', 'thingamafracasicle', 'like', '%']
}})
response = self.app.post(url('/corpora/%d/search' % corpus_id),
query,
self.json_headers,
self.extra_environ_admin,
status=400)
resp = json.loads(response.body)
assert resp['errors'][
'Form.thingamafracasicle'] == 'There is no attribute thingamafracasicle of Form'
# Request GET /corpora/new_search
response = self.app.get(url(controller='corpora', action='new_search'),
headers=self.json_headers,
extra_environ=self.extra_environ_admin)
resp = json.loads(response.body)
assert resp == {
'search_parameters':
h.get_search_parameters(SQLAQueryBuilder('Form'))
}