def test_aaa_initialize(self):
"""Initialize the database using pseudo-data generated from random
lorem ipsum sentences.
These are located in ``old/tests/data/corpora``.
The data contain morphologically analyzed sentences, their component
morphemes, and syntactic categories. The sentences have phrase
structure trees in bracket notation.
The test will try to load the lorem ipsum dataset from a MySQL/SQLite
dump file in ``onlinelinguisticdatabase/tests/data/corpora``. If the
dump file corresponding to ``loremipsum_path`` does not exist, it will
import the lorem ipsum data directly from the text files and create
the dump file so that future tests can run more speedily. The
``loremipsum100_path``, ``loremipsum1000_path``, ``loremipsum10000_path``
and ``loremipsum30000_path`` files are available and contain 100, 1000
and 10,000 sentences, respectively.
Setting the ``via_request`` variable to ``True`` will cause all of the
forms to be created via request, i.e., via
``self.app.post(url('forms))...``. This is much slower but may be
desirable since values for the morphological analysis attributes
will be generated.
.. note::
In order to run ``mysqldump`` the MySQL user must have permission
to lock and update tables (alter and file privileges may also be
required ...)::
mysql -u root -p<root_password>
grant lock tables, update on old_test.* to 'old'@'localhost';
.. warning::
Loading the .txt or .sql files with the ``via_request`` option set to
``True`` will take a very long time. This might be an argument for
separating the interface and logic components of the controllers so
that a "core" HTTP-less OLD application could be exposed. This
would facilitate the creation of models with system-generated data
and validation but without the HTTP overhead...
"""
self.create_db()
dbsession = self.dbsession
db = DBUtils(dbsession, self.settings)
###################################################################
# Configure lorem ipsum data set import
###################################################################
# Set ``loremipsum_path`` this to ``self.loremipsum100_path``,
# ``self.loremipsum1000_path`` or ``self.loremipsum10000_path``.
# WARNING: the larger ones will take a long time.
# Use the 10,000-sentence lorem ipsum dataset to ensure that
# very large corpora are handled correctly.
loremipsum_path = self.loremipsum100_path
# Set ``via_request`` to ``True`` to create all forms via HTTP requests.
via_request = True
add_SEARCH_to_web_test_valid_methods()
# Add an application settings so that morpheme references will work
# out right.
application_settings = omb.generate_default_application_settings()
dbsession.add(application_settings)
dbsession.commit()
def create_model(line, categories, via_request=False):
"""Create a model (form or syncat) using the string in ``line``."""
model = 'Form'
elements = str(line).split('\t')
non_empty_elements = list(filter(None, elements))
try:
ol, mb, mg, ml, sc, sx = non_empty_elements
except ValueError:
try:
ol, mb, mg, ml, sc = non_empty_elements
sx = ''
except ValueError:
try:
model = 'SyntacticCategory'
n, t = non_empty_elements
except ValueError:
return categories
if via_request:
if model == 'SyntacticCategory':
params = self.syntactic_category_create_params.copy()
params.update({'name': n, 'type': t})
params = json.dumps(params)
response = self.app.post(
'/{}/syntacticcategories'.format(self.old_name),
params, self.json_headers, self.extra_environ_admin)
cat_id = response.json_body['id']
categories[n] = cat_id
else:
params = self.form_create_params.copy()
params.update({
'transcription':
ol,
'morpheme_break':
mb,
'morpheme_gloss':
mg,
'translations': [{
'transcription': ml,
'grammaticality': ''
}],
'syntax':
sx,
'syntactic_category':
categories.get(sc, '')
})
params = json.dumps(params)
self.app.post('/{}/forms'.format(self.old_name), params,
self.json_headers, self.extra_environ_admin)
else:
if model == 'SyntacticCategory':
syntactic_category = old_models.SyntacticCategory()
syntactic_category.name = n
syntactic_category.type = t
dbsession.add(syntactic_category)
categories[n] = syntactic_category.id
else:
form = old_models.Form()
form.transcription = ol
form.morpheme_break = mb
form.morpheme_gloss = mg
translation = old_models.Translation()
translation.transcription = ml
form.translations.append(translation)
form.syntax = sx
form.syntacticcategory_id = categories.get(sc, None)
dbsession.add(form)
return categories
def add_loremipsum_to_db(loremipsum_path, via_request=False):
"""Add the contents of the file at ``loremipsum_path`` to the database."""
categories = {}
with open(loremipsum_path, 'r') as f:
i = 0
for l in f:
if i % 100 == 0:
if not via_request: dbsession.commit()
LOGGER.debug('%d lines processed' % i)
i = i + 1
categories = create_model(l.replace('\n', ''), categories,
via_request)
dbsession.commit()
loremipsum_path_no_ext = os.path.splitext(loremipsum_path)[0]
sqlalchemy_URL = self.settings['sqlalchemy.url']
sqlalchemy_URL_list = sqlalchemy_URL.split(':')
olddump_script_path = os.path.join(self.test_scripts_path,
'olddump.sh')
oldload_script_path = os.path.join(self.test_scripts_path,
'oldload.sh')
RDBMS = sqlalchemy_URL_list[0]
if RDBMS.startswith('mysql'):
RDBMS = 'mysql'
if RDBMS == 'mysql':
mysql_dump_path = '%s_mysql.sql' % loremipsum_path_no_ext
username = sqlalchemy_URL_list[1][2:]
password = sqlalchemy_URL_list[2].split('@')[0]
dbname = sqlalchemy_URL_list[3].split('/')[1]
# This is not an option anymore: too frustrated trying to load
# the dump file.
if False and os.path.exists(mysql_dump_path):
LOGGER.debug(
'The lorem ipsum MySQL dump file exists. Loading it...')
# Clear the current DB completely
db.clear_all_models(retain=[])
# Load the dump file to the DB
shell_script = '#!/bin/sh\nmysql -u %s -p%s %s < %s' % (
username, password, dbname, mysql_dump_path)
with open(oldload_script_path, 'w') as f:
f.write(shell_script)
os.chmod(oldload_script_path, 0o744)
# Load the DB
with open(os.devnull, 'w') as f:
call([oldload_script_path], stdout=f, stderr=f)
# Destroy the load script
os.remove(oldload_script_path)
LOGGER.debug('Loaded.')
else:
LOGGER.debug(
'Have to import the lorem ipsum dataset from the text file and create the MySQL dump file.'
)
# Populate the database from the loremipusm text file and dump it
add_loremipsum_to_db(loremipsum_path, via_request=via_request)
# Write the DB dump shell script
# Note: the --single-transaction option seems to be required (on Mac MySQL 5.6 using InnoDB tables ...)
# see http://forums.mysql.com/read.php?10,108835,112951#msg-112951
shell_script = '#!/bin/sh\nmysqldump -u %s -p%s --single-transaction --no-create-info --result-file=%s %s' % (
username, password, mysql_dump_path, dbname)
with open(olddump_script_path, 'w') as f:
f.write(shell_script)
os.chmod(olddump_script_path, 0o744)
# Dump the DB
with open(os.devnull, 'w') as f:
call([olddump_script_path], stdout=f, stderr=f)
# Destroy the dump script
os.remove(olddump_script_path)
LOGGER.debug('Imported and dumped.')
elif RDBMS == 'sqlite' and h.command_line_program_installed('sqlite3'):
sqlite_dump_path = '%s_sqlite.sql' % loremipsum_path_no_ext
sqlite_full_dump_path = '%s_full_dump_tmp_sqlite.sql' % (
loremipsum_path_no_ext, )
sqlite_schema_dump_path = '%s_schema_dump_tmp_sqlite.sql' % (
loremipsum_path_no_ext, )
sqlite_db = sqlalchemy_URL.split('/')[-1]
dbpath = os.path.join(self.here, sqlite_db)
if os.path.exists(sqlite_dump_path):
LOGGER.debug(
'The lorem ipsum SQLite dump file exists. Loading it...')
print(
'The lorem ipsum SQLite dump file exists. Loading it...')
# Destroy the sqlite db file
os.remove(dbpath)
# Load the dump file to the DB
shell_script = '#!/bin/sh\nsqlite3 %s < %s' % (
dbpath, sqlite_dump_path)
with open(oldload_script_path, 'w') as f:
f.write(shell_script)
os.chmod(oldload_script_path, 0o744)
# Load the DB
with open(os.devnull, 'w') as f:
call([oldload_script_path], stdout=f, stderr=f)
# Destroy the load script
os.remove(oldload_script_path)
LOGGER.debug('Loaded.')
else:
LOGGER.debug('Have to import the lorem ipsum dataset from'
' the text file and create the SQLite dump'
' file.')
# Populate the database from the loremipusm text file and
# dump it
add_loremipsum_to_db(loremipsum_path, via_request=via_request)
# Write the DB dump shell script
shell_script = (
'#!/bin/sh\n'
'sqlite3 {dbpath} .dump > {dump_path}\n'.format(
dbpath=dbpath, dump_path=sqlite_dump_path))
_shell_script = ('#!/bin/sh\n'
'sqlite3 {dbpath} .schema > {schema_path}\n'
'sqlite3 {dbpath} .dump > {full_dump_path}\n'
'grep -vx -f {schema_path} {full_dump_path} >'
' {dump_path}\n'
'rm {schema_path}\n'
'rm {full_dump_path}\n'.format(
dbpath=dbpath,
schema_path=sqlite_schema_dump_path,
full_dump_path=sqlite_full_dump_path,
dump_path=sqlite_dump_path))
# shell_script = ('#!/bin/sh\nsqlite3 %s ".dump" | grep -v'
# ' "^CREATE" > %s' % (dbpath,
# sqlite_dump_path))
with open(olddump_script_path, 'w') as f:
f.write(shell_script)
os.chmod(olddump_script_path, 0o744)
# Dump the DB
with open(os.devnull, 'w') as f:
call([olddump_script_path], stdout=f, stderr=f)
# Destroy the dump script
os.remove(olddump_script_path)
LOGGER.debug('Imported and dumped.')
forms = db.get_forms()
LOGGER.debug(
'Lorem ipsum data loaded. There are now %d forms in the db.' %
len(forms))
print('Lorem ipsum data loaded. There are now %d forms in the db.' %
len(forms))
# Restrict one sentential form in the db.
restricted_tag = omb.generate_restricted_tag()
dbsession.add(restricted_tag)
dbsession.commit()
a_form = dbsession.query(old_models.Form).\
filter(old_models.Form.syntactic_category.\
has(old_models.SyntacticCategory.name=='S')).first()
a_form_id = a_form.id
a_form.tags.append(restricted_tag)
dbsession.commit()
restricted_form = dbsession.query(old_models.Form).\
filter(old_models.Form.tags.any(
old_models.Tag.name=='restricted')).first()
assert a_form_id == restricted_form.id
def test_writetofile_all_sentences(self):
"""Tests file writing/retrieval of a corpus containing all sentences.
That is, that ``PUT /corpora/id/writetofile`` and
``GET /corpora/id/servefile`` both work with a corpus defined by a form
search model that returns all sentences.
"""
dbsession = self.dbsession
db = DBUtils(dbsession, self.settings)
forms = db.get_forms()
assert len(forms) > 0
restricted_form_id = dbsession.query(old_models.Form)\
.filter(old_models.Form.tags.any(
old_models.Tag.name=='restricted')).first().id
tgrep2_installed = h.command_line_program_installed('tgrep2')
# Create a form search model that retrieves all sentences
query = {'filter': ['Form', 'syntactic_category', 'name', '=', 'S']}
params = json.dumps({
'name': 'Get all sentences',
'description': 'Query to return all sentences in the database.',
'search': query
})
response = self.app.post(
'/{old_name}/formsearches'.format(old_name=self.old_name), params,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
form_search_id = resp['id']
# Perform the search to get the resulting forms.
params = json.dumps({
'query': query,
'paginator': {
'page': 1,
'items_per_page': 1
}
})
response = self.app.post(
'/{old_name}/forms/search'.format(old_name=self.old_name), params,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
sentence_count = resp['paginator']['count']
# Generate some valid corpus creation input parameters.
params = self.corpus_create_params.copy()
params.update({
'name': 'Corpus of sentences',
'description': 'No ordering, no duplicates.',
'form_search': form_search_id
})
params = json.dumps(params)
# Create the corpus
#assert os.listdir(self.corpora_path) == []
original_corpus_count = dbsession.query(Corpus).count()
response = self.app.post(url('create'), params, self.json_headers,
self.extra_environ_admin)
resp = response.json_body
corpus_id = resp['id']
new_corpus_count = dbsession.query(Corpus).count()
corpus = dbsession.query(Corpus).get(corpus_id)
corpus_dir = os.path.join(self.corpora_path, 'corpus_%d' % corpus_id)
corpus_dir_contents = os.listdir(corpus_dir)
assert new_corpus_count == original_corpus_count + 1
assert resp['name'] == 'Corpus of sentences'
assert resp['description'] == 'No ordering, no duplicates.'
assert corpus_dir_contents == []
assert response.content_type == 'application/json'
assert resp['content'] == ''
assert len(corpus.forms) == sentence_count
assert resp['form_search']['id'] == form_search_id
# Try to TGrep2-search the corpus without first writing it to file
# and expect to fail.
tgrep2pattern = json.dumps({'tgrep2pattern': 'S < NP-SBJ'})
if h.command_line_program_installed('tgrep2'):
# Failed tgrep2 search with invalid corpus id.
response = self.app.request('/{}/corpora/{}/tgrep2'.format(
self.old_name, corpus_id),
method='SEARCH',
body=tgrep2pattern.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin,
status=400)
tgrep2resp = response.json_body
assert tgrep2resp['error'] == (
'Corpus %d has not been written to file as a treebank.' %
(corpus_id, ))
# Write the corpus to file
sleep(1)
params = json.dumps({'format': 'treebank'})
response = self.app.put('/%s/corpora/%d/writetofile' %
(self.old_name, corpus_id),
params,
headers=self.json_headers,
extra_environ=self.extra_environ_admin)
resp2 = response.json_body
corpus_dir_contents = os.listdir(corpus_dir)
corpus_tbk_path = os.path.join(corpus_dir, 'corpus_%d.tbk' % corpus_id)
corpus_tbk_mod_time = h.get_file_modification_time(corpus_tbk_path)
corpus_tbk_gzipped_path = '%s.gz' % corpus_tbk_path
corpus_tbk_file_length = h.get_file_length(corpus_tbk_path)
corpus_tbk_t2c_path = os.path.join(corpus_dir,
'corpus_%d.tbk.t2c' % corpus_id)
corpus_file_id = resp2['files'][0]['id']
assert resp['id'] == resp2['id']
assert resp['name'] == resp2['name']
assert resp2['datetime_modified'] > resp['datetime_modified']
assert os.path.exists(corpus_tbk_path)
if tgrep2_installed:
assert os.path.exists(corpus_tbk_t2c_path)
else:
assert not os.path.exists(corpus_tbk_t2c_path)
assert os.path.exists(corpus_tbk_gzipped_path)
assert get_file_size(corpus_tbk_path) > get_file_size(
corpus_tbk_gzipped_path)
assert sentence_count == corpus_tbk_file_length
# Retrieve the corpus file directly from the filesystem.
with open(corpus_tbk_path, 'rb') as filei:
corpus_file_object = filei
corpus_file_content = corpus_file_object.read()
# Attempt to retrieve the gzipped corpus file via request as a restricted
# user and expect to fail. This is because there is one restricted
# sentential form in the db, cf. the ``initialize`` "test".
response = self.app.get('/%s/corpora/%d/servefile/%d' %
(self.old_name, corpus_id, corpus_file_id),
params,
status=403,
headers=self.json_headers,
extra_environ=self.extra_environ_contrib)
resp = response.json_body
assert resp == UNAUTHORIZED_MSG
# Retrieve the gzipped corpus file via request.
response = self.app.get('/%s/corpora/%d/servefile/%d' %
(self.old_name, corpus_id, corpus_file_id),
params,
headers=self.json_headers,
extra_environ=self.extra_environ_admin)
unzipped_corpus_file_content = decompress_gzip_string(response.body)
assert unzipped_corpus_file_content == corpus_file_content
assert response.content_type == 'application/x-gzip'
# Now update the corpus by changing the form search, re-write-to-file
# and make sure everything works.
# Create a form search model that retrieves all sentences with even-numbered
# ids and the restricted form.
query = {
'filter': [
'and',
[['Form', 'syntactic_category', 'name', '=', 'S'],
[
'or',
[['Form', 'id', '=', restricted_form_id],
['Form', 'id', 'regex', '[02468]$']]
]]
]
}
params = json.dumps({
'name': 'Get even-numbered or restricted sentences',
'description':
'Query to return all sentences in the database that have even-numbered ids or are restricted.',
'search': query
})
response = self.app.post(
'/{old_name}/formsearches'.format(old_name=self.old_name), params,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
form_search_id = resp['id']
# Perform the search to get the resulting forms.
params = json.dumps({
'query': query,
'paginator': {
'page': 1,
'items_per_page': 1
}
})
response = self.app.post('/%s/forms/search' % self.old_name, params,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
sentence_count = resp['paginator']['count']
# Update the above-created corpus.
dbsession.expire(corpus)
params = self.corpus_create_params.copy()
params.update({
'name': 'Corpus of even-numbered sentences',
'description': 'No ordering, no duplicates.',
'form_search': form_search_id
})
params = json.dumps(params)
original_corpus_count = dbsession.query(Corpus).count()
response = self.app.put(url('update', id=corpus_id), params,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
new_corpus_count = dbsession.query(Corpus).count()
corpus = dbsession.query(Corpus).get(corpus_id)
corpus_dir = os.path.join(self.corpora_path, 'corpus_%d' % corpus_id)
corpus_dir_contents = os.listdir(corpus_dir)
assert new_corpus_count == original_corpus_count
assert resp['name'] == 'Corpus of even-numbered sentences'
assert resp['description'] == 'No ordering, no duplicates.'
assert corpus_dir_contents != [
] # Already a previously written corpus file there
assert response.content_type == 'application/json'
assert resp['content'] == ''
assert len(corpus.forms) == sentence_count
assert resp['form_search']['id'] == form_search_id
# Write the corpus to file
sleep(1)
params = json.dumps({'format': 'treebank'})
response = self.app.put('/%s/corpora/%d/writetofile' %
(self.old_name, corpus_id),
params,
headers=self.json_headers,
extra_environ=self.extra_environ_admin)
resp2 = response.json_body # Response is a JSON repr. of the corpus
corpus_dir_contents = os.listdir(corpus_dir)
corpus_tbk_path = os.path.join(corpus_dir, 'corpus_%d.tbk' % corpus_id)
old_corpus_tbk_mod_time = corpus_tbk_mod_time
corpus_tbk_mod_time = h.get_file_modification_time(corpus_tbk_path)
corpus_tbk_gzipped_path = '%s.gz' % corpus_tbk_path
corpus_tbk_file_length = h.get_file_length(
corpus_tbk_path) # no. of lines
corpus_tbk_t2c_path = os.path.join(corpus_dir,
'corpus_%d.tbk.t2c' % corpus_id)
corpus_file_id = resp2['files'][0]['id']
assert old_corpus_tbk_mod_time < corpus_tbk_mod_time
assert len(resp2['files']) == 1
assert resp['id'] == resp2['id']
assert resp['name'] == resp2['name']
assert resp2['datetime_modified'] > resp['datetime_modified']
assert os.path.exists(corpus_tbk_path)
assert os.path.exists(corpus_tbk_gzipped_path)
if tgrep2_installed:
assert os.path.exists(corpus_tbk_t2c_path)
else:
assert not os.path.exists(corpus_tbk_t2c_path)
assert get_file_size(corpus_tbk_path) > get_file_size(
corpus_tbk_gzipped_path)
assert sentence_count == corpus_tbk_file_length
# Retrieve the corpus file directly from the filesystem.
with open(corpus_tbk_path, 'rb') as filei:
corpus_file_object = filei
corpus_file_content = corpus_file_object.read()
# Attempt to retrieve the gzipped corpus file via request as a restricted
# user and expect to fail. This is because the one restricted sentential
# form in the db is in the corpus.
response = self.app.get('/%s/corpora/%d/servefile/%d' %
(self.old_name, corpus_id, corpus_file_id),
params,
status=403,
headers=self.json_headers,
extra_environ=self.extra_environ_contrib)
resp = response.json_body
assert resp == UNAUTHORIZED_MSG
# Retrieve the gzipped corpus file via request.
response = self.app.get('/%s/corpora/%d/servefile/%d' %
(self.old_name, corpus_id, corpus_file_id),
params,
headers=self.json_headers,
extra_environ=self.extra_environ_admin)
unzipped_corpus_file_content = decompress_gzip_string(response.body)
assert unzipped_corpus_file_content == corpus_file_content
# Write the corpus to file again without any changes and expect a vacuous recreation
sleep(1)
params = json.dumps({'format': 'treebank'})
response = self.app.put('/%s/corpora/%d/writetofile' %
(self.old_name, corpus_id),
params,
headers=self.json_headers,
extra_environ=self.extra_environ_admin)
old_resp2 = resp2
resp2 = response.json_body # Response is a JSON repr. of the corpus
corpus_tbk_path = os.path.join(corpus_dir, 'corpus_%d.tbk' % corpus_id)
old_corpus_tbk_mod_time = corpus_tbk_mod_time
corpus_tbk_mod_time = h.get_file_modification_time(corpus_tbk_path)
assert old_corpus_tbk_mod_time < corpus_tbk_mod_time
assert len(resp2['files']) == 1
assert resp2['datetime_modified'] > old_resp2['datetime_modified']
assert os.path.exists(corpus_tbk_path)
# TGrep2-search the corpus-as-treebank
# {'order_by': {'order_by_model': '', 'order_by_attribute': '', 'order_by_direction': ''}}
# {'paginator': {'page': 0, 'items_per_page': 0}}
tgrep2pattern = 'S < NP-SBJ'
query = {
'paginator': {
'page': 1,
'items_per_page': 10
},
'tgrep2pattern': tgrep2pattern
}
json_query = json.dumps(query)
if not h.command_line_program_installed('tgrep2'):
response = self.app.request('/{}/corpora/{}/tgrep2'.format(
self.old_name, corpus_id),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin,
status=400)
resp = response.json_body
assert resp["error"] == "TGrep2 is not installed."
else:
# TGrep2-search the corpus-as-treebank
response = self.app.request('/{}/corpora/{}/tgrep2'.format(
self.old_name, corpus_id),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin)
resp = response.json_body
for f in resp['items']:
assert '(S ' in f['syntax'] and '(NP-SBJ ' in f['syntax']
# A slightly more complex TGrep2 search
tgrep2pattern = 'S < NP-SBJ << DT'
query['tgrep2pattern'] = tgrep2pattern
json_query = json.dumps(query)
response = self.app.request('/{}/corpora/{}/tgrep2'.format(
self.old_name, corpus_id),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin)
resp = response.json_body
for f in resp['items']:
assert ('(S ' in f['syntax'] and '(NP-SBJ ' in f['syntax']
and '(DT ' in f['syntax'])
# Another TGrep2 search
tgrep2pattern = 'NP-SBJ < DT . VP'
query['tgrep2pattern'] = tgrep2pattern
json_query = json.dumps(query)
response = self.app.request('/{}/corpora/{}/tgrep2'.format(
self.old_name, corpus_id),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin)
resp = response.json_body
match_count = resp['paginator']['count']
for f in resp['items']:
assert ('(NP-SBJ ' in f['syntax'] and '(DT ' in f['syntax']
and '(VP ' in f['syntax'])
# Failed tgrep2 search with invalid corpus id.
response = self.app.request('/{}/corpora/{}/tgrep2'.format(
self.old_name, 123456789),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin,
status=404)
resp = response.json_body
assert resp['error'] == 'There is no corpus with id 123456789'
# Restricted user will not get all of the results.
response = self.app.request('/{}/corpora/{}/tgrep2'.format(
self.old_name, corpus_id),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_view)
resp = response.json_body
restricted_match_count = resp['paginator']['count']
assert isinstance(restricted_match_count,
int) and restricted_match_count < match_count
# Failed TGrep2 search: bad JSON in request body
json_query = json_query[:-1]
response = self.app.request('/{}/corpora/{}/tgrep2'.format(
self.old_name, corpus_id),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin,
status=400)
resp = response.json_body
assert resp == JSONDecodeErrorResponse
# Failed TGrep2 search: malformed params
tgrep2pattern = json.dumps({'TGrep2pattern': 'NP-SBJ < DT . VP'})
response = self.app.request('/{}/corpora/{}/tgrep2'.format(
self.old_name, corpus_id),
method='SEARCH',
body=tgrep2pattern.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin,
status=400)
resp = response.json_body
assert resp['errors']['tgrep2pattern'] == \
"A tgrep2pattern attribute must be supplied and must have a string value"
# Empty string TGrep2 pattern results in no forms being returned.
tgrep2pattern = json.dumps({'tgrep2pattern': ''})
response = self.app.request('/{}/corpora/{}/tgrep2'.format(
self.old_name, corpus_id),
method='SEARCH',
body=tgrep2pattern.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin)
resp = response.json_body
assert resp == []
def test_d_search(self):
"""Tests that SEARCH /rememberedforms/id returns an array of the forms remembered by the user with id=id that match the search criteria.
Here we show the somewhat complex interplay of the unrestricted users, the
restricted tag and the remembered_forms relation between users and forms.
"""
dbsession = self.dbsession
db = DBUtils(dbsession, self.settings)
forms = json.loads(
json.dumps([self.fix_form(f.get_dict()) for f in db.get_forms()]))
mysql_engine = old_models.Model.__table_args__.get('mysql_engine')
viewer, contributor, administrator = get_users(db)
viewer_id = viewer.id
contributor_id = contributor.id
administrator_id = administrator.id
viewer_remembered_forms = [
f for f in forms
if 'restricted' not in [t['name'] for t in f['tags']]
]
contributor_remembered_forms = [f for f in forms if f['id'] % 2 != 0]
administrator_remembered_forms = [
f for f in forms if f['id'] % 2 == 0 and f['id'] > 25
]
RDBMSName = h.get_RDBMS_name(self.settings)
_today_timestamp = today_timestamp
# The query we will use over and over again
json_query = json.dumps({
'query': {
'filter': [
'and',
[['Translation', 'transcription', 'like', '%1%'],
['not', ['Form', 'morpheme_break', 'regex', '[18][5-7]']],
[
'or',
[[
'Form', 'datetime_modified', '=',
today_timestamp.isoformat()
], ['Form', 'date_elicited', '=',
jan1.isoformat()]]
]]
]
}
})
# A slight variation on the above query so that searches on the admin's
# remembered forms will return some values
json_query_admin = json.dumps({
'query': {
'filter': [
'and',
[['Translation', 'transcription', 'like', '%8%'],
['not', ['Form', 'morpheme_break', 'regex', '[18][5-7]']],
[
'or',
[[
'Form', 'datetime_modified', '=',
today_timestamp.isoformat()
], ['Form', 'date_elicited', '=',
jan1.isoformat()]]
]]
]
}
})
# The expected output of the above query on each of the user's remembered forms list
result_set_viewer = [
f for f in viewer_remembered_forms
if '1' in ' '.join([g['transcription'] for g in f['translations']])
and not re.search('[18][5-7]', f['morpheme_break']) and
(_today_timestamp.isoformat() == f['datetime_modified'] or
(f['date_elicited'] and jan1.isoformat() == f['date_elicited']))
]
result_set_contributor = [
f for f in contributor_remembered_forms
if '1' in ' '.join([g['transcription'] for g in f['translations']])
and not re.search('[18][5-7]', f['morpheme_break']) and
(_today_timestamp.isoformat() == f['datetime_modified'] or
(f['date_elicited'] and jan1.isoformat() == f['date_elicited']))
]
result_set_administrator = [
f for f in administrator_remembered_forms
if '8' in ' '.join([g['transcription'] for g in f['translations']])
and not re.search('[18][5-7]', f['morpheme_break']) and
(_today_timestamp.isoformat() == f['datetime_modified'] or
(f['date_elicited'] and jan1.isoformat() == f['date_elicited']))
]
# Search the viewer's remembered forms as the viewer
response = self.app.post(
'/{old_name}/rememberedforms/{id}/search'.format(
old_name=self.old_name, id=viewer_id), json_query,
self.json_headers, self.extra_environ_admin_appset)
resp = response.json_body
assert [f['id'] for f in result_set_viewer] == [f['id'] for f in resp]
assert response.content_type == 'application/json'
assert resp
# Perform the same search as above on the contributor's remembered forms,
# as the contributor.
response = self.app.request('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=contributor_id),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_contrib_appset)
resp = response.json_body
assert [f['id']
for f in result_set_contributor] == ([f['id'] for f in resp])
assert response.content_type == 'application/json'
assert resp
# Perform the same search as above on the contributor's remembered forms,
# but search as the viewer and expect not to see the restricted forms,
# i.e., those with ids > 50.
response = self.app.post(
'/{old_name}/rememberedforms/{id}/search'.format(
old_name=self.old_name, id=contributor_id), json_query,
self.json_headers, self.extra_environ_view_appset)
resp = response.json_body
result_set = [
f for f in result_set_contributor
if 'restricted' not in [t['name'] for t in f['tags']]
]
assert [f['id'] for f in result_set] == [f['id'] for f in resp]
assert response.content_type == 'application/json'
assert resp
# Perform the search on the administrator's remembered forms as the viewer.
response = self.app.request('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=administrator_id),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_view_appset)
resp = response.json_body
result_set = [
f for f in result_set_administrator
if 'restricted' not in [t['name'] for t in f['tags']]
]
assert [f['id'] for f in result_set] == [f['id'] for f in resp]
assert response.content_type == 'application/json'
# Perform the search on the administrator's remembered forms as the
# contributor.
response = self.app.post(
'/{old_name}/rememberedforms/{id}/search'.format(
old_name=self.old_name, id=administrator_id), json_query_admin,
self.json_headers, self.extra_environ_contrib_appset)
resp = response.json_body
result_set = result_set_administrator
assert [f['id'] for f in result_set] == [f['id'] for f in resp]
assert response.content_type == 'application/json'
assert resp
# Perform the search on the administrator's remembered forms as the
# administrator.
response = self.app.request('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=administrator_id),
method='SEARCH',
body=json_query_admin.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin_appset)
resp = response.json_body
result_set = result_set_administrator
assert [f['id'] for f in result_set] == [f['id'] for f in resp]
assert response.content_type == 'application/json'
assert resp
def test_c_show(self):
"""Tests that GET /rememberedforms/id returns an array of the forms remembered by the user with id=id."""
dbsession = self.dbsession
db = DBUtils(dbsession, self.settings)
forms = json.loads(
json.dumps([self.fix_form(f.get_dict()) for f in db.get_forms()]))
viewer, contributor, administrator = get_users(db)
viewer_id = viewer.id
contributor_id = contributor.id
administrator_id = administrator.id
########################################################################
# Viewer
########################################################################
# Get the viewer's remembered forms (show that a contributor can do this)
response = self.app.get(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id),
headers=self.json_headers,
extra_environ=self.extra_environ_contrib_appset)
resp = response.json_body
result_set = [
f for f in forms
if 'restricted' not in [t['name'] for t in f['tags']]
]
assert response.content_type == 'application/json'
assert set([f['id']
for f in result_set]) == set([f['id'] for f in resp])
# Test the pagination and order by
# Test the paginator GET params.
paginator = {'items_per_page': 7, 'page': 3}
response = self.app.get(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id),
paginator,
headers=self.json_headers,
extra_environ=self.extra_environ_contrib_appset)
resp = response.json_body
assert response.content_type == 'application/json'
assert len(resp['items']) == 7
assert resp['items'][0]['transcription'] == result_set[14][
'transcription']
# Test the order_by GET params.
order_by_params = {
'order_by_model': 'Form',
'order_by_attribute': 'transcription',
'order_by_direction': 'desc'
}
response = self.app.get(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id),
order_by_params,
headers=self.json_headers,
extra_environ=self.extra_environ_contrib_appset)
resp = response.json_body
result_set_ordered = sorted(result_set,
key=lambda f: f['transcription'],
reverse=True)
assert response.content_type == 'application/json'
assert result_set_ordered == resp
# Test the order_by *with* paginator.
params = {
'order_by_model': 'Form',
'order_by_attribute': 'transcription',
'order_by_direction': 'desc',
'items_per_page': 7,
'page': 3
}
response = self.app.get(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id),
params,
headers=self.json_headers,
extra_environ=self.extra_environ_contrib_appset)
resp = response.json_body
assert len(resp['items']) == 7
assert result_set_ordered[14]['transcription'] == resp['items'][0][
'transcription']
# Expect a 400 error when the order_by_direction param is invalid
order_by_params = {
'order_by_model': 'Form',
'order_by_attribute': 'transcription',
'order_by_direction': 'descending'
}
response = self.app.get(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id),
order_by_params,
headers=self.json_headers,
extra_environ=self.extra_environ_contrib_appset,
status=400)
resp = response.json_body
assert response.content_type == 'application/json'
assert resp['errors'][
'order_by_direction'] == "Value must be one of: asc; desc (not 'descending')"
# Expect the default BY id ASCENDING ordering when the order_by_model/Attribute
# param is invalid.
order_by_params = {
'order_by_model': 'Formosa',
'order_by_attribute': 'transcrumption',
'order_by_direction': 'desc'
}
response = self.app.get(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id),
order_by_params,
headers=self.json_headers,
extra_environ=self.extra_environ_contrib_appset)
resp = response.json_body
assert resp[0]['id'] == forms[0]['id']
# Expect a 400 error when the paginator GET params are, empty, not
# or integers that are less than 1
paginator = {'items_per_page': 'a', 'page': ''}
response = self.app.get(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id),
paginator,
headers=self.json_headers,
extra_environ=self.extra_environ_contrib_appset,
status=400)
resp = response.json_body
assert resp['errors'][
'items_per_page'] == 'Please enter an integer value'
assert resp['errors']['page'] == 'Please enter a value'
paginator = {'items_per_page': 0, 'page': -1}
response = self.app.get(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id),
paginator,
headers=self.json_headers,
extra_environ=self.extra_environ_contrib_appset,
status=400)
resp = response.json_body
assert resp['errors'][
'items_per_page'] == 'Please enter a number that is 1 or greater'
assert resp['errors'][
'page'] == 'Please enter a number that is 1 or greater'
########################################################################
# Contributor
########################################################################
# Get the contributor's remembered forms
response = self.app.get(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=contributor_id),
headers=self.json_headers,
extra_environ=self.extra_environ_contrib_appset)
resp = response.json_body
result_set = [f for f in forms if f['id'] % 2 != 0]
assert response.content_type == 'application/json'
assert set([f['id']
for f in result_set]) == set([f['id'] for f in resp])
# Invalid user id returns a 404 error
response = self.app.get(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=200987654),
headers=self.json_headers,
extra_environ=self.extra_environ_contrib_appset,
status=404)
resp = response.json_body
assert response.content_type == 'application/json'
assert resp['error'] == 'There is no user with id 200987654'
########################################################################
# Administrator
########################################################################
# Get the administrator's remembered forms
response = self.app.get('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=administrator_id),
headers=self.json_headers,
extra_environ=self.extra_environ_admin_appset)
resp = response.json_body
result_set = [f for f in forms if f['id'] % 2 == 0 and f['id'] > 25]
assert response.content_type == 'application/json'
assert set([f['id']
for f in result_set]) == set([f['id'] for f in resp])
def test_b_update(self):
"""Tests that PUT /rememberedforms/id correctly updates the set of forms remembered by the user with id=id."""
dbsession = self.dbsession
db = DBUtils(dbsession, self.settings)
forms = sorted(json.loads(
json.dumps([self.fix_form(f.get_dict()) for f in db.get_forms()])),
key=lambda f: f['id'])
viewer, contributor, administrator = get_users(db)
viewer_id = viewer.id
viewer_datetime_modified = viewer.datetime_modified
contributor_id = contributor.id
administrator_id = administrator.id
########################################################################
# Viewer -- play with the viewer's remembered forms
########################################################################
# Try to add every form in the database to the viewer's remembered
# forms. Since the viewer is restricted (i.e., not unrestricted),
# an error will be generated.
sleep(1)
params = json.dumps({'forms': [f['id'] for f in forms]})
response = self.app.put(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id), params,
self.json_headers, self.extra_environ_view_appset)
resp = response.json_body
viewer_remembered_forms = sorted(resp, key=lambda f: f['id'])
result_set = [
f for f in forms
if 'restricted' not in [t['name'] for t in f['tags']]
]
dbsession.expire(viewer)
viewer, contributor, administrator = get_users(db)
new_viewer_datetime_modified = viewer.datetime_modified
assert new_viewer_datetime_modified != viewer_datetime_modified
assert set([f['id']
for f in result_set]) == set([f['id'] for f in resp])
assert response.content_type == 'application/json'
# Try to clear the viewer's remembered forms as the contributor and
# expect the request to be denied.
params = json.dumps({'forms': []})
response = self.app.put('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=viewer_id),
params,
self.json_headers,
self.extra_environ_contrib_appset,
status=403)
resp = response.json_body
assert response.content_type == 'application/json'
assert resp[
'error'] == 'You are not authorized to access this resource.'
# Get the list of ids from the userforms relational table. This is used
# to show that resetting a user's remembered_forms attribute via SQLAlchemy
# does not wastefully recreate all relations. See below
user_forms = dbsession.query(UserForm).filter(
UserForm.user_id == viewer_id).all()
expected_new_user_form_ids = [
uf.id for uf in user_forms
if uf.form_id != viewer_remembered_forms[-1]['id']
]
# Remove the last of the viewer's remembered forms as the administrator.
params = json.dumps(
{'forms': [f['id'] for f in viewer_remembered_forms][:-1]})
response = self.app.put(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id), params,
self.json_headers, self.extra_environ_admin_appset)
resp = response.json_body
result_set = result_set[:-1]
assert set([f['id']
for f in result_set]) == set([f['id'] for f in resp])
assert response.content_type == 'application/json'
# See what happens when a large list of remembered forms is altered like this;
# are all the relations destroyed and recreated?
# Get the list of ids from the userforms relational table
user_forms = dbsession.query(UserForm).filter(
UserForm.user_id == viewer_id).all()
current_user_form_ids = sorted([uf.id for uf in user_forms])
assert set(expected_new_user_form_ids) == set(current_user_form_ids)
# Attempted update fails: bad user id
params = json.dumps({'forms': []})
response = self.app.put('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=100896),
params,
self.json_headers,
self.extra_environ_admin_appset,
status=404)
resp = response.json_body
assert response.content_type == 'application/json'
assert resp['error'] == 'There is no user with id 100896'
# Attempted update fails: invalid array of form ids
params = json.dumps({'forms': ['a', 1000000087654]})
response = self.app.put('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=viewer_id),
params,
self.json_headers,
self.extra_environ_admin_appset,
status=400)
resp = response.json_body
assert response.content_type == 'application/json'
assert resp['errors']['forms'] == [
u'Please enter an integer value',
'There is no form with id 1000000087654.'
]
# Attempted update fails: array of form ids is bad JSON
params = json.dumps({'forms': []})[:-1]
response = self.app.put('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=viewer_id),
params,
self.json_headers,
self.extra_environ_admin_appset,
status=400)
resp = response.json_body
assert response.content_type == 'application/json'
assert resp[
'error'] == 'JSON decode error: the parameters provided were not valid JSON.'
# Clear the forms
params = json.dumps({'forms': []})
response = self.app.put(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id), params,
self.json_headers, self.extra_environ_admin_appset)
resp = response.json_body
viewer = dbsession.query(
old_models.User).filter(old_models.User.role == 'viewer').first()
assert response.content_type == 'application/json'
assert viewer.remembered_forms == []
assert resp == []
# Attempt to clear the forms again and fail because the submitted data are not new.
params = json.dumps({'forms': []})
response = self.app.put('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=viewer_id),
params,
self.json_headers,
self.extra_environ_view_appset,
status=400)
resp = response.json_body
assert response.content_type == 'application/json'
assert resp[
'error'] == 'The update request failed because the submitted data were not new.'
# Attempt to add all unrestricted forms to the viewer's remembered forms.
# Fail because unauthenticated.
params = json.dumps({'forms': [f['id'] for f in forms]})
response = self.app.put('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=viewer_id),
params,
self.json_headers,
status=401)
resp = response.json_body
assert response.content_type == 'application/json'
assert resp[
'error'] == 'Authentication is required to access this resource.'
# Finally for the viewer, re-add all unrestricted forms to the viewer's
# remembered forms for subsequent searches and GETs.
params = json.dumps({'forms': [f['id'] for f in forms]})
response = self.app.put(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=viewer_id), params,
self.json_headers, self.extra_environ_view_appset)
resp = response.json_body
assert response.content_type == 'application/json'
result_set = [
f for f in forms
if 'restricted' not in [t['name'] for t in f['tags']]
]
assert set([f['id']
for f in result_set]) == set([f['id'] for f in resp])
########################################################################
# Contributor -- play with the contributor's remembered forms
########################################################################
# The contributor is unrestricted. Add all forms to this user's
# remembered forms.
params = json.dumps({'forms': [f['id'] for f in forms]})
response = self.app.put(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=contributor_id),
params, self.json_headers, self.extra_environ_contrib_appset)
resp = response.json_body
assert response.content_type == 'application/json'
assert set([f['id'] for f in forms]) == set([f['id'] for f in resp])
# Change the contributor's remembered forms to contain only the forms
# with odd numbered ids.
odd_numbered_form_ids = [f['id'] for f in forms if f['id'] % 2 != 0]
params = json.dumps({'forms': odd_numbered_form_ids})
response = self.app.put(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=contributor_id),
params, self.json_headers, self.extra_environ_contrib_appset)
resp = response.json_body
assert response.content_type == 'application/json'
assert set(odd_numbered_form_ids) == set([f['id'] for f in resp])
########################################################################
# Administrator -- play with the administrator's remembered forms
########################################################################
# Make sure even an unrestricted contributor cannot update another user's
# remembered forms.
form_ids_for_admin = [
f['id'] for f in forms if f['id'] % 2 != 0 and f['id'] > 25
]
params = json.dumps({'forms': form_ids_for_admin})
response = self.app.put('/{old_name}/rememberedforms/{id}'.format(
old_name=self.old_name, id=administrator_id),
params,
self.json_headers,
self.extra_environ_contrib_appset,
status=403)
resp = response.json_body
assert response.content_type == 'application/json'
assert resp[
'error'] == 'You are not authorized to access this resource.'
# The administrator's remembered forms are all the evenly id-ed ones with
# ids greater than 25.
form_ids_for_admin = [
f['id'] for f in forms if f['id'] % 2 == 0 and f['id'] > 25
]
params = json.dumps({'forms': form_ids_for_admin})
response = self.app.put(
'/{old_name}/rememberedforms/{id}'.format(old_name=self.old_name,
id=administrator_id),
params, self.json_headers, self.extra_environ_admin_appset)
resp = response.json_body
assert response.content_type == 'application/json'
assert set(form_ids_for_admin) == set([f['id'] for f in resp])
def test_index(self):
"""Tests that GET & SEARCH /corpusbackups behave correctly.
"""
dbsession = self.dbsession
db = DBUtils(dbsession, self.settings)
tag = old_models.Tag()
tag.name = 'random tag name'
dbsession.add(tag)
dbsession.flush()
tag_id = tag.id
dbsession.commit()
# Add 10 forms and use them to generate a valid value for ``test_corpus_content``
def create_form_from_index(index):
form = old_models.Form()
form.transcription = 'Form %d' % index
translation = old_models.Translation()
translation.transcription = 'Translation %d' % index
form.translation = translation
return form
forms = [create_form_from_index(i) for i in range(1, 10)]
dbsession.add_all(forms)
dbsession.commit()
forms = db.get_forms()
half_forms = forms[:5]
form_ids = [form.id for form in forms]
half_form_ids = [form.id for form in half_forms]
test_corpus_content = ','.join(map(str, form_ids))
test_corpus_half_content = ','.join(map(str, half_form_ids))
# Create a form search model
query = {'filter': ['Form', 'transcription', 'regex', '[a-zA-Z]{3,}']}
params = json.dumps({
'name': 'form search',
'description': 'This one\'s worth saving!',
'search': query
})
response = self.app.post(fs_url('create'), params, self.json_headers,
self.extra_environ_admin)
resp = response.json_body
form_search_id = resp['id']
# Generate some valid corpus creation input parameters.
params = self.corpus_create_params.copy()
params.update({
'name': 'Corpus',
'description': 'Covers a lot of the data.',
'content': test_corpus_content
})
params = json.dumps(params)
# Attempt to create a corpus as a viewer and expect to fail
response = self.app.post(crps_url('create'),
params,
self.json_headers,
self.extra_environ_view,
status=403)
resp = response.json_body
assert resp[
'error'] == 'You are not authorized to access this resource.'
assert response.content_type == 'application/json'
# Successfully create a corpus as the admin
assert os.listdir(self.corpora_path) == []
original_corpus_count = dbsession.query(Corpus).count()
response = self.app.post(crps_url('create'), params, self.json_headers,
self.extra_environ_admin)
resp = response.json_body
corpus_id = resp['id']
new_corpus_count = dbsession.query(Corpus).count()
corpus = dbsession.query(Corpus).get(corpus_id)
corpus_form_ids = sorted([f.id for f in corpus.forms])
corpus_dir = os.path.join(self.corpora_path, 'corpus_%d' % corpus_id)
corpus_dir_contents = os.listdir(corpus_dir)
assert new_corpus_count == original_corpus_count + 1
assert resp['name'] == 'Corpus'
assert resp['description'] == 'Covers a lot of the data.'
assert corpus_dir_contents == []
assert response.content_type == 'application/json'
assert resp['content'] == test_corpus_content
assert corpus_form_ids == sorted(form_ids)
# Update the corpus as the contributor -- now we should have one backup
dbsession.expire(corpus)
params = self.corpus_create_params.copy()
params.update({
'name': 'Corpus',
'description': 'Covers a little less data.',
'content': test_corpus_half_content
})
params = json.dumps(params)
response = self.app.put(crps_url('update', id=corpus_id), params,
self.json_headers, self.extra_environ_contrib)
resp = response.json_body
corpus_count = new_corpus_count
new_corpus_count = dbsession.query(Corpus).count()
corpus = dbsession.query(Corpus).get(corpus_id)
corpus_form_ids = sorted([f.id for f in corpus.forms])
assert new_corpus_count == corpus_count
assert resp['name'] == 'Corpus'
assert resp['description'] == 'Covers a little less data.'
assert response.content_type == 'application/json'
assert resp['content'] == test_corpus_half_content
assert corpus_form_ids == sorted(half_form_ids)
# Update the corpus again -- now we should have two backups
sleep(1)
params = self.corpus_create_params.copy()
params.update({
'name': 'Corpus',
'description': 'Covers a little less data.',
'content': test_corpus_half_content,
'tags': [tag_id]
})
params = json.dumps(params)
response = self.app.put(crps_url('update', id=corpus_id), params,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
corpus_count = new_corpus_count
new_corpus_count = dbsession.query(Corpus).count()
corpus = dbsession.query(Corpus).get(corpus_id)
corpus_form_ids = sorted([f.id for f in corpus.forms])
assert new_corpus_count == corpus_count
assert resp['name'] == 'Corpus'
assert resp['description'] == 'Covers a little less data.'
assert response.content_type == 'application/json'
assert resp['content'] == test_corpus_half_content
assert corpus_form_ids == sorted(half_form_ids)
all_corpus_backups = dbsession.query(CorpusBackup).order_by(
CorpusBackup.id).all()
all_corpus_backup_ids = [cb.id for cb in all_corpus_backups]
all_corpus_backup_descriptions = [
cb.description for cb in all_corpus_backups
]
# Now request the corpus backups as either the contributor or the viewer and
# expect to get them all.
response = self.app.get(url('index'),
headers=self.json_headers,
extra_environ=self.extra_environ_contrib)
resp = response.json_body
assert len(resp) == 2
assert response.content_type == 'application/json'
assert resp[0]['modifier']['role'] == 'administrator'
assert resp[1]['modifier']['role'] == 'contributor'
# The admin should get them all too.
response = self.app.get(url('index'),
headers=self.json_headers,
extra_environ=self.extra_environ_view)
resp = response.json_body
assert len(resp) == 2
assert [cb['id'] for cb in resp] == all_corpus_backup_ids
# Test the paginator GET params.
paginator = {'items_per_page': 1, 'page': 2}
response = self.app.get(url('index'),
paginator,
headers=self.json_headers,
extra_environ=self.extra_environ_admin)
resp = response.json_body
assert len(resp['items']) == 1
assert resp['paginator']['count'] == 2
assert response.content_type == 'application/json'
assert resp['items'][0]['id'] == all_corpus_backup_ids[1]
# Test the order_by GET params.
order_by_params = {
'order_by_model': 'CorpusBackup',
'order_by_attribute': 'id',
'order_by_direction': 'desc'
}
response = self.app.get(url('index'),
order_by_params,
headers=self.json_headers,
extra_environ=self.extra_environ_admin)
resp = response.json_body
result_set = list(reversed(all_corpus_backup_ids))
assert [cb['id'] for cb in resp] == result_set
# Test the order_by *with* paginator.
params = {
'order_by_model': 'CorpusBackup',
'order_by_attribute': 'id',
'order_by_direction': 'desc',
'items_per_page': 1,
'page': 1
}
response = self.app.get(url('index'),
params,
headers=self.json_headers,
extra_environ=self.extra_environ_admin)
resp = response.json_body
assert result_set[0] == resp['items'][0]['id']
# Now test the show action:
# Get a specific corpus backup.
response = self.app.get(url('show', id=all_corpus_backup_ids[0]),
headers=self.json_headers,
extra_environ=self.extra_environ_admin)
resp = response.json_body
assert resp['description'] == 'Covers a lot of the data.'
assert resp['content'] == test_corpus_content
assert response.content_type == 'application/json'
# A nonexistent cb id will return a 404 error
response = self.app.get(url('show', id=100987),
headers=self.json_headers,
extra_environ=self.extra_environ_view,
status=404)
resp = response.json_body
assert resp['error'] == 'There is no corpus backup with id 100987'
assert response.content_type == 'application/json'
# Test the search action
add_SEARCH_to_web_test_valid_methods()
# A search on corpus backup titles using POST /corpusbackups/search
json_query = json.dumps({
'query': {
'filter': ['CorpusBackup', 'description', 'like', '%less%']
}
})
response = self.app.post(url('search_post'),
json_query,
self.json_headers,
extra_environ=self.extra_environ_admin)
resp = response.json_body
result_set = [
name for name in all_corpus_backup_descriptions if 'less' in name
]
assert len(resp) == len(result_set) == 1
assert resp[0]['description'] == result_set[0]
assert response.content_type == 'application/json'
# A search on corpus backup titles using SEARCH /corpusbackups
json_query = json.dumps({
'query': {
'filter': ['CorpusBackup', 'description', 'like', '%less%']
}
})
response = self.app.request(url('search'),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin)
resp = response.json_body
assert len(resp) == len(result_set) == 1
assert resp[0]['description'] == result_set[0]
assert response.content_type == 'application/json'
# Attempting to call edit/new/create/delete/update on a read-only resource
# will return a 404 response
response = self.app.get(url('edit', id=2232),
status=404,
extra_environ=self.extra_environ_admin)
assert response.json_body['error'] == 'This resource is read-only.'
response = self.app.get(url('new', id=2232),
status=404,
extra_environ=self.extra_environ_admin)
assert response.json_body['error'] == 'This resource is read-only.'
response = self.app.post(url('create'),
status=404,
extra_environ=self.extra_environ_admin)
assert response.json_body['error'] == 'This resource is read-only.'
response = self.app.put(url('update', id=2232),
status=404,
extra_environ=self.extra_environ_admin)
assert response.json_body['error'] == 'This resource is read-only.'
response = self.app.delete(url('delete', id=2232),
status=404,
extra_environ=self.extra_environ_admin)
assert response.json_body['error'] == 'This resource is read-only.'
assert response.content_type == 'application/json'