def test_a_initialize(self): """Initialize the database for /rememberedforms tests.""" dbsession = self.dbsession db = DBUtils(dbsession, self.settings) self.create_db() db.clear_all_models() administrator = omb.generate_default_administrator( settings=self.settings) contributor = omb.generate_default_contributor(settings=self.settings) viewer = omb.generate_default_viewer(settings=self.settings) dbsession.add_all([administrator, contributor, viewer]) dbsession.commit() _create_test_data(dbsession, db, self.n) add_SEARCH_to_web_test_valid_methods() # Create an application settings where the contributor is unrestricted viewer, contributor, administrator = get_users(db) application_settings = omb.generate_default_application_settings() application_settings.unrestricted_users = [contributor] dbsession.add(application_settings) dbsession.commit()
def test_aaa_initialize(self): """Initialize the database using pseudo-data generated from random lorem ipsum sentences. These are located in ``old/tests/data/corpora``. The data contain morphologically analyzed sentences, their component morphemes, and syntactic categories. The sentences have phrase structure trees in bracket notation. The test will try to load the lorem ipsum dataset from a MySQL/SQLite dump file in ``onlinelinguisticdatabase/tests/data/corpora``. If the dump file corresponding to ``loremipsum_path`` does not exist, it will import the lorem ipsum data directly from the text files and create the dump file so that future tests can run more speedily. The ``loremipsum100_path``, ``loremipsum1000_path``, ``loremipsum10000_path`` and ``loremipsum30000_path`` files are available and contain 100, 1000 and 10,000 sentences, respectively. Setting the ``via_request`` variable to ``True`` will cause all of the forms to be created via request, i.e., via ``self.app.post(url('forms))...``. This is much slower but may be desirable since values for the morphological analysis attributes will be generated. .. note:: In order to run ``mysqldump`` the MySQL user must have permission to lock and update tables (alter and file privileges may also be required ...):: mysql -u root -p<root_password> grant lock tables, update on old_test.* to 'old'@'localhost'; .. warning:: Loading the .txt or .sql files with the ``via_request`` option set to ``True`` will take a very long time. This might be an argument for separating the interface and logic components of the controllers so that a "core" HTTP-less OLD application could be exposed. This would facilitate the creation of models with system-generated data and validation but without the HTTP overhead... """ self.create_db() dbsession = self.dbsession db = DBUtils(dbsession, self.settings) ################################################################### # Configure lorem ipsum data set import ################################################################### # Set ``loremipsum_path`` this to ``self.loremipsum100_path``, # ``self.loremipsum1000_path`` or ``self.loremipsum10000_path``. # WARNING: the larger ones will take a long time. # Use the 10,000-sentence lorem ipsum dataset to ensure that # very large corpora are handled correctly. loremipsum_path = self.loremipsum100_path # Set ``via_request`` to ``True`` to create all forms via HTTP requests. via_request = True add_SEARCH_to_web_test_valid_methods() # Add an application settings so that morpheme references will work # out right. application_settings = omb.generate_default_application_settings() dbsession.add(application_settings) dbsession.commit() def create_model(line, categories, via_request=False): """Create a model (form or syncat) using the string in ``line``.""" model = 'Form' elements = str(line).split('\t') non_empty_elements = list(filter(None, elements)) try: ol, mb, mg, ml, sc, sx = non_empty_elements except ValueError: try: ol, mb, mg, ml, sc = non_empty_elements sx = '' except ValueError: try: model = 'SyntacticCategory' n, t = non_empty_elements except ValueError: return categories if via_request: if model == 'SyntacticCategory': params = self.syntactic_category_create_params.copy() params.update({'name': n, 'type': t}) params = json.dumps(params) response = self.app.post( '/{}/syntacticcategories'.format(self.old_name), params, self.json_headers, self.extra_environ_admin) cat_id = response.json_body['id'] categories[n] = cat_id else: params = self.form_create_params.copy() params.update({ 'transcription': ol, 'morpheme_break': mb, 'morpheme_gloss': mg, 'translations': [{ 'transcription': ml, 'grammaticality': '' }], 'syntax': sx, 'syntactic_category': categories.get(sc, '') }) params = json.dumps(params) self.app.post('/{}/forms'.format(self.old_name), params, self.json_headers, self.extra_environ_admin) else: if model == 'SyntacticCategory': syntactic_category = old_models.SyntacticCategory() syntactic_category.name = n syntactic_category.type = t dbsession.add(syntactic_category) categories[n] = syntactic_category.id else: form = old_models.Form() form.transcription = ol form.morpheme_break = mb form.morpheme_gloss = mg translation = old_models.Translation() translation.transcription = ml form.translations.append(translation) form.syntax = sx form.syntacticcategory_id = categories.get(sc, None) dbsession.add(form) return categories def add_loremipsum_to_db(loremipsum_path, via_request=False): """Add the contents of the file at ``loremipsum_path`` to the database.""" categories = {} with open(loremipsum_path, 'r') as f: i = 0 for l in f: if i % 100 == 0: if not via_request: dbsession.commit() LOGGER.debug('%d lines processed' % i) i = i + 1 categories = create_model(l.replace('\n', ''), categories, via_request) dbsession.commit() loremipsum_path_no_ext = os.path.splitext(loremipsum_path)[0] sqlalchemy_URL = self.settings['sqlalchemy.url'] sqlalchemy_URL_list = sqlalchemy_URL.split(':') olddump_script_path = os.path.join(self.test_scripts_path, 'olddump.sh') oldload_script_path = os.path.join(self.test_scripts_path, 'oldload.sh') RDBMS = sqlalchemy_URL_list[0] if RDBMS.startswith('mysql'): RDBMS = 'mysql' if RDBMS == 'mysql': mysql_dump_path = '%s_mysql.sql' % loremipsum_path_no_ext username = sqlalchemy_URL_list[1][2:] password = sqlalchemy_URL_list[2].split('@')[0] dbname = sqlalchemy_URL_list[3].split('/')[1] # This is not an option anymore: too frustrated trying to load # the dump file. if False and os.path.exists(mysql_dump_path): LOGGER.debug( 'The lorem ipsum MySQL dump file exists. Loading it...') # Clear the current DB completely db.clear_all_models(retain=[]) # Load the dump file to the DB shell_script = '#!/bin/sh\nmysql -u %s -p%s %s < %s' % ( username, password, dbname, mysql_dump_path) with open(oldload_script_path, 'w') as f: f.write(shell_script) os.chmod(oldload_script_path, 0o744) # Load the DB with open(os.devnull, 'w') as f: call([oldload_script_path], stdout=f, stderr=f) # Destroy the load script os.remove(oldload_script_path) LOGGER.debug('Loaded.') else: LOGGER.debug( 'Have to import the lorem ipsum dataset from the text file and create the MySQL dump file.' ) # Populate the database from the loremipusm text file and dump it add_loremipsum_to_db(loremipsum_path, via_request=via_request) # Write the DB dump shell script # Note: the --single-transaction option seems to be required (on Mac MySQL 5.6 using InnoDB tables ...) # see http://forums.mysql.com/read.php?10,108835,112951#msg-112951 shell_script = '#!/bin/sh\nmysqldump -u %s -p%s --single-transaction --no-create-info --result-file=%s %s' % ( username, password, mysql_dump_path, dbname) with open(olddump_script_path, 'w') as f: f.write(shell_script) os.chmod(olddump_script_path, 0o744) # Dump the DB with open(os.devnull, 'w') as f: call([olddump_script_path], stdout=f, stderr=f) # Destroy the dump script os.remove(olddump_script_path) LOGGER.debug('Imported and dumped.') elif RDBMS == 'sqlite' and h.command_line_program_installed('sqlite3'): sqlite_dump_path = '%s_sqlite.sql' % loremipsum_path_no_ext sqlite_full_dump_path = '%s_full_dump_tmp_sqlite.sql' % ( loremipsum_path_no_ext, ) sqlite_schema_dump_path = '%s_schema_dump_tmp_sqlite.sql' % ( loremipsum_path_no_ext, ) sqlite_db = sqlalchemy_URL.split('/')[-1] dbpath = os.path.join(self.here, sqlite_db) if os.path.exists(sqlite_dump_path): LOGGER.debug( 'The lorem ipsum SQLite dump file exists. Loading it...') print( 'The lorem ipsum SQLite dump file exists. Loading it...') # Destroy the sqlite db file os.remove(dbpath) # Load the dump file to the DB shell_script = '#!/bin/sh\nsqlite3 %s < %s' % ( dbpath, sqlite_dump_path) with open(oldload_script_path, 'w') as f: f.write(shell_script) os.chmod(oldload_script_path, 0o744) # Load the DB with open(os.devnull, 'w') as f: call([oldload_script_path], stdout=f, stderr=f) # Destroy the load script os.remove(oldload_script_path) LOGGER.debug('Loaded.') else: LOGGER.debug('Have to import the lorem ipsum dataset from' ' the text file and create the SQLite dump' ' file.') # Populate the database from the loremipusm text file and # dump it add_loremipsum_to_db(loremipsum_path, via_request=via_request) # Write the DB dump shell script shell_script = ( '#!/bin/sh\n' 'sqlite3 {dbpath} .dump > {dump_path}\n'.format( dbpath=dbpath, dump_path=sqlite_dump_path)) _shell_script = ('#!/bin/sh\n' 'sqlite3 {dbpath} .schema > {schema_path}\n' 'sqlite3 {dbpath} .dump > {full_dump_path}\n' 'grep -vx -f {schema_path} {full_dump_path} >' ' {dump_path}\n' 'rm {schema_path}\n' 'rm {full_dump_path}\n'.format( dbpath=dbpath, schema_path=sqlite_schema_dump_path, full_dump_path=sqlite_full_dump_path, dump_path=sqlite_dump_path)) # shell_script = ('#!/bin/sh\nsqlite3 %s ".dump" | grep -v' # ' "^CREATE" > %s' % (dbpath, # sqlite_dump_path)) with open(olddump_script_path, 'w') as f: f.write(shell_script) os.chmod(olddump_script_path, 0o744) # Dump the DB with open(os.devnull, 'w') as f: call([olddump_script_path], stdout=f, stderr=f) # Destroy the dump script os.remove(olddump_script_path) LOGGER.debug('Imported and dumped.') forms = db.get_forms() LOGGER.debug( 'Lorem ipsum data loaded. There are now %d forms in the db.' % len(forms)) print('Lorem ipsum data loaded. There are now %d forms in the db.' % len(forms)) # Restrict one sentential form in the db. restricted_tag = omb.generate_restricted_tag() dbsession.add(restricted_tag) dbsession.commit() a_form = dbsession.query(old_models.Form).\ filter(old_models.Form.syntactic_category.\ has(old_models.SyntacticCategory.name=='S')).first() a_form_id = a_form.id a_form.tags.append(restricted_tag) dbsession.commit() restricted_form = dbsession.query(old_models.Form).\ filter(old_models.Form.tags.any( old_models.Tag.name=='restricted')).first() assert a_form_id == restricted_form.id
def test_index(self): """Tests that GET & SEARCH /collectionbackups behave correctly. """ db = DBUtils(self.dbsession, self.settings) # Add some test data to the database. application_settings = omb.generate_default_application_settings() source = omb.generate_default_source() restricted_tag = omb.generate_restricted_tag() file1 = omb.generate_default_file() file1.name = 'file1' file2 = omb.generate_default_file() file2.name = 'file2' speaker = omb.generate_default_speaker() self.dbsession.add_all([ application_settings, source, restricted_tag, file1, file2, speaker ]) self.dbsession.flush() speaker_id = speaker.id restricted_tag_id = restricted_tag.id file1_id = file1.id file2_id = file2.id self.dbsession.commit() tag_ids = [restricted_tag_id] file_ids = [file1_id, file2_id] # Create a restricted collection (via request) as the default contributor users = db.get_users() contributor_id = [u for u in users if u.role == 'contributor'][0].id administrator_id = [u for u in users if u.role == 'administrator'][0].id # Define some extra_environs view = { 'test.authentication.role': 'viewer', 'test.application_settings': True } contrib = { 'test.authentication.role': 'contributor', 'test.application_settings': True } admin = { 'test.authentication.role': 'administrator', 'test.application_settings': True } params = self.collection_create_params.copy() params.update({ 'title': 'Created by the Contributor', 'elicitor': contributor_id, 'tags': [restricted_tag_id] }) params = json.dumps(params) response = self.app.post(coll_url('create'), params, self.json_headers, contrib) collection_count = self.dbsession.query(old_models.Collection).count() resp = response.json_body collection_id = resp['id'] assert response.content_type == 'application/json' assert collection_count == 1 # Update our collection (via request) as the default administrator; this # will create one collection backup. params = self.collection_create_params.copy() params.update({ 'url': 'find/me/here', 'title': 'Updated by the Administrator', 'speaker': speaker_id, 'tags': tag_ids + [ None, '' ], # None and '' ('') will be ignored by collections.update_collection 'enterer': administrator_id # This should change nothing. }) params = json.dumps(params) response = self.app.put(coll_url('update', id=collection_id), params, self.json_headers, admin) resp = response.json_body collection_count = self.dbsession.query(old_models.Collection).count() assert response.content_type == 'application/json' assert collection_count == 1 # Finally, update our collection (via request) as the default contributor. # Now we will have two collection backups. params = self.collection_create_params.copy() params.update({ 'title': 'Updated by the Contributor', 'speaker': speaker_id, 'tags': tag_ids, 'files': file_ids }) params = json.dumps(params) response = self.app.put(coll_url('update', id=collection_id), params, self.json_headers, contrib) resp = response.json_body collection_count = self.dbsession.query(old_models.Collection).count() assert collection_count == 1 # Now GET the collection backups as the restricted enterer of the original # collection and expect to get them all. response = self.app.get(url('index'), headers=self.json_headers, extra_environ=contrib) resp = response.json_body assert len(resp) == 2 assert response.content_type == 'application/json' # The admin should get them all too. response = self.app.get(url('index'), headers=self.json_headers, extra_environ=admin) resp = response.json_body assert len(resp) == 2 # The viewer should get none because they're all restricted. response = self.app.get(url('index'), headers=self.json_headers, extra_environ=view) resp = response.json_body assert len(resp) == 0 # Now update the collection and de-restrict it. params = self.collection_create_params.copy() params.update({ 'title': 'Updated and de-restricted by the Contributor', 'speaker': speaker_id, 'tags': [], 'files': file_ids }) params = json.dumps(params) response = self.app.put(coll_url('update', id=collection_id), params, self.json_headers, contrib) resp = response.json_body collection_count = self.dbsession.query(old_models.Collection).count() assert collection_count == 1 # Now GET the collection backups. Admin and contrib should see 3 but the # viewer should still see none. response = self.app.get(url('index'), headers=self.json_headers, extra_environ=contrib) resp = response.json_body assert len(resp) == 3 response = self.app.get(url('index'), headers=self.json_headers, extra_environ=admin) resp = response.json_body assert len(resp) == 3 response = self.app.get(url('index'), headers=self.json_headers, extra_environ=view) resp = response.json_body assert len(resp) == 0 assert response.content_type == 'application/json' # Finally, update our collection in some trivial way. params = self.collection_create_params.copy() params.update({ 'title': 'Updated by the Contributor *again*', 'speaker': speaker_id, 'tags': [], 'files': file_ids }) params = json.dumps(params) response = self.app.put(coll_url('update', id=collection_id), params, self.json_headers, contrib) resp = response.json_body collection_count = self.dbsession.query(old_models.Collection).count() assert collection_count == 1 # Now GET the collection backups. Admin and contrib should see 4 and the # viewer should see 1 response = self.app.get(url('index'), headers=self.json_headers, extra_environ=contrib) resp = response.json_body assert len(resp) == 4 response = self.app.get(url('index'), headers=self.json_headers, extra_environ=admin) resp = response.json_body all_collection_backups = resp assert len(resp) == 4 response = self.app.get(url('index'), headers=self.json_headers, extra_environ=view) resp = response.json_body unrestricted_collection_backup = resp[0] assert len(resp) == 1 assert resp[0][ 'title'] == 'Updated and de-restricted by the Contributor' restricted_collection_backups = [ cb for cb in all_collection_backups if cb != unrestricted_collection_backup ] assert len(restricted_collection_backups) == 3 # Test the paginator GET params. paginator = {'items_per_page': 1, 'page': 2} response = self.app.get(url('index'), paginator, headers=self.json_headers, extra_environ=admin) resp = response.json_body assert len(resp['items']) == 1 assert resp['items'][0]['title'] == all_collection_backups[1]['title'] assert response.content_type == 'application/json' # Test the order_by GET params. order_by_params = { 'order_by_model': 'CollectionBackup', 'order_by_attribute': 'id', 'order_by_direction': 'desc' } response = self.app.get(url('index'), order_by_params, headers=self.json_headers, extra_environ=admin) resp = response.json_body result_set = sorted(all_collection_backups, key=lambda cb: cb['id'], reverse=True) assert [cb['id'] for cb in resp] == [cb['id'] for cb in result_set] # Test the order_by *with* paginator. params = { 'order_by_model': 'CollectionBackup', 'order_by_attribute': 'id', 'order_by_direction': 'desc', 'items_per_page': 1, 'page': 3 } response = self.app.get(url('index'), params, headers=self.json_headers, extra_environ=admin) resp = response.json_body assert result_set[2]['title'] == resp['items'][0]['title'] # Now test the show action: # Admin should be able to GET a particular restricted collection backup response = self.app.get(url('show', id=restricted_collection_backups[0]['id']), headers=self.json_headers, extra_environ=admin) resp = response.json_body assert resp['title'] == restricted_collection_backups[0]['title'] assert response.content_type == 'application/json' # Viewer should receive a 403 error when attempting to do so. response = self.app.get(url('show', id=restricted_collection_backups[0]['id']), headers=self.json_headers, extra_environ=view, status=403) resp = response.json_body assert resp[ 'error'] == 'You are not authorized to access this resource.' assert response.content_type == 'application/json' # Viewer should be able to GET the unrestricted collection backup response = self.app.get(url('show', id=unrestricted_collection_backup['id']), headers=self.json_headers, extra_environ=view) resp = response.json_body assert resp['title'] == unrestricted_collection_backup['title'] # A nonexistent cb id will return a 404 error response = self.app.get(url('show', id=100987), headers=self.json_headers, extra_environ=view, status=404) resp = response.json_body assert resp['error'] == 'There is no collection backup with id 100987' assert response.content_type == 'application/json' # Test the search action add_SEARCH_to_web_test_valid_methods() # A search on collection backup titles using POST /collectionbackups/search json_query = json.dumps({ 'query': { 'filter': ['CollectionBackup', 'title', 'like', '%Contributor%'] } }) response = self.app.post(url('search_post'), json_query, self.json_headers, admin) resp = response.json_body result_set = [ cb for cb in all_collection_backups if 'Contributor' in cb['title'] ] assert len(resp) == len(result_set) == 3 assert set([cb['id'] for cb in resp]) == set([cb['id'] for cb in result_set]) assert response.content_type == 'application/json' # A search on collection backup titles using SEARCH /collectionbackups json_query = json.dumps({ 'query': { 'filter': ['CollectionBackup', 'title', 'like', '%Administrator%'] } }) response = self.app.request(url('search'), method='SEARCH', body=json_query.encode('utf8'), headers=self.json_headers, environ=admin) resp = response.json_body result_set = [ cb for cb in all_collection_backups if 'Administrator' in cb['title'] ] assert len(resp) == len(result_set) == 1 assert set([cb['id'] for cb in resp]) == set([cb['id'] for cb in result_set]) # Perform the two previous searches as a restricted viewer to show that # the restricted tag is working correctly. json_query = json.dumps({ 'query': { 'filter': ['CollectionBackup', 'title', 'like', '%Contributor%'] } }) response = self.app.post(url('search_post'), json_query, self.json_headers, view) resp = response.json_body result_set = [ cb for cb in [unrestricted_collection_backup] if 'Contributor' in cb['title'] ] assert len(resp) == len(result_set) == 1 assert set([cb['id'] for cb in resp]) == set([cb['id'] for cb in result_set]) json_query = json.dumps({ 'query': { 'filter': ['CollectionBackup', 'title', 'like', '%Administrator%'] } }) response = self.app.request(url('search'), method='SEARCH', body=json_query.encode('utf8'), headers=self.json_headers, environ=view) resp = response.json_body result_set = [ cb for cb in [unrestricted_collection_backup] if 'Administrator' in cb['title'] ] assert len(resp) == len(result_set) == 0 # I'm just going to assume that the order by and pagination functions are # working correctly since the implementation is essentially equivalent # to that in the index action already tested above. # Attempting to call edit/new/create/delete/update on a read-only # resource will return a 404 response response = self.app.get(url('edit', id=2232), status=404, extra_environ=admin) assert response.json_body['error'] == 'This resource is read-only.' response = self.app.get(url('new', id=2232), status=404, extra_environ=admin) assert response.json_body['error'] == 'This resource is read-only.' response = self.app.post(url('create'), status=404, extra_environ=admin) assert response.json_body['error'] == 'This resource is read-only.' response = self.app.put(url('update', id=2232), status=404, extra_environ=admin) assert response.json_body['error'] == 'This resource is read-only.' response = self.app.delete(url('delete', id=2232), status=404, extra_environ=admin) assert response.json_body['error'] == 'This resource is read-only.' assert response.content_type == 'application/json'
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) add_SEARCH_to_web_test_valid_methods()
def test_search(self): """Tests that SEARCH /formsearches (a.k.a. POST /formsearches/search) correctly returns an array of formsearches based on search criteria.""" dbsession = self.dbsession db = DBUtils(dbsession, self.settings) # Create some form_searches (and other models) to search and add SEARCH to the list of allowable methods _create_test_data(db, dbsession, 100) add_SEARCH_to_web_test_valid_methods() RDBMSName = h.get_RDBMS_name(self.settings) form_searches = json.loads( json.dumps([ self.fix_formsearch(fs.get_dict()) for fs in db.get_form_searches(True) ])) # Searching where values may be NULL json_query = json.dumps( {'query': { 'filter': ['FormSearch', 'search', 'like', '%2%'] }}) response = self.app.post(url('search_post'), json_query, self.json_headers, self.extra_environ_admin) resp = response.json_body result_set = [ fs for fs in form_searches if '2' in json.dumps(fs['search']) ] assert resp assert len(resp) == len(result_set) assert set([s['id'] for s in resp]) == set([s['id'] for s in result_set]) assert response.content_type == 'application/json' # A fairly complex search json_query = json.dumps({ 'query': { 'filter': [ 'and', [['FormSearch', 'name', 'regex', '[13456]'], ['not', ['FormSearch', 'name', 'like', '%F%']], [ 'or', [['FormSearch', 'search', 'regex', '[1456]'], [ 'FormSearch', 'datetime_modified', '>', yesterday_timestamp.isoformat() ]] ]] ] } }) response = self.app.post(url('search_post'), json_query, self.json_headers, self.extra_environ_admin) resp = response.json_body mysql_engine = old_models.Model.__table_args__.get('mysql_engine') if RDBMSName == 'mysql' and mysql_engine == 'InnoDB': _yesterday_timestamp = h.round_datetime(yesterday_timestamp) else: _yesterday_timestamp = yesterday_timestamp result_set = [ fs for fs in form_searches if re.search('[13456]', fs['name']) and not 'F' in fs['name'] and ( re.search('[1456]', json.dumps(fs['search'])) or fs['datetime_modified'] > _yesterday_timestamp.isoformat()) ] assert resp assert len(resp) == len(result_set) assert set([s['id'] for s in resp]) == set([s['id'] for s in result_set]) # A basic search with a paginator provided. json_query = json.dumps({ 'query': { 'filter': ['FormSearch', 'search', 'like', '%3%'] }, 'paginator': { 'page': 2, 'items_per_page': 5 } }) response = self.app.request(url('search'), method='SEARCH', body=json_query.encode('utf8'), headers=self.json_headers, environ=self.extra_environ_admin) resp = response.json_body result_set = [ fs for fs in form_searches if json.dumps(fs['search']) and '3' in json.dumps(fs['search']) ] assert resp['paginator']['count'] == len(result_set) assert len(resp['items']) == 5 assert resp['items'][0]['id'] == result_set[5]['id'] assert resp['items'][-1]['id'] == result_set[9]['id'] # An invalid paginator (here 'page' is less than 1) will result in formencode.Invalid # being raised resulting in a response with a 400 status code and a JSON error msg. json_query = json.dumps({ 'query': { 'filter': ['FormSearch', 'search', 'like', '%3%'] }, 'paginator': { 'page': 0, 'items_per_page': 10 } }) response = self.app.request(url('search'), method='SEARCH', body=json_query.encode('utf8'), headers=self.json_headers, environ=self.extra_environ_admin, status=400) resp = response.json_body assert resp['errors'][ 'page'] == 'Please enter a number that is 1 or greater' assert response.content_type == 'application/json' # Some "invalid" paginators will silently fail. For example, if there is # no 'pages' key, then SEARCH /formsearches will just assume there is no paginator # and all of the results will be returned. json_query = json.dumps({ 'query': { 'filter': ['FormSearch', 'search', 'like', '%3%'] }, 'paginator': { 'pages': 1, 'items_per_page': 10 } }) response = self.app.request(url('search'), method='SEARCH', body=json_query.encode('utf8'), headers=self.json_headers, environ=self.extra_environ_admin) resp = response.json_body assert len(resp) == len([ fs for fs in form_searches if json.dumps(fs['search']) and '3' in json.dumps(fs['search']) ]) # Adding a 'count' key to the paginator object in the request will spare # the server from running query.count(). Note that the server will not # attempt to verify the count (since that would defeat the purpose) but # will simply pass it back. The server trusts that the client is passing # in a factual count. Here we pass in an inaccurate count for demonstration. json_query = json.dumps({ 'query': { 'filter': ['FormSearch', 'search', 'like', '%3%'] }, 'paginator': { 'page': 2, 'items_per_page': 4, 'count': 750 } }) response = self.app.request(url('search'), method='SEARCH', body=json_query.encode('utf8'), headers=self.json_headers, environ=self.extra_environ_admin) resp = response.json_body assert resp['paginator']['count'] == 750 assert len(resp['items']) == 4 assert resp['items'][0]['id'] == result_set[4]['id'] assert resp['items'][-1]['id'] == result_set[7]['id'] # Test order by: order by name descending json_query = json.dumps({ 'query': { 'filter': ['FormSearch', 'search', 'regex', '.'], 'order_by': ['FormSearch', 'name', 'desc'] } }) response = self.app.post(url('search_post'), json_query, self.json_headers, self.extra_environ_admin) resp = response.json_body result_set = sorted(form_searches, key=lambda fs: fs['name'].lower(), reverse=True) assert len(resp) == 100 rs_names = [fs['name'] for fs in result_set] r_names = [fs['name'] for fs in resp] assert rs_names == r_names assert resp[0]['name'] == 'form search 99' assert resp[-1]['name'] == 'form search 1' # order by with missing direction defaults to 'asc' json_query = json.dumps({ 'query': { 'filter': ['FormSearch', 'search', 'regex', '.'], 'order_by': ['FormSearch', 'name'] } }) response = self.app.post(url('search_post'), json_query, self.json_headers, self.extra_environ_admin) resp = response.json_body assert len(resp) == 100 assert resp[0]['name'] == 'form search 1' assert resp[-1]['name'] == 'form search 99' assert response.content_type == 'application/json' # order by with unknown direction defaults to 'asc' json_query = json.dumps({ 'query': { 'filter': ['FormSearch', 'search', 'regex', '.'], 'order_by': ['FormSearch', 'name', 'descending'] } }) response = self.app.post(url('search_post'), json_query, self.json_headers, self.extra_environ_admin) resp = response.json_body assert len(resp) == 100 assert resp[0]['name'] == 'form search 1' assert resp[-1]['name'] == 'form search 99' # syntactically malformed order by json_query = json.dumps({ 'query': { 'filter': ['FormSearch', 'search', 'regex', '.'], 'order_by': ['FormSearch'] } }) response = self.app.post(url('search_post'), json_query, self.json_headers, self.extra_environ_admin, status=400) resp = response.json_body assert resp['errors'][ 'OrderByError'] == 'The provided order by expression was invalid.' assert response.content_type == 'application/json' # searches with lexically malformed order bys json_query = json.dumps({ 'query': { 'filter': ['FormSearch', 'name', 'regex', '.'], 'order_by': ['FormSearch', 'foo', 'desc'] } }) response = self.app.post(url('search_post'), json_query, self.json_headers, self.extra_environ_admin, status=400) resp = response.json_body assert resp['errors'][ 'FormSearch.foo'] == 'Searching on FormSearch.foo is not permitted' assert resp['errors'][ 'OrderByError'] == 'The provided order by expression was invalid.' json_query = json.dumps({ 'query': { 'filter': ['FormSearch', 'name', 'regex', '.'], 'order_by': ['Foo', 'id', 'desc'] } }) response = self.app.post(url('search_post'), json_query, self.json_headers, self.extra_environ_admin, status=400) resp = response.json_body assert resp['errors'][ 'Foo'] == 'Searching the FormSearch model by joining on the Foo model is not possible' assert resp['errors'][ 'Foo.id'] == 'Searching on Foo.id is not permitted' assert resp['errors'][ 'OrderByError'] == 'The provided order by expression was invalid.'
def test_index(self): """Tests that GET & SEARCH /corpusbackups behave correctly. """ dbsession = self.dbsession db = DBUtils(dbsession, self.settings) tag = old_models.Tag() tag.name = 'random tag name' dbsession.add(tag) dbsession.flush() tag_id = tag.id dbsession.commit() # Add 10 forms and use them to generate a valid value for ``test_corpus_content`` def create_form_from_index(index): form = old_models.Form() form.transcription = 'Form %d' % index translation = old_models.Translation() translation.transcription = 'Translation %d' % index form.translation = translation return form forms = [create_form_from_index(i) for i in range(1, 10)] dbsession.add_all(forms) dbsession.commit() forms = db.get_forms() half_forms = forms[:5] form_ids = [form.id for form in forms] half_form_ids = [form.id for form in half_forms] test_corpus_content = ','.join(map(str, form_ids)) test_corpus_half_content = ','.join(map(str, half_form_ids)) # Create a form search model query = {'filter': ['Form', 'transcription', 'regex', '[a-zA-Z]{3,}']} params = json.dumps({ 'name': 'form search', 'description': 'This one\'s worth saving!', 'search': query }) response = self.app.post(fs_url('create'), params, self.json_headers, self.extra_environ_admin) resp = response.json_body form_search_id = resp['id'] # Generate some valid corpus creation input parameters. params = self.corpus_create_params.copy() params.update({ 'name': 'Corpus', 'description': 'Covers a lot of the data.', 'content': test_corpus_content }) params = json.dumps(params) # Attempt to create a corpus as a viewer and expect to fail response = self.app.post(crps_url('create'), params, self.json_headers, self.extra_environ_view, status=403) resp = response.json_body assert resp[ 'error'] == 'You are not authorized to access this resource.' assert response.content_type == 'application/json' # Successfully create a corpus as the admin assert os.listdir(self.corpora_path) == [] original_corpus_count = dbsession.query(Corpus).count() response = self.app.post(crps_url('create'), params, self.json_headers, self.extra_environ_admin) resp = response.json_body corpus_id = resp['id'] new_corpus_count = dbsession.query(Corpus).count() corpus = dbsession.query(Corpus).get(corpus_id) corpus_form_ids = sorted([f.id for f in corpus.forms]) corpus_dir = os.path.join(self.corpora_path, 'corpus_%d' % corpus_id) corpus_dir_contents = os.listdir(corpus_dir) assert new_corpus_count == original_corpus_count + 1 assert resp['name'] == 'Corpus' assert resp['description'] == 'Covers a lot of the data.' assert corpus_dir_contents == [] assert response.content_type == 'application/json' assert resp['content'] == test_corpus_content assert corpus_form_ids == sorted(form_ids) # Update the corpus as the contributor -- now we should have one backup dbsession.expire(corpus) params = self.corpus_create_params.copy() params.update({ 'name': 'Corpus', 'description': 'Covers a little less data.', 'content': test_corpus_half_content }) params = json.dumps(params) response = self.app.put(crps_url('update', id=corpus_id), params, self.json_headers, self.extra_environ_contrib) resp = response.json_body corpus_count = new_corpus_count new_corpus_count = dbsession.query(Corpus).count() corpus = dbsession.query(Corpus).get(corpus_id) corpus_form_ids = sorted([f.id for f in corpus.forms]) assert new_corpus_count == corpus_count assert resp['name'] == 'Corpus' assert resp['description'] == 'Covers a little less data.' assert response.content_type == 'application/json' assert resp['content'] == test_corpus_half_content assert corpus_form_ids == sorted(half_form_ids) # Update the corpus again -- now we should have two backups sleep(1) params = self.corpus_create_params.copy() params.update({ 'name': 'Corpus', 'description': 'Covers a little less data.', 'content': test_corpus_half_content, 'tags': [tag_id] }) params = json.dumps(params) response = self.app.put(crps_url('update', id=corpus_id), params, self.json_headers, self.extra_environ_admin) resp = response.json_body corpus_count = new_corpus_count new_corpus_count = dbsession.query(Corpus).count() corpus = dbsession.query(Corpus).get(corpus_id) corpus_form_ids = sorted([f.id for f in corpus.forms]) assert new_corpus_count == corpus_count assert resp['name'] == 'Corpus' assert resp['description'] == 'Covers a little less data.' assert response.content_type == 'application/json' assert resp['content'] == test_corpus_half_content assert corpus_form_ids == sorted(half_form_ids) all_corpus_backups = dbsession.query(CorpusBackup).order_by( CorpusBackup.id).all() all_corpus_backup_ids = [cb.id for cb in all_corpus_backups] all_corpus_backup_descriptions = [ cb.description for cb in all_corpus_backups ] # Now request the corpus backups as either the contributor or the viewer and # expect to get them all. response = self.app.get(url('index'), headers=self.json_headers, extra_environ=self.extra_environ_contrib) resp = response.json_body assert len(resp) == 2 assert response.content_type == 'application/json' assert resp[0]['modifier']['role'] == 'administrator' assert resp[1]['modifier']['role'] == 'contributor' # The admin should get them all too. response = self.app.get(url('index'), headers=self.json_headers, extra_environ=self.extra_environ_view) resp = response.json_body assert len(resp) == 2 assert [cb['id'] for cb in resp] == all_corpus_backup_ids # Test the paginator GET params. paginator = {'items_per_page': 1, 'page': 2} response = self.app.get(url('index'), paginator, headers=self.json_headers, extra_environ=self.extra_environ_admin) resp = response.json_body assert len(resp['items']) == 1 assert resp['paginator']['count'] == 2 assert response.content_type == 'application/json' assert resp['items'][0]['id'] == all_corpus_backup_ids[1] # Test the order_by GET params. order_by_params = { 'order_by_model': 'CorpusBackup', 'order_by_attribute': 'id', 'order_by_direction': 'desc' } response = self.app.get(url('index'), order_by_params, headers=self.json_headers, extra_environ=self.extra_environ_admin) resp = response.json_body result_set = list(reversed(all_corpus_backup_ids)) assert [cb['id'] for cb in resp] == result_set # Test the order_by *with* paginator. params = { 'order_by_model': 'CorpusBackup', 'order_by_attribute': 'id', 'order_by_direction': 'desc', 'items_per_page': 1, 'page': 1 } response = self.app.get(url('index'), params, headers=self.json_headers, extra_environ=self.extra_environ_admin) resp = response.json_body assert result_set[0] == resp['items'][0]['id'] # Now test the show action: # Get a specific corpus backup. response = self.app.get(url('show', id=all_corpus_backup_ids[0]), headers=self.json_headers, extra_environ=self.extra_environ_admin) resp = response.json_body assert resp['description'] == 'Covers a lot of the data.' assert resp['content'] == test_corpus_content assert response.content_type == 'application/json' # A nonexistent cb id will return a 404 error response = self.app.get(url('show', id=100987), headers=self.json_headers, extra_environ=self.extra_environ_view, status=404) resp = response.json_body assert resp['error'] == 'There is no corpus backup with id 100987' assert response.content_type == 'application/json' # Test the search action add_SEARCH_to_web_test_valid_methods() # A search on corpus backup titles using POST /corpusbackups/search json_query = json.dumps({ 'query': { 'filter': ['CorpusBackup', 'description', 'like', '%less%'] } }) response = self.app.post(url('search_post'), json_query, self.json_headers, extra_environ=self.extra_environ_admin) resp = response.json_body result_set = [ name for name in all_corpus_backup_descriptions if 'less' in name ] assert len(resp) == len(result_set) == 1 assert resp[0]['description'] == result_set[0] assert response.content_type == 'application/json' # A search on corpus backup titles using SEARCH /corpusbackups json_query = json.dumps({ 'query': { 'filter': ['CorpusBackup', 'description', 'like', '%less%'] } }) response = self.app.request(url('search'), method='SEARCH', body=json_query.encode('utf8'), headers=self.json_headers, environ=self.extra_environ_admin) resp = response.json_body assert len(resp) == len(result_set) == 1 assert resp[0]['description'] == result_set[0] assert response.content_type == 'application/json' # Attempting to call edit/new/create/delete/update on a read-only resource # will return a 404 response response = self.app.get(url('edit', id=2232), status=404, extra_environ=self.extra_environ_admin) assert response.json_body['error'] == 'This resource is read-only.' response = self.app.get(url('new', id=2232), status=404, extra_environ=self.extra_environ_admin) assert response.json_body['error'] == 'This resource is read-only.' response = self.app.post(url('create'), status=404, extra_environ=self.extra_environ_admin) assert response.json_body['error'] == 'This resource is read-only.' response = self.app.put(url('update', id=2232), status=404, extra_environ=self.extra_environ_admin) assert response.json_body['error'] == 'This resource is read-only.' response = self.app.delete(url('delete', id=2232), status=404, extra_environ=self.extra_environ_admin) assert response.json_body['error'] == 'This resource is read-only.' assert response.content_type == 'application/json'
def test_index(self): """Tests that GET & SEARCH /languages behave correctly. NOTE: during testing, the language table contains only 8 records. """ dbsession = self.dbsession db = DBUtils(dbsession, self.settings) languages = dbsession.query(old_models.Language).all() # GET the languages response = self.app.get(url('index'), headers=self.json_headers, extra_environ=self.extra_environ_view) resp = response.json_body assert len(resp) == len(languages) assert response.content_type == 'application/json' # Test the paginator GET params. paginator = {'items_per_page': 2, 'page': 2} response = self.app.get(url('index'), paginator, headers=self.json_headers, extra_environ=self.extra_environ_view) resp = response.json_body assert len(resp['items']) == 2 assert resp['items'][0]['Part2B'] == languages[2].Part2B assert response.content_type == 'application/json' # Test the order_by GET params. order_by_params = {'order_by_model': 'Language', 'order_by_attribute': 'Ref_Name', 'order_by_direction': 'desc'} response = self.app.get(url('index'), order_by_params, headers=self.json_headers, extra_environ=self.extra_environ_view) resp = response.json_body result_set = sorted(languages, key=lambda l: l.Ref_Name, reverse=True) assert [l['Id'] for l in resp] == [l.Id for l in result_set] # Test the order_by *with* paginator. params = {'order_by_model': 'Language', 'order_by_attribute': 'Ref_Name', 'order_by_direction': 'desc', 'items_per_page': 1, 'page': 3} response = self.app.get(url('index'), params, headers=self.json_headers, extra_environ=self.extra_environ_admin) resp = response.json_body assert result_set[2].Ref_Name == resp['items'][0]['Ref_Name'] assert response.content_type == 'application/json' # Now test the show action: response = self.app.get(url('show', id=languages[4].Id), headers=self.json_headers, extra_environ=self.extra_environ_view) resp = response.json_body assert resp['Ref_Name'] == languages[4].Ref_Name assert response.content_type == 'application/json' # A nonexistent language Id will return a 404 error response = self.app.get(url('show', id=100987), headers=self.json_headers, extra_environ=self.extra_environ_view, status=404) resp = response.json_body assert resp['error'] == 'There is no language with Id 100987' assert response.content_type == 'application/json' # Test the search action add_SEARCH_to_web_test_valid_methods() # A search on language transcriptions using POST /languages/search json_query = json.dumps({'query': {'filter': ['Language', 'Ref_Name', 'like', '%m%']}}) response = self.app.post(url('search_post'), json_query, self.json_headers, self.extra_environ_view) resp = response.json_body result_set = [l for l in languages if 'm' in l.Ref_Name] assert resp assert set([l['Id'] for l in resp]) == set([l.Id for l in result_set]) assert response.content_type == 'application/json' # A search on language Ref_Name using SEARCH /languages json_query = json.dumps({'query': {'filter': ['Language', 'Ref_Name', 'like', '%l%']}}) response = self.app.request(url('search'), method='SEARCH', body=json_query.encode('utf8'), headers=self.json_headers, environ=self.extra_environ_view) resp = response.json_body result_set = [l for l in languages if 'l' in l.Ref_Name] assert resp assert len(resp) == len(result_set) assert set([l['Id'] for l in resp]) == set([l.Id for l in result_set]) # I'm just going to assume that the order by and pagination functions are # working correctly since the implementation is essentially equivalent # to that in the index action already tested above. # Attempting to call edit/new/create/delete/update on a read-only resource # will return a 404 response response = self.app.get( url('edit', id=2232), extra_environ=self.extra_environ_contrib, status=404) assert response.json_body['error'] == 'This resource is read-only.' assert response.content_type == 'application/json' response = self.app.get( url('new', id=2232), extra_environ=self.extra_environ_contrib, status=404) assert response.json_body['error'] == 'This resource is read-only.' assert response.content_type == 'application/json' response = self.app.post( url('create'), extra_environ=self.extra_environ_contrib, status=404) assert response.json_body['error'] == 'This resource is read-only.' assert response.content_type == 'application/json' response = self.app.put( url('update', id=2232), extra_environ=self.extra_environ_contrib, status=404) assert response.json_body['error'] == 'This resource is read-only.' assert response.content_type == 'application/json' response = self.app.delete( url('delete', id=2232), extra_environ=self.extra_environ_contrib, status=404) assert response.json_body['error'] == 'This resource is read-only.' assert response.content_type == 'application/json'