def test_index(self):
"""Tests that GET /formsearches returns an array of all form searches and that order_by and pagination parameters work correctly."""
dbsession = self.dbsession
db = DBUtils(dbsession, self.settings)
# Add 100 form searches.
def create_form_search_from_index(index):
form_search = old_models.FormSearch()
form_search.name = 'form_search%d' % index
form_search.description = 'description %d' % index
form_search.search = str(
json.dumps({
'query': {
'filter':
['Form', 'transcription', 'regex',
'%d' % index]
}
}))
return form_search
form_searches = [
create_form_search_from_index(i) for i in range(1, 101)
]
dbsession.add_all(form_searches)
dbsession.commit()
form_searches = db.get_form_searches(True)
form_searches_count = len(form_searches)
# Test that GET /formsearches gives us all of the form searches.
response = self.app.get(url('index'),
headers=self.json_headers,
extra_environ=self.extra_environ_view)
resp = response.json_body
assert len(resp) == form_searches_count
assert resp[0]['name'] == 'form_search1'
assert resp[0]['id'] == form_searches[0].id
assert response.content_type == 'application/json'
# Test the paginator GET params.
paginator = {'items_per_page': 23, 'page': 3}
response = self.app.get(url('index'),
paginator,
headers=self.json_headers,
extra_environ=self.extra_environ_view)
resp = response.json_body
assert len(resp['items']) == 23
assert resp['items'][0]['name'] == form_searches[46].name
# Test the order_by GET params.
order_by_params = {
'order_by_model': 'FormSearch',
'order_by_attribute': 'name',
'order_by_direction': 'desc'
}
response = self.app.get(url('index'),
order_by_params,
headers=self.json_headers,
extra_environ=self.extra_environ_view)
resp = response.json_body
result_set = sorted([t.name for t in form_searches], reverse=True)
assert result_set == [t['name'] for t in resp]
# Test the order_by *with* paginator.
params = {
'order_by_model': 'FormSearch',
'order_by_attribute': 'name',
'order_by_direction': 'desc',
'items_per_page': 23,
'page': 3
}
response = self.app.get(url('index'),
params,
headers=self.json_headers,
extra_environ=self.extra_environ_view)
resp = response.json_body
assert result_set[46] == resp['items'][0]['name']
# Expect a 400 error when the order_by_direction param is invalid
order_by_params = {
'order_by_model': 'FormSearch',
'order_by_attribute': 'name',
'order_by_direction': 'descending'
}
response = self.app.get(url('index'),
order_by_params,
status=400,
headers=self.json_headers,
extra_environ=self.extra_environ_view)
resp = response.json_body
assert resp['errors'][
'order_by_direction'] == "Value must be one of: asc; desc (not 'descending')"
assert response.content_type == 'application/json'
# Expect the default BY id ASCENDING ordering when the order_by_model/Attribute
# param is invalid.
order_by_params = {
'order_by_model': 'FormSearchist',
'order_by_attribute': 'nominal',
'order_by_direction': 'desc'
}
response = self.app.get(url('index'),
order_by_params,
headers=self.json_headers,
extra_environ=self.extra_environ_view)
resp = response.json_body
assert resp[0]['id'] == form_searches[0].id
# Expect a 400 error when the paginator GET params are empty
# or are integers less than 1
paginator = {'items_per_page': 'a', 'page': ''}
response = self.app.get(url('index'),
paginator,
headers=self.json_headers,
extra_environ=self.extra_environ_view,
status=400)
resp = response.json_body
assert resp['errors'][
'items_per_page'] == 'Please enter an integer value'
assert resp['errors']['page'] == 'Please enter a value'
assert response.content_type == 'application/json'
paginator = {'items_per_page': 0, 'page': -1}
response = self.app.get(url('index'),
paginator,
headers=self.json_headers,
extra_environ=self.extra_environ_view,
status=400)
resp = response.json_body
assert resp['errors'][
'items_per_page'] == 'Please enter a number that is 1 or greater'
assert resp['errors'][
'page'] == 'Please enter a number that is 1 or greater'
def test_search(self):
"""Tests that SEARCH /formsearches (a.k.a. POST /formsearches/search) correctly returns an array of formsearches based on search criteria."""
dbsession = self.dbsession
db = DBUtils(dbsession, self.settings)
# Create some form_searches (and other models) to search and add SEARCH to the list of allowable methods
_create_test_data(db, dbsession, 100)
add_SEARCH_to_web_test_valid_methods()
RDBMSName = h.get_RDBMS_name(self.settings)
form_searches = json.loads(
json.dumps([
self.fix_formsearch(fs.get_dict())
for fs in db.get_form_searches(True)
]))
# Searching where values may be NULL
json_query = json.dumps(
{'query': {
'filter': ['FormSearch', 'search', 'like', '%2%']
}})
response = self.app.post(url('search_post'), json_query,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
result_set = [
fs for fs in form_searches if '2' in json.dumps(fs['search'])
]
assert resp
assert len(resp) == len(result_set)
assert set([s['id']
for s in resp]) == set([s['id'] for s in result_set])
assert response.content_type == 'application/json'
# A fairly complex search
json_query = json.dumps({
'query': {
'filter': [
'and',
[['FormSearch', 'name', 'regex', '[13456]'],
['not', ['FormSearch', 'name', 'like', '%F%']],
[
'or',
[['FormSearch', 'search', 'regex', '[1456]'],
[
'FormSearch', 'datetime_modified', '>',
yesterday_timestamp.isoformat()
]]
]]
]
}
})
response = self.app.post(url('search_post'), json_query,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
mysql_engine = old_models.Model.__table_args__.get('mysql_engine')
if RDBMSName == 'mysql' and mysql_engine == 'InnoDB':
_yesterday_timestamp = h.round_datetime(yesterday_timestamp)
else:
_yesterday_timestamp = yesterday_timestamp
result_set = [
fs for fs in form_searches
if re.search('[13456]', fs['name']) and not 'F' in fs['name'] and (
re.search('[1456]', json.dumps(fs['search']))
or fs['datetime_modified'] > _yesterday_timestamp.isoformat())
]
assert resp
assert len(resp) == len(result_set)
assert set([s['id']
for s in resp]) == set([s['id'] for s in result_set])
# A basic search with a paginator provided.
json_query = json.dumps({
'query': {
'filter': ['FormSearch', 'search', 'like', '%3%']
},
'paginator': {
'page': 2,
'items_per_page': 5
}
})
response = self.app.request(url('search'),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin)
resp = response.json_body
result_set = [
fs for fs in form_searches
if json.dumps(fs['search']) and '3' in json.dumps(fs['search'])
]
assert resp['paginator']['count'] == len(result_set)
assert len(resp['items']) == 5
assert resp['items'][0]['id'] == result_set[5]['id']
assert resp['items'][-1]['id'] == result_set[9]['id']
# An invalid paginator (here 'page' is less than 1) will result in formencode.Invalid
# being raised resulting in a response with a 400 status code and a JSON error msg.
json_query = json.dumps({
'query': {
'filter': ['FormSearch', 'search', 'like', '%3%']
},
'paginator': {
'page': 0,
'items_per_page': 10
}
})
response = self.app.request(url('search'),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin,
status=400)
resp = response.json_body
assert resp['errors'][
'page'] == 'Please enter a number that is 1 or greater'
assert response.content_type == 'application/json'
# Some "invalid" paginators will silently fail. For example, if there is
# no 'pages' key, then SEARCH /formsearches will just assume there is no paginator
# and all of the results will be returned.
json_query = json.dumps({
'query': {
'filter': ['FormSearch', 'search', 'like', '%3%']
},
'paginator': {
'pages': 1,
'items_per_page': 10
}
})
response = self.app.request(url('search'),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin)
resp = response.json_body
assert len(resp) == len([
fs for fs in form_searches
if json.dumps(fs['search']) and '3' in json.dumps(fs['search'])
])
# Adding a 'count' key to the paginator object in the request will spare
# the server from running query.count(). Note that the server will not
# attempt to verify the count (since that would defeat the purpose) but
# will simply pass it back. The server trusts that the client is passing
# in a factual count. Here we pass in an inaccurate count for demonstration.
json_query = json.dumps({
'query': {
'filter': ['FormSearch', 'search', 'like', '%3%']
},
'paginator': {
'page': 2,
'items_per_page': 4,
'count': 750
}
})
response = self.app.request(url('search'),
method='SEARCH',
body=json_query.encode('utf8'),
headers=self.json_headers,
environ=self.extra_environ_admin)
resp = response.json_body
assert resp['paginator']['count'] == 750
assert len(resp['items']) == 4
assert resp['items'][0]['id'] == result_set[4]['id']
assert resp['items'][-1]['id'] == result_set[7]['id']
# Test order by: order by name descending
json_query = json.dumps({
'query': {
'filter': ['FormSearch', 'search', 'regex', '.'],
'order_by': ['FormSearch', 'name', 'desc']
}
})
response = self.app.post(url('search_post'), json_query,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
result_set = sorted(form_searches,
key=lambda fs: fs['name'].lower(),
reverse=True)
assert len(resp) == 100
rs_names = [fs['name'] for fs in result_set]
r_names = [fs['name'] for fs in resp]
assert rs_names == r_names
assert resp[0]['name'] == 'form search 99'
assert resp[-1]['name'] == 'form search 1'
# order by with missing direction defaults to 'asc'
json_query = json.dumps({
'query': {
'filter': ['FormSearch', 'search', 'regex', '.'],
'order_by': ['FormSearch', 'name']
}
})
response = self.app.post(url('search_post'), json_query,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
assert len(resp) == 100
assert resp[0]['name'] == 'form search 1'
assert resp[-1]['name'] == 'form search 99'
assert response.content_type == 'application/json'
# order by with unknown direction defaults to 'asc'
json_query = json.dumps({
'query': {
'filter': ['FormSearch', 'search', 'regex', '.'],
'order_by': ['FormSearch', 'name', 'descending']
}
})
response = self.app.post(url('search_post'), json_query,
self.json_headers, self.extra_environ_admin)
resp = response.json_body
assert len(resp) == 100
assert resp[0]['name'] == 'form search 1'
assert resp[-1]['name'] == 'form search 99'
# syntactically malformed order by
json_query = json.dumps({
'query': {
'filter': ['FormSearch', 'search', 'regex', '.'],
'order_by': ['FormSearch']
}
})
response = self.app.post(url('search_post'),
json_query,
self.json_headers,
self.extra_environ_admin,
status=400)
resp = response.json_body
assert resp['errors'][
'OrderByError'] == 'The provided order by expression was invalid.'
assert response.content_type == 'application/json'
# searches with lexically malformed order bys
json_query = json.dumps({
'query': {
'filter': ['FormSearch', 'name', 'regex', '.'],
'order_by': ['FormSearch', 'foo', 'desc']
}
})
response = self.app.post(url('search_post'),
json_query,
self.json_headers,
self.extra_environ_admin,
status=400)
resp = response.json_body
assert resp['errors'][
'FormSearch.foo'] == 'Searching on FormSearch.foo is not permitted'
assert resp['errors'][
'OrderByError'] == 'The provided order by expression was invalid.'
json_query = json.dumps({
'query': {
'filter': ['FormSearch', 'name', 'regex', '.'],
'order_by': ['Foo', 'id', 'desc']
}
})
response = self.app.post(url('search_post'),
json_query,
self.json_headers,
self.extra_environ_admin,
status=400)
resp = response.json_body
assert resp['errors'][
'Foo'] == 'Searching the FormSearch model by joining on the Foo model is not possible'
assert resp['errors'][
'Foo.id'] == 'Searching on Foo.id is not permitted'
assert resp['errors'][
'OrderByError'] == 'The provided order by expression was invalid.'