def test_order(self):
def unique(seq):
result = []
for item in seq:
if item not in result:
result.append(item)
return result
response = self.app.get(
url(controller='api2',
action='aggregate',
dataset='cra',
order='year:asc',
drilldown='year'))
h.assert_equal(response.status, '200 OK')
result = json.loads(response.body)
order = [cell['year'] for cell in result['drilldown']]
h.assert_equal(
unique(order),
map(unicode, [2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010]))
response = self.app.get(
url(controller='api2',
action='aggregate',
dataset='cra',
order='year:desc',
drilldown='year'))
h.assert_equal(response.status, '200 OK')
result = json.loads(response.body)
order = [cell['year'] for cell in result['drilldown']]
h.assert_equal(
unique(order),
map(unicode, [2010, 2009, 2008, 2007, 2006, 2005, 2004, 2003]))
def test_aggregate_order(self):
def unique(seq):
result = []
for item in seq:
if item not in result:
result.append(item)
return result
response = self.app.get(url(controller='api/version2',
action='aggregate',
dataset='cra', order='year:asc',
drilldown='year'))
assert response.status == '200 OK'
result = json.loads(response.body)
order = [cell['year'] for cell in result['drilldown']]
expected_result = map(unicode, [2003, 2004, 2005, 2006, 2007,
2008, 2009, 2010])
assert unique(order) == expected_result
response = self.app.get(url(controller='api/version2',
action='aggregate',
dataset='cra', order='year:desc',
drilldown='year'))
assert response.status == '200 OK'
result = json.loads(response.body)
order = [cell['year'] for cell in result['drilldown']]
expected_result = map(unicode, [2010, 2009, 2008, 2007, 2006,
2005, 2004, 2003])
assert unique(order) == expected_result
def test_order(self):
def unique(seq):
result = []
for item in seq:
if item not in result:
result.append(item)
return result
response = self.app.get(url(controller='api2', action='aggregate',
dataset='cra', order='year:asc',
drilldown='year'))
h.assert_equal(response.status, '200 OK')
result = json.loads(response.body)
order = [cell['year'] for cell in result['drilldown']]
h.assert_equal(unique(order),
map(unicode, [2003, 2004, 2005, 2006, 2007, 2008, 2009,
2010]))
response = self.app.get(url(controller='api2', action='aggregate',
dataset='cra', order='year:desc',
drilldown='year'))
h.assert_equal(response.status, '200 OK')
result = json.loads(response.body)
order = [cell['year'] for cell in result['drilldown']]
h.assert_equal(unique(order),
map(unicode, [2010, 2009, 2008, 2007, 2006, 2005, 2004,
2003]))
def test_inflation(self):
"""
Test for inflation support in the aggregation api. Inflation works
by adding a url parameter containing the target year of inflation.
This test has hard coded values based on existing inflation data used
by an external module. This may therefore need updating should the
inflation data become more accurate with better data.
"""
response = self.app.get(
url(controller='api/version2',
action='aggregate',
dataset='cra',
cut='year:2009',
inflate='2011'))
assert '200' in response.status, \
"Inflation request didn't return successfully (status isn't 200)"
result = json.loads(response.body)
# Check for inflated amount
assert 'amount' in result['summary'], \
"Amount is absent for the result summary"
assert int(result['summary']['amount']) == 61836609, \
"Inflation amount is incorrect"
# Check for original amount
assert 'original' in result['summary'], \
"Original amount not in inflation request"
assert result['summary']['original'] == 57300000.0, \
"Original amount (for inflation) is incorrect"
# Check for inflation adjustment object in drilldown results
assert len(result['drilldown']) == 1, \
"Drilldown results were not of length 1"
assert 'inflation_adjustment' in result['drilldown'][0], \
"Inflation adjustment is not present in drilldown results"
# Check for what happens when inflation is not possible
response = self.app.get(
url(controller='api/version2',
action='aggregate',
dataset='cra',
cut='year:2009',
inflate='1000'))
assert '200' in response.status, \
"Incorrect inflation did not return sucessfully (status isn't 200)"
result = json.loads(response.body)
assert 'warning' in result, \
"No warning given when inflation not possible"
assert result['summary']['amount'] == 57300000.0, \
"Amount does not fall back to the original amount"
def test_inflation(self):
"""
Test for inflation support in the aggregation api. Inflation works
by adding a url parameter containing the target year of inflation.
This test has hard coded values based on existing inflation data used
by an external module. This may therefore need updating should the
inflation data become more accurate with better data.
"""
response = self.app.get(url(controller='api/version2',
action='aggregate',
dataset='cra', cut='year:2009',
inflate='2011'))
assert '200' in response.status, \
"Inflation request didn't return successfully (status isn't 200)"
result = json.loads(response.body)
# Check for inflated amount
assert 'amount' in result['summary'], \
"Amount is absent for the result summary"
assert int(result['summary']['amount']) == 61836609, \
"Inflation amount is incorrect"
# Check for original amount
assert 'original' in result['summary'], \
"Original amount not in inflation request"
assert result['summary']['original'] == 57300000.0, \
"Original amount (for inflation) is incorrect"
# Check for inflation adjustment object in drilldown results
assert len(result['drilldown']) == 1, \
"Drilldown results were not of length 1"
assert 'inflation_adjustment' in result['drilldown'][0], \
"Inflation adjustment is not present in drilldown results"
# Check for what happens when inflation is not possible
response = self.app.get(url(controller='api/version2',
action='aggregate',
dataset='cra', cut='year:2009',
inflate='1000'))
assert '200' in response.status, \
"Incorrect inflation did not return sucessfully (status isn't 200)"
result = json.loads(response.body)
assert 'warning' in result, \
"No warning given when inflation not possible"
assert result['summary']['amount'] == 57300000.0, \
"Amount does not fall back to the original amount"
def test_search_order(self):
response = self.app.get(url(controller='api/version2', action='search',
order="amount:asc"))
result = json.loads(response.body)
amounts = [r['amount'] for r in result['results']]
assert amounts == sorted(amounts)
response = self.app.get(url(controller='api/version2', action='search',
order="amount:desc"))
result = json.loads(response.body)
amounts = [r['amount'] for r in result['results']]
assert amounts == sorted(amounts)[::-1]
def test_search_02_query(self):
response = self.app.get(url(controller='api', action='search',
q='Children'))
out = json.loads(str(response.body))['response']
assert out['numFound'] == 7, out['numFound']
exp_entity = 'Department for Children, Schools and Families'
assert out['docs'][0]['from.label'] == exp_entity, out['docs'][0]
def test_aggregate_drilldown(self):
response = self.app.get(url(controller='api2', action='aggregate',
dataset='cra', drilldown='cofog1|cofog2'))
h.assert_equal(response.status, '200 OK')
result = json.loads(response.body)
h.assert_equal(result['summary']['num_drilldowns'], 6)
h.assert_equal(result['summary']['amount'], -371500000.0)
def update(self, dataset, name):
"""
Update dataset. Does nothing at the moment.
"""
# Get the dataset for the view
self._get_dataset(dataset)
# Get the named view
view = View.by_name(c.dataset, name)
# User must be allowed to update the named view
require.view.update(c.dataset, view)
# Possible update values
# We don't update the view's name because it might have been embedded
view.label = request.params.get('label', view.label)
try:
# Try to load the state
view.state = json.loads(request.params['state'])
except:
pass
view.description = request.params.get('description', view.description)
# Commit the changes
db.session.commit()
# Redirect to the view page for this view
redirect(h.url_for(controller='view', action='view',
dataset=c.dataset.name, name=view.name))
def test_aggregate_multiple_measures(self):
"""
Test whether multiple measures work. Multiple measures are
separated with | in the measure url parameter.
"""
# Get the aggregated amount and total values for year 2009
response = self.app.get(url(controller='api/version2',
action='aggregate',
dataset='cra', cut='year:2009',
measure='amount|total'))
# This should return a status code 200.
assert '200' in response.status, \
'Aggregation for multiple measures did not return successfully'
# Load the json body into a dict
result = json.loads(response.body)
# Only one drilldown should be made even if there are two measures
assert result['summary']['num_drilldowns'] == 1, \
"Aggregation of multiple measures wasn't done with one drilldown"
# Since amount measure and total measure are two different measures
# for the same field in the csv file they should contain the same
# amount but still be distinct.
assert result['summary']['total'] == 57300000.0, \
'Multiple measure aggregation of total measure is not correct'
assert result['summary']['amount'] == 57300000.0, \
'Multiple measure aggregation of amount measure is not correct'
def test_aggregate_cut(self):
response = self.app.get(url(controller='api2', action='aggregate',
dataset='cra', cut='year:2009'))
h.assert_equal(response.status, '200 OK')
result = json.loads(response.body)
h.assert_equal(result['summary']['num_drilldowns'], 1)
h.assert_equal(result['summary']['amount'], 57300000.0)
def test_search_q(self):
response = self.app.get(url(controller='api2', action='search', q="Ministry of Justice"))
result = json.loads(response.body)
h.assert_equal(result['stats']['results_count'], 5)
h.assert_equal(result['stats']['results_count_query'], 5)
h.assert_equal(result['results'][0]['id'], "06dafa7250420ab1dc616d2bbbe310c9ad6e485e")
def test_search_filter(self):
response = self.app.get(url(controller='api2', action='search', filter="pog:P13 S091105"))
result = json.loads(response.body)
h.assert_equal(result['stats']['results_count'], 5)
h.assert_equal(result['stats']['results_count_query'], 5)
h.assert_equal(result['results'][0]['id'], "06dafa7250420ab1dc616d2bbbe310c9ad6e485e")
def update(self, dataset, name):
"""
Update dataset. Does nothing at the moment.
"""
# Get the dataset for the view
self._get_dataset(dataset)
# Get the named view
view = View.by_name(c.dataset, name)
# User must be allowed to update the named view
require.view.update(c.dataset, view)
# Possible update values
# We don't update the view's name because it might have been embedded
view.label = request.params.get('label', view.label)
try:
# Try to load the state
view.state = json.loads(request.params['state'])
except:
pass
view.description = request.params.get('description', view.description)
# Commit the changes
db.session.commit()
# Redirect to the view page for this view
redirect(h.url_for(controller='view', action='view',
dataset=c.dataset.name, name=view.name))
def test_search_page_pagesize(self):
response = self.app.get(url(controller='api/version2', action='search',
page=2, pagesize=10))
result = json.loads(response.body)
assert result['stats']['results_count'] == 10
assert result['stats']['results_count_query'] == 36
def test_search_02_query(self):
response = self.app.get(
url(controller='api', action='search', q='Children'))
out = json.loads(str(response.body))['response']
assert out['numFound'] == 7, out['numFound']
exp_entity = 'Department for Children, Schools and Families'
assert out['docs'][0]['from.label'] == exp_entity, out['docs'][0]
def test_search_facet(self):
response = self.app.get(url(controller='api/version2', action='search',
pagesize=0, facet_field="dataset"))
result = json.loads(response.body)
assert len(result['facets']['dataset']) == 1
assert result['facets']['dataset'][0] == ['cra', 36]
def test_aggregate_cut(self):
response = self.app.get(url(controller='api/version2',
action='aggregate',
dataset='cra', cut='year:2009'))
assert response.status == '200 OK'
result = json.loads(response.body)
assert result['summary']['num_drilldowns'] == 1
assert result['summary']['amount'] == 57300000.0
def test_search(self):
response = self.app.get(url(controller='api2', action='search'))
result = json.loads(response.body)
h.assert_equal(result['stats']['results_count'], 36)
h.assert_equal(result['stats']['results_count_query'], 36)
h.assert_equal(result['facets'], {})
h.assert_equal(len(result['results']), 36)
def test_search_results_dataset(self):
response = self.app.get(
url(controller='api/version2', action='search'))
result = json.loads(response.body)
assert result['results'][0]['dataset']['name'] == 'cra'
expected_label = 'Country Regional Analysis v2009'
assert result['results'][0]['dataset']['label'] == expected_label
def test_search(self):
response = self.app.get(
url(controller='api/version2', action='search'))
result = json.loads(response.body)
assert result['stats']['results_count'] == 36
assert result['stats']['results_count_query'] == 36
assert result['facets'] == {}
assert len(result['results']) == 36
def test_aggregate_drilldown(self):
response = self.app.get(url(controller='api/version2',
action='aggregate',
dataset='cra', drilldown='cofog1|cofog2'))
assert response.status == '200 OK'
result = json.loads(response.body)
assert result['summary']['num_drilldowns'] == 6
assert result['summary']['amount'] == -371500000.0
def test_search_expand_facet_dimensions_no_dataset(self):
response = self.app.get(url(controller='api2',
action='search',
pagesize=0,
facet_field="from",
expand_facet_dimensions="1"))
result = json.loads(response.body)
# facets should *NOT* be expanded unless exactly 1 dataset was specified
h.assert_equal(result['facets']['from'][0][0], '999')
def test_search_filter(self):
response = self.app.get(url(controller='api/version2', action='search',
filter="pog:P13 S091105"))
result = json.loads(response.body)
assert result['stats']['results_count'] == 5
assert result['stats']['results_count_query'] == 5
id_value = '06dafa7250420ab1dc616d2bbbe310c9ad6e485e'
assert result['results'][0]['id'] == id_value
def embed(self, dataset):
self._get_dataset(dataset)
c.widget = request.params.get('widget')
if c.widget is None:
abort(400, _("No widget type has been specified."))
try:
c.widget = widgets.get_widget(c.widget)
c.state = json.loads(request.params.get('state', '{}'))
except ValueError as ve:
abort(400, unicode(ve))
return templating.render('view/embed.html')
def embed(self, dataset):
self._get_dataset(dataset)
c.widget = request.params.get('widget')
if c.widget is None:
abort(400, _("No widget type has been specified."))
try:
c.widget = widgets.get_widget(c.widget)
c.state = json.loads(request.params.get('state', '{}'))
except ValueError as ve:
abort(400, unicode(ve))
return render('view/embed.html')
def test_aggregate(self):
response = self.app.get(
url(controller='api2', action='aggregate', dataset='cra'))
h.assert_equal(response.status, '200 OK')
h.assert_equal(response.content_type, 'application/json')
result = json.loads(response.body)
h.assert_equal(sorted(result.keys()), [u'drilldown', u'summary'])
h.assert_equal(sorted(result['summary'].items()),
[(u'amount', -371500000.0), (u'num_drilldowns', 1),
(u'num_entries', 36), (u'page', 1), (u'pages', 1),
(u'pagesize', 10000)])
def test_search_expand_facet_dimensions(self):
response = self.app.get(url(controller='api2',
action='search',
dataset='cra',
pagesize=0,
facet_field="from|to.name",
expand_facet_dimensions="1"))
result = json.loads(response.body)
hra = {"taxonomy": "from", "description": "", "id": 5, "name": "999", "label": "ENG_HRA"}
h.assert_equal(result['facets']['from'][0][0], hra)
h.assert_equal(result['facets']['to.name'][0][0], 'society')
def test_view_by_taxonomy_name_json(self):
classifier = self.db['classifier'].find_one({'taxonomy': 'cofog',
'name': '03'})
url_ = classifier_url(classifier, format='json')
result = self.app.get(url_)
h.assert_equal(result.status, '200 OK')
h.assert_equal(result.content_type, 'application/json')
json_data = json.loads(result.body)
h.assert_equal(json_data['name'], u'03')
h.assert_equal(json_data['label'], classifier['label'])
h.assert_equal(json_data['_id'], str(classifier['_id']))
def test_view_entries_json(self):
classifier = self.db['classifier'].find_one({'taxonomy': 'cofog',
'name': '03'})
url_ = url(controller='classifier', action='entries', format='json',
taxonomy=classifier['taxonomy'],
name=classifier['name'])
result = self.app.get(url_)
h.assert_equal(result.status, '200 OK')
h.assert_equal(result.content_type, 'application/json')
json_data = json.loads(result.body)
h.assert_equal(len(json_data['results']), 5)
def test_aggregate(self):
response = self.app.get(
url(controller='api/version2', action='aggregate', dataset='cra'))
assert response.status == '200 OK'
assert response.content_type == 'application/json'
result = json.loads(response.body)
assert sorted(result.keys()) == [u'drilldown', u'summary']
expected_result = [(u'amount', -371500000.0),
(u'currency', {
u'amount': u'GBP'
}), (u'num_drilldowns', 1), (u'num_entries', 36),
(u'page', 1), (u'pages', 1), (u'pagesize', 10000)]
assert sorted(result['summary'].items()) == expected_result
def test_aggregate(self):
response = self.app.get(url(controller='api2', action='aggregate',
dataset='cra'))
h.assert_equal(response.status, '200 OK')
h.assert_equal(response.content_type, 'application/json')
result = json.loads(response.body)
h.assert_equal(sorted(result.keys()), [u'drilldown', u'summary'])
h.assert_equal(sorted(result['summary'].items()),
[(u'amount', -371500000.0),
(u'num_drilldowns', 1),
(u'num_entries', 36),
(u'page', 1),
(u'pages', 1),
(u'pagesize', 10000)])
def _write_json(self, resource, data, account):
try:
data = json.loads(data, object_hook=json_util.object_hook)
except ValueError as e:
abort(400, str(e))
if str(data["_id"]) != str(resource.id):
abort(400, _("Cannot change _id attribute!"))
resource.clear() # PUT is NOT an incremental update
resource.update(data)
save_kwargs = {}
resource.save(**save_kwargs)
return resource
def test_aggregate(self):
response = self.app.get(url(controller='api/version2',
action='aggregate',
dataset='cra'))
assert response.status == '200 OK'
assert response.content_type == 'application/json'
result = json.loads(response.body)
assert sorted(result.keys()) == [u'drilldown', u'summary']
expected_result = [(u'amount', -371500000.0),
(u'currency', {u'amount': u'GBP'}),
(u'num_drilldowns', 1),
(u'num_entries', 36),
(u'page', 1),
(u'pages', 1),
(u'pagesize', 10000)]
assert sorted(result['summary'].items()) == expected_result
def _write_json(self, resource, data, account):
try:
data = json.loads(data, object_hook=json_util.object_hook)
except ValueError as e:
abort(400, str(e))
if str(data["_id"]) != str(resource.id):
abort(400, _("Cannot change _id attribute!"))
resource.clear() # PUT is NOT an incremental update
resource.update(data)
save_kwargs = {}
if getattr(resource, "is_revisioned", False):
cs = Changeset(author=account.email)
save_kwargs["changeset"] = cs
resource.save(**save_kwargs)
return resource
def _query(self, **kwargs):
kwargs.update({'wt': 'json'})
data = solr.get_connection().raw_query(**kwargs)
return json.loads(data)
def test_search_02_query(self):
response = self.app.get(url(controller="api", action="search", q="Children"))
out = json.loads(str(response.body))["response"]
assert out["numFound"] == 7, out["numFound"]
exp_entity = "Department for Children, Schools and Families"
assert out["docs"][0]["from.label"] == exp_entity, out["docs"][0]
def deserialize(self, node, cstruct):
try:
return json.loads(cstruct)
except Exception as exc:
raise colander.Invalid(node, unicode(exc))
def test_search_01_no_query(self):
response = self.app.get(url(controller='api/version1',
action='search'))
out = json.loads(str(response.body))['response']
assert out['numFound'] == 36, out['numFound']
assert out['docs'][0]['dataset'] == 'cra', out
def test_search_page_pagesize(self):
response = self.app.get(url(controller='api2', action='search', page=2, pagesize=10))
result = json.loads(response.body)
h.assert_equal(result['stats']['results_count'], 10)
h.assert_equal(result['stats']['results_count_query'], 36)
def test_search_facet(self):
response = self.app.get(url(controller='api2', action='search', pagesize=0, facet_field="dataset"))
result = json.loads(response.body)
h.assert_equal(len(result['facets']['dataset']), 1)
h.assert_equal(result['facets']['dataset'][0], ['cra', 36])
def test_search_results_dataset(self):
response = self.app.get(url(controller='api2', action='search'))
result = json.loads(response.body)
h.assert_equal(result['results'][0]['dataset']['name'], 'cra')
h.assert_equal(result['results'][0]['dataset']['label'],'Country Regional Analysis v2009')
def test_permissions(self):
"""
Test permissions API which tells users if they are allowed to
perform CRUD operations on a given dataset
"""
# Create our users
make_account('test_admin', admin=True)
maintainer = make_account('maintainer')
make_account('test_user')
# Set maintainer as maintainer of cra dataset
dataset = Dataset.by_name('cra')
dataset.managers.append(maintainer)
db.session.add(dataset)
db.session.commit()
# Make the url reusable
permission = url(controller='api/version2', action='permissions')
# First we try to get permissions without dataset parameter
# This should return a 200 but include an error message and nothing
# else
response = self.app.get(permission)
json_response = json.loads(response.body)
assert len(json_response.keys()) == 1, \
'Parameterless call response includes more than one properties'
assert 'error' in json_response, \
'Error property not present in parameterless call response'
# Dataset is public by default
# Anonymous user
response = self.app.get(permission, params={'dataset': 'cra'})
anon_response = json.loads(response.body)
assert not anon_response['create'], \
'Anonymous user can create existing dataset'
assert anon_response['read'], \
'Anonymous user cannot read public dataset'
assert not anon_response['update'], \
'Anonymous user can update existing dataset'
assert not anon_response['delete'], \
'Anonymous user can delete existing dataset'
# Normal user
response = self.app.get(permission, params={'dataset': 'cra'},
extra_environ={'REMOTE_USER': '******'})
normal_response = json.loads(response.body)
assert anon_response == normal_response, \
'Normal user has wrong permissions for a public dataset'
# Maintainer
response = self.app.get(permission, params={'dataset': 'cra'},
extra_environ={'REMOTE_USER': '******'})
main_response = json.loads(response.body)
assert not main_response['create'], \
'Maintainer can create a dataset with an existing (public) name'
assert main_response['read'], \
'Maintainer is not able to read public dataset'
assert main_response['update'], \
'Maintainer is not able to update public dataset'
assert main_response['delete'], \
'Maintainer is not able to delete public dataset'
# Administrator
response = self.app.get(permission, params={'dataset': 'cra'},
extra_environ={'REMOTE_USER': '******'})
admin_response = json.loads(response.body)
# Permissions for admins should be the same as for maintainer
assert main_response == admin_response, \
'Admin and maintainer permissions differ on public datasets'
# Set cra dataset as private so only maintainer and admin should be
# able to 'read' (and 'update' and 'delete'). All others should get
# False on everything
dataset = Dataset.by_name('cra')
dataset.private = True
db.session.add(dataset)
db.session.commit()
# Anonymous user
response = self.app.get(permission, params={'dataset': 'cra'})
anon_response = json.loads(response.body)
assert True not in anon_response.values(), \
'Anonymous user has access to a private dataset'
# Normal user
response = self.app.get(permission, params={'dataset': 'cra'},
extra_environ={'REMOTE_USER': '******'})
normal_response = json.loads(response.body)
assert anon_response == normal_response, \
'Normal user has access to a private dataset'
# Maintainer
response = self.app.get(permission, params={'dataset': 'cra'},
extra_environ={'REMOTE_USER': '******'})
main_response = json.loads(response.body)
assert not main_response['create'], \
'Maintainer can create a dataset with an existing (private) name'
assert main_response['read'], \
'Maintainer is not able to read private dataset'
assert main_response['update'], \
'Maintainer is not able to update private dataset'
assert main_response['delete'], \
'Maintainer is not able to delete private dataset'
# Administrator
response = self.app.get(permission, params={'dataset': 'cra'},
extra_environ={'REMOTE_USER': '******'})
admin_response = json.loads(response.body)
# Permissions for admins should be the same as for maintainer
assert main_response == admin_response, \
'Admin does not have the same permissions as maintainer'
# Now we try accessing a nonexistent dataset
# Everyone except anonymous user should have the same permissions
# We don't need to check with maintainer or admin now since this
# applies to all logged in users
response = self.app.get(permission, params={'dataset': 'nonexistent'})
anon_response = json.loads(response.body)
assert True not in anon_response.values(), \
'Anonymous users has permissions on a nonexistent datasets'
# Any logged in user (we use normal user)
response = self.app.get(permission, params={'dataset': 'nonexistent'},
extra_environ={'REMOTE_USER': '******'})
normal_response = json.loads(response.body)
assert normal_response['create'], \
'User cannot create a nonexistent dataset'
assert not normal_response['read'], \
'User can read a nonexistent dataset'
assert not normal_response['update'], \
'User can update a nonexistent dataset'
assert not normal_response['delete'], \
'User can delete a nonexistent dataset'