class DataSetCountResource(DataCatalogResource):
"""
Shows how many data sets are currently in the index.
"""
def __init__(self):
super(DataSetCountResource, self).__init__()
self._elastic_search = Elasticsearch('{}:{}'.format(
self._config.elastic.elastic_hostname,
self._config.elastic.elastic_port))
self._search = DataSetSearch()
def get(self):
"""
Get the number of current data sets in the index per organisation.
"""
args = flask.request.args
params = self._search.get_params_from_request_args(args)
return self._search.search({}, flask.g.org_uuid_list,
params['dataset_filtering'],
flask.g.is_admin)['total']
def collect(self):
return self._search.search({}, [], DataSetFiltering.PRIVATE_AND_PUBLIC,
True)['total']
def setUp(self):
super(SearchTests, self).setUp()
self.fake_org_id = 'orgId001'
self.test_search_result = {'whateverField': 'something'}
self.test_total_hits = 666
self.test_id = 13
self.test_es_search_results = {
'hits': {
'hits': [
{
'_source': self.test_search_result,
'_id': self.test_id
}
],
'total': self.test_total_hits
},
'aggregations': {
'categories': {
'buckets': [
{
'key': 'health'
},
{
'key': 'science'
}
]
},
'formats': {
'buckets': [
{
'key': 'csv'
}
]
}
}
}
self._search_obj = DataSetSearch()
self._search_obj._translator.translate = self._mock_translate = MagicMock()
self._search_obj._elastic_search.search = self._mock_es_search = MagicMock()
self.request_context = self.app.test_request_context('/rest/datasets')
self.request_context.push()
class DataSetCountResource(DataCatalogResource):
"""
Shows how many data sets are currently in the index.
"""
def __init__(self):
super(DataSetCountResource, self).__init__()
self._elastic_search = Elasticsearch(
'{}:{}'.format(self._config.elastic.elastic_hostname,
self._config.elastic.elastic_port))
self._search = DataSetSearch()
def get(self):
"""
Get the number of current data sets in the index per organisation.
"""
args = flask.request.args
params = self._search.get_params_from_request_args(args)
return self._search.search({}, flask.g.org_uuid_list,
params['dataset_filtering'],
flask.g.is_admin)['total']
def __init__(self):
super(DataSetCountResource, self).__init__()
self._elastic_search = Elasticsearch('{}:{}'.format(
self._config.elastic.elastic_hostname,
self._config.elastic.elastic_port))
self._search = DataSetSearch()
def __init__(self):
super(DataSetCountResource, self).__init__()
self._elastic_search = Elasticsearch(
'{}:{}'.format(self._config.elastic.elastic_hostname,
self._config.elastic.elastic_port))
self._search = DataSetSearch()
class DataSetCountResource(DataCatalogResource):
"""
Shows how many data sets are currently in the index.
"""
def __init__(self):
super(DataSetCountResource, self).__init__()
self._elastic_search = Elasticsearch(
'{}:{}'.format(self._config.elastic.elastic_hostname,
self._config.elastic.elastic_port))
self._search = DataSetSearch()
@swagger.operation(
responseClass='integer',
nickname='data_set_count',
parameters=[
{
'name': 'orgs',
'description': 'A list of org UUIDs.',
'required': False,
'allowMultiple': True,
'dataType': 'list',
'paramType': 'query'
},
{
'name': 'onlyPrivate',
'description': 'Returns a list of the private data sets only',
'required': False,
'allowMultiple': False,
'dataType': 'boolean',
'paramType': 'query'
},
{
'name': 'onlyPublic',
'description': 'Returns a list of the public data sets only.',
'required': False,
'allowMultiple': False,
'dataType': 'boolean',
'paramType': 'query'
}
],
responseMessages=[
{
'code': 200,
'message': 'Data set count returned.'
}
]
)
def get(self):
"""
Get the number of current data sets in the index per organisation.
"""
args = flask.request.args
params = self._search.get_params_from_request_args(args)
return self._search.search({}, flask.g.org_uuid_list,
params['dataset_filtering'],
flask.g.is_admin)['total']
class SearchTests(DataCatalogTestCase):
def setUp(self):
super(SearchTests, self).setUp()
self.fake_org_id = 'orgId001'
self.test_search_result = {'whateverField': 'something'}
self.test_total_hits = 666
self.test_id = 13
self.test_es_search_results = {
'hits': {
'hits': [
{
'_source': self.test_search_result,
'_id': self.test_id
}
],
'total': self.test_total_hits
},
'aggregations': {
'categories': {
'buckets': [
{
'key': 'health'
},
{
'key': 'science'
}
]
},
'formats': {
'buckets': [
{
'key': 'csv'
}
]
}
}
}
self._search_obj = DataSetSearch()
self._search_obj._translator.translate = self._mock_translate = MagicMock()
self._search_obj._elastic_search.search = self._mock_es_search = MagicMock()
self.request_context = self.app.test_request_context('/rest/datasets')
self.request_context.push()
def tearDown(self):
super(SearchTests, self).tearDown()
self.request_context.pop()
def test_search_withQuery_queryPassedToIndex(self):
QUERY_STRING = 'fake data catalog query'
TRANSLATED_QUERY = 'fake translated query'
self._mock_es_search.return_value = dict(self.test_es_search_results)
self._mock_translate.return_value = str(TRANSLATED_QUERY)
response = self._search_obj.search(QUERY_STRING, self.fake_org_id, True, False)
self.assertListEqual([self.test_search_result], response['hits'])
self.assertEqual(self.test_total_hits, response['total'])
self.assertEqual(self.test_id, response['hits'][0]['id'])
self._mock_translate.assert_called_once_with(QUERY_STRING, self.fake_org_id, True, False)
self._mock_es_search.assert_called_once_with(
index=self._config.elastic.elastic_index,
doc_type=self._config.elastic.elastic_metadata_type,
body=TRANSLATED_QUERY)
def test_search_invalidQuery_invalidQueryErrorRaised(self):
self._mock_es_search.side_effect = RequestError
with self.assertRaises(InvalidQueryError):
self._search_obj.search('an invalid query string', self.fake_org_id, False, False)
def test_search_noIndexConnection_connectionErrorRaised(self):
self._mock_es_search.side_effect = ConnectionError
with self.assertRaises(IndexConnectionError):
self._search_obj.search('some query string', self.fake_org_id, False, False)
@patch.object(DataSetSearch, 'search')
def test_restSearch_withQuery_queryPassedToSearch(self, mock_search):
flask.g.org_uuid_list = '[orgid001]'
flask.g.is_admin = False
test_query = 'fake data catalog query'
mock_search.return_value = dict(self.test_es_search_results)
response = self.client.get('{}?query={}&orgs=orgid001'.format('/rest/datasets', test_query))
org_uuid_list = '[orgid001]'
self.assertEqual(200, response.status_code)
self.assertDictEqual(self.test_es_search_results, json.loads(response.data))
mock_search.assert_called_once_with(test_query, org_uuid_list, None, False)
@patch.object(DataSetSearch, 'search')
def test_restSearch_invalidQuery_400Returned(self, mock_search):
flask.g.is_admin = False
mock_search.side_effect = InvalidQueryError
response = self.client.get(self._config.app_base_path + '?query=some_invalid_query')
self.assertEqual(400, response.status_code)
@patch.object(DataSetSearch, 'search')
def test_restSearch_noIndexConnection_500Returned(self, mock_search):
flask.g.is_admin = False
mock_search.side_effect = IndexConnectionError
response = self.client.get(self._config.app_base_path + '?query=some_query')
self.assertEqual(500, response.status_code)