def _do_search(self, q, wanted_results):
options = QueryOptions()
options.order_by = 'rank'
result = self.backend.query_for(model.Package).run(query=q, options=options)
results = result['results']
err = 'Wanted %r, got %r' % (wanted_results, results)
assert wanted_results[0] == results[0], err
assert wanted_results[1] == results[1], err
def _do_search(self, q, wanted_results):
options = QueryOptions()
options.order_by = 'rank'
result = self.backend.query_for(model.Package).run(query=q,
options=options)
results = result['results']
err = 'Wanted %r, got %r' % (wanted_results, results)
assert wanted_results[0] == results[0], err
assert wanted_results[1] == results[1], err
def _do_search(self, q, expected_pkgs, count=None):
options = QueryOptions()
options.order_by = 'rank'
result = self.backend.query_for(model.Package).run(query=q, options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
if not (count is None):
assert result['count'] == count, result['count']
for expected_pkg in expected_pkgs:
assert expected_pkg in fields, expected_pkg
def _filtered_search(self, value, expected_pkgs, count=None):
options = QueryOptions()
options.order_by = 'rank'
result = self.backend.query_for(model.Package).run(fields={'geographic_coverage':value}, options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
if not (count is None):
assert result['count'] == count, result['count']
for expected_pkg in expected_pkgs:
assert expected_pkg in fields, expected_pkg
def _check_search_results(self, terms, expected_count, expected_packages=[], only_open=False, only_downloadable=False):
options = QueryOptions()
options.filter_by_openness = only_open
options.filter_by_downloadable = only_downloadable
result = self.backend.query_for(model.Package).run(query=unicode(terms))
pkgs = result['results']
count = result['count']
assert count == expected_count, (count, expected_count)
for expected_pkg in expected_packages:
assert expected_pkg in pkgs, '%s : %s' % (expected_pkg, result)
def _filtered_search(self, value, expected_pkgs, count=None):
options = QueryOptions()
options.order_by = 'rank'
result = self.backend.query_for(model.Package).run(
fields={'geographic_coverage': value}, options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
if not (count is None):
assert result['count'] == count, result['count']
for expected_pkg in expected_pkgs:
assert expected_pkg in fields, expected_pkg
def _do_search(self, q, expected_pkgs, count=None):
options = QueryOptions()
options.order_by = 'rank'
result = self.backend.query_for(model.Package).run(query=q,
options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
if not (count is None):
assert result['count'] == count, result['count']
for expected_pkg in expected_pkgs:
assert expected_pkg in fields, expected_pkg
def test_pagination(self):
# large search
all_results = self.backend.query_for(model.Package).run(query=self.q_all)
all_pkgs = all_results['results']
all_pkg_count = all_results['count']
# limit
options = QueryOptions()
options.limit = 2
result = self.backend.query_for(model.Package).run(query=self.q_all, options=options)
pkgs = result['results']
count = result['count']
assert len(pkgs) == 2, pkgs
assert count == all_pkg_count
assert pkgs == all_pkgs[:2]
# offset
options = QueryOptions()
options.limit = 2
options.offset = 2
result = self.backend.query_for(model.Package).run(query=self.q_all, options=options)
pkgs = result['results']
assert len(pkgs) == 2, pkgs
assert pkgs == all_pkgs[2:4]
# larger offset
options = QueryOptions()
options.limit = 2
options.offset = 4
result = self.backend.query_for(model.Package).run(query=self.q_all, options=options)
pkgs = result['results']
assert len(pkgs) == 2, pkgs
assert pkgs == all_pkgs[4:6]
def _check_search_results(self,
terms,
expected_count,
expected_packages=[],
only_open=False,
only_downloadable=False):
options = QueryOptions()
options.filter_by_openness = only_open
options.filter_by_downloadable = only_downloadable
result = self.backend.query_for(
model.Package).run(query=unicode(terms))
pkgs = result['results']
count = result['count']
assert count == expected_count, (count, expected_count)
for expected_pkg in expected_packages:
assert expected_pkg in pkgs, '%s : %s' % (expected_pkg, result)
def test_order_by(self):
# large search
all_results = self.backend.query_for(model.Package).run(query=self.q_all)
all_pkgs = all_results['results']
all_pkg_count = all_results['count']
# rank
options = QueryOptions()
options.order_by = 'rank'
result = self.backend.query_for(model.Package).run(query='penguin', options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
assert fields[0] == 'usa-courts-gov', fields # has penguin three times
assert pkgs == all_pkgs, pkgs #default ordering
# name
options = QueryOptions()
options.order_by = 'name'
result = self.backend.query_for(model.Package).run(query=self.q_all, options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
sorted_fields = fields; sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
# title
options = QueryOptions()
options.order_by = 'title'
result = self.backend.query_for(model.Package).run(query=self.q_all, options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).title for pkg_name in pkgs]
sorted_fields = fields; sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
# notes
options = QueryOptions()
options.order_by = 'notes'
result = self.backend.query_for(model.Package).run(query=self.q_all, options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).notes for pkg_name in pkgs]
sorted_fields = fields; sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
def test_13_pagination(self):
# large search
options = QueryOptions(order_by='hash')
fields = {'url':'site'}
all_results = self.backend.query_for(model.PackageResource).run(fields=fields, options=options)
all_resources = all_results['results']
all_resource_count = all_results['count']
assert all_resource_count >= 6, all_results
# limit
options = QueryOptions(order_by='hash')
options.limit = 2
result = self.backend.query_for(model.PackageResource).run(fields=fields, options=options)
resources = result['results']
count = result['count']
assert len(resources) == 2, resources
assert count == all_resource_count
assert resources == all_resources[:2], '%r, %r' % (resources, all_resources)
# offset
options = QueryOptions(order_by='hash')
options.limit = 2
options.offset = 2
result = self.backend.query_for(model.PackageResource).run(fields=fields, options=options)
resources = result['results']
assert len(resources) == 2, resources
assert resources == all_resources[2:4]
# larger offset
options = QueryOptions(order_by='hash')
options.limit = 2
options.offset = 4
result = self.backend.query_for(model.PackageResource).run(fields=fields, options=options)
resources = result['results']
assert len(resources) == 2, resources
assert resources == all_resources[4:6]
def test_12_search_all_fields(self):
fields = {'url':'a/b'}
options = QueryOptions(all_fields=True)
result = self.backend.query_for(model.Resource).run(fields=fields, options=options)
assert result['count'] == 1, result
res_dict = result['results'][0]
assert isinstance(res_dict, dict)
res_keys = set(res_dict.keys())
expected_res_keys = set(model.Resource.get_columns())
expected_res_keys.update(['id', 'resource_group_id', 'package_id', 'position', 'size'])
assert_equal(res_keys, expected_res_keys)
pkg1 = model.Package.by_name(u'pkg1')
ab = pkg1.resources[0]
assert res_dict['id'] == ab.id
assert res_dict['package_id'] == pkg1.id
assert res_dict['url'] == ab.url
assert res_dict['description'] == ab.description
assert res_dict['format'] == ab.format
assert res_dict['hash'] == ab.hash
assert res_dict['position'] == 0
params = MultiDict(self._get_search_params(request.params))
except ValueError, e:
return self._finish_bad_request(
gettext('Could not read parameters: %r' % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ['dataset', 'package'] and not params.get('fl'):
params['fl'] = 'id' if ver == '2' else 'name'
try:
if register == 'resource':
query = query_for(model.Resource)
# resource search still uses ckan query parser
options = QueryOptions()
for k, v in params.items():
if (k in DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
params = MultiDict(self._get_search_params(request.params))
except ValueError, e:
return self._finish_bad_request(
gettext('Could not read parameters: %r' % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ['dataset', 'package'] and not params.get('fl'):
params['fl'] = 'id' if ver == '2' else 'name'
try:
if register == 'resource':
query = query_for(model.Resource)
# resource search still uses ckan query parser
options = QueryOptions()
for k, v in params.items():
if (k in DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
def res_search(self, query='', fields={}, terms=[], options=QueryOptions()):
result = self.backend.query_for(model.Resource).run(query=query, fields=fields, terms=terms, options=options)
resources = [model.Session.query(model.Resource).get(resource_id) for resource_id in result['results']]
urls = set([resource.url for resource in resources])
return urls
try:
since_time = model.strptimestamp(since_time_str)
except ValueError, inst:
return self._finish_bad_request('ValueError: %s' % inst)
else:
return self._finish_bad_request(
gettext("Missing search term ('since_id=UUID' or 'since_time=TIMESTAMP')"))
revs = model.Session.query(model.Revision).filter(model.Revision.timestamp>since_time)
return self._finish_ok([rev.id for rev in revs])
elif register == 'package' or register == 'resource':
try:
params = self._get_search_params(request.params)
except ValueError, e:
return self._finish_bad_request(
gettext('Could not read parameters: %r' % e))
options = QueryOptions()
for k, v in params.items():
if (k in DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
return self._finish_bad_request('ValueError: %s' % inst)
else:
return self._finish_bad_request(
gettext(
"Missing search term ('since_id=UUID' or 'since_time=TIMESTAMP')"
))
revs = model.Session.query(
model.Revision).filter(model.Revision.timestamp > since_time)
return self._finish_ok([rev.id for rev in revs])
elif register == 'package' or register == 'resource':
try:
params = self._get_search_params(request.params)
except ValueError, e:
return self._finish_bad_request(
gettext('Could not read parameters: %r' % e))
options = QueryOptions()
for k, v in params.items():
if (k in DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
def test_order_by(self):
# large search
all_results = self.backend.query_for(
model.Package).run(query=self.q_all)
all_pkgs = all_results['results']
all_pkg_count = all_results['count']
# rank
options = QueryOptions()
options.order_by = 'rank'
result = self.backend.query_for(model.Package).run(query='penguin',
options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
assert fields[0] == 'usa-courts-gov', fields # has penguin three times
assert pkgs == all_pkgs, pkgs #default ordering
# name
options = QueryOptions()
options.order_by = 'name'
result = self.backend.query_for(model.Package).run(query=self.q_all,
options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
# title
options = QueryOptions()
options.order_by = 'title'
result = self.backend.query_for(model.Package).run(query=self.q_all,
options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).title for pkg_name in pkgs]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
# notes
options = QueryOptions()
options.order_by = 'notes'
result = self.backend.query_for(model.Package).run(query=self.q_all,
options=options)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).notes for pkg_name in pkgs]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
def test_pagination(self):
# large search
all_results = self.backend.query_for(
model.Package).run(query=self.q_all)
all_pkgs = all_results['results']
all_pkg_count = all_results['count']
# limit
options = QueryOptions()
options.limit = 2
result = self.backend.query_for(model.Package).run(query=self.q_all,
options=options)
pkgs = result['results']
count = result['count']
assert len(pkgs) == 2, pkgs
assert count == all_pkg_count
assert pkgs == all_pkgs[:2]
# offset
options = QueryOptions()
options.limit = 2
options.offset = 2
result = self.backend.query_for(model.Package).run(query=self.q_all,
options=options)
pkgs = result['results']
assert len(pkgs) == 2, pkgs
assert pkgs == all_pkgs[2:4]
# larger offset
options = QueryOptions()
options.limit = 2
options.offset = 4
result = self.backend.query_for(model.Package).run(query=self.q_all,
options=options)
pkgs = result['results']
assert len(pkgs) == 2, pkgs
assert pkgs == all_pkgs[4:6]
def test_13_pagination(self):
# large search
options = QueryOptions(order_by='hash')
fields = {'url':'site'}
all_results = self.backend.query_for(model.Resource).run(fields=fields, options=options)
all_resources = all_results['results']
all_resource_count = all_results['count']
assert all_resource_count >= 6, all_results
# limit
options = QueryOptions(order_by='hash')
options.limit = 2
result = self.backend.query_for(model.Resource).run(fields=fields, options=options)
resources = result['results']
count = result['count']
assert len(resources) == 2, resources
assert count == all_resource_count
assert resources == all_resources[:2], '%r, %r' % (resources, all_resources)
# offset
options = QueryOptions(order_by='hash')
options.limit = 2
options.offset = 2
result = self.backend.query_for(model.Resource).run(fields=fields, options=options)
resources = result['results']
assert len(resources) == 2, resources
assert resources == all_resources[2:4]
# larger offset
options = QueryOptions(order_by='hash')
options.limit = 2
options.offset = 4
result = self.backend.query_for(model.Resource).run(fields=fields, options=options)
resources = result['results']
assert len(resources) == 2, resources
assert resources == all_resources[4:6]
elif register == 'package' or register == 'resource':
if request.params.has_key('qjson'):
if not request.params['qjson']:
response.status_int = 400
return gettext('Blank qjson parameter')
params = json.loads(request.params['qjson'])
elif request.params.values() and request.params.values() != [u''] and request.params.values() != [u'1']:
params = request.params
else:
try:
params = self._get_request_data()
except ValueError, inst:
response.status_int = 400
return gettext(u'Search params: %s') % unicode(inst)
options = QueryOptions()
for k, v in params.items():
if (k in DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = self._get_username()
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
