def test_12_search_all_fields(self):
fields = {'url': 'a/b'}
options = search.QueryOptions(all_fields=True)
result = search.query_for(model.Resource).run(fields=fields,
options=options)
assert result['count'] == 1, result
res_dict = result['results'][0]
assert isinstance(res_dict, dict)
res_keys = set(res_dict.keys())
expected_res_keys = set(model.Resource.get_columns())
expected_res_keys.update([
'id', 'resource_group_id', 'package_id', 'position', 'size_extra',
'tracking_summary'
])
assert_equal(res_keys, expected_res_keys)
pkg1 = model.Package.by_name(u'pkg1')
ab = pkg1.resources[0]
assert res_dict['id'] == ab.id
assert res_dict['package_id'] == pkg1.id
assert res_dict['url'] == ab.url
assert res_dict['description'] == ab.description
# FIXME: This needs to be fixed before this branch is merged to master
from ckan.lib.dictization.model_dictize import _unified_resource_format
assert res_dict['format'] == _unified_resource_format(ab.format)
assert res_dict['hash'] == ab.hash
assert res_dict['position'] == 0
def test_pagination(self):
# large search
options = search.QueryOptions(order_by="id")
fields = {"url": "site"}
all_results = search.query_for(model.Resource).run(fields=fields,
options=options)
all_resources = all_results["results"]
all_resource_count = all_results["count"]
assert all_resource_count >= 6, all_results
# limit
options = search.QueryOptions(order_by="id")
options.limit = 2
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result["results"]
count = result["count"]
assert len(resources) == 2, resources
assert count == all_resource_count, (count, all_resource_count)
assert resources == all_resources[:2], "%r, %r" % (
resources,
all_resources,
)
# offset
options = search.QueryOptions(order_by="id")
options.limit = 2
options.offset = 2
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result["results"]
assert len(resources) == 2, resources
assert resources == all_resources[2:4]
# larger offset
options = search.QueryOptions(order_by="id")
options.limit = 2
options.offset = 4
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result["results"]
assert len(resources) == 2, resources
assert resources == all_resources[4:6]
def test_13_pagination(self):
# large search
options = search.QueryOptions(order_by='hash')
fields = {'url': 'site'}
all_results = search.query_for(model.Resource).run(fields=fields,
options=options)
all_resources = all_results['results']
all_resource_count = all_results['count']
assert all_resource_count >= 6, all_results
# limit
options = search.QueryOptions(order_by='hash')
options.limit = 2
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result['results']
count = result['count']
assert len(resources) == 2, resources
assert count == all_resource_count, (count, all_resource_count)
assert resources == all_resources[:2], '%r, %r' % (resources,
all_resources)
# offset
options = search.QueryOptions(order_by='hash')
options.limit = 2
options.offset = 2
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result['results']
assert len(resources) == 2, resources
assert resources == all_resources[2:4]
# larger offset
options = search.QueryOptions(order_by='hash')
options.limit = 2
options.offset = 4
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result['results']
assert len(resources) == 2, resources
assert resources == all_resources[4:6]
def res_search(self,
query="",
fields={},
terms=[],
options=search.QueryOptions()):
result = search.query_for(model.Resource).run(query=query,
fields=fields,
terms=terms,
options=options)
resources = [
model.Session.query(model.Resource).get(resource_id)
for resource_id in result["results"]
]
urls = set([resource.url for resource in resources])
return urls
def test_12_search_all_fields(self):
fields = {'url':'a/b'}
options = search.QueryOptions(all_fields=True)
result = search.query_for(model.Resource).run(fields=fields, options=options)
assert result['count'] == 1, result
res_dict = result['results'][0]
assert isinstance(res_dict, dict)
res_keys = set(res_dict.keys())
expected_res_keys = set(model.Resource.get_columns())
expected_res_keys.update(['id', 'package_id', 'position', 'size_extra'])
assert_equal(res_keys, expected_res_keys)
pkg1 = model.Package.by_name(u'pkg1')
ab = pkg1.resources[0]
assert res_dict['id'] == ab.id
assert res_dict['package_id'] == pkg1.id
assert res_dict['url'] == ab.url
assert res_dict['description'] == ab.description
assert res_dict['format'] == ab.format
assert res_dict['hash'] == ab.hash
assert res_dict['position'] == 0
def test_search_all_fields(self):
fields = {"url": "a/b"}
options = search.QueryOptions(all_fields=True)
result = search.query_for(model.Resource).run(fields=fields,
options=options)
assert result["count"] == 1, result
res_dict = result["results"][0]
assert isinstance(res_dict, dict)
res_keys = set(res_dict.keys())
expected_res_keys = set(model.Resource.get_columns())
expected_res_keys.update(["id", "package_id", "position"])
assert res_keys == expected_res_keys
pkg1 = model.Package.by_name(u"pkg1")
ab = [r for r in pkg1.resources if r.url == self.ab][0]
assert res_dict["id"] == ab.id
assert res_dict["package_id"] == pkg1.id
assert res_dict["url"] == ab.url
assert res_dict["description"] == ab.description
assert res_dict["format"] == ab.format
assert res_dict["hash"] == ab.hash
assert res_dict["position"] == 0
params = MultiDict(self._get_search_params(request.params))
except ValueError, e:
return self._finish_bad_request(
_('Could not read parameters: %r' % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ['dataset', 'package'] and not params.get('fl'):
params['fl'] = 'id' if ver == 2 else 'name'
try:
if register == 'resource':
query = search.query_for(model.Resource)
# resource search still uses ckan query parser
options = search.QueryOptions()
for k, v in params.items():
if (k in search.DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
def search(self, ver=None, register=None):
log.debug('search %s params: %r', register, request.params)
if register == 'revision':
since_time = None
if 'since_id' in request.params:
id = request.params['since_id']
if not id:
return self._finish_bad_request(
_(u'No revision specified'))
rev = model.Session.query(model.Revision).get(id)
if rev is None:
return self._finish_not_found(
_(u'There is no revision with id: %s') % id)
since_time = rev.timestamp
elif 'since_time' in request.params:
since_time_str = request.params['since_time']
try:
since_time = h.date_str_to_datetime(since_time_str)
except ValueError as inst:
return self._finish_bad_request('ValueError: %s' % inst)
else:
return self._finish_bad_request(
_("Missing search term ('since_id=UUID' or " +
" 'since_time=TIMESTAMP')"))
revs = model.Session.query(model.Revision) \
.filter(model.Revision.timestamp > since_time) \
.order_by(model.Revision.timestamp) \
.limit(50) # reasonable enough for a page
return self._finish_ok([rev.id for rev in revs])
elif register in ['dataset', 'package', 'resource']:
try:
params = MultiDict(self._get_search_params(request.params))
except ValueError as e:
return self._finish_bad_request(
_('Could not read parameters: %r' % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ['dataset', 'package'] and not params.get('fl'):
params['fl'] = 'id' if ver == 2 else 'name'
try:
if register == 'resource':
query = search.query_for(model.Resource)
# resource search still uses ckan query parser
options = search.QueryOptions()
for k, v in params.items():
if (k in search.DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
values = value
for v in values:
query_fields.add(field, v)
results = query.run(query=params.get('q'),
fields=query_fields,
options=options)
else:
# For package searches in API v3 and higher, we can pass
# parameters straight to Solr.
if ver in [1, 2]:
# Otherwise, put all unrecognised ones into the q
# parameter
params = search.\
convert_legacy_parameters_to_solr(params)
query = search.query_for(model.Package)
# Remove any existing fq param and set the capacity to
# public
if 'fq' in params:
del params['fq']
params['fq'] = '+capacity:public'
# if callback is specified we do not want to send that to
# the search
if 'callback' in params:
del params['callback']
results = query.run(params)
return self._finish_ok(results)
except search.SearchError as e:
log.exception(e)
return self._finish_bad_request(_('Bad search option: %s') % e)
else:
return self._finish_not_found(_('Unknown register: %s') % register)