def test_search_foreign_chars(self):
result = search.query_for(model.Package).run({"q": "umlaut"})
assert result["results"] == ["gils"], result["results"]
result = search.query_for(model.Package).run({"q": u"thumb"})
assert result["results"] == ["gils"], result["results"]
result = search.query_for(model.Package).run({"q": u"th\xfcmb"})
assert result["results"] == ["gils"], result["results"]
def test_pagination(self):
# large search
all_results = search.query_for(model.Package).run({"q": self.q_all})
all_pkgs = all_results["results"]
all_pkg_count = all_results["count"]
# limit
query = {"q": self.q_all, "rows": 2}
result = search.query_for(model.Package).run(query)
pkgs = result["results"]
count = result["count"]
assert len(pkgs) == 2, pkgs
assert count == all_pkg_count
assert pkgs == all_pkgs[:2]
# offset
query = {"q": self.q_all, "rows": 2, "start": 2}
result = search.query_for(model.Package).run(query)
pkgs = result["results"]
assert len(pkgs) == 2, pkgs
assert pkgs == all_pkgs[2:4]
# larger offset
query = {"q": self.q_all, "rows": 2, "start": 4}
result = search.query_for(model.Package).run(query)
pkgs = result["results"]
assert len(pkgs) == 2, pkgs
assert pkgs == all_pkgs[4:6]
def test_groups(self):
result = search.query_for(model.Package).run({"q": u"groups:random"})
assert self._pkg_names(result) == "", self._pkg_names(result)
result = search.query_for(model.Package).run({"q": u"groups:ukgov"})
assert result["count"] == 4, self._pkg_names(result)
result = search.query_for(model.Package).run({"q": u"groups:ukgov tags:us"})
assert result["count"] == 2, self._pkg_names(result)
def test_groups(self):
result = search.query_for(model.Package).run({'q': u'groups:random'})
assert self._pkg_names(result) == '', self._pkg_names(result)
result = search.query_for(model.Package).run({'q': u'groups:ukgov'})
assert result['count'] == 4, self._pkg_names(result)
result = search.query_for(model.Package).run({'q': u'groups:ukgov tags:us'})
assert result['count'] == 2, self._pkg_names(result)
def test_search_foreign_chars(self):
result = search.query_for(model.Package).run({'q': 'umlaut'})
assert result['results'] == ['gils'], result['results']
result = search.query_for(model.Package).run({'q': u'thumb'})
assert result['results'] == ['gils'], result['results']
result = search.query_for(model.Package).run({'q': u'th\xfcmb'})
assert result['results'] == ['gils'], result['results']
def test_quotation(self):
# multiple words quoted
result = search.query_for(model.Package).run({'q': u'"Government Expenditure"'})
assert self._pkg_names(result) == 'uk-government-expenditure', self._pkg_names(result)
# multiple words quoted wrong order
result = search.query_for(model.Package).run({'q': u'"Expenditure Government"'})
assert self._pkg_names(result) == '', self._pkg_names(result)
def test_good_search_fields(self):
result = search.query_for(model.Tag).run(fields={"tags": u"ru"})
assert result["count"] == 1, result
assert "russian" in result["results"], result
result = search.query_for(model.Tag).run(fields={"tags": u"s"})
assert result["count"] == 2, result
assert "russian" in result["results"], result
assert "tolstoy" in result["results"], result
def test_good_search_query(self):
result = search.query_for(model.Tag).run(query=u"ru")
assert result["count"] == 1, result
assert "russian" in result["results"], result
result = search.query_for(model.Tag).run(query=u"s")
assert result["count"] == 2, result
assert "russian" in result["results"], result
assert "tolstoy" in result["results"], result
def test_good_search_query(self):
result = search.query_for(model.Tag).run(query=u'ru')
assert result['count'] == 1, result
assert 'russian' in result['results'], result
result = search.query_for(model.Tag).run(query=u's')
assert result['count'] == 2, result
assert 'russian' in result['results'], result
assert 'tolstoy' in result['results'], result
def test_good_search_fields(self):
result = search.query_for(model.Tag).run(fields={'tags': u'ru'})
assert result['count'] == 1, result
assert 'russian' in result['results'], result
result = search.query_for(model.Tag).run(fields={'tags': u's'})
assert result['count'] == 2, result
assert 'russian' in result['results'], result
assert 'tolstoy' in result['results'], result
def test_2_title(self):
# exact title, one word
result = search.query_for(model.Package).run({'q': u'Opengov.se'})
assert self._pkg_names(result) == 'se-opengov', self._pkg_names(result)
# multiple words
result = search.query_for(model.Package).run({'q': u'Government Expenditure'})
assert self._pkg_names(result) == 'uk-government-expenditure', self._pkg_names(result)
# multiple words wrong order
result = search.query_for(model.Package).run({'q': u'Expenditure Government'})
assert self._pkg_names(result) == 'uk-government-expenditure', self._pkg_names(result)
# multiple words, one doesn't match
result = search.query_for(model.Package).run({'q': u'Expenditure Government China'})
assert len(result['results']) == 0, self._pkg_names(result)
def test_14_extra_info(self):
fields = {'alt_url':'alt_1'}
result = search.query_for(model.Resource).run(fields=fields)
assert result['count'] == 2, result
fields = {'alt_url':'alt_2'}
result = search.query_for(model.Resource).run(fields=fields)
assert result['count'] == 1, result
# Document that resource extras not in ckan.extra_resource_fields
# can't be searched
fields = {'size_extra':'100'}
assert_raises(search.SearchError, search.query_for(model.Resource).run, fields=fields)
def test_14_extra_info(self):
fields = {"alt_url": "alt1"}
result = search.query_for(model.Resource).run(fields=fields)
assert result["count"] == 2, result
fields = {"alt_url": "alt2"}
result = search.query_for(model.Resource).run(fields=fields)
assert result["count"] == 1, result
# Document that resource extras not in ckan.extra_resource_fields
# can't be searched
fields = {"size_extra": "100"}
assert_raises(search.SearchError, search.query_for(model.Resource).run, fields=fields)
def test_special_characters_folding(self):
# Search word with letter a with grave accent
result = search.query_for(model.Package).run({'q': u'metropolità'})
self._assert_one_package(result, u'test-catalan')
# Search same word with letter a without accent
result = search.query_for(model.Package).run({'q': u'metropolita'})
self._assert_one_package(result, u'test-catalan')
# Search word with two special characters
result = search.query_for(model.Package).run({'q': u'pražských'})
self._assert_one_package(result, u'test-czech')
# Search same word with just one special character
result = search.query_for(model.Package).run({'q': u'prazských'})
self._assert_one_package(result, u'test-czech')
# Search uppercase word with accents
result = search.query_for(model.Package).run({'q': u'SVĚTOVÁ'})
self._assert_one_package(result, u'test-czech')
# Search same uppercase word without accents
result = search.query_for(model.Package).run({'q': u'SVETOVA'})
self._assert_one_package(result, u'test-czech')
# Search greek word with accent
result = search.query_for(model.Package).run({'q': u'ηπειρωτικές'})
self._assert_one_package(result, u'test-greek')
# Search same greek word without accents: folding does not work
result = search.query_for(model.Package).run({'q': u'ηπειρωτικες'})
assert result['count'] == 0, result
def test_12_search_all_fields(self):
fields = {"url": "a/b"}
options = search.QueryOptions(all_fields=True)
result = search.query_for(model.Resource).run(fields=fields, options=options)
assert result["count"] == 1, result
res_dict = result["results"][0]
assert isinstance(res_dict, dict)
res_keys = set(res_dict.keys())
expected_res_keys = set(model.Resource.get_columns())
expected_res_keys.update(
["id", "resource_group_id", "package_id", "position", "size_extra", "tracking_summary"]
)
assert_equal(res_keys, expected_res_keys)
pkg1 = model.Package.by_name(u"pkg1")
ab = pkg1.resources[0]
assert res_dict["id"] == ab.id
assert res_dict["package_id"] == pkg1.id
assert res_dict["url"] == ab.url
assert res_dict["description"] == ab.description
# FIXME: This needs to be fixed before this branch is merged to master
from ckan.lib.dictization.model_dictize import _unified_resource_format
assert res_dict["format"] == _unified_resource_format(ab.format)
assert res_dict["hash"] == ab.hash
assert res_dict["position"] == 0
def package_search(context, data_dict):
model = context["model"]
session = context["session"]
user = context["user"]
check_access("package_search", context, data_dict)
# return a list of package ids
data_dict["fl"] = "id"
query = query_for(model.Package)
query.run(data_dict)
results = []
for package in query.results:
# get the package object
pkg_query = (
session.query(model.PackageRevision)
.filter(model.PackageRevision.id == package)
.filter(and_(model.PackageRevision.state == u"active", model.PackageRevision.current == True))
)
pkg = pkg_query.first()
## if the index has got a package that is not in ckan then
## ignore it.
if not pkg:
log.warning("package %s in index but not in database" % package)
continue
result_dict = package_dictize(pkg, context)
results.append(result_dict)
return {"count": query.count, "facets": query.facets, "results": results}
def tag_list(context, data_dict):
"""Returns a list of tags"""
model = context["model"]
user = context["user"]
all_fields = data_dict.get("all_fields", None)
check_access("tag_list", context, data_dict)
q = data_dict.get("q", "")
if q:
limit = data_dict.get("limit", 25)
offset = data_dict.get("offset", 0)
return_objects = data_dict.get("return_objects", True)
query = query_for(model.Tag)
query.run(query=q, limit=limit, offset=offset, return_objects=return_objects, username=user)
tags = query.results
else:
tags = model.Session.query(model.Tag).all()
tag_list = []
if all_fields:
for tag in tags:
result_dict = tag_dictize(tag, context)
tag_list.append(result_dict)
else:
tag_list = [tag.name for tag in tags]
return tag_list
def test_2_title(self):
# exact title, one word
result = search.query_for(model.Package).run({'q': u'Opengov'})
assert self._pkg_names(result) == 'se-opengov', self._pkg_names(result)
# multiple words
result = search.query_for(model.Package).run({'q': u'Government Expenditure'})
# uk-government-expenditure is the best match but all other results should be retured
assert self._pkg_names(result).startswith('uk-government-expenditure'), self._pkg_names(result)
# multiple words wrong order
result = search.query_for(model.Package).run({'q': u'Expenditure Government'})
assert self._pkg_names(result).startswith('uk-government-expenditure'), self._pkg_names(result)
# multiple words all should match government
result = search.query_for(model.Package).run({'q': u'Expenditure Government China'})
assert len(result['results']) == 1, self._pkg_names(result)
def index(self):
c.q = request.params.get('q', '')
if c.q:
page = int(request.params.get('page', 1))
query = query_for('tag', backend='sql')
query.run(query=c.q,
limit=LIMIT,
offset=(page-1)*LIMIT,
return_objects=True,
username=c.user)
c.page = h.Page(
collection=query.results,
page=page,
item_count=query.count,
items_per_page=LIMIT
)
c.page.items = query.results
else:
query = model.Tag.all()
c.page = AlphaPage(
collection=query,
page=request.params.get('page', 'A'),
alpha_attribute='name',
other_text=_('Other'),
)
return render('tag/index.html')
def _do_search(self, q, wanted_results):
query = {"q": q, "sort": "rank"}
result = search.query_for(model.Package).run(query)
results = result["results"]
err = "Wanted %r, got %r" % (wanted_results, results)
assert wanted_results[0] == results[0], err
assert wanted_results[1] == results[1], err
def tag_list(context, data_dict):
'''Returns a list of tags'''
model = context['model']
user = context['user']
all_fields = data_dict.get('all_fields',None)
check_access('tag_list',context, data_dict)
q = data_dict.get('q','')
if q:
limit = data_dict.get('limit',25)
offset = data_dict.get('offset',0)
return_objects = data_dict.get('return_objects',True)
query = query_for(model.Tag)
query.run(query=q,
limit=limit,
offset=offset,
return_objects=return_objects,
username=user)
tags = query.results
else:
tags = model.Session.query(model.Tag).all()
tag_list = []
if all_fields:
for tag in tags:
result_dict = tag_dictize(tag, context)
tag_list.append(result_dict)
else:
tag_list = [tag.name for tag in tags]
return tag_list
def test_01_search_url(self):
fields = {'url':'site.com'}
result = search.query_for(model.Resource).run(fields=fields)
assert result['count'] == 6, result
resources = [model.Session.query(model.Resource).get(resource_id) for resource_id in result['results']]
urls = set([resource.url for resource in resources])
assert set([self.ab, self.cd, self.ef]) == urls, urls
def package_search(context, data_dict):
model = context['model']
session = context['session']
user = context['user']
check_access('package_search', context, data_dict)
# check if some extension needs to modify the search params
for item in PluginImplementations(IPackageController):
data_dict = item.before_search(data_dict)
# the extension may have decided that it's no necessary to perform the query
abort = data_dict.get('abort_search',False)
results = []
if not abort:
# return a list of package ids
data_dict['fl'] = 'id'
query = query_for(model.Package)
query.run(data_dict)
for package in query.results:
# get the package object
pkg_query = session.query(model.PackageRevision)\
.filter(model.PackageRevision.id == package)\
.filter(and_(
model.PackageRevision.state == u'active',
model.PackageRevision.current == True
))
pkg = pkg_query.first()
## if the index has got a package that is not in ckan then
## ignore it.
if not pkg:
log.warning('package %s in index but not in database' % package)
continue
result_dict = package_dictize(pkg,context)
results.append(result_dict)
count = query.count
facets = query.facets
else:
count = 0
facets = {}
results = []
search_results = {
'count': count,
'facets': facets,
'results': results
}
# check if some extension needs to modify the search results
for item in PluginImplementations(IPackageController):
search_results = item.after_search(search_results,data_dict)
return search_results
def _do_search(self, department, expected_pkgs, count=None):
result = search.query_for(model.Package).run({'q': 'department: %s' % department})
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
if not (count is None):
assert result['count'] == count, result['count']
for expected_pkg in expected_pkgs:
assert expected_pkg in fields, expected_pkg
def test_2_title(self):
# exact title, one word
result = search.query_for(model.Package).run({"q": u"Opengov.se"})
assert self._pkg_names(result) == "se-opengov", self._pkg_names(result)
# multiple words
result = search.query_for(model.Package).run({"q": u"Government Expenditure"})
# uk-government-expenditure is the best match but all other results should be retured
assert self._pkg_names(result).startswith("uk-government-expenditure"), self._pkg_names(result)
# se-opengov has only government in tags, all others hav it in title.
assert self._pkg_names(result).endswith("se-opengov"), self._pkg_names(result)
# multiple words wrong order
result = search.query_for(model.Package).run({"q": u"Expenditure Government"})
assert self._pkg_names(result).startswith("uk-government-expenditure"), self._pkg_names(result)
assert self._pkg_names(result).endswith("se-opengov"), self._pkg_names(result)
# multiple words all should match government
result = search.query_for(model.Package).run({"q": u"Expenditure Government China"})
assert len(result["results"]) == 5, self._pkg_names(result)
def test_1_extras_in_all_fields(self):
response = search.query_for(model.Package).run({"q": "abc", "fl": "*"})
assert response["count"] == 2
results = response["results"]
for result in results:
assert "extras" in result.keys(), result
assert "department" in result["extras"], result["extras"]
assert result["extras"]["department"] in ["abc", "cde abc"], result["extras"]["department"]
def _filtered_search(self, value, expected_pkgs, count=None):
query = {"q": "geographic_coverage:%s" % value, "sort": "rank"}
result = search.query_for(model.Package).run(query)
pkgs = result["results"]
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
if not (count is None):
assert result["count"] == count, result["count"]
for expected_pkg in expected_pkgs:
assert expected_pkg in fields, expected_pkg
def _do_search(self, q, expected_pkgs, count=None):
query = {"q": q, "sort": "rank"}
result = search.query_for(model.Package).run(query)
pkgs = result["results"]
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
if not (count is None):
assert result["count"] == count, result["count"]
for expected_pkg in expected_pkgs:
assert expected_pkg in fields, expected_pkg
def test_1_extras_in_all_fields(self):
response = search.query_for(model.Package).run({'q': 'abc', 'fl': '*'})
assert response['count'] == 2
results = response['results']
for result in results:
assert 'extras' in result.keys(), result
assert 'department' in result['extras'], result['extras']
assert result['extras']['department'] in ['abc', 'cde abc'], result['extras']['department']
def _get_stars_from_solr(id):
results = query_for('package').run({
'q': 'id:' + id,
'fl': 'extras_five_star_rating'
})['results']
try:
return int(results[0]['extras']['five_star_rating'])
except Exception as e:
logger.warn('Unable to get rating of <{}>: {}'.format(id, e))
return 0
def _do_search(self, q, wanted_results):
query = {
'q': q,
'sort': 'score desc, name asc',
}
result = search.query_for(model.Package).run(query)
results = result['results']
err = 'Wanted %r, got %r' % (wanted_results, results)
assert wanted_results[0] == results[0], err
assert wanted_results[1] == results[1], err
def test_01_search_url(self):
fields = {'url': 'site.com'}
result = search.query_for(model.Resource).run(fields=fields)
assert result['count'] == 6, result
resources = [
model.Session.query(model.Resource).get(resource_id)
for resource_id in result['results']
]
urls = set([resource.url for resource in resources])
assert set([self.ab, self.cd, self.ef]) == urls, urls
def check_search_results(terms, expected_count, expected_packages=[]):
query = {
'q': text_type(terms),
}
result = search.query_for(model.Package).run(query)
pkgs = result['results']
count = result['count']
assert_equal(count, expected_count)
for expected_pkg in expected_packages:
assert expected_pkg in pkgs, '%s : %s' % (expected_pkg, result)
def check_search_results(terms, expected_count, expected_packages=[]):
query = {
'q': unicode(terms),
}
result = search.query_for(model.Package).run(query)
pkgs = result['results']
count = result['count']
assert_equal(count, expected_count)
for expected_pkg in expected_packages:
assert expected_pkg in pkgs, '%s : %s' % (expected_pkg, result)
def test_04_delete_package_from_dict():
factories.Dataset()
factories.Dataset(**get_data())
query = search.query_for(model.Package)
package = model.Package.by_name("council-owned-litter-bins")
# delete it
package.delete()
model.repo.commit_and_remove()
assert query.run({"q": ""})["count"] == 1
def test_1_extras_in_all_fields(self):
response = search.query_for(model.Package).run({'q': 'abc', 'fl': '*'})
assert response['count'] == 2
results = response['results']
for result in results:
assert 'extras' in result.keys(), result
assert 'department' in result['extras'], result['extras']
assert result['extras']['department'] in [
'abc', 'cde abc'
], result['extras']['department']
def test_13_pagination(self):
# large search
options = search.QueryOptions(order_by='hash')
fields = {'url': 'site'}
all_results = search.query_for(model.Resource).run(fields=fields,
options=options)
all_resources = all_results['results']
all_resource_count = all_results['count']
assert all_resource_count >= 6, all_results
# limit
options = search.QueryOptions(order_by='hash')
options.limit = 2
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result['results']
count = result['count']
assert len(resources) == 2, resources
assert count == all_resource_count, (count, all_resource_count)
assert resources == all_resources[:2], '%r, %r' % (resources,
all_resources)
# offset
options = search.QueryOptions(order_by='hash')
options.limit = 2
options.offset = 2
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result['results']
assert len(resources) == 2, resources
assert resources == all_resources[2:4]
# larger offset
options = search.QueryOptions(order_by='hash')
options.limit = 2
options.offset = 4
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result['results']
assert len(resources) == 2, resources
assert resources == all_resources[4:6]
def test_2_title(self):
# exact title, one word
result = search.query_for(model.Package).run({"q": u"Opengov"})
assert self._pkg_names(result) == "se-opengov", self._pkg_names(result)
# multiple words
result = search.query_for(model.Package).run(
{"q": u"Government Expenditure"})
# uk-government-expenditure is the best match but all other results should be retured
assert self._pkg_names(result).startswith(
"uk-government-expenditure"), self._pkg_names(result)
# multiple words wrong order
result = search.query_for(model.Package).run(
{"q": u"Expenditure Government"})
assert self._pkg_names(result).startswith(
"uk-government-expenditure"), self._pkg_names(result)
# multiple words all should match government
result = search.query_for(model.Package).run(
{"q": u"Expenditure Government China"})
assert len(result["results"]) == 1, self._pkg_names(result)
def test_2_title(self):
# exact title, one word
result = search.query_for(model.Package).run({'q': u'Opengov'})
assert self._pkg_names(result) == 'se-opengov', self._pkg_names(result)
# multiple words
result = search.query_for(model.Package).run(
{'q': u'Government Expenditure'})
# uk-government-expenditure is the best match but all other results should be retured
assert self._pkg_names(result).startswith(
'uk-government-expenditure'), self._pkg_names(result)
# multiple words wrong order
result = search.query_for(model.Package).run(
{'q': u'Expenditure Government'})
assert self._pkg_names(result).startswith(
'uk-government-expenditure'), self._pkg_names(result)
# multiple words all should match government
result = search.query_for(model.Package).run(
{'q': u'Expenditure Government China'})
assert len(result['results']) == 1, self._pkg_names(result)
def test_1_extras_in_all_fields(self):
response = search.query_for(model.Package).run({"q": "abc", "fl": "*"})
assert response["count"] == 2
results = response["results"]
for result in results:
assert "extras" in result.keys(), result
assert "department" in result["extras"], result["extras"]
assert result["extras"]["department"] in [
"abc",
"cde abc",
], result["extras"]["department"]
def test_order_by(self):
# TODO: fix this test
#
# as we are not using the 'edismax' query parser now (requires solr >= 3.*), the
# search weighting has been changed
from nose import SkipTest
raise SkipTest()
# large search
all_results = search.query_for(model.Package).run({"q": self.q_all})
all_pkgs = all_results["results"]
all_pkg_count = all_results["count"]
# rank
query = {"q": "government", "sort": "rank"}
result = search.query_for(model.Package).run(query)
pkgs = result["results"]
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
assert (fields[0] == "gils"
), fields # has government in tags, title and notes
# name
query = {"q": self.q_all, "sort": "name asc"}
result = search.query_for(model.Package).run(query)
pkgs = result["results"]
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
# title
query = {"q": self.q_all, "sort": "title asc"}
result = search.query_for(model.Package).run(query)
pkgs = result["results"]
fields = [model.Package.by_name(pkg_name).title for pkg_name in pkgs]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
# notes
query = {"q": self.q_all, "sort": "notes asc"}
result = search.query_for(model.Package).run(query)
pkgs = result["results"]
fields = [model.Package.by_name(pkg_name).notes for pkg_name in pkgs]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
# extra field
query = {"q": self.q_all, "sort": "date_released asc"}
result = search.query_for(model.Package).run(query)
pkgs = result["results"]
fields = [model.Package.by_name(pkg_name) for pkg_name in pkgs]
fields = [field.extras.get("date_released") for field in fields]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
def test_order_by(self):
# TODO: fix this test
#
# as we are not using the 'edismax' query parser now (requires solr >= 3.*), the
# search weighting has been changed
from nose import SkipTest
raise SkipTest()
# large search
all_results = search.query_for(model.Package).run({'q': self.q_all})
all_pkgs = all_results['results']
all_pkg_count = all_results['count']
# rank
query = {'q': 'government', 'sort': 'rank'}
result = search.query_for(model.Package).run(query)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
assert fields[
0] == 'gils', fields # has government in tags, title and notes
# name
query = {'q': self.q_all, 'sort': 'name asc'}
result = search.query_for(model.Package).run(query)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
# title
query = {'q': self.q_all, 'sort': 'title asc'}
result = search.query_for(model.Package).run(query)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).title for pkg_name in pkgs]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
# notes
query = {'q': self.q_all, 'sort': 'notes asc'}
result = search.query_for(model.Package).run(query)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name).notes for pkg_name in pkgs]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
# extra field
query = {'q': self.q_all, 'sort': 'date_released asc'}
result = search.query_for(model.Package).run(query)
pkgs = result['results']
fields = [model.Package.by_name(pkg_name) for pkg_name in pkgs]
fields = [field.extras.get('date_released') for field in fields]
sorted_fields = fields
sorted_fields.sort()
assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
def _refresh_pkg_count_on_org_list(orgs):
query_params = {
'start': 0,
'rows': 1,
# 'fl': ['id', 'name'],
'fl': 'id name',
'facet.field': ['organization'],
'facet.limit': 2000,
}
# search_result = tk.get_action('package_search')({}, query_params)
query = search.query_for(model.Package)
query.run(query_params)
org_name_to_pkg_count = query.facets.get('organization', {})
for org in orgs:
org['package_count'] = org_name_to_pkg_count.get(org['name'], 0)
def res_search(self,
query="",
fields={},
terms=[],
options=search.QueryOptions()):
result = search.query_for(model.Resource).run(query=query,
fields=fields,
terms=terms,
options=options)
resources = [
model.Session.query(model.Resource).get(resource_id)
for resource_id in result["results"]
]
urls = set([resource.url for resource in resources])
return urls
def tag_autocomplete(self):
incomplete = request.params.get('incomplete', '')
if incomplete:
query = query_for('tag', backend='sql')
query.run(query=incomplete,
return_objects=True,
limit=10,
username=c.user)
tagNames = [t.name for t in query.results]
else:
tagNames = []
resultSet = {"ResultSet": {"Result": []}}
for tagName in tagNames[:10]:
result = {"Name": tagName}
resultSet["ResultSet"]["Result"].append(result)
return self._finish_ok(resultSet)
def test_overall(self):
CreateTestData.create()
query = search.query_for(model.Package)
assert query.run({"q": "annakarenina"})["count"] == 1
assert query.run({"q": "warandpeace"})["count"] == 1
assert query.run({"q": ""})["count"] == 2
assert query.run({"q": "Tolstoy"})["count"] == 1
assert query.run({"q": "title:Novel"})["count"] == 1
assert query.run({"q": "title:peace"})["count"] == 0
assert query.run({"q": "name:warandpeace"})["count"] == 1
assert query.run({"q": "groups:david"})["count"] == 2
assert query.run({"q": "groups:roger"})["count"] == 1
assert query.run({"q": "groups:lenny"})["count"] == 0
assert query.run({"q": 'tags:"russian"'})["count"] == 2
assert query.run({"q": 'tags:"Flexible \u30a1"'})["count"] == 2
assert query.run({"q": "Flexible \u30a1"})["count"] == 2
assert query.run({"q": "Flexible"})["count"] == 2
assert query.run({"q": "flexible"})["count"] == 2
def tag_autocomplete(context, data_dict):
'''Returns tags containing the provided string'''
model = context['model']
session = context['session']
user = context['user']
check_access('tag_autocomplete', context, data_dict)
q = data_dict.get('q', None)
if not q:
return []
limit = data_dict.get('limit', 10)
query = query_for('tag')
query.run(query=q, return_objects=True, limit=10, username=user)
return [tag.name for tag in query.results]
def test_12_search_all_fields(self):
fields = {'url':'a/b'}
options = search.QueryOptions(all_fields=True)
result = search.query_for(model.Resource).run(fields=fields, options=options)
assert result['count'] == 1, result
res_dict = result['results'][0]
assert isinstance(res_dict, dict)
res_keys = set(res_dict.keys())
expected_res_keys = set(model.Resource.get_columns())
expected_res_keys.update(['id', 'package_id', 'position', 'size_extra'])
assert_equal(res_keys, expected_res_keys)
pkg1 = model.Package.by_name(u'pkg1')
ab = pkg1.resources[0]
assert res_dict['id'] == ab.id
assert res_dict['package_id'] == pkg1.id
assert res_dict['url'] == ab.url
assert res_dict['description'] == ab.description
assert res_dict['format'] == ab.format
assert res_dict['hash'] == ab.hash
assert res_dict['position'] == 0
def test_search_all_fields(self):
fields = {"url": "a/b"}
options = search.QueryOptions(all_fields=True)
result = search.query_for(model.Resource).run(fields=fields,
options=options)
assert result["count"] == 1, result
res_dict = result["results"][0]
assert isinstance(res_dict, dict)
res_keys = set(res_dict.keys())
expected_res_keys = set(model.Resource.get_columns())
expected_res_keys.update(["id", "package_id", "position"])
assert res_keys == expected_res_keys
pkg1 = model.Package.by_name(u"pkg1")
ab = [r for r in pkg1.resources if r.url == self.ab][0]
assert res_dict["id"] == ab.id
assert res_dict["package_id"] == pkg1.id
assert res_dict["url"] == ab.url
assert res_dict["description"] == ab.description
assert res_dict["format"] == ab.format
assert res_dict["hash"] == ab.hash
assert res_dict["position"] == 0
def index(self):
cache_key = self._home_cache_key()
etag_cache(cache_key)
query = query_for(model.Package)
query.run(query='*:*',
facet_by=g.facets,
limit=0,
offset=0,
username=c.user)
c.facets = query.facets
c.fields = []
c.package_count = query.count
c.latest_packages = current_package_list_with_resources(
{
'model': model,
'user': c.user,
'limit': 5
}, {})
return render('home/index.html',
cache_key=cache_key,
cache_expire=cache_expires)
def setup_class(cls):
cls.search = SearchIndexCommand('search-index')
cls.index = index_for(model.Package)
cls.query = query_for(model.Package)
CreateTestData.create()
def test_bad_search_fields(self):
result = search.query_for(model.Tag).run(fields={'tags': u'asdf'})
assert result['count'] == 0, result
def test_search_is_case_insensitive(self):
result = search.query_for(model.Tag).run(query=u'flexible')
assert u'Flexible \u30a1' in result['results']
def test_search_with_unicode_in_search_query(self):
"""
Asserts that search works with a unicode character above \u00ff.
"""
result = search.query_for(model.Tag).run(query=u' \u30a1')
assert u'Flexible \u30a1' in result['results']
def test_search_with_capital_letter_in_search_query(self):
"""
Asserts that search works with a capital letter in the search query.
"""
result = search.query_for(model.Tag).run(query=u'Flexible')
assert u'Flexible \u30a1' in result['results']
def test_search_with_capital_letter_in_tagname(self):
"""
Asserts that it doesn't matter if the tagname has capital letters in it.
"""
result = search.query_for(model.Tag).run(query=u'lexible')
assert u'Flexible \u30a1' in result['results']
def test_bad_search_query(self):
result = search.query_for(model.Tag).run(query=u'asdf')
assert result['count'] == 0, result
return self._finish_ok([rev.id for rev in revs])
elif register in ['dataset', 'package', 'resource']:
try:
params = MultiDict(self._get_search_params(request.params))
except ValueError, e:
return self._finish_bad_request(
_('Could not read parameters: %r' % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ['dataset', 'package'] and not params.get('fl'):
params['fl'] = 'id' if ver == 2 else 'name'
try:
if register == 'resource':
query = search.query_for(model.Resource)
# resource search still uses ckan query parser
options = search.QueryOptions()
for k, v in params.items():
if (k in search.DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
if not include_private:
data_dict['fq'] = '+capacity:public ' + data_dict['fq']
if include_drafts:
data_dict['fq'] += ' +state:(active OR draft)'
# Pop these ones as Solr does not need them
extras = data_dict.pop('extras', None)
# enforce permission filter based on user
if context.get('ignore_auth') or (user and authz.is_sysadmin(user)):
labels = None
else:
labels = lib_plugins.get_permission_labels(
).get_user_dataset_labels(context['auth_user_obj'])
query = search.query_for(model.Package)
query.run(data_dict, permission_labels=labels)
# Add them back so extensions can use them on after_search
data_dict['extras'] = extras
if result_fl:
for package in query.results:
if package.get('extras'):
package.update(package['extras'] )
package.pop('extras')
results.append(package)
else:
for package in query.results:
# get the package object
package_dict = package.get(data_source)
def test_good_search_queries(self):
result = search.query_for(model.Tag).run(query=[u'ru', u's'])
assert result['count'] == 1, result
assert 'russian' in result['results'], result
