def setup_class(cls):
if not tests.is_search_supported():
raise tests.SkipTest("Search not supported")
tests.setup_test_search_index()
ctd.CreateTestData.create()
model.repo.new_revision()
cls.vocab = model.Vocabulary(TEST_VOCAB_NAME)
model.Session.add(cls.vocab)
model.Session.commit()
vocab_tag_1 = model.Tag('tag1', cls.vocab.id)
vocab_tag_2 = model.Tag('tag2', cls.vocab.id)
model.Session.add(vocab_tag_1)
model.Session.add(vocab_tag_2)
pkg = model.Package.get('warandpeace')
pkg_tag1 = model.PackageTag(pkg, vocab_tag_1)
pkg_tag2 = model.PackageTag(pkg, vocab_tag_2)
model.Session.add(pkg_tag1)
model.Session.add(pkg_tag2)
model.Session.commit()
model.Session.remove()
def setup_class(cls):
if not tests.is_search_supported():
raise tests.SkipTest("Search not supported")
tests.setup_test_search_index()
ctd.CreateTestData.create()
model.repo.new_revision()
cls.vocab = model.Vocabulary(TEST_VOCAB_NAME)
model.Session.add(cls.vocab)
model.Session.commit()
vocab_tag_1 = model.Tag('tag1', cls.vocab.id)
vocab_tag_2 = model.Tag('tag2', cls.vocab.id)
model.Session.add(vocab_tag_1)
model.Session.add(vocab_tag_2)
pkg = model.Package.get('warandpeace')
pkg_tag1 = model.PackageTag(pkg, vocab_tag_1)
pkg_tag2 = model.PackageTag(pkg, vocab_tag_2)
model.Session.add(pkg_tag1)
model.Session.add(pkg_tag2)
model.Session.commit()
model.Session.remove()
def setup_class(self):
if not is_search_supported():
raise SkipTest("Search not supported")
self.ab = 'http://site.com/a/b.txt'
self.cd = 'http://site.com/c/d.txt'
self.ef = 'http://site.com/e/f.txt'
self.pkgs = [
{
'name':
'pkg1',
'resources': [
{
'url': self.ab,
'description': 'This is site ab.',
'format': 'Excel spreadsheet',
'hash': 'xyz-123',
'alt_url': 'alt_1',
'extras': {
'size_extra': '100'
},
},
{
'url': self.cd,
'description': 'This is site cd.',
'format': 'Office spreadsheet',
'hash': 'qwe-456',
'alt_url': 'alt_2',
'extras': {
'size_extra': '200'
},
},
]
},
{
'name':
'pkg2',
'resources': [
{
'url': self.cd,
'alt_url': 'alt_1',
'description': 'This is site cd.'
},
{
'url': self.ef,
'description': 'This is site ef.'
},
{
'url': self.ef,
'description': 'This is site gh.'
},
{
'url': self.ef,
'description': 'This is site ij.'
},
]
},
]
CreateTestData.create_arbitrary(self.pkgs)
def test_solr_url_exists(self):
if not is_search_supported():
from nose import SkipTest
raise SkipTest("Search not supported")
conn = search.make_connection()
try:
# solr.SolrConnection.query will throw a socket.error if it
# can't connect to the SOLR instance
q = conn.search(q="*:*", rows=1)
except pysolr.SolrError, e:
if not config.get('solr_url'):
raise AssertionError("Config option 'solr_url' needs to be defined in this CKAN's development.ini. Default of %s didn't work: %s" % (search.DEFAULT_SOLR_URL, e))
else:
raise AssertionError('SOLR connection problem. Connection defined in development.ini as: solr_url=%s Error: %s' % (config['solr_url'], e))
def test_solr_url_exists(self):
if not is_search_supported():
from nose import SkipTest
raise SkipTest("Search not supported")
conn = search.make_connection()
try:
# solr.SolrConnection.query will throw a socket.error if it
# can't connect to the SOLR instance
q = conn.search(q="*:*", rows=1)
except pysolr.SolrError, e:
if not config.get('solr_url'):
raise AssertionError("Config option 'solr_url' needs to be defined in this CKAN's development.ini. Default of %s didn't work: %s" % (search.DEFAULT_SOLR_URL, e))
else:
raise AssertionError('SOLR connection problem. Connection defined in development.ini as: solr_url=%s Error: %s' % (config['solr_url'], e))
def setup_class(self):
if not is_search_supported():
raise SkipTest("Search not supported")
self.ab = 'http://site.com/a/b.txt'
self.cd = 'http://site.com/c/d.txt'
self.ef = 'http://site.com/e/f.txt'
self.pkgs = [
{'name':'pkg1',
'resources':[
{'url':self.ab,
'description':'This is site ab.',
'format':'Excel spreadsheet',
'hash':'xyz-123',
'alt_url': 'alt_1',
'extras':{'size_extra': '100'},
},
{'url':self.cd,
'description':'This is site cd.',
'format':'Office spreadsheet',
'hash':'qwe-456',
'alt_url':'alt_2',
'extras':{'size_extra':'200'},
},
]
},
{'name':'pkg2',
'resources':[
{'url':self.cd,
'alt_url': 'alt_1',
'description':'This is site cd.'},
{'url':self.ef,
'description':'This is site ef.'},
{'url':self.ef,
'description':'This is site gh.'},
{'url':self.ef,
'description':'This is site ij.'},
]
},
]
CreateTestData.create_arbitrary(self.pkgs)
def initial_data(self, clean_db, clean_index):
if not tests.is_search_supported():
raise tests.SkipTest("Search not supported")
ctd.CreateTestData.create()
self.vocab = model.Vocabulary(TEST_VOCAB_NAME)
model.Session.add(self.vocab)
model.Session.commit()
vocab_tag_1 = model.Tag("tag1", self.vocab.id)
vocab_tag_2 = model.Tag("tag2", self.vocab.id)
model.Session.add(vocab_tag_1)
model.Session.add(vocab_tag_2)
pkg = model.Package.get("warandpeace")
pkg_tag1 = model.PackageTag(pkg, vocab_tag_1)
pkg_tag2 = model.PackageTag(pkg, vocab_tag_2)
model.Session.add(pkg_tag1)
model.Session.add(pkg_tag2)
model.Session.commit()
model.Session.remove()
def setup_class(self):
if not is_search_supported():
raise SkipTest("Search not supported")
CreateTestData.create()
def setup_class(self):
if not is_search_supported():
raise SkipTest("Search not supported")
CreateTestData.create()
class TestTagSearch(object):
@pytest.fixture(autouse=True)
@pytest.mark.skipif(not is_search_supported(),
reason="Search not supported")
def initial_data(self, clean_db, clean_index):
CreateTestData.create()
def test_good_search_query(self):
result = search.query_for(model.Tag).run(query=u"ru")
assert result["count"] == 1, result
assert "russian" in result["results"], result
result = search.query_for(model.Tag).run(query=u"s")
assert result["count"] == 2, result
assert "russian" in result["results"], result
assert "tolstoy" in result["results"], result
def test_good_search_queries(self):
result = search.query_for(model.Tag).run(query=[u"ru", u"s"])
assert result["count"] == 1, result
assert "russian" in result["results"], result
def test_bad_search_query(self):
result = search.query_for(model.Tag).run(query=u"asdf")
assert result["count"] == 0, result
def test_search_with_capital_letter_in_tagname(self):
"""
Asserts that it doesn't matter if the tagname has capital letters in it.
"""
result = search.query_for(model.Tag).run(query=u"lexible")
assert u"Flexible \u30a1" in result["results"]
def test_search_with_capital_letter_in_search_query(self):
"""
Asserts that search works with a capital letter in the search query.
"""
result = search.query_for(model.Tag).run(query=u"Flexible")
assert u"Flexible \u30a1" in result["results"]
def test_search_with_unicode_in_search_query(self):
"""
Asserts that search works with a unicode character above \u00ff.
"""
result = search.query_for(model.Tag).run(query=u" \u30a1")
assert u"Flexible \u30a1" in result["results"]
def test_search_is_case_insensitive(self):
result = search.query_for(model.Tag).run(query=u"flexible")
assert u"Flexible \u30a1" in result["results"]
def test_good_search_fields(self):
result = search.query_for(model.Tag).run(fields={"tags": u"ru"})
assert result["count"] == 1, result
assert "russian" in result["results"], result
result = search.query_for(model.Tag).run(fields={"tags": u"s"})
assert result["count"] == 2, result
assert "russian" in result["results"], result
assert "tolstoy" in result["results"], result
def test_bad_search_fields(self):
result = search.query_for(model.Tag).run(fields={"tags": u"asdf"})
assert result["count"] == 0, result
class TestSearch(object):
@pytest.fixture(autouse=True)
@pytest.mark.skipif(not is_search_supported(),
reason="Search not supported")
def initial_data(self, clean_db):
self.ab = "http://site.com/a/b.txt"
self.cd = "http://site.com/c/d.txt"
self.ef = "http://site.com/e/f.txt"
self.pkgs = [
{
"name":
"pkg1",
"resources": [
{
"url": self.ab,
"description": "This is site ab.",
"format": "Excel spreadsheet",
"hash": "xyz-123",
"alt_url": "alt_1",
"extras": {
"size_extra": "100"
},
},
{
"url": self.cd,
"description": "This is site cd.",
"format": "Office spreadsheet",
"hash": "qwe-456",
"alt_url": "alt_2",
"extras": {
"size_extra": "200"
},
},
],
},
{
"name":
"pkg2",
"resources": [
{
"url": self.cd,
"alt_url": "alt_1",
"description": "This is site cd.",
},
{
"url": self.ef,
"description": "This is site ef."
},
{
"url": self.ef,
"description": "This is site gh."
},
{
"url": self.ef,
"description": "This is site ij."
},
],
},
]
CreateTestData.create_arbitrary(self.pkgs)
def res_search(self,
query="",
fields={},
terms=[],
options=search.QueryOptions()):
result = search.query_for(model.Resource).run(query=query,
fields=fields,
terms=terms,
options=options)
resources = [
model.Session.query(model.Resource).get(resource_id)
for resource_id in result["results"]
]
urls = set([resource.url for resource in resources])
return urls
def test_01_search_url(self):
fields = {"url": "site.com"}
result = search.query_for(model.Resource).run(fields=fields)
assert result["count"] == 6, result
resources = [
model.Session.query(model.Resource).get(resource_id)
for resource_id in result["results"]
]
urls = set([resource.url for resource in resources])
assert set([self.ab, self.cd, self.ef]) == urls, urls
def test_02_search_url_2(self):
urls = self.res_search(fields={"url": "a/b"})
assert set([self.ab]) == urls, urls
def test_03_search_url_multiple_words(self):
fields = {"url": "e f"}
urls = self.res_search(fields=fields)
assert {self.ef} == urls
def test_04_search_url_none(self):
urls = self.res_search(fields={"url": "nothing"})
assert set() == urls, urls
def test_05_search_description(self):
urls = self.res_search(fields={"description": "cd"})
assert set([self.cd]) == urls, urls
def test_06_search_format(self):
urls = self.res_search(fields={"format": "excel"})
assert set([self.ab]) == urls, urls
def test_07_search_format_2(self):
urls = self.res_search(fields={"format": "sheet"})
assert set([self.ab, self.cd]) == urls, urls
def test_08_search_hash_complete(self):
urls = self.res_search(fields={"hash": "xyz-123"})
assert set([self.ab]) == urls, urls
def test_09_search_hash_partial(self):
urls = self.res_search(fields={"hash": "xyz"})
assert set([self.ab]) == urls, urls
def test_10_search_hash_partial_but_not_initial(self):
urls = self.res_search(fields={"hash": "123"})
assert set() == urls, urls
def test_11_search_several_fields(self):
urls = self.res_search(fields={"description": "ab", "format": "sheet"})
assert set([self.ab]) == urls, urls
def test_12_search_all_fields(self):
fields = {"url": "a/b"}
options = search.QueryOptions(all_fields=True)
result = search.query_for(model.Resource).run(fields=fields,
options=options)
assert result["count"] == 1, result
res_dict = result["results"][0]
assert isinstance(res_dict, dict)
res_keys = set(res_dict.keys())
expected_res_keys = set(model.Resource.get_columns())
expected_res_keys.update(
["id", "package_id", "position", "size_extra"])
assert res_keys == expected_res_keys
pkg1 = model.Package.by_name(u"pkg1")
ab = pkg1.resources[0]
assert res_dict["id"] == ab.id
assert res_dict["package_id"] == pkg1.id
assert res_dict["url"] == ab.url
assert res_dict["description"] == ab.description
assert res_dict["format"] == ab.format
assert res_dict["hash"] == ab.hash
assert res_dict["position"] == 0
def test_13_pagination(self):
# large search
options = search.QueryOptions(order_by="id")
fields = {"url": "site"}
all_results = search.query_for(model.Resource).run(fields=fields,
options=options)
all_resources = all_results["results"]
all_resource_count = all_results["count"]
assert all_resource_count >= 6, all_results
# limit
options = search.QueryOptions(order_by="id")
options.limit = 2
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result["results"]
count = result["count"]
assert len(resources) == 2, resources
assert count == all_resource_count, (count, all_resource_count)
assert resources == all_resources[:2], "%r, %r" % (
resources,
all_resources,
)
# offset
options = search.QueryOptions(order_by="id")
options.limit = 2
options.offset = 2
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result["results"]
assert len(resources) == 2, resources
assert resources == all_resources[2:4]
# larger offset
options = search.QueryOptions(order_by="id")
options.limit = 2
options.offset = 4
result = search.query_for(model.Resource).run(fields=fields,
options=options)
resources = result["results"]
assert len(resources) == 2, resources
assert resources == all_resources[4:6]
def test_14_extra_info(self):
fields = {"alt_url": "alt_1"}
result = search.query_for(model.Resource).run(fields=fields)
assert result["count"] == 2, result
fields = {"alt_url": "alt_2"}
result = search.query_for(model.Resource).run(fields=fields)
assert result["count"] == 1, result
# Document that resource extras not in ckan.extra_resource_fields
# can't be searched
fields = {"size_extra": "100"}
with pytest.raises(search.SearchError):
search.query_for(model.Resource).run(fields=fields)
