def _do_search(self, q, wanted_results): options = QueryOptions() options.order_by = 'rank' result = self.backend.query_for(model.Package).run(query=q, options=options) results = result['results'] err = 'Wanted %r, got %r' % (wanted_results, results) assert wanted_results[0] == results[0], err assert wanted_results[1] == results[1], err
def _do_search(self, q, expected_pkgs, count=None): options = QueryOptions() options.order_by = 'rank' result = self.backend.query_for(model.Package).run(query=q, options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs] if not (count is None): assert result['count'] == count, result['count'] for expected_pkg in expected_pkgs: assert expected_pkg in fields, expected_pkg
def _filtered_search(self, value, expected_pkgs, count=None): options = QueryOptions() options.order_by = 'rank' result = self.backend.query_for(model.Package).run(fields={'geographic_coverage':value}, options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs] if not (count is None): assert result['count'] == count, result['count'] for expected_pkg in expected_pkgs: assert expected_pkg in fields, expected_pkg
def _check_search_results(self, terms, expected_count, expected_packages=[], only_open=False, only_downloadable=False): options = QueryOptions() options.filter_by_openness = only_open options.filter_by_downloadable = only_downloadable result = self.backend.query_for(model.Package).run(query=unicode(terms)) pkgs = result['results'] count = result['count'] assert count == expected_count, (count, expected_count) for expected_pkg in expected_packages: assert expected_pkg in pkgs, '%s : %s' % (expected_pkg, result)
def _filtered_search(self, value, expected_pkgs, count=None): options = QueryOptions() options.order_by = 'rank' result = self.backend.query_for(model.Package).run( fields={'geographic_coverage': value}, options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs] if not (count is None): assert result['count'] == count, result['count'] for expected_pkg in expected_pkgs: assert expected_pkg in fields, expected_pkg
def test_pagination(self): # large search all_results = self.backend.query_for(model.Package).run(query=self.q_all) all_pkgs = all_results['results'] all_pkg_count = all_results['count'] # limit options = QueryOptions() options.limit = 2 result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] count = result['count'] assert len(pkgs) == 2, pkgs assert count == all_pkg_count assert pkgs == all_pkgs[:2] # offset options = QueryOptions() options.limit = 2 options.offset = 2 result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] assert len(pkgs) == 2, pkgs assert pkgs == all_pkgs[2:4] # larger offset options = QueryOptions() options.limit = 2 options.offset = 4 result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] assert len(pkgs) == 2, pkgs assert pkgs == all_pkgs[4:6]
def _check_search_results(self, terms, expected_count, expected_packages=[], only_open=False, only_downloadable=False): options = QueryOptions() options.filter_by_openness = only_open options.filter_by_downloadable = only_downloadable result = self.backend.query_for( model.Package).run(query=unicode(terms)) pkgs = result['results'] count = result['count'] assert count == expected_count, (count, expected_count) for expected_pkg in expected_packages: assert expected_pkg in pkgs, '%s : %s' % (expected_pkg, result)
def test_order_by(self): # large search all_results = self.backend.query_for(model.Package).run(query=self.q_all) all_pkgs = all_results['results'] all_pkg_count = all_results['count'] # rank options = QueryOptions() options.order_by = 'rank' result = self.backend.query_for(model.Package).run(query='penguin', options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs] assert fields[0] == 'usa-courts-gov', fields # has penguin three times assert pkgs == all_pkgs, pkgs #default ordering # name options = QueryOptions() options.order_by = 'name' result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs] sorted_fields = fields; sorted_fields.sort() assert fields == sorted_fields, repr(fields) + repr(sorted_fields) # title options = QueryOptions() options.order_by = 'title' result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).title for pkg_name in pkgs] sorted_fields = fields; sorted_fields.sort() assert fields == sorted_fields, repr(fields) + repr(sorted_fields) # notes options = QueryOptions() options.order_by = 'notes' result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).notes for pkg_name in pkgs] sorted_fields = fields; sorted_fields.sort() assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
def test_13_pagination(self): # large search options = QueryOptions(order_by='hash') fields = {'url':'site'} all_results = self.backend.query_for(model.PackageResource).run(fields=fields, options=options) all_resources = all_results['results'] all_resource_count = all_results['count'] assert all_resource_count >= 6, all_results # limit options = QueryOptions(order_by='hash') options.limit = 2 result = self.backend.query_for(model.PackageResource).run(fields=fields, options=options) resources = result['results'] count = result['count'] assert len(resources) == 2, resources assert count == all_resource_count assert resources == all_resources[:2], '%r, %r' % (resources, all_resources) # offset options = QueryOptions(order_by='hash') options.limit = 2 options.offset = 2 result = self.backend.query_for(model.PackageResource).run(fields=fields, options=options) resources = result['results'] assert len(resources) == 2, resources assert resources == all_resources[2:4] # larger offset options = QueryOptions(order_by='hash') options.limit = 2 options.offset = 4 result = self.backend.query_for(model.PackageResource).run(fields=fields, options=options) resources = result['results'] assert len(resources) == 2, resources assert resources == all_resources[4:6]
def test_12_search_all_fields(self): fields = {'url':'a/b'} options = QueryOptions(all_fields=True) result = self.backend.query_for(model.Resource).run(fields=fields, options=options) assert result['count'] == 1, result res_dict = result['results'][0] assert isinstance(res_dict, dict) res_keys = set(res_dict.keys()) expected_res_keys = set(model.Resource.get_columns()) expected_res_keys.update(['id', 'resource_group_id', 'package_id', 'position', 'size']) assert_equal(res_keys, expected_res_keys) pkg1 = model.Package.by_name(u'pkg1') ab = pkg1.resources[0] assert res_dict['id'] == ab.id assert res_dict['package_id'] == pkg1.id assert res_dict['url'] == ab.url assert res_dict['description'] == ab.description assert res_dict['format'] == ab.format assert res_dict['hash'] == ab.hash assert res_dict['position'] == 0
params = MultiDict(self._get_search_params(request.params)) except ValueError, e: return self._finish_bad_request( gettext('Could not read parameters: %r' % e)) # if using API v2, default to returning the package ID if # no field list is specified if register in ['dataset', 'package'] and not params.get('fl'): params['fl'] = 'id' if ver == '2' else 'name' try: if register == 'resource': query = query_for(model.Resource) # resource search still uses ckan query parser options = QueryOptions() for k, v in params.items(): if (k in DEFAULT_OPTIONS.keys()): options[k] = v options.update(params) options.username = c.user options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in DEFAULT_OPTIONS.keys() or \ field in IGNORE_FIELDS: continue values = [value] if isinstance(value, list):
def res_search(self, query='', fields={}, terms=[], options=QueryOptions()): result = self.backend.query_for(model.Resource).run(query=query, fields=fields, terms=terms, options=options) resources = [model.Session.query(model.Resource).get(resource_id) for resource_id in result['results']] urls = set([resource.url for resource in resources]) return urls
try: since_time = model.strptimestamp(since_time_str) except ValueError, inst: return self._finish_bad_request('ValueError: %s' % inst) else: return self._finish_bad_request( gettext("Missing search term ('since_id=UUID' or 'since_time=TIMESTAMP')")) revs = model.Session.query(model.Revision).filter(model.Revision.timestamp>since_time) return self._finish_ok([rev.id for rev in revs]) elif register == 'package' or register == 'resource': try: params = self._get_search_params(request.params) except ValueError, e: return self._finish_bad_request( gettext('Could not read parameters: %r' % e)) options = QueryOptions() for k, v in params.items(): if (k in DEFAULT_OPTIONS.keys()): options[k] = v options.update(params) options.username = c.user options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in DEFAULT_OPTIONS.keys() or \ field in IGNORE_FIELDS: continue values = [value]
return self._finish_bad_request('ValueError: %s' % inst) else: return self._finish_bad_request( gettext( "Missing search term ('since_id=UUID' or 'since_time=TIMESTAMP')" )) revs = model.Session.query( model.Revision).filter(model.Revision.timestamp > since_time) return self._finish_ok([rev.id for rev in revs]) elif register == 'package' or register == 'resource': try: params = self._get_search_params(request.params) except ValueError, e: return self._finish_bad_request( gettext('Could not read parameters: %r' % e)) options = QueryOptions() for k, v in params.items(): if (k in DEFAULT_OPTIONS.keys()): options[k] = v options.update(params) options.username = c.user options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in DEFAULT_OPTIONS.keys() or \ field in IGNORE_FIELDS: continue values = [value]
def test_order_by(self): # large search all_results = self.backend.query_for( model.Package).run(query=self.q_all) all_pkgs = all_results['results'] all_pkg_count = all_results['count'] # rank options = QueryOptions() options.order_by = 'rank' result = self.backend.query_for(model.Package).run(query='penguin', options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs] assert fields[0] == 'usa-courts-gov', fields # has penguin three times assert pkgs == all_pkgs, pkgs #default ordering # name options = QueryOptions() options.order_by = 'name' result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).name for pkg_name in pkgs] sorted_fields = fields sorted_fields.sort() assert fields == sorted_fields, repr(fields) + repr(sorted_fields) # title options = QueryOptions() options.order_by = 'title' result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).title for pkg_name in pkgs] sorted_fields = fields sorted_fields.sort() assert fields == sorted_fields, repr(fields) + repr(sorted_fields) # notes options = QueryOptions() options.order_by = 'notes' result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] fields = [model.Package.by_name(pkg_name).notes for pkg_name in pkgs] sorted_fields = fields sorted_fields.sort() assert fields == sorted_fields, repr(fields) + repr(sorted_fields)
def test_pagination(self): # large search all_results = self.backend.query_for( model.Package).run(query=self.q_all) all_pkgs = all_results['results'] all_pkg_count = all_results['count'] # limit options = QueryOptions() options.limit = 2 result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] count = result['count'] assert len(pkgs) == 2, pkgs assert count == all_pkg_count assert pkgs == all_pkgs[:2] # offset options = QueryOptions() options.limit = 2 options.offset = 2 result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] assert len(pkgs) == 2, pkgs assert pkgs == all_pkgs[2:4] # larger offset options = QueryOptions() options.limit = 2 options.offset = 4 result = self.backend.query_for(model.Package).run(query=self.q_all, options=options) pkgs = result['results'] assert len(pkgs) == 2, pkgs assert pkgs == all_pkgs[4:6]
def test_13_pagination(self): # large search options = QueryOptions(order_by='hash') fields = {'url':'site'} all_results = self.backend.query_for(model.Resource).run(fields=fields, options=options) all_resources = all_results['results'] all_resource_count = all_results['count'] assert all_resource_count >= 6, all_results # limit options = QueryOptions(order_by='hash') options.limit = 2 result = self.backend.query_for(model.Resource).run(fields=fields, options=options) resources = result['results'] count = result['count'] assert len(resources) == 2, resources assert count == all_resource_count assert resources == all_resources[:2], '%r, %r' % (resources, all_resources) # offset options = QueryOptions(order_by='hash') options.limit = 2 options.offset = 2 result = self.backend.query_for(model.Resource).run(fields=fields, options=options) resources = result['results'] assert len(resources) == 2, resources assert resources == all_resources[2:4] # larger offset options = QueryOptions(order_by='hash') options.limit = 2 options.offset = 4 result = self.backend.query_for(model.Resource).run(fields=fields, options=options) resources = result['results'] assert len(resources) == 2, resources assert resources == all_resources[4:6]
elif register == 'package' or register == 'resource': if request.params.has_key('qjson'): if not request.params['qjson']: response.status_int = 400 return gettext('Blank qjson parameter') params = json.loads(request.params['qjson']) elif request.params.values() and request.params.values() != [u''] and request.params.values() != [u'1']: params = request.params else: try: params = self._get_request_data() except ValueError, inst: response.status_int = 400 return gettext(u'Search params: %s') % unicode(inst) options = QueryOptions() for k, v in params.items(): if (k in DEFAULT_OPTIONS.keys()): options[k] = v options.update(params) options.username = self._get_username() options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in DEFAULT_OPTIONS.keys() or \ field in IGNORE_FIELDS: continue values = [value]