def index(self):
try:
# package search
context = {
'model': model,
'session': model.Session,
'user': c.user,
'auth_user_obj': c.userobj
}
data_dict = {
'q': '*:*',
'facet.field': h.facets(),
'rows': 4,
'start': 0,
'sort': 'views_recent desc',
'fq': 'capacity:"public"'
}
query = logic.get_action('package_search')(context, data_dict)
c.search_facets = query['search_facets']
c.package_count = query['count']
c.datasets = query['results']
org_label = h.humanize_entity_type(
u'organization', h.default_group_type(u'organization'),
u'facet label') or _(u'Organizations')
group_label = h.humanize_entity_type(
u'group', h.default_group_type(u'group'),
u'facet label') or _(u'Groups')
c.facet_titles = {
'organization': org_label,
'groups': group_label,
'tags': _('Tags'),
'res_format': _('Formats'),
'license': _('Licenses'),
}
except search.SearchError:
c.package_count = 0
if c.userobj and not c.userobj.email:
url = h.url_for('user.edit')
msg = _('Please <a href="%s">update your profile</a>'
' and add your email address. ') % url + \
_('%s uses your email address'
' if you need to reset your password.') \
% config.get('ckan.site_title')
h.flash_notice(msg, allow_html=True)
return base.render('home/index.html', cache_force=True)
def index() -> str:
u'''display home page'''
try:
context = cast(
Context, {
u'model': model,
u'session': model.Session,
u'user': current_user.name,
u'auth_user_obj': current_user
})
data_dict: dict[str, Any] = {
u'q': u'*:*',
u'facet.field': h.facets(),
u'rows': 4,
u'start': 0,
u'sort': u'view_recent desc',
u'fq': u'capacity:"public"'
}
query = logic.get_action(u'package_search')(context, data_dict)
g.package_count = query['count']
g.datasets = query['results']
org_label = h.humanize_entity_type(
u'organization', h.default_group_type(u'organization'),
u'facet label') or _(u'Organizations')
group_label = h.humanize_entity_type(u'group',
h.default_group_type(u'group'),
u'facet label') or _(u'Groups')
g.facet_titles = {
u'organization': org_label,
u'groups': group_label,
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license': _(u'Licenses'),
}
except search.SearchError:
g.package_count = 0
if current_user.is_authenticated and not current_user.email:
url = h.url_for('user.edit')
msg = _(u'Please <a href="%s">update your profile</a>'
u' and add your email address. ') % url + \
_(u'%s uses your email address'
u' if you need to reset your password.') \
% config.get_value(u'ckan.site_title')
h.flash_notice(msg, allow_html=True)
return base.render(u'home/index.html', extra_vars={})
def _generate_facet_name_to_title_map(self, package_type):
facets = OrderedDict()
default_facet_titles = get_default_facet_titles()
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
return facets
def index(self):
if c.user:
try:
# package search
context = {
'model': model,
'session': model.Session,
'user': c.user,
'auth_user_obj': c.userobj
}
data_dict = {
'q': '*:*',
'facet.field': h.facets(),
'rows': 4,
'start': 0,
'sort': 'views_recent desc',
'fq': 'capacity:"public"'
}
query = logic.get_action('package_search')(context, data_dict)
c.search_facets = query['search_facets']
c.package_count = query['count']
c.datasets = query['results']
c.facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license': _('Licenses'),
}
except search.SearchError:
c.package_count = 0
if c.userobj and not c.userobj.email:
url = h.url_for(controller='user', action='edit')
msg = _('Please <a href="%s">update your profile</a>'
' and add your email address. ') % url + \
_('%s uses your email address'
' if you need to reset your password.') \
% config.get('ckan.site_title')
h.flash_notice(msg, allow_html=True)
querytool_controller \
= 'ckanext.querytool.controllers.querytool:QueryToolController'
h.redirect_to(controller=querytool_controller,
action='querytool_public')
def index():
u'''display home page'''
try:
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'auth_user_obj': g.userobj
}
data_dict = {
u'q': u'*:*',
u'facet.field': h.facets(),
u'rows': 4,
u'start': 0,
u'sort': u'view_recent desc',
u'fq': u'capacity:"public"'
}
query = logic.get_action(u'package_search')(context, data_dict)
g.search_facets = query['search_facets']
g.package_count = query['count']
g.datasets = query['results']
g.facet_titles = {
u'organization': _(u'Organizations'),
u'groups': _(u'Groups'),
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license': _(u'Licenses'),
}
except search.SearchError:
g.package_count = 0
if g.userobj and not g.userobj.email:
url = h.url_for(controller=u'user', action=u'edit')
msg = _(u'Please <a href="%s">update your profile</a>'
u' and add your email address. ') % url + \
_(u'%s uses your email address'
u' if you need to reset your password.') \
% config.get(u'ckan.site_title')
h.flash_notice(msg, allow_html=True)
return base.render(u'home/index.html', extra_vars={})
def index(self):
try:
# package search
context = {'model': model, 'session': model.Session,
'user': c.user, 'auth_user_obj': c.userobj}
data_dict = {
'q': '*:*',
'facet.field': h.facets(),
'rows': 4,
'start': 0,
'sort': 'views_recent desc',
'fq': 'capacity:"public"'
}
query = logic.get_action('package_search')(
context, data_dict)
c.search_facets = query['search_facets']
c.package_count = query['count']
c.datasets = query['results']
c.facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license': _('Licenses'),
}
except search.SearchError:
c.package_count = 0
if c.userobj and not c.userobj.email:
url = h.url_for(controller='user', action='edit')
msg = _('Please <a href="%s">update your profile</a>'
' and add your email address. ') % url + \
_('%s uses your email address'
' if you need to reset your password.') \
% config.get('ckan.site_title')
h.flash_notice(msg, allow_html=True)
return base.render('home/index.html', cache_force=True)
def index():
u'''display home page'''
try:
context = {u'model': model, u'session': model.Session,
u'user': g.user, u'auth_user_obj': g.userobj}
data_dict = {u'q': u'*:*',
u'facet.field': h.facets(),
u'rows': 4,
u'start': 4,
u'sort': u'view_recent desc',
u'fq': u'capacity:"public"'}
query = logic.get_action(u'package_search')(context, data_dict)
g.search_facets = query['search_facets']
g.package_count = query['count']
g.datasets = query['results']
g.facet_titles = {
u'organization': _(u'Organizations'),
u'groups': _(u'Groups'),
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license': _(u'Licenses'),
}
except search.SearchError:
g.package_count = 0
if g.userobj and not g.userobj.email:
url = h.url_for(controller=u'user', action=u'edit')
msg = _(u'Please <a href="%s">update your profile</a>'
u' and add your email address. ') % url + \
_(u'%s uses your email address'
u' if you need to reset your password.') \
% config.get(u'ckan.site_title')
h.flash_notice(msg, allow_html=True)
return base.render(u'home/index.html', extra_vars={})
def _package_search(self, package_ids, params):
from ckan.lib.search import SearchError, SearchQueryError
package_type = 'dataset'
q = ' OR '.join(["id:{}".format(package_id) for package_id in package_ids])
c.query_error = False
page = h.get_page_number(params)
limit = int(config.get('ckan.datasets_per_page', 20))
params_nopage = [(k, v) for k, v in params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package', action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='package', action='search',
alternative_url='record')
c.remove_field = remove_field
sort_by = params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0]
for field in sort_by.split(',')]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {'model': model, 'session': model.Session,
'user': c.user, 'for_view': True,
'auth_user_obj': c.userobj}
search_all_type = config.get(
'ckan.search.show_all_types', 'dataset')
search_all = False
try:
search_all = asbool(search_all_type)
search_all_type = 'dataset'
except ValueError:
search_all = True
if not package_type:
package_type = 'dataset'
if not search_all or package_type != search_all_type:
fq += ' +dataset_type:{type}'.format(type=package_type)
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras,
'include_private': asbool(config.get(
'ckan.search.default_include_private', False)),
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchQueryError as se:
log.info('Dataset search query rejected: %r', se.args)
abort(400, _('Invalid search query: {error_message}')
.format(error_message=str(se)))
except SearchError as se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.search_facets = {}
c.page = h.Page(collection=[])
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
c.search_facets_limits = {}
for facet in c.search_facets.keys():
try:
limit = int(params.get('_%s_limit' % facet,
int(config.get('search.facets.default', 10))))
except ValueError:
abort(400, _('Parameter "{parameter_name}" is not '
'an integer').format(
parameter_name='_%s_limit' % facet))
c.search_facets_limits[facet] = limit
return {
'resource_filter': self.resource_filter,
'dataset_type': str(package_type),
'page': c.page,
'search_facets_limits': c.search_facets_limits,
'search_facets': c.search_facets,
'facet_titles': c.facet_titles,
'fields_grouped': c.fields_grouped,
'translated_fields': c.translated_fields,
'remove_field': c.remove_field,
'sort_by_selected': c.sort_by_selected
}
def _read(id: Optional[str], limit: int, group_type: str) -> dict[str, Any]:
u''' This is common code used by both read and bulk_process'''
extra_vars: dict[str, Any] = {}
context = cast(
Context, {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'schema': _db_to_form_schema(group_type=group_type),
u'for_view': True,
u'extras_as_string': True
})
q = request.args.get(u'q', u'')
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.q = q
# Search within group
if g.group_dict.get(u'is_organization'):
fq = u' owner_org:"%s"' % g.group_dict.get(u'id')
else:
fq = u' groups:"%s"' % g.group_dict.get(u'name')
extra_vars["q"] = q
g.description_formatted = \
h.render_markdown(g.group_dict.get(u'description'))
context['return_query'] = True
page = h.get_page_number(request.args)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.args.items(multi=True)
if k != u'page']
sort_by = request.args.get(u'sort', None)
def search_url(params: Any) -> str:
action = u'bulk_process' if getattr(
g, u'action', u'') == u'bulk_process' else u'read'
url = h.url_for(u'.'.join([group_type, action]), id=id)
params = [(k, v.encode(u'utf-8') if isinstance(v, str) else str(v))
for k, v in params]
return url + u'?' + urlencode(params)
def remove_field(key: str,
value: Optional[str] = None,
replace: Optional[str] = None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(key,
value=value,
replace=replace,
controller=controller,
action=u'read',
extras=dict(id=g.group_dict.get(u'name')))
extra_vars["remove_field"] = remove_field
def pager_url(q: Any = None, page: Optional[int] = None):
params: list[tuple[str, Any]] = list(params_nopage)
params.append((u'page', page))
return search_url(params)
details = _get_search_details()
extra_vars[u'fields'] = details[u'fields']
extra_vars[u'fields_grouped'] = details[u'fields_grouped']
fq += details[u'fq']
search_extras = details[u'search_extras']
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.fields = extra_vars[u'fields']
g.fields_grouped = extra_vars[u'fields_grouped']
facets: "OrderedDict[str, str]" = OrderedDict()
org_label = h.humanize_entity_type(u'organization',
h.default_group_type(u'organization'),
u'facet label') or _(u'Organizations')
group_label = h.humanize_entity_type(u'group',
h.default_group_type(u'group'),
u'facet label') or _(u'Groups')
default_facet_titles = {
u'organization': org_label,
u'groups': group_label,
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license_id': _(u'Licenses')
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
facets = _update_facet_titles(facets, group_type)
extra_vars["facet_titles"] = facets
data_dict: dict[str, Any] = {
u'q': q,
u'fq': fq,
u'include_private': True,
u'facet.field': list(facets.keys()),
u'rows': limit,
u'sort': sort_by,
u'start': (page - 1) * limit,
u'extras': search_extras
}
context_ = cast(
Context, dict((k, v) for (k, v) in context.items() if k != u'schema'))
try:
query = get_action(u'package_search')(context_, data_dict)
except search.SearchError as se:
log.error(u'Group search error: %r', se.args)
extra_vars["query_error"] = True
extra_vars["page"] = h.Page(collection=[])
else:
extra_vars["page"] = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.group_dict['package_count'] = query['count']
extra_vars["search_facets"] = query['search_facets']
extra_vars["search_facets_limits"] = g.search_facets_limits = {}
default_limit: int = config.get_value(u'search.facets.default')
for facet in extra_vars["search_facets"].keys():
limit = int(request.args.get(u'_%s_limit' % facet, default_limit))
g.search_facets_limits[facet] = limit
extra_vars["page"].items = query['results']
extra_vars["sort_by_selected"] = sort_by
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.facet_titles = facets
g.page = extra_vars["page"]
extra_vars["group_type"] = group_type
_setup_template_variables(context, {u'id': id}, group_type=group_type)
return extra_vars
def _add_dataset_search(self, showcase_id, showcase_name):
'''
Search logic for discovering datasets to add to a showcase.
'''
from ckan.lib.search import SearchError
package_type = 'dataset'
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
try:
page = self._get_page_number(request.params)
except AttributeError:
# in CKAN >= 2.5 _get_page_number has been moved
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package', action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='package', action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return self._search_url(params, showcase_name)
c.sort_by = _sort_by
if sort_by is None:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0]
for field in sort_by.split(',')]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return self._search_url(params, showcase_name)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'auth_user_obj': c.userobj}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not tk.asbool(config.get('ckan.search.show_all_types',
'False')):
fq += ' +dataset_type:dataset'
# Only search for packages that aren't already associated with the
# Showcase
associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase(showcase_id)
# flatten resulting list to space separated string
if associated_package_ids:
associated_package_ids_str = \
' OR '.join([id[0] for id in associated_package_ids])
fq += ' !id:({0})'.format(associated_package_ids_str)
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
# for CKAN-Versions that do not provide the facets-method from
# helper-context, import facets from ckan.common
if hasattr(h, 'facets'):
current_facets = h.facets()
else:
from ckan.common import g
current_facets = g.facets
for facet in current_facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
c.page = h.Page(collection=[])
def _add_dataset_search(self, showcase_id, showcase_name):
'''
Search logic for discovering datasets to add to a showcase.
'''
from ckan.lib.search import SearchError
package_type = 'dataset'
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
try:
page = self._get_page_number(request.params)
except AttributeError:
# in CKAN >= 2.5 _get_page_number has been moved
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return self._search_url(params, showcase_name)
c.sort_by = _sort_by
if sort_by is None:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return self._search_url(params, showcase_name)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'auth_user_obj': c.userobj
}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not tk.asbool(
config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
# Only search for packages that aren't already associated with the
# Showcase
associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase(
showcase_id)
# flatten resulting list to space separated string
if associated_package_ids:
associated_package_ids_str = \
' OR '.join([id[0] for id in associated_package_ids])
fq += ' !id:({0})'.format(associated_package_ids_str)
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
# for CKAN-Versions that do not provide the facets-method from
# helper-context, import facets from ckan.common
if hasattr(h, 'facets'):
current_facets = h.facets()
else:
from ckan.common import g
current_facets = g.facets
for facet in current_facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.facets = query['facets']
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
c.page = h.Page(collection=[])
def index(self, error=None, offset=0):
for item in p.PluginImplementations(p.IAuthenticator):
item.login()
if 'error' in request.params:
h.flash_error(request.params['error'])
if not c.user:
came_from = request.params.get('came_from')
if not came_from:
came_from = h.url_for(controller='user', action='logged_in')
c.login_handler = h.url_for(
self._get_repoze_handler('login_handler_path'),
came_from=came_from)
if error:
vars = {'error_summary': {'': error}}
else:
vars = {}
h.redirect_to(controller='user', action='login')
# return base.render('user/login.html', extra_vars=vars)
else:
try:
# package search
context = {
'model': model,
'session': model.Session,
'user': c.user,
'auth_user_obj': c.userobj,
'for_view': True
}
data_dict = {
'q': '*:*',
'facet.field': h.facets(),
'rows': 4,
'start': 0,
'sort': 'views_recent desc',
'fq': 'capacity:"public"'
}
query = logic.get_action('package_search')(context, data_dict)
c.search_facets = query['search_facets']
c.package_count = query['count']
c.datasets = query['results']
c.facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license': _('Licenses'),
}
except search.SearchError:
c.package_count = 0
if c.userobj and not c.userobj.email:
url = h.url_for(controller='user', action='edit')
msg = _('Please <a href="%s">update your profile</a>'
' and add your email address. ') % url + \
_('%s uses your email address'
' if you need to reset your password.') \
% config.get('ckan.site_title')
h.flash_notice(msg, allow_html=True)
q = request.params.get('q', u'')
filter_type = request.params.get('type', u'')
filter_id = request.params.get('name', u'')
c.followee_list = logic.get_action('followee_list')(
context, {
'id': c.userobj.id,
'q': q
})
# c.dashboard_activity_stream_context = self._get_dashboard_context(
# filter_type, filter_id, q)
c.dashboard_activity_stream = h.dashboard_activity_stream(
c.userobj.id, filter_type, filter_id, offset)
return base.render('home/index.html', cache_force=True)
def _add_dataset_search(showcase_id, showcase_name):
'''
Search logic for discovering datasets to add to a showcase.
'''
from ckan.lib.search import SearchError
package_type = 'dataset'
# unicode format (decoded from utf8)
q = tk.g.q = tk.request.params.get('q', u'')
tk.g.query_error = False
page = h.get_page_number(tk.request.params)
limit = int(tk.config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in tk.request.params.items()
if k != 'page']
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='dataset' if
tk.check_ckan_version('2.9') else 'package',
action='search')
tk.g.remove_field = remove_field
sort_by = tk.request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return _search_url(params, showcase_name)
tk.g.sort_by = _sort_by
if sort_by is None:
tk.g.sort_by_fields = []
else:
tk.g.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return _search_url(params, showcase_name)
tk.g.search_url_params = urlencode(_encode_params(params_nopage))
try:
tk.g.fields = []
# tk.g.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
tk.g.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in tk.request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
tk.g.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in tk.g.fields_grouped:
tk.g.fields_grouped[param] = [value]
else:
tk.g.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': tk.g.user or tk.g.author,
'for_view': True,
'auth_user_obj': tk.g.userobj
}
# Unless changed via config options, don't show other dataset
# types any search page. Potential alternatives are do show them
# on the default search page (dataset) or on one other search page
search_all_type = tk.config.get('ckan.search.show_all_types')
search_all = False
try:
# If the "type" is set to True or False, convert to bool
# and we know that no type was specified, so use traditional
# behaviour of applying this only to dataset type
search_all = tk.asbool(search_all_type)
search_all_type = 'dataset'
# Otherwise we treat as a string representing a type
except ValueError:
search_all = True
if not search_all or package_type != search_all_type:
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
# Only search for packages that aren't already associated with the
# Showcase
associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase(
showcase_id)
# flatten resulting list to space separated string
if associated_package_ids:
associated_package_ids_str = \
' OR '.join([id[0] for id in associated_package_ids])
fq += ' !id:({0})'.format(associated_package_ids_str)
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
# for CKAN-Versions that do not provide the facets-method from
# helper-context, import facets from ckan.common
if hasattr(h, 'facets'):
current_facets = h.facets()
else:
from ckan.common import g
current_facets = g.facets
for facet in current_facets:
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
tk.g.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': list(facets.keys()),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = tk.get_action('package_search')(context, data_dict)
tk.g.sort_by_selected = query['sort']
tk.g.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
tk.g.facets = query['facets']
tk.g.search_facets = query['search_facets']
tk.g.page.items = query['results']
except SearchError as se:
log.error('Dataset search error: %r', se.args)
tk.g.query_error = True
tk.g.facets = {}
tk.g.search_facets = {}
tk.g.page = h.Page(collection=[])
tk.g.search_facets_limits = {}
for facet in tk.g.search_facets.keys():
try:
limit = int(
tk.request.params.get(
'_%s_limit' % facet,
int(tk.config.get('search.facets.default', 10))))
except tk.ValueError:
abort(
400,
_("Parameter '{parameter_name}' is not an integer").format(
parameter_name='_%s_limit' % facet))
tk.g.search_facets_limits[facet] = limit
def search(package_type):
extra_vars = {}
try:
context = {
u'model': model,
u'user': g.user,
u'auth_user_obj': g.userobj
}
check_access(u'site_read', context)
except NotAuthorized:
base.abort(403, _(u'Not authorized to see this page'))
# unicode format (decoded from utf8)
extra_vars[u'q'] = q = request.args.get(u'q', u'')
extra_vars['query_error'] = False
page = h.get_page_number(request.args)
limit = int(config.get(u'ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.args.items() if k != u'page']
extra_vars[u'drill_down_url'] = drill_down_url
extra_vars[u'remove_field'] = partial(remove_field, package_type)
sort_by = request.args.get(u'sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != u'sort']
extra_vars[u'sort_by'] = partial(_sort_by, params_nosort, package_type)
if not sort_by:
sort_by_fields = []
else:
sort_by_fields = [field.split()[0] for field in sort_by.split(u',')]
extra_vars[u'sort_by_fields'] = sort_by_fields
pager_url = partial(_pager_url, params_nopage, package_type)
search_url_params = urlencode(_encode_params(params_nopage))
extra_vars[u'search_url_params'] = search_url_params
details = _get_search_details()
extra_vars[u'fields'] = details[u'fields']
extra_vars[u'fields_grouped'] = details[u'fields_grouped']
fq = details[u'fq']
search_extras = details[u'search_extras']
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'for_view': True,
u'auth_user_obj': g.userobj
}
# Unless changed via config options, don't show other dataset
# types any search page. Potential alternatives are do show them
# on the default search page (dataset) or on one other search page
search_all_type = config.get(u'ckan.search.show_all_types', u'dataset')
search_all = False
try:
# If the "type" is set to True or False, convert to bool
# and we know that no type was specified, so use traditional
# behaviour of applying this only to dataset type
search_all = asbool(search_all_type)
search_all_type = u'dataset'
# Otherwise we treat as a string representing a type
except ValueError:
search_all = True
if not search_all or package_type != search_all_type:
# Only show datasets of this particular type
fq += u' +dataset_type:{type}'.format(type=package_type)
facets = OrderedDict()
org_label = h.humanize_entity_type(
u'organization',
h.default_group_type(u'organization'),
u'facet label') or _(u'Organizations')
group_label = h.humanize_entity_type(
u'group',
h.default_group_type(u'group'),
u'facet label') or _(u'Groups')
default_facet_titles = {
u'organization': org_label,
u'groups': group_label,
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license_id': _(u'Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in plugins.PluginImplementations(plugins.IFacets):
facets = plugin.dataset_facets(facets, package_type)
extra_vars[u'facet_titles'] = facets
data_dict = {
u'q': q,
u'fq': fq.strip(),
u'facet.field': list(facets.keys()),
u'rows': limit,
u'start': (page - 1) * limit,
u'sort': sort_by,
u'extras': search_extras,
u'include_private': asbool(
config.get(u'ckan.search.default_include_private', True)
),
}
try:
query = get_action(u'package_search')(context, data_dict)
extra_vars[u'sort_by_selected'] = query[u'sort']
extra_vars[u'page'] = h.Page(
collection=query[u'results'],
page=page,
url=pager_url,
item_count=query[u'count'],
items_per_page=limit
)
extra_vars[u'search_facets'] = query[u'search_facets']
extra_vars[u'page'].items = query[u'results']
except SearchQueryError as se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info(u'Dataset search query rejected: %r', se.args)
base.abort(
400,
_(u'Invalid search query: {error_message}')
.format(error_message=str(se))
)
except SearchError as se:
# May be bad input from the user, but may also be more serious like
# bad code causing a SOLR syntax error, or a problem connecting to
# SOLR
log.error(u'Dataset search error: %r', se.args)
extra_vars[u'query_error'] = True
extra_vars[u'search_facets'] = {}
extra_vars[u'page'] = h.Page(collection=[])
# FIXME: try to avoid using global variables
g.search_facets_limits = {}
for facet in extra_vars[u'search_facets'].keys():
try:
limit = int(
request.args.get(
u'_%s_limit' % facet,
int(config.get(u'search.facets.default', 10))
)
)
except ValueError:
base.abort(
400,
_(u'Parameter u"{parameter_name}" is not '
u'an integer').format(parameter_name=u'_%s_limit' % facet)
)
g.search_facets_limits[facet] = limit
_setup_template_variables(context, {}, package_type=package_type)
extra_vars[u'dataset_type'] = package_type
# TODO: remove
for key, value in six.iteritems(extra_vars):
setattr(g, key, value)
return base.render(
_get_pkg_template(u'search_template', package_type), extra_vars
)
def _search(self, package_type, pager_url, additional_fq='', additional_facets=None,
default_sort_by=DEFAULT_SORTING, num_of_items=NUM_OF_ITEMS,
ignore_capacity_check=False, use_solr_collapse=False, enable_update_status_facet=False):
from ckan.lib.search import SearchError
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
page = self._page_number()
# Commenting below parts as it doesn't seem to be used
# def drill_down_url(alternative_url=None, **by):
# return h.add_url_param(alternative_url=alternative_url,
# controller='package', action='search',
# new_params=by)
#
# c.drill_down_url = drill_down_url
# self._set_remove_field_function()
req_sort_by = request.params.get('sort', None)
if not req_sort_by and q:
req_sort_by = 'score desc, ' + DEFAULT_SORTING
if req_sort_by:
sort_by = req_sort_by
c.used_default_sort_by = False
else:
sort_by = default_sort_by
c.used_default_sort_by = True
# params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
# The sort_by function seems to not be used anymore
# def _sort_by(fields):
# """
# Sort by the given list of fields.
#
# Each entry in the list is a 2-tuple: (fieldname, sort_order)
#
# eg - [('metadata_modified', 'desc'), ('name', 'asc')]
#
# If fields is empty, then the default ordering is used.
# """
# params = params_nosort[:]
#
# if fields:
# sort_string = ', '.join('%s %s' % f for f in fields)
# params.append(('sort', sort_string))
# return self._search_url(params, package_type)
#
# c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0]
for field in sort_by.split(',')]
self._set_other_links()
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
# limit = g.datasets_per_page
fq = additional_fq
tagged_fq_dict = {}
featured_filters_set = False
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if param == 'fq':
fq += ' %s' % (value,)
elif not param.startswith('ext_'):
c.fields.append((param, value))
# fq += ' {!tag=%s}%s:"%s"' % (param, param, value)
if param not in tagged_fq_dict:
tagged_fq_dict[param] = []
tagged_fq_dict[param].append(u'{}:"{}"'.format(param, value))
self.append_selected_facet_to_group(c.fields_grouped, param, value)
elif param == UPDATE_STATUS_URL_FILTER:
self.append_selected_facet_to_group(c.fields_grouped, param, value)
else:
if param in ['ext_cod', 'ext_subnational', 'ext_quickcharts', 'ext_geodata', 'ext_requestdata',
'ext_hxl', 'ext_showcases', 'ext_archived', 'ext_administrative_divisions']:
featured_filters_set = True
search_extras[param] = value
if c.fields_grouped.get(UPDATE_STATUS_URL_FILTER):
search_extras[UPDATE_STATUS_URL_FILTER] = c.fields_grouped[UPDATE_STATUS_URL_FILTER]
self._set_filters_are_selected_flag()
fq_list = [u'{{!tag={tag}}}{value}'.format(tag=key, category=key, value=' OR '.join(value_list))
for key, value_list in tagged_fq_dict.items()]
# if the search is not filtered by query or facet group datasets
solr_expand = 'false'
if use_solr_collapse and not fq_list and not q and not featured_filters_set:
fq_list = ['{!tag=batch}{!collapse field=batch nullPolicy=expand} ']
solr_expand = 'true'
try:
limit = 1 if self._is_facet_only_request() else int(request.params.get('ext_page_size', num_of_items))
except:
limit = num_of_items
c.ext_page_size = limit
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'auth_user_obj': c.userobj}
if ignore_capacity_check:
context['ignore_capacity_check'] = ignore_capacity_check
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not asbool(config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = get_default_facet_titles()
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
if additional_facets:
facets.update(additional_facets)
c.facet_titles = facets
# TODO Line below to be removed after refactoring
c.tab = 'all'
#adding site_id to facets to facilitate totals counts in case of batch/collapse
facet_keys = ['{!ex=batch}site_id'] + facets.keys()
self._performing_search(q, fq, facet_keys, limit, page, sort_by, search_extras, pager_url, context,
fq_list=fq_list, expand=solr_expand,
enable_update_status_facet=enable_update_status_facet)
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.facets = {}
c.search_facets = {}
c.page = h.Page(collection=[])
def _read(id, limit, group_type):
u''' This is common code used by both read and bulk_process'''
extra_vars = {}
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'schema': _db_to_form_schema(group_type=group_type),
u'for_view': True,
u'extras_as_string': True
}
q = request.params.get(u'q', u'')
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.q = q
# Search within group
if g.group_dict.get(u'is_organization'):
fq = u' owner_org:"%s"' % g.group_dict.get(u'id')
else:
fq = u' groups:"%s"' % g.group_dict.get(u'name')
extra_vars["q"] = q
g.description_formatted = \
h.render_markdown(g.group_dict.get(u'description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items() if k != u'page']
sort_by = request.params.get(u'sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = u'bulk_process' if getattr(
g, u'action', u'') == u'bulk_process' else u'read'
url = h.url_for(u'.'.join([controller, action]), id=id)
params = [(k, v.encode(u'utf-8')
if isinstance(v, string_types) else str(v))
for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(
alternative_url=None,
controller=u'group',
action=u'read',
extras=dict(id=g.group_dict.get(u'name')),
new_params=by)
extra_vars["drill_down_url"] = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(
key,
value=value,
replace=replace,
controller=controller,
action=u'read',
extras=dict(id=g.group_dict.get(u'name')))
extra_vars["remove_field"] = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append((u'page', page))
return search_url(params)
try:
extra_vars["fields"] = fields = []
extra_vars["fields_grouped"] = fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in [u'q', u'page', u'sort'] \
and len(value) and not param.startswith(u'_'):
if not param.startswith(u'ext_'):
fields.append((param, value))
q += u' %s: "%s"' % (param, value)
if param not in fields_grouped:
fields_grouped[param] = [value]
else:
fields_grouped[param].append(value)
else:
search_extras[param] = value
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.fields = fields
g.fields_grouped = fields_grouped
facets = OrderedDict()
default_facet_titles = {
u'organization': _(u'Organizations'),
u'groups': _(u'Groups'),
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license_id': _(u'Licenses')
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
_update_facet_titles(facets, group_type)
extra_vars["facet_titles"] = facets
data_dict = {
u'q': q,
u'fq': fq,
u'include_private': True,
u'facet.field': facets.keys(),
u'rows': limit,
u'sort': sort_by,
u'start': (page - 1) * limit,
u'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items() if k != u'schema')
query = get_action(u'package_search')(context_, data_dict)
extra_vars["page"] = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.group_dict['package_count'] = query['count']
extra_vars["search_facets"] = g.search_facets = query['search_facets']
extra_vars["search_facets_limits"] = g.search_facets_limits = {}
for facet in g.search_facets.keys():
limit = int(
request.params.get(u'_%s_limit' % facet,
config.get(u'search.facets.default', 10)))
g.search_facets_limits[facet] = limit
extra_vars["page"].items = query['results']
extra_vars["sort_by_selected"] = sort_by
except search.SearchError as se:
log.error(u'Group search error: %r', se.args)
extra_vars["query_error"] = True
extra_vars["page"] = h.Page(collection=[])
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.facet_titles = facets
g.page = extra_vars["page"]
extra_vars["group_type"] = group_type
_setup_template_variables(context, {u'id': id}, group_type=group_type)
return extra_vars
def search(self):
from ckan.lib.search import SearchError, SearchQueryError
package_type = self._guess_package_type()
extra_vars = {'dataset_type': package_type}
try:
context = {
'model': model,
'user': c.user,
'auth_user_obj': c.userobj
}
check_access('site_read', context)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search',
alternative_url=package_type)
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': c.user,
'for_view': True,
'auth_user_obj': c.userobj
}
# Unless changed via config options, don't show other dataset
# types any search page. Potential alternatives are do show them
# on the default search page (dataset) or on one other search page
search_all_type = config.get('ckan.search.show_all_types',
'dataset')
search_all = False
try:
# If the "type" is set to True or False, convert to bool
# and we know that no type was specified, so use traditional
# behaviour of applying this only to dataset type
search_all = asbool(search_all_type)
search_all_type = 'dataset'
# Otherwise we treat as a string representing a type
except ValueError:
search_all = True
if not package_type:
package_type = 'dataset'
if not search_all or package_type != search_all_type:
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
# Date interval search
start_date = None
end_date = None
if request.params.get('_start_date'):
start_date = extract_date(request.params.get('_start_date'))
if start_date:
extra_vars['start_date'] = request.params.get(
'_start_date')
if request.params.get('_end_date'):
end_date = extract_date(request.params.get('_end_date'))
if end_date:
extra_vars['end_date'] = request.params.get('_end_date')
if start_date or end_date:
if not start_date:
start_date = datetime(year=1900, month=1, day=1)
if not end_date:
end_date = datetime(year=2100,
month=1,
day=1,
hour=23,
minute=59,
second=59,
microsecond=999999)
else:
end_date = datetime_to_utc(end_date)
end_date = end_date.replace(hour=23,
minute=59,
second=59,
microsecond=999999)
fq = fq + ' +metadata_modified:[%s TO %s]' % (
datetime_to_utc(start_date).strftime('%Y-%m-%dT%H:%M:%SZ'),
datetime_to_utc(end_date).strftime('%Y-%m-%dT%H:%M:%SZ'))
data_dict = {
'q':
q,
'fq':
fq.strip(),
'facet.field':
facets.keys(),
'rows':
limit,
'start': (page - 1) * limit,
'sort':
sort_by,
'extras':
search_extras,
'include_private':
asbool(config.get('ckan.search.default_include_private',
True)),
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchQueryError as se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info('Dataset search query rejected: %r', se.args)
abort(
400,
_('Invalid search query: {error_message}').format(
error_message=str(se)))
except SearchError as se:
# May be bad input from the user, but may also be more serious like
# bad code causing a SOLR syntax error, or a problem connecting to
# SOLR
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.search_facets = {}
c.page = h.Page(collection=[])
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
c.search_facets_limits = {}
for facet in c.search_facets.keys():
try:
limit = int(
request.params.get(
'_%s_limit' % facet,
int(config.get('search.facets.default', 10))))
except ValueError:
abort(
400,
_('Parameter "{parameter_name}" is not '
'an integer').format(parameter_name='_%s_limit' % facet))
c.search_facets_limits[facet] = limit
self._setup_template_variables(context, {}, package_type=package_type)
extra_vars['dataset_type'] = package_type
return render(self._search_template(package_type),
extra_vars=extra_vars)
def _read(self, id, limit, group_type): # noqa
c.include_children_selected = False
if not c.group_dict.get('is_organization'):
return
''' This is common code used by both read and bulk_process'''
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True, 'extras_as_string': True}
c.description_formatted = \
h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
# c.group_admins is used by CKAN's legacy (Genshi) templates only,
# if we drop support for those then we can delete this line.
c.group_admins = authz.get_group_or_org_admin_ids(c.group.id)
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
params = [(k, v.encode('utf-8') if isinstance(v, basestring)
else str(v)) for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
q = c.q = request.params.get('q', '')
fq = c.fq = request.params.get('fq', '')
c.fields = []
search_extras = {}
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s: "%s"' % (param, value)
else:
search_extras[param] = value
user_member_of_orgs = [org['id'] for org
in h.organizations_available('read')]
if (c.group and c.group.id in user_member_of_orgs):
context['ignore_capacity_check'] = True
else:
fq += ' capacity:"public"'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
self._update_facet_titles(facets, group_type)
if 'capacity' in facets and (group_type != 'organization' or
not user_member_of_orgs):
del facets['capacity']
c.facet_titles = facets
# filter by organization with fq (filter query)
c.include_children_selected = True
children = _children_name_list(
hierarchy_helpers.group_tree_section(
c.group_dict.get('id'),
include_parents=False,
include_siblings=False
).get('children', [])
)
if not children:
fq += ' organization:"%s"' % c.group_dict.get('name')
else:
fq += ' organization:("%s"' % c.group_dict.get('name')
for name in children:
if name:
fq += ' OR "%s"' % name
fq += ")"
data_dict = {
'q': q,
'fq': fq,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items()
if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.group_dict['package_count'] = query['count']
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.search_facets.keys():
limit = int(request.params.get('_%s_limit' % facet,
config.get(
'search.facets.default',
10)))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.search_facets = {}
c.page = h.Page(collection=[])
def search(self):
from ckan.lib.search import SearchError, SearchQueryError
package_type = self._guess_package_type()
try:
context = {
'model': model,
'user': c.user,
'auth_user_obj': c.userobj
}
check_access('site_read', context)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
org = request.params.get('organization', None)
c.query_error = False
page = h.get_page_number(request.params)
try:
limit = int(config.get('ckan.datasets_per_page', 20))
except:
limit = 20
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search',
alternative_url=package_type)
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': c.user,
'for_view': True,
'auth_user_obj': c.userobj
}
# Unless changed via config options, don't show other dataset
# types any search page. Potential alternatives are do show them
# on the default search page (dataset) or on one other search page
search_all_type = config.get('ckan.search.show_all_types',
'dataset')
search_all = False
map_results = None
try:
# If the "type" is set to True or False, convert to bool
# and we know that no type was specified, so use traditional
# behaviour of applying this only to dataset type
search_all = asbool(search_all_type)
search_all_type = 'dataset'
# Otherwise we treat as a string representing a type
except ValueError:
search_all = True
if not package_type:
package_type = 'dataset'
if not search_all or package_type != search_all_type:
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q':
q,
'fq':
fq.strip(),
'facet.field':
facets.keys(),
'rows':
limit,
'start': (page - 1) * limit,
'sort':
sort_by,
'extras':
search_extras,
'include_private':
asbool(config.get('ckan.search.default_include_private',
True)),
}
query = get_action('package_search')(context, data_dict)
# loop the search query and get all the results
# this workaround the 1000 rows solr hard limit
HARD_LIMIT = 1000
conn = get_connection_redis()
pager = 0
# crank up the pager limit high as using points cluster for better performance
PAGER_LIMIT = 10000
data_dict_full_result = {
'q':
q,
'fq':
fq.strip(),
'facet.field':
facets.keys(),
'rows':
HARD_LIMIT,
'start':
0,
'sort':
sort_by,
'extras':
search_extras,
'include_private':
asbool(config.get('ckan.search.default_include_private',
True)),
}
if not org:
# if no q, it is an init load or direct visit on / dataset
log.info('### Not org ###')
if not q:
log.info('### Not q ###')
if not conn.exists('redis_full_results'):
log.info('### generating full results ###')
# get full results and add to redis when there is not full results in redis
full_results = self.get_full_results(
context, data_dict_full_result, pager, PAGER_LIMIT,
q, fq, facets, HARD_LIMIT, sort_by, search_extras)
map_results = self.get_map_result(full_results)
# adding to redis
log.info('adding full results to redis')
conn.set('redis_full_results', json.dumps(map_results))
# log.info(c.full_results)
else:
log.info('### using cached full results ###')
map_results = json.loads(
conn.get('redis_full_results'))
# log.info(c.full_results)
else:
log.info('### With q ###')
full_results = self.get_full_results(
context, data_dict_full_result, pager, PAGER_LIMIT, q,
fq, facets, HARD_LIMIT, sort_by, search_extras)
map_results = self.get_map_result(full_results)
else:
log.info('### With org ###')
if not q:
log.info('### Not q ###')
if not conn.exists('redis_full_results_%s' % org):
log.info('### generating %s results ###' % org)
# get full results and add to redis when there is not full results in redis
full_results = self.get_full_results(
context, data_dict_full_result, pager, PAGER_LIMIT,
q, fq, facets, HARD_LIMIT, sort_by, search_extras)
map_results = self.get_map_result(full_results)
# adding to redis
log.info('adding %s results to redis' % org)
conn.set('redis_full_results_%s' % org,
json.dumps(map_results))
# log.info(c.full_results)
else:
log.info('### using cached %s results ###' % org)
map_results = json.loads(
conn.get('redis_full_results_%s' % org))
# log.info(c.full_results)
else:
log.info('### With q ###')
full_results = self.get_full_results(
context, data_dict_full_result, pager, PAGER_LIMIT, q,
fq, facets, HARD_LIMIT, sort_by, search_extras)
map_results = self.get_map_result(full_results)
# log.info(c.full_results)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchQueryError as se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info('Dataset search query rejected: %r', se.args)
abort(
400,
_('Invalid search query: {error_message}').format(
error_message=str(se)))
except SearchError as se:
# May be bad input from the user, but may also be more serious like
# bad code causing a SOLR syntax error, or a problem connecting to
# SOLR
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.search_facets = {}
c.page = h.Page(collection=[])
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
c.search_facets_limits = {}
for facet in c.search_facets.keys():
try:
limit = int(
request.params.get(
'_%s_limit' % facet,
int(config.get('search.facets.default', 10))))
except ValueError:
abort(
400,
_('Parameter "{parameter_name}" is not '
'an integer').format(parameter_name='_%s_limit' % facet))
c.search_facets_limits[facet] = limit
self._setup_template_variables(context, {}, package_type=package_type)
return render(self._search_template(package_type),
extra_vars={
'dataset_type': package_type,
'map_results': map_results
})
def _read(self, id, limit, group_type):
''' This is common code used by both read and bulk_process'''
context = {'model': model, 'session': model.Session,
'user': c.user,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True, 'extras_as_string': True}
q = c.q = request.params.get('q', '')
# Search within group
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
else:
q += ' groups:"%s"' % c.group_dict.get('name')
c.description_formatted = \
h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
params = [(k, v.encode('utf-8') if isinstance(v, basestring)
else str(v)) for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(key, value=value, replace=replace,
controller=controller, action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
def collection_information(collection_id=None):
from ckanext.opensearch import config
collections = config.load_settings("collections_list")
collection_items = collections.items()
for collection in collection_items:
if collection[0] == collection_id:
return dict(collection[1])
group_extras = c.group_dict.get('extras')
group_collections = []
for extra in group_extras:
if extra['key'] == 'collections':
col_value = extra['value'].split(", ")
for a in col_value:
group_collections.append(a)
c.collections = []
for collection_id in group_collections:
c.collections.append(collection_information(collection_id))
collection_params = ''
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
facets = OrderedDict()
search_parameters = OrderedDict()
default_facet_titles = {
'collection_name': _('Data Collections'),
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
sentinel_facet_titles = {
'FamilyName': _('Family Name'),
'ProductType': _('Product Type'),
'OrbitDirection': _('Orbit Direction'),
'Swath': _('Acquisition Mode'),
'TransmitterReceiverPolarisation': _('Polarisation'),
}
default_search_titles = {
'timerange_start': _('Timerange Start'),
'timerange_end': _('Timerange End'),
}
collection_names = ['Sentinel-1 Level-1 (SLC)', 'Sentinel-1 Level-1 (GRD)', 'Sentinel-1 Level-2 (OCN)',
'Sentinel-2 Level-1C', 'Sentinel-2 Level-2A', 'Sentinel-3 SRAL Level-1 SRA']
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
for search_param in helpers.search_params():
if search_param in default_search_titles:
search_parameters[search_param] = default_search_titles[search_param]
# Facet titles
self._update_facet_titles(facets, group_type)
if collection_params not in collection_names:
for f in facets:
if f in sentinel_facet_titles:
del facets[f]
c.facet_titles = facets
data_dict = {
'q': q,
'fq': '',
'include_private': True,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items()
if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.group_dict['package_count'] = query['count']
c.group_dict['collection_count'] = len(c.collections)
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.search_facets.keys():
limit = int(request.params.get('_%s_limit' % facet,
config.get('search.facets.default', 10)))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.page = h.Page(collection=[])
def _read(self, id, limit, group_type):
''' This is common code used by both read and bulk_process'''
context = {
'model': model,
'session': model.Session,
'user': c.user,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True,
'extras_as_string': True
}
q = c.q = request.params.get('q', '')
# Search within group
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
else:
q += ' groups:"%s"' % c.group_dict.get('name')
c.description_formatted = \
h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
params = [
(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
for k, v in params
]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group',
action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(key,
value=value,
replace=replace,
controller=controller,
action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
self._update_facet_titles(facets, group_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': '',
'include_private': True,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict(
(k, v) for (k, v) in context.items() if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.group_dict['package_count'] = query['count']
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.search_facets.keys():
limit = int(
request.params.get('_%s_limit' % facet,
config.get('search.facets.default',
10)))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError as se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.page = h.Page(collection=[])
self._setup_template_variables(context, {'id': id},
group_type=group_type)
def search(self):
custom_base.g_analitics()
from ckan.lib.search import SearchError, SearchQueryError
package_type = self._guess_package_type()
try:
context = {'model': model, 'user': c.user,
'auth_user_obj': c.userobj}
check_access('site_read', context)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package', action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key, value=value, replace=replace,
controller='package', action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [field.split()[0]
for field in sort_by.split(',')]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {'model': model, 'session': model.Session,
'user': c.user, 'for_view': True,
'auth_user_obj': c.userobj}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not asbool(
config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras,
'include_private': asbool(config.get(
'ckan.search.default_include_private', True)),
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchQueryError, se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info('Dataset search query rejected: %r', se.args)
abort(400, _('Invalid search query: {error_message}')
.format(error_message=str(se)))
def _read(self, id, limit, group_type):
''' This is common code used by both read and bulk_process'''
context = {
'model': model,
'session': model.Session,
'user': c.user,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True,
'extras_as_string': True
}
q = c.q = request.params.get('q', '')
# Search within group
fq = 'collections:"%s"' % c.group_dict.get('name')
c.description_formatted = \
h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
# ORIGINAL
# controller = lookup_group_controller(group_type)
controller = 'ckanext.collection.controller:CollectionController'
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
params = [
(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
for k, v in params
]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group',
action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='group',
action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
# Set the custom default sort parameter here
# Might need a rewrite when ckan is updated
if q and not sort_by:
sort_by = 'score desc, metadata_modified desc'
elif not sort_by:
sort_by = 'metadata_created desc'
c.fields = []
search_extras = {}
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
else:
search_extras[param] = value
include_private = False
user_member_of_orgs = [
org['id'] for org in h.organizations_available('read')
]
if (c.group and c.group.id in user_member_of_orgs):
include_private = True
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Collections'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.group_facets(facets, group_type, None)
if 'capacity' in facets and (group_type != 'organization'
or not user_member_of_orgs):
del facets['capacity']
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq,
'include_private': include_private,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict(
(k, v) for (k, v) in context.items() if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.group_dict['package_count'] = query['count']
c.facets = query['facets']
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.facets.keys():
limit = int(
request.params.get(
'_%s_limit' % facet,
int(config.get('search.facets.default', 10))))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Collection search error: %r', se.args)
c.query_error = True
c.facets = {}
c.page = h.Page(collection=[])
def search(package_type):
extra_vars = {}
try:
context = {
u'model': model,
u'user': g.user,
u'auth_user_obj': g.userobj
}
check_access(u'site_read', context)
except NotAuthorized:
base.abort(403, _(u'Not authorized to see this page'))
# unicode format (decoded from utf8)
extra_vars[u'q'] = q = request.args.get(u'q', u'')
extra_vars['query_error'] = False
page = h.get_page_number(request.args)
limit = int(config.get(u'ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.args.items() if k != u'page']
extra_vars[u'drill_down_url'] = drill_down_url
extra_vars[u'remove_field'] = partial(remove_field, package_type)
sort_by = request.args.get(u'sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != u'sort']
extra_vars[u'sort_by'] = partial(_sort_by, params_nosort, package_type)
if not sort_by:
sort_by_fields = []
else:
sort_by_fields = [field.split()[0] for field in sort_by.split(u',')]
extra_vars[u'sort_by_fields'] = sort_by_fields
pager_url = partial(_pager_url, params_nopage, package_type)
search_url_params = urlencode(_encode_params(params_nopage))
extra_vars[u'search_url_params'] = search_url_params
try:
# fields_grouped will contain a dict of params containing
# a list of values eg {u'tags':[u'tag1', u'tag2']}
extra_vars[u'fields'] = fields = []
extra_vars[u'fields_grouped'] = fields_grouped = {}
search_extras = {}
fq = u''
for (param, value) in request.args.items():
if param not in [u'q', u'page', u'sort'] \
and len(value) and not param.startswith(u'_'):
if not param.startswith(u'ext_'):
fields.append((param, value))
fq += u' %s:"%s"' % (param, value)
if param not in fields_grouped:
fields_grouped[param] = [value]
else:
fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'for_view': True,
u'auth_user_obj': g.userobj
}
# Unless changed via config options, don't show other dataset
# types any search page. Potential alternatives are do show them
# on the default search page (dataset) or on one other search page
search_all_type = config.get(u'ckan.search.show_all_types', u'dataset')
search_all = False
try:
# If the "type" is set to True or False, convert to bool
# and we know that no type was specified, so use traditional
# behaviour of applying this only to dataset type
search_all = asbool(search_all_type)
search_all_type = u'dataset'
# Otherwise we treat as a string representing a type
except ValueError:
search_all = True
if not search_all or package_type != search_all_type:
# Only show datasets of this particular type
fq += u' +dataset_type:{type}'.format(type=package_type)
facets = OrderedDict()
default_facet_titles = {
u'organization': _(u'Organizations'),
u'groups': _(u'Groups'),
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license_id': _(u'Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in plugins.PluginImplementations(plugins.IFacets):
facets = plugin.dataset_facets(facets, package_type)
extra_vars[u'facet_titles'] = facets
data_dict = {
u'q': q,
u'fq': fq.strip(),
u'facet.field': facets.keys(),
u'rows': limit,
u'start': (page - 1) * limit,
u'sort': sort_by,
u'extras': search_extras,
u'include_private': asbool(
config.get(u'ckan.search.default_include_private', True)
),
}
query = get_action(u'package_search')(context, data_dict)
extra_vars[u'sort_by_selected'] = query[u'sort']
extra_vars[u'page'] = h.Page(
collection=query[u'results'],
page=page,
url=pager_url,
item_count=query[u'count'],
items_per_page=limit
)
extra_vars[u'search_facets'] = query[u'search_facets']
extra_vars[u'page'].items = query[u'results']
except SearchQueryError as se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info(u'Dataset search query rejected: %r', se.args)
base.abort(
400,
_(u'Invalid search query: {error_message}')
.format(error_message=str(se))
)
except SearchError as se:
# May be bad input from the user, but may also be more serious like
# bad code causing a SOLR syntax error, or a problem connecting to
# SOLR
log.error(u'Dataset search error: %r', se.args)
extra_vars[u'query_error'] = True
extra_vars[u'search_facets'] = {}
extra_vars[u'page'] = h.Page(collection=[])
# FIXME: try to avoid using global variables
g.search_facets_limits = {}
for facet in extra_vars[u'search_facets'].keys():
try:
limit = int(
request.args.get(
u'_%s_limit' % facet,
int(config.get(u'search.facets.default', 10))
)
)
except ValueError:
base.abort(
400,
_(u'Parameter u"{parameter_name}" is not '
u'an integer').format(parameter_name=u'_%s_limit' % facet)
)
g.search_facets_limits[facet] = limit
_setup_template_variables(context, {}, package_type=package_type)
extra_vars[u'dataset_type'] = package_type
# TODO: remove
for key, value in extra_vars.iteritems():
setattr(g, key, value)
return base.render(
_get_pkg_template(u'search_template', package_type), extra_vars
)
def _read(self, id, limit, group_type):
''' This is common code used by both read and bulk_process'''
context = {'model': model, 'session': model.Session,
'user': c.user,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True, 'extras_as_string': True}
q = c.q = request.params.get('q', '')
# Search within group
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
else:
q += ' groups:"%s"' % c.group_dict.get('name')
c.description_formatted = \
h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
params = [(k, v.encode('utf-8') if isinstance(v, basestring)
else str(v)) for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group', action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(key, value=value, replace=replace,
controller=controller, action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
facets = OrderedDict()
default_facet_titles = {'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
self._update_facet_titles(facets, group_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': '',
'include_private': True,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items()
if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit
)
c.group_dict['package_count'] = query['count']
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.search_facets.keys():
limit = int(request.params.get('_%s_limit' % facet,
config.get('search.facets.default', 10)))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.page = h.Page(collection=[])
def search(self):
from ckan.lib.search import SearchError, SearchQueryError
package_type = self._guess_package_type()
try:
context = {
'model': model,
'user': c.user,
'auth_user_obj': c.userobj
}
check_access('site_read', context)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
timerange = []
collection_params = ''
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
c.parameters_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort', 'timerange_start', 'timerange_end'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
if param == 'collection_name':
collection_params = value
else:
search_extras[param] = value
from ckanext.opensearch import converters
from ckanext.opensearch.config import PARAMETERS
if param == 'timerange_start':
if len(value) > 0:
c.parameters_grouped[param] = value
for converter in PARAMETERS['collection'][param].get(
"converters", []):
value = getattr(converters, converter)(value)
c.fields.append((param, value))
fq += " %s:%s" % (param, value)
timerange.append(param)
if param == 'timerange_end':
if len(value) > 0:
c.parameters_grouped[param] = value
for converter in PARAMETERS['collection'][param].get(
"converters", []):
value = getattr(converters, converter)(value)
c.fields.append((param, value))
fq += " %s:%s" % (param, value)
context = {
'model': model,
'session': model.Session,
'user': c.user,
'for_view': True,
'auth_user_obj': c.userobj
}
if package_type and package_type != 'dataset':
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
else:
# Unless changed via config options, don't show non standard
# dataset types on the default search page
if not asbool(config.get('ckan.search.show_all_types',
'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
search_parameters = OrderedDict()
default_facet_titles = {
'collection_name': _('Data Collections'),
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
sentinel_facet_titles = {
'FamilyName': _('Family Name'),
'ProductType': _('Product Type'),
'OrbitDirection': _('Orbit Direction'),
'Swath': _('Acquisition Mode'),
'TransmitterReceiverPolarisation': _('Polarisation'),
}
default_search_titles = {
'timerange_start': _('Timerange Start'),
'timerange_end': _('Timerange End'),
}
collection_names = [
'Sentinel-1 Level-1 (SLC)', 'Sentinel-1 Level-1 (GRD)',
'Sentinel-1 Level-2 (OCN)', 'Sentinel-2 Level-1C',
'Sentinel-2 Level-2A', 'Sentinel-3 SRAL Level-1 SRA'
]
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
for search_param in helpers.search_params():
if search_param in default_search_titles:
search_parameters[search_param] = default_search_titles[
search_param]
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
if collection_params not in collection_names:
for f in facets:
if f in sentinel_facet_titles:
del facets[f]
c.facet_titles = facets
c.search_parameters = search_parameters
data_dict = {
'q':
q,
'fq':
fq.strip(),
'facet.field':
facets.keys(),
'rows':
limit,
'start': (page - 1) * limit,
'sort':
sort_by,
'extras':
search_extras,
'include_private':
asbool(config.get('ckan.search.default_include_private',
True)),
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.search_facets = query['search_facets']
c.search_search_parameters = search_parameters
c.page.items = query['results']
except SearchQueryError, se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info('Dataset search query rejected: %r', se.args)
abort(
400,
_('Invalid search query: {error_message}').format(
error_message=str(se)))
def _read(id, limit, group_type):
u''' This is common code used by both read and bulk_process'''
extra_vars = {}
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'schema': _db_to_form_schema(group_type=group_type),
u'for_view': True,
u'extras_as_string': True
}
q = request.params.get(u'q', u'')
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.q = q
# Search within group
if g.group_dict.get(u'is_organization'):
fq = u' owner_org:"%s"' % g.group_dict.get(u'id')
else:
fq = u' groups:"%s"' % g.group_dict.get(u'name')
extra_vars["q"] = q
g.description_formatted = \
h.render_markdown(g.group_dict.get(u'description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items() if k != u'page']
sort_by = request.params.get(u'sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = u'bulk_process' if getattr(
g, u'action', u'') == u'bulk_process' else u'read'
url = h.url_for(u'.'.join([controller, action]), id=id)
params = [(k, v.encode(u'utf-8')
if isinstance(v, string_types) else str(v))
for k, v in params]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(
alternative_url=None,
controller=u'group',
action=u'read',
extras=dict(id=g.group_dict.get(u'name')),
new_params=by)
extra_vars["drill_down_url"] = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(
key,
value=value,
replace=replace,
controller=controller,
action=u'read',
extras=dict(id=g.group_dict.get(u'name')))
extra_vars["remove_field"] = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append((u'page', page))
return search_url(params)
try:
extra_vars["fields"] = fields = []
extra_vars["fields_grouped"] = fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in [u'q', u'page', u'sort'] \
and len(value) and not param.startswith(u'_'):
if not param.startswith(u'ext_'):
fields.append((param, value))
q += u' %s: "%s"' % (param, value)
if param not in fields_grouped:
fields_grouped[param] = [value]
else:
fields_grouped[param].append(value)
else:
search_extras[param] = value
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.fields = fields
g.fields_grouped = fields_grouped
facets = OrderedDict()
default_facet_titles = {
u'organization': _(u'Organizations'),
u'groups': _(u'Groups'),
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license_id': _(u'Licenses')
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
_update_facet_titles(facets, group_type)
extra_vars["facet_titles"] = facets
data_dict = {
u'q': q,
u'fq': fq,
u'include_private': True,
u'facet.field': facets.keys(),
u'rows': limit,
u'sort': sort_by,
u'start': (page - 1) * limit,
u'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items() if k != u'schema')
query = get_action(u'package_search')(context_, data_dict)
extra_vars["page"] = h.Page(
collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.group_dict['package_count'] = query['count']
extra_vars["search_facets"] = g.search_facets = query['search_facets']
extra_vars["search_facets_limits"] = g.search_facets_limits = {}
for facet in g.search_facets.keys():
limit = int(
request.params.get(u'_%s_limit' % facet,
config.get(u'search.facets.default', 10)))
g.search_facets_limits[facet] = limit
extra_vars["page"].items = query['results']
extra_vars["sort_by_selected"] = sort_by
except search.SearchError as se:
log.error(u'Group search error: %r', se.args)
extra_vars["query_error"] = True
extra_vars["page"] = h.Page(collection=[])
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.facet_titles = facets
g.page = extra_vars["page"]
extra_vars["group_type"] = group_type
_setup_template_variables(context, {u'id': id}, group_type=group_type)
return extra_vars
def search(self):
package_type = self._guess_package_type()
try:
context = {
'model': model,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
q = c.q = request.params.get('q', u'')
c.query_error = False
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and value and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
# Modificación para andino: usamos una función para buscar dependencias entre organizaciones
if param != 'organization':
fq += ' %s:"%s"' % (param, value)
else:
fq += custom_organization_filter(value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'auth_user_obj': c.userobj
}
if package_type and package_type != 'dataset':
fq += ' +dataset_type:{type}'.format(type=package_type)
elif not asbool(config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = logic.action.get.package_search(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.search_facets = {}
c.page = h.Page(collection=[])
