def dashboard_organizations(self): context = { 'model': model, 'session': model.Session, 'for_view': True, 'user': c.user or c.author, 'auth_user_obj': c.userobj } data_dict = {'user_obj': c.userobj} self._setup_template_variables(context, data_dict) (user_orgs, usr_suborgs) = get_user_toporgs(c.userobj.id) facets = OrderedDict() #Add the organization facet to get the number of records for each organization facets['organization'] = _('Organizations') data_dict = { 'facet.field': facets.keys(), } query = get_action('package_search')(context, data_dict) c.org_pkg_count = query['facets'].get('organization') c.top_orgs_items = user_orgs c.suborgs_items = usr_suborgs return render('user/dashboard_organizations.html')
def dashboard_organizations(self): context = {'model': model, 'session': model.Session, 'for_view': True, 'user': c.user or c.author, 'auth_user_obj': c.userobj} data_dict = {'user_obj': c.userobj} self._setup_template_variables(context, data_dict) (user_orgs, usr_suborgs) = get_user_toporgs(c.userobj.id) facets = OrderedDict() #Add the organization facet to get the number of records for each organization facets['organization'] = _('Organizations') data_dict = { 'facet.field': facets.keys(), } query = get_action('package_search')(context, data_dict) c.org_pkg_count = query['facets'].get('organization') c.top_orgs_items = user_orgs c.suborgs_items = usr_suborgs return render('user/dashboard_organizations.html')
def cioos_get_facets(package_type='dataset'): ''' get all dataset for the given package type, including private ones. This function works similarly to code found in ckan/ckan/controllers/package.py in that it does a search of all datasets and populates the following globals for later use: c.facet_titles c.search_facets ''' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in toolkit.h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'facet.field': facets.keys(), 'rows': 0, 'include_private': asbool(config.get('ckan.search.default_include_private', True)), } context = { 'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'auth_user_obj': c.userobj } query = get_action('package_search')(context, data_dict) c.search_facets = query['search_facets']
def index(self): # FIXME: index copied from GroupController and modified to # show only parent groups context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'with_private': False } data_dict = {'all_fields': False} try: self._check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # pass user info to context as needed to view private datasets of # orgs correctly if c.userobj: context['user_id'] = c.userobj.id context['user_is_admin'] = c.userobj.sysadmin search_result = self._action('organization_list')(context, data_dict) org_model = context['model'] #Get the list of all groups of type "organization" that have no parents. top_level_orgs = org_model.Group.get_top_level_groups( type="organization") top_results = [ org for org in top_level_orgs if org.name in search_result ] facets = OrderedDict() facets['organization'] = _('Organizations') data_dict = { 'facet.field': facets.keys(), } query = get_action('package_search')(context, data_dict) c.org_pkg_count = query['facets'].get('organization') c.top_orgs_items = top_results return render('organization/index.html')
def index(self): # FIXME: index copied from GroupController and modified to # show only parent groups context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'with_private': False} data_dict = {'all_fields': False} try: self._check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # pass user info to context as needed to view private datasets of # orgs correctly if c.userobj: context['user_id'] = c.userobj.id context['user_is_admin'] = c.userobj.sysadmin search_result = self._action('organization_list')(context, data_dict) org_model = context['model'] #Get the list of all groups of type "organization" that have no parents. top_level_orgs = org_model.Group.get_top_level_groups(type="organization") top_results = [org for org in top_level_orgs if org.name in search_result ] facets = OrderedDict() facets['organization'] = _('Organizations') data_dict = { 'facet.field': facets.keys(), } query = get_action('package_search')(context, data_dict) c.org_pkg_count = query['facets'].get('organization') c.top_orgs_items = top_results return render('organization/index.html')
def search(self): from ckan.lib.search import SearchError, SearchQueryError # Get package type name package_type = self._guess_package_type()[:-1] c.package_type = package_type # Get page content from Wordpress # ========================================= import ckanext.dadosgovbr.helpers.wordpress as wp wp_page_slug = 'scheming_'+package_type+'s' c.wp_page = type('Nothing', (object,), {}) c.wp_page.content = type('Nothing', (object,), {}) c.wp_page.content.rendered = "Conteudo da pagina nao encontrado..." try: c.wp_page = wp.page(wp_page_slug) except: pass # DEBUG # from pprint import pprint # pprint(c.concursos) # Package type facets (filters) # ========================================= package_type_facets = u'organization groups tags res_format license_id' if(package_type == 'inventario'): package_type_facets = u'organization situacao_base informacoes_sigilosas_base informacoes_publicas_base atualizacoes_base dados_abertos_base' if(package_type == 'concurso'): package_type_facets = u'organization datasets_used' if(package_type == 'aplicativo'): package_type_facets = u'organization groups tags res_format license_id' try: context = {'model': model, 'user': c.user, 'auth_user_obj': c.userobj} check_access('site_read', context) except NotAuthorized: abort(403, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False page = h.get_page_number(request.params) limit = int(config.get('ckan.datasets_per_page', 20)) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = {'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'auth_user_obj': c.userobj} if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not asbool( config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { # Default package 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), # Inventário package 'situacao_base': _(u'Situação da base'), 'informacoes_sigilosas_base': _(u'Base possui informações sigilosas?'), 'vocab_sim': _(u'Sim'), 'vocab_nao': _(u'Não'), 'informacoes_publicas_base': _(u'Base possui informações públicas?'), 'informacoes_publicas_base_publico': _(u'Público'), 'atualizacoes_base': _(u'Período de atualização dos dados'), 'dados_abertos_base': _(u'Exporta para dados abertos?'), # Concurso package 'datasets_used': _(u'Dados utilizados'), 'tags': _(u'Tags'), 'date': _(u'Data de início'), 'end_date': _(u'Data final'), 'publico': _(u'Público'), 'sim': _(u'Sim'), 'nao': _(u'Não'), } for facet in config.get(u'search.facets', package_type_facets.split()): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras, 'include_private': asbool(config.get( 'ckan.search.default_include_private', True)), } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchQueryError, se: # User's search parameters are invalid, in such a way that is not # achievable with the web interface, so return a proper error to # discourage spiders which are the main cause of this. log.info('Dataset search query rejected: %r', se.args) abort(400, _('Invalid search query: {error_message}') .format(error_message=str(se)))
for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit
def search(self): package_type = self._guess_package_type() try: context = { 'model': model, 'user': c.user or c.author, 'auth_user_obj': c.userobj } check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) q = c.q = request.params.get('q', u'') c.query_error = False page = self._get_page_number(request.params) limit = g.datasets_per_page params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) if param != 'organization': fq += ' %s:"%s"' % (param, value) else: fq += custom_organization_filter(value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'auth_user_obj': c.userobj } if package_type and package_type != 'dataset': fq += ' +dataset_type:{type}'.format(type=package_type) elif not asbool(config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = h.Page(collection=[])
def search(self): from ckan.lib.search import SearchError, SearchQueryError package_type = self._guess_package_type() try: context = {'model': model, 'user': c.user, 'auth_user_obj': c.userobj} check_access('site_read', context) except NotAuthorized: abort(403, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False page = h.get_page_number(request.params) limit = int(config.get('ckan.datasets_per_page', 20)) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='ckanext.sixodp_showcase.controller:Sixodp_ShowcaseController', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='ckanext.sixodp_showcase.controller:Sixodp_ShowcaseController', action='search') c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = {'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'auth_user_obj': c.userobj} if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not asbool( config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in ckan.plugins.PluginImplementations(ckan.plugins.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets # Set the custom default sort parameter here # Might need a rewrite when ckan is updated if not sort_by: sort_by = 'metadata_created desc' data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras, 'include_private': asbool(config.get( 'ckan.search.default_include_private', True)), } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchQueryError, se: # User's search parameters are invalid, in such a way that is not # achievable with the web interface, so return a proper error to # discourage spiders which are the main cause of this. log.info('Dataset search query rejected: %r', se.args) abort(400, _('Invalid search query: {error_message}') .format(error_message=str(se)))
def search(self): package_type = self._guess_package_type() try: context = {'model': model, 'user': c.user or c.author, 'auth_user_obj': c.userobj} check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) q = c.q = request.params.get('q', u'') c.query_error = False page = self._get_page_number(request.params) limit = g.datasets_per_page params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param( alternative_url=alternative_url, controller='package', action='search', new_params=by ) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) if param != 'organization': fq += ' %s:"%s"' % (param, value) else: fq += custom_organization_filter(value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'auth_user_obj': c.userobj} if package_type and package_type != 'dataset': fq += ' +dataset_type:{type}'.format(type=package_type) elif not asbool(config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = h.Page(collection=[])
def _read(self, id, limit, group_type): ''' This is common code used by both read and bulk_process''' context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True} q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): q += ' owner_org:"%s"' % c.group_dict.get('id') else: q += ' groups:"%s"' % c.group_dict.get('name') c.description_formatted = h.render_markdown(c.group_dict.get('description')) context['return_query'] = True # c.group_admins is used by CKAN's legacy (Genshi) templates only, # if we drop support for those then we can delete this line. c.group_admins = authz.get_group_or_org_admin_ids(c.group.id) page = self._get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='group', action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) else: search_extras[param] = value fq = 'capacity:"public"' user_member_of_orgs = [org['id'] for org in h.organizations_available('read')] if (c.group and c.group.id in user_member_of_orgs): fq = '' context['ignore_capacity_check'] = True facets = OrderedDict() default_facet_titles = {'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses')} for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles self._update_facet_titles(facets, group_type) if 'capacity' in facets and (group_type != 'organization' or not user_member_of_orgs): del facets['capacity'] c.facet_titles = facets data_dict = { 'q': q, 'fq': fq, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict((k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.group_dict['package_count'] = query['count'] lang = get_lang()[0] q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.group_id == c.group_dict.get('id'), GroupMultilang.lang == lang).all() if q_results: for result in q_results: c.group_dict[result.field] = result.text if result.field == 'title': c.group_dict['display_name'] = result.text c.facets = query['facets'] maintain.deprecate_context_item('facets', 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.facets.keys(): limit = int(request.params.get('_%s_limit' % facet, g.facets_default_number)) c.search_facets_limits[facet] = limit # MULTILANG - Localizing Organizations display names in Facet list organizations = c.search_facets.get('organization') for org in organizations.get('items'): q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.name == org.get('name'), GroupMultilang.lang == lang).all() if q_results: for result in q_results: if result.field == 'title': org['display_name'] = result.text # MULTILANG - Localizing Groups display names in Facet list groups = c.search_facets.get('groups') for group in groups.get('items'): q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.name == group.get('name'), GroupMultilang.lang == lang).all() if q_results: for result in q_results: if result.field == 'title': group['display_name'] = result.text # MULTILANG - Localizing Tags display names in Facet list tags = c.search_facets.get('tags') for tag in tags.get('items'): localized_tag = TagMultilang.by_name(tag.get('name'), lang) if localized_tag: tag['display_name'] = localized_tag.text c.page.items = query['results'] # MULTILANG - Localizing Datasets names and descriptions in search list log.info(':::::::::::: Retrieving the corresponding localized title and abstract :::::::::::::::') for item in c.page.items: lang = get_lang()[0] q_results = model.Session.query(PackageMultilang).filter(PackageMultilang.package_id == item.get('id'), PackageMultilang.lang == lang).all() if q_results: for result in q_results: item[result.field] = result.text c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])
def search(self): from ckan.lib.search import SearchError package_type = self._guess_package_type() try: context = {'model': model, 'user': c.user or c.author, 'auth_user_obj': c.userobj} check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False page = self._get_page_number(request.params) limit = g.datasets_per_page # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if sort_by is None: c.sort_by_fields = [] else: c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) api_search_url_params = None try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'auth_user_obj': c.userobj} if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not asbool(config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } ##### EXTENDING ORIGINAL <<<<<<<<<< # API Params api_url_params = { 'q': q, 'fq': fq.strip(), 'rows': limit, 'start': (page - 1) * limit, } if sort_by: api_url_params['sort'] = sort_by if facets.keys(): api_url_params['facet.field'] = json.dumps(facets.keys()) # if search_extras: # api_url_params['extras'] = ','.join(search_extras) api_search_url_params = urlencode(_encode_params(api_url_params.items())) ##### EXTENDING ORIGINAL >>>>>>>>>>>>>> query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = h.Page(collection=[])
def _add_dataset_search(self, showcase_id, showcase_name): ''' Search logic for discovering datasets to add to a showcase. ''' from ckan.lib.search import SearchError package_type = 'dataset' # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False try: page = self._get_page_number(request.params) except AttributeError: # in CKAN >= 2.5 _get_page_number has been moved page = h.get_page_number(request.params) limit = int(config.get('ckan.datasets_per_page', 20)) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return self._search_url(params, showcase_name) c.sort_by = _sort_by if sort_by is None: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return self._search_url(params, showcase_name) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'auth_user_obj': c.userobj } if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not tk.asbool( config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' # Only search for packages that aren't already associated with the # Showcase associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase( showcase_id) # flatten resulting list to space separated string if associated_package_ids: associated_package_ids_str = \ ' OR '.join([id[0] for id in associated_package_ids]) fq += ' !id:({0})'.format(associated_package_ids_str) facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } # for CKAN-Versions that do not provide the facets-method from # helper-context, import facets from ckan.common if hasattr(h, 'facets'): current_facets = h.facets() else: from ckan.common import g current_facets = g.facets for facet in current_facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = h.Page(collection=[])
def _read(self, id, limit, group_type): ''' This is common code used by both read and bulk_process''' context = {'model': model, 'session': model.Session, 'user': c.user, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True} q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): fq = 'owner_org:"%s"' % c.group_dict.get('id') else: fq = 'groups:"%s"' % c.group_dict.get('name') c.description_formatted = \ h.render_markdown(c.group_dict.get('description')) context['return_query'] = True page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) params = [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): controller = lookup_group_controller(group_type) return h.remove_url_param(key, value=value, replace=replace, controller=controller, action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: # Set the custom default sort parameter here # Might need a rewrite when ckan is updated if q and not sort_by: sort_by = 'score desc, metadata_modified desc' elif not sort_by: sort_by = 'metadata_created desc' c.fields = [] c.fields_grouped = {} search_extras = {} for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value facets = OrderedDict() default_facet_titles = {'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses')} for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles facets = self._update_facet_titles(facets, group_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq, 'include_private': True, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict((k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.group_dict['package_count'] = query['count'] c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.search_facets.keys(): limit = int(request.params.get('_%s_limit' % facet, config.get('search.facets.default', 10))) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError as se: log.error('Group search error: %r', se.args) c.query_error = True c.page = h.Page(collection=[]) self._setup_template_variables(context, {'id': id}, group_type=group_type)
def _search(self, package_type, pager_url, additional_fq='', additional_facets=None, default_sort_by=DEFAULT_SORTING, num_of_items=NUM_OF_ITEMS, ignore_capacity_check=False, use_solr_collapse=False, enable_update_status_facet=False): from ckan.lib.search import SearchError # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False page = self._page_number() # Commenting below parts as it doesn't seem to be used # def drill_down_url(alternative_url=None, **by): # return h.add_url_param(alternative_url=alternative_url, # controller='package', action='search', # new_params=by) # # c.drill_down_url = drill_down_url # self._set_remove_field_function() req_sort_by = request.params.get('sort', None) if not req_sort_by and q: req_sort_by = 'score desc, ' + DEFAULT_SORTING if req_sort_by: sort_by = req_sort_by c.used_default_sort_by = False else: sort_by = default_sort_by c.used_default_sort_by = True # params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] # The sort_by function seems to not be used anymore # def _sort_by(fields): # """ # Sort by the given list of fields. # # Each entry in the list is a 2-tuple: (fieldname, sort_order) # # eg - [('metadata_modified', 'desc'), ('name', 'asc')] # # If fields is empty, then the default ordering is used. # """ # params = params_nosort[:] # # if fields: # sort_string = ', '.join('%s %s' % f for f in fields) # params.append(('sort', sort_string)) # return self._search_url(params, package_type) # # c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')] self._set_other_links() try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} # limit = g.datasets_per_page fq = additional_fq tagged_fq_dict = {} featured_filters_set = False for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if param == 'fq': fq += ' %s' % (value,) elif not param.startswith('ext_'): c.fields.append((param, value)) # fq += ' {!tag=%s}%s:"%s"' % (param, param, value) if param not in tagged_fq_dict: tagged_fq_dict[param] = [] tagged_fq_dict[param].append(u'{}:"{}"'.format(param, value)) self.append_selected_facet_to_group(c.fields_grouped, param, value) elif param == UPDATE_STATUS_URL_FILTER: self.append_selected_facet_to_group(c.fields_grouped, param, value) else: if param in ['ext_cod', 'ext_subnational', 'ext_quickcharts', 'ext_geodata', 'ext_requestdata', 'ext_hxl', 'ext_showcases', 'ext_archived', 'ext_administrative_divisions']: featured_filters_set = True search_extras[param] = value if c.fields_grouped.get(UPDATE_STATUS_URL_FILTER): search_extras[UPDATE_STATUS_URL_FILTER] = c.fields_grouped[UPDATE_STATUS_URL_FILTER] self._set_filters_are_selected_flag() fq_list = [u'{{!tag={tag}}}{value}'.format(tag=key, category=key, value=' OR '.join(value_list)) for key, value_list in tagged_fq_dict.items()] # if the search is not filtered by query or facet group datasets solr_expand = 'false' if use_solr_collapse and not fq_list and not q and not featured_filters_set: fq_list = ['{!tag=batch}{!collapse field=batch nullPolicy=expand} '] solr_expand = 'true' try: limit = 1 if self._is_facet_only_request() else int(request.params.get('ext_page_size', num_of_items)) except: limit = num_of_items c.ext_page_size = limit context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'auth_user_obj': c.userobj} if ignore_capacity_check: context['ignore_capacity_check'] = ignore_capacity_check if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not asbool(config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = get_default_facet_titles() for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) if additional_facets: facets.update(additional_facets) c.facet_titles = facets # TODO Line below to be removed after refactoring c.tab = 'all' #adding site_id to facets to facilitate totals counts in case of batch/collapse facet_keys = ['{!ex=batch}site_id'] + facets.keys() self._performing_search(q, fq, facet_keys, limit, page, sort_by, search_extras, pager_url, context, fq_list=fq_list, expand=solr_expand, enable_update_status_facet=enable_update_status_facet) except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = h.Page(collection=[])
# Facet titles for plugin in plugins.PluginImplementations(plugins.IFacets): if self.group_type == "organization": facets = plugin.organization_facets(facets, self.group_type, None) else: facets = plugin.group_facets(facets, self.group_type, None) if "capacity" in facets and (self.group_type != "organization" or not user_member_of_orgs): del facets["capacity"] c.facet_titles = facets data_dict = { "q": q, "fq": fq, "facet.field": facets.keys(), "rows": limit, "sort": sort_by, "start": (page - 1) * limit, "extras": search_extras, } query = get_action("package_search")(context, data_dict) c.page = h.Page( collection=query["results"], page=page, url=pager_url, item_count=query["count"], items_per_page=limit ) c.facets = query["facets"] maintain.deprecate_context_item("facets", "Use `c.search_facets` instead.")
def _read(self, id, limit, group_type): # noqa c.include_children_selected = False if not c.group_dict.get('is_organization'): return ''' This is common code used by both read and bulk_process''' context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True} c.description_formatted = \ h.render_markdown(c.group_dict.get('description')) context['return_query'] = True # c.group_admins is used by CKAN's legacy (Genshi) templates only, # if we drop support for those then we can delete this line. c.group_admins = authz.get_group_or_org_admin_ids(c.group.id) page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='group', action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: q = c.q = request.params.get('q', '') fq = c.fq = request.params.get('fq', '') c.fields = [] search_extras = {} for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s: "%s"' % (param, value) else: search_extras[param] = value user_member_of_orgs = [org['id'] for org in h.organizations_available('read')] if (c.group and c.group.id in user_member_of_orgs): context['ignore_capacity_check'] = True else: fq += ' capacity:"public"' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses') } for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles self._update_facet_titles(facets, group_type) if 'capacity' in facets and (group_type != 'organization' or not user_member_of_orgs): del facets['capacity'] c.facet_titles = facets # filter by organization with fq (filter query) c.include_children_selected = True children = _children_name_list( hierarchy_helpers.group_tree_section( c.group_dict.get('id'), include_parents=False, include_siblings=False ).get('children', []) ) if not children: fq += ' organization:"%s"' % c.group_dict.get('name') else: fq += ' organization:("%s"' % c.group_dict.get('name') for name in children: if name: fq += ' OR "%s"' % name fq += ")" data_dict = { 'q': q, 'fq': fq, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict((k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.group_dict['package_count'] = query['count'] c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.search_facets.keys(): limit = int(request.params.get('_%s_limit' % facet, config.get( 'search.facets.default', 10))) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.search_facets = {} c.page = h.Page(collection=[])
def search(self): from ckan.lib.search import SearchError, SearchQueryError package_type = self._guess_package_type() extra_vars = {'dataset_type': package_type} try: context = { 'model': model, 'user': c.user, 'auth_user_obj': c.userobj } check_access('site_read', context) except NotAuthorized: abort(403, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False page = h.get_page_number(request.params) limit = int(config.get('ckan.datasets_per_page', 20)) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search', alternative_url=package_type) c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'auth_user_obj': c.userobj } # Unless changed via config options, don't show other dataset # types any search page. Potential alternatives are do show them # on the default search page (dataset) or on one other search page search_all_type = config.get('ckan.search.show_all_types', 'dataset') search_all = False try: # If the "type" is set to True or False, convert to bool # and we know that no type was specified, so use traditional # behaviour of applying this only to dataset type search_all = asbool(search_all_type) search_all_type = 'dataset' # Otherwise we treat as a string representing a type except ValueError: search_all = True if not package_type: package_type = 'dataset' if not search_all or package_type != search_all_type: # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets # Date interval search start_date = None end_date = None if request.params.get('_start_date'): start_date = extract_date(request.params.get('_start_date')) if start_date: extra_vars['start_date'] = request.params.get( '_start_date') if request.params.get('_end_date'): end_date = extract_date(request.params.get('_end_date')) if end_date: extra_vars['end_date'] = request.params.get('_end_date') if start_date or end_date: if not start_date: start_date = datetime(year=1900, month=1, day=1) if not end_date: end_date = datetime(year=2100, month=1, day=1, hour=23, minute=59, second=59, microsecond=999999) else: end_date = datetime_to_utc(end_date) end_date = end_date.replace(hour=23, minute=59, second=59, microsecond=999999) fq = fq + ' +metadata_modified:[%s TO %s]' % ( datetime_to_utc(start_date).strftime('%Y-%m-%dT%H:%M:%SZ'), datetime_to_utc(end_date).strftime('%Y-%m-%dT%H:%M:%SZ')) data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras, 'include_private': asbool(config.get('ckan.search.default_include_private', True)), } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchQueryError as se: # User's search parameters are invalid, in such a way that is not # achievable with the web interface, so return a proper error to # discourage spiders which are the main cause of this. log.info('Dataset search query rejected: %r', se.args) abort( 400, _('Invalid search query: {error_message}').format( error_message=str(se))) except SearchError as se: # May be bad input from the user, but may also be more serious like # bad code causing a SOLR syntax error, or a problem connecting to # SOLR log.error('Dataset search error: %r', se.args) c.query_error = True c.search_facets = {} c.page = h.Page(collection=[]) except NotAuthorized: abort(403, _('Not authorized to see this page')) c.search_facets_limits = {} for facet in c.search_facets.keys(): try: limit = int( request.params.get( '_%s_limit' % facet, int(config.get('search.facets.default', 10)))) except ValueError: abort( 400, _('Parameter "{parameter_name}" is not ' 'an integer').format(parameter_name='_%s_limit' % facet)) c.search_facets_limits[facet] = limit self._setup_template_variables(context, {}, package_type=package_type) extra_vars['dataset_type'] = package_type return render(self._search_template(package_type), extra_vars=extra_vars)
def search(self): from ckan.lib.search import SearchError, SearchQueryError package_type = self._guess_package_type() try: context = { 'model': model, 'user': c.user, 'auth_user_obj': c.userobj } check_access('site_read', context) except NotAuthorized: abort(403, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False page = h.get_page_number(request.params) limit = g.datasets_per_page # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field # This is the only difference between this and core implementation sort_by = request.params.get('sort', 'metadata_modified desc') # <- params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'auth_user_obj': c.userobj } if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not asbool(config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras, 'include_private': asbool(config.get('ckan.search.default_include_private', True)), } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchQueryError, se: # User's search parameters are invalid, in such a way that is not # achievable with the web interface, so return a proper error to # discourage spiders which are the main cause of this. log.info('Dataset search query rejected: %r', se.args) abort( 400, _('Invalid search query: {error_message}').format( error_message=str(se)))
def _read(self, id, limit, group_type): """ This is common code used by both read and bulk_process""" context = { "model": model, "session": model.Session, "user": c.user, "schema": self._db_to_form_schema(group_type=group_type), "for_view": True, "extras_as_string": True, } q = c.q = request.params.get("q", "") # Search within group if c.group_dict.get("is_organization"): q += ' owner_org:"%s"' % c.group_dict.get("id") else: q += ' groups:"%s"' % c.group_dict.get("name") c.description_formatted = h.render_markdown(c.group_dict.get("description")) context["return_query"] = True page = self._get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != "page"] sort_by = request.params.get("sort", None) def search_url(params): controller = lookup_group_controller(group_type) action = "bulk_process" if c.action == "bulk_process" else "read" url = h.url_for(controller=controller, action=action, id=id) params = [(k, v.encode("utf-8") if isinstance(v, basestring) else str(v)) for k, v in params] return url + u"?" + urlencode(params) def drill_down_url(**by): return h.add_url_param( alternative_url=None, controller="group", action="read", extras=dict(id=c.group_dict.get("name")), new_params=by, ) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param( key, value=value, replace=replace, controller="group", action="read", extras=dict(id=c.group_dict.get("name")), ) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(("page", page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if not param in ["q", "page", "sort"] and len(value) and not param.startswith("_"): if not param.startswith("ext_"): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) else: search_extras[param] = value fq = 'capacity:"public"' user_member_of_orgs = [org["id"] for org in h.organizations_available("read")] if c.group and c.group.id in user_member_of_orgs: fq = "" context["ignore_capacity_check"] = True facets = OrderedDict() default_facet_titles = { "organization": _("Organizations"), "groups": _("Groups"), "tags": _("Tags"), "res_format": _("Formats"), "license_id": _("Licenses"), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles self._update_facet_titles(facets, group_type) if "capacity" in facets and (group_type != "organization" or not user_member_of_orgs): del facets["capacity"] c.facet_titles = facets data_dict = { "q": q, "fq": fq, "facet.field": facets.keys(), "rows": limit, "sort": sort_by, "start": (page - 1) * limit, "extras": search_extras, } context_ = dict((k, v) for (k, v) in context.items() if k != "schema") query = get_action("package_search")(context_, data_dict) c.page = h.Page( collection=query["results"], page=page, url=pager_url, item_count=query["count"], items_per_page=limit ) c.group_dict["package_count"] = query["count"] c.facets = query["facets"] maintain.deprecate_context_item("facets", "Use `c.search_facets` instead.") c.search_facets = query["search_facets"] c.search_facets_limits = {} for facet in c.facets.keys(): limit = int(request.params.get("_%s_limit" % facet, g.facets_default_number)) c.search_facets_limits[facet] = limit c.page.items = query["results"] c.sort_by_selected = sort_by except search.SearchError, se: log.error("Group search error: %r", se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])
def _add_dataset_search(self, showcase_id, showcase_name): ''' Search logic for discovering datasets to add to a showcase. ''' from ckan.lib.search import SearchError package_type = 'dataset' # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False try: page = self._get_page_number(request.params) except AttributeError: # in CKAN >= 2.5 _get_page_number has been moved page = h.get_page_number(request.params) limit = int(config.get('ckan.datasets_per_page', 20)) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return self._search_url(params, showcase_name) c.sort_by = _sort_by if sort_by is None: c.sort_by_fields = [] else: c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return self._search_url(params, showcase_name) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'auth_user_obj': c.userobj} if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not tk.asbool(config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' # Only search for packages that aren't already associated with the # Showcase associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase(showcase_id) # flatten resulting list to space separated string if associated_package_ids: associated_package_ids_str = \ ' OR '.join([id[0] for id in associated_package_ids]) fq += ' !id:({0})'.format(associated_package_ids_str) facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } # for CKAN-Versions that do not provide the facets-method from # helper-context, import facets from ckan.common if hasattr(h, 'facets'): current_facets = h.facets() else: from ckan.common import g current_facets = g.facets for facet in current_facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = h.Page(collection=[])
def _read(self, id, limit): ''' This is common code used by both read and bulk_process''' group_type = self._get_group_type(id.split('@')[0]) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True} q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): q += ' owner_org:"%s"' % c.group_dict.get('id') else: q += ' groups:"%s"' % c.group_dict.get('name') description = c.group_dict.get('description') if description: if group_type == 'group': c.description_formatted = h.render_markdown(description.get(h.lang())) else: c.description_formatted = h.render_markdown(description) context['return_query'] = True # c.group_admins is used by CKAN's legacy (Genshi) templates only, # if we drop support for those then we can delete this line. c.group_admins = new_authz.get_group_or_org_admin_ids(c.group.id) page = self._get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): if group_type == 'organization': if c.action == 'bulk_process': url = self._url_for(controller='organization', action='bulk_process', id=id) else: url = self._url_for(controller='organization', action='read', id=id) else: url = self._url_for(controller='group', action='read', id=id) params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='group', action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) else: search_extras[param] = value fq = 'capacity:"public"' user_member_of_orgs = [org['id'] for org in h.organizations_available('read')] if (c.group and c.group.id in user_member_of_orgs): fq = '' context['ignore_capacity_check'] = True facets = OrderedDict() default_facet_titles = {'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses')} for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in plugins.PluginImplementations(plugins.IFacets): if self.group_type == 'organization': facets = plugin.organization_facets( facets, self.group_type, None) else: facets = plugin.group_facets( facets, self.group_type, None) if 'capacity' in facets and (self.group_type != 'organization' or not user_member_of_orgs): del facets['capacity'] c.facet_titles = facets data_dict = { 'q': q, 'fq': fq, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict((k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.group_dict['package_count'] = query['count'] c.facets = query['facets'] maintain.deprecate_context_item('facets', 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.facets.keys(): limit = int(request.params.get('_%s_limit' % facet, g.facets_default_number)) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])
def search(self): from ckan.lib.search import SearchError package_type = self._guess_package_type() try: context = { 'model': model, 'user': c.user or c.author, 'auth_user_obj': c.userobj } check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False page = self._get_page_number(request.params) limit = g.datasets_per_page # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field if package_type == 'dataset': default_sort_by = 'metadata_modified desc' if g.tracking_enabled else None else: default_sort_by = 'metadata_created desc' sort_by = request.params.get('sort', default_sort_by) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if sort_by is None: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) api_search_url_params = None try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' fq_list = [ ] # filter statements that will be sent in separate "fq" params (because we want to be able to exclude them selectively from facets) for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq_list.append('{!tag=%s}%s:"%s"' % (param, param, value)) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'auth_user_obj': c.userobj } if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) if (package_type == 'application' ) and not (new_authz.is_sysadmin(c.user)): fq += ' +status:verified' else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not asbool(config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets facets_keys = [] for f in facets: facets_keys.append("{!ex=" + f + "}" + f) data_dict = { 'q': q, 'fq': fq.strip(), 'fq_list': fq_list, 'facet.field': facets_keys, 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } ##### EXTENDING ORIGINAL <<<<<<<<<< # API Params api_url_params = { 'q': q, 'fq': fq.strip(), 'rows': limit, 'start': (page - 1) * limit, } if sort_by: api_url_params['sort'] = sort_by if facets.keys(): api_url_params['facet.field'] = json.dumps(facets.keys()) # if search_extras: # api_url_params['extras'] = ','.join(search_extras) api_search_url_params = urlencode( _encode_params(api_url_params.items())) ##### EXTENDING ORIGINAL >>>>>>>>>>>>>> query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = h.Page(collection=[])
def _read(self, id, limit, group_type): ''' This is common code used by both read and bulk_process''' context = {'model': model, 'session': model.Session, 'user': c.user, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True} q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): q += ' owner_org:"%s"' % c.group_dict.get('id') else: q += ' groups:"%s"' % c.group_dict.get('name') c.description_formatted = \ h.render_markdown(c.group_dict.get('description')) context['return_query'] = True page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): controller = lookup_group_controller(group_type) return h.remove_url_param(key, value=value, replace=replace, controller=controller, action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] c.fields_grouped = {} search_extras = {} for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value facets = OrderedDict() default_facet_titles = {'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses')} for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles self._update_facet_titles(facets, group_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': '', 'include_private': True, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict((k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.group_dict['package_count'] = query['count'] c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.search_facets.keys(): limit = int(request.params.get('_%s_limit' % facet, config.get('search.facets.default', 10))) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.page = h.Page(collection=[])
def _read(self, id, limit, group_type): # noqa """ This controller replaces the GroupController controller from CKAN. It uses the filter query (fq) parameter to query facets in Solr instead of the query parameter (q). If the query parameter is used the search always returns "no results found" when combining a search term with facets. Unfortunately there are no clean extension points in the GroupController, so that the _read() method had to be overridden completely. This is common code used by both read and bulk_process """ context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True} q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): fq = 'owner_org:"%s"' % c.group_dict.get('id') else: fq = 'groups:"%s"' % c.group_dict.get('name') c.description_formatted = \ h.render_markdown(c.group_dict.get('description')) context['return_query'] = True # c.group_admins is used by CKAN's legacy (Genshi) templates only, # if we drop support for those then we can delete this line. c.group_admins = authz.get_group_or_org_admin_ids(c.group.id) page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) params = [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): controller = lookup_group_controller(group_type) return h.remove_url_param(key, value=value, replace=replace, controller=controller, action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] c.fields_grouped = {} search_extras = {} for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s: "%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value facets = OrderedDict() default_facet_titles = {'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses')} for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles self._update_facet_titles(facets, group_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq, 'include_private': True, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict((k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.group_dict['package_count'] = query['count'] c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.search_facets.keys(): limit = int(request.params.get('_%s_limit' % facet, config.get( 'search.facets.default', 10))) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError as se: log.error('Group search error: %r', se.args) c.query_error = True c.search_facets = {} c.page = h.Page(collection=[]) self._setup_template_variables( context, {'id': id}, group_type=group_type )
def _read(self, id, limit, group_type): ''' This is common code used by both read and bulk_process''' context = { 'model': model, 'session': model.Session, 'user': c.user, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True } q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): q += ' owner_org:"%s"' % c.group_dict.get('id') else: q += ' groups:"%s"' % c.group_dict.get('name') c.description_formatted = \ h.render_markdown(c.group_dict.get('description')) context['return_query'] = True page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) params = [ (k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params ] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='group', action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) else: search_extras[param] = value include_private = False user_member_of_orgs = [ org['id'] for org in h.organizations_available('read') ] if (c.group and c.group.id in user_member_of_orgs): include_private = True facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses') } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles self._update_facet_titles(facets, group_type) if 'capacity' in facets and (group_type != 'organization' or not user_member_of_orgs): del facets['capacity'] c.facet_titles = facets data_dict = { 'q': q, 'fq': '', 'include_private': include_private, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict( (k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.group_dict['package_count'] = query['count'] c.facets = query['facets'] maintain.deprecate_context_item('facets', 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.facets.keys(): limit = int( request.params.get('_%s_limit' % facet, g.facets_default_number)) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])
def _read(self, id, limit, group_type): ''' This is common code used by both read and bulk_process''' context = {'model': model, 'session': model.Session, 'user': c.user, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True} q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): q += ' owner_org:"%s"' % c.group_dict.get('id') else: q += ' groups:"%s"' % c.group_dict.get('name') c.description_formatted = \ h.render_markdown(c.group_dict.get('description')) context['return_query'] = True page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] sort_by = request.params.get('sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): controller = lookup_group_controller(group_type) return h.remove_url_param(key, value=value, replace=replace, controller=controller, action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) def collection_information(collection_id=None): from ckanext.opensearch import config collections = config.load_settings("collections_list") collection_items = collections.items() for collection in collection_items: if collection[0] == collection_id: return dict(collection[1]) group_extras = c.group_dict.get('extras') group_collections = [] for extra in group_extras: if extra['key'] == 'collections': col_value = extra['value'].split(", ") for a in col_value: group_collections.append(a) c.collections = [] for collection_id in group_collections: c.collections.append(collection_information(collection_id)) collection_params = '' try: c.fields = [] c.fields_grouped = {} search_extras = {} for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value facets = OrderedDict() search_parameters = OrderedDict() default_facet_titles = { 'collection_name': _('Data Collections'), 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } sentinel_facet_titles = { 'FamilyName': _('Family Name'), 'ProductType': _('Product Type'), 'OrbitDirection': _('Orbit Direction'), 'Swath': _('Acquisition Mode'), 'TransmitterReceiverPolarisation': _('Polarisation'), } default_search_titles = { 'timerange_start': _('Timerange Start'), 'timerange_end': _('Timerange End'), } collection_names = ['Sentinel-1 Level-1 (SLC)', 'Sentinel-1 Level-1 (GRD)', 'Sentinel-1 Level-2 (OCN)', 'Sentinel-2 Level-1C', 'Sentinel-2 Level-2A', 'Sentinel-3 SRAL Level-1 SRA'] for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet for search_param in helpers.search_params(): if search_param in default_search_titles: search_parameters[search_param] = default_search_titles[search_param] # Facet titles self._update_facet_titles(facets, group_type) if collection_params not in collection_names: for f in facets: if f in sentinel_facet_titles: del facets[f] c.facet_titles = facets data_dict = { 'q': q, 'fq': '', 'include_private': True, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict((k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit ) c.group_dict['package_count'] = query['count'] c.group_dict['collection_count'] = len(c.collections) c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.search_facets.keys(): limit = int(request.params.get('_%s_limit' % facet, config.get('search.facets.default', 10))) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.page = h.Page(collection=[])
def _read(self, id, limit): ''' This is common code used by both read and bulk_process''' group_type = self._get_group_type(id.split('@')[0]) context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._db_to_form_schema(group_type=group_type), 'for_view': True, 'extras_as_string': True } q = c.q = request.params.get('q', '') # Search within group if c.group_dict.get('is_organization'): q += ' owner_org:"%s"' % c.group_dict.get('id') else: q += ' groups:"%s"' % c.group_dict.get('name') description = c.group_dict.get('description') if description: if group_type == 'group': c.description_formatted = h.render_markdown( description.get(h.lang())) else: c.description_formatted = h.render_markdown(description) context['return_query'] = True # c.group_admins is used by CKAN's legacy (Genshi) templates only, # if we drop support for those then we can delete this line. c.group_admins = new_authz.get_group_or_org_admin_ids(c.group.id) page = self._get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] default_sort_by = 'metadata_modified desc' sort_by = request.params.get('sort', default_sort_by) def search_url(params): if group_type == 'organization': if c.action == 'bulk_process': url = self._url_for(controller='organization', action='bulk_process', id=id) else: url = self._url_for(controller='organization', action='read', id=id) else: url = self._url_for(controller='group', action='read', id=id) params = [ (k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params ] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param(alternative_url=None, controller='group', action='read', extras=dict(id=c.group_dict.get('name')), new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='group', action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) try: c.fields = [] search_extras = {} for (param, value) in request.params.items(): if not param in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) else: search_extras[param] = value fq = 'capacity:"public"' user_member_of_orgs = [ org['id'] for org in h.organizations_available('read') ] if (c.group and c.group.id in user_member_of_orgs): fq = '' context['ignore_capacity_check'] = True facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses') } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in plugins.PluginImplementations(plugins.IFacets): if self.group_type == 'organization': facets = plugin.organization_facets( facets, self.group_type, None) else: facets = plugin.group_facets(facets, self.group_type, None) if 'capacity' in facets and (self.group_type != 'organization' or not user_member_of_orgs): del facets['capacity'] c.facet_titles = facets data_dict = { 'q': q, 'fq': fq, 'facet.field': facets.keys(), 'rows': limit, 'sort': sort_by, 'start': (page - 1) * limit, 'extras': search_extras } context_ = dict( (k, v) for (k, v) in context.items() if k != 'schema') query = get_action('package_search')(context_, data_dict) c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.group_dict['package_count'] = query['count'] c.facets = query['facets'] maintain.deprecate_context_item('facets', 'Use `c.search_facets` instead.') c.search_facets = query['search_facets'] c.search_facets_limits = {} for facet in c.search_facets.keys(): limit = int( request.params.get('_%s_limit' % facet, g.facets_default_number)) c.search_facets_limits[facet] = limit c.page.items = query['results'] c.sort_by_selected = sort_by except search.SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True c.facets = {} c.page = h.Page(collection=[])
# Add 'author' facet facets['author'] = _('Authors') # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets fq += ' +organization:"{}"'.format(org_name) data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras } query = get_action('package_search')(context, data_dict) # Override the "author" list, to include full name authors query['search_facets']['author']['items'] =\ self._get_full_name_authors(context, org_name, None) c.sort_by_selected = query['sort'] c.page = h.Page(
def search(self): from ckan.lib.search import SearchError, SearchQueryError package_type = self._guess_package_type() try: context = { 'model': model, 'user': c.user, 'auth_user_obj': c.userobj } check_access('site_read', context) except NotAuthorized: abort(403, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') org = request.params.get('organization', None) c.query_error = False page = h.get_page_number(request.params) try: limit = int(config.get('ckan.datasets_per_page', 20)) except: limit = 20 # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search', alternative_url=package_type) c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'auth_user_obj': c.userobj } # Unless changed via config options, don't show other dataset # types any search page. Potential alternatives are do show them # on the default search page (dataset) or on one other search page search_all_type = config.get('ckan.search.show_all_types', 'dataset') search_all = False map_results = None try: # If the "type" is set to True or False, convert to bool # and we know that no type was specified, so use traditional # behaviour of applying this only to dataset type search_all = asbool(search_all_type) search_all_type = 'dataset' # Otherwise we treat as a string representing a type except ValueError: search_all = True if not package_type: package_type = 'dataset' if not search_all or package_type != search_all_type: # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras, 'include_private': asbool(config.get('ckan.search.default_include_private', True)), } query = get_action('package_search')(context, data_dict) # loop the search query and get all the results # this workaround the 1000 rows solr hard limit HARD_LIMIT = 1000 conn = get_connection_redis() pager = 0 # crank up the pager limit high as using points cluster for better performance PAGER_LIMIT = 10000 data_dict_full_result = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': HARD_LIMIT, 'start': 0, 'sort': sort_by, 'extras': search_extras, 'include_private': asbool(config.get('ckan.search.default_include_private', True)), } if not org: # if no q, it is an init load or direct visit on / dataset log.info('### Not org ###') if not q: log.info('### Not q ###') if not conn.exists('redis_full_results'): log.info('### generating full results ###') # get full results and add to redis when there is not full results in redis full_results = self.get_full_results( context, data_dict_full_result, pager, PAGER_LIMIT, q, fq, facets, HARD_LIMIT, sort_by, search_extras) map_results = self.get_map_result(full_results) # adding to redis log.info('adding full results to redis') conn.set('redis_full_results', json.dumps(map_results)) # log.info(c.full_results) else: log.info('### using cached full results ###') map_results = json.loads( conn.get('redis_full_results')) # log.info(c.full_results) else: log.info('### With q ###') full_results = self.get_full_results( context, data_dict_full_result, pager, PAGER_LIMIT, q, fq, facets, HARD_LIMIT, sort_by, search_extras) map_results = self.get_map_result(full_results) else: log.info('### With org ###') if not q: log.info('### Not q ###') if not conn.exists('redis_full_results_%s' % org): log.info('### generating %s results ###' % org) # get full results and add to redis when there is not full results in redis full_results = self.get_full_results( context, data_dict_full_result, pager, PAGER_LIMIT, q, fq, facets, HARD_LIMIT, sort_by, search_extras) map_results = self.get_map_result(full_results) # adding to redis log.info('adding %s results to redis' % org) conn.set('redis_full_results_%s' % org, json.dumps(map_results)) # log.info(c.full_results) else: log.info('### using cached %s results ###' % org) map_results = json.loads( conn.get('redis_full_results_%s' % org)) # log.info(c.full_results) else: log.info('### With q ###') full_results = self.get_full_results( context, data_dict_full_result, pager, PAGER_LIMIT, q, fq, facets, HARD_LIMIT, sort_by, search_extras) map_results = self.get_map_result(full_results) # log.info(c.full_results) c.sort_by_selected = query['sort'] c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchQueryError as se: # User's search parameters are invalid, in such a way that is not # achievable with the web interface, so return a proper error to # discourage spiders which are the main cause of this. log.info('Dataset search query rejected: %r', se.args) abort( 400, _('Invalid search query: {error_message}').format( error_message=str(se))) except SearchError as se: # May be bad input from the user, but may also be more serious like # bad code causing a SOLR syntax error, or a problem connecting to # SOLR log.error('Dataset search error: %r', se.args) c.query_error = True c.search_facets = {} c.page = h.Page(collection=[]) except NotAuthorized: abort(403, _('Not authorized to see this page')) c.search_facets_limits = {} for facet in c.search_facets.keys(): try: limit = int( request.params.get( '_%s_limit' % facet, int(config.get('search.facets.default', 10)))) except ValueError: abort( 400, _('Parameter "{parameter_name}" is not ' 'an integer').format(parameter_name='_%s_limit' % facet)) c.search_facets_limits[facet] = limit self._setup_template_variables(context, {}, package_type=package_type) return render(self._search_template(package_type), extra_vars={ 'dataset_type': package_type, 'map_results': map_results })
def show(self): #This is based on the CKAN package search controller #There are probably still some pieces we don't need #And using the facets to get the info we need is not ideal #because normal facet limits are then not applied on the page from ckan.lib.search import SearchError # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search') c.remove_field = remove_field params_nopage = [(k, v) for k, v in request.params.items()] def pager_url(q=None, page=None): params = list(params_nopage) return self.search_url(params, 'dataset') c.search_url_params = urlencode(self._encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'auth_user_obj': c.userobj} facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, 'dataset') c.facet_titles = facets #We use the organization facet to get the list of organizations detected for this query #The actual data is irrelevant data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'facet.limit': -1, 'extras': search_extras, 'rows': 1 } query = get_action('package_search')(context, data_dict) orglist = query['facets']['organization'].keys() #Now get the data concerning the orgs q = Session.query(Group).filter(Group.name.in_(orglist)) values = [] qres = q.all() #Combine in extras, add counts #Make the data look like what we get with a org search for d in qres: dobj = d.as_dict() dobj['extras'] = d.extras dobj['package_count'] = query['facets']['organization'][dobj['name']] values.append(dobj) c.page = h.Page( collection=query['results'], url=pager_url, item_count=query['count'] ) c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search (map) error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {}
def _read(id, limit, group_type): u''' This is common code used by both read and bulk_process''' extra_vars = {} context = { u'model': model, u'session': model.Session, u'user': g.user, u'schema': _db_to_form_schema(group_type=group_type), u'for_view': True, u'extras_as_string': True } q = request.params.get(u'q', u'') # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.q = q # Search within group if g.group_dict.get(u'is_organization'): fq = u' owner_org:"%s"' % g.group_dict.get(u'id') else: fq = u' groups:"%s"' % g.group_dict.get(u'name') extra_vars["q"] = q g.description_formatted = \ h.render_markdown(g.group_dict.get(u'description')) context['return_query'] = True page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != u'page'] sort_by = request.params.get(u'sort', None) def search_url(params): controller = lookup_group_controller(group_type) action = u'bulk_process' if getattr( g, u'action', u'') == u'bulk_process' else u'read' url = h.url_for(u'.'.join([controller, action]), id=id) params = [(k, v.encode(u'utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] return url + u'?' + urlencode(params) def drill_down_url(**by): return h.add_url_param( alternative_url=None, controller=u'group', action=u'read', extras=dict(id=g.group_dict.get(u'name')), new_params=by) extra_vars["drill_down_url"] = drill_down_url def remove_field(key, value=None, replace=None): controller = lookup_group_controller(group_type) return h.remove_url_param( key, value=value, replace=replace, controller=controller, action=u'read', extras=dict(id=g.group_dict.get(u'name'))) extra_vars["remove_field"] = remove_field def pager_url(q=None, page=None): params = list(params_nopage) params.append((u'page', page)) return search_url(params) try: extra_vars["fields"] = fields = [] extra_vars["fields_grouped"] = fields_grouped = {} search_extras = {} for (param, value) in request.params.items(): if param not in [u'q', u'page', u'sort'] \ and len(value) and not param.startswith(u'_'): if not param.startswith(u'ext_'): fields.append((param, value)) q += u' %s: "%s"' % (param, value) if param not in fields_grouped: fields_grouped[param] = [value] else: fields_grouped[param].append(value) else: search_extras[param] = value # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.fields = fields g.fields_grouped = fields_grouped facets = OrderedDict() default_facet_titles = { u'organization': _(u'Organizations'), u'groups': _(u'Groups'), u'tags': _(u'Tags'), u'res_format': _(u'Formats'), u'license_id': _(u'Licenses') } for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles _update_facet_titles(facets, group_type) extra_vars["facet_titles"] = facets data_dict = { u'q': q, u'fq': fq, u'include_private': True, u'facet.field': facets.keys(), u'rows': limit, u'sort': sort_by, u'start': (page - 1) * limit, u'extras': search_extras } context_ = dict((k, v) for (k, v) in context.items() if k != u'schema') query = get_action(u'package_search')(context_, data_dict) extra_vars["page"] = h.Page( collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.group_dict['package_count'] = query['count'] extra_vars["search_facets"] = g.search_facets = query['search_facets'] extra_vars["search_facets_limits"] = g.search_facets_limits = {} for facet in g.search_facets.keys(): limit = int( request.params.get(u'_%s_limit' % facet, config.get(u'search.facets.default', 10))) g.search_facets_limits[facet] = limit extra_vars["page"].items = query['results'] extra_vars["sort_by_selected"] = sort_by except search.SearchError as se: log.error(u'Group search error: %r', se.args) extra_vars["query_error"] = True extra_vars["page"] = h.Page(collection=[]) # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.facet_titles = facets g.page = extra_vars["page"] extra_vars["group_type"] = group_type _setup_template_variables(context, {u'id': id}, group_type=group_type) return extra_vars
def search(self): from ckan.lib.search import SearchError package_type = self._guess_package_type() if package_type == 'search': package_type = 'dataset' try: context = {'model': model, 'user': c.user or c.author, 'auth_user_obj': c.userobj} check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False page = self._page_number() params_nopage = self._params_nopage() # Commenting below parts as it doesn't seem to be used # def drill_down_url(alternative_url=None, **by): # return h.add_url_param(alternative_url=alternative_url, # controller='package', action='search', # new_params=by) # # c.drill_down_url = drill_down_url self._set_remove_field_function() sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return self._search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return self._search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) self._set_other_links() try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} #limit = g.datasets_per_page fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value self._set_filters_are_selected_flag() limit = self._allowed_num_of_items(search_extras) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'auth_user_obj': c.userobj} if package_type and package_type != 'dataset': # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) else: # Unless changed via config options, don't show non standard # dataset types on the default search page if not asbool(config.get('ckan.search.show_all_types', 'False')): fq += ' +dataset_type:dataset' facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in g.facets: if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets self._which_tab_is_selected(search_extras) self._performing_search(q, fq, facets.keys(), limit, page, sort_by, search_extras, pager_url, context) except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True c.facets = {} c.search_facets = {} c.page = h.Page(collection=[])