Beispiel #1
0
        c.sort_by = _sort_by

        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0]
                                for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))

            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
Beispiel #2
0
class MultieditController(PackageController):
    def __before__(self, action, **params):
        super(MultieditController, self).__before__(action, **params)

    # Checks that user is logged in. We need users api key to perform updates.
    def authorize(self):
        try:
            context = {
                'model': model,
                'user': c.user,
                'auth_user_obj': c.userobj
            }
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))
        return model.User.get(c.user)

    def get_dataset_fields(self):
        fields = model.Package.get_fields(core_only=True)

        scheming_schema = scheming_get_dataset_schema(
            'dataset')['dataset_fields']

        scheming_fields = []
        for field in scheming_schema:
            scheming_fields.append(field['field_name'].encode('utf8'))

        # Remove duplicate fields, since scheming can contain fields named similarly to CKAN core fields
        for field in scheming_fields:
            if field not in fields:
                fields.append(field)

        log.info(fields)
        return fields

    def render_package_form(self):

        data = dict(request.params)

        errors = {}
        error_summary = {}
        c.licenses = [('', '')] + model.Package.get_license_options()
        # c.groups_authz = []

        # query = model.Session.query(model.Group)
        # groups = set(query.all())
        # if config.get('ckan.auth.profile', '') == 'publisher':
        #     c.groups_available = [{'id':group.id,'name':group.name} for group in c.userobj.get_groups('organization')]#c.userobj and c.userobj.get_groups('organization') or []
        #     c.groups_available.extend([{'id':group.id,'name':group.name} for group in groups if not filter(lambda org: org['id'] == group.id, c.groups_available)])
        # else:
        #     c.groups_available = [{'id':group.id,'name':group.name} for group in groups]

        # c.group_packages = {}
        # for group in c.groups_available:
        #     pkg_ids = toolkit.get_action('package_list')(data_dict={'id': group['id'], 'object_type': 'dataset'})
        #     c.group_packages[group['name']] = pkg_ids

        extra_vars = {
            'data': data,
            'errors': errors,
            'error_summary': error_summary,
            'stage': ['complete', 'complete', 'complete']
        }

        return render(self._package_form(), extra_vars=extra_vars)

    def limit(self, mode='datasets', limit='0'):

        if limit == '0':
            c.limit = limit = config.get('multiedit.limit', '100')

        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            params = set(params_nopage)
            params |= set(by.items())
            if alternative_url:
                return url_with_params(alternative_url, params)
            return search_url(params, mode + '/')

        c.drill_down_url = drill_down_url

        def remove_field(key, value):
            params = list(params_nopage)
            params.remove((key, value))
            return search_url(params, mode + '/')

        c.remove_field = remove_field

        return self.perform_query(mode, int(limit))

    def perform_query(self, mode, limit):
        from ckan.lib.search import SearchError

        package_type = self._guess_package_type()

        c.limit = limit

        try:
            context = {'model': model, 'user': c.user or c.author}
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))

        q = c.q = request.params.get('q',
                                     u'')  # unicode format (decoded from utf8)
        c.query_error = False
        try:
            page = int(request.params.get('page', 1))
        except ValueError, e:
            abort(400, ('"page" parameter must be an integer'))

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params)

        c.sort_by = _sort_by
        if sort_by is None:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [
                field.split()[0] for field in sort_by.split(',')
            ]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params,
                              package_type).replace('/multiedit', '', 1)

        def no_limit_url():
            url = h.url_for(
                controller='ckanext.multiedit.controller:MultieditController',
                mode=mode,
                action='nolimit')
            params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) \
                            for k, v in list(params_nopage)]
            if len(params) > 0:
                return url + u'?' + urlencode(params)
            else:
                return url

        c.no_limit_url = no_limit_url

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.query_error = False
            user = self.authorize()
            c.apikey = user.apikey

            c.core_fields = self.get_dataset_fields()

            c.fields = []
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                    else:
                        search_extras[param] = value

            fq += ' capacity:"public"'
            context = {
                'model': model,
                'session': model.Session,
                'user': c.user or c.author,
                'for_view': True
            }

            data_dict = {
                'q': q,
                'fq': fq,
                'facet.field': g.facets,
                'rows': int(limit),
                'start': (int(page) - 1) * int(limit),
                'sort': sort_by,
                'extras': search_extras
            }

            query = get_action('package_search')(context, data_dict)

            if query['count'] is not None:
                c.query_count = query['count']
            else:
                c.query_count = 0

            #if query['count'] > int(limit):
            #    c.too_many_results = True

            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page = h.Page(collection=query['results'],
                            page=int(page),
                            url=pager_url,
                            item_count=query['count'],
                            items_per_page=int(limit))
            c.page.items = query['results']
            c.results = c.page.items

            c.form = self.render_package_form()

        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.query_count = 0
            c.facets = {}
            c.page = h.Page(collection=[])
Beispiel #3
0
    def _add_dataset_search(self, showcase_id, showcase_name):
        '''
        Search logic for discovering datasets to add to a showcase.
        '''

        from ckan.lib.search import SearchError

        package_type = 'dataset'

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        try:
            page = self._get_page_number(request.params)
        except AttributeError:
            # in CKAN >= 2.5 _get_page_number has been moved
            page = h.get_page_number(request.params)

        limit = int(config.get('ckan.datasets_per_page', 20))

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package',
                                   action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key,
                                      value=value,
                                      replace=replace,
                                      controller='package',
                                      action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return self._search_url(params, showcase_name)

        c.sort_by = _sort_by
        if sort_by is None:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [
                field.split()[0] for field in sort_by.split(',')
            ]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return self._search_url(params, showcase_name)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {
                'model': model,
                'session': model.Session,
                'user': c.user or c.author,
                'for_view': True,
                'auth_user_obj': c.userobj
            }

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not tk.asbool(
                        config.get('ckan.search.show_all_types', 'False')):
                    fq += ' +dataset_type:dataset'

            # Only search for packages that aren't already associated with the
            # Showcase
            associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase(
                showcase_id)
            # flatten resulting list to space separated string
            if associated_package_ids:
                associated_package_ids_str = \
                    ' OR '.join([id[0] for id in associated_package_ids])
                fq += ' !id:({0})'.format(associated_package_ids_str)

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            # for CKAN-Versions that do not provide the facets-method from
            # helper-context, import facets from ckan.common
            if hasattr(h, 'facets'):
                current_facets = h.facets()
            else:
                from ckan.common import g
                current_facets = g.facets

            for facet in current_facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(collection=query['results'],
                            page=page,
                            url=pager_url,
                            item_count=query['count'],
                            items_per_page=limit)
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = h.Page(collection=[])
    def search(self):
        package_type = self._guess_package_type()

        try:
            context = {'model': model, 'user': c.user or c.author, 'auth_user_obj': c.userobj}
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))

        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = self._get_page_number(request.params)
        limit = g.datasets_per_page
        params_nopage = [(k, v) for k, v in request.params.items() if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(
                alternative_url=alternative_url,
                controller='package',
                action='search',
                new_params=by
            )

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            params = params_nosort[:]
            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        if param != 'organization':
                            fq += ' %s:"%s"' % (param, value)
                        else:
                            fq += custom_organization_filter(value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user or c.author, 'for_view': True,
                       'auth_user_obj': c.userobj}

            if package_type and package_type != 'dataset':
                fq += ' +dataset_type:{type}'.format(type=package_type)
            elif not asbool(config.get('ckan.search.show_all_types', 'False')):
                fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = h.Page(collection=[])
Beispiel #5
0
class ECPortalUserController(core_user.UserController):
    def logged_in(self):
        #override the logged_in method after the access was granted or denied
        #if login was successfull check the group membership
        if c.user:
            util.add_missing_user_to_all_groups()
        return super(ECPortalUserController, self).logged_in()

    def login_user(self, tup):
        if tup:
            user = tup[0]

            user_object = model.User.get(user)
            if not user_object:
                user_object = model.User.get('api')
                c.userobj = user_object
                c.user = '******'

                try:
                    # Create user
                    user_attributes = tup[1]
                    user_name = user
                    user_email = user_attributes["email"].text
                    user_password = "******"
                    user_fullname = user_attributes[
                        "firstName"].text + " " + user_attributes[
                            "lastName"].text

                    h.get_action(
                        'user_create', {
                            "name": user_name,
                            "email": user_email,
                            "password": user_password,
                            "fullname": user_fullname
                        })

                    rememberer = request.environ['repoze.who.plugins'][
                        'friendlyform']
                    identity = {'repoze.who.userid': user}
                    response.headerlist += rememberer.remember(
                        request.environ, identity)
                except logic.ActionError:
                    base.abort(401, _('Error while creating new user'))
            else:
                c.userobj = user_object
                c.user = user

                rememberer = request.environ['repoze.who.plugins'][
                    'friendlyform']
                identity = {'repoze.who.userid': user}
                response.headerlist += rememberer.remember(
                    request.environ, identity)

            self.logged_in()

    def read(self, id=None):
        context = {
            'model': model,
            'session': model.Session,
            'user': c.user or c.author,
            'auth_user_obj': c.userobj,
            'for_view': True,
            'return_minimal': True,
            'save': 'save' in request.params
        }
        data_dict = {'id': id, 'user_obj': c.userobj}

        context['with_related'] = True

        self._setup_user_template_variables(context, data_dict)

        if context['save']:
            request.POST.pop('save')
            data = ecportal_logic.transform_to_data_dict(request.POST)

            credential_validator = Validator({
                'contact_mailbox': [email]
                #'contact_phone_number': [IntPhoneNumberRule]
            })
            errors = {}
            succeed = credential_validator.validate(data, results=errors)

            if succeed is True:
                contact_info = Package_contact_info.get_by_user(c.userobj.id)
                if not contact_info:
                    contact_info = Package_contact_info(c.userobj.id)

                contact_info.from_dict(data)
                contact_info.save()
                h.flash_success(_('ecodp.user.save.success'))
            elif errors:
                h.flash_error(_('ecodp.user.save.error'))

            c.user_dict.update(data)

            c.errors = errors

        # The legacy templates have the user's activity stream on the user
        # profile page, new templates do not.
        if h.asbool(config.get('ckan.legacy_templates', False)):
            c.user_activity_stream = get_action('user_activity_list_html')(
                context, {
                    'id': c.user_dict['id']
                })

        #vars = {'data': {}, 'errors': {}, 'error_summary': {}}
        #c.form = base.render('user/edit_contact_info_form.html', extra_vars=vars)
        return base.render('user/read.html')

    def read_contact_info(self, id=None):
        context = {
            'model': model,
            'session': model.Session,
            'user': c.user or c.author,
            'auth_user_obj': c.userobj,
            'for_view': True,
            'return_minimal': True,
            'save': 'save' in request.params
        }
        data_dict = {'id': id, 'user_obj': c.userobj}

        context['with_related'] = True

        self._setup_user_template_variables(context, data_dict)

        if context['save']:
            request.POST.pop('save')
            data = ecportal_logic.transform_to_data_dict(request.POST)

            credential_validator = Validator({
                'contact_mailbox': [email]
                #'contact_phone_number': [IntPhoneNumberRule]
            })
            errors = {}
            succeed = credential_validator.validate(data, results=errors)

            if succeed is True:
                contact_info = Package_contact_info.get_by_user(c.userobj.id)
                if not contact_info:
                    contact_info = Package_contact_info(c.userobj.id)

                contact_info.from_dict(data)
                contact_info.save()
                h.flash_success(_('ecodp.user.save.success'))
            elif errors:
                h.flash_error(_('ecodp.user.save.error'))

            c.user_dict.update(data)

            c.errors = errors

        # The legacy templates have the user's activity stream on the user
        # profile page, new templates do not.
        if h.asbool(config.get('ckan.legacy_templates', False)):
            c.user_activity_stream = get_action('user_activity_list_html')(
                context, {
                    'id': c.user_dict['id']
                })

        #vars = {'data': {}, 'errors': {}, 'error_summary': {}}
        #c.form = base.render('user/edit_contact_info_form.html', extra_vars=vars)
        return base.render('user/read_contact_info.html')

    def dashboard(self):
        def query_for_datasets():
            query = ''

            if not new_authz.is_sysadmin(c.user):
                organisation_ids = self.__get_users_organizations_ids(c.user)

                if not organisation_ids:
                    # return an empty query result with the needed (i.e.: accessed later) elements
                    return {'results': [], 'count': 0}

                query = build_owner_filter(organisation_ids)

            if c.q != u'':
                search = c.q.strip().split(' ')
                result = ''
                if request.params.get('ext_boolean') == 'any':
                    result = build_search_for_any_words(search)
                elif request.params.get('ext_boolean') == 'all':
                    result = build_search_for_all_words(search)
                elif request.params.get('ext_boolean') == 'exact':
                    result = '("%s")' % (c.q)
                    log.info("%s" % (request.params.get('ext_boolean')))

                query = query + " + title:" + result

            return query

        log.debug("Entering MDT dashboard")

        c.is_sysadmin = new_authz.is_sysadmin(c.user)

        context = {
            'for_view': True,
            'user': c.user or c.author,
            'auth_user_obj': c.userobj,
            'return_minimal': True,
            'ignore_capacity_check': True  # Display also private datasets
        }
        user_data = {"id": c.userobj.id, 'user_obj': c.userobj}

        try:
            user_dict = logic.get_action('user_show')(context, user_data)
        except logic.NotFound:
            base.abort(404, _('User not found'))
        except logic.NotAuthorized:
            base.abort(401, _('Not authorized to see this page'))

        c.user_dict = user_dict
        c.is_myself = user_dict['name'] == c.user
        c.about_formatted = h.render_markdown(user_dict['about'])
        c.search_choice = request.params.get('ext_boolean')

        # Save the used search paramters in a proper format to the exchange variable
        user = user_dict['name']

        log.debug("Start getting the datasets")
        # ---------------------------------------------------------------------------------------------------------------
        # Get the datasets
        from ckan.lib.search import SearchError

        package_type = self._guess_package_type()

        if request.GET.get('ext_boolean') in ['all', 'any', 'exact']:
            session['ext_boolean'] = request.GET['ext_boolean']
            session.save()

        # unicode format (decoded from utf8)
        c.q = request.params.get('q', u'')
        c.query_error = False

        q = query_for_datasets()

        try:
            page = int(request.params.get('page', 1))
        except ValueError, e:
            abort(400, ('"page" parameter must be an integer'))
        limit = g.datasets_per_page

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        new_params_nopage = []
        for key, value in params_nopage:
            if key == 'eurovoc_domains':
                new_params_nopage.append(('groups', value))
            else:
                new_params_nopage.append((key, value))

        params_nopage = new_params_nopage

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package',
                                   action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(
                key,
                value=value,
                replace=replace,
                controller=
                'ckanext.ecportal.controllers.user:ECPortalUserController',
                action='dashboard')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', 'views_total desc')
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [
                field.split()[0] for field in sort_by.split(',')
            ]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(
            core_package._encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''

            # if request.params.get('private'):
            #    fq = 'private: %s' % request.params.get('private')

            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort', 'id'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        # paramFQ = 'groups' if (param == 'eurovoc_domains') else param;
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

                        # if package_type and package_type != 'dataset':
                        # Only show datasets of this particular type
                        #     fq += ' +dataset_type:{type}'.format(type=package_type)
                        #  else:
                        # Unless changed via config options, don't show non standard
                        # dataset types on the default search page
                        #     if not asbool(config.get('ckan.search.show_all_types', 'False')):
                        #         fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dashboard_facets(facets, package_type)

            c.facet_titles = facets

            facet_fields = facets.keys()
            facet_fields.append('private')
            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facet_fields,
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            if sort_by == 'modified_date desc':
                # This is the customized part for ODP-570
                # add the group parameter to the solr query
                data_dict['group'] = 'true'
                data_dict['group.query'] = [
                    '-organization:estat AND -organization:comp AND -organization:grow',
                    'organization:estat', 'organization:comp',
                    'organization:grow'
                ]
                data_dict['group.format'] = 'simple'
                data_dict['rows'] = 2147483646

                query = get_action('custom_package_search')(context, data_dict)

                cached_result = []

                for name, group in query['groups'].iteritems():
                    cached_result += group['doclist']['docs']

                start = (page - 1) * limit
                result_list = customsearch.check_solr_result(
                    context, cached_result[start:], limit)

            else:
                query = get_action('package_search')(context, data_dict)
                result_list = query['results']

            c.sort_by_selected = query['sort']

            c.page = page_util.Page(collection=result_list,
                                    page=page,
                                    url=pager_url,
                                    item_count=query['count'],
                                    items_per_page=limit)
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = result_list
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = page_util.Page(collection=[])
Beispiel #6
0
    def _add_dataset_search(self, showcase_id, showcase_name):
        '''
        Search logic for discovering datasets to add to a showcase.
        '''

        from ckan.lib.search import SearchError

        package_type = 'dataset'

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        try:
            page = self._get_page_number(request.params)
        except AttributeError:
            # in CKAN >= 2.5 _get_page_number has been moved
            page = h.get_page_number(request.params)

        limit = int(config.get('ckan.datasets_per_page', 20))

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package', action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace,
                                      controller='package', action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return self._search_url(params, showcase_name)

        c.sort_by = _sort_by
        if sort_by is None:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0]
                                for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return self._search_url(params, showcase_name)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user or c.author, 'for_view': True,
                       'auth_user_obj': c.userobj}

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not tk.asbool(config.get('ckan.search.show_all_types',
                                            'False')):
                    fq += ' +dataset_type:dataset'

            # Only search for packages that aren't already associated with the
            # Showcase
            associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase(showcase_id)
            # flatten resulting list to space separated string
            if associated_package_ids:
                associated_package_ids_str = \
                    ' OR '.join([id[0] for id in associated_package_ids])
                fq += ' !id:({0})'.format(associated_package_ids_str)

            facets = OrderedDict()

            default_facet_titles = {
                    'organization': _('Organizations'),
                    'groups': _('Groups'),
                    'tags': _('Tags'),
                    'res_format': _('Formats'),
                    'license_id': _('Licenses'),
                    }

            # for CKAN-Versions that do not provide the facets-method from
            # helper-context, import facets from ckan.common
            if hasattr(h, 'facets'):
                current_facets = h.facets()
            else:
                from ckan.common import g
                current_facets = g.facets

            for facet in current_facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = h.Page(collection=[])
Beispiel #7
0
def url_with_params(url, params):
    params = _encode_params(params)
    return url + u'?' + urlencode(params)
Beispiel #8
0
    def search(self):
        from ckan.lib.search import SearchError, SearchQueryError

        package_type = self._guess_package_type()

        try:
            context = {
                'model': model,
                'user': c.user,
                'auth_user_obj': c.userobj
            }
            check_access('site_read', context)
        except NotAuthorized:
            abort(403, _('Not authorized to see this page'))

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = h.get_page_number(request.params)

        limit = int(config.get('ckan.datasets_per_page', 20))

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package',
                                   action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key,
                                      value=value,
                                      replace=replace,
                                      controller='package',
                                      action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [
                field.split()[0] for field in sort_by.split(',')
            ]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))
        timerange = []
        collection_params = ''

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            c.parameters_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort', 'timerange_start', 'timerange_end'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                        if param == 'collection_name':
                            collection_params = value
                    else:
                        search_extras[param] = value

                from ckanext.opensearch import converters
                from ckanext.opensearch.config import PARAMETERS

                if param == 'timerange_start':
                    if len(value) > 0:
                        c.parameters_grouped[param] = value
                        for converter in PARAMETERS['collection'][param].get(
                                "converters", []):
                            value = getattr(converters, converter)(value)
                        c.fields.append((param, value))
                        fq += " %s:%s" % (param, value)
                        timerange.append(param)

                if param == 'timerange_end':
                    if len(value) > 0:
                        c.parameters_grouped[param] = value
                        for converter in PARAMETERS['collection'][param].get(
                                "converters", []):
                            value = getattr(converters, converter)(value)
                        c.fields.append((param, value))
                        fq += " %s:%s" % (param, value)

            context = {
                'model': model,
                'session': model.Session,
                'user': c.user,
                'for_view': True,
                'auth_user_obj': c.userobj
            }

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not asbool(config.get('ckan.search.show_all_types',
                                         'False')):
                    fq += ' +dataset_type:dataset'

            facets = OrderedDict()
            search_parameters = OrderedDict()

            default_facet_titles = {
                'collection_name': _('Data Collections'),
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            sentinel_facet_titles = {
                'FamilyName': _('Family Name'),
                'ProductType': _('Product Type'),
                'OrbitDirection': _('Orbit Direction'),
                'Swath': _('Acquisition Mode'),
                'TransmitterReceiverPolarisation': _('Polarisation'),
            }

            default_search_titles = {
                'timerange_start': _('Timerange Start'),
                'timerange_end': _('Timerange End'),
            }

            collection_names = [
                'Sentinel-1 Level-1 (SLC)', 'Sentinel-1 Level-1 (GRD)',
                'Sentinel-1 Level-2 (OCN)', 'Sentinel-2 Level-1C',
                'Sentinel-2 Level-2A', 'Sentinel-3 SRAL Level-1 SRA'
            ]

            for facet in h.facets():
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            for search_param in helpers.search_params():
                if search_param in default_search_titles:
                    search_parameters[search_param] = default_search_titles[
                        search_param]

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            if collection_params not in collection_names:
                for f in facets:
                    if f in sentinel_facet_titles:
                        del facets[f]

            c.facet_titles = facets

            c.search_parameters = search_parameters

            data_dict = {
                'q':
                q,
                'fq':
                fq.strip(),
                'facet.field':
                facets.keys(),
                'rows':
                limit,
                'start': (page - 1) * limit,
                'sort':
                sort_by,
                'extras':
                search_extras,
                'include_private':
                asbool(config.get('ckan.search.default_include_private',
                                  True)),
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(collection=query['results'],
                            page=page,
                            url=pager_url,
                            item_count=query['count'],
                            items_per_page=limit)

            c.search_facets = query['search_facets']
            c.search_search_parameters = search_parameters
            c.page.items = query['results']
        except SearchQueryError, se:
            # User's search parameters are invalid, in such a way that is not
            # achievable with the web interface, so return a proper error to
            # discourage spiders which are the main cause of this.
            log.info('Dataset search query rejected: %r', se.args)
            abort(
                400,
                _('Invalid search query: {error_message}').format(
                    error_message=str(se)))
Beispiel #9
0
class package(PackageController):

    def search(self):
        '''
        See ckan.controllers.package:PackageController for source, this is HACKed to fix broken scandic parameters.
        '''
        from ckan.lib.search import SearchError

        package_type = self._guess_package_type()

        try:
            context = {'model': model, 'user': c.user or c.author}
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        try:
            page = int(request.params.get('page', 1))
        except ValueError, e:
            abort(400, ('"page" parameter must be an integer'))
        limit = g.datasets_per_page

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package', action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            # HACK When removing scandic error prone values, check for their broken couterpart's presence in the URL parameters.
            #      If found, replace given fixed value with broken one, otherwise remove_url_param won't find it and crashes. 
            if value == u'Kaavat ja kiinteistöt' and ('categories', u'Kaavat ja kiinteist' + u"\ufffd" + 't') in request.params.items():
                value = u'Kaavat ja kiinteist' + u"\ufffd" + 't'
            if value == u'Työ ja elinkeinot' and ('categories', u'Ty' + u"\ufffd" + u' ja elinkeinot') in request.params.items():
                value = u'Ty' + u"\ufffd" + u' ja elinkeinot'
            if value == u'Väestö' and ('categories', u'V' + u"\ufffd" + u'est' + u"\ufffd") in request.params.items():
                value = u'V' + u"\ufffd" + u'est' + u"\ufffd"
            if value == u'Ympäristö ja luonto' and ('categories', u'Ymp' + u"\ufffd" + u'rist' + u"\ufffd" + u' ja luonto') in request.params.items():
                value = u'Ymp' + u"\ufffd" + u'rist' + u"\ufffd" + u' ja luonto'
                
            return h.remove_url_param(key, value=value, replace=replace,
                                  controller='package', action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if sort_by is None:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0]
                                for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        # HACK Hardcoded replace for parameters with scandics that sometimes get replaced with unknown char (u"\ufffd").
                        if param == 'categories':
                            if value == u'Kaavat ja kiinteist' + u"\ufffd" + 't':
                                value = u'Kaavat ja kiinteistöt'
                            if value == u'Ty' + u"\ufffd" + u' ja elinkeinot':
                                value = u'Työ ja elinkeinot'
                            if value == u'V' + u"\ufffd" + u'est' + u"\ufffd":
                                value = u'Väestö'
                            if value == u'Ymp' + u"\ufffd" + u'rist' + u"\ufffd" + u' ja luonto':
                                value = u'Ympäristö ja luonto'
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user or c.author, 'for_view': True}

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not asbool(config.get('ckan.search.show_all_types', 'False')):
                    fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                    'organization': _('Organizations'),
                    'groups': _('Groups'),
                    'tags': _('Tags'),
                    'res_format': _('Formats'),
                    'license_id': _('License'),
                    }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )
            c.facets = query['facets']            
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = h.Page(collection=[])
    def search(self):
        from ckan.lib.search import SearchError, SearchQueryError

        package_type = self._guess_package_type()

        extra_vars = {'dataset_type': package_type}

        try:
            context = {
                'model': model,
                'user': c.user,
                'auth_user_obj': c.userobj
            }
            check_access('site_read', context)
        except NotAuthorized:
            abort(403, _('Not authorized to see this page'))

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = h.get_page_number(request.params)

        limit = int(config.get('ckan.datasets_per_page', 20))

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package',
                                   action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key,
                                      value=value,
                                      replace=replace,
                                      controller='package',
                                      action='search',
                                      alternative_url=package_type)

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [
                field.split()[0] for field in sort_by.split(',')
            ]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {
                'model': model,
                'session': model.Session,
                'user': c.user,
                'for_view': True,
                'auth_user_obj': c.userobj
            }

            # Unless changed via config options, don't show other dataset
            # types any search page. Potential alternatives are do show them
            # on the default search page (dataset) or on one other search page
            search_all_type = config.get('ckan.search.show_all_types',
                                         'dataset')
            search_all = False

            try:
                # If the "type" is set to True or False, convert to bool
                # and we know that no type was specified, so use traditional
                # behaviour of applying this only to dataset type
                search_all = asbool(search_all_type)
                search_all_type = 'dataset'
            # Otherwise we treat as a string representing a type
            except ValueError:
                search_all = True

            if not package_type:
                package_type = 'dataset'

            if not search_all or package_type != search_all_type:
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in h.facets():
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            # Date interval search

            start_date = None
            end_date = None

            if request.params.get('_start_date'):
                start_date = extract_date(request.params.get('_start_date'))
                if start_date:
                    extra_vars['start_date'] = request.params.get(
                        '_start_date')

            if request.params.get('_end_date'):
                end_date = extract_date(request.params.get('_end_date'))
                if end_date:
                    extra_vars['end_date'] = request.params.get('_end_date')

            if start_date or end_date:
                if not start_date:
                    start_date = datetime(year=1900, month=1, day=1)
                if not end_date:
                    end_date = datetime(year=2100,
                                        month=1,
                                        day=1,
                                        hour=23,
                                        minute=59,
                                        second=59,
                                        microsecond=999999)
                else:
                    end_date = datetime_to_utc(end_date)
                    end_date = end_date.replace(hour=23,
                                                minute=59,
                                                second=59,
                                                microsecond=999999)

                fq = fq + ' +metadata_modified:[%s TO %s]' % (
                    datetime_to_utc(start_date).strftime('%Y-%m-%dT%H:%M:%SZ'),
                    datetime_to_utc(end_date).strftime('%Y-%m-%dT%H:%M:%SZ'))

            data_dict = {
                'q':
                q,
                'fq':
                fq.strip(),
                'facet.field':
                facets.keys(),
                'rows':
                limit,
                'start': (page - 1) * limit,
                'sort':
                sort_by,
                'extras':
                search_extras,
                'include_private':
                asbool(config.get('ckan.search.default_include_private',
                                  True)),
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(collection=query['results'],
                            page=page,
                            url=pager_url,
                            item_count=query['count'],
                            items_per_page=limit)
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchQueryError as se:
            # User's search parameters are invalid, in such a way that is not
            # achievable with the web interface, so return a proper error to
            # discourage spiders which are the main cause of this.
            log.info('Dataset search query rejected: %r', se.args)
            abort(
                400,
                _('Invalid search query: {error_message}').format(
                    error_message=str(se)))
        except SearchError as se:
            # May be bad input from the user, but may also be more serious like
            # bad code causing a SOLR syntax error, or a problem connecting to
            # SOLR
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.search_facets = {}
            c.page = h.Page(collection=[])
        except NotAuthorized:
            abort(403, _('Not authorized to see this page'))

        c.search_facets_limits = {}
        for facet in c.search_facets.keys():
            try:
                limit = int(
                    request.params.get(
                        '_%s_limit' % facet,
                        int(config.get('search.facets.default', 10))))
            except ValueError:
                abort(
                    400,
                    _('Parameter "{parameter_name}" is not '
                      'an integer').format(parameter_name='_%s_limit' % facet))
            c.search_facets_limits[facet] = limit

        self._setup_template_variables(context, {}, package_type=package_type)

        extra_vars['dataset_type'] = package_type
        return render(self._search_template(package_type),
                      extra_vars=extra_vars)
Beispiel #11
0
    def search(self):
        package_type = self._guess_package_type()

        try:
            context = {
                'model': model,
                'user': c.user or c.author,
                'auth_user_obj': c.userobj
            }
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))

        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = self._get_page_number(request.params)
        limit = g.datasets_per_page
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package',
                                   action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key,
                                      value=value,
                                      replace=replace,
                                      controller='package',
                                      action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            params = params_nosort[:]
            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [
                field.split()[0] for field in sort_by.split(',')
            ]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        if param != 'organization':
                            fq += ' %s:"%s"' % (param, value)
                        else:
                            fq += custom_organization_filter(value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {
                'model': model,
                'session': model.Session,
                'user': c.user or c.author,
                'for_view': True,
                'auth_user_obj': c.userobj
            }

            if package_type and package_type != 'dataset':
                fq += ' +dataset_type:{type}'.format(type=package_type)
            elif not asbool(config.get('ckan.search.show_all_types', 'False')):
                fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(collection=query['results'],
                            page=page,
                            url=pager_url,
                            item_count=query['count'],
                            items_per_page=limit)
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = h.Page(collection=[])
Beispiel #12
0
class DashboardsController(PackageController):
    def organization_dashboard(self, name):

        org = get_action('organization_show')({}, {'id': name, 'include_extras': True})

        if 'orgdashboards_is_active' in org and org['orgdashboards_is_active'] == '0':
            return plugins.toolkit.render('dashboards/snippets/not_active.html')

        from ckan.lib.search import SearchError

        package_type = 'dataset'

        try:
            context = {'model': model, 'user': c.user or c.author,
                       'auth_user_obj': c.userobj}
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        if p.toolkit.check_ckan_version(min_version='2.5.0', max_version='2.5.3'):
            page = self._get_page_number(request.params)
        else:
            page = h.get_page_number(request.params)

        try:
            limit = int(org['orgdashboards_datasets_per_page'])
        except KeyError, ValueError:
            limit = int(config.get('ckanext.orgdashboards.datasets_per_page', '6'))

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package', action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace,
                                      controller='package', action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0]
                                for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user or c.author, 'for_view': True,
                       'auth_user_obj': c.userobj}

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not asbool(
                        config.get('ckan.search.show_all_types', 'False')):
                    fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Add 'author' facet
            facets['author'] = _('Authors')

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets
            c.name = name

            fq += ' +organization:"{}"'.format(name)

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            query = get_action('package_search')(context, data_dict)

            # Override the "author" list, to include full name authors
            query['search_facets']['author']['items'] = self._get_full_name_authors(context, name)

            c.sort_by_selected = query['sort']

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = h.Page(collection=[])
    def search(self):
        from ckan.lib.search import SearchError, SearchQueryError

        package_type = self._guess_package_type()

        try:
            context = {'model': model, 'user': c.user,
                       'auth_user_obj': c.userobj}
            check_access('site_read', context)
        except NotAuthorized:
            abort(403, _('Not authorized to see this page'))

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = h.get_page_number(request.params)

        limit = int(config.get('ckan.datasets_per_page', 20))

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='ckanext.sixodp_showcase.controller:Sixodp_ShowcaseController', action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace,
                                      controller='ckanext.sixodp_showcase.controller:Sixodp_ShowcaseController',
                                      action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.
            Each entry in the list is a 2-tuple: (fieldname, sort_order)
            eg - [('metadata_modified', 'desc'), ('name', 'asc')]
            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0]
                                for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user, 'for_view': True,
                       'auth_user_obj': c.userobj}

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not asbool(
                        config.get('ckan.search.show_all_types', 'False')):
                    fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in h.facets():
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in ckan.plugins.PluginImplementations(ckan.plugins.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            # Set the custom default sort parameter here
            # Might need a rewrite when ckan is updated
            if not sort_by:
                sort_by = 'metadata_created desc'

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras,
                'include_private': asbool(config.get(
                    'ckan.search.default_include_private', True)),
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchQueryError, se:
            # User's search parameters are invalid, in such a way that is not
            # achievable with the web interface, so return a proper error to
            # discourage spiders which are the main cause of this.
            log.info('Dataset search query rejected: %r', se.args)
            abort(400, _('Invalid search query: {error_message}')
                  .format(error_message=str(se)))