Beispiel #1
0
    def dashboard_organizations(self):
        context = {'model': model, 'session': model.Session,
                   'for_view': True, 'user': c.user or c.author,
                   'auth_user_obj': c.userobj}
        
        data_dict = {'user_obj': c.userobj}
        self._setup_template_variables(context, data_dict)
        (user_orgs, usr_suborgs) = get_user_toporgs(c.userobj.id) 
        
        facets = OrderedDict()

        #Add the organization facet to get the number of records for each organization
        facets['organization'] = _('Organizations')

        data_dict = {
                'facet.field': facets.keys(),
        }

        query = get_action('package_search')(context, data_dict)
        c.org_pkg_count = query['facets'].get('organization')
        c.top_orgs_items = user_orgs
        c.suborgs_items = usr_suborgs
        
               
        return render('user/dashboard_organizations.html')
                    
def cioos_get_facets(package_type='dataset'):
    ''' get all dataset for the given package type, including private ones.
        This function works similarly to code found in ckan/ckan/controllers/package.py
        in that it does a search of all datasets and populates the following
        globals for later use:
            c.facet_titles
            c.search_facets
    '''
    facets = OrderedDict()

    default_facet_titles = {
        'organization': _('Organizations'),
        'groups': _('Groups'),
        'tags': _('Tags'),
        'res_format': _('Formats'),
        'license_id': _('Licenses'),
    }

    for facet in toolkit.h.facets():
        if facet in default_facet_titles:
            facets[facet] = default_facet_titles[facet]
        else:
            facets[facet] = facet

    # Facet titles
    for plugin in p.PluginImplementations(p.IFacets):
        facets = plugin.dataset_facets(facets, package_type)

    c.facet_titles = facets

    data_dict = {
        'facet.field':
        facets.keys(),
        'rows':
        0,
        'include_private':
        asbool(config.get('ckan.search.default_include_private', True)),
    }

    context = {
        'model': model,
        'session': model.Session,
        'user': c.user,
        'for_view': True,
        'auth_user_obj': c.userobj
    }

    query = get_action('package_search')(context, data_dict)
    c.search_facets = query['search_facets']
Beispiel #3
0
def datasets_without_resources(organization, include_sub_organizations=False):
    pkg_dicts = []
    query = model.Session.query(model.Package)\
                .filter_by(state='active')\
                .order_by(model.Package.title)
    if organization:
        query = lib.filter_by_organizations(query, organization,
                                            include_sub_organizations)
    for pkg in query.all():
        if len(pkg.resources) != 0 or \
                pkg.extras.get('unpublished', '').lower() == 'true':
            continue
        deleted, url = last_resource_deleted(pkg)
        pkg_dict = OrderedDict((
            ('name', pkg.name),
            ('title', pkg.title),
            ('metadata created', pkg.metadata_created.isoformat()),
            ('metadata modified', pkg.metadata_modified.isoformat()),
            ('last resource deleted',
             deleted.isoformat() if deleted else None),
            ('last resource url', url),
            ('dataset_notes', lib.dataset_notes(pkg)),
        ))
        pkg_dicts.append(pkg_dict)
    return {'table': pkg_dicts}
Beispiel #4
0
def update_report():
    with open(update_log_location) as f:
        content = f.readlines()
    full_history = []
    grouped_history = []
    failed_history = []

    group_dict = {}
    names = {}

    for line in content:
        json_line = json.loads(line)
        id = json_line["id"]
        name = json_line["name"]
        time = json_line["time"]
        success = json_line["success"]
        details = json_line["details"]

        table_line = OrderedDict((("id", id), ("name", name), ("time", time),
                                  ("success", success), ("details", details)))

        group_line = OrderedDict(
            (("time", time), ("success", success), ("details", details)))

        if id in group_dict.keys():
            group_dict[id].append(group_line)
        else:
            group_dict[id] = [group_line]
        names[id] = name

        full_history.append(table_line)
        if success == "Fail":
            failed_history.append(table_line)

    for id in group_dict.keys():
        if len(group_dict[id]) > 10:
            group_dict[id] = group_dict[id][len(group_dict[id]) -
                                            11:len(group_dict[id]) - 1]
        grouped_history.append(
            OrderedDict((("id", id), ("name", names[id]), ("history",
                                                           group_dict[id]))))

    return {
        'table': list(reversed(full_history)),
        'fail_table': failed_history,
        'grouped_table': grouped_history
    }
Beispiel #5
0
def tagless_report(organization, include_sub_organizations=False):
    '''
    Produces a report on packages without tags.
    Returns something like this:
        {
         'table': [
            {'name': 'river-levels', 'title': 'River levels', 'notes': 'Harvested', 'user': '******',
             'created': '2008-06-13T10:24:59.435631'},
            {'name': 'co2-monthly', 'title' 'CO2 monthly', 'notes': '', 'user': '******',
             'created': '2009-12-14T08:42:45.473827'},
            ],
         'num_packages': 56,
         'packages_without_tags_percent': 4,
         'average_tags_per_package': 3.5,
        }
    '''
    # Find the packages without tags
    q = model.Session.query(model.Package) \
        .outerjoin(model.PackageTag) \
        .filter(
        model.PackageTag.id == None  # noqa: E711
    )
    if organization:
        q = lib.filter_by_organizations(q, organization,
                                        include_sub_organizations)
    tagless_pkgs = [
        OrderedDict((
            ('name', pkg.name),
            ('title', pkg.title),
            ('notes', lib.dataset_notes(pkg)),
            ('user', pkg.creator_user_id),
            ('created', pkg.metadata_created.isoformat()),
        )) for pkg in q.slice(0, 100)
    ]  # First 100 only for this demo

    # Average number of tags per package
    q = model.Session.query(model.Package)
    q = lib.filter_by_organizations(q, organization, include_sub_organizations)
    num_packages = q.count()
    q = q.join(model.PackageTag)
    num_taggings = q.count()
    if num_packages:
        average_tags_per_package = round(float(num_taggings) / num_packages, 1)
    else:
        average_tags_per_package = None
    packages_without_tags_percent = lib.percent(len(tagless_pkgs),
                                                num_packages)

    return {
        'table': tagless_pkgs,
        'num_packages': num_packages,
        'packages_without_tags_percent': packages_without_tags_percent,
        'average_tags_per_package': average_tags_per_package,
    }
Beispiel #6
0
def html_datasets_report(organization, include_sub_organizations=False):
    '''
    Returns datasets that only have an HTML link, by organization.
    '''

    # Get packages
    query = model.Session.query(model.Package)\
                .filter_by(state='active')
    if organization:
        query = lib.filter_by_organizations(query, organization,
                                            include_sub_organizations)

    pkgs = query.all()
    # See if HTML
    num_datasets_published = 0
    num_datasets_only_html = 0
    datasets_only_html = []
    # use yield_per, otherwise memory use just goes up til the script is killed
    # by the os.
    for pkg in pkgs:
        if p.toolkit.asbool(pkg.extras.get('unpublished')):
            continue
        num_datasets_published += 1

        formats = set([
            res.format.lower() for res in pkg.resources
            if res.resource_type != 'documentation'
        ])
        if 'html' not in formats:
            continue
        #org = pkg.get_organization().name

        data_formats = formats - set(('asp', '', None))
        if data_formats == set(('html', )):
            num_datasets_only_html += 1
            datasets_only_html.append(pkg)

    rows = []
    for pkg in datasets_only_html:
        row = OrderedDict((
            ('name', pkg.name),
            ('title', pkg.title),
            ('metadata created', pkg.metadata_created.isoformat()),
            ('metadata modified', pkg.metadata_modified.isoformat()),
            ('dataset_notes', lib.dataset_notes(pkg)),
        ))
        rows.append(row)

    return {
        'table': rows,
        'num_datasets_published': num_datasets_published,
        'num_datasets_only_html': num_datasets_only_html,
    }
    def index(self):
        # FIXME: index copied from GroupController and modified to
        # show only parent groups
        context = {'model': model, 'session': model.Session,
                   'user': c.user or c.author, 'for_view': True,
                   'with_private': False}

        data_dict = {'all_fields': False}
        try:
            self._check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))

        # pass user info to context as needed to view private datasets of
        # orgs correctly
        if c.userobj:
            context['user_id'] = c.userobj.id
            context['user_is_admin'] = c.userobj.sysadmin

        search_result = self._action('organization_list')(context, data_dict)
        
        org_model = context['model']
        
        #Get the list of all groups of type "organization" that have no parents.             
        top_level_orgs = org_model.Group.get_top_level_groups(type="organization")
        top_results = [org for org in top_level_orgs if org.name in search_result ]
        
        facets = OrderedDict()

        facets['organization'] = _('Organizations')

        data_dict = {
                'facet.field': facets.keys(),
        }

        query = get_action('package_search')(context, data_dict)
        c.org_pkg_count = query['facets'].get('organization')
        
        c.top_orgs_items = top_results
        return render('organization/index.html')
def _add_facets(facets_dict, group=False):
    new_facets_dict = OrderedDict([
        ('keywords_facet', ''),
        facets_dict.items()[2],
        ('data_type_facet', p.toolkit._('Data Type')),
        ('organization', p.toolkit._('Projects')),
        ('groups', p.toolkit._('Topics')),
        ('language_facet', p.toolkit._('Language'))
    ] + facets_dict.items()[3:])

    if not group:
        new_facets_dict['date_facet'] = ''

    return new_facets_dict
Beispiel #9
0
    def dump(self, dump_file_obj):
        query = model.Session.query(model.User)
        query = query.order_by(model.User.created.asc())

        columns = (('id', 'name', 'openid', 'fullname', 'email', 'created',
                    'about'))
        row_dicts = []
        for user in query:
            row = OrderedDict()
            for col in columns:
                value = getattr(user, col)
                if not value:
                    value = ''
                if col == 'created':
                    value = str(value)  # or maybe dd/mm/yyyy?
                row[col] = value
            row_dicts.append(row)

        writer = CsvWriter(row_dicts)
        writer.save(dump_file_obj)
        dump_file_obj.close()
Beispiel #10
0
            search_extras = {}
            fq = ''

            if c.facet_choose_item:
                # Build query: in dataset browsing the query is limited to a single facet only
                fq = ' %s:"%s"' % (c.facet_choose_title, c.facet_choose_item)
                c.fields.append((c.facet_choose_title, c.facet_choose_item))
                c.fields_grouped[c.facet_choose_title] = [c.facet_choose_item]

            context = {'model': model, 'session': model.Session,
                       'user': c.user or c.author, 'for_view': True}

            if not asbool(config.get('ckan.search.show_all_types', 'False')):
                fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                    'organization': _('Organizations'),
                    'groups': _('Groups'),
                    'tags': _('Tags'),
                    'res_format': _('Formats'),
                    'license_id': _('License'),
                    }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet
    def search(self):
        from ckan.lib.search import SearchError, SearchQueryError

        # Get package type name
        package_type = self._guess_package_type()[:-1]
        c.package_type = package_type


        # Get page content from Wordpress
        # =========================================
        import ckanext.dadosgovbr.helpers.wordpress as wp
        wp_page_slug = 'scheming_'+package_type+'s'
        c.wp_page = type('Nothing', (object,), {})  
        c.wp_page.content = type('Nothing', (object,), {})  
        c.wp_page.content.rendered = "Conteudo da pagina nao encontrado..."
        try:
            c.wp_page = wp.page(wp_page_slug)
        except:
            pass

        # DEBUG
        # from pprint import pprint 
        # pprint(c.concursos)

        # Package type facets (filters)
        # =========================================
        package_type_facets = u'organization groups tags res_format license_id'
        if(package_type == 'inventario'):
            package_type_facets = u'organization situacao_base informacoes_sigilosas_base informacoes_publicas_base atualizacoes_base dados_abertos_base'

        if(package_type == 'concurso'):
            package_type_facets = u'organization datasets_used'

        if(package_type == 'aplicativo'):
            package_type_facets = u'organization groups tags res_format license_id'
        

        try:
            context = {'model': model, 'user': c.user,
                       'auth_user_obj': c.userobj}
            check_access('site_read', context)
        except NotAuthorized:
            abort(403, _('Not authorized to see this page'))

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = h.get_page_number(request.params)

        limit = int(config.get('ckan.datasets_per_page', 20))

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package', action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace,
                                      controller='package', action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.
            Each entry in the list is a 2-tuple: (fieldname, sort_order)
            eg - [('metadata_modified', 'desc'), ('name', 'asc')]
            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0]
                                for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user, 'for_view': True,
                       'auth_user_obj': c.userobj}

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not asbool(
                        config.get('ckan.search.show_all_types', 'False')):
                    fq += ' +dataset_type:dataset'

            facets = OrderedDict()
            

            default_facet_titles = {
                # Default package
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),

                # Inventário package
                'situacao_base': _(u'Situação da base'),
                'informacoes_sigilosas_base': _(u'Base possui informações sigilosas?'),
                'vocab_sim': _(u'Sim'),
                'vocab_nao': _(u'Não'),
                'informacoes_publicas_base': _(u'Base possui informações públicas?'),
                'informacoes_publicas_base_publico': _(u'Público'),
                'atualizacoes_base': _(u'Período de atualização dos dados'),
                'dados_abertos_base': _(u'Exporta para dados abertos?'),



                # Concurso package
                'datasets_used': _(u'Dados utilizados'),
                'tags': _(u'Tags'),
                'date': _(u'Data de início'),
                'end_date': _(u'Data final'),




                'publico': _(u'Público'),
                'sim': _(u'Sim'),
                'nao': _(u'Não'),
                }


            for facet in config.get(u'search.facets', package_type_facets.split()):
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras,
                'include_private': asbool(config.get(
                    'ckan.search.default_include_private', True)),
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchQueryError, se:
            # User's search parameters are invalid, in such a way that is not
            # achievable with the web interface, so return a proper error to
            # discourage spiders which are the main cause of this.
            log.info('Dataset search query rejected: %r', se.args)
            abort(400, _('Invalid search query: {error_message}')
                  .format(error_message=str(se)))
Beispiel #12
0
    def search(self):
        from ckan.lib.search import SearchError

        package_type = self._guess_package_type()

        if package_type == 'search':
            package_type = 'dataset'

        try:
            context = {'model': model, 'user': c.user or c.author,
                       'auth_user_obj': c.userobj}
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False

        page = self._page_number()

        params_nopage = self._params_nopage()

        # Commenting below parts as it doesn't seem to be used
        # def drill_down_url(alternative_url=None, **by):
        #     return h.add_url_param(alternative_url=alternative_url,
        #                            controller='package', action='search',
        #                            new_params=by)
        #
        # c.drill_down_url = drill_down_url

        self._set_remove_field_function()

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return self._search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0]
                                for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return self._search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))
        self._set_other_links()

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            #limit = g.datasets_per_page

            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            self._set_filters_are_selected_flag()

            limit = self._allowed_num_of_items(search_extras)

            context = {'model': model, 'session': model.Session,
                       'user': c.user or c.author, 'for_view': True,
                       'auth_user_obj': c.userobj}

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not asbool(config.get('ckan.search.show_all_types', 'False')):
                    fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            self._which_tab_is_selected(search_extras)
            self._performing_search(q, fq, facets.keys(), limit, page, sort_by,
                                    search_extras, pager_url, context)

        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = h.Page(collection=[])
Beispiel #13
0
    def _read(self, id, limit, group_type):
        ''' This is common code used by both read and bulk_process'''
        context = {'model': model, 'session': model.Session,
                   'user': c.user,
                   'schema': self._db_to_form_schema(group_type=group_type),
                   'for_view': True, 'extras_as_string': True}

        q = c.q = request.params.get('q', '')
        # Search within group
        if c.group_dict.get('is_organization'):
            q += ' owner_org:"%s"' % c.group_dict.get('id')
        else:
            q += ' groups:"%s"' % c.group_dict.get('name')

        c.description_formatted = \
            h.render_markdown(c.group_dict.get('description'))

        context['return_query'] = True

        page = h.get_page_number(request.params)

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']
        sort_by = request.params.get('sort', None)

        def search_url(params):
            controller = lookup_group_controller(group_type)
            action = 'bulk_process' if c.action == 'bulk_process' else 'read'
            url = h.url_for(controller=controller, action=action, id=id)
            params = [(k, v.encode('utf-8') if isinstance(v, basestring)
                       else str(v)) for k, v in params]
            return url + u'?' + urlencode(params)

        def drill_down_url(**by):
            return h.add_url_param(alternative_url=None,
                                   controller='group', action='read',
                                   extras=dict(id=c.group_dict.get('name')),
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            controller = lookup_group_controller(group_type)
            return h.remove_url_param(key, value=value, replace=replace,
                                      controller=controller, action='read',
                                      extras=dict(id=c.group_dict.get('name')))

        c.remove_field = remove_field

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params)

        try:
            c.fields = []
            c.fields_grouped = {}
            search_extras = {}
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        q += ' %s: "%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            facets = OrderedDict()

            default_facet_titles = {'organization': _('Organizations'),
                                    'groups': _('Groups'),
                                    'tags': _('Tags'),
                                    'res_format': _('Formats'),
                                    'license_id': _('Licenses')}

            for facet in h.facets():
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            self._update_facet_titles(facets, group_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': '',
                'include_private': True,
                'facet.field': facets.keys(),
                'rows': limit,
                'sort': sort_by,
                'start': (page - 1) * limit,
                'extras': search_extras
            }

            context_ = dict((k, v) for (k, v) in context.items()
                            if k != 'schema')
            query = get_action('package_search')(context_, data_dict)

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )

            c.group_dict['package_count'] = query['count']

            c.search_facets = query['search_facets']
            c.search_facets_limits = {}
            for facet in c.search_facets.keys():
                limit = int(request.params.get('_%s_limit' % facet,
                            config.get('search.facets.default', 10)))
                c.search_facets_limits[facet] = limit
            c.page.items = query['results']

            c.sort_by_selected = sort_by

        except search.SearchError, se:
            log.error('Group search error: %r', se.args)
            c.query_error = True
            c.page = h.Page(collection=[])
Beispiel #14
0
            for (param, value) in request.params.items():
                if not param in ["q", "page", "sort"] and len(value) and not param.startswith("_"):
                    if not param.startswith("ext_"):
                        c.fields.append((param, value))
                        q += ' %s: "%s"' % (param, value)
                    else:
                        search_extras[param] = value

            fq = 'capacity:"public"'
            user_member_of_orgs = [org["id"] for org in h.organizations_available("read")]

            if c.group and c.group.id in user_member_of_orgs:
                fq = ""
                context["ignore_capacity_check"] = True

            facets = OrderedDict()

            default_facet_titles = {
                "groups": _("Groups"),
                "tags": _("Tags"),
                "res_format": _("Formats"),
                "license": _("Licence"),
            }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
    def search(self):
        from ckan.lib.search import SearchError, SearchQueryError

        package_type = self._guess_package_type()

        try:
            context = {'model': model, 'user': c.user,
                       'auth_user_obj': c.userobj}
            check_access('site_read', context)
        except NotAuthorized:
            abort(403, _('Not authorized to see this page'))

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = h.get_page_number(request.params)

        limit = int(config.get('ckan.datasets_per_page', 20))

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='ckanext.sixodp_showcase.controller:Sixodp_ShowcaseController', action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace,
                                      controller='ckanext.sixodp_showcase.controller:Sixodp_ShowcaseController',
                                      action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.
            Each entry in the list is a 2-tuple: (fieldname, sort_order)
            eg - [('metadata_modified', 'desc'), ('name', 'asc')]
            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0]
                                for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user, 'for_view': True,
                       'auth_user_obj': c.userobj}

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not asbool(
                        config.get('ckan.search.show_all_types', 'False')):
                    fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in h.facets():
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in ckan.plugins.PluginImplementations(ckan.plugins.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            # Set the custom default sort parameter here
            # Might need a rewrite when ckan is updated
            if not sort_by:
                sort_by = 'metadata_created desc'

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras,
                'include_private': asbool(config.get(
                    'ckan.search.default_include_private', True)),
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchQueryError, se:
            # User's search parameters are invalid, in such a way that is not
            # achievable with the web interface, so return a proper error to
            # discourage spiders which are the main cause of this.
            log.info('Dataset search query rejected: %r', se.args)
            abort(400, _('Invalid search query: {error_message}')
                  .format(error_message=str(se)))
Beispiel #16
0
    def _add_dataset_search(self, showcase_id, showcase_name):
        '''
        Search logic for discovering datasets to add to a showcase.
        '''

        from ckan.lib.search import SearchError

        package_type = 'dataset'

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        try:
            page = self._get_page_number(request.params)
        except AttributeError:
            # in CKAN >= 2.5 _get_page_number has been moved
            page = h.get_page_number(request.params)

        limit = int(config.get('ckan.datasets_per_page', 20))

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package', action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace,
                                      controller='package', action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return self._search_url(params, showcase_name)

        c.sort_by = _sort_by
        if sort_by is None:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0]
                                for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return self._search_url(params, showcase_name)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user or c.author, 'for_view': True,
                       'auth_user_obj': c.userobj}

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not tk.asbool(config.get('ckan.search.show_all_types',
                                            'False')):
                    fq += ' +dataset_type:dataset'

            # Only search for packages that aren't already associated with the
            # Showcase
            associated_package_ids = ShowcasePackageAssociation.get_package_ids_for_showcase(showcase_id)
            # flatten resulting list to space separated string
            if associated_package_ids:
                associated_package_ids_str = \
                    ' OR '.join([id[0] for id in associated_package_ids])
                fq += ' !id:({0})'.format(associated_package_ids_str)

            facets = OrderedDict()

            default_facet_titles = {
                    'organization': _('Organizations'),
                    'groups': _('Groups'),
                    'tags': _('Tags'),
                    'res_format': _('Formats'),
                    'license_id': _('Licenses'),
                    }

            # for CKAN-Versions that do not provide the facets-method from
            # helper-context, import facets from ckan.common
            if hasattr(h, 'facets'):
                current_facets = h.facets()
            else:
                from ckan.common import g
                current_facets = g.facets

            for facet in current_facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = h.Page(collection=[])
    def search(self):
        package_type = self._guess_package_type()

        try:
            context = {'model': model, 'user': c.user or c.author, 'auth_user_obj': c.userobj}
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))

        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = self._get_page_number(request.params)
        limit = g.datasets_per_page
        params_nopage = [(k, v) for k, v in request.params.items() if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(
                alternative_url=alternative_url,
                controller='package',
                action='search',
                new_params=by
            )

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            params = params_nosort[:]
            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0] for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        if param != 'organization':
                            fq += ' %s:"%s"' % (param, value)
                        else:
                            fq += custom_organization_filter(value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user or c.author, 'for_view': True,
                       'auth_user_obj': c.userobj}

            if package_type and package_type != 'dataset':
                fq += ' +dataset_type:{type}'.format(type=package_type)
            elif not asbool(config.get('ckan.search.show_all_types', 'False')):
                fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = h.Page(collection=[])
    def search(self):
        from ckan.lib.search import SearchError

        package_type = self._guess_package_type()

        try:
            context = {'model': model, 'user': c.user or c.author,
                       'auth_user_obj': c.userobj}
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = self._get_page_number(request.params)

        limit = g.datasets_per_page

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package', action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace,
                                  controller='package', action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            """
            Sort by the given list of fields.

            Each entry in the list is a 2-tuple: (fieldname, sort_order)

            eg - [('metadata_modified', 'desc'), ('name', 'asc')]

            If fields is empty, then the default ordering is used.
            """
            params = params_nosort[:]

            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if sort_by is None:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [field.split()[0]
                                for field in sort_by.split(',')]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))
        api_search_url_params = None

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user or c.author, 'for_view': True,
                       'auth_user_obj': c.userobj}

            if package_type and package_type != 'dataset':
                # Only show datasets of this particular type
                fq += ' +dataset_type:{type}'.format(type=package_type)
            else:
                # Unless changed via config options, don't show non standard
                # dataset types on the default search page
                if not asbool(config.get('ckan.search.show_all_types', 'False')):
                    fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                    'organization': _('Organizations'),
                    'groups': _('Groups'),
                    'tags': _('Tags'),
                    'res_format': _('Formats'),
                    'license_id': _('Licenses'),
                    }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            ##### EXTENDING ORIGINAL <<<<<<<<<<
            # API Params
            api_url_params = {
                'q': q,
                'fq': fq.strip(),
                'rows': limit,
                'start': (page - 1) * limit,
            }
            if sort_by:
                api_url_params['sort'] = sort_by
            if facets.keys():
                 api_url_params['facet.field'] = json.dumps(facets.keys())
            # if search_extras:
            #      api_url_params['extras'] = ','.join(search_extras)

            api_search_url_params = urlencode(_encode_params(api_url_params.items()))
            ##### EXTENDING ORIGINAL >>>>>>>>>>>>>>

            query = get_action('package_search')(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
            c.page = h.Page(collection=[])
    def _read(self, id, limit, group_type):
        ''' This is common code used by both read and bulk_process'''
        context = {'model': model, 'session': model.Session,
                   'user': c.user or c.author,
                   'schema': self._db_to_form_schema(group_type=group_type),
                   'for_view': True, 'extras_as_string': True}

        q = c.q = request.params.get('q', '')
        # Search within group
        if c.group_dict.get('is_organization'):
            q += ' owner_org:"%s"' % c.group_dict.get('id')
        else:
            q += ' groups:"%s"' % c.group_dict.get('name')

        c.description_formatted = h.render_markdown(c.group_dict.get('description'))

        context['return_query'] = True

        # c.group_admins is used by CKAN's legacy (Genshi) templates only,
        # if we drop support for those then we can delete this line.
        c.group_admins = authz.get_group_or_org_admin_ids(c.group.id)

        page = self._get_page_number(request.params)

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']
        sort_by = request.params.get('sort', None)

        def search_url(params):
            controller = lookup_group_controller(group_type)
            action = 'bulk_process' if c.action == 'bulk_process' else 'read'
            url = h.url_for(controller=controller, action=action, id=id)
            params = [(k, v.encode('utf-8') if isinstance(v, basestring)
                       else str(v)) for k, v in params]
            return url + u'?' + urlencode(params)

        def drill_down_url(**by):
            return h.add_url_param(alternative_url=None,
                                   controller='group', action='read',
                                   extras=dict(id=c.group_dict.get('name')),
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace,
                                      controller='group', action='read',
                                      extras=dict(id=c.group_dict.get('name')))

        c.remove_field = remove_field

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params)

        try:
            c.fields = []
            search_extras = {}
            for (param, value) in request.params.items():
                if not param in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        q += ' %s: "%s"' % (param, value)
                    else:
                        search_extras[param] = value

            fq = 'capacity:"public"'
            user_member_of_orgs = [org['id'] for org
                                   in h.organizations_available('read')]

            if (c.group and c.group.id in user_member_of_orgs):
                fq = ''
                context['ignore_capacity_check'] = True

            facets = OrderedDict()

            default_facet_titles = {'organization': _('Organizations'),
                                    'groups': _('Groups'),
                                    'tags': _('Tags'),
                                    'res_format': _('Formats'),
                                    'license_id': _('Licenses')}

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            self._update_facet_titles(facets, group_type)

            if 'capacity' in facets and (group_type != 'organization' or
                                         not user_member_of_orgs):
                del facets['capacity']

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq,
                'facet.field': facets.keys(),
                'rows': limit,
                'sort': sort_by,
                'start': (page - 1) * limit,
                'extras': search_extras
            }

            context_ = dict((k, v) for (k, v) in context.items() if k != 'schema')
            query = get_action('package_search')(context_, data_dict)

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )

            c.group_dict['package_count'] = query['count']

            lang = get_lang()[0]
            
            q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.group_id == c.group_dict.get('id'), GroupMultilang.lang == lang).all() 

            if q_results:
                for result in q_results:
                    c.group_dict[result.field] = result.text
                    if result.field == 'title':
                        c.group_dict['display_name'] = result.text
            
            c.facets = query['facets']
            maintain.deprecate_context_item('facets',
                                            'Use `c.search_facets` instead.')

            c.search_facets = query['search_facets']
            c.search_facets_limits = {}
            for facet in c.facets.keys():
                limit = int(request.params.get('_%s_limit' % facet,
                                               g.facets_default_number))
                c.search_facets_limits[facet] = limit

            # MULTILANG - Localizing Organizations display names in Facet list
            organizations = c.search_facets.get('organization')

            for org in organizations.get('items'):
                q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.name == org.get('name'), GroupMultilang.lang == lang).all() 

                if q_results:
                    for result in q_results:
                        if result.field == 'title':
                            org['display_name'] = result.text

            # MULTILANG - Localizing Groups display names in Facet list
            groups = c.search_facets.get('groups')
            for group in groups.get('items'):
                q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.name == group.get('name'), GroupMultilang.lang == lang).all() 

                if q_results:
                    for result in q_results:
                        if result.field == 'title':
                            group['display_name'] = result.text

             # MULTILANG - Localizing Tags display names in Facet list
            tags = c.search_facets.get('tags')
            for tag in tags.get('items'):
                localized_tag = TagMultilang.by_name(tag.get('name'), lang)

                if localized_tag:
                    tag['display_name'] = localized_tag.text

            c.page.items = query['results']
            
            # MULTILANG - Localizing Datasets names and descriptions in search list
            log.info(':::::::::::: Retrieving the corresponding localized title and abstract :::::::::::::::')

            for item in c.page.items:
                lang = get_lang()[0]

                q_results = model.Session.query(PackageMultilang).filter(PackageMultilang.package_id == item.get('id'), PackageMultilang.lang == lang).all() 

                if q_results:
                    for result in q_results:
                        item[result.field] = result.text
                
            c.sort_by_selected = sort_by

        except search.SearchError, se:
            log.error('Group search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.page = h.Page(collection=[])
Beispiel #20
0
    def _read(self, id, limit, group_type):
        """ This is common code used by both read and bulk_process"""
        context = {
            "model": model,
            "session": model.Session,
            "user": c.user,
            "schema": self._db_to_form_schema(group_type=group_type),
            "for_view": True,
            "extras_as_string": True,
        }

        q = c.q = request.params.get("q", "")
        # Search within group
        if c.group_dict.get("is_organization"):
            q += ' owner_org:"%s"' % c.group_dict.get("id")
        else:
            q += ' groups:"%s"' % c.group_dict.get("name")

        c.description_formatted = h.render_markdown(c.group_dict.get("description"))

        context["return_query"] = True

        page = self._get_page_number(request.params)

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items() if k != "page"]
        sort_by = request.params.get("sort", None)

        def search_url(params):
            controller = lookup_group_controller(group_type)
            action = "bulk_process" if c.action == "bulk_process" else "read"
            url = h.url_for(controller=controller, action=action, id=id)
            params = [(k, v.encode("utf-8") if isinstance(v, basestring) else str(v)) for k, v in params]
            return url + u"?" + urlencode(params)

        def drill_down_url(**by):
            return h.add_url_param(
                alternative_url=None,
                controller="group",
                action="read",
                extras=dict(id=c.group_dict.get("name")),
                new_params=by,
            )

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(
                key,
                value=value,
                replace=replace,
                controller="group",
                action="read",
                extras=dict(id=c.group_dict.get("name")),
            )

        c.remove_field = remove_field

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(("page", page))
            return search_url(params)

        try:
            c.fields = []
            search_extras = {}
            for (param, value) in request.params.items():
                if not param in ["q", "page", "sort"] and len(value) and not param.startswith("_"):
                    if not param.startswith("ext_"):
                        c.fields.append((param, value))
                        q += ' %s: "%s"' % (param, value)
                    else:
                        search_extras[param] = value

            fq = 'capacity:"public"'
            user_member_of_orgs = [org["id"] for org in h.organizations_available("read")]

            if c.group and c.group.id in user_member_of_orgs:
                fq = ""
                context["ignore_capacity_check"] = True

            facets = OrderedDict()

            default_facet_titles = {
                "organization": _("Organizations"),
                "groups": _("Groups"),
                "tags": _("Tags"),
                "res_format": _("Formats"),
                "license_id": _("Licenses"),
            }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            self._update_facet_titles(facets, group_type)

            if "capacity" in facets and (group_type != "organization" or not user_member_of_orgs):
                del facets["capacity"]

            c.facet_titles = facets

            data_dict = {
                "q": q,
                "fq": fq,
                "facet.field": facets.keys(),
                "rows": limit,
                "sort": sort_by,
                "start": (page - 1) * limit,
                "extras": search_extras,
            }

            context_ = dict((k, v) for (k, v) in context.items() if k != "schema")
            query = get_action("package_search")(context_, data_dict)

            c.page = h.Page(
                collection=query["results"], page=page, url=pager_url, item_count=query["count"], items_per_page=limit
            )

            c.group_dict["package_count"] = query["count"]
            c.facets = query["facets"]
            maintain.deprecate_context_item("facets", "Use `c.search_facets` instead.")

            c.search_facets = query["search_facets"]
            c.search_facets_limits = {}
            for facet in c.facets.keys():
                limit = int(request.params.get("_%s_limit" % facet, g.facets_default_number))
                c.search_facets_limits[facet] = limit
            c.page.items = query["results"]

            c.sort_by_selected = sort_by

        except search.SearchError, se:
            log.error("Group search error: %r", se.args)
            c.query_error = True
            c.facets = {}
            c.page = h.Page(collection=[])
Beispiel #21
0
    def _read(self, id, limit):
        ''' This is common code used by both read and bulk_process'''
        group_type = self._get_group_type(id.split('@')[0])
        context = {'model': model, 'session': model.Session,
                   'user': c.user or c.author,
                   'schema': self._db_to_form_schema(group_type=group_type),
                   'for_view': True, 'extras_as_string': True}

        q = c.q = request.params.get('q', '')
        # Search within group
        if c.group_dict.get('is_organization'):
            q += ' owner_org:"%s"' % c.group_dict.get('id')
        else:
            q += ' groups:"%s"' % c.group_dict.get('name')

        description = c.group_dict.get('description')
        if description:
            if group_type == 'group':
                c.description_formatted = h.render_markdown(description.get(h.lang()))
            else:
                c.description_formatted = h.render_markdown(description)

        context['return_query'] = True

        # c.group_admins is used by CKAN's legacy (Genshi) templates only,
        # if we drop support for those then we can delete this line.
        c.group_admins = new_authz.get_group_or_org_admin_ids(c.group.id)

        page = self._get_page_number(request.params)

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']
        sort_by = request.params.get('sort', None)

        def search_url(params):
            if group_type == 'organization':
                if c.action == 'bulk_process':
                    url = self._url_for(controller='organization',
                                        action='bulk_process',
                                        id=id)
                else:
                    url = self._url_for(controller='organization',
                                        action='read',
                                        id=id)
            else:
                url = self._url_for(controller='group', action='read', id=id)
            params = [(k, v.encode('utf-8') if isinstance(v, basestring)
                       else str(v)) for k, v in params]
            return url + u'?' + urlencode(params)

        def drill_down_url(**by):
            return h.add_url_param(alternative_url=None,
                                   controller='group', action='read',
                                   extras=dict(id=c.group_dict.get('name')),
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace,
                                      controller='group', action='read',
                                      extras=dict(id=c.group_dict.get('name')))

        c.remove_field = remove_field

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params)

        try:
            c.fields = []
            search_extras = {}
            for (param, value) in request.params.items():
                if not param in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        q += ' %s: "%s"' % (param, value)
                    else:
                        search_extras[param] = value

            fq = 'capacity:"public"'
            user_member_of_orgs = [org['id'] for org
                                   in h.organizations_available('read')]

            if (c.group and c.group.id in user_member_of_orgs):
                fq = ''
                context['ignore_capacity_check'] = True

            facets = OrderedDict()

            default_facet_titles = {'organization': _('Organizations'),
                                    'groups': _('Groups'),
                                    'tags': _('Tags'),
                                    'res_format': _('Formats'),
                                    'license_id': _('Licenses')}

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in plugins.PluginImplementations(plugins.IFacets):
                if self.group_type == 'organization':
                    facets = plugin.organization_facets(
                        facets, self.group_type, None)
                else:
                    facets = plugin.group_facets(
                        facets, self.group_type, None)

            if 'capacity' in facets and (self.group_type != 'organization' or
                                         not user_member_of_orgs):
                del facets['capacity']

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq,
                'facet.field': facets.keys(),
                'rows': limit,
                'sort': sort_by,
                'start': (page - 1) * limit,
                'extras': search_extras
            }

            context_ = dict((k, v) for (k, v) in context.items() if k != 'schema')
            query = get_action('package_search')(context_, data_dict)

            c.page = h.Page(
                collection=query['results'],
                page=page,
                url=pager_url,
                item_count=query['count'],
                items_per_page=limit
            )

            c.group_dict['package_count'] = query['count']
            c.facets = query['facets']
            maintain.deprecate_context_item('facets',
                                            'Use `c.search_facets` instead.')

            c.search_facets = query['search_facets']
            c.search_facets_limits = {}
            for facet in c.facets.keys():
                limit = int(request.params.get('_%s_limit' % facet,
                                               g.facets_default_number))
                c.search_facets_limits[facet] = limit
            c.page.items = query['results']

            c.sort_by_selected = sort_by

        except search.SearchError, se:
            log.error('Group search error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.page = h.Page(collection=[])
Beispiel #22
0
def _read(id, limit, group_type):
    u''' This is common code used by both read and bulk_process'''
    extra_vars = {}
    context = {
        u'model': model,
        u'session': model.Session,
        u'user': g.user,
        u'schema': _db_to_form_schema(group_type=group_type),
        u'for_view': True,
        u'extras_as_string': True
    }

    q = request.params.get(u'q', u'')

    # TODO: Remove
    # ckan 2.9: Adding variables that were removed from c object for
    # compatibility with templates in existing extensions
    g.q = q

    # Search within group
    if g.group_dict.get(u'is_organization'):
        fq = u' owner_org:"%s"' % g.group_dict.get(u'id')
    else:
        fq = u' groups:"%s"' % g.group_dict.get(u'name')

    extra_vars["q"] = q

    g.description_formatted = \
        h.render_markdown(g.group_dict.get(u'description'))

    context['return_query'] = True

    page = h.get_page_number(request.params)

    # most search operations should reset the page counter:
    params_nopage = [(k, v) for k, v in request.params.items() if k != u'page']
    sort_by = request.params.get(u'sort', None)

    def search_url(params):
        controller = lookup_group_controller(group_type)
        action = u'bulk_process' if getattr(
            g, u'action', u'') == u'bulk_process' else u'read'
        url = h.url_for(u'.'.join([controller, action]), id=id)
        params = [(k, v.encode(u'utf-8')
                   if isinstance(v, string_types) else str(v))
                  for k, v in params]
        return url + u'?' + urlencode(params)

    def drill_down_url(**by):
        return h.add_url_param(
            alternative_url=None,
            controller=u'group',
            action=u'read',
            extras=dict(id=g.group_dict.get(u'name')),
            new_params=by)

    extra_vars["drill_down_url"] = drill_down_url

    def remove_field(key, value=None, replace=None):
        controller = lookup_group_controller(group_type)
        return h.remove_url_param(
            key,
            value=value,
            replace=replace,
            controller=controller,
            action=u'read',
            extras=dict(id=g.group_dict.get(u'name')))

    extra_vars["remove_field"] = remove_field

    def pager_url(q=None, page=None):
        params = list(params_nopage)
        params.append((u'page', page))
        return search_url(params)

    try:
        extra_vars["fields"] = fields = []
        extra_vars["fields_grouped"] = fields_grouped = {}
        search_extras = {}
        for (param, value) in request.params.items():
            if param not in [u'q', u'page', u'sort'] \
                    and len(value) and not param.startswith(u'_'):
                if not param.startswith(u'ext_'):
                    fields.append((param, value))
                    q += u' %s: "%s"' % (param, value)
                    if param not in fields_grouped:
                        fields_grouped[param] = [value]
                    else:
                        fields_grouped[param].append(value)
                else:
                    search_extras[param] = value

        # TODO: Remove
        # ckan 2.9: Adding variables that were removed from c object for
        # compatibility with templates in existing extensions
        g.fields = fields
        g.fields_grouped = fields_grouped

        facets = OrderedDict()

        default_facet_titles = {
            u'organization': _(u'Organizations'),
            u'groups': _(u'Groups'),
            u'tags': _(u'Tags'),
            u'res_format': _(u'Formats'),
            u'license_id': _(u'Licenses')
        }

        for facet in h.facets():
            if facet in default_facet_titles:
                facets[facet] = default_facet_titles[facet]
            else:
                facets[facet] = facet

        # Facet titles
        _update_facet_titles(facets, group_type)

        extra_vars["facet_titles"] = facets

        data_dict = {
            u'q': q,
            u'fq': fq,
            u'include_private': True,
            u'facet.field': facets.keys(),
            u'rows': limit,
            u'sort': sort_by,
            u'start': (page - 1) * limit,
            u'extras': search_extras
        }

        context_ = dict((k, v) for (k, v) in context.items() if k != u'schema')
        query = get_action(u'package_search')(context_, data_dict)

        extra_vars["page"] = h.Page(
            collection=query['results'],
            page=page,
            url=pager_url,
            item_count=query['count'],
            items_per_page=limit)

        # TODO: Remove
        # ckan 2.9: Adding variables that were removed from c object for
        # compatibility with templates in existing extensions
        g.group_dict['package_count'] = query['count']

        extra_vars["search_facets"] = g.search_facets = query['search_facets']
        extra_vars["search_facets_limits"] = g.search_facets_limits = {}
        for facet in g.search_facets.keys():
            limit = int(
                request.params.get(u'_%s_limit' % facet,
                                   config.get(u'search.facets.default', 10)))
            g.search_facets_limits[facet] = limit
        extra_vars["page"].items = query['results']

        extra_vars["sort_by_selected"] = sort_by

    except search.SearchError as se:
        log.error(u'Group search error: %r', se.args)
        extra_vars["query_error"] = True
        extra_vars["page"] = h.Page(collection=[])

    # TODO: Remove
    # ckan 2.9: Adding variables that were removed from c object for
    # compatibility with templates in existing extensions
    g.facet_titles = facets
    g.page = extra_vars["page"]

    extra_vars["group_type"] = group_type
    _setup_template_variables(context, {u'id': id}, group_type=group_type)
    return extra_vars
Beispiel #23
0
    def show(self):
        #This is based on the CKAN package search controller
        #There are probably still some pieces we don't need
        #And using the facets to get the info we need is not ideal
        #because normal facet limits are then not applied on the page
        from ckan.lib.search import SearchError

        # unicode format (decoded from utf8)
        q = c.q = request.params.get('q', u'')
        c.query_error = False

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key, value=value, replace=replace,
                                  controller='package', action='search')

        c.remove_field = remove_field
        
        params_nopage = [(k, v) for k, v in request.params.items()]
        
        def pager_url(q=None, page=None):
            params = list(params_nopage)
            return self.search_url(params, 'dataset')
            
        c.search_url_params = urlencode(self._encode_params(params_nopage))

        try:
            c.fields = []
            # c.fields_grouped will contain a dict of params containing
            # a list of values eg {'tags':['tag1', 'tag2']}
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        fq += ' %s:"%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {'model': model, 'session': model.Session,
                       'user': c.user or c.author, 'for_view': True,
                       'auth_user_obj': c.userobj}

            facets = OrderedDict()

            default_facet_titles = {
                    'organization': _('Organizations'),
                    'groups': _('Groups'),
                    'tags': _('Tags'),
                    'res_format': _('Formats'),
                    'license_id': _('Licenses'),
                    }

            for facet in g.facets:
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, 'dataset')

            c.facet_titles = facets

            #We use the organization facet to get the list of organizations detected for this query
            #The actual data is irrelevant
            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'facet.limit': -1,
                'extras': search_extras,
                'rows': 1
            }

            query = get_action('package_search')(context, data_dict)

            orglist = query['facets']['organization'].keys()
                
            #Now get the data concerning the orgs
            q = Session.query(Group).filter(Group.name.in_(orglist))
            
            values = []
            qres = q.all()

            #Combine in extras, add counts
            #Make the data look like what we get with a org search
            for d in qres:
                dobj = d.as_dict()
                dobj['extras'] = d.extras
                dobj['package_count'] = query['facets']['organization'][dobj['name']]
                values.append(dobj)

            c.page = h.Page(
                collection=query['results'],
                url=pager_url,
                item_count=query['count']
            )
            c.facets = query['facets']
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search (map) error: %r', se.args)
            c.query_error = True
            c.facets = {}
            c.search_facets = {}
    def get_pkg_dict(cls, resource, ldr):
        from ckan import model

        pkg_dict = OrderedDict()
        extras = OrderedDict()
        uri = str(resource.identifier)
        pkg_dict['title'] = unicode(resource[RDFS.label].next())
        extras['registry_uri'] = uri

        # Create or update?
        pkg = model.Session.query(model.Package) \
                .filter_by(state='active') \
                .join(model.PackageExtra) \
                .filter_by(state='active') \
                .filter_by(key='registry_uri') \
                .filter_by(value=uri).first()
        if pkg:
            pkg_dict['id'] = pkg.id
            pkg_dict['name'] = pkg.name
            action = 'update'
        else:
            pkg_dict['id'] = unicode(uuid.uuid4())
            pkg_dict['name'] = cls._gen_new_name(pkg_dict['title'])
            action = 'new'

        dgu_type = cls.get_dgu_type(resource)
        extras['data_standard_type'] = dgu_type

        resources = []
        for format_display, format_extension, format_dgu in (
                ('RDF ttl', 'ttl', 'RDF'),
                ('RDF/XML', 'rdf', 'RDF'),
                ('JSON-LD', 'jsonld', 'JSON')):
            url = uri + '?_format=%s' % format_extension
            resources.append({'description': '%s as %s' % (dgu_type, format_display),
                              'url': url,
                              'format': format_dgu,
                              'resource_type': 'file'})
            resources.append({'description': '%s and metadata as %s' % (dgu_type, format_display),
                              'url': url + METADATA_PARAM,
                              'format': format_dgu,
                              'resource_type': 'file'})

        pkg_dict['notes'] = unicode(resource[DCT.description].next())
        licence_url = str(resource[DCT.license].next())
        if 'open-government-licence' in licence_url:
            pkg_dict['licence_id'] = 'uk-ogl'
        else:
            extras['licence_url'] = licence_url
            # not sure how this will display as just as URL
        pkg_dict['owner_org'] = cls.get_publisher(resource).id
        resources.append({'description': 'Web page for this %s on a Linked Data Registry' % dgu_type,
                          'url': uri,
                          'format': 'HTML',
                          'resource_type': 'documentation'})
        metadata = cls.get_resource_metadata(uri)
        status = metadata[REG.status].next()
        extras['status'] = str(status).split('#')[-1]
        extras['harvested_version'] = str(metadata[OWL.versionInfo].next())
        extras['data_standard_type'] = dgu_type
        pkg_dict['type'] = 'data-standard'

        pkg_dict['extras'] = [{'key': k, 'value': v} for k, v in extras.items()]
        pkg_dict['resources'] = resources
        return pkg_dict, action