Пример #1
0
class YtpDrupalPlugin(plugins.SingletonPlugin, DefaultTranslation):
    plugins.implements(plugins.IConfigurable)
    plugins.implements(plugins.ITemplateHelpers)
    plugins.implements(plugins.IConfigurer, inherit=True)
    plugins.implements(plugins.IAuthFunctions, inherit=True)
    plugins.implements(plugins.IRoutes, inherit=True)
    plugins.implements(plugins.ITranslation)

    _config_template = "ckanext.ytp.drupal.%s"
    _node_type = 'service_alert'
    _node_status = '1'  # published content has status 1, unpublised has status 0
    _language_fallback_order = ['fi', 'en', 'sv']
    cancel_url = None

    def before_map(self, m):
        """ Override delete page """
        controller = 'ckanext.ytp_drupal.controller:YtpDrupalController'
        m.connect('user_delete_me',
                  '/user/delete-me',
                  action='delete_me',
                  controller=controller)
        return m

    def configure(self, config):
        connection_variable = self._config_template % "connection"
        self.drupal_connection_url = config.get(connection_variable)
        if not self.drupal_connection_url:
            raise Exception(
                'YtpDrupalPlugin: required configuration variable missing: %s'
                % (connection_variable))

        self.cancel_url = config.get(self._config_template % "cancel_url",
                                     "/cancel-user.php")
        self._node_type = config.get(self._config_template % "node_type",
                                     self._node_type)
        self._translations_disabled = asbool(
            config.get(self._config_template % "translations_disabled",
                       "false"))

        self.engine = sqlalchemy.create_engine(self.drupal_connection_url)

    def update_config(self, config):
        toolkit.add_template_directory(config, 'templates')

    def _service_alerts(self):
        """ Get service alerts from Drupal """
        language = None if self._translations_disabled else helpers.lang()
        return self.engine.execute(
            """SELECT nid, title FROM node WHERE type = %(type)s AND
                                      language = %(language)s AND status = %(status)s""",
            {
                'type': self._node_type,
                'language': language,
                'status': self._node_status
            })

    def _fetch_drupal_content(self, identifier, language=None, fallback=True):
        """ This helper fetches content from Drupal database using url alias identifier.
            Return content as dictionary containing language, title, body, node_id and edit link. None if not found.
            Tries to fallback to different language if fallback is True.

            Not cached.
        """
        if not language:
            language = helpers.lang()

        query = """SELECT url_alias.language, node.title, field_revision_body.body_value, node.nid from url_alias
                       INNER JOIN node ON node.nid = split_part(url_alias.source, '/', 2)::integer
                       INNER JOIN field_revision_body ON field_revision_body.entity_id = split_part(url_alias.source, '/', 2)::integer
                   WHERE url_alias.alias = %(identifier)s"""  # noqa: E501

        results = {}
        for content_language, title, body, node_id in self.engine.execute(
                query, {'identifier': identifier}):
            results[content_language] = {
                'language': content_language,
                'title': title,
                'body': body,
                'node_id': node_id
            }

        result = results.get(language, None)

        if not result and fallback and results:
            for fallback_language in self._language_fallback_order:
                result = results.get(fallback_language)
                if result:
                    break
            if not result:
                result = results.itervalues().next()

        if result:
            result['edit'] = urllib.quote("/%s/node/%s/edit" %
                                          (language, str(result['node_id'])))
            result['body'] = literal(result['body'])

        return result

    def get_drupal_user_id(self, username):
        result = self.engine.execute(
            "SELECT uid FROM users_field_data WHERE name = %(name)s",
            {'name': username})
        for row in result:
            return row[0]
        raise NotFound

    def get_drupal_session_cookie(self):
        '''returns tuple of (cookie_name, cookie_value)'''
        request_cookies = request.cookies
        session_cookie = None
        for cookie_key in request_cookies:
            if cookie_key[:5] == 'SSESS':
                session_cookie = (cookie_key, request_cookies[cookie_key])
        return session_cookie

    def get_drupal_session_token(self, domain, service, cookie_header=''):
        '''return text of X-CSRF-Token)'''
        token_url = 'https://' + domain + '/' + service + '/?q=services/session/token'
        verify_cert = config.get('ckanext.drupal8.development_cert',
                                 '') or True
        token_request = requests.get(token_url,
                                     headers={"Cookie": cookie_header},
                                     verify=verify_cert)
        token = token_request.text
        return token

    def get_helpers(self):
        return {
            'service_alerts': self._service_alerts,
            'fetch_drupal_content': self._fetch_drupal_content
        }

    def get_auth_functions(self):
        return {'user_delete_me': user_delete_me}
Пример #2
0
class SpatialMetadata(SingletonPlugin):
    ''' '''

    implements(interfaces.IPackageController, inherit=True)
    implements(interfaces.IConfigurable, inherit=True)
    implements(interfaces.IConfigurer, inherit=True)
    implements(interfaces.ITemplateHelpers, inherit=True)

    def configure(self, config):
        '''

        :param config: 

        '''
        from ckanext.spatial.model.package_extent import setup as setup_model

        if not toolkit.asbool(config.get(u'ckan.spatial.testing', u'False')):
            log.debug(u'Setting up the spatial model')
            setup_model()

    def update_config(self, config):
        '''Set up the resource library, public directory and
        template directory for all the spatial extensions

        :param config: 

        '''
        toolkit.add_public_directory(config, u'public')
        toolkit.add_template_directory(config, u'templates')
        toolkit.add_resource(u'public', u'ckanext-spatial')

        # Add media types for common extensions not included in the mimetypes
        # module
        mimetypes.add_type(u'application/json', u'.geojson')
        mimetypes.add_type(u'application/gml+xml', u'.gml')

    def create(self, package):
        '''

        :param package: 

        '''
        self.check_spatial_extra(package)

    def edit(self, package):
        '''

        :param package: 

        '''
        self.check_spatial_extra(package)

    def check_spatial_extra(self, package):
        '''For a given package, looks at the spatial extent (as given in the
        extra "spatial" in GeoJSON format) and records it in PostGIS.

        :param package: 

        '''
        from ckanext.spatial.lib import save_package_extent

        if not package.id:
            log.warning(
                u'Couldn\'t store spatial extent because no id was provided for the '
                u'package')
            return

        # TODO: deleted extra
        for extra in package.extras_list:
            if extra.key == u'spatial':
                if extra.state == u'active' and extra.value:
                    try:
                        log.debug(u'Received: %r' % extra.value)
                        geometry = json.loads(extra.value)
                    except ValueError, e:
                        error_dict = {
                            u'spatial':
                            [u'Error decoding JSON object: %s' % str(e)]
                        }
                        raise toolkit.ValidationError(
                            error_dict,
                            error_summary=package_error_summary(error_dict))
                    except TypeError, e:
                        error_dict = {
                            u'spatial':
                            [u'Error decoding JSON object: %s' % str(e)]
                        }
                        raise toolkit.ValidationError(
                            error_dict,
                            error_summary=package_error_summary(error_dict))

                    try:
                        save_package_extent(package.id, geometry)

                    except ValueError, e:
                        error_dict = {
                            u'spatial':
                            [u'Error creating geometry: %s' % str(e)]
                        }
                        raise toolkit.ValidationError(
                            error_dict,
                            error_summary=package_error_summary(error_dict))
                    except Exception, e:
                        if bool(os.getenv(u'DEBUG')):
                            raise
                        error_dict = {u'spatial': [u'Error: %s' % str(e)]}
                        raise toolkit.ValidationError(
                            error_dict,
                            error_summary=package_error_summary(error_dict))
Пример #3
0
class GoogleAnalyticsPlugin(p.SingletonPlugin):
    p.implements(p.IConfigurable, inherit=True)
    p.implements(p.IGenshiStreamFilter, inherit=True)
    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IConfigurer, inherit=True)
    p.implements(p.ITemplateHelpers)

    def configure(self, config):
        '''Load config settings for this extension from config file.

        See IConfigurable.

        '''
        if 'googleanalytics.id' not in config:
            msg = "Missing googleanalytics.id in config"
            raise GoogleAnalyticsException(msg)
        self.googleanalytics_id = config['googleanalytics.id']
        self.googleanalytics_domain = config.get('googleanalytics.domain',
                                                 'auto')
        self.googleanalytics_javascript_url = h.url_for_static(
            '/scripts/ckanext-googleanalytics.js')

        # If resource_prefix is not in config file then write the default value
        # to the config dict, otherwise templates seem to get 'true' when they
        # try to read resource_prefix from config.
        if 'googleanalytics_resource_prefix' not in config:
            config['googleanalytics_resource_prefix'] = (
                commands.DEFAULT_RESOURCE_URL_TAG)
        self.googleanalytics_resource_prefix = config[
            'googleanalytics_resource_prefix']

        self.show_downloads = converters.asbool(
            config.get('googleanalytics.show_downloads', True))
        self.track_events = converters.asbool(
            config.get('googleanalytics.track_events', False))

        if not converters.asbool(config.get('ckan.legacy_templates', 'false')):
            p.toolkit.add_resource('fanstatic_library',
                                   'ckanext-googleanalytics')

    def update_config(self, config):
        '''Change the CKAN (Pylons) environment configuration.

        See IConfigurer.

        '''
        if converters.asbool(config.get('ckan.legacy_templates', 'false')):
            p.toolkit.add_template_directory(config, 'legacy_templates')
            p.toolkit.add_public_directory(config, 'legacy_public')
        else:
            p.toolkit.add_template_directory(config, 'templates')

    def after_map(self, map):
        '''Add new routes that this extension's controllers handle.

        See IRoutes.

        '''
        map.redirect("/analytics/package/top", "/analytics/dataset/top")
        map.connect(
            'analytics',
            '/analytics/dataset/top',
            controller='ckanext.googleanalytics.controller:GAController',
            action='view')
        return map

    def filter(self, stream):
        '''Insert Google Analytics code into legacy Genshi templates.

        This is called by CKAN whenever any page is rendered, _if_ using old
        CKAN 1.x legacy templates. If using new CKAN 2.0 Jinja templates, the
        template helper methods below are used instead.

        See IGenshiStreamFilter.

        '''
        log.info("Inserting Google Analytics code into template")

        # Add the Google Analytics tracking code into the page header.
        header_code = genshi.HTML(
            gasnippet.header_code %
            (self.googleanalytics_id, self.googleanalytics_domain))
        stream = stream | genshi.filters.Transformer('head').append(
            header_code)

        # Add the Google Analytics Event Tracking script into the page footer.
        if self.track_events:
            footer_code = genshi.HTML(gasnippet.footer_code %
                                      self.googleanalytics_javascript_url)
            stream = stream | genshi.filters.Transformer(
                'body/div[@id="scripts"]').append(footer_code)

        routes = pylons.request.environ.get('pylons.routes_dict')
        action = routes.get('action')
        controller = routes.get('controller')

        if ((controller == 'package'
             and action in ['search', 'read', 'resource_read'])
                or (controller == 'group' and action == 'read')):

            log.info("Tracking of resource downloads")

            # add download tracking link
            def js_attr(name, event):
                attrs = event[1][1]
                href = attrs.get('href').encode('utf-8')
                link = '%s%s' % (self.googleanalytics_resource_prefix,
                                 urllib.quote(href))
                js = "javascript: _gaq.push(['_trackPageview', '%s']);" % link
                return js

            # add some stats
            def download_adder(stream):
                download_html = '''<span class="downloads-count">
                [downloaded %s times]</span>'''
                count = None
                for mark, (kind, data, pos) in stream:
                    if mark and kind == genshi.core.START:
                        href = data[1].get('href')
                        if href:
                            count = dbutil.get_resource_visits_for_url(href)
                    if count and mark is genshi.filters.transform.EXIT:
                        # emit count
                        yield genshi.filters.transform.INSIDE, (
                            genshi.core.TEXT,
                            genshi.HTML(download_html % count), pos)
                    yield mark, (kind, data, pos)

            # perform the stream transform
            stream = stream | genshi.filters.Transformer(
                '//a[contains(@class, "resource-url-analytics")]').attr(
                    'onclick', js_attr)

            if (self.show_downloads and action == 'read'
                    and controller == 'package'):
                stream = stream | genshi.filters.Transformer(
                    '//a[contains(@class, "resource-url-analytics")]').apply(
                        download_adder)
                stream = stream | genshi.filters.Transformer('//head').append(
                    genshi.HTML(gasnippet.download_style))

        return stream

    def get_helpers(self):
        '''Return the CKAN 2.0 template helper functions this plugin provides.

        See ITemplateHelpers.

        '''
        return {'googleanalytics_header': self.googleanalytics_header}

    def googleanalytics_header(self):
        '''Render the googleanalytics_header snippet for CKAN 2.0 templates.

        This is a template helper function that renders the
        googleanalytics_header jinja snippet. To be called from the jinja
        templates in this extension, see ITemplateHelpers.

        '''
        data = {
            'googleanalytics_id': self.googleanalytics_id,
            'googleanalytics_domain': self.googleanalytics_domain
        }
        return p.toolkit.render_snippet(
            'googleanalytics/snippets/googleanalytics_header.html', data)
Пример #4
0
class UebPackagePlugins(p.SingletonPlugin):
    # Set inherit=True so that we don't have to implement all functions of this interface
    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IConfigurer)
    p.implements(p.ITemplateHelpers, inherit=True)

    #Ref: http://docs.ckan.org/sq/latest/resources.html#resources-within-extensions
    p.toolkit.add_resource('public', 'uebresources')

    # Update CKAN's config settings, see the IConfigurer plugin interface.
    def update_config(self, config):

        # Tell CKAN to use the template files in
        # ckanext-uebpackage/ckanext/uebpackage/templates.
        p.toolkit.add_template_directory(config, 'templates')

        # add the extension's public dir path so that
        # ckan can find any resources used from this path
        # get the current dir path (here) for this plugin
        here = os.path.dirname(__file__)
        rootdir = os.path.dirname(os.path.dirname(here))
        our_public_dir = os.path.join(rootdir, 'ckanext', 'uebpackage',
                                      'public')
        config['extra_public_paths'] = ','.join(
            [our_public_dir,
             config.get('extra_public_paths', '')])

    # Update CKAN's map settings, see the IRoutes plugin interface.
    def before_map(self, map):
        # Here create route shortcuts to all your extension's controllers and their actions
        map.connect(
            '/uebpackage/createform',
            controller=
            'ckanext.uebpackage.controllers.packagecreate:PackagecreateController',
            action='packagecreateform')
        map.connect(
            '/uebpackage/createformsubmit',
            controller=
            'ckanext.uebpackage.controllers.packagecreate:PackagecreateController',
            action='submit')
        map.connect(
            '/uebpackage/selectpackagetoexecute',
            controller=
            'ckanext.uebpackage.controllers.uebexecute:UEBexecuteController',
            action='select_model_package')
        map.connect(
            '/uebpackage/ueb_execute/{pkg_id}',
            controller=
            'ckanext.uebpackage.controllers.uebexecute:UEBexecuteController',
            action='execute')
        map.connect(
            '/uebpackage/ueb_execute_status/{pkg_id}',
            controller=
            'ckanext.uebpackage.controllers.uebexecute:UEBexecuteController',
            action='check_package_run_status')
        map.connect(
            '/uebpackage/check_package_build_status/{pkg_id}',
            controller=
            'ckanext.uebpackage.controllers.packagecreate:PackagecreateController',
            action='check_package_build_status')
        map.connect(
            '/uebpackage/retrieve_input_package/{pkg_id}',
            controller=
            'ckanext.uebpackage.controllers.packagecreate:PackagecreateController',
            action='retrieve_input_package')
        map.connect(
            '/uebpackage/retrieve_output_package/{pkg_id}',
            controller=
            'ckanext.uebpackage.controllers.uebexecute:UEBexecuteController',
            action='retrieve_output_package')
        return map

    # ITemplateHelpers method implementation
    def get_helpers(self):

        # Template helper function names (e.g: 'uebpackage_build_nav_main')should begin with the name of the
        # extension they belong to, to avoid clashing with functions from
        # other extensions.
        return {
            'uebpackage_build_nav_main': uebpackage_build_main_navigation,
            'get_package_extras': get_package_extras
        }
Пример #5
0
class GoogleAnalyticsPlugin(p.SingletonPlugin):
    p.implements(p.IConfigurable, inherit=True)
    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IConfigurer, inherit=True)
    p.implements(p.ITemplateHelpers)

    analytics_queue = Queue.Queue()

    def configure(self, config):
        '''Load config settings for this extension from config file.

        See IConfigurable.

        '''
        if 'googleanalytics.id' not in config:
            msg = "Missing googleanalytics.id in config"
            raise GoogleAnalyticsException(msg)
        self.googleanalytics_id = config['googleanalytics.id']
        self.googleanalytics_id2 = config.get('googleanalytics.id2')
        self.googleanalytics_domain = config.get('googleanalytics.domain',
                                                 'auto')
        self.googleanalytics_fields = ast.literal_eval(
            config.get('googleanalytics.fields', '{}'))

        googleanalytics_linked_domains = config.get(
            'googleanalytics.linked_domains', '')
        self.googleanalytics_linked_domains = [
            x.strip() for x in googleanalytics_linked_domains.split(',') if x
        ]

        if self.googleanalytics_linked_domains:
            self.googleanalytics_fields['allowLinker'] = 'true'

        self.googleanalytics_javascript_url = h.url_for_static(
            '/scripts/ckanext-googleanalytics.js')

        # If resource_prefix is not in config file then write the default value
        # to the config dict, otherwise templates seem to get 'true' when they
        # try to read resource_prefix from config.
        if 'googleanalytics_resource_prefix' not in config:
            config['googleanalytics_resource_prefix'] = (
                commands.DEFAULT_RESOURCE_URL_TAG)
        self.googleanalytics_resource_prefix = config[
            'googleanalytics_resource_prefix']

        self.show_downloads = converters.asbool(
            config.get('googleanalytics.show_downloads', True))
        self.track_events = converters.asbool(
            config.get('googleanalytics.track_events', False))

        if not converters.asbool(config.get('ckan.legacy_templates', 'false')):
            p.toolkit.add_resource('fanstatic_library',
                                   'ckanext-googleanalytics')

            # spawn a pool of 5 threads, and pass them queue instance
        for i in range(5):
            t = AnalyticsPostThread(self.analytics_queue)
            t.setDaemon(True)
            t.start()

    def update_config(self, config):
        '''Change the CKAN (Pylons) environment configuration.

        See IConfigurer.

        '''
        if converters.asbool(config.get('ckan.legacy_templates', 'false')):
            p.toolkit.add_template_directory(config, 'legacy_templates')
            p.toolkit.add_public_directory(config, 'legacy_public')
        else:
            p.toolkit.add_template_directory(config, 'templates')

    def before_map(self, map):
        '''Add new routes that this extension's controllers handle.

        See IRoutes.

        '''
        # Helpers to reduce code clutter
        GET = dict(method=['GET'])
        PUT = dict(method=['PUT'])
        POST = dict(method=['POST'])
        DELETE = dict(method=['DELETE'])
        GET_POST = dict(method=['GET', 'POST'])
        # intercept API calls that we want to capture analytics on
        register_list = [
            'package', 'dataset', 'resource', 'tag', 'group', 'related',
            'revision', 'licenses', 'rating', 'user', 'activity'
        ]
        register_list_str = '|'.join(register_list)
        # /api ver 3 or none
        with SubMapper(
                map,
                controller='ckanext.googleanalytics.controller:GAApiController',
                path_prefix='/api{ver:/3|}',
                ver='/3') as m:
            m.connect('/action/{logic_function}',
                      action='action',
                      conditions=GET_POST)

        # /api ver 1, 2, 3 or none
        with SubMapper(
                map,
                controller='ckanext.googleanalytics.controller:GAApiController',
                path_prefix='/api{ver:/1|/2|/3|}',
                ver='/1') as m:
            m.connect('/search/{register}', action='search')

        # /api/rest ver 1, 2 or none
        with SubMapper(
                map,
                controller='ckanext.googleanalytics.controller:GAApiController',
                path_prefix='/api{ver:/1|/2|}',
                ver='/1',
                requirements=dict(register=register_list_str)) as m:

            m.connect('/rest/{register}', action='list', conditions=GET)
            m.connect('/rest/{register}', action='create', conditions=POST)
            m.connect('/rest/{register}/{id}', action='show', conditions=GET)
            m.connect('/rest/{register}/{id}', action='update', conditions=PUT)
            m.connect('/rest/{register}/{id}',
                      action='update',
                      conditions=POST)
            m.connect('/rest/{register}/{id}',
                      action='delete',
                      conditions=DELETE)

        with SubMapper(
                map,
                controller=
                'ckanext.googleanalytics.controller:GADatastoreController'
        ) as m:
            m.connect('/datastore/dump/{resource_id}', action='dump')
            m.connect('/datastore/download/{resource_id}', action='dump')

        return map

    def after_map(self, map):
        '''Add new routes that this extension's controllers handle.

        See IRoutes.

        '''
        self.modify_resource_download_route(map)
        map.redirect("/analytics/package/top", "/analytics/dataset/top")
        map.connect(
            'analytics',
            '/analytics/dataset/top',
            controller='ckanext.googleanalytics.controller:GAController',
            action='view')
        return map

    def get_helpers(self):
        '''Return the CKAN 2.0 template helper functions this plugin provides.

        See ITemplateHelpers.

        '''
        return {'googleanalytics_header': self.googleanalytics_header}

    def googleanalytics_header(self):
        '''Render the googleanalytics_header snippet for CKAN 2.0 templates.

        This is a template helper function that renders the
        googleanalytics_header jinja snippet. To be called from the jinja
        templates in this extension, see ITemplateHelpers.

        '''
        data = {
            'googleanalytics_id': self.googleanalytics_id,
            'googleanalytics_id2': self.googleanalytics_id2,
            'googleanalytics_domain': self.googleanalytics_domain,
            'googleanalytics_fields': str(self.googleanalytics_fields),
            'googleanalytics_linked_domains':
            self.googleanalytics_linked_domains
        }
        return p.toolkit.render_snippet(
            'googleanalytics/snippets/googleanalytics_header.html', data)

    def modify_resource_download_route(self, map):
        '''Modifies resource_download method in related controller
        to attach GA tracking code.
        '''

        if '_routenames' in map.__dict__:
            if 'resource_download' in map.__dict__['_routenames']:
                route_data = map.__dict__['_routenames'][
                    'resource_download'].__dict__
                route_controller = route_data['defaults']['controller'].split(
                    ':')
                module = importlib.import_module(route_controller[0])
                controller_class = getattr(module, route_controller[1])
                controller_class.resource_download = post_analytics_decorator(
                    controller_class.resource_download)
            else:
                # If no custom uploader applied, use the default one
                PackageController.resource_download = post_analytics_decorator(
                    PackageController.resource_download)
Пример #6
0
class SchemaPlugin(plugins.SingletonPlugin, tk.DefaultDatasetForm):
    plugins.implements(plugins.IValidators)
    plugins.implements(plugins.IDatasetForm)
    plugins.implements(plugins.IPackageController, inherit=True)

    # IValidators

    def get_validators(self):  # noqa
        return {
            'convert_list_to_string': converters.convert_list_to_string,
            'convert_string_to_list': converters.convert_string_to_list,
            'default_conversion': converters.default,
            'single_value': validators.single_valued,
            'multi_value': validators.multi_valued,
            'is_string': validators.string,
            'is_bool': validators.boolean,
            'is_uri': validators.uri,
            'is_date': validators.date,
            'is_number': validators.number,
            'controlled_vocabulary': validators.in_vocabulary,
            'taxonomy': validators.in_taxonomy,
            'determine_communities': validators.extract_communities,
            'contact_point': validators.contact_point,
            'temporal': validators.temporal,
            'date_planned': validators.date_planned,
            'legal_foundation': validators.legal_foundation,
            'checksum': validators.checksum,
            'rights': validators.rights,
            'spatial': validators.spatial,
            'epsg_28992': validators.epsg28992,
            'postcode_huisnummer': validators.postcode_huisnummer
        }

    # IDatasetForm

    def is_fallback(self):  # noqa
        return tk.asbool(
            config.get('ckan.ckanext-dataoverheid.is_fallback', True))

    def package_types(self):  # noqa
        return tk.aslist(
            config.get('ckan.ckanext-dataoverheid.package_types', []))

    def create_package_schema(self):
        schema = super(SchemaPlugin, self).create_package_schema()
        schema = dcat_ap_donl.create_schema(schema)
        schema = dataoverheid.create_schema(schema)

        return schema

    def update_package_schema(self):
        schema = super(SchemaPlugin, self).update_package_schema()
        schema = dcat_ap_donl.update_schema(schema)
        schema = dataoverheid.update_schema(schema)

        return schema

    def show_package_schema(self):
        schema = super(SchemaPlugin, self).show_package_schema()
        schema = dcat_ap_donl.show_schema(schema)
        schema = dataoverheid.show_schema(schema)

        return schema

    # IPackageController

    def before_index(self, data_dict):  # noqa
        return transformers.transform_multivalued_properties(data_dict)

    def after_show(self, context, data_dict):  # noqa
        return context, transformers.remove_properties(data_dict)
Пример #7
0
class OLGeoView(GeoViewMixin, GeoViewBase):

    p.implements(p.ITemplateHelpers)

    GEOVIEW_FORMATS = [
        "kml",
        "geojson",
        "gml",
        "wms",
        "wfs",
        "esrigeojson",
        "gft",
        "arcgis_rest",
        "wmts",
        "esri rest",
    ]

    # ITemplateHelpers

    def get_helpers(self):
        return {
            "get_common_map_config_geoviews": utils.get_common_map_config,
            "get_openlayers_viewer_config": utils.get_openlayers_viewer_config,
        }

    # IResourceView

    def info(self):
        return {
            "name": "geo_view",
            "title": "Map viewer (OpenLayers)",
            "icon": "globe",
            "iframed": True,
            "default_title": toolkit._("Map viewer"),
            "schema": {
                "feature_hoveron": [ignore_empty, boolean_validator],
                "feature_style": [ignore_empty],
            },
        }

    def can_view(self, data_dict):
        format_lower = data_dict["resource"].get("format", "").lower()
        same_domain = on_same_domain(data_dict)

        # Guess from file extension
        if not format_lower and data_dict["resource"].get("url"):
            format_lower = self._guess_format_from_extension(
                data_dict["resource"]["url"])

        if not format_lower:
            return False

        view_formats = toolkit.config.get("ckanext.geoview.ol_viewer.formats",
                                          "")
        if view_formats:
            view_formats = view_formats.split(" ")
        else:
            view_formats = self.GEOVIEW_FORMATS

        correct_format = format_lower in view_formats
        can_preview_from_domain = self.proxy_enabled or same_domain

        return correct_format and can_preview_from_domain

    def view_template(self, context, data_dict):
        return "dataviewer/openlayers.html"

    def form_template(self, context, data_dict):
        return "dataviewer/openlayers_form.html"

    def _guess_format_from_extension(self, url):
        try:
            parsed_url = urlparse(url)
            format_lower = (os.path.splitext(parsed_url.path)[1][1:].encode(
                "ascii", "ignore").lower())
        except ValueError as e:
            log.error("Invalid URL: {0}, {1}".format(url, e))
            format_lower = ""

        return format_lower

    def setup_template_variables(self, context, data_dict):
        import ckanext.resourceproxy.plugin as proxy

        same_domain = on_same_domain(data_dict)

        if not data_dict["resource"].get("format"):
            data_dict["resource"][
                "format"] = self._guess_format_from_extension(
                    data_dict["resource"]["url"])

        if self.proxy_enabled and not same_domain:
            proxy_url = proxy.get_proxified_resource_url(data_dict)
            proxy_service_url = utils.get_proxified_service_url(data_dict)
        else:
            proxy_url = data_dict["resource"]["url"]
            proxy_service_url = data_dict["resource"]["url"]

        gapi_key = toolkit.config.get("ckanext.geoview.gapi_key")
        return {
            "resource_view_json":
            "resource_view" in data_dict
            and json.dumps(data_dict["resource_view"]),
            "proxy_service_url":
            proxy_service_url,
            "proxy_url":
            proxy_url,
            "gapi_key":
            gapi_key,
            "basemapsConfig":
            self.basemapsConfig,
        }
Пример #8
0
class GeoJSONView(GeoViewBase):
    p.implements(p.ITemplateHelpers, inherit=True)

    GeoJSON = ['gjson', 'geojson']

    def update_config(self, config):

        super(GeoJSONView, self).update_config(config)

        mimetypes.add_type('application/geo+json', '.geojson')

    # IResourceView (CKAN >=2.3)
    def info(self):
        return {
            'name': 'geojson_view',
            'title': 'GeoJSON',
            'icon': 'map-marker',
            'iframed': True,
            'default_title': toolkit._('GeoJSON'),
        }

    def can_view(self, data_dict):
        resource = data_dict['resource']

        format_lower = resource.get('format', '').lower()

        same_domain = on_same_domain(data_dict)

        if format_lower in self.GeoJSON:
            return same_domain or self.proxy_enabled
        return False

    def view_template(self, context, data_dict):
        return 'dataviewer/geojson.html'

    # IResourcePreview (CKAN < 2.3)

    def can_preview(self, data_dict):
        format_lower = data_dict['resource']['format'].lower()

        correct_format = format_lower in self.GeoJSON
        can_preview_from_domain = (self.proxy_enabled or
                                   data_dict['resource'].get('on_same_domain'))
        quality = 2

        if toolkit.check_ckan_version('2.1'):
            if correct_format:
                if can_preview_from_domain:
                    return {'can_preview': True, 'quality': quality}
                else:
                    return {
                        'can_preview': False,
                        'fixable': 'Enable resource_proxy',
                        'quality': quality
                    }
            else:
                return {'can_preview': False, 'quality': quality}

        return correct_format and can_preview_from_domain

    def preview_template(self, context, data_dict):
        return 'dataviewer/geojson.html'

    def setup_template_variables(self, context, data_dict):
        import ckanext.resourceproxy.plugin as proxy
        self.same_domain = data_dict['resource'].get('on_same_domain')
        if self.proxy_enabled and not self.same_domain:
            data_dict['resource']['original_url'] = \
                data_dict['resource'].get('url')
            data_dict['resource']['url'] = \
                proxy.get_proxified_resource_url(data_dict)

    # ITemplateHelpers

    def get_helpers(self):
        return {
            'get_common_map_config_geojson': get_common_map_config,
            'geojson_get_max_file_size': get_max_file_size,
        }
Пример #9
0
class Cioos_HarvestPlugin(plugins.SingletonPlugin):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(ISpatialHarvester, inherit=True)
    plugins.implements(plugins.IOrganizationController, inherit=True)

    # IOrganizationController
    def read(self, entity):
        pass

    def create(self, entity):
        if hasattr(entity, 'title_translated'):
            if entity.title_translated == '{}' or not entity.title_translated:
                toolkit.get_action('organization_patch')(data_dict={
                    'id':
                    entity.id,
                    'title':
                    entity.title,
                    'title_translated':
                    '{"en":"%s", "fr":"%s"}' % (entity.title, entity.title)
                })
        return entity

    def edit(self, entity):
        pass

    def delete(self, entity):
        pass

    def before_view(self, pkg_dict):
        return pkg_dict

    # IConfigurer
    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')
        toolkit.add_public_directory(config_, 'public')
        toolkit.add_resource('fanstatic', 'cioos_harvest')

    # ISpatialHarvester
    def get_validators(self):
        return [MyValidator]

    def from_json(self, val):
        try:
            new_val = json.loads(val)
        except Exception:
            new_val = val
        return new_val

    def _get_object_extra(self, harvest_object, key):
        '''
        Helper function for retrieving the value from a harvest object extra,
        given the key, copied from ckanext-spatial/ckanext/spatial/harvesters/base.py
        '''
        for extra in harvest_object.extras:
            if extra.key == key:
                return extra.value
        return None

    def trim_values(self, values):
        if (isinstance(values, Number)):
            return values
        elif (isinstance(values, list)):
            return [self.trim_values(x) for x in values]
        elif (isinstance(values, dict)):
            return {k.strip(): self.trim_values(v) for k, v in values.items()}
        elif (isinstance(values, str)):
            try:
                json_object = json.loads(values)
            except ValueError:
                return values.strip()
            else:
                return json.dumps(self.trim_values(json_object))
        return values

    def cioos_guess_resource_format(self, url, use_mimetypes=True):
        '''
        Given a URL try to guess the best format to assign to the resource

        This function does not replace the guess_resource_format() in the base
        spatial harvester. In stead it adds some resource and file types that
        are missing from that function.

        Returns None if no format could be guessed.

        '''
        url = url.lower().strip()
        resource_types = {
            # ERDDAP
            'ERDDAP': ('/erddap/', ),
        }

        for resource_type, parts in resource_types.items():
            if any(part in url for part in parts):
                return resource_type

        file_types = {
            'CSV': ('csv', ),
            'PDF': ('pdf', ),
            'TXT': ('txt', ),
            'XML': ('xml', ),
            'HTML': ('html', ),
            'JSON': ('json', ),
        }

        for file_type, extensions in file_types.items():
            if any(url.endswith(extension) for extension in extensions):
                return file_type

        return None

    def get_package_dict(self, context, data_dict):
        package_dict = data_dict['package_dict']
        iso_values = data_dict['iso_values']
        harvest_object = data_dict['harvest_object']
        source_config = json.loads(data_dict['harvest_object'].source.config)
        xml_location_url = self._get_object_extra(data_dict['harvest_object'],
                                                  'waf_location')
        xml_modified_date = self._get_object_extra(data_dict['harvest_object'],
                                                   'waf_modified_date')

        # convert extras key:value list to dictinary
        extras = {x['key']: x['value'] for x in package_dict.get('extras', [])}

        extras['xml_location_url'] = xml_location_url
        if xml_modified_date:
            extras['xml_modified_date'] = xml_modified_date

        # copy some fields over from iso_values if they exist
        if (iso_values.get('limitations-on-public-access')):
            extras['limitations-on-public-access'] = iso_values.get(
                'limitations-on-public-access')
        if (iso_values.get('access-constraints')):
            extras['access-constraints'] = iso_values.get('access-constraints')
        if (iso_values.get('use-constraints')):
            extras['use-constraints'] = iso_values.get('use-constraints')
        if (iso_values.get('use-constraints-code')):
            extras['use-constraints-code'] = iso_values.get(
                'use-constraints-code')
        if (iso_values.get('legal-constraints-reference-code')):
            extras['legal-constraints-reference-code'] = iso_values.get(
                'legal-constraints-reference-code')
        if (iso_values.get('distributor')):
            extras['distributor'] = iso_values.get('distributor')

        # load remote xml content
        package_dict = _extract_xml_from_harvest_object(
            package_dict, harvest_object)

        # Handle Scheming, Composit, and Fluent extensions
        loaded_plugins = plugins.toolkit.config.get("ckan.plugins")
        if 'scheming_datasets' in loaded_plugins:
            # composite = 'composite' in loaded_plugins
            fluent = 'fluent' in loaded_plugins

            log.debug(
                '#### Scheming, Composite, or Fluent extensions found, processing dictinary ####'
            )
            schema = plugins.toolkit.h.scheming_get_dataset_schema('dataset')

            # Package name, default harvester uses title or guid in that order.
            # we want to reverse that order, so guid or title. Also use english
            # title only for name
            title_as_name = self.from_json(package_dict.get(
                'title', '{}')).get('en', package_dict['name'])
            name = munge.munge_name(extras.get('guid', title_as_name)).lower()
            package_dict['name'] = name

            # populate license_id
            package_dict['license_id'] = iso_values.get(
                'legal-constraints-reference-code') or iso_values.get(
                    'use-constraints') or 'CC-BY-4.0'

            # populate citation
            package_dict['citation'] = iso_values.get('citation')

            # populate trlanslation method for bilingual field
            notes_translation_method = iso_values.get(
                'abstract_translation_method')
            title_translation_method = iso_values.get(
                'title_translation_method')
            if notes_translation_method:
                extras['notes_translation_method'] = notes_translation_method
            if title_translation_method:
                extras['title_translation_method'] = title_translation_method

            # iterate over schema fields and update package dictionary as needed
            for field in schema['dataset_fields']:
                handled_fields = []
                self.handle_composite_harvest_dictinary(
                    field, iso_values, extras, package_dict, handled_fields)

                if fluent:
                    self.handle_fluent_harvest_dictinary(
                        field, iso_values, package_dict, schema,
                        handled_fields, source_config)

                self.handle_scheming_harvest_dictinary(field, iso_values,
                                                       extras, package_dict,
                                                       handled_fields)

            # populate resource format if missing
            for resource in package_dict.get('resources', []):
                if not resource.get('format'):
                    if (resource.get('resource_locator_protocol').startswith(
                            'http') or resource.get('url').startswith('http')):
                        resource['format'] = 'text/html'

            # set default values
            package_dict['progress'] = extras.get('progress', 'onGoing')
            package_dict['frequency-of-update'] = extras.get(
                'frequency-of-update', 'asNeeded')

        extras_as_list = []
        for key, value in extras.items():
            if package_dict.get(key, ''):
                log.error('extras %s found in package dict: key:%s value:%s',
                          key, key, value)
            if isinstance(value, (list, dict)):
                extras_as_list.append({'key': key, 'value': json.dumps(value)})
            else:
                extras_as_list.append({'key': key, 'value': value})

        package_dict['extras'] = extras_as_list

        # update resource format
        resources = package_dict.get('resources', [])
        if len(resources):
            for resource in resources:
                url = resource.get('url', '').strip()
                format = resource.get('format') or ''
                if url:
                    format = self.cioos_guess_resource_format(url) or format
                resource['format'] = format
        package_dict['resources'] = resources
        return self.trim_values(package_dict)

    def handle_fluent_harvest_dictinary(self, field, iso_values, package_dict,
                                        schema, handled_fields,
                                        harvest_config):
        field_name = field['field_name']
        if field_name in handled_fields:
            return

        field_value = {}

        if not field.get('preset', '').startswith(u'fluent'):
            return

        # set default language, default to english
        default_language = iso_values.get('metadata-language', 'en')[0:2]
        if not default_language:
            default_language = 'en'

        # handle tag fields
        if field.get('preset', '') == u'fluent_tags':
            fluent_tags = iso_values.get(field_name, [])
            schema_languages = plugins.toolkit.h.fluent_form_languages(
                schema=schema)
            do_clean = toolkit.asbool(harvest_config.get('clean_tags', False))

            # init language key
            field_value = {sl: [] for sl in schema_languages}

            # process fluent_tags by convert list of language dictionaries into
            # a dictionary of language lists
            for t in fluent_tags:
                tobj = self.from_json(t.get('keyword', t))
                if isinstance(tobj, Number):
                    tobj = str(tobj)
                if isinstance(tobj, dict):
                    for key, value in tobj.items():
                        if key in schema_languages:
                            if do_clean:
                                if isinstance(value, list):
                                    value = [
                                        munge.munge_tag(kw) for kw in value
                                    ]
                                else:
                                    value = munge.munge_tag(value)
                            field_value[key].append(value)
                else:
                    if do_clean:
                        tobj = munge.munge_tag(tobj)
                    field_value[default_language].append(tobj)

            package_dict[field_name] = field_value

            # update tags with all values from fluent_tags
            tag_list = [t['name'] for t in package_dict['tags']]
            for item in field_value.get('en', []) + field_value.get('fr', []):
                if item not in tag_list:
                    tag_list.append(item)
            package_dict['tags'] = [{'name': t} for t in tag_list]

        else:
            # Populate translated fields from core. this could have been done in
            # the spatial extensions. example 'title' -> 'title_translated'

            # strip trailing _translated part of field name
            if field_name.endswith(u'_translated'):
                package_fn = field_name[:-11]
            else:
                package_fn = field_name

            package_val = package_dict.get(package_fn, '')
            field_value = self.from_json(package_val)

            if isinstance(field_value,
                          dict):  # assume bilingual values already in data
                package_dict[field_name] = field_value
            else:
                # create bilingual dictionary. This will likely fail validation as it does not contain all the languages
                package_dict[field_name] = {}
                package_dict[field_name][default_language] = field_value

        handled_fields.append(field_name)

    def flatten_composite_keys(self, obj, new_obj={}, keys=[]):
        for key, value in obj.items():
            if isinstance(value, dict):
                self.flatten_composite_keys(obj[key], new_obj, keys + [key])
            else:
                new_obj['_'.join(keys + [key])] = value
        return new_obj

    def handle_composite_harvest_dictinary(self, field, iso_values, extras,
                                           package_dict, handled_fields):
        sep = plugins.toolkit.h.scheming_composite_separator()
        field_name = field['field_name']
        if field_name in handled_fields:
            return

        field_value = iso_values.get(field_name, {})

        # populate composite fields from multi-level dictionary
        if field_value and field.get('simple_subfields'):
            if isinstance(field_value, list):
                field_value = field_value[0]
            field_value = self.flatten_composite_keys(field_value, {}, [])

            for key, value in field_value.items():
                newKey = field_name + sep + key
                package_dict[newKey] = value

            # remove from extras so as not to duplicate fields
            if extras.get(field_name):
                del extras[field_name]
            handled_fields.append(field_name)

        # populate composite repeating fields
        elif field_value and field.get('repeating_subfields'):
            if isinstance(field_value, dict):
                field_value[0] = field_value

            for idx, subitem in enumerate(field_value):
                # collapse subfields into one key value pair
                subitem = self.flatten_composite_keys(subitem, {}, [])
                for key, value in subitem.items():
                    newKey = field_name + sep + str(idx + 1) + sep + key
                    package_dict[newKey] = value

            # remove from extras so as not to duplicate fields
            if extras.get(field_name):
                del extras[field_name]
            handled_fields.append(field_name)

    def handle_scheming_harvest_dictinary(self, field, iso_values, extras,
                                          package_dict, handled_fields):
        field_name = field['field_name']
        if field_name in handled_fields:
            return
        iso_field_value = iso_values.get(field_name, {})
        extra_field_value = extras.get(field_name, "")

        # move schema fields, in extras, to package dictionary
        if field_name in extras and not package_dict.get(field_name, ''):
            package_dict[field_name] = extra_field_value
            del extras[field_name]
            handled_fields.append(field_name)
        # move schema fields, in iso_values, to package dictionary
        elif iso_field_value and not package_dict.get(field_name, ''):
            # convert list to single value for select fields (not multi-select)
            if field.get('preset', '') == 'select' and isinstance(
                    iso_field_value, list):
                iso_field_value = iso_field_value[0]
            package_dict[field_name] = iso_field_value
            # remove from extras so as not to duplicate fields
            if extras.get(field_name):
                del extras[field_name]
            handled_fields.append(field_name)
Пример #10
0
class ChartsPlugin(plugins.SingletonPlugin):
    plugins.implements(plugins.IConfigurer, inherit=True)
    plugins.implements(plugins.IResourceView, inherit=True)

    # IConfigurer

    def update_config(self, config):
        toolkit.add_template_directory(config, 'templates')
        toolkit.add_public_directory(config, 'public')
        toolkit.add_resource('fanstatic', 'c3charts')

    def info(self):
        schema = {
            'chart_type': [not_empty],
            'key_fields': [not_empty],
            'x_fields': [ignore_missing],
            'color_scheme': [not_empty],
            'header': [ignore_missing],
            'measure_unit_x': [ignore_missing],
            'measure_unit_y': [ignore_missing],
            'text_chart_number_action': [not_empty],
            'legend': [not_empty],
            'rotated': [ignore_missing],
            'data_labels': [ignore_missing],
            'x_grid': [ignore_missing],
            'y_grid': [ignore_missing],
            'remap_key': [ignore_missing],
            'aggregate': [ignore_missing]
        }

        return {
            'name': 'Chart builder',
            'icon': 'bar-chart-o',
            'filterable': True,
            'iframed': False,
            'schema': schema
        }

    def can_view(self, data_dict):
        return data_dict['resource'].get('datastore_active', False)

    def setup_template_variables(self, context, data_dict):
        resource = data_dict['resource']
        resource_view = data_dict['resource_view']

        fields = _get_fields_without_id(resource)
        remap_keys = list(fields)
        remap_keys.insert(0, {'value': ''})
        logger.debug(remap_keys)

        return {
            'resource':
            resource,
            'resource_view':
            resource_view,
            'fields':
            fields,
            'remap_keys':
            remap_keys,
            'chart_types': [{
                'value': 'Bar Chart'
            }, {
                'value': 'Stacked Bar Chart'
            }, {
                'value': 'Donut Chart'
            }, {
                'value': 'Line Chart'
            }, {
                'value': 'Pie Chart'
            }, {
                'value': 'Spline Chart'
            }, {
                'value': 'Table Chart'
            }, {
                'value': 'Simple Chart'
            }],
            'color_schemes': [{
                'value':
                '#B80000, #995522, #556677, #118888, #115588, '
                '#4C3D3D, #2B2B2B, #660000, #221100',
                'text':
                'Saturated'
            }, {
                'value':
                '#DDBBAA, #79E6F2, #88AA99, #00A864, #228899, '
                '#3F797F, #775555, #118855, #008751, #3D4C46',
                'text':
                'Light'
            }, {
                'value':
                '#ADC0D8, #79AFF2, #8899AA, #0EAAB2, #00A0A8, '
                '#776655, #118888, #885511, #3F5C7F, #225599',
                'text':
                'Pastel'
            }, {
                'value':
                '#ADB1D8, #8899AA, #7983F2, #777752, #887711, '
                '#0070C0, #0062A8, #3F457F, #115588, #3D464C',
                'text':
                'Pastel 2'
            }, {
                'value':
                '#AA9988, #A88600, #779922, #6C7F3F, #887711, '
                '#555577, #665500, #665100, #4C493D, #2B2B2V',
                'text':
                'Contrast'
            }],
            'text_chart_number_actions': [{
                'value': 'substract',
                'text': 'Substract last two entries'
            }, {
                'value': 'average',
                'text': 'Average'
            }, {
                'value': 'last',
                'text': 'Show last'
            }],
            'legend_options': [{
                'text': 'Hide',
                'value': 'hide'
            }, {
                'text': 'Right',
                'value': 'right'
            }, {
                'text': 'Bottom',
                'value': 'bottom'
            }]
        }

    def view_template(self, context, data_dict):
        return 'charts_view.html'

    def form_template(self, context, data_dict):
        return 'charts_form.html'
Пример #11
0
class TagmanagerPlugin(plugins.SingletonPlugin):
    plugins.implements(plugins.IConfigurer)

    # IConfigurer

    plugins.implements(plugins.IConfigurable)
    plugins.implements(plugins.IRoutes, inherit=True)
    #p.implements(p.IDomainObjectModification, inherit=True)
    #p.implements(p.IResourceUrlChange)
    plugins.implements(plugins.ITemplateHelpers)

    def configure(self, config):
        self.site_url = config.get('ckan.site_url')

    def before_map(self, map):
        tagmanager = 'ckanext.tagmanager.controller:TagmanagerController'

        map.connect('/tagmanager',
                    'tagmanager',
                    controller=tagmanager,
                    action='index')
        map.connect('/tagmanager/edit', controller=tagmanager, action='edit')
        map.connect('/tagmanager/index_process_suggestions',
                    controller=tagmanager,
                    action='index_process_suggestions')
        map.connect('/tagmanager/save_merge_suggestions',
                    controller=tagmanager,
                    action='save_merge_suggestions')
        map.connect('/tagmanager/merge_0',
                    controller=tagmanager,
                    action='merge_0')
        map.connect('/tagmanager/merge_1',
                    controller=tagmanager,
                    action='merge_1')
        map.connect('/tagmanager/merge_2',
                    controller=tagmanager,
                    action='merge_2')
        map.connect('/tagmanager/merge_form',
                    controller=tagmanager,
                    action='merge_form')
        map.connect('/tagmanager', controller=tagmanager, action='index')
        map.connect('/tagmanager/merge_confirm',
                    controller=tagmanager,
                    action='merge_confirm')
        map.connect('/tagmanager/merge', controller=tagmanager, action='merge')
        map.connect('/tagmanager/merge_do',
                    controller=tagmanager,
                    action='merge_do')
        map.connect('/tagmanager/delete_confirm',
                    controller=tagmanager,
                    action='delete_confirm')
        map.connect('/tagmanager/delete',
                    controller=tagmanager,
                    action='delete')
        return map

    def after_map(self, map):
        return map

    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')
        toolkit.add_public_directory(config_, 'public')
        toolkit.add_resource('fanstatic', 'tagmanager')

    def get_helpers(self):
        return {
            'tagmanager_list_tags': list_tags,
            'tagmanager_tag_show': tag_show,
            'tagmanager_tags_stats': tags_stats,
            'tagmanager_tag_count': tag_count,
            'tagmanager_has_suggestions': has_suggestions,
            'tagmanager_get_suggestions': get_suggestions,
            'tagmanager_get_name': get_name
        }
Пример #12
0
class SchemaPlugin(plugins.SingletonPlugin):

    plugins.implements(plugins.IConfigurer)

    plugins.implements(plugins.IRoutes, inherit=True)

    plugins.implements(plugins.ITemplateHelpers, inherit=False)

    plugins.implements(plugins.IPackageController, inherit=True)

    plugins.implements(plugins.IFacets, inherit=True)

    plugins.implements(plugins.IActions, inherit=True)

    plugins.implements(plugins.IAuthFunctions)

    plugins.implements(plugins.IResourceController, inherit=True)

    def get_helpers(self):
        return {
            "dataset_type": get_dataset_type,
            "edc_tags": get_edc_tags,
            "edc_orgs": get_organizations,
            "edc_org_branches": get_organization_branches,
            "edc_org_title": get_organization_title,
            "edc_type_label": edc_type_label,
            "edc_state_values": get_state_values,
            "edc_username": get_username,
            "get_sector": get_suborg_sector,
            "get_user_orgs": get_user_orgs,
            "get_user_orgs_id": get_user_orgs_id,
            "get_user_toporgs": get_user_toporgs,
            "get_suborg_sector": get_suborg_sector,
            "get_user_dataset_num": get_user_dataset_num,
            "get_edc_package": get_package_data,
            "is_license_open": is_license_open,
            "record_type_label": get_record_type_label,
            "get_suborgs": get_suborgs,
            "record_is_viewable": record_is_viewable,
            "get_espg_id": get_espg_id,
            "orgs_user_can_edit": get_orgs_user_can_edit,
            "get_facets_selected": get_facets_selected,
            "get_facets_unselected": get_facets_unselected,
            "get_sectors_list": get_sectors_list,
            "get_edc_org": get_edc_org,
            "get_iso_topic_values": get_iso_topic_values,
            "get_eas_login_url": get_eas_login_url,
            "get_fqdn": get_fqdn,
            "get_environment_name": get_environment_name,
            "get_version": get_version,
            "get_bcgov_commit_id": get_bcgov_commit_id,
            "googleanalytics_resource_prefix": resource_prefix,
            "get_parent_org": get_org_parent,
            "size_or_link": size_or_link,
            "display_pacific_time": display_pacific_time,
            "sort_vocab_list": sort_vocab_list,
            "debug_full_info_as_list": debug_full_info_as_list,
            "remove_user_link": remove_user_link,
            "get_dashboard_config": get_dashboard_config,
            "get_ofi_config": get_ofi_config,
            "get_ofi_resources": get_ofi_resources,
            "get_non_ofi_resources": get_non_ofi_resources,
            "can_view_resource": can_view_resource,
            "get_pow_config": get_pow_config,
            "get_package_tracking": get_package_tracking,
            "get_resource_tracking": get_resource_tracking,
            "log": log_this
        }

    def update_config(self, config):
        toolkit.add_public_directory(config, 'public')
        toolkit.add_template_directory(config, 'templates')
        toolkit.add_resource('fanstatic', 'edc_resource')
        toolkit.add_resource('public/scripts', 'theme_scripts')

    # Customizing action mapping
    def before_map(self, map):
        from routes.mapper import SubMapper

        package_controller = 'ckanext.bcgov.controllers.package:EDCPackageController'
        user_controller = 'ckanext.bcgov.controllers.user:EDCUserController'
        org_controller = 'ckanext.bcgov.controllers.organization:EDCOrganizationController'
        site_map_controller = 'ckanext.bcgov.controllers.site_map:GsaSitemapController'
        api_controller = 'ckanext.bcgov.controllers.api:EDCApiController'
        ofi_controller = 'ckanext.bcgov.controllers.ofi:EDCOfiController'

        GET_POST = dict(method=['GET', 'POST'])

        map.connect('package_index',
                    '/',
                    controller=package_controller,
                    action='index')

        with SubMapper(map, controller=package_controller) as m:
            m.connect('/dataset/add', action='typeSelect')
            m.connect('add dataset', '/dataset/new', action='new')
            m.connect(
                'new edc dataset',
                '/{dataset_type}/new',
                action='new',
                requirements=dict(dataset_type='|'.join(
                    ['Dataset', 'Geographic', 'WebService', 'Application'])))
            m.connect('search',
                      '/dataset',
                      action='search',
                      highlight_actions='index search')
            m.connect('dataset_read',
                      '/dataset/{id}',
                      action='read',
                      ckan_icon='sitemap')
            m.connect(
                'read edc dataset',
                '/{dataset_type}/{id}',
                action='read',
                ckan_icon='sitemap',
                requirements=dict(dataset_type='|'.join(
                    ['Dataset', 'Geographic', 'WebService', 'Application'])))
            m.connect('duplicate',
                      '/dataset/duplicate/{id}/{package_type}',
                      action='duplicate')
            m.connect('/dataset/{id}/resource/{resource_id}',
                      action='resource_read')
            m.connect('/dataset/{id}/resource_delete/{resource_id}',
                      action='resource_delete')
            m.connect('/authorization-error', action='auth_error')
            m.connect('resource_edit',
                      '/dataset/{id}/resource_edit/{resource_id}',
                      action='resource_edit',
                      ckan_icon='edit')
            m.connect('new_resource',
                      '/dataset/new_resource/{id}',
                      action='new_resource')
            m.connect('resources',
                      '/dataset/resources/{id}',
                      action='resources')

        with SubMapper(map, controller=user_controller) as m:
            m.connect('user_dashboard_unpublished',
                      '/dashboard/unpublished',
                      action='dashboard_unpublished',
                      ckan_icon='group')
            m.connect('/user/edit', action='edit')
            m.connect('/user/activity/{id}/{offset}', action='activity')
            m.connect('user_activity_stream',
                      '/user/activity/{id}',
                      action='activity',
                      ckan_icon='time')
            m.connect('user_dashboard',
                      '/dashboard',
                      action='dashboard',
                      ckan_icon='list')
            m.connect('user_dashboard_datasets',
                      '/dashboard/datasets',
                      action='dashboard_datasets',
                      ckan_icon='sitemap')
            m.connect('user_dashboard_organizations',
                      '/dashboard/organizations',
                      action='dashboard_organizations',
                      ckan_icon='building')
            m.connect('/dashboard/{offset}', action='dashboard')
            m.connect('user_follow', '/user/follow/{id}', action='follow')
            m.connect('/user/unfollow/{id}', action='unfollow')
            m.connect('user_followers',
                      '/user/followers/{id:.*}',
                      action='followers',
                      ckan_icon='group')
            m.connect('user_edit',
                      '/user/edit/{id:.*}',
                      action='edit',
                      ckan_icon='cog')
            m.connect('user_delete', '/user/delete/{id}', action='delete')
            m.connect('/user/reset/{id:.*}', action='perform_reset')
            m.connect('register', '/user/register', action='register')
            m.connect('login', '/user/login', action='login')
            m.connect('/user/_logout', action='logout')
            m.connect('/user/logged_in', action='logged_in')
            m.connect('/user/logged_out', action='logged_out')
            m.connect('/user/logged_out_redirect', action='logged_out_page')
            m.connect('/user/reset', action='request_reset')
            m.connect('/user/me', action='me')
            m.connect('/user/set_lang/{lang}', action='set_lang')
            m.connect('user_datasets',
                      '/user/{id:.*}',
                      action='read',
                      ckan_icon='sitemap')
            m.connect('user_index', '/user', action='index')

        with SubMapper(map, controller=org_controller) as m:
            m.connect('organizations_index', '/organization', action='index')
            m.connect('/organization/list', action='list')
            m.connect('/organization/new', action='new')
            m.connect('/organization/{action}/{id}',
                      requirements=dict(action='|'.join([
                          'delete', 'admins', 'member_new', 'member_delete',
                          'history'
                      ])))
            m.connect('organization_activity',
                      '/organization/activity/{id}',
                      action='activity',
                      ckan_icon='time')
            m.connect('organization_about',
                      '/organization/about/{id}',
                      action='about',
                      ckan_icon='info-sign')
            m.connect('organization_edit',
                      '/organization/edit/{id}',
                      action='edit',
                      ckan_icon='edit')
            m.connect('organization_members',
                      '/organization/members/{id}',
                      action='members',
                      ckan_icon='group')
            m.connect('organization_bulk_process',
                      '/organization/bulk_process/{id}',
                      action='bulk_process',
                      ckan_icon='sitemap')
            m.connect('organization_read',
                      '/organization/{id}',
                      action='read',
                      ckan_icon='sitemap')

        map.connect('sitemap',
                    '/sitemap.html',
                    controller=site_map_controller,
                    action='view')
        map.connect('sitemap',
                    '/sitemap.xml',
                    controller=site_map_controller,
                    action='read')

        with SubMapper(map,
                       controller=api_controller,
                       path_prefix='/api{ver:/1|/2|/3|}',
                       ver='/1') as m:
            m.connect('/i18n/{lang}', action='i18n_js_translations')
            m.connect('/')

        map.connect('ofi api',
                    '/api/ofi/{call_action}',
                    controller=ofi_controller,
                    action='action',
                    conditions=GET_POST)
        map.connect('ofi resource',
                    '/api/ofi/{format}/{object_name}',
                    action='action')

        m.connect('/action/organization_list_related',
                  action='organization_list_related',
                  conditions=GET_POST)
        m.connect('/action/{logic_function}',
                  action='action',
                  conditions=GET_POST)

        map.connect('/admin/trash', controller='admin', action='trash')
        map.connect('ckanadmin_trash',
                    '/admin/trash',
                    controller='admin',
                    action='trash',
                    ckan_icon='trash')

        return map

    def after_map(self, map):
        return map

    def before_index(self, pkg_dict):
        '''
        Makes the sort by name case insensitive.
        Note that the search index must be rebuild for the first time in order for the changes to take affect.
        '''
        title = pkg_dict['title']
        if title:
            #Assign title to title_string with all characters switched to lower case.
            pkg_dict['title_string'] = title.lower()

        res_format = pkg_dict.get('res_format', [])
        if 'other' in res_format:
            # custom download (other) supports a number of formats
            res_format.remove('other')
            res_format.extend(['shp', 'fgdb', 'e00'])

        return pkg_dict

    def before_search(self, search_params):
        '''
        Customizes package search and applies filters based on the dataset metadata-visibility
        and user roles.
        '''

        #Change the default sort order when no query passed
        if not search_params.get('q') and search_params.get('sort') in (
                None, 'rank'):
            search_params[
                'sort'] = 'record_publish_date desc, metadata_modified desc'

        #Change the query filter depending on the user

        if 'fq' in search_params:
            fq = search_params['fq']
        else:
            fq = ''

        #need to append solr param q.op to force an AND query
        if 'q' in search_params:
            q = search_params['q']
        else:
            q = ''

        try:
            user_name = c.user or 'visitor'

            #  There are no restrictions for sysadmin
            if c.userobj and c.userobj.sysadmin == True:
                fq += ' '
                fq = filter_query_regex.sub(r'+\1', fq)
            else:
                if user_name != 'visitor':
                    if 'edc_state' not in fq:
                        fq = filter_query_regex.sub(r'+\1', fq)
                        fq += ' +(edc_state:("PUBLISHED" OR "PENDING ARCHIVE")'

                        if 'owner_org' not in fq:
                            #IDIR users can also see private records of their organizations
                            user_id = c.userobj.id
                            #Get the list of orgs that the user is an admin or editor of
                            user_orgs = get_orgs_user_can_edit(
                                c.userobj
                            )  #['"' + org + '"' for org in get_orgs_user_can_edit()]
                            #user_orgs = ['"' + org.get('id') + '"' for org in get_user_orgs(user_id, 'admin')]
                            #user_orgs += ['"' + org.get('id') + '"' for org in get_user_orgs(user_id, 'editor')]
                            if user_orgs != []:
                                fq += ' OR ' + 'owner_org:(' + ' OR '.join(
                                    user_orgs) + ')'

                        fq += ')'

                else:
                    if fq:
                        # make all fieds in Filter Query minditory with '+'
                        fq = filter_query_regex.sub(r'+\1', fq)

                    # Public user can only view public and published records
                    fq += ' +(edc_state:("PUBLISHED" OR "PENDING ARCHIVE") AND metadata_visibility:("Public"))'

        except Exception:
            if 'fq' in search_params:
                fq = search_params['fq']
            else:
                fq = ''
            fq += ' +edc_state:("PUBLISHED" OR "PENDING ARCHIVE") +metadata_visibility:("Public")'

        search_params['fq'] = fq

        return search_params

    def before_view(self, pkg_dict):
        # CITZEDC808
        if not record_is_viewable(pkg_dict, c.userobj):
            abort(401,
                  _('Unauthorized to read package %s') % pkg_dict.get("title"))

        return pkg_dict

    #def after_update(self, context, pkg_dict):
    # If there are no resources added, redirect to the "add resource" page after saving
    # if len(pkg_dict.get('resources', [])) == 0:
    #    toolkit.redirect_to(controller='package', action='new_resource', id=pkg_dict['id'])

    def dataset_facets(self, facet_dict, package_type):
        '''
        Customizes search facet list.
        '''

        from collections import OrderedDict
        facet_dict = OrderedDict()
        #Add dataset types and organization sectors to the facet list
        facet_dict['license_id'] = _('License')
        facet_dict['sector'] = _('Sectors')
        facet_dict['type'] = _('Dataset types')
        facet_dict['res_format'] = _('Format')
        facet_dict['organization'] = _('Organizations')
        facet_dict['download_audience'] = _('Download permission')

        if c.userobj and c.userobj.sysadmin:
            facet_dict['edc_state'] = _('States')

        return facet_dict

    def group_facets(self, facet_dict, group_type, package_type):
        '''
        Use the same facets for filtering datasets within group pages
        '''
        return self.dataset_facets(facet_dict, package_type)

    def get_actions(self):
        import ckanext.bcgov.logic.action as edc_action
        from ckanext.bcgov.logic.ofi import call_action as ofi
        return {
            'organization_list': edc_action.organization_list,
            'edc_package_update': edc_action.edc_package_update,
            'edc_package_update_bcgw': edc_action.edc_package_update_bcgw,
            'package_update': edc_action.package_update,
            'package_autocomplete': edc_action.package_autocomplete,
            'check_object_name': ofi.check_object_name,
            'file_formats': ofi.file_formats,
            'crs_types': ofi.crs_types,
            'populate_dataset_with_ofi': ofi.populate_dataset_with_ofi,
            'geo_resource_form': ofi.geo_resource_form,
            'remove_ofi_resources': ofi.remove_ofi_resources,
            'edit_ofi_resources': ofi.edit_ofi_resources,
            'get_max_aoi': ofi.get_max_aoi,
            'ofi_create_order': ofi.ofi_create_order
        }

    def get_auth_functions(self):
        from ckanext.bcgov.logic.auth import create as edc_auth_create
        from ckanext.bcgov.logic.auth.ofi import call_action as ofi
        return {
            'package_create': edc_auth_create.package_create,
            'check_object_name': ofi.check_object_name,
            'file_formats': ofi.file_formats,
            'crs_types': ofi.crs_types,
            'populate_dataset_with_ofi': ofi.populate_dataset_with_ofi,
            'geo_resource_form': ofi.geo_resource_form,
            'remove_ofi_resources': ofi.remove_ofi_resources,
            'edit_ofi_resources': ofi.edit_ofi_resources,
            'get_max_aoi': ofi.get_max_aoi,
            'ofi_create_order': ofi.ofi_create_order
        }

    # IResourceController
    def before_create(self, context, resource):
        # preventative fix for #386 - make sure facet format types are always lowercase;
        resource['format'] = resource.get('format', '').lower()
Пример #13
0
class CollectionPlugin(plugins.SingletonPlugin, DefaultTranslation):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IRoutes, inherit=True)
    plugins.implements(plugins.IPackageController, inherit=True)
    if toolkit.check_ckan_version(min_version='2.5.0'):
        plugins.implements(plugins.ITranslation, inherit=True)
    plugins.implements(plugins.IActions, inherit=True)
    plugins.implements(plugins.IFacets, inherit=True)

    # IConfigurer

    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')
        toolkit.add_public_directory(config_, 'public')
        toolkit.add_resource('fanstatic', 'collection')

    def update_config_schema(self, schema):
        ignore_missing = toolkit.get_validator('ignore_missing')

        schema.update({
            'ckanext.collection.api_collection_name_or_id':
            [ignore_missing, unicode],
        })

        return schema

    # IRoutes

    def before_map(self, map):
        with SubMapper(
                map,
                controller='ckanext.collection.controller:CollectionController'
        ) as m:
            m.connect('collection.index', '/collection', action='index')

            m.connect('collection.new', '/collection/new', action='new')

            m.connect('collection.read', '/collection/:id', action='read')

            m.connect('collection.members',
                      '/collection/:id/members',
                      action='members')

            m.connect('collection.edit', '/collection/edit/:id', action='edit')

            m.connect('collection.delete',
                      '/collection/delete/:id',
                      action='delete')

            m.connect('collection.about',
                      '/collection/about/:id',
                      action='about')

            m.connect('dataset_collection_list',
                      '/dataset/collections/{id}',
                      action='dataset_collection_list',
                      ckan_icon='picture')

        map.redirect('/collections', '/collection')

        return map

    def before_index(self, data_dict):
        groups = json.loads(data_dict.get('data_dict', {})).get('groups', [])

        data_dict['collections'] = [
            group.get('name', '') for group in groups
            if group.get('type', "") == 'collection'
        ]

        groups_to_remove = [
            group.get('name', '') for group in groups
            if group.get('type', "") == 'collection'
        ]
        data_dict['groups'] = [
            group for group in data_dict['groups']
            if group not in groups_to_remove
        ]

        return data_dict

    def after_search(self, search_results, search_params):
        if search_results['search_facets'].get('collections'):
            context = {'for_view': True, 'with_private': False}
            data_dict = {
                'all_fields': True,
                'include_extras': True,
                'type': 'collection'
            }
            collections_with_extras = get_action('group_list')(context,
                                                               data_dict)

            for i, facet in enumerate(
                    search_results['search_facets']['collections'].get(
                        'items', [])):
                for collection in collections_with_extras:
                    if facet['name'] == collection['name']:
                        search_results['search_facets']['collections'][
                            'items'][i]['title_translated'] = collection.get(
                                'title_translated')
                        if not collection.get('title_translated').get('en'):
                            search_results['search_facets']['collections'][
                                'items'][i]['title_translated'][
                                    'en'] = collection.get('title')
                        if not collection.get('title_translated').get('sv'):
                            search_results['search_facets']['collections'][
                                'items'][i]['title_translated'][
                                    'sv'] = collection.get('title')

        return search_results

    # IActions

    def get_actions(self):
        return {
            'group_list_authz': action.group_list_authz,
            'api_collection_show': action.api_collection_show
        }

    # IFacets

    def group_facets(self, facets_dict, group_type, package_type):

        if (group_type == 'collection'):
            facets_dict = OrderedDict()
            facets_dict.update({'res_format': _('Formats')})
            facets_dict.update(
                {'vocab_geographical_coverage': _('Geographical Coverage')})
            facets_dict.update({'groups': _('Groups')})
            facets_dict.update({'organization': _('Organizations')})
            facets_dict.update({'collections': _('Collections')})

        return facets_dict
Пример #14
0
        try:
            if g.facet_json_data:
                print "global value is there..."
        except AttributeError:
            print "Facet json config is not available. Returning the default facets."
            helpers.load_ngds_facets()
            #print "search_results: ",search_results
        return search_results

    def before_view(self,pkg):
        # pkg['title'] = "Muhaha"
        # print pkg
        # TODO - Use for rendering packages. Process resources to get more responsible party information from the email.
        return pkg

    implements(IFacets, inherit=True)

    def dataset_facets(self, facets_dict, package_type):

        if package_type == 'harvest':
            return OrderedDict([('frequency', 'Frequency'), ('source_type', 'Type')])

        ngds_facets = helpers.load_ngds_facets()

        if ngds_facets:
            facets_dict = ngds_facets

        return facets_dict

    def organization_facets(self, facets_dict, organization_type, package_type):
Пример #15
0
class PrivateDatasets(p.SingletonPlugin, tk.DefaultDatasetForm):

    p.implements(p.IDatasetForm)
    p.implements(p.IAuthFunctions)
    p.implements(p.IConfigurer)
    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IActions)
    p.implements(p.IPackageController, inherit=True)
    p.implements(p.ITemplateHelpers)

    ######################################################################
    ############################ DATASET FORM ############################
    ######################################################################

    def __init__(self, name=None):
        self.indexer = search.PackageSearchIndex()

    def _modify_package_schema(self):
        return {
            # remove datasets_with_no_organization_cannot_be_private validator
            'private': [
                tk.get_validator('ignore_missing'),
                tk.get_validator('boolean_validator')
            ],
            constants.ALLOWED_USERS_STR: [
                tk.get_validator('ignore_missing'),
                conv_val.private_datasets_metadata_checker
            ],
            constants.ALLOWED_USERS: [
                conv_val.allowed_users_convert,
                tk.get_validator('ignore_missing'),
                conv_val.private_datasets_metadata_checker
            ],
            constants.ACQUIRE_URL: [
                tk.get_validator('ignore_missing'),
                conv_val.private_datasets_metadata_checker,
                conv_val.url_checker,
                tk.get_converter('convert_to_extras')
            ],
            constants.SEARCHABLE: [
                tk.get_validator('ignore_missing'),
                conv_val.private_datasets_metadata_checker,
                tk.get_converter('convert_to_extras'),
                tk.get_validator('boolean_validator')
            ]
        }

    def create_package_schema(self):
        # grab the default schema in our plugin
        schema = super(PrivateDatasets, self).create_package_schema()
        schema.update(self._modify_package_schema())
        # update resource
        # Add our custom_resource_text metadata field to the schema
        schema['resources'].update({
            'allowed_users': [
                tk.get_validator('ignore_missing'),
                tk.get_converter('convert_to_extras')
            ]
        })
        return schema

    def update_package_schema(self):
        # grab the default schema in our plugin
        schema = super(PrivateDatasets, self).update_package_schema()
        schema.update(self._modify_package_schema())
        # update resource
        # Add our custom_resource_text metadata field to the schema
        schema['resources'].update({
            'allowed_users': [
                tk.get_validator('ignore_missing'),
                tk.get_converter('convert_to_extras')
            ]
        })
        return schema

    def show_package_schema(self):
        schema = super(PrivateDatasets, self).show_package_schema()
        schema.update({
            constants.ALLOWED_USERS:
            [conv_val.get_allowed_users,
             tk.get_validator('ignore_missing')],
            constants.ACQUIRE_URL: [
                tk.get_converter('convert_from_extras'),
                tk.get_validator('ignore_missing')
            ],
            constants.SEARCHABLE: [
                tk.get_converter('convert_from_extras'),
                tk.get_validator('ignore_missing')
            ]
        })
        # update resource
        # Add our custom_resource_text metadata field to the schema
        schema['resources'].update({
            'allowed_users': [
                tk.get_validator('ignore_missing'),
                tk.get_converter('convert_from_extras')
            ]
        })
        return schema

    def is_fallback(self):
        # Return True to register this plugin as the default handler for
        # package types not handled by any other IDatasetForm plugin.
        return True

    def package_types(self):
        # This plugin doesn't handle any special package types, it just
        # registers itself as the default (above).
        return []

    ######################################################################
    ########################### AUTH FUNCTIONS ###########################
    ######################################################################

    def get_auth_functions(self):
        auth_functions = {
            'package_show':
            auth.package_show,
            'package_update':
            auth.package_update,
            # 'resource_show': auth.resource_show,
            constants.PACKAGE_ACQUIRED:
            auth.package_acquired,
            constants.ACQUISITIONS_LIST:
            auth.acquisitions_list
        }

        # resource_show is not required in CKAN 2.3 because it delegates to
        # package_show
        if not tk.check_ckan_version(min_version='2.3'):
            auth_functions['resource_show'] = auth.resource_show

        return auth_functions

    ######################################################################
    ############################ ICONFIGURER #############################
    ######################################################################

    def update_config(self, config):
        # Add this plugin's templates dir to CKAN's extra_template_paths, so
        # that CKAN will use this plugin's custom templates.
        tk.add_template_directory(config, 'templates')

        # Register this plugin's fanstatic directory with CKAN.
        tk.add_resource('fanstatic', 'privatedatasets')

    ######################################################################
    ############################## IROUTES ###############################
    ######################################################################

    def before_map(self, m):
        # DataSet acquired notification
        m.connect(
            'user_acquired_datasets',
            '/dashboard/acquired',
            ckan_icon='shopping-cart',
            controller=
            'ckanext.privatedatasets.controllers.ui_controller:AcquiredDatasetsControllerUI',
            action='user_acquired_datasets',
            conditions=dict(method=['GET']))

        return m

    ######################################################################
    ############################## IACTIONS ##############################
    ######################################################################

    def get_actions(self):
        return {
            constants.PACKAGE_ACQUIRED: actions.package_acquired,
            constants.ACQUISITIONS_LIST: actions.acquisitions_list
        }

    ######################################################################
    ######################### IPACKAGECONTROLLER #########################
    ######################################################################

    def _delete_pkg_atts(self, pkg_dict, attrs):
        for attr in attrs:
            if attr in pkg_dict:
                del pkg_dict[attr]

    def before_index(self, pkg_dict):

        if 'extras_' + constants.SEARCHABLE in pkg_dict:
            if pkg_dict['extras_searchable'] == 'False':
                pkg_dict['capacity'] = 'private'
            else:
                pkg_dict['capacity'] = 'public'

        return pkg_dict

    def after_create(self, context, pkg_dict):
        session = context['session']
        update_cache = False

        db.init_db(context['model'])

        # Get the users and the package ID
        if constants.ALLOWED_USERS in pkg_dict:

            allowed_users = pkg_dict[constants.ALLOWED_USERS]
            package_id = pkg_dict['id']

            # Get current users
            users = db.AllowedUser.get(package_id=package_id)

            # Delete users and save the list of current users
            current_users = []
            for user in users:
                current_users.append(user.user_name)
                if user.user_name not in allowed_users:
                    session.delete(user)
                    update_cache = True

            # Add non existing users
            for user_name in allowed_users:
                if user_name not in current_users:
                    out = db.AllowedUser()
                    out.package_id = package_id
                    out.user_name = user_name
                    out.save()
                    session.add(out)
                    update_cache = True

            session.commit()

            # The cache should be updated. Otherwise, the system may return
            # outdated information in future requests
            if update_cache:
                new_pkg_dict = tk.get_action('package_show')(
                    {
                        'model': context['model'],
                        'ignore_auth': True,
                        'validate': False,
                        'use_cache': False
                    }, {
                        'id': package_id
                    })

                # Prevent acquired datasets jumping to the first position
                revision = tk.get_action('revision_show')(
                    {
                        'ignore_auth': True
                    }, {
                        'id': new_pkg_dict['revision_id']
                    })
                new_pkg_dict['metadata_modified'] = revision.get(
                    'timestamp', '')
                self.indexer.update_dict(new_pkg_dict)

        return pkg_dict

    def after_update(self, context, pkg_dict):
        return self.after_create(context, pkg_dict)

    def after_show(self, context, pkg_dict):

        user_obj = context.get('auth_user_obj')
        updating_via_api = context.get(constants.CONTEXT_CALLBACK, False)

        # allowed_users and searchable fileds can be only viewed by (and only if the dataset is private):
        # * the dataset creator
        # * the sysadmin
        # * users allowed to update the allowed_users list via the notification API
        if pkg_dict.get('private') is False or not updating_via_api and (
                not user_obj or (pkg_dict['creator_user_id'] != user_obj.id
                                 and not user_obj.sysadmin)):
            # The original list cannot be modified
            attrs = list(HIDDEN_FIELDS)
            self._delete_pkg_atts(pkg_dict, attrs)

        return pkg_dict

    def after_delete(self, context, pkg_dict):
        session = context['session']
        package_id = pkg_dict['id']

        # Get current users
        db.init_db(context['model'])
        users = db.AllowedUser.get(package_id=package_id)

        # Delete all the users
        for user in users:
            session.delete(user)
        session.commit()

        return pkg_dict

    def after_search(self, search_results, search_params):
        for result in search_results['results']:
            # Extra fields should not be returned
            # The original list cannot be modified
            attrs = list(HIDDEN_FIELDS)

            # Additionally, resources should not be included if the user is not allowed
            # to show the resource
            context = {
                'model': model,
                'session': model.Session,
                'user': tk.c.user,
                'user_obj': tk.c.userobj
            }

            try:
                tk.check_access('package_show', context, result)
            except tk.NotAuthorized:
                # NotAuthorized exception is risen when the user is not allowed
                # to read the package.
                attrs.append('resources')

            # Delete
            self._delete_pkg_atts(result, attrs)

        return search_results

    ######################################################################
    ######################### ITEMPLATESHELPER ###########################
    ######################################################################

    def get_helpers(self):
        return {
            'is_dataset_acquired': helpers.is_dataset_acquired,
            'get_allowed_users_str': helpers.get_allowed_users_str,
            'is_owner': helpers.is_owner,
            'can_read': helpers.can_read,
            'show_acquire_url_on_create': helpers.show_acquire_url_on_create,
            'show_acquire_url_on_edit': helpers.show_acquire_url_on_edit,
            'acquire_button': helpers.acquire_button,
            'is_dataresource_acquired': helpers.is_dataresource_acquired
        }
Пример #16
0
class PluginObserverPlugin(MockSingletonPlugin):
    implements(IPluginObserver)
Пример #17
0
class WPRDCPlugin(p.SingletonPlugin):

    p.implements(p.IConfigurer, inherit=True)
    p.implements(p.ITemplateHelpers, inherit=True)
    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IPackageController, inherit=True)

    # IConfigurer
    def update_config(self, config):
        p.toolkit.add_template_directory(config, 'templates')
        p.toolkit.add_public_directory(config, 'public')
        p.toolkit.add_resource('fanstatic', 'wprdc_theme')

    # ITemplateHelpers
    def get_helpers(self):
        return {
            'wprdc_user_terms': self.check_user_terms,
            'wprdc_get_year': self.get_current_year,
            'wprdc_wordpress_url': self.get_wordpress_url,
            'wprdc_google_tracking': self.get_google_tracking,
            'render_datetime': self.convert_to_local,
        }

    def check_user_terms(self):
        if check_if_google():
            return True
        else:
            if 'wprdc_user_terms' in request.cookies:
                return True
            else:
                controller = 'ckanext.wprdc.controller:WPRDCController'
                h.redirect_to(controller=controller,
                              action='view_terms',
                              came_from=request.url)

    def get_current_year(self):
        return datetime.date.today().year

    def get_wordpress_url(self):
        url = config.get('ckan.wordpress_url', 'http://www.wprdc.org')
        return url

    def get_google_tracking(self):
        url = config.get('ckan.google_tracking', '')
        return url

    def convert_to_local(self, time, date_format=None, with_hours=False):
        from_zone = tz.tzutc()
        to_zone = tz.tzlocal()
        utc = h._datestamp_to_datetime(str(time))

        if not utc:
            return ''

        utc = utc.replace(tzinfo=from_zone)
        time = utc.astimezone(to_zone)

        # if date_format was supplied we use it
        if date_format:
            return time.strftime(date_format)

        # if with_hours was supplied show them
        if with_hours:
            return time.strftime('%B %-d, %Y, %-I:%M %p')
        else:
            return time.strftime('%B %-d, %Y')

    # IRoutes
    def before_map(self, map):
        controller = 'ckanext.wprdc.controller:WPRDCController'
        map.redirect('/', '/dataset')
        map.connect('terms',
                    '/terms-of-use',
                    controller=controller,
                    action='view_terms',
                    conditions=dict(method=['GET']))
        map.connect('terms',
                    '/terms-of-use',
                    controller=controller,
                    action='submit_terms',
                    conditions=dict(method=['POST']))
        return map

    # IPackageController
    def after_create(self, context, pkg_dict):
        if 'group' in pkg_dict:
            if pkg_dict['group']:
                data = {
                    'id': pkg_dict['group'],
                    'object': pkg_dict['id'],
                    'object_type': 'package',
                    'capacity': 'public'
                }
                p.toolkit.get_action('member_create')(context, data)

    def after_update(self, context, pkg_dict):
        if 'group' in pkg_dict:
            if pkg_dict['group']:
                data = {
                    'id': pkg_dict['group'],
                    'object': pkg_dict['id'],
                    'object_type': 'package',
                    'capacity': 'public'
                }
                p.toolkit.get_action('member_create')(context, data)
            self.remove_from_other_groups(context, pkg_dict['id'])

    def remove_from_other_groups(self, context, package_id):
        package = p.toolkit.get_action('package_show')(context, {
            'id': package_id
        })
        for group in package['groups']:
            if group['name'] != package['group']:
                p.toolkit.get_action('member_delete')(context, {
                    'id': group['id'],
                    'object': package['id'],
                    'object_type': 'package'
                })
Пример #18
0
class SHPView(GeoViewBase):
    p.implements(p.ITemplateHelpers, inherit=True)

    SHP = ['shp', 'shapefile']

    # IResourceView (CKAN >=2.3)
    def info(self):
        return {
            'name': 'shp_view',
            'title': 'Shapefile',
            'icon': 'map-marker',
            'iframed': True,
            'default_title': p.toolkit._('Shapefile'),
        }

    def can_view(self, data_dict):
        resource = data_dict['resource']
        format_lower = resource.get('format', '').lower()

        if format_lower in self.SHP:
            return self.same_domain or self.proxy_enabled
        return False

    def view_template(self, context, data_dict):
        return 'dataviewer/shp.html'

    # IResourcePreview (CKAN < 2.3)
    def can_preview(self, data_dict):
        format_lower = data_dict['resource']['format'].lower()

        correct_format = format_lower in self.SHP
        can_preview_from_domain = (self.proxy_enabled or
                                   data_dict['resource'].get('on_same_domain'))
        quality = 2

        if p.toolkit.check_ckan_version('2.1'):
            if correct_format:
                if can_preview_from_domain:
                    return {'can_preview': True, 'quality': quality}
                else:
                    return {
                        'can_preview': False,
                        'fixable': 'Enable resource_proxy',
                        'quality': quality
                    }
            else:
                return {'can_preview': False, 'quality': quality}

        return correct_format and can_preview_from_domain

    def preview_template(self, context, data_dict):
        return 'dataviewer/shp.html'

    def setup_template_variables(self, context, data_dict):
        import ckanext.resourceproxy.plugin as proxy
        self.same_domain = data_dict['resource'].get('on_same_domain')
        if self.proxy_enabled and not self.same_domain:
            data_dict['resource']['original_url'] = \
                data_dict['resource'].get('url')
            data_dict['resource']['url'] = \
                proxy.get_proxified_resource_url(data_dict)

    ## ITemplateHelpers

    def get_helpers(self):
        return {
            'get_common_map_config_shp': get_common_map_config,
            'get_shapefile_viewer_config': get_shapefile_viewer_config,
        }
Пример #19
0
class OAuth2Plugin(plugins.SingletonPlugin):

    plugins.implements(plugins.IAuthenticator, inherit=True)
    plugins.implements(plugins.IAuthFunctions, inherit=True)
    plugins.implements(plugins.IBlueprint)
    plugins.implements(plugins.IConfigurer)

    def __init__(self, name=None):
        """Store the OAuth 2 client configuration"""
        log.debug("Init OAuth2 extension")

        self.verify_https = os.environ.get("OAUTHLIB_INSECURE_TRANSPORT",
                                           "") == ""

        if self.verify_https and os.environ.get("REQUESTS_CA_BUNDLE",
                                                "").strip() != "":
            self.verify_https = os.environ["REQUESTS_CA_BUNDLE"].strip()

        self.jwt_enable = os.environ.get(
            "CKAN_OAUTH2_JWT_ENABLE",
            toolkit.config.get("ckan.oauth2.jwt.enable",
                               "")).strip().lower() in ("true", "1", "on")

        self.authorization_endpoint = os.environ.get(
            "CKAN_OAUTH2_AUTHORIZATION_ENDPOINT",
            toolkit.config.get("ckan.oauth2.authorization_endpoint", ""),
        ).strip()

        self.token_endpoint = os.environ.get(
            "CKAN_OAUTH2_TOKEN_ENDPOINT",
            toolkit.config.get("ckan.oauth2.token_endpoint", ""),
        ).strip()

        self.profile_api_url = os.environ.get(
            "CKAN_OAUTH2_PROFILE_API_URL",
            toolkit.config.get("ckan.oauth2.profile_api_url", "")).strip()

        self.client_id = os.environ.get(
            "CKAN_OAUTH2_CLIENT_ID",
            toolkit.config.get("ckan.oauth2.client_id", "")).strip()

        self.client_secret = os.environ.get(
            "CKAN_OAUTH2_CLIENT_SECRET",
            toolkit.config.get("ckan.oauth2.client_secret", "")).strip()

        self.scope = os.environ.get(
            "CKAN_OAUTH2_SCOPE", toolkit.config.get("ckan.oauth2.scope",
                                                    "")).strip()

        self.rememberer_name = os.environ.get(
            "CKAN_OAUTH2_REMEMBER_NAME",
            toolkit.config.get("ckan.oauth2.rememberer_name", "auth_tkt"),
        ).strip()

        self.profile_api_user_field = os.environ.get(
            "CKAN_OAUTH2_PROFILE_API_USER_FIELD",
            toolkit.config.get("ckan.oauth2.profile_api_user_field", ""),
        ).strip()

        self.profile_api_fullname_field = os.environ.get(
            "CKAN_OAUTH2_PROFILE_API_FULLNAME_FIELD",
            toolkit.config.get("ckan.oauth2.profile_api_fullname_field", ""),
        ).strip()

        self.profile_api_mail_field = os.environ.get(
            "CKAN_OAUTH2_PROFILE_API_MAIL_FIELD",
            toolkit.config.get("ckan.oauth2.profile_api_mail_field", ""),
        ).strip()

        self.profile_api_groupmembership_field = os.environ.get(
            "CKAN_OAUTH2_PROFILE_API_GROUPMEMBERSHIP_FIELD",
            toolkit.config.get("ckan.oauth2.profile_api_groupmembership_field",
                               ""),
        ).strip()

        self.sysadmin_group_name = os.environ.get(
            "CKAN_OAUTH2_SYSADMIN_GROUP_NAME",
            toolkit.config.get("ckan.oauth2.sysadmin_group_name", ""),
        ).strip()

        self.redirect_uri = urljoin(
            urljoin(
                toolkit.config.get("ckan.site_url", "http://localhost:5000"),
                toolkit.config.get("ckan.root_path"),
            ),
            REDIRECT_URL,
        )

        # Init db
        db.init_db(model)

        missing = [
            key for key in REQUIRED_CONF if getattr(self, key, "") == ""
        ]
        if missing:
            raise ValueError("Missing required oauth2 conf: %s" %
                             ", ".join(missing))
        elif self.scope == "":
            self.scope = None
        """
        # these still need to be added as rules to the blueprint
        # Redirect the user to the OAuth service register page
        if self.register_url:
            redirect("/user/register", self.register_url)

        # Redirect the user to the OAuth service reset page
        if self.reset_url:
            redirect("/user/reset", self.reset_url)

        # Redirect the user to the OAuth service reset page
        if self.edit_url:
            redirect("/user/edit/{user}", self.edit_url)
        """

    def get_blueprint(self):
        """Create Flask blueprint."""
        blueprint = Blueprint("oauth2", self.__module__)
        rules = [
            ("/user/login", "login", self.login),
            ("/oauth2/callback", "callback", self.callback),
        ]

        for rule in rules:
            blueprint.add_url_rule(*rule)

        return blueprint

    def get_auth_functions(self):
        """Prevent some actions from being authorized."""
        return {
            "user_create": user_create,
            "user_update": user_update,
            "user_reset": user_reset,
            "request_reset": request_reset,
        }

    def update_config(self, config):
        """Update configuration."""
        self.register_url = os.environ.get(
            "CKAN_OAUTH2_REGISTER_URL",
            config.get("ckan.oauth2.register_url", None))
        self.reset_url = os.environ.get(
            "CKAN_OAUTH2_RESET_URL", config.get("ckan.oauth2.reset_url", None))
        self.edit_url = os.environ.get(
            "CKAN_OAUTH2_EDIT_URL", config.get("ckan.oauth2.edit_url", None))
        self.authorization_header = os.environ.get(
            "CKAN_OAUTH2_AUTHORIZATION_HEADER",
            config.get("ckan.oauth2.authorization_header", "Authorization"),
        ).lower()

        # Add plugin's templates dir to CKAN's extra_template_paths, so CKAN will use them
        plugins.toolkit.add_template_directory(config, "templates")

    def login(self):
        """Start log in process."""
        log.debug("login")

        state = generate_state(get_previous_page())
        oauth = OAuth2Session(self.client_id,
                              redirect_uri=self.redirect_uri,
                              scope=self.scope,
                              state=state)
        auth_url, _ = oauth.authorization_url(self.authorization_endpoint)
        log.debug(f"Challenge: Redirecting challenge to page {auth_url}")

        return toolkit.redirect_to(auth_url)

    def callback(self):
        """Resume login process from authorization service."""
        log.debug("callback")
        try:
            token = self.get_token()
        except InsecureTransportError as e:
            log.warn(f"Error in getting token: {e}")
            helpers.flash_error(
                "Authentication error; please contact the administrator.")
            return toolkit.redirect_to("/")

        try:
            user_name = self.authenticate(token)
        except InsecureTransportError as e:
            log.warn(f"Error authenticating user: {e}")
            helpers.flash_error(
                "Authentication error; please contact the administrator.")
            return toolkit.redirect_to("/")

        # create headers from remember token to be added to redirected response
        remember_headers = self.remember(user_name)

        self.update_token(user_name, token)
        state = toolkit.request.params.get("state")
        redirect = toolkit.redirect_to(get_came_from(state))

        for header, value in remember_headers:
            redirect.headers[header] = value

        return redirect

    def identify(self):
        log.debug("identify")

        def _refresh_and_save_token(user_name):
            new_token = self.refresh_token(user_name)
            if new_token:
                toolkit.g.usertoken = new_token

        environ = toolkit.request.environ
        apikey = toolkit.request.headers.get(self.authorization_header, "")
        user_name = None

        if self.authorization_header == "authorization":
            if apikey.startswith("Bearer "):
                apikey = apikey[7:].strip()
            else:
                apikey = ""

        # This API Key is not the one of CKAN, it's the one provided by the OAuth2 Service
        if apikey:
            try:
                token = {"access_token": apikey}
                user_name = self.authenticate(token)
            except Exception:
                pass

        # If the authentication via API fails, we can still log in the user using session.
        if user_name is None and "repoze.who.identity" in environ:
            user_name = environ["repoze.who.identity"]["repoze.who.userid"]
            log.info(f"User {user_name} logged using session")
        # If we have been able to log in the user (via API or Session)
        if user_name:
            g.user = user_name
            toolkit.g.user = user_name
            toolkit.g.usertoken = self.get_stored_token(user_name)
            toolkit.g.usertoken_refresh = partial(_refresh_and_save_token,
                                                  user_name)
        else:
            g.user = None
            log.warn("The user is not currently logged in...")

    def get_token(self):
        """Get token from authorization service."""
        log.debug("get_token")
        oauth = OAuth2Session(self.client_id,
                              redirect_uri=self.redirect_uri,
                              scope=self.scope)

        try:
            # NOTE: authorization_response/toolkit.request.url was using http instead of https
            # hacked to replace http with https
            authorization_response = toolkit.request.url
            authorization_response = authorization_response.replace(
                "http", "https")
            token = oauth.fetch_token(
                self.token_endpoint,
                client_secret=self.client_secret,
                authorization_response=authorization_response,
                verify=self.verify_https,
            )
        except InsecureTransportError:
            raise

        return token

    def authenticate(self, token):
        log.debug("authenticate")

        if self.jwt_enable:
            access_token = token["access_token"]
            user_data = jwt.decode(access_token, verify=False)
            user = self.user_json(user_data)
        else:

            try:
                oauth = OAuth2Session(self.client_id, token=token)
                profile_response = oauth.get(self.profile_api_url,
                                             verify=self.verify_https)
            except InsecureTransportError:
                raise

            # Token can be invalid
            if not profile_response.ok:
                error = profile_response.json()
                if error.get("error", "") == "invalid_token":
                    raise ValueError(error.get("error_description"))
                else:
                    profile_response.raise_for_status()
            else:
                user_data = profile_response.json()
                user = self.user_json(user_data)

        # Save the user in the database
        model.Session.add(user)
        model.Session.commit()
        model.Session.remove()

        return user.name

    def user_json(self, user_data):
        email = user_data[self.profile_api_mail_field]
        user_name = user_data[self.profile_api_user_field]

        # In CKAN can exists more than one user associated with the same email
        # Some providers, like Google and FIWARE only allows one account per email
        user = None
        users = model.User.by_email(email)
        if len(users) == 1:
            user = users[0]

        # If the user does not exist, we have to create it...
        if user is None:
            user = model.User(email=email)

        # Now we update his/her user_name with the one provided by the OAuth2 service
        # In the future, users will be obtained based on this field
        user.name = user_name

        # Update fullname
        if self.profile_api_fullname_field != "" and self.profile_api_fullname_field in user_data:
            user.fullname = user_data[self.profile_api_fullname_field]

        # Update sysadmin status
        if (self.profile_api_groupmembership_field != ""
                and self.profile_api_groupmembership_field in user_data):
            user.sysadmin = (
                self.sysadmin_group_name
                in user_data[self.profile_api_groupmembership_field])

        return user

    def _get_rememberer(self, environ):
        plugins = environ.get("repoze.who.plugins", {})
        return plugins.get(self.rememberer_name)

    def remember(self, user_name):
        """
        Remember the authenticated identity.

        This method simply delegates to another IIdentifier plugin if configured.

        Return headers so they can be added to redirected response.
        """
        log.debug("Repoze OAuth remember")
        environ = toolkit.request.environ
        rememberer = self._get_rememberer(environ)
        identity = {"repoze.who.userid": user_name}
        headers = rememberer.remember(environ, identity)

        return headers

    def get_stored_token(self, user_name):
        user_token = db.UserToken.by_user_name(user_name=user_name)
        if user_token:
            return {
                "access_token": user_token.access_token,
                "refresh_token": user_token.refresh_token,
                "expires_in": user_token.expires_in,
                "token_type": user_token.token_type,
            }

    def update_token(self, user_name, token):
        user_token = db.UserToken.by_user_name(user_name=user_name)

        # Create the user if it does not exist
        if not user_token:
            user_token = db.UserToken()
            user_token.user_name = user_name

        # Save the new token
        user_token.access_token = token["access_token"]
        user_token.token_type = token["token_type"]
        user_token.refresh_token = token.get("refresh_token")
        if "expires_in" in token:
            user_token.expires_in = token["expires_in"]
        else:
            access_token = jwt.decode(user_token.access_token, verify=False)
            user_token.expires_in = access_token["exp"] - access_token["iat"]

        model.Session.add(user_token)
        model.Session.commit()

    def refresh_token(self, user_name):
        token = self.get_stored_token(user_name)
        if token:
            client = OAuth2Session(self.client_id,
                                   token=token,
                                   scope=self.scope)
            try:
                token = client.refresh_token(
                    self.token_endpoint,
                    client_secret=self.client_secret,
                    client_id=self.client_id,
                    verify=self.verify_https,
                )
            except InsecureTransportError:
                raise
            self.update_token(user_name, token)
            log.info(f"Token for user {user_name} has been updated properly")
            return token
        else:
            log.warn(f"User {user_name} has no refresh token")
Пример #20
0
class NapThemePlugin(plugins.SingletonPlugin, toolkit.DefaultDatasetForm):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.ITemplateHelpers)
    plugins.implements(plugins.IBlueprint)
    plugins.implements(plugins.IPackageController, inherit=True)
    plugins.implements(plugins.IValidators)
    plugins.implements(plugins.IDatasetForm)

    # IConfigurer
    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')
        toolkit.add_public_directory(config_, 'fanstatic')
        toolkit.add_resource('fanstatic', 'nap_theme')

    # ITemplateHelpers
    def get_helpers(self):
        return {
            'nap_theme_get_extra': get_extra,
            'nap_theme_get_orgs': get_orgs,
            'nap_theme_get_tags': get_tags,
            'nap_theme_get_tag_names': get_tag_names,
            'nap_theme_get_sorted_error_summary': get_sorted_error_summary,
            'nap_theme_get_form_data': get_form_data,
            'nap_theme_get_package_display_name': get_package_display_name,
            'nap_theme_get_topics': get_topics,
            'nap_theme_get_transport_modes': get_transport_modes,
            'nap_theme_get_road_networks': get_road_networks,
            'nap_theme_get_user_with_datasets': get_user_with_datasets,
            'nap_theme_get_update_frequencies': get_update_frequencies,
            'nap_theme_get_update_frequency_name': get_update_frequency_name,
            'nap_theme_get_licences': get_licences,
            'nap_theme_get_licence_name': get_licence_name,
            'nap_theme_get_licence_type': get_licence_type,
            'nap_theme_get_time_period': get_time_period,
            'nap_theme_get_date_formatted': get_date_formatted,
            'nap_theme_get_facets': get_facets,
        }

    # IBlueprint
    def get_blueprint(self):
        u'''Return a Flask Blueprint object to be registered by the app.'''
        # Create Blueprint for plugin
        blueprint = Blueprint(self.name, self.__module__)
        blueprint.template_folder = u'templates'
        # Add plugin url rules to Blueprint object
        blueprint.add_url_rule(u'/cookies', u'cookies', cookies)
        return blueprint

    def _create_search_param_dict(self, filter_query):
        from collections import defaultdict
        # fix issue with spaces in params such as tags we need to differentiate spaces between search terms and spaces
        filter_query = filter_query.replace('" ', '"|||')
        # fix issue with : in params
        filter_query = filter_query.replace(':"', '~~~"')
        filter_params = filter_query.split('|||')
        filter_dict = list(s.split('~~~') for s in filter_params)
        new_dict = defaultdict(list)
        for (key, value) in filter_dict:
            new_dict[key].append(value)
        return new_dict

    # IPackageController
    def before_search(self, search_params):
        def make_filters_or(filter_dict):
            filter_string = ""
            for (key, value) in filter_dict.items():
                string = f'{key}:({" OR ".join(value)})'
                filter_string = filter_string + " " + string
            return filter_string

        if (search_params.get('fq', None)
                and not '+owner_org' in search_params['fq']):
            fq = make_filters_or(
                self._create_search_param_dict(search_params['fq']))
            search_params["fq"] = fq

        extras = search_params.get('extras')
        start_date = extras.get('ext_startdate')
        end_date = extras.get('ext_enddate')
        if not start_date or not end_date:
            return search_params

        start_date = datetime.strptime(
            start_date, '%Y-%m-%d').strftime("%Y-%m-%dT%H:%M:%SZ")
        end_date = datetime.strptime(end_date,
                                     '%Y-%m-%d').strftime("%Y-%m-%dT%H:%M:%SZ")

        fq = search_params["fq"]
        fq = '{fq} (start_date:[* TO {end_date}] AND end_date:[{start_date} TO *])'.format(
            fq=fq, start_date=start_date, end_date=end_date)
        search_params["fq"] = fq
        return search_params

    # Schema Changes
    def is_fallback(self):
        # Return True to register this plugin as the default handler for
        # package types not handled by any other IDatasetForm plugin.
        return True

    def package_types(self):
        # This plugin doesn't handle any special package types, it just
        # registers itself as the default (above).
        return []

    def create_package_schema(self):

        schema = super(NapThemePlugin, self).create_package_schema()
        schema = self._modify_package_schema(schema)

        return schema

    def update_package_schema(self):
        schema = super(NapThemePlugin, self).update_package_schema()
        schema = self._modify_package_schema(schema)
        return schema

    def _modify_package_schema(self, schema):
        schema.update({
            'url':
            [custom_url_validator,
             toolkit.get_converter('unicode_safe')],
            'title': [
                custom_description_validator,
                toolkit.get_converter('unicode_safe')
            ],
            'author_email': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('unicode_safe'),
                custom_author_email_validator,
                custom_required_author_email_validator
            ],
            'author': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('unicode_safe'),
                custom_required_author_name_validator
            ],
            'name': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('unicode_safe')
            ],
            'location': [
                custom_location_validator,
                toolkit.get_converter('convert_to_extras')
            ],
            'data_formats': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_to_extras')
            ],
            'update_frequency': [
                custom_update_frequency_validator,
                toolkit.get_converter('convert_to_extras')
            ],
            'regularly_updated': [
                custom_regularly_updated_validatior,
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_to_extras')
            ],
            'date_range_earliest_day': [
                custom_earliest_day_validator,
                toolkit.get_converter('convert_to_extras')
            ],
            'date_range_earliest_month': [
                custom_earliest_month_validator,
                toolkit.get_converter('convert_to_extras')
            ],
            'date_range_earliest_year': [
                custom_earliest_year_validator,
                toolkit.get_converter('convert_to_extras')
            ],
            'date_range_latest_day': [
                custom_latest_day_validator,
                toolkit.get_converter('convert_to_extras')
            ],
            'date_range_latest_month': [
                custom_latest_month_validator,
                toolkit.get_converter('convert_to_extras')
            ],
            'date_range_latest_year': [
                custom_latest_year_validator,
                toolkit.get_converter('convert_to_extras')
            ],
            'date_range_earliest': [custom_date_range_earliest_validator],
            'date_range_latest': [custom_date_range_latest_validator],
            'regularly_updated_earliest_day': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_to_extras')
            ],
            'regularly_updated_earliest_month': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_to_extras')
            ],
            'regularly_updated_earliest_year': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_to_extras')
            ],
            'data_available': [
                custom_data_available_validator,
                toolkit.get_converter('convert_to_extras')
            ],
            'topics': [
                custom_topics_validator,
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_to_tags')('nap_topics')
            ],
            'transport_modes': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_to_tags')('nap_transport_modes')
            ],
            'road_networks': [
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_to_tags')('nap_road_networks')
            ],
            'start_date': [
                custom_date_range_start_converter,
                toolkit.get_converter('convert_to_extras')
            ],
            'end_date': [
                custom_date_range_end_converter,
                toolkit.get_converter('convert_to_extras')
            ]
        })
        return schema

    def show_package_schema(self):
        schema = super(NapThemePlugin, self).show_package_schema()

        schema.update({
            'url':
            [toolkit.get_converter('unicode_safe'), custom_url_validator],
            'title': [
                toolkit.get_converter('unicode_safe'),
                custom_description_validator
            ],
            'author_email': [
                toolkit.get_converter('unicode_safe'),
                toolkit.get_validator('ignore_missing'),
                custom_author_email_validator,
                custom_required_author_email_validator
            ],
            'author': [
                toolkit.get_converter('unicode_safe'),
                toolkit.get_validator('ignore_missing'),
                custom_required_author_name_validator
            ],
            'name': [
                toolkit.get_converter('unicode_safe'),
                toolkit.get_validator('ignore_missing')
            ],
            'location': [
                toolkit.get_converter('convert_from_extras'),
                custom_location_validator
            ],
            'data_formats': [
                toolkit.get_converter('unicode_safe'),
                toolkit.get_validator('convert_from_extras')
            ],
            'update_frequency': [
                toolkit.get_converter('unicode_safe'),
                toolkit.get_validator('convert_from_extras')
            ],
            'regularly_updated': [
                toolkit.get_converter('unicode_safe'),
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_from_extras')
            ],
            'date_range_earliest_day': [
                toolkit.get_converter('unicode_safe'),
                custom_earliest_day_validator,
                toolkit.get_converter('convert_from_extras')
            ],
            'date_range_earliest_month': [
                toolkit.get_converter('unicode_safe'),
                custom_earliest_month_validator,
                toolkit.get_converter('convert_from_extras')
            ],
            'date_range_earliest_year': [
                toolkit.get_converter('unicode_safe'),
                custom_earliest_year_validator,
                toolkit.get_converter('convert_from_extras')
            ],
            'date_range_latest_day': [
                toolkit.get_converter('unicode_safe'),
                custom_latest_day_validator,
                toolkit.get_converter('convert_from_extras')
            ],
            'date_range_latest_month': [
                toolkit.get_converter('unicode_safe'),
                custom_latest_month_validator,
                toolkit.get_converter('convert_from_extras')
            ],
            'date_range_latest_year': [
                toolkit.get_converter('unicode_safe'),
                custom_latest_year_validator,
                toolkit.get_converter('convert_from_extras')
            ],
            'regularly_updated_earliest_day': [
                toolkit.get_converter('unicode_safe'),
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_from_extras')
            ],
            'regularly_updated_earliest_month': [
                toolkit.get_converter('unicode_safe'),
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_from_extras')
            ],
            'regularly_updated_earliest_year': [
                toolkit.get_converter('unicode_safe'),
                toolkit.get_validator('ignore_missing'),
                toolkit.get_converter('convert_from_extras')
            ],
            'data_available': [
                custom_data_available_validator,
                toolkit.get_converter('unicode_safe'),
                toolkit.get_converter('convert_from_extras')
            ],
            'topics': [
                custom_topics_validator,
                toolkit.get_converter('convert_from_tags')('nap_topics'),
                toolkit.get_validator('ignore_missing')
            ],
            'transport_modes': [
                toolkit.get_converter('convert_from_tags')(
                    'nap_transport_modes'),
                toolkit.get_validator('ignore_missing')
            ],
            'road_networks': [
                toolkit.get_converter('convert_from_tags')(
                    'nap_road_networks'),
                toolkit.get_validator('ignore_missing')
            ],
            'start_date': [toolkit.get_converter('convert_from_extras')],
            'end_date': [toolkit.get_converter('convert_from_extras')]
        })
        return schema

    # IValidators
    def get_validators(self):
        return {
            u'owner_org_validator': custom_owner_org_validator,
            u'url_validator': custom_url_validator,
            u'title_validator': custom_title_validator,
            u'location_validator': custom_location_validator
        }
Пример #21
0
class VectorStorer(SingletonPlugin):
    STATE_DELETED = 'deleted'

    resource_delete_action = None
    resource_update_action = None

    implements(IRoutes, inherit=True)
    implements(IConfigurer, inherit=True)
    implements(IConfigurable, inherit=True)
    implements(IResourceUrlChange)
    implements(ITemplateHelpers)
    implements(IDomainObjectModification, inherit=True)

    def get_helpers(self):
        return {
            'vectorstore_is_in_vectorstore': isInVectorStore,
            'vectorstore_supported_format': supportedFormat
        }

    def configure(self, config):
        ''' Extend the resource_delete action in order to get notification of deleted resources'''
        if self.resource_delete_action is None:

            resource_delete = toolkit.get_action('resource_delete')

            @logic.side_effect_free
            def new_resource_delete(context, data_dict):
                resource = ckan.model.Session.query(model.Resource).get(
                    data_dict['id'])
                self.notify(resource,
                            model.domain_object.DomainObjectOperation.deleted)
                res_delete = resource_delete(context, data_dict)

                return res_delete

            logic._actions['resource_delete'] = new_resource_delete
            self.resource_delete_action = new_resource_delete
        ''' Extend the resource_update action in order to pass the extra keys to vectorstorer resources
        when they are being updated'''
        if self.resource_update_action is None:

            resource_update = toolkit.get_action('resource_update')

            @logic.side_effect_free
            def new_resource_update(context, data_dict):
                resource = ckan.model.Session.query(model.Resource).get(
                    data_dict['id']).as_dict()
                if resource.has_key('vectorstorer_resource'):
                    if resource['format'].lower() == settings.WMS_FORMAT:
                        data_dict['parent_resource_id'] = resource[
                            'parent_resource_id']
                        data_dict['vectorstorer_resource'] = resource[
                            'vectorstorer_resource']
                        data_dict['wms_server'] = resource['wms_server']
                        data_dict['wms_layer'] = resource['wms_layer']
                    if resource['format'].lower() == settings.DB_TABLE_FORMAT:
                        data_dict['vectorstorer_resource'] = resource[
                            'vectorstorer_resource']
                        data_dict['parent_resource_id'] = resource[
                            'parent_resource_id']
                        data_dict['geometry'] = resource['geometry']

                    if not data_dict['url'] == resource['url']:
                        abort(
                            400,
                            _('You cant upload a file to a ' +
                              resource['format'] + ' resource.'))
                res_update = resource_update(context, data_dict)

                return res_update

            logic._actions['resource_update'] = new_resource_update
            self.resource_update_action = new_resource_update

    def before_map(self, map):
        map.connect(
            'style',
            '/dataset/{id}/resource/{resource_id}/style/{operation}',
            controller='ckanext.vectorstorer.controllers.style:StyleController',
            action='style',
            operation='operation')
        map.connect('export',
                    '/dataset/{id}/resource/{resource_id}/export/{operation}',
                    controller=
                    'ckanext.vectorstorer.controllers.export:ExportController',
                    action='export',
                    operation='{operation}')
        map.connect('search_epsg',
                    '/api/search_epsg',
                    controller=
                    'ckanext.vectorstorer.controllers.export:ExportController',
                    action='search_epsg')
        map.connect('publish',
                    '/api/vector/publish',
                    controller=
                    'ckanext.vectorstorer.controllers.vector:VectorController',
                    action='publish')

        return map

    def update_config(self, config):

        toolkit.add_public_directory(config, 'public')
        toolkit.add_template_directory(config, 'templates')
        toolkit.add_resource('public', 'ckanext-vectorstorer')

    def notify(self, entity, operation=None):

        if isinstance(entity, model.resource.Resource):

            if operation == model.domain_object.DomainObjectOperation.new and entity.format.lower(
            ) in settings.SUPPORTED_DATA_FORMATS:
                #A new vector resource has been created
                #resource_actions.create_vector_storer_task(entity)
                resource_actions.identify_resource(entity)
            #elif operation==model.domain_object.DomainObjectOperation.deleted:
            ##A vectorstorer resource has been deleted
            #resource_actions.delete_vector_storer_task(entity.as_dict())

            #elif operation is None:
            ##Resource Url has changed

            #if entity.format.lower() in settings.SUPPORTED_DATA_FORMATS:
            ##Vector file was updated

            #resource_actions.update_vector_storer_task(entity)

            #else :
            ##Resource File updated but not in supported formats

            #resource_actions.delete_vector_storer_task(entity.as_dict())

        elif isinstance(entity, model.Package):

            if entity.state == self.STATE_DELETED:

                resource_actions.pkg_delete_vector_storer_task(
                    entity.as_dict())
class YtpCommentsPlugin(plugins.SingletonPlugin):
    implements(plugins.IRoutes, inherit=True)
    implements(plugins.IConfigurer, inherit=True)
    implements(plugins.IPackageController, inherit=True)
    implements(plugins.ITemplateHelpers, inherit=True)
    implements(plugins.IActions, inherit=True)
    implements(plugins.IAuthFunctions, inherit=True)

    # IConfigurer

    def configure(self, config):
        log.debug("Configuring comments module")

    def update_config(self, config):
        toolkit.add_template_directory(config, "templates")
        toolkit.add_public_directory(config, 'public')
        toolkit.add_resource('public/javascript/', 'comments_js')

    def get_helpers(self):
        return {
            'get_comment_thread': self._get_comment_thread,
            'get_comment_count_for_dataset': self._get_comment_count_for_dataset
        }

    def get_actions(self):
        from ckanext.ytp.comments.logic.action import get, create, delete, update

        return {
            "comment_create": create.comment_create,
            "thread_show": get.thread_show,
            "comment_update": update.comment_update,
            "comment_show": get.comment_show,
            "comment_delete": delete.comment_delete,
            "comment_count": get.comment_count
        }

    def get_auth_functions(self):
        from ckanext.ytp.comments.logic.auth import get, create, delete, update

        return {
            'comment_create': create.comment_create,
            'comment_update': update.comment_update,
            'comment_show': get.comment_show,
            'comment_delete': delete.comment_delete,
            "comment_count": get.comment_count
        }
    # IPackageController

    def before_view(self, pkg_dict):
        # TODO: append comments from model to pkg_dict
        return pkg_dict

    # IRoutes

    def before_map(self, map):
        """
            /dataset/NAME/comments/reply/PARENT_ID
            /dataset/NAME/comments/add
        """
        controller = 'ckanext.ytp.comments.controller:CommentController'
        map.connect('/dataset/{dataset_id}/comments/add', controller=controller, action='add')
        map.connect('/dataset/{dataset_id}/comments/{comment_id}/edit', controller=controller, action='edit')
        map.connect('/dataset/{dataset_id}/comments/{parent_id}/reply', controller=controller, action='reply')
        map.connect('/dataset/{dataset_id}/comments/{comment_id}/delete', controller=controller, action='delete')
        return map

    def _get_comment_thread(self, dataset_name):
        import ckan.model as model
        from ckan.logic import get_action
        url = '/dataset/%s' % dataset_name
        return get_action('thread_show')({'model': model, 'with_deleted': True}, {'url': url})

    def _get_comment_count_for_dataset(self, dataset_name):
        import ckan.model as model
        from ckan.logic import get_action
        url = '/dataset/%s' % dataset_name
        count = get_action('comment_count')({'model': model}, {'url': url})
        return count
Пример #23
0
class IntroExamplePlugin(p.SingletonPlugin):

    p.implements(p.IConfigurer)
    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IAuthFunctions)
    p.implements(p.IActions)

    ## IConfigurer
    def update_config(self, config):
        '''
        This method allows to access and modify the CKAN configuration object
        '''

        log.info('You are using the following plugins: {0}'
                 .format(config.get('ckan.plugins')))

        # Check CKAN version
        # To raise an exception instead, use:
        #   p.toolkit.require_ckan_version('2.1')
        if not p.toolkit.check_ckan_version('2.1'):
            log.warn('This extension has only been tested on CKAN 2.1!')

        # Add the extension templates directory so it overrides the CKAN core
        # one
        p.toolkit.add_template_directory(config, 'theme/templates')

        # Add the extension public directory so we can serve our own content
        p.toolkit.add_public_directory(config, 'theme/public')

    ## IRoutes
    def after_map(self, map):

        controller = 'ckanext.intro.plugin:CustomController'
        map.connect('/custom', controller=controller, action='custom_page')

        map.connect('/csv', controller=controller, action='datasets_report')
        
        # /changes obtiene los cambios recientes
        map.connect('/changes', controller=controller, action='changes_recently')

        return map

    ## IAuthFunctions
    def get_auth_functions(self):

        # Return a dict with the auth functions that we want to override
        # or add
        return {
            'group_create': group_create,
            'datasets_report_csv': datasets_report_csv_auth,
            'changes_recently_csv' : changes_recently_auth,
        }

    ## IActions
    def get_actions(self):
        # Return a dict with the action functions that we want to add
        return {
            'datasets_report_csv': datasets_report_csv,
            'changes_recently_csv' : changes_recently_csv, 
            # asocia el action con el método 
        }
Пример #24
0
class LIREPlugin(p.SingletonPlugin):

    p.implements(p.IConfigurer, inherit=True)
    p.implements(p.IConfigurable)
    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IDomainObjectModification, inherit=True)
    p.implements(p.IResourceUrlChange)
    p.implements(p.ITemplateHelpers)

    def configure(self, config):
        self.site_url = config.get('ckan.site_url')

    def update_config(self, config):
        # p.toolkit.add_resource('fanstatic', 'rem')
        # check if new templates
        if p.toolkit.check_ckan_version(min_version='2.0'):
            if not p.toolkit.asbool(config.get('ckan.legacy_templates', False)):
                # add the extend templates
                p.toolkit.add_template_directory(config, 'templates_extend')
            else:
                # legacy templates
                p.toolkit.add_template_directory(config, 'templates')
            # templates for helper functions
            p.toolkit.add_template_directory(config, 'templates_new')
        else:
            # FIXME we don't support ckan < 2.0
            p.toolkit.add_template_directory(config, 'templates')
        p.toolkit.add_public_directory(config, 'public')

    def before_map(self, map):
        lire = 'ckanext.lire.controller:LIREController'
        rem = 'ckanext.lire.controllers.rem:REMController'
        ace = 'ckanext.lire.controllers.ace:ACEController'
        semre = 'ckanext.lire.controllers.semre:SEMREController'

        map.connect('/lire', controller=lire, action='index')

        map.connect('/lire/manager', controller=lire, action='manager')

        map.connect('/lire/semantic', controller=semre, action='semantic')

        map.connect('/lire/linksets.:id', controller=semre, action='linksets')

        map.connect('/lire/organization/:org.:ext', controller=semre, action='org_rdf')

        map.connect('/lire/examineDatasets', controller=rem, action='examineDatasets')

        map.connect('/lire/storeRelationships', controller=ace, action='storeRelationships')
    
        map.connect('/lire/checkDataset', controller=semre, action='checkDataset')

        return map

    #return random int necessary to create position of graphical element (dataset)
    def randomNum(self,num):

      randNum = random.randint(0,num) 

      return randNum

    ################
    ## LIRE SEMRE ##
    ################
    # Use this function in custom template to get dataset relationships
    # It will enable us to represent them semantically
    def semre_dataset(self,datasetName):

      semre = SEMREController()

      # To get dataset relationships we call SEMRE to whom we forward the name of the dataset
      datasetRelationships = semre.semre_create(datasetName)

      return datasetRelationships

    def get_helpers(self):
        return {'randomNum': self.randomNum,'semre_dataset': self.semre_dataset}
Пример #25
0
class Dset_HarvesterPlugin(p.SingletonPlugin):
    p.implements(ISpatialHarvester, inherit=True)
    p.implements(p.IConfigurer)
    p.implements(p.ITemplateHelpers)

    # ISpatialHarvester

    def get_package_dict(self, context, data_dict):
        package_dict = data_dict['package_dict']
        iso_values = data_dict['iso_values']
        xml_tree = data_dict['xml_tree']

        # Add ISO Topic Category from spatial harvester 
        package_dict['extras'].append(
            {'key': 'topic-category', 'value': iso_values.get('topic-category')}
        )
	
        # Add author field
	authorList = getNamesByRole(xml_tree, 'author', 'gmd:individualName/gco:CharacterString')
        authorString = json.dumps(authorList)
        package_dict['extras'].append(
            {'key': 'harvest-author', 'value': authorString}
        )

        # Examine iso_values if there are authors, just to get a better idea of harvester data structures.
	#if len(authorString) > 1:
            #log.debug("START iso_values print:")
            #log.debug(pprint.pformat(iso_values))
            #log.debug("END iso_values print.")
	
        # Add publisher field
	publisherList = getNamesByRole(xml_tree, 'publisher', 'gmd:organisationName/gco:CharacterString')
        publisherString = json.dumps(publisherList)
        package_dict['extras'].append({'key': 'publisher', 'value': publisherString})
	
        # Add Support Contact fields
        # TODO: Resource Support Contact Object?

        resourceSupportIsoPath = './/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:pointOfContact/gmd:CI_ResponsibleParty'
        resourceSupportName = getSupportContactName(xml_tree, resourceSupportIsoPath)
        package_dict['extras'].append({'key': 'resource-support-name', 'value': resourceSupportName})

        resourceSupportOrg = getSupportContactOrg(xml_tree, resourceSupportIsoPath)
        package_dict['extras'].append({'key': 'resource-support-organization', 'value': resourceSupportOrg})

        resourceSupportEmail = getSupportContactEmail(xml_tree, resourceSupportIsoPath)
        package_dict['extras'].append({'key': 'resource-support-email', 'value': resourceSupportEmail})

        # Metadata point of contact
	# TODO: don't build string here. Esp check for null org.
        pointOfContactIsoPath = './/gmd:contact/gmd:CI_ResponsibleParty'

        pointOfContactName = getSupportContactName(xml_tree, pointOfContactIsoPath)
        package_dict['extras'].append({'key': 'metadata-point-of-contact-name', 'value': pointOfContactName})

        pointOfContactOrg = getSupportContactOrg(xml_tree, pointOfContactIsoPath)
        package_dict['extras'].append({'key': 'metadata-point-of-contact-organization', 'value': pointOfContactOrg})

        # Set CKAN Resource Type to first DataCite ResourceType keyword (CKAN only allows one keyword)
	resourceTypeList = getDataCiteResourceTypes(xml_tree)
        for extra in package_dict['extras']:
            if extra['key'] == 'resource-type':
                extra['value'] = resourceTypeList[0]
	
        # Add Harvester-related values
        harvest_object = data_dict['harvest_object']
        package_dict['extras'].append({'key': 'harvested_object_id', 'value': harvest_object.id})
        package_dict['extras'].append({'key': 'harvester_source_id', 'value': harvest_object.harvest_source_id})
        package_dict['extras'].append({'key': 'harvester_source_title', 'value': harvest_object.source.title})

        # Add Publication Date field
        publication_date = getPublicationDate(iso_values['dataset-reference-date'])
        package_dict['extras'].append({'key': 'publication-date', 'value': publication_date})

        # Convert GCMD Keywords to split form
        splitKeywords = getSplitKeywordsGCMD(iso_values.pop('tags'))
        package_dict['tags'] = [{'name': t} for t in splitKeywords]

        # Convert some harvested fields from lists to JSON (spatial harvester is inconsistent in its output)
        for key in ('topic-category','access-constraints'):
           for extra in package_dict['extras']:
               if extra['key'] == key:
                   extra['value'] = json.dumps(extra['value'])
	
        #log.debug("START data_dict print:")
        #log.debug(pprint.pformat(data_dict))
        #log.debug("END data_dict print.")
        return package_dict

    # IConfigurer

    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')
        toolkit.add_public_directory(config_, 'public')
        toolkit.add_resource('fanstatic', 'dset_harvester')

   # ITemplateHelpers

    def get_helpers(self):

        function_names = (
            'string_to_json',
            'dset_sorted_extras',
            'dset_render_datetime',
            'dset_valid_temporal_extent',
        )
        return _get_module_functions(helpers, function_names)
class ExtendSearchPlugin(plugins.SingletonPlugin):
    '''
    Extends the Ckan dataset/package search
    '''
    print "loading ckanext-extend_search"

    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IPackageController, inherit=True)

    def update_config(self, config):
        toolkit.add_template_directory(config, 'templates')
        toolkit.add_resource('fanstatic', 'ckanext-datesearch')
        toolkit.add_resource('fanstatic', 'custodianpicker-module')

    # Add the custom parameters to Solr's facet queries
    def before_search(self, search_params):

        extras = search_params.get('extras')
        if not extras:
            # There are no extras in the search params, so do nothing.
            return search_params

        start_date = extend_search_convert_local_to_utc_timestamp(
            extras.get('ext_startdate'))
        end_date = extend_search_convert_local_to_utc_timestamp(
            extras.get('ext_enddate'))
        cust_id = extras.get('ext_cust_id')

        if not cust_id:
            if not start_date or not end_date:
                # The user didn't select any additional params, so do nothing.
                return search_params

        fq = search_params['fq']

        if start_date and end_date:
            # Add a date-range query with the selected start and end dates into the
            # Solr facet queries.
            fq = '{fq} +metadata_modified:[{start_date} TO {end_date}]'.format(
                fq=fq, start_date=start_date, end_date=end_date)

        #Add creator (user) id query to the Solr facet queries
        if cust_id:
            fq = '{fq} +maintainer:{cust_id}'.format(fq=fq, cust_id=cust_id)

        #return modified facet queries
        search_params['fq'] = fq

        return search_params

    def after_search(self, search_params, search_results):

        context = {
            'model': ckan.model,
            'session': ckan.model.Session,
            'user': pylons.c.user
        }

        #set permission level: read (default is edit)
        data_dict = {'user': pylons.c.user, 'permission': 'read'}
        #get list of organisations that the user is a member of
        orgs = ckan.logic.get_action('organization_list_for_user')(context,
                                                                   data_dict)

        #user doesn't belong to an organisation
        if not orgs:
            print('User is not a member of any organisations!')
            c.maintainers = []
            return search_params

        #get a distinct list of members who belong to the organisations
        members = []
        for org in orgs:
            params = {'id': org['id'], 'object_type': 'user'}
            member_list = ckan.logic.get_action('member_list')(context, params)
            for m in member_list:
                members.append(m)

        memberset = set(members)

        #need the user name to match with the maintainer field
        current_user_name = None
        member_names = []
        for member in memberset:
            user = User.get(member[0])  #user id
            member_names.append(user.name)

        #get all maintainers
        maintainers = [
            p[0] for p in meta.Session.query(distinct(Package.maintainer))
            if p[0]
        ]
        maintset = set(maintainers)

        #filter maintainers by user-related organisation members
        results = maintset.intersection(member_names)
        c.maintainers = results

        return search_params
Пример #27
0
class SchemingDatasetsPlugin(p.SingletonPlugin, DefaultDatasetForm,
                             _SchemingMixin):
    p.implements(p.IConfigurer)
    p.implements(p.ITemplateHelpers)
    p.implements(p.IDatasetForm, inherit=True)
    p.implements(p.IActions)
    p.implements(p.IValidators)

    SCHEMA_OPTION = 'scheming.dataset_schemas'
    FALLBACK_OPTION = 'scheming.dataset_fallback'
    SCHEMA_TYPE_FIELD = 'dataset_type'

    @classmethod
    def _store_instance(cls, self):
        SchemingDatasetsPlugin.instance = self

    def read_template(self):
        return 'scheming/package/read.html'

    def resource_template(self):
        return 'scheming/package/resource_read.html'

    def package_form(self):
        return 'scheming/package/snippets/package_form.html'

    def resource_form(self):
        return 'scheming/package/snippets/resource_form.html'

    def package_types(self):
        return list(self._schemas)

    def validate(self, context, data_dict, schema, action):
        """
        Validate and convert for package_create, package_update and
        package_show actions.
        """
        thing, action_type = action.split('_')
        t = data_dict.get('type')
        if not t or t not in self._schemas:
            return data_dict, {
                'type': ["Unsupported dataset type: {t}".format(t=t)]
            }
        scheming_schema = self._expanded_schemas[t]

        if action_type == 'show':
            get_validators = _field_output_validators
        elif action_type == 'create':
            get_validators = _field_create_validators
        else:
            get_validators = _field_validators

        for f in scheming_schema['dataset_fields']:
            schema[f['field_name']] = get_validators(
                f, scheming_schema, f['field_name'] not in schema)

        resource_schema = schema['resources']
        for f in scheming_schema.get('resource_fields', []):
            resource_schema[f['field_name']] = get_validators(
                f, scheming_schema, False)

        return navl_validate(data_dict, schema, context)

    def get_actions(self):
        """
        publish dataset schemas
        """
        return {
            'scheming_dataset_schema_list': scheming_dataset_schema_list,
            'scheming_dataset_schema_show': scheming_dataset_schema_show,
        }

    def setup_template_variables(self, context, data_dict):
        super(SchemingDatasetsPlugin,
              self).setup_template_variables(context, data_dict)
        # do not override licenses if they were already added by some
        # other extension. We just want to make sure, that licenses
        # are not empty.
        if not hasattr(c, 'licenses'):
            c.licenses = [('', '')] + model.Package.get_license_options()
Пример #28
0
class SpatialQuery(SingletonPlugin):
    ''' '''

    implements(interfaces.IRoutes, inherit=True)
    implements(interfaces.IPackageController, inherit=True)
    implements(interfaces.IConfigurable, inherit=True)

    search_backend = None

    def configure(self, config):
        '''

        :param config: 

        '''

        self.search_backend = config.get(u'ckanext.spatial.search_backend',
                                         u'postgis')
        if self.search_backend != u'postgis' and not toolkit.check_ckan_version(
                u'2.0.1'):
            msg = u'The Solr backends for the spatial search require CKAN 2.0.1 or ' \
                  u'higher. ' + \
                  u'Please upgrade CKAN or select the \'postgis\' backend.'
            raise toolkit.CkanVersionException(msg)

    def before_map(self, map):
        '''

        :param map: 

        '''

        map.connect(
            u'api_spatial_query',
            '/api/2/search/{register:dataset|package}/geo',
            controller=u'ckanext.spatial.controllers.api:ApiController',
            action=u'spatial_query')
        return map

    def before_index(self, pkg_dict):
        '''

        :param pkg_dict: 

        '''
        import shapely.geometry

        if pkg_dict.get(u'extras_spatial', None) and self.search_backend in (
                u'solr', u'solr-spatial-field'):
            try:
                geometry = json.loads(pkg_dict[u'extras_spatial'])
            except ValueError, e:
                log.error(u'Geometry not valid GeoJSON, not indexing')
                return pkg_dict

            if self.search_backend == u'solr':
                # Only bbox supported for this backend
                if not (geometry[u'type'] == u'Polygon'
                        and len(geometry[u'coordinates']) == 1
                        and len(geometry[u'coordinates'][0]) == 5):
                    log.error(
                        u'Solr backend only supports bboxes (Polygons with 5 points), '
                        u'ignoring geometry {0}'.format(
                            pkg_dict[u'extras_spatial']))
                    return pkg_dict

                coords = geometry[u'coordinates']
                pkg_dict[u'maxy'] = max(coords[0][2][1], coords[0][0][1])
                pkg_dict[u'miny'] = min(coords[0][2][1], coords[0][0][1])
                pkg_dict[u'maxx'] = max(coords[0][2][0], coords[0][0][0])
                pkg_dict[u'minx'] = min(coords[0][2][0], coords[0][0][0])
                pkg_dict[u'bbox_area'] = (pkg_dict[u'maxx'] - pkg_dict[u'minx']) * \
                                         (pkg_dict[u'maxy'] - pkg_dict[u'miny'])

            elif self.search_backend == u'solr-spatial-field':
                wkt = None

                # Check potential problems with bboxes
                if geometry[u'type'] == u'Polygon' \
                        and len(geometry[u'coordinates']) == 1 \
                        and len(geometry[u'coordinates'][0]) == 5:

                    # Check wrong bboxes (4 same points)
                    xs = [p[0] for p in geometry[u'coordinates'][0]]
                    ys = [p[1] for p in geometry[u'coordinates'][0]]

                    if xs.count(xs[0]) == 5 and ys.count(ys[0]) == 5:
                        wkt = u'POINT({x} {y})'.format(x=xs[0], y=ys[0])
                    else:
                        # Check if coordinates are defined counter-clockwise,
                        # otherwise we'll get wrong results from Solr
                        lr = shapely.geometry.polygon.LinearRing(
                            geometry[u'coordinates'][0])
                        if not lr.is_ccw:
                            lr.coords = list(lr.coords)[::-1]
                        polygon = shapely.geometry.polygon.Polygon(lr)
                        wkt = polygon.wkt

                if not wkt:
                    shape = shapely.geometry.asShape(geometry)
                    if not shape.is_valid:
                        log.error(u'Wrong geometry, not indexing')
                        return pkg_dict
                    wkt = shape.wkt

                pkg_dict[u'spatial_geom'] = wkt

        return pkg_dict
Пример #29
0
class ExampleIAuthFunctionsPlugin(plugins.SingletonPlugin):
    plugins.implements(plugins.IAuthFunctions)

    def get_auth_functions(self):
        return {'group_create': group_create}
Пример #30
0
        # unfinished email trigger:
        # self._email_on_change(context,resource,'privacy_contains_pii')
        self._redirect_to_edit_on_change(resource, 'resource_type')
        # reset monitored keys 
        for key in self.changed:
            self.changed[key] = False
        return
    def before_delete(self, context, resource, resources):
        return
    def after_delete(self, context, resources):
        return
    def before_show(self, resource_dict):
        return


    p.implements(p.ITemplateHelpers)
    def get_helpers(self):
        return {'clean_select_multi': v.clean_select_multi,
                'options_format': opts.format,
                'options_storage_location': opts.storage_location,
                'options_legal_authority_for_collection': opts.legal_authority_for_collection,
                'options_privacy_pia_title': opts.privacy_pia_title,
                'options_privacy_sorn_number': opts.privacy_sorn_number,
                'tag_relevant_governing_documents': tag_relevant_governing_documents, 
                'options_relevant_governing_documents': opts.relevant_governing_documents,
                'options_content_periodicity': opts.content_periodicity,
                'options_update_frequency': opts.update_frequency,
                'options_content_spatial': opts.content_spatial,
                'options_pra_exclusion': opts.pra_exclusion,
                'options_privacy_pia_notes': opts.privacy_pia_notes,
                'options_transfer_method': opts.transfer_method,
Пример #31
0
class OLGeoView(GeoViewBase):

    p.implements(p.IRoutes, inherit=True)
    p.implements(p.ITemplateHelpers, inherit=True)

    # IRoutes

    def before_map(self, m):
        controller = \
            'ckanext.geoview.controllers.service_proxy:ServiceProxyController'
        m.connect('/dataset/{id}/resource/{resource_id}/service_proxy',
                  controller=controller,
                  action='proxy_service')

        m.connect('/basemap_service/{map_id}',
                  controller=controller,
                  action='proxy_service_url')
        return m

    # ITemplateHelpers

    def get_helpers(self):
        return {
            'get_common_map_config_geoviews': get_common_map_config,
            'get_openlayers_viewer_config': get_openlayers_viewer_config,
        }

    # IResourceView (CKAN >=2.3)

    def info(self):
        return {
            'name': 'geo_view',
            'title': 'Map viewer (OpenLayers)',
            'icon': 'globe',
            'iframed': True,
            'default_title': toolkit._('Map viewer'),
            'schema': {
                'feature_hoveron': [ignore_empty, boolean_validator],
                'feature_style': [ignore_empty]
            },
        }

    def can_view(self, data_dict):
        format_lower = data_dict['resource'].get('format', '').lower()
        same_domain = on_same_domain(data_dict)

        # Guess from file extension
        if not format_lower and data_dict['resource'].get('url'):
            format_lower = self._guess_format_from_extension(
                data_dict['resource']['url'])

        if not format_lower:
            resource_locator_protocol = data_dict['resource'].get(
                'resource_locator_protocol', '').lower()
            if resource_locator_protocol and resource_locator_protocol.startswith(
                    'ogc:'):
                format_lower = resource_locator_protocol.split(':')[1]
                data_dict['resource']['format'] = format_lower
            else:
                return False

        view_formats = toolkit.config.get('ckanext.geoview.ol_viewer.formats',
                                          '')
        if view_formats:
            view_formats = view_formats.split(' ')
        else:
            view_formats = GEOVIEW_FORMATS

        correct_format = format_lower in view_formats
        can_preview_from_domain = self.proxy_enabled or same_domain

        return correct_format and can_preview_from_domain

    def view_template(self, context, data_dict):
        return 'dataviewer/openlayers.html'

    def form_template(self, context, data_dict):
        return 'dataviewer/openlayers_form.html'

    # IResourcePreview (CKAN < 2.3)

    def can_preview(self, data_dict):

        return self.can_view(data_dict)

    def preview_template(self, context, data_dict):
        return 'dataviewer/openlayers.html'

    # Common for IResourceView and IResourcePreview

    def _guess_format_from_extension(self, url):
        try:
            parsed_url = urlparse.urlparse(url)
            format_lower = (os.path.splitext(parsed_url.path)[1][1:].encode(
                'ascii', 'ignore').lower())
        except ValueError, e:
            log.error('Invalid URL: {0}, {1}'.format(url, e))
            format_lower = ''

        return format_lower
Пример #32
0
        # upload the file into the dataset
        resource = ckan.action.resource_create(
            package_id=resource.get('package_id'),
            name=resource.get('name'),
            description=resource.get('description'),
            format='CSV',
            mimetype='CSV',
            mimetype_inner='CSV',
            url='dummy-value', # ignored, but required by ckan
            upload=tmpfile
        )

        tmpfile.close()


    p.implements(p.IResourceController, inherit=True)

    def after_create(self, context, resource):

        self.check_and_create_csv(context, resource)

    def after_update(self, context, resource):

        resource_url = resource.get('url')

        self.check_and_create_csv(context, resource)

    # TODO
    # def after_delete(context, resource):
    #     pass
Пример #33
0
class MapperPlugin2(MapperPlugin):
    implements(IMapper)