Exemple #1
0
def get_storage_path():
    '''Function to cache storage path'''
    global _storage_path

    # None means it has not been set. False means not in config.
    if _storage_path is None:
        storage_path = config.get('ckan.storage_path')
        ofs_impl = config.get('ofs.impl')
        ofs_storage_dir = config.get('ofs.storage_dir')
        if storage_path:
            _storage_path = storage_path
        elif ofs_impl == 'pairtree' and ofs_storage_dir:
            log.warn('''Please use config option ckan.storage_path instead of
                     ofs.storage_dir''')
            _storage_path = ofs_storage_dir
            return _storage_path
        elif ofs_impl:
            log.critical('''We only support local file storage form version 2.2
                         of ckan please specify ckan.storage_path in your
                         config for your uploads''')
            _storage_path = False
        else:
            log.critical('''Please specify a ckan.storage_path in your config
                         for your uploads''')
            _storage_path = False

    return _storage_path
Exemple #2
0
def mail_recipient(recipient_name, recipient_email, subject,
                   body, headers={}):
    site_title = config.get('ckan.site_title')
    site_url = config.get('ckan.site_url')
    return _mail_recipient(recipient_name, recipient_email,
                           site_title, site_url, subject, body,
                           headers=headers)
Exemple #3
0
def _setup_error_mail_handler(app):

    class ContextualFilter(logging.Filter):
        def filter(self, log_record):
            log_record.url = request.path
            log_record.method = request.method
            log_record.ip = request.environ.get("REMOTE_ADDR")
            log_record.headers = request.headers
            return True

    mailhost = tuple(config.get('smtp.server', 'localhost').split(":"))
    mail_handler = SMTPHandler(
        mailhost=mailhost,
        fromaddr=config.get('error_email_from'),
        toaddrs=[config.get('email_to')],
        subject='Application Error'
    )

    mail_handler.setFormatter(logging.Formatter('''
Time:               %(asctime)s
URL:                %(url)s
Method:             %(method)s
IP:                 %(ip)s
Headers:            %(headers)s

'''))

    context_provider = ContextualFilter()
    app.logger.addFilter(context_provider)
    app.logger.addHandler(mail_handler)
Exemple #4
0
    def before_search(self, search_params):
        lang_set = set(self.LANGS)

        try:
            current_lang = request.environ['CKAN_LANG']
        except TypeError as err:
            if err.message == ('No object (name: request) has been registered '
                               'for this thread'):
                # This happens when this code gets called as part of a paster
                # command rather then as part of an HTTP request.
                current_lang = config.get('ckan.locale_default')
            else:
                raise
        except KeyError:
            current_lang = config.get('ckan.locale_default')

        # fallback to default locale if locale not in suported langs
        if not current_lang in lang_set:
            current_lang = config.get('ckan.locale_default')
        # fallback to english if default locale is not supported
        if not current_lang in lang_set:
            current_lang = 'en'
        # treat current lang differenly so remove from set
        lang_set.remove(current_lang)

        # weight current lang more highly
        query_fields = 'title_%s^8 text_%s^4' % (current_lang, current_lang)

        for lang in lang_set:
            query_fields += ' title_%s^2 text_%s' % (lang, lang)

        search_params['qf'] = query_fields

        return search_params
Exemple #5
0
def webassets_init():
    global env

    static_path = get_webassets_path()

    public = config.get(u'ckan.base_public_folder')

    public_folder = os.path.abspath(os.path.join(
        os.path.dirname(__file__), u'..', public))

    base_path = os.path.join(public_folder, u'base')

    env = Environment()
    env.directory = static_path
    env.debug = config.get(u'debug', False)
    env.url = u'/webassets/'

    env.append_path(base_path, u'/base/')

    logger.debug(u'Base path {0}'.format(base_path))
    create_library(u'vendor', os.path.join(
        base_path, u'vendor'))

    create_library(u'base', os.path.join(base_path, u'javascript'))

    create_library(u'datapreview', os.path.join(base_path, u'datapreview'))

    create_library(u'css', os.path.join(base_path, u'css'))
Exemple #6
0
def set_cors_headers_for_response(response):
    u'''
    Set up Access Control Allow headers if either origin_allow_all is True, or
    the request Origin is in the origin_whitelist.
    '''
    if config.get(u'ckan.cors.origin_allow_all') \
       and request.headers.get(u'Origin'):

        cors_origin_allowed = None
        if asbool(config.get(u'ckan.cors.origin_allow_all')):
            cors_origin_allowed = b'*'
        elif config.get(u'ckan.cors.origin_whitelist') and \
                request.headers.get(u'Origin') \
                in config[u'ckan.cors.origin_whitelist'].split(u' '):
            # set var to the origin to allow it.
            cors_origin_allowed = request.headers.get(u'Origin')

        if cors_origin_allowed is not None:
            response.headers[b'Access-Control-Allow-Origin'] = \
                cors_origin_allowed
            response.headers[b'Access-Control-Allow-Methods'] = \
                b'POST, PUT, GET, DELETE, OPTIONS'
            response.headers[b'Access-Control-Allow-Headers'] = \
                b'X-CKAN-API-KEY, Authorization, Content-Type'

    return response
Exemple #7
0
def _mail_recipient(recipient_name, recipient_email,
                    sender_name, sender_url, subject,
                    body, headers={}):
    mail_from = config.get('smtp.mail_from')
    msg = MIMEText(body.encode('utf-8'), 'plain', 'utf-8')
    for k, v in headers.items():
        msg[k] = v
    subject = Header(subject.encode('utf-8'), 'utf-8')
    msg['Subject'] = subject
    msg['From'] = _("%s <%s>") % (sender_name, mail_from)
    recipient = u"%s <%s>" % (recipient_name, recipient_email)
    msg['To'] = Header(recipient, 'utf-8')
    msg['Date'] = Utils.formatdate(time())
    msg['X-Mailer'] = "CKAN %s" % ckan.__version__

    # Send the email using Python's smtplib.
    smtp_connection = smtplib.SMTP()
    if 'smtp.test_server' in config:
        # If 'smtp.test_server' is configured we assume we're running tests,
        # and don't use the smtp.server, starttls, user, password etc. options.
        smtp_server = config['smtp.test_server']
        smtp_starttls = False
        smtp_user = None
        smtp_password = None
    else:
        smtp_server = config.get('smtp.server', 'localhost')
        smtp_starttls = paste.deploy.converters.asbool(
            config.get('smtp.starttls'))
        smtp_user = config.get('smtp.user')
        smtp_password = config.get('smtp.password')
    smtp_connection.connect(smtp_server)
    try:
        # Identify ourselves and prompt the server for supported features.
        smtp_connection.ehlo()

        # If 'smtp.starttls' is on in CKAN config, try to put the SMTP
        # connection into TLS mode.
        if smtp_starttls:
            if smtp_connection.has_extn('STARTTLS'):
                smtp_connection.starttls()
                # Re-identify ourselves over TLS connection.
                smtp_connection.ehlo()
            else:
                raise MailerException("SMTP server does not support STARTTLS")

        # If 'smtp.user' is in CKAN config, try to login to SMTP server.
        if smtp_user:
            assert smtp_password, ("If smtp.user is configured then "
                                   "smtp.password must be configured as well.")
            smtp_connection.login(smtp_user, smtp_password)

        smtp_connection.sendmail(mail_from, [recipient_email], msg.as_string())
        log.info("Sent email to {0}".format(recipient_email))

    except smtplib.SMTPException, e:
        msg = '%r' % e
        log.exception(msg)
        raise MailerException(msg)
def upgrade(migrate_engine):

    datastore_connection_url = config.get(
        'ckan.datastore.read_url', config.get('ckan.datastore.write_url'))

    if not datastore_connection_url:
        return

    try:
        datastore_engine = create_engine(datastore_connection_url)
    except SQLAlchemyError:
        return

    try:
        datastore_connection = datastore_engine.connect()
    except SQLAlchemyError:
        datastore_engine.dispose()
        return

    try:

        resources_in_datastore = datastore_connection.execute('''
            SELECT table_name
            FROM information_schema.tables
            WHERE table_schema = 'public'
            AND table_name != '_table_metadata'
        ''')

        if resources_in_datastore.rowcount:

            resources = migrate_engine.execute('''
                SELECT id, extras
                FROM resource
                WHERE id IN ({0}) AND extras IS NOT NULL
            '''.format(
                ','.join(['\'{0}\''.format(_id[0])
                          for _id
                          in resources_in_datastore])
                )
            )
            if resources.rowcount:
                params = []
                for resource in resources:
                    new_extras = json.loads(resource[1])
                    new_extras.update({'datastore_active': True})
                    params.append(
                        {'id': resource[0],
                         'extras': json.dumps(new_extras)})

                migrate_engine.execute(
                    text('''
                    UPDATE resource
                    SET extras = :extras
                    WHERE id = :id'''),
                    params)
    finally:
        datastore_connection.close()
        datastore_engine.dispose()
Exemple #9
0
def get_reset_link_body(user):
    extra_vars = {
        'reset_link': get_reset_link(user),
        'site_title': config.get('ckan.site_title'),
        'site_url': config.get('ckan.site_url'),
        'user_name': user.name,
    }
    # NOTE: This template is translated
    return render_jinja2('emails/reset_password.txt', extra_vars)
Exemple #10
0
def get_all_resources_ids_in_datastore():
    read_url = config.get('ckan.datastore.read_url')
    write_url = config.get('ckan.datastore.write_url')
    data_dict = {
        'connection_url': read_url or write_url
    }
    resources_sql = sqlalchemy.text(u'''SELECT name FROM "_table_metadata"
                                        WHERE alias_of IS NULL''')
    query = _get_engine(data_dict).execute(resources_sql)
    return [q[0] for q in query.fetchall()]
Exemple #11
0
 def delete_package(self, pkg_dict):
     conn = make_connection()
     query = "+%s:%s (+id:\"%s\" OR +name:\"%s\") +site_id:\"%s\"" % (TYPE_FIELD, PACKAGE_TYPE,
                                                    pkg_dict.get('id'), pkg_dict.get('id'),
                                                    config.get('ckan.site_id'))
     try:
         commit = asbool(config.get('ckan.search.solr_commit', 'true'))
         conn.delete(q=query, commit=commit)
     except Exception, e:
         log.exception(e)
         raise SearchIndexError(e)
Exemple #12
0
    def set_active_backend(cls, config):
        """Choose most suitable backend depending on configuration

        :param config: configuration object
        :rtype: ckan.common.CKANConfig

        """
        schema = config.get(u'ckan.datastore.write_url').split(u':')[0]
        read_schema = config.get(u'ckan.datastore.read_url').split(u':')[0]
        assert read_schema == schema, u'Read and write engines are different'
        cls._active_backend = cls._backends[schema]()
 def get_helpers(self):
     return {
         'ozwillo_theme_get_last_datasets': lambda: logic.get_action('package_search')({}, {"rows": 8})['results'],
         'ozwillo_theme_get_resource_number': ozwillo_theme_get_resource_number,
         'ozwillo_theme_get_popular_datasets': lambda: logic.get_action('package_search')({}, {"rows": 4, 'sort': 'views_total desc'})['results'],
         'ozwillo_theme_display_date': ozwillo_theme_display_date,
         'ozwillo_theme_get_map': ozwillo_theme_get_map,
         'ozwillo_theme_get_groups': lambda: logic.get_action('group_list')({}, {"all_fields": True}),
         'ozwillo_theme_spatial_installed': lambda: config.get('ckanext.ozwillo_theme.spatial_installed', 'False'),
         'ozwillo_theme_osmnames_key': lambda: config.get('ckanext.ozwillo_theme.osmnames_key', '')
     }
Exemple #14
0
def get_locales_from_config():
    """ despite the name of this function it gets the locales defined by
    the config AND also the locals available subject to the config. """
    locales_offered = config.get("ckan.locales_offered", "").split()
    filtered_out = config.get("ckan.locales_filtered_out", "").split()
    locale_default = [config.get("ckan.locale_default", "en")]
    locale_order = config.get("ckan.locale_order", "").split()

    known_locales = get_locales()
    all_locales = set(known_locales) | set(locales_offered) | set(locale_order) | set(locale_default)
    all_locales -= set(filtered_out)
    return all_locales
Exemple #15
0
def get_locales_from_config():
    ''' despite the name of this function it gets the locales defined by
    the config AND also the locals available subject to the config. '''
    locales_offered = config.get('ckan.locales_offered', '').split()
    filtered_out = config.get('ckan.locales_filtered_out', '').split()
    locale_default = config.get('ckan.locale_default', 'en')
    locale_order = config.get('ckan.locale_order', '').split()
    known_locales = get_locales()
    all_locales = (set(known_locales) |
                   set(locales_offered) |
                   set(locale_order) |
                   set(locale_default))
    all_locales -= set(filtered_out)
    return all_locales
Exemple #16
0
def _is_legacy_mode(config):
    '''
        Decides if the DataStore should run on legacy mode

        Returns True if `ckan.datastore.read_url` is not set in the provided
        config object or CKAN is running on Postgres < 9.x
    '''
    write_url = config.get('ckan.datastore.write_url')

    engine = db._get_engine({'connection_url': write_url})
    connection = engine.connect()

    return (not config.get('ckan.datastore.read_url') or
            not db._pg_version_is_at_least(connection, '9.0'))
Exemple #17
0
def check_recaptcha(request):
    '''Check a user\'s recaptcha submission is valid, and raise CaptchaError
    on failure.'''
    recaptcha_private_key = config.get('ckan.recaptcha.privatekey', '')
    if not recaptcha_private_key:
        # Recaptcha not enabled
        return
    
    client_ip_address = request.environ.get('REMOTE_ADDR', 'Unknown IP Address')
    
    recaptcha_version = config.get('ckan.recaptcha.version', '1')
    if recaptcha_version is '1':
        recaptcha_response_field = request.params.get('recaptcha_response_field', '')
        recaptcha_server_name = 'http://api-verify.recaptcha.net/verify'
        recaptcha_challenge_field = request.params.get('recaptcha_challenge_field')

        # recaptcha_response_field will be unicode if there are foreign chars in
        # the user input. So we need to encode it as utf8 before urlencoding or
        # we get an exception (#1431).
        params = urllib.urlencode(dict(privatekey=recaptcha_private_key,
                                       remoteip=client_ip_address,
                                       challenge=recaptcha_challenge_field,
                                       response=recaptcha_response_field.encode('utf8')))
        f = urllib2.urlopen(recaptcha_server_name, params)
        data = f.read()
        f.close()
        
        if not data.lower().startswith('true'):
            raise CaptchaError()
    elif recaptcha_version is '2':
        recaptcha_response_field = request.params.get('g-recaptcha-response', '')
        recaptcha_server_name = 'https://www.google.com/recaptcha/api/siteverify'

        # recaptcha_response_field will be unicode if there are foreign chars in
        # the user input. So we need to encode it as utf8 before urlencoding or
        # we get an exception (#1431).
        params = urllib.urlencode(dict(secret=recaptcha_private_key,
                                       remoteip=client_ip_address,
                                       response=recaptcha_response_field.encode('utf8')))
        f = urllib2.urlopen(recaptcha_server_name, params)
        data = json.load(f)   
        f.close()
    
        try:
            if not data['success']:
                raise CaptchaError()
        except IndexError:
            # Something weird with recaptcha response
            raise CaptchaError()
Exemple #18
0
def load_all():
    '''
    Load all plugins listed in the 'ckan.plugins' config directive.
    '''
    # Clear any loaded plugins
    unload_all()

    plugins = config.get('ckan.plugins', '').split() + find_system_plugins()
    # Add the synchronous search plugin, unless already loaded or
    # explicitly disabled
    if 'synchronous_search' not in plugins and \
            asbool(config.get('ckan.search.automatic_indexing', True)):
        log.debug('Loading the synchronous search plugin')
        plugins.append('synchronous_search')

    load(*plugins)
Exemple #19
0
 def as_dict(self, ref_package_by='name', ref_group_by='name'):
     _dict = domain_object.DomainObject.as_dict(self)
     # Set 'license' in _dict to cater for old clients.
     # Todo: Remove from Version 2?
     _dict['license'] = self.license.title if self.license else _dict.get('license_id', '')
     _dict['isopen'] = self.isopen()
     tags = [tag.name for tag in self.get_tags()]
     tags.sort() # so it is determinable
     _dict['tags'] = tags
     groups = [getattr(group, ref_group_by) for group in self.get_groups()]
     groups.sort()
     _dict['groups'] = groups
     _dict['extras'] = {key: value for key, value in self.extras.items()}
     _dict['ratings_average'] = self.get_average_rating()
     _dict['ratings_count'] = len(self.ratings)
     _dict['resources'] = [res.as_dict(core_columns_only=False) \
                           for res in self.resources]
     site_url = config.get('ckan.site_url', None)
     if site_url:
         _dict['ckan_url'] = '%s/dataset/%s' % (site_url, self.name)
     _dict['relationships'] = [rel.as_dict(self, ref_package_by=ref_package_by) for rel in self.get_relationships()]
     _dict['metadata_modified'] = self.metadata_modified.isoformat() \
         if self.metadata_modified else None
     _dict['metadata_created'] = self.metadata_created.isoformat() \
         if self.metadata_created else None
     import ckan.lib.helpers as h
     _dict['notes_rendered'] = h.render_markdown(self.notes)
     _dict['type'] = self.type or u'dataset'
     return _dict
Exemple #20
0
def render_snippet(*template_names, **kw):
    ''' Helper function for rendering snippets. Rendered html has
    comment tags added to show the template used. NOTE: unlike other
    render functions this takes a list of keywords instead of a dict for
    the extra template variables.

    :param template_names: the template to render, optionally with fallback
        values, for when the template can't be found. For each, specify the
        relative path to the template inside the registered tpl_dir.
    :type template_names: str
    :param kw: extra template variables to supply to the template
    :type kw: named arguments of any type that are supported by the template
    '''

    exc = None
    for template_name in template_names:
        try:
            output = render(template_name, extra_vars=kw)
            if config.get('debug'):
                output = (
                    '\n<!-- Snippet %s start -->\n%s\n<!-- Snippet %s end -->'
                    '\n' % (template_name, output, template_name))
            return literal(output)
        except TemplateNotFound as exc:
            if exc.name == template_name:
                # the specified template doesn't exist - try the next fallback
                continue
            # a nested template doesn't exist - don't fallback
            raise exc
    else:
        raise exc or TemplateNotFound
Exemple #21
0
def check_config_permission(permission):
    '''Returns the configuration value for the provided permission

    Permission is a string indentifying the auth permission (eg
    `anon_create_dataset`), optionally prefixed with `ckan.auth.`.

    The possible values for `permission` are the keys of
    CONFIG_PERMISSIONS_DEFAULTS. These can be overriden in the config file
    by prefixing them with `ckan.auth.`.

    Returns the permission value, generally True or False, except on
    `roles_that_cascade_to_sub_groups` which is a list of strings.

    '''

    key = permission.replace('ckan.auth.', '')

    if key not in CONFIG_PERMISSIONS_DEFAULTS:
        return False

    default_value = CONFIG_PERMISSIONS_DEFAULTS.get(key)

    config_key = 'ckan.auth.' + key

    value = config.get(config_key, default_value)

    if key == 'roles_that_cascade_to_sub_groups':
        # This permission is set as a list of strings (space separated)
        value = value.split() if value else []
    else:
        value = asbool(value)

    return value
Exemple #22
0
    def get_config_value(key, default=''):
        if model.meta.engine.has_table('system_info'):
            value = model.get_system_info(key)
        else:
            value = None
        config_value = config.get(key)
        # sort encodeings if needed
        if isinstance(config_value, str):
            try:
                config_value = config_value.decode('utf-8')
            except UnicodeDecodeError:
                config_value = config_value.decode('latin-1')
        # we want to store the config the first time we get here so we can
        # reset them if needed
        if key not in _CONFIG_CACHE:
            _CONFIG_CACHE[key] = config_value
        if value is not None:
            log.debug('config `%s` set to `%s` from db' % (key, value))
        else:
            value = _CONFIG_CACHE[key]
            if value:
                log.debug('config `%s` set to `%s` from config' % (key, value))
            else:
                value = default

        set_app_global(key, value)

        # update the config
        config[key] = value

        return value
 def setup_class(cls):
     smtp_server = config.get('smtp.test_server')
     if smtp_server:
         host, port = smtp_server.split(':')
         port = int(port) + int(str(hashlib.md5(cls.__name__).hexdigest())[0], 16)
         config['smtp.test_server'] = '%s:%s' % (host, port)
     SmtpServerHarness.setup_class()
Exemple #24
0
def _build_fts_indexes(connection, data_dict, sql_index_str_method, fields):
    fts_indexes = []
    resource_id = data_dict['resource_id']
    # FIXME: This is repeated on the plugin.py, we should keep it DRY
    default_fts_lang = config.get('ckan.datastore.default_fts_lang')
    if default_fts_lang is None:
        default_fts_lang = u'english'
    fts_lang = data_dict.get('lang', default_fts_lang)

    # create full-text search indexes
    def to_tsvector(x):
        return u"to_tsvector('{0}', {1})".format(fts_lang, x)

    def cast_as_text(x):
        return u'cast("{0}" AS text)'.format(x)

    full_text_field = {'type': 'tsvector', 'id': '_full_text'}
    for field in [full_text_field] + fields:
        if not datastore_helpers.should_fts_index_field_type(field['type']):
            continue

        field_str = field['id']
        if field['type'] not in ['text', 'tsvector']:
            field_str = cast_as_text(field_str)
        else:
            field_str = u'"{0}"'.format(field_str)
        if field['type'] != 'tsvector':
            field_str = to_tsvector(field_str)
        fts_indexes.append(sql_index_str_method.format(
            res_id=resource_id,
            unique='',
            name=_generate_index_name(resource_id, field_str),
            method=_get_fts_index_method(), fields=field_str))

    return fts_indexes
Exemple #25
0
    def edit(self, id=None, data=None, errors=None, error_summary=None):
        context = {'save': 'save' in request.params,
                   'schema': self._edit_form_to_db_schema(),
                   'model': model, 'session': model.Session,
                   'user': c.user, 'auth_user_obj': c.userobj
                   }
        if id is None:
            if c.userobj:
                id = c.userobj.id
            else:
                abort(400, _('No user specified'))
        data_dict = {'id': id}

        try:
            check_access('user_update', context, data_dict)
        except NotAuthorized:
            abort(403, _('Unauthorized to edit a user.'))

        if (context['save']) and not data:
            return self._save_edit(id, context)

        try:
            old_data = get_action('user_show')(context, data_dict)

            schema = self._db_to_edit_form_schema()
            if schema:
                old_data, errors = \
                    dictization_functions.validate(old_data, schema, context)

            c.display_name = old_data.get('display_name')
            c.user_name = old_data.get('name')

            data = data or old_data

        except NotAuthorized:
            abort(403, _('Unauthorized to edit user %s') % '')
        except NotFound:
            abort(404, _('User not found'))

        user_obj = context.get('user_obj')

        if not (authz.is_sysadmin(c.user)
                or c.user == user_obj.name):
            abort(403, _('User %s not authorized to edit %s') %
                  (str(c.user), id))

        errors = errors or {}
        vars = {'data': data, 'errors': errors, 'error_summary': error_summary}

        self._setup_template_variables({'model': model,
                                        'session': model.Session,
                                        'user': c.user},
                                       data_dict)

        c.is_myself = True
        c.show_email_notifications = asbool(
            config.get('ckan.activity_streams_email_notifications'))
        c.form = render(self.edit_user_form, extra_vars=vars)

        return render('user/edit.html')
Exemple #26
0
    def general(self):
        data_dict, params = self._parse_url_params()
        data_dict['q'] = '*:*'

        item_count, results = _package_search(data_dict)

        navigation_urls = self._navigation_urls(params,
                                                item_count=item_count,
                                                limit=data_dict['rows'],
                                                controller='feed',
                                                action='general')

        feed_url = self._feed_url(params,
                                  controller='feed',
                                  action='general')

        alternate_url = self._alternate_url(params)

        site_title = config.get('ckan.site_title', 'CKAN')

        return self.output_feed(results,
                                feed_title=site_title,
                                feed_description=u'Recently created or '
                                'updated datasets on %s' % site_title,
                                feed_link=alternate_url,
                                feed_guid=_create_atom_id
                                (u'/feeds/dataset.atom'),
                                feed_url=feed_url,
                                navigation_urls=navigation_urls)
Exemple #27
0
    def before_view(self, dataset_dict):

        # Translate any selected search facets (e.g. if we are rendering a
        # group read page or the dataset index page): lookup translations of
        # all the terms in c.fields (c.fields contains the selected facets)
        # and save them in c.translated_fields where the templates can
        # retrieve them later.
        desired_lang_code = request.environ['CKAN_LANG']
        fallback_lang_code = config.get('ckan.locale_default', 'en')
        try:
            fields = c.fields
        except AttributeError:
            return translate_data_dict(dataset_dict)
        terms = [value for param, value in fields]
        translations = get_action('term_translation_show')(
                {'model': ckan.model},
                {'terms': terms,
                 'lang_codes': (desired_lang_code, fallback_lang_code)})
        c.translated_fields = {}
        for param, value in fields:
            matching_translations = [translation for translation in
                    translations if translation['term'] == value and
                    translation['lang_code'] == desired_lang_code]
            if not matching_translations:
                matching_translations = [translation for translation in
                        translations if translation['term'] == value and
                        translation['lang_code'] == fallback_lang_code]
            if matching_translations:
                assert len(matching_translations) == 1
                translation = matching_translations[0]['term_translation']
                c.translated_fields[(param, value)] = translation

        # Now translate the fields of the dataset itself.
        return translate_data_dict(dataset_dict)
Exemple #28
0
    def render_template():
        globs = extra_vars or {}
        globs.update(pylons_globals())

        # Using pylons.url() directly destroys the localisation stuff so
        # we remove it so any bad templates crash and burn
        del globs['url']

        try:
            template_path, template_type = render_.template_info(template_name)
        except render_.TemplateNotFound:
            raise

        log.debug('rendering %s [%s]' % (template_path, template_type))
        if config.get('debug'):
            context_vars = globs.get('c')
            if context_vars:
                context_vars = dir(context_vars)
            debug_info = {'template_name': template_name,
                          'template_path': template_path,
                          'template_type': template_type,
                          'vars': globs,
                          'c_vars': context_vars,
                          'renderer': renderer}
            if 'CKAN_DEBUG_INFO' not in request.environ:
                request.environ['CKAN_DEBUG_INFO'] = []
            request.environ['CKAN_DEBUG_INFO'].append(debug_info)

        del globs['config']
        return render_jinja2(template_name, globs)
Exemple #29
0
    def index(self):
        page = h.get_page_number(request.params)
        c.q = request.params.get('q', '')
        c.order_by = request.params.get('order_by', 'name')

        context = {'return_query': True, 'user': c.user,
                   'auth_user_obj': c.userobj}

        data_dict = {'q': c.q,
                     'order_by': c.order_by}

        limit = int(
            request.params.get('limit', config.get('ckan.user_list_limit', 20))
        )
        try:
            check_access('user_list', context, data_dict)
        except NotAuthorized:
            abort(403, _('Not authorized to see this page'))

        users_list = get_action('user_list')(context, data_dict)

        c.page = h.Page(
            collection=users_list,
            page=page,
            url=h.pager_url,
            item_count=users_list.count(),
            items_per_page=limit
        )
        return render('user/list.html')
Exemple #30
0
    def tag(self, id):
        data_dict, params = self._parse_url_params()
        data_dict['fq'] = 'tags:"%s"' % id

        item_count, results = _package_search(data_dict)

        navigation_urls = self._navigation_urls(params,
                                                item_count=item_count,
                                                limit=data_dict['rows'],
                                                controller='feed',
                                                action='tag',
                                                id=id)

        feed_url = self._feed_url(params,
                                  controller='feed',
                                  action='tag',
                                  id=id)

        alternate_url = self._alternate_url(params, tags=id)

        site_title = config.get('ckan.site_title', 'CKAN')

        return self.output_feed(results,
                                feed_title=u'%s - Tag: "%s"' %
                                (site_title, id),
                                feed_description=u'Recently created or '
                                'updated datasets on %s by tag: "%s"' %
                                (site_title, id),
                                feed_link=alternate_url,
                                feed_guid=_create_atom_id
                                (u'/feeds/tag/%s.atom' % id),
                                feed_url=feed_url,
                                navigation_urls=navigation_urls)
def get_i18n_path():
    return config.get(u'ckan.i18n_directory', os.path.join(ckan_path, u'i18n'))
Exemple #32
0
def get_jira_script():
    jira_script = config.get('ckanext.nextgeoss.jira_issue_tracker')
    return jira_script
Exemple #33
0
You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""

import datetime

from sqlalchemy import Table, select, join, func, and_

from ckanext.stats.stats import Stats as CoreStats, RevisionStats as CoreRevisionStats, table, datetime2date, DATE_FORMAT

import ckan.model as model
import ckan.plugins as p
from ckan.common import config

cache_enabled = p.toolkit.asbool(config.get('ckanext.stats.cache_enabled', 'True'))

if cache_enabled:
    from pylons import cache
    from ckanext.stats.stats import our_cache

class Stats(CoreStats):
    pass

class RevisionStats(CoreRevisionStats):
    
    @classmethod
    def get_num_packages_by_week(cls):
        """
        overriden
        """
Exemple #34
0
def _create_atom_id(resource_path, authority_name=None, date_string=None):
    """
    Helper method that creates an atom id for a feed or entry.

    An id must be unique, and must not change over time.  ie - once published,
    it represents an atom feed or entry uniquely, and forever.  See [4]:

        When an Atom Document is relocated, migrated, syndicated,
        republished, exported, or imported, the content of its atom:id
        element MUST NOT change.  Put another way, an atom:id element
        pertains to all instantiations of a particular Atom entry or feed;
        revisions retain the same content in their atom:id elements.  It is
        suggested that the atom:id element be stored along with the
        associated resource.

    resource_path
        The resource path that uniquely identifies the feed or element.  This
        mustn't be something that changes over time for a given entry or feed.
        And does not necessarily need to be resolvable.

        e.g. ``"/group/933f3857-79fd-4beb-a835-c0349e31ce76"`` could represent
        the feed of datasets belonging to the identified group.

    authority_name
        The domain name or email address of the publisher of the feed.  See [3]
        for more details.  If ``None`` then the domain name is taken from the
        config file.  First trying ``ckan.feeds.authority_name``, and failing
        that, it uses ``ckan.site_url``.  Again, this should not change over
        time.

    date_string
        A string representing a date on which the authority_name is owned by
        the publisher of the feed.

        e.g. ``"2012-03-22"``

        Again, this should not change over time.

        If date_string is None, then an attempt is made to read the config
        option ``ckan.feeds.date``.  If that's not available,
        then the date_string is not used in the generation of the atom id.

    Following the methods outlined in [1], [2] and [3], this function produces
    tagURIs like:
    ``"tag:thedatahub.org,2012:/group/933f3857-79fd-4beb-a835-c0349e31ce76"``.

    If not enough information is provide to produce a valid tagURI, then only
    the resource_path is used, e.g.: ::

        "http://thedatahub.org/group/933f3857-79fd-4beb-a835-c0349e31ce76"

    or

        "/group/933f3857-79fd-4beb-a835-c0349e31ce76"

    The latter of which is only used if no site_url is available.   And it
    should be noted will result in an invalid feed.

    [1] http://web.archive.org/web/20110514113830/http://diveintomark.org/\
    archives/2004/05/28/howto-atom-id
    [2] http://www.taguri.org/
    [3] http://tools.ietf.org/html/rfc4151#section-2.1
    [4] http://www.ietf.org/rfc/rfc4287
    """
    if authority_name is None:
        authority_name = config.get('ckan.feeds.authority_name', '').strip()
        if not authority_name:
            site_url = config.get('ckan.site_url', '').strip()
            authority_name = urlparse.urlparse(site_url).netloc

    if not authority_name:
        log.warning('No authority_name available for feed generation.  '
                    'Generated feed will be invalid.')

    if date_string is None:
        date_string = config.get('ckan.feeds.date', '')

    if not date_string:
        log.warning('No date_string available for feed generation.  '
                    'Please set the "ckan.feeds.date" config value.')

        # Don't generate a tagURI without a date as it wouldn't be valid.
        # This is best we can do, and if the site_url is not set, then
        # this still results in an invalid feed.
        site_url = config.get('ckan.site_url', '')
        return '/'.join([site_url, resource_path])

    tagging_entity = ','.join([authority_name, date_string])
    return ':'.join(['tag', tagging_entity, resource_path])
Exemple #35
0
def get_featured_image_url(default):
    return config.get('ckanext.lacounts.featured_image') or default
Exemple #36
0
def sysadmin_email():
    return text_type(config.get('smtp.mail_from', ''))
Exemple #37
0
def get_max_image_size():
    global _max_image_size
    if _max_image_size is None:
        _max_image_size = int(config.get('ckan.max_image_size', 2))
    return _max_image_size
Exemple #38
0
def get_max_resource_size():
    global _max_resource_size
    if _max_resource_size is None:
        _max_resource_size = int(config.get('ckan.max_resource_size', 10))
    return _max_resource_size
def upgrade(migrate_engine):
    schema = ckan_config.get(u'ckan.migrations.target_schema') or 'public'
    # we specify the schema here because of a clash with another 'state' table
    # in the mdillon/postgis container. You only need to change the value in the
    # config if you've altered the default schema from 'public' in your
    # postgresql.conf. Because this is such a rarely needed option, it is
    # otherwise undocumented.
    meta = MetaData(schema=schema)

    state = Table(
        'state',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('name', Unicode(100)),
    )

    revision = Table(
        'revision',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('timestamp', DateTime(timezone=False)),
        Column('author', Unicode(200)),
        Column('message', UnicodeText()),
        Column('state_id', Integer()),
    )

    apikey = Table(
        'apikey',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('name', UnicodeText()),
        Column('key', UnicodeText()),
    )

    license = Table(
        'license',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('name', Unicode(100)),
        # Column('state_id', Integer(), ForeignKey('state.id')),
        Column('state_id', Integer()))

    package = Table(
        'package',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('name', Unicode(100), nullable=False, unique=True),
        Column('title', UnicodeText()),
        Column('version', Unicode(100)),
        Column('url', UnicodeText()),
        Column('download_url', UnicodeText()),
        Column('notes', UnicodeText()),
        Column('license_id', Integer(), ForeignKey('license.id')),
        Column('state_id', Integer(), ForeignKey('state.id')),
        Column('revision_id', Integer(), ForeignKey('revision.id')),
    )

    package_revision = Table(
        'package_revision',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('name', Unicode(100), nullable=False),
        Column('title', UnicodeText()),
        Column('version', Unicode(100)),
        Column('url', UnicodeText()),
        Column('download_url', UnicodeText()),
        Column('notes', UnicodeText()),
        Column('license_id', Integer(), ForeignKey('license.id')),
        Column('state_id', Integer(), ForeignKey('state.id')),
        Column('revision_id',
               Integer(),
               ForeignKey('revision.id'),
               primary_key=True),
        Column('continuity_id', Integer(), ForeignKey('package.id')),
    )

    tag = Table(
        'tag',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('name', Unicode(100), nullable=False, unique=True),
    )

    package_tag = Table(
        'package_tag',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('package_id', Integer(), ForeignKey('package.id')),
        Column('tag_id', Integer(), ForeignKey('tag.id')),
        Column('state_id', Integer(), ForeignKey('state.id')),
        Column('revision_id', Integer(), ForeignKey('revision.id')),
    )

    package_tag_revision = Table(
        'package_tag_revision',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('package_id', Integer(), ForeignKey('package.id')),
        Column('tag_id', Integer(), ForeignKey('tag.id')),
        Column('state_id', Integer(), ForeignKey('state.id')),
        Column('revision_id',
               Integer(),
               ForeignKey('revision.id'),
               primary_key=True),
        Column('continuity_id', Integer(), ForeignKey('package_tag.id')),
    )

    package_extra = Table(
        'package_extra',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('package_id', Integer(), ForeignKey('package.id')),
        Column('key', UnicodeText()),
        Column('value', UnicodeText()),
        Column('state_id', Integer(), ForeignKey('state.id')),
        Column('revision_id', Integer(), ForeignKey('revision.id')),
    )

    package_extra_revision = Table(
        'package_extra_revision',
        meta,
        Column('id', Integer(), primary_key=True, nullable=False),
        Column('package_id', Integer(), ForeignKey('package.id')),
        Column('key', UnicodeText()),
        Column('value', UnicodeText()),
        Column('state_id', Integer(), ForeignKey('state.id')),
        Column('revision_id',
               Integer(),
               ForeignKey('revision.id'),
               primary_key=True),
        Column('continuity_id', Integer(), ForeignKey('package_extra.id')),
    )

    meta.bind = migrate_engine
    meta.create_all()
Exemple #40
0
def make_map():
    """Create, configure and return the routes Mapper"""
    # import controllers here rather than at root level because
    # pylons config is initialised by this point.

    # Helpers to reduce code clutter
    GET = dict(method=['GET'])
    PUT = dict(method=['PUT'])
    POST = dict(method=['POST'])
    DELETE = dict(method=['DELETE'])
    GET_POST = dict(method=['GET', 'POST'])
    PUT_POST = dict(method=['PUT', 'POST'])
    PUT_POST_DELETE = dict(method=['PUT', 'POST', 'DELETE'])
    OPTIONS = dict(method=['OPTIONS'])

    import ckan.lib.plugins as lib_plugins
    lib_plugins.reset_package_plugins()

    map = Mapper(directory=config['pylons.paths']['controllers'],
                 always_scan=config['debug'])
    map.minimization = False
    map.explicit = True

    # The ErrorController route (handles 404/500 error pages); it should
    # likely stay at the top, ensuring it can always be resolved.
    map.connect('/error/{action}', controller='error', ckan_core=True)
    map.connect('/error/{action}/{id}', controller='error', ckan_core=True)

    map.connect('*url',
                controller='home',
                action='cors_options',
                conditions=OPTIONS,
                ckan_core=True)

    # CUSTOM ROUTES HERE
    for plugin in p.PluginImplementations(p.IRoutes):
        map = plugin.before_map(map)

    # Mark all routes added from extensions on the `before_map` extension point
    # as non-core
    for route in map.matchlist:
        if not hasattr(route, '_ckan_core'):
            route._ckan_core = False

    map.connect('invite',
                '/__invite__/',
                controller='partyline',
                action='join_party')

    map.connect('home', '/', controller='home', action='index')
    map.connect('about', '/about', controller='home', action='about')
    map.connect('visual', '/vis', controller='home', action='visual')

    map.connect('suggest', '/suggest', controller='home', action='suggest')
    # map.connect('user_database','/user/userdatabase/{id:.*}',controller = 'user', action = 'dbsearch')

    # CKAN API versioned.
    register_list = [
        'package', 'dataset', 'resource', 'tag', 'group', 'revision',
        'licenses', 'rating', 'user', 'activity'
    ]
    register_list_str = '|'.join(register_list)

    # /api ver 3 or none
    with SubMapper(map,
                   controller='api',
                   path_prefix='/api{ver:/3|}',
                   ver='/3') as m:
        m.connect('/action/{logic_function}',
                  action='action',
                  conditions=GET_POST)

    # /api ver 1, 2, 3 or none
    with SubMapper(map,
                   controller='api',
                   path_prefix='/api{ver:/1|/2|/3|}',
                   ver='/1') as m:
        m.connect('', action='get_api')
        m.connect('/search/{register}', action='search')

    # /api ver 1, 2 or none
    with SubMapper(map,
                   controller='api',
                   path_prefix='/api{ver:/1|/2|}',
                   ver='/1') as m:
        m.connect('/tag_counts', action='tag_counts')
        m.connect('/rest', action='index')
        m.connect('/qos/throughput/', action='throughput', conditions=GET)

    # /api/rest ver 1, 2 or none
    with SubMapper(map,
                   controller='api',
                   path_prefix='/api{ver:/1|/2|}',
                   ver='/1',
                   requirements=dict(register=register_list_str)) as m:

        m.connect('/rest/{register}', action='list', conditions=GET)
        m.connect('/rest/{register}', action='create', conditions=POST)
        m.connect('/rest/{register}/{id}', action='show', conditions=GET)
        m.connect('/rest/{register}/{id}', action='update', conditions=PUT)
        m.connect('/rest/{register}/{id}', action='update', conditions=POST)
        m.connect('/rest/{register}/{id}', action='delete', conditions=DELETE)
        m.connect('/rest/{register}/{id}/:subregister',
                  action='list',
                  conditions=GET)
        m.connect('/rest/{register}/{id}/:subregister',
                  action='create',
                  conditions=POST)
        m.connect('/rest/{register}/{id}/:subregister/{id2}',
                  action='create',
                  conditions=POST)
        m.connect('/rest/{register}/{id}/:subregister/{id2}',
                  action='show',
                  conditions=GET)
        m.connect('/rest/{register}/{id}/:subregister/{id2}',
                  action='update',
                  conditions=PUT)
        m.connect('/rest/{register}/{id}/:subregister/{id2}',
                  action='delete',
                  conditions=DELETE)

    # /api/util ver 1, 2 or none
    with SubMapper(map,
                   controller='api',
                   path_prefix='/api{ver:/1|/2|}',
                   ver='/1') as m:
        m.connect('/util/user/autocomplete', action='user_autocomplete')
        m.connect('/util/is_slug_valid',
                  action='is_slug_valid',
                  conditions=GET)
        m.connect('/util/dataset/autocomplete',
                  action='dataset_autocomplete',
                  conditions=GET)
        m.connect('/util/tag/autocomplete',
                  action='tag_autocomplete',
                  conditions=GET)
        m.connect('/util/resource/format_autocomplete',
                  action='format_autocomplete',
                  conditions=GET)
        m.connect('/util/resource/format_icon',
                  action='format_icon',
                  conditions=GET)
        m.connect('/util/group/autocomplete', action='group_autocomplete')
        m.connect('/util/organization/autocomplete',
                  action='organization_autocomplete',
                  conditions=GET)
        m.connect('/util/markdown', action='markdown')
        m.connect('/util/dataset/munge_name', action='munge_package_name')
        m.connect('/util/dataset/munge_title_to_name',
                  action='munge_title_to_package_name')
        m.connect('/util/tag/munge', action='munge_tag')
        m.connect('/util/status', action='status')
        m.connect('/util/snippet/{snippet_path:.*}', action='snippet')
        m.connect('/i18n/{lang}', action='i18n_js_translations')

    ###########
    ## /END API
    ###########

    map.redirect('/packages', '/dataset')
    map.redirect('/packages/{url:.*}', '/dataset/{url}')
    map.redirect('/package', '/dataset')
    map.redirect('/package/{url:.*}', '/dataset/{url}')

    with SubMapper(map, controller='package') as m:
        m.connect('search',
                  '/dataset',
                  action='search',
                  highlight_actions='index search')
        m.connect('add dataset', '/dataset/new', action='new')
        m.connect('/dataset/{action}',
                  requirements=dict(
                      action='|'.join(['list', 'autocomplete', 'search'])))

        m.connect('/dataset/{action}/{id}/{revision}',
                  action='read_ajax',
                  requirements=dict(action='|'.join([
                      'read',
                      'edit',
                      'history',
                  ])))
        m.connect('/dataset/{action}/{id}',
                  requirements=dict(action='|'.join([
                      'new_resource', 'history', 'read_ajax', 'history_ajax',
                      'follow', 'activity', 'groups', 'unfollow', 'delete',
                      'api_data'
                  ])))
        m.connect(
            '/dataset/add_button/{package_name}/{id}/{pack_r_id}/{user_name}',
            action='add_button')

        m.connect('dataset_edit',
                  '/dataset/edit/{id}',
                  action='edit',
                  ckan_icon='edit')
        m.connect('dataset_followers',
                  '/dataset/followers/{id}',
                  action='followers',
                  ckan_icon='group')
        m.connect('dataset_activity',
                  '/dataset/activity/{id}',
                  action='activity',
                  ckan_icon='time')
        m.connect('/dataset/activity/{id}/{offset}', action='activity')
        m.connect('dataset_groups',
                  '/dataset/groups/{id}',
                  action='groups',
                  ckan_icon='group')
        m.connect('dataset_resources',
                  '/dataset/resources/{id}',
                  action='resources',
                  ckan_icon='reorder')
        m.connect('dataset_read',
                  '/dataset/{id}',
                  action='read',
                  ckan_icon='sitemap')
        m.connect('/dataset/{id}/resource/{resource_id}',
                  action='resource_read')
        m.connect('/dataset/{id}/resource_delete/{resource_id}',
                  action='resource_delete')
        m.connect('resource_edit',
                  '/dataset/{id}/resource_edit/{resource_id}',
                  action='resource_edit',
                  ckan_icon='edit')
        m.connect('/dataset/{id}/resource/{resource_id}/download',
                  action='resource_download')
        m.connect('/dataset/{id}/resource/{resource_id}/download/{filename}',
                  action='resource_download')
        m.connect('/dataset/{id}/resource/{resource_id}/embed',
                  action='resource_embedded_dataviewer')
        m.connect('/dataset/{id}/resource/{resource_id}/viewer',
                  action='resource_embedded_dataviewer',
                  width="960",
                  height="800")
        m.connect('/dataset/{id}/resource/{resource_id}/preview',
                  action='resource_datapreview')
        m.connect('views',
                  '/dataset/{id}/resource/{resource_id}/views',
                  action='resource_views',
                  ckan_icon='reorder')
        m.connect('new_view',
                  '/dataset/{id}/resource/{resource_id}/new_view',
                  action='edit_view',
                  ckan_icon='edit')
        m.connect('edit_view',
                  '/dataset/{id}/resource/{resource_id}/edit_view/{view_id}',
                  action='edit_view',
                  ckan_icon='edit')
        m.connect('resource_view',
                  '/dataset/{id}/resource/{resource_id}/view/{view_id}',
                  action='resource_view')
        m.connect('/dataset/{id}/resource/{resource_id}/view/',
                  action='resource_view')

    # group
    map.redirect('/groups', '/group')
    map.redirect('/groups/{url:.*}', '/group/{url}')

    # These named routes are used for custom group forms which will use the
    # names below based on the group.type ('group' is the default type)
    with SubMapper(map, controller='group') as m:
        m.connect('group_index',
                  '/group',
                  action='index',
                  highlight_actions='index search')
        m.connect('group_list', '/group/list', action='list')
        m.connect('group_new', '/group/new', action='new')
        m.connect('group_action',
                  '/group/{action}/{id}',
                  requirements=dict(action='|'.join([
                      'edit',
                      'delete',
                      'member_new',
                      'member_delete',
                      'history',
                      'followers',
                      'follow',
                      'unfollow',
                      'admins',
                      'activity',
                  ])))
        m.connect('group_about',
                  '/group/about/{id}',
                  action='about',
                  ckan_icon='info-sign'),
        m.connect('group_edit',
                  '/group/edit/{id}',
                  action='edit',
                  ckan_icon='edit')
        m.connect('group_members',
                  '/group/members/{id}',
                  action='members',
                  ckan_icon='group'),
        m.connect('group_activity',
                  '/group/activity/{id}/{offset}',
                  action='activity',
                  ckan_icon='time'),
        m.connect('group_read',
                  '/group/{id}',
                  action='read',
                  ckan_icon='sitemap')

    # organizations these basically end up being the same as groups
    with SubMapper(map, controller='organization') as m:
        m.connect('organizations_index', '/organization', action='index')
        m.connect('/organization/list', action='list')
        m.connect('/organization/new', action='new')
        m.connect(
            '/organization/{action}/{id}',
            requirements=dict(action='|'.join([
                'delete', 'admins', 'member_new', 'member_delete', 'history'
            ])))
        m.connect('organization_activity',
                  '/organization/activity/{id}/{offset}',
                  action='activity',
                  ckan_icon='time')
        m.connect('organization_read', '/organization/{id}', action='read')
        m.connect('organization_about',
                  '/organization/about/{id}',
                  action='about',
                  ckan_icon='info-sign')
        m.connect('organization_read',
                  '/organization/{id}',
                  action='read',
                  ckan_icon='sitemap')
        m.connect('organization_edit',
                  '/organization/edit/{id}',
                  action='edit',
                  ckan_icon='edit')
        m.connect('organization_members',
                  '/organization/members/{id}',
                  action='members',
                  ckan_icon='group')
        m.connect('organization_bulk_process',
                  '/organization/bulk_process/{id}',
                  action='bulk_process',
                  ckan_icon='sitemap')
    lib_plugins.register_package_plugins(map)
    lib_plugins.register_group_plugins(map)

    # tags
    map.redirect('/tags', '/tag')
    map.redirect('/tags/{url:.*}', '/tag/{url}')
    map.redirect('/tag/read/{url:.*}',
                 '/tag/{url}',
                 _redirect_code='301 Moved Permanently')
    map.connect('/tag', controller='tag', action='index')
    map.connect('/tag/{id}', controller='tag', action='read')
    # users
    map.redirect('/users/{url:.*}', '/user/{url}')
    map.redirect('/user/', '/user')
    with SubMapper(map, controller='user') as m:
        m.connect('/user/edit', action='edit')
        # Note: openid users have slashes in their ids, so need the wildcard
        # in the route.
        m.connect('user_database',
                  '/user/userdatabase/{id:.*}',
                  action='dbsearch')
        m.connect('user_generate_apikey',
                  '/user/generate_key/{id}',
                  action='generate_apikey')
        m.connect('/user/activity/{id}/{offset}', action='activity')
        m.connect('user_activity_stream',
                  '/user/activity/{id}',
                  action='activity',
                  ckan_icon='time')
        m.connect('user_dashboard',
                  '/dashboard',
                  action='dashboard',
                  ckan_icon='list')
        m.connect('user_dashboard_datasets',
                  '/dashboard/datasets',
                  action='dashboard_datasets',
                  ckan_icon='sitemap')
        m.connect('user_dashboard_groups',
                  '/dashboard/groups',
                  action='dashboard_groups',
                  ckan_icon='group')
        m.connect('user_dashboard_organizations',
                  '/dashboard/organizations',
                  action='dashboard_organizations',
                  ckan_icon='building')
        m.connect('/dashboard/{offset}', action='dashboard')
        m.connect('user_follow', '/user/follow/{id}', action='follow')
        m.connect('/user/unfollow/{id}', action='unfollow')
        m.connect('user_followers',
                  '/user/followers/{id:.*}',
                  action='followers',
                  ckan_icon='group')
        m.connect('user_edit',
                  '/user/edit/{id:.*}',
                  action='edit',
                  ckan_icon='cog')
        m.connect('user_delete', '/user/delete/{id}', action='delete')
        m.connect('/user/reset/{id:.*}', action='perform_reset')
        m.connect('register', '/user/register', action='register')
        m.connect('login', '/user/login', action='login')
        m.connect('/user/_logout', action='logout')
        m.connect('/user/logged_in', action='logged_in')
        m.connect('/user/logged_out', action='logged_out')
        m.connect('/user/logged_out_redirect', action='logged_out_page')
        m.connect('/user/reset', action='request_reset')
        m.connect('/user/me', action='me')
        m.connect('/user/set_lang/{lang}', action='set_lang')
        m.connect('user_datasets',
                  '/user/{id:.*}',
                  action='read',
                  ckan_icon='sitemap')
        m.connect('user_index', '/user', action='index')
        m.connect('user_ffdatabase',
                  '/user111/userdatabase/{id:.*}',
                  action='dbsearch')

    with SubMapper(map, controller='revision') as m:
        m.connect('/revision', action='index')
        m.connect('/revision/edit/{id}', action='edit')
        m.connect('/revision/diff/{id}', action='diff')
        m.connect('/revision/list', action='list')
        m.connect('/revision/{id}', action='read')

    # feeds
    with SubMapper(map, controller='feed') as m:
        m.connect('/feeds/group/{id}.atom', action='group')
        m.connect('/feeds/organization/{id}.atom', action='organization')
        m.connect('/feeds/tag/{id}.atom', action='tag')
        m.connect('/feeds/dataset.atom', action='general')
        m.connect('/feeds/custom.atom', action='custom')

    map.connect('ckanadmin_index',
                '/ckan-admin',
                controller='admin',
                action='index',
                ckan_icon='legal')
    map.connect('ckanadmin_config',
                '/ckan-admin/config',
                controller='admin',
                action='config',
                ckan_icon='check')
    map.connect('ckanadmin_trash',
                '/ckan-admin/trash',
                controller='admin',
                action='trash',
                ckan_icon='trash')
    map.connect('ckanadmin', '/ckan-admin/{action}', controller='admin')

    with SubMapper(
            map, controller='ckan.controllers.storage:StorageController') as m:
        m.connect('storage_file', '/storage/f/{label:.*}', action='file')

    with SubMapper(map, controller='util') as m:
        m.connect('/i18n/strings_{lang}.js', action='i18n_js_strings')
        m.connect('/util/redirect', action='redirect')
        m.connect('/testing/primer', action='primer')
        m.connect('/testing/markup', action='markup')

    # Mark all unmarked routes added up until now as core routes
    for route in map.matchlist:
        if not hasattr(route, '_ckan_core'):
            route._ckan_core = True

    for plugin in p.PluginImplementations(p.IRoutes):
        map = plugin.after_map(map)

    # Mark all routes added from extensions on the `after_map` extension point
    # as non-core
    for route in map.matchlist:
        if not hasattr(route, '_ckan_core'):
            route._ckan_core = False

    # sometimes we get requests for favicon.ico we should redirect to
    # the real favicon location.
    map.redirect('/favicon.ico', config.get('ckan.favicon'))

    map.redirect('/*(url)/', '/{url}', _redirect_code='301 Moved Permanently')
    map.connect('/*url', controller='template', action='view', ckan_core=True)

    return map
Exemple #41
0
def make_flask_stack(conf):
    """ This has to pass the flask app through all the same middleware that
    Pylons used """

    root = os.path.dirname(
        os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

    debug = asbool(conf.get('debug', conf.get('DEBUG', False)))
    testing = asbool(conf.get('testing', conf.get('TESTING', False)))
    app = flask_app = CKANFlask(__name__, static_url_path='')

    # Register storage for accessing group images, site logo, etc.
    storage_folder = []
    storage = uploader.get_storage_path()
    if storage:
        storage_folder = [os.path.join(storage, 'storage')]

    # Static files folders (core and extensions)
    public_folder = config.get(u'ckan.base_public_folder')
    app.static_folder = config.get('extra_public_paths', '').split(',') + [
        os.path.join(root, public_folder)
    ] + storage_folder

    app.jinja_options = jinja_extensions.get_jinja_env_options()
    app.jinja_env.policies['ext.i18n.trimmed'] = True

    app.debug = debug
    app.testing = testing
    app.template_folder = os.path.join(root, 'templates')
    app.app_ctx_globals_class = CKAN_AppCtxGlobals
    app.url_rule_class = CKAN_Rule

    # Update Flask config with the CKAN values. We use the common config
    # object as values might have been modified on `load_environment`
    if config:
        app.config.update(config)
    else:
        app.config.update(conf)

    # Do all the Flask-specific stuff before adding other middlewares

    # Secret key needed for flask-debug-toolbar and sessions
    if not app.config.get('SECRET_KEY'):
        app.config['SECRET_KEY'] = config.get('beaker.session.secret')
    if not app.config.get('SECRET_KEY'):
        raise RuntimeError(u'You must provide a value for the secret key'
                           ' with the SECRET_KEY config option')

    root_path = config.get('ckan.root_path', None)
    if debug:
        from flask_debugtoolbar import DebugToolbarExtension
        app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
        debug_ext = DebugToolbarExtension()

        # register path that includes `ckan.site_root` before
        # initializing debug app. In such a way, our route receives
        # higher precedence.

        # TODO: After removal of Pylons code, switch to
        # `APPLICATION_ROOT` config value for flask application. Right
        # now it's a bad option because we are handling both pylons
        # and flask urls inside helpers and splitting this logic will
        # bring us tons of headache.
        if root_path:
            app.add_url_rule(
                root_path.replace('{{LANG}}', '').rstrip('/') +
                '/_debug_toolbar/static/<path:filename>',
                '_debug_toolbar.static', debug_ext.send_static_file)
        debug_ext.init_app(app)

        from werkzeug.debug import DebuggedApplication
        app.wsgi_app = DebuggedApplication(app.wsgi_app, True)

    # Use Beaker as the Flask session interface
    class BeakerSessionInterface(SessionInterface):
        def open_session(self, app, request):
            if 'beaker.session' in request.environ:
                return request.environ['beaker.session']

        def save_session(self, app, session, response):
            session.save()

    namespace = 'beaker.session.'
    session_opts = {
        k.replace('beaker.', ''): v
        for k, v in config.items() if k.startswith(namespace)
    }
    if (not session_opts.get('session.data_dir')
            and session_opts.get('session.type', 'file') == 'file'):
        cache_dir = conf.get('cache_dir') or conf.get('cache.dir')
        session_opts['session.data_dir'] = '{data_dir}/sessions'.format(
            data_dir=cache_dir)

    app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts)
    app.session_interface = BeakerSessionInterface()

    # Add Jinja2 extensions and filters
    app.jinja_env.filters['empty_and_escape'] = \
        jinja_extensions.empty_and_escape

    # Common handlers for all requests
    app.before_request(ckan_before_request)
    app.after_request(ckan_after_request)

    # Template context processors
    app.context_processor(helper_functions)
    app.context_processor(c_object)

    @app.context_processor
    def ungettext_alias():
        u'''
        Provide `ungettext` as an alias of `ngettext` for backwards
        compatibility
        '''
        return dict(ungettext=ungettext)

    # Babel
    _ckan_i18n_dir = i18n.get_ckan_i18n_dir()

    pairs = [(_ckan_i18n_dir, u'ckan')
             ] + [(p.i18n_directory(), p.i18n_domain())
                  for p in reversed(list(PluginImplementations(ITranslation)))]

    i18n_dirs, i18n_domains = zip(*pairs)

    app.config[u'BABEL_TRANSLATION_DIRECTORIES'] = ';'.join(i18n_dirs)
    app.config[u'BABEL_DOMAIN'] = 'ckan'
    app.config[u'BABEL_MULTIPLE_DOMAINS'] = ';'.join(i18n_domains)
    app.config[u'BABEL_DEFAULT_TIMEZONE'] = str(helpers.get_display_timezone())

    babel = CKANBabel(app)

    babel.localeselector(get_locale)

    # WebAssets
    _setup_webassets(app)

    # Auto-register all blueprints defined in the `views` folder
    _register_core_blueprints(app)
    _register_error_handler(app)

    # Set up each IBlueprint extension as a Flask Blueprint
    for plugin in PluginImplementations(IBlueprint):
        if hasattr(plugin, 'get_blueprint'):
            plugin_blueprints = plugin.get_blueprint()
            if not isinstance(plugin_blueprints, list):
                plugin_blueprints = [plugin_blueprints]
            for blueprint in plugin_blueprints:
                app.register_extension_blueprint(blueprint)

    lib_plugins.register_package_blueprints(app)
    lib_plugins.register_group_blueprints(app)

    # Start other middleware
    for plugin in PluginImplementations(IMiddleware):
        app = plugin.make_middleware(app, config)

    for plugin in PluginImplementations(IMiddleware):
        try:
            app = plugin.make_error_log_middleware(app, config)
        except AttributeError:
            log.critical('Middleware class {0} is missing the method'
                         'make_error_log_middleware.'.format(
                             plugin.__class__.__name__))

    # Initialize repoze.who
    who_parser = WhoConfig(conf['here'])
    who_parser.parse(open(conf['who.config_file']))

    app = PluggableAuthenticationMiddleware(
        RepozeAdapterMiddleware(app),
        who_parser.identifiers,
        who_parser.authenticators,
        who_parser.challengers,
        who_parser.mdproviders,
        who_parser.request_classifier,
        who_parser.challenge_decider,
        logging.getLogger('repoze.who'),
        logging.WARN,  # ignored
        who_parser.remote_user_key)

    # Update the main CKAN config object with the Flask specific keys
    # that were set here or autogenerated
    flask_config_keys = set(flask_app.config.keys()) - set(config.keys())
    for key in flask_config_keys:
        config[key] = flask_app.config[key]

    # Prevent the host from request to be added to the new header location.
    app = HostHeaderMiddleware(app)

    app = I18nMiddleware(app)

    if asbool(config.get('ckan.tracking_enabled', 'false')):
        app = TrackingMiddleware(app, config)

    # Add a reference to the actual Flask app so it's easier to access
    app._wsgi_app = flask_app

    return app
Exemple #42
0
def update_config():
    ''' This code needs to be run when the config is changed to take those
    changes into account. It is called whenever a plugin is loaded as the
    plugin might have changed the config values (for instance it might
    change ckan.site_url) '''

    for plugin in p.PluginImplementations(p.IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # Set whitelisted env vars on config object
    # This is set up before globals are initialized

    ckan_db = os.environ.get('CKAN_DB', None)
    if ckan_db:
        msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
            ' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
        log.warn(msg)
        config['sqlalchemy.url'] = ckan_db

    for option in CONFIG_FROM_ENV_VARS:
        from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
        if from_env:
            config[option] = from_env

    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

    site_url = config.get('ckan.site_url', '')
    if not site_url:
        raise RuntimeError(
            'ckan.site_url is not configured and it must have a value.'
            ' Please amend your .ini file.')
    if not site_url.lower().startswith('http'):
        raise RuntimeError(
            'ckan.site_url should be a full URL, including the schema '
            '(http or https)')

    display_timezone = config.get('ckan.display_timezone', '')
    if (display_timezone and display_timezone != 'server'
            and display_timezone not in pytz.all_timezones):
        raise CkanConfigurationException(
            "ckan.display_timezone is not 'server' or a valid timezone")

    # Remove backslash from site_url if present
    config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')

    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # ensure that a favicon has been set
    favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
    config['ckan.favicon'] = favicon

    # Init SOLR settings and check if the schema is compatible
    # from ckan.lib.search import SolrSettings, check_solr_schema_version

    # lib.search is imported here as we need the config enabled and parsed
    search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
                             config.get('solr_password'))
    search.check_solr_schema_version()

    routes_map = routing.make_map()
    config['routes.map'] = routes_map
    # The RoutesMiddleware needs its mapper updating if it exists
    if 'routes.middleware' in config:
        config['routes.middleware'].mapper = routes_map
    # routes.named_routes is a CKAN thing
    config['routes.named_routes'] = routing.named_routes
    config['pylons.app_globals'] = app_globals.app_globals
    # initialise the globals
    app_globals.app_globals._init()

    helpers.load_plugin_helpers()
    config['pylons.h'] = helpers.helper_functions

    jinja2_templates_path = os.path.join(root, 'templates')
    template_paths = [jinja2_templates_path]

    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths
    config['pylons.app_globals'].template_paths = template_paths

    # Set the default language for validation messages from formencode
    # to what is set as the default locale in the config
    default_lang = config.get('ckan.locale_default', 'en')
    formencode.api.set_stdtranslation(domain="FormEncode",
                                      languages=[default_lang])

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create Jinja2 environment
    env = jinja_extensions.Environment(
        loader=jinja_extensions.CkanFileSystemLoader(template_paths),
        autoescape=True,
        extensions=[
            'jinja2.ext.do', 'jinja2.ext.with_',
            jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend,
            jinja_extensions.CkanInternationalizationExtension,
            jinja_extensions.LinkForExtension,
            jinja_extensions.ResourceExtension,
            jinja_extensions.UrlForStaticExtension,
            jinja_extensions.UrlForExtension
        ])
    env.install_gettext_callables(_, ungettext, newstyle=True)
    # custom filters
    env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
    env.filters['truncate'] = jinja_extensions.truncate
    config['pylons.app_globals'].jinja_env = env

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # for postgresql we want to enforce utf-8
    sqlalchemy_url = config.get('sqlalchemy.url', '')
    if sqlalchemy_url.startswith('postgresql://'):
        extras = {'client_encoding': 'utf8'}
    else:
        extras = {}

    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)

    if not model.meta.engine:
        model.init_model(engine)

    for plugin in p.PluginImplementations(p.IConfigurable):
        plugin.configure(config)

    # reset the template cache - we do this here so that when we load the
    # environment it is clean
    render.reset_template_info_cache()

    # clear other caches
    logic.clear_actions_cache()
    logic.clear_validators_cache()
    authz.clear_auth_functions_cache()

    # Here we create the site user if they are not already in the database
    try:
        logic.get_action('get_site_user')({'ignore_auth': True}, None)
    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
        # (ProgrammingError for Postgres, OperationalError for SQLite)
        # The database is not initialised.  This is a bit dirty.  This occurs
        # when running tests.
        pass
    except sqlalchemy.exc.InternalError:
        # The database is not initialised.  Travis hits this
        pass
    # if an extension or our code does not finish
    # transaction properly db cli commands can fail
    model.Session.remove()
Exemple #43
0
def get_minimum_views_for_trending():
    return int(config.get('ckanext.lacounts.trending_min') or '10') or 10
Exemple #44
0
def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf):
    """Create a Pylons WSGI application and return it

    ``conf``
        The inherited configuration for this application. Normally from
        the [DEFAULT] section of the Paste ini file.

    ``full_stack``
        Whether this application provides a full WSGI stack (by default,
        meaning it handles its own exceptions and errors). Disable
        full_stack when this application is "managed" by another WSGI
        middleware.

    ``static_files``
        Whether this application serves its own static files; disable
        when another web server is responsible for serving them.

    ``app_conf``
        The application's local configuration. Normally specified in
        the [app:<name>] section of the Paste ini file (where <name>
        defaults to main).

    """
    # The Pylons WSGI app
    app = pylons_app = CKANPylonsApp()

    for plugin in PluginImplementations(IMiddleware):
        app = plugin.make_middleware(app, config)

    app = common_middleware.CloseWSGIInputMiddleware(app, config)
    app = common_middleware.RootPathMiddleware(app, config)
    # Routing/Session/Cache Middleware
    app = RoutesMiddleware(app, config['routes.map'])
    # we want to be able to retrieve the routes middleware to be able to update
    # the mapper.  We store it in the pylons config to allow this.
    config['routes.middleware'] = app
    app = SessionMiddleware(app, config)
    app = CacheMiddleware(app, config)

    # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
    # app = QueueLogMiddleware(app)
    if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)):
        app = execute_on_completion(app, config,
                                    cleanup_pylons_response_string)

    # Fanstatic
    if asbool(config.get('debug', False)):
        fanstatic_config = {
            'versioning': True,
            'recompute_hashes': True,
            'minified': False,
            'bottom': True,
            'bundle': False,
        }
    else:
        fanstatic_config = {
            'versioning': True,
            'recompute_hashes': False,
            'minified': True,
            'bottom': True,
            'bundle': True,
        }
    root_path = config.get('ckan.root_path', None)
    if root_path:
        root_path = re.sub('/{{LANG}}', '', root_path)
        fanstatic_config['base_url'] = root_path
    app = Fanstatic(app, **fanstatic_config)

    for plugin in PluginImplementations(IMiddleware):
        try:
            app = plugin.make_error_log_middleware(app, config)
        except AttributeError:
            log.critical('Middleware class {0} is missing the method'
                         'make_error_log_middleware.'.format(
                             plugin.__class__.__name__))

    if asbool(full_stack):
        # Handle Python exceptions
        app = ErrorHandler(app, conf, **config['pylons.errorware'])

        # Display error documents for 400, 403, 404 status codes (and
        # 500 when debug is disabled)
        if asbool(config['debug']):
            app = StatusCodeRedirect(app, [400, 403, 404])
        else:
            app = StatusCodeRedirect(app, [400, 403, 404, 500])

    # Initialize repoze.who
    who_parser = WhoConfig(conf['here'])
    who_parser.parse(open(app_conf['who.config_file']))

    app = PluggableAuthenticationMiddleware(
        app,
        who_parser.identifiers,
        who_parser.authenticators,
        who_parser.challengers,
        who_parser.mdproviders,
        who_parser.request_classifier,
        who_parser.challenge_decider,
        logging.getLogger('repoze.who'),
        logging.WARN,  # ignored
        who_parser.remote_user_key)

    # Establish the Registry for this application
    # The RegistryManager includes code to pop
    # registry values after the stream has completed,
    # so we need to prevent this with `streaming` set to True.
    app = RegistryManager(app, streaming=True)

    if asbool(static_files):
        # Serve static files
        static_max_age = None if not asbool(
            config.get('ckan.cache_enabled')) \
            else int(config.get('ckan.static_max_age', 3600))

        static_app = StaticURLParser(config['pylons.paths']['static_files'],
                                     cache_max_age=static_max_age)
        static_parsers = [static_app, app]

        storage_directory = uploader.get_storage_path()
        if storage_directory:
            path = os.path.join(storage_directory, 'storage')
            try:
                os.makedirs(path)
            except OSError as e:
                # errno 17 is file already exists
                if e.errno != 17:
                    raise

            storage_app = StaticURLParser(path, cache_max_age=static_max_age)
            static_parsers.insert(0, storage_app)

        # Configurable extra static file paths
        extra_static_parsers = []
        for public_path in config.get('extra_public_paths', '').split(','):
            if public_path.strip():
                extra_static_parsers.append(
                    StaticURLParser(public_path.strip(),
                                    cache_max_age=static_max_age))
        app = Cascade(extra_static_parsers + static_parsers)

    # Tracking
    if asbool(config.get('ckan.tracking_enabled', 'false')):
        app = common_middleware.TrackingMiddleware(app, config)

    # Add a reference to the actual Pylons app so it's easier to access
    app._wsgi_app = pylons_app

    return app
Exemple #45
0
def datapusher_status(context, data_dict):
    ''' Get the status of a datapusher job for a certain resource.

    :param resource_id: The resource id of the resource that you want the
        datapusher status for.
    :type resource_id: string
    '''

    p.toolkit.check_access('datapusher_status', context, data_dict)

    if 'id' in data_dict:
        data_dict['resource_id'] = data_dict['id']
    res_id = _get_or_bust(data_dict, 'resource_id')

    task = p.toolkit.get_action('task_status_show')(context, {
        'entity_id': res_id,
        'task_type': 'datapusher',
        'key': 'datapusher'
    })

    datapusher_url = config.get('ckan.datapusher.url')
    if not datapusher_url:
        raise p.toolkit.ValidationError(
            {'configuration': ['ckan.datapusher.url not in config file']})

    value = json.loads(task['value'])
    job_key = value.get('job_key')
    job_id = value.get('job_id')
    url = None
    job_detail = None

    if job_id:
        url = urlparse.urljoin(datapusher_url, 'job' + '/' + job_id)
        try:
            r = requests.get(url,
                             headers={
                                 'Content-Type': 'application/json',
                                 'Authorization': job_key
                             })
            r.raise_for_status()
            job_detail = r.json()
            for log in job_detail['logs']:
                if 'timestamp' in log:
                    date = time.strptime(log['timestamp'],
                                         "%Y-%m-%dT%H:%M:%S.%f")
                    date = datetime.datetime.utcfromtimestamp(
                        time.mktime(date))
                    log['timestamp'] = date
        except (requests.exceptions.ConnectionError,
                requests.exceptions.HTTPError):
            job_detail = {'error': 'cannot connect to datapusher'}

    return {
        'status': task['state'],
        'job_id': job_id,
        'job_url': url,
        'last_updated': task['last_updated'],
        'job_key': job_key,
        'task_info': job_detail,
        'error': json.loads(task['error'])
    }
Exemple #46
0
class FeedController(base.BaseController):
    base_url = config.get('ckan.site_url')

    def _alternate_url(self, params, **kwargs):
        search_params = params.copy()
        search_params.update(kwargs)

        # Can't count on the page sizes being the same on the search results
        # view.  So provide an alternate link to the first page, regardless
        # of the page we're looking at in the feed.
        search_params.pop('page', None)
        return self._feed_url(search_params,
                              controller='package',
                              action='search')

    def _group_or_organization(self, obj_dict, is_org):

        data_dict, params = self._parse_url_params()
        if is_org:
            key = 'owner_org'
            value = obj_dict['id']
            group_type = 'organization'
        else:
            key = 'groups'
            value = obj_dict['name']
            group_type = 'group'

        data_dict['fq'] = '{0}:"{1}"'.format(key, value)

        item_count, results = _package_search(data_dict)

        navigation_urls = self._navigation_urls(params,
                                                item_count=item_count,
                                                limit=data_dict['rows'],
                                                controller='feed',
                                                action=group_type,
                                                id=obj_dict['name'])
        feed_url = self._feed_url(params,
                                  controller='feed',
                                  action=group_type,
                                  id=obj_dict['name'])

        site_title = config.get('ckan.site_title', 'CKAN')
        if is_org:
            guid = _create_atom_id(u'/feeds/organization/%s.atom' %
                                   obj_dict['name'])
            alternate_url = self._alternate_url(params,
                                                organization=obj_dict['name'])
            desc = u'Recently created or updated datasets on %s '\
                'by organization: "%s"' % (site_title, obj_dict['title'])
            title = u'%s - Organization: "%s"' % (site_title,
                                                  obj_dict['title'])

        else:  # is group
            guid = _create_atom_id(u'/feeds/group/%s.atom' %
                                   obj_dict['name'])
            alternate_url = self._alternate_url(params,
                                                groups=obj_dict['name'])
            desc = u'Recently created or updated datasets on %s '\
                'by group: "%s"' % (site_title, obj_dict['title'])
            title = u'%s - Group: "%s"' %\
                (site_title, obj_dict['title'])

        return self.output_feed(results,
                                feed_title=title,
                                feed_description=desc,
                                feed_link=alternate_url,
                                feed_guid=guid,
                                feed_url=feed_url,
                                navigation_urls=navigation_urls)

    def group(self, id):
        try:
            context = {'model': model, 'session': model.Session,
                       'user': c.user, 'auth_user_obj': c.userobj}
            group_dict = logic.get_action('group_show')(context, {'id': id})
        except logic.NotFound:
            base.abort(404, _('Group not found'))

        return self._group_or_organization(group_dict, is_org=False)

    def organization(self, id):
        try:
            context = {'model': model, 'session': model.Session,
                       'user': c.user, 'auth_user_obj': c.userobj}
            group_dict = logic.get_action('organization_show')(context,
                                                               {'id': id})
        except logic.NotFound:
            base.abort(404, _('Organization not found'))

        return self._group_or_organization(group_dict, is_org=True)

    def tag(self, id):
        data_dict, params = self._parse_url_params()
        data_dict['fq'] = 'tags:"%s"' % id

        item_count, results = _package_search(data_dict)

        navigation_urls = self._navigation_urls(params,
                                                item_count=item_count,
                                                limit=data_dict['rows'],
                                                controller='feed',
                                                action='tag',
                                                id=id)

        feed_url = self._feed_url(params,
                                  controller='feed',
                                  action='tag',
                                  id=id)

        alternate_url = self._alternate_url(params, tags=id)

        site_title = config.get('ckan.site_title', 'CKAN')

        return self.output_feed(results,
                                feed_title=u'%s - Tag: "%s"' %
                                (site_title, id),
                                feed_description=u'Recently created or '
                                'updated datasets on %s by tag: "%s"' %
                                (site_title, id),
                                feed_link=alternate_url,
                                feed_guid=_create_atom_id
                                (u'/feeds/tag/%s.atom' % id),
                                feed_url=feed_url,
                                navigation_urls=navigation_urls)

    def general(self):
        data_dict, params = self._parse_url_params()
        data_dict['q'] = '*:*'

        item_count, results = _package_search(data_dict)

        navigation_urls = self._navigation_urls(params,
                                                item_count=item_count,
                                                limit=data_dict['rows'],
                                                controller='feed',
                                                action='general')

        feed_url = self._feed_url(params,
                                  controller='feed',
                                  action='general')

        alternate_url = self._alternate_url(params)

        site_title = config.get('ckan.site_title', 'CKAN')

        return self.output_feed(results,
                                feed_title=site_title,
                                feed_description=u'Recently created or '
                                'updated datasets on %s' % site_title,
                                feed_link=alternate_url,
                                feed_guid=_create_atom_id
                                (u'/feeds/dataset.atom'),
                                feed_url=feed_url,
                                navigation_urls=navigation_urls)

    # TODO check search params
    def custom(self):
        q = request.params.get('q', u'')
        fq = ''
        search_params = {}
        for (param, value) in request.params.items():
            if param not in ['q', 'page', 'sort'] \
                    and len(value) and not param.startswith('_'):
                search_params[param] = value
                fq += ' %s:"%s"' % (param, value)

        page = h.get_page_number(request.params)

        limit = ITEMS_LIMIT
        data_dict = {
            'q': q,
            'fq': fq,
            'start': (page - 1) * limit,
            'rows': limit,
            'sort': request.params.get('sort', None),
        }

        item_count, results = _package_search(data_dict)

        navigation_urls = self._navigation_urls(request.params,
                                                item_count=item_count,
                                                limit=data_dict['rows'],
                                                controller='feed',
                                                action='custom')

        feed_url = self._feed_url(request.params,
                                  controller='feed',
                                  action='custom')

        atom_url = h._url_with_params('/feeds/custom.atom',
                                      search_params.items())

        alternate_url = self._alternate_url(request.params)

        site_title = config.get('ckan.site_title', 'CKAN')

        return self.output_feed(results,
                                feed_title=u'%s - Custom query' % site_title,
                                feed_description=u'Recently created or updated'
                                ' datasets on %s. Custom query: \'%s\'' %
                                (site_title, q),
                                feed_link=alternate_url,
                                feed_guid=_create_atom_id(atom_url),
                                feed_url=feed_url,
                                navigation_urls=navigation_urls)

    def output_feed(self, results, feed_title, feed_description,
                    feed_link, feed_url, navigation_urls, feed_guid):
        author_name = config.get('ckan.feeds.author_name', '').strip() or \
            config.get('ckan.site_id', '').strip()
        author_link = config.get('ckan.feeds.author_link', '').strip() or \
            config.get('ckan.site_url', '').strip()

        # TODO language
        feed_class = None
        for plugin in plugins.PluginImplementations(plugins.IFeed):
            if hasattr(plugin, 'get_feed_class'):
                feed_class = plugin.get_feed_class()

        if not feed_class:
            feed_class = _FixedAtom1Feed

        feed = feed_class(
            feed_title,
            feed_link,
            feed_description,
            language=u'en',
            author_name=author_name,
            author_link=author_link,
            feed_guid=feed_guid,
            feed_url=feed_url,
            previous_page=navigation_urls['previous'],
            next_page=navigation_urls['next'],
            first_page=navigation_urls['first'],
            last_page=navigation_urls['last'],
        )

        for pkg in results:
            additional_fields = {}

            for plugin in plugins.PluginImplementations(plugins.IFeed):
                if hasattr(plugin, 'get_item_additional_fields'):
                    additional_fields = plugin.get_item_additional_fields(pkg)

            feed.add_item(
                title=pkg.get('title', ''),
                link=self.base_url + h.url_for('dataset.read',
                                               id=pkg['id']),
                description=pkg.get('notes', ''),
                updated=h.date_str_to_datetime(pkg.get('metadata_modified')),
                published=h.date_str_to_datetime(pkg.get('metadata_created')),
                unique_id=_create_atom_id(u'/dataset/%s' % pkg['id']),
                author_name=pkg.get('author', ''),
                author_email=pkg.get('author_email', ''),
                categories=[t['name'] for t in pkg.get('tags', [])],
                enclosure=webhelpers.feedgenerator.Enclosure(
                    h.url_for(controller='api',
                              register='package',
                              action='show',
                              id=pkg['name'],
                              ver='3',
                              qualified=True),
                    text_type(len(json.dumps(pkg))),   # TODO fix this
                    u'application/json'),
                **additional_fields
            )
        response.content_type = feed.mime_type
        return feed.writeString('utf-8')

    # CLASS PRIVATE METHODS #

    def _feed_url(self, query, controller, action, **kwargs):
        """
        Constructs the url for the given action.  Encoding the query
        parameters.
        """
        path = h.url_for(controller=controller, action=action, **kwargs)
        return h._url_with_params(self.base_url + path, query.items())

    def _navigation_urls(self, query, controller, action,
                         item_count, limit, **kwargs):
        """
        Constructs and returns first, last, prev and next links for paging
        """
        urls = dict((rel, None) for rel in 'previous next first last'.split())

        page = int(query.get('page', 1))

        # first: remove any page parameter
        first_query = query.copy()
        first_query.pop('page', None)
        urls['first'] = self._feed_url(first_query, controller,
                                       action, **kwargs)

        # last: add last page parameter
        last_page = (item_count / limit) + min(1, item_count % limit)
        last_query = query.copy()
        last_query['page'] = last_page
        urls['last'] = self._feed_url(last_query, controller,
                                      action, **kwargs)

        # previous
        if page > 1:
            previous_query = query.copy()
            previous_query['page'] = page - 1
            urls['previous'] = self._feed_url(previous_query, controller,
                                              action, **kwargs)
        else:
            urls['previous'] = None

        # next
        if page < last_page:
            next_query = query.copy()
            next_query['page'] = page + 1
            urls['next'] = self._feed_url(next_query, controller,
                                          action, **kwargs)
        else:
            urls['next'] = None

        return urls

    def _parse_url_params(self):
        """
        Constructs a search-query dict from the URL query parameters.

        Returns the constructed search-query dict, and the valid URL
        query parameters.
        """
        page = h.get_page_number(request.params)

        limit = ITEMS_LIMIT
        data_dict = {
            'start': (page - 1) * limit,
            'rows': limit
        }

        # Filter ignored query parameters
        valid_params = ['page']
        params = dict((p, request.params.get(p)) for p in valid_params
                      if p in request.params)
        return data_dict, params
Exemple #47
0
def me():
    return h.redirect_to(
        config.get(u'ckan.route_after_login', u'dashboard.index'))
Exemple #48
0
    def output_feed(self, results, feed_title, feed_description,
                    feed_link, feed_url, navigation_urls, feed_guid):
        author_name = config.get('ckan.feeds.author_name', '').strip() or \
            config.get('ckan.site_id', '').strip()
        author_link = config.get('ckan.feeds.author_link', '').strip() or \
            config.get('ckan.site_url', '').strip()

        # TODO language
        feed_class = None
        for plugin in plugins.PluginImplementations(plugins.IFeed):
            if hasattr(plugin, 'get_feed_class'):
                feed_class = plugin.get_feed_class()

        if not feed_class:
            feed_class = _FixedAtom1Feed

        feed = feed_class(
            feed_title,
            feed_link,
            feed_description,
            language=u'en',
            author_name=author_name,
            author_link=author_link,
            feed_guid=feed_guid,
            feed_url=feed_url,
            previous_page=navigation_urls['previous'],
            next_page=navigation_urls['next'],
            first_page=navigation_urls['first'],
            last_page=navigation_urls['last'],
        )

        for pkg in results:
            additional_fields = {}

            for plugin in plugins.PluginImplementations(plugins.IFeed):
                if hasattr(plugin, 'get_item_additional_fields'):
                    additional_fields = plugin.get_item_additional_fields(pkg)

            feed.add_item(
                title=pkg.get('title', ''),
                link=self.base_url + h.url_for('dataset.read',
                                               id=pkg['id']),
                description=pkg.get('notes', ''),
                updated=h.date_str_to_datetime(pkg.get('metadata_modified')),
                published=h.date_str_to_datetime(pkg.get('metadata_created')),
                unique_id=_create_atom_id(u'/dataset/%s' % pkg['id']),
                author_name=pkg.get('author', ''),
                author_email=pkg.get('author_email', ''),
                categories=[t['name'] for t in pkg.get('tags', [])],
                enclosure=webhelpers.feedgenerator.Enclosure(
                    h.url_for(controller='api',
                              register='package',
                              action='show',
                              id=pkg['name'],
                              ver='3',
                              qualified=True),
                    text_type(len(json.dumps(pkg))),   # TODO fix this
                    u'application/json'),
                **additional_fields
            )
        response.content_type = feed.mime_type
        return feed.writeString('utf-8')
    def show_datalineage(self, id):
        """
        Retrieves data lineage information for a specific package
        """
        reinitialize_results()
        context = {
            'model': model,
            'session': model.Session,
            'user': c.user,
            'for_view': True,
            'auth_user_obj': c.userobj
        }
        data_dict = {'id': id}
        extra_vars = {}
        try:
            c.pkg_dict = get_action('package_show')(context, data_dict)
            c.pkg = context['package']

            # this is a DS
            if c.pkg_dict.get('parent'):
                q = 'extras_code:%s' % (c.pkg_dict.get('parent').replace(
                    ':', '\:'))
            else:
                # this is a process/activity/model
                q = 'extras_parent:%s' % (c.pkg_dict.get('code').replace(
                    ':', '\:'))

            search_extras = {}
            data_dict = {
                'q':
                q,
                'fq':
                q,
                'extras':
                search_extras,
                'include_private':
                asbool(config.get('ckan.search.default_include_private',
                                  True)),
            }
            query = get_action('package_search')(context, data_dict)

            results = query['results'][0] if query['results'] else {}
            if c.pkg_dict.get('parent'):
                extra_vars['datalineage_wasgeneratedby'] = results
            else:
                extra_vars['datalineage_generates'] = results

            usage_models = get_usage_models(
                context, c.pkg_dict if c.pkg_dict.get('parent') else results)
            usage_datasets = get_usage_datasets(context, usage_models)
            extra_vars['usage_models'] = usage_models
            extra_vars['usage_datasets'] = usage_datasets
            extra_vars['detail_data'] = c.pkg_dict if c.pkg_dict.get(
                'parent') else results
            extra_vars['current_model'] = results if c.pkg_dict.get(
                'parent') else c.pkg_dict

            # get the producers DSs
            producers = c.pkg_dict.get('producers', '') or results.get(
                'producers', '')
            producers_info = []
            if producers:
                for ds_code in producers.split(','):
                    q = 'extras_code:%s' % (ds_code.replace(':', '\:'))
                    data_dict = {
                        'q':
                        q,
                        'fq':
                        q,
                        'extras': {},
                        'include_private':
                        asbool(
                            config.get('ckan.search.default_include_private',
                                       True)),
                    }
                    query = get_action('package_search')(context, data_dict)
                    if query['count'] == 0:
                        logger.warning(
                            'No result found for producer [%s] of package [%s]'
                            % (ds_code, c.pkg_dict['code']))
                    else:
                        producers_info.append(query['results'][0])

            extra_vars['datalineage_producers'] = producers_info
            convert_extra_vars_to_metaviz(extra_vars)

            dataset_type = c.pkg_dict['type'] or 'dataset'
            extra_vars['dataset_type'] = dataset_type
        except NotFound:
            abort(404, _('Dataset not found'))
        except NotAuthorized:
            abort(403, _('Unauthorized to read dataset %s') % id)

        return render(
            'package/datalineage.html',
            # extra_vars = {'data': json.dumps(extra_vars)},
            extra_vars={'data': json.dumps(RESULTS)})
Exemple #50
0
def search(package_type):
    extra_vars = {}

    try:
        context = {
            u'model': model,
            u'user': g.user,
            u'auth_user_obj': g.userobj
        }
        check_access(u'site_read', context)
    except NotAuthorized:
        base.abort(403, _(u'Not authorized to see this page'))

    # unicode format (decoded from utf8)
    extra_vars[u'q'] = q = request.args.get(u'q', u'')

    extra_vars['query_error'] = False
    page = h.get_page_number(request.args)

    limit = int(config.get(u'ckan.datasets_per_page', 20))

    # most search operations should reset the page counter:
    params_nopage = [(k, v) for k, v in request.args.items() if k != u'page']

    extra_vars[u'drill_down_url'] = drill_down_url
    extra_vars[u'remove_field'] = partial(remove_field, package_type)

    sort_by = request.args.get(u'sort', None)
    params_nosort = [(k, v) for k, v in params_nopage if k != u'sort']

    extra_vars[u'sort_by'] = partial(_sort_by, params_nosort, package_type)

    if not sort_by:
        sort_by_fields = []
    else:
        sort_by_fields = [field.split()[0] for field in sort_by.split(u',')]
    extra_vars[u'sort_by_fields'] = sort_by_fields

    pager_url = partial(_pager_url, params_nopage, package_type)

    search_url_params = urlencode(_encode_params(params_nopage))
    extra_vars[u'search_url_params'] = search_url_params

    try:
        # fields_grouped will contain a dict of params containing
        # a list of values eg {u'tags':[u'tag1', u'tag2']}

        extra_vars[u'fields'] = fields = []
        extra_vars[u'fields_grouped'] = fields_grouped = {}
        search_extras = {}
        fq = u''
        for (param, value) in request.args.items(multi=True):
            if param not in [u'q', u'page', u'sort'] \
                    and len(value) and not param.startswith(u'_'):
                if not param.startswith(u'ext_'):
                    fields.append((param, value))
                    fq += u' %s:"%s"' % (param, value)
                    if param not in fields_grouped:
                        fields_grouped[param] = [value]
                    else:
                        fields_grouped[param].append(value)
                else:
                    search_extras[param] = value

        context = {
            u'model': model,
            u'session': model.Session,
            u'user': g.user,
            u'for_view': True,
            u'auth_user_obj': g.userobj
        }

        # Unless changed via config options, don't show other dataset
        # types any search page. Potential alternatives are do show them
        # on the default search page (dataset) or on one other search page
        search_all_type = config.get(u'ckan.search.show_all_types', u'dataset')
        search_all = False

        try:
            # If the "type" is set to True or False, convert to bool
            # and we know that no type was specified, so use traditional
            # behaviour of applying this only to dataset type
            search_all = asbool(search_all_type)
            search_all_type = u'dataset'
        # Otherwise we treat as a string representing a type
        except ValueError:
            search_all = True

        if not search_all or package_type != search_all_type:
            # Only show datasets of this particular type
            fq += u' +dataset_type:{type}'.format(type=package_type)

        facets = OrderedDict()

        default_facet_titles = {
            u'organization': _(u'Organizations'),
            u'groups': _(u'Groups'),
            u'tags': _(u'Tags'),
            u'res_format': _(u'Formats'),
            u'license_id': _(u'Licenses'),
        }

        for facet in h.facets():
            if facet in default_facet_titles:
                facets[facet] = default_facet_titles[facet]
            else:
                facets[facet] = facet

        # Facet titles
        for plugin in plugins.PluginImplementations(plugins.IFacets):
            facets = plugin.dataset_facets(facets, package_type)

        extra_vars[u'facet_titles'] = facets
        data_dict = {
            u'q': q,
            u'fq': fq.strip(),
            u'facet.field': facets.keys(),
            u'rows': limit,
            u'start': (page - 1) * limit,
            u'sort': sort_by,
            u'extras': search_extras,
            u'include_private': asbool(
                config.get(u'ckan.search.default_include_private', True)
            ),
        }

        query = get_action(u'package_search')(context, data_dict)

        extra_vars[u'sort_by_selected'] = query[u'sort']

        extra_vars[u'page'] = h.Page(
            collection=query[u'results'],
            page=page,
            url=pager_url,
            item_count=query[u'count'],
            items_per_page=limit
        )
        extra_vars[u'search_facets'] = query[u'search_facets']
        extra_vars[u'page'].items = query[u'results']
    except SearchQueryError as se:
        # User's search parameters are invalid, in such a way that is not
        # achievable with the web interface, so return a proper error to
        # discourage spiders which are the main cause of this.
        log.info(u'Dataset search query rejected: %r', se.args)
        base.abort(
            400,
            _(u'Invalid search query: {error_message}')
            .format(error_message=str(se))
        )
    except SearchError as se:
        # May be bad input from the user, but may also be more serious like
        # bad code causing a SOLR syntax error, or a problem connecting to
        # SOLR
        log.error(u'Dataset search error: %r', se.args)
        extra_vars[u'query_error'] = True
        extra_vars[u'search_facets'] = {}
        extra_vars[u'page'] = h.Page(collection=[])

    # FIXME: try to avoid using global variables
    g.search_facets_limits = {}
    for facet in extra_vars[u'search_facets'].keys():
        try:
            limit = int(
                request.args.get(
                    u'_%s_limit' % facet,
                    int(config.get(u'search.facets.default', 10))
                )
            )
        except ValueError:
            base.abort(
                400,
                _(u'Parameter u"{parameter_name}" is not '
                  u'an integer').format(parameter_name=u'_%s_limit' % facet)
            )

        g.search_facets_limits[facet] = limit

    _setup_template_variables(context, {}, package_type=package_type)

    extra_vars[u'dataset_type'] = package_type

    # TODO: remove
    for key, value in extra_vars.iteritems():
        setattr(g, key, value)

    return base.render(
        _get_pkg_template(u'search_template', package_type), extra_vars
    )
Exemple #51
0
def package_publish_microdata(context, data_dict):
    default_error = 'Unknown microdata error'

    # Get data
    dataset_id = data_dict.get('id')
    nation = data_dict.get('nation')
    repoid = data_dict.get('repoid')

    # Check access
    toolkit.check_access('sysadmin', context)
    api_key = config.get('ckanext.unhcr.microdata_api_key')
    if not api_key:
        raise toolkit.NotAuthorized('Microdata API Key is not set')

    # Get dataset/survey
    headers = {'X-Api-Key': api_key}
    dataset = toolkit.get_action('package_show')(context, {'id': dataset_id})
    survey = helpers.convert_dataset_to_microdata_survey(
        dataset, nation, repoid)
    idno = survey['study_desc']['title_statement']['idno']

    try:

        # Publish dataset
        url = 'https://microdata.unhcr.org/index.php/api/datasets/create/survey/%s' % idno
        response = requests.post(url, headers=headers, json=survey).json()
        if response.get('status') != 'success':
            raise RuntimeError(str(response.get('errors', default_error)))
        template = 'https://microdata.unhcr.org/index.php/catalog/%s'
        survey['url'] = template % response['dataset']['id']
        survey['resources'] = []
        survey['files'] = []

        # Pubish resources/files
        file_name_counter = {}
        if dataset.get('resources', []):
            url = 'https://microdata.unhcr.org/index.php/api/datasets/%s/%s'
            for resource in dataset.get('resources', []):

                # resource
                resouce_url = url % (idno, 'resources')
                md_resource = helpers.convert_resource_to_microdata_resource(
                    resource)
                response = requests.post(resouce_url,
                                         headers=headers,
                                         json=md_resource).json()
                if response.get('status') != 'success':
                    raise RuntimeError(
                        str(response.get('errors', default_error)))
                survey['resources'].append(response['resource'])

                # file
                file_url = url % (idno, 'files')
                file_name = resource['url'].split('/')[-1]
                file_path = helpers.get_resource_file_path(resource)
                file_mime = resource['mimetype']
                if not file_name or not file_path:
                    continue
                file_name_counter.setdefault(file_name, 0)
                file_name_counter[file_name] += 1
                if file_name_counter[file_name] > 1:
                    file_name = helpers.add_file_name_suffix(
                        file_name, file_name_counter[file_name] - 1)
                with open(file_path, 'rb') as file_obj:
                    file = (file_name, file_obj, file_mime)
                    response = requests.post(file_url,
                                             headers=headers,
                                             files={
                                                 'file': file
                                             }).json()
                # TODO: update
                # it's a hack to overcome incorrect Microdata responses
                # usopported file types fail this way and we are skipping them
                if not isinstance(response, dict):
                    continue
                if response.get('status') != 'success':
                    raise RuntimeError(
                        str(response.get('errors', default_error)))
                survey['files'].append(response)

    except requests.exceptions.HTTPError:
        log.exception(exception)
        raise RuntimeError('Microdata connection failed')

    return survey
Exemple #52
0
    def _read(self, id, limit, group_type):
        ''' This is common code used by both read and bulk_process'''
        context = {
            'model': model,
            'session': model.Session,
            'user': c.user,
            'schema': self._db_to_form_schema(group_type=group_type),
            'for_view': True,
            'extras_as_string': True
        }

        q = c.q = request.params.get('q', '')
        # Search within group
        if c.group_dict.get('is_organization'):
            q += ' owner_org:"%s"' % c.group_dict.get('id')
        else:
            q += ' groups:"%s"' % c.group_dict.get('name')

        c.description_formatted = \
            h.render_markdown(c.group_dict.get('description'))

        context['return_query'] = True

        page = h.get_page_number(request.params)

        # most search operations should reset the page counter:
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']
        sort_by = request.params.get('sort', None)

        def search_url(params):
            controller = lookup_group_controller(group_type)
            action = 'bulk_process' if c.action == 'bulk_process' else 'read'
            url = h.url_for(controller=controller, action=action, id=id)
            params = [
                (k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
                for k, v in params
            ]
            return url + u'?' + urlencode(params)

        def drill_down_url(**by):
            return h.add_url_param(alternative_url=None,
                                   controller='group',
                                   action='read',
                                   extras=dict(id=c.group_dict.get('name')),
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            controller = lookup_group_controller(group_type)
            return h.remove_url_param(key,
                                      value=value,
                                      replace=replace,
                                      controller=controller,
                                      action='read',
                                      extras=dict(id=c.group_dict.get('name')))

        c.remove_field = remove_field

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params)

        try:
            c.fields = []
            c.fields_grouped = {}
            search_extras = {}
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and len(value) and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        q += ' %s: "%s"' % (param, value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses')
            }

            for facet in h.facets():
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            # Facet titles
            self._update_facet_titles(facets, group_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': '',
                'include_private': True,
                'facet.field': facets.keys(),
                'rows': limit,
                'sort': sort_by,
                'start': (page - 1) * limit,
                'extras': search_extras
            }

            context_ = dict(
                (k, v) for (k, v) in context.items() if k != 'schema')
            query = get_action('package_search')(context_, data_dict)

            c.page = h.Page(collection=query['results'],
                            page=page,
                            url=pager_url,
                            item_count=query['count'],
                            items_per_page=limit)

            c.group_dict['package_count'] = query['count']

            c.search_facets = query['search_facets']
            c.search_facets_limits = {}
            for facet in c.search_facets.keys():
                limit = int(
                    request.params.get('_%s_limit' % facet,
                                       config.get('search.facets.default',
                                                  10)))
                c.search_facets_limits[facet] = limit
            c.page.items = query['results']

            c.sort_by_selected = sort_by

        except search.SearchError, se:
            log.error('Group search error: %r', se.args)
            c.query_error = True
            c.page = h.Page(collection=[])
Exemple #53
0
def datapusher_submit(context, data_dict):
    ''' Submit a job to the datapusher. The datapusher is a service that
    imports tabular data into the datastore.

    :param resource_id: The resource id of the resource that the data
        should be imported in. The resource's URL will be used to get the data.
    :type resource_id: string
    :param set_url_type: If set to True, the ``url_type`` of the resource will
        be set to ``datastore`` and the resource URL will automatically point
        to the :ref:`datastore dump <dump>` URL. (optional, default: False)
    :type set_url_type: bool
    :param ignore_hash: If set to True, the datapusher will reload the file
        even if it haven't changed. (optional, default: False)
    :type ignore_hash: bool

    Returns ``True`` if the job has been submitted and ``False`` if the job
    has not been submitted, i.e. when the datapusher is not configured.

    :rtype: bool
    '''
    schema = context.get('schema', dpschema.datapusher_submit_schema())
    data_dict, errors = _validate(data_dict, schema, context)
    if errors:
        raise p.toolkit.ValidationError(errors)

    res_id = data_dict['resource_id']

    p.toolkit.check_access('datapusher_submit', context, data_dict)

    try:
        resource_dict = p.toolkit.get_action('resource_show')(context, {
            'id': res_id,
        })
    except logic.NotFound:
        return False

    datapusher_url = config.get('ckan.datapusher.url')

    site_url = h.url_for('/', qualified=True)
    callback_url = h.url_for('/api/3/action/datapusher_hook', qualified=True)

    user = p.toolkit.get_action('user_show')(context, {'id': context['user']})

    for plugin in p.PluginImplementations(interfaces.IDataPusher):
        upload = plugin.can_upload(res_id)
        if not upload:
            msg = "Plugin {0} rejected resource {1}"\
                .format(plugin.__class__.__name__, res_id)
            log.info(msg)
            return False

    task = {
        'entity_id': res_id,
        'entity_type': 'resource',
        'task_type': 'datapusher',
        'last_updated': str(datetime.datetime.utcnow()),
        'state': 'submitting',
        'key': 'datapusher',
        'value': '{}',
        'error': '{}',
    }
    try:
        existing_task = p.toolkit.get_action('task_status_show')(
            context, {
                'entity_id': res_id,
                'task_type': 'datapusher',
                'key': 'datapusher'
            })
        assume_task_stale_after = datetime.timedelta(seconds=int(
            config.get('ckan.datapusher.assume_task_stale_after', 3600)))
        if existing_task.get('state') == 'pending':
            updated = datetime.datetime.strptime(existing_task['last_updated'],
                                                 '%Y-%m-%dT%H:%M:%S.%f')
            time_since_last_updated = datetime.datetime.utcnow() - updated
            if time_since_last_updated > assume_task_stale_after:
                # it's been a while since the job was last updated - it's more
                # likely something went wrong with it and the state wasn't
                # updated than its still in progress. Let it be restarted.
                log.info(
                    'A pending task was found %r, but it is only %s hours'
                    'old', existing_task['id'], time_since_last_updated)
            else:
                log.info(
                    'A pending task was found %s for this resource, so '
                    'skipping this duplicate task', existing_task['id'])
                return False

        task['id'] = existing_task['id']
    except logic.NotFound:
        pass

    context['ignore_auth'] = True
    p.toolkit.get_action('task_status_update')(context, task)

    try:
        r = requests.post(urlparse.urljoin(datapusher_url, 'job'),
                          headers={'Content-Type': 'application/json'},
                          data=json.dumps({
                              'api_key': user['apikey'],
                              'job_type': 'push_to_datastore',
                              'result_url': callback_url,
                              'metadata': {
                                  'ignore_hash':
                                  data_dict.get('ignore_hash', False),
                                  'ckan_url':
                                  site_url,
                                  'resource_id':
                                  res_id,
                                  'set_url_type':
                                  data_dict.get('set_url_type', False),
                                  'task_created':
                                  task['last_updated'],
                                  'original_url':
                                  resource_dict.get('url'),
                              }
                          }))
        r.raise_for_status()
    except requests.exceptions.ConnectionError, e:
        error = {
            'message': 'Could not connect to DataPusher.',
            'details': str(e)
        }
        task['error'] = json.dumps(error)
        task['state'] = 'error'
        task['last_updated'] = str(datetime.datetime.utcnow()),
        p.toolkit.get_action('task_status_update')(context, task)
        raise p.toolkit.ValidationError(error)
Exemple #54
0
    def index_package(self, pkg_dict, defer_commit=False):
        if pkg_dict is None:
            return

        # tracking summary values will be stale, never store them
        tracking_summary = pkg_dict.pop('tracking_summary', None)
        for r in pkg_dict.get('resources', []):
            r.pop('tracking_summary', None)

        data_dict_json = json.dumps(pkg_dict)

        if config.get('ckan.cache_validated_datasets', True):
            package_plugin = lib_plugins.lookup_package_plugin(
                pkg_dict.get('type'))

            schema = package_plugin.show_package_schema()
            validated_pkg_dict, errors = lib_plugins.plugin_validate(
                package_plugin, {'model': model, 'session': model.Session},
                pkg_dict, schema, 'package_show')
            pkg_dict['validated_data_dict'] = json.dumps(validated_pkg_dict,
                cls=ckan.lib.navl.dictization_functions.MissingNullEncoder)

        pkg_dict['data_dict'] = data_dict_json

        # add to string field for sorting
        title = pkg_dict.get('title')
        if title:
            pkg_dict['title_string'] = title

        # delete the package if there is no state, or the state is `deleted`
        if (not pkg_dict.get('state') or 'deleted' in pkg_dict.get('state')):
            return self.delete_package(pkg_dict)

        index_fields = RESERVED_FIELDS + pkg_dict.keys()

        # include the extras in the main namespace
        extras = pkg_dict.get('extras', [])
        for extra in extras:
            key, value = extra['key'], extra['value']
            if isinstance(value, (tuple, list)):
                value = " ".join(map(text_type, value))
            key = ''.join([c for c in key if c in KEY_CHARS])
            pkg_dict['extras_' + key] = value
            if key not in index_fields:
                pkg_dict[key] = value
        pkg_dict.pop('extras', None)

        # add tags, removing vocab tags from 'tags' list and adding them as
        # vocab_<tag name> so that they can be used in facets
        non_vocab_tag_names = []
        tags = pkg_dict.pop('tags', [])
        context = {'model': model}

        for tag in tags:
            if tag.get('vocabulary_id'):
                data = {'id': tag['vocabulary_id']}
                vocab = logic.get_action('vocabulary_show')(context, data)
                key = u'vocab_%s' % vocab['name']
                if key in pkg_dict:
                    pkg_dict[key].append(tag['name'])
                else:
                    pkg_dict[key] = [tag['name']]
            else:
                non_vocab_tag_names.append(tag['name'])

        pkg_dict['tags'] = non_vocab_tag_names

        # add groups
        groups = pkg_dict.pop('groups', [])

        # we use the capacity to make things private in the search index
        if pkg_dict['private']:
            pkg_dict['capacity'] = 'private'
        else:
            pkg_dict['capacity'] = 'public'

        pkg_dict['groups'] = [group['name'] for group in groups]

        # if there is an owner_org we want to add this to groups for index
        # purposes
        if pkg_dict.get('organization'):
           pkg_dict['organization'] = pkg_dict['organization']['name']
        else:
           pkg_dict['organization'] = None

        # tracking
        if not tracking_summary:
            tracking_summary = model.TrackingSummary.get_for_package(
                pkg_dict['id'])
        pkg_dict['views_total'] = tracking_summary['total']
        pkg_dict['views_recent'] = tracking_summary['recent']

        resource_fields = [('name', 'res_name'),
                           ('description', 'res_description'),
                           ('format', 'res_format'),
                           ('url', 'res_url'),
                           ('resource_type', 'res_type')]
        resource_extras = [(e, 'res_extras_' + e) for e
                            in model.Resource.get_extra_columns()]
        # flatten the structure for indexing:
        for resource in pkg_dict.get('resources', []):
            for (okey, nkey) in resource_fields + resource_extras:
                pkg_dict[nkey] = pkg_dict.get(nkey, []) + [resource.get(okey, u'')]
        pkg_dict.pop('resources', None)

        rel_dict = collections.defaultdict(list)
        subjects = pkg_dict.pop("relationships_as_subject", [])
        objects = pkg_dict.pop("relationships_as_object", [])
        for rel in objects:
            type = model.PackageRelationship.forward_to_reverse_type(rel['type'])
            rel_dict[type].append(model.Package.get(rel['subject_package_id']).name)
        for rel in subjects:
            type = rel['type']
            rel_dict[type].append(model.Package.get(rel['object_package_id']).name)
        for key, value in rel_dict.iteritems():
            if key not in pkg_dict:
                pkg_dict[key] = value

        pkg_dict[TYPE_FIELD] = PACKAGE_TYPE

        # Save dataset type
        pkg_dict['dataset_type'] = pkg_dict['type']

        # clean the dict fixing keys and dates
        # FIXME where are we getting these dirty keys from?  can we not just
        # fix them in the correct place or is this something that always will
        # be needed?  For my data not changing the keys seems to not cause a
        # problem.
        new_dict = {}
        bogus_date = datetime.datetime(1, 1, 1)
        for key, value in pkg_dict.items():
            key = key.encode('ascii', 'ignore')
            if key.endswith('_date'):
                try:
                    date = parse(value, default=bogus_date)
                    if date != bogus_date:
                        value = date.isoformat() + 'Z'
                    else:
                        # The date field was empty, so dateutil filled it with
                        # the default bogus date
                        value = None
                except ValueError:
                    continue
            new_dict[key] = value
        pkg_dict = new_dict

        for k in ('title', 'notes', 'title_string'):
            if k in pkg_dict and pkg_dict[k]:
                pkg_dict[k] = escape_xml_illegal_chars(pkg_dict[k])

        # modify dates (SOLR is quite picky with dates, and only accepts ISO dates
        # with UTC time (i.e trailing Z)
        # See http://lucene.apache.org/solr/api/org/apache/solr/schema/DateField.html
        pkg_dict['metadata_created'] += 'Z'
        pkg_dict['metadata_modified'] += 'Z'

        # mark this CKAN instance as data source:
        pkg_dict['site_id'] = config.get('ckan.site_id')

        # Strip a selection of the fields.
        # These fields are possible candidates for sorting search results on,
        # so we strip leading spaces because solr will sort " " before "a" or "A".
        for field_name in ['title']:
            try:
                value = pkg_dict.get(field_name)
                if value:
                    pkg_dict[field_name] = value.lstrip()
            except KeyError:
                pass

        # add a unique index_id to avoid conflicts
        import hashlib
        pkg_dict['index_id'] = hashlib.md5('%s%s' % (pkg_dict['id'],config.get('ckan.site_id'))).hexdigest()

        for item in PluginImplementations(IPackageController):
            pkg_dict = item.before_index(pkg_dict)

        assert pkg_dict, 'Plugin must return non empty package dict on index'

        # permission labels determine visibility in search, can't be set
        # in original dataset or before_index plugins
        labels = lib_plugins.get_permission_labels()
        dataset = model.Package.get(pkg_dict['id'])
        pkg_dict['permission_labels'] = labels.get_dataset_labels(
            dataset) if dataset else [] # TestPackageSearchIndex-workaround

        # send to solr:
        try:
            conn = make_connection()
            commit = not defer_commit
            if not asbool(config.get('ckan.search.solr_commit', 'true')):
                commit = False
            conn.add(docs=[pkg_dict], commit=commit)
        except pysolr.SolrError as e:
            msg = 'Solr returned an error: {0}'.format(
                e[:1000] # limit huge responses
            )
            raise SearchIndexError(msg)
        except socket.error as e:
            err = 'Could not connect to Solr using {0}: {1}'.format(conn.url, str(e))
            log.error(err)
            raise SearchIndexError(err)

        commit_debug_msg = 'Not committed yet' if defer_commit else 'Committed'
        log.debug('Updated index for %s [%s]' % (pkg_dict.get('name'), commit_debug_msg))
def disable_user_access():
    return toolkit.asbool(config.get('ckan.senasa_theme.disable_user_access', False))
Exemple #56
0
def _read(id, limit, group_type):
    u''' This is common code used by both read and bulk_process'''
    extra_vars = {}
    context = {
        u'model': model,
        u'session': model.Session,
        u'user': g.user,
        u'schema': _db_to_form_schema(group_type=group_type),
        u'for_view': True,
        u'extras_as_string': True
    }

    q = request.params.get(u'q', u'')

    # TODO: Remove
    # ckan 2.9: Adding variables that were removed from c object for
    # compatibility with templates in existing extensions
    g.q = q

    # Search within group
    if g.group_dict.get(u'is_organization'):
        fq = u' owner_org:"%s"' % g.group_dict.get(u'id')
    else:
        fq = u' groups:"%s"' % g.group_dict.get(u'name')

    extra_vars["q"] = q

    g.description_formatted = \
        h.render_markdown(g.group_dict.get(u'description'))

    context['return_query'] = True

    page = h.get_page_number(request.params)

    # most search operations should reset the page counter:
    params_nopage = [(k, v) for k, v in request.params.items() if k != u'page']
    sort_by = request.params.get(u'sort', None)

    def search_url(params):
        controller = lookup_group_controller(group_type)
        action = u'bulk_process' if getattr(
            g, u'action', u'') == u'bulk_process' else u'read'
        url = h.url_for(u'.'.join([controller, action]), id=id)
        params = [
            (k, v.encode(u'utf-8') if isinstance(v, string_types) else str(v))
            for k, v in params
        ]
        return url + u'?' + urlencode(params)

    def drill_down_url(**by):
        return h.add_url_param(alternative_url=None,
                               controller=u'group',
                               action=u'read',
                               extras=dict(id=g.group_dict.get(u'name')),
                               new_params=by)

    extra_vars["drill_down_url"] = drill_down_url

    def remove_field(key, value=None, replace=None):
        controller = lookup_group_controller(group_type)
        return h.remove_url_param(key,
                                  value=value,
                                  replace=replace,
                                  controller=controller,
                                  action=u'read',
                                  extras=dict(id=g.group_dict.get(u'name')))

    extra_vars["remove_field"] = remove_field

    def pager_url(q=None, page=None):
        params = list(params_nopage)
        params.append((u'page', page))
        return search_url(params)

    try:
        extra_vars["fields"] = fields = []
        extra_vars["fields_grouped"] = fields_grouped = {}
        search_extras = {}
        for (param, value) in request.params.items():
            if param not in [u'q', u'page', u'sort'] \
                    and len(value) and not param.startswith(u'_'):
                if not param.startswith(u'ext_'):
                    fields.append((param, value))
                    q += u' %s: "%s"' % (param, value)
                    if param not in fields_grouped:
                        fields_grouped[param] = [value]
                    else:
                        fields_grouped[param].append(value)
                else:
                    search_extras[param] = value

        # TODO: Remove
        # ckan 2.9: Adding variables that were removed from c object for
        # compatibility with templates in existing extensions
        g.fields = fields
        g.fields_grouped = fields_grouped

        facets = OrderedDict()

        default_facet_titles = {
            u'organization': _(u'Organizations'),
            u'groups': _(u'Groups'),
            u'tags': _(u'Tags'),
            u'res_format': _(u'Formats'),
            u'license_id': _(u'Licenses')
        }

        for facet in h.facets():
            if facet in default_facet_titles:
                facets[facet] = default_facet_titles[facet]
            else:
                facets[facet] = facet

        # Facet titles
        _update_facet_titles(facets, group_type)

        extra_vars["facet_titles"] = facets

        data_dict = {
            u'q': q,
            u'fq': fq,
            u'include_private': True,
            u'facet.field': facets.keys(),
            u'rows': limit,
            u'sort': sort_by,
            u'start': (page - 1) * limit,
            u'extras': search_extras
        }

        context_ = dict((k, v) for (k, v) in context.items() if k != u'schema')
        query = get_action(u'package_search')(context_, data_dict)

        extra_vars["page"] = h.Page(collection=query['results'],
                                    page=page,
                                    url=pager_url,
                                    item_count=query['count'],
                                    items_per_page=limit)

        # TODO: Remove
        # ckan 2.9: Adding variables that were removed from c object for
        # compatibility with templates in existing extensions
        g.group_dict['package_count'] = query['count']

        extra_vars["search_facets"] = g.search_facets = query['search_facets']
        extra_vars["search_facets_limits"] = g.search_facets_limits = {}
        for facet in g.search_facets.keys():
            limit = int(
                request.params.get(u'_%s_limit' % facet,
                                   config.get(u'search.facets.default', 10)))
            g.search_facets_limits[facet] = limit
        extra_vars["page"].items = query['results']

        extra_vars["sort_by_selected"] = sort_by

    except search.SearchError as se:
        log.error(u'Group search error: %r', se.args)
        extra_vars["query_error"] = True
        extra_vars["page"] = h.Page(collection=[])

    # TODO: Remove
    # ckan 2.9: Adding variables that were removed from c object for
    # compatibility with templates in existing extensions
    g.facet_titles = facets
    g.page = extra_vars["page"]

    extra_vars["group_type"] = group_type
    _setup_template_variables(context, {u'id': id}, group_type=group_type)
    return extra_vars
Exemple #57
0
def index(group_type, is_organization):
    extra_vars = {}
    set_org(is_organization)
    page = h.get_page_number(request.params) or 1
    items_per_page = int(config.get(u'ckan.datasets_per_page', 20))

    context = {
        u'model': model,
        u'session': model.Session,
        u'user': g.user,
        u'for_view': True,
        u'with_private': False
    }

    try:
        _check_access(u'site_read', context)
        _check_access(u'group_list', context)
    except NotAuthorized:
        base.abort(403, _(u'Not authorized to see this page'))

    q = request.params.get(u'q', u'')
    sort_by = request.params.get(u'sort')

    # TODO: Remove
    # ckan 2.9: Adding variables that were removed from c object for
    # compatibility with templates in existing extensions
    g.q = q
    g.sort_by_selected = sort_by

    extra_vars["q"] = q
    extra_vars["sort_by_selected"] = sort_by

    # pass user info to context as needed to view private datasets of
    # orgs correctly
    if g.userobj:
        context['user_id'] = g.userobj.id
        context['user_is_admin'] = g.userobj.sysadmin

    try:
        data_dict_global_results = {
            u'all_fields': False,
            u'q': q,
            u'sort': sort_by,
            u'type': group_type or u'group',
        }
        global_results = _action(u'group_list')(context,
                                                data_dict_global_results)
    except ValidationError as e:
        if e.error_dict and e.error_dict.get(u'message'):
            msg = e.error_dict['message']
        else:
            msg = str(e)
        h.flash_error(msg)
        extra_vars["page"] = h.Page([], 0)
        extra_vars["group_type"] = group_type
        return base.render(_index_template(group_type), extra_vars)

    data_dict_page_results = {
        u'all_fields': True,
        u'q': q,
        u'sort': sort_by,
        u'type': group_type or u'group',
        u'limit': items_per_page,
        u'offset': items_per_page * (page - 1),
        u'include_extras': True
    }
    page_results = _action(u'group_list')(context, data_dict_page_results)

    extra_vars["page"] = h.Page(
        collection=global_results,
        page=page,
        url=h.pager_url,
        items_per_page=items_per_page,
    )

    extra_vars["page"].items = page_results
    extra_vars["group_type"] = group_type

    # TODO: Remove
    # ckan 2.9: Adding variables that were removed from c object for
    # compatibility with templates in existing extensions
    g.page = extra_vars["page"]
    return base.render(_index_template(group_type), extra_vars)
Exemple #58
0
def search_params():
    u'''Returns a list of the current search names'''
    return config.get(u'search.search_param', DEFAULT_SEARCH_NAMES).split()
Exemple #59
0
def make_flask_stack(conf, **app_conf):
    """ This has to pass the flask app through all the same middleware that
    Pylons used """

    root = os.path.dirname(
        os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

    debug = asbool(conf.get('debug', conf.get('DEBUG', False)))
    testing = asbool(app_conf.get('testing', app_conf.get('TESTING', False)))
    app = flask_app = CKANFlask(__name__)
    app.debug = debug
    app.testing = testing
    app.template_folder = os.path.join(root, 'templates')
    app.app_ctx_globals_class = CKAN_AppCtxGlobals
    app.url_rule_class = CKAN_Rule

    app.jinja_options = jinja_extensions.get_jinja_env_options()
    # Update Flask config with the CKAN values. We use the common config
    # object as values might have been modified on `load_environment`
    if config:
        app.config.update(config)
    else:
        app.config.update(conf)
        app.config.update(app_conf)

    # Do all the Flask-specific stuff before adding other middlewares

    # Secret key needed for flask-debug-toolbar and sessions
    if not app.config.get('SECRET_KEY'):
        app.config['SECRET_KEY'] = config.get('beaker.session.secret')
    if not app.config.get('SECRET_KEY'):
        raise RuntimeError(u'You must provide a value for the secret key'
                           ' with the SECRET_KEY config option')

    if debug:
        from flask_debugtoolbar import DebugToolbarExtension
        app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
        DebugToolbarExtension(app)

        from werkzeug.debug import DebuggedApplication
        app = DebuggedApplication(app, True)
        app = app.app

        log = logging.getLogger('werkzeug')
        log.setLevel(logging.DEBUG)

    # Use Beaker as the Flask session interface
    class BeakerSessionInterface(SessionInterface):
        def open_session(self, app, request):
            if 'beaker.session' in request.environ:
                return request.environ['beaker.session']

        def save_session(self, app, session, response):
            session.save()

    namespace = 'beaker.session.'
    session_opts = dict([(k.replace('beaker.', ''), v)
                         for k, v in config.iteritems()
                         if k.startswith(namespace)])
    if (not session_opts.get('session.data_dir')
            and session_opts.get('session.type', 'file') == 'file'):
        cache_dir = app_conf.get('cache_dir') or app_conf.get('cache.dir')
        session_opts['session.data_dir'] = '{data_dir}/sessions'.format(
            data_dir=cache_dir)

    app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts)
    app.session_interface = BeakerSessionInterface()

    # Add Jinja2 extensions and filters
    app.jinja_env.filters['empty_and_escape'] = \
        jinja_extensions.empty_and_escape

    # Common handlers for all requests
    app.before_request(ckan_before_request)
    app.after_request(ckan_after_request)

    # Template context processors
    app.context_processor(helper_functions)
    app.context_processor(c_object)

    @app.context_processor
    def ungettext_alias():
        u'''
        Provide `ungettext` as an alias of `ngettext` for backwards
        compatibility
        '''
        return dict(ungettext=ungettext)

    # Babel
    pairs = [(os.path.join(root, u'i18n'), 'ckan')
             ] + [(p.i18n_directory(), p.i18n_domain())
                  for p in PluginImplementations(ITranslation)]

    i18n_dirs, i18n_domains = zip(*pairs)

    app.config[u'BABEL_TRANSLATION_DIRECTORIES'] = ';'.join(i18n_dirs)
    app.config[u'BABEL_DOMAIN'] = 'ckan'
    app.config[u'BABEL_MULTIPLE_DOMAINS'] = ';'.join(i18n_domains)

    babel = CKANBabel(app)

    babel.localeselector(get_locale)

    @app.route('/hello', methods=['GET'])
    def hello_world():
        return 'Hello World, this is served by Flask'

    @app.route('/hello', methods=['POST'])
    def hello_world_post():
        return 'Hello World, this was posted to Flask'

    # Auto-register all blueprints defined in the `views` folder
    _register_core_blueprints(app)
    _register_error_handler(app)

    # Set up each IBlueprint extension as a Flask Blueprint
    for plugin in PluginImplementations(IBlueprint):
        if hasattr(plugin, 'get_blueprint'):
            plugin_blueprints = plugin.get_blueprint()
            if not isinstance(plugin_blueprints, list):
                plugin_blueprints = [plugin_blueprints]
            for blueprint in plugin_blueprints:
                app.register_extension_blueprint(blueprint)

    lib_plugins.register_package_blueprints(app)
    lib_plugins.register_group_blueprints(app)

    # Set flask routes in named_routes
    for rule in app.url_map.iter_rules():
        if '.' not in rule.endpoint:
            continue
        controller, action = rule.endpoint.split('.')
        needed = list(rule.arguments - set(rule.defaults or {}))
        route = {
            rule.endpoint: {
                'action': action,
                'controller': controller,
                'highlight_actions': action,
                'needed': needed
            }
        }
        config['routes.named_routes'].update(route)

    # Start other middleware
    for plugin in PluginImplementations(IMiddleware):
        app = plugin.make_middleware(app, config)

    # Fanstatic
    fanstatic_enable_rollup = asbool(
        app_conf.get('fanstatic_enable_rollup', False))
    if debug:
        fanstatic_config = {
            'versioning': True,
            'recompute_hashes': True,
            'minified': False,
            'bottom': True,
            'bundle': False,
            'rollup': fanstatic_enable_rollup,
        }
    else:
        fanstatic_config = {
            'versioning': True,
            'recompute_hashes': False,
            'minified': True,
            'bottom': True,
            'bundle': True,
            'rollup': fanstatic_enable_rollup,
        }
    root_path = config.get('ckan.root_path', None)
    if root_path:
        root_path = re.sub('/{{LANG}}', '', root_path)
        fanstatic_config['base_url'] = root_path
    app = Fanstatic(app, **fanstatic_config)

    for plugin in PluginImplementations(IMiddleware):
        try:
            app = plugin.make_error_log_middleware(app, config)
        except AttributeError:
            log.critical('Middleware class {0} is missing the method'
                         'make_error_log_middleware.'.format(
                             plugin.__class__.__name__))

    # Initialize repoze.who
    who_parser = WhoConfig(conf['here'])
    who_parser.parse(open(app_conf['who.config_file']))

    app = PluggableAuthenticationMiddleware(
        app,
        who_parser.identifiers,
        who_parser.authenticators,
        who_parser.challengers,
        who_parser.mdproviders,
        who_parser.request_classifier,
        who_parser.challenge_decider,
        logging.getLogger('repoze.who'),
        logging.WARN,  # ignored
        who_parser.remote_user_key)

    # Update the main CKAN config object with the Flask specific keys
    # that were set here or autogenerated
    flask_config_keys = set(flask_app.config.keys()) - set(config.keys())
    for key in flask_config_keys:
        config[key] = flask_app.config[key]

    # Add a reference to the actual Flask app so it's easier to access
    app._wsgi_app = flask_app

    return app
def send_email(content, to, subject, file=None):
    '''Sends email
       :param content: The body content for the mail.
       :type string:
       :param to: To whom will be mail sent
       :type string:
       :param subject: The subject of mail.
       :type string:


       :rtype: string

       '''
    SMTP_SERVER = config.get('smtp.server')
    SMTP_USER = config.get('smtp.user')
    SMTP_PASSWORD = config.get('smtp.password')
    SMTP_FROM = config.get('smtp.mail_from')

    msg = MIMEMultipart()

    from_ = SMTP_FROM

    if isinstance(to, basestring):
        to = [to]

    msg['Subject'] = subject
    msg['From'] = from_
    msg['To'] = ','.join(to)

    content = """\
        <html>
          <head></head>
          <body>
            <p>""" + content + """</p>
          </body>
        </html>
    """

    msg.attach(MIMEText(content, 'html', _charset='utf-8'))

    if isinstance(file, cgi.FieldStorage):
        part = MIMEBase('application', 'octet-stream')
        part.set_payload(file.file.read())
        Encoders.encode_base64(part)

        extension = file.filename.split('.')[-1]

        part.add_header('Content-Disposition',
                        'attachment; filename=attachment.{0}'.format(extension))

        msg.attach(part)

    try:
        s = smtplib.SMTP(SMTP_SERVER)
        if SMTP_USER:
            s.login(SMTP_USER, SMTP_PASSWORD)
        s.sendmail(from_, to, msg.as_string())
        s.quit()
        response_dict = {
            'success': True,
            'message': 'Email message was successfully sent.'
        }
        return response_dict
    except socket_error:
        log.critical(
            'Could not connect to email server. Have you configured the SMTP settings?')
        error_dict = {
            'success': False,
            'message': 'An error occured while sending the email. Try again.'
        }
        return error_dict