def check_recaptcha(token, expected_action):
    '''
    Check a user's recaptcha token is valid, and raise RecaptchaError if it's not. If the token is
    valid then this function has no side effects.

    :param token: the token as returned in the frontend by a call to execute
    :param expected_action: the expected action associated with the token
    '''
    key = config.get(u'ckanext.contact.recaptcha_v3_key', False)
    secret = config.get(u'ckanext.contact.recaptcha_v3_secret', False)

    if not key or not secret:
        # recaptcha not enabled
        return

    post_params = {
        u'secret': secret,
        u'response': token,
    }
    client_ip_address = request.environ.get(u'REMOTE_ADDR', None)
    if client_ip_address:
        post_params[u'remoteip'] = client_ip_address

    response = requests.post(u'https://www.google.com/recaptcha/api/siteverify', params=post_params)
    response.raise_for_status()
    result = response.json()

    if not result[u'success']:
        raise RecaptchaError(u', '.join(result[u'error-codes']))
    if expected_action != result[u'action']:
        raise RecaptchaError(_(u'Action mismatch'))
Example #2
0
def static(url):
    static_path = config.get("openspending.static_path", "/static/")
    url_ = "%s%s" % (static_path, url)
    version = config.get("openspending.static_cache_version", "")
    if version:
        url_ = "%s?%s" % (url_, version)
    return url_
def get_profile_id(service):
    """
    Get the profile ID for this user and the service specified by the
    'googleanalytics.id' configuration option. This function iterates
    over all of the accounts available to the user who invoked the
    service to find one where the account name matches (in case the
    user has several).
    """
    accounts = service.management().accounts().list().execute()

    if not accounts.get('items'):
        return None

    accountName = config.get('googleanalytics.account')
    webPropertyId = config.get('googleanalytics.id')
    for acc in accounts.get('items'):
        if acc.get('name') == accountName:
            accountId = acc.get('id')

    webproperties = service.management().webproperties().list(accountId=accountId).execute()

    profiles = service.management().profiles().list(
        accountId=accountId, webPropertyId=webPropertyId).execute()

    if profiles.get('items'):
        return profiles.get('items')[0].get('id')

    return None
Example #4
0
    def by_name(cls, name):
        """Return a Domain instance by name."""

        if os.path.exists(os.path.join(config.get('herder.po_dir'), name)):
            return Domain(name, os.path.join(config.get('herder.po_dir'), name))

        raise KeyError("Unknown domain name.")
Example #5
0
def send(to_addrs, from_addr, subject, body):
    """A simple method to send a simple email.

    :param to_addrs: Comma separated list of email addresses to send to.
    :type to_addrs: unicode

    :param from_addr: Email address to put in the 'from' field
    :type from_addr: unicode

    :param subject: Subject line of the email.
    :type subject: unicode

    :param body: Body text of the email, optionally marked up with HTML.
    :type body: unicode
    """
    smtp_server = config.get('smtp_server', 'localhost')
    server = smtplib.SMTP(smtp_server)
    if isinstance(to_addrs, basestring):
        to_addrs = parse_email_string(to_addrs)

    to_addrs = ", ".join(to_addrs)

    msg = ("To: %(to_addrs)s\n"
           "From: %(from_addr)s\n"
           "Subject: %(subject)s\n\n"
           "%(body)s\n") % locals()

    smtp_username = config.get('smtp_username')
    smtp_password = config.get('smtp_password')
    if smtp_username and smtp_password:
        server.login(smtp_username, smtp_password)

    server.sendmail(from_addr, to_addrs, msg.encode('utf-8'))
    server.quit()
Example #6
0
def get_value(key, converter, default=None, config=config,
              converter_kwargs={}, instance_overwrites=True):

    allow_overwrite = (instance_overwrites
                       and get_bool('%s.allow_overwrite' % key,
                                    instance_overwrites=False))

    if allow_overwrite:
        from adhocracy.model import instance_filter as ifilter
        current_instance = ifilter.get_instance()
        if current_instance is not None:
            value = config.get('%s.%s' % (key, current_instance.key))
        else:
            value = None
    else:
        value = None

    if value is None:
        value = config.get(key)

    if value is None:
        if default is not None:
            return default
        else:
            return DEFAULTS.get(key)

    if converter is None:
        return value
    else:
        return converter(value, **converter_kwargs)
Example #7
0
    def __init__(self, profiles=None, compatibility_mode=False):
        '''
        Creates a parser instance

        You can optionally pass a list of profiles to be used.

        In compatibility mode, some fields are modified to maintain
        compatibility with previous versions of the ckanext-dcat parsers
        (eg adding the `dcat_` prefix or storing comma separated lists instead
        of JSON dumps).

        '''
        if not profiles:
            profiles = config.get(RDF_PROFILES_CONFIG_OPTION, None)
            if profiles:
                profiles = profiles.split(' ')
            else:
                profiles = DEFAULT_RDF_PROFILES
        self._profiles = self._load_profiles(profiles)
        if not self._profiles:
            raise RDFProfileException(
                'No suitable RDF profiles could be loaded')

        if not compatibility_mode:
            compatibility_mode = p.toolkit.asbool(
                config.get(COMPAT_MODE_CONFIG_OPTION, False))
        self.compatibility_mode = compatibility_mode

        self.g = rdflib.Graph()
Example #8
0
def catalog_uri():
    '''
    Returns an URI for the whole catalog

    This will be used to uniquely reference the CKAN instance on the RDF
    serializations and as a basis for eg datasets URIs (if not present on
    the metadata).

    The value will be the first found of:

        1. The `ckanext.dcat.base_uri` config option (recommended)
        2. The `ckan.site_url` config option
        3. `http://` + the `app_instance_uuid` config option (minus brackets)

    A warning is emited if the third option is used.

    Returns a string with the catalog URI.
    '''

    uri = config.get('ckanext.dcat.base_uri')
    if not uri:
        uri = config.get('ckan.site_url')
    if not uri:
        app_uuid = config.get('app_instance_uuid')
        if app_uuid:
            uri = 'http://' + app_uuid.replace('{', '').replace('}', '')
            log.critical('Using app id as catalog URI, you should set the ' +
                         '`ckanext.dcat.base_uri` or `ckan.site_url` option')
        else:
            uri = 'http://' + str(uuid.uuid4())
            log.critical('Using a random id as catalog URI, you should set ' +
                         'the `ckanext.dcat.base_uri` or `ckan.site_url` ' +
                         'option')

    return uri
Example #9
0
def setSort(spectra, config, package):
    sort = None

    # backword compatibility.  But moving away from config object
    # all 'advanced data' should be stored in package
    extras = package.get('extras')
    if extras != None and extras.get('sort') != None:
        sort = json.loads(extras.get('sort'))
    elif config.get("sort") != None:
        sort = config.get("sort")

    if sort == None:
        return

    on = sort.get('on')
    type = sort.get('type')

    if on is None:
        return

    if on not in spectra:
        return

    if type == 'datetime':
        try:
            spectra['ecosis']['sort'] = dateutil.parser.parse(spectra[on])
        except:
            pass
    elif type == 'numeric':
        try:
            spectra['ecosis']['sort'] = float(spectra[on])
        except:
            pass
    else:
        spectra['ecosis']['sort'] = spectra[on]
Example #10
0
    def graph_from_catalog(self, catalog_dict, catalog_ref):

        g = self.g

        for prefix, namespace in namespaces.iteritems():
            g.bind(prefix, namespace)

        g.add((catalog_ref, RDF.type, DCAT.Catalog))

        # Basic fields
        items = [
            ('title', DCT.title, config.get('ckan.site_title')),
            ('description', DCT.description, config.get('ckan.site_description')),
            ('homepage', FOAF.homepage, config.get('ckan.site_url')),
            ('language', DCT.language, config.get('ckan.locale_default', 'en')),
        ]
        for item in items:
            key, predicate, fallback = item
            if catalog_dict:
                value = catalog_dict.get(key, fallback)
            else:
                value = fallback
            if value:
                g.add((catalog_ref, predicate, Literal(value)))

        # Dates
        modified = self._last_catalog_modification()
        if modified:
            self._add_date_triple(catalog_ref, DCT.modified, modified)
Example #11
0
    def upload(self):
        if 'datafile' in request.params:
            myfile = request.params.get('datafile')

            # Store file in user directory on Longboard
            source_dir = config.get('cw.cwdata_loc')
            source = os.path.join(source_dir, myfile.filename)
            ###target = os.path.sep.join([config.get('cw.cwproj_loc'), session.get('user', 'guest')])
            target = os.path.sep.join([config.get('cw.cwproj_rem'), session.get('user', 'guest')])
            file_contents = ''
            for i in myfile.file:
                file_contents += i
            if file.type.find('text') > -1:
                try:
                    fh = open(source, 'w')
                    fh.write(file_contents)
                    fh.close()
                except Exception as _:
                    log.error('Cannot write file to disk.')
            else:
                try:
                    fh = open(source, 'wb')
                    fh.write(file_contents)
                    fh.close()
                except Exception:
                    log.error('Cannot write file to disk.')
            host = config.get('cw.arch_host', 'longboard.acel')
            resource = app_globals.jodis.manager.getResource(host)
            resource.ssh.scp(None, None, source, resource.ssh.user, host, target)
        return render('/data/upload.mako')
Example #12
0
    def _call(self, **kwargs):

        account_name = config.get("ckanext.doi.account_name")
        account_password = config.get("ckanext.doi.account_password")
        endpoint = os.path.join(get_endpoint(), self.path)

        try:
            path_extra = kwargs.pop('path_extra')
        except KeyError:
            pass
        else:
            endpoint = os.path.join(endpoint, path_extra)

        try:
            method = kwargs.pop('method')
        except KeyError:
            method = 'get'

        # Add authorisation to request
        kwargs['auth'] = (account_name, account_password)

        log.info("Calling %s:%s - %s", endpoint, method, kwargs)

        r = getattr(requests, method)(endpoint, **kwargs)
        r.raise_for_status()
        # Return the result
        return r
Example #13
0
    def before_create(self, context, resource):
        disallowed_extensions = toolkit.aslist(config.get('ckanext.romania_theme.disallowed_extensions',[]))
        disallowed_mimetypes = [mimetypes.types_map["." + x] for x in disallowed_extensions]

        allowed_extensions = toolkit.aslist(config.get('ckanext.romania_theme.allowed_extensions',[]))
        allowed_mimetypes = [mimetypes.types_map["." + x] for x in allowed_extensions]

        is_resource_extension_allowed = False
        error_message = ''
        if allowed_mimetypes:
            if resource['upload'].type in allowed_mimetypes:
                is_resource_extension_allowed = True
            else:
                error_message="Doar urmatoarele extensii sunt permise: " + ", ".join(allowed_extensions) + "."
        else:
            if resource['upload'].type not in disallowed_mimetypes:
                is_resource_extension_allowed = True
            else:
                error_message= "Urmatoarele extensii sunt nepermise: " + ", ".join(disallowed_extensions) + "."

        if ('upload' in resource) and (type(resource['upload']) is not unicode) and not is_resource_extension_allowed:
            # If we did not do this, the URL field would contain the filename
            # and people can press finalize afterwards.
            resource['url'] = ''


            raise toolkit.ValidationError(['Fisierul are o extensie nepermisa! ' + error_message])
Example #14
0
def ssl_check(ssl_required=[], ssl_allowed=[], ssl_required_all=False, ssl_allowed_all=False):

    if not asbool(config.get('enable_ssl_requirement', False)):
        return

    action = request.environ['pylons.routes_dict']['action']

    if action in ssl_allowed or ssl_allowed_all:             # We don't care if they use http or https
        return
    elif action in ssl_required or ssl_required_all:     # Must have ssl
        protocol = 'https'
    else:
        protocol = 'http'

    if current_protocol() == protocol:
        return

    if request.method.upper() != 'POST':
        log.debug('Redirecting to %s, request: %s', protocol, request.path_info)
        host = config.get('ssl_host')
        if host:
            redirect_to(protocol=protocol, host=host)
        else:
            redirect_to(protocol=protocol)
    else:
        abort(405, headers=[('Allow', 'GET')]) # don't allow POSTs.
Example #15
0
    def update_config(self, config):
        """Update CKAN's config with settings needed by this plugin.

        """
        toolkit.add_template_directory(config, "templates")
        self.header_parameter = config.get("ckan.simplesso.header_parameter", "user-id")
        self.email_domain = config.get("ckan.simplesso.email_domain")
def get_name_from_siret(siret):

    apiKey = config.get('ckanext.ozwillo_organization_api.verifsiret_apikey', '')
    secretKey = config.get('ckanext.ozwillo_organization_api.verifsiret_secretkey', '')
    url = "http://www.verif-siret.com/api/siret?siret=" + siret
    result = None

    if apiKey == '' or secretKey == '':
        log.error('Verif-siret config incomplete, please register your api key and api secret in the config file')
        result = ''
        return result

    try:
        get = requests.get(url, auth=(apiKey, secretKey))
        if get.status_code != 200:
            raise requests.ConnectionError()
    except requests.exceptions.RequestException as err:
        log.error('An error occurred: {0}'.format(err))
    else:
        try:
            result = get.json()['array_return'][0]['LIBCOM']
        except (IndexError, TypeError, AttributeError):
            log.error('No organization found for this siret number')
    finally:
        return result
Example #17
0
    def send_mail(self,to,subject, _body):              
        body = _body.replace('\n', '\r\n')
        guid=str(uuid.uuid4())
        body = config.get('ckan.site_url') + "/user/register?guid=" + guid + "&activate"
        # Prepare actual message
        message = """From: %s
        To: %s
        Subject: %s

        %s 
        """ % (config.get('dk.aarhuskommune.odaa_from'),str(to),subject,body)

        try:
            result=self.connection()       
            connectString = """dbname='%s' user='******' host='%s' password='******'""" % (result["database"],result["user"],result["host"],result["password"])
            conn = psycopg2.connect(connectString)
            c = conn.cursor()            
            p=request.params["password1"]            
            password=self._encode(result["password"],p)
            now = datetime.datetime.now()                                    
            sql="INSERT INTO mailnotifikation VALUES ('" + guid + "','" + str(now) + "','" + request.params["name"] + "','" +request.params["fullname"] +   "','" + request.params["email"] + "','" + password + "','False')"
            c.execute(sql)
	    conn.commit()

            nowlm = now - datetime.timedelta(days=int(config.get('dk.aarhuskommune.odaa_days')))
            sNowlm=nowlm.strftime('%Y-%m-%d')            
            sql="delete from mailnotifikation where _date<'%s'" % (sNowlm);           
            c.execute(sql)
	    conn.commit()
            conn.close()
            smtpObj = smtplib.SMTP('localhost')        
            to=to.split(',')
            smtpObj.sendmail(config.get('dk.aarhuskommune.odaa_from'), to, message)
        except Exception as e:
            logging.error('Error: unable to send email. %s ',e)
Example #18
0
 def __before__(self):
     "before"
     if 'theme' not in session:
         session['theme'] = ''
         basedir = config.get('baruwa.themes.base', None)
         if basedir:
             # Default theme
             defaultdir = os.path.join(basedir, 'templates', 'default')
             if os.path.exists(defaultdir):
                 session['theme'] = 'default'
             # Host theme
             themedir = os.path.join(basedir, 'templates',
                         request.server_name)
             if os.path.exists(themedir):
                 session['theme'] = request.server_name
         session.save()
     self.theme = session.get('theme')
     if 'lang' in session:
         set_lang(session['lang'])
     else:
         try:
             languages = [lang.split('-')[0] for lang in request.languages
                     if check_language(lang.split('-')[0])]
             set_lang(languages)
         except AttributeError:
             default_lang = config.get('baruwa.default.language', 'en')
             if check_language(default_lang):
                 set_lang([default_lang])
             else:
                 set_lang(['en'])
     # pylint: disable-msg=W0201
     self.invalidate = request.GET.get('uc', None)
     self.langchange = request.GET.get('lc', None)
Example #19
0
    def before_search(self, search_params):
        lang_set = set(LANGS)

        try:
            current_lang = pylons.request.environ['CKAN_LANG']
        except TypeError as err:
            if err.message == ('No object (name: request) has been registered '
                               'for this thread'):
                # This happens when this code gets called as part of a paster
                # command rather then as part of an HTTP request.
                current_lang = config.get('ckan.locale_default')
            else:
                raise

        # fallback to default locale if locale not in suported langs
        if not current_lang in lang_set:
            current_lang = config.get('ckan.locale_default')
        # fallback to english if default locale is not supported
        if not current_lang in lang_set:
            current_lang = 'en'
        # treat current lang differenly so remove from set
        lang_set.remove(current_lang)

        # weight current lang more highly
        query_fields = 'title_%s^8 text_%s^4' % (current_lang, current_lang)

        for lang in lang_set:
            query_fields += ' title_%s^2 text_%s' % (lang, lang)

        search_params['qf'] = query_fields

        return search_params
Example #20
0
    def getPosts(self, ident):

        thread = self.getThread(ident)
        request = thread
        fresh_thread = False

        if thread['code'] != 0:
            thread = self.createThreadFromIdent(ident)
            fresh_thread = True

        if thread['code'] == 0:
            thread_id = thread['response']['id']
            api_key = config.get('edcdisqus.api_key')
            forum_name = config.get('edcdisqus.forum_name')
            data_string = urllib.urlencode({'forum': forum_name, 'thread': thread_id, 'api_key': api_key, 'limit': 100 })
            request_json = self.request('https://disqus.com/api/3.0/posts/list.json', data_string, 'get')
            request = json.loads(request_json)

            # For some reason, the disqus API likes to return all of the comments when a thread id doesn't exist.  we're just emptying the result set
            # when a new thread is created.  (There shouldn't be any comments anyway, the thread was just made!)
            if fresh_thread:
                request['response'] = []

        else:
            request = thread

        return request
Example #21
0
 def setup_class(cls):
     reset_db()
     config
     cls.fake_context = {
         'site_url': config.get('ckan.site_url_internally') or config['ckan.site_url'],
         'cache_url_root': config.get('ckanext-archiver.cache_url_root'),
     }
Example #22
0
def _mail_recipient(recipient_name, recipient_email,
        sender_name, sender_url, subject,
        body, headers={}):
    mail_from = config.get('ckan.mail_from')
    body = add_msg_niceties(recipient_name, body, sender_name, sender_url)
    msg = MIMEText(body.encode('utf-8'), 'plain', 'utf-8')
    for k, v in headers.items(): msg[k] = v
    subject = Header(subject.encode('utf-8'), 'utf-8')
    msg['Subject'] = subject
    msg['From'] = _("%s <%s>") % (sender_name, mail_from)
    recipient = u"%s <%s>" % (recipient_name, recipient_email)
    msg['To'] = Header(recipient, 'utf-8')
    msg['Date'] = Utils.formatdate(time())
    msg['X-Mailer'] = "CKAN %s" % __version__
    try:
        server = smtplib.SMTP(
            config.get('test_smtp_server',
                       config.get('smtp_server', 'localhost')))
        #server.set_debuglevel(1)
        server.sendmail(mail_from, [recipient_email], msg.as_string())
        server.quit()
    except Exception, e:
        msg = '%r' % e
        log.exception(msg)
        raise MailerException(msg)
Example #23
0
    def _get_allowed_sender_options(cls, user):
        sender_options = {
            'user': {
                'email': user.email,
                'checked': False,
                'enabled': user.is_email_activated(),
                'reason': _("Email isn't activated"),
            },
            'system': {
                'email': config.get('adhocracy.email.from'),
                'checked': False,
                'enabled': asbool(config.get(
                    'allow_system_email_in_mass_messages', 'true')),
                'reason': _("Not permitted in system settings"),
            },
            'support': {
                'email': config.get('adhocracy.registration_support_email'),
                'checked': False,
                'enabled': (config.get('adhocracy.registration_support_email')
                            is not None),
                'reason': _("adhocracy.registration_support_email not set"),
            }
        }

        if sender_options['user']['enabled']:
            sender_options['user']['checked'] = True
        elif sender_options['system']['enabled']:
            sender_options['system']['checked'] = True

        return sender_options
Example #24
0
    def __init__(self):

        connect_string = config.get("linotpOpenID.sql.url")

        implicit_returning = config.get("linotpSQL.implicit_returning", True)
        self.engine = None

        if connect_string is None:
            log.info("[__init__] Missing linotpOpenID.sql.url parameter in "
                     "config file! Using the sqlalchemy.url")
            # raise Exception("Missing linotpOpenID.sql.url parameter in "
            # "config file!")
            connect_string = config.get("sqlalchemy.url")
        ########################## SESSION ##################################

        # Create an engine and create all the tables we need
        if implicit_returning:
            # If implicit_returning is explicitly set to True, we
            # get lots of mysql errors:
            # AttributeError: 'MySQLCompiler_mysqldb' object has no attribute
            # 'returning_clause' So we do not mention explicit_returning at all
            self.engine = create_engine(connect_string)
        else:
            self.engine = create_engine(connect_string,
                                        implicit_returning=False)

        metadata.bind = self.engine
        metadata.create_all()

        # Set up the session
        self.sm = orm.sessionmaker(bind=self.engine, autoflush=True,
                                   autocommit=False,
            expire_on_commit=True)
        self.session = orm.scoped_session(self.sm)
Example #25
0
def shorten_link(long_url):
    longUrl = cgi.escape(long_url)
    bitly_userid= config.get('bitly.userid')
    bitly_key = config.get('bitly.key')
    bitly_result = urllib.urlopen("http://api.bit.ly/v3/shorten?login=%(bitly_userid)s&apiKey=%(bitly_key)s&longUrl=%(longUrl)s&format=json" % locals()).read()
    bitly_data = json.loads(bitly_result)['data']
    return bitly_data["url"]
Example #26
0
def user_language(user, fallbacks=[]):
    # find out the locale
    locale = None
    if user and user.locale:
        locale = user.locale

    if locale is None:
        locales = map(str, LOCALES)
        locale = Locale.parse(Locale.negotiate(fallbacks, locales)) \
                 or get_default_locale()

    # determinate from which path we load the translations
    translations_module = config.get('adhocracy.translations', 'adhocracy')
    translations_module_loader = pkgutil.get_loader(translations_module)
    if translations_module_loader is None:
        raise ValueError(('Cannot import the module "%s" configured for '
                          '"adhocracy.translations". Make sure it is an '
                          'importable module (and contains the '
                          'translation files in a subdirectory '
                          '"i18n"') % translations_module)

    translations_root = translations_module_loader.filename
    translations_config = {'pylons.paths': {'root': translations_root},
                           'pylons.package': config.get('pylons.package')}

    # set language and fallback
    set_lang(locale.language, pylons_config=translations_config)
    add_fallback(get_default_locale().language,
                 pylons_config=translations_config)
    formencode.api.set_stdtranslation(domain="FormEncode",
                                      languages=[locale.language])
    return locale
Example #27
0
 def checkTarget(environ, result):
     back = parseBoolString(config.get('ckan.gov_theme.is_back', False));
     ignoreList =  config.get('ckan.api.ignore', 'NotInList')
     path = environ['PATH_INFO'].split("/")[-1]
     if  back or ignoreList.find(path) > 0:
         return False
     return True
Example #28
0
def _mail_recipient(recipient_name, recipient_email,
        sender_name, sender_url, subject,
        body, headers={}):
    mail_from = config.get('smtp.mail_from')
    body = add_msg_niceties(recipient_name, body, sender_name, sender_url)
    msg = MIMEText(body.encode('utf-8'), 'plain', 'utf-8')
    for k, v in headers.items(): msg[k] = v
    subject = Header(subject.encode('utf-8'), 'utf-8')
    msg['Subject'] = subject
    msg['From'] = _("%s <%s>") % (sender_name, mail_from)
    recipient = u"%s <%s>" % (recipient_name, recipient_email)
    msg['To'] = Header(recipient, 'utf-8')
    msg['Date'] = Utils.formatdate(time())
    msg['X-Mailer'] = "CKAN %s" % ckan.__version__

    # Send the email using Python's smtplib.
    smtp_connection = smtplib.SMTP()
    if 'smtp.test_server' in config:
        # If 'smtp.test_server' is configured we assume we're running tests,
        # and don't use the smtp.server, starttls, user, password etc. options.
        smtp_server = config['smtp.test_server']
        smtp_starttls = False
        smtp_user = None
        smtp_password = None
    else:
        smtp_server = config.get('smtp.server', 'localhost')
        smtp_starttls = paste.deploy.converters.asbool(
                config.get('smtp.starttls'))
        smtp_user = config.get('smtp.user')
        smtp_password = config.get('smtp.password')
    smtp_connection.connect(smtp_server)
    try:
        #smtp_connection.set_debuglevel(True)

        # Identify ourselves and prompt the server for supported features.
        smtp_connection.ehlo()

        # If 'smtp.starttls' is on in CKAN config, try to put the SMTP
        # connection into TLS mode.
        if smtp_starttls:
            if smtp_connection.has_extn('STARTTLS'):
                smtp_connection.starttls()
                # Re-identify ourselves over TLS connection.
                smtp_connection.ehlo()
            else:
                raise MailerException("SMTP server does not support STARTTLS")

        # If 'smtp.user' is in CKAN config, try to login to SMTP server.
        if smtp_user:
            assert smtp_password, ("If smtp.user is configured then "
                    "smtp.password must be configured as well.")
            smtp_connection.login(smtp_user, smtp_password)

        smtp_connection.sendmail(mail_from, [recipient_email], msg.as_string())
        log.info("Sent email to {0}".format(recipient_email))

    except smtplib.SMTPException, e:
        msg = '%r' % e
        log.exception(msg)
        raise MailerException(msg)
Example #29
0
    def all(cls):
        """Return a sequence of all available domains."""

        return [Domain(n, os.path.join(config.get('herder.po_dir'), n)) 
                for n in os.listdir(config.get('herder.po_dir'))
                if n not in cls._IGNORE_DIRS and 
                   os.path.isdir(os.path.join(config.get('herder.po_dir'), n))]
Example #30
0
def harvest_source_index_clear(context, data_dict):
    '''
    Clears all datasets, jobs and objects related to a harvest source, but
    keeps the source itself.  This is useful to clean history of long running
    harvest sources to start again fresh.

    :param id: the id of the harvest source to clear
    :type id: string
    '''

    check_access('harvest_source_clear', context, data_dict)
    harvest_source_id = data_dict.get('id')

    source = HarvestSource.get(harvest_source_id)
    if not source:
        log.error('Harvest source %s does not exist', harvest_source_id)
        raise NotFound('Harvest source %s does not exist' % harvest_source_id)

    harvest_source_id = source.id

    conn = make_connection()
    query = ''' +%s:"%s" +site_id:"%s" ''' % (
        'harvest_source_id', harvest_source_id, config.get('ckan.site_id'))
    try:
        conn.delete_query(query)
        if asbool(config.get('ckan.search.solr_commit', 'true')):
            conn.commit()
    except Exception, e:
        log.exception(e)
        raise SearchIndexError(e)
    def search(self):
        package_type = self._guess_package_type()
        try:
            context = {
                'model': model,
                'user': c.user or c.author,
                'auth_user_obj': c.userobj
            }
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = h.get_page_number(request.params)
        limit = int(config.get('ckan.datasets_per_page', 20))
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package',
                                   action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key,
                                      value=value,
                                      replace=replace,
                                      controller='package',
                                      action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            params = params_nosort[:]
            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [
                field.split()[0] for field in sort_by.split(',')
            ]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and value and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        # Modificación para andino: usamos una función para buscar dependencias entre organizaciones
                        if param != 'organization':
                            fq += ' %s:"%s"' % (param, value)
                        else:
                            fq += custom_organization_filter(value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {
                'model': model,
                'session': model.Session,
                'user': c.user or c.author,
                'for_view': True,
                'auth_user_obj': c.userobj
            }

            if package_type and package_type != 'dataset':
                fq += ' +dataset_type:{type}'.format(type=package_type)
            elif not asbool(config.get('ckan.search.show_all_types', 'False')):
                fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in h.facets():
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            query = logic.action.get.package_search(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(collection=query['results'],
                            page=page,
                            url=pager_url,
                            item_count=query['count'],
                            items_per_page=limit)
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.search_facets = {}
            c.page = h.Page(collection=[])
Example #32
0
def get_syndicated_name_prefix():
    return config.get('ckan.syndicate.name_prefix', '')
Example #33
0
def get_syndicated_author():
    return config.get('ckan.syndicate.author')
Example #34
0
def get_syndicated_id():
    return config.get('ckan.syndicate.id', 'syndicated_id')
Example #35
0
def get_syndicate_flag():
    return config.get('ckan.syndicate.flag', 'syndicate')
Example #36
0
import sys
import cgitb
import warnings


def text_traceback():
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        res = 'the original traceback:'.join(
            cgitb.text(
                sys.exc_info()).split('the original traceback:')[1:]).strip()
    return res


SIMPLE_SEARCH = config.get('ckan.simple_search', False)

SUPPORTED_SCHEMA_VERSIONS = ['1.3']

DEFAULT_OPTIONS = {
    'limit': 20,
    'offset': 0,
    # about presenting the results
    'order_by': 'rank',
    'return_objects': False,
    'ref_entity_with_attr': 'name',
    'all_fields': False,
    'search_tags': True,
    'callback': None,  # simply passed through
}
Example #37
0
import urllib2
import pylons.config as config
import base64
from urllib2 import HTTPError

# doc https://team.eea.sk/wiki/pages/viewpage.action?pageId=108660564
# TODO POST /pipelines/<pipeline_id>/schedules
# TODO GET /pipelines/<pipeline_id>/schedules/<id>
# TODO GET /pipelines/<pipeline_id>/executions
# TODO /pipelines/<pipeline_id>/executions/<execution_id>
# TODO /pipelines/<pipeline_id>/executions
# TODO /pipelines/<pipeline_id>/executions/<execution_id>/events

# TODO this class to ckancommons

TIMEOUT = int(config.get(u'odn.uv.timeout', 5))
AUTH_HEADER_FIELD_NAME = u'Authorization'
USER_EXT_ID = u'userExternalId'
USER_ACTOR_EXT_ID = u'userActorExternalId'

log = logging.getLogger('ckanext')


class UVRestAPIWrapper():
    def __init__(self, uv_url, auth=None):
        assert uv_url
        self.url = uv_url
        self.auth = None
        if auth:
            self.auth = base64.b64encode(auth)
    def new_resource(self, id, data=None, errors=None, error_summary=None):
        ''' FIXME: This is a temporary action to allow styling of the
        forms. '''
        if request.method == 'POST' and not data:
            save_action = request.params.get('save')
            data = data or clean_dict(
                dict_fns.unflatten(tuplize_dict(parse_params(request.POST))))
            # we don't want to include save as it is part of the form
            del data['save']
            resource_id = data['id']
            del data['id']

            # Guardo los campos issued y modified
            time_now = moment.now().isoformat()
            data['issued'] = time_now
            data['modified'] = time_now

            self._validate_resource(data)

            context = {
                'model': model,
                'session': model.Session,
                'user': c.user,
                'auth_user_obj': c.userobj
            }

            if save_action == 'go-dataset':
                # go to first stage of add dataset
                h.redirect_to(controller='package', action='edit', id=id)

            # see if we have any data that we are trying to save
            data_provided = False
            for key, value in data.iteritems():
                if ((value
                     or isinstance(value, cgi.FieldStorage)) and key not in [
                         'resource_type', 'license_id', 'attributesDescription'
                     ]):
                    data_provided = True
                    break

            if not data_provided and save_action != "go-dataset-complete":
                if save_action == 'go-dataset':
                    # go to first stage of add dataset
                    h.redirect_to(controller='package', action='edit', id=id)
                try:
                    data_dict = get_action('package_show')(context, {'id': id})
                except NotAuthorized:
                    abort(403, _('Unauthorized to update dataset'))
                except NotFound:
                    abort(
                        404,
                        _('The dataset {id} could not be found.').format(
                            id=id))
                if not len(data_dict['resources']):
                    # no data so keep on page
                    msg = _('You must add at least one data resource')
                    # On new templates do not use flash message

                    if asbool(config.get('ckan.legacy_templates')):
                        h.flash_error(msg)
                        h.redirect_to(controller='package',
                                      action='new_resource',
                                      id=id)
                    else:
                        errors = {}
                        error_summary = {_('Error'): msg}
                        return self.new_resource(id, data, errors,
                                                 error_summary)
                # XXX race condition if another user edits/deletes
                data_dict = get_action('package_show')(context, {'id': id})
                get_action('package_update')(dict(context,
                                                  allow_state_change=True),
                                             dict(data_dict, state='active'))
                h.redirect_to(controller='package', action='read', id=id)

            data['package_id'] = id
            try:
                if resource_id:
                    data['id'] = resource_id
                    get_action('resource_update')(context, data)
                else:
                    get_action('resource_create')(context, data)
            except ValidationError, e:
                errors = e.error_dict
                error_summary = e.error_summary
                return self.new_resource(id, data, errors, error_summary)
            except NotAuthorized:
                abort(403, _('Unauthorized to create a resource'))
Example #39
0
def load_environment(global_conf, app_conf):
    """Configure the Pylons environment via the ``pylons.config``
    object
    """

    ######  Pylons monkey-patch
    # this must be run at a time when the env is semi-setup, thus inlined here.
    # Required by the deliverance plugin and iATI
    from pylons.wsgiapp import PylonsApp
    import pkg_resources
    find_controller_generic = PylonsApp.find_controller

    # This is from pylons 1.0 source, will monkey-patch into 0.9.7
    def find_controller(self, controller):
        if controller in self.controller_classes:
            return self.controller_classes[controller]
        # Check to see if its a dotted name
        if '.' in controller or ':' in controller:
            mycontroller = pkg_resources.EntryPoint.parse(
                'x=%s' % controller).load(False)
            self.controller_classes[controller] = mycontroller
            return mycontroller
        return find_controller_generic(self, controller)

    PylonsApp.find_controller = find_controller
    ###### END evil monkey-patch

    os.environ['CKAN_CONFIG'] = global_conf['__file__']

    # Pylons paths
    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    paths = dict(root=root,
                 controllers=os.path.join(root, 'controllers'),
                 static_files=os.path.join(root, 'public'),
                 templates=[os.path.join(root, 'templates')])

    # Initialize config with the basic options

    config.init_app(global_conf, app_conf, package='ckan', paths=paths)

    # load all CKAN plugins
    plugins.load_all(config)

    from ckan.plugins import PluginImplementations
    from ckan.plugins.interfaces import IConfigurer

    for plugin in PluginImplementations(IConfigurer):
        # must do update in place as this does not work:
        # config = plugin.update_config(config)
        plugin.update_config(config)

    # This is set up before globals are initialized
    site_id = os.environ.get('CKAN_SITE_ID')
    if site_id:
        config['ckan.site_id'] = site_id

    site_url = config.get('ckan.site_url', '')
    ckan_host = config['ckan.host'] = urlparse(site_url).netloc
    if config.get('ckan.site_id') is None:
        if ':' in ckan_host:
            ckan_host, port = ckan_host.split(':')
        assert ckan_host, 'You need to configure ckan.site_url or ' \
                          'ckan.site_id for SOLR search-index rebuild to work.'
        config['ckan.site_id'] = ckan_host

    # Init SOLR settings and check if the schema is compatible
    from ckan.lib.search import SolrSettings, check_solr_schema_version
    SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
                      config.get('solr_password'))
    check_solr_schema_version()

    config['routes.map'] = make_map()
    config['pylons.app_globals'] = app_globals.Globals()
    if asbool(config.get('ckan.restrict_template_vars', 'false')):
        import ckan.lib.helpers_clean
        config['pylons.h'] = ckan.lib.helpers_clean
    else:
        config['pylons.h'] = h

    ## redo template setup to use genshi.search_path (so remove std template setup)
    template_paths = [paths['templates'][0]]
    extra_template_paths = config.get('extra_template_paths', '')
    if extra_template_paths:
        # must be first for them to override defaults
        template_paths = extra_template_paths.split(',') + template_paths

    # Translator (i18n)
    translator = Translator(pylons.translator)

    def template_loaded(template):
        translator.setup(template)

    # Markdown ignores the logger config, so to get rid of excessive
    # markdown debug messages in the log, set it to the level of the
    # root logger.
    logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)

    # Create the Genshi TemplateLoader
    config['pylons.app_globals'].genshi_loader = TemplateLoader(
        template_paths, auto_reload=True, callback=template_loaded)

    # CONFIGURATION OPTIONS HERE (note: all config options will override
    # any Pylons config options)

    # Setup the SQLAlchemy database engine
    # Suppress a couple of sqlalchemy warnings
    warnings.filterwarnings(
        'ignore', '^Unicode type received non-unicode bind param value',
        sqlalchemy.exc.SAWarning)
    warnings.filterwarnings(
        'ignore', "^Did not recognize type 'BIGINT' of column 'size'",
        sqlalchemy.exc.SAWarning)
    warnings.filterwarnings(
        'ignore',
        "^Did not recognize type 'tsvector' of column 'search_vector'",
        sqlalchemy.exc.SAWarning)

    ckan_db = os.environ.get('CKAN_DB')

    if ckan_db:
        engine = sqlalchemy.create_engine(ckan_db)
    else:
        engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')

    if not model.meta.engine:
        model.init_model(engine)

    from ckan.plugins import PluginImplementations
    from ckan.plugins.interfaces import IConfigurable

    for plugin in PluginImplementations(IConfigurable):
        plugin.configure(config)
class GobArPackageController(PackageController):
    def __generate_spatial_extra_field(self, data_dict):
        extras = data_dict['extras']

        def __find_extra(extras, key, create=False):
            for extra in extras:
                if extra['key'] == key:
                    return extra

            if create:
                extra = {'key': key}
                extras.append(extra)
                return extra

        country = __find_extra(extras, 'country')
        if country:
            spatial = __find_extra(extras, 'spatial', True)
            spatial['value'] = [country['value']]

            province = __find_extra(extras, 'province')
            if province and province['value']:
                spatial['value'].append(province['value'])

            district = __find_extra(extras, 'district')
            if district and district['value']:
                spatial['value'].append(district['value'])

            spatial['value'] = ','.join(spatial['value'])

    def search(self):
        package_type = self._guess_package_type()
        try:
            context = {
                'model': model,
                'user': c.user or c.author,
                'auth_user_obj': c.userobj
            }
            check_access('site_read', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))
        q = c.q = request.params.get('q', u'')
        c.query_error = False
        page = h.get_page_number(request.params)
        limit = int(config.get('ckan.datasets_per_page', 20))
        params_nopage = [(k, v) for k, v in request.params.items()
                         if k != 'page']

        def drill_down_url(alternative_url=None, **by):
            return h.add_url_param(alternative_url=alternative_url,
                                   controller='package',
                                   action='search',
                                   new_params=by)

        c.drill_down_url = drill_down_url

        def remove_field(key, value=None, replace=None):
            return h.remove_url_param(key,
                                      value=value,
                                      replace=replace,
                                      controller='package',
                                      action='search')

        c.remove_field = remove_field

        sort_by = request.params.get('sort', None)
        params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']

        def _sort_by(fields):
            params = params_nosort[:]
            if fields:
                sort_string = ', '.join('%s %s' % f for f in fields)
                params.append(('sort', sort_string))
            return search_url(params, package_type)

        c.sort_by = _sort_by
        if not sort_by:
            c.sort_by_fields = []
        else:
            c.sort_by_fields = [
                field.split()[0] for field in sort_by.split(',')
            ]

        def pager_url(q=None, page=None):
            params = list(params_nopage)
            params.append(('page', page))
            return search_url(params, package_type)

        c.search_url_params = urlencode(_encode_params(params_nopage))

        try:
            c.fields = []
            c.fields_grouped = {}
            search_extras = {}
            fq = ''
            for (param, value) in request.params.items():
                if param not in ['q', 'page', 'sort'] \
                        and value and not param.startswith('_'):
                    if not param.startswith('ext_'):
                        c.fields.append((param, value))
                        # Modificación para andino: usamos una función para buscar dependencias entre organizaciones
                        if param != 'organization':
                            fq += ' %s:"%s"' % (param, value)
                        else:
                            fq += custom_organization_filter(value)
                        if param not in c.fields_grouped:
                            c.fields_grouped[param] = [value]
                        else:
                            c.fields_grouped[param].append(value)
                    else:
                        search_extras[param] = value

            context = {
                'model': model,
                'session': model.Session,
                'user': c.user or c.author,
                'for_view': True,
                'auth_user_obj': c.userobj
            }

            if package_type and package_type != 'dataset':
                fq += ' +dataset_type:{type}'.format(type=package_type)
            elif not asbool(config.get('ckan.search.show_all_types', 'False')):
                fq += ' +dataset_type:dataset'

            facets = OrderedDict()

            default_facet_titles = {
                'organization': _('Organizations'),
                'groups': _('Groups'),
                'tags': _('Tags'),
                'res_format': _('Formats'),
                'license_id': _('Licenses'),
            }

            for facet in h.facets():
                if facet in default_facet_titles:
                    facets[facet] = default_facet_titles[facet]
                else:
                    facets[facet] = facet

            for plugin in p.PluginImplementations(p.IFacets):
                facets = plugin.dataset_facets(facets, package_type)

            c.facet_titles = facets

            data_dict = {
                'q': q,
                'fq': fq.strip(),
                'facet.field': facets.keys(),
                'rows': limit,
                'start': (page - 1) * limit,
                'sort': sort_by,
                'extras': search_extras
            }

            query = logic.action.get.package_search(context, data_dict)
            c.sort_by_selected = query['sort']

            c.page = h.Page(collection=query['results'],
                            page=page,
                            url=pager_url,
                            item_count=query['count'],
                            items_per_page=limit)
            c.search_facets = query['search_facets']
            c.page.items = query['results']
        except SearchError, se:
            log.error('Dataset search error: %r', se.args)
            c.query_error = True
            c.search_facets = {}
            c.page = h.Page(collection=[])
        c.search_facets_limits = {}
        for facet in c.search_facets.keys():
            try:
                # Modificación para andino: chequeo si el facet es 'organization'
                if facet != 'organization':
                    limit = int(
                        request.params.get(
                            '_%s_limit' % facet,
                            int(config.get('search.facets.default', 10))))
                else:
                    limit = None
            except ValueError:
                error_description = _(
                    'Parameter "{parameter_name}" is not an integer')
                error_description = error_description.format(
                    parameter_name='_%s_limit' % facet)
                abort(400, error_description)
            c.search_facets_limits[facet] = limit

        self._setup_template_variables(context, {}, package_type=package_type)

        return render(self._search_template(package_type),
                      extra_vars={'dataset_type': package_type})
Example #41
0
def drupal_site_url():
  from pylons import config
  return config.get('drupal.site_url')
Example #42
0
    def index_package(self, pkg_dict, defer_commit=False):
        import solr.core
        if pkg_dict is None:
            return

        pkg_dict['data_dict'] = json.dumps(pkg_dict)

        if config.get('ckan.cache_validated_datasets', True):
            package_plugin = lib_plugins.lookup_package_plugin(
                pkg_dict.get('type'))

            schema = package_plugin.show_package_schema()
            validated_pkg_dict, errors = _validate(pkg_dict, schema, {
                'model': model,
                'session': model.Session
            })
            pkg_dict['validated_data_dict'] = json.dumps(
                validated_pkg_dict,
                cls=ckan.lib.navl.dictization_functions.MissingNullEncoder)

        # add to string field for sorting
        title = pkg_dict.get('title')
        if title:
            pkg_dict['title_string'] = title

        if (not pkg_dict.get('state')) or ('active'
                                           not in pkg_dict.get('state')):
            return self.delete_package(pkg_dict)

        index_fields = RESERVED_FIELDS + pkg_dict.keys()

        # include the extras in the main namespace
        extras = pkg_dict.get('extras', [])
        for extra in extras:
            key, value = extra['key'], extra['value']
            if isinstance(value, (tuple, list)):
                value = " ".join(map(unicode, value))
            key = ''.join([c for c in key if c in KEY_CHARS])
            pkg_dict['extras_' + key] = value
            if key not in index_fields:
                pkg_dict[key] = value
        pkg_dict.pop('extras', None)

        # add tags, removing vocab tags from 'tags' list and adding them as
        # vocab_<tag name> so that they can be used in facets
        non_vocab_tag_names = []
        tags = pkg_dict.pop('tags', [])
        context = {'model': model}

        for tag in tags:
            if tag.get('vocabulary_id'):
                data = {'id': tag['vocabulary_id']}
                vocab = logic.get_action('vocabulary_show')(context, data)
                key = u'vocab_%s' % vocab['name']
                if key in pkg_dict:
                    pkg_dict[key].append(tag['name'])
                else:
                    pkg_dict[key] = [tag['name']]
            else:
                non_vocab_tag_names.append(tag['name'])

        pkg_dict['tags'] = non_vocab_tag_names

        # add groups
        groups = pkg_dict.pop('groups', [])

        # we use the capacity to make things private in the search index
        if pkg_dict['private']:
            pkg_dict['capacity'] = 'private'
        else:
            pkg_dict['capacity'] = 'public'

        pkg_dict['groups'] = [group['name'] for group in groups]

        # if there is an owner_org we want to add this to groups for index
        # purposes
        if pkg_dict.get('organization'):
            pkg_dict['organization'] = pkg_dict['organization']['name']
        else:
            pkg_dict['organization'] = None

        # tracking
        tracking_summary = pkg_dict.pop('tracking_summary', None)
        if tracking_summary:
            pkg_dict['views_total'] = tracking_summary['total']
            pkg_dict['views_recent'] = tracking_summary['recent']

        # flatten the structure for indexing:
        for resource in pkg_dict.get('resources', []):
            for (okey, nkey) in [('description', 'res_description'),
                                 ('format', 'res_format'), ('url', 'res_url')]:
                pkg_dict[nkey] = pkg_dict.get(nkey,
                                              []) + [resource.get(okey, u'')]
        pkg_dict.pop('resources', None)

        rel_dict = collections.defaultdict(list)
        subjects = pkg_dict.pop("relationships_as_subject", [])
        objects = pkg_dict.pop("relationships_as_object", [])
        for rel in objects:
            type = model.PackageRelationship.forward_to_reverse_type(
                rel['type'])
            rel_dict[type].append(
                model.Package.get(rel['subject_package_id']).name)
        for rel in subjects:
            type = rel['type']
            rel_dict[type].append(
                model.Package.get(rel['object_package_id']).name)
        for key, value in rel_dict.iteritems():
            if key not in pkg_dict:
                pkg_dict[key] = value

        pkg_dict[TYPE_FIELD] = PACKAGE_TYPE

        # Save dataset type
        pkg_dict['dataset_type'] = pkg_dict['type']

        # clean the dict fixing keys and dates
        # FIXME where are we getting these dirty keys from?  can we not just
        # fix them in the correct place or is this something that always will
        # be needed?  For my data not changing the keys seems to not cause a
        # problem.
        new_dict = {}
        for key, value in pkg_dict.items():
            key = key.encode('ascii', 'ignore')
            if key.endswith('_date'):
                try:
                    value = parse(value).isoformat() + 'Z'
                except ValueError:
                    continue
            new_dict[key] = value
        pkg_dict = new_dict

        for k in ('title', 'notes', 'title_string'):
            if k in pkg_dict and pkg_dict[k]:
                pkg_dict[k] = escape_xml_illegal_chars(pkg_dict[k])

        # modify dates (SOLR is quite picky with dates, and only accepts ISO dates
        # with UTC time (i.e trailing Z)
        # See http://lucene.apache.org/solr/api/org/apache/solr/schema/DateField.html
        pkg_dict['metadata_created'] += 'Z'
        pkg_dict['metadata_modified'] += 'Z'

        # mark this CKAN instance as data source:
        pkg_dict['site_id'] = config.get('ckan.site_id')

        # Strip a selection of the fields.
        # These fields are possible candidates for sorting search results on,
        # so we strip leading spaces because solr will sort " " before "a" or "A".
        for field_name in ['title']:
            try:
                value = pkg_dict.get(field_name)
                if value:
                    pkg_dict[field_name] = value.lstrip()
            except KeyError:
                pass

        # add a unique index_id to avoid conflicts
        import hashlib
        pkg_dict['index_id'] = hashlib.md5(
            '%s%s' % (pkg_dict['id'], config.get('ckan.site_id'))).hexdigest()

        for item in PluginImplementations(IPackageController):
            pkg_dict = item.before_index(pkg_dict)

        assert pkg_dict, 'Plugin must return non empty package dict on index'

        # send to solr:
        try:
            conn = make_connection()
            commit = not defer_commit
            if not asbool(config.get('ckan.search.solr_commit', 'true')):
                commit = False
            conn.add_many([pkg_dict], _commit=commit)
        except solr.core.SolrException, e:
            msg = 'Solr returned an error: {0} {1} - {2}'.format(
                e.httpcode,
                e.reason,
                e.body[:1000]  # limit huge responses
            )
            raise SearchIndexError(msg)
Example #43
0
def drupal_get_secondary_header():
  import urllib2
  from pylons import config
  secondary_header = urllib2.urlopen(config.get('drupal.site_url') + '/koop_theme/get/secondary_header')
  return secondary_header.read().decode('utf-8')
Example #44
0
def drupal_get_dataset_quality(table, id):
  import urllib2
  from pylons import config
  text = urllib2.urlopen(config.get('drupal.site_url') + '/service/linkcheck/' + table + '/' + id)
  return text.read().decode('utf-8')
Example #45
0
    def configure(self, config):
        from ckanext.spatial.model.package_extent import setup as setup_model

        if not p.toolkit.asbool(config.get('ckan.spatial.testing', 'False')):
            log.debug('Setting up the spatial model')
            setup_model()
Example #46
0
def drupal_get_footer():
  import urllib2
  from pylons import config
  footer = urllib2.urlopen(config.get('drupal.site_url') + '/koop_theme/get/footer')
  return footer.read().decode('utf-8')
Example #47
0
    def validator(key, data, errors, context):
        if errors[key]:
            return

        value = data[key]
        if value is not missing:
            if value:
                return

        output = {}

        prefix = field['autogeneration_field']
        if not prefix:
            prefix = DEFAULT_TITLE_FIELD

        log.debug('[multilanguage_url] Creating field using the field %s', prefix)

        prefix = prefix + '-'

        extras = data.get(key[:-1] + ('__extras',), {})

        locale_default = config.get('ckan.locale_default', 'es')
        if locale_default:
            title_lang = prefix + locale_default

            if title_lang in extras and extras[title_lang]:
                dataset_title = extras[title_lang]

                # Generate title prefix
                field_prefix = field['organization_field']
                organization_prefix = field['organization_prefix']

                publisher_id = data.get((field_prefix,))
                if not publisher_id and field_prefix in extras:
                    publisher_id = extras[field_prefix]

                if publisher_id and organization_prefix:

                    organization = h.get_organization(publisher_id)
                    if not organization:
                        organization = toolkit.get_action('dge_organization_publisher')({'model': model}, {'id': publisher_id})
                    if organization and organization['extras']:
                        for extra in organization['extras']:
                            if extra['key'] == organization_prefix and extra['state'] == 'active':
                                dataset_title = extra['value'] + '-' + dataset_title
                                break
                data[key] = munge.munge_title_to_name(dataset_title)

                log.debug('[multilanguage_url] Created name "%s" for package from language %s',
                            data[key], locale_default)
                return

        locale_order = config.get('ckan.locale_order', '').split()
        for l in locale_order:
            title_lang = prefix + l
            if title_lang in extras and extras[title_lang]:
                dataset_title = extras[title_lang]

                # Generate title prefix
                field_prefix = field['organization_field']
                organization_prefix = field['organization_prefix']

                publisher_id = data.get((field_prefix,))
                if not publisher_id and field_prefix in extras:
                    publisher_id = extras[field_prefix]

                if publisher_id and organization_prefix:
                    organization = h.get_organization(publisher_id)
                    if not organization:
                        organization = toolkit.get_action('dge_organization_publisher')({'model': model}, {'id': publisher_id})
                    if organization and organization['extras']:
                        for extra in organization['extras']:
                            if extra['key'] == organization_prefix and extra['state'] == 'active':
                                dataset_title = extra['value'] + '-' + dataset_title
                                break

                data[key] = munge.munge_title_to_name(dataset_title)

                log.debug('[multilanguage_url] Created name "%s" for package from language %s',
                            data[key], l)
                break
Example #48
0
def drupal_get_header():
  import urllib2
  from pylons import config
  navigation = urllib2.urlopen(config.get('drupal.site_url') + '/koop_theme/get/header')
  return navigation.read().decode('utf-8')
Example #49
0
    def _get_content_and_type(self,
                              url,
                              harvest_job,
                              page=1,
                              content_type=None):
        '''
        Gets the content and type of the given url.

        :param url: a web url (starting with http) or a local path
        :param harvest_job: the job, used for error reporting
        :param page: adds paging to the url
        :param content_type: will be returned as type
        :return: a tuple containing the content and content-type
        '''
        method_log_prefix = '[%s][_get_content_and_type]' % type(self).__name__

        if not url.lower().startswith('http'):
            # Check local file
            if os.path.exists(url):
                with open(url, 'r') as f:
                    content = f.read()
                content_type = content_type or rdflib.util.guess_format(url)
                return content, content_type
            else:
                msg = u'No se pudo obtener contenido de esta url'  #'Could not get content for this url'
                errormsg = dhc.CATALOG_ACCESS_ERROR % (msg)
                if url:
                    errormsg = dhc.CATALOG_ACCESS_ERROR_URL % (url, msg)
                self._save_gather_error(errormsg, harvest_job)
                return None, None

        try:
            if page > 1:
                url = url + '&' if '?' in url else url + '?'
                url = url + 'page={0}'.format(page)

            log.debug('%s Getting file %s' % (method_log_prefix, url))

            http_proxy = pc.get(dhc.CKAN_PROP_HTTP_PROXY, None)
            https_proxy = pc.get(dhc.CKAN_PROP_HTTPS_PROXY, None)
            proxies = None
            if http_proxy or https_proxy:
                proxies = {}
                if http_proxy:
                    proxies['http'] = http_proxy
                if https_proxy:
                    proxies['https'] = https_proxy
                log.debug("%s Using proxies %s" % (method_log_prefix, proxies))
            else:
                log.debug("%s No proxies" % (method_log_prefix))

            # first we try a HEAD request which may not be supported
            did_get = False
            r = requests.head(url, proxies=proxies, verify=False)
            if r.status_code == 405 or r.status_code == 400 or r.status_code == 404:
                r = requests.get(url,
                                 proxies=proxies,
                                 stream=True,
                                 verify=False)
                did_get = True
            r.raise_for_status()

            cl = r.headers.get('content-length')
            if cl and int(cl) > self.MAX_FILE_SIZE:
                #                 msg = '''Remote file is too big. Allowed
                #                     file size: {allowed}, Content-Length: {actual}.'''.format(
                #                     allowed=self.MAX_FILE_SIZE, actual=cl)
                msg = u'''El fichero remoto es demasiado grande. Tama\u00F1o 
                de fichero permitido: {allowed], Longitud del contenido: {actual}'''.format(
                    allowed=self.MAX_FILE_SIZE, actual=cl)
                errormsg = dhc.CATALOG_ACCESS_ERROR % (msg)
                if url:
                    errormsg = dhc.CATALOG_ACCESS_ERROR_URL % (url, msg)
                self._save_gather_error(errormsg, harvest_job)
                return None, None

            if not did_get:
                r = requests.get(url,
                                 proxies=proxies,
                                 stream=True,
                                 verify=False)

            length = 0
            content = ''
            for chunk in r.iter_content(chunk_size=self.CHUNK_SIZE):
                content = content + chunk
                length += len(chunk)

                if length >= self.MAX_FILE_SIZE:
                    msg = u'El fichero remoto es demasiado grande'  #'Remote file is too big.'
                    errormsg = dhc.CATALOG_ACCESS_ERROR % (msg)
                    if url:
                        errormsg = dhc.CATALOG_ACCESS_ERROR_URL % (url, msg)
                    self._save_gather_error(errormsg, harvest_job)
                    return None, None

            if content_type is None and r.headers.get('content-type'):
                content_type = r.headers.get('content-type').split(";", 1)[0]

            return content, content_type

        except requests.exceptions.HTTPError, error:
            if page > 1 and error.response.status_code == 404:
                # We want to catch these ones later on
                raise

#             msg = 'Could not get content. Server responded with %s %s' % (
#                 error.response.status_code, error.response.reason)
            msg = u'No se pudo obtener contenido. El servidor respondi\u00F3 con %s %s' % (
                error.response.status_code, error.response.reason)
            errormsg = dhc.CATALOG_ACCESS_ERROR % (msg)
            if url:
                errormsg = dhc.CATALOG_ACCESS_ERROR_URL % (url, msg)
            self._save_gather_error(errormsg, harvest_job)
            return None, None
Example #50
0
    def validator(key, data, errors, context):
        if errors[key]:
            return

        values = []
        languages = []

        required_language = field['required_language']
        if not required_language:
            required_language = config.get('ckan.locale_default', 'es')

        required_field = field.get('required')
        if not required_field:
            required_field = False

        #Get value from dict or JSON encoded string
        value = data[key]
        if value is not missing:
            if isinstance(value, basestring):
                try:
                    value = json.loads(value)
                except ValueError:
                    errors[key].append(_('Failed to decode JSON string'))
                    return
                except UnicodeDecodeError:
                    errors[key].append(_('Invalid encoding for JSON string'))
                    return
            if not isinstance(value, dict):
                errors[key].append(_('Expecting JSON object'))
                return

            for lang, text in value.iteritems():
                try:
                    m = re.match(ISO_639_LANGUAGE, lang)
                except TypeError:
                    errors[key].append(_('Invalid type for language code: %r')
                        % lang)
                    continue

                #Register the language
                languages += [lang]
                if not isinstance(text, basestring):
                    errors[key].append(_('Invalid type for "%s" value') % lang)
                    continue
                if isinstance(text, str) or isinstance(text, unicode):
                    try:
                        #Not register empty values
                        if text.strip():
                            values += [lang]
                    except UnicodeDecodeError:
                        errors[key]. append(_('Invalid encoding for "%s" value')
                            % lang)

            #Error if texts exist but not in required language
            if values and not required_language in values:
                errors[key[:-1] + (key[-1] + '-' + required_language,)] = [_('Missing required language value')]
            else:
                if not values and required_field:
                    errors[key[:-1] + (key[-1] + '-' + required_language,)] = [_('Missing required language value')]

            return


        #Get values from formulary
        prefix = key[-1] + '-'
        extras = data.get(key[:-1] + ('__extras',), {})

        for name, text in extras.iteritems():
            if not name.startswith(prefix):
                continue

            lang = name.split('-', 1)[1]
            m = re.match(ISO_639_LANGUAGE, lang)
            if not m:
                errors[name] = [_('Invalid language code: "%s"') % lang]
                values = None
                continue

            languages += [lang]
            if text:
                values += [lang]

        #Error if texts exist but not in required language
        if values and not required_language in values:
            errors[key[:-1] + (key[-1] + '-' + required_language,)] = [_('Missing required language value')]
        else:
            if not values and required_field:
                errors[key[:-1] + (key[-1] + '-' + required_language,)] = [_('Missing required language value')]
Example #51
0
 def _private_key(self):
     return config.get('recaptcha.private_key')
Example #52
0
from linotp.lib.user import getUserInfo

from linotp.lib.util import check_session
from linotp.lib.util import getLowerParams
from linotp.lib.util import get_client

from linotp.lib.context import request_context
import linotp.model

from linotp.lib.error import UserError

import linotp.model.meta

Session = linotp.model.meta.Session

audit = config.get('audit')

log = logging.getLogger(__name__)


class OcraController(BaseController):
    '''
    The OcraController implements challenges/response tokens
    according to RFC 6287
    '''
    def __before__(self, action, **params):
        '''
        Here we see, what action is to be called and check the authorization
        '''

        try:
Example #53
0
import datetime

from pylons import config
from sqlalchemy import Table, select, func, and_
from sqlalchemy.sql.expression import text

import ckan.plugins as p
import ckan.model as model

import re

cache_enabled = p.toolkit.asbool(
    config.get('ckanext.stats.cache_enabled', 'True'))

if cache_enabled:
    from pylons import cache
    our_cache = cache.get_cache('stats', type='dbm')

DATE_FORMAT = '%Y-%m-%d'


def table(name):
    return Table(name, model.meta.metadata, autoload=True)


def datetime2date(datetime_):
    return datetime.date(datetime_.year, datetime_.month, datetime_.day)


class Stats(object):
    @classmethod
Example #54
0
    def transform_to_iso(self, original_document, original_format,
                         harvest_object):

        if original_format != 'fgdc':
            return None

        transform_service = config.get('ckanext.geodatagov.fgdc2iso_service')
        if not transform_service:
            self._save_object_error('No FGDC to ISO transformation service',
                                    harvest_object, 'Import')
            return None

        # Validate against FGDC schema
        if self.source_config.get('validator_profiles'):
            profiles = self.source_config.get('validator_profiles')
        else:
            profiles = ['fgdc_minimal']

        validator = Validators(profiles=profiles)
        for custom_validator in custom_validators:
            validator.add_validator(custom_validator)

        is_valid, profile, errors = self._validate_document(
            original_document, harvest_object, validator=validator)
        if not is_valid:
            # TODO: Provide an option to continue anyway
            return None

        original_document = re.sub('<\?xml(.*)\?>', '', original_document)

        tree = etree.fromstring(original_document)
        comments = tree.xpath('//comment()')

        for comment in comments:
            p = comment.getparent()
            if p:
                p.remove(comment)

        ptvctcnt = tree.xpath('//ptvctcnt')
        for node in ptvctcnt:
            p = node.getparent()
            if p and not node.text:
                p.remove(node)

        themekt = tree.xpath('//placekt')
        for num, node in enumerate(themekt):
            p = node.getparent()
            ###remove all but first
            if p and num > 0:
                p.remove(node)

        original_document = etree.tostring(tree)

        response = requests.post(
            transform_service,
            data=original_document.encode('utf8'),
            headers={'content-type': 'text/xml; charset=utf-8'})

        if response.status_code == 200:
            # XML coming from the conversion tool is already declared and encoded as utf-8
            return response.content
        else:
            msg = 'The transformation service returned an error for object {0}'
            if response.status_code and response.content:
                msg += ': [{0}] {1}'.format(response.status_code,
                                            response.content)
            elif response.error:
                msg += ': {0}'.format(response.error)
            self._save_object_error(msg, harvest_object, 'Import')
            return None
Example #55
0
def render(template_name,
           extra_vars=None,
           cache_key=None,
           cache_type=None,
           cache_expire=None,
           method='xhtml',
           loader_class=MarkupTemplate,
           cache_force=None,
           renderer=None):
    '''Render a template and return the output.

    This is CKAN's main template rendering function.

    .. todo::

       Document the parameters of :py:func:`ckan.plugins.toolkit.render`.

    '''
    def render_template():
        globs = extra_vars or {}
        globs.update(pylons_globals())
        globs['actions'] = model.Action

        # Using pylons.url() directly destroys the localisation stuff so
        # we remove it so any bad templates crash and burn
        del globs['url']

        try:
            template_path, template_type = render_.template_info(template_name)
        except render_.TemplateNotFound:
            raise

        log.debug('rendering %s [%s]' % (template_path, template_type))
        if config.get('debug'):
            context_vars = globs.get('c')
            if context_vars:
                context_vars = dir(context_vars)
            debug_info = {
                'template_name': template_name,
                'template_path': template_path,
                'template_type': template_type,
                'vars': globs,
                'c_vars': context_vars,
                'renderer': renderer
            }
            if 'CKAN_DEBUG_INFO' not in request.environ:
                request.environ['CKAN_DEBUG_INFO'] = []
            request.environ['CKAN_DEBUG_INFO'].append(debug_info)

        # Jinja2 templates
        if template_type == 'jinja2':
            # We don't want to have the config in templates it should be
            # accessed via g (app_globals) as this gives us flexability such
            # as changing via database settings.
            del globs['config']
            # TODO should we raise error if genshi filters??
            return render_jinja2(template_name, globs)

        # Genshi templates
        template = globs['app_globals'].genshi_loader.load(
            template_name.encode('utf-8'), cls=loader_class)
        stream = template.generate(**globs)

        for item in p.PluginImplementations(p.IGenshiStreamFilter):
            stream = item.filter(stream)

        if loader_class == NewTextTemplate:
            return literal(stream.render(method="text", encoding=None))

        return literal(
            stream.render(method=method, encoding=None, strip_whitespace=True))

    if 'Pragma' in response.headers:
        del response.headers["Pragma"]

    ## Caching Logic
    allow_cache = True
    # Force cache or not if explicit.
    if cache_force is not None:
        allow_cache = cache_force
    # Do not allow caching of pages for logged in users/flash messages etc.
    elif session.last_accessed:
        allow_cache = False
    # Tests etc.
    elif 'REMOTE_USER' in request.environ:
        allow_cache = False
    # Don't cache if based on a non-cachable template used in this.
    elif request.environ.get('__no_cache__'):
        allow_cache = False
    # Don't cache if we have set the __no_cache__ param in the query string.
    elif request.params.get('__no_cache__'):
        allow_cache = False
    # Don't cache if we have extra vars containing data.
    elif extra_vars:
        for k, v in extra_vars.iteritems():
            allow_cache = False
            break
    # Record cachability for the page cache if enabled
    request.environ['CKAN_PAGE_CACHABLE'] = allow_cache

    if allow_cache:
        response.headers["Cache-Control"] = "public"
        try:
            cache_expire = int(config.get('ckan.cache_expires', 0))
            response.headers["Cache-Control"] += \
                ", max-age=%s, must-revalidate" % cache_expire
        except ValueError:
            pass
    else:
        # We do not want caching.
        response.headers["Cache-Control"] = "private"
        # Prevent any further rendering from being cached.
        request.environ['__no_cache__'] = True
    log.debug('Template cache-control: %s' % response.headers["Cache-Control"])

    # Render Time :)
    try:
        return cached_template(template_name,
                               render_template,
                               loader_class=loader_class)
    except ckan.exceptions.CkanUrlException, e:
        raise ckan.exceptions.CkanUrlException(
            '\nAn Exception has been raised for template %s\n%s' %
            (template_name, e.message))
Example #56
0
 def _public_key(self):
     return config.get('recaptcha.public_key')
Example #57
0
class ListController(BaseController):

    request_path = config.get('list.staticmax_url')

    @classmethod
    @jsonify
    def overview(cls, slug):
        game = get_game_by_slug(slug)
        if not game:
            response.status_int = 404
            return {'ok': False, 'msg': 'Game does not exist: %s' % slug}

        return {
            'ok': True,
            'data': {
                'slug': game.slug,
                'staticFilePrefix' : 'staticmax',
                'mappingTable': game.mapping_table
            }
        }

    @classmethod
    @jsonify
    def assets(cls, slug, path=''):
        game = get_game_by_slug(slug)
        if not game:
            response.status_int = 404
            return {'ok': False, 'msg': 'Game does not exist: %s' % slug}

        try:
            asset_list = game.get_asset_list(cls.request_path, path)
        except (GamePathError, GamePathNotFoundError) as e:
            response.status_int = 404
            return {'ok': False, 'msg': str(e)}
        else:
            return {
                'ok': True,
                'data': {
                    'items': [i.as_dict() for i in asset_list],
                    'path': path.strip('/'),
                    'mappingTable': game.has_mapping_table
                }
            }

    @classmethod
    @jsonify
    def files(cls, slug, path=''):
        game = get_game_by_slug(slug)
        if not game:
            response.status_int = 404
            return {'ok': False, 'msg': 'Game does not exist: %s' % slug}

        try:
            asset_list = game.get_static_files(game.path, cls.request_path, path)
        except GamePathNotFoundError as e:
            response.status_int = 404
            return {'ok': False, 'msg': str(e)}
        else:
            return {
                'ok': True,
                'data': {
                    'items': [ i.as_dict() for i in asset_list ],
                    'path': path.strip('/'),
                    'mappingTable': game.has_mapping_table
                }
            }
Example #58
0
    def publisher_list(self):
        package_type = 'dataset'
        try:
            context = {'model': model, 'user': c.user or c.author,
                       'auth_user_obj': c.userobj}

            check_access('openness', context)
        except NotAuthorized:
            abort(401, _('Not authorized to see this page'))

        c.pkg_dict = {}
        c.publishers = logic.get_action('organization_list')(context, {'all_fields': True})

        report = {}

        locale = tk.request.environ['CKAN_LANG'] or config.get('ckan.locale_default', 'en')
        cache_key = 'global_openness:{0}'.format(locale)
        g_start_time = time.time()
        dict_string = cache.get_from_cache(cache_key, pool=cache.MISC_POOL)
        if dict_string:
            start_time = time.time()
            report = pickle.loads(dict_string)
            duration = time.time()-start_time
            log.info("Loading json took {0}".format(duration))
        else:
            osp_start  = time.time()
            rows = self._openness_sores_for_publisher()
            osp_duration = time.time()- osp_start
            log.info("_openness_sores_for_publisher took {0}".format(osp_duration))

            ps_start  = time.time()
            publishers = model.Group.all(group_type='organization')
            dict_pub = {}
            for publisher in publishers:
                dict_pub[publisher.name] = publisher

            for ds_name, obj in rows.iteritems():
                if not obj.get('owner_org'):
                    continue
                publisher = dict_pub.get(obj.get('owner_org').split('/')[-1].lower(),model.Group())
                #publisher = next((pub for pub in publishers if pub.name == model.Group.get(obj.get('owner_org').split('/')[-1].lower())),None)

                publ_report = report.get(publisher.name or publisher.id, {'publisher_name': publisher.title,
                                                                          'publisher_id': publisher.name or publisher.id,
                                                                          'zero': 0,
                                                                          'one': 0,
                                                                          'two': 0,
                                                                          'three': 0,
                                                                          'four': 0,
                                                                          'five': 0,
                                                                          'sum': 0,
                                                                          'avg': 0})

                column_key = self._set_dataset_score(obj['sum_value'])
                publ_report[column_key] += 1
                publ_report = self.calculate_sum(publ_report)
                publ_report = self.calculate_avg(publ_report)
                report[publisher.name] = publ_report


            #cache.set_value_in_cache(cache_key,pickle.dumps(report), pool=redis_cache.MISC_POOL)
            ps_duration = time.time()- ps_start
            log.info("publisher list 1st loop took {0}".format(ps_duration))

        totals = {'zero': 0,
              'one': 0,
              'two': 0,
              'three': 0,
              'four': 0,
              'five': 0}
        for publ, value in report.iteritems():
            totals['zero'] = value['zero'] + totals['zero']
            totals['one'] = value['one'] + totals['one']
            totals['two'] = value['two'] + totals['two']
            totals['three'] = value['three'] + totals['three']
            totals['four'] = value['four'] + totals['four']
            totals['five'] = value['five'] + totals['five']


        common_formats = self.get_format_summary_list()

        result = {}
        result['table'] = report
        result['totals'] = totals
        result['json'] = json.dumps(totals)
        result['common_formats'] = json.dumps(common_formats)

        c.pkg_dict = result

        self._setup_template_variables(context, {},
                                       package_type=package_type)
        g_duration = time.time()-g_start_time
        log.info("Global Loading took {0}".format(g_duration))
        # c.form = base.render(self._package_form(package_type='upload_package'), extra_vars=vars)
        return base.render('openness/publisher_list.html')
Example #59
0
def _get_default_locale():
    return config.get('ckan.locale_default', 'en')
Example #60
0
def _pagination_info(query, data_dict):
    '''
    Creates a pagination_info dict to be passed to the serializers

    `query` is the output of `package_search` and `data_dict`
    contains the request params.

    The keys for the dictionary are:

    * `count` (total elements)
    * `items_per_page` (`ckanext.dcat.datasets_per_page` or 100)
    * `current`
    * `first`
    * `last`
    * `next`
    * `previous`

    Returns a dict
    '''
    def _page_url(page):

        base_url = config.get('ckan.site_url', '').strip('/')
        if not base_url:
            base_url = toolkit.request.host_url
        base_url = '%s%s' % (base_url, toolkit.request.path)

        params = [
            p for p in toolkit.request.params.iteritems() if p[0] != 'page'
        ]
        if params:
            qs = '&'.join(['{0}={1}'.format(p[0], p[1]) for p in params])
            return '{0}?{1}&page={2}'.format(base_url, qs, page)
        else:
            return '{0}?page={1}'.format(base_url, page)

    try:
        page = int(data_dict.get('page', 1) or 1)
        if page < 1:
            raise wrong_page_exception
    except ValueError:
        raise wrong_page_exception

    if query['count'] == 0:
        return {}

    items_per_page = int(
        config.get('ckanext.dcat.datasets_per_page', DATASETS_PER_PAGE))
    pagination_info = {
        'count': query['count'],
        'items_per_page': items_per_page,
    }

    pagination_info['current'] = _page_url(page)
    pagination_info['first'] = _page_url(1)

    last_page = int(math.ceil(query['count'] / items_per_page)) or 1
    pagination_info['last'] = _page_url(last_page)

    if page > 1:
        if ((page - 1) * items_per_page +
                len(query['results'])) <= query['count']:
            previous_page = page - 1
        else:
            previous_page = last_page

        pagination_info['previous'] = _page_url(previous_page)

    if page * items_per_page < query['count']:
        pagination_info['next'] = _page_url(page + 1)

    return pagination_info