def check_recaptcha(token, expected_action):
'''
Check a user's recaptcha token is valid, and raise RecaptchaError if it's not. If the token is
valid then this function has no side effects.
:param token: the token as returned in the frontend by a call to execute
:param expected_action: the expected action associated with the token
'''
key = config.get(u'ckanext.contact.recaptcha_v3_key', False)
secret = config.get(u'ckanext.contact.recaptcha_v3_secret', False)
if not key or not secret:
# recaptcha not enabled
return
post_params = {
u'secret': secret,
u'response': token,
}
client_ip_address = request.environ.get(u'REMOTE_ADDR', None)
if client_ip_address:
post_params[u'remoteip'] = client_ip_address
response = requests.post(u'https://www.google.com/recaptcha/api/siteverify', params=post_params)
response.raise_for_status()
result = response.json()
if not result[u'success']:
raise RecaptchaError(u', '.join(result[u'error-codes']))
if expected_action != result[u'action']:
raise RecaptchaError(_(u'Action mismatch'))
def static(url):
static_path = config.get("openspending.static_path", "/static/")
url_ = "%s%s" % (static_path, url)
version = config.get("openspending.static_cache_version", "")
if version:
url_ = "%s?%s" % (url_, version)
return url_
def get_profile_id(service):
"""
Get the profile ID for this user and the service specified by the
'googleanalytics.id' configuration option. This function iterates
over all of the accounts available to the user who invoked the
service to find one where the account name matches (in case the
user has several).
"""
accounts = service.management().accounts().list().execute()
if not accounts.get('items'):
return None
accountName = config.get('googleanalytics.account')
webPropertyId = config.get('googleanalytics.id')
for acc in accounts.get('items'):
if acc.get('name') == accountName:
accountId = acc.get('id')
webproperties = service.management().webproperties().list(accountId=accountId).execute()
profiles = service.management().profiles().list(
accountId=accountId, webPropertyId=webPropertyId).execute()
if profiles.get('items'):
return profiles.get('items')[0].get('id')
return None
def by_name(cls, name):
"""Return a Domain instance by name."""
if os.path.exists(os.path.join(config.get('herder.po_dir'), name)):
return Domain(name, os.path.join(config.get('herder.po_dir'), name))
raise KeyError("Unknown domain name.")
def send(to_addrs, from_addr, subject, body):
"""A simple method to send a simple email.
:param to_addrs: Comma separated list of email addresses to send to.
:type to_addrs: unicode
:param from_addr: Email address to put in the 'from' field
:type from_addr: unicode
:param subject: Subject line of the email.
:type subject: unicode
:param body: Body text of the email, optionally marked up with HTML.
:type body: unicode
"""
smtp_server = config.get('smtp_server', 'localhost')
server = smtplib.SMTP(smtp_server)
if isinstance(to_addrs, basestring):
to_addrs = parse_email_string(to_addrs)
to_addrs = ", ".join(to_addrs)
msg = ("To: %(to_addrs)s\n"
"From: %(from_addr)s\n"
"Subject: %(subject)s\n\n"
"%(body)s\n") % locals()
smtp_username = config.get('smtp_username')
smtp_password = config.get('smtp_password')
if smtp_username and smtp_password:
server.login(smtp_username, smtp_password)
server.sendmail(from_addr, to_addrs, msg.encode('utf-8'))
server.quit()
def get_value(key, converter, default=None, config=config,
converter_kwargs={}, instance_overwrites=True):
allow_overwrite = (instance_overwrites
and get_bool('%s.allow_overwrite' % key,
instance_overwrites=False))
if allow_overwrite:
from adhocracy.model import instance_filter as ifilter
current_instance = ifilter.get_instance()
if current_instance is not None:
value = config.get('%s.%s' % (key, current_instance.key))
else:
value = None
else:
value = None
if value is None:
value = config.get(key)
if value is None:
if default is not None:
return default
else:
return DEFAULTS.get(key)
if converter is None:
return value
else:
return converter(value, **converter_kwargs)
def __init__(self, profiles=None, compatibility_mode=False):
'''
Creates a parser instance
You can optionally pass a list of profiles to be used.
In compatibility mode, some fields are modified to maintain
compatibility with previous versions of the ckanext-dcat parsers
(eg adding the `dcat_` prefix or storing comma separated lists instead
of JSON dumps).
'''
if not profiles:
profiles = config.get(RDF_PROFILES_CONFIG_OPTION, None)
if profiles:
profiles = profiles.split(' ')
else:
profiles = DEFAULT_RDF_PROFILES
self._profiles = self._load_profiles(profiles)
if not self._profiles:
raise RDFProfileException(
'No suitable RDF profiles could be loaded')
if not compatibility_mode:
compatibility_mode = p.toolkit.asbool(
config.get(COMPAT_MODE_CONFIG_OPTION, False))
self.compatibility_mode = compatibility_mode
self.g = rdflib.Graph()
def catalog_uri():
'''
Returns an URI for the whole catalog
This will be used to uniquely reference the CKAN instance on the RDF
serializations and as a basis for eg datasets URIs (if not present on
the metadata).
The value will be the first found of:
1. The `ckanext.dcat.base_uri` config option (recommended)
2. The `ckan.site_url` config option
3. `http://` + the `app_instance_uuid` config option (minus brackets)
A warning is emited if the third option is used.
Returns a string with the catalog URI.
'''
uri = config.get('ckanext.dcat.base_uri')
if not uri:
uri = config.get('ckan.site_url')
if not uri:
app_uuid = config.get('app_instance_uuid')
if app_uuid:
uri = 'http://' + app_uuid.replace('{', '').replace('}', '')
log.critical('Using app id as catalog URI, you should set the ' +
'`ckanext.dcat.base_uri` or `ckan.site_url` option')
else:
uri = 'http://' + str(uuid.uuid4())
log.critical('Using a random id as catalog URI, you should set ' +
'the `ckanext.dcat.base_uri` or `ckan.site_url` ' +
'option')
return uri
def setSort(spectra, config, package):
sort = None
# backword compatibility. But moving away from config object
# all 'advanced data' should be stored in package
extras = package.get('extras')
if extras != None and extras.get('sort') != None:
sort = json.loads(extras.get('sort'))
elif config.get("sort") != None:
sort = config.get("sort")
if sort == None:
return
on = sort.get('on')
type = sort.get('type')
if on is None:
return
if on not in spectra:
return
if type == 'datetime':
try:
spectra['ecosis']['sort'] = dateutil.parser.parse(spectra[on])
except:
pass
elif type == 'numeric':
try:
spectra['ecosis']['sort'] = float(spectra[on])
except:
pass
else:
spectra['ecosis']['sort'] = spectra[on]
def graph_from_catalog(self, catalog_dict, catalog_ref):
g = self.g
for prefix, namespace in namespaces.iteritems():
g.bind(prefix, namespace)
g.add((catalog_ref, RDF.type, DCAT.Catalog))
# Basic fields
items = [
('title', DCT.title, config.get('ckan.site_title')),
('description', DCT.description, config.get('ckan.site_description')),
('homepage', FOAF.homepage, config.get('ckan.site_url')),
('language', DCT.language, config.get('ckan.locale_default', 'en')),
]
for item in items:
key, predicate, fallback = item
if catalog_dict:
value = catalog_dict.get(key, fallback)
else:
value = fallback
if value:
g.add((catalog_ref, predicate, Literal(value)))
# Dates
modified = self._last_catalog_modification()
if modified:
self._add_date_triple(catalog_ref, DCT.modified, modified)
def upload(self):
if 'datafile' in request.params:
myfile = request.params.get('datafile')
# Store file in user directory on Longboard
source_dir = config.get('cw.cwdata_loc')
source = os.path.join(source_dir, myfile.filename)
###target = os.path.sep.join([config.get('cw.cwproj_loc'), session.get('user', 'guest')])
target = os.path.sep.join([config.get('cw.cwproj_rem'), session.get('user', 'guest')])
file_contents = ''
for i in myfile.file:
file_contents += i
if file.type.find('text') > -1:
try:
fh = open(source, 'w')
fh.write(file_contents)
fh.close()
except Exception as _:
log.error('Cannot write file to disk.')
else:
try:
fh = open(source, 'wb')
fh.write(file_contents)
fh.close()
except Exception:
log.error('Cannot write file to disk.')
host = config.get('cw.arch_host', 'longboard.acel')
resource = app_globals.jodis.manager.getResource(host)
resource.ssh.scp(None, None, source, resource.ssh.user, host, target)
return render('/data/upload.mako')
def _call(self, **kwargs):
account_name = config.get("ckanext.doi.account_name")
account_password = config.get("ckanext.doi.account_password")
endpoint = os.path.join(get_endpoint(), self.path)
try:
path_extra = kwargs.pop('path_extra')
except KeyError:
pass
else:
endpoint = os.path.join(endpoint, path_extra)
try:
method = kwargs.pop('method')
except KeyError:
method = 'get'
# Add authorisation to request
kwargs['auth'] = (account_name, account_password)
log.info("Calling %s:%s - %s", endpoint, method, kwargs)
r = getattr(requests, method)(endpoint, **kwargs)
r.raise_for_status()
# Return the result
return r
def before_create(self, context, resource):
disallowed_extensions = toolkit.aslist(config.get('ckanext.romania_theme.disallowed_extensions',[]))
disallowed_mimetypes = [mimetypes.types_map["." + x] for x in disallowed_extensions]
allowed_extensions = toolkit.aslist(config.get('ckanext.romania_theme.allowed_extensions',[]))
allowed_mimetypes = [mimetypes.types_map["." + x] for x in allowed_extensions]
is_resource_extension_allowed = False
error_message = ''
if allowed_mimetypes:
if resource['upload'].type in allowed_mimetypes:
is_resource_extension_allowed = True
else:
error_message="Doar urmatoarele extensii sunt permise: " + ", ".join(allowed_extensions) + "."
else:
if resource['upload'].type not in disallowed_mimetypes:
is_resource_extension_allowed = True
else:
error_message= "Urmatoarele extensii sunt nepermise: " + ", ".join(disallowed_extensions) + "."
if ('upload' in resource) and (type(resource['upload']) is not unicode) and not is_resource_extension_allowed:
# If we did not do this, the URL field would contain the filename
# and people can press finalize afterwards.
resource['url'] = ''
raise toolkit.ValidationError(['Fisierul are o extensie nepermisa! ' + error_message])
def ssl_check(ssl_required=[], ssl_allowed=[], ssl_required_all=False, ssl_allowed_all=False):
if not asbool(config.get('enable_ssl_requirement', False)):
return
action = request.environ['pylons.routes_dict']['action']
if action in ssl_allowed or ssl_allowed_all: # We don't care if they use http or https
return
elif action in ssl_required or ssl_required_all: # Must have ssl
protocol = 'https'
else:
protocol = 'http'
if current_protocol() == protocol:
return
if request.method.upper() != 'POST':
log.debug('Redirecting to %s, request: %s', protocol, request.path_info)
host = config.get('ssl_host')
if host:
redirect_to(protocol=protocol, host=host)
else:
redirect_to(protocol=protocol)
else:
abort(405, headers=[('Allow', 'GET')]) # don't allow POSTs.
def update_config(self, config):
"""Update CKAN's config with settings needed by this plugin.
"""
toolkit.add_template_directory(config, "templates")
self.header_parameter = config.get("ckan.simplesso.header_parameter", "user-id")
self.email_domain = config.get("ckan.simplesso.email_domain")
def get_name_from_siret(siret):
apiKey = config.get('ckanext.ozwillo_organization_api.verifsiret_apikey', '')
secretKey = config.get('ckanext.ozwillo_organization_api.verifsiret_secretkey', '')
url = "http://www.verif-siret.com/api/siret?siret=" + siret
result = None
if apiKey == '' or secretKey == '':
log.error('Verif-siret config incomplete, please register your api key and api secret in the config file')
result = ''
return result
try:
get = requests.get(url, auth=(apiKey, secretKey))
if get.status_code != 200:
raise requests.ConnectionError()
except requests.exceptions.RequestException as err:
log.error('An error occurred: {0}'.format(err))
else:
try:
result = get.json()['array_return'][0]['LIBCOM']
except (IndexError, TypeError, AttributeError):
log.error('No organization found for this siret number')
finally:
return result
def send_mail(self,to,subject, _body):
body = _body.replace('\n', '\r\n')
guid=str(uuid.uuid4())
body = config.get('ckan.site_url') + "/user/register?guid=" + guid + "&activate"
# Prepare actual message
message = """From: %s
To: %s
Subject: %s
%s
""" % (config.get('dk.aarhuskommune.odaa_from'),str(to),subject,body)
try:
result=self.connection()
connectString = """dbname='%s' user='******' host='%s' password='******'""" % (result["database"],result["user"],result["host"],result["password"])
conn = psycopg2.connect(connectString)
c = conn.cursor()
p=request.params["password1"]
password=self._encode(result["password"],p)
now = datetime.datetime.now()
sql="INSERT INTO mailnotifikation VALUES ('" + guid + "','" + str(now) + "','" + request.params["name"] + "','" +request.params["fullname"] + "','" + request.params["email"] + "','" + password + "','False')"
c.execute(sql)
conn.commit()
nowlm = now - datetime.timedelta(days=int(config.get('dk.aarhuskommune.odaa_days')))
sNowlm=nowlm.strftime('%Y-%m-%d')
sql="delete from mailnotifikation where _date<'%s'" % (sNowlm);
c.execute(sql)
conn.commit()
conn.close()
smtpObj = smtplib.SMTP('localhost')
to=to.split(',')
smtpObj.sendmail(config.get('dk.aarhuskommune.odaa_from'), to, message)
except Exception as e:
logging.error('Error: unable to send email. %s ',e)
def __before__(self):
"before"
if 'theme' not in session:
session['theme'] = ''
basedir = config.get('baruwa.themes.base', None)
if basedir:
# Default theme
defaultdir = os.path.join(basedir, 'templates', 'default')
if os.path.exists(defaultdir):
session['theme'] = 'default'
# Host theme
themedir = os.path.join(basedir, 'templates',
request.server_name)
if os.path.exists(themedir):
session['theme'] = request.server_name
session.save()
self.theme = session.get('theme')
if 'lang' in session:
set_lang(session['lang'])
else:
try:
languages = [lang.split('-')[0] for lang in request.languages
if check_language(lang.split('-')[0])]
set_lang(languages)
except AttributeError:
default_lang = config.get('baruwa.default.language', 'en')
if check_language(default_lang):
set_lang([default_lang])
else:
set_lang(['en'])
# pylint: disable-msg=W0201
self.invalidate = request.GET.get('uc', None)
self.langchange = request.GET.get('lc', None)
def before_search(self, search_params):
lang_set = set(LANGS)
try:
current_lang = pylons.request.environ['CKAN_LANG']
except TypeError as err:
if err.message == ('No object (name: request) has been registered '
'for this thread'):
# This happens when this code gets called as part of a paster
# command rather then as part of an HTTP request.
current_lang = config.get('ckan.locale_default')
else:
raise
# fallback to default locale if locale not in suported langs
if not current_lang in lang_set:
current_lang = config.get('ckan.locale_default')
# fallback to english if default locale is not supported
if not current_lang in lang_set:
current_lang = 'en'
# treat current lang differenly so remove from set
lang_set.remove(current_lang)
# weight current lang more highly
query_fields = 'title_%s^8 text_%s^4' % (current_lang, current_lang)
for lang in lang_set:
query_fields += ' title_%s^2 text_%s' % (lang, lang)
search_params['qf'] = query_fields
return search_params
def getPosts(self, ident):
thread = self.getThread(ident)
request = thread
fresh_thread = False
if thread['code'] != 0:
thread = self.createThreadFromIdent(ident)
fresh_thread = True
if thread['code'] == 0:
thread_id = thread['response']['id']
api_key = config.get('edcdisqus.api_key')
forum_name = config.get('edcdisqus.forum_name')
data_string = urllib.urlencode({'forum': forum_name, 'thread': thread_id, 'api_key': api_key, 'limit': 100 })
request_json = self.request('https://disqus.com/api/3.0/posts/list.json', data_string, 'get')
request = json.loads(request_json)
# For some reason, the disqus API likes to return all of the comments when a thread id doesn't exist. we're just emptying the result set
# when a new thread is created. (There shouldn't be any comments anyway, the thread was just made!)
if fresh_thread:
request['response'] = []
else:
request = thread
return request
def setup_class(cls):
reset_db()
config
cls.fake_context = {
'site_url': config.get('ckan.site_url_internally') or config['ckan.site_url'],
'cache_url_root': config.get('ckanext-archiver.cache_url_root'),
}
def _mail_recipient(recipient_name, recipient_email,
sender_name, sender_url, subject,
body, headers={}):
mail_from = config.get('ckan.mail_from')
body = add_msg_niceties(recipient_name, body, sender_name, sender_url)
msg = MIMEText(body.encode('utf-8'), 'plain', 'utf-8')
for k, v in headers.items(): msg[k] = v
subject = Header(subject.encode('utf-8'), 'utf-8')
msg['Subject'] = subject
msg['From'] = _("%s <%s>") % (sender_name, mail_from)
recipient = u"%s <%s>" % (recipient_name, recipient_email)
msg['To'] = Header(recipient, 'utf-8')
msg['Date'] = Utils.formatdate(time())
msg['X-Mailer'] = "CKAN %s" % __version__
try:
server = smtplib.SMTP(
config.get('test_smtp_server',
config.get('smtp_server', 'localhost')))
#server.set_debuglevel(1)
server.sendmail(mail_from, [recipient_email], msg.as_string())
server.quit()
except Exception, e:
msg = '%r' % e
log.exception(msg)
raise MailerException(msg)
def _get_allowed_sender_options(cls, user):
sender_options = {
'user': {
'email': user.email,
'checked': False,
'enabled': user.is_email_activated(),
'reason': _("Email isn't activated"),
},
'system': {
'email': config.get('adhocracy.email.from'),
'checked': False,
'enabled': asbool(config.get(
'allow_system_email_in_mass_messages', 'true')),
'reason': _("Not permitted in system settings"),
},
'support': {
'email': config.get('adhocracy.registration_support_email'),
'checked': False,
'enabled': (config.get('adhocracy.registration_support_email')
is not None),
'reason': _("adhocracy.registration_support_email not set"),
}
}
if sender_options['user']['enabled']:
sender_options['user']['checked'] = True
elif sender_options['system']['enabled']:
sender_options['system']['checked'] = True
return sender_options
def __init__(self):
connect_string = config.get("linotpOpenID.sql.url")
implicit_returning = config.get("linotpSQL.implicit_returning", True)
self.engine = None
if connect_string is None:
log.info("[__init__] Missing linotpOpenID.sql.url parameter in "
"config file! Using the sqlalchemy.url")
# raise Exception("Missing linotpOpenID.sql.url parameter in "
# "config file!")
connect_string = config.get("sqlalchemy.url")
########################## SESSION ##################################
# Create an engine and create all the tables we need
if implicit_returning:
# If implicit_returning is explicitly set to True, we
# get lots of mysql errors:
# AttributeError: 'MySQLCompiler_mysqldb' object has no attribute
# 'returning_clause' So we do not mention explicit_returning at all
self.engine = create_engine(connect_string)
else:
self.engine = create_engine(connect_string,
implicit_returning=False)
metadata.bind = self.engine
metadata.create_all()
# Set up the session
self.sm = orm.sessionmaker(bind=self.engine, autoflush=True,
autocommit=False,
expire_on_commit=True)
self.session = orm.scoped_session(self.sm)
def shorten_link(long_url):
longUrl = cgi.escape(long_url)
bitly_userid= config.get('bitly.userid')
bitly_key = config.get('bitly.key')
bitly_result = urllib.urlopen("http://api.bit.ly/v3/shorten?login=%(bitly_userid)s&apiKey=%(bitly_key)s&longUrl=%(longUrl)s&format=json" % locals()).read()
bitly_data = json.loads(bitly_result)['data']
return bitly_data["url"]
def user_language(user, fallbacks=[]):
# find out the locale
locale = None
if user and user.locale:
locale = user.locale
if locale is None:
locales = map(str, LOCALES)
locale = Locale.parse(Locale.negotiate(fallbacks, locales)) \
or get_default_locale()
# determinate from which path we load the translations
translations_module = config.get('adhocracy.translations', 'adhocracy')
translations_module_loader = pkgutil.get_loader(translations_module)
if translations_module_loader is None:
raise ValueError(('Cannot import the module "%s" configured for '
'"adhocracy.translations". Make sure it is an '
'importable module (and contains the '
'translation files in a subdirectory '
'"i18n"') % translations_module)
translations_root = translations_module_loader.filename
translations_config = {'pylons.paths': {'root': translations_root},
'pylons.package': config.get('pylons.package')}
# set language and fallback
set_lang(locale.language, pylons_config=translations_config)
add_fallback(get_default_locale().language,
pylons_config=translations_config)
formencode.api.set_stdtranslation(domain="FormEncode",
languages=[locale.language])
return locale
def checkTarget(environ, result):
back = parseBoolString(config.get('ckan.gov_theme.is_back', False));
ignoreList = config.get('ckan.api.ignore', 'NotInList')
path = environ['PATH_INFO'].split("/")[-1]
if back or ignoreList.find(path) > 0:
return False
return True
def _mail_recipient(recipient_name, recipient_email,
sender_name, sender_url, subject,
body, headers={}):
mail_from = config.get('smtp.mail_from')
body = add_msg_niceties(recipient_name, body, sender_name, sender_url)
msg = MIMEText(body.encode('utf-8'), 'plain', 'utf-8')
for k, v in headers.items(): msg[k] = v
subject = Header(subject.encode('utf-8'), 'utf-8')
msg['Subject'] = subject
msg['From'] = _("%s <%s>") % (sender_name, mail_from)
recipient = u"%s <%s>" % (recipient_name, recipient_email)
msg['To'] = Header(recipient, 'utf-8')
msg['Date'] = Utils.formatdate(time())
msg['X-Mailer'] = "CKAN %s" % ckan.__version__
# Send the email using Python's smtplib.
smtp_connection = smtplib.SMTP()
if 'smtp.test_server' in config:
# If 'smtp.test_server' is configured we assume we're running tests,
# and don't use the smtp.server, starttls, user, password etc. options.
smtp_server = config['smtp.test_server']
smtp_starttls = False
smtp_user = None
smtp_password = None
else:
smtp_server = config.get('smtp.server', 'localhost')
smtp_starttls = paste.deploy.converters.asbool(
config.get('smtp.starttls'))
smtp_user = config.get('smtp.user')
smtp_password = config.get('smtp.password')
smtp_connection.connect(smtp_server)
try:
#smtp_connection.set_debuglevel(True)
# Identify ourselves and prompt the server for supported features.
smtp_connection.ehlo()
# If 'smtp.starttls' is on in CKAN config, try to put the SMTP
# connection into TLS mode.
if smtp_starttls:
if smtp_connection.has_extn('STARTTLS'):
smtp_connection.starttls()
# Re-identify ourselves over TLS connection.
smtp_connection.ehlo()
else:
raise MailerException("SMTP server does not support STARTTLS")
# If 'smtp.user' is in CKAN config, try to login to SMTP server.
if smtp_user:
assert smtp_password, ("If smtp.user is configured then "
"smtp.password must be configured as well.")
smtp_connection.login(smtp_user, smtp_password)
smtp_connection.sendmail(mail_from, [recipient_email], msg.as_string())
log.info("Sent email to {0}".format(recipient_email))
except smtplib.SMTPException, e:
msg = '%r' % e
log.exception(msg)
raise MailerException(msg)
def all(cls):
"""Return a sequence of all available domains."""
return [Domain(n, os.path.join(config.get('herder.po_dir'), n))
for n in os.listdir(config.get('herder.po_dir'))
if n not in cls._IGNORE_DIRS and
os.path.isdir(os.path.join(config.get('herder.po_dir'), n))]
def harvest_source_index_clear(context, data_dict):
'''
Clears all datasets, jobs and objects related to a harvest source, but
keeps the source itself. This is useful to clean history of long running
harvest sources to start again fresh.
:param id: the id of the harvest source to clear
:type id: string
'''
check_access('harvest_source_clear', context, data_dict)
harvest_source_id = data_dict.get('id')
source = HarvestSource.get(harvest_source_id)
if not source:
log.error('Harvest source %s does not exist', harvest_source_id)
raise NotFound('Harvest source %s does not exist' % harvest_source_id)
harvest_source_id = source.id
conn = make_connection()
query = ''' +%s:"%s" +site_id:"%s" ''' % (
'harvest_source_id', harvest_source_id, config.get('ckan.site_id'))
try:
conn.delete_query(query)
if asbool(config.get('ckan.search.solr_commit', 'true')):
conn.commit()
except Exception, e:
log.exception(e)
raise SearchIndexError(e)
def search(self):
package_type = self._guess_package_type()
try:
context = {
'model': model,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
q = c.q = request.params.get('q', u'')
c.query_error = False
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and value and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
# Modificación para andino: usamos una función para buscar dependencias entre organizaciones
if param != 'organization':
fq += ' %s:"%s"' % (param, value)
else:
fq += custom_organization_filter(value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'auth_user_obj': c.userobj
}
if package_type and package_type != 'dataset':
fq += ' +dataset_type:{type}'.format(type=package_type)
elif not asbool(config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = logic.action.get.package_search(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.search_facets = {}
c.page = h.Page(collection=[])
def get_syndicated_name_prefix():
return config.get('ckan.syndicate.name_prefix', '')
def get_syndicated_author():
return config.get('ckan.syndicate.author')
def get_syndicated_id():
return config.get('ckan.syndicate.id', 'syndicated_id')
def get_syndicate_flag():
return config.get('ckan.syndicate.flag', 'syndicate')
import sys
import cgitb
import warnings
def text_traceback():
with warnings.catch_warnings():
warnings.simplefilter("ignore")
res = 'the original traceback:'.join(
cgitb.text(
sys.exc_info()).split('the original traceback:')[1:]).strip()
return res
SIMPLE_SEARCH = config.get('ckan.simple_search', False)
SUPPORTED_SCHEMA_VERSIONS = ['1.3']
DEFAULT_OPTIONS = {
'limit': 20,
'offset': 0,
# about presenting the results
'order_by': 'rank',
'return_objects': False,
'ref_entity_with_attr': 'name',
'all_fields': False,
'search_tags': True,
'callback': None, # simply passed through
}
import urllib2
import pylons.config as config
import base64
from urllib2 import HTTPError
# doc https://team.eea.sk/wiki/pages/viewpage.action?pageId=108660564
# TODO POST /pipelines/<pipeline_id>/schedules
# TODO GET /pipelines/<pipeline_id>/schedules/<id>
# TODO GET /pipelines/<pipeline_id>/executions
# TODO /pipelines/<pipeline_id>/executions/<execution_id>
# TODO /pipelines/<pipeline_id>/executions
# TODO /pipelines/<pipeline_id>/executions/<execution_id>/events
# TODO this class to ckancommons
TIMEOUT = int(config.get(u'odn.uv.timeout', 5))
AUTH_HEADER_FIELD_NAME = u'Authorization'
USER_EXT_ID = u'userExternalId'
USER_ACTOR_EXT_ID = u'userActorExternalId'
log = logging.getLogger('ckanext')
class UVRestAPIWrapper():
def __init__(self, uv_url, auth=None):
assert uv_url
self.url = uv_url
self.auth = None
if auth:
self.auth = base64.b64encode(auth)
def new_resource(self, id, data=None, errors=None, error_summary=None):
''' FIXME: This is a temporary action to allow styling of the
forms. '''
if request.method == 'POST' and not data:
save_action = request.params.get('save')
data = data or clean_dict(
dict_fns.unflatten(tuplize_dict(parse_params(request.POST))))
# we don't want to include save as it is part of the form
del data['save']
resource_id = data['id']
del data['id']
# Guardo los campos issued y modified
time_now = moment.now().isoformat()
data['issued'] = time_now
data['modified'] = time_now
self._validate_resource(data)
context = {
'model': model,
'session': model.Session,
'user': c.user,
'auth_user_obj': c.userobj
}
if save_action == 'go-dataset':
# go to first stage of add dataset
h.redirect_to(controller='package', action='edit', id=id)
# see if we have any data that we are trying to save
data_provided = False
for key, value in data.iteritems():
if ((value
or isinstance(value, cgi.FieldStorage)) and key not in [
'resource_type', 'license_id', 'attributesDescription'
]):
data_provided = True
break
if not data_provided and save_action != "go-dataset-complete":
if save_action == 'go-dataset':
# go to first stage of add dataset
h.redirect_to(controller='package', action='edit', id=id)
try:
data_dict = get_action('package_show')(context, {'id': id})
except NotAuthorized:
abort(403, _('Unauthorized to update dataset'))
except NotFound:
abort(
404,
_('The dataset {id} could not be found.').format(
id=id))
if not len(data_dict['resources']):
# no data so keep on page
msg = _('You must add at least one data resource')
# On new templates do not use flash message
if asbool(config.get('ckan.legacy_templates')):
h.flash_error(msg)
h.redirect_to(controller='package',
action='new_resource',
id=id)
else:
errors = {}
error_summary = {_('Error'): msg}
return self.new_resource(id, data, errors,
error_summary)
# XXX race condition if another user edits/deletes
data_dict = get_action('package_show')(context, {'id': id})
get_action('package_update')(dict(context,
allow_state_change=True),
dict(data_dict, state='active'))
h.redirect_to(controller='package', action='read', id=id)
data['package_id'] = id
try:
if resource_id:
data['id'] = resource_id
get_action('resource_update')(context, data)
else:
get_action('resource_create')(context, data)
except ValidationError, e:
errors = e.error_dict
error_summary = e.error_summary
return self.new_resource(id, data, errors, error_summary)
except NotAuthorized:
abort(403, _('Unauthorized to create a resource'))
def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
"""
###### Pylons monkey-patch
# this must be run at a time when the env is semi-setup, thus inlined here.
# Required by the deliverance plugin and iATI
from pylons.wsgiapp import PylonsApp
import pkg_resources
find_controller_generic = PylonsApp.find_controller
# This is from pylons 1.0 source, will monkey-patch into 0.9.7
def find_controller(self, controller):
if controller in self.controller_classes:
return self.controller_classes[controller]
# Check to see if its a dotted name
if '.' in controller or ':' in controller:
mycontroller = pkg_resources.EntryPoint.parse(
'x=%s' % controller).load(False)
self.controller_classes[controller] = mycontroller
return mycontroller
return find_controller_generic(self, controller)
PylonsApp.find_controller = find_controller
###### END evil monkey-patch
os.environ['CKAN_CONFIG'] = global_conf['__file__']
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[os.path.join(root, 'templates')])
# Initialize config with the basic options
config.init_app(global_conf, app_conf, package='ckan', paths=paths)
# load all CKAN plugins
plugins.load_all(config)
from ckan.plugins import PluginImplementations
from ckan.plugins.interfaces import IConfigurer
for plugin in PluginImplementations(IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# This is set up before globals are initialized
site_id = os.environ.get('CKAN_SITE_ID')
if site_id:
config['ckan.site_id'] = site_id
site_url = config.get('ckan.site_url', '')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# Init SOLR settings and check if the schema is compatible
from ckan.lib.search import SolrSettings, check_solr_schema_version
SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
check_solr_schema_version()
config['routes.map'] = make_map()
config['pylons.app_globals'] = app_globals.Globals()
if asbool(config.get('ckan.restrict_template_vars', 'false')):
import ckan.lib.helpers_clean
config['pylons.h'] = ckan.lib.helpers_clean
else:
config['pylons.h'] = h
## redo template setup to use genshi.search_path (so remove std template setup)
template_paths = [paths['templates'][0]]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
# Translator (i18n)
translator = Translator(pylons.translator)
def template_loaded(template):
translator.setup(template)
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create the Genshi TemplateLoader
config['pylons.app_globals'].genshi_loader = TemplateLoader(
template_paths, auto_reload=True, callback=template_loaded)
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# Setup the SQLAlchemy database engine
# Suppress a couple of sqlalchemy warnings
warnings.filterwarnings(
'ignore', '^Unicode type received non-unicode bind param value',
sqlalchemy.exc.SAWarning)
warnings.filterwarnings(
'ignore', "^Did not recognize type 'BIGINT' of column 'size'",
sqlalchemy.exc.SAWarning)
warnings.filterwarnings(
'ignore',
"^Did not recognize type 'tsvector' of column 'search_vector'",
sqlalchemy.exc.SAWarning)
ckan_db = os.environ.get('CKAN_DB')
if ckan_db:
engine = sqlalchemy.create_engine(ckan_db)
else:
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
if not model.meta.engine:
model.init_model(engine)
from ckan.plugins import PluginImplementations
from ckan.plugins.interfaces import IConfigurable
for plugin in PluginImplementations(IConfigurable):
plugin.configure(config)
class GobArPackageController(PackageController):
def __generate_spatial_extra_field(self, data_dict):
extras = data_dict['extras']
def __find_extra(extras, key, create=False):
for extra in extras:
if extra['key'] == key:
return extra
if create:
extra = {'key': key}
extras.append(extra)
return extra
country = __find_extra(extras, 'country')
if country:
spatial = __find_extra(extras, 'spatial', True)
spatial['value'] = [country['value']]
province = __find_extra(extras, 'province')
if province and province['value']:
spatial['value'].append(province['value'])
district = __find_extra(extras, 'district')
if district and district['value']:
spatial['value'].append(district['value'])
spatial['value'] = ','.join(spatial['value'])
def search(self):
package_type = self._guess_package_type()
try:
context = {
'model': model,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
q = c.q = request.params.get('q', u'')
c.query_error = False
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search')
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and value and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
# Modificación para andino: usamos una función para buscar dependencias entre organizaciones
if param != 'organization':
fq += ' %s:"%s"' % (param, value)
else:
fq += custom_organization_filter(value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'auth_user_obj': c.userobj
}
if package_type and package_type != 'dataset':
fq += ' +dataset_type:{type}'.format(type=package_type)
elif not asbool(config.get('ckan.search.show_all_types', 'False')):
fq += ' +dataset_type:dataset'
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': fq.strip(),
'facet.field': facets.keys(),
'rows': limit,
'start': (page - 1) * limit,
'sort': sort_by,
'extras': search_extras
}
query = logic.action.get.package_search(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchError, se:
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.search_facets = {}
c.page = h.Page(collection=[])
c.search_facets_limits = {}
for facet in c.search_facets.keys():
try:
# Modificación para andino: chequeo si el facet es 'organization'
if facet != 'organization':
limit = int(
request.params.get(
'_%s_limit' % facet,
int(config.get('search.facets.default', 10))))
else:
limit = None
except ValueError:
error_description = _(
'Parameter "{parameter_name}" is not an integer')
error_description = error_description.format(
parameter_name='_%s_limit' % facet)
abort(400, error_description)
c.search_facets_limits[facet] = limit
self._setup_template_variables(context, {}, package_type=package_type)
return render(self._search_template(package_type),
extra_vars={'dataset_type': package_type})
def drupal_site_url():
from pylons import config
return config.get('drupal.site_url')
def index_package(self, pkg_dict, defer_commit=False):
import solr.core
if pkg_dict is None:
return
pkg_dict['data_dict'] = json.dumps(pkg_dict)
if config.get('ckan.cache_validated_datasets', True):
package_plugin = lib_plugins.lookup_package_plugin(
pkg_dict.get('type'))
schema = package_plugin.show_package_schema()
validated_pkg_dict, errors = _validate(pkg_dict, schema, {
'model': model,
'session': model.Session
})
pkg_dict['validated_data_dict'] = json.dumps(
validated_pkg_dict,
cls=ckan.lib.navl.dictization_functions.MissingNullEncoder)
# add to string field for sorting
title = pkg_dict.get('title')
if title:
pkg_dict['title_string'] = title
if (not pkg_dict.get('state')) or ('active'
not in pkg_dict.get('state')):
return self.delete_package(pkg_dict)
index_fields = RESERVED_FIELDS + pkg_dict.keys()
# include the extras in the main namespace
extras = pkg_dict.get('extras', [])
for extra in extras:
key, value = extra['key'], extra['value']
if isinstance(value, (tuple, list)):
value = " ".join(map(unicode, value))
key = ''.join([c for c in key if c in KEY_CHARS])
pkg_dict['extras_' + key] = value
if key not in index_fields:
pkg_dict[key] = value
pkg_dict.pop('extras', None)
# add tags, removing vocab tags from 'tags' list and adding them as
# vocab_<tag name> so that they can be used in facets
non_vocab_tag_names = []
tags = pkg_dict.pop('tags', [])
context = {'model': model}
for tag in tags:
if tag.get('vocabulary_id'):
data = {'id': tag['vocabulary_id']}
vocab = logic.get_action('vocabulary_show')(context, data)
key = u'vocab_%s' % vocab['name']
if key in pkg_dict:
pkg_dict[key].append(tag['name'])
else:
pkg_dict[key] = [tag['name']]
else:
non_vocab_tag_names.append(tag['name'])
pkg_dict['tags'] = non_vocab_tag_names
# add groups
groups = pkg_dict.pop('groups', [])
# we use the capacity to make things private in the search index
if pkg_dict['private']:
pkg_dict['capacity'] = 'private'
else:
pkg_dict['capacity'] = 'public'
pkg_dict['groups'] = [group['name'] for group in groups]
# if there is an owner_org we want to add this to groups for index
# purposes
if pkg_dict.get('organization'):
pkg_dict['organization'] = pkg_dict['organization']['name']
else:
pkg_dict['organization'] = None
# tracking
tracking_summary = pkg_dict.pop('tracking_summary', None)
if tracking_summary:
pkg_dict['views_total'] = tracking_summary['total']
pkg_dict['views_recent'] = tracking_summary['recent']
# flatten the structure for indexing:
for resource in pkg_dict.get('resources', []):
for (okey, nkey) in [('description', 'res_description'),
('format', 'res_format'), ('url', 'res_url')]:
pkg_dict[nkey] = pkg_dict.get(nkey,
[]) + [resource.get(okey, u'')]
pkg_dict.pop('resources', None)
rel_dict = collections.defaultdict(list)
subjects = pkg_dict.pop("relationships_as_subject", [])
objects = pkg_dict.pop("relationships_as_object", [])
for rel in objects:
type = model.PackageRelationship.forward_to_reverse_type(
rel['type'])
rel_dict[type].append(
model.Package.get(rel['subject_package_id']).name)
for rel in subjects:
type = rel['type']
rel_dict[type].append(
model.Package.get(rel['object_package_id']).name)
for key, value in rel_dict.iteritems():
if key not in pkg_dict:
pkg_dict[key] = value
pkg_dict[TYPE_FIELD] = PACKAGE_TYPE
# Save dataset type
pkg_dict['dataset_type'] = pkg_dict['type']
# clean the dict fixing keys and dates
# FIXME where are we getting these dirty keys from? can we not just
# fix them in the correct place or is this something that always will
# be needed? For my data not changing the keys seems to not cause a
# problem.
new_dict = {}
for key, value in pkg_dict.items():
key = key.encode('ascii', 'ignore')
if key.endswith('_date'):
try:
value = parse(value).isoformat() + 'Z'
except ValueError:
continue
new_dict[key] = value
pkg_dict = new_dict
for k in ('title', 'notes', 'title_string'):
if k in pkg_dict and pkg_dict[k]:
pkg_dict[k] = escape_xml_illegal_chars(pkg_dict[k])
# modify dates (SOLR is quite picky with dates, and only accepts ISO dates
# with UTC time (i.e trailing Z)
# See http://lucene.apache.org/solr/api/org/apache/solr/schema/DateField.html
pkg_dict['metadata_created'] += 'Z'
pkg_dict['metadata_modified'] += 'Z'
# mark this CKAN instance as data source:
pkg_dict['site_id'] = config.get('ckan.site_id')
# Strip a selection of the fields.
# These fields are possible candidates for sorting search results on,
# so we strip leading spaces because solr will sort " " before "a" or "A".
for field_name in ['title']:
try:
value = pkg_dict.get(field_name)
if value:
pkg_dict[field_name] = value.lstrip()
except KeyError:
pass
# add a unique index_id to avoid conflicts
import hashlib
pkg_dict['index_id'] = hashlib.md5(
'%s%s' % (pkg_dict['id'], config.get('ckan.site_id'))).hexdigest()
for item in PluginImplementations(IPackageController):
pkg_dict = item.before_index(pkg_dict)
assert pkg_dict, 'Plugin must return non empty package dict on index'
# send to solr:
try:
conn = make_connection()
commit = not defer_commit
if not asbool(config.get('ckan.search.solr_commit', 'true')):
commit = False
conn.add_many([pkg_dict], _commit=commit)
except solr.core.SolrException, e:
msg = 'Solr returned an error: {0} {1} - {2}'.format(
e.httpcode,
e.reason,
e.body[:1000] # limit huge responses
)
raise SearchIndexError(msg)
def drupal_get_secondary_header():
import urllib2
from pylons import config
secondary_header = urllib2.urlopen(config.get('drupal.site_url') + '/koop_theme/get/secondary_header')
return secondary_header.read().decode('utf-8')
def drupal_get_dataset_quality(table, id):
import urllib2
from pylons import config
text = urllib2.urlopen(config.get('drupal.site_url') + '/service/linkcheck/' + table + '/' + id)
return text.read().decode('utf-8')
def configure(self, config):
from ckanext.spatial.model.package_extent import setup as setup_model
if not p.toolkit.asbool(config.get('ckan.spatial.testing', 'False')):
log.debug('Setting up the spatial model')
setup_model()
def drupal_get_footer():
import urllib2
from pylons import config
footer = urllib2.urlopen(config.get('drupal.site_url') + '/koop_theme/get/footer')
return footer.read().decode('utf-8')
def validator(key, data, errors, context):
if errors[key]:
return
value = data[key]
if value is not missing:
if value:
return
output = {}
prefix = field['autogeneration_field']
if not prefix:
prefix = DEFAULT_TITLE_FIELD
log.debug('[multilanguage_url] Creating field using the field %s', prefix)
prefix = prefix + '-'
extras = data.get(key[:-1] + ('__extras',), {})
locale_default = config.get('ckan.locale_default', 'es')
if locale_default:
title_lang = prefix + locale_default
if title_lang in extras and extras[title_lang]:
dataset_title = extras[title_lang]
# Generate title prefix
field_prefix = field['organization_field']
organization_prefix = field['organization_prefix']
publisher_id = data.get((field_prefix,))
if not publisher_id and field_prefix in extras:
publisher_id = extras[field_prefix]
if publisher_id and organization_prefix:
organization = h.get_organization(publisher_id)
if not organization:
organization = toolkit.get_action('dge_organization_publisher')({'model': model}, {'id': publisher_id})
if organization and organization['extras']:
for extra in organization['extras']:
if extra['key'] == organization_prefix and extra['state'] == 'active':
dataset_title = extra['value'] + '-' + dataset_title
break
data[key] = munge.munge_title_to_name(dataset_title)
log.debug('[multilanguage_url] Created name "%s" for package from language %s',
data[key], locale_default)
return
locale_order = config.get('ckan.locale_order', '').split()
for l in locale_order:
title_lang = prefix + l
if title_lang in extras and extras[title_lang]:
dataset_title = extras[title_lang]
# Generate title prefix
field_prefix = field['organization_field']
organization_prefix = field['organization_prefix']
publisher_id = data.get((field_prefix,))
if not publisher_id and field_prefix in extras:
publisher_id = extras[field_prefix]
if publisher_id and organization_prefix:
organization = h.get_organization(publisher_id)
if not organization:
organization = toolkit.get_action('dge_organization_publisher')({'model': model}, {'id': publisher_id})
if organization and organization['extras']:
for extra in organization['extras']:
if extra['key'] == organization_prefix and extra['state'] == 'active':
dataset_title = extra['value'] + '-' + dataset_title
break
data[key] = munge.munge_title_to_name(dataset_title)
log.debug('[multilanguage_url] Created name "%s" for package from language %s',
data[key], l)
break
def drupal_get_header():
import urllib2
from pylons import config
navigation = urllib2.urlopen(config.get('drupal.site_url') + '/koop_theme/get/header')
return navigation.read().decode('utf-8')
def _get_content_and_type(self,
url,
harvest_job,
page=1,
content_type=None):
'''
Gets the content and type of the given url.
:param url: a web url (starting with http) or a local path
:param harvest_job: the job, used for error reporting
:param page: adds paging to the url
:param content_type: will be returned as type
:return: a tuple containing the content and content-type
'''
method_log_prefix = '[%s][_get_content_and_type]' % type(self).__name__
if not url.lower().startswith('http'):
# Check local file
if os.path.exists(url):
with open(url, 'r') as f:
content = f.read()
content_type = content_type or rdflib.util.guess_format(url)
return content, content_type
else:
msg = u'No se pudo obtener contenido de esta url' #'Could not get content for this url'
errormsg = dhc.CATALOG_ACCESS_ERROR % (msg)
if url:
errormsg = dhc.CATALOG_ACCESS_ERROR_URL % (url, msg)
self._save_gather_error(errormsg, harvest_job)
return None, None
try:
if page > 1:
url = url + '&' if '?' in url else url + '?'
url = url + 'page={0}'.format(page)
log.debug('%s Getting file %s' % (method_log_prefix, url))
http_proxy = pc.get(dhc.CKAN_PROP_HTTP_PROXY, None)
https_proxy = pc.get(dhc.CKAN_PROP_HTTPS_PROXY, None)
proxies = None
if http_proxy or https_proxy:
proxies = {}
if http_proxy:
proxies['http'] = http_proxy
if https_proxy:
proxies['https'] = https_proxy
log.debug("%s Using proxies %s" % (method_log_prefix, proxies))
else:
log.debug("%s No proxies" % (method_log_prefix))
# first we try a HEAD request which may not be supported
did_get = False
r = requests.head(url, proxies=proxies, verify=False)
if r.status_code == 405 or r.status_code == 400 or r.status_code == 404:
r = requests.get(url,
proxies=proxies,
stream=True,
verify=False)
did_get = True
r.raise_for_status()
cl = r.headers.get('content-length')
if cl and int(cl) > self.MAX_FILE_SIZE:
# msg = '''Remote file is too big. Allowed
# file size: {allowed}, Content-Length: {actual}.'''.format(
# allowed=self.MAX_FILE_SIZE, actual=cl)
msg = u'''El fichero remoto es demasiado grande. Tama\u00F1o
de fichero permitido: {allowed], Longitud del contenido: {actual}'''.format(
allowed=self.MAX_FILE_SIZE, actual=cl)
errormsg = dhc.CATALOG_ACCESS_ERROR % (msg)
if url:
errormsg = dhc.CATALOG_ACCESS_ERROR_URL % (url, msg)
self._save_gather_error(errormsg, harvest_job)
return None, None
if not did_get:
r = requests.get(url,
proxies=proxies,
stream=True,
verify=False)
length = 0
content = ''
for chunk in r.iter_content(chunk_size=self.CHUNK_SIZE):
content = content + chunk
length += len(chunk)
if length >= self.MAX_FILE_SIZE:
msg = u'El fichero remoto es demasiado grande' #'Remote file is too big.'
errormsg = dhc.CATALOG_ACCESS_ERROR % (msg)
if url:
errormsg = dhc.CATALOG_ACCESS_ERROR_URL % (url, msg)
self._save_gather_error(errormsg, harvest_job)
return None, None
if content_type is None and r.headers.get('content-type'):
content_type = r.headers.get('content-type').split(";", 1)[0]
return content, content_type
except requests.exceptions.HTTPError, error:
if page > 1 and error.response.status_code == 404:
# We want to catch these ones later on
raise
# msg = 'Could not get content. Server responded with %s %s' % (
# error.response.status_code, error.response.reason)
msg = u'No se pudo obtener contenido. El servidor respondi\u00F3 con %s %s' % (
error.response.status_code, error.response.reason)
errormsg = dhc.CATALOG_ACCESS_ERROR % (msg)
if url:
errormsg = dhc.CATALOG_ACCESS_ERROR_URL % (url, msg)
self._save_gather_error(errormsg, harvest_job)
return None, None
def validator(key, data, errors, context):
if errors[key]:
return
values = []
languages = []
required_language = field['required_language']
if not required_language:
required_language = config.get('ckan.locale_default', 'es')
required_field = field.get('required')
if not required_field:
required_field = False
#Get value from dict or JSON encoded string
value = data[key]
if value is not missing:
if isinstance(value, basestring):
try:
value = json.loads(value)
except ValueError:
errors[key].append(_('Failed to decode JSON string'))
return
except UnicodeDecodeError:
errors[key].append(_('Invalid encoding for JSON string'))
return
if not isinstance(value, dict):
errors[key].append(_('Expecting JSON object'))
return
for lang, text in value.iteritems():
try:
m = re.match(ISO_639_LANGUAGE, lang)
except TypeError:
errors[key].append(_('Invalid type for language code: %r')
% lang)
continue
#Register the language
languages += [lang]
if not isinstance(text, basestring):
errors[key].append(_('Invalid type for "%s" value') % lang)
continue
if isinstance(text, str) or isinstance(text, unicode):
try:
#Not register empty values
if text.strip():
values += [lang]
except UnicodeDecodeError:
errors[key]. append(_('Invalid encoding for "%s" value')
% lang)
#Error if texts exist but not in required language
if values and not required_language in values:
errors[key[:-1] + (key[-1] + '-' + required_language,)] = [_('Missing required language value')]
else:
if not values and required_field:
errors[key[:-1] + (key[-1] + '-' + required_language,)] = [_('Missing required language value')]
return
#Get values from formulary
prefix = key[-1] + '-'
extras = data.get(key[:-1] + ('__extras',), {})
for name, text in extras.iteritems():
if not name.startswith(prefix):
continue
lang = name.split('-', 1)[1]
m = re.match(ISO_639_LANGUAGE, lang)
if not m:
errors[name] = [_('Invalid language code: "%s"') % lang]
values = None
continue
languages += [lang]
if text:
values += [lang]
#Error if texts exist but not in required language
if values and not required_language in values:
errors[key[:-1] + (key[-1] + '-' + required_language,)] = [_('Missing required language value')]
else:
if not values and required_field:
errors[key[:-1] + (key[-1] + '-' + required_language,)] = [_('Missing required language value')]
def _private_key(self):
return config.get('recaptcha.private_key')
from linotp.lib.user import getUserInfo
from linotp.lib.util import check_session
from linotp.lib.util import getLowerParams
from linotp.lib.util import get_client
from linotp.lib.context import request_context
import linotp.model
from linotp.lib.error import UserError
import linotp.model.meta
Session = linotp.model.meta.Session
audit = config.get('audit')
log = logging.getLogger(__name__)
class OcraController(BaseController):
'''
The OcraController implements challenges/response tokens
according to RFC 6287
'''
def __before__(self, action, **params):
'''
Here we see, what action is to be called and check the authorization
'''
try:
import datetime
from pylons import config
from sqlalchemy import Table, select, func, and_
from sqlalchemy.sql.expression import text
import ckan.plugins as p
import ckan.model as model
import re
cache_enabled = p.toolkit.asbool(
config.get('ckanext.stats.cache_enabled', 'True'))
if cache_enabled:
from pylons import cache
our_cache = cache.get_cache('stats', type='dbm')
DATE_FORMAT = '%Y-%m-%d'
def table(name):
return Table(name, model.meta.metadata, autoload=True)
def datetime2date(datetime_):
return datetime.date(datetime_.year, datetime_.month, datetime_.day)
class Stats(object):
@classmethod
def transform_to_iso(self, original_document, original_format,
harvest_object):
if original_format != 'fgdc':
return None
transform_service = config.get('ckanext.geodatagov.fgdc2iso_service')
if not transform_service:
self._save_object_error('No FGDC to ISO transformation service',
harvest_object, 'Import')
return None
# Validate against FGDC schema
if self.source_config.get('validator_profiles'):
profiles = self.source_config.get('validator_profiles')
else:
profiles = ['fgdc_minimal']
validator = Validators(profiles=profiles)
for custom_validator in custom_validators:
validator.add_validator(custom_validator)
is_valid, profile, errors = self._validate_document(
original_document, harvest_object, validator=validator)
if not is_valid:
# TODO: Provide an option to continue anyway
return None
original_document = re.sub('<\?xml(.*)\?>', '', original_document)
tree = etree.fromstring(original_document)
comments = tree.xpath('//comment()')
for comment in comments:
p = comment.getparent()
if p:
p.remove(comment)
ptvctcnt = tree.xpath('//ptvctcnt')
for node in ptvctcnt:
p = node.getparent()
if p and not node.text:
p.remove(node)
themekt = tree.xpath('//placekt')
for num, node in enumerate(themekt):
p = node.getparent()
###remove all but first
if p and num > 0:
p.remove(node)
original_document = etree.tostring(tree)
response = requests.post(
transform_service,
data=original_document.encode('utf8'),
headers={'content-type': 'text/xml; charset=utf-8'})
if response.status_code == 200:
# XML coming from the conversion tool is already declared and encoded as utf-8
return response.content
else:
msg = 'The transformation service returned an error for object {0}'
if response.status_code and response.content:
msg += ': [{0}] {1}'.format(response.status_code,
response.content)
elif response.error:
msg += ': {0}'.format(response.error)
self._save_object_error(msg, harvest_object, 'Import')
return None
def render(template_name,
extra_vars=None,
cache_key=None,
cache_type=None,
cache_expire=None,
method='xhtml',
loader_class=MarkupTemplate,
cache_force=None,
renderer=None):
'''Render a template and return the output.
This is CKAN's main template rendering function.
.. todo::
Document the parameters of :py:func:`ckan.plugins.toolkit.render`.
'''
def render_template():
globs = extra_vars or {}
globs.update(pylons_globals())
globs['actions'] = model.Action
# Using pylons.url() directly destroys the localisation stuff so
# we remove it so any bad templates crash and burn
del globs['url']
try:
template_path, template_type = render_.template_info(template_name)
except render_.TemplateNotFound:
raise
log.debug('rendering %s [%s]' % (template_path, template_type))
if config.get('debug'):
context_vars = globs.get('c')
if context_vars:
context_vars = dir(context_vars)
debug_info = {
'template_name': template_name,
'template_path': template_path,
'template_type': template_type,
'vars': globs,
'c_vars': context_vars,
'renderer': renderer
}
if 'CKAN_DEBUG_INFO' not in request.environ:
request.environ['CKAN_DEBUG_INFO'] = []
request.environ['CKAN_DEBUG_INFO'].append(debug_info)
# Jinja2 templates
if template_type == 'jinja2':
# We don't want to have the config in templates it should be
# accessed via g (app_globals) as this gives us flexability such
# as changing via database settings.
del globs['config']
# TODO should we raise error if genshi filters??
return render_jinja2(template_name, globs)
# Genshi templates
template = globs['app_globals'].genshi_loader.load(
template_name.encode('utf-8'), cls=loader_class)
stream = template.generate(**globs)
for item in p.PluginImplementations(p.IGenshiStreamFilter):
stream = item.filter(stream)
if loader_class == NewTextTemplate:
return literal(stream.render(method="text", encoding=None))
return literal(
stream.render(method=method, encoding=None, strip_whitespace=True))
if 'Pragma' in response.headers:
del response.headers["Pragma"]
## Caching Logic
allow_cache = True
# Force cache or not if explicit.
if cache_force is not None:
allow_cache = cache_force
# Do not allow caching of pages for logged in users/flash messages etc.
elif session.last_accessed:
allow_cache = False
# Tests etc.
elif 'REMOTE_USER' in request.environ:
allow_cache = False
# Don't cache if based on a non-cachable template used in this.
elif request.environ.get('__no_cache__'):
allow_cache = False
# Don't cache if we have set the __no_cache__ param in the query string.
elif request.params.get('__no_cache__'):
allow_cache = False
# Don't cache if we have extra vars containing data.
elif extra_vars:
for k, v in extra_vars.iteritems():
allow_cache = False
break
# Record cachability for the page cache if enabled
request.environ['CKAN_PAGE_CACHABLE'] = allow_cache
if allow_cache:
response.headers["Cache-Control"] = "public"
try:
cache_expire = int(config.get('ckan.cache_expires', 0))
response.headers["Cache-Control"] += \
", max-age=%s, must-revalidate" % cache_expire
except ValueError:
pass
else:
# We do not want caching.
response.headers["Cache-Control"] = "private"
# Prevent any further rendering from being cached.
request.environ['__no_cache__'] = True
log.debug('Template cache-control: %s' % response.headers["Cache-Control"])
# Render Time :)
try:
return cached_template(template_name,
render_template,
loader_class=loader_class)
except ckan.exceptions.CkanUrlException, e:
raise ckan.exceptions.CkanUrlException(
'\nAn Exception has been raised for template %s\n%s' %
(template_name, e.message))
def _public_key(self):
return config.get('recaptcha.public_key')
class ListController(BaseController):
request_path = config.get('list.staticmax_url')
@classmethod
@jsonify
def overview(cls, slug):
game = get_game_by_slug(slug)
if not game:
response.status_int = 404
return {'ok': False, 'msg': 'Game does not exist: %s' % slug}
return {
'ok': True,
'data': {
'slug': game.slug,
'staticFilePrefix' : 'staticmax',
'mappingTable': game.mapping_table
}
}
@classmethod
@jsonify
def assets(cls, slug, path=''):
game = get_game_by_slug(slug)
if not game:
response.status_int = 404
return {'ok': False, 'msg': 'Game does not exist: %s' % slug}
try:
asset_list = game.get_asset_list(cls.request_path, path)
except (GamePathError, GamePathNotFoundError) as e:
response.status_int = 404
return {'ok': False, 'msg': str(e)}
else:
return {
'ok': True,
'data': {
'items': [i.as_dict() for i in asset_list],
'path': path.strip('/'),
'mappingTable': game.has_mapping_table
}
}
@classmethod
@jsonify
def files(cls, slug, path=''):
game = get_game_by_slug(slug)
if not game:
response.status_int = 404
return {'ok': False, 'msg': 'Game does not exist: %s' % slug}
try:
asset_list = game.get_static_files(game.path, cls.request_path, path)
except GamePathNotFoundError as e:
response.status_int = 404
return {'ok': False, 'msg': str(e)}
else:
return {
'ok': True,
'data': {
'items': [ i.as_dict() for i in asset_list ],
'path': path.strip('/'),
'mappingTable': game.has_mapping_table
}
}
def publisher_list(self):
package_type = 'dataset'
try:
context = {'model': model, 'user': c.user or c.author,
'auth_user_obj': c.userobj}
check_access('openness', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
c.pkg_dict = {}
c.publishers = logic.get_action('organization_list')(context, {'all_fields': True})
report = {}
locale = tk.request.environ['CKAN_LANG'] or config.get('ckan.locale_default', 'en')
cache_key = 'global_openness:{0}'.format(locale)
g_start_time = time.time()
dict_string = cache.get_from_cache(cache_key, pool=cache.MISC_POOL)
if dict_string:
start_time = time.time()
report = pickle.loads(dict_string)
duration = time.time()-start_time
log.info("Loading json took {0}".format(duration))
else:
osp_start = time.time()
rows = self._openness_sores_for_publisher()
osp_duration = time.time()- osp_start
log.info("_openness_sores_for_publisher took {0}".format(osp_duration))
ps_start = time.time()
publishers = model.Group.all(group_type='organization')
dict_pub = {}
for publisher in publishers:
dict_pub[publisher.name] = publisher
for ds_name, obj in rows.iteritems():
if not obj.get('owner_org'):
continue
publisher = dict_pub.get(obj.get('owner_org').split('/')[-1].lower(),model.Group())
#publisher = next((pub for pub in publishers if pub.name == model.Group.get(obj.get('owner_org').split('/')[-1].lower())),None)
publ_report = report.get(publisher.name or publisher.id, {'publisher_name': publisher.title,
'publisher_id': publisher.name or publisher.id,
'zero': 0,
'one': 0,
'two': 0,
'three': 0,
'four': 0,
'five': 0,
'sum': 0,
'avg': 0})
column_key = self._set_dataset_score(obj['sum_value'])
publ_report[column_key] += 1
publ_report = self.calculate_sum(publ_report)
publ_report = self.calculate_avg(publ_report)
report[publisher.name] = publ_report
#cache.set_value_in_cache(cache_key,pickle.dumps(report), pool=redis_cache.MISC_POOL)
ps_duration = time.time()- ps_start
log.info("publisher list 1st loop took {0}".format(ps_duration))
totals = {'zero': 0,
'one': 0,
'two': 0,
'three': 0,
'four': 0,
'five': 0}
for publ, value in report.iteritems():
totals['zero'] = value['zero'] + totals['zero']
totals['one'] = value['one'] + totals['one']
totals['two'] = value['two'] + totals['two']
totals['three'] = value['three'] + totals['three']
totals['four'] = value['four'] + totals['four']
totals['five'] = value['five'] + totals['five']
common_formats = self.get_format_summary_list()
result = {}
result['table'] = report
result['totals'] = totals
result['json'] = json.dumps(totals)
result['common_formats'] = json.dumps(common_formats)
c.pkg_dict = result
self._setup_template_variables(context, {},
package_type=package_type)
g_duration = time.time()-g_start_time
log.info("Global Loading took {0}".format(g_duration))
# c.form = base.render(self._package_form(package_type='upload_package'), extra_vars=vars)
return base.render('openness/publisher_list.html')
def _get_default_locale():
return config.get('ckan.locale_default', 'en')
def _pagination_info(query, data_dict):
'''
Creates a pagination_info dict to be passed to the serializers
`query` is the output of `package_search` and `data_dict`
contains the request params.
The keys for the dictionary are:
* `count` (total elements)
* `items_per_page` (`ckanext.dcat.datasets_per_page` or 100)
* `current`
* `first`
* `last`
* `next`
* `previous`
Returns a dict
'''
def _page_url(page):
base_url = config.get('ckan.site_url', '').strip('/')
if not base_url:
base_url = toolkit.request.host_url
base_url = '%s%s' % (base_url, toolkit.request.path)
params = [
p for p in toolkit.request.params.iteritems() if p[0] != 'page'
]
if params:
qs = '&'.join(['{0}={1}'.format(p[0], p[1]) for p in params])
return '{0}?{1}&page={2}'.format(base_url, qs, page)
else:
return '{0}?page={1}'.format(base_url, page)
try:
page = int(data_dict.get('page', 1) or 1)
if page < 1:
raise wrong_page_exception
except ValueError:
raise wrong_page_exception
if query['count'] == 0:
return {}
items_per_page = int(
config.get('ckanext.dcat.datasets_per_page', DATASETS_PER_PAGE))
pagination_info = {
'count': query['count'],
'items_per_page': items_per_page,
}
pagination_info['current'] = _page_url(page)
pagination_info['first'] = _page_url(1)
last_page = int(math.ceil(query['count'] / items_per_page)) or 1
pagination_info['last'] = _page_url(last_page)
if page > 1:
if ((page - 1) * items_per_page +
len(query['results'])) <= query['count']:
previous_page = page - 1
else:
previous_page = last_page
pagination_info['previous'] = _page_url(previous_page)
if page * items_per_page < query['count']:
pagination_info['next'] = _page_url(page + 1)
return pagination_info