def get_storage_path():
'''Function to cache storage path'''
global _storage_path
# None means it has not been set. False means not in config.
if _storage_path is None:
storage_path = config.get('ckan.storage_path')
ofs_impl = config.get('ofs.impl')
ofs_storage_dir = config.get('ofs.storage_dir')
if storage_path:
_storage_path = storage_path
elif ofs_impl == 'pairtree' and ofs_storage_dir:
log.warn('''Please use config option ckan.storage_path instead of
ofs.storage_dir''')
_storage_path = ofs_storage_dir
return _storage_path
elif ofs_impl:
log.critical('''We only support local file storage form version 2.2
of ckan please specify ckan.storage_path in your
config for your uploads''')
_storage_path = False
else:
log.critical('''Please specify a ckan.storage_path in your config
for your uploads''')
_storage_path = False
return _storage_path
def mail_recipient(recipient_name, recipient_email, subject,
body, headers={}):
site_title = config.get('ckan.site_title')
site_url = config.get('ckan.site_url')
return _mail_recipient(recipient_name, recipient_email,
site_title, site_url, subject, body,
headers=headers)
def _setup_error_mail_handler(app):
class ContextualFilter(logging.Filter):
def filter(self, log_record):
log_record.url = request.path
log_record.method = request.method
log_record.ip = request.environ.get("REMOTE_ADDR")
log_record.headers = request.headers
return True
mailhost = tuple(config.get('smtp.server', 'localhost').split(":"))
mail_handler = SMTPHandler(
mailhost=mailhost,
fromaddr=config.get('error_email_from'),
toaddrs=[config.get('email_to')],
subject='Application Error'
)
mail_handler.setFormatter(logging.Formatter('''
Time: %(asctime)s
URL: %(url)s
Method: %(method)s
IP: %(ip)s
Headers: %(headers)s
'''))
context_provider = ContextualFilter()
app.logger.addFilter(context_provider)
app.logger.addHandler(mail_handler)
def before_search(self, search_params):
lang_set = set(self.LANGS)
try:
current_lang = request.environ['CKAN_LANG']
except TypeError as err:
if err.message == ('No object (name: request) has been registered '
'for this thread'):
# This happens when this code gets called as part of a paster
# command rather then as part of an HTTP request.
current_lang = config.get('ckan.locale_default')
else:
raise
except KeyError:
current_lang = config.get('ckan.locale_default')
# fallback to default locale if locale not in suported langs
if not current_lang in lang_set:
current_lang = config.get('ckan.locale_default')
# fallback to english if default locale is not supported
if not current_lang in lang_set:
current_lang = 'en'
# treat current lang differenly so remove from set
lang_set.remove(current_lang)
# weight current lang more highly
query_fields = 'title_%s^8 text_%s^4' % (current_lang, current_lang)
for lang in lang_set:
query_fields += ' title_%s^2 text_%s' % (lang, lang)
search_params['qf'] = query_fields
return search_params
def webassets_init():
global env
static_path = get_webassets_path()
public = config.get(u'ckan.base_public_folder')
public_folder = os.path.abspath(os.path.join(
os.path.dirname(__file__), u'..', public))
base_path = os.path.join(public_folder, u'base')
env = Environment()
env.directory = static_path
env.debug = config.get(u'debug', False)
env.url = u'/webassets/'
env.append_path(base_path, u'/base/')
logger.debug(u'Base path {0}'.format(base_path))
create_library(u'vendor', os.path.join(
base_path, u'vendor'))
create_library(u'base', os.path.join(base_path, u'javascript'))
create_library(u'datapreview', os.path.join(base_path, u'datapreview'))
create_library(u'css', os.path.join(base_path, u'css'))
def set_cors_headers_for_response(response):
u'''
Set up Access Control Allow headers if either origin_allow_all is True, or
the request Origin is in the origin_whitelist.
'''
if config.get(u'ckan.cors.origin_allow_all') \
and request.headers.get(u'Origin'):
cors_origin_allowed = None
if asbool(config.get(u'ckan.cors.origin_allow_all')):
cors_origin_allowed = b'*'
elif config.get(u'ckan.cors.origin_whitelist') and \
request.headers.get(u'Origin') \
in config[u'ckan.cors.origin_whitelist'].split(u' '):
# set var to the origin to allow it.
cors_origin_allowed = request.headers.get(u'Origin')
if cors_origin_allowed is not None:
response.headers[b'Access-Control-Allow-Origin'] = \
cors_origin_allowed
response.headers[b'Access-Control-Allow-Methods'] = \
b'POST, PUT, GET, DELETE, OPTIONS'
response.headers[b'Access-Control-Allow-Headers'] = \
b'X-CKAN-API-KEY, Authorization, Content-Type'
return response
def _mail_recipient(recipient_name, recipient_email,
sender_name, sender_url, subject,
body, headers={}):
mail_from = config.get('smtp.mail_from')
msg = MIMEText(body.encode('utf-8'), 'plain', 'utf-8')
for k, v in headers.items():
msg[k] = v
subject = Header(subject.encode('utf-8'), 'utf-8')
msg['Subject'] = subject
msg['From'] = _("%s <%s>") % (sender_name, mail_from)
recipient = u"%s <%s>" % (recipient_name, recipient_email)
msg['To'] = Header(recipient, 'utf-8')
msg['Date'] = Utils.formatdate(time())
msg['X-Mailer'] = "CKAN %s" % ckan.__version__
# Send the email using Python's smtplib.
smtp_connection = smtplib.SMTP()
if 'smtp.test_server' in config:
# If 'smtp.test_server' is configured we assume we're running tests,
# and don't use the smtp.server, starttls, user, password etc. options.
smtp_server = config['smtp.test_server']
smtp_starttls = False
smtp_user = None
smtp_password = None
else:
smtp_server = config.get('smtp.server', 'localhost')
smtp_starttls = paste.deploy.converters.asbool(
config.get('smtp.starttls'))
smtp_user = config.get('smtp.user')
smtp_password = config.get('smtp.password')
smtp_connection.connect(smtp_server)
try:
# Identify ourselves and prompt the server for supported features.
smtp_connection.ehlo()
# If 'smtp.starttls' is on in CKAN config, try to put the SMTP
# connection into TLS mode.
if smtp_starttls:
if smtp_connection.has_extn('STARTTLS'):
smtp_connection.starttls()
# Re-identify ourselves over TLS connection.
smtp_connection.ehlo()
else:
raise MailerException("SMTP server does not support STARTTLS")
# If 'smtp.user' is in CKAN config, try to login to SMTP server.
if smtp_user:
assert smtp_password, ("If smtp.user is configured then "
"smtp.password must be configured as well.")
smtp_connection.login(smtp_user, smtp_password)
smtp_connection.sendmail(mail_from, [recipient_email], msg.as_string())
log.info("Sent email to {0}".format(recipient_email))
except smtplib.SMTPException, e:
msg = '%r' % e
log.exception(msg)
raise MailerException(msg)
def upgrade(migrate_engine):
datastore_connection_url = config.get(
'ckan.datastore.read_url', config.get('ckan.datastore.write_url'))
if not datastore_connection_url:
return
try:
datastore_engine = create_engine(datastore_connection_url)
except SQLAlchemyError:
return
try:
datastore_connection = datastore_engine.connect()
except SQLAlchemyError:
datastore_engine.dispose()
return
try:
resources_in_datastore = datastore_connection.execute('''
SELECT table_name
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name != '_table_metadata'
''')
if resources_in_datastore.rowcount:
resources = migrate_engine.execute('''
SELECT id, extras
FROM resource
WHERE id IN ({0}) AND extras IS NOT NULL
'''.format(
','.join(['\'{0}\''.format(_id[0])
for _id
in resources_in_datastore])
)
)
if resources.rowcount:
params = []
for resource in resources:
new_extras = json.loads(resource[1])
new_extras.update({'datastore_active': True})
params.append(
{'id': resource[0],
'extras': json.dumps(new_extras)})
migrate_engine.execute(
text('''
UPDATE resource
SET extras = :extras
WHERE id = :id'''),
params)
finally:
datastore_connection.close()
datastore_engine.dispose()
def get_reset_link_body(user):
extra_vars = {
'reset_link': get_reset_link(user),
'site_title': config.get('ckan.site_title'),
'site_url': config.get('ckan.site_url'),
'user_name': user.name,
}
# NOTE: This template is translated
return render_jinja2('emails/reset_password.txt', extra_vars)
def get_all_resources_ids_in_datastore():
read_url = config.get('ckan.datastore.read_url')
write_url = config.get('ckan.datastore.write_url')
data_dict = {
'connection_url': read_url or write_url
}
resources_sql = sqlalchemy.text(u'''SELECT name FROM "_table_metadata"
WHERE alias_of IS NULL''')
query = _get_engine(data_dict).execute(resources_sql)
return [q[0] for q in query.fetchall()]
def delete_package(self, pkg_dict):
conn = make_connection()
query = "+%s:%s (+id:\"%s\" OR +name:\"%s\") +site_id:\"%s\"" % (TYPE_FIELD, PACKAGE_TYPE,
pkg_dict.get('id'), pkg_dict.get('id'),
config.get('ckan.site_id'))
try:
commit = asbool(config.get('ckan.search.solr_commit', 'true'))
conn.delete(q=query, commit=commit)
except Exception, e:
log.exception(e)
raise SearchIndexError(e)
def set_active_backend(cls, config):
"""Choose most suitable backend depending on configuration
:param config: configuration object
:rtype: ckan.common.CKANConfig
"""
schema = config.get(u'ckan.datastore.write_url').split(u':')[0]
read_schema = config.get(u'ckan.datastore.read_url').split(u':')[0]
assert read_schema == schema, u'Read and write engines are different'
cls._active_backend = cls._backends[schema]()
def get_helpers(self):
return {
'ozwillo_theme_get_last_datasets': lambda: logic.get_action('package_search')({}, {"rows": 8})['results'],
'ozwillo_theme_get_resource_number': ozwillo_theme_get_resource_number,
'ozwillo_theme_get_popular_datasets': lambda: logic.get_action('package_search')({}, {"rows": 4, 'sort': 'views_total desc'})['results'],
'ozwillo_theme_display_date': ozwillo_theme_display_date,
'ozwillo_theme_get_map': ozwillo_theme_get_map,
'ozwillo_theme_get_groups': lambda: logic.get_action('group_list')({}, {"all_fields": True}),
'ozwillo_theme_spatial_installed': lambda: config.get('ckanext.ozwillo_theme.spatial_installed', 'False'),
'ozwillo_theme_osmnames_key': lambda: config.get('ckanext.ozwillo_theme.osmnames_key', '')
}
def get_locales_from_config():
""" despite the name of this function it gets the locales defined by
the config AND also the locals available subject to the config. """
locales_offered = config.get("ckan.locales_offered", "").split()
filtered_out = config.get("ckan.locales_filtered_out", "").split()
locale_default = [config.get("ckan.locale_default", "en")]
locale_order = config.get("ckan.locale_order", "").split()
known_locales = get_locales()
all_locales = set(known_locales) | set(locales_offered) | set(locale_order) | set(locale_default)
all_locales -= set(filtered_out)
return all_locales
def get_locales_from_config():
''' despite the name of this function it gets the locales defined by
the config AND also the locals available subject to the config. '''
locales_offered = config.get('ckan.locales_offered', '').split()
filtered_out = config.get('ckan.locales_filtered_out', '').split()
locale_default = config.get('ckan.locale_default', 'en')
locale_order = config.get('ckan.locale_order', '').split()
known_locales = get_locales()
all_locales = (set(known_locales) |
set(locales_offered) |
set(locale_order) |
set(locale_default))
all_locales -= set(filtered_out)
return all_locales
def _is_legacy_mode(config):
'''
Decides if the DataStore should run on legacy mode
Returns True if `ckan.datastore.read_url` is not set in the provided
config object or CKAN is running on Postgres < 9.x
'''
write_url = config.get('ckan.datastore.write_url')
engine = db._get_engine({'connection_url': write_url})
connection = engine.connect()
return (not config.get('ckan.datastore.read_url') or
not db._pg_version_is_at_least(connection, '9.0'))
def check_recaptcha(request):
'''Check a user\'s recaptcha submission is valid, and raise CaptchaError
on failure.'''
recaptcha_private_key = config.get('ckan.recaptcha.privatekey', '')
if not recaptcha_private_key:
# Recaptcha not enabled
return
client_ip_address = request.environ.get('REMOTE_ADDR', 'Unknown IP Address')
recaptcha_version = config.get('ckan.recaptcha.version', '1')
if recaptcha_version is '1':
recaptcha_response_field = request.params.get('recaptcha_response_field', '')
recaptcha_server_name = 'http://api-verify.recaptcha.net/verify'
recaptcha_challenge_field = request.params.get('recaptcha_challenge_field')
# recaptcha_response_field will be unicode if there are foreign chars in
# the user input. So we need to encode it as utf8 before urlencoding or
# we get an exception (#1431).
params = urllib.urlencode(dict(privatekey=recaptcha_private_key,
remoteip=client_ip_address,
challenge=recaptcha_challenge_field,
response=recaptcha_response_field.encode('utf8')))
f = urllib2.urlopen(recaptcha_server_name, params)
data = f.read()
f.close()
if not data.lower().startswith('true'):
raise CaptchaError()
elif recaptcha_version is '2':
recaptcha_response_field = request.params.get('g-recaptcha-response', '')
recaptcha_server_name = 'https://www.google.com/recaptcha/api/siteverify'
# recaptcha_response_field will be unicode if there are foreign chars in
# the user input. So we need to encode it as utf8 before urlencoding or
# we get an exception (#1431).
params = urllib.urlencode(dict(secret=recaptcha_private_key,
remoteip=client_ip_address,
response=recaptcha_response_field.encode('utf8')))
f = urllib2.urlopen(recaptcha_server_name, params)
data = json.load(f)
f.close()
try:
if not data['success']:
raise CaptchaError()
except IndexError:
# Something weird with recaptcha response
raise CaptchaError()
def load_all():
'''
Load all plugins listed in the 'ckan.plugins' config directive.
'''
# Clear any loaded plugins
unload_all()
plugins = config.get('ckan.plugins', '').split() + find_system_plugins()
# Add the synchronous search plugin, unless already loaded or
# explicitly disabled
if 'synchronous_search' not in plugins and \
asbool(config.get('ckan.search.automatic_indexing', True)):
log.debug('Loading the synchronous search plugin')
plugins.append('synchronous_search')
load(*plugins)
def as_dict(self, ref_package_by='name', ref_group_by='name'):
_dict = domain_object.DomainObject.as_dict(self)
# Set 'license' in _dict to cater for old clients.
# Todo: Remove from Version 2?
_dict['license'] = self.license.title if self.license else _dict.get('license_id', '')
_dict['isopen'] = self.isopen()
tags = [tag.name for tag in self.get_tags()]
tags.sort() # so it is determinable
_dict['tags'] = tags
groups = [getattr(group, ref_group_by) for group in self.get_groups()]
groups.sort()
_dict['groups'] = groups
_dict['extras'] = {key: value for key, value in self.extras.items()}
_dict['ratings_average'] = self.get_average_rating()
_dict['ratings_count'] = len(self.ratings)
_dict['resources'] = [res.as_dict(core_columns_only=False) \
for res in self.resources]
site_url = config.get('ckan.site_url', None)
if site_url:
_dict['ckan_url'] = '%s/dataset/%s' % (site_url, self.name)
_dict['relationships'] = [rel.as_dict(self, ref_package_by=ref_package_by) for rel in self.get_relationships()]
_dict['metadata_modified'] = self.metadata_modified.isoformat() \
if self.metadata_modified else None
_dict['metadata_created'] = self.metadata_created.isoformat() \
if self.metadata_created else None
import ckan.lib.helpers as h
_dict['notes_rendered'] = h.render_markdown(self.notes)
_dict['type'] = self.type or u'dataset'
return _dict
def render_snippet(*template_names, **kw):
''' Helper function for rendering snippets. Rendered html has
comment tags added to show the template used. NOTE: unlike other
render functions this takes a list of keywords instead of a dict for
the extra template variables.
:param template_names: the template to render, optionally with fallback
values, for when the template can't be found. For each, specify the
relative path to the template inside the registered tpl_dir.
:type template_names: str
:param kw: extra template variables to supply to the template
:type kw: named arguments of any type that are supported by the template
'''
exc = None
for template_name in template_names:
try:
output = render(template_name, extra_vars=kw)
if config.get('debug'):
output = (
'\n<!-- Snippet %s start -->\n%s\n<!-- Snippet %s end -->'
'\n' % (template_name, output, template_name))
return literal(output)
except TemplateNotFound as exc:
if exc.name == template_name:
# the specified template doesn't exist - try the next fallback
continue
# a nested template doesn't exist - don't fallback
raise exc
else:
raise exc or TemplateNotFound
def check_config_permission(permission):
'''Returns the configuration value for the provided permission
Permission is a string indentifying the auth permission (eg
`anon_create_dataset`), optionally prefixed with `ckan.auth.`.
The possible values for `permission` are the keys of
CONFIG_PERMISSIONS_DEFAULTS. These can be overriden in the config file
by prefixing them with `ckan.auth.`.
Returns the permission value, generally True or False, except on
`roles_that_cascade_to_sub_groups` which is a list of strings.
'''
key = permission.replace('ckan.auth.', '')
if key not in CONFIG_PERMISSIONS_DEFAULTS:
return False
default_value = CONFIG_PERMISSIONS_DEFAULTS.get(key)
config_key = 'ckan.auth.' + key
value = config.get(config_key, default_value)
if key == 'roles_that_cascade_to_sub_groups':
# This permission is set as a list of strings (space separated)
value = value.split() if value else []
else:
value = asbool(value)
return value
def get_config_value(key, default=''):
if model.meta.engine.has_table('system_info'):
value = model.get_system_info(key)
else:
value = None
config_value = config.get(key)
# sort encodeings if needed
if isinstance(config_value, str):
try:
config_value = config_value.decode('utf-8')
except UnicodeDecodeError:
config_value = config_value.decode('latin-1')
# we want to store the config the first time we get here so we can
# reset them if needed
if key not in _CONFIG_CACHE:
_CONFIG_CACHE[key] = config_value
if value is not None:
log.debug('config `%s` set to `%s` from db' % (key, value))
else:
value = _CONFIG_CACHE[key]
if value:
log.debug('config `%s` set to `%s` from config' % (key, value))
else:
value = default
set_app_global(key, value)
# update the config
config[key] = value
return value
def setup_class(cls):
smtp_server = config.get('smtp.test_server')
if smtp_server:
host, port = smtp_server.split(':')
port = int(port) + int(str(hashlib.md5(cls.__name__).hexdigest())[0], 16)
config['smtp.test_server'] = '%s:%s' % (host, port)
SmtpServerHarness.setup_class()
def _build_fts_indexes(connection, data_dict, sql_index_str_method, fields):
fts_indexes = []
resource_id = data_dict['resource_id']
# FIXME: This is repeated on the plugin.py, we should keep it DRY
default_fts_lang = config.get('ckan.datastore.default_fts_lang')
if default_fts_lang is None:
default_fts_lang = u'english'
fts_lang = data_dict.get('lang', default_fts_lang)
# create full-text search indexes
def to_tsvector(x):
return u"to_tsvector('{0}', {1})".format(fts_lang, x)
def cast_as_text(x):
return u'cast("{0}" AS text)'.format(x)
full_text_field = {'type': 'tsvector', 'id': '_full_text'}
for field in [full_text_field] + fields:
if not datastore_helpers.should_fts_index_field_type(field['type']):
continue
field_str = field['id']
if field['type'] not in ['text', 'tsvector']:
field_str = cast_as_text(field_str)
else:
field_str = u'"{0}"'.format(field_str)
if field['type'] != 'tsvector':
field_str = to_tsvector(field_str)
fts_indexes.append(sql_index_str_method.format(
res_id=resource_id,
unique='',
name=_generate_index_name(resource_id, field_str),
method=_get_fts_index_method(), fields=field_str))
return fts_indexes
def edit(self, id=None, data=None, errors=None, error_summary=None):
context = {'save': 'save' in request.params,
'schema': self._edit_form_to_db_schema(),
'model': model, 'session': model.Session,
'user': c.user, 'auth_user_obj': c.userobj
}
if id is None:
if c.userobj:
id = c.userobj.id
else:
abort(400, _('No user specified'))
data_dict = {'id': id}
try:
check_access('user_update', context, data_dict)
except NotAuthorized:
abort(403, _('Unauthorized to edit a user.'))
if (context['save']) and not data:
return self._save_edit(id, context)
try:
old_data = get_action('user_show')(context, data_dict)
schema = self._db_to_edit_form_schema()
if schema:
old_data, errors = \
dictization_functions.validate(old_data, schema, context)
c.display_name = old_data.get('display_name')
c.user_name = old_data.get('name')
data = data or old_data
except NotAuthorized:
abort(403, _('Unauthorized to edit user %s') % '')
except NotFound:
abort(404, _('User not found'))
user_obj = context.get('user_obj')
if not (authz.is_sysadmin(c.user)
or c.user == user_obj.name):
abort(403, _('User %s not authorized to edit %s') %
(str(c.user), id))
errors = errors or {}
vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
self._setup_template_variables({'model': model,
'session': model.Session,
'user': c.user},
data_dict)
c.is_myself = True
c.show_email_notifications = asbool(
config.get('ckan.activity_streams_email_notifications'))
c.form = render(self.edit_user_form, extra_vars=vars)
return render('user/edit.html')
def general(self):
data_dict, params = self._parse_url_params()
data_dict['q'] = '*:*'
item_count, results = _package_search(data_dict)
navigation_urls = self._navigation_urls(params,
item_count=item_count,
limit=data_dict['rows'],
controller='feed',
action='general')
feed_url = self._feed_url(params,
controller='feed',
action='general')
alternate_url = self._alternate_url(params)
site_title = config.get('ckan.site_title', 'CKAN')
return self.output_feed(results,
feed_title=site_title,
feed_description=u'Recently created or '
'updated datasets on %s' % site_title,
feed_link=alternate_url,
feed_guid=_create_atom_id
(u'/feeds/dataset.atom'),
feed_url=feed_url,
navigation_urls=navigation_urls)
def before_view(self, dataset_dict):
# Translate any selected search facets (e.g. if we are rendering a
# group read page or the dataset index page): lookup translations of
# all the terms in c.fields (c.fields contains the selected facets)
# and save them in c.translated_fields where the templates can
# retrieve them later.
desired_lang_code = request.environ['CKAN_LANG']
fallback_lang_code = config.get('ckan.locale_default', 'en')
try:
fields = c.fields
except AttributeError:
return translate_data_dict(dataset_dict)
terms = [value for param, value in fields]
translations = get_action('term_translation_show')(
{'model': ckan.model},
{'terms': terms,
'lang_codes': (desired_lang_code, fallback_lang_code)})
c.translated_fields = {}
for param, value in fields:
matching_translations = [translation for translation in
translations if translation['term'] == value and
translation['lang_code'] == desired_lang_code]
if not matching_translations:
matching_translations = [translation for translation in
translations if translation['term'] == value and
translation['lang_code'] == fallback_lang_code]
if matching_translations:
assert len(matching_translations) == 1
translation = matching_translations[0]['term_translation']
c.translated_fields[(param, value)] = translation
# Now translate the fields of the dataset itself.
return translate_data_dict(dataset_dict)
def render_template():
globs = extra_vars or {}
globs.update(pylons_globals())
# Using pylons.url() directly destroys the localisation stuff so
# we remove it so any bad templates crash and burn
del globs['url']
try:
template_path, template_type = render_.template_info(template_name)
except render_.TemplateNotFound:
raise
log.debug('rendering %s [%s]' % (template_path, template_type))
if config.get('debug'):
context_vars = globs.get('c')
if context_vars:
context_vars = dir(context_vars)
debug_info = {'template_name': template_name,
'template_path': template_path,
'template_type': template_type,
'vars': globs,
'c_vars': context_vars,
'renderer': renderer}
if 'CKAN_DEBUG_INFO' not in request.environ:
request.environ['CKAN_DEBUG_INFO'] = []
request.environ['CKAN_DEBUG_INFO'].append(debug_info)
del globs['config']
return render_jinja2(template_name, globs)
def index(self):
page = h.get_page_number(request.params)
c.q = request.params.get('q', '')
c.order_by = request.params.get('order_by', 'name')
context = {'return_query': True, 'user': c.user,
'auth_user_obj': c.userobj}
data_dict = {'q': c.q,
'order_by': c.order_by}
limit = int(
request.params.get('limit', config.get('ckan.user_list_limit', 20))
)
try:
check_access('user_list', context, data_dict)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
users_list = get_action('user_list')(context, data_dict)
c.page = h.Page(
collection=users_list,
page=page,
url=h.pager_url,
item_count=users_list.count(),
items_per_page=limit
)
return render('user/list.html')
def tag(self, id):
data_dict, params = self._parse_url_params()
data_dict['fq'] = 'tags:"%s"' % id
item_count, results = _package_search(data_dict)
navigation_urls = self._navigation_urls(params,
item_count=item_count,
limit=data_dict['rows'],
controller='feed',
action='tag',
id=id)
feed_url = self._feed_url(params,
controller='feed',
action='tag',
id=id)
alternate_url = self._alternate_url(params, tags=id)
site_title = config.get('ckan.site_title', 'CKAN')
return self.output_feed(results,
feed_title=u'%s - Tag: "%s"' %
(site_title, id),
feed_description=u'Recently created or '
'updated datasets on %s by tag: "%s"' %
(site_title, id),
feed_link=alternate_url,
feed_guid=_create_atom_id
(u'/feeds/tag/%s.atom' % id),
feed_url=feed_url,
navigation_urls=navigation_urls)
def get_i18n_path():
return config.get(u'ckan.i18n_directory', os.path.join(ckan_path, u'i18n'))
def get_jira_script():
jira_script = config.get('ckanext.nextgeoss.jira_issue_tracker')
return jira_script
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
import datetime
from sqlalchemy import Table, select, join, func, and_
from ckanext.stats.stats import Stats as CoreStats, RevisionStats as CoreRevisionStats, table, datetime2date, DATE_FORMAT
import ckan.model as model
import ckan.plugins as p
from ckan.common import config
cache_enabled = p.toolkit.asbool(config.get('ckanext.stats.cache_enabled', 'True'))
if cache_enabled:
from pylons import cache
from ckanext.stats.stats import our_cache
class Stats(CoreStats):
pass
class RevisionStats(CoreRevisionStats):
@classmethod
def get_num_packages_by_week(cls):
"""
overriden
"""
def _create_atom_id(resource_path, authority_name=None, date_string=None):
"""
Helper method that creates an atom id for a feed or entry.
An id must be unique, and must not change over time. ie - once published,
it represents an atom feed or entry uniquely, and forever. See [4]:
When an Atom Document is relocated, migrated, syndicated,
republished, exported, or imported, the content of its atom:id
element MUST NOT change. Put another way, an atom:id element
pertains to all instantiations of a particular Atom entry or feed;
revisions retain the same content in their atom:id elements. It is
suggested that the atom:id element be stored along with the
associated resource.
resource_path
The resource path that uniquely identifies the feed or element. This
mustn't be something that changes over time for a given entry or feed.
And does not necessarily need to be resolvable.
e.g. ``"/group/933f3857-79fd-4beb-a835-c0349e31ce76"`` could represent
the feed of datasets belonging to the identified group.
authority_name
The domain name or email address of the publisher of the feed. See [3]
for more details. If ``None`` then the domain name is taken from the
config file. First trying ``ckan.feeds.authority_name``, and failing
that, it uses ``ckan.site_url``. Again, this should not change over
time.
date_string
A string representing a date on which the authority_name is owned by
the publisher of the feed.
e.g. ``"2012-03-22"``
Again, this should not change over time.
If date_string is None, then an attempt is made to read the config
option ``ckan.feeds.date``. If that's not available,
then the date_string is not used in the generation of the atom id.
Following the methods outlined in [1], [2] and [3], this function produces
tagURIs like:
``"tag:thedatahub.org,2012:/group/933f3857-79fd-4beb-a835-c0349e31ce76"``.
If not enough information is provide to produce a valid tagURI, then only
the resource_path is used, e.g.: ::
"http://thedatahub.org/group/933f3857-79fd-4beb-a835-c0349e31ce76"
or
"/group/933f3857-79fd-4beb-a835-c0349e31ce76"
The latter of which is only used if no site_url is available. And it
should be noted will result in an invalid feed.
[1] http://web.archive.org/web/20110514113830/http://diveintomark.org/\
archives/2004/05/28/howto-atom-id
[2] http://www.taguri.org/
[3] http://tools.ietf.org/html/rfc4151#section-2.1
[4] http://www.ietf.org/rfc/rfc4287
"""
if authority_name is None:
authority_name = config.get('ckan.feeds.authority_name', '').strip()
if not authority_name:
site_url = config.get('ckan.site_url', '').strip()
authority_name = urlparse.urlparse(site_url).netloc
if not authority_name:
log.warning('No authority_name available for feed generation. '
'Generated feed will be invalid.')
if date_string is None:
date_string = config.get('ckan.feeds.date', '')
if not date_string:
log.warning('No date_string available for feed generation. '
'Please set the "ckan.feeds.date" config value.')
# Don't generate a tagURI without a date as it wouldn't be valid.
# This is best we can do, and if the site_url is not set, then
# this still results in an invalid feed.
site_url = config.get('ckan.site_url', '')
return '/'.join([site_url, resource_path])
tagging_entity = ','.join([authority_name, date_string])
return ':'.join(['tag', tagging_entity, resource_path])
def get_featured_image_url(default):
return config.get('ckanext.lacounts.featured_image') or default
def sysadmin_email():
return text_type(config.get('smtp.mail_from', ''))
def get_max_image_size():
global _max_image_size
if _max_image_size is None:
_max_image_size = int(config.get('ckan.max_image_size', 2))
return _max_image_size
def get_max_resource_size():
global _max_resource_size
if _max_resource_size is None:
_max_resource_size = int(config.get('ckan.max_resource_size', 10))
return _max_resource_size
def upgrade(migrate_engine):
schema = ckan_config.get(u'ckan.migrations.target_schema') or 'public'
# we specify the schema here because of a clash with another 'state' table
# in the mdillon/postgis container. You only need to change the value in the
# config if you've altered the default schema from 'public' in your
# postgresql.conf. Because this is such a rarely needed option, it is
# otherwise undocumented.
meta = MetaData(schema=schema)
state = Table(
'state',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('name', Unicode(100)),
)
revision = Table(
'revision',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('timestamp', DateTime(timezone=False)),
Column('author', Unicode(200)),
Column('message', UnicodeText()),
Column('state_id', Integer()),
)
apikey = Table(
'apikey',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('name', UnicodeText()),
Column('key', UnicodeText()),
)
license = Table(
'license',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('name', Unicode(100)),
# Column('state_id', Integer(), ForeignKey('state.id')),
Column('state_id', Integer()))
package = Table(
'package',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('name', Unicode(100), nullable=False, unique=True),
Column('title', UnicodeText()),
Column('version', Unicode(100)),
Column('url', UnicodeText()),
Column('download_url', UnicodeText()),
Column('notes', UnicodeText()),
Column('license_id', Integer(), ForeignKey('license.id')),
Column('state_id', Integer(), ForeignKey('state.id')),
Column('revision_id', Integer(), ForeignKey('revision.id')),
)
package_revision = Table(
'package_revision',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('name', Unicode(100), nullable=False),
Column('title', UnicodeText()),
Column('version', Unicode(100)),
Column('url', UnicodeText()),
Column('download_url', UnicodeText()),
Column('notes', UnicodeText()),
Column('license_id', Integer(), ForeignKey('license.id')),
Column('state_id', Integer(), ForeignKey('state.id')),
Column('revision_id',
Integer(),
ForeignKey('revision.id'),
primary_key=True),
Column('continuity_id', Integer(), ForeignKey('package.id')),
)
tag = Table(
'tag',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('name', Unicode(100), nullable=False, unique=True),
)
package_tag = Table(
'package_tag',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('package_id', Integer(), ForeignKey('package.id')),
Column('tag_id', Integer(), ForeignKey('tag.id')),
Column('state_id', Integer(), ForeignKey('state.id')),
Column('revision_id', Integer(), ForeignKey('revision.id')),
)
package_tag_revision = Table(
'package_tag_revision',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('package_id', Integer(), ForeignKey('package.id')),
Column('tag_id', Integer(), ForeignKey('tag.id')),
Column('state_id', Integer(), ForeignKey('state.id')),
Column('revision_id',
Integer(),
ForeignKey('revision.id'),
primary_key=True),
Column('continuity_id', Integer(), ForeignKey('package_tag.id')),
)
package_extra = Table(
'package_extra',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('package_id', Integer(), ForeignKey('package.id')),
Column('key', UnicodeText()),
Column('value', UnicodeText()),
Column('state_id', Integer(), ForeignKey('state.id')),
Column('revision_id', Integer(), ForeignKey('revision.id')),
)
package_extra_revision = Table(
'package_extra_revision',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('package_id', Integer(), ForeignKey('package.id')),
Column('key', UnicodeText()),
Column('value', UnicodeText()),
Column('state_id', Integer(), ForeignKey('state.id')),
Column('revision_id',
Integer(),
ForeignKey('revision.id'),
primary_key=True),
Column('continuity_id', Integer(), ForeignKey('package_extra.id')),
)
meta.bind = migrate_engine
meta.create_all()
def make_map():
"""Create, configure and return the routes Mapper"""
# import controllers here rather than at root level because
# pylons config is initialised by this point.
# Helpers to reduce code clutter
GET = dict(method=['GET'])
PUT = dict(method=['PUT'])
POST = dict(method=['POST'])
DELETE = dict(method=['DELETE'])
GET_POST = dict(method=['GET', 'POST'])
PUT_POST = dict(method=['PUT', 'POST'])
PUT_POST_DELETE = dict(method=['PUT', 'POST', 'DELETE'])
OPTIONS = dict(method=['OPTIONS'])
import ckan.lib.plugins as lib_plugins
lib_plugins.reset_package_plugins()
map = Mapper(directory=config['pylons.paths']['controllers'],
always_scan=config['debug'])
map.minimization = False
map.explicit = True
# The ErrorController route (handles 404/500 error pages); it should
# likely stay at the top, ensuring it can always be resolved.
map.connect('/error/{action}', controller='error', ckan_core=True)
map.connect('/error/{action}/{id}', controller='error', ckan_core=True)
map.connect('*url',
controller='home',
action='cors_options',
conditions=OPTIONS,
ckan_core=True)
# CUSTOM ROUTES HERE
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.before_map(map)
# Mark all routes added from extensions on the `before_map` extension point
# as non-core
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = False
map.connect('invite',
'/__invite__/',
controller='partyline',
action='join_party')
map.connect('home', '/', controller='home', action='index')
map.connect('about', '/about', controller='home', action='about')
map.connect('visual', '/vis', controller='home', action='visual')
map.connect('suggest', '/suggest', controller='home', action='suggest')
# map.connect('user_database','/user/userdatabase/{id:.*}',controller = 'user', action = 'dbsearch')
# CKAN API versioned.
register_list = [
'package', 'dataset', 'resource', 'tag', 'group', 'revision',
'licenses', 'rating', 'user', 'activity'
]
register_list_str = '|'.join(register_list)
# /api ver 3 or none
with SubMapper(map,
controller='api',
path_prefix='/api{ver:/3|}',
ver='/3') as m:
m.connect('/action/{logic_function}',
action='action',
conditions=GET_POST)
# /api ver 1, 2, 3 or none
with SubMapper(map,
controller='api',
path_prefix='/api{ver:/1|/2|/3|}',
ver='/1') as m:
m.connect('', action='get_api')
m.connect('/search/{register}', action='search')
# /api ver 1, 2 or none
with SubMapper(map,
controller='api',
path_prefix='/api{ver:/1|/2|}',
ver='/1') as m:
m.connect('/tag_counts', action='tag_counts')
m.connect('/rest', action='index')
m.connect('/qos/throughput/', action='throughput', conditions=GET)
# /api/rest ver 1, 2 or none
with SubMapper(map,
controller='api',
path_prefix='/api{ver:/1|/2|}',
ver='/1',
requirements=dict(register=register_list_str)) as m:
m.connect('/rest/{register}', action='list', conditions=GET)
m.connect('/rest/{register}', action='create', conditions=POST)
m.connect('/rest/{register}/{id}', action='show', conditions=GET)
m.connect('/rest/{register}/{id}', action='update', conditions=PUT)
m.connect('/rest/{register}/{id}', action='update', conditions=POST)
m.connect('/rest/{register}/{id}', action='delete', conditions=DELETE)
m.connect('/rest/{register}/{id}/:subregister',
action='list',
conditions=GET)
m.connect('/rest/{register}/{id}/:subregister',
action='create',
conditions=POST)
m.connect('/rest/{register}/{id}/:subregister/{id2}',
action='create',
conditions=POST)
m.connect('/rest/{register}/{id}/:subregister/{id2}',
action='show',
conditions=GET)
m.connect('/rest/{register}/{id}/:subregister/{id2}',
action='update',
conditions=PUT)
m.connect('/rest/{register}/{id}/:subregister/{id2}',
action='delete',
conditions=DELETE)
# /api/util ver 1, 2 or none
with SubMapper(map,
controller='api',
path_prefix='/api{ver:/1|/2|}',
ver='/1') as m:
m.connect('/util/user/autocomplete', action='user_autocomplete')
m.connect('/util/is_slug_valid',
action='is_slug_valid',
conditions=GET)
m.connect('/util/dataset/autocomplete',
action='dataset_autocomplete',
conditions=GET)
m.connect('/util/tag/autocomplete',
action='tag_autocomplete',
conditions=GET)
m.connect('/util/resource/format_autocomplete',
action='format_autocomplete',
conditions=GET)
m.connect('/util/resource/format_icon',
action='format_icon',
conditions=GET)
m.connect('/util/group/autocomplete', action='group_autocomplete')
m.connect('/util/organization/autocomplete',
action='organization_autocomplete',
conditions=GET)
m.connect('/util/markdown', action='markdown')
m.connect('/util/dataset/munge_name', action='munge_package_name')
m.connect('/util/dataset/munge_title_to_name',
action='munge_title_to_package_name')
m.connect('/util/tag/munge', action='munge_tag')
m.connect('/util/status', action='status')
m.connect('/util/snippet/{snippet_path:.*}', action='snippet')
m.connect('/i18n/{lang}', action='i18n_js_translations')
###########
## /END API
###########
map.redirect('/packages', '/dataset')
map.redirect('/packages/{url:.*}', '/dataset/{url}')
map.redirect('/package', '/dataset')
map.redirect('/package/{url:.*}', '/dataset/{url}')
with SubMapper(map, controller='package') as m:
m.connect('search',
'/dataset',
action='search',
highlight_actions='index search')
m.connect('add dataset', '/dataset/new', action='new')
m.connect('/dataset/{action}',
requirements=dict(
action='|'.join(['list', 'autocomplete', 'search'])))
m.connect('/dataset/{action}/{id}/{revision}',
action='read_ajax',
requirements=dict(action='|'.join([
'read',
'edit',
'history',
])))
m.connect('/dataset/{action}/{id}',
requirements=dict(action='|'.join([
'new_resource', 'history', 'read_ajax', 'history_ajax',
'follow', 'activity', 'groups', 'unfollow', 'delete',
'api_data'
])))
m.connect(
'/dataset/add_button/{package_name}/{id}/{pack_r_id}/{user_name}',
action='add_button')
m.connect('dataset_edit',
'/dataset/edit/{id}',
action='edit',
ckan_icon='edit')
m.connect('dataset_followers',
'/dataset/followers/{id}',
action='followers',
ckan_icon='group')
m.connect('dataset_activity',
'/dataset/activity/{id}',
action='activity',
ckan_icon='time')
m.connect('/dataset/activity/{id}/{offset}', action='activity')
m.connect('dataset_groups',
'/dataset/groups/{id}',
action='groups',
ckan_icon='group')
m.connect('dataset_resources',
'/dataset/resources/{id}',
action='resources',
ckan_icon='reorder')
m.connect('dataset_read',
'/dataset/{id}',
action='read',
ckan_icon='sitemap')
m.connect('/dataset/{id}/resource/{resource_id}',
action='resource_read')
m.connect('/dataset/{id}/resource_delete/{resource_id}',
action='resource_delete')
m.connect('resource_edit',
'/dataset/{id}/resource_edit/{resource_id}',
action='resource_edit',
ckan_icon='edit')
m.connect('/dataset/{id}/resource/{resource_id}/download',
action='resource_download')
m.connect('/dataset/{id}/resource/{resource_id}/download/{filename}',
action='resource_download')
m.connect('/dataset/{id}/resource/{resource_id}/embed',
action='resource_embedded_dataviewer')
m.connect('/dataset/{id}/resource/{resource_id}/viewer',
action='resource_embedded_dataviewer',
width="960",
height="800")
m.connect('/dataset/{id}/resource/{resource_id}/preview',
action='resource_datapreview')
m.connect('views',
'/dataset/{id}/resource/{resource_id}/views',
action='resource_views',
ckan_icon='reorder')
m.connect('new_view',
'/dataset/{id}/resource/{resource_id}/new_view',
action='edit_view',
ckan_icon='edit')
m.connect('edit_view',
'/dataset/{id}/resource/{resource_id}/edit_view/{view_id}',
action='edit_view',
ckan_icon='edit')
m.connect('resource_view',
'/dataset/{id}/resource/{resource_id}/view/{view_id}',
action='resource_view')
m.connect('/dataset/{id}/resource/{resource_id}/view/',
action='resource_view')
# group
map.redirect('/groups', '/group')
map.redirect('/groups/{url:.*}', '/group/{url}')
# These named routes are used for custom group forms which will use the
# names below based on the group.type ('group' is the default type)
with SubMapper(map, controller='group') as m:
m.connect('group_index',
'/group',
action='index',
highlight_actions='index search')
m.connect('group_list', '/group/list', action='list')
m.connect('group_new', '/group/new', action='new')
m.connect('group_action',
'/group/{action}/{id}',
requirements=dict(action='|'.join([
'edit',
'delete',
'member_new',
'member_delete',
'history',
'followers',
'follow',
'unfollow',
'admins',
'activity',
])))
m.connect('group_about',
'/group/about/{id}',
action='about',
ckan_icon='info-sign'),
m.connect('group_edit',
'/group/edit/{id}',
action='edit',
ckan_icon='edit')
m.connect('group_members',
'/group/members/{id}',
action='members',
ckan_icon='group'),
m.connect('group_activity',
'/group/activity/{id}/{offset}',
action='activity',
ckan_icon='time'),
m.connect('group_read',
'/group/{id}',
action='read',
ckan_icon='sitemap')
# organizations these basically end up being the same as groups
with SubMapper(map, controller='organization') as m:
m.connect('organizations_index', '/organization', action='index')
m.connect('/organization/list', action='list')
m.connect('/organization/new', action='new')
m.connect(
'/organization/{action}/{id}',
requirements=dict(action='|'.join([
'delete', 'admins', 'member_new', 'member_delete', 'history'
])))
m.connect('organization_activity',
'/organization/activity/{id}/{offset}',
action='activity',
ckan_icon='time')
m.connect('organization_read', '/organization/{id}', action='read')
m.connect('organization_about',
'/organization/about/{id}',
action='about',
ckan_icon='info-sign')
m.connect('organization_read',
'/organization/{id}',
action='read',
ckan_icon='sitemap')
m.connect('organization_edit',
'/organization/edit/{id}',
action='edit',
ckan_icon='edit')
m.connect('organization_members',
'/organization/members/{id}',
action='members',
ckan_icon='group')
m.connect('organization_bulk_process',
'/organization/bulk_process/{id}',
action='bulk_process',
ckan_icon='sitemap')
lib_plugins.register_package_plugins(map)
lib_plugins.register_group_plugins(map)
# tags
map.redirect('/tags', '/tag')
map.redirect('/tags/{url:.*}', '/tag/{url}')
map.redirect('/tag/read/{url:.*}',
'/tag/{url}',
_redirect_code='301 Moved Permanently')
map.connect('/tag', controller='tag', action='index')
map.connect('/tag/{id}', controller='tag', action='read')
# users
map.redirect('/users/{url:.*}', '/user/{url}')
map.redirect('/user/', '/user')
with SubMapper(map, controller='user') as m:
m.connect('/user/edit', action='edit')
# Note: openid users have slashes in their ids, so need the wildcard
# in the route.
m.connect('user_database',
'/user/userdatabase/{id:.*}',
action='dbsearch')
m.connect('user_generate_apikey',
'/user/generate_key/{id}',
action='generate_apikey')
m.connect('/user/activity/{id}/{offset}', action='activity')
m.connect('user_activity_stream',
'/user/activity/{id}',
action='activity',
ckan_icon='time')
m.connect('user_dashboard',
'/dashboard',
action='dashboard',
ckan_icon='list')
m.connect('user_dashboard_datasets',
'/dashboard/datasets',
action='dashboard_datasets',
ckan_icon='sitemap')
m.connect('user_dashboard_groups',
'/dashboard/groups',
action='dashboard_groups',
ckan_icon='group')
m.connect('user_dashboard_organizations',
'/dashboard/organizations',
action='dashboard_organizations',
ckan_icon='building')
m.connect('/dashboard/{offset}', action='dashboard')
m.connect('user_follow', '/user/follow/{id}', action='follow')
m.connect('/user/unfollow/{id}', action='unfollow')
m.connect('user_followers',
'/user/followers/{id:.*}',
action='followers',
ckan_icon='group')
m.connect('user_edit',
'/user/edit/{id:.*}',
action='edit',
ckan_icon='cog')
m.connect('user_delete', '/user/delete/{id}', action='delete')
m.connect('/user/reset/{id:.*}', action='perform_reset')
m.connect('register', '/user/register', action='register')
m.connect('login', '/user/login', action='login')
m.connect('/user/_logout', action='logout')
m.connect('/user/logged_in', action='logged_in')
m.connect('/user/logged_out', action='logged_out')
m.connect('/user/logged_out_redirect', action='logged_out_page')
m.connect('/user/reset', action='request_reset')
m.connect('/user/me', action='me')
m.connect('/user/set_lang/{lang}', action='set_lang')
m.connect('user_datasets',
'/user/{id:.*}',
action='read',
ckan_icon='sitemap')
m.connect('user_index', '/user', action='index')
m.connect('user_ffdatabase',
'/user111/userdatabase/{id:.*}',
action='dbsearch')
with SubMapper(map, controller='revision') as m:
m.connect('/revision', action='index')
m.connect('/revision/edit/{id}', action='edit')
m.connect('/revision/diff/{id}', action='diff')
m.connect('/revision/list', action='list')
m.connect('/revision/{id}', action='read')
# feeds
with SubMapper(map, controller='feed') as m:
m.connect('/feeds/group/{id}.atom', action='group')
m.connect('/feeds/organization/{id}.atom', action='organization')
m.connect('/feeds/tag/{id}.atom', action='tag')
m.connect('/feeds/dataset.atom', action='general')
m.connect('/feeds/custom.atom', action='custom')
map.connect('ckanadmin_index',
'/ckan-admin',
controller='admin',
action='index',
ckan_icon='legal')
map.connect('ckanadmin_config',
'/ckan-admin/config',
controller='admin',
action='config',
ckan_icon='check')
map.connect('ckanadmin_trash',
'/ckan-admin/trash',
controller='admin',
action='trash',
ckan_icon='trash')
map.connect('ckanadmin', '/ckan-admin/{action}', controller='admin')
with SubMapper(
map, controller='ckan.controllers.storage:StorageController') as m:
m.connect('storage_file', '/storage/f/{label:.*}', action='file')
with SubMapper(map, controller='util') as m:
m.connect('/i18n/strings_{lang}.js', action='i18n_js_strings')
m.connect('/util/redirect', action='redirect')
m.connect('/testing/primer', action='primer')
m.connect('/testing/markup', action='markup')
# Mark all unmarked routes added up until now as core routes
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = True
for plugin in p.PluginImplementations(p.IRoutes):
map = plugin.after_map(map)
# Mark all routes added from extensions on the `after_map` extension point
# as non-core
for route in map.matchlist:
if not hasattr(route, '_ckan_core'):
route._ckan_core = False
# sometimes we get requests for favicon.ico we should redirect to
# the real favicon location.
map.redirect('/favicon.ico', config.get('ckan.favicon'))
map.redirect('/*(url)/', '/{url}', _redirect_code='301 Moved Permanently')
map.connect('/*url', controller='template', action='view', ckan_core=True)
return map
def make_flask_stack(conf):
""" This has to pass the flask app through all the same middleware that
Pylons used """
root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
debug = asbool(conf.get('debug', conf.get('DEBUG', False)))
testing = asbool(conf.get('testing', conf.get('TESTING', False)))
app = flask_app = CKANFlask(__name__, static_url_path='')
# Register storage for accessing group images, site logo, etc.
storage_folder = []
storage = uploader.get_storage_path()
if storage:
storage_folder = [os.path.join(storage, 'storage')]
# Static files folders (core and extensions)
public_folder = config.get(u'ckan.base_public_folder')
app.static_folder = config.get('extra_public_paths', '').split(',') + [
os.path.join(root, public_folder)
] + storage_folder
app.jinja_options = jinja_extensions.get_jinja_env_options()
app.jinja_env.policies['ext.i18n.trimmed'] = True
app.debug = debug
app.testing = testing
app.template_folder = os.path.join(root, 'templates')
app.app_ctx_globals_class = CKAN_AppCtxGlobals
app.url_rule_class = CKAN_Rule
# Update Flask config with the CKAN values. We use the common config
# object as values might have been modified on `load_environment`
if config:
app.config.update(config)
else:
app.config.update(conf)
# Do all the Flask-specific stuff before adding other middlewares
# Secret key needed for flask-debug-toolbar and sessions
if not app.config.get('SECRET_KEY'):
app.config['SECRET_KEY'] = config.get('beaker.session.secret')
if not app.config.get('SECRET_KEY'):
raise RuntimeError(u'You must provide a value for the secret key'
' with the SECRET_KEY config option')
root_path = config.get('ckan.root_path', None)
if debug:
from flask_debugtoolbar import DebugToolbarExtension
app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
debug_ext = DebugToolbarExtension()
# register path that includes `ckan.site_root` before
# initializing debug app. In such a way, our route receives
# higher precedence.
# TODO: After removal of Pylons code, switch to
# `APPLICATION_ROOT` config value for flask application. Right
# now it's a bad option because we are handling both pylons
# and flask urls inside helpers and splitting this logic will
# bring us tons of headache.
if root_path:
app.add_url_rule(
root_path.replace('{{LANG}}', '').rstrip('/') +
'/_debug_toolbar/static/<path:filename>',
'_debug_toolbar.static', debug_ext.send_static_file)
debug_ext.init_app(app)
from werkzeug.debug import DebuggedApplication
app.wsgi_app = DebuggedApplication(app.wsgi_app, True)
# Use Beaker as the Flask session interface
class BeakerSessionInterface(SessionInterface):
def open_session(self, app, request):
if 'beaker.session' in request.environ:
return request.environ['beaker.session']
def save_session(self, app, session, response):
session.save()
namespace = 'beaker.session.'
session_opts = {
k.replace('beaker.', ''): v
for k, v in config.items() if k.startswith(namespace)
}
if (not session_opts.get('session.data_dir')
and session_opts.get('session.type', 'file') == 'file'):
cache_dir = conf.get('cache_dir') or conf.get('cache.dir')
session_opts['session.data_dir'] = '{data_dir}/sessions'.format(
data_dir=cache_dir)
app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts)
app.session_interface = BeakerSessionInterface()
# Add Jinja2 extensions and filters
app.jinja_env.filters['empty_and_escape'] = \
jinja_extensions.empty_and_escape
# Common handlers for all requests
app.before_request(ckan_before_request)
app.after_request(ckan_after_request)
# Template context processors
app.context_processor(helper_functions)
app.context_processor(c_object)
@app.context_processor
def ungettext_alias():
u'''
Provide `ungettext` as an alias of `ngettext` for backwards
compatibility
'''
return dict(ungettext=ungettext)
# Babel
_ckan_i18n_dir = i18n.get_ckan_i18n_dir()
pairs = [(_ckan_i18n_dir, u'ckan')
] + [(p.i18n_directory(), p.i18n_domain())
for p in reversed(list(PluginImplementations(ITranslation)))]
i18n_dirs, i18n_domains = zip(*pairs)
app.config[u'BABEL_TRANSLATION_DIRECTORIES'] = ';'.join(i18n_dirs)
app.config[u'BABEL_DOMAIN'] = 'ckan'
app.config[u'BABEL_MULTIPLE_DOMAINS'] = ';'.join(i18n_domains)
app.config[u'BABEL_DEFAULT_TIMEZONE'] = str(helpers.get_display_timezone())
babel = CKANBabel(app)
babel.localeselector(get_locale)
# WebAssets
_setup_webassets(app)
# Auto-register all blueprints defined in the `views` folder
_register_core_blueprints(app)
_register_error_handler(app)
# Set up each IBlueprint extension as a Flask Blueprint
for plugin in PluginImplementations(IBlueprint):
if hasattr(plugin, 'get_blueprint'):
plugin_blueprints = plugin.get_blueprint()
if not isinstance(plugin_blueprints, list):
plugin_blueprints = [plugin_blueprints]
for blueprint in plugin_blueprints:
app.register_extension_blueprint(blueprint)
lib_plugins.register_package_blueprints(app)
lib_plugins.register_group_blueprints(app)
# Start other middleware
for plugin in PluginImplementations(IMiddleware):
app = plugin.make_middleware(app, config)
for plugin in PluginImplementations(IMiddleware):
try:
app = plugin.make_error_log_middleware(app, config)
except AttributeError:
log.critical('Middleware class {0} is missing the method'
'make_error_log_middleware.'.format(
plugin.__class__.__name__))
# Initialize repoze.who
who_parser = WhoConfig(conf['here'])
who_parser.parse(open(conf['who.config_file']))
app = PluggableAuthenticationMiddleware(
RepozeAdapterMiddleware(app),
who_parser.identifiers,
who_parser.authenticators,
who_parser.challengers,
who_parser.mdproviders,
who_parser.request_classifier,
who_parser.challenge_decider,
logging.getLogger('repoze.who'),
logging.WARN, # ignored
who_parser.remote_user_key)
# Update the main CKAN config object with the Flask specific keys
# that were set here or autogenerated
flask_config_keys = set(flask_app.config.keys()) - set(config.keys())
for key in flask_config_keys:
config[key] = flask_app.config[key]
# Prevent the host from request to be added to the new header location.
app = HostHeaderMiddleware(app)
app = I18nMiddleware(app)
if asbool(config.get('ckan.tracking_enabled', 'false')):
app = TrackingMiddleware(app, config)
# Add a reference to the actual Flask app so it's easier to access
app._wsgi_app = flask_app
return app
def update_config():
''' This code needs to be run when the config is changed to take those
changes into account. It is called whenever a plugin is loaded as the
plugin might have changed the config values (for instance it might
change ckan.site_url) '''
for plugin in p.PluginImplementations(p.IConfigurer):
# must do update in place as this does not work:
# config = plugin.update_config(config)
plugin.update_config(config)
# Set whitelisted env vars on config object
# This is set up before globals are initialized
ckan_db = os.environ.get('CKAN_DB', None)
if ckan_db:
msg = 'Setting CKAN_DB as an env var is deprecated and will be' \
' removed in a future release. Use CKAN_SQLALCHEMY_URL instead.'
log.warn(msg)
config['sqlalchemy.url'] = ckan_db
for option in CONFIG_FROM_ENV_VARS:
from_env = os.environ.get(CONFIG_FROM_ENV_VARS[option], None)
if from_env:
config[option] = from_env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
site_url = config.get('ckan.site_url', '')
if not site_url:
raise RuntimeError(
'ckan.site_url is not configured and it must have a value.'
' Please amend your .ini file.')
if not site_url.lower().startswith('http'):
raise RuntimeError(
'ckan.site_url should be a full URL, including the schema '
'(http or https)')
display_timezone = config.get('ckan.display_timezone', '')
if (display_timezone and display_timezone != 'server'
and display_timezone not in pytz.all_timezones):
raise CkanConfigurationException(
"ckan.display_timezone is not 'server' or a valid timezone")
# Remove backslash from site_url if present
config['ckan.site_url'] = config['ckan.site_url'].rstrip('/')
ckan_host = config['ckan.host'] = urlparse(site_url).netloc
if config.get('ckan.site_id') is None:
if ':' in ckan_host:
ckan_host, port = ckan_host.split(':')
assert ckan_host, 'You need to configure ckan.site_url or ' \
'ckan.site_id for SOLR search-index rebuild to work.'
config['ckan.site_id'] = ckan_host
# ensure that a favicon has been set
favicon = config.get('ckan.favicon', '/base/images/ckan.ico')
config['ckan.favicon'] = favicon
# Init SOLR settings and check if the schema is compatible
# from ckan.lib.search import SolrSettings, check_solr_schema_version
# lib.search is imported here as we need the config enabled and parsed
search.SolrSettings.init(config.get('solr_url'), config.get('solr_user'),
config.get('solr_password'))
search.check_solr_schema_version()
routes_map = routing.make_map()
config['routes.map'] = routes_map
# The RoutesMiddleware needs its mapper updating if it exists
if 'routes.middleware' in config:
config['routes.middleware'].mapper = routes_map
# routes.named_routes is a CKAN thing
config['routes.named_routes'] = routing.named_routes
config['pylons.app_globals'] = app_globals.app_globals
# initialise the globals
app_globals.app_globals._init()
helpers.load_plugin_helpers()
config['pylons.h'] = helpers.helper_functions
jinja2_templates_path = os.path.join(root, 'templates')
template_paths = [jinja2_templates_path]
extra_template_paths = config.get('extra_template_paths', '')
if extra_template_paths:
# must be first for them to override defaults
template_paths = extra_template_paths.split(',') + template_paths
config['pylons.app_globals'].template_paths = template_paths
# Set the default language for validation messages from formencode
# to what is set as the default locale in the config
default_lang = config.get('ckan.locale_default', 'en')
formencode.api.set_stdtranslation(domain="FormEncode",
languages=[default_lang])
# Markdown ignores the logger config, so to get rid of excessive
# markdown debug messages in the log, set it to the level of the
# root logger.
logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
# Create Jinja2 environment
env = jinja_extensions.Environment(
loader=jinja_extensions.CkanFileSystemLoader(template_paths),
autoescape=True,
extensions=[
'jinja2.ext.do', 'jinja2.ext.with_',
jinja_extensions.SnippetExtension, jinja_extensions.CkanExtend,
jinja_extensions.CkanInternationalizationExtension,
jinja_extensions.LinkForExtension,
jinja_extensions.ResourceExtension,
jinja_extensions.UrlForStaticExtension,
jinja_extensions.UrlForExtension
])
env.install_gettext_callables(_, ungettext, newstyle=True)
# custom filters
env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
env.filters['truncate'] = jinja_extensions.truncate
config['pylons.app_globals'].jinja_env = env
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
# for postgresql we want to enforce utf-8
sqlalchemy_url = config.get('sqlalchemy.url', '')
if sqlalchemy_url.startswith('postgresql://'):
extras = {'client_encoding': 'utf8'}
else:
extras = {}
engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.', **extras)
if not model.meta.engine:
model.init_model(engine)
for plugin in p.PluginImplementations(p.IConfigurable):
plugin.configure(config)
# reset the template cache - we do this here so that when we load the
# environment it is clean
render.reset_template_info_cache()
# clear other caches
logic.clear_actions_cache()
logic.clear_validators_cache()
authz.clear_auth_functions_cache()
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
pass
# if an extension or our code does not finish
# transaction properly db cli commands can fail
model.Session.remove()
def get_minimum_views_for_trending():
return int(config.get('ckanext.lacounts.trending_min') or '10') or 10
def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf):
"""Create a Pylons WSGI application and return it
``conf``
The inherited configuration for this application. Normally from
the [DEFAULT] section of the Paste ini file.
``full_stack``
Whether this application provides a full WSGI stack (by default,
meaning it handles its own exceptions and errors). Disable
full_stack when this application is "managed" by another WSGI
middleware.
``static_files``
Whether this application serves its own static files; disable
when another web server is responsible for serving them.
``app_conf``
The application's local configuration. Normally specified in
the [app:<name>] section of the Paste ini file (where <name>
defaults to main).
"""
# The Pylons WSGI app
app = pylons_app = CKANPylonsApp()
for plugin in PluginImplementations(IMiddleware):
app = plugin.make_middleware(app, config)
app = common_middleware.CloseWSGIInputMiddleware(app, config)
app = common_middleware.RootPathMiddleware(app, config)
# Routing/Session/Cache Middleware
app = RoutesMiddleware(app, config['routes.map'])
# we want to be able to retrieve the routes middleware to be able to update
# the mapper. We store it in the pylons config to allow this.
config['routes.middleware'] = app
app = SessionMiddleware(app, config)
app = CacheMiddleware(app, config)
# CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
# app = QueueLogMiddleware(app)
if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)):
app = execute_on_completion(app, config,
cleanup_pylons_response_string)
# Fanstatic
if asbool(config.get('debug', False)):
fanstatic_config = {
'versioning': True,
'recompute_hashes': True,
'minified': False,
'bottom': True,
'bundle': False,
}
else:
fanstatic_config = {
'versioning': True,
'recompute_hashes': False,
'minified': True,
'bottom': True,
'bundle': True,
}
root_path = config.get('ckan.root_path', None)
if root_path:
root_path = re.sub('/{{LANG}}', '', root_path)
fanstatic_config['base_url'] = root_path
app = Fanstatic(app, **fanstatic_config)
for plugin in PluginImplementations(IMiddleware):
try:
app = plugin.make_error_log_middleware(app, config)
except AttributeError:
log.critical('Middleware class {0} is missing the method'
'make_error_log_middleware.'.format(
plugin.__class__.__name__))
if asbool(full_stack):
# Handle Python exceptions
app = ErrorHandler(app, conf, **config['pylons.errorware'])
# Display error documents for 400, 403, 404 status codes (and
# 500 when debug is disabled)
if asbool(config['debug']):
app = StatusCodeRedirect(app, [400, 403, 404])
else:
app = StatusCodeRedirect(app, [400, 403, 404, 500])
# Initialize repoze.who
who_parser = WhoConfig(conf['here'])
who_parser.parse(open(app_conf['who.config_file']))
app = PluggableAuthenticationMiddleware(
app,
who_parser.identifiers,
who_parser.authenticators,
who_parser.challengers,
who_parser.mdproviders,
who_parser.request_classifier,
who_parser.challenge_decider,
logging.getLogger('repoze.who'),
logging.WARN, # ignored
who_parser.remote_user_key)
# Establish the Registry for this application
# The RegistryManager includes code to pop
# registry values after the stream has completed,
# so we need to prevent this with `streaming` set to True.
app = RegistryManager(app, streaming=True)
if asbool(static_files):
# Serve static files
static_max_age = None if not asbool(
config.get('ckan.cache_enabled')) \
else int(config.get('ckan.static_max_age', 3600))
static_app = StaticURLParser(config['pylons.paths']['static_files'],
cache_max_age=static_max_age)
static_parsers = [static_app, app]
storage_directory = uploader.get_storage_path()
if storage_directory:
path = os.path.join(storage_directory, 'storage')
try:
os.makedirs(path)
except OSError as e:
# errno 17 is file already exists
if e.errno != 17:
raise
storage_app = StaticURLParser(path, cache_max_age=static_max_age)
static_parsers.insert(0, storage_app)
# Configurable extra static file paths
extra_static_parsers = []
for public_path in config.get('extra_public_paths', '').split(','):
if public_path.strip():
extra_static_parsers.append(
StaticURLParser(public_path.strip(),
cache_max_age=static_max_age))
app = Cascade(extra_static_parsers + static_parsers)
# Tracking
if asbool(config.get('ckan.tracking_enabled', 'false')):
app = common_middleware.TrackingMiddleware(app, config)
# Add a reference to the actual Pylons app so it's easier to access
app._wsgi_app = pylons_app
return app
def datapusher_status(context, data_dict):
''' Get the status of a datapusher job for a certain resource.
:param resource_id: The resource id of the resource that you want the
datapusher status for.
:type resource_id: string
'''
p.toolkit.check_access('datapusher_status', context, data_dict)
if 'id' in data_dict:
data_dict['resource_id'] = data_dict['id']
res_id = _get_or_bust(data_dict, 'resource_id')
task = p.toolkit.get_action('task_status_show')(context, {
'entity_id': res_id,
'task_type': 'datapusher',
'key': 'datapusher'
})
datapusher_url = config.get('ckan.datapusher.url')
if not datapusher_url:
raise p.toolkit.ValidationError(
{'configuration': ['ckan.datapusher.url not in config file']})
value = json.loads(task['value'])
job_key = value.get('job_key')
job_id = value.get('job_id')
url = None
job_detail = None
if job_id:
url = urlparse.urljoin(datapusher_url, 'job' + '/' + job_id)
try:
r = requests.get(url,
headers={
'Content-Type': 'application/json',
'Authorization': job_key
})
r.raise_for_status()
job_detail = r.json()
for log in job_detail['logs']:
if 'timestamp' in log:
date = time.strptime(log['timestamp'],
"%Y-%m-%dT%H:%M:%S.%f")
date = datetime.datetime.utcfromtimestamp(
time.mktime(date))
log['timestamp'] = date
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError):
job_detail = {'error': 'cannot connect to datapusher'}
return {
'status': task['state'],
'job_id': job_id,
'job_url': url,
'last_updated': task['last_updated'],
'job_key': job_key,
'task_info': job_detail,
'error': json.loads(task['error'])
}
class FeedController(base.BaseController):
base_url = config.get('ckan.site_url')
def _alternate_url(self, params, **kwargs):
search_params = params.copy()
search_params.update(kwargs)
# Can't count on the page sizes being the same on the search results
# view. So provide an alternate link to the first page, regardless
# of the page we're looking at in the feed.
search_params.pop('page', None)
return self._feed_url(search_params,
controller='package',
action='search')
def _group_or_organization(self, obj_dict, is_org):
data_dict, params = self._parse_url_params()
if is_org:
key = 'owner_org'
value = obj_dict['id']
group_type = 'organization'
else:
key = 'groups'
value = obj_dict['name']
group_type = 'group'
data_dict['fq'] = '{0}:"{1}"'.format(key, value)
item_count, results = _package_search(data_dict)
navigation_urls = self._navigation_urls(params,
item_count=item_count,
limit=data_dict['rows'],
controller='feed',
action=group_type,
id=obj_dict['name'])
feed_url = self._feed_url(params,
controller='feed',
action=group_type,
id=obj_dict['name'])
site_title = config.get('ckan.site_title', 'CKAN')
if is_org:
guid = _create_atom_id(u'/feeds/organization/%s.atom' %
obj_dict['name'])
alternate_url = self._alternate_url(params,
organization=obj_dict['name'])
desc = u'Recently created or updated datasets on %s '\
'by organization: "%s"' % (site_title, obj_dict['title'])
title = u'%s - Organization: "%s"' % (site_title,
obj_dict['title'])
else: # is group
guid = _create_atom_id(u'/feeds/group/%s.atom' %
obj_dict['name'])
alternate_url = self._alternate_url(params,
groups=obj_dict['name'])
desc = u'Recently created or updated datasets on %s '\
'by group: "%s"' % (site_title, obj_dict['title'])
title = u'%s - Group: "%s"' %\
(site_title, obj_dict['title'])
return self.output_feed(results,
feed_title=title,
feed_description=desc,
feed_link=alternate_url,
feed_guid=guid,
feed_url=feed_url,
navigation_urls=navigation_urls)
def group(self, id):
try:
context = {'model': model, 'session': model.Session,
'user': c.user, 'auth_user_obj': c.userobj}
group_dict = logic.get_action('group_show')(context, {'id': id})
except logic.NotFound:
base.abort(404, _('Group not found'))
return self._group_or_organization(group_dict, is_org=False)
def organization(self, id):
try:
context = {'model': model, 'session': model.Session,
'user': c.user, 'auth_user_obj': c.userobj}
group_dict = logic.get_action('organization_show')(context,
{'id': id})
except logic.NotFound:
base.abort(404, _('Organization not found'))
return self._group_or_organization(group_dict, is_org=True)
def tag(self, id):
data_dict, params = self._parse_url_params()
data_dict['fq'] = 'tags:"%s"' % id
item_count, results = _package_search(data_dict)
navigation_urls = self._navigation_urls(params,
item_count=item_count,
limit=data_dict['rows'],
controller='feed',
action='tag',
id=id)
feed_url = self._feed_url(params,
controller='feed',
action='tag',
id=id)
alternate_url = self._alternate_url(params, tags=id)
site_title = config.get('ckan.site_title', 'CKAN')
return self.output_feed(results,
feed_title=u'%s - Tag: "%s"' %
(site_title, id),
feed_description=u'Recently created or '
'updated datasets on %s by tag: "%s"' %
(site_title, id),
feed_link=alternate_url,
feed_guid=_create_atom_id
(u'/feeds/tag/%s.atom' % id),
feed_url=feed_url,
navigation_urls=navigation_urls)
def general(self):
data_dict, params = self._parse_url_params()
data_dict['q'] = '*:*'
item_count, results = _package_search(data_dict)
navigation_urls = self._navigation_urls(params,
item_count=item_count,
limit=data_dict['rows'],
controller='feed',
action='general')
feed_url = self._feed_url(params,
controller='feed',
action='general')
alternate_url = self._alternate_url(params)
site_title = config.get('ckan.site_title', 'CKAN')
return self.output_feed(results,
feed_title=site_title,
feed_description=u'Recently created or '
'updated datasets on %s' % site_title,
feed_link=alternate_url,
feed_guid=_create_atom_id
(u'/feeds/dataset.atom'),
feed_url=feed_url,
navigation_urls=navigation_urls)
# TODO check search params
def custom(self):
q = request.params.get('q', u'')
fq = ''
search_params = {}
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
search_params[param] = value
fq += ' %s:"%s"' % (param, value)
page = h.get_page_number(request.params)
limit = ITEMS_LIMIT
data_dict = {
'q': q,
'fq': fq,
'start': (page - 1) * limit,
'rows': limit,
'sort': request.params.get('sort', None),
}
item_count, results = _package_search(data_dict)
navigation_urls = self._navigation_urls(request.params,
item_count=item_count,
limit=data_dict['rows'],
controller='feed',
action='custom')
feed_url = self._feed_url(request.params,
controller='feed',
action='custom')
atom_url = h._url_with_params('/feeds/custom.atom',
search_params.items())
alternate_url = self._alternate_url(request.params)
site_title = config.get('ckan.site_title', 'CKAN')
return self.output_feed(results,
feed_title=u'%s - Custom query' % site_title,
feed_description=u'Recently created or updated'
' datasets on %s. Custom query: \'%s\'' %
(site_title, q),
feed_link=alternate_url,
feed_guid=_create_atom_id(atom_url),
feed_url=feed_url,
navigation_urls=navigation_urls)
def output_feed(self, results, feed_title, feed_description,
feed_link, feed_url, navigation_urls, feed_guid):
author_name = config.get('ckan.feeds.author_name', '').strip() or \
config.get('ckan.site_id', '').strip()
author_link = config.get('ckan.feeds.author_link', '').strip() or \
config.get('ckan.site_url', '').strip()
# TODO language
feed_class = None
for plugin in plugins.PluginImplementations(plugins.IFeed):
if hasattr(plugin, 'get_feed_class'):
feed_class = plugin.get_feed_class()
if not feed_class:
feed_class = _FixedAtom1Feed
feed = feed_class(
feed_title,
feed_link,
feed_description,
language=u'en',
author_name=author_name,
author_link=author_link,
feed_guid=feed_guid,
feed_url=feed_url,
previous_page=navigation_urls['previous'],
next_page=navigation_urls['next'],
first_page=navigation_urls['first'],
last_page=navigation_urls['last'],
)
for pkg in results:
additional_fields = {}
for plugin in plugins.PluginImplementations(plugins.IFeed):
if hasattr(plugin, 'get_item_additional_fields'):
additional_fields = plugin.get_item_additional_fields(pkg)
feed.add_item(
title=pkg.get('title', ''),
link=self.base_url + h.url_for('dataset.read',
id=pkg['id']),
description=pkg.get('notes', ''),
updated=h.date_str_to_datetime(pkg.get('metadata_modified')),
published=h.date_str_to_datetime(pkg.get('metadata_created')),
unique_id=_create_atom_id(u'/dataset/%s' % pkg['id']),
author_name=pkg.get('author', ''),
author_email=pkg.get('author_email', ''),
categories=[t['name'] for t in pkg.get('tags', [])],
enclosure=webhelpers.feedgenerator.Enclosure(
h.url_for(controller='api',
register='package',
action='show',
id=pkg['name'],
ver='3',
qualified=True),
text_type(len(json.dumps(pkg))), # TODO fix this
u'application/json'),
**additional_fields
)
response.content_type = feed.mime_type
return feed.writeString('utf-8')
# CLASS PRIVATE METHODS #
def _feed_url(self, query, controller, action, **kwargs):
"""
Constructs the url for the given action. Encoding the query
parameters.
"""
path = h.url_for(controller=controller, action=action, **kwargs)
return h._url_with_params(self.base_url + path, query.items())
def _navigation_urls(self, query, controller, action,
item_count, limit, **kwargs):
"""
Constructs and returns first, last, prev and next links for paging
"""
urls = dict((rel, None) for rel in 'previous next first last'.split())
page = int(query.get('page', 1))
# first: remove any page parameter
first_query = query.copy()
first_query.pop('page', None)
urls['first'] = self._feed_url(first_query, controller,
action, **kwargs)
# last: add last page parameter
last_page = (item_count / limit) + min(1, item_count % limit)
last_query = query.copy()
last_query['page'] = last_page
urls['last'] = self._feed_url(last_query, controller,
action, **kwargs)
# previous
if page > 1:
previous_query = query.copy()
previous_query['page'] = page - 1
urls['previous'] = self._feed_url(previous_query, controller,
action, **kwargs)
else:
urls['previous'] = None
# next
if page < last_page:
next_query = query.copy()
next_query['page'] = page + 1
urls['next'] = self._feed_url(next_query, controller,
action, **kwargs)
else:
urls['next'] = None
return urls
def _parse_url_params(self):
"""
Constructs a search-query dict from the URL query parameters.
Returns the constructed search-query dict, and the valid URL
query parameters.
"""
page = h.get_page_number(request.params)
limit = ITEMS_LIMIT
data_dict = {
'start': (page - 1) * limit,
'rows': limit
}
# Filter ignored query parameters
valid_params = ['page']
params = dict((p, request.params.get(p)) for p in valid_params
if p in request.params)
return data_dict, params
def me():
return h.redirect_to(
config.get(u'ckan.route_after_login', u'dashboard.index'))
def output_feed(self, results, feed_title, feed_description,
feed_link, feed_url, navigation_urls, feed_guid):
author_name = config.get('ckan.feeds.author_name', '').strip() or \
config.get('ckan.site_id', '').strip()
author_link = config.get('ckan.feeds.author_link', '').strip() or \
config.get('ckan.site_url', '').strip()
# TODO language
feed_class = None
for plugin in plugins.PluginImplementations(plugins.IFeed):
if hasattr(plugin, 'get_feed_class'):
feed_class = plugin.get_feed_class()
if not feed_class:
feed_class = _FixedAtom1Feed
feed = feed_class(
feed_title,
feed_link,
feed_description,
language=u'en',
author_name=author_name,
author_link=author_link,
feed_guid=feed_guid,
feed_url=feed_url,
previous_page=navigation_urls['previous'],
next_page=navigation_urls['next'],
first_page=navigation_urls['first'],
last_page=navigation_urls['last'],
)
for pkg in results:
additional_fields = {}
for plugin in plugins.PluginImplementations(plugins.IFeed):
if hasattr(plugin, 'get_item_additional_fields'):
additional_fields = plugin.get_item_additional_fields(pkg)
feed.add_item(
title=pkg.get('title', ''),
link=self.base_url + h.url_for('dataset.read',
id=pkg['id']),
description=pkg.get('notes', ''),
updated=h.date_str_to_datetime(pkg.get('metadata_modified')),
published=h.date_str_to_datetime(pkg.get('metadata_created')),
unique_id=_create_atom_id(u'/dataset/%s' % pkg['id']),
author_name=pkg.get('author', ''),
author_email=pkg.get('author_email', ''),
categories=[t['name'] for t in pkg.get('tags', [])],
enclosure=webhelpers.feedgenerator.Enclosure(
h.url_for(controller='api',
register='package',
action='show',
id=pkg['name'],
ver='3',
qualified=True),
text_type(len(json.dumps(pkg))), # TODO fix this
u'application/json'),
**additional_fields
)
response.content_type = feed.mime_type
return feed.writeString('utf-8')
def show_datalineage(self, id):
"""
Retrieves data lineage information for a specific package
"""
reinitialize_results()
context = {
'model': model,
'session': model.Session,
'user': c.user,
'for_view': True,
'auth_user_obj': c.userobj
}
data_dict = {'id': id}
extra_vars = {}
try:
c.pkg_dict = get_action('package_show')(context, data_dict)
c.pkg = context['package']
# this is a DS
if c.pkg_dict.get('parent'):
q = 'extras_code:%s' % (c.pkg_dict.get('parent').replace(
':', '\:'))
else:
# this is a process/activity/model
q = 'extras_parent:%s' % (c.pkg_dict.get('code').replace(
':', '\:'))
search_extras = {}
data_dict = {
'q':
q,
'fq':
q,
'extras':
search_extras,
'include_private':
asbool(config.get('ckan.search.default_include_private',
True)),
}
query = get_action('package_search')(context, data_dict)
results = query['results'][0] if query['results'] else {}
if c.pkg_dict.get('parent'):
extra_vars['datalineage_wasgeneratedby'] = results
else:
extra_vars['datalineage_generates'] = results
usage_models = get_usage_models(
context, c.pkg_dict if c.pkg_dict.get('parent') else results)
usage_datasets = get_usage_datasets(context, usage_models)
extra_vars['usage_models'] = usage_models
extra_vars['usage_datasets'] = usage_datasets
extra_vars['detail_data'] = c.pkg_dict if c.pkg_dict.get(
'parent') else results
extra_vars['current_model'] = results if c.pkg_dict.get(
'parent') else c.pkg_dict
# get the producers DSs
producers = c.pkg_dict.get('producers', '') or results.get(
'producers', '')
producers_info = []
if producers:
for ds_code in producers.split(','):
q = 'extras_code:%s' % (ds_code.replace(':', '\:'))
data_dict = {
'q':
q,
'fq':
q,
'extras': {},
'include_private':
asbool(
config.get('ckan.search.default_include_private',
True)),
}
query = get_action('package_search')(context, data_dict)
if query['count'] == 0:
logger.warning(
'No result found for producer [%s] of package [%s]'
% (ds_code, c.pkg_dict['code']))
else:
producers_info.append(query['results'][0])
extra_vars['datalineage_producers'] = producers_info
convert_extra_vars_to_metaviz(extra_vars)
dataset_type = c.pkg_dict['type'] or 'dataset'
extra_vars['dataset_type'] = dataset_type
except NotFound:
abort(404, _('Dataset not found'))
except NotAuthorized:
abort(403, _('Unauthorized to read dataset %s') % id)
return render(
'package/datalineage.html',
# extra_vars = {'data': json.dumps(extra_vars)},
extra_vars={'data': json.dumps(RESULTS)})
def search(package_type):
extra_vars = {}
try:
context = {
u'model': model,
u'user': g.user,
u'auth_user_obj': g.userobj
}
check_access(u'site_read', context)
except NotAuthorized:
base.abort(403, _(u'Not authorized to see this page'))
# unicode format (decoded from utf8)
extra_vars[u'q'] = q = request.args.get(u'q', u'')
extra_vars['query_error'] = False
page = h.get_page_number(request.args)
limit = int(config.get(u'ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.args.items() if k != u'page']
extra_vars[u'drill_down_url'] = drill_down_url
extra_vars[u'remove_field'] = partial(remove_field, package_type)
sort_by = request.args.get(u'sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != u'sort']
extra_vars[u'sort_by'] = partial(_sort_by, params_nosort, package_type)
if not sort_by:
sort_by_fields = []
else:
sort_by_fields = [field.split()[0] for field in sort_by.split(u',')]
extra_vars[u'sort_by_fields'] = sort_by_fields
pager_url = partial(_pager_url, params_nopage, package_type)
search_url_params = urlencode(_encode_params(params_nopage))
extra_vars[u'search_url_params'] = search_url_params
try:
# fields_grouped will contain a dict of params containing
# a list of values eg {u'tags':[u'tag1', u'tag2']}
extra_vars[u'fields'] = fields = []
extra_vars[u'fields_grouped'] = fields_grouped = {}
search_extras = {}
fq = u''
for (param, value) in request.args.items(multi=True):
if param not in [u'q', u'page', u'sort'] \
and len(value) and not param.startswith(u'_'):
if not param.startswith(u'ext_'):
fields.append((param, value))
fq += u' %s:"%s"' % (param, value)
if param not in fields_grouped:
fields_grouped[param] = [value]
else:
fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'for_view': True,
u'auth_user_obj': g.userobj
}
# Unless changed via config options, don't show other dataset
# types any search page. Potential alternatives are do show them
# on the default search page (dataset) or on one other search page
search_all_type = config.get(u'ckan.search.show_all_types', u'dataset')
search_all = False
try:
# If the "type" is set to True or False, convert to bool
# and we know that no type was specified, so use traditional
# behaviour of applying this only to dataset type
search_all = asbool(search_all_type)
search_all_type = u'dataset'
# Otherwise we treat as a string representing a type
except ValueError:
search_all = True
if not search_all or package_type != search_all_type:
# Only show datasets of this particular type
fq += u' +dataset_type:{type}'.format(type=package_type)
facets = OrderedDict()
default_facet_titles = {
u'organization': _(u'Organizations'),
u'groups': _(u'Groups'),
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license_id': _(u'Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in plugins.PluginImplementations(plugins.IFacets):
facets = plugin.dataset_facets(facets, package_type)
extra_vars[u'facet_titles'] = facets
data_dict = {
u'q': q,
u'fq': fq.strip(),
u'facet.field': facets.keys(),
u'rows': limit,
u'start': (page - 1) * limit,
u'sort': sort_by,
u'extras': search_extras,
u'include_private': asbool(
config.get(u'ckan.search.default_include_private', True)
),
}
query = get_action(u'package_search')(context, data_dict)
extra_vars[u'sort_by_selected'] = query[u'sort']
extra_vars[u'page'] = h.Page(
collection=query[u'results'],
page=page,
url=pager_url,
item_count=query[u'count'],
items_per_page=limit
)
extra_vars[u'search_facets'] = query[u'search_facets']
extra_vars[u'page'].items = query[u'results']
except SearchQueryError as se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info(u'Dataset search query rejected: %r', se.args)
base.abort(
400,
_(u'Invalid search query: {error_message}')
.format(error_message=str(se))
)
except SearchError as se:
# May be bad input from the user, but may also be more serious like
# bad code causing a SOLR syntax error, or a problem connecting to
# SOLR
log.error(u'Dataset search error: %r', se.args)
extra_vars[u'query_error'] = True
extra_vars[u'search_facets'] = {}
extra_vars[u'page'] = h.Page(collection=[])
# FIXME: try to avoid using global variables
g.search_facets_limits = {}
for facet in extra_vars[u'search_facets'].keys():
try:
limit = int(
request.args.get(
u'_%s_limit' % facet,
int(config.get(u'search.facets.default', 10))
)
)
except ValueError:
base.abort(
400,
_(u'Parameter u"{parameter_name}" is not '
u'an integer').format(parameter_name=u'_%s_limit' % facet)
)
g.search_facets_limits[facet] = limit
_setup_template_variables(context, {}, package_type=package_type)
extra_vars[u'dataset_type'] = package_type
# TODO: remove
for key, value in extra_vars.iteritems():
setattr(g, key, value)
return base.render(
_get_pkg_template(u'search_template', package_type), extra_vars
)
def package_publish_microdata(context, data_dict):
default_error = 'Unknown microdata error'
# Get data
dataset_id = data_dict.get('id')
nation = data_dict.get('nation')
repoid = data_dict.get('repoid')
# Check access
toolkit.check_access('sysadmin', context)
api_key = config.get('ckanext.unhcr.microdata_api_key')
if not api_key:
raise toolkit.NotAuthorized('Microdata API Key is not set')
# Get dataset/survey
headers = {'X-Api-Key': api_key}
dataset = toolkit.get_action('package_show')(context, {'id': dataset_id})
survey = helpers.convert_dataset_to_microdata_survey(
dataset, nation, repoid)
idno = survey['study_desc']['title_statement']['idno']
try:
# Publish dataset
url = 'https://microdata.unhcr.org/index.php/api/datasets/create/survey/%s' % idno
response = requests.post(url, headers=headers, json=survey).json()
if response.get('status') != 'success':
raise RuntimeError(str(response.get('errors', default_error)))
template = 'https://microdata.unhcr.org/index.php/catalog/%s'
survey['url'] = template % response['dataset']['id']
survey['resources'] = []
survey['files'] = []
# Pubish resources/files
file_name_counter = {}
if dataset.get('resources', []):
url = 'https://microdata.unhcr.org/index.php/api/datasets/%s/%s'
for resource in dataset.get('resources', []):
# resource
resouce_url = url % (idno, 'resources')
md_resource = helpers.convert_resource_to_microdata_resource(
resource)
response = requests.post(resouce_url,
headers=headers,
json=md_resource).json()
if response.get('status') != 'success':
raise RuntimeError(
str(response.get('errors', default_error)))
survey['resources'].append(response['resource'])
# file
file_url = url % (idno, 'files')
file_name = resource['url'].split('/')[-1]
file_path = helpers.get_resource_file_path(resource)
file_mime = resource['mimetype']
if not file_name or not file_path:
continue
file_name_counter.setdefault(file_name, 0)
file_name_counter[file_name] += 1
if file_name_counter[file_name] > 1:
file_name = helpers.add_file_name_suffix(
file_name, file_name_counter[file_name] - 1)
with open(file_path, 'rb') as file_obj:
file = (file_name, file_obj, file_mime)
response = requests.post(file_url,
headers=headers,
files={
'file': file
}).json()
# TODO: update
# it's a hack to overcome incorrect Microdata responses
# usopported file types fail this way and we are skipping them
if not isinstance(response, dict):
continue
if response.get('status') != 'success':
raise RuntimeError(
str(response.get('errors', default_error)))
survey['files'].append(response)
except requests.exceptions.HTTPError:
log.exception(exception)
raise RuntimeError('Microdata connection failed')
return survey
def _read(self, id, limit, group_type):
''' This is common code used by both read and bulk_process'''
context = {
'model': model,
'session': model.Session,
'user': c.user,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True,
'extras_as_string': True
}
q = c.q = request.params.get('q', '')
# Search within group
if c.group_dict.get('is_organization'):
q += ' owner_org:"%s"' % c.group_dict.get('id')
else:
q += ' groups:"%s"' % c.group_dict.get('name')
c.description_formatted = \
h.render_markdown(c.group_dict.get('description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
sort_by = request.params.get('sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
params = [
(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
for k, v in params
]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller='group',
action='read',
extras=dict(id=c.group_dict.get('name')),
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(key,
value=value,
replace=replace,
controller=controller,
action='read',
extras=dict(id=c.group_dict.get('name')))
c.remove_field = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
try:
c.fields = []
c.fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
q += ' %s: "%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses')
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
self._update_facet_titles(facets, group_type)
c.facet_titles = facets
data_dict = {
'q': q,
'fq': '',
'include_private': True,
'facet.field': facets.keys(),
'rows': limit,
'sort': sort_by,
'start': (page - 1) * limit,
'extras': search_extras
}
context_ = dict(
(k, v) for (k, v) in context.items() if k != 'schema')
query = get_action('package_search')(context_, data_dict)
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.group_dict['package_count'] = query['count']
c.search_facets = query['search_facets']
c.search_facets_limits = {}
for facet in c.search_facets.keys():
limit = int(
request.params.get('_%s_limit' % facet,
config.get('search.facets.default',
10)))
c.search_facets_limits[facet] = limit
c.page.items = query['results']
c.sort_by_selected = sort_by
except search.SearchError, se:
log.error('Group search error: %r', se.args)
c.query_error = True
c.page = h.Page(collection=[])
def datapusher_submit(context, data_dict):
''' Submit a job to the datapusher. The datapusher is a service that
imports tabular data into the datastore.
:param resource_id: The resource id of the resource that the data
should be imported in. The resource's URL will be used to get the data.
:type resource_id: string
:param set_url_type: If set to True, the ``url_type`` of the resource will
be set to ``datastore`` and the resource URL will automatically point
to the :ref:`datastore dump <dump>` URL. (optional, default: False)
:type set_url_type: bool
:param ignore_hash: If set to True, the datapusher will reload the file
even if it haven't changed. (optional, default: False)
:type ignore_hash: bool
Returns ``True`` if the job has been submitted and ``False`` if the job
has not been submitted, i.e. when the datapusher is not configured.
:rtype: bool
'''
schema = context.get('schema', dpschema.datapusher_submit_schema())
data_dict, errors = _validate(data_dict, schema, context)
if errors:
raise p.toolkit.ValidationError(errors)
res_id = data_dict['resource_id']
p.toolkit.check_access('datapusher_submit', context, data_dict)
try:
resource_dict = p.toolkit.get_action('resource_show')(context, {
'id': res_id,
})
except logic.NotFound:
return False
datapusher_url = config.get('ckan.datapusher.url')
site_url = h.url_for('/', qualified=True)
callback_url = h.url_for('/api/3/action/datapusher_hook', qualified=True)
user = p.toolkit.get_action('user_show')(context, {'id': context['user']})
for plugin in p.PluginImplementations(interfaces.IDataPusher):
upload = plugin.can_upload(res_id)
if not upload:
msg = "Plugin {0} rejected resource {1}"\
.format(plugin.__class__.__name__, res_id)
log.info(msg)
return False
task = {
'entity_id': res_id,
'entity_type': 'resource',
'task_type': 'datapusher',
'last_updated': str(datetime.datetime.utcnow()),
'state': 'submitting',
'key': 'datapusher',
'value': '{}',
'error': '{}',
}
try:
existing_task = p.toolkit.get_action('task_status_show')(
context, {
'entity_id': res_id,
'task_type': 'datapusher',
'key': 'datapusher'
})
assume_task_stale_after = datetime.timedelta(seconds=int(
config.get('ckan.datapusher.assume_task_stale_after', 3600)))
if existing_task.get('state') == 'pending':
updated = datetime.datetime.strptime(existing_task['last_updated'],
'%Y-%m-%dT%H:%M:%S.%f')
time_since_last_updated = datetime.datetime.utcnow() - updated
if time_since_last_updated > assume_task_stale_after:
# it's been a while since the job was last updated - it's more
# likely something went wrong with it and the state wasn't
# updated than its still in progress. Let it be restarted.
log.info(
'A pending task was found %r, but it is only %s hours'
'old', existing_task['id'], time_since_last_updated)
else:
log.info(
'A pending task was found %s for this resource, so '
'skipping this duplicate task', existing_task['id'])
return False
task['id'] = existing_task['id']
except logic.NotFound:
pass
context['ignore_auth'] = True
p.toolkit.get_action('task_status_update')(context, task)
try:
r = requests.post(urlparse.urljoin(datapusher_url, 'job'),
headers={'Content-Type': 'application/json'},
data=json.dumps({
'api_key': user['apikey'],
'job_type': 'push_to_datastore',
'result_url': callback_url,
'metadata': {
'ignore_hash':
data_dict.get('ignore_hash', False),
'ckan_url':
site_url,
'resource_id':
res_id,
'set_url_type':
data_dict.get('set_url_type', False),
'task_created':
task['last_updated'],
'original_url':
resource_dict.get('url'),
}
}))
r.raise_for_status()
except requests.exceptions.ConnectionError, e:
error = {
'message': 'Could not connect to DataPusher.',
'details': str(e)
}
task['error'] = json.dumps(error)
task['state'] = 'error'
task['last_updated'] = str(datetime.datetime.utcnow()),
p.toolkit.get_action('task_status_update')(context, task)
raise p.toolkit.ValidationError(error)
def index_package(self, pkg_dict, defer_commit=False):
if pkg_dict is None:
return
# tracking summary values will be stale, never store them
tracking_summary = pkg_dict.pop('tracking_summary', None)
for r in pkg_dict.get('resources', []):
r.pop('tracking_summary', None)
data_dict_json = json.dumps(pkg_dict)
if config.get('ckan.cache_validated_datasets', True):
package_plugin = lib_plugins.lookup_package_plugin(
pkg_dict.get('type'))
schema = package_plugin.show_package_schema()
validated_pkg_dict, errors = lib_plugins.plugin_validate(
package_plugin, {'model': model, 'session': model.Session},
pkg_dict, schema, 'package_show')
pkg_dict['validated_data_dict'] = json.dumps(validated_pkg_dict,
cls=ckan.lib.navl.dictization_functions.MissingNullEncoder)
pkg_dict['data_dict'] = data_dict_json
# add to string field for sorting
title = pkg_dict.get('title')
if title:
pkg_dict['title_string'] = title
# delete the package if there is no state, or the state is `deleted`
if (not pkg_dict.get('state') or 'deleted' in pkg_dict.get('state')):
return self.delete_package(pkg_dict)
index_fields = RESERVED_FIELDS + pkg_dict.keys()
# include the extras in the main namespace
extras = pkg_dict.get('extras', [])
for extra in extras:
key, value = extra['key'], extra['value']
if isinstance(value, (tuple, list)):
value = " ".join(map(text_type, value))
key = ''.join([c for c in key if c in KEY_CHARS])
pkg_dict['extras_' + key] = value
if key not in index_fields:
pkg_dict[key] = value
pkg_dict.pop('extras', None)
# add tags, removing vocab tags from 'tags' list and adding them as
# vocab_<tag name> so that they can be used in facets
non_vocab_tag_names = []
tags = pkg_dict.pop('tags', [])
context = {'model': model}
for tag in tags:
if tag.get('vocabulary_id'):
data = {'id': tag['vocabulary_id']}
vocab = logic.get_action('vocabulary_show')(context, data)
key = u'vocab_%s' % vocab['name']
if key in pkg_dict:
pkg_dict[key].append(tag['name'])
else:
pkg_dict[key] = [tag['name']]
else:
non_vocab_tag_names.append(tag['name'])
pkg_dict['tags'] = non_vocab_tag_names
# add groups
groups = pkg_dict.pop('groups', [])
# we use the capacity to make things private in the search index
if pkg_dict['private']:
pkg_dict['capacity'] = 'private'
else:
pkg_dict['capacity'] = 'public'
pkg_dict['groups'] = [group['name'] for group in groups]
# if there is an owner_org we want to add this to groups for index
# purposes
if pkg_dict.get('organization'):
pkg_dict['organization'] = pkg_dict['organization']['name']
else:
pkg_dict['organization'] = None
# tracking
if not tracking_summary:
tracking_summary = model.TrackingSummary.get_for_package(
pkg_dict['id'])
pkg_dict['views_total'] = tracking_summary['total']
pkg_dict['views_recent'] = tracking_summary['recent']
resource_fields = [('name', 'res_name'),
('description', 'res_description'),
('format', 'res_format'),
('url', 'res_url'),
('resource_type', 'res_type')]
resource_extras = [(e, 'res_extras_' + e) for e
in model.Resource.get_extra_columns()]
# flatten the structure for indexing:
for resource in pkg_dict.get('resources', []):
for (okey, nkey) in resource_fields + resource_extras:
pkg_dict[nkey] = pkg_dict.get(nkey, []) + [resource.get(okey, u'')]
pkg_dict.pop('resources', None)
rel_dict = collections.defaultdict(list)
subjects = pkg_dict.pop("relationships_as_subject", [])
objects = pkg_dict.pop("relationships_as_object", [])
for rel in objects:
type = model.PackageRelationship.forward_to_reverse_type(rel['type'])
rel_dict[type].append(model.Package.get(rel['subject_package_id']).name)
for rel in subjects:
type = rel['type']
rel_dict[type].append(model.Package.get(rel['object_package_id']).name)
for key, value in rel_dict.iteritems():
if key not in pkg_dict:
pkg_dict[key] = value
pkg_dict[TYPE_FIELD] = PACKAGE_TYPE
# Save dataset type
pkg_dict['dataset_type'] = pkg_dict['type']
# clean the dict fixing keys and dates
# FIXME where are we getting these dirty keys from? can we not just
# fix them in the correct place or is this something that always will
# be needed? For my data not changing the keys seems to not cause a
# problem.
new_dict = {}
bogus_date = datetime.datetime(1, 1, 1)
for key, value in pkg_dict.items():
key = key.encode('ascii', 'ignore')
if key.endswith('_date'):
try:
date = parse(value, default=bogus_date)
if date != bogus_date:
value = date.isoformat() + 'Z'
else:
# The date field was empty, so dateutil filled it with
# the default bogus date
value = None
except ValueError:
continue
new_dict[key] = value
pkg_dict = new_dict
for k in ('title', 'notes', 'title_string'):
if k in pkg_dict and pkg_dict[k]:
pkg_dict[k] = escape_xml_illegal_chars(pkg_dict[k])
# modify dates (SOLR is quite picky with dates, and only accepts ISO dates
# with UTC time (i.e trailing Z)
# See http://lucene.apache.org/solr/api/org/apache/solr/schema/DateField.html
pkg_dict['metadata_created'] += 'Z'
pkg_dict['metadata_modified'] += 'Z'
# mark this CKAN instance as data source:
pkg_dict['site_id'] = config.get('ckan.site_id')
# Strip a selection of the fields.
# These fields are possible candidates for sorting search results on,
# so we strip leading spaces because solr will sort " " before "a" or "A".
for field_name in ['title']:
try:
value = pkg_dict.get(field_name)
if value:
pkg_dict[field_name] = value.lstrip()
except KeyError:
pass
# add a unique index_id to avoid conflicts
import hashlib
pkg_dict['index_id'] = hashlib.md5('%s%s' % (pkg_dict['id'],config.get('ckan.site_id'))).hexdigest()
for item in PluginImplementations(IPackageController):
pkg_dict = item.before_index(pkg_dict)
assert pkg_dict, 'Plugin must return non empty package dict on index'
# permission labels determine visibility in search, can't be set
# in original dataset or before_index plugins
labels = lib_plugins.get_permission_labels()
dataset = model.Package.get(pkg_dict['id'])
pkg_dict['permission_labels'] = labels.get_dataset_labels(
dataset) if dataset else [] # TestPackageSearchIndex-workaround
# send to solr:
try:
conn = make_connection()
commit = not defer_commit
if not asbool(config.get('ckan.search.solr_commit', 'true')):
commit = False
conn.add(docs=[pkg_dict], commit=commit)
except pysolr.SolrError as e:
msg = 'Solr returned an error: {0}'.format(
e[:1000] # limit huge responses
)
raise SearchIndexError(msg)
except socket.error as e:
err = 'Could not connect to Solr using {0}: {1}'.format(conn.url, str(e))
log.error(err)
raise SearchIndexError(err)
commit_debug_msg = 'Not committed yet' if defer_commit else 'Committed'
log.debug('Updated index for %s [%s]' % (pkg_dict.get('name'), commit_debug_msg))
def disable_user_access():
return toolkit.asbool(config.get('ckan.senasa_theme.disable_user_access', False))
def _read(id, limit, group_type):
u''' This is common code used by both read and bulk_process'''
extra_vars = {}
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'schema': _db_to_form_schema(group_type=group_type),
u'for_view': True,
u'extras_as_string': True
}
q = request.params.get(u'q', u'')
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.q = q
# Search within group
if g.group_dict.get(u'is_organization'):
fq = u' owner_org:"%s"' % g.group_dict.get(u'id')
else:
fq = u' groups:"%s"' % g.group_dict.get(u'name')
extra_vars["q"] = q
g.description_formatted = \
h.render_markdown(g.group_dict.get(u'description'))
context['return_query'] = True
page = h.get_page_number(request.params)
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items() if k != u'page']
sort_by = request.params.get(u'sort', None)
def search_url(params):
controller = lookup_group_controller(group_type)
action = u'bulk_process' if getattr(
g, u'action', u'') == u'bulk_process' else u'read'
url = h.url_for(u'.'.join([controller, action]), id=id)
params = [
(k, v.encode(u'utf-8') if isinstance(v, string_types) else str(v))
for k, v in params
]
return url + u'?' + urlencode(params)
def drill_down_url(**by):
return h.add_url_param(alternative_url=None,
controller=u'group',
action=u'read',
extras=dict(id=g.group_dict.get(u'name')),
new_params=by)
extra_vars["drill_down_url"] = drill_down_url
def remove_field(key, value=None, replace=None):
controller = lookup_group_controller(group_type)
return h.remove_url_param(key,
value=value,
replace=replace,
controller=controller,
action=u'read',
extras=dict(id=g.group_dict.get(u'name')))
extra_vars["remove_field"] = remove_field
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append((u'page', page))
return search_url(params)
try:
extra_vars["fields"] = fields = []
extra_vars["fields_grouped"] = fields_grouped = {}
search_extras = {}
for (param, value) in request.params.items():
if param not in [u'q', u'page', u'sort'] \
and len(value) and not param.startswith(u'_'):
if not param.startswith(u'ext_'):
fields.append((param, value))
q += u' %s: "%s"' % (param, value)
if param not in fields_grouped:
fields_grouped[param] = [value]
else:
fields_grouped[param].append(value)
else:
search_extras[param] = value
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.fields = fields
g.fields_grouped = fields_grouped
facets = OrderedDict()
default_facet_titles = {
u'organization': _(u'Organizations'),
u'groups': _(u'Groups'),
u'tags': _(u'Tags'),
u'res_format': _(u'Formats'),
u'license_id': _(u'Licenses')
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
_update_facet_titles(facets, group_type)
extra_vars["facet_titles"] = facets
data_dict = {
u'q': q,
u'fq': fq,
u'include_private': True,
u'facet.field': facets.keys(),
u'rows': limit,
u'sort': sort_by,
u'start': (page - 1) * limit,
u'extras': search_extras
}
context_ = dict((k, v) for (k, v) in context.items() if k != u'schema')
query = get_action(u'package_search')(context_, data_dict)
extra_vars["page"] = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.group_dict['package_count'] = query['count']
extra_vars["search_facets"] = g.search_facets = query['search_facets']
extra_vars["search_facets_limits"] = g.search_facets_limits = {}
for facet in g.search_facets.keys():
limit = int(
request.params.get(u'_%s_limit' % facet,
config.get(u'search.facets.default', 10)))
g.search_facets_limits[facet] = limit
extra_vars["page"].items = query['results']
extra_vars["sort_by_selected"] = sort_by
except search.SearchError as se:
log.error(u'Group search error: %r', se.args)
extra_vars["query_error"] = True
extra_vars["page"] = h.Page(collection=[])
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.facet_titles = facets
g.page = extra_vars["page"]
extra_vars["group_type"] = group_type
_setup_template_variables(context, {u'id': id}, group_type=group_type)
return extra_vars
def index(group_type, is_organization):
extra_vars = {}
set_org(is_organization)
page = h.get_page_number(request.params) or 1
items_per_page = int(config.get(u'ckan.datasets_per_page', 20))
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'for_view': True,
u'with_private': False
}
try:
_check_access(u'site_read', context)
_check_access(u'group_list', context)
except NotAuthorized:
base.abort(403, _(u'Not authorized to see this page'))
q = request.params.get(u'q', u'')
sort_by = request.params.get(u'sort')
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.q = q
g.sort_by_selected = sort_by
extra_vars["q"] = q
extra_vars["sort_by_selected"] = sort_by
# pass user info to context as needed to view private datasets of
# orgs correctly
if g.userobj:
context['user_id'] = g.userobj.id
context['user_is_admin'] = g.userobj.sysadmin
try:
data_dict_global_results = {
u'all_fields': False,
u'q': q,
u'sort': sort_by,
u'type': group_type or u'group',
}
global_results = _action(u'group_list')(context,
data_dict_global_results)
except ValidationError as e:
if e.error_dict and e.error_dict.get(u'message'):
msg = e.error_dict['message']
else:
msg = str(e)
h.flash_error(msg)
extra_vars["page"] = h.Page([], 0)
extra_vars["group_type"] = group_type
return base.render(_index_template(group_type), extra_vars)
data_dict_page_results = {
u'all_fields': True,
u'q': q,
u'sort': sort_by,
u'type': group_type or u'group',
u'limit': items_per_page,
u'offset': items_per_page * (page - 1),
u'include_extras': True
}
page_results = _action(u'group_list')(context, data_dict_page_results)
extra_vars["page"] = h.Page(
collection=global_results,
page=page,
url=h.pager_url,
items_per_page=items_per_page,
)
extra_vars["page"].items = page_results
extra_vars["group_type"] = group_type
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.page = extra_vars["page"]
return base.render(_index_template(group_type), extra_vars)
def search_params():
u'''Returns a list of the current search names'''
return config.get(u'search.search_param', DEFAULT_SEARCH_NAMES).split()
def make_flask_stack(conf, **app_conf):
""" This has to pass the flask app through all the same middleware that
Pylons used """
root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
debug = asbool(conf.get('debug', conf.get('DEBUG', False)))
testing = asbool(app_conf.get('testing', app_conf.get('TESTING', False)))
app = flask_app = CKANFlask(__name__)
app.debug = debug
app.testing = testing
app.template_folder = os.path.join(root, 'templates')
app.app_ctx_globals_class = CKAN_AppCtxGlobals
app.url_rule_class = CKAN_Rule
app.jinja_options = jinja_extensions.get_jinja_env_options()
# Update Flask config with the CKAN values. We use the common config
# object as values might have been modified on `load_environment`
if config:
app.config.update(config)
else:
app.config.update(conf)
app.config.update(app_conf)
# Do all the Flask-specific stuff before adding other middlewares
# Secret key needed for flask-debug-toolbar and sessions
if not app.config.get('SECRET_KEY'):
app.config['SECRET_KEY'] = config.get('beaker.session.secret')
if not app.config.get('SECRET_KEY'):
raise RuntimeError(u'You must provide a value for the secret key'
' with the SECRET_KEY config option')
if debug:
from flask_debugtoolbar import DebugToolbarExtension
app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
DebugToolbarExtension(app)
from werkzeug.debug import DebuggedApplication
app = DebuggedApplication(app, True)
app = app.app
log = logging.getLogger('werkzeug')
log.setLevel(logging.DEBUG)
# Use Beaker as the Flask session interface
class BeakerSessionInterface(SessionInterface):
def open_session(self, app, request):
if 'beaker.session' in request.environ:
return request.environ['beaker.session']
def save_session(self, app, session, response):
session.save()
namespace = 'beaker.session.'
session_opts = dict([(k.replace('beaker.', ''), v)
for k, v in config.iteritems()
if k.startswith(namespace)])
if (not session_opts.get('session.data_dir')
and session_opts.get('session.type', 'file') == 'file'):
cache_dir = app_conf.get('cache_dir') or app_conf.get('cache.dir')
session_opts['session.data_dir'] = '{data_dir}/sessions'.format(
data_dir=cache_dir)
app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts)
app.session_interface = BeakerSessionInterface()
# Add Jinja2 extensions and filters
app.jinja_env.filters['empty_and_escape'] = \
jinja_extensions.empty_and_escape
# Common handlers for all requests
app.before_request(ckan_before_request)
app.after_request(ckan_after_request)
# Template context processors
app.context_processor(helper_functions)
app.context_processor(c_object)
@app.context_processor
def ungettext_alias():
u'''
Provide `ungettext` as an alias of `ngettext` for backwards
compatibility
'''
return dict(ungettext=ungettext)
# Babel
pairs = [(os.path.join(root, u'i18n'), 'ckan')
] + [(p.i18n_directory(), p.i18n_domain())
for p in PluginImplementations(ITranslation)]
i18n_dirs, i18n_domains = zip(*pairs)
app.config[u'BABEL_TRANSLATION_DIRECTORIES'] = ';'.join(i18n_dirs)
app.config[u'BABEL_DOMAIN'] = 'ckan'
app.config[u'BABEL_MULTIPLE_DOMAINS'] = ';'.join(i18n_domains)
babel = CKANBabel(app)
babel.localeselector(get_locale)
@app.route('/hello', methods=['GET'])
def hello_world():
return 'Hello World, this is served by Flask'
@app.route('/hello', methods=['POST'])
def hello_world_post():
return 'Hello World, this was posted to Flask'
# Auto-register all blueprints defined in the `views` folder
_register_core_blueprints(app)
_register_error_handler(app)
# Set up each IBlueprint extension as a Flask Blueprint
for plugin in PluginImplementations(IBlueprint):
if hasattr(plugin, 'get_blueprint'):
plugin_blueprints = plugin.get_blueprint()
if not isinstance(plugin_blueprints, list):
plugin_blueprints = [plugin_blueprints]
for blueprint in plugin_blueprints:
app.register_extension_blueprint(blueprint)
lib_plugins.register_package_blueprints(app)
lib_plugins.register_group_blueprints(app)
# Set flask routes in named_routes
for rule in app.url_map.iter_rules():
if '.' not in rule.endpoint:
continue
controller, action = rule.endpoint.split('.')
needed = list(rule.arguments - set(rule.defaults or {}))
route = {
rule.endpoint: {
'action': action,
'controller': controller,
'highlight_actions': action,
'needed': needed
}
}
config['routes.named_routes'].update(route)
# Start other middleware
for plugin in PluginImplementations(IMiddleware):
app = plugin.make_middleware(app, config)
# Fanstatic
fanstatic_enable_rollup = asbool(
app_conf.get('fanstatic_enable_rollup', False))
if debug:
fanstatic_config = {
'versioning': True,
'recompute_hashes': True,
'minified': False,
'bottom': True,
'bundle': False,
'rollup': fanstatic_enable_rollup,
}
else:
fanstatic_config = {
'versioning': True,
'recompute_hashes': False,
'minified': True,
'bottom': True,
'bundle': True,
'rollup': fanstatic_enable_rollup,
}
root_path = config.get('ckan.root_path', None)
if root_path:
root_path = re.sub('/{{LANG}}', '', root_path)
fanstatic_config['base_url'] = root_path
app = Fanstatic(app, **fanstatic_config)
for plugin in PluginImplementations(IMiddleware):
try:
app = plugin.make_error_log_middleware(app, config)
except AttributeError:
log.critical('Middleware class {0} is missing the method'
'make_error_log_middleware.'.format(
plugin.__class__.__name__))
# Initialize repoze.who
who_parser = WhoConfig(conf['here'])
who_parser.parse(open(app_conf['who.config_file']))
app = PluggableAuthenticationMiddleware(
app,
who_parser.identifiers,
who_parser.authenticators,
who_parser.challengers,
who_parser.mdproviders,
who_parser.request_classifier,
who_parser.challenge_decider,
logging.getLogger('repoze.who'),
logging.WARN, # ignored
who_parser.remote_user_key)
# Update the main CKAN config object with the Flask specific keys
# that were set here or autogenerated
flask_config_keys = set(flask_app.config.keys()) - set(config.keys())
for key in flask_config_keys:
config[key] = flask_app.config[key]
# Add a reference to the actual Flask app so it's easier to access
app._wsgi_app = flask_app
return app
def send_email(content, to, subject, file=None):
'''Sends email
:param content: The body content for the mail.
:type string:
:param to: To whom will be mail sent
:type string:
:param subject: The subject of mail.
:type string:
:rtype: string
'''
SMTP_SERVER = config.get('smtp.server')
SMTP_USER = config.get('smtp.user')
SMTP_PASSWORD = config.get('smtp.password')
SMTP_FROM = config.get('smtp.mail_from')
msg = MIMEMultipart()
from_ = SMTP_FROM
if isinstance(to, basestring):
to = [to]
msg['Subject'] = subject
msg['From'] = from_
msg['To'] = ','.join(to)
content = """\
<html>
<head></head>
<body>
<p>""" + content + """</p>
</body>
</html>
"""
msg.attach(MIMEText(content, 'html', _charset='utf-8'))
if isinstance(file, cgi.FieldStorage):
part = MIMEBase('application', 'octet-stream')
part.set_payload(file.file.read())
Encoders.encode_base64(part)
extension = file.filename.split('.')[-1]
part.add_header('Content-Disposition',
'attachment; filename=attachment.{0}'.format(extension))
msg.attach(part)
try:
s = smtplib.SMTP(SMTP_SERVER)
if SMTP_USER:
s.login(SMTP_USER, SMTP_PASSWORD)
s.sendmail(from_, to, msg.as_string())
s.quit()
response_dict = {
'success': True,
'message': 'Email message was successfully sent.'
}
return response_dict
except socket_error:
log.critical(
'Could not connect to email server. Have you configured the SMTP settings?')
error_dict = {
'success': False,
'message': 'An error occured while sending the email. Try again.'
}
return error_dict