def issues_enabled(dataset):
'''Returns whether issues are enabled for the given dataset (dict)'''
# config options allow you to only enable issues for particular datasets or
# organizations
datasets_with_issues_enabled = set(
toolkit.aslist(config.get('ckanext.issues.enabled_for_datasets')))
organizations_with_issues_enabled = set(
toolkit.aslist(config.get('ckanext.issues.enabled_for_organizations')))
if datasets_with_issues_enabled or organizations_with_issues_enabled:
if datasets_with_issues_enabled and \
dataset['name'] in datasets_with_issues_enabled:
return True
elif organizations_with_issues_enabled and \
(dataset.get('organization') or {}).get('name') in \
organizations_with_issues_enabled:
return True
return False
else:
extras = dataset.get('extras', [])
for extra in extras:
if extra.get('key') == 'issues_enabled':
return toolkit.asbool(extra.get('value'))
else:
return toolkit.asbool(
config.get('ckanext.issues.enabled_without_extra', True))
def is_resource_blacklisted(resource):
package = toolkit.get_action('package_show')({'ignore_auth': True}, {
'id': resource['package_id'],
})
org_blacklist = list(set(toolkit.aslist(config.get('ckan.spatialingestor.org_blacklist', []))))
pkg_blacklist = list(set(toolkit.aslist(config.get('ckan.spatialingestor.pkg_blacklist', []))))
user_blacklist = list(
set(map(lambda x: model.User.get(x).id, toolkit.aslist(config.get('ckan.spatialingestor.user_blacklist', [])))))
if package['organization']['name'] in org_blacklist:
log.error("{0} in organization blacklist".format(package['organization']['name']))
return True
elif package['name'] in pkg_blacklist:
log.error("{0} in package blacklist".format(package['name']))
return True
else:
activity_list = toolkit.get_action('package_activity_list')({'ignore_auth': True}, {
'id': package['id'],
})
last_user = package['creator_user_id']
if activity_list:
last_user = activity_list[0]['user_id']
if last_user in user_blacklist:
log.error("{0} was last edited by blacklisted user".format(activity_list[0]['user_id']))
return True
return False
def before_create(self, context, resource):
disallowed_extensions = toolkit.aslist(config.get('ckanext.romania_theme.disallowed_extensions',[]))
disallowed_mimetypes = [mimetypes.types_map["." + x] for x in disallowed_extensions]
allowed_extensions = toolkit.aslist(config.get('ckanext.romania_theme.allowed_extensions',[]))
allowed_mimetypes = [mimetypes.types_map["." + x] for x in allowed_extensions]
is_resource_extension_allowed = False
error_message = ''
if allowed_mimetypes:
if resource['upload'].type in allowed_mimetypes:
is_resource_extension_allowed = True
else:
error_message="Doar urmatoarele extensii sunt permise: " + ", ".join(allowed_extensions) + "."
else:
if resource['upload'].type not in disallowed_mimetypes:
is_resource_extension_allowed = True
else:
error_message= "Urmatoarele extensii sunt nepermise: " + ", ".join(disallowed_extensions) + "."
if ('upload' in resource) and (type(resource['upload']) is not unicode) and not is_resource_extension_allowed:
# If we did not do this, the URL field would contain the filename
# and people can press finalize afterwards.
resource['url'] = ''
raise toolkit.ValidationError(['Fisierul are o extensie nepermisa! ' + error_message])
def before_create(self, context, resource):
disallowed_extensions = toolkit.aslist(config.get('ckanext.datagovro_theme.disallowed_extensions',[]))
disallowed_mimetypes = [mimetypes.types_map["." + x] for x in disallowed_extensions]
allowed_extensions = toolkit.aslist(config.get('ckanext.datagovro_theme.allowed_extensions',[]))
allowed_mimetypes = [mimetypes.types_map["." + x] for x in allowed_extensions]
is_resource_extension_allowed = False
error_message = ''
if (type(resource['upload']) is not unicode):
if allowed_mimetypes:
if resource['upload'].type in allowed_mimetypes:
is_resource_extension_allowed = True
else:
error_message="Doar urmatoarele extensii sunt permise: " + ", ".join(allowed_extensions) + "."
else:
if resource['upload'].type not in disallowed_mimetypes:
is_resource_extension_allowed = True
else:
error_message= "Urmatoarele extensii sunt nepermise: " + ", ".join(disallowed_extensions) + "."
# is_resource_extension_allowed = True
if ('upload' in resource) and (type(resource['upload']) is not unicode) and not is_resource_extension_allowed:
# If we did not do this, the URL field would contain the filename
# and people can press finalize afterwards.
resource['url'] = ''
raise toolkit.ValidationError(['Fisierul are o extensie nepermisa! ' + error_message])
is_resource_extension_allowed = False
def issues_enabled(dataset):
'''Returns whether issues are enabled for the given dataset (dict)'''
# config options allow you to only enable issues for particular datasets or
# organizations
datasets_with_issues_enabled = set(toolkit.aslist(
config.get('ckanext.issues.enabled_for_datasets')
))
organizations_with_issues_enabled = set(toolkit.aslist(
config.get('ckanext.issues.enabled_for_organizations')
))
if datasets_with_issues_enabled or organizations_with_issues_enabled:
if datasets_with_issues_enabled and \
dataset['name'] in datasets_with_issues_enabled:
return True
elif organizations_with_issues_enabled and \
(dataset.get('organization') or {}).get('name') in \
organizations_with_issues_enabled:
return True
return False
else:
extras = dataset.get('extras', [])
for extra in extras:
if extra.get('key') == 'issues_enabled':
return toolkit.asbool(extra.get('value'))
else:
return toolkit.asbool(
config.get('ckanext.issues.enabled_without_extra', True)
)
def update_config(self, config):
u'''
Load config and set up the resource library,
public directory and template directory for the view
'''
# https://datatables.net/reference/option/lengthMenu
self.page_length_choices = toolkit.aslist(
config.get(u'ckan.datatables.page_length_choices',
DEFAULT_PAGE_LENGTH_CHOICES))
self.page_length_choices = [int(i) for i in self.page_length_choices]
self.top_pagination_controls = toolkit.asbool(
config.get(u'ckan.datatables.top_pagination_controls', False))
self.search_delay = toolkit.asint(
config.get(u'ckan.datatables.search_delay', DEFAULT_SEARCH_DELAY))
self.state_saving = toolkit.asbool(
config.get(u'ckan.datatables.state_saving', True))
# https://datatables.net/reference/option/stateDuration
self.state_duration = toolkit.asint(
config.get(u'ckan.datatables.state_duration',
DEFAULT_STATE_DURATION))
self.data_dictionary_labels = toolkit.asbool(
config.get(u'ckan.datatables.data_dictionary_labels', True))
toolkit.add_template_directory(config, u'templates')
toolkit.add_public_directory(config, u'public')
toolkit.add_resource(u'public', u'ckanext-datatablesview')
def get_resource_info_by_id(cls, resource_id):
resource = model.Session.query(
model.Resource).filter(model.Resource.id == resource_id).first()
res_name = None
res_package_name = None
res_package_id = None
if resource is not None:
res_package_name = resource.package.title or resource.package.name
res_package_id = resource.package.name
name_translated_json = resource.extras.get('name_translated')
if name_translated_json is not None:
try:
name_translated = json.loads(name_translated_json)
languages = aslist(config.get('ckan.locales_offered', []))
res_name = next(
(name_translated[lang]
for lang in languages if name_translated.get(lang)),
None)
except json.JSONDecodeError:
pass
res_name = res_name or resource.description or resource.format
return [res_name, res_package_name, res_package_id]
def update_config(self, config):
u'''
Set up the resource library, public directory and
template directory for the view
'''
# https://datatables.net/reference/option/lengthMenu
self.page_length_choices = toolkit.aslist(
config.get(u'ckan.datatables.page_length_choices',
DEFAULT_PAGE_LENGTH_CHOICES))
self.page_length_choices = [int(i) for i in self.page_length_choices]
self.state_saving = toolkit.asbool(
config.get(u'ckan.datatables.state_saving', True))
# https://datatables.net/reference/option/stateDuration
self.state_duration = toolkit.asint(
config.get(u'ckan.datatables.state_duration',
DEFAULT_STATE_DURATION))
self.data_dictionary_labels = toolkit.asbool(
config.get(u'ckan.datatables.data_dictionary_labels', True))
self.ellipsis_length = toolkit.asint(
config.get(u'ckan.datatables.ellipsis_length',
DEFAULT_ELLIPSIS_LENGTH))
self.date_format = config.get(u'ckan.datatables.date_format',
DEFAULT_DATE_FORMAT)
self.default_view = config.get(u'ckan.datatables.default_view',
'table')
toolkit.add_template_directory(config, u'templates')
toolkit.add_public_directory(config, u'public')
toolkit.add_resource(u'public', u'ckanext-datatablesview')
def run(ctx, host, port, disable_reloader, threaded, extra_files, processes):
u"""Runs the Werkzeug development server"""
use_reloader = not disable_reloader
threaded = threaded or tk.asbool(config.get(u"ckan.devserver.threaded"))
processes = processes or tk.asint(
config.get(u"ckan.devserver.multiprocess", 1)
)
if threaded and processes > 1:
tk.error_shout(u"Cannot have a multithreaded and multi process server")
raise click.Abort()
log.info(u"Running server {0} on port {1}".format(host, port))
config_extra_files = tk.aslist(
config.get(u"ckan.devserver.watch_patterns")
)
extra_files = list(extra_files) + [
config[u"__file__"]
] + config_extra_files
run_simple(
host,
port,
ctx.obj.app,
use_reloader=use_reloader,
use_evalex=True,
threaded=threaded,
processes=processes,
extra_files=extra_files,
)
def update_config_schema(self, schema):
ignore_missing = toolkit.get_validator('ignore_missing')
charts = {}
for _ in range(1, 5):
charts.update({
'ckanext.eds.chart_{idx}'.format(idx=_):
[ignore_missing, unicode],
'ckanext.eds.chart_{idx}_subheader'.format(idx=_):
[ignore_missing, unicode]
})
for l in get_available_locales():
if l.short_name in toolkit.aslist(
config.get('ckanext.eds.available_locales', 'en da_DK')):
schema.update({
'ckan.site_intro_text_{0}'.format(l):
[ignore_missing, unicode],
'ckan.site_description_{0}'.format(l):
[ignore_missing, unicode],
'ckan.site_about_{0}'.format(l): [ignore_missing, unicode],
'ckan.site_title_{0}'.format(l): [ignore_missing, unicode],
'ckan.cookie_notice_content_{0}'.format(l):
[ignore_missing, unicode],
'api_guides_content_{0}'.format(l):
[ignore_missing, unicode],
'simple_guides_content_{0}'.format(l):
[ignore_missing, unicode]
})
schema.update(charts)
return schema
def syndicate_configs_from_config(config) -> Iterable[Profile]:
prefix = "ckan.syndicate."
keys = (
"api_key",
"author",
"extras",
"field_id",
"flag",
"organization",
"predicate",
"name_prefix",
"replicate_organization",
"ckan_url",
)
profile_lists = zip_longest(
*[tk.aslist(config.get(prefix + key)) for key in keys])
for idx, item in enumerate(profile_lists):
deprecated(f"Deprecated profile definition: {item}. Use"
f" {PROFILE_PREFIX}*.OPTION form")
data = dict((k, v) for k, v in zip(keys, item) if v is not None)
try:
data["extras"] = json.loads(data.get("extras", "{}"))
except (TypeError, ValueError):
data["extras"] = {}
yield Profile(id=str(idx), **data)
yield from _parse_profiles(config)
def mimetype_autocomplete(context, data_dict):
'''Return a list of MIME types whose names contain a string.
:param q: the string to search for
:type q: string
:param limit: the maximum number of resource formats to return (optional,default: 5)
:type limit: int
:rtype: list of strings
'''
_check_access('site_read', context, data_dict)
q = data_dict.get('q', None)
if not q:
return []
limit = int(data_dict.get('limit', 5))
mime_types = toolkit.aslist(
config.get('ckanext.publicamundi.mime_types', ''))
results = []
n = 0
for t in mime_types:
if t.find(q) >= 0:
results.append(t)
n += 1
if n == limit:
break
return results
def run(ctx, host, port, disable_reloader, threaded, extra_files, processes,
ssl_cert, ssl_key):
u"""Runs the Werkzeug development server"""
if tk.asbool(config.get("debug")):
warnings.filterwarnings("default", category=CkanDeprecationWarning)
# Reloading
use_reloader = not disable_reloader
config_extra_files = tk.aslist(
config.get(u"ckan.devserver.watch_patterns")
)
extra_files = list(extra_files) + [
config[u"__file__"]
] + config_extra_files
# Threads and processes
threaded = threaded or tk.asbool(config.get(u"ckan.devserver.threaded"))
processes = processes or tk.asint(
config.get(u"ckan.devserver.multiprocess", 1)
)
if threaded and processes > 1:
tk.error_shout(u"Cannot have a multithreaded and multi process server")
raise click.Abort()
# SSL
cert_file = ssl_cert or config.get('ckan.devserver.ssl_cert')
key_file = ssl_key or config.get('ckan.devserver.ssl_key')
if cert_file and key_file:
if cert_file == key_file == 'adhoc':
ssl_context = 'adhoc'
else:
ssl_context = (ssl_cert, ssl_key)
else:
ssl_context = None
host = host or config.get('ckan.devserver.host', DEFAULT_HOST)
port = port or config.get('ckan.devserver.port', DEFAULT_PORT)
try:
port = int(port)
except ValueError:
tk.error_shout(u"Server port must be an integer, not {}".format(port))
raise click.Abort()
log.info(u"Running CKAN on {scheme}://{host}:{port}".format(
scheme='https' if ssl_context else 'http', host=host, port=port))
run_simple(
host,
port,
ctx.obj.app,
use_reloader=use_reloader,
use_evalex=True,
threaded=threaded,
processes=processes,
extra_files=extra_files,
ssl_context=ssl_context,
)
def issues_enabled_for_organization(organization):
'''Returns whether issues are enabled for the given organization (dict)'''
organizations_with_issues_enabled = set(
toolkit.aslist(config.get('ckanext.issues.enabled_for_organizations')))
if organizations_with_issues_enabled:
return organization and \
organization.get('name') in organizations_with_issues_enabled
return True
def issues_enabled_for_organization(organization):
'''Returns whether issues are enabled for the given organization (dict)'''
organizations_with_issues_enabled = set(toolkit.aslist(
config.get('ckanext.issues.enabled_for_organizations')
))
if organizations_with_issues_enabled:
return organization and \
organization.get('name') in organizations_with_issues_enabled
return True
def hierarchy_enable_tree_search():
if not tk.request:
return False
endpoint = ".".join(tk.get_endpoint())
supported_endpoints = tk.aslist(
tk.config.get(
CONFIG_TREE_SEARCH_EDNPOINTS, DEFAULT_TREE_SEARCH_ENDPOINTS
)
)
return endpoint in supported_endpoints
def select_indexable_resources(resources):
"""
Filter out resources with unsupported formats
Returns a list of resources dicts
"""
supported = tk.aslist(
tk.config.get("ckanext.resource_indexer.indexable_formats"))
return [
res for res in resources if res.get("format", "").lower() in supported
]
def setup_template_variables(self, context, data_dict):
layers = toolkit.aslist(config.get('ckanext.mvt.mapbox.style', ''))
labels = toolkit.aslist(config.get('ckanext.mvt.mapbox.style_label', ''))
cache = lib.CacheHandler(MvtPlugin.TEMPDIR)
if len(labels) != len(layers):
log.warn('The config file contains {0} layer id\'s and {1} labels. The extension needs an equal amount and can only use the first {2}'.format(len(labels), len(layers), min(len(layers),len(labels))))
baseLayers = [];
for i in range(min(len(layers),len(labels))):
baseLayers.append({"id": layers[i], "label": labels[i]})
return {
'tilejson': data_dict['resource'].get('tilejson', ''),
'mapbox': {
'account': config.get('ckanext.mvt.mapbox.account', ''),
'key': config.get('ckanext.mvt.mapbox.key', ''),
'defaultStyle': toolkit.aslist(config.get('ckanext.mvt.mapbox.style', ''))[0],
'styles': baseLayers
},
'resource_job_status': cache.get_job_status(data_dict['resource'].get('id'))
}
def configure(self, config_):
# Setup database tables
db_setup()
# Load and parse user attributes mapping
user_mapping = t.aslist(config_.get('ckanext.cas.user_mapping'))
for attr in user_mapping:
key, val = attr.split('~')
if '+' in val:
val = val.split('+')
self.USER_ATTR_MAP.update({key: val})
if not any(self.USER_ATTR_MAP):
raise RuntimeError, 'User attribute mapping is required for plugin: {0}'.format(self.name)
if 'email' not in self.USER_ATTR_MAP.keys():
raise RuntimeError, '"email" attribute mapping is required for plugin: {0}'.format(self.name)
service_validation_url = config.get('ckanext.cas.service_validation_url', None)
saml_validation_url = config.get('ckanext.cas.saml_validation_url', None)
if (service_validation_url and saml_validation_url) or \
(not service_validation_url and not saml_validation_url):
msg = 'Configure either "ckanext.cas.service_validation_url" or "ckanext.cas.saml_validation_url" but not both.'
raise RuntimeError, msg
if not config.get('ckanext.cas.login_url', None):
raise RuntimeError, '"ckanext.cas.login_url" is required for plugin: {0}'.format(self.name)
if not config.get('ckanext.cas.logout_url', None):
raise RuntimeError, '"ckanext.cas.logout_url" is required for plugin: {0}'.format(self.name)
if not config.get('ckanext.cas.application_url', None):
raise RuntimeError, '"ckanext.cas.application_url" is required for plugin: {0}'.format(self.name)
if service_validation_url:
self.SERVICE_VALIDATION_URL = config.get('ckanext.cas.service_validation_url')
self.CAS_VERSION = 2
elif saml_validation_url:
self.SAML_VALIDATION_URL = config.get('ckanext.cas.saml_validation_url')
self.CAS_VERSION = 3
self.CAS_LOGIN_URL = config.get('ckanext.cas.login_url')
self.CAS_LOGOUT_URL = config.get('ckanext.cas.logout_url')
self.CAS_APP_URL = config.get('ckanext.cas.application_url')
self.CAS_COOKIE_NAME = config.get('ckanext.cas.cookie_name', 'sessionid')
self.TICKET_KEY = config.get('ckanext.cas.ticket_key', 'ticket')
self.SERVICE_KEY = config.get('ckanext.cas.service_key', 'service')
self.REDIRECT_ON_UNSUCCESSFUL_LOGIN = config.get('ckanext.cas.unsuccessful_login_redirect_url', None)
self.LOGIN_CHECKUP_TIME = t.asint(config.get('ckanext.cas.login_checkup_time', 600))
self.LOGIN_CHECKUP_COOKIE = config.get('ckanext.cas.login_checkup_cookie', 'cas_login_check')
self.VERIFY_CERTIFICATE = t.asbool(config.get('ckanext.cas.verify_certificate', True))
def get_helpers(self):
return {
'str_to_dict':
str_to_dict,
'resource_excluded_fields':
lambda: toolkit.aslist(
config.get('ckanext.edsmetadata.resource_excluded_fields',
'attributes filters format ')),
'split_string':
lambda text, delim: text.split(delim),
'markdown_fields':
_h.markdown_fields
}
def get_microservice_metadata():
for config_option in ('ckan.spatialingestor.postgis_url', 'ckan.spatialingestor.internal_geoserver_url',):
if not config.get(config_option):
raise Exception(
'Config option `{0}` must be set to use the SpatialIngestor.'.format(config_option))
core_url = config.get('ckan.site_url', 'http://localhost:8000/')
return {'postgis': cli.parse_db_config('ckan.spatialingestor.postgis_url'),
'geoserver': cli.parse_db_config('ckan.spatialingestor.internal_geoserver_url'),
'geoserver_public_url': config.get('ckan.spatialingestor.public_geoserver_url',
core_url + '/geoserver'),
'target_spatial_formats': list(set([x.upper() for x in toolkit.aslist(config.get('ckan.spatialingestor.target_formats', []))]))
}
def configure(self, config_):
results.create_database_table()
# Update the class variables for the config settings with the values
# from the config file, *if* they're in the config file.
config.recheck_resources_after = toolkit.asint(
config_.get("ckanext.deadoralive.recheck_resources_after",
config.recheck_resources_after))
config.resend_pending_resources_after = toolkit.asint(
config_.get("ckanext.deadoralive.resend_pending_resources_after",
config.resend_pending_resources_after))
config.broken_resource_min_fails = toolkit.asint(
config_.get("ckanext.deadoralive.broken_resource_min_fails",
config.broken_resource_min_fails))
config.broken_resource_min_hours = toolkit.asint(
config_.get("ckanext.deadoralive.broken_resource_min_hours",
config.broken_resource_min_hours))
config.authorized_users = toolkit.aslist(
config_.get("ckanext.deadoralive.authorized_users",
config.authorized_users))
config.organization_to_filter = toolkit.aslist(
config_.get("ckanext.deadoralive.organization_to_filter",
config.organization_to_filter))
def configure(self, config):
formats = toolkit.aslist(
config.get('ckanext.datastorer.formats', interesting_formats))
self.interesting_formats = set(interesting_formats) & set(formats)
logger.info('Interested in resource formats: %s' %(
', '.join(self.interesting_formats)))
try:
sample_size = int(config.get('ckanext.datastorer.sample_size'))
except:
self.sample_size = None
else:
self.sample_size = sample_size
def dataset_facets(self, facets_dict, package_type):
# 'Stikkord' and 'Formater' in Norwegian.
default_included_facets = ['tags', 'res_format']
# Gets the facets from config properties, if property is defined. Otherwise sets to default values.
included_facets = toolkit.aslist(
config.get('tolltheme.dataset.filters', default_included_facets))
if '*' in included_facets:
return facets_dict
facets = OrderedDict()
for key, value in facets_dict.items():
if key in included_facets or value in included_facets:
facets[key] = value
return facets
def configure(self, config):
formats = toolkit.aslist(
config.get('ckanext.datastorer.formats', interesting_formats))
self.interesting_formats = set(interesting_formats) & set(formats)
logger.info('Interested in resource formats: %s' %
(', '.join(self.interesting_formats)))
try:
sample_size = int(config.get('ckanext.datastorer.sample_size'))
except:
self.sample_size = None
else:
self.sample_size = sample_size
def issues_enabled(dataset):
datasets_with_issues_enabled = set(
toolkit.aslist(config.get('ckanext.issues.enabled_for_datasets')))
# if the config option 'ckanext.issues.enabled_for_dataset' is
# set with a list of datasets, only enabled for the listed datasets
if datasets_with_issues_enabled:
if dataset['name'] in datasets_with_issues_enabled:
return True
else:
extras = dataset.get('extras')
if extras is not None:
for extra in extras:
if extra.get('key') == 'issues_enabled':
return toolkit.asbool(extra.get('value'))
else:
return toolkit.asbool(
config.get('ckanext.issues.enabled_per_dataset_default', True))
def __call__(self, environ, start_response):
# if logged in via browser cookies all pages accessible
if 'repoze.who.identity' not in environ:
# If api request check for api key
if environ['PATH_INFO'].startswith('/api/action/') \
or environ['PATH_INFO'].startswith('/api/util/') \
or environ['PATH_INFO'].startswith('/api/i18n/'):
# All API requests require authenticated api keys
if not self._get_user_for_apikey(environ):
log.debug(f"Unauthorized api accessed: {environ['PATH_INFO']}")
return_dict = {}
return_dict[u'error'] = {u'__type': u'Authorization Error',
u'message': toolkit._(u'Access denied')}
return_dict[u'success'] = False
response_msg = json.dumps(return_dict, for_json=True).encode('utf8')
status = '403 Unauthorized'
headers = [('Content-type', 'application/json;charset=utf-8')]
start_response(status, headers)
# Return now as we want to end the request
return [response_msg]
else:
# The only pages unauthorized users have access to are below
# service login, saml2login, acs (SAML Assertion Consumer Service), user unauthorised and logout pages
# But still allow unauthorized access to assets
unauthorized_pages_allowed = toolkit.aslist(toolkit.config.get('ckanext.qdes_access.unauthorized_pages_allowed', ''))
if environ['PATH_INFO'] not in unauthorized_pages_allowed \
and not environ['PATH_INFO'].startswith('/base') \
and not environ['PATH_INFO'].startswith('/webassets') \
and not environ['PATH_INFO'].startswith('/images') \
and not environ['PATH_INFO'].startswith('/css') \
and not environ['PATH_INFO'].startswith('/js') \
and not environ['PATH_INFO'].startswith('/_debug') \
and not environ['PATH_INFO'].startswith('/uploads'):
log.debug(f"Unauthorized page accessed: {environ['PATH_INFO']}")
# Status code needs to be 3xx (redirection) for Location header to be used
status = "302 Unauthorized"
location = toolkit.config.get('ckanext.qdes_access.unauthorized_redirect_location', '/user/saml2login')
headers = [('Location', location),
('Content-Length', '0')]
log.debug(f"Redirecting to: {location}")
start_response(status, headers)
# Return now as we want to end the request
return []
return self.app(environ, start_response)
def issues_enabled(dataset):
datasets_with_issues_enabled = set(toolkit.aslist(
config.get('ckanext.issues.enabled_for_datasets')
))
# if the config option 'ckanext.issues.enabled_for_dataset' is
# set with a list of datasets, only enabled for the listed datasets
if datasets_with_issues_enabled:
if dataset['name'] in datasets_with_issues_enabled:
return True
else:
extras = dataset.get('extras')
if extras is not None:
for extra in extras:
if extra.get('key') == 'issues_enabled':
return toolkit.asbool(extra.get('value'))
else:
return toolkit.asbool(config.get('ckanext.issues.enabled_per_dataset_default', True))
def run(ctx, host, port, disable_reloader, threaded, extra_files, processes,
ssl_cert, ssl_key):
u"""Runs the Werkzeug development server"""
# Reloading
use_reloader = not disable_reloader
config_extra_files = tk.aslist(
config.get(u"ckan.devserver.watch_patterns"))
extra_files = list(extra_files) + [config[u"__file__"]
] + config_extra_files
# Threads and processes
threaded = threaded or tk.asbool(config.get(u"ckan.devserver.threaded"))
processes = processes or tk.asint(
config.get(u"ckan.devserver.multiprocess", 1))
if threaded and processes > 1:
tk.error_shout(u"Cannot have a multithreaded and multi process server")
raise click.Abort()
# SSL
cert_file = ssl_cert or config.get('ckan.devserver.ssl_cert')
key_file = ssl_key or config.get('ckan.devserver.ssl_key')
if cert_file and key_file:
if cert_file == key_file == 'adhoc':
ssl_context = 'adhoc'
else:
ssl_context = (ssl_cert, ssl_key)
else:
ssl_context = None
log.info(u"Running CKAN on {scheme}://{host}:{port}".format(
scheme='https' if ssl_context else 'http', host=host, port=port))
run_simple(
host,
port,
ctx.obj.app,
use_reloader=use_reloader,
use_evalex=True,
threaded=threaded,
processes=processes,
extra_files=extra_files,
ssl_context=ssl_context,
)
def is_resource_updatable(id, package_id=None):
if package_id:
pkg = model.Package.get(id)
else:
pkg = model.Resource.get(id).package
org = pkg.get_groups('organization')[0]
org_names = set(
map(attrgetter('name'), org.get_parent_groups('organization')))
include_owner = tk.asbool(
tk.config.get('ckanext.spc.orgs_with_publications_include_root'))
if include_owner:
org_names.add(org.name)
restricted_orgs = set(
tk.aslist(tk.config.get('ckanext.spc.orgs_with_publications')))
if org_names & restricted_orgs:
return pkg.private
return True
def get_user_details(sid: str) -> Optional[Details]:
"""Fetch user data from Drupal's database."""
import ckanext.drupal_idp.drupal as drupal
adapter = drupal.get_adapter(
tk.config.get(CONFIG_DRUPAL_VERSION, DEFAULT_DRUPAL_VERSION)
)
user = adapter.get_user_by_sid(sid)
# check if session has username,
# otherwise is unauthenticated user session
if not user:
return
details_data = DetailsData(**user)
roles = adapter.get_user_roles(user.id)
details_data["avatar"] = adapter.get_avatar(user.id)
details_data["roles"] = roles
extra_fields = tk.aslist(tk.config.get(CONFIG_EXTRA_FIELDS, DEFAULT_EXTRA_FIELDS))
details_data["fields"] = adapter.get_fields(user.id, extra_fields)
return Details(**details_data)
def upload_to_ckan(package_id, filename):
try:
resource = registry.action.resource_create(package_id=package_id,
upload=open(filename, 'rb'))
email_notification_recipients = t.aslist(
t.config.get('ckanext.prh_tools.mail_recipients', ''))
site_title = t.config.get('ckan.site_title', '')
today = datetime.now().date().isoformat()
msg = '%(site_title)s - PRH data uploaded %(today)s\n\n%(status)s' % {
'site_title':
site_title,
'today':
today,
'status':
"New data available in https://www.avoindata.fi/data/dataset/%s/resource/%s"
% (package_id, resource.get('id'))
}
for recipient in email_notification_recipients:
email = {
'recipient_name': '',
'recipient_email': recipient,
'subject': '%s - PRH data uploaded %s' % (site_title, today),
'body': msg
}
try:
mailer.mail_recipient(**email)
except mailer.MailerException as e:
print 'Sending prh data notification to %s failed: %s' % (
recipient, e)
except ValidationError as e:
print("Resource patch failed: %s" % e.error_summary)
def configure(self, config_):
results.create_database_table()
# Update the class variables for the config settings with the values
# from the config file, *if* they're in the config file.
config.recheck_resources_after = toolkit.asint(config_.get(
"ckanext.deadoralive.recheck_resources_after",
config.recheck_resources_after))
config.resend_pending_resources_after = toolkit.asint(
config_.get(
"ckanext.deadoralive.resend_pending_resources_after",
config.resend_pending_resources_after))
config.broken_resource_min_fails = toolkit.asint(
config_.get(
"ckanext.deadoralive.broken_resource_min_fails",
config.broken_resource_min_fails))
config.broken_resource_min_hours = toolkit.asint(
config_.get(
"ckanext.deadoralive.broken_resource_min_hours",
config.broken_resource_min_hours))
config.authorized_users = toolkit.aslist(
config_.get(
"ckanext.deadoralive.authorized_users",
config.authorized_users))
def dataguide_available_locales():
return toolkit.aslist(
config.get('ckanext.eds.available_locales',
'en da_DK')
)
from ckanext.publicamundi.lib import uploader
from ckanext.publicamundi.lib import vocabularies
log = logging.getLogger(__name__)
_ = toolkit._
_url = toolkit.url_for
_get_action = toolkit.get_action
_check_access = toolkit.check_access
content_types = {
'json': 'application/json; charset=utf8',
'xml': 'application/xml; charset=utf8',
}
resource_formats = toolkit.aslist(
config.get('ckanext.publicamundi.resource_formats'))
class Controller(BaseController):
'''Publicamundi-specific api actions'''
@staticmethod
def _make_context(**opts):
ctx = {'model': model, 'session': model.Session, 'api_version': 3}
if opts:
ctx.update(opts)
return ctx
#
# Autocomplete helpers
#
def _save_new_pending(self, context):
errors = {}
error_summary = {}
params = request.params
password = str(binascii.b2a_hex(os.urandom(15)))
data = dict(
fullname=params['fullname'],
name=params['name'],
password1=password,
password2=password,
state=model.State.PENDING,
email=params['email'],
organization_request=params['organization-for-request'],
reason_to_access=params['reason-to-access'],
role=params['role']
)
try:
# captcha.check_recaptcha(request)
user_dict = logic.get_action('user_create')(context, data)
if params['organization-for-request']:
organization = model.Group.get(data['organization_request'])
sys_admin = model.Session.query(model.User).filter(
sqlalchemy.and_(
model.User.sysadmin == True, # noqa
model.User.state == 'active'
)
).first().name
logic.get_action('organization_member_create')({
"user": sys_admin
}, {
"id": organization.id,
"username": user_dict['name'],
"role": data['role'] if data['role'] else 'member'
})
role = data['role'].title() if data['role'] else 'Member'
else:
organization = None
msg = (
"A request for a new user account has been submitted:"
"\nUsername: {}"
"\nName: {}\nEmail: {}\nOrganisation: {}\nRole: {}"
"\nReason for access: {}"
"\nThis request can be approved or rejected at {}"
).format(
data['name'], data['fullname'], data['email'],
organization.display_name if organization else None,
role if organization else None, data['reason_to_access'],
g.site_url + h.url_for(
controller=(
'ckanext.accessrequests.controller'
':AccessRequestsController'
),
action='account_requests'
)
)
list_admin_emails = tk.aslist(
config.get('ckanext.accessrequests.approver_email')
)
for admin_email in list_admin_emails:
try:
mailer.mail_recipient(
'Admin', admin_email, 'Account request', msg
)
except mailer.MailerException as e:
h.flash(
"Email error: {0}".format(e.message), allow_html=False
)
h.flash_success(
'Your request for access to the {0} has been submitted.'.
format(config.get('ckan.site_title'))
)
except ValidationError as e:
# return validation failures to the form
if e.error_dict:
errors = e.error_dict
error_summary = e.error_summary
return self.request_account(data, errors, error_summary)
except CaptchaError:
errors['Captcha'] = [_(u'Bad Captcha. Please try again.')]
error_summary['Captcha'] = 'Bad Captcha. Please try again.'
return self.request_account(data, errors, error_summary)
h.redirect_to('/')
def account_requests_management(self):
''' Approve or reject an account request
'''
action = request.params['action']
user_id = request.params['id']
user_name = request.params['name']
user = model.User.get(user_id)
context = {
'model': model,
'user': c.user,
'session': model.Session,
}
activity_create_context = {
'model': model,
'user': user_name,
'defer_commit': True,
'ignore_auth': True,
'session': model.Session
}
activity_dict = {'user_id': c.userobj.id, 'object_id': user_id}
list_admin_emails = tk.aslist(
config.get('ckanext.accessrequests.approver_email')
)
if action == 'forbid':
object_id_validators['reject new user'] = user_id_exists
activity_dict['activity_type'] = 'reject new user'
logic.get_action('activity_create')(
activity_create_context, activity_dict
)
org = logic.get_action('organization_list_for_user')({
'user': user_name
}, {
"permission": "read"
})
if org:
logic.get_action('organization_member_delete')(
context, {
"id": org[0]['id'],
"username": user_name
}
)
logic.get_action('user_delete')(context, {'id': user_id})
msg = (
"Your account request for {0} has been rejected by {1}"
"\n\nFor further clarification "
"as to why your request has been "
"rejected please contact the NSW Flood Data Portal ({2})"
)
mailer.mail_recipient(
user.fullname, user.email, 'Account request',
msg.format(
config.get('ckan.site_title'), c.userobj.fullname,
c.userobj.email
)
)
msg = ("User account request for {0} "
"has been rejected by {1}"
).format(user.fullname or user_name, c.userobj.fullname)
for admin_email in list_admin_emails:
try:
mailer.mail_recipient(
'Admin', admin_email, 'Account request feedback', msg
)
except mailer.MailerException as e:
h.flash(
"Email error: {0}".format(e.message), allow_html=False
)
elif action == 'approve':
user_org = request.params['org']
user_role = request.params['role']
object_id_validators['approve new user'] = user_id_exists
activity_dict['activity_type'] = 'approve new user'
logic.get_action('activity_create')(
activity_create_context, activity_dict
)
org_display_name, org_role = assign_user_to_org(
user_id, user_org, user_role, context
)
# Send invitation to complete registration
msg = (
"User account request for {0} "
"(Organization : {1}, Role: {2}) "
"has been approved by {3}"
).format(
user.fullname or user_name, org_display_name, org_role,
c.userobj.fullname
)
for admin_email in list_admin_emails:
try:
mailer.mail_recipient(
'Admin', admin_email, 'Account request feedback', msg
)
except mailer.MailerException as e:
h.flash(
"Email error: {0}".format(e.message), allow_html=False
)
try:
org_dict = tk.get_action('organization_show')(context, {'id': user_org})
user.name = user.name
mailer.send_invite(user, org_dict, user_role)
except Exception as e:
log.error('Error emailing invite to user: %s', e)
abort(500, _('Error: couldn' 't email invite to user'))
response.status = 200
return render('user/account_requests_management.html')
from ckanext.publicamundi.lib import uploader
from ckanext.publicamundi.lib import vocabularies
log = logging.getLogger(__name__)
_ = toolkit._
_url = toolkit.url_for
_get_action = toolkit.get_action
_check_access = toolkit.check_access
content_types = {
'json': 'application/json; charset=utf8',
'xml': 'application/xml; charset=utf8',
}
resource_formats = toolkit.aslist(config.get('ckanext.publicamundi.resource_formats'))
class Controller(BaseController):
'''Publicamundi-specific api actions'''
@staticmethod
def _make_context(**opts):
ctx = {
'model': model,
'session': model.Session,
'api_version': 3
}
if opts:
ctx.update(opts)
return ctx
def send_harvester_status_emails(ctx, config, dryrun, force, all_harvesters):
load_config(config or ctx.obj['config'])
email_notification_recipients = t.aslist(
t.config.get('ckanext.ytp.harvester_status_recipients', ''))
if not email_notification_recipients and not dryrun:
print 'No recipients configured'
return
status_opts = {} if not all_harvesters else {
'include_manual': True,
'include_never_run': True
}
status = get_action('harvester_status')({}, status_opts)
errored_runs = any(item.get('errors') != 0 for item in status.values())
running = (item.get('started') for item in status.values()
if item.get('status') == 'running')
stuck_runs = any(_elapsed_since(started).days > 1 for started in running)
if not (errored_runs or stuck_runs) and not force:
print 'Nothing to report'
return
if len(status) == 0:
print 'No harvesters matching criteria found'
return
site_title = t.config.get('ckan.site_title', '')
today = datetime.now().date().isoformat()
status_templates = {
'running':
'%%(title)-%ds | Running since %%(time)s with %%(errors)d errors',
'finished':
'%%(title)-%ds | Finished %%(time)s with %%(errors)d errors',
'pending': '%%(title)-%ds | Pending since %%(time)s'
}
unknown_status_template = '%%(title)-%ds | Unknown status: %%(status)s'
max_title_length = max(len(title) for title in status)
def status_string(title, values):
template = status_templates.get(values.get('status'),
unknown_status_template)
status = values.get('status')
time_field = 'finished' if status == 'finished' else 'started'
return template % max_title_length % {
'title': title,
'time': _pretty_time(values.get(time_field)),
'status': status,
'errors': values.get('errors')
}
msg = '%(site_title)s - Harvester summary %(today)s\n\n%(status)s' % {
'site_title':
site_title,
'today':
today,
'status':
'\n'.join(
status_string(title, values) for title, values in status.items())
}
for recipient in email_notification_recipients:
email = {
'recipient_name': recipient,
'recipient_email': recipient,
'subject': '%s - Harvester summary %s' % (site_title, today),
'body': msg
}
if dryrun:
print 'to: %s' % recipient
else:
try:
mailer.mail_recipient(**email)
except mailer.MailerException as e:
print 'Sending harvester summary to %s failed: %s' % (
recipient, e)
if dryrun:
print msg
