def read(self, id, format='html'):
# Check we know the content type, if not then it is likely a revision
# and therefore we should merge the format onto the end of id
ctype,extension,loader = self._content_type_for_format(format)
if not ctype:
# Reconstitute the ID if we don't know what content type to use
ctype = "text/html; charset=utf-8"
id = "%s.%s" % (id, format)
format = 'html'
else:
format = extension
response.headers['Content-Type'] = ctype
package_type = self._get_package_type(id.split('@')[0])
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'for_view': True}
data_dict = {'id': id}
# interpret @<revision_id> or @<date> suffix
split = id.split('@')
if len(split) == 2:
data_dict['id'], revision_ref = split
if model.is_id(revision_ref):
context['revision_id'] = revision_ref
else:
try:
date = date_str_to_datetime(revision_ref)
context['revision_date'] = date
except TypeError, e:
abort(400, _('Invalid revision format: %r') % e.args)
except ValueError, e:
abort(400, _('Invalid revision format: %r') % e.args)
def __before__(self, action, **params):
super(AdminController, self).__before__(action, **params)
context = {'model': model,
'user': c.user}
if not ckan.new_authz.is_authorized('sysadmin', context, {})['success']:
base.abort(401, _('Need to be system administrator to administer'))
c.revision_change_state_allowed = True
def read(self, id):
group_type = self._get_group_type(id.split('@')[0])
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': self._form_to_db_schema(group_type=type)}
data_dict = {'id': id}
q = c.q = request.params.get('q', '') # unicode format (decoded from utf8)
try:
c.group_dict = get_action('group_show')(context, data_dict)
c.group = context['group']
except NotFound:
abort(404, _('Group not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read group %s') % id)
# Search within group
q += ' groups: "%s"' % c.group_dict.get('name')
try:
description_formatted = ckan.misc.MarkdownFormat().to_html(c.group_dict.get('description',''))
c.description_formatted = genshi.HTML(description_formatted)
except Exception, e:
error_msg = "<span class='inline-warning'>%s</span>" % _("Cannot render description")
c.description_formatted = genshi.HTML(error_msg)
def revisions(self):
'''
Similar to the revision search API, lists all revisions for which
a dataset or group changed in some way.
URL Params:
since-revision-id
since-timestamp (utc)
in-the-last-x-minutes
'''
# parse options
rev_id = request.params.get('since-revision-id')
since_timestamp = request.params.get('since-timestamp')
in_the_last_x_minutes = request.params.get('in-the-last-x-minutes')
now = datetime.datetime.utcnow()
if rev_id is not None:
rev = model.Session.query(model.Revision).get(rev_id)
if not rev:
abort(400, 'Revision ID "%s" does not exist' % rev_id)
since_timestamp = rev.timestamp
elif since_timestamp is not None:
try:
since_timestamp = date_str_to_datetime(since_timestamp)
except (ValueError, TypeError), inst:
example = now.strftime('%Y-%m-%d%%20%H:%M') # e.g. 2013-11-30%2023:15
abort(400, 'Could not parse timestamp "%s": %s. Must be UTC. Example: since-time=%s' % (since_timestamp, inst, example))
def history_ajax(self, id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'extras_as_string': True,}
data_dict = {'id':id}
try:
pkg_revisions = get_action('package_revision_list')(context, data_dict)
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound:
abort(404, _('Dataset not found'))
data = []
approved = False
for num, revision in enumerate(pkg_revisions):
if not approved and revision['approved_timestamp']:
current_approved, approved = True, True
else:
current_approved = False
data.append({'revision_id': revision['id'],
'message': revision['message'],
'timestamp': revision['timestamp'],
'author': revision['author'],
'approved': bool(revision['approved_timestamp']),
'current_approved': current_approved})
response.headers['Content-Type'] = 'application/json;charset=utf-8'
return json.dumps(data)
def new(self,data = None,errors = None, error_summary = None):
context = {'model': model, 'user': c.user or c.author,
'auth_user_obj': c.userobj}
try:
p.toolkit.check_access('harvest_source_create', context)
except p.toolkit.NotAuthorized:
abort(401, _('Unauthorized to create a harvest source'))
if ('save' in request.params) and not data:
return self._save_new()
# #1433 URL params pre-populate fields
param_data = {}
for field_name in ('url', 'type', 'title', 'description', 'publisher_id'):
if field_name in request.params:
param_data[field_name] = request.params[field_name]
data = data or param_data or {}
errors = errors or {}
error_summary = error_summary or {}
try:
harvesters_info = p.toolkit.get_action('harvesters_info_show')(context,{})
except p.toolkit.NotAuthorized,e:
abort(401,self.not_auth_message)
def new(self, data=None, errors=None, error_summary=None):
group_type = self._guess_group_type(True)
if data:
data['type'] = group_type
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'extras_as_string': True,
'save': 'save' in request.params,
'parent': request.params.get('parent', None)}
try:
check_access('group_create', context)
except NotAuthorized:
abort(401, _('Unauthorized to create a group'))
if context['save'] and not data:
return self._save_new(context, group_type)
data = data or {}
errors = errors or {}
error_summary = error_summary or {}
vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
self._setup_template_variables(context, data, group_type=group_type)
c.form = render(self._group_form(group_type=group_type),
extra_vars=vars)
return render(self._new_template(group_type))
def register(self, data=None, errors=None, error_summary=None):
'''GET to display a form for registering a new user.
or POST the form data to actually do the user registration.
The bulk of this code is pulled directly from ckan/controlllers/user.py
'''
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': schema.user_new_form_schema(),
'save': 'save' in request.params}
try:
check_access('user_create', context)
except NotAuthorized:
abort(401, _('Unauthorized to create a user'))
if context['save'] and not data:
uc = UserController()
return uc._save_new(context)
if c.user and not data:
# #1799 Don't offer the registration form if already logged in
return render('user/logout_first.html')
data = data or {}
errors = errors or {}
error_summary = error_summary or {}
vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
c.is_sysadmin = new_authz.is_sysadmin(c.user)
c.form = render('user/new_user_form.html', extra_vars=vars)
return render('user/new.html')
def organization_index(self):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'with_private': False}
data_dict = {'all_fields': True}
try:
check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
results = get_action('organization_list')(context, data_dict)
def org_key(org):
title = org['title'].split(' | ')[-1 if c.language == 'fr' else 0]
return normalize_strip_accents(title)
results.sort(key=org_key)
c.page = Page(
collection=results,
page=request.params.get('page', 1),
url=h.pager_url,
items_per_page=1000
)
return render('organization/index.html')
def index(self):
context = {'model':model, 'user':c.user,'session':model.Session}
try:
# Request all harvest sources
c.sources = get_action('harvest_source_list')(context,{})
except NotAuthorized,e:
abort(401,self.not_auth_message)
def view(self, url):
"""By default, the final controller tried to fulfill the request
when no other routes match. It may be used to display a template
when all else fails, e.g.::
def view(self, url):
return render('/%s' % url)
Or if you're using Mako and want to explicitly send a 404 (Not
Found) response code when the requested template doesn't exist::
import mako.exceptions
def view(self, url):
try:
return render('/%s' % url)
except mako.exceptions.TopLevelLookupException:
abort(404)
By default this controller aborts the request with a 404 (Not
Found)
"""
try:
return base.render(url)
except ckan.lib.render.TemplateNotFound:
if url.endswith('.html'):
base.abort(404)
url += '.html'
try:
return base.render(url)
except ckan.lib.render.TemplateNotFound:
base.abort(404)
def new (self, data=None, errors=None, error_summary=None):
#q = model.Session.query(model.User).filter(model.User.sysadmin==True)
#c.sysadmins = [a.name for a in q.all()]
'''GET to display a form for registering a new user.
or POST the form data to actually do the user registration.
'''
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj,
'schema': self._new_form_to_db_schema(),
'save': 'save' in request.params}
c.is_sysadmin = new_authz.is_sysadmin(c.user)
if not c.user or not c.is_sysadmin:
return base.render('user/logout_first.html')
try:
logic.check_access('user_create', context)
except logic.NotAuthorized:
base.abort(401, _('Unauthorized to create a user'))
if context['save'] and not data:
return self._save_new(context)
c.data = data or {}
c.errors = errors or {}
c.error_summary = error_summary or {}
#vars = {'data': data, 'errors': errors, 'error_summary': error_summary}
#c.form = render(self.new_user_form, extra_vars=vars)
#return render('user/new.html')
return base.render('muser/new.html')
def index (self):
LIMIT = 20
page = int(request.params.get('page', 1))
c.q = request.params.get('q', '')
c.order_by = request.params.get('order_by', 'name')
context = {'return_query': True, 'user': c.user or c.author,
'auth_user_obj': c.userobj}
data_dict = {'q': c.q,
'limit': LIMIT,
'offset': (page - 1) * LIMIT,
'order_by': c.order_by}
try:
logic.check_access('user_list', context, data_dict)
except logic.NotAuthorized:
base.abort(401, _('Not authorized to see this page'))
users_list = logic.get_action('user_list')(context, data_dict)
c.users = users_list
c.page = h.Page(
collection=users_list,
page=page,
url=h.pager_url,
item_count=users_list.count(),
items_per_page=LIMIT
)
return base.render('muser/index.html')
def serve(self, path):
root = os.path.join(config.get('ckanext-archiver.archive_dir', '/tmp'),
path)
if not os.path.exists(root):
abort(404)
response.content_type = 'application/json'
return str(open(root).read())
def dataset_csv(self, id='all', month='all'):
'''
Returns a CSV with the number of views & visits for each dataset.
:param id: A Publisher ID or None if you want for all
:param month: The time period, or 'all'
'''
c.month = month if not month == 'all' else ''
if id != 'all':
c.publisher = model.Group.get(id)
if not c.publisher:
abort(404, 'A publisher with that name could not be found')
packages = self._get_packages(publisher=c.publisher, month=c.month)
response.headers['Content-Type'] = "text/csv; charset=utf-8"
response.headers['Content-Disposition'] = \
str('attachment; filename=datasets_%s_%s.csv' % (c.publisher_name, month,))
writer = csv.writer(response)
writer.writerow(["Dataset Title", "Dataset Name", "Views", "Visits", "Resource downloads", "Period Name"])
for package,view,visit,downloads in packages:
writer.writerow([package.title.encode('utf-8'),
package.name.encode('utf-8'),
view,
visit,
downloads,
month])
def contracts_archive(self, relative_url='/'):
import requests
from pylons import response
if request.method == 'POST':
abort(405) # Method Not Allowed
headers = {'X-Script-Name': '/data/contracts-finder-archive'}
contracts_url = pylons.config.get('dgu.contracts_url')
url = urlparse.urljoin(contracts_url, relative_url)
r = requests.get(url,
headers=headers,
params=dict(request.params),
stream=True)
if r.status_code != 200:
abort(r.status_code)
if relative_url.startswith(('/static/', '/download/')):
# CSS will only get loaded if it has the right content type
response.content_type = r.headers.get('Content-Type', 'text/html')
if r.headers.get('Content-Disposition'):
response.headers['Content-Disposition'] = r.headers.get('Content-Disposition')
return r.raw.read() # Some of the static files are binary
else:
extra_vars = {'content': r.text}
return render('contracts_archive/front_page.html',
extra_vars=extra_vars)
def _parse_url_params(self):
"""
Constructs a search-query dict from the URL query parameters.
Returns the constructed search-query dict, and the valid URL
query parameters.
"""
try:
page = int(request.params.get('page', 1)) or 1
except ValueError:
base.abort(400, _('"page" parameter must be a positive integer'))
if page < 0:
base.abort(400, _('"page" parameter must be a positive integer'))
limit = ITEMS_LIMIT
data_dict = {
'start': (page - 1) * limit,
'rows': limit
}
# Filter ignored query parameters
valid_params = ['page']
params = dict((p, request.params.get(p)) for p in valid_params
if p in request.params)
return data_dict, params
def refresh(self, id):
try:
context = {'model': model, 'user': c.user, 'session': model.Session}
p.toolkit.get_action('harvest_job_create')(
context, {'source_id': id, 'run': True}
)
h.flash_success(
_('Harvest will start shortly. Refresh this page for updates.')
)
except p.toolkit.ObjectNotFound:
abort(404, _('Harvest source not found'))
except p.toolkit.NotAuthorized:
abort(401, self.not_auth_message)
except HarvestSourceInactiveError:
h.flash_error(
_(
'Cannot create new harvest jobs on inactive '
'sources. First, please change the source status '
'to "active".'
)
)
except HarvestJobExists:
h.flash_notice(
_('A harvest job has already been scheduled for ' 'this source')
)
except Exception as e:
msg = 'An error occurred: [%s]' % str(e)
h.flash_error(msg)
h.redirect_to(h.url_for('{0}_admin'.format(DATASET_TYPE_NAME), id=id))
def openspending_report(self):
if not dgu_helpers.is_sysadmin():
abort(401, 'User must be a sysadmin to view this page.')
self._set_openspending_reports_dir()
c.content = open (c.openspending_report_dir + "/index.html").read()
return render('data/openspending_report.html')
def comment_update_moderation(context, data_dict):
import ckanext.comments.model as comment_model
model = context['model']
user = context['user']
cid = logic.get_or_bust(data_dict, 'id')
comment = comment_model.Comment.get(cid)
if not comment:
abort(404)
# If sysadmin, then remove instantly
if c.userobj.sysadmin:
if len(comment.children) > 0:
txt = config.get('ckan.comments.deleted.text', 'This message was deleted')
comment.comment = txt
else:
comment.state = 'deleted'
comment.approval_status = comment_model.COMMENT_PENDING
comment.moderated_by = c.userobj.id
comment.moderation_date = datetime.datetime.now()
model.Session.add(comment)
model.Session.commit()
elif not comment.moderated_by:
comment.spam_votes = comment.spam_votes + 1
comment.approval_status = comment_model.COMMENT_PENDING
model.Session.add(comment)
model.Session.commit()
return {}
def __before__(self, action, **env):
base.BaseController.__before__(self, action, **env)
try:
context = {'model': model, 'user': c.user or c.author}
logic.check_access('site_read', context)
except logic.NotAuthorized:
base.abort(401, _('Not authorized to see this page'))
def member_new(self, id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
#self._check_access('group_delete', context, {'id': id})
try:
if request.method == 'POST':
data_dict = clean_dict(unflatten(
tuplize_dict(parse_params(request.params))))
data_dict['id'] = id
c.group_dict = self._action('group_member_create')(context, data_dict)
self._redirect_to(controller='group', action='members', id=id)
else:
user = request.params.get('user')
if user:
c.user_dict = get_action('user_show')(context, {'id': user})
c.user_role = ckan.new_authz.users_role_for_group_or_org(id, user) or 'member'
else:
c.user_role = 'member'
c.group_dict = self._action('group_show')(context, {'id': id})
c.roles = self._action('member_roles_list')(context, {})
except NotAuthorized:
abort(401, _('Unauthorized to add member to group %s') % '')
except NotFound:
abort(404, _('Group not found'))
return self._render_template('group/member_new.html')
def resource_read(self, id, resource_id):
context = {"model": model, "session": model.Session, "user": c.user or c.author}
try:
c.resource = get_action("resource_show")(context, {"id": resource_id})
c.package = get_action("package_show")(context, {"id": id})
# required for nav menu
c.pkg = context["package"]
c.resource_json = json.dumps(c.resource)
c.pkg_dict = c.package
except NotFound:
abort(404, _("Resource not found"))
except NotAuthorized:
abort(401, _("Unauthorized to read resource %s") % id)
# get package license info
license_id = c.package.get("license_id")
try:
c.package["isopen"] = model.Package.get_license_register()[license_id].isopen()
except KeyError:
c.package["isopen"] = False
# TODO: find a nicer way of doing this
c.datastore_api = "%s/api/action" % config.get("ckan.site_url", "").rstrip("/")
c.related_count = c.pkg.related_count
return render("package/resource_read.html")
def read_ajax(self, id, revision=None):
package_type = self._get_package_type(id)
context = {
"model": model,
"session": model.Session,
"user": c.user or c.author,
"extras_as_string": True,
"schema": self._form_to_db_schema(package_type=package_type),
"revision_id": revision,
}
try:
data = get_action("package_show")(context, {"id": id})
schema = self._db_to_form_schema(package_type=package_type)
if schema:
data, errors = validate(data, schema)
except NotAuthorized:
abort(401, _("Unauthorized to read package %s") % "")
except NotFound:
abort(404, _("Dataset not found"))
## hack as db_to_form schema should have this
data["tag_string"] = ", ".join([tag["name"] for tag in data.get("tags", [])])
data.pop("tags")
data = flatten_to_string_key(data)
response.headers["Content-Type"] = "application/json;charset=utf-8"
return json.dumps(data)
def history_ajax(self, id):
context = {"model": model, "session": model.Session, "user": c.user or c.author, "extras_as_string": True}
data_dict = {"id": id}
try:
pkg_revisions = get_action("package_revision_list")(context, data_dict)
except NotAuthorized:
abort(401, _("Unauthorized to read package %s") % "")
except NotFound:
abort(404, _("Dataset not found"))
data = []
approved = False
for num, revision in enumerate(pkg_revisions):
if not approved and revision["approved_timestamp"]:
current_approved, approved = True, True
else:
current_approved = False
data.append(
{
"revision_id": revision["id"],
"message": revision["message"],
"timestamp": revision["timestamp"],
"author": revision["author"],
"approved": bool(revision["approved_timestamp"]),
"current_approved": current_approved,
}
)
response.headers["Content-Type"] = "application/json;charset=utf-8"
return json.dumps(data)
def download(package_type, id, resource_id, filename=None):
"""
Provides a direct download by either redirecting the user to the url
stored or downloading an uploaded file directly.
"""
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'auth_user_obj': g.userobj
}
try:
rsc = get_action(u'resource_show')(context, {u'id': resource_id})
get_action(u'package_show')(context, {u'id': id})
except (NotFound, NotAuthorized):
return base.abort(404, _(u'Resource not found'))
if rsc.get(u'url_type') == u'upload':
upload = uploader.get_resource_uploader(rsc)
filepath = upload.get_path(rsc[u'id'])
return flask.send_file(filepath)
elif u'url' not in rsc:
return base.abort(404, _(u'No download is available'))
return h.redirect_to(rsc[u'url'])
def get(self, package_type, id, resource_id):
context = self._prepare(id)
try:
resource_dict = get_action(u'resource_show')(
context, {
u'id': resource_id
}
)
pkg_id = id
except NotAuthorized:
return base.abort(
403,
_(u'Unauthorized to delete resource %s') % u''
)
except NotFound:
return base.abort(404, _(u'Resource not found'))
# TODO: remove
g.resource_dict = resource_dict
g.pkg_id = pkg_id
return base.render(
u'package/confirm_delete_resource.html', {
u'dataset_type': _get_package_type(id),
u'resource_dict': resource_dict,
u'pkg_id': pkg_id
}
)
def index(self):
group_type = self._guess_group_type()
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'with_private': False}
q = c.q = request.params.get('q', '')
data_dict = {'all_fields': True,
'q': q,}
sort_by = c.sort_by_selected = request.params.get('sort')
if sort_by:
data_dict['sort'] = sort_by
try:
self._check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
# pass user info to context as needed to view private datasets of
# orgs correctly
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
results = self._action('group_list')(context, data_dict)
c.page = h.Page(
collection=results,
page=request.params.get('page', 1),
url=h.pager_url,
items_per_page=21
)
return render(self._index_template(group_type))
def comment_update(context, data_dict):
model = context['model']
logic.check_access("comment_update", context, data_dict)
cid = logic.get_or_bust(data_dict, 'id')
comment = comment_model.Comment.get(cid)
if not comment:
abort(404)
# Validate that we have the required fields.
if not all([data_dict.get('comment')]):
raise logic.ValidationError("Comment text is required")
# Cleanup the comment
cleaned_comment = util.clean_input(data_dict.get('comment'))
comment.subject = data_dict.get('subject')
comment.comment = cleaned_comment
comment.modified_date = datetime.datetime.now()
model.Session.add(comment)
model.Session.commit()
return comment.as_dict()
def render_request_form(self, pkg_id):
"""
Render the access request contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
if not c.package:
abort(404, _(u"Dataset not found"))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts]
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/dataset_request_form.html')
class YtpDatasetController(PackageController):
def ytp_tag_autocomplete(self):
""" CKAN autocomplete discards vocabulary_id from request.
This is modification from tag_autocomplete function from CKAN.
Takes vocabulary_id as parameter.
"""
q = request.params.get('incomplete', '')
limit = request.params.get('limit', 10)
vocabulary_id = request.params.get('vocabulary_id', None)
tag_names = []
if q:
context = {'model': model, 'session': model.Session, 'user': c.user or c.author}
data_dict = {'q': q, 'limit': limit}
if vocabulary_id:
data_dict['vocabulary_id'] = vocabulary_id
try:
tag_names = get_action('tag_autocomplete')(context, data_dict)
except NotFound:
pass # return empty when vocabulary is not found
resultSet = {
'ResultSet': {
'Result': [{'Name': tag} for tag in tag_names]
}
}
status_int = 200
response.status_int = status_int
response.headers['Content-Type'] = 'application/json;charset=utf-8'
return helpers.json.dumps(resultSet)
def new_metadata(self, id, data=None, errors=None, error_summary=None):
""" Fake metadata creation. Change status to active and redirect to read. """
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj}
data_dict = get_action('package_show')(context, {'id': id})
data_dict['id'] = id
data_dict['state'] = 'active'
context['allow_state_change'] = True
get_action('package_update')(context, data_dict)
success_message = ('<div style="display: inline-block"><p>' + _("Dataset was saved successfully.") + '</p>' +
'<p>' + _("Fill additional info") + ':</p>' +
'<p><a href="/data/' + h.lang() + '/dataset/' + data_dict.get('name') + '/related/new">>' + _("Add related") + '</a></p>' +
'<p><a href="/data/' + h.lang() + '/dataset/edit/' + data_dict.get('name') + '">>' + _("Edit or add language versions") + '</a> ' +
'<a href="/data/' + h.lang() + '/dataset/delete/' + id + '">>' + _('Delete') + '</a></p>' +
'<p><a href="/data/' + h.lang() + '/dataset/new/">' + _('Create Dataset') + '</a></p></div>')
helpers.flash_success(success_message, True)
helpers.redirect_to(controller='package', action='read', id=id)
# Modified from original ckan function
def edit(self, id, data=None, errors=None, error_summary=None):
package_type = self._get_package_type(id)
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj,
'save': 'save' in request.params}
if context['save'] and not data:
return self._save_edit(id, context, package_type=package_type)
try:
c.pkg_dict = get_action('package_show')(context, {'id': id})
context['for_edit'] = True
old_data = get_action('package_show')(context, {'id': id})
# old data is from the database and data is passed from the
# user if there is a validation error. Use users data if there.
if data:
old_data.update(data)
data = old_data
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound:
abort(404, _('Dataset not found'))
# are we doing a multiphase add?
if data.get('state', '').startswith('draft') and len(data.get('resources')) == 0:
c.form_action = h.url_for(controller='package', action='new')
c.form_style = 'new'
return self.new(data=data, errors=errors,
error_summary=error_summary)
c.pkg = context.get("package")
c.resources_json = h.json.dumps(data.get('resources', []))
try:
check_access('package_update', context)
except NotAuthorized:
abort(401, _('User %r not authorized to edit %s') % (c.user, id))
# convert tags if not supplied in data
if data and not data.get('tag_string'):
data['tag_string'] = ', '.join(h.dict_list_reduce(
c.pkg_dict.get('tags', {}), 'name'))
errors = errors or {}
form_snippet = self._package_form(package_type=package_type)
form_vars = {'data': data, 'errors': errors,
'error_summary': error_summary, 'action': 'edit',
'dataset_type': package_type,
}
c.errors_json = h.json.dumps(errors)
self._setup_template_variables(context, {'id': id},
package_type=package_type)
c.related_count = c.pkg.related_count
# we have already completed stage 1
form_vars['stage'] = ['active']
if data.get('state', '').startswith('draft') and len(data.get('resources')) == 0:
form_vars['stage'] = ['active', 'complete']
edit_template = self._edit_template(package_type)
c.form = ckan.lib.render.deprecated_lazy_render(
edit_template,
form_snippet,
lambda: render(form_snippet, extra_vars=form_vars),
'use of c.form is deprecated. please see '
'ckan/templates/package/edit.html for an example '
'of the new way to include the form snippet'
)
return render(edit_template,
extra_vars={'form_vars': form_vars,
'form_snippet': form_snippet,
'dataset_type': package_type})
# original ckan new resource
def new_resource(self, id, data=None, errors=None, error_summary=None):
''' FIXME: This is a temporary action to allow styling of the
forms. '''
if request.method == 'POST' and not data:
save_action = request.params.get('save')
data = data or clean_dict(dict_fns.unflatten(tuplize_dict(parse_params(
request.POST))))
# we don't want to include save as it is part of the form
del data['save']
resource_id = data['id']
del data['id']
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj}
# see if we have any data that we are trying to save
data_provided = False
for key, value in data.iteritems():
if (value or isinstance(value, cgi.FieldStorage)) and key != 'resource_type':
data_provided = True
break
if not data_provided and save_action != "go-dataset-complete":
if save_action == 'go-dataset':
# go to final stage of adddataset
h.redirect_to(controller='package',
action='edit', id=id)
# see if we have added any resources
try:
data_dict = get_action('package_show')(context, {'id': id})
except NotAuthorized:
abort(401, _('Unauthorized to update dataset'))
except NotFound:
abort(404,
_('The dataset {id} could not be found.').format(id=id))
if not len(data_dict['resources']):
# no data so keep on page
msg = _('You must add at least one data resource')
# On new templates do not use flash message
if g.legacy_templates:
h.flash_error(msg)
h.redirect_to(controller='package',
action='new_resource', id=id)
else:
errors = {}
error_summary = {_('Error'): msg}
return self.new_resource(id, data, errors, error_summary)
# we have a resource so let them add metadata
h.redirect_to(controller='package',
action='new_metadata', id=id)
data['package_id'] = id
try:
if resource_id:
data['id'] = resource_id
get_action('resource_update')(context, data)
else:
get_action('resource_create')(context, data)
except ValidationError, e:
errors = e.error_dict
error_summary = e.error_summary
return self.new_resource(id, data, errors, error_summary)
except NotAuthorized:
abort(401, _('Unauthorized to create a resource'))
except NotFound:
abort(404,
_('The dataset {id} could not be found.').format(id=id))
def before_request():
try:
context = dict(model=model, user=g.user, auth_user_obj=g.userobj)
logic.check_access(u'sysadmin', context)
except logic.NotAuthorized:
base.abort(403, _(u'Need to be system administrator to administer'))
def post(self, id, group_type, is_organization, data=None):
set_org(is_organization)
context = self._prepare(group_type, id)
data_dict = {u'id': id, u'type': group_type}
try:
# Do not query for the group datasets when dictizing, as they will
# be ignored and get requested on the controller anyway
data_dict['include_datasets'] = False
group_dict = _action(u'group_show')(context, data_dict)
group = context['group']
except NotFound:
group_label = h.humanize_entity_type(
u'organization' if is_organization else u'group',
group_type,
u'default label') or _(
u'Organization' if is_organization else u'Group')
base.abort(404, _(u'{} not found'.format(group_label)))
except NotAuthorized:
base.abort(403,
_(u'User %r not authorized to edit %s') % (g.user, id))
if not group_dict['is_organization']:
# FIXME: better error
raise Exception(u'Must be an organization')
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.group_dict = group_dict
g.group = group
# use different form names so that ie7 can be detected
form_names = set([
u"bulk_action.public",
u"bulk_action.delete",
u"bulk_action.private"
])
actions_in_form = set(request.form.keys())
actions = form_names.intersection(actions_in_form)
# ie7 puts all buttons in form params but puts submitted one twice
for key, value in request.form.to_dict().items():
if value in [u'private', u'public']:
action = key.split(u'.')[-1]
break
else:
# normal good browser form submission
action = actions.pop().split(u'.')[-1]
# process the action first find the datasets to perform the action on.
# they are prefixed by dataset_ in the form data
datasets = []
for param in request.form:
if param.startswith(u'dataset_'):
datasets.append(param[8:])
action_functions = {
u'private': u'bulk_update_private',
u'public': u'bulk_update_public',
u'delete': u'bulk_update_delete',
}
data_dict = {u'datasets': datasets, u'org_id': group_dict['id']}
try:
get_action(action_functions[action])(context, data_dict)
except NotAuthorized:
base.abort(403, _(u'Not authorized to perform bulk update'))
return h.redirect_to(u'{}.bulk_process'.format(group_type), id=id)
def changes_multiple(is_organization, group_type=None):
'''
Called when a user specifies a range of versions they want to look at
changes between. Verifies that the range is valid and finds the set of
activity diffs for the changes in the given version range, then
re-renders changes.html with the list.
'''
set_org(is_organization)
extra_vars = {}
new_id = h.get_request_param(u'new_id')
old_id = h.get_request_param(u'old_id')
context = {
u'model': model, u'session': model.Session,
u'user': g.user, u'auth_user_obj': g.userobj
}
# check to ensure that the old activity is actually older than
# the new activity
old_activity = get_action(u'activity_show')(context, {
u'id': old_id,
u'include_data': False})
new_activity = get_action(u'activity_show')(context, {
u'id': new_id,
u'include_data': False})
old_timestamp = old_activity[u'timestamp']
new_timestamp = new_activity[u'timestamp']
t1 = datetime.strptime(old_timestamp, u'%Y-%m-%dT%H:%M:%S.%f')
t2 = datetime.strptime(new_timestamp, u'%Y-%m-%dT%H:%M:%S.%f')
time_diff = t2 - t1
# if the time difference is negative, just return the change that put us
# at the more recent ID we were just looking at
# TODO: do something better here - go back to the previous page,
# display a warning that the user can't look at a sequence where
# the newest item is older than the oldest one, etc
if time_diff.total_seconds() < 0:
return changes(h.get_request_param(u'current_new_id'))
done = False
current_id = new_id
diff_list = []
while not done:
try:
activity_diff = get_action(u'activity_diff')(
context, {
u'id': current_id,
u'object_type': u'group',
u'diff_type': u'html'})
except NotFound as e:
log.info(
u'Activity not found: {} - {}'.format(str(e), current_id)
)
return base.abort(404, _(u'Activity not found'))
except NotAuthorized:
return base.abort(403, _(u'Unauthorized to view activity data'))
diff_list.append(activity_diff)
if activity_diff['activities'][0]['id'] == old_id:
done = True
else:
current_id = activity_diff['activities'][0]['id']
group_id = diff_list[0][u'activities'][1][u'data'][u'group'][u'id']
current_group_dict = get_action(group_type + u'_show')(
context, {u'id': group_id})
group_activity_list = get_action(group_type + u'_activity_list')(context, {
u'id': group_id,
u'limit': 100})
extra_vars = {
u'activity_diffs': diff_list,
u'group_dict': current_group_dict,
u'group_activity_list': group_activity_list,
u'group_type': current_group_dict[u'type'],
}
return base.render(_replace_group_org(u'group/changes.html'), extra_vars)
def index(group_type, is_organization):
extra_vars = {}
set_org(is_organization)
page = h.get_page_number(request.params) or 1
items_per_page = int(config.get(u'ckan.datasets_per_page', 20))
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'for_view': True,
u'with_private': False
}
try:
_check_access(u'site_read', context)
_check_access(u'group_list', context)
except NotAuthorized:
base.abort(403, _(u'Not authorized to see this page'))
q = request.params.get(u'q', u'')
sort_by = request.params.get(u'sort')
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.q = q
g.sort_by_selected = sort_by
extra_vars["q"] = q
extra_vars["sort_by_selected"] = sort_by
# pass user info to context as needed to view private datasets of
# orgs correctly
if g.userobj:
context['user_id'] = g.userobj.id
context['user_is_admin'] = g.userobj.sysadmin
try:
data_dict_global_results = {
u'all_fields': False,
u'q': q,
u'sort': sort_by,
u'type': group_type or u'group',
}
global_results = _action(u'group_list')(context,
data_dict_global_results)
except ValidationError as e:
if e.error_dict and e.error_dict.get(u'message'):
msg = e.error_dict['message']
else:
msg = str(e)
h.flash_error(msg)
extra_vars["page"] = h.Page([], 0)
extra_vars["group_type"] = group_type
return base.render(
_get_group_template(u'index_template', group_type), extra_vars)
data_dict_page_results = {
u'all_fields': True,
u'q': q,
u'sort': sort_by,
u'type': group_type or u'group',
u'limit': items_per_page,
u'offset': items_per_page * (page - 1),
u'include_extras': True
}
page_results = _action(u'group_list')(context, data_dict_page_results)
extra_vars["page"] = h.Page(
collection=global_results,
page=page,
url=h.pager_url,
items_per_page=items_per_page, )
extra_vars["page"].items = page_results
extra_vars["group_type"] = group_type
# TODO: Remove
# ckan 2.9: Adding variables that were removed from c object for
# compatibility with templates in existing extensions
g.page = extra_vars["page"]
return base.render(
_get_group_template(u'index_template', group_type), extra_vars)
def edit(self, id, data=None, errors=None, error_summary=None):
package_type = self._get_package_type(id)
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj,
'save': 'save' in request.params}
if context['save'] and not data:
return self._save_edit(id, context, package_type=package_type)
try:
c.pkg_dict = get_action('package_show')(context, {'id': id})
context['for_edit'] = True
old_data = get_action('package_show')(context, {'id': id})
# old data is from the database and data is passed from the
# user if there is a validation error. Use users data if there.
if data:
old_data.update(data)
data = old_data
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound:
abort(404, _('Dataset not found'))
# are we doing a multiphase add?
if data.get('state', '').startswith('draft') and len(data.get('resources')) == 0:
c.form_action = h.url_for(controller='package', action='new')
c.form_style = 'new'
return self.new(data=data, errors=errors,
error_summary=error_summary)
c.pkg = context.get("package")
c.resources_json = h.json.dumps(data.get('resources', []))
try:
check_access('package_update', context)
except NotAuthorized:
abort(401, _('User %r not authorized to edit %s') % (c.user, id))
# convert tags if not supplied in data
if data and not data.get('tag_string'):
data['tag_string'] = ', '.join(h.dict_list_reduce(
c.pkg_dict.get('tags', {}), 'name'))
errors = errors or {}
form_snippet = self._package_form(package_type=package_type)
form_vars = {'data': data, 'errors': errors,
'error_summary': error_summary, 'action': 'edit',
'dataset_type': package_type,
}
c.errors_json = h.json.dumps(errors)
self._setup_template_variables(context, {'id': id},
package_type=package_type)
c.related_count = c.pkg.related_count
# we have already completed stage 1
form_vars['stage'] = ['active']
if data.get('state', '').startswith('draft') and len(data.get('resources')) == 0:
form_vars['stage'] = ['active', 'complete']
edit_template = self._edit_template(package_type)
c.form = ckan.lib.render.deprecated_lazy_render(
edit_template,
form_snippet,
lambda: render(form_snippet, extra_vars=form_vars),
'use of c.form is deprecated. please see '
'ckan/templates/package/edit.html for an example '
'of the new way to include the form snippet'
)
return render(edit_template,
extra_vars={'form_vars': form_vars,
'form_snippet': form_snippet,
'dataset_type': package_type})
def _edit_or_new(self, id, related_id, is_edit):
"""
Edit and New were too similar and so I've put the code together
and try and do as much up front as possible.
"""
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj,
'for_view': True}
if is_edit:
tpl = 'related/edit.html'
auth_name = 'related_update'
auth_dict = {'id': related_id}
action_name = 'related_update'
try:
related = get_action('related_show')(
context, {'id': related_id})
except NotFound:
abort(404, _('Related item not found'))
else:
tpl = 'related/new.html'
auth_name = 'related_create'
auth_dict = {}
action_name = 'related_create'
try:
check_access(auth_name, context, auth_dict)
except NotAuthorized:
abort(401, _('Not authorized'))
try:
c.pkg_dict = get_action('package_show')(context, {'id': id})
except NotFound:
abort(404, _('Package not found'))
data, errors, error_summary = {}, {}, {}
if request.method == "POST":
try:
data = clean_dict(
dict_fns.unflatten(
tuplize_dict(
parse_params(request.params))))
if is_edit:
data['id'] = related_id
else:
data['dataset_id'] = id
data['owner_id'] = c.userobj.id
related = get_action(action_name)(context, data)
if not is_edit:
h.flash_success(_("Related item was successfully created"))
else:
h.flash_success(_("Related item was successfully updated"))
h.redirect_to(
controller='package', action='read', id=c.pkg_dict['name'])
except dict_fns.DataError:
abort(400, _(u'Integrity Error'))
except ValidationError, e:
errors = e.error_dict
error_summary = e.error_summary
elif save_action == 'go-dataset-complete':
# go to first stage of add dataset
h.redirect_to(controller='package',
action='read', id=id)
else:
# add more resources
h.redirect_to(controller='package',
action='new_resource', id=id)
# get resources for sidebar
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj}
try:
pkg_dict = get_action('package_show')(context, {'id': id})
except NotFound:
abort(404, _('The dataset {id} could not be found.').format(id=id))
try:
check_access('resource_create', context, {'package_id': pkg_dict['id']})
except NotAuthorized:
abort(401, _('Unauthorized to create a resource for this package'))
package_type = pkg_dict['type'] or 'dataset'
errors = errors or {}
error_summary = error_summary or {}
vars = {'data': data, 'errors': errors,
'error_summary': error_summary, 'action': 'new',
'resource_form_snippet': self._resource_form(package_type),
'dataset_type': package_type}
vars['pkg_name'] = id
# required for nav menu
def _import_metadata(self, post):
'''Handle a submitted import_metadata form.
Return a redirection URL.
'''
redirect_url = None
#
# Read and validate post parameters
#
# Note Authorization is enforced by the underlying action.
owner_org = post.get('owner_org')
if not owner_org:
abort(400, 'The owner organization is not given')
uv = {'group': owner_org}
uv.update(request.urlvars)
redirect_url = _url(**uv)
dtype = post.get('dataset_type')
if not dtype in ext_metadata.dataset_types:
abort(400, 'Unknown metadata schema')
rename_if_conflict = post.get('rename', '') == 'y'
continue_on_errors = post.get('force_create', '') == 'y'
# Examine source (an upload or a link)
# Todo Remember orig_metadata_source (as source_url)
source_url = None
source = post.get('source')
source_upload = post.get('source-upload')
if source:
# Assume source is provided as a URL
source_url = source
elif isinstance(source_upload, cgi.FieldStorage):
# Assume source is an uploaded file
up = uploader.MetadataUpload(source_upload.filename)
up.update_data_dict(dict(post), 'source-upload')
try:
up.upload(max_size=1)
except Exception as ex:
log.error('Failed to save uploaded file %r: %s',
source_upload.filename, ex.message)
abort(400, 'Failed to upload file')
source_url = _url(
controller='ckanext.publicamundi.controllers.files:Controller',
action='download_file',
object_type=up.object_type,
name_or_id=up.filename,
filename=source_upload.filename)
source = source_upload.file
source.seek(0, 0)
# Provide a file-like object as source
#source = source_upload.file
#log.debug('\n\n source: %s ,type: %s, up is %s , source url is %s, source_upload is %s \n\n', source, type(source),up, source_url, source_upload)
#log.debug('\n source_upload: %s , value is %s\n\n', source_upload, source.getvalue())
#for attr in dir(source_upload):
# log.info(" source upload.%s = %r" % (attr, getattr(source_upload, attr)))
#for attr in dir(source):
# log.info("source.%s = %r" % (attr, getattr(source, attr)))
#source.seek(0, 0)
else:
# No source given
session['error_summary'] = _(
'No source specified: Upload or link to an XML file.')
return redirect_url
#
# Invoke dataset_import action
#
context = {'model': model, 'session': model.Session, 'api_version': 3}
data_dict = {
'source': source,
'owner_org': owner_org,
'dtype': dtype,
'rename_if_conflict': rename_if_conflict,
'continue_on_errors': continue_on_errors,
}
try:
result = _get_action('dataset_import')(context, data_dict)
except ext_actions.Invalid as ex:
log.error('Cannot import package (invalid input): %r' %
(ex.error_dict))
if len(ex.error_dict) > 1:
session['error_summary'] = _('Received invalid input (%s)' %
(','.join(ex.error_dict.keys())))
session['errors'] = ex.error_dict
else:
session['error_summary'] = next(ex.error_dict.itervalues())
except (ext_actions.IdentifierConflict,
ext_actions.NameConflict) as ex:
log.error('Cannot import package (name/id conflict): %r' %
(ex.error_dict))
session['error_summary'] = ex.error_summary
except toolkit.ValidationError as ex:
# The input is valid, but results in an invalid package
log.error('Cannot import package (metadata are invalid): %r' %
(ex.error_dict))
session['error_summary'] = _('The given metadata are invalid.')
session['errors'] = ex.error_dict
except AssertionError as ex:
raise
except Exception as ex:
log.error('Cannot import package (unexpected error): %s' % (ex))
abort(400, 'Cannot import package')
else:
# Success: save result and redirect to success page
session['result'] = result
# Done
return redirect_url
def new_resource(self, id, data=None, errors=None, error_summary=None):
''' FIXME: This is a temporary action to allow styling of the
forms. '''
if request.method == 'POST' and not data:
save_action = request.params.get('save')
data = data or clean_dict(dict_fns.unflatten(tuplize_dict(parse_params(
request.POST))))
# we don't want to include save as it is part of the form
del data['save']
resource_id = data['id']
del data['id']
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj}
# see if we have any data that we are trying to save
data_provided = False
for key, value in data.iteritems():
if (value or isinstance(value, cgi.FieldStorage)) and key != 'resource_type':
data_provided = True
break
if not data_provided and save_action != "go-dataset-complete":
if save_action == 'go-dataset':
# go to final stage of adddataset
h.redirect_to(controller='package',
action='edit', id=id)
# see if we have added any resources
try:
data_dict = get_action('package_show')(context, {'id': id})
except NotAuthorized:
abort(401, _('Unauthorized to update dataset'))
except NotFound:
abort(404,
_('The dataset {id} could not be found.').format(id=id))
if not len(data_dict['resources']):
# no data so keep on page
msg = _('You must add at least one data resource')
# On new templates do not use flash message
if g.legacy_templates:
h.flash_error(msg)
h.redirect_to(controller='package',
action='new_resource', id=id)
else:
errors = {}
error_summary = {_('Error'): msg}
return self.new_resource(id, data, errors, error_summary)
# we have a resource so let them add metadata
h.redirect_to(controller='package',
action='new_metadata', id=id)
data['package_id'] = id
try:
if resource_id:
data['id'] = resource_id
get_action('resource_update')(context, data)
else:
get_action('resource_create')(context, data)
except ValidationError, e:
errors = e.error_dict
error_summary = e.error_summary
return self.new_resource(id, data, errors, error_summary)
except NotAuthorized:
abort(401, _('Unauthorized to create a resource'))
def get(self,
package_type: str,
id: str,
resource_id: str,
view_id: Optional[str] = None,
post_extra: Optional[dict[str, Any]] = None) -> str:
context, extra_vars = self._prepare(id, resource_id)
to_preview = extra_vars[u'to_preview']
if post_extra:
extra_vars.update(post_extra)
package_type = _get_package_type(id)
data = extra_vars[u'data'] if u'data' in extra_vars else None
if data and u'view_type' in data:
view_type = data.get(u'view_type')
else:
view_type = request.args.get(u'view_type')
# view_id exists only when updating
if view_id:
if not data or not view_type:
try:
view_data = get_action(u'resource_view_show')(
context, {
u'id': view_id
}
)
view_type = view_data[u'view_type']
if data:
data.update(view_data)
else:
data = view_data
except (NotFound, NotAuthorized):
return base.abort(404, _(u'View not found'))
# might as well preview when loading good existing view
if not extra_vars[u'errors']:
to_preview = True
if data is not None:
data[u'view_type'] = view_type
view_plugin = lib_datapreview.get_view_plugin(view_type)
if not view_plugin:
return base.abort(404, _(u'View Type Not found'))
_setup_template_variables(
context, {u'id': id}, package_type=package_type
)
data_dict: dict[str, Any] = {
u'package': extra_vars[u'pkg_dict'],
u'resource': extra_vars[u'resource'],
u'resource_view': data
}
view_template = view_plugin.view_template(context, data_dict)
form_template = view_plugin.form_template(context, data_dict)
extra_vars.update({
u'form_template': form_template,
u'view_template': view_template,
u'data': data,
u'to_preview': to_preview,
u'datastore_available': plugins.plugin_loaded(u'datastore')
})
extra_vars.update(
view_plugin.setup_template_variables(context, data_dict) or {}
)
extra_vars.update(data_dict)
if view_id:
return base.render(u'package/edit_view.html', extra_vars)
return base.render(u'package/new_view.html', extra_vars)
def list_requests():
context = _get_context()
try:
check_access('get_request_log', context, {})
except logic.NotAuthorized:
return base.abort(403, _('Not authorized to view requests audit.'))
page = _safe_int(request.args.get('page', 1), 1)
limit = _safe_int(request.args.get('limit', 20), 20)
if page < 1:
page = 1
if limit < 1:
limit = 20
if limit > 500:
limit = 500
tab = request.args.get('tab', 'all')
timespan = request.args.get('timespan')
query = request.args.get('q', '').strip()
errors = {}
date_start = None
date_end = None
if timespan == 'last-hour':
date_start = datetime.now() - timedelta(hours=1)
elif timespan == 'last-day':
date_start = datetime.now() - timedelta(days=1)
elif timespan == 'last-week':
date_start = datetime.now() - timedelta(days=7)
elif timespan == 'last-month':
date_start = datetime.now() - timedelta(days=30)
elif timespan == 'custom':
date_vals = {}
for key in ['date_start', 'date_end']:
value = request.args.get(key)
if value:
try:
value = datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
date_vals[key] = value
except Exception as e:
errors[key] = [
_('Invalid value. Should be a formatted '
'date string like "2020-10-21 10:35:43"')
]
date_start = date_vals.get('date_start')
date_end = date_vals.get('date_end')
extra_vars = _extra_template_variables(context, {
'user_obj': g.userobj,
'user': g.userobj,
})
extra_vars['errors'] = errors
if tab == 'all':
try:
log_data = get_action('get_request_log')(context, {
'page': page,
'limit': limit,
'date_start': date_start,
'date_end': date_end,
'q': query,
})
except Exception as e:
log.exception(e)
errors['general'] = _('An error occured while trying to '
'get the requests log.')
log_data = {
'results': [],
'count': 0,
}
else:
report = {
'page_count': 'endpoint',
'user_count': 'remote_user',
'ip_count': 'remote_ip',
}.get(tab)
try:
log_data = get_action('get_request_log_report')(context, {
'report': report,
'date_start': date_start,
'date_end': date_end,
'page': page,
'limit': limit,
'q': query,
})
except Exception as e:
log.exception(e)
errors['general'] = _('An error occured while trying to '
'get the report.')
log_data = {
'results': [],
'count': 0,
}
extra_vars['results'] = log_data.get('results')
extra_vars.update({
'tab':
tab,
'page':
page,
'limit':
limit,
'timespan':
timespan,
'date_start':
date_start.strftime('%Y-%m-%d %H:%M:%S') if date_start else '',
'date_end':
date_start.strftime('%Y-%m-%d %H:%M:%S') if date_end else '',
'q':
query,
})
def _get_page_data():
current_page = page or 1
query_params = dict(
filter(lambda (k, v): k != 'page', request.args.items()))
pages = []
total_pages = int(math.ceil(log_data.get('count', 0) / float(limit)))
pagination = {
'current_page': current_page,
'pages': pages,
'previous_url': '',
'next_url': '',
'total_pages': total_pages,
'total_items': log_data.get('count', 0),
'limit': limit,
}
start = 1
if current_page > 3:
start = current_page - 3
end = current_page + 3
if end > total_pages:
end = total_pages
for i in range(start, end + 1):
query_params['page'] = i
url = h.url_for('user_dashboard.audit_requests', **query_params)
pages.append({
'page': i,
'url': url,
})
if current_page > 1:
query_params['page'] = current_page - 1
pagination['previous_url'] = h.url_for(
'user_dashboard.audit_requests', **query_params)
if current_page < total_pages:
query_params['page'] = current_page + 1
pagination['next_url'] = h.url_for('user_dashboard.audit_requests',
**query_params)
return pagination
extra_vars['pagination'] = _get_page_data()
if tab == 'all':
return base.render('request_audit/requests_list.html',
extra_vars=extra_vars)
return base.render('request_audit/requests_report.html',
extra_vars=extra_vars)
def post(self, package_type: str, id: str) -> Union[str, Response]:
save_action = request.form.get(u'save')
data = clean_dict(
dict_fns.unflatten(tuplize_dict(parse_params(request.form)))
)
data.update(clean_dict(
dict_fns.unflatten(tuplize_dict(parse_params(request.files)))
))
# we don't want to include save as it is part of the form
del data[u'save']
resource_id = data.pop(u'id')
context = cast(Context, {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'auth_user_obj': g.userobj
})
# see if we have any data that we are trying to save
data_provided = False
for key, value in data.items():
if (
(value or isinstance(value, cgi.FieldStorage))
and key != u'resource_type'):
data_provided = True
break
if not data_provided and save_action != u"go-dataset-complete":
if save_action == u'go-dataset':
# go to final stage of adddataset
return h.redirect_to(u'{}.edit'.format(package_type), id=id)
# see if we have added any resources
try:
data_dict = get_action(u'package_show')(context, {u'id': id})
except NotAuthorized:
return base.abort(403, _(u'Unauthorized to update dataset'))
except NotFound:
return base.abort(
404,
_(u'The dataset {id} could not be found.').format(id=id)
)
if not len(data_dict[u'resources']):
# no data so keep on page
msg = _(u'You must add at least one data resource')
# On new templates do not use flash message
errors: dict[str, Any] = {}
error_summary = {_(u'Error'): msg}
return self.get(package_type, id, data, errors, error_summary)
# XXX race condition if another user edits/deletes
data_dict = get_action(u'package_show')(context, {u'id': id})
get_action(u'package_update')(
cast(Context, dict(context, allow_state_change=True)),
dict(data_dict, state=u'active')
)
return h.redirect_to(u'{}.read'.format(package_type), id=id)
data[u'package_id'] = id
try:
if resource_id:
data[u'id'] = resource_id
get_action(u'resource_update')(context, data)
else:
get_action(u'resource_create')(context, data)
except ValidationError as e:
errors = e.error_dict
error_summary = e.error_summary
if data.get(u'url_type') == u'upload' and data.get(u'url'):
data[u'url'] = u''
data[u'url_type'] = u''
data[u'previous_upload'] = True
return self.get(package_type, id, data, errors, error_summary)
except NotAuthorized:
return base.abort(403, _(u'Unauthorized to create a resource'))
except NotFound:
return base.abort(
404, _(u'The dataset {id} could not be found.').format(id=id)
)
if save_action == u'go-metadata':
# XXX race condition if another user edits/deletes
data_dict = get_action(u'package_show')(context, {u'id': id})
get_action(u'package_update')(
cast(Context, dict(context, allow_state_change=True)),
dict(data_dict, state=u'active')
)
return h.redirect_to(u'{}.read'.format(package_type), id=id)
elif save_action == u'go-dataset':
# go to first stage of add dataset
return h.redirect_to(u'{}.edit'.format(package_type), id=id)
elif save_action == u'go-dataset-complete':
return h.redirect_to(u'{}.read'.format(package_type), id=id)
else:
# add more resources
return h.redirect_to(
u'{}_resource.new'.format(package_type),
id=id
)
length = 0
for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
base.response.body_file.write(chunk)
length += len(chunk)
if length >= MAX_FILE_SIZE:
base.abort(409, ('''Content is too large to be proxied. Allowed
file size: {allowed}, Content-Length: {actual}. Url: '''+url).format(
allowed=MAX_FILE_SIZE, actual=length))
except requests.exceptions.HTTPError, error:
details = 'Could not proxy resource. Server responded with %s %s' % (
error.response.status_code, error.response.reason)
base.abort(409, detail=details)
except requests.exceptions.ConnectionError, error:
details = '''Could not proxy resource because a
connection error occurred. %s''' % error
base.abort(502, detail=details)
except requests.exceptions.Timeout, error:
details = 'Could not proxy resource because the connection timed out.'
base.abort(504, detail=details)
class ServiceProxyController(base.BaseController):
def proxy_service(self, resource_id):
data_dict = {'resource_id': resource_id}
context = {'model': base.model, 'session': base.model.Session,
'user': base.c.user or base.c.author}
return proxy_service(self, context, data_dict)
content = u'<?xml version="1.0" encoding="UTF-8"?>\n' + content
except xml_parser_exception:
try:
json.loads(obj['content'])
response.content_type = 'application/json; charset=utf-8'
except ValueError:
# Just return whatever it is
pass
response.headers['Content-Length'] = len(content)
return content.encode('utf-8')
except p.toolkit.ObjectNotFound, e:
abort(404, _(str(e)))
except p.toolkit.NotAuthorized:
abort(401, self.not_auth_message)
except Exception, e:
msg = 'An error occurred: [%s]' % str(e)
abort(500, msg)
def _get_source_for_job(self, source_id):
try:
context = {'model': model, 'user': c.user}
source_dict = p.toolkit.get_action('harvest_source_show')(
context, {
'id': source_id
})
except p.toolkit.ObjectNotFound:
abort(404, p.toolkit._('Harvest source not found'))
except p.toolkit.NotAuthorized:
def read(package_type: str, id: str, resource_id: str) -> str:
context = cast(Context, {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'auth_user_obj': g.userobj,
u'for_view': True
})
try:
package = get_action(u'package_show')(context, {u'id': id})
except (NotFound, NotAuthorized):
return base.abort(404, _(u'Dataset not found'))
activity_id = request.args.get(u'activity_id')
if activity_id:
# view an 'old' version of the package, as recorded in the
# activity stream
current_pkg = package
try:
activity = context['session'].query(model.Activity).get(
activity_id
)
assert activity
package = activity.data['package']
except AttributeError:
base.abort(404, _(u'Dataset not found'))
if package['id'] != current_pkg['id']:
log.info(u'Mismatch between pkg id in activity and URL {} {}'
.format(package['id'], current_pkg['id']))
# the activity is not for the package in the URL - don't allow
# misleading URLs as could be malicious
base.abort(404, _(u'Activity not found'))
# The name is used lots in the template for links, so fix it to be
# the current one. It's not displayed to the user anyway.
package['name'] = current_pkg['name']
# Don't crash on old (unmigrated) activity records, which do not
# include resources or extras.
package.setdefault(u'resources', [])
resource = None
for res in package.get(u'resources', []):
if res[u'id'] == resource_id:
resource = res
break
if not resource:
return base.abort(404, _(u'Resource not found'))
# get package license info
license_id = package.get(u'license_id')
try:
package[u'isopen'] = model.Package.get_license_register()[license_id
].isopen()
except KeyError:
package[u'isopen'] = False
resource_views = get_action(u'resource_view_list')(
context, {
u'id': resource_id
}
)
resource[u'has_views'] = len(resource_views) > 0
current_resource_view = None
view_id = request.args.get(u'view_id')
if resource[u'has_views']:
if view_id:
current_resource_view = [
rv for rv in resource_views if rv[u'id'] == view_id
]
if len(current_resource_view) == 1:
current_resource_view = current_resource_view[0]
else:
return base.abort(404, _(u'Resource view not found'))
else:
current_resource_view = resource_views[0]
# required for nav menu
pkg = context[u'package']
dataset_type = pkg.type or package_type
# TODO: remove
g.package = package
g.resource = resource
g.pkg = pkg
g.pkg_dict = package
extra_vars: dict[str, Any] = {
u'resource_views': resource_views,
u'current_resource_view': current_resource_view,
u'dataset_type': dataset_type,
u'pkg_dict': package,
u'package': package,
u'resource': resource,
u'pkg': pkg, # NB it is the current version of the dataset, so ignores
# activity_id. Still used though in resource views for
# backward compatibility
u'is_activity_archive': bool(activity_id),
}
template = _get_pkg_template(u'resource_template', dataset_type)
return base.render(template, extra_vars)
def read_publisher(self, id):
'''
Lists the most popular datasets for a publisher (or across all publishers)
'''
count = 20
c.publishers = _get_publishers()
id = request.params.get('publisher', id)
if id and id != 'all':
c.publisher = model.Group.get(id)
if not c.publisher:
abort(404, 'A publisher with that name could not be found')
c.publisher_name = c.publisher.name
c.top_packages = [] # package, dataset_views in c.top_packages
# Get the month details by fetching distinct values and determining the
# month names from the values.
c.months, c.day = _month_details(GA_Url)
# Work out which month to show, based on query params of the first item
c.month = request.params.get('month', '')
if not c.month:
c.month_desc = 'all months'
else:
c.month_desc = ''.join([m[1] for m in c.months if m[0] == c.month])
month = c.month or 'All'
c.publisher_page_views = 0
q = model.Session.query(GA_Url).\
filter(GA_Url.url=='/publisher/%s' % c.publisher_name)
entry = q.filter(GA_Url.period_name == c.month).first()
c.publisher_page_views = entry.pageviews if entry else 0
c.top_packages = self._get_packages(publisher=c.publisher,
count=20,
month=c.month)
# Graph query
top_packages_all_time = self._get_packages(publisher=c.publisher,
count=20,
month='All')
top_package_names = [x[0].name for x in top_packages_all_time]
graph_query = model.Session.query(GA_Url,model.Package)\
.filter(model.Package.name==GA_Url.package_id)\
.filter(GA_Url.url.like('/dataset/%'))\
.filter(GA_Url.package_id.in_(top_package_names))
all_series = {}
for entry, package in graph_query:
if not package: continue
if entry.period_name == 'All': continue
all_series[package.name] = all_series.get(package.name, {
'name': package.title,
'raw': {}
})
all_series[package.name]['raw'][entry.period_name] = int(
entry.pageviews)
graph = [all_series[series_name] for series_name in top_package_names]
c.graph_data = json.dumps(_to_rickshaw(graph))
return render('ga_report/publisher/read.html')
def members(self, id):
'''
Modified core method from 'group' controller.
Added search & sort functionality.
:param id: id of the organization for which the member list is requested
:type id: string
:return: the rendered template
:rtype: unicode
'''
context = self._get_context()
q, sort = self._find_filter_params()
reverse = True if sort == u'title desc' else False
org_meta = org_meta_dao.OrgMetaDao(id, c.user or c.author, c.userobj)
org_meta.fetch_all()
try:
member_list = self._action('member_list')(context, {
'id': id,
'object_type': 'user',
'q': q,
'user_info': True
})
member_list.sort(key=lambda y: y[4].lower(), reverse=reverse)
member_groups = {}
for m in member_list:
role = m[3]
if not member_groups.get(role):
member_groups[role] = []
member_groups[role].append(m)
member_groups = collections.OrderedDict(
sorted(member_groups.items()))
data_dict = {'id': id}
data_dict['include_datasets'] = False
current_user = self._current_user_info(member_list)
is_sysadmin = c.userobj and c.userobj.sysadmin
c_params = {
'sort':
sort,
'members': [a[0:4] for a in member_list],
'member_groups':
member_groups,
'org_meta':
org_meta,
'current_user':
current_user,
'allow_view_right_side':
is_sysadmin or bool(current_user.get('role')),
'allow_approve':
is_sysadmin or current_user.get('role') == 'admin',
'request_list':
self._get_member_requests_for_org(id)
}
self._set_c_params(c_params)
except NotAuthorized:
base.abort(401, _('Unauthorized to view member list %s') % '')
except NotFound:
base.abort(404, _('Group not found'))
except Exception, ex:
log.error(str(ex))
base.abort(404, _('Server error'))
def proxy_ngsi_resource(self, resource_id):
# Chunked proxy for ngsi resources.
context = {'model': base.model, 'session': base.model.Session, 'user': base.c.user or base.c.author}
log.info('Proxify resource {id}'.format(id=resource_id))
resource = logic.get_action('resource_show')(context, {'id': resource_id})
headers = {
'Accept': 'application/json'
}
resource.setdefault('auth_type', 'none')
self.process_auth_credentials(resource, headers)
if resource.get('tenant', '') != '':
headers['FIWARE-Service'] = resource['tenant']
if resource.get('service_path', '') != '':
headers['FIWARE-ServicePath'] = resource['service_path']
url = resource['url']
parsed_url = urlparse.urlsplit(url)
if parsed_url.scheme not in ("http", "https") or not parsed_url.netloc:
base.abort(409, detail='Invalid URL.')
# Process verify configuration
verify_conf = os.environ.get('CKAN_RIGHT_TIME_CONTEXT_VERIFY_REQUESTS', toolkit.config.get('ckan.right_time_context.verify_requests'))
if verify_conf is None or (isinstance(verify_conf, six.string_types) and verify_conf.strip() == ""):
verify_conf = os.environ.get('CKAN_VERIFY_REQUESTS', toolkit.config.get('ckan.verify_requests'))
if isinstance(verify_conf, six.string_types) and verify_conf.strip() != "":
compare_env = verify_conf.lower().strip()
if compare_env in ("true", "1", "on"):
verify = True
elif compare_env in ("false", "0", "off"):
verify = False
else:
verify = verify_conf
elif isinstance(verify_conf, bool):
verify = verify_conf
else:
verify = True
# Make the request to the server
try:
if resource['format'].lower() == NGSI_REG_FORMAT:
r = self._proxy_registration_resource(resource, parsed_url, headers, verify=verify)
else:
r = self._proxy_query_resource(resource, parsed_url, headers, verify=verify)
except requests.HTTPError:
details = 'Could not proxy ngsi_resource. We are working to resolve this issue as quickly as possible'
base.abort(409, detail=details)
except requests.ConnectionError:
details = 'Could not proxy ngsi_resource because a connection error occurred.'
base.abort(502, detail=details)
except requests.Timeout:
details = 'Could not proxy ngsi_resource because the connection timed out.'
base.abort(504, detail=details)
if r.status_code == 401:
if resource.get('auth_type', 'none') != 'none':
details = 'ERROR 401 token expired. Retrieving new token, reload please.'
log.info(details)
toolkit.c.usertoken_refresh()
base.abort(409, detail=details)
elif resource.get('auth_type', 'none') == 'none':
details = 'Authentication requested by server, please check resource configuration.'
log.info(details)
base.abort(409, detail=details)
elif r.status_code == 400:
response = r.json()
details = response['description']
log.info(details)
base.abort(422, detail=details)
else:
r.raise_for_status()
base.response.content_type = r.headers['content-type']
base.response.charset = r.encoding
for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
base.response.body_file.write(chunk)
def _prepare_and_send(self, pkg_id, recipient_id, subject, prefix_template, suffix):
"""
Sends a message by email from the logged in user to the appropriate
contact address of the given dataset.
The prefix template should have formatting placeholders for the following arguments:
{sender_name}, {sender_email}, {package_title}, {package_id}
:param pkg_id: package id
:type pkg_id: string
:param recipient_id: id of the recipient, as returned by utils.get_package_contacts
:type recipient_id: string
:param subject: the subject of the message
:type subject: string
:param prefix_template: the template for the prefix to be automatically included before the user message
:type prefix_template: unicode
:param suffix: an additional note to be automatically included after the user message
:type suffix: unicode
"""
url = h.url_for(controller='package', action='read', id=pkg_id)
if asbool(config.get('kata.contact_captcha')):
try:
captcha.check_recaptcha(request)
except captcha.CaptchaError:
h.flash_error(_(u'Bad Captcha. Please try again.'))
redirect(url)
if not request.params.get('accept_logging'):
h.flash_error(_(u"Message not sent as logging wasn't permitted"))
return redirect(url)
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. Please try again later or contact customer "
u"service."))
return redirect(url)
package = Package.get(pkg_id)
package_title = package.title if package.title else package.name
sender_addr = request.params.get('from_address')
sender_name = request.params.get('from_name')
recipient = self._get_contact_email(pkg_id, recipient_id)
if not recipient:
abort(404, _('Recipient not found'))
user_msg = request.params.get('msg', '')
if request.params.get('hp'):
h.flash_error(_(u"Message not sent. Couldn't confirm human interaction (spam bot control)"))
return redirect(url)
if sender_addr and sender_name and \
isinstance(sender_name, basestring) and len(sender_name) >= 3:
if user_msg:
prefix = prefix_template.format(
sender_name=sender_name,
sender_email=sender_addr,
package_title=package_title,
package_id=pkg_id
# above line was: metadata_pid=utils.get_primary_data_pid_from_package(package)
)
log.info(u"Message {m} sent from {a} ({b}) to {r} about {c}, IP: {d}"
.format(m=user_msg, a=sender_name, b=sender_addr, r=recipient, c=pkg_id,
d=request.environ.get('REMOTE_ADDR', 'No remote address')))
full_msg = u"{a}{b}{c}".format(a=prefix, b=user_msg, c=suffix)
self._send_message(subject, full_msg, recipient.get('email'), recipient.get('name'))
h.flash_success(_(u"Message sent"))
else:
h.flash_error(_(u"No message"))
else:
h.flash_error(_(u"Message not sent. Please, provide reply address and name. Name must contain \
at least three letters."))
return redirect(url)
def before_request():
try:
context = dict(model=model, user=g.user, auth_user_obj=g.userobj)
logic.check_access(u'site_read', context)
except logic.NotAuthorized:
base.abort(403, _(u'Not authorized to see this page'))
def edit(self, id):
def validate(data):
error_dict = {}
has_owner = data.get('owner')
has_key = data.get('key')
if tk.asbool(data.get('integration', 'False')):
if not has_owner:
error_dict['owner'] = ['Required']
if not has_key:
error_dict['key'] = ['Required']
if tk.asbool(data.get('show_links', 'False')):
if not has_owner or not has_key:
error_dict['show_links'] = [
'This option available only '
'if credentials are provided'
]
if not error_dict:
api = API(has_owner, has_key)
check = api.check_credentials()
if not check:
error_dict['key'] = ['Incorrect key']
if error_dict:
raise logic.ValidationError(error_dict)
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj
}
data_dict = {'id': id}
stats = {}
extra = {'errors': {}, 'error_summary': None, 'stats': stats}
try:
if not h.check_access('organization_update', data_dict):
raise logic.NotAuthorized
c.group_dict = logic.get_action('organization_show')(context,
data_dict)
c.group = context['group']
c.credentials = c.group.datadotworld_credentials
if c.credentials is None:
c.credentials = Credentials(organization=c.group)
model.Session.add(c.credentials)
except logic.NotFound:
base.abort(404, _('Organization not found'))
except logic.NotAuthorized:
base.abort(401,
_('User %r not authorized to edit %s') % (c.user, id))
if request.method == 'POST':
data = dict(request.POST)
c.credentials.update(data)
try:
validate(data)
except logic.ValidationError as e:
extra['errors'] = e.error_dict
extra['error_summary'] = e.error_summary
else:
query = model.Session.query(Extras).join(model.Package).join(
model.Group,
model.Package.owner_org == model.Group.id).filter(
model.Group.id == c.group.id)
for item in query:
item.state = 'pending'
model.Session.commit()
h.flash_success('Saved')
if tk.asbool(c.credentials.integration):
syncronize_org(c.group.id)
return base.redirect_to('organization_dataworld', id=id)
query = model.Session.query(
func.count(model.Package.id).label('total'),
Extras.state).join(model.Group,
model.Package.owner_org == model.Group.id).join(
Extras).group_by(Extras.state).filter(
model.Package.owner_org == c.group.id)
for amount, state in query:
stats[state] = amount
return base.render('organization/edit_credentials.html',
extra_vars=extra)
def not_found(self, url):
from ckan.lib.base import abort
abort(404)
def map_preview_auth(self, resource_id, user_gs_layer):
"""
Authorization check for the given resource ID and Geoserver layer name
:param resource_id: resource_id embedded in map preview URL
:param user_gs_layer: Geoserver layer name requested by user in the URL
:return:
"""
"""
Test cases:
User mpeterman-msea-read.
Testing with dataset: https://www.gis-hub.ca/dataset/test05. This dataset has a mixture of
resources with restrictions. Test the following URLs:
1. This resource is accessible to all users in the MSEA org:
https://www.gis-hub.ca/dataset/test05/map_preview/7fad5dba-b70e-4a01-b692-a59a9b1b100c
Result: 200 Access OK
2. Accessible only to user dfo or admins:
https://www.gis-hub.ca/dataset/test05/map_preview/b1a24c96-eaa6-4433-a706-99854894be13
Experimenting with NGINX URL rewrite and variable capture
https://www.gis-hub.ca/nginx_auth/geoserver/rest/workspaces/hubdata/datastores/maps.gis-hub.ca/featuretypes/ia_geography_pncima_ia_oceanography_pncima.json
https://www.gis-hub.ca/geoserver/rest/workspaces/hubdata/datastores/maps.gis-hub.ca/featuretypes/ia_geography_pncima_ia_oceanography_pncima.json
Capture block in nginx has changed, the above URLs should now be:
https://www.gis-hub.ca/map_preview/test05/7fad5dba-b70e-4a01-b692-a59a9b1b100c
https://www.gis-hub.ca/map_preview/test05/b1a24c96-eaa6-4433-a706-99854894be13
This NGINX location block matches after /geoserver, but it removes all the URL params after ?
So, /hubdata/wms?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap... etc
gets reduced to:
/hubdata/wms
# location ~ ^/map_preview/(?<package_id>.+)/(?<resource_id>.+)/geoserver/(?<geo_params>.+)$ {
Need to add $query_string to the end of the proxy URI that gets passed on to geoserver on maps.gis-hub.ca
set $geoserver_uri "https://maps.gis-hub.ca/geoserver/$geo_params?$query_string";
Have to open this in a Chrome Private window, because there was some cached HTTP basic auth that it tried to use.
This works! We get the dataset and resource id, check it against CKAN auth, if it passes, we construct the correct geoserver
URL and pass it to the proxy.
With a geoserver request appended, test proxy.
https://www.gis-hub.ca/map_preview/test05/7fad5dba-b70e-4a01-b692-a59a9b1b100c/geoserver/hubdata/wms?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image%2Fpng&TRANSPARENT=true&LAYERS=hubdata%3Aenv_layers_nsbssb_bpi_medium&exceptions=application%2Fvnd.ogc.se_inimage&SRS=EPSG%3A3857&STYLES=&WIDTH=873&HEIGHT=801&BBOX=-15384022.060787713%2C5832451.006272089%2C-13248677.23861303%2C7791684.915277727
Test with a restricted resource, should fail:
https://www.gis-hub.ca/map_preview/test05/b1a24c96-eaa6-4433-a706-99854894be13/geoserver/hubdata/wms?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image%2Fpng&TRANSPARENT=true&LAYERS=hubdata%3Aenv_layers_nsbssb_bpi_medium&exceptions=application%2Fvnd.ogc.se_inimage&SRS=EPSG%3A3857&STYLES=&WIDTH=873&HEIGHT=801&BBOX=-15384022.060787713%2C5832451.006272089%2C-13248677.23861303%2C7791684.915277727
Yes, this request never makes it to the proxy, the auth check in nginx fails with error 500.
For now, just during development, we can keep the previous /geoserver proxy open, so that map previews will continue to work.
TODO: ensure that the resource being requested matches the WMS or vector layer in Geoserver. This can be deferred for now.
We dont care about requests like /vector/:dataset/:layer_name. The only thing we care about is securing actual map tile requests.
Probably need to also add the resource ID to the map preview URL, something like:
A: /vector/:dataset/:layer_name/:resource_id.
Then, inside the vector tiles template, use a URL like this:
B: {{ host }}/map_preview/{{dataset}}/{{resource_id}}/geoserver/gwc/service/tms/1.0.0/hubdata:{{ layer_name }}...
This will be sent to nginx auth, then proxy directly to:
C: https://maps.gis-hub.ca/geoserver/$geo_params?$query_string << this part does not touch the Node.js mapserver middleware at all.
And this all happens *inside NGINX*, the only URL the user's browser sees is A and B
** Update May 20. For some reason, the map_preview endpoint has stopped working. e.g.
https://www.gis-hub.ca/dataset/test05/map_preview/7fad5dba-b70e-4a01-b692-a59a9b1b100c throws not found, this controller function is
not called at all. > Because I renamed the function to map_preview_auth!
Test this with:
https://maps.gis-hub.ca/vector/ia_geography_pncima/ia_oceanography_pncima/f63a3614-84ae-4168-b340-6919eac458c6
DOES NOT WORK because: ia_geography_pncima > in CKAN this should be ia-geography-pncima with HYPHENS
However in mapserver backend, it is using ia_geography_pncima + resource title with UNDERSCORES to check map extent and so on.
Need to replace the dataset part of the map preview URL with ia-geography-pncima, then in the Node backend, convert this to
underscores before generating postgis layer name. Postgis is not happy with table names containing hyphens. What a pain.
Change to:
https://maps.gis-hub.ca/vector/ia-geography-pncima/ia_oceanography_pncima/f63a3614-84ae-4168-b340-6919eac458c6
Now the last issue we have is that when the request gets proxied from maps.gis-hub.ca back to gis-hub, it no longer has the CKAN
context object with the user. Is is possible to pass the session with the proxied headers? If not, we might have to redo the
map preview code inside a CKAN extension (where we know for sure that we have the context). This would simplify things somewhat,
no need for all the proxying between the two servers.
https://maps.gis-hub.ca/vector/ia-geography-pncima/ia_oceanography_pncima/f63a3614-84ae-4168-b340-6919eac458c6
https://gis-hub.ca/vector/ia-geography-pncima/ia_oceanography_pncima/f63a3614-84ae-4168-b340-6919eac458c6
Finally, in the nginx conf inside mapserver, allow access /geoserver/gwc and /geoserver/hubdata/wms only through host gis-hub.ca.
Geoserver admins need to use the Geoserver web interface, which is accessed directly through maps.gis-hub.ca:8080,
which bypasses Nginx entirely. That interface is already secured with an admin pass.
"""
logger.info('Check if user is admin')
if auth.is_admin(c.user):
# Return success immediately if user is admin
logger.warning('User %s is admin. Skipping all auth checks' %
c.user)
return render('restricted/restricted_ok_page.html')
auth_status = False
try:
# Get resource object
resource = model.Resource.get(resource_id)
if not resource:
logger.warning('Resource %s not found!' % resource_id)
flask_abort(401, 'Access denied')
resource = resource.as_dict()
# Ensure that the Geoserver layer name requested by the user matches the resource
# Get the internal layer name used in Geoserver, directly from metadata if exists
gs_layer_name = resource.get('geoserver_layer')
if gs_layer_name:
if user_gs_layer != gs_layer_name:
logger.warning(
'User requested a layer name not from this resource! %s, expected %s'
% (user_gs_layer, gs_layer_name))
flask_abort(401, 'Invalid geoserver layer name')
else:
# If the geoserver_layer field does not exist, get layer name from map_preview_link field
logger.warning(
'Field geoserver_layer is empty, try map_preview_link')
match_ok = self.check_geoserver_layer_name(
resource, user_gs_layer)
if not match_ok:
logger.warning(
'User requested a layer name not from this resource! %s'
% user_gs_layer)
flask_abort(401, 'Invalid geoserver layer name')
package_id = resource.get('package_id')
# Get package object using package_id from the resource
package = model.Package.get(package_id)
if not package:
msg = 'Dataset %s not found!' % package_id
logger.warning(msg)
logger.warning(
'Were HYPHENS in the dataset name changed to UNDERSCORES?')
flask_abort(401, msg)
package = package.as_dict()
logger.info(
'Checking access for User: %s, Resource: %s, Geoserver layer: %s, Package: %s'
% (c.user, resource.get('title'), user_gs_layer,
package.get('name')))
# Ready to check access
authorized = restricted_logic.restricted_check_user_resource_access(
c.user, resource, package)
auth_status = authorized.get('success')
logger.info('Status: %s' % auth_status)
except:
# If anything else goes wrong, throw an error back to Nginx
logger.warning(
'Something went wrong during auth check. Return error 401')
import traceback
logger.error(traceback.format_exc())
flask_abort(401, 'Error in auth check')
if auth_status:
"""
Return a normal page with status code 2xx IF the user is authorized for this resource.
Nginx responds to a 2xx status code by passing the auth request, then continues with
the proxy to maps.gis-hub.ca/geoserver.
"""
return render('restricted/restricted_ok_page.html')
else:
"""
This causes the server to throw error 500, which is OK because anything other than
a 2xx code causes Nginx to fail the auth request, and deny the proxy. User will see an
error in their browser.
https://docs.nginx.com/nginx/admin-guide/security-controls/configuring-subrequest-authentication/
"""
return base.abort(404, _(u'User not authorized'))
def search(self):
from ckan.lib.search import SearchError, SearchQueryError
package_type = self._guess_package_type()
extra_vars = {'dataset_type': package_type}
try:
context = {
'model': model,
'user': c.user,
'auth_user_obj': c.userobj
}
check_access('site_read', context)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
# unicode format (decoded from utf8)
q = c.q = request.params.get('q', u'')
c.query_error = False
page = h.get_page_number(request.params)
limit = int(config.get('ckan.datasets_per_page', 20))
# most search operations should reset the page counter:
params_nopage = [(k, v) for k, v in request.params.items()
if k != 'page']
def drill_down_url(alternative_url=None, **by):
return h.add_url_param(alternative_url=alternative_url,
controller='package',
action='search',
new_params=by)
c.drill_down_url = drill_down_url
def remove_field(key, value=None, replace=None):
return h.remove_url_param(key,
value=value,
replace=replace,
controller='package',
action='search',
alternative_url=package_type)
c.remove_field = remove_field
sort_by = request.params.get('sort', None)
params_nosort = [(k, v) for k, v in params_nopage if k != 'sort']
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return search_url(params, package_type)
c.sort_by = _sort_by
if not sort_by:
c.sort_by_fields = []
else:
c.sort_by_fields = [
field.split()[0] for field in sort_by.split(',')
]
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params, package_type)
c.search_url_params = urlencode(_encode_params(params_nopage))
try:
c.fields = []
# c.fields_grouped will contain a dict of params containing
# a list of values eg {'tags':['tag1', 'tag2']}
c.fields_grouped = {}
search_extras = {}
fq = ''
for (param, value) in request.params.items():
if param not in ['q', 'page', 'sort'] \
and len(value) and not param.startswith('_'):
if not param.startswith('ext_'):
c.fields.append((param, value))
fq += ' %s:"%s"' % (param, value)
if param not in c.fields_grouped:
c.fields_grouped[param] = [value]
else:
c.fields_grouped[param].append(value)
else:
search_extras[param] = value
context = {
'model': model,
'session': model.Session,
'user': c.user,
'for_view': True,
'auth_user_obj': c.userobj
}
# Unless changed via config options, don't show other dataset
# types any search page. Potential alternatives are do show them
# on the default search page (dataset) or on one other search page
search_all_type = config.get('ckan.search.show_all_types',
'dataset')
search_all = False
try:
# If the "type" is set to True or False, convert to bool
# and we know that no type was specified, so use traditional
# behaviour of applying this only to dataset type
search_all = asbool(search_all_type)
search_all_type = 'dataset'
# Otherwise we treat as a string representing a type
except ValueError:
search_all = True
if not package_type:
package_type = 'dataset'
if not search_all or package_type != search_all_type:
# Only show datasets of this particular type
fq += ' +dataset_type:{type}'.format(type=package_type)
facets = OrderedDict()
default_facet_titles = {
'organization': _('Organizations'),
'groups': _('Groups'),
'tags': _('Tags'),
'res_format': _('Formats'),
'license_id': _('Licenses'),
}
for facet in h.facets():
if facet in default_facet_titles:
facets[facet] = default_facet_titles[facet]
else:
facets[facet] = facet
# Facet titles
for plugin in p.PluginImplementations(p.IFacets):
facets = plugin.dataset_facets(facets, package_type)
c.facet_titles = facets
# Date interval search
start_date = None
end_date = None
if request.params.get('_start_date'):
start_date = extract_date(request.params.get('_start_date'))
if start_date:
extra_vars['start_date'] = request.params.get(
'_start_date')
if request.params.get('_end_date'):
end_date = extract_date(request.params.get('_end_date'))
if end_date:
extra_vars['end_date'] = request.params.get('_end_date')
if start_date or end_date:
if not start_date:
start_date = datetime(year=1900, month=1, day=1)
if not end_date:
end_date = datetime(year=2100,
month=1,
day=1,
hour=23,
minute=59,
second=59,
microsecond=999999)
else:
end_date = datetime_to_utc(end_date)
end_date = end_date.replace(hour=23,
minute=59,
second=59,
microsecond=999999)
fq = fq + ' +metadata_modified:[%s TO %s]' % (
datetime_to_utc(start_date).strftime('%Y-%m-%dT%H:%M:%SZ'),
datetime_to_utc(end_date).strftime('%Y-%m-%dT%H:%M:%SZ'))
data_dict = {
'q':
q,
'fq':
fq.strip(),
'facet.field':
facets.keys(),
'rows':
limit,
'start': (page - 1) * limit,
'sort':
sort_by,
'extras':
search_extras,
'include_private':
asbool(config.get('ckan.search.default_include_private',
True)),
}
query = get_action('package_search')(context, data_dict)
c.sort_by_selected = query['sort']
c.page = h.Page(collection=query['results'],
page=page,
url=pager_url,
item_count=query['count'],
items_per_page=limit)
c.search_facets = query['search_facets']
c.page.items = query['results']
except SearchQueryError as se:
# User's search parameters are invalid, in such a way that is not
# achievable with the web interface, so return a proper error to
# discourage spiders which are the main cause of this.
log.info('Dataset search query rejected: %r', se.args)
abort(
400,
_('Invalid search query: {error_message}').format(
error_message=str(se)))
except SearchError as se:
# May be bad input from the user, but may also be more serious like
# bad code causing a SOLR syntax error, or a problem connecting to
# SOLR
log.error('Dataset search error: %r', se.args)
c.query_error = True
c.search_facets = {}
c.page = h.Page(collection=[])
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
c.search_facets_limits = {}
for facet in c.search_facets.keys():
try:
limit = int(
request.params.get(
'_%s_limit' % facet,
int(config.get('search.facets.default', 10))))
except ValueError:
abort(
400,
_('Parameter "{parameter_name}" is not '
'an integer').format(parameter_name='_%s_limit' % facet))
c.search_facets_limits[facet] = limit
self._setup_template_variables(context, {}, package_type=package_type)
extra_vars['dataset_type'] = package_type
return render(self._search_template(package_type),
extra_vars=extra_vars)
split = id.split('@')
if len(split) == 2:
data_dict['id'], revision_ref = split
if model.is_id(revision_ref):
context['revision_id'] = revision_ref
else:
try:
date = h.date_str_to_datetime(revision_ref)
context['revision_date'] = date
except TypeError, e:
base.abort(400, _('Invalid revision format: %r') % e.args)
except ValueError, e:
base.abort(400, _('Invalid revision format: %r') % e.args)
elif len(split) > 2:
base.abort(
400,
_('Invalid revision format: %r') % 'Too many "@" symbols')
# check if package exists
try:
c.pkg_dict = logic.get_action('package_show')(context, data_dict)
c.pkg = context['package']
except NotFound:
base.abort(404, _('Dataset not found'))
except NotAuthorized:
base.abort(401, _('Unauthorized to read package %s') % id)
# used by disqus plugin
c.current_package_id = c.pkg.id
c.related_count = c.pkg.related_count
def dashboard(self):
""" List all related items regardless of dataset """
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'auth_user_obj': c.userobj,
'for_view': True
}
data_dict = {
'type_filter': base.request.params.get('type', ''),
'sort': base.request.params.get('sort', ''),
'featured': base.request.params.get('featured', '')
}
params_nopage = [(k, v) for k, v in base.request.params.items()
if k != 'page']
try:
page = int(base.request.params.get('page', 1))
except ValueError:
base.abort(400, ('"page" parameter must be an integer'))
# Update ordering in the context
related_list = logic.get_action('related_list')(context, data_dict)
def search_url(params):
url = h.url_for(controller='related', action='dashboard')
params = [
(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
for k, v in params
]
return url + u'?' + urllib.urlencode(params)
def pager_url(q=None, page=None):
params = list(params_nopage)
params.append(('page', page))
return search_url(params)
c.page = h.Page(collection=related_list,
page=page,
url=pager_url,
item_count=len(related_list),
items_per_page=9)
c.filters = dict(params_nopage)
c.type_options = self._type_options()
c.sort_options = ({
'value': '',
'text': _('Most viewed')
}, {
'value': 'view_count_desc',
'text': _('Most Viewed')
}, {
'value': 'view_count_asc',
'text': _('Least Viewed')
}, {
'value': 'created_desc',
'text': _('Newest')
}, {
'value': 'created_asc',
'text': _('Oldest')
})
return base.render("related/dashboard.html")
def post(self, id: Optional[str] = None) -> Union[Response, str]:
context, id = self._prepare(id)
if not context[u'save']:
return self.get(id)
# checks if user id match with the current logged user
if id in (g.userobj.id, g.userobj.name):
current_user = True
else:
current_user = False
# we save the username for later use.. in case the current
# logged in user change his username
old_username = g.userobj.name
try:
data_dict = logic.clean_dict(
dictization_functions.unflatten(
logic.tuplize_dict(logic.parse_params(request.form))))
data_dict.update(
logic.clean_dict(
dictization_functions.unflatten(
logic.tuplize_dict(logic.parse_params(
request.files)))))
except dictization_functions.DataError:
base.abort(400, _(u'Integrity Error'))
data_dict.setdefault(u'activity_streams_email_notifications', False)
data_dict[u'id'] = id
# we need this comparison when sysadmin edits a user,
# this will return True
# and we can utilize it for later use.
email_changed = data_dict[u'email'] != g.userobj.email
# common users can edit their own profiles without providing
# password, but if they want to change
# their old password with new one... old password must be provided..
# so we are checking here if password1
# and password2 are filled so we can enter the validation process.
# when sysadmins edits a user he MUST provide sysadmin password.
# We are recognizing sysadmin user
# by email_changed variable.. this returns True
# and we are entering the validation.
if (data_dict[u'password1']
and data_dict[u'password2']) or email_changed:
# getting the identity for current logged user
identity = {
u'login': g.user,
u'password': data_dict[u'old_password']
}
auth = authenticator.UsernamePasswordAuthenticator()
# we are checking if the identity is not the
# same with the current logged user if so raise error.
if auth.authenticate(request.environ, identity) != g.user:
errors: ErrorDict = {
u'oldpassword': [_(u'Password entered was incorrect')]
}
error_summary = {_(u'Old Password'): _(u'incorrect password')}\
if not g.userobj.sysadmin \
else {_(u'Sysadmin Password'): _(u'incorrect password')}
return self.get(id, data_dict, errors, error_summary)
try:
user = logic.get_action(u'user_update')(context, data_dict)
except logic.NotAuthorized:
base.abort(403, _(u'Unauthorized to edit user %s') % id)
except logic.NotFound:
base.abort(404, _(u'User not found'))
except logic.ValidationError as e:
errors = e.error_dict
error_summary = e.error_summary
return self.get(id, data_dict, errors, error_summary)
h.flash_success(_(u'Profile updated'))
resp = h.redirect_to(u'user.read', id=user[u'name'])
if current_user and data_dict[u'name'] != old_username:
# Changing currently logged in user's name.
# Update repoze.who cookie to match
set_repoze_user(data_dict[u'name'], resp)
return resp
def read(package_type, id, resource_id):
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'auth_user_obj': g.userobj,
u'for_view': True
}
try:
package = get_action(u'package_show')(context, {u'id': id})
except (NotFound, NotAuthorized):
return base.abort(404, _(u'Dataset not found'))
resource = None
for res in package.get(u'resources', []):
if res[u'id'] == resource_id:
resource = res
break
if not resource:
return base.abort(404, _(u'Resource not found'))
# get package license info
license_id = package.get(u'license_id')
try:
package[u'isopen'] = model.Package.get_license_register(
)[license_id].isopen()
except KeyError:
package[u'isopen'] = False
resource_views = get_action(u'resource_view_list')(context, {
u'id': resource_id
})
resource[u'has_views'] = len(resource_views) > 0
current_resource_view = None
view_id = request.args.get(u'view_id')
if resource[u'has_views']:
if view_id:
current_resource_view = [
rv for rv in resource_views if rv[u'id'] == view_id
]
if len(current_resource_view) == 1:
current_resource_view = current_resource_view[0]
else:
return base.abort(404, _(u'Resource view not found'))
else:
current_resource_view = resource_views[0]
# required for nav menu
pkg = context[u'package']
dataset_type = pkg.type or package_type
# TODO: remove
g.package = package
g.resource = resource
g.pkg = pkg
g.pkg_dict = package
extra_vars = {
u'resource_views': resource_views,
u'current_resource_view': current_resource_view,
u'dataset_type': dataset_type,
u'pkg_dict': package,
u'package': package,
u'resource': resource,
u'pkg': pkg
}
template = _get_pkg_template(u'resource_template', dataset_type)
return base.render(template, extra_vars)
remote_ip = request.environ.get('REMOTE_ADDR',
'Unknown IP Address')
if (_check_recaptcha(remote_ip, recaptcha_response)):
user = get_action('user_create')(context, data_dict)
else:
error_msg = _(u'Bad Captcha. Please try again.')
h.flash_error(error_msg)
return self.new(data_dict)
except NotAuthorized:
abort(403, _('Unauthorized to create user %s') % '')
except NotFound, e:
abort(404, _('User not found'))
except DataError:
abort(400, _(u'Integrity Error'))
except ValidationError, e:
errors = e.error_dict
error_summary = e.error_summary
return self.new(data_dict, errors, error_summary)
h.flash_success(
_('A confirmation email has been sent to %s. '
'Please use the link in the email to continue.') %
data_dict['email'])
if not c.user:
# Do not log in the user programatically
# set_repoze_user(data_dict['name'])
if came_from:
h.redirect_to(came_from)
else:
