def read(self, id, format='html'): # Check we know the content type, if not then it is likely a revision # and therefore we should merge the format onto the end of id ctype,extension,loader = self._content_type_for_format(format) if not ctype: # Reconstitute the ID if we don't know what content type to use ctype = "text/html; charset=utf-8" id = "%s.%s" % (id, format) format = 'html' else: format = extension response.headers['Content-Type'] = ctype package_type = self._get_package_type(id.split('@')[0]) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, 'for_view': True} data_dict = {'id': id} # interpret @<revision_id> or @<date> suffix split = id.split('@') if len(split) == 2: data_dict['id'], revision_ref = split if model.is_id(revision_ref): context['revision_id'] = revision_ref else: try: date = date_str_to_datetime(revision_ref) context['revision_date'] = date except TypeError, e: abort(400, _('Invalid revision format: %r') % e.args) except ValueError, e: abort(400, _('Invalid revision format: %r') % e.args)
def __before__(self, action, **params): super(AdminController, self).__before__(action, **params) context = {'model': model, 'user': c.user} if not ckan.new_authz.is_authorized('sysadmin', context, {})['success']: base.abort(401, _('Need to be system administrator to administer')) c.revision_change_state_allowed = True
def read(self, id): group_type = self._get_group_type(id.split('@')[0]) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': self._form_to_db_schema(group_type=type)} data_dict = {'id': id} q = c.q = request.params.get('q', '') # unicode format (decoded from utf8) try: c.group_dict = get_action('group_show')(context, data_dict) c.group = context['group'] except NotFound: abort(404, _('Group not found')) except NotAuthorized: abort(401, _('Unauthorized to read group %s') % id) # Search within group q += ' groups: "%s"' % c.group_dict.get('name') try: description_formatted = ckan.misc.MarkdownFormat().to_html(c.group_dict.get('description','')) c.description_formatted = genshi.HTML(description_formatted) except Exception, e: error_msg = "<span class='inline-warning'>%s</span>" % _("Cannot render description") c.description_formatted = genshi.HTML(error_msg)
def revisions(self): ''' Similar to the revision search API, lists all revisions for which a dataset or group changed in some way. URL Params: since-revision-id since-timestamp (utc) in-the-last-x-minutes ''' # parse options rev_id = request.params.get('since-revision-id') since_timestamp = request.params.get('since-timestamp') in_the_last_x_minutes = request.params.get('in-the-last-x-minutes') now = datetime.datetime.utcnow() if rev_id is not None: rev = model.Session.query(model.Revision).get(rev_id) if not rev: abort(400, 'Revision ID "%s" does not exist' % rev_id) since_timestamp = rev.timestamp elif since_timestamp is not None: try: since_timestamp = date_str_to_datetime(since_timestamp) except (ValueError, TypeError), inst: example = now.strftime('%Y-%m-%d%%20%H:%M') # e.g. 2013-11-30%2023:15 abort(400, 'Could not parse timestamp "%s": %s. Must be UTC. Example: since-time=%s' % (since_timestamp, inst, example))
def history_ajax(self, id): context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True,} data_dict = {'id':id} try: pkg_revisions = get_action('package_revision_list')(context, data_dict) except NotAuthorized: abort(401, _('Unauthorized to read package %s') % '') except NotFound: abort(404, _('Dataset not found')) data = [] approved = False for num, revision in enumerate(pkg_revisions): if not approved and revision['approved_timestamp']: current_approved, approved = True, True else: current_approved = False data.append({'revision_id': revision['id'], 'message': revision['message'], 'timestamp': revision['timestamp'], 'author': revision['author'], 'approved': bool(revision['approved_timestamp']), 'current_approved': current_approved}) response.headers['Content-Type'] = 'application/json;charset=utf-8' return json.dumps(data)
def new(self,data = None,errors = None, error_summary = None): context = {'model': model, 'user': c.user or c.author, 'auth_user_obj': c.userobj} try: p.toolkit.check_access('harvest_source_create', context) except p.toolkit.NotAuthorized: abort(401, _('Unauthorized to create a harvest source')) if ('save' in request.params) and not data: return self._save_new() # #1433 URL params pre-populate fields param_data = {} for field_name in ('url', 'type', 'title', 'description', 'publisher_id'): if field_name in request.params: param_data[field_name] = request.params[field_name] data = data or param_data or {} errors = errors or {} error_summary = error_summary or {} try: harvesters_info = p.toolkit.get_action('harvesters_info_show')(context,{}) except p.toolkit.NotAuthorized,e: abort(401,self.not_auth_message)
def new(self, data=None, errors=None, error_summary=None): group_type = self._guess_group_type(True) if data: data['type'] = group_type context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, 'save': 'save' in request.params, 'parent': request.params.get('parent', None)} try: check_access('group_create', context) except NotAuthorized: abort(401, _('Unauthorized to create a group')) if context['save'] and not data: return self._save_new(context, group_type) data = data or {} errors = errors or {} error_summary = error_summary or {} vars = {'data': data, 'errors': errors, 'error_summary': error_summary} self._setup_template_variables(context, data, group_type=group_type) c.form = render(self._group_form(group_type=group_type), extra_vars=vars) return render(self._new_template(group_type))
def register(self, data=None, errors=None, error_summary=None): '''GET to display a form for registering a new user. or POST the form data to actually do the user registration. The bulk of this code is pulled directly from ckan/controlllers/user.py ''' context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'schema': schema.user_new_form_schema(), 'save': 'save' in request.params} try: check_access('user_create', context) except NotAuthorized: abort(401, _('Unauthorized to create a user')) if context['save'] and not data: uc = UserController() return uc._save_new(context) if c.user and not data: # #1799 Don't offer the registration form if already logged in return render('user/logout_first.html') data = data or {} errors = errors or {} error_summary = error_summary or {} vars = {'data': data, 'errors': errors, 'error_summary': error_summary} c.is_sysadmin = new_authz.is_sysadmin(c.user) c.form = render('user/new_user_form.html', extra_vars=vars) return render('user/new.html')
def organization_index(self): context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'with_private': False} data_dict = {'all_fields': True} try: check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # pass user info to context as needed to view private datasets of # orgs correctly if c.userobj: context['user_id'] = c.userobj.id context['user_is_admin'] = c.userobj.sysadmin results = get_action('organization_list')(context, data_dict) def org_key(org): title = org['title'].split(' | ')[-1 if c.language == 'fr' else 0] return normalize_strip_accents(title) results.sort(key=org_key) c.page = Page( collection=results, page=request.params.get('page', 1), url=h.pager_url, items_per_page=1000 ) return render('organization/index.html')
def index(self): context = {'model':model, 'user':c.user,'session':model.Session} try: # Request all harvest sources c.sources = get_action('harvest_source_list')(context,{}) except NotAuthorized,e: abort(401,self.not_auth_message)
def view(self, url): """By default, the final controller tried to fulfill the request when no other routes match. It may be used to display a template when all else fails, e.g.:: def view(self, url): return render('/%s' % url) Or if you're using Mako and want to explicitly send a 404 (Not Found) response code when the requested template doesn't exist:: import mako.exceptions def view(self, url): try: return render('/%s' % url) except mako.exceptions.TopLevelLookupException: abort(404) By default this controller aborts the request with a 404 (Not Found) """ try: return base.render(url) except ckan.lib.render.TemplateNotFound: if url.endswith('.html'): base.abort(404) url += '.html' try: return base.render(url) except ckan.lib.render.TemplateNotFound: base.abort(404)
def new (self, data=None, errors=None, error_summary=None): #q = model.Session.query(model.User).filter(model.User.sysadmin==True) #c.sysadmins = [a.name for a in q.all()] '''GET to display a form for registering a new user. or POST the form data to actually do the user registration. ''' context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj, 'schema': self._new_form_to_db_schema(), 'save': 'save' in request.params} c.is_sysadmin = new_authz.is_sysadmin(c.user) if not c.user or not c.is_sysadmin: return base.render('user/logout_first.html') try: logic.check_access('user_create', context) except logic.NotAuthorized: base.abort(401, _('Unauthorized to create a user')) if context['save'] and not data: return self._save_new(context) c.data = data or {} c.errors = errors or {} c.error_summary = error_summary or {} #vars = {'data': data, 'errors': errors, 'error_summary': error_summary} #c.form = render(self.new_user_form, extra_vars=vars) #return render('user/new.html') return base.render('muser/new.html')
def index (self): LIMIT = 20 page = int(request.params.get('page', 1)) c.q = request.params.get('q', '') c.order_by = request.params.get('order_by', 'name') context = {'return_query': True, 'user': c.user or c.author, 'auth_user_obj': c.userobj} data_dict = {'q': c.q, 'limit': LIMIT, 'offset': (page - 1) * LIMIT, 'order_by': c.order_by} try: logic.check_access('user_list', context, data_dict) except logic.NotAuthorized: base.abort(401, _('Not authorized to see this page')) users_list = logic.get_action('user_list')(context, data_dict) c.users = users_list c.page = h.Page( collection=users_list, page=page, url=h.pager_url, item_count=users_list.count(), items_per_page=LIMIT ) return base.render('muser/index.html')
def serve(self, path): root = os.path.join(config.get('ckanext-archiver.archive_dir', '/tmp'), path) if not os.path.exists(root): abort(404) response.content_type = 'application/json' return str(open(root).read())
def dataset_csv(self, id='all', month='all'): ''' Returns a CSV with the number of views & visits for each dataset. :param id: A Publisher ID or None if you want for all :param month: The time period, or 'all' ''' c.month = month if not month == 'all' else '' if id != 'all': c.publisher = model.Group.get(id) if not c.publisher: abort(404, 'A publisher with that name could not be found') packages = self._get_packages(publisher=c.publisher, month=c.month) response.headers['Content-Type'] = "text/csv; charset=utf-8" response.headers['Content-Disposition'] = \ str('attachment; filename=datasets_%s_%s.csv' % (c.publisher_name, month,)) writer = csv.writer(response) writer.writerow(["Dataset Title", "Dataset Name", "Views", "Visits", "Resource downloads", "Period Name"]) for package,view,visit,downloads in packages: writer.writerow([package.title.encode('utf-8'), package.name.encode('utf-8'), view, visit, downloads, month])
def contracts_archive(self, relative_url='/'): import requests from pylons import response if request.method == 'POST': abort(405) # Method Not Allowed headers = {'X-Script-Name': '/data/contracts-finder-archive'} contracts_url = pylons.config.get('dgu.contracts_url') url = urlparse.urljoin(contracts_url, relative_url) r = requests.get(url, headers=headers, params=dict(request.params), stream=True) if r.status_code != 200: abort(r.status_code) if relative_url.startswith(('/static/', '/download/')): # CSS will only get loaded if it has the right content type response.content_type = r.headers.get('Content-Type', 'text/html') if r.headers.get('Content-Disposition'): response.headers['Content-Disposition'] = r.headers.get('Content-Disposition') return r.raw.read() # Some of the static files are binary else: extra_vars = {'content': r.text} return render('contracts_archive/front_page.html', extra_vars=extra_vars)
def _parse_url_params(self): """ Constructs a search-query dict from the URL query parameters. Returns the constructed search-query dict, and the valid URL query parameters. """ try: page = int(request.params.get('page', 1)) or 1 except ValueError: base.abort(400, _('"page" parameter must be a positive integer')) if page < 0: base.abort(400, _('"page" parameter must be a positive integer')) limit = ITEMS_LIMIT data_dict = { 'start': (page - 1) * limit, 'rows': limit } # Filter ignored query parameters valid_params = ['page'] params = dict((p, request.params.get(p)) for p in valid_params if p in request.params) return data_dict, params
def refresh(self, id): try: context = {'model': model, 'user': c.user, 'session': model.Session} p.toolkit.get_action('harvest_job_create')( context, {'source_id': id, 'run': True} ) h.flash_success( _('Harvest will start shortly. Refresh this page for updates.') ) except p.toolkit.ObjectNotFound: abort(404, _('Harvest source not found')) except p.toolkit.NotAuthorized: abort(401, self.not_auth_message) except HarvestSourceInactiveError: h.flash_error( _( 'Cannot create new harvest jobs on inactive ' 'sources. First, please change the source status ' 'to "active".' ) ) except HarvestJobExists: h.flash_notice( _('A harvest job has already been scheduled for ' 'this source') ) except Exception as e: msg = 'An error occurred: [%s]' % str(e) h.flash_error(msg) h.redirect_to(h.url_for('{0}_admin'.format(DATASET_TYPE_NAME), id=id))
def openspending_report(self): if not dgu_helpers.is_sysadmin(): abort(401, 'User must be a sysadmin to view this page.') self._set_openspending_reports_dir() c.content = open (c.openspending_report_dir + "/index.html").read() return render('data/openspending_report.html')
def comment_update_moderation(context, data_dict): import ckanext.comments.model as comment_model model = context['model'] user = context['user'] cid = logic.get_or_bust(data_dict, 'id') comment = comment_model.Comment.get(cid) if not comment: abort(404) # If sysadmin, then remove instantly if c.userobj.sysadmin: if len(comment.children) > 0: txt = config.get('ckan.comments.deleted.text', 'This message was deleted') comment.comment = txt else: comment.state = 'deleted' comment.approval_status = comment_model.COMMENT_PENDING comment.moderated_by = c.userobj.id comment.moderation_date = datetime.datetime.now() model.Session.add(comment) model.Session.commit() elif not comment.moderated_by: comment.spam_votes = comment.spam_votes + 1 comment.approval_status = comment_model.COMMENT_PENDING model.Session.add(comment) model.Session.commit() return {}
def __before__(self, action, **env): base.BaseController.__before__(self, action, **env) try: context = {'model': model, 'user': c.user or c.author} logic.check_access('site_read', context) except logic.NotAuthorized: base.abort(401, _('Not authorized to see this page'))
def member_new(self, id): context = {'model': model, 'session': model.Session, 'user': c.user or c.author} #self._check_access('group_delete', context, {'id': id}) try: if request.method == 'POST': data_dict = clean_dict(unflatten( tuplize_dict(parse_params(request.params)))) data_dict['id'] = id c.group_dict = self._action('group_member_create')(context, data_dict) self._redirect_to(controller='group', action='members', id=id) else: user = request.params.get('user') if user: c.user_dict = get_action('user_show')(context, {'id': user}) c.user_role = ckan.new_authz.users_role_for_group_or_org(id, user) or 'member' else: c.user_role = 'member' c.group_dict = self._action('group_show')(context, {'id': id}) c.roles = self._action('member_roles_list')(context, {}) except NotAuthorized: abort(401, _('Unauthorized to add member to group %s') % '') except NotFound: abort(404, _('Group not found')) return self._render_template('group/member_new.html')
def resource_read(self, id, resource_id): context = {"model": model, "session": model.Session, "user": c.user or c.author} try: c.resource = get_action("resource_show")(context, {"id": resource_id}) c.package = get_action("package_show")(context, {"id": id}) # required for nav menu c.pkg = context["package"] c.resource_json = json.dumps(c.resource) c.pkg_dict = c.package except NotFound: abort(404, _("Resource not found")) except NotAuthorized: abort(401, _("Unauthorized to read resource %s") % id) # get package license info license_id = c.package.get("license_id") try: c.package["isopen"] = model.Package.get_license_register()[license_id].isopen() except KeyError: c.package["isopen"] = False # TODO: find a nicer way of doing this c.datastore_api = "%s/api/action" % config.get("ckan.site_url", "").rstrip("/") c.related_count = c.pkg.related_count return render("package/resource_read.html")
def read_ajax(self, id, revision=None): package_type = self._get_package_type(id) context = { "model": model, "session": model.Session, "user": c.user or c.author, "extras_as_string": True, "schema": self._form_to_db_schema(package_type=package_type), "revision_id": revision, } try: data = get_action("package_show")(context, {"id": id}) schema = self._db_to_form_schema(package_type=package_type) if schema: data, errors = validate(data, schema) except NotAuthorized: abort(401, _("Unauthorized to read package %s") % "") except NotFound: abort(404, _("Dataset not found")) ## hack as db_to_form schema should have this data["tag_string"] = ", ".join([tag["name"] for tag in data.get("tags", [])]) data.pop("tags") data = flatten_to_string_key(data) response.headers["Content-Type"] = "application/json;charset=utf-8" return json.dumps(data)
def history_ajax(self, id): context = {"model": model, "session": model.Session, "user": c.user or c.author, "extras_as_string": True} data_dict = {"id": id} try: pkg_revisions = get_action("package_revision_list")(context, data_dict) except NotAuthorized: abort(401, _("Unauthorized to read package %s") % "") except NotFound: abort(404, _("Dataset not found")) data = [] approved = False for num, revision in enumerate(pkg_revisions): if not approved and revision["approved_timestamp"]: current_approved, approved = True, True else: current_approved = False data.append( { "revision_id": revision["id"], "message": revision["message"], "timestamp": revision["timestamp"], "author": revision["author"], "approved": bool(revision["approved_timestamp"]), "current_approved": current_approved, } ) response.headers["Content-Type"] = "application/json;charset=utf-8" return json.dumps(data)
def download(package_type, id, resource_id, filename=None): """ Provides a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = { u'model': model, u'session': model.Session, u'user': g.user, u'auth_user_obj': g.userobj } try: rsc = get_action(u'resource_show')(context, {u'id': resource_id}) get_action(u'package_show')(context, {u'id': id}) except (NotFound, NotAuthorized): return base.abort(404, _(u'Resource not found')) if rsc.get(u'url_type') == u'upload': upload = uploader.get_resource_uploader(rsc) filepath = upload.get_path(rsc[u'id']) return flask.send_file(filepath) elif u'url' not in rsc: return base.abort(404, _(u'No download is available')) return h.redirect_to(rsc[u'url'])
def get(self, package_type, id, resource_id): context = self._prepare(id) try: resource_dict = get_action(u'resource_show')( context, { u'id': resource_id } ) pkg_id = id except NotAuthorized: return base.abort( 403, _(u'Unauthorized to delete resource %s') % u'' ) except NotFound: return base.abort(404, _(u'Resource not found')) # TODO: remove g.resource_dict = resource_dict g.pkg_id = pkg_id return base.render( u'package/confirm_delete_resource.html', { u'dataset_type': _get_package_type(id), u'resource_dict': resource_dict, u'pkg_id': pkg_id } )
def index(self): group_type = self._guess_group_type() context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'for_view': True, 'with_private': False} q = c.q = request.params.get('q', '') data_dict = {'all_fields': True, 'q': q,} sort_by = c.sort_by_selected = request.params.get('sort') if sort_by: data_dict['sort'] = sort_by try: self._check_access('site_read', context) except NotAuthorized: abort(401, _('Not authorized to see this page')) # pass user info to context as needed to view private datasets of # orgs correctly if c.userobj: context['user_id'] = c.userobj.id context['user_is_admin'] = c.userobj.sysadmin results = self._action('group_list')(context, data_dict) c.page = h.Page( collection=results, page=request.params.get('page', 1), url=h.pager_url, items_per_page=21 ) return render(self._index_template(group_type))
def comment_update(context, data_dict): model = context['model'] logic.check_access("comment_update", context, data_dict) cid = logic.get_or_bust(data_dict, 'id') comment = comment_model.Comment.get(cid) if not comment: abort(404) # Validate that we have the required fields. if not all([data_dict.get('comment')]): raise logic.ValidationError("Comment text is required") # Cleanup the comment cleaned_comment = util.clean_input(data_dict.get('comment')) comment.subject = data_dict.get('subject') comment.comment = cleaned_comment comment.modified_date = datetime.datetime.now() model.Session.add(comment) model.Session.commit() return comment.as_dict()
def render_request_form(self, pkg_id): """ Render the access request contact form if allowed. :param pkg_id: package id :type pkg_id: string """ c.package = Package.get(pkg_id) if asbool(config.get('kata.disable_contact')): h.flash_error(_(u"Sending contact emails is prohibited for now. " u"Please try again later or contact customer service.")) return redirect(h.url_for(controller='package', action="read", id=c.package.name)) if not c.package: abort(404, _(u"Dataset not found")) contacts = utils.get_package_contacts(c.package.id) c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts] c.recipient_index = request.params.get('recipient', '') c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time()))))) return render('contact/dataset_request_form.html')
class YtpDatasetController(PackageController): def ytp_tag_autocomplete(self): """ CKAN autocomplete discards vocabulary_id from request. This is modification from tag_autocomplete function from CKAN. Takes vocabulary_id as parameter. """ q = request.params.get('incomplete', '') limit = request.params.get('limit', 10) vocabulary_id = request.params.get('vocabulary_id', None) tag_names = [] if q: context = {'model': model, 'session': model.Session, 'user': c.user or c.author} data_dict = {'q': q, 'limit': limit} if vocabulary_id: data_dict['vocabulary_id'] = vocabulary_id try: tag_names = get_action('tag_autocomplete')(context, data_dict) except NotFound: pass # return empty when vocabulary is not found resultSet = { 'ResultSet': { 'Result': [{'Name': tag} for tag in tag_names] } } status_int = 200 response.status_int = status_int response.headers['Content-Type'] = 'application/json;charset=utf-8' return helpers.json.dumps(resultSet) def new_metadata(self, id, data=None, errors=None, error_summary=None): """ Fake metadata creation. Change status to active and redirect to read. """ context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} data_dict = get_action('package_show')(context, {'id': id}) data_dict['id'] = id data_dict['state'] = 'active' context['allow_state_change'] = True get_action('package_update')(context, data_dict) success_message = ('<div style="display: inline-block"><p>' + _("Dataset was saved successfully.") + '</p>' + '<p>' + _("Fill additional info") + ':</p>' + '<p><a href="/data/' + h.lang() + '/dataset/' + data_dict.get('name') + '/related/new">>' + _("Add related") + '</a></p>' + '<p><a href="/data/' + h.lang() + '/dataset/edit/' + data_dict.get('name') + '">>' + _("Edit or add language versions") + '</a> ' + '<a href="/data/' + h.lang() + '/dataset/delete/' + id + '">>' + _('Delete') + '</a></p>' + '<p><a href="/data/' + h.lang() + '/dataset/new/">' + _('Create Dataset') + '</a></p></div>') helpers.flash_success(success_message, True) helpers.redirect_to(controller='package', action='read', id=id) # Modified from original ckan function def edit(self, id, data=None, errors=None, error_summary=None): package_type = self._get_package_type(id) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj, 'save': 'save' in request.params} if context['save'] and not data: return self._save_edit(id, context, package_type=package_type) try: c.pkg_dict = get_action('package_show')(context, {'id': id}) context['for_edit'] = True old_data = get_action('package_show')(context, {'id': id}) # old data is from the database and data is passed from the # user if there is a validation error. Use users data if there. if data: old_data.update(data) data = old_data except NotAuthorized: abort(401, _('Unauthorized to read package %s') % '') except NotFound: abort(404, _('Dataset not found')) # are we doing a multiphase add? if data.get('state', '').startswith('draft') and len(data.get('resources')) == 0: c.form_action = h.url_for(controller='package', action='new') c.form_style = 'new' return self.new(data=data, errors=errors, error_summary=error_summary) c.pkg = context.get("package") c.resources_json = h.json.dumps(data.get('resources', [])) try: check_access('package_update', context) except NotAuthorized: abort(401, _('User %r not authorized to edit %s') % (c.user, id)) # convert tags if not supplied in data if data and not data.get('tag_string'): data['tag_string'] = ', '.join(h.dict_list_reduce( c.pkg_dict.get('tags', {}), 'name')) errors = errors or {} form_snippet = self._package_form(package_type=package_type) form_vars = {'data': data, 'errors': errors, 'error_summary': error_summary, 'action': 'edit', 'dataset_type': package_type, } c.errors_json = h.json.dumps(errors) self._setup_template_variables(context, {'id': id}, package_type=package_type) c.related_count = c.pkg.related_count # we have already completed stage 1 form_vars['stage'] = ['active'] if data.get('state', '').startswith('draft') and len(data.get('resources')) == 0: form_vars['stage'] = ['active', 'complete'] edit_template = self._edit_template(package_type) c.form = ckan.lib.render.deprecated_lazy_render( edit_template, form_snippet, lambda: render(form_snippet, extra_vars=form_vars), 'use of c.form is deprecated. please see ' 'ckan/templates/package/edit.html for an example ' 'of the new way to include the form snippet' ) return render(edit_template, extra_vars={'form_vars': form_vars, 'form_snippet': form_snippet, 'dataset_type': package_type}) # original ckan new resource def new_resource(self, id, data=None, errors=None, error_summary=None): ''' FIXME: This is a temporary action to allow styling of the forms. ''' if request.method == 'POST' and not data: save_action = request.params.get('save') data = data or clean_dict(dict_fns.unflatten(tuplize_dict(parse_params( request.POST)))) # we don't want to include save as it is part of the form del data['save'] resource_id = data['id'] del data['id'] context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} # see if we have any data that we are trying to save data_provided = False for key, value in data.iteritems(): if (value or isinstance(value, cgi.FieldStorage)) and key != 'resource_type': data_provided = True break if not data_provided and save_action != "go-dataset-complete": if save_action == 'go-dataset': # go to final stage of adddataset h.redirect_to(controller='package', action='edit', id=id) # see if we have added any resources try: data_dict = get_action('package_show')(context, {'id': id}) except NotAuthorized: abort(401, _('Unauthorized to update dataset')) except NotFound: abort(404, _('The dataset {id} could not be found.').format(id=id)) if not len(data_dict['resources']): # no data so keep on page msg = _('You must add at least one data resource') # On new templates do not use flash message if g.legacy_templates: h.flash_error(msg) h.redirect_to(controller='package', action='new_resource', id=id) else: errors = {} error_summary = {_('Error'): msg} return self.new_resource(id, data, errors, error_summary) # we have a resource so let them add metadata h.redirect_to(controller='package', action='new_metadata', id=id) data['package_id'] = id try: if resource_id: data['id'] = resource_id get_action('resource_update')(context, data) else: get_action('resource_create')(context, data) except ValidationError, e: errors = e.error_dict error_summary = e.error_summary return self.new_resource(id, data, errors, error_summary) except NotAuthorized: abort(401, _('Unauthorized to create a resource')) except NotFound: abort(404, _('The dataset {id} could not be found.').format(id=id))
def before_request(): try: context = dict(model=model, user=g.user, auth_user_obj=g.userobj) logic.check_access(u'sysadmin', context) except logic.NotAuthorized: base.abort(403, _(u'Need to be system administrator to administer'))
def post(self, id, group_type, is_organization, data=None): set_org(is_organization) context = self._prepare(group_type, id) data_dict = {u'id': id, u'type': group_type} try: # Do not query for the group datasets when dictizing, as they will # be ignored and get requested on the controller anyway data_dict['include_datasets'] = False group_dict = _action(u'group_show')(context, data_dict) group = context['group'] except NotFound: group_label = h.humanize_entity_type( u'organization' if is_organization else u'group', group_type, u'default label') or _( u'Organization' if is_organization else u'Group') base.abort(404, _(u'{} not found'.format(group_label))) except NotAuthorized: base.abort(403, _(u'User %r not authorized to edit %s') % (g.user, id)) if not group_dict['is_organization']: # FIXME: better error raise Exception(u'Must be an organization') # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.group_dict = group_dict g.group = group # use different form names so that ie7 can be detected form_names = set([ u"bulk_action.public", u"bulk_action.delete", u"bulk_action.private" ]) actions_in_form = set(request.form.keys()) actions = form_names.intersection(actions_in_form) # ie7 puts all buttons in form params but puts submitted one twice for key, value in request.form.to_dict().items(): if value in [u'private', u'public']: action = key.split(u'.')[-1] break else: # normal good browser form submission action = actions.pop().split(u'.')[-1] # process the action first find the datasets to perform the action on. # they are prefixed by dataset_ in the form data datasets = [] for param in request.form: if param.startswith(u'dataset_'): datasets.append(param[8:]) action_functions = { u'private': u'bulk_update_private', u'public': u'bulk_update_public', u'delete': u'bulk_update_delete', } data_dict = {u'datasets': datasets, u'org_id': group_dict['id']} try: get_action(action_functions[action])(context, data_dict) except NotAuthorized: base.abort(403, _(u'Not authorized to perform bulk update')) return h.redirect_to(u'{}.bulk_process'.format(group_type), id=id)
def changes_multiple(is_organization, group_type=None): ''' Called when a user specifies a range of versions they want to look at changes between. Verifies that the range is valid and finds the set of activity diffs for the changes in the given version range, then re-renders changes.html with the list. ''' set_org(is_organization) extra_vars = {} new_id = h.get_request_param(u'new_id') old_id = h.get_request_param(u'old_id') context = { u'model': model, u'session': model.Session, u'user': g.user, u'auth_user_obj': g.userobj } # check to ensure that the old activity is actually older than # the new activity old_activity = get_action(u'activity_show')(context, { u'id': old_id, u'include_data': False}) new_activity = get_action(u'activity_show')(context, { u'id': new_id, u'include_data': False}) old_timestamp = old_activity[u'timestamp'] new_timestamp = new_activity[u'timestamp'] t1 = datetime.strptime(old_timestamp, u'%Y-%m-%dT%H:%M:%S.%f') t2 = datetime.strptime(new_timestamp, u'%Y-%m-%dT%H:%M:%S.%f') time_diff = t2 - t1 # if the time difference is negative, just return the change that put us # at the more recent ID we were just looking at # TODO: do something better here - go back to the previous page, # display a warning that the user can't look at a sequence where # the newest item is older than the oldest one, etc if time_diff.total_seconds() < 0: return changes(h.get_request_param(u'current_new_id')) done = False current_id = new_id diff_list = [] while not done: try: activity_diff = get_action(u'activity_diff')( context, { u'id': current_id, u'object_type': u'group', u'diff_type': u'html'}) except NotFound as e: log.info( u'Activity not found: {} - {}'.format(str(e), current_id) ) return base.abort(404, _(u'Activity not found')) except NotAuthorized: return base.abort(403, _(u'Unauthorized to view activity data')) diff_list.append(activity_diff) if activity_diff['activities'][0]['id'] == old_id: done = True else: current_id = activity_diff['activities'][0]['id'] group_id = diff_list[0][u'activities'][1][u'data'][u'group'][u'id'] current_group_dict = get_action(group_type + u'_show')( context, {u'id': group_id}) group_activity_list = get_action(group_type + u'_activity_list')(context, { u'id': group_id, u'limit': 100}) extra_vars = { u'activity_diffs': diff_list, u'group_dict': current_group_dict, u'group_activity_list': group_activity_list, u'group_type': current_group_dict[u'type'], } return base.render(_replace_group_org(u'group/changes.html'), extra_vars)
def index(group_type, is_organization): extra_vars = {} set_org(is_organization) page = h.get_page_number(request.params) or 1 items_per_page = int(config.get(u'ckan.datasets_per_page', 20)) context = { u'model': model, u'session': model.Session, u'user': g.user, u'for_view': True, u'with_private': False } try: _check_access(u'site_read', context) _check_access(u'group_list', context) except NotAuthorized: base.abort(403, _(u'Not authorized to see this page')) q = request.params.get(u'q', u'') sort_by = request.params.get(u'sort') # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.q = q g.sort_by_selected = sort_by extra_vars["q"] = q extra_vars["sort_by_selected"] = sort_by # pass user info to context as needed to view private datasets of # orgs correctly if g.userobj: context['user_id'] = g.userobj.id context['user_is_admin'] = g.userobj.sysadmin try: data_dict_global_results = { u'all_fields': False, u'q': q, u'sort': sort_by, u'type': group_type or u'group', } global_results = _action(u'group_list')(context, data_dict_global_results) except ValidationError as e: if e.error_dict and e.error_dict.get(u'message'): msg = e.error_dict['message'] else: msg = str(e) h.flash_error(msg) extra_vars["page"] = h.Page([], 0) extra_vars["group_type"] = group_type return base.render( _get_group_template(u'index_template', group_type), extra_vars) data_dict_page_results = { u'all_fields': True, u'q': q, u'sort': sort_by, u'type': group_type or u'group', u'limit': items_per_page, u'offset': items_per_page * (page - 1), u'include_extras': True } page_results = _action(u'group_list')(context, data_dict_page_results) extra_vars["page"] = h.Page( collection=global_results, page=page, url=h.pager_url, items_per_page=items_per_page, ) extra_vars["page"].items = page_results extra_vars["group_type"] = group_type # TODO: Remove # ckan 2.9: Adding variables that were removed from c object for # compatibility with templates in existing extensions g.page = extra_vars["page"] return base.render( _get_group_template(u'index_template', group_type), extra_vars)
def edit(self, id, data=None, errors=None, error_summary=None): package_type = self._get_package_type(id) context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj, 'save': 'save' in request.params} if context['save'] and not data: return self._save_edit(id, context, package_type=package_type) try: c.pkg_dict = get_action('package_show')(context, {'id': id}) context['for_edit'] = True old_data = get_action('package_show')(context, {'id': id}) # old data is from the database and data is passed from the # user if there is a validation error. Use users data if there. if data: old_data.update(data) data = old_data except NotAuthorized: abort(401, _('Unauthorized to read package %s') % '') except NotFound: abort(404, _('Dataset not found')) # are we doing a multiphase add? if data.get('state', '').startswith('draft') and len(data.get('resources')) == 0: c.form_action = h.url_for(controller='package', action='new') c.form_style = 'new' return self.new(data=data, errors=errors, error_summary=error_summary) c.pkg = context.get("package") c.resources_json = h.json.dumps(data.get('resources', [])) try: check_access('package_update', context) except NotAuthorized: abort(401, _('User %r not authorized to edit %s') % (c.user, id)) # convert tags if not supplied in data if data and not data.get('tag_string'): data['tag_string'] = ', '.join(h.dict_list_reduce( c.pkg_dict.get('tags', {}), 'name')) errors = errors or {} form_snippet = self._package_form(package_type=package_type) form_vars = {'data': data, 'errors': errors, 'error_summary': error_summary, 'action': 'edit', 'dataset_type': package_type, } c.errors_json = h.json.dumps(errors) self._setup_template_variables(context, {'id': id}, package_type=package_type) c.related_count = c.pkg.related_count # we have already completed stage 1 form_vars['stage'] = ['active'] if data.get('state', '').startswith('draft') and len(data.get('resources')) == 0: form_vars['stage'] = ['active', 'complete'] edit_template = self._edit_template(package_type) c.form = ckan.lib.render.deprecated_lazy_render( edit_template, form_snippet, lambda: render(form_snippet, extra_vars=form_vars), 'use of c.form is deprecated. please see ' 'ckan/templates/package/edit.html for an example ' 'of the new way to include the form snippet' ) return render(edit_template, extra_vars={'form_vars': form_vars, 'form_snippet': form_snippet, 'dataset_type': package_type})
def _edit_or_new(self, id, related_id, is_edit): """ Edit and New were too similar and so I've put the code together and try and do as much up front as possible. """ context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj, 'for_view': True} if is_edit: tpl = 'related/edit.html' auth_name = 'related_update' auth_dict = {'id': related_id} action_name = 'related_update' try: related = get_action('related_show')( context, {'id': related_id}) except NotFound: abort(404, _('Related item not found')) else: tpl = 'related/new.html' auth_name = 'related_create' auth_dict = {} action_name = 'related_create' try: check_access(auth_name, context, auth_dict) except NotAuthorized: abort(401, _('Not authorized')) try: c.pkg_dict = get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('Package not found')) data, errors, error_summary = {}, {}, {} if request.method == "POST": try: data = clean_dict( dict_fns.unflatten( tuplize_dict( parse_params(request.params)))) if is_edit: data['id'] = related_id else: data['dataset_id'] = id data['owner_id'] = c.userobj.id related = get_action(action_name)(context, data) if not is_edit: h.flash_success(_("Related item was successfully created")) else: h.flash_success(_("Related item was successfully updated")) h.redirect_to( controller='package', action='read', id=c.pkg_dict['name']) except dict_fns.DataError: abort(400, _(u'Integrity Error')) except ValidationError, e: errors = e.error_dict error_summary = e.error_summary
elif save_action == 'go-dataset-complete': # go to first stage of add dataset h.redirect_to(controller='package', action='read', id=id) else: # add more resources h.redirect_to(controller='package', action='new_resource', id=id) # get resources for sidebar context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} try: pkg_dict = get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('The dataset {id} could not be found.').format(id=id)) try: check_access('resource_create', context, {'package_id': pkg_dict['id']}) except NotAuthorized: abort(401, _('Unauthorized to create a resource for this package')) package_type = pkg_dict['type'] or 'dataset' errors = errors or {} error_summary = error_summary or {} vars = {'data': data, 'errors': errors, 'error_summary': error_summary, 'action': 'new', 'resource_form_snippet': self._resource_form(package_type), 'dataset_type': package_type} vars['pkg_name'] = id # required for nav menu
def _import_metadata(self, post): '''Handle a submitted import_metadata form. Return a redirection URL. ''' redirect_url = None # # Read and validate post parameters # # Note Authorization is enforced by the underlying action. owner_org = post.get('owner_org') if not owner_org: abort(400, 'The owner organization is not given') uv = {'group': owner_org} uv.update(request.urlvars) redirect_url = _url(**uv) dtype = post.get('dataset_type') if not dtype in ext_metadata.dataset_types: abort(400, 'Unknown metadata schema') rename_if_conflict = post.get('rename', '') == 'y' continue_on_errors = post.get('force_create', '') == 'y' # Examine source (an upload or a link) # Todo Remember orig_metadata_source (as source_url) source_url = None source = post.get('source') source_upload = post.get('source-upload') if source: # Assume source is provided as a URL source_url = source elif isinstance(source_upload, cgi.FieldStorage): # Assume source is an uploaded file up = uploader.MetadataUpload(source_upload.filename) up.update_data_dict(dict(post), 'source-upload') try: up.upload(max_size=1) except Exception as ex: log.error('Failed to save uploaded file %r: %s', source_upload.filename, ex.message) abort(400, 'Failed to upload file') source_url = _url( controller='ckanext.publicamundi.controllers.files:Controller', action='download_file', object_type=up.object_type, name_or_id=up.filename, filename=source_upload.filename) source = source_upload.file source.seek(0, 0) # Provide a file-like object as source #source = source_upload.file #log.debug('\n\n source: %s ,type: %s, up is %s , source url is %s, source_upload is %s \n\n', source, type(source),up, source_url, source_upload) #log.debug('\n source_upload: %s , value is %s\n\n', source_upload, source.getvalue()) #for attr in dir(source_upload): # log.info(" source upload.%s = %r" % (attr, getattr(source_upload, attr))) #for attr in dir(source): # log.info("source.%s = %r" % (attr, getattr(source, attr))) #source.seek(0, 0) else: # No source given session['error_summary'] = _( 'No source specified: Upload or link to an XML file.') return redirect_url # # Invoke dataset_import action # context = {'model': model, 'session': model.Session, 'api_version': 3} data_dict = { 'source': source, 'owner_org': owner_org, 'dtype': dtype, 'rename_if_conflict': rename_if_conflict, 'continue_on_errors': continue_on_errors, } try: result = _get_action('dataset_import')(context, data_dict) except ext_actions.Invalid as ex: log.error('Cannot import package (invalid input): %r' % (ex.error_dict)) if len(ex.error_dict) > 1: session['error_summary'] = _('Received invalid input (%s)' % (','.join(ex.error_dict.keys()))) session['errors'] = ex.error_dict else: session['error_summary'] = next(ex.error_dict.itervalues()) except (ext_actions.IdentifierConflict, ext_actions.NameConflict) as ex: log.error('Cannot import package (name/id conflict): %r' % (ex.error_dict)) session['error_summary'] = ex.error_summary except toolkit.ValidationError as ex: # The input is valid, but results in an invalid package log.error('Cannot import package (metadata are invalid): %r' % (ex.error_dict)) session['error_summary'] = _('The given metadata are invalid.') session['errors'] = ex.error_dict except AssertionError as ex: raise except Exception as ex: log.error('Cannot import package (unexpected error): %s' % (ex)) abort(400, 'Cannot import package') else: # Success: save result and redirect to success page session['result'] = result # Done return redirect_url
def new_resource(self, id, data=None, errors=None, error_summary=None): ''' FIXME: This is a temporary action to allow styling of the forms. ''' if request.method == 'POST' and not data: save_action = request.params.get('save') data = data or clean_dict(dict_fns.unflatten(tuplize_dict(parse_params( request.POST)))) # we don't want to include save as it is part of the form del data['save'] resource_id = data['id'] del data['id'] context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} # see if we have any data that we are trying to save data_provided = False for key, value in data.iteritems(): if (value or isinstance(value, cgi.FieldStorage)) and key != 'resource_type': data_provided = True break if not data_provided and save_action != "go-dataset-complete": if save_action == 'go-dataset': # go to final stage of adddataset h.redirect_to(controller='package', action='edit', id=id) # see if we have added any resources try: data_dict = get_action('package_show')(context, {'id': id}) except NotAuthorized: abort(401, _('Unauthorized to update dataset')) except NotFound: abort(404, _('The dataset {id} could not be found.').format(id=id)) if not len(data_dict['resources']): # no data so keep on page msg = _('You must add at least one data resource') # On new templates do not use flash message if g.legacy_templates: h.flash_error(msg) h.redirect_to(controller='package', action='new_resource', id=id) else: errors = {} error_summary = {_('Error'): msg} return self.new_resource(id, data, errors, error_summary) # we have a resource so let them add metadata h.redirect_to(controller='package', action='new_metadata', id=id) data['package_id'] = id try: if resource_id: data['id'] = resource_id get_action('resource_update')(context, data) else: get_action('resource_create')(context, data) except ValidationError, e: errors = e.error_dict error_summary = e.error_summary return self.new_resource(id, data, errors, error_summary) except NotAuthorized: abort(401, _('Unauthorized to create a resource'))
def get(self, package_type: str, id: str, resource_id: str, view_id: Optional[str] = None, post_extra: Optional[dict[str, Any]] = None) -> str: context, extra_vars = self._prepare(id, resource_id) to_preview = extra_vars[u'to_preview'] if post_extra: extra_vars.update(post_extra) package_type = _get_package_type(id) data = extra_vars[u'data'] if u'data' in extra_vars else None if data and u'view_type' in data: view_type = data.get(u'view_type') else: view_type = request.args.get(u'view_type') # view_id exists only when updating if view_id: if not data or not view_type: try: view_data = get_action(u'resource_view_show')( context, { u'id': view_id } ) view_type = view_data[u'view_type'] if data: data.update(view_data) else: data = view_data except (NotFound, NotAuthorized): return base.abort(404, _(u'View not found')) # might as well preview when loading good existing view if not extra_vars[u'errors']: to_preview = True if data is not None: data[u'view_type'] = view_type view_plugin = lib_datapreview.get_view_plugin(view_type) if not view_plugin: return base.abort(404, _(u'View Type Not found')) _setup_template_variables( context, {u'id': id}, package_type=package_type ) data_dict: dict[str, Any] = { u'package': extra_vars[u'pkg_dict'], u'resource': extra_vars[u'resource'], u'resource_view': data } view_template = view_plugin.view_template(context, data_dict) form_template = view_plugin.form_template(context, data_dict) extra_vars.update({ u'form_template': form_template, u'view_template': view_template, u'data': data, u'to_preview': to_preview, u'datastore_available': plugins.plugin_loaded(u'datastore') }) extra_vars.update( view_plugin.setup_template_variables(context, data_dict) or {} ) extra_vars.update(data_dict) if view_id: return base.render(u'package/edit_view.html', extra_vars) return base.render(u'package/new_view.html', extra_vars)
def list_requests(): context = _get_context() try: check_access('get_request_log', context, {}) except logic.NotAuthorized: return base.abort(403, _('Not authorized to view requests audit.')) page = _safe_int(request.args.get('page', 1), 1) limit = _safe_int(request.args.get('limit', 20), 20) if page < 1: page = 1 if limit < 1: limit = 20 if limit > 500: limit = 500 tab = request.args.get('tab', 'all') timespan = request.args.get('timespan') query = request.args.get('q', '').strip() errors = {} date_start = None date_end = None if timespan == 'last-hour': date_start = datetime.now() - timedelta(hours=1) elif timespan == 'last-day': date_start = datetime.now() - timedelta(days=1) elif timespan == 'last-week': date_start = datetime.now() - timedelta(days=7) elif timespan == 'last-month': date_start = datetime.now() - timedelta(days=30) elif timespan == 'custom': date_vals = {} for key in ['date_start', 'date_end']: value = request.args.get(key) if value: try: value = datetime.strptime(value, '%Y-%m-%d %H:%M:%S') date_vals[key] = value except Exception as e: errors[key] = [ _('Invalid value. Should be a formatted ' 'date string like "2020-10-21 10:35:43"') ] date_start = date_vals.get('date_start') date_end = date_vals.get('date_end') extra_vars = _extra_template_variables(context, { 'user_obj': g.userobj, 'user': g.userobj, }) extra_vars['errors'] = errors if tab == 'all': try: log_data = get_action('get_request_log')(context, { 'page': page, 'limit': limit, 'date_start': date_start, 'date_end': date_end, 'q': query, }) except Exception as e: log.exception(e) errors['general'] = _('An error occured while trying to ' 'get the requests log.') log_data = { 'results': [], 'count': 0, } else: report = { 'page_count': 'endpoint', 'user_count': 'remote_user', 'ip_count': 'remote_ip', }.get(tab) try: log_data = get_action('get_request_log_report')(context, { 'report': report, 'date_start': date_start, 'date_end': date_end, 'page': page, 'limit': limit, 'q': query, }) except Exception as e: log.exception(e) errors['general'] = _('An error occured while trying to ' 'get the report.') log_data = { 'results': [], 'count': 0, } extra_vars['results'] = log_data.get('results') extra_vars.update({ 'tab': tab, 'page': page, 'limit': limit, 'timespan': timespan, 'date_start': date_start.strftime('%Y-%m-%d %H:%M:%S') if date_start else '', 'date_end': date_start.strftime('%Y-%m-%d %H:%M:%S') if date_end else '', 'q': query, }) def _get_page_data(): current_page = page or 1 query_params = dict( filter(lambda (k, v): k != 'page', request.args.items())) pages = [] total_pages = int(math.ceil(log_data.get('count', 0) / float(limit))) pagination = { 'current_page': current_page, 'pages': pages, 'previous_url': '', 'next_url': '', 'total_pages': total_pages, 'total_items': log_data.get('count', 0), 'limit': limit, } start = 1 if current_page > 3: start = current_page - 3 end = current_page + 3 if end > total_pages: end = total_pages for i in range(start, end + 1): query_params['page'] = i url = h.url_for('user_dashboard.audit_requests', **query_params) pages.append({ 'page': i, 'url': url, }) if current_page > 1: query_params['page'] = current_page - 1 pagination['previous_url'] = h.url_for( 'user_dashboard.audit_requests', **query_params) if current_page < total_pages: query_params['page'] = current_page + 1 pagination['next_url'] = h.url_for('user_dashboard.audit_requests', **query_params) return pagination extra_vars['pagination'] = _get_page_data() if tab == 'all': return base.render('request_audit/requests_list.html', extra_vars=extra_vars) return base.render('request_audit/requests_report.html', extra_vars=extra_vars)
def post(self, package_type: str, id: str) -> Union[str, Response]: save_action = request.form.get(u'save') data = clean_dict( dict_fns.unflatten(tuplize_dict(parse_params(request.form))) ) data.update(clean_dict( dict_fns.unflatten(tuplize_dict(parse_params(request.files))) )) # we don't want to include save as it is part of the form del data[u'save'] resource_id = data.pop(u'id') context = cast(Context, { u'model': model, u'session': model.Session, u'user': g.user, u'auth_user_obj': g.userobj }) # see if we have any data that we are trying to save data_provided = False for key, value in data.items(): if ( (value or isinstance(value, cgi.FieldStorage)) and key != u'resource_type'): data_provided = True break if not data_provided and save_action != u"go-dataset-complete": if save_action == u'go-dataset': # go to final stage of adddataset return h.redirect_to(u'{}.edit'.format(package_type), id=id) # see if we have added any resources try: data_dict = get_action(u'package_show')(context, {u'id': id}) except NotAuthorized: return base.abort(403, _(u'Unauthorized to update dataset')) except NotFound: return base.abort( 404, _(u'The dataset {id} could not be found.').format(id=id) ) if not len(data_dict[u'resources']): # no data so keep on page msg = _(u'You must add at least one data resource') # On new templates do not use flash message errors: dict[str, Any] = {} error_summary = {_(u'Error'): msg} return self.get(package_type, id, data, errors, error_summary) # XXX race condition if another user edits/deletes data_dict = get_action(u'package_show')(context, {u'id': id}) get_action(u'package_update')( cast(Context, dict(context, allow_state_change=True)), dict(data_dict, state=u'active') ) return h.redirect_to(u'{}.read'.format(package_type), id=id) data[u'package_id'] = id try: if resource_id: data[u'id'] = resource_id get_action(u'resource_update')(context, data) else: get_action(u'resource_create')(context, data) except ValidationError as e: errors = e.error_dict error_summary = e.error_summary if data.get(u'url_type') == u'upload' and data.get(u'url'): data[u'url'] = u'' data[u'url_type'] = u'' data[u'previous_upload'] = True return self.get(package_type, id, data, errors, error_summary) except NotAuthorized: return base.abort(403, _(u'Unauthorized to create a resource')) except NotFound: return base.abort( 404, _(u'The dataset {id} could not be found.').format(id=id) ) if save_action == u'go-metadata': # XXX race condition if another user edits/deletes data_dict = get_action(u'package_show')(context, {u'id': id}) get_action(u'package_update')( cast(Context, dict(context, allow_state_change=True)), dict(data_dict, state=u'active') ) return h.redirect_to(u'{}.read'.format(package_type), id=id) elif save_action == u'go-dataset': # go to first stage of add dataset return h.redirect_to(u'{}.edit'.format(package_type), id=id) elif save_action == u'go-dataset-complete': return h.redirect_to(u'{}.read'.format(package_type), id=id) else: # add more resources return h.redirect_to( u'{}_resource.new'.format(package_type), id=id )
length = 0 for chunk in r.iter_content(chunk_size=CHUNK_SIZE): base.response.body_file.write(chunk) length += len(chunk) if length >= MAX_FILE_SIZE: base.abort(409, ('''Content is too large to be proxied. Allowed file size: {allowed}, Content-Length: {actual}. Url: '''+url).format( allowed=MAX_FILE_SIZE, actual=length)) except requests.exceptions.HTTPError, error: details = 'Could not proxy resource. Server responded with %s %s' % ( error.response.status_code, error.response.reason) base.abort(409, detail=details) except requests.exceptions.ConnectionError, error: details = '''Could not proxy resource because a connection error occurred. %s''' % error base.abort(502, detail=details) except requests.exceptions.Timeout, error: details = 'Could not proxy resource because the connection timed out.' base.abort(504, detail=details) class ServiceProxyController(base.BaseController): def proxy_service(self, resource_id): data_dict = {'resource_id': resource_id} context = {'model': base.model, 'session': base.model.Session, 'user': base.c.user or base.c.author} return proxy_service(self, context, data_dict)
content = u'<?xml version="1.0" encoding="UTF-8"?>\n' + content except xml_parser_exception: try: json.loads(obj['content']) response.content_type = 'application/json; charset=utf-8' except ValueError: # Just return whatever it is pass response.headers['Content-Length'] = len(content) return content.encode('utf-8') except p.toolkit.ObjectNotFound, e: abort(404, _(str(e))) except p.toolkit.NotAuthorized: abort(401, self.not_auth_message) except Exception, e: msg = 'An error occurred: [%s]' % str(e) abort(500, msg) def _get_source_for_job(self, source_id): try: context = {'model': model, 'user': c.user} source_dict = p.toolkit.get_action('harvest_source_show')( context, { 'id': source_id }) except p.toolkit.ObjectNotFound: abort(404, p.toolkit._('Harvest source not found')) except p.toolkit.NotAuthorized:
def read(package_type: str, id: str, resource_id: str) -> str: context = cast(Context, { u'model': model, u'session': model.Session, u'user': g.user, u'auth_user_obj': g.userobj, u'for_view': True }) try: package = get_action(u'package_show')(context, {u'id': id}) except (NotFound, NotAuthorized): return base.abort(404, _(u'Dataset not found')) activity_id = request.args.get(u'activity_id') if activity_id: # view an 'old' version of the package, as recorded in the # activity stream current_pkg = package try: activity = context['session'].query(model.Activity).get( activity_id ) assert activity package = activity.data['package'] except AttributeError: base.abort(404, _(u'Dataset not found')) if package['id'] != current_pkg['id']: log.info(u'Mismatch between pkg id in activity and URL {} {}' .format(package['id'], current_pkg['id'])) # the activity is not for the package in the URL - don't allow # misleading URLs as could be malicious base.abort(404, _(u'Activity not found')) # The name is used lots in the template for links, so fix it to be # the current one. It's not displayed to the user anyway. package['name'] = current_pkg['name'] # Don't crash on old (unmigrated) activity records, which do not # include resources or extras. package.setdefault(u'resources', []) resource = None for res in package.get(u'resources', []): if res[u'id'] == resource_id: resource = res break if not resource: return base.abort(404, _(u'Resource not found')) # get package license info license_id = package.get(u'license_id') try: package[u'isopen'] = model.Package.get_license_register()[license_id ].isopen() except KeyError: package[u'isopen'] = False resource_views = get_action(u'resource_view_list')( context, { u'id': resource_id } ) resource[u'has_views'] = len(resource_views) > 0 current_resource_view = None view_id = request.args.get(u'view_id') if resource[u'has_views']: if view_id: current_resource_view = [ rv for rv in resource_views if rv[u'id'] == view_id ] if len(current_resource_view) == 1: current_resource_view = current_resource_view[0] else: return base.abort(404, _(u'Resource view not found')) else: current_resource_view = resource_views[0] # required for nav menu pkg = context[u'package'] dataset_type = pkg.type or package_type # TODO: remove g.package = package g.resource = resource g.pkg = pkg g.pkg_dict = package extra_vars: dict[str, Any] = { u'resource_views': resource_views, u'current_resource_view': current_resource_view, u'dataset_type': dataset_type, u'pkg_dict': package, u'package': package, u'resource': resource, u'pkg': pkg, # NB it is the current version of the dataset, so ignores # activity_id. Still used though in resource views for # backward compatibility u'is_activity_archive': bool(activity_id), } template = _get_pkg_template(u'resource_template', dataset_type) return base.render(template, extra_vars)
def read_publisher(self, id): ''' Lists the most popular datasets for a publisher (or across all publishers) ''' count = 20 c.publishers = _get_publishers() id = request.params.get('publisher', id) if id and id != 'all': c.publisher = model.Group.get(id) if not c.publisher: abort(404, 'A publisher with that name could not be found') c.publisher_name = c.publisher.name c.top_packages = [] # package, dataset_views in c.top_packages # Get the month details by fetching distinct values and determining the # month names from the values. c.months, c.day = _month_details(GA_Url) # Work out which month to show, based on query params of the first item c.month = request.params.get('month', '') if not c.month: c.month_desc = 'all months' else: c.month_desc = ''.join([m[1] for m in c.months if m[0] == c.month]) month = c.month or 'All' c.publisher_page_views = 0 q = model.Session.query(GA_Url).\ filter(GA_Url.url=='/publisher/%s' % c.publisher_name) entry = q.filter(GA_Url.period_name == c.month).first() c.publisher_page_views = entry.pageviews if entry else 0 c.top_packages = self._get_packages(publisher=c.publisher, count=20, month=c.month) # Graph query top_packages_all_time = self._get_packages(publisher=c.publisher, count=20, month='All') top_package_names = [x[0].name for x in top_packages_all_time] graph_query = model.Session.query(GA_Url,model.Package)\ .filter(model.Package.name==GA_Url.package_id)\ .filter(GA_Url.url.like('/dataset/%'))\ .filter(GA_Url.package_id.in_(top_package_names)) all_series = {} for entry, package in graph_query: if not package: continue if entry.period_name == 'All': continue all_series[package.name] = all_series.get(package.name, { 'name': package.title, 'raw': {} }) all_series[package.name]['raw'][entry.period_name] = int( entry.pageviews) graph = [all_series[series_name] for series_name in top_package_names] c.graph_data = json.dumps(_to_rickshaw(graph)) return render('ga_report/publisher/read.html')
def members(self, id): ''' Modified core method from 'group' controller. Added search & sort functionality. :param id: id of the organization for which the member list is requested :type id: string :return: the rendered template :rtype: unicode ''' context = self._get_context() q, sort = self._find_filter_params() reverse = True if sort == u'title desc' else False org_meta = org_meta_dao.OrgMetaDao(id, c.user or c.author, c.userobj) org_meta.fetch_all() try: member_list = self._action('member_list')(context, { 'id': id, 'object_type': 'user', 'q': q, 'user_info': True }) member_list.sort(key=lambda y: y[4].lower(), reverse=reverse) member_groups = {} for m in member_list: role = m[3] if not member_groups.get(role): member_groups[role] = [] member_groups[role].append(m) member_groups = collections.OrderedDict( sorted(member_groups.items())) data_dict = {'id': id} data_dict['include_datasets'] = False current_user = self._current_user_info(member_list) is_sysadmin = c.userobj and c.userobj.sysadmin c_params = { 'sort': sort, 'members': [a[0:4] for a in member_list], 'member_groups': member_groups, 'org_meta': org_meta, 'current_user': current_user, 'allow_view_right_side': is_sysadmin or bool(current_user.get('role')), 'allow_approve': is_sysadmin or current_user.get('role') == 'admin', 'request_list': self._get_member_requests_for_org(id) } self._set_c_params(c_params) except NotAuthorized: base.abort(401, _('Unauthorized to view member list %s') % '') except NotFound: base.abort(404, _('Group not found')) except Exception, ex: log.error(str(ex)) base.abort(404, _('Server error'))
def proxy_ngsi_resource(self, resource_id): # Chunked proxy for ngsi resources. context = {'model': base.model, 'session': base.model.Session, 'user': base.c.user or base.c.author} log.info('Proxify resource {id}'.format(id=resource_id)) resource = logic.get_action('resource_show')(context, {'id': resource_id}) headers = { 'Accept': 'application/json' } resource.setdefault('auth_type', 'none') self.process_auth_credentials(resource, headers) if resource.get('tenant', '') != '': headers['FIWARE-Service'] = resource['tenant'] if resource.get('service_path', '') != '': headers['FIWARE-ServicePath'] = resource['service_path'] url = resource['url'] parsed_url = urlparse.urlsplit(url) if parsed_url.scheme not in ("http", "https") or not parsed_url.netloc: base.abort(409, detail='Invalid URL.') # Process verify configuration verify_conf = os.environ.get('CKAN_RIGHT_TIME_CONTEXT_VERIFY_REQUESTS', toolkit.config.get('ckan.right_time_context.verify_requests')) if verify_conf is None or (isinstance(verify_conf, six.string_types) and verify_conf.strip() == ""): verify_conf = os.environ.get('CKAN_VERIFY_REQUESTS', toolkit.config.get('ckan.verify_requests')) if isinstance(verify_conf, six.string_types) and verify_conf.strip() != "": compare_env = verify_conf.lower().strip() if compare_env in ("true", "1", "on"): verify = True elif compare_env in ("false", "0", "off"): verify = False else: verify = verify_conf elif isinstance(verify_conf, bool): verify = verify_conf else: verify = True # Make the request to the server try: if resource['format'].lower() == NGSI_REG_FORMAT: r = self._proxy_registration_resource(resource, parsed_url, headers, verify=verify) else: r = self._proxy_query_resource(resource, parsed_url, headers, verify=verify) except requests.HTTPError: details = 'Could not proxy ngsi_resource. We are working to resolve this issue as quickly as possible' base.abort(409, detail=details) except requests.ConnectionError: details = 'Could not proxy ngsi_resource because a connection error occurred.' base.abort(502, detail=details) except requests.Timeout: details = 'Could not proxy ngsi_resource because the connection timed out.' base.abort(504, detail=details) if r.status_code == 401: if resource.get('auth_type', 'none') != 'none': details = 'ERROR 401 token expired. Retrieving new token, reload please.' log.info(details) toolkit.c.usertoken_refresh() base.abort(409, detail=details) elif resource.get('auth_type', 'none') == 'none': details = 'Authentication requested by server, please check resource configuration.' log.info(details) base.abort(409, detail=details) elif r.status_code == 400: response = r.json() details = response['description'] log.info(details) base.abort(422, detail=details) else: r.raise_for_status() base.response.content_type = r.headers['content-type'] base.response.charset = r.encoding for chunk in r.iter_content(chunk_size=CHUNK_SIZE): base.response.body_file.write(chunk)
def _prepare_and_send(self, pkg_id, recipient_id, subject, prefix_template, suffix): """ Sends a message by email from the logged in user to the appropriate contact address of the given dataset. The prefix template should have formatting placeholders for the following arguments: {sender_name}, {sender_email}, {package_title}, {package_id} :param pkg_id: package id :type pkg_id: string :param recipient_id: id of the recipient, as returned by utils.get_package_contacts :type recipient_id: string :param subject: the subject of the message :type subject: string :param prefix_template: the template for the prefix to be automatically included before the user message :type prefix_template: unicode :param suffix: an additional note to be automatically included after the user message :type suffix: unicode """ url = h.url_for(controller='package', action='read', id=pkg_id) if asbool(config.get('kata.contact_captcha')): try: captcha.check_recaptcha(request) except captcha.CaptchaError: h.flash_error(_(u'Bad Captcha. Please try again.')) redirect(url) if not request.params.get('accept_logging'): h.flash_error(_(u"Message not sent as logging wasn't permitted")) return redirect(url) if asbool(config.get('kata.disable_contact')): h.flash_error(_(u"Sending contact emails is prohibited for now. Please try again later or contact customer " u"service.")) return redirect(url) package = Package.get(pkg_id) package_title = package.title if package.title else package.name sender_addr = request.params.get('from_address') sender_name = request.params.get('from_name') recipient = self._get_contact_email(pkg_id, recipient_id) if not recipient: abort(404, _('Recipient not found')) user_msg = request.params.get('msg', '') if request.params.get('hp'): h.flash_error(_(u"Message not sent. Couldn't confirm human interaction (spam bot control)")) return redirect(url) if sender_addr and sender_name and \ isinstance(sender_name, basestring) and len(sender_name) >= 3: if user_msg: prefix = prefix_template.format( sender_name=sender_name, sender_email=sender_addr, package_title=package_title, package_id=pkg_id # above line was: metadata_pid=utils.get_primary_data_pid_from_package(package) ) log.info(u"Message {m} sent from {a} ({b}) to {r} about {c}, IP: {d}" .format(m=user_msg, a=sender_name, b=sender_addr, r=recipient, c=pkg_id, d=request.environ.get('REMOTE_ADDR', 'No remote address'))) full_msg = u"{a}{b}{c}".format(a=prefix, b=user_msg, c=suffix) self._send_message(subject, full_msg, recipient.get('email'), recipient.get('name')) h.flash_success(_(u"Message sent")) else: h.flash_error(_(u"No message")) else: h.flash_error(_(u"Message not sent. Please, provide reply address and name. Name must contain \ at least three letters.")) return redirect(url)
def before_request(): try: context = dict(model=model, user=g.user, auth_user_obj=g.userobj) logic.check_access(u'site_read', context) except logic.NotAuthorized: base.abort(403, _(u'Not authorized to see this page'))
def edit(self, id): def validate(data): error_dict = {} has_owner = data.get('owner') has_key = data.get('key') if tk.asbool(data.get('integration', 'False')): if not has_owner: error_dict['owner'] = ['Required'] if not has_key: error_dict['key'] = ['Required'] if tk.asbool(data.get('show_links', 'False')): if not has_owner or not has_key: error_dict['show_links'] = [ 'This option available only ' 'if credentials are provided' ] if not error_dict: api = API(has_owner, has_key) check = api.check_credentials() if not check: error_dict['key'] = ['Incorrect key'] if error_dict: raise logic.ValidationError(error_dict) context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj } data_dict = {'id': id} stats = {} extra = {'errors': {}, 'error_summary': None, 'stats': stats} try: if not h.check_access('organization_update', data_dict): raise logic.NotAuthorized c.group_dict = logic.get_action('organization_show')(context, data_dict) c.group = context['group'] c.credentials = c.group.datadotworld_credentials if c.credentials is None: c.credentials = Credentials(organization=c.group) model.Session.add(c.credentials) except logic.NotFound: base.abort(404, _('Organization not found')) except logic.NotAuthorized: base.abort(401, _('User %r not authorized to edit %s') % (c.user, id)) if request.method == 'POST': data = dict(request.POST) c.credentials.update(data) try: validate(data) except logic.ValidationError as e: extra['errors'] = e.error_dict extra['error_summary'] = e.error_summary else: query = model.Session.query(Extras).join(model.Package).join( model.Group, model.Package.owner_org == model.Group.id).filter( model.Group.id == c.group.id) for item in query: item.state = 'pending' model.Session.commit() h.flash_success('Saved') if tk.asbool(c.credentials.integration): syncronize_org(c.group.id) return base.redirect_to('organization_dataworld', id=id) query = model.Session.query( func.count(model.Package.id).label('total'), Extras.state).join(model.Group, model.Package.owner_org == model.Group.id).join( Extras).group_by(Extras.state).filter( model.Package.owner_org == c.group.id) for amount, state in query: stats[state] = amount return base.render('organization/edit_credentials.html', extra_vars=extra)
def not_found(self, url): from ckan.lib.base import abort abort(404)
def map_preview_auth(self, resource_id, user_gs_layer): """ Authorization check for the given resource ID and Geoserver layer name :param resource_id: resource_id embedded in map preview URL :param user_gs_layer: Geoserver layer name requested by user in the URL :return: """ """ Test cases: User mpeterman-msea-read. Testing with dataset: https://www.gis-hub.ca/dataset/test05. This dataset has a mixture of resources with restrictions. Test the following URLs: 1. This resource is accessible to all users in the MSEA org: https://www.gis-hub.ca/dataset/test05/map_preview/7fad5dba-b70e-4a01-b692-a59a9b1b100c Result: 200 Access OK 2. Accessible only to user dfo or admins: https://www.gis-hub.ca/dataset/test05/map_preview/b1a24c96-eaa6-4433-a706-99854894be13 Experimenting with NGINX URL rewrite and variable capture https://www.gis-hub.ca/nginx_auth/geoserver/rest/workspaces/hubdata/datastores/maps.gis-hub.ca/featuretypes/ia_geography_pncima_ia_oceanography_pncima.json https://www.gis-hub.ca/geoserver/rest/workspaces/hubdata/datastores/maps.gis-hub.ca/featuretypes/ia_geography_pncima_ia_oceanography_pncima.json Capture block in nginx has changed, the above URLs should now be: https://www.gis-hub.ca/map_preview/test05/7fad5dba-b70e-4a01-b692-a59a9b1b100c https://www.gis-hub.ca/map_preview/test05/b1a24c96-eaa6-4433-a706-99854894be13 This NGINX location block matches after /geoserver, but it removes all the URL params after ? So, /hubdata/wms?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap... etc gets reduced to: /hubdata/wms # location ~ ^/map_preview/(?<package_id>.+)/(?<resource_id>.+)/geoserver/(?<geo_params>.+)$ { Need to add $query_string to the end of the proxy URI that gets passed on to geoserver on maps.gis-hub.ca set $geoserver_uri "https://maps.gis-hub.ca/geoserver/$geo_params?$query_string"; Have to open this in a Chrome Private window, because there was some cached HTTP basic auth that it tried to use. This works! We get the dataset and resource id, check it against CKAN auth, if it passes, we construct the correct geoserver URL and pass it to the proxy. With a geoserver request appended, test proxy. https://www.gis-hub.ca/map_preview/test05/7fad5dba-b70e-4a01-b692-a59a9b1b100c/geoserver/hubdata/wms?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image%2Fpng&TRANSPARENT=true&LAYERS=hubdata%3Aenv_layers_nsbssb_bpi_medium&exceptions=application%2Fvnd.ogc.se_inimage&SRS=EPSG%3A3857&STYLES=&WIDTH=873&HEIGHT=801&BBOX=-15384022.060787713%2C5832451.006272089%2C-13248677.23861303%2C7791684.915277727 Test with a restricted resource, should fail: https://www.gis-hub.ca/map_preview/test05/b1a24c96-eaa6-4433-a706-99854894be13/geoserver/hubdata/wms?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&FORMAT=image%2Fpng&TRANSPARENT=true&LAYERS=hubdata%3Aenv_layers_nsbssb_bpi_medium&exceptions=application%2Fvnd.ogc.se_inimage&SRS=EPSG%3A3857&STYLES=&WIDTH=873&HEIGHT=801&BBOX=-15384022.060787713%2C5832451.006272089%2C-13248677.23861303%2C7791684.915277727 Yes, this request never makes it to the proxy, the auth check in nginx fails with error 500. For now, just during development, we can keep the previous /geoserver proxy open, so that map previews will continue to work. TODO: ensure that the resource being requested matches the WMS or vector layer in Geoserver. This can be deferred for now. We dont care about requests like /vector/:dataset/:layer_name. The only thing we care about is securing actual map tile requests. Probably need to also add the resource ID to the map preview URL, something like: A: /vector/:dataset/:layer_name/:resource_id. Then, inside the vector tiles template, use a URL like this: B: {{ host }}/map_preview/{{dataset}}/{{resource_id}}/geoserver/gwc/service/tms/1.0.0/hubdata:{{ layer_name }}... This will be sent to nginx auth, then proxy directly to: C: https://maps.gis-hub.ca/geoserver/$geo_params?$query_string << this part does not touch the Node.js mapserver middleware at all. And this all happens *inside NGINX*, the only URL the user's browser sees is A and B ** Update May 20. For some reason, the map_preview endpoint has stopped working. e.g. https://www.gis-hub.ca/dataset/test05/map_preview/7fad5dba-b70e-4a01-b692-a59a9b1b100c throws not found, this controller function is not called at all. > Because I renamed the function to map_preview_auth! Test this with: https://maps.gis-hub.ca/vector/ia_geography_pncima/ia_oceanography_pncima/f63a3614-84ae-4168-b340-6919eac458c6 DOES NOT WORK because: ia_geography_pncima > in CKAN this should be ia-geography-pncima with HYPHENS However in mapserver backend, it is using ia_geography_pncima + resource title with UNDERSCORES to check map extent and so on. Need to replace the dataset part of the map preview URL with ia-geography-pncima, then in the Node backend, convert this to underscores before generating postgis layer name. Postgis is not happy with table names containing hyphens. What a pain. Change to: https://maps.gis-hub.ca/vector/ia-geography-pncima/ia_oceanography_pncima/f63a3614-84ae-4168-b340-6919eac458c6 Now the last issue we have is that when the request gets proxied from maps.gis-hub.ca back to gis-hub, it no longer has the CKAN context object with the user. Is is possible to pass the session with the proxied headers? If not, we might have to redo the map preview code inside a CKAN extension (where we know for sure that we have the context). This would simplify things somewhat, no need for all the proxying between the two servers. https://maps.gis-hub.ca/vector/ia-geography-pncima/ia_oceanography_pncima/f63a3614-84ae-4168-b340-6919eac458c6 https://gis-hub.ca/vector/ia-geography-pncima/ia_oceanography_pncima/f63a3614-84ae-4168-b340-6919eac458c6 Finally, in the nginx conf inside mapserver, allow access /geoserver/gwc and /geoserver/hubdata/wms only through host gis-hub.ca. Geoserver admins need to use the Geoserver web interface, which is accessed directly through maps.gis-hub.ca:8080, which bypasses Nginx entirely. That interface is already secured with an admin pass. """ logger.info('Check if user is admin') if auth.is_admin(c.user): # Return success immediately if user is admin logger.warning('User %s is admin. Skipping all auth checks' % c.user) return render('restricted/restricted_ok_page.html') auth_status = False try: # Get resource object resource = model.Resource.get(resource_id) if not resource: logger.warning('Resource %s not found!' % resource_id) flask_abort(401, 'Access denied') resource = resource.as_dict() # Ensure that the Geoserver layer name requested by the user matches the resource # Get the internal layer name used in Geoserver, directly from metadata if exists gs_layer_name = resource.get('geoserver_layer') if gs_layer_name: if user_gs_layer != gs_layer_name: logger.warning( 'User requested a layer name not from this resource! %s, expected %s' % (user_gs_layer, gs_layer_name)) flask_abort(401, 'Invalid geoserver layer name') else: # If the geoserver_layer field does not exist, get layer name from map_preview_link field logger.warning( 'Field geoserver_layer is empty, try map_preview_link') match_ok = self.check_geoserver_layer_name( resource, user_gs_layer) if not match_ok: logger.warning( 'User requested a layer name not from this resource! %s' % user_gs_layer) flask_abort(401, 'Invalid geoserver layer name') package_id = resource.get('package_id') # Get package object using package_id from the resource package = model.Package.get(package_id) if not package: msg = 'Dataset %s not found!' % package_id logger.warning(msg) logger.warning( 'Were HYPHENS in the dataset name changed to UNDERSCORES?') flask_abort(401, msg) package = package.as_dict() logger.info( 'Checking access for User: %s, Resource: %s, Geoserver layer: %s, Package: %s' % (c.user, resource.get('title'), user_gs_layer, package.get('name'))) # Ready to check access authorized = restricted_logic.restricted_check_user_resource_access( c.user, resource, package) auth_status = authorized.get('success') logger.info('Status: %s' % auth_status) except: # If anything else goes wrong, throw an error back to Nginx logger.warning( 'Something went wrong during auth check. Return error 401') import traceback logger.error(traceback.format_exc()) flask_abort(401, 'Error in auth check') if auth_status: """ Return a normal page with status code 2xx IF the user is authorized for this resource. Nginx responds to a 2xx status code by passing the auth request, then continues with the proxy to maps.gis-hub.ca/geoserver. """ return render('restricted/restricted_ok_page.html') else: """ This causes the server to throw error 500, which is OK because anything other than a 2xx code causes Nginx to fail the auth request, and deny the proxy. User will see an error in their browser. https://docs.nginx.com/nginx/admin-guide/security-controls/configuring-subrequest-authentication/ """ return base.abort(404, _(u'User not authorized'))
def search(self): from ckan.lib.search import SearchError, SearchQueryError package_type = self._guess_package_type() extra_vars = {'dataset_type': package_type} try: context = { 'model': model, 'user': c.user, 'auth_user_obj': c.userobj } check_access('site_read', context) except NotAuthorized: abort(403, _('Not authorized to see this page')) # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False page = h.get_page_number(request.params) limit = int(config.get('ckan.datasets_per_page', 20)) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() if k != 'page'] def drill_down_url(alternative_url=None, **by): return h.add_url_param(alternative_url=alternative_url, controller='package', action='search', new_params=by) c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): return h.remove_url_param(key, value=value, replace=replace, controller='package', action='search', alternative_url=package_type) c.remove_field = remove_field sort_by = request.params.get('sort', None) params_nosort = [(k, v) for k, v in params_nopage if k != 'sort'] def _sort_by(fields): """ Sort by the given list of fields. Each entry in the list is a 2-tuple: (fieldname, sort_order) eg - [('metadata_modified', 'desc'), ('name', 'asc')] If fields is empty, then the default ordering is used. """ params = params_nosort[:] if fields: sort_string = ', '.join('%s %s' % f for f in fields) params.append(('sort', sort_string)) return search_url(params, package_type) c.sort_by = _sort_by if not sort_by: c.sort_by_fields = [] else: c.sort_by_fields = [ field.split()[0] for field in sort_by.split(',') ] def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params, package_type) c.search_url_params = urlencode(_encode_params(params_nopage)) try: c.fields = [] # c.fields_grouped will contain a dict of params containing # a list of values eg {'tags':['tag1', 'tag2']} c.fields_grouped = {} search_extras = {} fq = '' for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ and len(value) and not param.startswith('_'): if not param.startswith('ext_'): c.fields.append((param, value)) fq += ' %s:"%s"' % (param, value) if param not in c.fields_grouped: c.fields_grouped[param] = [value] else: c.fields_grouped[param].append(value) else: search_extras[param] = value context = { 'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'auth_user_obj': c.userobj } # Unless changed via config options, don't show other dataset # types any search page. Potential alternatives are do show them # on the default search page (dataset) or on one other search page search_all_type = config.get('ckan.search.show_all_types', 'dataset') search_all = False try: # If the "type" is set to True or False, convert to bool # and we know that no type was specified, so use traditional # behaviour of applying this only to dataset type search_all = asbool(search_all_type) search_all_type = 'dataset' # Otherwise we treat as a string representing a type except ValueError: search_all = True if not package_type: package_type = 'dataset' if not search_all or package_type != search_all_type: # Only show datasets of this particular type fq += ' +dataset_type:{type}'.format(type=package_type) facets = OrderedDict() default_facet_titles = { 'organization': _('Organizations'), 'groups': _('Groups'), 'tags': _('Tags'), 'res_format': _('Formats'), 'license_id': _('Licenses'), } for facet in h.facets(): if facet in default_facet_titles: facets[facet] = default_facet_titles[facet] else: facets[facet] = facet # Facet titles for plugin in p.PluginImplementations(p.IFacets): facets = plugin.dataset_facets(facets, package_type) c.facet_titles = facets # Date interval search start_date = None end_date = None if request.params.get('_start_date'): start_date = extract_date(request.params.get('_start_date')) if start_date: extra_vars['start_date'] = request.params.get( '_start_date') if request.params.get('_end_date'): end_date = extract_date(request.params.get('_end_date')) if end_date: extra_vars['end_date'] = request.params.get('_end_date') if start_date or end_date: if not start_date: start_date = datetime(year=1900, month=1, day=1) if not end_date: end_date = datetime(year=2100, month=1, day=1, hour=23, minute=59, second=59, microsecond=999999) else: end_date = datetime_to_utc(end_date) end_date = end_date.replace(hour=23, minute=59, second=59, microsecond=999999) fq = fq + ' +metadata_modified:[%s TO %s]' % ( datetime_to_utc(start_date).strftime('%Y-%m-%dT%H:%M:%SZ'), datetime_to_utc(end_date).strftime('%Y-%m-%dT%H:%M:%SZ')) data_dict = { 'q': q, 'fq': fq.strip(), 'facet.field': facets.keys(), 'rows': limit, 'start': (page - 1) * limit, 'sort': sort_by, 'extras': search_extras, 'include_private': asbool(config.get('ckan.search.default_include_private', True)), } query = get_action('package_search')(context, data_dict) c.sort_by_selected = query['sort'] c.page = h.Page(collection=query['results'], page=page, url=pager_url, item_count=query['count'], items_per_page=limit) c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchQueryError as se: # User's search parameters are invalid, in such a way that is not # achievable with the web interface, so return a proper error to # discourage spiders which are the main cause of this. log.info('Dataset search query rejected: %r', se.args) abort( 400, _('Invalid search query: {error_message}').format( error_message=str(se))) except SearchError as se: # May be bad input from the user, but may also be more serious like # bad code causing a SOLR syntax error, or a problem connecting to # SOLR log.error('Dataset search error: %r', se.args) c.query_error = True c.search_facets = {} c.page = h.Page(collection=[]) except NotAuthorized: abort(403, _('Not authorized to see this page')) c.search_facets_limits = {} for facet in c.search_facets.keys(): try: limit = int( request.params.get( '_%s_limit' % facet, int(config.get('search.facets.default', 10)))) except ValueError: abort( 400, _('Parameter "{parameter_name}" is not ' 'an integer').format(parameter_name='_%s_limit' % facet)) c.search_facets_limits[facet] = limit self._setup_template_variables(context, {}, package_type=package_type) extra_vars['dataset_type'] = package_type return render(self._search_template(package_type), extra_vars=extra_vars)
split = id.split('@') if len(split) == 2: data_dict['id'], revision_ref = split if model.is_id(revision_ref): context['revision_id'] = revision_ref else: try: date = h.date_str_to_datetime(revision_ref) context['revision_date'] = date except TypeError, e: base.abort(400, _('Invalid revision format: %r') % e.args) except ValueError, e: base.abort(400, _('Invalid revision format: %r') % e.args) elif len(split) > 2: base.abort( 400, _('Invalid revision format: %r') % 'Too many "@" symbols') # check if package exists try: c.pkg_dict = logic.get_action('package_show')(context, data_dict) c.pkg = context['package'] except NotFound: base.abort(404, _('Dataset not found')) except NotAuthorized: base.abort(401, _('Unauthorized to read package %s') % id) # used by disqus plugin c.current_package_id = c.pkg.id c.related_count = c.pkg.related_count
def dashboard(self): """ List all related items regardless of dataset """ context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj, 'for_view': True } data_dict = { 'type_filter': base.request.params.get('type', ''), 'sort': base.request.params.get('sort', ''), 'featured': base.request.params.get('featured', '') } params_nopage = [(k, v) for k, v in base.request.params.items() if k != 'page'] try: page = int(base.request.params.get('page', 1)) except ValueError: base.abort(400, ('"page" parameter must be an integer')) # Update ordering in the context related_list = logic.get_action('related_list')(context, data_dict) def search_url(params): url = h.url_for(controller='related', action='dashboard') params = [ (k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) for k, v in params ] return url + u'?' + urllib.urlencode(params) def pager_url(q=None, page=None): params = list(params_nopage) params.append(('page', page)) return search_url(params) c.page = h.Page(collection=related_list, page=page, url=pager_url, item_count=len(related_list), items_per_page=9) c.filters = dict(params_nopage) c.type_options = self._type_options() c.sort_options = ({ 'value': '', 'text': _('Most viewed') }, { 'value': 'view_count_desc', 'text': _('Most Viewed') }, { 'value': 'view_count_asc', 'text': _('Least Viewed') }, { 'value': 'created_desc', 'text': _('Newest') }, { 'value': 'created_asc', 'text': _('Oldest') }) return base.render("related/dashboard.html")
def post(self, id: Optional[str] = None) -> Union[Response, str]: context, id = self._prepare(id) if not context[u'save']: return self.get(id) # checks if user id match with the current logged user if id in (g.userobj.id, g.userobj.name): current_user = True else: current_user = False # we save the username for later use.. in case the current # logged in user change his username old_username = g.userobj.name try: data_dict = logic.clean_dict( dictization_functions.unflatten( logic.tuplize_dict(logic.parse_params(request.form)))) data_dict.update( logic.clean_dict( dictization_functions.unflatten( logic.tuplize_dict(logic.parse_params( request.files))))) except dictization_functions.DataError: base.abort(400, _(u'Integrity Error')) data_dict.setdefault(u'activity_streams_email_notifications', False) data_dict[u'id'] = id # we need this comparison when sysadmin edits a user, # this will return True # and we can utilize it for later use. email_changed = data_dict[u'email'] != g.userobj.email # common users can edit their own profiles without providing # password, but if they want to change # their old password with new one... old password must be provided.. # so we are checking here if password1 # and password2 are filled so we can enter the validation process. # when sysadmins edits a user he MUST provide sysadmin password. # We are recognizing sysadmin user # by email_changed variable.. this returns True # and we are entering the validation. if (data_dict[u'password1'] and data_dict[u'password2']) or email_changed: # getting the identity for current logged user identity = { u'login': g.user, u'password': data_dict[u'old_password'] } auth = authenticator.UsernamePasswordAuthenticator() # we are checking if the identity is not the # same with the current logged user if so raise error. if auth.authenticate(request.environ, identity) != g.user: errors: ErrorDict = { u'oldpassword': [_(u'Password entered was incorrect')] } error_summary = {_(u'Old Password'): _(u'incorrect password')}\ if not g.userobj.sysadmin \ else {_(u'Sysadmin Password'): _(u'incorrect password')} return self.get(id, data_dict, errors, error_summary) try: user = logic.get_action(u'user_update')(context, data_dict) except logic.NotAuthorized: base.abort(403, _(u'Unauthorized to edit user %s') % id) except logic.NotFound: base.abort(404, _(u'User not found')) except logic.ValidationError as e: errors = e.error_dict error_summary = e.error_summary return self.get(id, data_dict, errors, error_summary) h.flash_success(_(u'Profile updated')) resp = h.redirect_to(u'user.read', id=user[u'name']) if current_user and data_dict[u'name'] != old_username: # Changing currently logged in user's name. # Update repoze.who cookie to match set_repoze_user(data_dict[u'name'], resp) return resp
def read(package_type, id, resource_id): context = { u'model': model, u'session': model.Session, u'user': g.user, u'auth_user_obj': g.userobj, u'for_view': True } try: package = get_action(u'package_show')(context, {u'id': id}) except (NotFound, NotAuthorized): return base.abort(404, _(u'Dataset not found')) resource = None for res in package.get(u'resources', []): if res[u'id'] == resource_id: resource = res break if not resource: return base.abort(404, _(u'Resource not found')) # get package license info license_id = package.get(u'license_id') try: package[u'isopen'] = model.Package.get_license_register( )[license_id].isopen() except KeyError: package[u'isopen'] = False resource_views = get_action(u'resource_view_list')(context, { u'id': resource_id }) resource[u'has_views'] = len(resource_views) > 0 current_resource_view = None view_id = request.args.get(u'view_id') if resource[u'has_views']: if view_id: current_resource_view = [ rv for rv in resource_views if rv[u'id'] == view_id ] if len(current_resource_view) == 1: current_resource_view = current_resource_view[0] else: return base.abort(404, _(u'Resource view not found')) else: current_resource_view = resource_views[0] # required for nav menu pkg = context[u'package'] dataset_type = pkg.type or package_type # TODO: remove g.package = package g.resource = resource g.pkg = pkg g.pkg_dict = package extra_vars = { u'resource_views': resource_views, u'current_resource_view': current_resource_view, u'dataset_type': dataset_type, u'pkg_dict': package, u'package': package, u'resource': resource, u'pkg': pkg } template = _get_pkg_template(u'resource_template', dataset_type) return base.render(template, extra_vars)
remote_ip = request.environ.get('REMOTE_ADDR', 'Unknown IP Address') if (_check_recaptcha(remote_ip, recaptcha_response)): user = get_action('user_create')(context, data_dict) else: error_msg = _(u'Bad Captcha. Please try again.') h.flash_error(error_msg) return self.new(data_dict) except NotAuthorized: abort(403, _('Unauthorized to create user %s') % '') except NotFound, e: abort(404, _('User not found')) except DataError: abort(400, _(u'Integrity Error')) except ValidationError, e: errors = e.error_dict error_summary = e.error_summary return self.new(data_dict, errors, error_summary) h.flash_success( _('A confirmation email has been sent to %s. ' 'Please use the link in the email to continue.') % data_dict['email']) if not c.user: # Do not log in the user programatically # set_repoze_user(data_dict['name']) if came_from: h.redirect_to(came_from) else: