def success(self, label=None):
label = request.params.get('label', label)
h.flash_success('Upload successful')
c.file_url = h.url_for('storage_file',
label=label,
qualified=True)
c.upload_url = h.url_for('storage_upload')
return render('storage/success.html')
def _download_file(self, res, label):
# We need this as a resource object to check access so create a dummy
# obj and trick CKAN
resource = model.Resource()
for k in res.keys():
setattr(resource, k, res[k])
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'auth_user_obj': c.userobj, 'resource': resource}
data_dict = {'id': resource.id}
try:
logic.check_access('resource_show', context, data_dict)
except logic.NotAuthorized:
redirect_url = h.url_for(controller='user', action='login',
came_from=resource.url)
r = generate_response(303, u'Not authorized to read file ' + resource.id,
other_headers={'Location': redirect_url, 'X-CKAN-Error': '403 Access Denied'})
return r
exists = self.ofs.exists(BUCKET, label)
if not exists:
# handle erroneous trailing slash by redirecting to url w/o slash
if label.endswith('/'):
label = label[:-1]
# This may be best being cached_url until we have moved it into
# permanent storage
file_url = h.url_for('storage_file', label=label)
h.redirect_to(file_url)
else:
# abort(404)
r = generate_response(404, u'File not found')
return r
file_url = self.ofs.get_url(BUCKET, label)
if file_url.startswith("file://") or file_url.endswith('xlsx'):
metadata = self.ofs.get_metadata(BUCKET, label)
filepath = file_url[len("file://"):]
headers = {
# 'Content-Disposition':'attachment; filename="%s"' % label,
'Pragma': 'no-cache',
'Cache-Control': 'max-age=0, no-store, no-cache',
'Content-Type': metadata.get('_format', 'text/plain')}
if resource.name:
res_name = resource.name.replace('"', '_')
res_name_encoded = res_name.encode('utf-8', 'ignore')
file_name, file_extension = os.path.splitext(res_name_encoded)
if file_extension == '' and resource.format:
file_name = file_name + '.' + resource.format
else:
file_name = res_name_encoded
headers[
'Content-Disposition'] = 'inline; filename="{}"'.format(file_name)
fapp = FileApp(filepath, headers=None, **headers)
return fapp(request.environ, self.start_response)
else:
h.redirect_to(file_url.encode('ascii', 'ignore'))
def success(self, label=None):
label=request.params.get('label', label)
h.flash_success('Upload successful')
c.file_url = h.url_for('storage_file',
label=label,
qualified=True
)
c.upload_url = h.url_for('storage_upload')
return render('storage/success.html')
def _download_file(self, res, label):
# We need this as a resource object to check access so create a dummy
# obj and trick CKAN
resource = model.Resource()
for k in res.keys():
setattr(resource, k, res[k])
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'auth_user_obj': c.userobj, 'resource': resource}
data_dict = {'id': resource.id}
try:
logic.check_access('resource_show', context, data_dict)
except logic.NotAuthorized:
redirect_url = h.url_for(controller='user', action='login',
came_from=resource.url)
r = generate_response(303, u'Not authorized to read file ' + resource.id,
other_headers={'Location': redirect_url, 'X-CKAN-Error': '403 Access Denied'})
return r
exists = self.ofs.exists(BUCKET, label)
if not exists:
# handle erroneous trailing slash by redirecting to url w/o slash
if label.endswith('/'):
label = label[:-1]
# This may be best being cached_url until we have moved it into
# permanent storage
file_url = h.url_for('storage_file', label=label)
h.redirect_to(file_url)
else:
# abort(404)
r = generate_response(404, u'File not found')
return r
file_url = self.ofs.get_url(BUCKET, label)
if file_url.startswith("file://") or file_url.endswith('xlsx'):
metadata = self.ofs.get_metadata(BUCKET, label)
filepath = file_url[len("file://"):]
headers = {
# 'Content-Disposition':'attachment; filename="%s"' % label,
'Pragma': 'no-cache',
'Cache-Control': 'max-age=0, no-store, no-cache',
'Content-Type': metadata.get('_format', 'text/plain')}
if resource.name:
res_name = resource.name.replace('"', '_')
res_name_encoded = res_name.encode('utf-8', 'ignore')
headers[
'Content-Disposition'] = 'inline; filename="{}"'.format(res_name_encoded)
fapp = FileApp(filepath, headers=None, **headers)
return fapp(request.environ, self.start_response)
else:
h.redirect_to(file_url.encode('ascii', 'ignore'))
def create_request(self, pkg_id):
pkg = Package.get(pkg_id)
user = c.userobj if c.userobj else None
if user:
req = KataAccessRequest(user.id, pkg.id)
req.save()
url = h.url_for(controller='package', action='read', id=pkg_id)
h.flash_success(_("You now requested editor rights to package %s" % pkg.name))
redirect(url)
else:
url = h.url_for(controller='package', action='read', id=pkg_id)
h.flash_error(_("Please log in!"))
redirect(url)
def render_request_form(self, pkg_id):
"""
Render the access request contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if not c.package:
abort(404, _(u"Dataset not found"))
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts]
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/dataset_request_form.html')
def file(self, label):
exists = self.ofs.exists(BUCKET, label)
if not exists:
# handle erroneous trailing slash by redirecting to url w/o slash
if label.endswith('/'):
label = label[:-1]
# This may be best being cached_url until we have moved it into
# permanent storage
file_url = h.url_for('storage_file', label=label)
h.redirect_to(file_url)
else:
abort(404)
file_url = self.ofs.get_url(BUCKET, label)
if file_url.startswith("file://"):
metadata = self.ofs.get_metadata(BUCKET, label)
filepath = file_url[len("file://"):]
headers = {
# 'Content-Disposition':'attachment; filename="%s"' % label,
'Content-Type': metadata.get('_format', 'text/plain')
}
fapp = FileApp(filepath, headers=None, **headers)
return fapp(request.environ, self.start_response)
else:
h.redirect_to(file_url.encode('ascii', 'ignore'))
def file(self, label):
exists = self.ofs.exists(BUCKET, label)
if not exists:
# handle erroneous trailing slash by redirecting to url w/o slash
if label.endswith("/"):
label = label[:-1]
# This may be best being cached_url until we have moved it into
# permanent storage
file_url = h.url_for("storage_file", label=label)
h.redirect_to(file_url)
else:
abort(404)
file_url = self.ofs.get_url(BUCKET, label)
if file_url.startswith("file://"):
metadata = self.ofs.get_metadata(BUCKET, label)
filepath = file_url[len("file://") :]
headers = {
# 'Content-Disposition':'attachment; filename="%s"' % label,
"Content-Type": metadata.get("_format", "text/plain")
}
fapp = FileApp(filepath, headers=None, **headers)
return fapp(request.environ, self.start_response)
else:
h.redirect_to(file_url)
def send(self, pkg_id):
package = Package.get(pkg_id)
url = h.url_for(controller='package',
action="read",
id=package.id)
if c.user:
userid = None
for role in package.roles:
if role.role == "admin":
userid = role.user_id
break
if userid:
owner = User.get(userid)
msg = request.params.get('msg', '')
if msg:
send_contact_email(owner, c.userobj, package,\
msg)
else:
h.flash_error(_("No message"))
return redirect(url)
else:
h.flash_error(_("No owner found"))
return redirect(url)
h.flash_notice(_("Message sent"))
else:
h.flash_error(_("Please login"))
return redirect(url)
def _get_remote_form_data(self, label):
method = "POST"
content_length_range = int(config.get("ckan.storage.max_content_length", 50000000))
acl = "public-read"
fields = [{"name": self.ofs.conn.provider.metadata_prefix + "uploaded-by", "value": c.userobj.id}]
conditions = ['{"%s": "%s"}' % (x["name"], x["value"]) for x in fields]
# In FF redirect to this breaks js upload as FF attempts to open file
# (presumably because mimetype = javascript) and this stops js
# success_action_redirect = h.url_for('storage_api_get_metadata', qualified=True,
# label=label)
success_action_redirect = h.url_for("storage_upload_success_empty", qualified=True, label=label)
data = self.ofs.conn.build_post_form_args(
BUCKET,
label,
expires_in=72000,
max_content_length=content_length_range,
success_action_redirect=success_action_redirect,
acl=acl,
fields=fields,
conditions=conditions,
)
# HACK: fix up some broken stuff from boto
# e.g. should not have content-length-range in list of fields!
storage_backend = config["ofs.impl"]
for idx, field in enumerate(data["fields"]):
if storage_backend == "google":
if field["name"] == "AWSAccessKeyId":
field["name"] = "GoogleAccessId"
if field["name"] == "content-length-range":
del data["fields"][idx]
return data
def logged_out_page(self):
"""
This renders a page that informs the user of a successful logout.
"""
url = h.url_for(controller='ckanext.ngds.ngdsui.controllers.home:HomeController', action='render_index')
h.flash_notice(_('You are now Logged out'), allow_html=True)
redirect(url)
def search_url(params):
url = h.url_for(controller='group',
action='read',
id=c.group_dict.get('name'))
params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) \
for k, v in params]
return url + u'?' + urlencode(params)
def resource_read(self, id, resource_id):
context = {"model": model, "session": model.Session, "user": c.user or c.author}
try:
c.resource = get_action("resource_show")(context, {"id": resource_id})
c.package = get_action("package_show")(context, {"id": id})
# required for nav menu
c.pkg = context["package"]
c.resource_json = json.dumps(c.resource)
c.pkg_dict = c.package
except NotFound:
abort(404, _("Resource not found"))
except NotAuthorized:
abort(401, _("Unauthorized to read resource %s") % id)
# get package license info
license_id = c.package.get("license_id")
try:
c.package["isopen"] = model.Package.get_license_register()[license_id].isopen()
except KeyError:
c.package["isopen"] = False
c.datastore_api = h.url_for("datastore_read", id=c.resource.get("id"), qualified=True)
c.related_count = len(c.pkg.related)
c.num_followers = get_action("dataset_follower_count")(context, {"id": c.pkg.id})
self._setup_follow_button(context)
return render("package/resource_read.html")
def upload(self, id):
package_type = self._get_package_type(id)
geno = get_geno(package_type)
lc = ckanapi.LocalCKAN(username=c.user)
dataset = lc.action.package_show(id=id)
try:
if request.POST['xls_update'] == '':
raise BadExcelData('You must provide a valid file')
_process_upload_file(
lc,
dataset,
request.POST['xls_update'].file,
geno)
h.flash_success(_(
"Your file was successfully uploaded into the central system."
))
redirect(h.url_for(controller='package', action='read', id=id))
except BadExcelData, e:
org = lc.action.organization_show(id=dataset['owner_org'])
return self.preview_table(
resource_name=dataset['resources'][0]['name'],
owner_org=org['name'],
errors=[e.message])
def get_metadata(self, label):
log.info("child")
bucket = BUCKET
storage_backend = config['ofs.impl']
if storage_backend in ['google', 's3']:
if not label.startswith("/"):
label = "/" + label
url = "https://%s%s" % (self.ofs.conn.calling_format.build_host(
self.ofs.conn.server_name(), bucket), label)
k = get_key(self.ofs, label)
url = k.generate_url(60 * 60 * 24 * 365 * 10, force_http=True)
log.info("get_metadata. url={}".format(url))
else:
url = h.url_for('storage_file', label=label, qualified=False)
if url.startswith('/'):
url = config.get('ckan.site_url', '').rstrip('/') + url
if not self.ofs.exists(bucket, label):
abort(404)
metadata = self.ofs.get_metadata(bucket, label)
metadata["_location"] = url
return metadata
def render_request_form(self, pkg_id):
"""
Render the access request contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
if not c.package:
abort(404, _(u"Dataset not found"))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts]
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/dataset_request_form.html')
def pager_url(q=None, page=None):
return h.url_for(
controller="ckanext.dgu.controllers.tag:TagController",
action="index",
q=request.params["q"],
page=page,
)
def _get_remote_form_data(self, label):
method = 'POST'
content_length_range = int(
config.get('ckan.storage.max_content_length',
50000000))
acl = 'public-read'
fields = [ {
'name': self.ofs.conn.provider.metadata_prefix + 'uploaded-by',
'value': c.userobj.name
}]
conditions = [ '{"%s": "%s"}' % (x['name'], x['value']) for x in
fields ]
# In FF redirect to this breaks js upload as FF attempts to open file
# (presumably because mimetype = javascript) and this stops js
# success_action_redirect = h.url_for('storage_api_get_metadata', qualified=True,
# label=label)
success_action_redirect = h.url_for('storage_upload_success_empty', qualified=True,
label=label)
data = self.ofs.conn.build_post_form_args(
BUCKET,
label,
expires_in=72000,
max_content_length=content_length_range,
success_action_redirect=success_action_redirect,
acl=acl,
fields=fields,
conditions=conditions
)
# HACK: fix up some broken stuff from boto
# e.g. should not have content-length-range in list of fields!
for idx,field in enumerate(data['fields']):
if field['name'] == 'content-length-range':
del data['fields'][idx]
return data
def upload(self):
label = key_prefix + request.params.get("filepath", str(uuid.uuid4()))
c.data = {
"action": h.url_for("storage_upload_handle", qualified=False),
"fields": [{"name": "key", "value": label}],
}
return render("storage/index.html")
def resource_read(self, id, resource_id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author}
try:
c.resource = get_action('resource_show')(context, {'id': resource_id})
c.package = get_action('package_show')(context, {'id': id})
# required for nav menu
c.pkg = context['package']
c.resource_json = json.dumps(c.resource)
c.pkg_dict = c.package
except NotFound:
abort(404, _('Resource not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read resource %s') % id)
# get package license info
license_id = c.package.get('license_id')
try:
c.package['isopen'] = model.Package.get_license_register()[license_id].isopen()
except KeyError:
c.package['isopen'] = False
c.datastore_api = h.url_for('datastore_read', id=c.resource.get('id'),
qualified=True)
c.related_count = len(c.pkg.related)
c.num_followers = get_action('dataset_follower_count')(context,
{'id':c.pkg.id})
self._setup_follow_button(context)
return render('package/resource_read.html')
def filter(self, stream):
"""
Implements IGenshiStreamFilter.
"""
routes = request.environ.get('pylons.routes_dict')
# add a 'Todo' link to the menu bar
menu_data = {'href':
h.link_to("Todo", h.url_for('todo_page'),
class_ = ('active' if c.controller == 'ckanext.todo.controller:TodoController' else ''))}
stream = stream | Transformer('body//div[@id="mainmenu"]')\
.append(HTML(html.MENU_CODE % menu_data))
# if this is the read action of a package, show todo info
if(routes.get('controller') == 'package' and
routes.get('action') == 'read' and
c.pkg.id):
user_id = controller.get_user_id(request.environ.get('REMOTE_USER')) or ""
data = {'package': c.pkg.name,
'user_id': user_id}
# add CSS style
stream = stream | Transformer('head').append(HTML(html.HEAD_CODE))
# add jquery and todo.js links
stream = stream | Transformer('body').append(HTML(html.BODY_CODE % data))
# add todo subsection
stream = stream | Transformer('//div[@id="dataset"]')\
.append(HTML(html.TODO_CODE))
return stream
def get_metadata(self, label):
log.info("child")
bucket = BUCKET
storage_backend = config['ofs.impl']
if storage_backend in ['google', 's3']:
if not label.startswith("/"):
label = "/" + label
url = "https://%s%s" % (
self.ofs.conn.calling_format.build_host(
self.ofs.conn.server_name(), bucket), label)
k = get_key(self.ofs, label)
url = k.generate_url(60*60*24*365*10, force_http=True)
log.info("get_metadata. url={}".format(url))
else:
url = h.url_for('storage_file',
label=label,
qualified=False
)
if url.startswith('/'):
url = config.get('ckan.site_url','').rstrip('/') + url
if not self.ofs.exists(bucket, label):
abort(404)
metadata = self.ofs.get_metadata(bucket, label)
metadata["_location"] = url
return metadata
def delete_record(self, id):
lc = ckanapi.LocalCKAN(username=c.user)
filters = {}
package_type = self._get_package_type(id)
for f in primary_key_fields(package_type):
filters[f['datastore_id']] = request.POST.get(f['datastore_id'], '')
package = lc.action.package_show(id=id)
result = lc.action.datastore_search(
resource_id=package['resources'][0]['id'],
filters=filters,
rows=2) # only need two to know if there are multiple matches
records = result['records']
x_vars = {'filters': filters, 'action': 'edit'}
if not records:
x_vars['delete_errors'] = [_('No matching records found')]
elif len(records) > 1:
x_vars['delete_errors'] = [_('Multiple matching records found')]
if 'delete_errors' in x_vars:
c.pkg_dict = package
return render(self._edit_template(package_type), extra_vars=x_vars)
# XXX: can't avoid the race here with the existing datastore API.
# datastore_delete doesn't support _id filters
lc.action.datastore_delete(
resource_id=package['resources'][0]['id'],
filters=filters,
)
h.flash_success(_(
"Record deleted."
))
redirect(h.url_for(controller='package', action='read', id=id))
def resource_read(self, id, resource_id):
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author
}
try:
c.resource = get_action('resource_show')(context, {
'id': resource_id
})
c.package = get_action('package_show')(context, {'id': id})
# required for nav menu
c.pkg = context['package']
c.resource_json = json.dumps(c.resource)
c.pkg_dict = c.package
except NotFound:
abort(404, _('Resource not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read resource %s') % id)
# get package license info
license_id = c.package.get('license_id')
try:
c.package['isopen'] = model.Package.\
get_license_register()[license_id].isopen()
except KeyError:
c.package['isopen'] = False
c.datastore_api = h.url_for('datastore_read',
id=c.resource.get('id'),
qualified=True)
c.related_count = c.pkg.related_count
return render('package/resource_read.html')
def index(self):
group_type = None
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'for_view': True,
'with_private': False}
q = c.q = request.params.get('q', '')
data_dict = {'all_fields': True, 'q': q}
sort_by = c.sort_by_selected = request.params.get('sort')
if sort_by:
data_dict['sort'] = sort_by
try:
self._check_access('site_read', context)
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
c.group_package_stuff = caching.cached_get_group_package_stuff()
##Removing groups without geojson for the map
c.group_map = []
for gp in c.group_package_stuff:
'''
Removed check for geojson data because in the new version this information
does not come from the group_list action and for now we are not using the map.
If we'll need this we should implement some caching functionality for this too.
'''
c.group_map.append(gp)
#print c.group_package_stuff
if c.userobj is not None:
msg = None
url = h.url_for(controller='user', action='edit')
is_google_id = \
c.userobj.name.startswith('https://www.google.com/accounts/o8/id')
if not c.userobj.email and (is_google_id and not c.userobj.fullname):
msg = _(u'Please <a href="{link}">update your profile</a>'
u' and add your email address and your full name. '
u'{site} uses your email address'
u' if you need to reset your password.'.format(
link=url, site=g.site_title))
elif not c.userobj.email:
msg = _('Please <a href="%s">update your profile</a>'
' and add your email address. ') % url + \
_('%s uses your email address'
' if you need to reset your password.') \
% g.site_title
elif is_google_id and not c.userobj.fullname:
msg = _('Please <a href="%s">update your profile</a>'
' and add your full name.') % (url)
if msg:
h.flash_notice(msg, allow_html=True)
return base.render('home/index.html', cache_force=True)
def edit(self, id, data=None, errors=None, error_summary=None):
package_type = self._get_package_type(id)
context = {'model': model, 'session': model.Session,
'user': c.user, 'auth_user_obj': c.userobj,
'save': 'save' in request.params}
if context['save'] and not data:
return self._save_edit(id, context, package_type=package_type)
try:
c.pkg_dict = get_action('package_show')(dict(context,
for_view=True),
{'id': id})
context['for_edit'] = True
old_data = get_action('package_show')(context, {'id': id})
# old data is from the database and data is passed from the
# user if there is a validation error. Use users data if there.
if data:
old_data.update(data)
data = old_data
except (NotFound, NotAuthorized):
abort(404, _('Dataset not found'))
# are we doing a multiphase add?
if data.get('state', '').startswith('draft'):
c.form_action = h.url_for(controller='package', action='new')
c.form_style = 'new'
return self.new(data=data, errors=errors,
error_summary=error_summary)
c.pkg = context.get("package")
try:
check_access('package_update', context)
except NotAuthorized:
abort(403, _('User %r not authorized to edit %s') % (c.user, id))
# convert tags if not supplied in data
if data and not data.get('tag_string'):
data['tag_string'] = ', '.join(h.dict_list_reduce(
c.pkg_dict.get('tags', {}), 'name'))
errors = errors or {}
form_snippet = self._package_form(package_type=package_type)
form_vars = {'data': data, 'errors': errors,
'error_summary': error_summary, 'action': 'edit',
'dataset_type': package_type,
}
#c.errors_json = h.json.dumps(errors)
self._setup_template_variables(context, {'id': id},
package_type=package_type)
# we have already completed stage 1
form_vars['stage'] = ['active']
if data.get('state', '').startswith('draft'):
form_vars['stage'] = ['active', 'complete']
edit_template = self._edit_template(package_type)
return render(edit_template,
extra_vars={'form_vars': form_vars,
'form_snippet': form_snippet,
'dataset_type': package_type})
def pager_url(q=None, page=None):
url = h.url_for(
controller=
'ckanext.dgu_orgs.controllers.organisation:OrganisationController',
action='read',
id=c.group_dict.get('name'))
params = [('page', str(page))]
return url + u'?' + urlencode(params)
def bulk_process(self, id):
''' Allow bulk processing of datasets for an organization. Make
private/public or delete. For organization admins.'''
group_type = self._get_group_type(id.split('@')[0])
if group_type != 'organization':
# FIXME: better error
raise Exception('Must be an organization')
# check we are org admin
context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': self._db_to_form_schema(group_type=group_type),
'for_view': True, 'extras_as_string': True}
data_dict = {'id': id}
try:
c.group_dict = self._action('group_show')(context, data_dict)
c.group = context['group']
except NotFound:
abort(404, _('Group not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read group %s') % id)
# Search within group
action = request.params.get('bulk_action')
# If no action then just show the datasets
if not action:
# unicode format (decoded from utf8)
limit = 500
self._read(id, limit)
c.packages = c.page.items
return render(self._bulk_process_template(group_type))
# process the action first find the datasets to perform the action on. they are prefixed by dataset_ in the form data
datasets = []
for param in request.params:
if param.startswith('dataset_'):
datasets.append(param[8:])
action_functions = {
'private': 'bulk_update_private',
'public': 'bulk_update_public',
'delete': 'bulk_update_delete',
}
data_dict = {'datasets': datasets, 'org_id': c.group_dict['id']}
try:
get_action(action_functions[action])(context, data_dict)
except NotAuthorized:
abort(401, _('Not authorized to perform bulk update'))
base.redirect(h.url_for(controller='organization', action='bulk_process',
id=id))
def upload(self):
label = key_prefix + request.params.get('filepath', str(uuid.uuid4()))
c.data = {
'action': h.url_for('storage_upload_handle', qualified=False),
'fields': [{
'name': 'key',
'value': label
}]
}
return render('storage/index.html')
def _get_form_data(self, label):
storage_backend = config["ofs.impl"]
if storage_backend in ["google", "s3"]:
return self._get_remote_form_data(label)
else:
data = {
"action": h.url_for("storage_upload_handle", qualified=True),
"fields": [{"name": "key", "value": label}],
}
return data
def _get_form_data(self, label):
data = {
'action': h.url_for('storage_upload_handle', qualified=True),
'fields': [
{
'name': 'key',
'value': label
}
]
}
return data
def get_metadata(self, label):
bucket = BUCKET
url = h.url_for('storage_file',
label=label,
qualified=True
)
if not self.ofs.exists(bucket, label):
abort(404)
metadata = self.ofs.get_metadata(bucket, label)
metadata["_location"] = url
return metadata
def upload(self):
label = key_prefix + request.params.get('filepath', str(uuid.uuid4()))
c.data = {
'action': h.url_for('storage_upload_handle'),
'fields': [
{
'name': 'key',
'value': label
}
]
}
return render('storage/index.html')
def type_redirect(self, resource_name):
orgs = h.organizations_available('read')
if not orgs:
abort(404, _('No organizations found'))
try:
chromo = get_chromo(resource_name)
except RecombinantException:
abort(404, _('Recombinant resource_name not found'))
return redirect(h.url_for('recombinant_resource',
resource_name=resource_name, owner_org=orgs[0]['name']))
def _get_form_data(self, label):
storage_backend = config['ofs.impl']
if storage_backend in ['google', 's3']:
return self._get_remote_form_data(label)
else:
data = {
'action': h.url_for('storage_upload_handle', qualified=False),
'fields': [{
'name': 'key',
'value': label
}]
}
return data
def _form_save_redirect(self, pkgname, action):
'''This redirects the user to the CKAN package/read page,
unless there is request parameter giving an alternate location,
perhaps an external website.
@param pkgname - Name of the package just edited
@param action - What the action of the edit was
'''
assert action in ('new', 'edit')
if action == 'new':
msg = _('<span class="new-dataset">Congratulations, your dataset has been created. ' \
'<a href="%s">Upload or link ' \
'some data now »</a></span>')
msg = msg % h.url_for(controller='package', action='edit',
id=pkgname, anchor='section-resources')
h.flash_success(msg,allow_html=True)
url = request.params.get('return_to') or \
config.get('package_%s_return_url' % action)
if url:
url = url.replace('<NAME>', pkgname)
else:
url = h.url_for(controller='package', action='read', id=pkgname)
redirect(url)
def _form_save_redirect(self, pkgname, action):
'''This redirects the user to the CKAN package/read page,
unless there is request parameter giving an alternate location,
perhaps an external website.
@param pkgname - Name of the package just edited
@param action - What the action of the edit was
'''
assert action in ('new', 'edit')
if action == 'new':
msg = _('<span class="new-dataset">Congratulations, your dataset has been created. ' \
'<a href="%s">Upload or link ' \
'some data now »</a></span>')
msg = msg % h.url_for(controller='package', action='edit',
id=pkgname, anchor='section-resources')
h.flash_success(msg,allow_html=True)
url = request.params.get('return_to') or \
config.get('package_%s_return_url' % action)
if url:
url = url.replace('<NAME>', pkgname)
else:
url = h.url_for(controller='package', action='read', id=pkgname)
redirect(url)
def type_redirect(self, resource_name):
orgs = h.organizations_available('read')
if not orgs:
abort(404, _('No organizations found'))
try:
chromo = get_chromo(resource_name)
except RecombinantException:
abort(404, _('Recombinant resource_name not found'))
return redirect(
h.url_for('recombinant_resource',
resource_name=resource_name,
owner_org=orgs[0]['name']))
def _form_save_redirect(self, pkgname, action):
"""This redirects the user to the CKAN package/read page,
unless there is request parameter giving an alternate location,
perhaps an external website.
@param pkgname - Name of the package just edited
@param action - What the action of the edit was
"""
assert action in ("new", "edit")
url = request.params.get("return_to") or config.get("package_%s_return_url" % action)
if url:
url = url.replace("<NAME>", pkgname)
else:
url = h.url_for(controller="package", action="read", id=pkgname)
redirect(url)
def upload(self, id):
package_type = self._get_package_type(id)
try:
if request.POST['xls_update'] == u'':
raise ValidationError({'xls_update': 'You must provide a valid file'})
upload_data = read_xls('', file_contents = request.POST['xls_update'].file.read())
sheet_name, org_name = next(upload_data)
lc = ckanapi.LocalCKAN(username = c.user)
package = lc.action.package_show(id = id)
owner_org = package['organization']['name']
#is this the right sheet for this organization?
if org_name != owner_org:
msg = ('Invalid sheet for this organization. Sheet must be labeled for{0}, ' +
'but you supplied a sheet for {1}').format(owner_org, org_name)
raise ValidationError({'xls_update': msg})
for t in _get_tables():
if t['xls_sheet_name'] == sheet_name:
break
else:
msg = "Sheet name '{0}' not found in list of valid tables".format(sheet_name)
raise ValidationError({'xls_update': msg})
resource_id = package['resources'][0]['id']
records = []
fields = t['datastore_table']['fields']
for n, row in enumerate(upload_data):
if len(row) != len(fields):
msg = ("Row {0} of this sheet has {1} columns, "
"expecting {2}").format(n+3, len(row), len(fields))
raise ValidationError({'xls_update': msg})
records.append(dict(
(f['id'], v) for f, v in zip(fields, row)))
lc.action.datastore_upsert(resource_id=resource_id, records=records)
redirect(h.url_for(controller='package',
action='read', id=id))
except ValidationError, e:
errors = []
for error in e.error_dict.values():
errors.append(str(error).decode('utf-8'))
vars = {'errors': errors, 'action': 'edit'}
c.pkg_dict = package
return render(self._edit_template(package_type), extra_vars = vars)
def get_metadata(self, label):
bucket = BUCKET
storage_backend = config["ofs.impl"]
if storage_backend in ["google", "s3"]:
if not label.startswith("/"):
label = "/" + label
url = "https://%s/%s%s" % (self.ofs.conn.server_name(), bucket, label)
else:
url = h.url_for("storage_file", label=label, qualified=True)
if not self.ofs.exists(bucket, label):
abort(404)
metadata = self.ofs.get_metadata(bucket, label)
metadata["_location"] = url
return metadata
def new(self):
c.error = ''
api_url = config.get('ckan.api_url', '/').rstrip('/')
c.package_create_slug_api_url = api_url+h.url_for(controller='api', action='create_slug')
is_admin = self.authorizer.is_sysadmin(c.user)
# Check access control for user to create a package.
auth_for_create = self.authorizer.am_authorized(c, model.Action.PACKAGE_CREATE, model.System())
if not auth_for_create:
abort(401, _('Unauthorized to create a package'))
# Get the name of the package form.
try:
fs = self._get_package_fieldset(is_admin=is_admin)
except ValueError, e:
abort(400, e)
def get_metadata(self, label):
bucket = BUCKET
storage_backend = config['ofs.impl']
if storage_backend in ['google', 's3']:
if not label.startswith("/"):
label = "/" + label
url = "https://%s/%s%s" % (self.ofs.conn.server_name(), bucket,
label)
else:
url = h.url_for('storage_file', label=label, qualified=True)
if not self.ofs.exists(bucket, label):
abort(404)
metadata = self.ofs.get_metadata(bucket, label)
metadata["_location"] = url
return metadata
def _get_form_data(self, label):
storage_backend = config['ofs.impl']
if storage_backend in ['google', 's3']:
return self._get_remote_form_data(label)
else:
data = {
'action': h.url_for('storage_upload_handle', qualified=True),
'fields': [
{
'name': 'key',
'value': label
}
]
}
return data
def _form_save_redirect(self, pkgname, action):
'''This redirects the user to the CKAN package/read page,
unless there is request parameter giving an alternate location,
perhaps an external website.
@param pkgname - Name of the package just edited
@param action - What the action of the edit was
'''
assert action in ('new', 'edit')
url = request.params.get('return_to') or \
config.get('package_%s_return_url' % action)
if url:
url = url.replace('<NAME>', pkgname)
else:
url = h.url_for(controller='package', action='read', id=pkgname)
redirect(url)
def new(self):
c.error = ''
api_url = config.get('ckan.api_url', '/').rstrip('/')
c.package_create_slug_api_url = api_url + h.url_for(
controller='api', action='create_slug')
is_admin = self.authorizer.is_sysadmin(c.user)
# Check access control for user to create a package.
auth_for_create = self.authorizer.am_authorized(
c, model.Action.PACKAGE_CREATE, model.System())
if not auth_for_create:
abort(401, _('Unauthorized to create a package'))
# Get the name of the package form.
try:
fs = self._get_package_fieldset(is_admin=is_admin)
except ValueError, e:
abort(400, e)
def _sort_by(fields):
"""
Sort by the given list of fields.
Each entry in the list is a 2-tuple: (fieldname, sort_order)
eg - [('metadata_modified', 'desc'), ('name', 'asc')]
If fields is empty, then the default ordering is used.
"""
params = params_nosort[:]
if fields:
sort_string = ', '.join('%s %s' % f for f in fields)
params.append(('sort', sort_string))
return url_with_params(h.url_for(controller='ckanext.kata.controllers:KataPackageController', action='browse'), params)
def get_metadata(self, label):
bucket = BUCKET
storage_backend = config['ofs.impl']
if storage_backend in ['google', 's3']:
if not label.startswith("/"):
label = "/" + label
url = "https://%s%s" % (self.ofs.conn.calling_format.build_host(
self.ofs.conn.server_name(), bucket), label)
else:
url = h.url_for('storage_file', label=label, qualified=False)
if url.startswith('/'):
url = config.get('ckan.site_url', '').rstrip('/') + url
if not self.ofs.exists(bucket, label):
abort(404)
metadata = self.ofs.get_metadata(bucket, label)
metadata["_location"] = url
return metadata
def dgu_linked_user(user, maxlength=16): # Overwrite h.linked_user
from ckan import model
from ckan.lib.base import h
from ckanext.dgu.plugins_toolkit import c
if user in [model.PSEUDO_USER__LOGGED_IN, model.PSEUDO_USER__VISITOR]:
return user
if not isinstance(user, model.User):
user_name = unicode(user)
user = model.User.get(user_name)
if not user:
# may be in the format "NHS North Staffordshire (uid 6107 )"
match = re.match('.*\(uid (\d+)\s?\)', user_name)
if match:
drupal_user_id = match.groups()[0]
user = model.User.get('user_d%s' % drupal_user_id)
if (c.is_an_official):
# only officials can see the actual user name
if user:
publisher = ', '.join([group.title for group in user.get_groups('publisher')])
display_name = '%s (%s)' % (user.fullname, publisher)
link_text = truncate(user.fullname or user.name, length=maxlength)
return h.link_to(link_text,
h.url_for(controller='user', action='read', id=user.name))
else:
return truncate(user_name, length=maxlength)
else:
# joe public just gets a link to the user's publisher(s)
import ckan.authz
if user:
groups = user.get_groups('publisher')
if groups:
return h.literal(' '.join([h.link_to(truncate(group.title, length=maxlength),
'/publisher/%s' % group.name) \
for group in groups]))
elif ckan.authz.Authorizer().is_sysadmin(user):
return 'System Administrator'
else:
return 'Staff'
else:
return 'Staff'
def index(self):
"""
This function renders the contribute landing page for a node-in-a-box and the harvest page for the central node.
"""
if not c.user:
h.redirect_to(
controller='user',
action='login',
came_from=h.url_for(
controller=
'ckanext.ngds.ngdsui.controllers.contribute:ContributeController',
action='index'))
if g.central:
#TODO: Need to change this to point the correct controller
url = h.url_for_static(
controller='ckanext.harvest.controllers.view:ViewController')
h.redirect_to(url)
else:
return render('contribute/contribute.html')
def _download_file(self, relative_path, label):
file_path = BASE_PATH + relative_path + label
exists = os.path.isfile(file_path)
if not exists:
# handle erroneous trailing slash by redirecting to url w/o slash
if label.endswith('/'):
label = label[:-1]
# This may be best being cached_url until we have moved it into
# permanent storage
file_url = h.url_for('storage_file', label=label)
h.redirect_to(file_url)
else:
# abort(404)
r = generate_response(404, None, error=u'File not found')
return r
with open(file_path, 'rb') as f:
image = f.read()
return generate_response(200,
image,
other_headers={'Content-type': 'image'})
def upload_handle(self):
bucket_id = BUCKET
params = dict(request.params.items())
stream = params.get('file')
label = params.get('key')
authorize('POST', BUCKET, label, c.userobj, self.ofs)
if not label:
abort(400, "No label")
if not isinstance(stream, FieldStorage):
abort(400, "No file stream.")
del params['file']
params['filename-original'] = stream.filename
#params['_owner'] = c.userobj.name if c.userobj else ""
params['uploaded-by'] = c.userobj.name if c.userobj else ""
self.ofs.put_stream(bucket_id, label, stream.file, params)
success_action_redirect = h.url_for('storage_upload_success', qualified=True,
bucket=BUCKET, label=label)
# Do not redirect here as it breaks js file uploads (get infinite loop
# in FF and crash in Chrome)
return self.success(label)
def _get_remote_form_data(self, label):
method = 'POST'
content_length_range = int(
config.get('ckan.storage.max_content_length',
50000000))
acl = 'public-read'
fields = [ {
'name': self.ofs.conn.provider.metadata_prefix + 'uploaded-by',
'value': c.userobj.id
}]
conditions = [ '{"%s": "%s"}' % (x['name'], x['value']) for x in
fields ]
# In FF redirect to this breaks js upload as FF attempts to open file
# (presumably because mimetype = javascript) and this stops js
# success_action_redirect = h.url_for('storage_api_get_metadata', qualified=True,
# label=label)
success_action_redirect = h.url_for('storage_upload_success_empty', qualified=True,
label=label)
data = self.ofs.conn.build_post_form_args(
BUCKET,
label,
expires_in=72000,
max_content_length=content_length_range,
success_action_redirect=success_action_redirect,
acl=acl,
fields=fields,
conditions=conditions
)
# HACK: fix up some broken stuff from boto
# e.g. should not have content-length-range in list of fields!
storage_backend = config['ofs.impl']
for idx,field in enumerate(data['fields']):
if storage_backend == 'google':
if field['name'] == 'AWSAccessKeyId':
field['name'] = 'GoogleAccessId'
if field['name'] == 'content-length-range':
del data['fields'][idx]
return data
def upload(self, id):
package_type = self._get_package_type(id)
geno = get_geno(package_type)
lc = ckanapi.LocalCKAN(username=c.user)
dataset = lc.action.package_show(id=id)
try:
if request.POST['xls_update'] == '':
raise BadExcelData('You must provide a valid file')
_process_upload_file(lc, dataset, request.POST['xls_update'].file,
geno)
h.flash_success(
_("Your file was successfully uploaded into the central system."
))
redirect(h.url_for(controller='package', action='read', id=id))
except BadExcelData, e:
org = lc.action.organization_show(id=dataset['owner_org'])
return self.preview_table(
resource_name=dataset['resources'][0]['name'],
owner_org=org['name'],
errors=[e.message])
def render_contact_form(self, pkg_id):
"""
Render the contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if not c.package:
abort(404, _(u"Dataset not found"))
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = []
for contact in contacts:
if 'name' in contact:
text_val = contact['name']
else:
at_idx = contact['email'].find('@')
text_val = contact['email'][0:at_idx]
text_val = text_val.replace(".", " ").title()
print(text_val)
c.recipient_options.append({'text': text_val, 'value': contact['id']})
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/contact_form.html')
def delete_record(self, id, resource_id):
lc = ckanapi.LocalCKAN(username=c.user)
filters = {}
res = lc.action.resource_show(id=resource_id)
for f in recombinant_primary_key_fields(res['name']):
filters[f['datastore_id']] = request.POST.get(f['datastore_id'], '')
result = lc.action.datastore_search(
resource_id=resource_id,
filters=filters,
rows=2) # only need two to know if there are multiple matches
records = result['records']
x_vars = {'filters': filters, 'action': 'edit'}
if not records:
x_vars['delete_errors'] = [_('No matching records found')]
elif len(records) > 1:
x_vars['delete_errors'] = [_('Multiple matching records found')]
if 'delete_errors' in x_vars:
c.pkg_dict = dataset = lc.action.package_show(id=id)
return render('recombinant/resource_edit.html',
extra_vars=dict(x_vars, dataset=dataset, resource=res))
# XXX: can't avoid the race here with the existing datastore API.
# datastore_delete doesn't support _id filters
lc.action.datastore_delete(
resource_id=resource_id,
filters=filters,
)
h.flash_success(_(
"Record deleted."
))
redirect(h.url_for(
controller='ckanext.recombinant.controller:PreviewController',
action='preview_table', id=id, resource_id=resource_id))
def upload(self, id):
package_type = self._get_package_type(id)
geno = get_geno(package_type)
lc = ckanapi.LocalCKAN(username=c.user)
dataset = lc.action.package_show(id=id)
try:
if request.POST['xls_update'] == '':
raise BadExcelData('You must provide a valid file')
_process_upload_file(
lc,
dataset,
request.POST['xls_update'].file,
geno)
h.flash_success(_(
"Your file was successfully uploaded into the central system."
))
redirect(h.url_for(controller='package', action='read', id=id))
except BadExcelData, e:
x_vars = {'errors': [e.message], 'action': 'edit'}
c.pkg_dict = dataset
return render(self._edit_template(package_type), extra_vars=x_vars)
def index(self):
group_type = None
context = {
'model': model,
'session': model.Session,
'user': c.user or c.author,
'for_view': True,
'with_private': False
}
q = c.q = request.params.get('q', '')
data_dict = {'all_fields': True, 'q': q}
sort_by = c.sort_by_selected = request.params.get('sort')
if sort_by:
data_dict['sort'] = sort_by
try:
self._check_access('site_read', context)
except NotAuthorized:
abort(403, _('Not authorized to see this page'))
if c.userobj:
context['user_id'] = c.userobj.id
context['user_is_admin'] = c.userobj.sysadmin
# c.group_package_stuff = caching.cached_get_group_package_stuff()
#
# ##Removing groups without geojson for the map
# c.group_map = []
# for gp in c.group_package_stuff:
# '''
# Removed check for geojson data because in the new version this information
# does not come from the group_list action and for now we are not using the map.
# If we'll need this we should implement some caching functionality for this too.
# '''
# c.group_map.append(gp)
if c.userobj is not None:
site_title = config.get('ckan.site_title', 'CKAN')
msg = None
url = h.url_for(controller='user', action='edit')
is_google_id = \
c.userobj.name.startswith('https://www.google.com/accounts/o8/id')
if not c.userobj.email and (is_google_id
and not c.userobj.fullname):
msg = _(u'Please <a href="{link}">update your profile</a>'
u' and add your email address and your full name. '
u'{site} uses your email address'
u' if you need to reset your password.'.format(
link=url, site=site_title))
elif not c.userobj.email:
msg = _('Please <a href="%s">update your profile</a>'
' and add your email address. ') % url + \
_('%s uses your email address'
' if you need to reset your password.') \
% site_title
elif is_google_id and not c.userobj.fullname:
msg = _('Please <a href="%s">update your profile</a>'
' and add your full name.') % (url)
if msg:
h.flash_notice(msg, allow_html=True)
# template_data = {
# 'data': {
# 'hdx.carousel.config': logic.get_action('hdx_carousel_settings_show')({}, {})
# }
# }
c.structured_data = structured_data
return base.render('home/index.html')
def search_url(params):
url = h.url_for(controller='ckanext.dgu.controllers.inventory:InventoryController', action='search')
return url_with_params(url, params)
