def render_request_form(self, pkg_id):
"""
Render the access request contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if not c.package:
abort(404, _(u"Dataset not found"))
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts]
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/dataset_request_form.html')
def create_access_request(context, data_dict):
if not context.get('ignore_auth'):
check_access('create_access_request', context, data_dict)
pkg_id_or_name, reason, user = get_or_bust(data_dict,
['id', 'reason', 'user'])
user_org = data_dict.get('user_org')
# check if the package with such id exists to use it's ID
pkg = Package.get(pkg_id_or_name)
if not pkg:
raise ObjectNotFound()
req = AccessRequest.create_or_get_request(user_id=user,
package_id=pkg.id,
reason=reason,
org_id=pkg.owner_org)
# send email notifications to org custodians
if config.get('spc.access_request.send_admin_notification'):
notify_org_members(pkg.owner_org, {
'pkg': pkg,
'user': user,
'reason': reason,
'user_org': user_org
})
return req.as_dict()
def send(self, pkg_id):
package = Package.get(pkg_id)
url = h.url_for(controller='package',
action="read",
id=package.id)
if c.user:
userid = None
for role in package.roles:
if role.role == "admin":
userid = role.user_id
break
if userid:
owner = User.get(userid)
msg = request.params.get('msg', '')
if msg:
send_contact_email(owner, c.userobj, package,\
msg)
else:
h.flash_error(_("No message"))
return redirect(url)
else:
h.flash_error(_("No owner found"))
return redirect(url)
h.flash_notice(_("Message sent"))
else:
h.flash_error(_("Please login"))
return redirect(url)
def getRecord(self, metadataPrefix, identifier):
'''Simple getRecord for a dataset.
'''
package = Package.get(identifier)
if not package:
raise IdDoesNotExistError("No dataset with id %s" % identifier)
return self._record_for_dataset(package)
def test_update_extent(self):
package = Package.get('annakarenina')
geojson = json.loads(self.geojson_examples['point'])
shape = asShape(geojson)
package_extent = PackageExtent(package_id=package.id,
the_geom=WKTSpatialElement(
shape.wkt, self.db_srid))
package_extent.save()
assert Session.scalar(
package_extent.the_geom.geometry_type) == 'ST_Point'
# Update the geometry (Point -> Polygon)
geojson = json.loads(self.geojson_examples['polygon'])
shape = asShape(geojson)
package_extent.the_geom = WKTSpatialElement(shape.wkt, self.db_srid)
package_extent.save()
assert package_extent.package_id == package.id
assert Session.scalar(
package_extent.the_geom.geometry_type) == 'ST_Polygon'
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def send_email(req):
requestee = User.get(req.user_id)
pkg = Package.get(req.pkg_id)
selrole = False
for role in pkg.roles:
if role.role == "admin":
selrole = role
if not selrole:
return
admin = User.get(selrole.user_id)
msg = _("""%s (%s) is requesting editor access to a dataset you have created
%s.
Please click this link if you want to give this user write access:
%s%s""")
controller = 'ckanext.kata.controllers:AccessRequestController'
body = msg % (requestee.name, requestee.email, pkg.title if pkg.title else pkg.name,
config.get('ckan.site_url', ''),
h.url_for(controller=controller,
action="unlock_access",
id=req.id))
email_dict = {}
email_dict["subject"] = _("Access request for dataset %s" % pkg.title if pkg.title else pkg.name)
email_dict["body"] = body
send_notification(admin.as_dict(), email_dict)
def accessreq_show(context, data_dict):
ret = {}
ret['title'] = _('Request edit rights')
smtp = config.get('smtp.server', '')
if not len(smtp):
ret['ret'] = 'Yes'
return ret
pkg = Package.get(data_dict['id'])
selrole = False
ret['id'] = pkg.id
for role in pkg.roles:
if role.role == "admin":
selrole = True
ret['no_owner'] = not selrole
if c.userobj:
if 'id' in data_dict:
req = KataAccessRequest.is_requesting(c.userobj.id, data_dict['id'])
if req:
ret['ret'] = 'Yes'
return ret
else:
ret['ret'] = 'No'
return ret
else:
ret['ret'] = 'No'
return ret
ret['ret'] = 'No'
return ret
def test_new(self):
name = "test-spatial-dataset-1"
offset = url_for(controller="package", action="new")
res = self.app.get(offset, extra_environ=self.extra_environ)
assert "Add - Datasets" in res
fv = res.forms["dataset-edit"]
prefix = ""
fv[prefix + "name"] = name
fv[prefix + "extras__0__key"] = u"spatial"
fv[prefix + "extras__0__value"] = self.geojson_examples["point"]
res = fv.submit("save", extra_environ=self.extra_environ)
assert not "Error" in res, res
package = Package.get(name)
# Check that a PackageExtent object has been created
package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id == package.id).first()
geojson = json.loads(self.geojson_examples["point"])
assert package_extent
assert package_extent.package_id == package.id
assert Session.scalar(package_extent.the_geom.x) == geojson["coordinates"][0]
assert Session.scalar(package_extent.the_geom.y) == geojson["coordinates"][1]
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def render_request_form(self, pkg_id):
"""
Render the access request contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
if not c.package:
abort(404, _(u"Dataset not found"))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts]
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/dataset_request_form.html')
def is_owner(context, data_dict):
'''
This is used in "request edit rights" feature.
Checks if the user is creator, admin or editor of the
package in question
:param context: context
:param data_dict: package data
:type data_dict: dictionary
:rtype: dictionary
'''
# Package creator is always owner regardless of organizations
pkg = context.get('package', None) or Package.get(data_dict['id'])
user = context.get('user', False)
# If user id can't be resolved, user can't be owner
try:
user_id = convert_user_name_or_id_to_id(user, context)
except:
return {'success': False}
if pkg.creator_user_id == user_id:
return {'success': True}
# Check if the user has editor rights to this dataset through an organization
package = get_package_object(context, data_dict)
if authz.has_user_permission_for_group_or_org(package.owner_org, user, 'delete_dataset'):
return {'success': True}
return {'success': False}
def is_owner(context, data_dict):
'''
This is used in "request edit rights" feature.
Checks if the user is creator, admin or editor of the
package in question
:param context: context
:param data_dict: package data
:type data_dict: dictionary
:rtype: dictionary
'''
# Package creator is always owner regardless of organizations
pkg = context.get('package', None) or Package.get(data_dict['id'])
user = context.get('user', False)
# If user id can't be resolved, user can't be owner
try:
user_id = convert_user_name_or_id_to_id(user, context)
except:
return {'success': False}
if pkg.creator_user_id == user_id:
return {'success': True}
# Check if the user has editor rights to this dataset through an organization
package = get_package_object(context, data_dict)
if authz.has_user_permission_for_group_or_org(package.owner_org, user,
'delete_dataset'):
return {'success': True}
return {'success': False}
def test_new(self):
name = 'test-spatial-dataset-1'
offset = url_for(controller='package', action='new')
res = self.app.get(offset, extra_environ=self.extra_environ)
assert 'Add - Datasets' in res
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'name'] = name
fv[prefix + 'extras__0__key'] = u'spatial'
fv[prefix + 'extras__0__value'] = self.geojson_examples['point']
res = fv.submit('save', extra_environ=self.extra_environ)
assert not 'Error' in res, res
package = Package.get(name)
# Check that a PackageExtent object has been created
package_extent = Session.query(PackageExtent).filter(
PackageExtent.package_id == package.id).first()
geojson = json.loads(self.geojson_examples['point'])
assert package_extent
assert package_extent.package_id == package.id
assert Session.scalar(
package_extent.the_geom.x) == geojson['coordinates'][0]
assert Session.scalar(
package_extent.the_geom.y) == geojson['coordinates'][1]
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def is_owner(context, data_dict):
'''
This is used in "request edit rights" feature.
Checks if the user is admin or editor of the
package in question
:param context: context
:param data_dict: package data
:type data_dict: dictionary
:rtype: dictionary
'''
pkg = context.get('package', None)
roles = pkg.roles if pkg else Package.get(data_dict['id']).roles
user = context.get('user', False)
if user:
for role in roles:
ruser = User.get(role.user.id)
if user == ruser.name and role.role in ('admin', 'editor'):
return {'success': True}
# Check if the user has editor rights to this dataset through an organization
package = get_package_object(context, data_dict)
if new_authz.has_user_permission_for_group_or_org(package.owner_org, user, 'delete_dataset'):
return {'success': True}
return {'success': False}
def _create_pkg(self):
model.repo.new_revision()
pkg = Package.get("annakarenina")
pkg.name = "fookarenina"
pkg.add_resource("www.google.com", description="foo", name="foo")
Session.add(pkg)
Session.commit()
return pkg
def _create_pkg(self):
model.repo.new_revision()
pkg = Package.get('annakarenina')
pkg.name = "fookarenina"
pkg.add_resource('www.google.com', description='foo', name="foo")
Session.add(pkg)
Session.commit()
return pkg
def get_ancestral_license(session, package_id):
license_id = None
if package_id:
package = Package.get(package_id)
extras = package.extras
if 'ancestral_license' in extras:
license_id = extras['ancestral_license']
return license_id
def get_resources_licenses(session, pkg_dict):
license_ids = []
package = Package.get(pkg_dict['id'])
for resource in package.resources:
res_license = session.query(CeonResourceLicense).filter(CeonResourceLicense.resource_id == resource.id).first()
if res_license:
license_ids.append(res_license.license_id)
return license_ids
def get_resources_licenses(session, pkg_dict):
license_ids = []
package = Package.get(pkg_dict['id'])
for resource in package.resources:
res_license = session.query(CeonResourceLicense).filter(CeonResourceLicense.resource_id == resource.id).first()
if res_license:
license_ids.append(res_license.license_id)
return license_ids
def get_ancestral_license(session, package_id):
license_id = None
if package_id:
package = Package.get(package_id)
extras = package.extras
if 'ancestral_license' in extras:
license_id = extras['ancestral_license']
return license_id
def package_show(context, data_dict):
'''
Return the metadata of a dataset (package) and its resources.
Called before showing the dataset in some interface (browser, API),
or when adding package to Solr index (no validation / conversions then).
:param id: the id or name of the dataset
:type id: string
:rtype: dictionary
'''
if data_dict.get('type') == 'harvest':
context['schema'] = Schemas.harvest_source_show_package_schema()
if not data_dict.get('id') and not data_dict.get('name'):
# Get package by data PIDs
data_dict['id'] = utils.get_package_id_by_data_pids(data_dict)
pkg_dict1 = ckan.logic.action.get.package_show(context, data_dict)
pkg_dict1 = utils.resource_to_dataset(pkg_dict1)
# Remove empty agents that come from padding the agent list in converters
if 'agent' in pkg_dict1:
agents = filter(None, pkg_dict1.get('agent', []))
pkg_dict1['agent'] = agents or []
# Normally logic function should not catch the raised errors
# but here it is needed so action package_show won't catch it instead
# Hiding information from API calls
try:
check_access('package_update', context)
except NotAuthorized:
pkg_dict1 = utils.hide_sensitive_fields(pkg_dict1)
pkg = Package.get(pkg_dict1['id'])
if 'erelated' in pkg.extras:
erelated = pkg.extras['erelated']
if len(erelated):
for value in erelated.split(';'):
if len(Session.query(Related).filter(Related.title == value).all()) == 0:
data_dict = {'title': value,
'type': _("Paper"),
'dataset_id': pkg.id}
related_create(context, data_dict)
# Update package.title to match package.extras.title_0
extras_title = pkg.extras.get(u'title_0')
if extras_title and extras_title != pkg.title:
repo.new_revision()
pkg.title = pkg.extras[u'title_0']
pkg.save()
rebuild(pkg.id) # Rebuild solr-index for this dataset
return pkg_dict1
def getMetadataDDI(self,id):
toolkit.response.content_type = 'application/xml'
c.pkg = Package.get(id)
if c.pkg is None:
abort(404, 'Dataset not found')
pkginfo = toolkit.get_action('package_show')({}, {'id': id})
try:
return createDDIXML(pkginfo,toolkit.request.url)
except Exception,e:
print str(e)
return "[]"
def getMetadataDDI(self, id):
toolkit.response.content_type = 'application/xml'
c.pkg = Package.get(id)
if c.pkg is None:
abort(404, 'Dataset not found')
pkginfo = toolkit.get_action('package_show')({}, {'id': id})
try:
return createDDIXML(pkginfo, toolkit.request.url)
except Exception, e:
print str(e)
return "[]"
def getRecord(self, metadataPrefix, identifier):
'''Simple getRecord for a dataset.
'''
package = Package.get(identifier)
if not package:
raise IdDoesNotExistError("No dataset with id %s" % identifier)
spec = package.name
if package.owner_org:
group = Group.get(package.owner_org)
if group and group.name:
spec = group.name
return self._record_for_dataset(package, spec)
def googleanalytics_dataset_visits(context=None, data_dict=None):
"""
Fetch the amount of times a dataset hs been visited
:param id: Dataset id
:type id: string
:returns: The number of times the dataset has been viewed
:rtype: integer
"""
package = Package.get(data_dict['id'])
return PackageStats.get_all_visits(package.id)
def show(self,id):
package = Package.get(id)
if package:
graph = pkg_produce(package)
doc = graph.serialize(format='pretty-xml')
response.content_type = 'application/rdf+xml'
response.headers['Content-Length'] = len(doc)
return doc
else:
abort(404)
def is_owner(context, data_dict):
pkg = context.get('package', None)
roles = pkg.roles if pkg else Package.get(data_dict['id']).roles
user = context.get('user', False)
if user:
for role in roles:
ruser = User.get(role.user_id)
if user == ruser.name and role.role in ('admin', 'editor'):
return {'success': True}
else:
return {'success': False}
return {'success': False}
def create_request(self, pkg_id):
pkg = Package.get(pkg_id)
user = c.userobj if c.userobj else None
if user:
req = KataAccessRequest(user.id, pkg.id)
req.save()
url = h.url_for(controller='package', action='read', id=pkg_id)
h.flash_success(_("You now requested editor rights to package %s" % pkg.name))
redirect(url)
else:
url = h.url_for(controller='package', action='read', id=pkg_id)
h.flash_error(_("Please log in!"))
redirect(url)
def request_access(self, pkg_id):
# If the user is logged in show the access request button
pkg = Package.get(pkg_id)
if c.user and not user_has_role(c.userobj, 'admin', pkg) and\
not user_has_role(c.userobj, 'editor', pkg) and\
not c.userobj.sysadmin and\
not config.get('smtp_server', False):
following = KataAccessRequest.is_requesting(c.userobj.id, pkg_id)
if not following:
return snippet('snippets/access_button.html',
obj_id=pkg_id,
following=following)
return ''
def wms_preview(self,id):
#check if package exists
c.pkg = Package.get(id)
if c.pkg is None:
abort(404, 'Dataset not found')
for res in c.pkg.resources:
if res.format == "WMS":
c.wms_url = res.url if not '?' in res.url else res.url.split('?')[0]
break
if not c.wms_url:
abort(400, 'This dataset does not have a WMS resource')
return render('ckanext/spatial/wms_preview.html')
def package_show(context, data_dict):
'''
Return the metadata of a dataset (package) and its resources.
Called before showing the dataset in some interface (browser, API),
or when adding package to Solr index (no validation / conversions then).
:param id: the id or name of the dataset
:type id: string
:rtype: dictionary
'''
if data_dict.get('type') == 'harvest':
context['schema'] = Schemas.harvest_source_show_package_schema()
context['use_cache'] = False # Disable package retrieval directly from Solr as contact.email is not there.
if not data_dict.get('id') and not data_dict.get('name'):
# Get package by data PIDs
data_dict['id'] = utils.get_package_id_by_primary_pid(data_dict)
pkg_dict1 = ckan.logic.action.get.package_show(context, data_dict)
pkg_dict1 = utils.resource_to_dataset(pkg_dict1)
# Remove empty agents that come from padding the agent list in converters
if 'agent' in pkg_dict1:
agents = filter(None, pkg_dict1.get('agent', []))
pkg_dict1['agent'] = agents or []
# Normally logic function should not catch the raised errors
# but here it is needed so action package_show won't catch it instead
# Hiding information from API calls
try:
check_access('package_update', context)
except NotAuthorized:
pkg_dict1 = utils.hide_sensitive_fields(pkg_dict1)
pkg = Package.get(pkg_dict1['id'])
if 'erelated' in pkg.extras:
erelated = pkg.extras['erelated']
if len(erelated):
for value in erelated.split(';'):
if len(Session.query(Related).filter(Related.title == value).all()) == 0:
data_dict = {'title': value,
'type': _("Paper"),
'dataset_id': pkg.id}
related_create(context, data_dict)
return pkg_dict1
def getRecord(self, metadataPrefix, identifier):
'''Simple getRecord for a dataset.
'''
package = Package.get(identifier)
if not package:
raise IdDoesNotExistError("No dataset with id %s" % identifier)
spec = package.name
if package.owner_org:
group = Group.get(package.owner_org)
if group and group.name:
spec = group.name
if metadataPrefix == 'rdf':
return self._record_for_dataset_dcat(package, spec)
return self._record_for_dataset(package, spec)
def test_create_extent(self):
package = Package.get('annakarenina')
assert package
geojson = json.loads(self.geojson_examples['point'])
shape = asShape(geojson)
package_extent = PackageExtent(package_id=package.id,the_geom=WKTSpatialElement(shape.wkt, self.db_srid))
package_extent.save()
assert package_extent.package_id == package.id
assert Session.scalar(package_extent.the_geom.x) == geojson['coordinates'][0]
assert Session.scalar(package_extent.the_geom.y) == geojson['coordinates'][1]
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def wms_preview(self,id):
#check if package exists
c.pkg = Package.get(id)
if c.pkg is None:
abort(404, 'Dataset not found')
for res in c.pkg.resources:
if res.format.lower() == 'wms':
c.wms_url = res.url if not '?' in res.url else res.url.split('?')[0]
break
if not c.wms_url:
abort(400, 'This dataset does not have a WMS resource')
return render('ckanext/spatial/wms_preview.html')
def test_edit(self):
name = 'annakarenina'
offset = url_for(controller='package', action='edit', id=name)
res = self.app.get(offset, extra_environ=self.extra_environ)
assert 'Edit - Datasets' in res
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'extras__1__key'] = u'spatial'
fv[prefix + 'extras__1__value'] = self.geojson_examples['point']
res = fv.submit('save', extra_environ=self.extra_environ)
assert not 'Error' in res, res
package = Package.get(name)
# Check that a PackageExtent object has been created
package_extent = Session.query(PackageExtent).filter(
PackageExtent.package_id == package.id).first()
geojson = json.loads(self.geojson_examples['point'])
assert package_extent
assert package_extent.package_id == package.id
assert Session.scalar(
package_extent.the_geom.x) == geojson['coordinates'][0]
assert Session.scalar(
package_extent.the_geom.y) == geojson['coordinates'][1]
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
# Update the spatial extra
offset = url_for(controller='package', action='edit', id=name)
res = self.app.get(offset, extra_environ=self.extra_environ)
assert 'Edit - Datasets' in res
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'extras__1__value'] = self.geojson_examples['polygon']
res = fv.submit('save', extra_environ=self.extra_environ)
assert not 'Error' in res, res
# Check that the PackageExtent object has been updated
package_extent = Session.query(PackageExtent).filter(
PackageExtent.package_id == package.id).first()
assert package_extent
assert package_extent.package_id == package.id
assert Session.scalar(
package_extent.the_geom.geometry_type) == 'ST_Polygon'
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def send_request_message(self, pkg_id):
"""
Sends a data request message by email from the logged in user to the appropriate
contact address of the dataset.
:param pkg_id: package id
:type pkg_id: string
"""
package = Package.get(pkg_id)
package_title = package.title if package.title else package.name
subject = u"Data access request for dataset / Datapyyntö tietoaineistolle %s" % package_title
recipient_id = request.params.get('recipient', '')
return self._prepare_and_send(pkg_id, recipient_id, subject, settings.DATA_REQUEST_PREFIX_TEMPLATE, settings.REPLY_TO_SENDER_NOTE)
def view(self, id):
"""
Render the import form, see if there is a POST request, then call
import_irods_collection.
"""
context = {'model': model, 'user': c.user or c.author}
try:
check_access('package_update', context, {'id': id})
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
pkg = Package.get(id)
c.pkg = pkg.name
if ('save' in request.params):
import_collection_to_package(request.params, id)
return render('ckanext/irods/irods.html')
def send_request_message(self, pkg_id):
"""
Sends a data request message by email from the logged in user to the appropriate
contact address of the dataset.
:param pkg_id: package id
:type pkg_id: string
"""
package = Package.get(pkg_id)
package_title = package.title if package.title else package.name
subject = u"Data access request for dataset / Datapyyntö tietoaineistolle %s" % package_title
recipient_id = request.params.get('recipient', '')
return self._prepare_and_send(pkg_id, recipient_id, subject, settings.DATA_REQUEST_PREFIX_TEMPLATE, settings.REPLY_TO_SENDER_NOTE)
def view(self, id):
"""
Render the import form, see if there is a POST request, then call
import_irods_collection.
"""
context = {'model':model, 'user': c.user or c.author}
try:
check_access('package_update', context, {'id' : id})
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
pkg = Package.get(id)
c.pkg = pkg.name
if ('save' in request.params):
import_collection_to_package(request.params, id)
return render('ckanext/irods/irods.html')
def unlock_access(self, id):
q = model.Session.query(KataAccessRequest)
q = q.filter_by(id=id)
req = q.first()
if req:
user = User.get(req.user_id)
pkg = Package.get(req.pkg_id)
add_user_to_role(user, 'editor', pkg)
url = h.url_for(controller='package', action='read', id=req.pkg_id)
h.flash_success(_("%s now has editor rights to package %s" % (user.name, pkg.name)))
req.delete()
meta.Session.commit()
redirect(url)
else:
h.flash_error(_("No such request found!"))
redirect('/')
def update_ancestral_license(context, pkg_dict, license_id):
session = context['session']
package = Package.get(pkg_dict['id'])
log.debug(u'Updating license for package {}: {}'.format(pkg_dict['name'], PKG_LICENSE_ID))
pkg_license = package.get_license_register()[PKG_LICENSE_ID]
package.set_license(pkg_license)
session.merge(package)
if not license_id:
return
log.debug(u'Updating ancestral license for package {}: {}'.format(pkg_dict['name'], license_id))
for resource in package.resources:
res_license = session.query(CeonResourceLicense).filter(CeonResourceLicense.resource_id == resource.id).first()
if res_license:
session.delete(res_license)
new_res_license = CeonResourceLicense(resource_id = resource.id, license_id = license_id)
session.add(new_res_license)
def update_oa_tag(context, pkg_dict, vocabulary_name, tag_value):
if not isinstance(tag_value, basestring):
try:
tag_value = tag_value[0]
except:
pass
if not tag_value:
return
log.debug(u'Updating {} tag in package {}: {}'.format(vocabulary_name,
pkg_dict['name'], tag_value))
tag = Tag.get(tag_value, vocabulary_name)
if tag:
package = Package.get(pkg_dict['id'])
package.add_tag(tag)
else:
raise Exception(u'Tag "{}" not found within vocabulary "{}"'.format(tag_value, vocabulary_name))
def update_oa_tag(context, pkg_dict, vocabulary_name, tag_value):
if not isinstance(tag_value, basestring):
try:
tag_value = tag_value[0]
except:
pass
if not tag_value:
return
log.debug(u'Updating {} tag in package {}: {}'.format(vocabulary_name,
pkg_dict['name'], tag_value))
tag = Tag.get(tag_value, vocabulary_name)
if tag:
package = Package.get(pkg_dict['id'])
package.add_tag(tag)
else:
raise Exception(u'Tag "{}" not found within vocabulary "{}"'.format(tag_value, vocabulary_name))
def send_contact_message(self, pkg_id):
"""
Sends a message by email from the logged in user to the appropriate
contact address of the dataset.
:param pkg_id: package id
:type pkg_id: string
"""
package = Package.get(pkg_id)
if not package:
ckan.lib.base.abort(404, _("Dataset not found"))
package_title = package.title if package.title else package.name
subject = u"Message regarding dataset / Viesti koskien tietoaineistoa %s" % package_title
recipient_id = request.params.get('recipient', '')
return self._prepare_and_send(pkg_id, recipient_id, subject, settings.USER_MESSAGE_PREFIX_TEMPLATE, settings.REPLY_TO_SENDER_NOTE)
def send_contact_message(self, pkg_id):
"""
Sends a message by email from the logged in user to the appropriate
contact address of the dataset.
:param pkg_id: package id
:type pkg_id: string
"""
package = Package.get(pkg_id)
if not package:
ckan.lib.base.abort(404, _("Dataset not found"))
package_title = package.title if package.title else package.name
subject = u"Message regarding dataset / Viesti koskien tietoaineistoa %s" % package_title
recipient_id = request.params.get('recipient', '')
return self._prepare_and_send(pkg_id, recipient_id, subject, settings.USER_MESSAGE_PREFIX_TEMPLATE, settings.REPLY_TO_SENDER_NOTE)
def import_collection_to_package(params, id):
"""
Import a collection to dataset. Does not import whole file data but
rather the metadata.
"""
from irods import irodsCollection
path = params['path']
pkg = Package.get(id)
conn = get_connection_from_params(params)
if (conn):
coll = irodsCollection(conn, path)
from irods import iRodsOpen
rev = model.repo.new_revision()
i = 0
for obj in coll.getObjects():
extras = {}
fname, _ = obj
fpath = "%s/%s" % (coll.getCollName(), fname)
f = iRodsOpen(conn, fpath, 'r')
if f:
i += 1
res = Resource.by_name(fname)
if not res:
res = Resource(url = '', name=fname, extras=extras, \
resource_type='file')
for met in f.getUserMetadata():
key, value, _ = met
extras[key] = value
res.extras = extras
resgrp = pkg.resource_groups[0]
resgrp.resources.append(res)
Session.add(res)
Session.add(resgrp)
rev.message = "Update from iRODS, matched file %s" % fname
for met in coll.getUserMetadata():
key, value, _ = met
pkg.extras[key] = value
Session.add(pkg)
model.repo.commit()
conn.disconnect()
h.flash_success("iRODS import to dataset OK! Imported %s resources." %
i)
else:
h.flash_error("Could not connect to iRODS!")
h.redirect_to(controller='package', action='read', id=id)
def import_collection_to_package(params, id):
"""
Import a collection to dataset. Does not import whole file data but
rather the metadata.
"""
from irods import irodsCollection
path = params['path']
pkg = Package.get(id)
conn = get_connection_from_params(params)
if (conn):
coll = irodsCollection(conn, path)
from irods import iRodsOpen
rev = model.repo.new_revision()
i = 0
for obj in coll.getObjects():
extras = {}
fname, _ = obj
fpath = "%s/%s" % (coll.getCollName(), fname)
f = iRodsOpen(conn, fpath, 'r')
if f:
i += 1
res = Resource.by_name(fname)
if not res:
res = Resource(url = '', name=fname, extras=extras, \
resource_type='file')
for met in f.getUserMetadata():
key, value, _ = met
extras[key] = value
res.extras = extras
resgrp = pkg.resource_groups[0]
resgrp.resources.append(res)
Session.add(res)
Session.add(resgrp)
rev.message = "Update from iRODS, matched file %s" % fname
for met in coll.getUserMetadata():
key, value, _ = met
pkg.extras[key] = value
Session.add(pkg)
model.repo.commit()
conn.disconnect()
h.flash_success("iRODS import to dataset OK! Imported %s resources." % i)
else:
h.flash_error("Could not connect to iRODS!")
h.redirect_to(controller='package', action='read', id=id)
def test_edit(self):
name = "annakarenina"
offset = url_for(controller="package", action="edit", id=name)
res = self.app.get(offset, extra_environ=self.extra_environ)
assert "Edit - Datasets" in res
fv = res.forms["dataset-edit"]
prefix = ""
fv[prefix + "extras__1__key"] = u"spatial"
fv[prefix + "extras__1__value"] = self.geojson_examples["point"]
res = fv.submit("save", extra_environ=self.extra_environ)
assert not "Error" in res, res
package = Package.get(name)
# Check that a PackageExtent object has been created
package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id == package.id).first()
geojson = json.loads(self.geojson_examples["point"])
assert package_extent
assert package_extent.package_id == package.id
assert Session.scalar(package_extent.the_geom.x) == geojson["coordinates"][0]
assert Session.scalar(package_extent.the_geom.y) == geojson["coordinates"][1]
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
# Update the spatial extra
offset = url_for(controller="package", action="edit", id=name)
res = self.app.get(offset, extra_environ=self.extra_environ)
assert "Edit - Datasets" in res
fv = res.forms["dataset-edit"]
prefix = ""
fv[prefix + "extras__1__value"] = self.geojson_examples["polygon"]
res = fv.submit("save", extra_environ=self.extra_environ)
assert not "Error" in res, res
# Check that the PackageExtent object has been updated
package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id == package.id).first()
assert package_extent
assert package_extent.package_id == package.id
assert Session.scalar(package_extent.the_geom.geometry_type) == "ST_Polygon"
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def update_ancestral_license(context, pkg_dict, license_id):
session = context['session']
package = Package.get(pkg_dict['id'])
log.debug(u'Updating license for package {}: {}'.format(pkg_dict['name'], PKG_LICENSE_ID))
pkg_license = package.get_license_register()[PKG_LICENSE_ID]
package.set_license(pkg_license)
session.merge(package)
if not license_id:
return
log.debug(u'Updating ancestral license for package {}: {}'.format(pkg_dict['name'], license_id))
for resource in package.resources:
res_license = session.query(CeonResourceLicense).filter(CeonResourceLicense.resource_id == resource.id).first()
if res_license:
#session.delete(res_license)
res_license.license_id = license_id
session.merge(res_license)
else:
new_res_license = CeonResourceLicense(resource_id = resource.id, license_id = license_id)
session.add(new_res_license)
def test_new_bad_geojson(self):
name = 'test-spatial-dataset-3'
offset = url_for(controller='package', action='new')
res = self.app.get(offset, extra_environ=self.extra_environ)
assert 'Add - Datasets' in res
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'name'] = name
fv[prefix + 'extras__0__key'] = u'spatial'
fv[prefix + 'extras__0__value'] = u'{"Type":"Bad_GeoJSON","a":2}'
res = fv.submit('save', extra_environ=self.extra_environ)
assert 'Error' in res, res
assert 'Spatial' in res
assert 'Error creating geometry' in res
# Check that package was not created
assert not Package.get(name)
def test_new_bad_geojson(self):
name = "test-spatial-dataset-3"
offset = url_for(controller="package", action="new")
res = self.app.get(offset, extra_environ=self.extra_environ)
assert "Add - Datasets" in res
fv = res.forms["dataset-edit"]
prefix = ""
fv[prefix + "name"] = name
fv[prefix + "extras__0__key"] = u"spatial"
fv[prefix + "extras__0__value"] = u'{"Type":"Bad_GeoJSON","a":2}'
res = fv.submit("save", extra_environ=self.extra_environ)
assert "Error" in res, res
assert "Spatial" in res
assert "Error creating geometry" in res
# Check that package was not created
assert not Package.get(name)
def wms_preview(self, id):
'''
:param id:
'''
# check if package exists
toolkit.c.pkg = Package.get(id)
if toolkit.c.pkg is None:
toolkit.abort(404, u'Dataset not found')
for res in toolkit.c.pkg.resources:
if res.format.lower() == u'wms':
toolkit.c.wms_url = res.url \
if u'?' not in res.url else res.url.split(u'?')[0]
break
if not toolkit.c.wms_url:
toolkit.abort(400, u'This dataset does not have a WMS resource')
return toolkit.render(u'ckanext/spatial/wms_preview.html')
def getRecord(self, metadataPrefix, identifier):
'''Simple getRecord for a dataset.
'''
package = Package.get(identifier)
if not package:
raise IdDoesNotExistError("No dataset with id %s" % identifier)
set_spec = []
if package.owner_org:
group = Group.get(package.owner_org)
if group and group.name:
set_spec.append(group.name)
if 'openaire_data' in package.as_dict().get('tags'):
set_spec.append('openaire_data')
if not set_spec:
set_spec = [package.name]
if metadataPrefix == 'rdf':
return self._record_for_dataset_dcat(package, set_spec)
if metadataPrefix == 'oai_openaire':
return self._record_for_dataset_datacite(package, set_spec)
return self._record_for_dataset(package, set_spec)
def test_link_and_map_shown(self):
name = u'annakarenina'
wms_url = 'http://maps.bgs.ac.uk/ArcGIS/services/BGS_Detailed_Geology/MapServer/WMSServer?'
rev = model.repo.new_revision()
pkg = Package.get(name)
pr = Resource(url=wms_url, format='WMS')
pkg.resources.append(pr)
pkg.save()
model.repo.commit_and_remove()
# Load the dataset page and check if link appears
offset = url_for(controller='package', action='read', id=name)
res = self.app.get(offset)
assert 'View available WMS layers' in res, res
# Load the dataset map preview page and check if libraries are loaded
offset = '/dataset/%s/map' % name
res = self.app.get(offset)
assert '<script type="text/javascript" src="/ckanext/spatial/js/wms_preview.js"></script>' in res, res
assert 'CKAN.WMSPreview.setup("%s");' % wms_url.split('?')[0] in res
def render_contact_form(self, pkg_id):
"""
Render the contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if not c.package:
abort(404, _(u"Dataset not found"))
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = []
for contact in contacts:
if 'name' in contact:
text_val = contact['name']
else:
at_idx = contact['email'].find('@')
text_val = contact['email'][0:at_idx]
text_val = text_val.replace(".", " ").title()
print(text_val)
c.recipient_options.append({'text': text_val, 'value': contact['id']})
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/contact_form.html')
def send_package_deprecation_emails(packages):
grouped_by_maintainer = {}
for package in packages:
fullPackage = Package.get(package)
maintainer_email = fullPackage.maintainer_email
packageInfoForEmail = {
"title": fullPackage.title,
"id": fullPackage.id,
"valid_till": fullPackage.extras.get("valid_till"),
}
if maintainer_email not in grouped_by_maintainer:
grouped_by_maintainer[maintainer_email] = {"maintainer": fullPackage.maintainer, "packages": [packageInfoForEmail]}
else:
grouped_by_maintainer[maintainer_email]["packages"].append(packageInfoForEmail)
for maintainer_email, details in grouped_by_maintainer.iteritems():
send_deprecation_email_user(
maintainer_email,
details["packages"],
details["maintainer"],
details["packages"][0]["valid_till"]
)
def test_clean_tags(self):
# Create source
source_fixture = {
'title': 'Test Source',
'name': 'test-source',
'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml',
'source_type': u'gemini-single',
'owner_org': 'test-org',
'metadata_created': datetime.now().strftime('%YYYY-%MM-%DD %HH:%MM:%s'),
'metadata_modified': datetime.now().strftime('%YYYY-%MM-%DD %HH:%MM:%s'),
}
user = User.get('dummy')
if not user:
user = call_action('user_create',
name='dummy',
password='******',
email='*****@*****.**')
user_name = user['name']
else:
user_name = user.name
org = Group.by_name('test-org')
if org is None:
org = call_action('organization_create',
context={'user': user_name},
name='test-org')
existing_g = Group.by_name('existing-group')
if existing_g is None:
existing_g = call_action('group_create',
context={'user': user_name},
name='existing-group')
context = {'user': '******'}
package_schema = default_update_package_schema()
context['schema'] = package_schema
package_dict = {'frequency': 'manual',
'publisher_name': 'dummy',
'extras': [{'key':'theme', 'value':['non-mappable', 'thememap1']}],
'groups': [],
'title': 'fakename',
'holder_name': 'dummy',
'holder_identifier': 'dummy',
'name': 'fakename',
'notes': 'dummy',
'owner_org': 'test-org',
'modified': datetime.now(),
'publisher_identifier': 'dummy',
'metadata_created' : datetime.now(),
'metadata_modified' : datetime.now(),
'guid': unicode(uuid4()),
'identifier': 'dummy'}
package_data = call_action('package_create', context=context, **package_dict)
package = Package.get('fakename')
source, job = self._create_source_and_job(source_fixture)
job.package = package
job.guid = uuid4()
harvester = SpatialHarvester()
with open(os.path.join('..', 'data', 'dataset.json')) as f:
dataset = json.load(f)
# long tags are invalid in all cases
TAG_LONG_INVALID = 'abcdefghij' * 20
# if clean_tags is not set to true, tags will be truncated to 50 chars
TAG_LONG_VALID = TAG_LONG_INVALID[:50]
# default truncate to 100
TAG_LONG_VALID_LONG = TAG_LONG_INVALID[:100]
assert len(TAG_LONG_VALID) == 50
assert TAG_LONG_VALID[-1] == 'j'
TAG_CHARS_INVALID = '[email protected]!'
TAG_CHARS_VALID = 'pretty-invlidtag'
dataset['tags'].append(TAG_LONG_INVALID)
dataset['tags'].append(TAG_CHARS_INVALID)
harvester.source_config = {'clean_tags': False}
out = harvester.get_package_dict(dataset, job)
tags = out['tags']
# no clean tags, so invalid chars are in
# but tags are truncated to 50 chars
assert {'name': TAG_CHARS_VALID} not in tags
assert {'name': TAG_CHARS_INVALID} in tags
assert {'name': TAG_LONG_VALID_LONG} in tags
assert {'name': TAG_LONG_INVALID} not in tags
harvester.source_config = {'clean_tags': True}
out = harvester.get_package_dict(dataset, job)
tags = out['tags']
assert {'name': TAG_CHARS_VALID} in tags
assert {'name': TAG_LONG_VALID_LONG} in tags
def _prepare_and_send(self, pkg_id, recipient_id, subject, prefix_template, suffix):
"""
Sends a message by email from the logged in user to the appropriate
contact address of the given dataset.
The prefix template should have formatting placeholders for the following arguments:
{sender_name}, {sender_email}, {package_title}, {package_id}
:param pkg_id: package id
:type pkg_id: string
:param recipient_id: id of the recipient, as returned by utils.get_package_contacts
:type recipient_id: string
:param subject: the subject of the message
:type subject: string
:param prefix_template: the template for the prefix to be automatically included before the user message
:type prefix_template: unicode
:param suffix: an additional note to be automatically included after the user message
:type suffix: unicode
"""
url = h.url_for(controller='package', action='read', id=pkg_id)
if asbool(config.get('kata.contact_captcha')):
try:
captcha.check_recaptcha(request)
except captcha.CaptchaError:
h.flash_error(_(u'Bad Captcha. Please try again.'))
redirect(url)
if not request.params.get('accept_logging'):
h.flash_error(_(u"Message not sent as logging wasn't permitted"))
return redirect(url)
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. Please try again later or contact customer "
u"service."))
return redirect(url)
package = Package.get(pkg_id)
package_title = package.title if package.title else package.name
sender_addr = request.params.get('from_address')
sender_name = request.params.get('from_name')
recipient = self._get_contact_email(pkg_id, recipient_id)
if not recipient:
abort(404, _('Recipient not found'))
user_msg = request.params.get('msg', '')
if request.params.get('hp'):
h.flash_error(_(u"Message not sent. Couldn't confirm human interaction (spam bot control)"))
return redirect(url)
if sender_addr and sender_name and \
isinstance(sender_name, basestring) and len(sender_name) >= 3:
if user_msg:
prefix = prefix_template.format(
sender_name=sender_name,
sender_email=sender_addr,
package_title=package_title,
package_id=pkg_id
# above line was: metadata_pid=utils.get_primary_data_pid_from_package(package)
)
log.info(u"Message {m} sent from {a} ({b}) to {r} about {c}, IP: {d}"
.format(m=user_msg, a=sender_name, b=sender_addr, r=recipient, c=pkg_id,
d=request.environ.get('REMOTE_ADDR', 'No remote address')))
full_msg = u"{a}{b}{c}".format(a=prefix, b=user_msg, c=suffix)
self._send_message(subject, full_msg, recipient.get('email'), recipient.get('name'))
h.flash_success(_(u"Message sent"))
else:
h.flash_error(_(u"No message"))
else:
h.flash_error(_(u"Message not sent. Please, provide reply address and name. Name must contain \
at least three letters."))
return redirect(url)
def get_package_name_by_id(package_id):
from ckan.model import Package
return Package.get(package_id).title
def reimport_batch(self, package_ids, context):
'''Batch-reimport all packages in `package_ids` from their original
harvest source.'''
ckan_fb_mapping = {}
# first, do checks that can be done without connection to FIS-Broker
for package_id in package_ids:
package = Package.get(package_id)
if not package:
raise PackageIdDoesNotExistError(package_id)
if not dataset_was_harvested(package):
raise PackageNotHarvestedError(package_id)
harvester = harvester_for_package(package)
harvester_url = harvester.url
harvester_type = harvester.type
if not harvester_type == HARVESTER_ID:
raise PackageNotHarvestedInFisbrokerError(package_id)
fb_guid = fisbroker_guid(package)
if not fb_guid:
raise NoFisbrokerIdError(package_id)
ckan_fb_mapping[package.id] = fb_guid
# get the harvest source for FIS-Broker datasets
fb_source = get_fisbroker_source()
if not fb_source:
raise NoFBHarvesterDefined()
source_id = fb_source.get('id', None)
# Create and start a new harvest job
job_dict = toolkit.get_action('harvest_job_create')(context, {'source_id': source_id})
harvest_job = HarvestJob.get(job_dict['id'])
harvest_job.gather_started = datetime.datetime.utcnow()
assert harvest_job
# instatiate the CSW connector (on the reasonable assumption that harvester_url is
# the same for all package_ids)
package_id = None
reimported_packages = []
try:
csw = CatalogueServiceWeb(harvester_url)
for package_id, fb_guid in ckan_fb_mapping.items():
# query connector to get resource document
csw.getrecordbyid([fb_guid], outputschema=namespaces['gmd'])
# show resource document
record = csw.records.get(fb_guid, None)
if record:
obj = HarvestObject(guid=fb_guid,
job=harvest_job,
content=record.xml,
package_id=package_id,
extras=[
HarvestObjectExtra(key='status',value='change'),
HarvestObjectExtra(key='type',value='reimport'),
])
obj.save()
assert obj, obj.content
harvester = FisbrokerPlugin()
harvester.force_import = True
harvester.import_stage(obj)
rejection_reason = self._dataset_rejected(obj)
if rejection_reason:
raise FBImportError(package_id, rejection_reason)
harvester.force_import = False
Session.refresh(obj)
reimported_packages.append(record)
else:
raise NotFoundInFisbrokerError(package_id, fb_guid)
except RequestException as error:
raise NoConnectionError(package_id, harvester_url, str(error.__class__.__name__))
# successfully finish harvest job
harvest_job.status = u'Finished'
harvest_job.finished = datetime.datetime.utcnow()
harvest_job.save()
return reimported_packages
