def test_zfaulty_xml_unknown_errors(self):
harv, job = self._create_harvester()
res = "http://www.fsd.uta.fi/fi/aineistot/luettelo/FSD0115/FSD0115.xml"
urllib2.urlopen = mock.Mock(return_value=StringIO(res))
gathered = harv.gather_stage(job)
urllib2.urlopen = mock.Mock(return_value=open("FSD2355.xml"))
harvest_obj = HarvestObject.get(gathered[0])
self.assert_(harv.fetch_stage(harvest_obj))
self.assert_(harv.import_stage(harvest_obj))
print Package.text_search(\
Session.query(Package),
'Kansalaiskeskustelu ydinvoimasta 2006').all()
self.assert_(len(Package.text_search(\
Session.query(Package),
'Kansalaiskeskustelu ydinvoimasta 2006').all()) >= 1)
res = "http://www.fsd.uta.fi/fi/aineistot/luettelo/FSD0115/FSD0115.xml"
urllib2.urlopen = mock.Mock(return_value=StringIO(res))
gathered = harv.gather_stage(job)
urllib2.urlopen = mock.Mock(return_value=open("FSD2362.xml"))
harvest_obj = HarvestObject.get(gathered[0])
self.assert_(harv.fetch_stage(harvest_obj))
self.assert_(harv.import_stage(harvest_obj))
self.assert_(len(Package.text_search(\
Session.query(Package),
'Energia-asennetutkimus 2004').all()) >= 1)
def test_zaincremental_harvester(self):
client = CKANServer()
metadata_registry = metadata.MetadataRegistry()
metadata_registry.registerReader('oai_dc', oai_dc_reader)
metadata_registry.registerWriter('oai_dc', oai_dc_writer)
serv = BatchingServer(client, metadata_registry=metadata_registry)
oaipmh.client.Client = mock.Mock(return_value=ServerClient(serv, metadata_registry))
harv = OAIPMHHarvester()
harvest_job = HarvestJob()
harvest_job.source = HarvestSource()
harvest_job.source.title = "Test"
harvest_job.source.url = "http://helda.helsinki.fi/oai/request"
harvest_job.gather_started = ((datetime.now() + timedelta(days=1)))
harvest_job.source.config = '{"incremental":"True"}'
harvest_job.source.type = "OAI-PMH"
Session.add(harvest_job)
rev = model.repo.new_revision()
rev.timestamp = ((datetime.now() + timedelta(days=2)))
pkg = Package(name='footest', revision=rev)
Session.add(pkg)
pkg.save()
roger = Group.get('roger')
roger.add_package_by_name('footest')
Session.add(roger)
roger.save()
gathered = harv.gather_stage(harvest_job)
harvest_object = HarvestObject.get(gathered[0])
harv.fetch_stage(harvest_object)
harvobj = json.loads(harvest_object.content)
self.assert_(harvobj['records'])
def accessreq_show(context, data_dict):
ret = {}
ret['title'] = _('Request edit rights')
smtp = config.get('smtp.server', '')
if not len(smtp):
ret['ret'] = 'Yes'
return ret
pkg = Package.get(data_dict['id'])
selrole = False
ret['id'] = pkg.id
for role in pkg.roles:
if role.role == "admin":
selrole = True
ret['no_owner'] = not selrole
if c.userobj:
if 'id' in data_dict:
req = KataAccessRequest.is_requesting(c.userobj.id, data_dict['id'])
if req:
ret['ret'] = 'Yes'
return ret
else:
ret['ret'] = 'No'
return ret
else:
ret['ret'] = 'No'
return ret
ret['ret'] = 'No'
return ret
def send(self, pkg_id):
package = Package.get(pkg_id)
url = h.url_for(controller='package',
action="read",
id=package.id)
if c.user:
userid = None
for role in package.roles:
if role.role == "admin":
userid = role.user_id
break
if userid:
owner = User.get(userid)
msg = request.params.get('msg', '')
if msg:
send_contact_email(owner, c.userobj, package,\
msg)
else:
h.flash_error(_("No message"))
return redirect(url)
else:
h.flash_error(_("No owner found"))
return redirect(url)
h.flash_notice(_("Message sent"))
else:
h.flash_error(_("Please login"))
return redirect(url)
def is_owner(context, data_dict):
'''
This is used in "request edit rights" feature.
Checks if the user is creator, admin or editor of the
package in question
:param context: context
:param data_dict: package data
:type data_dict: dictionary
:rtype: dictionary
'''
# Package creator is always owner regardless of organizations
pkg = context.get('package', None) or Package.get(data_dict['id'])
user = context.get('user', False)
# If user id can't be resolved, user can't be owner
try:
user_id = convert_user_name_or_id_to_id(user, context)
except:
return {'success': False}
if pkg.creator_user_id == user_id:
return {'success': True}
# Check if the user has editor rights to this dataset through an organization
package = get_package_object(context, data_dict)
if authz.has_user_permission_for_group_or_org(package.owner_org, user,
'delete_dataset'):
return {'success': True}
return {'success': False}
def is_owner(context, data_dict):
'''
This is used in "request edit rights" feature.
Checks if the user is creator, admin or editor of the
package in question
:param context: context
:param data_dict: package data
:type data_dict: dictionary
:rtype: dictionary
'''
# Package creator is always owner regardless of organizations
pkg = context.get('package', None) or Package.get(data_dict['id'])
user = context.get('user', False)
# If user id can't be resolved, user can't be owner
try:
user_id = convert_user_name_or_id_to_id(user, context)
except:
return {'success': False}
if pkg.creator_user_id == user_id:
return {'success': True}
# Check if the user has editor rights to this dataset through an organization
package = get_package_object(context, data_dict)
if authz.has_user_permission_for_group_or_org(package.owner_org, user, 'delete_dataset'):
return {'success': True}
return {'success': False}
def alert(self, dataset_name):
'''
Put an alert aka. a signalement on a dataset.
'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized('Alert creation requires an user')
dataset = Package.by_name(dataset_name)
alert_type = toolkit.request.POST['type']
comment = toolkit.request.POST['comment']
alert = DatasetAlert(dataset, user, alert_type, comment)
DB.add(alert)
DB.commit()
alert.notify_admins()
return self.json_response({
'id': alert.id,
'user_id': alert.user_id,
'dataset_id': alert.dataset_id,
'type': alert.type,
'comment': alert.comment,
'created': alert.created
})
def get_licenses():
licenses = [('', '')]
register = Package.get_license_register()
for l in register.values():
if l.status == 'active':
licenses.append((l.title, l.id))
return licenses
def test_update_extent(self):
package = Package.get('annakarenina')
geojson = json.loads(self.geojson_examples['point'])
shape = asShape(geojson)
package_extent = PackageExtent(package_id=package.id,
the_geom=WKTSpatialElement(
shape.wkt, self.db_srid))
package_extent.save()
assert Session.scalar(
package_extent.the_geom.geometry_type) == 'ST_Point'
# Update the geometry (Point -> Polygon)
geojson = json.loads(self.geojson_examples['polygon'])
shape = asShape(geojson)
package_extent.the_geom = WKTSpatialElement(shape.wkt, self.db_srid)
package_extent.save()
assert package_extent.package_id == package.id
assert Session.scalar(
package_extent.the_geom.geometry_type) == 'ST_Polygon'
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def render_request_form(self, pkg_id):
"""
Render the access request contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
if not c.package:
abort(404, _(u"Dataset not found"))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts]
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/dataset_request_form.html')
def test_new(self):
name = 'test-spatial-dataset-1'
offset = url_for(controller='package', action='new')
res = self.app.get(offset, extra_environ=self.extra_environ)
assert 'Add - Datasets' in res
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'name'] = name
fv[prefix + 'extras__0__key'] = u'spatial'
fv[prefix + 'extras__0__value'] = self.geojson_examples['point']
res = fv.submit('save', extra_environ=self.extra_environ)
assert not 'Error' in res, res
package = Package.get(name)
# Check that a PackageExtent object has been created
package_extent = Session.query(PackageExtent).filter(
PackageExtent.package_id == package.id).first()
geojson = json.loads(self.geojson_examples['point'])
assert package_extent
assert package_extent.package_id == package.id
assert Session.scalar(
package_extent.the_geom.x) == geojson['coordinates'][0]
assert Session.scalar(
package_extent.the_geom.y) == geojson['coordinates'][1]
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def render_request_form(self, pkg_id):
"""
Render the access request contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if not c.package:
abort(404, _(u"Dataset not found"))
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts]
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/dataset_request_form.html')
def alert(self, dataset_name):
'''
Put an alert aka. a signalement on a dataset.
'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized('Alert creation requires an user')
dataset = Package.by_name(dataset_name)
alert_type = toolkit.request.POST['type']
comment = toolkit.request.POST['comment']
alert = DatasetAlert(dataset, user, alert_type, comment)
DB.add(alert)
DB.commit()
alert.notify_admins()
return self.json_response({
'id': alert.id,
'user_id': alert.user_id,
'dataset_id': alert.dataset_id,
'type': alert.type,
'comment': alert.comment,
'created': alert.created
})
def is_owner(context, data_dict):
'''
This is used in "request edit rights" feature.
Checks if the user is admin or editor of the
package in question
:param context: context
:param data_dict: package data
:type data_dict: dictionary
:rtype: dictionary
'''
pkg = context.get('package', None)
roles = pkg.roles if pkg else Package.get(data_dict['id']).roles
user = context.get('user', False)
if user:
for role in roles:
ruser = User.get(role.user.id)
if user == ruser.name and role.role in ('admin', 'editor'):
return {'success': True}
# Check if the user has editor rights to this dataset through an organization
package = get_package_object(context, data_dict)
if new_authz.has_user_permission_for_group_or_org(package.owner_org, user, 'delete_dataset'):
return {'success': True}
return {'success': False}
def send_email(req):
requestee = User.get(req.user_id)
pkg = Package.get(req.pkg_id)
selrole = False
for role in pkg.roles:
if role.role == "admin":
selrole = role
if not selrole:
return
admin = User.get(selrole.user_id)
msg = _("""%s (%s) is requesting editor access to a dataset you have created
%s.
Please click this link if you want to give this user write access:
%s%s""")
controller = 'ckanext.kata.controllers:AccessRequestController'
body = msg % (requestee.name, requestee.email, pkg.title if pkg.title else pkg.name,
config.get('ckan.site_url', ''),
h.url_for(controller=controller,
action="unlock_access",
id=req.id))
email_dict = {}
email_dict["subject"] = _("Access request for dataset %s" % pkg.title if pkg.title else pkg.name)
email_dict["body"] = body
send_notification(admin.as_dict(), email_dict)
def get_licenses():
licenses = [('', '')]
register = Package.get_license_register()
for l in register.values():
if l.status == 'active':
licenses.append((l.title, l.id))
return licenses
def create_access_request(context, data_dict):
if not context.get('ignore_auth'):
check_access('create_access_request', context, data_dict)
pkg_id_or_name, reason, user = get_or_bust(data_dict,
['id', 'reason', 'user'])
user_org = data_dict.get('user_org')
# check if the package with such id exists to use it's ID
pkg = Package.get(pkg_id_or_name)
if not pkg:
raise ObjectNotFound()
req = AccessRequest.create_or_get_request(user_id=user,
package_id=pkg.id,
reason=reason,
org_id=pkg.owner_org)
# send email notifications to org custodians
if config.get('spc.access_request.send_admin_notification'):
notify_org_members(pkg.owner_org, {
'pkg': pkg,
'user': user,
'reason': reason,
'user_org': user_org
})
return req.as_dict()
def test_new(self):
name = "test-spatial-dataset-1"
offset = url_for(controller="package", action="new")
res = self.app.get(offset, extra_environ=self.extra_environ)
assert "Add - Datasets" in res
fv = res.forms["dataset-edit"]
prefix = ""
fv[prefix + "name"] = name
fv[prefix + "extras__0__key"] = u"spatial"
fv[prefix + "extras__0__value"] = self.geojson_examples["point"]
res = fv.submit("save", extra_environ=self.extra_environ)
assert not "Error" in res, res
package = Package.get(name)
# Check that a PackageExtent object has been created
package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id == package.id).first()
geojson = json.loads(self.geojson_examples["point"])
assert package_extent
assert package_extent.package_id == package.id
assert Session.scalar(package_extent.the_geom.x) == geojson["coordinates"][0]
assert Session.scalar(package_extent.the_geom.y) == geojson["coordinates"][1]
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def getRecord(self, metadataPrefix, identifier):
'''Simple getRecord for a dataset.
'''
package = Package.get(identifier)
if not package:
raise IdDoesNotExistError("No dataset with id %s" % identifier)
return self._record_for_dataset(package)
def get_ancestral_license(session, package_id):
license_id = None
if package_id:
package = Package.get(package_id)
extras = package.extras
if 'ancestral_license' in extras:
license_id = extras['ancestral_license']
return license_id
def get_resources_licenses(session, pkg_dict):
license_ids = []
package = Package.get(pkg_dict['id'])
for resource in package.resources:
res_license = session.query(CeonResourceLicense).filter(CeonResourceLicense.resource_id == resource.id).first()
if res_license:
license_ids.append(res_license.license_id)
return license_ids
def _create_pkg(self):
model.repo.new_revision()
pkg = Package.get("annakarenina")
pkg.name = "fookarenina"
pkg.add_resource("www.google.com", description="foo", name="foo")
Session.add(pkg)
Session.commit()
return pkg
def _create_pkg(self):
model.repo.new_revision()
pkg = Package.get('annakarenina')
pkg.name = "fookarenina"
pkg.add_resource('www.google.com', description='foo', name="foo")
Session.add(pkg)
Session.commit()
return pkg
def validate_config(self,config):
super(GuiaHarvesterPlugin, self).validate_config(config)
config_obj = json.loads(config)
if 'license_id' in config:
license_id = config_obj.get('license_id', None)
if license_id not in Package.get_license_register().keys():
raise ValueError('license_id not found')
return config
def get_resources_licenses(session, pkg_dict):
license_ids = []
package = Package.get(pkg_dict['id'])
for resource in package.resources:
res_license = session.query(CeonResourceLicense).filter(CeonResourceLicense.resource_id == resource.id).first()
if res_license:
license_ids.append(res_license.license_id)
return license_ids
def get_ancestral_license(session, package_id):
license_id = None
if package_id:
package = Package.get(package_id)
extras = package.extras
if 'ancestral_license' in extras:
license_id = extras['ancestral_license']
return license_id
def validate_config(self, config):
super(GuiaHarvesterPlugin, self).validate_config(config)
config_obj = json.loads(config)
if 'license_id' in config:
license_id = config_obj.get('license_id', None)
if license_id not in Package.get_license_register().keys():
raise ValueError('license_id not found')
return config
def package_show(context, data_dict):
'''
Return the metadata of a dataset (package) and its resources.
Called before showing the dataset in some interface (browser, API),
or when adding package to Solr index (no validation / conversions then).
:param id: the id or name of the dataset
:type id: string
:rtype: dictionary
'''
if data_dict.get('type') == 'harvest':
context['schema'] = Schemas.harvest_source_show_package_schema()
if not data_dict.get('id') and not data_dict.get('name'):
# Get package by data PIDs
data_dict['id'] = utils.get_package_id_by_data_pids(data_dict)
pkg_dict1 = ckan.logic.action.get.package_show(context, data_dict)
pkg_dict1 = utils.resource_to_dataset(pkg_dict1)
# Remove empty agents that come from padding the agent list in converters
if 'agent' in pkg_dict1:
agents = filter(None, pkg_dict1.get('agent', []))
pkg_dict1['agent'] = agents or []
# Normally logic function should not catch the raised errors
# but here it is needed so action package_show won't catch it instead
# Hiding information from API calls
try:
check_access('package_update', context)
except NotAuthorized:
pkg_dict1 = utils.hide_sensitive_fields(pkg_dict1)
pkg = Package.get(pkg_dict1['id'])
if 'erelated' in pkg.extras:
erelated = pkg.extras['erelated']
if len(erelated):
for value in erelated.split(';'):
if len(Session.query(Related).filter(Related.title == value).all()) == 0:
data_dict = {'title': value,
'type': _("Paper"),
'dataset_id': pkg.id}
related_create(context, data_dict)
# Update package.title to match package.extras.title_0
extras_title = pkg.extras.get(u'title_0')
if extras_title and extras_title != pkg.title:
repo.new_revision()
pkg.title = pkg.extras[u'title_0']
pkg.save()
rebuild(pkg.id) # Rebuild solr-index for this dataset
return pkg_dict1
def getMetadataDDI(self,id):
toolkit.response.content_type = 'application/xml'
c.pkg = Package.get(id)
if c.pkg is None:
abort(404, 'Dataset not found')
pkginfo = toolkit.get_action('package_show')({}, {'id': id})
try:
return createDDIXML(pkginfo,toolkit.request.url)
except Exception,e:
print str(e)
return "[]"
def getMetadataDDI(self, id):
toolkit.response.content_type = 'application/xml'
c.pkg = Package.get(id)
if c.pkg is None:
abort(404, 'Dataset not found')
pkginfo = toolkit.get_action('package_show')({}, {'id': id})
try:
return createDDIXML(pkginfo, toolkit.request.url)
except Exception, e:
print str(e)
return "[]"
def test_menuitem_for_resource(self):
"""
See if there is a 'IRODS' menuitem in the main menu for IRODS import,
specifically for when a resource is seen.
"""
name = 'annakarenina'
pkg = Package.by_name(name)
res = pkg.resource_groups[0].resources[0]
offset = url_for(controller='package', action='resource_read', \
id = name, resource_id = res.id)
res = self.app.get(offset)
assert 'IRODS' in res
def getRecord(self, metadataPrefix, identifier):
'''Simple getRecord for a dataset.
'''
package = Package.get(identifier)
if not package:
raise IdDoesNotExistError("No dataset with id %s" % identifier)
spec = package.name
if package.owner_org:
group = Group.get(package.owner_org)
if group and group.name:
spec = group.name
return self._record_for_dataset(package, spec)
def show(self,id):
package = Package.get(id)
if package:
graph = pkg_produce(package)
doc = graph.serialize(format='pretty-xml')
response.content_type = 'application/rdf+xml'
response.headers['Content-Length'] = len(doc)
return doc
else:
abort(404)
def test_menuitem_for_resource(self):
"""
See if there is a 'IRODS' menuitem in the main menu for IRODS import,
specifically for when a resource is seen.
"""
name = 'annakarenina'
pkg = Package.by_name(name)
res = pkg.resources[0]
offset = url_for(controller='package', action='resource_read', \
id = name, resource_id = res.id)
res = self.app.get(offset)
assert 'IRODS' in res
def is_owner(context, data_dict):
pkg = context.get('package', None)
roles = pkg.roles if pkg else Package.get(data_dict['id']).roles
user = context.get('user', False)
if user:
for role in roles:
ruser = User.get(role.user_id)
if user == ruser.name and role.role in ('admin', 'editor'):
return {'success': True}
else:
return {'success': False}
return {'success': False}
def googleanalytics_dataset_visits(context=None, data_dict=None):
"""
Fetch the amount of times a dataset hs been visited
:param id: Dataset id
:type id: string
:returns: The number of times the dataset has been viewed
:rtype: integer
"""
package = Package.get(data_dict['id'])
return PackageStats.get_all_visits(package.id)
def create_request(self, pkg_id):
pkg = Package.get(pkg_id)
user = c.userobj if c.userobj else None
if user:
req = KataAccessRequest(user.id, pkg.id)
req.save()
url = h.url_for(controller='package', action='read', id=pkg_id)
h.flash_success(_("You now requested editor rights to package %s" % pkg.name))
redirect(url)
else:
url = h.url_for(controller='package', action='read', id=pkg_id)
h.flash_error(_("Please log in!"))
redirect(url)
def request_access(self, pkg_id):
# If the user is logged in show the access request button
pkg = Package.get(pkg_id)
if c.user and not user_has_role(c.userobj, 'admin', pkg) and\
not user_has_role(c.userobj, 'editor', pkg) and\
not c.userobj.sysadmin and\
not config.get('smtp_server', False):
following = KataAccessRequest.is_requesting(c.userobj.id, pkg_id)
if not following:
return snippet('snippets/access_button.html',
obj_id=pkg_id,
following=following)
return ''
def package_show(context, data_dict):
'''
Return the metadata of a dataset (package) and its resources.
Called before showing the dataset in some interface (browser, API),
or when adding package to Solr index (no validation / conversions then).
:param id: the id or name of the dataset
:type id: string
:rtype: dictionary
'''
if data_dict.get('type') == 'harvest':
context['schema'] = Schemas.harvest_source_show_package_schema()
context['use_cache'] = False # Disable package retrieval directly from Solr as contact.email is not there.
if not data_dict.get('id') and not data_dict.get('name'):
# Get package by data PIDs
data_dict['id'] = utils.get_package_id_by_primary_pid(data_dict)
pkg_dict1 = ckan.logic.action.get.package_show(context, data_dict)
pkg_dict1 = utils.resource_to_dataset(pkg_dict1)
# Remove empty agents that come from padding the agent list in converters
if 'agent' in pkg_dict1:
agents = filter(None, pkg_dict1.get('agent', []))
pkg_dict1['agent'] = agents or []
# Normally logic function should not catch the raised errors
# but here it is needed so action package_show won't catch it instead
# Hiding information from API calls
try:
check_access('package_update', context)
except NotAuthorized:
pkg_dict1 = utils.hide_sensitive_fields(pkg_dict1)
pkg = Package.get(pkg_dict1['id'])
if 'erelated' in pkg.extras:
erelated = pkg.extras['erelated']
if len(erelated):
for value in erelated.split(';'):
if len(Session.query(Related).filter(Related.title == value).all()) == 0:
data_dict = {'title': value,
'type': _("Paper"),
'dataset_id': pkg.id}
related_create(context, data_dict)
return pkg_dict1
def test_create_extent(self):
package = Package.get('annakarenina')
assert package
geojson = json.loads(self.geojson_examples['point'])
shape = asShape(geojson)
package_extent = PackageExtent(package_id=package.id,the_geom=WKTSpatialElement(shape.wkt, self.db_srid))
package_extent.save()
assert package_extent.package_id == package.id
assert Session.scalar(package_extent.the_geom.x) == geojson['coordinates'][0]
assert Session.scalar(package_extent.the_geom.y) == geojson['coordinates'][1]
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def wms_preview(self,id):
#check if package exists
c.pkg = Package.get(id)
if c.pkg is None:
abort(404, 'Dataset not found')
for res in c.pkg.resources:
if res.format == "WMS":
c.wms_url = res.url if not '?' in res.url else res.url.split('?')[0]
break
if not c.wms_url:
abort(400, 'This dataset does not have a WMS resource')
return render('ckanext/spatial/wms_preview.html')
def getRecord(self, metadataPrefix, identifier):
'''Simple getRecord for a dataset.
'''
package = Package.get(identifier)
if not package:
raise IdDoesNotExistError("No dataset with id %s" % identifier)
spec = package.name
if package.owner_org:
group = Group.get(package.owner_org)
if group and group.name:
spec = group.name
if metadataPrefix == 'rdf':
return self._record_for_dataset_dcat(package, spec)
return self._record_for_dataset(package, spec)
def wms_preview(self,id):
#check if package exists
c.pkg = Package.get(id)
if c.pkg is None:
abort(404, 'Dataset not found')
for res in c.pkg.resources:
if res.format.lower() == 'wms':
c.wms_url = res.url if not '?' in res.url else res.url.split('?')[0]
break
if not c.wms_url:
abort(400, 'This dataset does not have a WMS resource')
return render('ckanext/spatial/wms_preview.html')
def test_edit(self):
name = 'annakarenina'
offset = url_for(controller='package', action='edit', id=name)
res = self.app.get(offset, extra_environ=self.extra_environ)
assert 'Edit - Datasets' in res
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'extras__1__key'] = u'spatial'
fv[prefix + 'extras__1__value'] = self.geojson_examples['point']
res = fv.submit('save', extra_environ=self.extra_environ)
assert not 'Error' in res, res
package = Package.get(name)
# Check that a PackageExtent object has been created
package_extent = Session.query(PackageExtent).filter(
PackageExtent.package_id == package.id).first()
geojson = json.loads(self.geojson_examples['point'])
assert package_extent
assert package_extent.package_id == package.id
assert Session.scalar(
package_extent.the_geom.x) == geojson['coordinates'][0]
assert Session.scalar(
package_extent.the_geom.y) == geojson['coordinates'][1]
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
# Update the spatial extra
offset = url_for(controller='package', action='edit', id=name)
res = self.app.get(offset, extra_environ=self.extra_environ)
assert 'Edit - Datasets' in res
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'extras__1__value'] = self.geojson_examples['polygon']
res = fv.submit('save', extra_environ=self.extra_environ)
assert not 'Error' in res, res
# Check that the PackageExtent object has been updated
package_extent = Session.query(PackageExtent).filter(
PackageExtent.package_id == package.id).first()
assert package_extent
assert package_extent.package_id == package.id
assert Session.scalar(
package_extent.the_geom.geometry_type) == 'ST_Polygon'
assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def send_request_message(self, pkg_id):
"""
Sends a data request message by email from the logged in user to the appropriate
contact address of the dataset.
:param pkg_id: package id
:type pkg_id: string
"""
package = Package.get(pkg_id)
package_title = package.title if package.title else package.name
subject = u"Data access request for dataset / Datapyyntö tietoaineistolle %s" % package_title
recipient_id = request.params.get('recipient', '')
return self._prepare_and_send(pkg_id, recipient_id, subject, settings.DATA_REQUEST_PREFIX_TEMPLATE, settings.REPLY_TO_SENDER_NOTE)
def view(self, id):
"""
Render the import form, see if there is a POST request, then call
import_irods_collection.
"""
context = {'model':model, 'user': c.user or c.author}
try:
check_access('package_update', context, {'id' : id})
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
pkg = Package.get(id)
c.pkg = pkg.name
if ('save' in request.params):
import_collection_to_package(request.params, id)
return render('ckanext/irods/irods.html')
def send_request_message(self, pkg_id):
"""
Sends a data request message by email from the logged in user to the appropriate
contact address of the dataset.
:param pkg_id: package id
:type pkg_id: string
"""
package = Package.get(pkg_id)
package_title = package.title if package.title else package.name
subject = u"Data access request for dataset / Datapyyntö tietoaineistolle %s" % package_title
recipient_id = request.params.get('recipient', '')
return self._prepare_and_send(pkg_id, recipient_id, subject, settings.DATA_REQUEST_PREFIX_TEMPLATE, settings.REPLY_TO_SENDER_NOTE)
def view(self, id):
"""
Render the import form, see if there is a POST request, then call
import_irods_collection.
"""
context = {'model': model, 'user': c.user or c.author}
try:
check_access('package_update', context, {'id': id})
except NotAuthorized:
abort(401, _('Not authorized to see this page'))
pkg = Package.get(id)
c.pkg = pkg.name
if ('save' in request.params):
import_collection_to_package(request.params, id)
return render('ckanext/irods/irods.html')
def update_oa_tag(context, pkg_dict, vocabulary_name, tag_value):
if not isinstance(tag_value, basestring):
try:
tag_value = tag_value[0]
except:
pass
if not tag_value:
return
log.debug(u'Updating {} tag in package {}: {}'.format(vocabulary_name,
pkg_dict['name'], tag_value))
tag = Tag.get(tag_value, vocabulary_name)
if tag:
package = Package.get(pkg_dict['id'])
package.add_tag(tag)
else:
raise Exception(u'Tag "{}" not found within vocabulary "{}"'.format(tag_value, vocabulary_name))
def test_dataset_fetch(self):
"""
A simple controller test for dataset fetching from iRODS
"""
name = 'annakarenina'
pkg = Package.by_name(name)
result = self.app.get('/irods_import/' + pkg.id)
form = result.forms['irods-form']
form['server'] = 'irods.lan'
form['uname'] = 'rods'
form['pw'] = 'rods'
form['port'] = '1247'
form['zone'] = 'omaZone'
form['path'] = '/omaZone/home/rods/ale.spec'
formres = form.submit('save')
assert not 'Could not' in formres
def update_oa_tag(context, pkg_dict, vocabulary_name, tag_value):
if not isinstance(tag_value, basestring):
try:
tag_value = tag_value[0]
except:
pass
if not tag_value:
return
log.debug(u'Updating {} tag in package {}: {}'.format(vocabulary_name,
pkg_dict['name'], tag_value))
tag = Tag.get(tag_value, vocabulary_name)
if tag:
package = Package.get(pkg_dict['id'])
package.add_tag(tag)
else:
raise Exception(u'Tag "{}" not found within vocabulary "{}"'.format(tag_value, vocabulary_name))
def send_contact_message(self, pkg_id):
"""
Sends a message by email from the logged in user to the appropriate
contact address of the dataset.
:param pkg_id: package id
:type pkg_id: string
"""
package = Package.get(pkg_id)
if not package:
ckan.lib.base.abort(404, _("Dataset not found"))
package_title = package.title if package.title else package.name
subject = u"Message regarding dataset / Viesti koskien tietoaineistoa %s" % package_title
recipient_id = request.params.get('recipient', '')
return self._prepare_and_send(pkg_id, recipient_id, subject, settings.USER_MESSAGE_PREFIX_TEMPLATE, settings.REPLY_TO_SENDER_NOTE)
def import_collection_to_package(params, id):
"""
Import a collection to dataset. Does not import whole file data but
rather the metadata.
"""
from irods import irodsCollection
path = params['path']
pkg = Package.get(id)
conn = get_connection_from_params(params)
if (conn):
coll = irodsCollection(conn, path)
from irods import iRodsOpen
rev = model.repo.new_revision()
i = 0
for obj in coll.getObjects():
extras = {}
fname, _ = obj
fpath = "%s/%s" % (coll.getCollName(), fname)
f = iRodsOpen(conn, fpath, 'r')
if f:
i += 1
res = Resource.by_name(fname)
if not res:
res = Resource(url = '', name=fname, extras=extras, \
resource_type='file')
for met in f.getUserMetadata():
key, value, _ = met
extras[key] = value
res.extras = extras
resgrp = pkg.resource_groups[0]
resgrp.resources.append(res)
Session.add(res)
Session.add(resgrp)
rev.message = "Update from iRODS, matched file %s" % fname
for met in coll.getUserMetadata():
key, value, _ = met
pkg.extras[key] = value
Session.add(pkg)
model.repo.commit()
conn.disconnect()
h.flash_success("iRODS import to dataset OK! Imported %s resources." %
i)
else:
h.flash_error("Could not connect to iRODS!")
h.redirect_to(controller='package', action='read', id=id)
def test_new_bad_geojson(self):
name = 'test-spatial-dataset-3'
offset = url_for(controller='package', action='new')
res = self.app.get(offset, extra_environ=self.extra_environ)
assert 'Add - Datasets' in res
fv = res.forms['dataset-edit']
prefix = ''
fv[prefix + 'name'] = name
fv[prefix + 'extras__0__key'] = u'spatial'
fv[prefix + 'extras__0__value'] = u'{"Type":"Bad_GeoJSON","a":2}'
res = fv.submit('save', extra_environ=self.extra_environ)
assert 'Error' in res, res
assert 'Spatial' in res
assert 'Error creating geometry' in res
# Check that package was not created
assert not Package.get(name)
def update_ancestral_license(context, pkg_dict, license_id):
session = context['session']
package = Package.get(pkg_dict['id'])
log.debug(u'Updating license for package {}: {}'.format(pkg_dict['name'], PKG_LICENSE_ID))
pkg_license = package.get_license_register()[PKG_LICENSE_ID]
package.set_license(pkg_license)
session.merge(package)
if not license_id:
return
log.debug(u'Updating ancestral license for package {}: {}'.format(pkg_dict['name'], license_id))
for resource in package.resources:
res_license = session.query(CeonResourceLicense).filter(CeonResourceLicense.resource_id == resource.id).first()
if res_license:
#session.delete(res_license)
res_license.license_id = license_id
session.merge(res_license)
else:
new_res_license = CeonResourceLicense(resource_id = resource.id, license_id = license_id)
session.add(new_res_license)
def wms_preview(self, id):
'''
:param id:
'''
# check if package exists
toolkit.c.pkg = Package.get(id)
if toolkit.c.pkg is None:
toolkit.abort(404, u'Dataset not found')
for res in toolkit.c.pkg.resources:
if res.format.lower() == u'wms':
toolkit.c.wms_url = res.url \
if u'?' not in res.url else res.url.split(u'?')[0]
break
if not toolkit.c.wms_url:
toolkit.abort(400, u'This dataset does not have a WMS resource')
return toolkit.render(u'ckanext/spatial/wms_preview.html')
def getRecord(self, metadataPrefix, identifier):
'''Simple getRecord for a dataset.
'''
package = Package.get(identifier)
if not package:
raise IdDoesNotExistError("No dataset with id %s" % identifier)
set_spec = []
if package.owner_org:
group = Group.get(package.owner_org)
if group and group.name:
set_spec.append(group.name)
if 'openaire_data' in package.as_dict().get('tags'):
set_spec.append('openaire_data')
if not set_spec:
set_spec = [package.name]
if metadataPrefix == 'rdf':
return self._record_for_dataset_dcat(package, set_spec)
if metadataPrefix == 'oai_openaire':
return self._record_for_dataset_datacite(package, set_spec)
return self._record_for_dataset(package, set_spec)
def test_link_and_map_shown(self):
name = u'annakarenina'
wms_url = 'http://maps.bgs.ac.uk/ArcGIS/services/BGS_Detailed_Geology/MapServer/WMSServer?'
rev = model.repo.new_revision()
pkg = Package.get(name)
pr = Resource(url=wms_url, format='WMS')
pkg.resources.append(pr)
pkg.save()
model.repo.commit_and_remove()
# Load the dataset page and check if link appears
offset = url_for(controller='package', action='read', id=name)
res = self.app.get(offset)
assert 'View available WMS layers' in res, res
# Load the dataset map preview page and check if libraries are loaded
offset = '/dataset/%s/map' % name
res = self.app.get(offset)
assert '<script type="text/javascript" src="/ckanext/spatial/js/wms_preview.js"></script>' in res, res
assert 'CKAN.WMSPreview.setup("%s");' % wms_url.split('?')[0] in res
def render_contact_form(self, pkg_id):
"""
Render the contact form if allowed.
:param pkg_id: package id
:type pkg_id: string
"""
c.package = Package.get(pkg_id)
if not c.package:
abort(404, _(u"Dataset not found"))
if asbool(config.get('kata.disable_contact')):
h.flash_error(_(u"Sending contact emails is prohibited for now. "
u"Please try again later or contact customer service."))
return redirect(h.url_for(controller='package',
action="read",
id=c.package.name))
contacts = utils.get_package_contacts(c.package.id)
c.recipient_options = []
for contact in contacts:
if 'name' in contact:
text_val = contact['name']
else:
at_idx = contact['email'].find('@')
text_val = contact['email'][0:at_idx]
text_val = text_val.replace(".", " ").title()
print(text_val)
c.recipient_options.append({'text': text_val, 'value': contact['id']})
c.recipient_index = request.params.get('recipient', '')
c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time())))))
return render('contact/contact_form.html')
