def render_request_form(self, pkg_id): """ Render the access request contact form if allowed. :param pkg_id: package id :type pkg_id: string """ c.package = Package.get(pkg_id) if not c.package: abort(404, _(u"Dataset not found")) if asbool(config.get('kata.disable_contact')): h.flash_error(_(u"Sending contact emails is prohibited for now. " u"Please try again later or contact customer service.")) return redirect(h.url_for(controller='package', action="read", id=c.package.name)) contacts = utils.get_package_contacts(c.package.id) c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts] c.recipient_index = request.params.get('recipient', '') c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time()))))) return render('contact/dataset_request_form.html')
def create_access_request(context, data_dict): if not context.get('ignore_auth'): check_access('create_access_request', context, data_dict) pkg_id_or_name, reason, user = get_or_bust(data_dict, ['id', 'reason', 'user']) user_org = data_dict.get('user_org') # check if the package with such id exists to use it's ID pkg = Package.get(pkg_id_or_name) if not pkg: raise ObjectNotFound() req = AccessRequest.create_or_get_request(user_id=user, package_id=pkg.id, reason=reason, org_id=pkg.owner_org) # send email notifications to org custodians if config.get('spc.access_request.send_admin_notification'): notify_org_members(pkg.owner_org, { 'pkg': pkg, 'user': user, 'reason': reason, 'user_org': user_org }) return req.as_dict()
def send(self, pkg_id): package = Package.get(pkg_id) url = h.url_for(controller='package', action="read", id=package.id) if c.user: userid = None for role in package.roles: if role.role == "admin": userid = role.user_id break if userid: owner = User.get(userid) msg = request.params.get('msg', '') if msg: send_contact_email(owner, c.userobj, package,\ msg) else: h.flash_error(_("No message")) return redirect(url) else: h.flash_error(_("No owner found")) return redirect(url) h.flash_notice(_("Message sent")) else: h.flash_error(_("Please login")) return redirect(url)
def getRecord(self, metadataPrefix, identifier): '''Simple getRecord for a dataset. ''' package = Package.get(identifier) if not package: raise IdDoesNotExistError("No dataset with id %s" % identifier) return self._record_for_dataset(package)
def test_update_extent(self): package = Package.get('annakarenina') geojson = json.loads(self.geojson_examples['point']) shape = asShape(geojson) package_extent = PackageExtent(package_id=package.id, the_geom=WKTSpatialElement( shape.wkt, self.db_srid)) package_extent.save() assert Session.scalar( package_extent.the_geom.geometry_type) == 'ST_Point' # Update the geometry (Point -> Polygon) geojson = json.loads(self.geojson_examples['polygon']) shape = asShape(geojson) package_extent.the_geom = WKTSpatialElement(shape.wkt, self.db_srid) package_extent.save() assert package_extent.package_id == package.id assert Session.scalar( package_extent.the_geom.geometry_type) == 'ST_Polygon' assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def send_email(req): requestee = User.get(req.user_id) pkg = Package.get(req.pkg_id) selrole = False for role in pkg.roles: if role.role == "admin": selrole = role if not selrole: return admin = User.get(selrole.user_id) msg = _("""%s (%s) is requesting editor access to a dataset you have created %s. Please click this link if you want to give this user write access: %s%s""") controller = 'ckanext.kata.controllers:AccessRequestController' body = msg % (requestee.name, requestee.email, pkg.title if pkg.title else pkg.name, config.get('ckan.site_url', ''), h.url_for(controller=controller, action="unlock_access", id=req.id)) email_dict = {} email_dict["subject"] = _("Access request for dataset %s" % pkg.title if pkg.title else pkg.name) email_dict["body"] = body send_notification(admin.as_dict(), email_dict)
def accessreq_show(context, data_dict): ret = {} ret['title'] = _('Request edit rights') smtp = config.get('smtp.server', '') if not len(smtp): ret['ret'] = 'Yes' return ret pkg = Package.get(data_dict['id']) selrole = False ret['id'] = pkg.id for role in pkg.roles: if role.role == "admin": selrole = True ret['no_owner'] = not selrole if c.userobj: if 'id' in data_dict: req = KataAccessRequest.is_requesting(c.userobj.id, data_dict['id']) if req: ret['ret'] = 'Yes' return ret else: ret['ret'] = 'No' return ret else: ret['ret'] = 'No' return ret ret['ret'] = 'No' return ret
def test_new(self): name = "test-spatial-dataset-1" offset = url_for(controller="package", action="new") res = self.app.get(offset, extra_environ=self.extra_environ) assert "Add - Datasets" in res fv = res.forms["dataset-edit"] prefix = "" fv[prefix + "name"] = name fv[prefix + "extras__0__key"] = u"spatial" fv[prefix + "extras__0__value"] = self.geojson_examples["point"] res = fv.submit("save", extra_environ=self.extra_environ) assert not "Error" in res, res package = Package.get(name) # Check that a PackageExtent object has been created package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id == package.id).first() geojson = json.loads(self.geojson_examples["point"]) assert package_extent assert package_extent.package_id == package.id assert Session.scalar(package_extent.the_geom.x) == geojson["coordinates"][0] assert Session.scalar(package_extent.the_geom.y) == geojson["coordinates"][1] assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def render_request_form(self, pkg_id): """ Render the access request contact form if allowed. :param pkg_id: package id :type pkg_id: string """ c.package = Package.get(pkg_id) if asbool(config.get('kata.disable_contact')): h.flash_error(_(u"Sending contact emails is prohibited for now. " u"Please try again later or contact customer service.")) return redirect(h.url_for(controller='package', action="read", id=c.package.name)) if not c.package: abort(404, _(u"Dataset not found")) contacts = utils.get_package_contacts(c.package.id) c.recipient_options = [{'text': contact['name'], 'value': contact['id']} for contact in contacts] c.recipient_index = request.params.get('recipient', '') c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time()))))) return render('contact/dataset_request_form.html')
def is_owner(context, data_dict): ''' This is used in "request edit rights" feature. Checks if the user is creator, admin or editor of the package in question :param context: context :param data_dict: package data :type data_dict: dictionary :rtype: dictionary ''' # Package creator is always owner regardless of organizations pkg = context.get('package', None) or Package.get(data_dict['id']) user = context.get('user', False) # If user id can't be resolved, user can't be owner try: user_id = convert_user_name_or_id_to_id(user, context) except: return {'success': False} if pkg.creator_user_id == user_id: return {'success': True} # Check if the user has editor rights to this dataset through an organization package = get_package_object(context, data_dict) if authz.has_user_permission_for_group_or_org(package.owner_org, user, 'delete_dataset'): return {'success': True} return {'success': False}
def test_new(self): name = 'test-spatial-dataset-1' offset = url_for(controller='package', action='new') res = self.app.get(offset, extra_environ=self.extra_environ) assert 'Add - Datasets' in res fv = res.forms['dataset-edit'] prefix = '' fv[prefix + 'name'] = name fv[prefix + 'extras__0__key'] = u'spatial' fv[prefix + 'extras__0__value'] = self.geojson_examples['point'] res = fv.submit('save', extra_environ=self.extra_environ) assert not 'Error' in res, res package = Package.get(name) # Check that a PackageExtent object has been created package_extent = Session.query(PackageExtent).filter( PackageExtent.package_id == package.id).first() geojson = json.loads(self.geojson_examples['point']) assert package_extent assert package_extent.package_id == package.id assert Session.scalar( package_extent.the_geom.x) == geojson['coordinates'][0] assert Session.scalar( package_extent.the_geom.y) == geojson['coordinates'][1] assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def is_owner(context, data_dict): ''' This is used in "request edit rights" feature. Checks if the user is admin or editor of the package in question :param context: context :param data_dict: package data :type data_dict: dictionary :rtype: dictionary ''' pkg = context.get('package', None) roles = pkg.roles if pkg else Package.get(data_dict['id']).roles user = context.get('user', False) if user: for role in roles: ruser = User.get(role.user.id) if user == ruser.name and role.role in ('admin', 'editor'): return {'success': True} # Check if the user has editor rights to this dataset through an organization package = get_package_object(context, data_dict) if new_authz.has_user_permission_for_group_or_org(package.owner_org, user, 'delete_dataset'): return {'success': True} return {'success': False}
def _create_pkg(self): model.repo.new_revision() pkg = Package.get("annakarenina") pkg.name = "fookarenina" pkg.add_resource("www.google.com", description="foo", name="foo") Session.add(pkg) Session.commit() return pkg
def _create_pkg(self): model.repo.new_revision() pkg = Package.get('annakarenina') pkg.name = "fookarenina" pkg.add_resource('www.google.com', description='foo', name="foo") Session.add(pkg) Session.commit() return pkg
def get_ancestral_license(session, package_id): license_id = None if package_id: package = Package.get(package_id) extras = package.extras if 'ancestral_license' in extras: license_id = extras['ancestral_license'] return license_id
def get_resources_licenses(session, pkg_dict): license_ids = [] package = Package.get(pkg_dict['id']) for resource in package.resources: res_license = session.query(CeonResourceLicense).filter(CeonResourceLicense.resource_id == resource.id).first() if res_license: license_ids.append(res_license.license_id) return license_ids
def package_show(context, data_dict): ''' Return the metadata of a dataset (package) and its resources. Called before showing the dataset in some interface (browser, API), or when adding package to Solr index (no validation / conversions then). :param id: the id or name of the dataset :type id: string :rtype: dictionary ''' if data_dict.get('type') == 'harvest': context['schema'] = Schemas.harvest_source_show_package_schema() if not data_dict.get('id') and not data_dict.get('name'): # Get package by data PIDs data_dict['id'] = utils.get_package_id_by_data_pids(data_dict) pkg_dict1 = ckan.logic.action.get.package_show(context, data_dict) pkg_dict1 = utils.resource_to_dataset(pkg_dict1) # Remove empty agents that come from padding the agent list in converters if 'agent' in pkg_dict1: agents = filter(None, pkg_dict1.get('agent', [])) pkg_dict1['agent'] = agents or [] # Normally logic function should not catch the raised errors # but here it is needed so action package_show won't catch it instead # Hiding information from API calls try: check_access('package_update', context) except NotAuthorized: pkg_dict1 = utils.hide_sensitive_fields(pkg_dict1) pkg = Package.get(pkg_dict1['id']) if 'erelated' in pkg.extras: erelated = pkg.extras['erelated'] if len(erelated): for value in erelated.split(';'): if len(Session.query(Related).filter(Related.title == value).all()) == 0: data_dict = {'title': value, 'type': _("Paper"), 'dataset_id': pkg.id} related_create(context, data_dict) # Update package.title to match package.extras.title_0 extras_title = pkg.extras.get(u'title_0') if extras_title and extras_title != pkg.title: repo.new_revision() pkg.title = pkg.extras[u'title_0'] pkg.save() rebuild(pkg.id) # Rebuild solr-index for this dataset return pkg_dict1
def getMetadataDDI(self,id): toolkit.response.content_type = 'application/xml' c.pkg = Package.get(id) if c.pkg is None: abort(404, 'Dataset not found') pkginfo = toolkit.get_action('package_show')({}, {'id': id}) try: return createDDIXML(pkginfo,toolkit.request.url) except Exception,e: print str(e) return "[]"
def getMetadataDDI(self, id): toolkit.response.content_type = 'application/xml' c.pkg = Package.get(id) if c.pkg is None: abort(404, 'Dataset not found') pkginfo = toolkit.get_action('package_show')({}, {'id': id}) try: return createDDIXML(pkginfo, toolkit.request.url) except Exception, e: print str(e) return "[]"
def getRecord(self, metadataPrefix, identifier): '''Simple getRecord for a dataset. ''' package = Package.get(identifier) if not package: raise IdDoesNotExistError("No dataset with id %s" % identifier) spec = package.name if package.owner_org: group = Group.get(package.owner_org) if group and group.name: spec = group.name return self._record_for_dataset(package, spec)
def googleanalytics_dataset_visits(context=None, data_dict=None): """ Fetch the amount of times a dataset hs been visited :param id: Dataset id :type id: string :returns: The number of times the dataset has been viewed :rtype: integer """ package = Package.get(data_dict['id']) return PackageStats.get_all_visits(package.id)
def show(self,id): package = Package.get(id) if package: graph = pkg_produce(package) doc = graph.serialize(format='pretty-xml') response.content_type = 'application/rdf+xml' response.headers['Content-Length'] = len(doc) return doc else: abort(404)
def is_owner(context, data_dict): pkg = context.get('package', None) roles = pkg.roles if pkg else Package.get(data_dict['id']).roles user = context.get('user', False) if user: for role in roles: ruser = User.get(role.user_id) if user == ruser.name and role.role in ('admin', 'editor'): return {'success': True} else: return {'success': False} return {'success': False}
def create_request(self, pkg_id): pkg = Package.get(pkg_id) user = c.userobj if c.userobj else None if user: req = KataAccessRequest(user.id, pkg.id) req.save() url = h.url_for(controller='package', action='read', id=pkg_id) h.flash_success(_("You now requested editor rights to package %s" % pkg.name)) redirect(url) else: url = h.url_for(controller='package', action='read', id=pkg_id) h.flash_error(_("Please log in!")) redirect(url)
def request_access(self, pkg_id): # If the user is logged in show the access request button pkg = Package.get(pkg_id) if c.user and not user_has_role(c.userobj, 'admin', pkg) and\ not user_has_role(c.userobj, 'editor', pkg) and\ not c.userobj.sysadmin and\ not config.get('smtp_server', False): following = KataAccessRequest.is_requesting(c.userobj.id, pkg_id) if not following: return snippet('snippets/access_button.html', obj_id=pkg_id, following=following) return ''
def wms_preview(self,id): #check if package exists c.pkg = Package.get(id) if c.pkg is None: abort(404, 'Dataset not found') for res in c.pkg.resources: if res.format == "WMS": c.wms_url = res.url if not '?' in res.url else res.url.split('?')[0] break if not c.wms_url: abort(400, 'This dataset does not have a WMS resource') return render('ckanext/spatial/wms_preview.html')
def package_show(context, data_dict): ''' Return the metadata of a dataset (package) and its resources. Called before showing the dataset in some interface (browser, API), or when adding package to Solr index (no validation / conversions then). :param id: the id or name of the dataset :type id: string :rtype: dictionary ''' if data_dict.get('type') == 'harvest': context['schema'] = Schemas.harvest_source_show_package_schema() context['use_cache'] = False # Disable package retrieval directly from Solr as contact.email is not there. if not data_dict.get('id') and not data_dict.get('name'): # Get package by data PIDs data_dict['id'] = utils.get_package_id_by_primary_pid(data_dict) pkg_dict1 = ckan.logic.action.get.package_show(context, data_dict) pkg_dict1 = utils.resource_to_dataset(pkg_dict1) # Remove empty agents that come from padding the agent list in converters if 'agent' in pkg_dict1: agents = filter(None, pkg_dict1.get('agent', [])) pkg_dict1['agent'] = agents or [] # Normally logic function should not catch the raised errors # but here it is needed so action package_show won't catch it instead # Hiding information from API calls try: check_access('package_update', context) except NotAuthorized: pkg_dict1 = utils.hide_sensitive_fields(pkg_dict1) pkg = Package.get(pkg_dict1['id']) if 'erelated' in pkg.extras: erelated = pkg.extras['erelated'] if len(erelated): for value in erelated.split(';'): if len(Session.query(Related).filter(Related.title == value).all()) == 0: data_dict = {'title': value, 'type': _("Paper"), 'dataset_id': pkg.id} related_create(context, data_dict) return pkg_dict1
def getRecord(self, metadataPrefix, identifier): '''Simple getRecord for a dataset. ''' package = Package.get(identifier) if not package: raise IdDoesNotExistError("No dataset with id %s" % identifier) spec = package.name if package.owner_org: group = Group.get(package.owner_org) if group and group.name: spec = group.name if metadataPrefix == 'rdf': return self._record_for_dataset_dcat(package, spec) return self._record_for_dataset(package, spec)
def test_create_extent(self): package = Package.get('annakarenina') assert package geojson = json.loads(self.geojson_examples['point']) shape = asShape(geojson) package_extent = PackageExtent(package_id=package.id,the_geom=WKTSpatialElement(shape.wkt, self.db_srid)) package_extent.save() assert package_extent.package_id == package.id assert Session.scalar(package_extent.the_geom.x) == geojson['coordinates'][0] assert Session.scalar(package_extent.the_geom.y) == geojson['coordinates'][1] assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def wms_preview(self,id): #check if package exists c.pkg = Package.get(id) if c.pkg is None: abort(404, 'Dataset not found') for res in c.pkg.resources: if res.format.lower() == 'wms': c.wms_url = res.url if not '?' in res.url else res.url.split('?')[0] break if not c.wms_url: abort(400, 'This dataset does not have a WMS resource') return render('ckanext/spatial/wms_preview.html')
def test_edit(self): name = 'annakarenina' offset = url_for(controller='package', action='edit', id=name) res = self.app.get(offset, extra_environ=self.extra_environ) assert 'Edit - Datasets' in res fv = res.forms['dataset-edit'] prefix = '' fv[prefix + 'extras__1__key'] = u'spatial' fv[prefix + 'extras__1__value'] = self.geojson_examples['point'] res = fv.submit('save', extra_environ=self.extra_environ) assert not 'Error' in res, res package = Package.get(name) # Check that a PackageExtent object has been created package_extent = Session.query(PackageExtent).filter( PackageExtent.package_id == package.id).first() geojson = json.loads(self.geojson_examples['point']) assert package_extent assert package_extent.package_id == package.id assert Session.scalar( package_extent.the_geom.x) == geojson['coordinates'][0] assert Session.scalar( package_extent.the_geom.y) == geojson['coordinates'][1] assert Session.scalar(package_extent.the_geom.srid) == self.db_srid # Update the spatial extra offset = url_for(controller='package', action='edit', id=name) res = self.app.get(offset, extra_environ=self.extra_environ) assert 'Edit - Datasets' in res fv = res.forms['dataset-edit'] prefix = '' fv[prefix + 'extras__1__value'] = self.geojson_examples['polygon'] res = fv.submit('save', extra_environ=self.extra_environ) assert not 'Error' in res, res # Check that the PackageExtent object has been updated package_extent = Session.query(PackageExtent).filter( PackageExtent.package_id == package.id).first() assert package_extent assert package_extent.package_id == package.id assert Session.scalar( package_extent.the_geom.geometry_type) == 'ST_Polygon' assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def send_request_message(self, pkg_id): """ Sends a data request message by email from the logged in user to the appropriate contact address of the dataset. :param pkg_id: package id :type pkg_id: string """ package = Package.get(pkg_id) package_title = package.title if package.title else package.name subject = u"Data access request for dataset / Datapyyntö tietoaineistolle %s" % package_title recipient_id = request.params.get('recipient', '') return self._prepare_and_send(pkg_id, recipient_id, subject, settings.DATA_REQUEST_PREFIX_TEMPLATE, settings.REPLY_TO_SENDER_NOTE)
def view(self, id): """ Render the import form, see if there is a POST request, then call import_irods_collection. """ context = {'model': model, 'user': c.user or c.author} try: check_access('package_update', context, {'id': id}) except NotAuthorized: abort(401, _('Not authorized to see this page')) pkg = Package.get(id) c.pkg = pkg.name if ('save' in request.params): import_collection_to_package(request.params, id) return render('ckanext/irods/irods.html')
def view(self, id): """ Render the import form, see if there is a POST request, then call import_irods_collection. """ context = {'model':model, 'user': c.user or c.author} try: check_access('package_update', context, {'id' : id}) except NotAuthorized: abort(401, _('Not authorized to see this page')) pkg = Package.get(id) c.pkg = pkg.name if ('save' in request.params): import_collection_to_package(request.params, id) return render('ckanext/irods/irods.html')
def unlock_access(self, id): q = model.Session.query(KataAccessRequest) q = q.filter_by(id=id) req = q.first() if req: user = User.get(req.user_id) pkg = Package.get(req.pkg_id) add_user_to_role(user, 'editor', pkg) url = h.url_for(controller='package', action='read', id=req.pkg_id) h.flash_success(_("%s now has editor rights to package %s" % (user.name, pkg.name))) req.delete() meta.Session.commit() redirect(url) else: h.flash_error(_("No such request found!")) redirect('/')
def update_ancestral_license(context, pkg_dict, license_id): session = context['session'] package = Package.get(pkg_dict['id']) log.debug(u'Updating license for package {}: {}'.format(pkg_dict['name'], PKG_LICENSE_ID)) pkg_license = package.get_license_register()[PKG_LICENSE_ID] package.set_license(pkg_license) session.merge(package) if not license_id: return log.debug(u'Updating ancestral license for package {}: {}'.format(pkg_dict['name'], license_id)) for resource in package.resources: res_license = session.query(CeonResourceLicense).filter(CeonResourceLicense.resource_id == resource.id).first() if res_license: session.delete(res_license) new_res_license = CeonResourceLicense(resource_id = resource.id, license_id = license_id) session.add(new_res_license)
def update_oa_tag(context, pkg_dict, vocabulary_name, tag_value): if not isinstance(tag_value, basestring): try: tag_value = tag_value[0] except: pass if not tag_value: return log.debug(u'Updating {} tag in package {}: {}'.format(vocabulary_name, pkg_dict['name'], tag_value)) tag = Tag.get(tag_value, vocabulary_name) if tag: package = Package.get(pkg_dict['id']) package.add_tag(tag) else: raise Exception(u'Tag "{}" not found within vocabulary "{}"'.format(tag_value, vocabulary_name))
def send_contact_message(self, pkg_id): """ Sends a message by email from the logged in user to the appropriate contact address of the dataset. :param pkg_id: package id :type pkg_id: string """ package = Package.get(pkg_id) if not package: ckan.lib.base.abort(404, _("Dataset not found")) package_title = package.title if package.title else package.name subject = u"Message regarding dataset / Viesti koskien tietoaineistoa %s" % package_title recipient_id = request.params.get('recipient', '') return self._prepare_and_send(pkg_id, recipient_id, subject, settings.USER_MESSAGE_PREFIX_TEMPLATE, settings.REPLY_TO_SENDER_NOTE)
def import_collection_to_package(params, id): """ Import a collection to dataset. Does not import whole file data but rather the metadata. """ from irods import irodsCollection path = params['path'] pkg = Package.get(id) conn = get_connection_from_params(params) if (conn): coll = irodsCollection(conn, path) from irods import iRodsOpen rev = model.repo.new_revision() i = 0 for obj in coll.getObjects(): extras = {} fname, _ = obj fpath = "%s/%s" % (coll.getCollName(), fname) f = iRodsOpen(conn, fpath, 'r') if f: i += 1 res = Resource.by_name(fname) if not res: res = Resource(url = '', name=fname, extras=extras, \ resource_type='file') for met in f.getUserMetadata(): key, value, _ = met extras[key] = value res.extras = extras resgrp = pkg.resource_groups[0] resgrp.resources.append(res) Session.add(res) Session.add(resgrp) rev.message = "Update from iRODS, matched file %s" % fname for met in coll.getUserMetadata(): key, value, _ = met pkg.extras[key] = value Session.add(pkg) model.repo.commit() conn.disconnect() h.flash_success("iRODS import to dataset OK! Imported %s resources." % i) else: h.flash_error("Could not connect to iRODS!") h.redirect_to(controller='package', action='read', id=id)
def test_edit(self): name = "annakarenina" offset = url_for(controller="package", action="edit", id=name) res = self.app.get(offset, extra_environ=self.extra_environ) assert "Edit - Datasets" in res fv = res.forms["dataset-edit"] prefix = "" fv[prefix + "extras__1__key"] = u"spatial" fv[prefix + "extras__1__value"] = self.geojson_examples["point"] res = fv.submit("save", extra_environ=self.extra_environ) assert not "Error" in res, res package = Package.get(name) # Check that a PackageExtent object has been created package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id == package.id).first() geojson = json.loads(self.geojson_examples["point"]) assert package_extent assert package_extent.package_id == package.id assert Session.scalar(package_extent.the_geom.x) == geojson["coordinates"][0] assert Session.scalar(package_extent.the_geom.y) == geojson["coordinates"][1] assert Session.scalar(package_extent.the_geom.srid) == self.db_srid # Update the spatial extra offset = url_for(controller="package", action="edit", id=name) res = self.app.get(offset, extra_environ=self.extra_environ) assert "Edit - Datasets" in res fv = res.forms["dataset-edit"] prefix = "" fv[prefix + "extras__1__value"] = self.geojson_examples["polygon"] res = fv.submit("save", extra_environ=self.extra_environ) assert not "Error" in res, res # Check that the PackageExtent object has been updated package_extent = Session.query(PackageExtent).filter(PackageExtent.package_id == package.id).first() assert package_extent assert package_extent.package_id == package.id assert Session.scalar(package_extent.the_geom.geometry_type) == "ST_Polygon" assert Session.scalar(package_extent.the_geom.srid) == self.db_srid
def update_ancestral_license(context, pkg_dict, license_id): session = context['session'] package = Package.get(pkg_dict['id']) log.debug(u'Updating license for package {}: {}'.format(pkg_dict['name'], PKG_LICENSE_ID)) pkg_license = package.get_license_register()[PKG_LICENSE_ID] package.set_license(pkg_license) session.merge(package) if not license_id: return log.debug(u'Updating ancestral license for package {}: {}'.format(pkg_dict['name'], license_id)) for resource in package.resources: res_license = session.query(CeonResourceLicense).filter(CeonResourceLicense.resource_id == resource.id).first() if res_license: #session.delete(res_license) res_license.license_id = license_id session.merge(res_license) else: new_res_license = CeonResourceLicense(resource_id = resource.id, license_id = license_id) session.add(new_res_license)
def test_new_bad_geojson(self): name = 'test-spatial-dataset-3' offset = url_for(controller='package', action='new') res = self.app.get(offset, extra_environ=self.extra_environ) assert 'Add - Datasets' in res fv = res.forms['dataset-edit'] prefix = '' fv[prefix + 'name'] = name fv[prefix + 'extras__0__key'] = u'spatial' fv[prefix + 'extras__0__value'] = u'{"Type":"Bad_GeoJSON","a":2}' res = fv.submit('save', extra_environ=self.extra_environ) assert 'Error' in res, res assert 'Spatial' in res assert 'Error creating geometry' in res # Check that package was not created assert not Package.get(name)
def test_new_bad_geojson(self): name = "test-spatial-dataset-3" offset = url_for(controller="package", action="new") res = self.app.get(offset, extra_environ=self.extra_environ) assert "Add - Datasets" in res fv = res.forms["dataset-edit"] prefix = "" fv[prefix + "name"] = name fv[prefix + "extras__0__key"] = u"spatial" fv[prefix + "extras__0__value"] = u'{"Type":"Bad_GeoJSON","a":2}' res = fv.submit("save", extra_environ=self.extra_environ) assert "Error" in res, res assert "Spatial" in res assert "Error creating geometry" in res # Check that package was not created assert not Package.get(name)
def wms_preview(self, id): ''' :param id: ''' # check if package exists toolkit.c.pkg = Package.get(id) if toolkit.c.pkg is None: toolkit.abort(404, u'Dataset not found') for res in toolkit.c.pkg.resources: if res.format.lower() == u'wms': toolkit.c.wms_url = res.url \ if u'?' not in res.url else res.url.split(u'?')[0] break if not toolkit.c.wms_url: toolkit.abort(400, u'This dataset does not have a WMS resource') return toolkit.render(u'ckanext/spatial/wms_preview.html')
def getRecord(self, metadataPrefix, identifier): '''Simple getRecord for a dataset. ''' package = Package.get(identifier) if not package: raise IdDoesNotExistError("No dataset with id %s" % identifier) set_spec = [] if package.owner_org: group = Group.get(package.owner_org) if group and group.name: set_spec.append(group.name) if 'openaire_data' in package.as_dict().get('tags'): set_spec.append('openaire_data') if not set_spec: set_spec = [package.name] if metadataPrefix == 'rdf': return self._record_for_dataset_dcat(package, set_spec) if metadataPrefix == 'oai_openaire': return self._record_for_dataset_datacite(package, set_spec) return self._record_for_dataset(package, set_spec)
def test_link_and_map_shown(self): name = u'annakarenina' wms_url = 'http://maps.bgs.ac.uk/ArcGIS/services/BGS_Detailed_Geology/MapServer/WMSServer?' rev = model.repo.new_revision() pkg = Package.get(name) pr = Resource(url=wms_url, format='WMS') pkg.resources.append(pr) pkg.save() model.repo.commit_and_remove() # Load the dataset page and check if link appears offset = url_for(controller='package', action='read', id=name) res = self.app.get(offset) assert 'View available WMS layers' in res, res # Load the dataset map preview page and check if libraries are loaded offset = '/dataset/%s/map' % name res = self.app.get(offset) assert '<script type="text/javascript" src="/ckanext/spatial/js/wms_preview.js"></script>' in res, res assert 'CKAN.WMSPreview.setup("%s");' % wms_url.split('?')[0] in res
def render_contact_form(self, pkg_id): """ Render the contact form if allowed. :param pkg_id: package id :type pkg_id: string """ c.package = Package.get(pkg_id) if not c.package: abort(404, _(u"Dataset not found")) if asbool(config.get('kata.disable_contact')): h.flash_error(_(u"Sending contact emails is prohibited for now. " u"Please try again later or contact customer service.")) return redirect(h.url_for(controller='package', action="read", id=c.package.name)) contacts = utils.get_package_contacts(c.package.id) c.recipient_options = [] for contact in contacts: if 'name' in contact: text_val = contact['name'] else: at_idx = contact['email'].find('@') text_val = contact['email'][0:at_idx] text_val = text_val.replace(".", " ").title() print(text_val) c.recipient_options.append({'text': text_val, 'value': contact['id']}) c.recipient_index = request.params.get('recipient', '') c.current_time = base64.b64encode(self.crypto.encrypt(self._pad(str(int(time.time()))))) return render('contact/contact_form.html')
def send_package_deprecation_emails(packages): grouped_by_maintainer = {} for package in packages: fullPackage = Package.get(package) maintainer_email = fullPackage.maintainer_email packageInfoForEmail = { "title": fullPackage.title, "id": fullPackage.id, "valid_till": fullPackage.extras.get("valid_till"), } if maintainer_email not in grouped_by_maintainer: grouped_by_maintainer[maintainer_email] = {"maintainer": fullPackage.maintainer, "packages": [packageInfoForEmail]} else: grouped_by_maintainer[maintainer_email]["packages"].append(packageInfoForEmail) for maintainer_email, details in grouped_by_maintainer.iteritems(): send_deprecation_email_user( maintainer_email, details["packages"], details["maintainer"], details["packages"][0]["valid_till"] )
def test_clean_tags(self): # Create source source_fixture = { 'title': 'Test Source', 'name': 'test-source', 'url': u'http://127.0.0.1:8999/gemini2.1/dataset1.xml', 'source_type': u'gemini-single', 'owner_org': 'test-org', 'metadata_created': datetime.now().strftime('%YYYY-%MM-%DD %HH:%MM:%s'), 'metadata_modified': datetime.now().strftime('%YYYY-%MM-%DD %HH:%MM:%s'), } user = User.get('dummy') if not user: user = call_action('user_create', name='dummy', password='******', email='*****@*****.**') user_name = user['name'] else: user_name = user.name org = Group.by_name('test-org') if org is None: org = call_action('organization_create', context={'user': user_name}, name='test-org') existing_g = Group.by_name('existing-group') if existing_g is None: existing_g = call_action('group_create', context={'user': user_name}, name='existing-group') context = {'user': '******'} package_schema = default_update_package_schema() context['schema'] = package_schema package_dict = {'frequency': 'manual', 'publisher_name': 'dummy', 'extras': [{'key':'theme', 'value':['non-mappable', 'thememap1']}], 'groups': [], 'title': 'fakename', 'holder_name': 'dummy', 'holder_identifier': 'dummy', 'name': 'fakename', 'notes': 'dummy', 'owner_org': 'test-org', 'modified': datetime.now(), 'publisher_identifier': 'dummy', 'metadata_created' : datetime.now(), 'metadata_modified' : datetime.now(), 'guid': unicode(uuid4()), 'identifier': 'dummy'} package_data = call_action('package_create', context=context, **package_dict) package = Package.get('fakename') source, job = self._create_source_and_job(source_fixture) job.package = package job.guid = uuid4() harvester = SpatialHarvester() with open(os.path.join('..', 'data', 'dataset.json')) as f: dataset = json.load(f) # long tags are invalid in all cases TAG_LONG_INVALID = 'abcdefghij' * 20 # if clean_tags is not set to true, tags will be truncated to 50 chars TAG_LONG_VALID = TAG_LONG_INVALID[:50] # default truncate to 100 TAG_LONG_VALID_LONG = TAG_LONG_INVALID[:100] assert len(TAG_LONG_VALID) == 50 assert TAG_LONG_VALID[-1] == 'j' TAG_CHARS_INVALID = '[email protected]!' TAG_CHARS_VALID = 'pretty-invlidtag' dataset['tags'].append(TAG_LONG_INVALID) dataset['tags'].append(TAG_CHARS_INVALID) harvester.source_config = {'clean_tags': False} out = harvester.get_package_dict(dataset, job) tags = out['tags'] # no clean tags, so invalid chars are in # but tags are truncated to 50 chars assert {'name': TAG_CHARS_VALID} not in tags assert {'name': TAG_CHARS_INVALID} in tags assert {'name': TAG_LONG_VALID_LONG} in tags assert {'name': TAG_LONG_INVALID} not in tags harvester.source_config = {'clean_tags': True} out = harvester.get_package_dict(dataset, job) tags = out['tags'] assert {'name': TAG_CHARS_VALID} in tags assert {'name': TAG_LONG_VALID_LONG} in tags
def _prepare_and_send(self, pkg_id, recipient_id, subject, prefix_template, suffix): """ Sends a message by email from the logged in user to the appropriate contact address of the given dataset. The prefix template should have formatting placeholders for the following arguments: {sender_name}, {sender_email}, {package_title}, {package_id} :param pkg_id: package id :type pkg_id: string :param recipient_id: id of the recipient, as returned by utils.get_package_contacts :type recipient_id: string :param subject: the subject of the message :type subject: string :param prefix_template: the template for the prefix to be automatically included before the user message :type prefix_template: unicode :param suffix: an additional note to be automatically included after the user message :type suffix: unicode """ url = h.url_for(controller='package', action='read', id=pkg_id) if asbool(config.get('kata.contact_captcha')): try: captcha.check_recaptcha(request) except captcha.CaptchaError: h.flash_error(_(u'Bad Captcha. Please try again.')) redirect(url) if not request.params.get('accept_logging'): h.flash_error(_(u"Message not sent as logging wasn't permitted")) return redirect(url) if asbool(config.get('kata.disable_contact')): h.flash_error(_(u"Sending contact emails is prohibited for now. Please try again later or contact customer " u"service.")) return redirect(url) package = Package.get(pkg_id) package_title = package.title if package.title else package.name sender_addr = request.params.get('from_address') sender_name = request.params.get('from_name') recipient = self._get_contact_email(pkg_id, recipient_id) if not recipient: abort(404, _('Recipient not found')) user_msg = request.params.get('msg', '') if request.params.get('hp'): h.flash_error(_(u"Message not sent. Couldn't confirm human interaction (spam bot control)")) return redirect(url) if sender_addr and sender_name and \ isinstance(sender_name, basestring) and len(sender_name) >= 3: if user_msg: prefix = prefix_template.format( sender_name=sender_name, sender_email=sender_addr, package_title=package_title, package_id=pkg_id # above line was: metadata_pid=utils.get_primary_data_pid_from_package(package) ) log.info(u"Message {m} sent from {a} ({b}) to {r} about {c}, IP: {d}" .format(m=user_msg, a=sender_name, b=sender_addr, r=recipient, c=pkg_id, d=request.environ.get('REMOTE_ADDR', 'No remote address'))) full_msg = u"{a}{b}{c}".format(a=prefix, b=user_msg, c=suffix) self._send_message(subject, full_msg, recipient.get('email'), recipient.get('name')) h.flash_success(_(u"Message sent")) else: h.flash_error(_(u"No message")) else: h.flash_error(_(u"Message not sent. Please, provide reply address and name. Name must contain \ at least three letters.")) return redirect(url)
def get_package_name_by_id(package_id): from ckan.model import Package return Package.get(package_id).title
def reimport_batch(self, package_ids, context): '''Batch-reimport all packages in `package_ids` from their original harvest source.''' ckan_fb_mapping = {} # first, do checks that can be done without connection to FIS-Broker for package_id in package_ids: package = Package.get(package_id) if not package: raise PackageIdDoesNotExistError(package_id) if not dataset_was_harvested(package): raise PackageNotHarvestedError(package_id) harvester = harvester_for_package(package) harvester_url = harvester.url harvester_type = harvester.type if not harvester_type == HARVESTER_ID: raise PackageNotHarvestedInFisbrokerError(package_id) fb_guid = fisbroker_guid(package) if not fb_guid: raise NoFisbrokerIdError(package_id) ckan_fb_mapping[package.id] = fb_guid # get the harvest source for FIS-Broker datasets fb_source = get_fisbroker_source() if not fb_source: raise NoFBHarvesterDefined() source_id = fb_source.get('id', None) # Create and start a new harvest job job_dict = toolkit.get_action('harvest_job_create')(context, {'source_id': source_id}) harvest_job = HarvestJob.get(job_dict['id']) harvest_job.gather_started = datetime.datetime.utcnow() assert harvest_job # instatiate the CSW connector (on the reasonable assumption that harvester_url is # the same for all package_ids) package_id = None reimported_packages = [] try: csw = CatalogueServiceWeb(harvester_url) for package_id, fb_guid in ckan_fb_mapping.items(): # query connector to get resource document csw.getrecordbyid([fb_guid], outputschema=namespaces['gmd']) # show resource document record = csw.records.get(fb_guid, None) if record: obj = HarvestObject(guid=fb_guid, job=harvest_job, content=record.xml, package_id=package_id, extras=[ HarvestObjectExtra(key='status',value='change'), HarvestObjectExtra(key='type',value='reimport'), ]) obj.save() assert obj, obj.content harvester = FisbrokerPlugin() harvester.force_import = True harvester.import_stage(obj) rejection_reason = self._dataset_rejected(obj) if rejection_reason: raise FBImportError(package_id, rejection_reason) harvester.force_import = False Session.refresh(obj) reimported_packages.append(record) else: raise NotFoundInFisbrokerError(package_id, fb_guid) except RequestException as error: raise NoConnectionError(package_id, harvester_url, str(error.__class__.__name__)) # successfully finish harvest job harvest_job.status = u'Finished' harvest_job.finished = datetime.datetime.utcnow() harvest_job.save() return reimported_packages