def alert(self, dataset_name): ''' Put an alert aka. a signalement on a dataset. ''' if not toolkit.request.method == 'POST': raise toolkit.abort(400, 'Expected POST method') user = toolkit.c.userobj if not user: raise toolkit.NotAuthorized('Alert creation requires an user') dataset = Package.by_name(dataset_name) alert_type = toolkit.request.POST['type'] comment = toolkit.request.POST['comment'] alert = DatasetAlert(dataset, user, alert_type, comment) DB.add(alert) DB.commit() alert.notify_admins() return self.json_response({ 'id': alert.id, 'user_id': alert.user_id, 'dataset_id': alert.dataset_id, 'type': alert.type, 'comment': alert.comment, 'created': alert.created })
def test_menuitem_for_resource(self): """ See if there is a 'IRODS' menuitem in the main menu for IRODS import, specifically for when a resource is seen. """ name = 'annakarenina' pkg = Package.by_name(name) res = pkg.resources[0] offset = url_for(controller='package', action='resource_read', \ id = name, resource_id = res.id) res = self.app.get(offset) assert 'IRODS' in res
def test_menuitem_for_resource(self): """ See if there is a 'IRODS' menuitem in the main menu for IRODS import, specifically for when a resource is seen. """ name = 'annakarenina' pkg = Package.by_name(name) res = pkg.resource_groups[0].resources[0] offset = url_for(controller='package', action='resource_read', \ id = name, resource_id = res.id) res = self.app.get(offset) assert 'IRODS' in res
def test_dataset_fetch(self): """ A simple controller test for dataset fetching from iRODS """ name = 'annakarenina' pkg = Package.by_name(name) result = self.app.get('/irods_import/' + pkg.id) form = result.forms['irods-form'] form['server'] = 'irods.lan' form['uname'] = 'rods' form['pw'] = 'rods' form['port'] = '1247' form['zone'] = 'omaZone' form['path'] = '/omaZone/home/rods/ale.spec' formres = form.submit('save') assert not 'Could not' in formres
def test_dataset_fetch(self): """ A simple controller test for dataset fetching from iRODS """ name = 'annakarenina' pkg = Package.by_name(name) result = self.app.get('/irods_import/'+pkg.id, extra_environ={'REMOTE_USER': str(self.sysadmin.name)}) log.debug(result) form = result.forms['irods-form'] form['server'] = 'irods.lan' form['uname'] = 'rods' form['pw'] = 'rods' form['port'] = '1247' form['zone'] = 'omaZone' form['path'] = '/omaZone/home/rods/ale.spec' formres = form.submit('save') assert not 'Could not' in formres
def fork(self, dataset_name): ''' Fork this package by duplicating it. The new owner will be the user parameter. The new package is created and the original will have a new related reference to the fork. ''' if not toolkit.request.method == 'POST': raise toolkit.abort(400, 'Expected POST method') user = toolkit.c.userobj if not user: raise toolkit.NotAuthorized('Membership request requires an user') dataset = Package.by_name(dataset_name) name_width = min(len(dataset.name), 88) forked_name = '{name}-fork-{hash}'.format( name=dataset.name[:name_width], hash=str(uuid1())[:6], ) orgs = user.get_groups('organization') resources = [{ 'url': r.url, 'description': r.description, 'format': r.format, 'name': r.name, 'resource_type': r.resource_type, 'mimetype': r.mimetype, } for r in dataset.resources] groups = [{'id': g.id} for g in dataset.get_groups()] tags = [{ 'name': t.name, 'vocabulary_id': t.vocabulary_id } for t in dataset.get_tags()] extras = [{ 'key': key, 'value': value } for key, value in dataset.extras.items() if key != 'supplier_id'] forked = toolkit.get_action('package_create')( data_dict={ 'name': forked_name, 'title': dataset.title, 'maintainer': user.fullname, 'maintainer_email': user.email, 'license_id': dataset.license_id, 'notes': dataset.notes, 'url': dataset.url, 'version': dataset.version, 'type': dataset.type, 'owner_org': orgs[0].id if len(orgs) else None, 'resources': resources, 'groups': groups, 'tags': tags, 'extras': extras, }) # PackageRole.add_user_to_role(user, model.Role.ADMIN, forked) # Manually add the groups to bypass CKAN authorization # TODO: Find a better way to handle open groups for group in dataset.get_groups(): group.add_package_by_name(forked_name) self.commit() # Create the fork relationship toolkit.get_action('package_relationship_create')( data_dict={ 'type': 'has_derivation', 'subject': dataset.id, 'object': forked['id'], 'comment': 'Fork', }) return self.json_response(Package.by_name(forked_name))
def import_stage(self, harvest_object): ''' The import stage will receive a HarvestObject object and will be responsible for: - performing any necessary action with the fetched object (e.g create a CKAN package). Note: if this stage creates or updates a package, a reference to the package must be added to the HarvestObject. Additionally, the HarvestObject must be flagged as current. - creating the HarvestObject - Package relation (if necessary) - creating and storing any suitable HarvestObjectErrors that may occur. - returning True if everything went as expected, False otherwise. :param harvest_object: HarvestObject object :returns: True if everything went right, False if errors were found ''' model.repo.new_revision() master_data = json.loads(harvest_object.content) domain = master_data['domain'] group = Group.get(domain) if not group: group = Group(name=domain, description=domain) if 'records' in master_data: records = master_data['records'] set_name = master_data['set_name'] for rec in records: identifier, metadata, _ = rec if metadata: name = metadata['title'][0] if len(metadata['title'])\ else identifier title = name norm_title = unicodedata.normalize('NFKD', name)\ .encode('ASCII', 'ignore')\ .lower().replace(' ', '_')[:35] slug = ''.join(e for e in norm_title if e in string.ascii_letters + '_') name = slug creator = metadata['creator'][0]\ if len(metadata['creator']) else '' description = metadata['description'][0]\ if len(metadata['description']) else '' pkg = Package.by_name(name) if not pkg: pkg = Package(name=name, title=title) extras = {} for met in metadata.items(): key, value = met if len(value) > 0: if key == 'subject' or key == 'type': for tag in value: if tag: tag = munge_tag(tag[:100]) tag_obj = model.Tag.by_name(tag) if not tag_obj: tag_obj = model.Tag(name=tag) if tag_obj: pkgtag = model.PackageTag( tag=tag_obj, package=pkg) Session.add(tag_obj) Session.add(pkgtag) else: extras[key] = ' '.join(value) pkg.author = creator pkg.author_email = creator pkg.title = title pkg.notes = description pkg.extras = extras pkg.url = \ "%s?verb=GetRecord&identifier=%s&metadataPrefix=oai_dc"\ % (harvest_object.job.source.url, identifier) pkg.save() harvest_object.package_id = pkg.id Session.add(harvest_object) setup_default_user_roles(pkg) url = '' for ids in metadata['identifier']: if ids.startswith('http://'): url = ids title = metadata['title'][0] if len(metadata['title'])\ else '' description = metadata['description'][0]\ if len(metadata['description']) else '' pkg.add_resource(url, description=description, name=title) group.add_package_by_name(pkg.name) subg_name = "%s - %s" % (domain, set_name) subgroup = Group.by_name(subg_name) if not subgroup: subgroup = Group(name=subg_name, description=subg_name) subgroup.add_package_by_name(pkg.name) Session.add(group) Session.add(subgroup) setup_default_user_roles(group) setup_default_user_roles(subgroup) model.repo.commit() else: self._save_object_error('Could not receive any objects from fetch!' , harvest_object, stage='Import') return False return True
def fork(self, dataset_name): ''' Fork this package by duplicating it. The new owner will be the user parameter. The new package is created and the original will have a new related reference to the fork. ''' if not toolkit.request.method == 'POST': raise toolkit.abort(400, 'Expected POST method') user = toolkit.c.userobj if not user: raise toolkit.NotAuthorized('Membership request requires an user') dataset = Package.by_name(dataset_name) name_width = min(len(dataset.name), 88) forked_name = '{name}-fork-{hash}'.format( name=dataset.name[:name_width], hash=str(uuid1())[:6], ) orgs = user.get_groups('organization') resources = [{ 'url': r.url, 'description': r.description, 'format': r.format, 'name': r.name, 'resource_type': r.resource_type, 'mimetype': r.mimetype, } for r in dataset.resources ] groups = [{'id': g.id} for g in dataset.get_groups()] tags = [{'name': t.name, 'vocabulary_id': t.vocabulary_id} for t in dataset.get_tags()] extras = [{'key': key, 'value': value} for key, value in dataset.extras.items() if key != 'supplier_id'] forked = toolkit.get_action('package_create')(data_dict={ 'name': forked_name, 'title': dataset.title, 'maintainer': user.fullname, 'maintainer_email': user.email, 'license_id': dataset.license_id, 'notes': dataset.notes, 'url': dataset.url, 'version': dataset.version, 'type': dataset.type, 'owner_org': orgs[0].id if len(orgs) else None, 'resources': resources, 'groups': groups, 'tags': tags, 'extras': extras, }) # PackageRole.add_user_to_role(user, model.Role.ADMIN, forked) # Manually add the groups to bypass CKAN authorization # TODO: Find a better way to handle open groups for group in dataset.get_groups(): group.add_package_by_name(forked_name) self.commit() # Create the fork relationship toolkit.get_action('package_relationship_create')(data_dict={ 'type': 'has_derivation', 'subject': dataset.id, 'object': forked['id'], 'comment': 'Fork', }) return self.json_response(Package.by_name(forked_name))
def is_central_gov_organogram(pkg_name): pkg_dict = Package.by_name(pkg_name).as_dict() pkg_extras = pkg_dict.get('extras', {}) return pkg_extras.get( 'schema-vocabulary') == 'd3c0b23f-6979-45e4-88ed-d2ab59b005d0'
def activate_upload(pkg_name): pkg_dict = Package.by_name(pkg_name).as_dict() pkg_extras = pkg_dict.get('extras', {}) return pkg_extras.get('schema-vocabulary') in ( '538b857a-64ba-490e-8440-0e32094a28a7', 'd3c0b23f-6979-45e4-88ed-d2ab59b005d0')