def alert(self, dataset_name):
'''
Put an alert aka. a signalement on a dataset.
'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized('Alert creation requires an user')
dataset = Package.by_name(dataset_name)
alert_type = toolkit.request.POST['type']
comment = toolkit.request.POST['comment']
alert = DatasetAlert(dataset, user, alert_type, comment)
DB.add(alert)
DB.commit()
alert.notify_admins()
return self.json_response({
'id': alert.id,
'user_id': alert.user_id,
'dataset_id': alert.dataset_id,
'type': alert.type,
'comment': alert.comment,
'created': alert.created
})
def alert(self, dataset_name):
'''
Put an alert aka. a signalement on a dataset.
'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized('Alert creation requires an user')
dataset = Package.by_name(dataset_name)
alert_type = toolkit.request.POST['type']
comment = toolkit.request.POST['comment']
alert = DatasetAlert(dataset, user, alert_type, comment)
DB.add(alert)
DB.commit()
alert.notify_admins()
return self.json_response({
'id': alert.id,
'user_id': alert.user_id,
'dataset_id': alert.dataset_id,
'type': alert.type,
'comment': alert.comment,
'created': alert.created
})
def test_menuitem_for_resource(self):
"""
See if there is a 'IRODS' menuitem in the main menu for IRODS import,
specifically for when a resource is seen.
"""
name = 'annakarenina'
pkg = Package.by_name(name)
res = pkg.resources[0]
offset = url_for(controller='package', action='resource_read', \
id = name, resource_id = res.id)
res = self.app.get(offset)
assert 'IRODS' in res
def test_menuitem_for_resource(self):
"""
See if there is a 'IRODS' menuitem in the main menu for IRODS import,
specifically for when a resource is seen.
"""
name = 'annakarenina'
pkg = Package.by_name(name)
res = pkg.resource_groups[0].resources[0]
offset = url_for(controller='package', action='resource_read', \
id = name, resource_id = res.id)
res = self.app.get(offset)
assert 'IRODS' in res
def test_dataset_fetch(self):
"""
A simple controller test for dataset fetching from iRODS
"""
name = 'annakarenina'
pkg = Package.by_name(name)
result = self.app.get('/irods_import/' + pkg.id)
form = result.forms['irods-form']
form['server'] = 'irods.lan'
form['uname'] = 'rods'
form['pw'] = 'rods'
form['port'] = '1247'
form['zone'] = 'omaZone'
form['path'] = '/omaZone/home/rods/ale.spec'
formres = form.submit('save')
assert not 'Could not' in formres
def test_dataset_fetch(self):
"""
A simple controller test for dataset fetching from iRODS
"""
name = 'annakarenina'
pkg = Package.by_name(name)
result = self.app.get('/irods_import/'+pkg.id, extra_environ={'REMOTE_USER':
str(self.sysadmin.name)})
log.debug(result)
form = result.forms['irods-form']
form['server'] = 'irods.lan'
form['uname'] = 'rods'
form['pw'] = 'rods'
form['port'] = '1247'
form['zone'] = 'omaZone'
form['path'] = '/omaZone/home/rods/ale.spec'
formres = form.submit('save')
assert not 'Could not' in formres
def fork(self, dataset_name):
'''
Fork this package by duplicating it.
The new owner will be the user parameter.
The new package is created and the original will have a new related reference to the fork.
'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized('Membership request requires an user')
dataset = Package.by_name(dataset_name)
name_width = min(len(dataset.name), 88)
forked_name = '{name}-fork-{hash}'.format(
name=dataset.name[:name_width],
hash=str(uuid1())[:6],
)
orgs = user.get_groups('organization')
resources = [{
'url': r.url,
'description': r.description,
'format': r.format,
'name': r.name,
'resource_type': r.resource_type,
'mimetype': r.mimetype,
} for r in dataset.resources]
groups = [{'id': g.id} for g in dataset.get_groups()]
tags = [{
'name': t.name,
'vocabulary_id': t.vocabulary_id
} for t in dataset.get_tags()]
extras = [{
'key': key,
'value': value
} for key, value in dataset.extras.items() if key != 'supplier_id']
forked = toolkit.get_action('package_create')(
data_dict={
'name': forked_name,
'title': dataset.title,
'maintainer': user.fullname,
'maintainer_email': user.email,
'license_id': dataset.license_id,
'notes': dataset.notes,
'url': dataset.url,
'version': dataset.version,
'type': dataset.type,
'owner_org': orgs[0].id if len(orgs) else None,
'resources': resources,
'groups': groups,
'tags': tags,
'extras': extras,
})
# PackageRole.add_user_to_role(user, model.Role.ADMIN, forked)
# Manually add the groups to bypass CKAN authorization
# TODO: Find a better way to handle open groups
for group in dataset.get_groups():
group.add_package_by_name(forked_name)
self.commit()
# Create the fork relationship
toolkit.get_action('package_relationship_create')(
data_dict={
'type': 'has_derivation',
'subject': dataset.id,
'object': forked['id'],
'comment': 'Fork',
})
return self.json_response(Package.by_name(forked_name))
def import_stage(self, harvest_object):
'''
The import stage will receive a HarvestObject object and will be
responsible for:
- performing any necessary action with the fetched object (e.g
create a CKAN package).
Note: if this stage creates or updates a package, a reference
to the package must be added to the HarvestObject.
Additionally, the HarvestObject must be flagged as current.
- creating the HarvestObject - Package relation (if necessary)
- creating and storing any suitable HarvestObjectErrors that may
occur.
- returning True if everything went as expected, False otherwise.
:param harvest_object: HarvestObject object
:returns: True if everything went right, False if errors were found
'''
model.repo.new_revision()
master_data = json.loads(harvest_object.content)
domain = master_data['domain']
group = Group.get(domain)
if not group:
group = Group(name=domain, description=domain)
if 'records' in master_data:
records = master_data['records']
set_name = master_data['set_name']
for rec in records:
identifier, metadata, _ = rec
if metadata:
name = metadata['title'][0] if len(metadata['title'])\
else identifier
title = name
norm_title = unicodedata.normalize('NFKD', name)\
.encode('ASCII', 'ignore')\
.lower().replace(' ', '_')[:35]
slug = ''.join(e for e in norm_title
if e in string.ascii_letters + '_')
name = slug
creator = metadata['creator'][0]\
if len(metadata['creator']) else ''
description = metadata['description'][0]\
if len(metadata['description']) else ''
pkg = Package.by_name(name)
if not pkg:
pkg = Package(name=name, title=title)
extras = {}
for met in metadata.items():
key, value = met
if len(value) > 0:
if key == 'subject' or key == 'type':
for tag in value:
if tag:
tag = munge_tag(tag[:100])
tag_obj = model.Tag.by_name(tag)
if not tag_obj:
tag_obj = model.Tag(name=tag)
if tag_obj:
pkgtag = model.PackageTag(
tag=tag_obj,
package=pkg)
Session.add(tag_obj)
Session.add(pkgtag)
else:
extras[key] = ' '.join(value)
pkg.author = creator
pkg.author_email = creator
pkg.title = title
pkg.notes = description
pkg.extras = extras
pkg.url = \
"%s?verb=GetRecord&identifier=%s&metadataPrefix=oai_dc"\
% (harvest_object.job.source.url, identifier)
pkg.save()
harvest_object.package_id = pkg.id
Session.add(harvest_object)
setup_default_user_roles(pkg)
url = ''
for ids in metadata['identifier']:
if ids.startswith('http://'):
url = ids
title = metadata['title'][0] if len(metadata['title'])\
else ''
description = metadata['description'][0]\
if len(metadata['description']) else ''
pkg.add_resource(url, description=description, name=title)
group.add_package_by_name(pkg.name)
subg_name = "%s - %s" % (domain, set_name)
subgroup = Group.by_name(subg_name)
if not subgroup:
subgroup = Group(name=subg_name, description=subg_name)
subgroup.add_package_by_name(pkg.name)
Session.add(group)
Session.add(subgroup)
setup_default_user_roles(group)
setup_default_user_roles(subgroup)
model.repo.commit()
else:
self._save_object_error('Could not receive any objects from fetch!'
, harvest_object, stage='Import')
return False
return True
def fork(self, dataset_name):
'''
Fork this package by duplicating it.
The new owner will be the user parameter.
The new package is created and the original will have a new related reference to the fork.
'''
if not toolkit.request.method == 'POST':
raise toolkit.abort(400, 'Expected POST method')
user = toolkit.c.userobj
if not user:
raise toolkit.NotAuthorized('Membership request requires an user')
dataset = Package.by_name(dataset_name)
name_width = min(len(dataset.name), 88)
forked_name = '{name}-fork-{hash}'.format(
name=dataset.name[:name_width],
hash=str(uuid1())[:6],
)
orgs = user.get_groups('organization')
resources = [{
'url': r.url,
'description': r.description,
'format': r.format,
'name': r.name,
'resource_type': r.resource_type,
'mimetype': r.mimetype,
}
for r in dataset.resources
]
groups = [{'id': g.id} for g in dataset.get_groups()]
tags = [{'name': t.name, 'vocabulary_id': t.vocabulary_id} for t in dataset.get_tags()]
extras = [{'key': key, 'value': value} for key, value in dataset.extras.items() if key != 'supplier_id']
forked = toolkit.get_action('package_create')(data_dict={
'name': forked_name,
'title': dataset.title,
'maintainer': user.fullname,
'maintainer_email': user.email,
'license_id': dataset.license_id,
'notes': dataset.notes,
'url': dataset.url,
'version': dataset.version,
'type': dataset.type,
'owner_org': orgs[0].id if len(orgs) else None,
'resources': resources,
'groups': groups,
'tags': tags,
'extras': extras,
})
# PackageRole.add_user_to_role(user, model.Role.ADMIN, forked)
# Manually add the groups to bypass CKAN authorization
# TODO: Find a better way to handle open groups
for group in dataset.get_groups():
group.add_package_by_name(forked_name)
self.commit()
# Create the fork relationship
toolkit.get_action('package_relationship_create')(data_dict={
'type': 'has_derivation',
'subject': dataset.id,
'object': forked['id'],
'comment': 'Fork',
})
return self.json_response(Package.by_name(forked_name))
def is_central_gov_organogram(pkg_name):
pkg_dict = Package.by_name(pkg_name).as_dict()
pkg_extras = pkg_dict.get('extras', {})
return pkg_extras.get(
'schema-vocabulary') == 'd3c0b23f-6979-45e4-88ed-d2ab59b005d0'
def activate_upload(pkg_name):
pkg_dict = Package.by_name(pkg_name).as_dict()
pkg_extras = pkg_dict.get('extras', {})
return pkg_extras.get('schema-vocabulary') in (
'538b857a-64ba-490e-8440-0e32094a28a7',
'd3c0b23f-6979-45e4-88ed-d2ab59b005d0')
