def package_relationship_update(context, data_dict):
model = context['model']
user = context['user']
id = data_dict["id"]
id2 = data_dict["id2"]
rel = data_dict["rel"]
api = context.get('api_version') or '1'
ref_package_by = 'id' if api == '2' else 'name'
pkg1 = model.Package.get(id)
pkg2 = model.Package.get(id2)
if not pkg1:
raise NotFound('First package named in address was not found.')
if not pkg2:
return NotFound('Second package named in address was not found.')
authorizer = ckan.authz.Authorizer()
am_authorized = authorizer.authorized_package_relationship(
user, pkg1, pkg2, action=model.Action.EDIT)
if not am_authorized:
raise NotAuthorized
existing_rels = pkg1.get_relationships_with(pkg2, rel)
if not existing_rels:
raise NotFound('This relationship between the packages was not found.')
entity = existing_rels[0]
comment = data_dict.get('comment', u'')
return _update_package_relationship(entity, comment, context)
def package_relationship_update(context, data_dict):
model = context['model']
user = context['user']
id = data_dict["id"]
id2 = data_dict["id2"]
rel = data_dict["rel"]
api = context.get('api_version') or '1'
ref_package_by = 'id' if api == '2' else 'name'
pkg1 = model.Package.get(id)
pkg2 = model.Package.get(id2)
if not pkg1:
raise NotFound('First package named in address was not found.')
if not pkg2:
return NotFound('Second package named in address was not found.')
check_access('package_relationship_update', context, data_dict)
existing_rels = pkg1.get_relationships_with(pkg2, rel)
if not existing_rels:
raise NotFound('This relationship between the packages was not found.')
entity = existing_rels[0]
comment = data_dict.get('comment', u'')
return _update_package_relationship(entity, comment, context)
def package_relationship_create(context, data_dict):
model = context['model']
user = context['user']
id = data_dict["id"]
id2 = data_dict["id2"]
rel_type = data_dict["rel"]
api = context.get('api_version') or '1'
ref_package_by = 'id' if api == '2' else 'name'
# Create a Package Relationship.
pkg1 = model.Package.get(id)
pkg2 = model.Package.get(id2)
if not pkg1:
raise NotFound('First package named in address was not found.')
if not pkg2:
return NotFound('Second package named in address was not found.')
check_access('package_relationship_create', context, data_dict)
##FIXME should have schema
comment = data_dict.get('comment', u'')
existing_rels = pkg1.get_relationships_with(pkg2, rel_type)
if existing_rels:
return _update_package_relationship(existing_rels[0], comment, context)
rev = model.repo.new_revision()
rev.author = user
rev.message = _(u'REST API: Create package relationship: %s %s %s') % (
pkg1, rel_type, pkg2)
rel = pkg1.add_relationship(rel_type, pkg2, comment=comment)
if not context.get('defer_commit'):
model.repo.commit_and_remove()
relationship_dicts = rel.as_dict(ref_package_by=ref_package_by)
return relationship_dicts
def package_relationship_delete(context, data_dict):
model = context['model']
user = context['user']
id = data_dict['id']
id2 = data_dict['id2']
rel = data_dict['rel']
pkg1 = model.Package.get(id)
pkg2 = model.Package.get(id2)
if not pkg1:
raise NotFound('First package named in address was not found.')
if not pkg2:
return NotFound('Second package named in address was not found.')
check_access('package_relationship_delete', context, data_dict)
existing_rels = pkg1.get_relationships_with(pkg2, rel)
if not existing_rels:
raise NotFound
relationship = existing_rels[0]
revisioned_details = 'Package Relationship: %s %s %s' % (id, rel, id2)
context['relationship'] = relationship
check_access('relationship_delete', context, data_dict)
rev = model.repo.new_revision()
rev.author = user
rev.message = _(u'REST API: Delete %s') % revisioned_details
relationship.delete()
model.repo.commit()
def user_update(context, data_dict):
'''Updates the user\'s details'''
model = context['model']
user = context['user']
schema = context.get('schema') or default_update_user_schema()
id = data_dict['id']
user_obj = model.User.get(id)
context['user_obj'] = user_obj
if user_obj is None:
raise NotFound('User was not found.')
check_access('user_update', context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, group_error_summary(errors))
user = user_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return user_dictize(user, context)
def task_status_update(context, data_dict):
model = context['model']
session = model.meta.create_local_session()
context['session'] = session
user = context['user']
id = data_dict.get("id")
schema = context.get('schema') or default_task_status_schema()
if id:
task_status = model.TaskStatus.get(id)
context["task_status"] = task_status
if task_status is None:
raise NotFound(_('TaskStatus was not found.'))
check_access('task_status_update', context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
session.rollback()
raise ValidationError(errors, task_status_error_summary(errors))
task_status = task_status_dict_save(data, context)
session.commit()
session.close()
return task_status_dictize(task_status, context)
def request_show(context, data_dict):
'''Return the metadata of a requestdata.
:param id: The id of a requestdata.
:type id: string
:rtype: dictionary
'''
data, errors = df.validate(data_dict, schema.request_show_schema(),
context)
if errors:
raise toolkit.ValidationError(errors)
check_access('requestdata_request_show', context, data_dict)
id = data.get('id')
requestdata = ckanextRequestdata.get(id=id)
if requestdata is None:
raise NotFound('Request with provided \'id\' cannot be found')
out = requestdata.as_dict()
return out
def resource_update(context, data_dict):
model = context['model']
user = context.get('user')
resource = get_resource_object(context, data_dict)
# check authentication against package
query = model.Session.query(model.Package)\
.join(model.ResourceGroup)\
.join(model.Resource)\
.filter(model.ResourceGroup.id == resource.resource_group_id)
pkg = query.first()
if not pkg:
raise NotFound(
_('No package found for this resource, cannot check auth.'))
pkg_dict = {'id': pkg.id}
authorized = package_update(context, pkg_dict).get('success')
if not authorized:
return {
'success':
False,
'msg':
_('User %s not authorized to read edit %s') %
(str(user), resource.id)
}
else:
return {'success': True}
def check_group_auth(context, data_dict):
if not data_dict:
return True
model = context['model']
pkg = context.get("package")
api_version = context.get('api_version') or '1'
group_blobs = data_dict.get("groups", [])
groups = set()
for group_blob in group_blobs:
# group_blob might be a dict or a group_ref
if isinstance(group_blob, dict):
if api_version == '1':
id = group_blob.get('name')
else:
id = group_blob.get('id')
if not id:
continue
else:
id = group_blob
grp = model.Group.get(id)
if grp is None:
raise NotFound(_('Group was not found.'))
groups.add(grp)
if pkg:
groups = groups - set(pkg.groups)
for group in groups:
if not check_access_old(group, model.Action.EDIT, context):
return False
return True
def harvest_job_create(context, data_dict):
log.info('Harvest job create: %r', data_dict)
check_access('harvest_job_create', context, data_dict)
source_id = data_dict['source_id']
# Check if source exists
source = HarvestSource.get(source_id)
if not source:
log.warn('Harvest source %s does not exist', source_id)
raise NotFound('Harvest source %s does not exist' % source_id)
# Check if the source is active
if not source.active:
log.warn('Harvest job cannot be created for inactive source %s',
source_id)
raise Exception('Can not create jobs on inactive sources')
# Check if there already is an unrun or currently running job for this source
exists = _check_for_existing_jobs(context, source_id)
if exists:
log.warn('There is already an unrun job %r for this source %s', exists,
source_id)
raise HarvestJobExists('There already is an unrun job for this source')
job = HarvestJob()
job.source = source
job.save()
log.info('Harvest job saved %s', job.id)
return harvest_job_dictize(job, context)
def harvest_source_index_clear(context, data_dict):
'''
Clears all datasets, jobs and objects related to a harvest source, but
keeps the source itself. This is useful to clean history of long running
harvest sources to start again fresh.
:param id: the id of the harvest source to clear
:type id: string
'''
check_access('harvest_source_clear', context, data_dict)
harvest_source_id = data_dict.get('id')
source = HarvestSource.get(harvest_source_id)
if not source:
log.error('Harvest source %s does not exist', harvest_source_id)
raise NotFound('Harvest source %s does not exist' % harvest_source_id)
harvest_source_id = source.id
conn = make_connection()
query = ''' +%s:"%s" +site_id:"%s" ''' % (
'harvest_source_id', harvest_source_id, config.get('ckan.site_id'))
solr_commit = toolkit.asbool(config.get('ckan.search.solr_commit', 'true'))
if toolkit.check_ckan_version(max_version='2.5.99'):
# conn is solrpy
try:
conn.delete_query(query)
if solr_commit:
conn.commit()
except Exception, e:
log.exception(e)
raise SearchIndexError(e)
finally:
def resource_update(context, data_dict):
model = context['model']
session = context['session']
user = context['user']
id = data_dict["id"]
schema = context.get('schema') or default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
raise NotFound(_('Resource was not found.'))
check_access('resource_update', context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, resource_error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
resource = resource_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return resource_dictize(resource, context)
def resource_delete(context, data_dict):
'''
Nearly plain copying of resource_delete: CKAN calls local package_delete, thus passing our local version of it.
:param context: context
:param data_dict: data_dict
:return: dict with success and optional msg
'''
model = context['model']
user = context.get('user')
resource = logic_auth.get_resource_object(context, data_dict)
# check authentication against package
pkg = model.Package.get(resource.package_id)
if not pkg:
raise NotFound(
_('No package found for this resource, cannot check auth.'))
pkg_dict = {'id': pkg.id}
authorized = package_delete(context, pkg_dict).get('success')
if not authorized:
return {
'success':
False,
'msg':
_('User %s not authorized to delete resource %s') %
(user, resource.id)
}
else:
return {'success': True}
def harvest_source_clear(context, data_dict):
'''
Clears all datasets, jobs and objects related to a harvest source, but keeps the source itself.
This is useful to clean history of long running harvest sources to start again fresh.
:param id: the id of the harvest source to clear
:type id: string
'''
check_access('harvest_source_clear', context, data_dict)
harvest_source_id = data_dict.get('id', None)
source = HarvestSource.get(harvest_source_id)
if not source:
log.error('Harvest source %s does not exist', harvest_source_id)
raise NotFound('Harvest source %s does not exist' % harvest_source_id)
harvest_source_id = source.id
# Clear all datasets from this source from the index
harvest_source_index_clear(context, data_dict)
sql = '''begin; update package set state = 'to_delete' where id in (select package_id from harvest_object where harvest_source_id = '{harvest_source_id}');
delete from harvest_object_error where harvest_object_id in (select id from harvest_object where harvest_source_id = '{harvest_source_id}');
delete from harvest_object_extra where harvest_object_id in (select id from harvest_object where harvest_source_id = '{harvest_source_id}');
delete from harvest_object where harvest_source_id = '{harvest_source_id}';
delete from harvest_gather_error where harvest_job_id in (select id from harvest_job where source_id = '{harvest_source_id}');
delete from harvest_job where source_id = '{harvest_source_id}';
delete from package_role where package_id in (select id from package where state = 'to_delete' );
delete from user_object_role where id not in (select user_object_role_id from package_role) and context = 'Package';
delete from resource_revision where resource_group_id in (select id from resource_group where package_id in (select id from package where state = 'to_delete'));
delete from resource_group_revision where package_id in (select id from package where state = 'to_delete');
delete from package_tag_revision where package_id in (select id from package where state = 'to_delete');
delete from member_revision where table_id in (select id from package where state = 'to_delete');
delete from package_extra_revision where package_id in (select id from package where state = 'to_delete');
delete from package_revision where id in (select id from package where state = 'to_delete');
delete from package_tag where package_id in (select id from package where state = 'to_delete');
delete from resource where resource_group_id in (select id from resource_group where package_id in (select id from package where state = 'to_delete'));
delete from package_extra where package_id in (select id from package where state = 'to_delete');
delete from member where table_id in (select id from package where state = 'to_delete');
delete from resource_group where package_id in (select id from package where state = 'to_delete');
delete from package where id in (select id from package where state = 'to_delete'); commit;'''.format(
harvest_source_id=harvest_source_id)
model = context['model']
model.Session.execute(sql)
# Refresh the index for this source to update the status object
context.update({'validate': False, 'ignore_auth': True})
package_dict = logic.get_action('package_show')(context, {
'id': harvest_source_id
})
if package_dict:
package_index = PackageSearchIndex()
package_index.index_package(package_dict)
return {'id': harvest_source_id}
def package_update_validate(context, data_dict):
model = context['model']
user = context['user']
id = data_dict["id"]
schema = context.get('schema') or default_update_package_schema()
model.Session.remove()
model.Session()._context = context
pkg = model.Package.get(id)
context["package"] = pkg
if pkg is None:
raise NotFound(_('Package was not found.'))
data_dict["id"] = pkg.id
check_access('package_update', context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, package_error_summary(errors))
return data
def harvest_job_create(context, data_dict):
log.info('Harvest job create: %r', data_dict)
check_access('harvest_job_create', context, data_dict)
source_id = data_dict['source_id']
# Check if source exists
source = HarvestSource.get(source_id)
if not source:
log.warn('Harvest source %s does not exist', source_id)
raise NotFound('Harvest source %s does not exist' % source_id)
# Check if the source is active
if not source.active:
log.warn('Harvest job cannot be created for inactive source %s',
source_id)
raise HarvestError('Can not create jobs on inactive sources')
# Check if there already is an unrun job for this source
data_dict = {'source_id': source_id, 'status': u'New'}
exists = harvest_job_list(context, data_dict)
if len(exists):
log.warn('There is already an unrun job %r for this source %s', exists,
source_id)
raise HarvestError('There already is an unrun job for this source')
job = HarvestJob()
job.source = source
job.save()
log.info('Harvest job saved %s', job.id)
return harvest_job_dictize(job, context)
def resource_acl_patch(context, data_dict):
'''Patch the resource acl
:param id: the id of the resource acl
:param auth_type: user, org
:param auth_id: the id of user or organization
:param permission: none, read
'''
reference = get_or_bust(data_dict, 'id')
acl = ResourceAcl.get(reference)
if not acl:
raise NotFound('acl <{id}> was not found.'.format(id=reference))
data_dict['resource_id'] = acl.resource_id
check_access('resource_acl_patch', context, data_dict)
data, errors = validate(data_dict, resource_acl_patch_schema(), context)
if errors:
raise ValidationError(errors)
acl.auth_type = data.get('auth_type', acl.auth_type)
acl.auth_id = data.get('auth_id', acl.auth_id)
acl.permission = data.get('permission', acl.permission)
acl.last_modified = datetime.datetime.utcnow()
acl.modifier_user_id = context.get('user')
acl.commit()
return acl.as_dict()
def _user_has_organization(username):
user = model.User.get(username)
if not user:
raise NotFound("Failed to find user")
query = model.Session.query(
model.Member).filter(model.Member.table_name == 'user').filter(
model.Member.table_id == user.id)
return query.count() > 0
def resource_view_show(context, data_dict):
model = context['model']
resource_view = model.ResourceView.get(data_dict['id'])
if not resource_view:
raise NotFound(_('Resource view not found, cannot check auth.'))
package = context['package']
return is_authorized('restrict_dataset_show', context, package.as_dict())
def _find_entity(self):
views = self._outer._get_views_map()
try:
id = views[self._eid]
except KeyError:
raise NotFound('No view with EID {!r} in {}'.format(
self._eid, self._outer))
view_dict = self._outer._api.action.resource_view_show(id=id)
return View(self._eid, view_dict, self._outer)
def harvest_job_abort(context, data_dict):
'''
Aborts a harvest job. Given a harvest source_id, it looks for the latest
one and (assuming it not already Finished) marks it as Finished. It also
marks any of that source's harvest objects and (if not complete or error)
marks them "ERROR", so any left in limbo are cleaned up. Does not actually
stop running any queued harvest fetchs/objects.
:param source_id: the name or id of the harvest source with a job to abort
:type source_id: string
'''
check_access('harvest_job_abort', context, data_dict)
model = context['model']
source_id = data_dict.get('source_id')
source = harvest_source_show(context, {'id': source_id})
# HarvestJob set status to 'Finished'
# Don not use harvest_job_list since it can use a lot of memory
last_job = model.Session.query(HarvestJob) \
.filter_by(source_id=source['id']) \
.order_by(HarvestJob.created.desc()).first()
if not last_job:
raise NotFound('Error: source has no jobs')
job = get_action('harvest_job_show')(context, {'id': last_job.id})
if job['status'] != 'Finished':
# i.e. New or Running
job_obj = HarvestJob.get(job['id'])
job_obj.status = new_status = 'Finished'
model.repo.commit_and_remove()
log.info('Harvest job changed status from "%s" to "%s"', job['status'],
new_status)
else:
log.info('Harvest job unchanged. Source %s status is: "%s"', job['id'],
job['status'])
# HarvestObjects set to ERROR
job_obj = HarvestJob.get(job['id'])
objs = job_obj.objects
for obj in objs:
if obj.state not in ('COMPLETE', 'ERROR'):
old_state = obj.state
obj.state = 'ERROR'
log.info('Harvest object changed state from "%s" to "%s": %s',
old_state, obj.state, obj.id)
else:
log.info('Harvest object not changed from "%s": %s', obj.state,
obj.id)
model.repo.commit_and_remove()
job_obj = HarvestJob.get(job['id'])
return harvest_job_dictize(job_obj, context)
def featured_show(context, data_dict):
data, errors = df.validate(data_dict, schema_get, context)
if errors:
raise ValidationError(errors)
featured = db.Featured.get(resource_view_id=data['resource_view_id'])
if featured is None:
raise NotFound()
return table_dictize(featured, context)
def featured_show(context, data_dict):
data, errors = df.validate(data_dict, schema_get, context)
if errors:
raise ValidationError(errors)
featuredn = db.Featurednumbers.get(id_fn=data['id_fn'])
if featuredn is None:
raise NotFound()
return table_dictize(featuredn, context)
def resource_update(context, data_dict):
if not tk.asbool(
config.get('ckan.cloud_storage_enable')) or data_dict.get('url'):
return origin.resource_update(context, data_dict)
model = context['model']
user = context['user']
id = _get_or_bust(data_dict, "id")
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
_check_access('resource_update', context, data_dict)
del context["resource"]
package_id = resource.resource_group.package.id
pkg_dict = _get_action('package_show')(context, {'id': package_id})
for n, p in enumerate(pkg_dict['resources']):
if p['id'] == id:
break
else:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
upload = uploader.S3Upload(data_dict)
pkg_dict['resources'][n] = data_dict
try:
context['defer_commit'] = True
context['use_cache'] = False
pkg_dict = _get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][n]
raise ValidationError(errors)
def _find_entity(self):
res_dicts = [
r for r in self._outer['resources']
if r['ckanext_importer_resource_eid'] == self._eid
]
if not res_dicts:
raise NotFound('No resource with EID {!r} in {}'.format(
self._eid, self._outer))
if len(res_dicts) > 1:
raise ValueError('Multiple resources for EID {} in {}'.format(
self._eid, self._outer))
return Resource(self._eid, res_dicts[0], self._outer)
def webhook_show(context, data_dict):
check_access("webhook_show", context, data_dict)
data, errors = df.validate(data_dict, schema_get, context)
if errors:
raise ValidationError(errors)
webhook = db.Webhook.get(id=data['id'])
if webhook is None:
raise NotFound()
return table_dictize(webhook, context)
def harvest_source_update(context, data_dict):
check_access('harvest_source_update', context, data_dict)
model = context['model']
session = context['session']
source_id = data_dict.get('id')
schema = context.get('schema') or default_harvest_source_schema()
log.info('Harvest source %s update: %r', source_id, data_dict)
source = HarvestSource.get(source_id)
if not source:
log.error('Harvest source %s does not exist', source_id)
raise NotFound('Harvest source %s does not exist' % source_id)
data, errors = validate(data_dict, schema)
if errors:
session.rollback()
raise ValidationError(errors, _error_summary(errors))
fields = ['url', 'title', 'type', 'description', 'user_id', 'publisher_id']
for f in fields:
if f in data and data[f] is not None:
if f == 'url':
data[f] = data[f].strip()
source.__setattr__(f, data[f])
if 'active' in data_dict:
source.active = data['active']
if 'config' in data_dict:
source.config = data['config']
source.save()
# Abort any pending jobs
if not source.active:
jobs = HarvestJob.filter(source=source, status=u'New')
log.info(
'Harvest source %s not active, so aborting %i outstanding jobs',
source_id, jobs.count())
if jobs:
for job in jobs:
job.status = u'Aborted'
job.save()
# Ensure sqlalchemy writes to the db immediately, since the gather/fetch
# runs in a different process and needs the latest source info. Not sure if
# this works, but try it.
model.repo.commit_and_remove()
return harvest_source_dictize(source, context)
def package_relationships_list(context, data_dict):
##TODO needs to work with dictization layer
model = context['model']
user = context['user']
api = context.get('api_version') or '1'
id = data_dict["id"]
id2 = data_dict.get("id2")
rel = data_dict.get("rel")
ref_package_by = 'id' if api == '2' else 'name'
pkg1 = model.Package.get(id)
pkg2 = None
if not pkg1:
raise NotFound('First package named in request was not found.')
if id2:
pkg2 = model.Package.get(id2)
if not pkg2:
raise NotFound('Second package named in address was not found.')
if rel == 'relationships':
rel = None
check_access('package_relationships_list', context, data_dict)
# TODO: How to handle this object level authz?
relationships = Authorizer().\
authorized_package_relationships(\
user, pkg1, pkg2, rel, model.Action.READ)
if rel and not relationships:
raise NotFound('Relationship "%s %s %s" not found.' % (id, rel, id2))
relationship_dicts = [
rel.as_dict(pkg1, ref_package_by=ref_package_by)
for rel in relationships
]
return relationship_dicts
def ogdch_shacl_validate(context, data_dict): # noqa
"""
validates a harvest source against a shacl shape
"""
# get sources from data_dict
if 'harvest_source_id' in data_dict:
harvest_source_id = data_dict['harvest_source_id']
harvest_source = HarvestSource.get(harvest_source_id)
if not harvest_source:
raise NotFound(
'Harvest source {} does not exist'.format(harvest_source_id))
else:
raise NotFound('Configuration missing for harvest source')
datapath = data_dict['datapath']
resultpath = data_dict['resultpath']
shapefilepath = data_dict['shapefilepath']
csvpath = data_dict['csvpath']
shaclcommand = data_dict['shaclcommand']
log.info('shacl_validate called for source: {},'
'configuration: {}'.format(harvest_source_id, data_dict))
# get rdf parse config for harvest source
rdf_format = json.loads(harvest_source.config)\
.get("rdf_format", "xml")
# parse harvest_source
data_rdfgraph = rdflib.Graph()
# parse data from harvest source url
try:
data_rdfgraph.parse(harvest_source.url, format=rdf_format)
except RDFParserException, e:
raise RDFParserException(
'Error parsing the RDF file during shacl validation: {0}'.format(
e))
def webhook_list(context, data_dict):
check_access("webhook_list", context, data_dict)
data, errors = df.validate(data_dict, schema_list, context)
if errors:
raise ValidationError(errors)
webhooks = db.Webhook.find(topic=data['topic']).all()
if webhooks is None:
raise NotFound()
ids = [webhook.id for webhook in webhooks]
return ids
