def resource_create(context, data_dict):
if not tk.asbool(config.get('ckan.cloud_storage_enable')) or data_dict.get('url'):
return origin.resource_create(context, data_dict)
model = context['model']
user = context['user']
package_id = _get_or_bust(data_dict, 'package_id')
data_dict.pop('package_id')
pkg_dict = _get_action('package_show')(context, {'id': package_id})
_check_access('resource_create', context, data_dict)
if not 'resources' in pkg_dict:
pkg_dict['resources'] = []
upload = uploader.S3Upload(data_dict)
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
_get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][-1]
raise ValidationError(errors)
def resource_create(context, data_dict):
'''
.. sealso https://github.com/ckan/ckan/blob/master/ckan/logic/action/create.py
'''
model = context['model']
user = context['user']
package_id = _get_or_bust(data_dict, 'package_id')
_get_or_bust(data_dict, 'url')
pkg_dict = _get_action('package_show')(
dict(context, return_type='dict'),
{'id': package_id})
_check_access('resource_create', context, data_dict)
for plugin in plugins.PluginImplementations(plugins.IResourceController):
plugin.before_create(context, data_dict)
if not 'resources' in pkg_dict:
pkg_dict['resources'] = []
upload = uploader.get_resource_uploader(data_dict)
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
_get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][-1]
raise ValidationError(errors)
def resource_patch(context: Context,
data_dict: DataDict) -> ActionResult.ResourcePatch:
'''Patch a resource
:param id: the id of the resource
:type id: string
The difference between the update and patch methods is that the patch will
perform an update of the provided parameters, while leaving all other
parameters unchanged, whereas the update methods deletes all parameters
not explicitly provided in the data_dict
'''
_check_access('resource_patch', context, data_dict)
show_context: Context = {
'model': context['model'],
'session': context['session'],
'user': context['user'],
'auth_user_obj': context['auth_user_obj'],
}
resource_dict = _get_action('resource_show')(
show_context, {
'id': _get_or_bust(data_dict, 'id')
})
patched = dict(resource_dict)
patched.update(data_dict)
return _update.resource_update(context, patched)
def hdx_qa_resource_patch(context, data_dict):
_check_access('hdx_qa_resource_patch', context, data_dict)
context['allow_resource_qa_script_field'] = True
context[BATCH_MODE] = BATCH_MODE_KEEP_OLD
return _get_action('resource_patch')(context, data_dict)
def package_patch(context, data_dict):
'''Patch a dataset (package).
:param id: the id or name of the dataset
:type id: string
The difference between the update and patch methods is that the patch will
perform an update of the provided parameters, while leaving all other
parameters unchanged, whereas the update methods deletes all parameters
not explicitly provided in the data_dict
You must be authorized to edit the dataset and the groups that it belongs
to.
'''
_check_access('package_patch', context, data_dict)
show_context = {
'model': context['model'],
'session': context['session'],
'user': context['user'],
'auth_user_obj': context['auth_user_obj'],
}
package_dict = _get_action('package_show')(
show_context,
{'id': _get_or_bust(data_dict, 'id')})
patched = dict(package_dict)
patched.update(data_dict)
patched['id'] = package_dict['id']
return _update.package_update(context, patched)
def group_patch(context, data_dict):
'''Patch a group
:param id: the id or name of the group
:type id: string
The difference between the update and patch methods is that the patch will
perform an update of the provided parameters, while leaving all other
parameters unchanged, whereas the update methods deletes all parameters
not explicitly provided in the data_dict
'''
_check_access('group_patch', context, data_dict)
show_context = {
'model': context['model'],
'session': context['session'],
'user': context['user'],
'auth_user_obj': context['auth_user_obj'],
}
group_dict = _get_action('group_show')(
show_context,
{'id': _get_or_bust(data_dict, 'id')})
patched = dict(group_dict)
patched.pop('display_name', None)
patched.update(data_dict)
return _update.group_update(context, patched)
def package_patch(context, data_dict):
'''Patch a dataset (package).
:param id: the id or name of the dataset
:type id: string
The difference between the update and patch methods is that the patch will
perform an update of the provided parameters, while leaving all other
parameters unchanged, whereas the update methods deletes all parameters
not explicitly provided in the data_dict
You must be authorized to edit the dataset and the groups that it belongs
to.
'''
_check_access('package_patch', context, data_dict)
show_context = {
'model': context['model'],
'session': context['session'],
'user': context['user'],
'auth_user_obj': context['auth_user_obj'],
}
package_dict = _get_action('package_show')(
show_context, {
'id': _get_or_bust(data_dict, 'id')
})
patched = dict(package_dict)
patched.update(data_dict)
patched['id'] = package_dict['id']
return _update.package_update(context, patched)
def organization_patch(context: Context,
data_dict: DataDict) -> ActionResult.OrganizationPatch:
'''Patch an organization
:param id: the id or name of the organization
:type id: string
The difference between the update and patch methods is that the patch will
perform an update of the provided parameters, while leaving all other
parameters unchanged, whereas the update methods deletes all parameters
not explicitly provided in the data_dict
'''
_check_access('organization_patch', context, data_dict)
show_context: Context = {
'model': context['model'],
'session': context['session'],
'user': context['user'],
'auth_user_obj': context['auth_user_obj'],
}
organization_dict = _get_action('organization_show')(
show_context, {
'id': _get_or_bust(data_dict, 'id')
})
patched = dict(organization_dict)
patched.pop('display_name', None)
patched.update(data_dict)
patch_context = context.copy()
patch_context['allow_partial_update'] = True
return _update.organization_update(patch_context, patched)
def group_patch(context, data_dict):
'''Patch a group
:param id: the id or name of the group
:type id: string
The difference between the update and patch methods is that the patch will
perform an update of the provided parameters, while leaving all other
parameters unchanged, whereas the update methods deletes all parameters
not explicitly provided in the data_dict
'''
_check_access('group_patch', context, data_dict)
show_context = {
'model': context['model'],
'session': context['session'],
'user': context['user'],
'auth_user_obj': context['auth_user_obj'],
}
group_dict = _get_action('group_show')(show_context, {
'id': _get_or_bust(data_dict, 'id')
})
patched = dict(group_dict)
patched.pop('display_name', None)
patched.update(data_dict)
return _update.group_update(context, patched)
def hdx_mark_qa_completed(context, data_dict):
_check_access('hdx_mark_qa_completed', context, data_dict)
_get_or_bust(data_dict, 'qa_completed')
context['allow_qa_completed_field'] = True
context[BATCH_MODE] = BATCH_MODE_KEEP_OLD
return _get_action('package_patch')(context, data_dict)
def resource_update(context, data_dict):
'''Update a resource.
::seealso https://github.com/ckan/ckan/blob/master/ckan/logic/action/update.py
'''
model = context['model']
user = context['user']
id = _get_or_bust(data_dict, "id")
resource = model.Resource.get(id)
previous_s3_object_url = resource.url
context["resource"] = resource
if not resource:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
_check_access('resource_update', context, data_dict)
del context["resource"]
package_id = resource.package.id
pkg_dict = _get_action('package_show')(dict(context, return_type='dict'),
{'id': package_id})
for n, p in enumerate(pkg_dict['resources']):
if p['id'] == id:
break
else:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
for plugin in plugins.PluginImplementations(plugins.IResourceController):
plugin.before_update(context, pkg_dict['resources'][n], data_dict)
upload = uploader.get_resource_uploader(data_dict)
pkg_dict['resources'][n] = data_dict
try:
context['defer_commit'] = True
context['use_cache'] = False
updated_pkg_dict = _get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][n]
raise ValidationError(errors)
def resource_update(context, data_dict):
if not tk.asbool(
config.get('ckan.cloud_storage_enable')) or data_dict.get('url'):
return origin.resource_update(context, data_dict)
model = context['model']
user = context['user']
id = _get_or_bust(data_dict, "id")
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
_check_access('resource_update', context, data_dict)
del context["resource"]
package_id = resource.resource_group.package.id
pkg_dict = _get_action('package_show')(context, {'id': package_id})
for n, p in enumerate(pkg_dict['resources']):
if p['id'] == id:
break
else:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
upload = uploader.S3Upload(data_dict)
pkg_dict['resources'][n] = data_dict
try:
context['defer_commit'] = True
context['use_cache'] = False
pkg_dict = _get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][n]
raise ValidationError(errors)
def resource_update(context, data_dict):
if not config.get('ckan.cloud_storage_enable') or data_dict.get('url'):
return origin.resource_update(context, data_dict)
model = context['model']
user = context['user']
id = _get_or_bust(data_dict, "id")
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
_check_access('resource_update', context, data_dict)
del context["resource"]
package_id = resource.resource_group.package.id
pkg_dict = _get_action('package_show')(context, {'id': package_id})
for n, p in enumerate(pkg_dict['resources']):
if p['id'] == id:
break
else:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
upload = uploader.S3Upload(data_dict)
pkg_dict['resources'][n] = data_dict
try:
context['defer_commit'] = True
context['use_cache'] = False
pkg_dict = _get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][n]
raise ValidationError(errors)
def hdx_mark_broken_link_in_resource(context, data_dict):
'''
Does a resource patch to change the 'broken_link' to True. Also sets a field in the context so that the value
of the 'broken_link' field is kept. Otherwise it would be reset on validation.
:param id: the id of the resource
:type id: str
:return:
:rtype: dict
'''
data_dict['broken_link'] = True
context['allow_broken_link_field'] = True
context[BATCH_MODE] = BATCH_MODE_KEEP_OLD
return _get_action('resource_patch')(context, data_dict)
def resource_patch(context, data_dict):
'''
Cloned from core. It adds a 'no_compute_extra_hdx_show_properties' in contexts to
make the update faster (less computation in the custom package_show)
Also used to parse validation parameters (SKIP_VALIDATION) for special cases.
Patch a resource
:param id: the id of the resource
:type id: string
The difference between the update and patch methods is that the patch will
perform an update of the provided parameters, while leaving all other
parameters unchanged, whereas the update methods deletes all parameters
not explicitly provided in the data_dict
'''
_check_access('resource_patch', context, data_dict)
context['no_compute_extra_hdx_show_properties'] = True
process_batch_mode(context, data_dict)
process_skip_validation(context, data_dict)
show_context = {
'model':
context['model'],
'session':
context['session'],
'user':
context['user'],
'auth_user_obj':
context['auth_user_obj'],
'no_compute_extra_hdx_show_properties':
context.get('no_compute_extra_hdx_show_properties')
}
resource_dict = _get_action('resource_show')(
show_context, {
'id': _get_or_bust(data_dict, 'id')
})
patched = dict(resource_dict)
patched.update(data_dict)
return _update.resource_update(context, patched)
def package_patch(context: Context,
data_dict: DataDict) -> ActionResult.PackagePatch:
'''Patch a dataset (package).
:param id: the id or name of the dataset
:type id: string
The difference between the update and patch methods is that the patch will
perform an update of the provided parameters, while leaving all other
parameters unchanged, whereas the update methods deletes all parameters
not explicitly provided in the data_dict.
You are able to partially update and/or create resources with
package_patch. If you are updating existing resources be sure to provide
the resource id. Existing resources excluded from the package_patch
data_dict will be removed. Resources in the package data_dict without
an id will be treated as new resources and will be added. New resources
added with the patch method do not create the default views.
You must be authorized to edit the dataset and the groups that it belongs
to.
'''
_check_access('package_patch', context, data_dict)
show_context: Context = {
'model': context['model'],
'session': context['session'],
'user': context['user'],
'auth_user_obj': context['auth_user_obj'],
'ignore_auth': context.get('ignore_auth', False),
'for_update': True
}
package_dict = _get_action('package_show')(
show_context, {
'id': _get_or_bust(data_dict, 'id')
})
patched = dict(package_dict)
patched.update(data_dict)
patched['id'] = package_dict['id']
return _update.package_update(context, patched)
def package_patch(context, data_dict):
'''
Cloned from core. It's used to parse validation parameters (SKIP_VALIDATION) for special cases
Also, changed so that it now calls "our" package_update instead of the core package_update.
Patch a dataset (package).
:param id: the id or name of the dataset
:type id: string
The difference between the update and patch methods is that the patch will
perform an update of the provided parameters, while leaving all other
parameters unchanged, whereas the update methods deletes all parameters
not explicitly provided in the data_dict
You must be authorized to edit the dataset and the groups that it belongs
to.
'''
process_skip_validation(context, data_dict)
# Original package patch from CKAN
_check_access('package_patch', context, data_dict)
show_context = {
'model': context['model'],
'session': context['session'],
'user': context['user'],
'auth_user_obj': context['auth_user_obj'],
}
package_dict = _get_action('package_show')(
show_context, {
'id': _get_or_bust(data_dict, 'id')
})
patched = dict(package_dict)
patched.update(data_dict)
patched['id'] = package_dict['id']
# slightly modified to call "our" package_update
return package_update(context, patched)
def package_patch(context, data_dict):
'''Patch a dataset (package).
:param id: the id or name of the dataset
:type id: string
The difference between the update and patch methods is that the patch will
perform an update of the provided parameters, while leaving all other
parameters unchanged, whereas the update methods deletes all parameters
not explicitly provided in the data_dict.
You are able to partially update and/or create resources with
package_patch. If you are updating existing resources be sure to provide
the resource id. Existing resources excluded from the package_patch
data_dict will be removed. Resources in the package data_dict without
an id will be treated as new resources and will be added. New resources
added with the patch method do not create the default views.
You must be authorized to edit the dataset and the groups that it belongs
to.
'''
_check_access('package_patch', context, data_dict)
show_context = {
'model': context['model'],
'session': context['session'],
'user': context['user'],
'auth_user_obj': context['auth_user_obj'],
}
package_dict = _get_action('package_show')(
show_context,
{'id': _get_or_bust(data_dict, 'id')})
patched = dict(package_dict)
patched.update(data_dict)
patched['id'] = package_dict['id']
return _update.package_update(context, patched)
def resource_delete(context, data_dict):
''' Delete a resource.
.. seealso https://github.com/ckan/ckan/blob/master/ckan/logic/action/delete.py
'''
model = context['model']
user = context['user']
id = _get_or_bust(data_dict, "id")
log.debug(id)
resource = model.Resource.get(id)
previous_s3_object_url = resource.url
################################################################################################################
if tk.asbool(config.get('ckanext.cloud_storage.enable')) and previous_s3_object_url.startswith("https://s3.amazonaws.com/") :
log.debug('Deleting Remote Resource')
log.debug(previous_s3_object_url)
context["resource"] = resource
if not resource:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
_check_access('resource_delete', context, data_dict)
package_id = resource.package.id
pkg_dict = _get_action('package_show')(context, {'id': package_id})
for n, p in enumerate(pkg_dict['resources']):
if p['id'] == id:
break
else:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
upload = uploader.get_resource_uploader(data_dict)
upload.delete(previous_s3_object_url)
else:
log.debug('Plugin Not Enabled or External Link')
################################################################################################################
return origin.resource_delete(context, data_dict)
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
_get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][-1]
raise ValidationError(errors)
## Get out resource_id resource from model as it will not appear in
## package_show until after commit
s3_link = upload.upload(context['package'].resources[-1].id,
uploader.get_max_resource_size())
if s3_link:
pkg_dict['resources'][-1]['url_type'] = ''
pkg_dict['resources'][-1]['url'] = 'https://s3.amazonaws.com/' + s3_link
_get_action('package_update')(context, pkg_dict)
model.repo.commit()
## Run package show again to get out actual last_resource
pkg_dict = _get_action('package_show')(context, {'id': package_id})
resource = pkg_dict['resources'][-1]
return resource
resource_create.__doc__ = origin.resource_create.__doc__
upload = uploader.get_resource_uploader(data_dict)
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
_get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][-1]
raise ValidationError(errors)
## Get out resource_id resource from model as it will not appear in
## package_show until after commit
remote_filepath = upload.upload(context['package'].resources[-1].id, uploader.get_max_resource_size())
if remote_filepath:
log.debug(remote_filepath)
pkg_dict['resources'][-1]['url_type'] = ''
pkg_dict['resources'][-1]['url'] = remote_filepath
_get_action('package_update')(context, pkg_dict)
model.repo.commit()
## Run package show again to get out actual last_resource
updated_pkg_dict = _get_action('package_show')(context, {'id': package_id})
resource = updated_pkg_dict['resources'][-1]
for plugin in plugins.PluginImplementations(plugins.IResourceController):
plugin.after_create(context, resource)
return resource
if p['id'] == id:
break
else:
log.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
upload = uploader.S3Upload(data_dict)
pkg_dict['resources'][n] = data_dict
try:
context['defer_commit'] = True
context['use_cache'] = False
pkg_dict = _get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][n]
raise ValidationError(errors)
s3_link = upload.upload(id, uploader.get_max_resource_size())
if s3_link:
pkg_dict['resources'][n]['url_type'] = ''
pkg_dict['resources'][n]['url'] = 'https://s3.amazonaws.com/' + s3_link
_get_action('package_update')(context, pkg_dict)
model.repo.commit()
return _get_action('resource_show')(context, {'id': id})
resource_update.__doc__ = origin.resource_update.__doc__
context['use_cache'] = False
updated_pkg_dict = _get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except ValidationError, e:
errors = e.error_dict['resources'][n]
raise ValidationError(errors)
#Delete previous file (?)
log.debug("Delete previous file: "+previous_s3_object_url)
upload.delete(previous_s3_object_url)
remote_filepath = upload.upload(id, uploader.get_max_resource_size())
log.debug(remote_filepath)
if remote_filepath:
pkg_dict['resources'][n]['url_type'] = ''
pkg_dict['resources'][n]['url'] = remote_filepath
_get_action('package_update')(context, pkg_dict)
model.repo.commit()
resource = _get_action('resource_show')(context, {'id': id})
for plugin in plugins.PluginImplementations(plugins.IResourceController):
plugin.after_update(context, resource)
return resource
resource_update.__doc__ = origin.resource_update.__doc__
