def after_create(self, context, data_dict):
if not get_create_mode_from_config() == u'async':
return
if data_dict.get(u'resources'):
# This is a dataset
for resource in data_dict[u'resources']:
self._handle_validation_for_resource(resource)
else:
# This is a resource
self._handle_validation_for_resource(data_dict)
def get_actions(self):
new_actions = {
u'resource_validation_run': resource_validation_run,
u'resource_validation_show': resource_validation_show,
u'resource_validation_delete': resource_validation_delete,
}
if get_create_mode_from_config() == u'sync':
new_actions[u'resource_create'] = custom_resource_create
if get_update_mode_from_config() == u'sync':
new_actions[u'resource_update'] = custom_resource_update
return new_actions
def after_update(self, context, data_dict):
is_dataset = self._data_dict_is_dataset(data_dict)
# Need to allow create as well because resource_create calls
# package_update
if (not get_update_mode_from_config() == u'async'
and not get_create_mode_from_config() == u'async'):
return
if context.pop('_validation_performed', None):
# Ugly, but needed to avoid circular loops caused by the
# validation job calling resource_patch (which calls
# package_update)
return
if is_dataset:
package_id = data_dict.get('id')
if self.packages_to_skip.pop(package_id, None) or context.get(
'save', False):
# Either we're updating an individual resource,
# or we're updating the package metadata via the web form;
# in both cases, we don't need to validate every resource.
return
for resource in data_dict.get(u'resources', []):
if resource[u'id'] in self.resources_to_validate:
# This is part of a resource_update call, it will be
# handled on the next `after_update` call
continue
else:
# This is an actual package_update call, validate the
# resources if necessary
self._handle_validation_for_resource(context, resource)
else:
# This is a resource
resource_id = data_dict[u'id']
if resource_id in self.resources_to_validate:
for plugin in p.PluginImplementations(IDataValidation):
if not plugin.can_validate(context, data_dict):
log.debug('Skipping validation for resource %s',
data_dict['id'])
return
del self.resources_to_validate[resource_id]
_run_async_validation(resource_id)
def after_create(self, context, data_dict):
is_dataset = self._data_dict_is_dataset(data_dict)
if not get_create_mode_from_config() == u'async':
return
if is_dataset:
for resource in data_dict.get(u'resources', []):
self._handle_validation_for_resource(context, resource)
else:
# This is a resource. Resources don't need to be handled here
# as there is always a previous `package_update` call that will
# trigger the `before_update` and `after_update` hooks
pass
def after_update(self, context, data_dict):
is_dataset = self._data_dict_is_dataset(data_dict)
# Need to allow create as well because resource_create calls
# package_update
if (not get_update_mode_from_config() == u'async'
and not get_create_mode_from_config() == u'async'):
return
if context.get('_validation_performed'):
# Ugly, but needed to avoid circular loops caused by the
# validation job calling resource_patch (which calls
# package_update)
del context['_validation_performed']
return
if is_dataset:
for resource in data_dict.get(u'resources', []):
if resource[u'id'] in self.resources_to_validate:
# This is part of a resource_update call, it will be
# handled on the next `after_update` call
continue
else:
# This is an actual package_update call, validate the
# resources if necessary
self._handle_validation_for_resource(context, resource)
else:
# This is a resource
resource_id = data_dict[u'id']
if resource_id in self.resources_to_validate:
for plugin in p.PluginImplementations(IDataValidation):
if not plugin.can_validate(context, data_dict):
log.debug('Skipping validation for resource {}'.format(
data_dict['id']))
return
del self.resources_to_validate[resource_id]
_run_async_validation(resource_id)
def after_create(self, context, resource):
if not get_create_mode_from_config() == u'async':
return
needs_validation = False
if ((
# File uploaded
resource.get(u'url_type') == u'upload' or
# URL defined
resource.get(u'url')
) and (
# Make sure format is supported
resource.get(u'format', u'').lower() in
settings.SUPPORTED_FORMATS
)):
needs_validation = True
if needs_validation:
_run_async_validation(resource['id'])
def after_update(self, context, data_dict):
is_dataset = self._data_dict_is_dataset(data_dict)
# Need to allow create as well because resource_create calls
# package_update
if (not get_update_mode_from_config() == u'async'
and not get_create_mode_from_config() == u'async'):
return
if context.get('_validation_performed'):
# Ugly, but needed to avoid circular loops caused by the
# validation job calling resource_patch (which calls
# package_update)
del context['_validation_performed']
return
if not is_dataset:
if context.get('_dont_validate'):
# Ugly, but needed to avoid circular loops caused by the
# validation job calling resource_patch (which calls
# package_update)
del context['_dont_validate']
return
# This is a resource
resource_id = data_dict[u'id']
if resource_id in self.resources_to_validate:
for plugin in p.PluginImplementations(IDataValidation):
if not plugin.can_validate(context, data_dict):
log.debug('Skipping validation for resource {}'.format(
data_dict['id']))
return
del self.resources_to_validate[resource_id]
_run_async_validation(resource_id)
if data_dict.get('validate_package'):
t.get_action('resource_validation_run_batch')(
context, {
'dataset_ids': data_dict.get('package_id')
})
def after_create(self, context, data_dict):
is_dataset = self._data_dict_is_dataset(data_dict)
if not get_create_mode_from_config() == u'async':
return
if is_dataset:
for resource in data_dict.get(u'resources', []):
self._handle_validation_for_resource(context, resource)
else:
for plugin in p.PluginImplementations(IDataValidation):
if not plugin.can_validate(context, data_dict):
log.debug('Skipping validation for resource {}'.format(
data_dict['id']))
return
_run_async_validation(data_dict['id'])
if data_dict.get('validate_package'):
t.get_action('resource_validation_run_batch')(
context, {
'dataset_ids': data_dict.get('package_id')
})
def test_config_both_false(self):
assert_equals(get_update_mode_from_config(), None)
assert_equals(get_create_mode_from_config(), None)
def test_config_create_false_async(self):
assert_equals(get_create_mode_from_config(), None)
def test_config_create_true_async(self):
assert_equals(get_create_mode_from_config(), 'async')
def test_config_defaults(self):
assert_equals(get_update_mode_from_config(), 'async')
assert_equals(get_create_mode_from_config(), 'async')
def resource_create(context, data_dict):
'''Appends a new resource to a datasets list of resources.
This is duplicate of the CKAN core resource_create action, with just the
addition of a synchronous data validation step.
This is of course not ideal but it's the only way right now to hook
reliably into the creation process without overcomplicating things.
Hopefully future versions of CKAN will incorporate more flexible hook
points that will allow a better approach.
'''
model = context['model']
package_id = t.get_or_bust(data_dict, 'package_id')
if not data_dict.get('url'):
data_dict['url'] = ''
pkg_dict = t.get_action('package_show')(
dict(context, return_type='dict'),
{'id': package_id})
t.check_access('resource_create', context, data_dict)
for plugin in plugins.PluginImplementations(plugins.IResourceController):
plugin.before_create(context, data_dict)
if 'resources' not in pkg_dict:
pkg_dict['resources'] = []
upload = uploader.get_resource_uploader(data_dict)
if 'mimetype' not in data_dict:
if hasattr(upload, 'mimetype'):
data_dict['mimetype'] = upload.mimetype
if 'size' not in data_dict:
if hasattr(upload, 'filesize'):
data_dict['size'] = upload.filesize
pkg_dict['resources'].append(data_dict)
try:
context['defer_commit'] = True
context['use_cache'] = False
t.get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except t.ValidationError as e:
try:
raise t.ValidationError(e.error_dict['resources'][-1])
except (KeyError, IndexError):
raise t.ValidationError(e.error_dict)
# Get out resource_id resource from model as it will not appear in
# package_show until after commit
resource_id = context['package'].resources[-1].id
upload.upload(resource_id,
uploader.get_max_resource_size())
# Custom code starts
if get_create_mode_from_config() == u'sync':
run_validation = True
for plugin in plugins.PluginImplementations(IDataValidation):
if not plugin.can_validate(context, data_dict):
log.debug('Skipping validation for resource %s', resource_id)
run_validation = False
if run_validation:
is_local_upload = (
hasattr(upload, 'filename') and
upload.filename is not None and
isinstance(upload, uploader.ResourceUpload))
_run_sync_validation(
resource_id, local_upload=is_local_upload, new_resource=True)
# Custom code ends
model.repo.commit()
# Run package show again to get out actual last_resource
updated_pkg_dict = t.get_action('package_show')(
context, {'id': package_id})
resource = updated_pkg_dict['resources'][-1]
# Add the default views to the new resource
t.get_action('resource_create_default_resource_views')(
{'model': context['model'],
'user': context['user'],
'ignore_auth': True
},
{'resource': resource,
'package': updated_pkg_dict
})
for plugin in plugins.PluginImplementations(plugins.IResourceController):
plugin.after_create(context, resource)
return resource
def test_config_create_true_async(self):
assert get_create_mode_from_config() == 'async'
def test_config_both_false(self):
assert get_update_mode_from_config() is None
assert get_create_mode_from_config() is None
def test_config_create_false_async(self):
assert get_create_mode_from_config() is None
t.get_action('package_update')(context, pkg_dict)
context.pop('defer_commit')
except t.ValidationError, e:
try:
raise t.ValidationError(e.error_dict['resources'][-1])
except (KeyError, IndexError):
raise t.ValidationError(e.error_dict)
# Get out resource_id resource from model as it will not appear in
# package_show until after commit
resource_id = context['package'].resources[-1].id
upload.upload(resource_id, uploader.get_max_resource_size())
# Custom code starts
if get_create_mode_from_config() == u'sync':
is_upload = (hasattr(upload, 'filename')
and upload.filename is not None)
_run_sync_validation(resource_id, upload=is_upload)
# Custom code ends
model.repo.commit()
# Run package show again to get out actual last_resource
updated_pkg_dict = t.get_action('package_show')(context, {
'id': package_id
})
resource = updated_pkg_dict['resources'][-1]
# Add the default views to the new resource
def test_config_defaults(self):
assert get_update_mode_from_config() == 'async'
assert get_create_mode_from_config() == 'async'
