def after_create(self, context, data_dict): if not get_create_mode_from_config() == u'async': return if data_dict.get(u'resources'): # This is a dataset for resource in data_dict[u'resources']: self._handle_validation_for_resource(resource) else: # This is a resource self._handle_validation_for_resource(data_dict)
def get_actions(self): new_actions = { u'resource_validation_run': resource_validation_run, u'resource_validation_show': resource_validation_show, u'resource_validation_delete': resource_validation_delete, } if get_create_mode_from_config() == u'sync': new_actions[u'resource_create'] = custom_resource_create if get_update_mode_from_config() == u'sync': new_actions[u'resource_update'] = custom_resource_update return new_actions
def after_update(self, context, data_dict): is_dataset = self._data_dict_is_dataset(data_dict) # Need to allow create as well because resource_create calls # package_update if (not get_update_mode_from_config() == u'async' and not get_create_mode_from_config() == u'async'): return if context.pop('_validation_performed', None): # Ugly, but needed to avoid circular loops caused by the # validation job calling resource_patch (which calls # package_update) return if is_dataset: package_id = data_dict.get('id') if self.packages_to_skip.pop(package_id, None) or context.get( 'save', False): # Either we're updating an individual resource, # or we're updating the package metadata via the web form; # in both cases, we don't need to validate every resource. return for resource in data_dict.get(u'resources', []): if resource[u'id'] in self.resources_to_validate: # This is part of a resource_update call, it will be # handled on the next `after_update` call continue else: # This is an actual package_update call, validate the # resources if necessary self._handle_validation_for_resource(context, resource) else: # This is a resource resource_id = data_dict[u'id'] if resource_id in self.resources_to_validate: for plugin in p.PluginImplementations(IDataValidation): if not plugin.can_validate(context, data_dict): log.debug('Skipping validation for resource %s', data_dict['id']) return del self.resources_to_validate[resource_id] _run_async_validation(resource_id)
def after_create(self, context, data_dict): is_dataset = self._data_dict_is_dataset(data_dict) if not get_create_mode_from_config() == u'async': return if is_dataset: for resource in data_dict.get(u'resources', []): self._handle_validation_for_resource(context, resource) else: # This is a resource. Resources don't need to be handled here # as there is always a previous `package_update` call that will # trigger the `before_update` and `after_update` hooks pass
def after_update(self, context, data_dict): is_dataset = self._data_dict_is_dataset(data_dict) # Need to allow create as well because resource_create calls # package_update if (not get_update_mode_from_config() == u'async' and not get_create_mode_from_config() == u'async'): return if context.get('_validation_performed'): # Ugly, but needed to avoid circular loops caused by the # validation job calling resource_patch (which calls # package_update) del context['_validation_performed'] return if is_dataset: for resource in data_dict.get(u'resources', []): if resource[u'id'] in self.resources_to_validate: # This is part of a resource_update call, it will be # handled on the next `after_update` call continue else: # This is an actual package_update call, validate the # resources if necessary self._handle_validation_for_resource(context, resource) else: # This is a resource resource_id = data_dict[u'id'] if resource_id in self.resources_to_validate: for plugin in p.PluginImplementations(IDataValidation): if not plugin.can_validate(context, data_dict): log.debug('Skipping validation for resource {}'.format( data_dict['id'])) return del self.resources_to_validate[resource_id] _run_async_validation(resource_id)
def after_create(self, context, resource): if not get_create_mode_from_config() == u'async': return needs_validation = False if (( # File uploaded resource.get(u'url_type') == u'upload' or # URL defined resource.get(u'url') ) and ( # Make sure format is supported resource.get(u'format', u'').lower() in settings.SUPPORTED_FORMATS )): needs_validation = True if needs_validation: _run_async_validation(resource['id'])
def after_update(self, context, data_dict): is_dataset = self._data_dict_is_dataset(data_dict) # Need to allow create as well because resource_create calls # package_update if (not get_update_mode_from_config() == u'async' and not get_create_mode_from_config() == u'async'): return if context.get('_validation_performed'): # Ugly, but needed to avoid circular loops caused by the # validation job calling resource_patch (which calls # package_update) del context['_validation_performed'] return if not is_dataset: if context.get('_dont_validate'): # Ugly, but needed to avoid circular loops caused by the # validation job calling resource_patch (which calls # package_update) del context['_dont_validate'] return # This is a resource resource_id = data_dict[u'id'] if resource_id in self.resources_to_validate: for plugin in p.PluginImplementations(IDataValidation): if not plugin.can_validate(context, data_dict): log.debug('Skipping validation for resource {}'.format( data_dict['id'])) return del self.resources_to_validate[resource_id] _run_async_validation(resource_id) if data_dict.get('validate_package'): t.get_action('resource_validation_run_batch')( context, { 'dataset_ids': data_dict.get('package_id') })
def after_create(self, context, data_dict): is_dataset = self._data_dict_is_dataset(data_dict) if not get_create_mode_from_config() == u'async': return if is_dataset: for resource in data_dict.get(u'resources', []): self._handle_validation_for_resource(context, resource) else: for plugin in p.PluginImplementations(IDataValidation): if not plugin.can_validate(context, data_dict): log.debug('Skipping validation for resource {}'.format( data_dict['id'])) return _run_async_validation(data_dict['id']) if data_dict.get('validate_package'): t.get_action('resource_validation_run_batch')( context, { 'dataset_ids': data_dict.get('package_id') })
def test_config_both_false(self): assert_equals(get_update_mode_from_config(), None) assert_equals(get_create_mode_from_config(), None)
def test_config_create_false_async(self): assert_equals(get_create_mode_from_config(), None)
def test_config_create_true_async(self): assert_equals(get_create_mode_from_config(), 'async')
def test_config_defaults(self): assert_equals(get_update_mode_from_config(), 'async') assert_equals(get_create_mode_from_config(), 'async')
def resource_create(context, data_dict): '''Appends a new resource to a datasets list of resources. This is duplicate of the CKAN core resource_create action, with just the addition of a synchronous data validation step. This is of course not ideal but it's the only way right now to hook reliably into the creation process without overcomplicating things. Hopefully future versions of CKAN will incorporate more flexible hook points that will allow a better approach. ''' model = context['model'] package_id = t.get_or_bust(data_dict, 'package_id') if not data_dict.get('url'): data_dict['url'] = '' pkg_dict = t.get_action('package_show')( dict(context, return_type='dict'), {'id': package_id}) t.check_access('resource_create', context, data_dict) for plugin in plugins.PluginImplementations(plugins.IResourceController): plugin.before_create(context, data_dict) if 'resources' not in pkg_dict: pkg_dict['resources'] = [] upload = uploader.get_resource_uploader(data_dict) if 'mimetype' not in data_dict: if hasattr(upload, 'mimetype'): data_dict['mimetype'] = upload.mimetype if 'size' not in data_dict: if hasattr(upload, 'filesize'): data_dict['size'] = upload.filesize pkg_dict['resources'].append(data_dict) try: context['defer_commit'] = True context['use_cache'] = False t.get_action('package_update')(context, pkg_dict) context.pop('defer_commit') except t.ValidationError as e: try: raise t.ValidationError(e.error_dict['resources'][-1]) except (KeyError, IndexError): raise t.ValidationError(e.error_dict) # Get out resource_id resource from model as it will not appear in # package_show until after commit resource_id = context['package'].resources[-1].id upload.upload(resource_id, uploader.get_max_resource_size()) # Custom code starts if get_create_mode_from_config() == u'sync': run_validation = True for plugin in plugins.PluginImplementations(IDataValidation): if not plugin.can_validate(context, data_dict): log.debug('Skipping validation for resource %s', resource_id) run_validation = False if run_validation: is_local_upload = ( hasattr(upload, 'filename') and upload.filename is not None and isinstance(upload, uploader.ResourceUpload)) _run_sync_validation( resource_id, local_upload=is_local_upload, new_resource=True) # Custom code ends model.repo.commit() # Run package show again to get out actual last_resource updated_pkg_dict = t.get_action('package_show')( context, {'id': package_id}) resource = updated_pkg_dict['resources'][-1] # Add the default views to the new resource t.get_action('resource_create_default_resource_views')( {'model': context['model'], 'user': context['user'], 'ignore_auth': True }, {'resource': resource, 'package': updated_pkg_dict }) for plugin in plugins.PluginImplementations(plugins.IResourceController): plugin.after_create(context, resource) return resource
def test_config_create_true_async(self): assert get_create_mode_from_config() == 'async'
def test_config_both_false(self): assert get_update_mode_from_config() is None assert get_create_mode_from_config() is None
def test_config_create_false_async(self): assert get_create_mode_from_config() is None
t.get_action('package_update')(context, pkg_dict) context.pop('defer_commit') except t.ValidationError, e: try: raise t.ValidationError(e.error_dict['resources'][-1]) except (KeyError, IndexError): raise t.ValidationError(e.error_dict) # Get out resource_id resource from model as it will not appear in # package_show until after commit resource_id = context['package'].resources[-1].id upload.upload(resource_id, uploader.get_max_resource_size()) # Custom code starts if get_create_mode_from_config() == u'sync': is_upload = (hasattr(upload, 'filename') and upload.filename is not None) _run_sync_validation(resource_id, upload=is_upload) # Custom code ends model.repo.commit() # Run package show again to get out actual last_resource updated_pkg_dict = t.get_action('package_show')(context, { 'id': package_id }) resource = updated_pkg_dict['resources'][-1] # Add the default views to the new resource
def test_config_defaults(self): assert get_update_mode_from_config() == 'async' assert get_create_mode_from_config() == 'async'