def find_translatable_fields(cls): for dtype in cls.packages: md_cls = class_for_metadata(dtype) flattened_fields = md_cls.get_flattened_fields() for kp, uf in flattened_fields.items(): if uf.queryTaggedValue('translatable'): cls.translatable_fields[dtype].append(kp)
def after_search(self, search_results, search_params): '''Try to replace displayed fields with their translations (if any). ''' from ckanext.publicamundi.lib.metadata import fields, bound_field from ckanext.publicamundi.lib.metadata import class_for_metadata, translator_for uf = fields.TextField() lang = self.target_language() for pkg in search_results['results']: source_lang = pkg.get('language') if not source_lang or (source_lang == lang): continue # no need to translate dtype = pkg['dataset_type'] md = class_for_metadata(dtype)(identifier=pkg['id']) translator = translator_for(md, source_lang) # Lookup translations in the context of this package translated = False for k in ('title', 'notes'): tr = translator.get_field_translator(bound_field(uf, (k,), pkg[k])) yf = tr.get(lang) if tr else None if yf: pkg[k] = yf.context.value translated = True # If at least one translation was found, mark as translated if translated: pkg['translated_to_language'] = lang return search_results
def after_search(self, search_results, search_params): '''Try to replace displayed fields with their translations (if any). ''' from ckanext.publicamundi.lib.metadata import fields, bound_field from ckanext.publicamundi.lib.metadata import class_for_metadata, translator_for uf = fields.TextField() lang = self.target_language() for pkg in search_results['results']: source_lang = pkg.get('language') if not source_lang or (source_lang == lang): continue # no need to translate dtype = pkg['dataset_type'] md = class_for_metadata(dtype)(identifier=pkg['id']) translator = translator_for(md, source_lang) # Lookup translations in the context of this package translated = False for k in ('title', 'notes'): tr = translator.get_field_translator( bound_field(uf, (k, ), pkg[k])) yf = tr.get(lang) if tr else None if yf: pkg[k] = yf.context.value translated = True # If at least one translation was found, mark as translated if translated: pkg['translated_to_language'] = lang return search_results
def postprocess_dataset_for_edit(key, data, errors, context): assert key[0] == '__after', ( 'This validator can only be invoked in the __after stage') def debug(msg): logger.debug('Post-processing dataset for editing: %s' % (msg)) # The state we are moving to state = data.get(('state', ), '') # The previous state (if exists) pkg = context.get('package') prev_state = pkg.state if pkg else '' requested_with_api = 'api_version' in context is_new = not pkg if is_new and not requested_with_api: return # only core metadata are expected key_prefix = dtype = data[('dataset_type', )] if not dtype in ext_metadata.dataset_types: raise Invalid('Unknown dataset-type: %s' % (dtype)) # 1. Build metadata object cls = ext_metadata.class_for_metadata(dtype) md = cls.from_converted_data(data, for_edit=True) if not md: return # failed to create (in resources form ?) data[(key_prefix, )] = md # 2. Validate as an object if not 'skip_validation' in context: validation_errors = md.validate(dictize_errors=True) # Fixme Map validation_errors to errors #assert not validation_errors # 3. Convert fields to extras extras_list = data[('extras', )] extras_list.extend(({'key': k, 'value': v} for k, v in md.to_extras())) # 4. Compute next state if 'skip_validation' in context: state = data[('state', )] = 'invalid' #data[('private',)] = True if not state: if prev_state == 'invalid': state = data[('state', )] = 'active' return
def postprocess_dataset_for_edit(key, data, errors, context): assert key[0] == '__after', ( 'This validator can only be invoked in the __after stage') def debug(msg): logger.debug('Post-processing dataset for editing: %s' %(msg)) # The state we are moving to state = data.get(('state',), '') # The previous state (if exists) pkg = context.get('package') prev_state = pkg.state if pkg else '' requested_with_api = 'api_version' in context is_new = not pkg if is_new and not requested_with_api: return # only core metadata are expected key_prefix = dtype = data[('dataset_type',)] if not dtype in ext_metadata.dataset_types: raise Invalid('Unknown dataset-type: %s' %(dtype)) # 1. Build metadata object cls = ext_metadata.class_for_metadata(dtype) md = cls.from_converted_data(data, for_edit=True) if not md: return # failed to create (in resources form ?) data[(key_prefix,)] = md # 2. Validate as an object if not 'skip_validation' in context: validation_errors = md.validate(dictize_errors=True) # Fixme Map validation_errors to errors #assert not validation_errors # 3. Convert fields to extras extras_list = data[('extras',)] extras_list.extend(({'key': k, 'value': v} for k, v in md.to_extras())) # 4. Compute next state if 'skip_validation' in context: state = data[('state',)] = 'invalid' #data[('private',)] = True if not state: if prev_state == 'invalid': state = data[('state',)] = 'active' return
def _check_result_for_edit(self, data, result): key_prefix = dtype = result.get('dataset_type') obj_cls = class_for_metadata(dtype) keys = data.keys() core_keys = set(keys) & self.core_keys for key in core_keys: assert data[key] == result[key] if data.get('tags') and result.get('tags'): tags = set(map(lambda t: t['name'], data['tags'])) result_tags = set(map(lambda t: t['name'], result['tags'])) assert tags == result_tags dt_keys = filter(lambda t: t.startswith(key_prefix + '.'), keys) # Note The input data may be in either flat or nested format expected_obj = obj_cls() if key_prefix in data: # Load from nested input data expected_obj.from_dict(data[key_prefix], is_flat=0, opts={'unserialize-values': 'default'}) else: # Load from flattened input data expected_obj.from_dict(data, is_flat=1, opts={ 'unserialize-keys': True, 'key-prefix': key_prefix, 'unserialize-values': 'default' }) result_obj = result[key_prefix] # Check if expected and result objects differ. The only acceptaple # changes are empty fields linked to (non-empty) core CKAN metadata. linked_fields = { '.'.join(map(str, k)): k1 for k, f, k1 in obj_cls.iter_linked_fields() } for change, key, (initial_value, result_value) in dictdiffer.diff( expected_obj.to_dict(), result_obj.to_dict()): if (key in linked_fields) and (not initial_value) and \ (result_value == result[linked_fields[key]]): pass # is a linked field else: assert False, 'Unexpected change for key %s' % (key) pass
def show_package_schema(self): schema = super(DatasetForm, self).show_package_schema() # Don't show vocab tags mixed in with normal 'free' tags # (e.g. on dataset pages, or on the search page) schema['tags']['__extras'].append( toolkit.get_converter('free_tags_only')) check_not_empty = toolkit.get_validator('not_empty') ignore_missing = toolkit.get_validator('ignore_missing') convert_from_extras = toolkit.get_converter('convert_from_extras') schema['dataset_type'] = [convert_from_extras, check_not_empty] # Add package field-level converters get_field_processor = ext_validators.get_field_read_processor for dtype in self._dataset_types: cls1 = ext_metadata.class_for_metadata(dtype) opts1 = {'serialize-keys': True, 'key-prefix': dtype} for field_name, field in cls1.get_flattened_fields( opts=opts1).items(): schema[field_name] = [ convert_from_extras, ignore_missing, get_field_processor(field) ] # Add before/after package-level processors preprocess_dataset = ext_validators.preprocess_dataset_for_read postprocess_dataset = ext_validators.postprocess_dataset_for_read schema['__before'].insert(-1, preprocess_dataset) if not '__after' in schema: schema['__after'] = [] schema['__after'].append(postprocess_dataset) # Add extra top-level fields (i.e. not under a schema) for field_name in self._extra_fields: schema[field_name] = [convert_from_extras, ignore_missing] # Done, return updated schema return schema
def show_package_schema(self): schema = super(DatasetForm, self).show_package_schema() # Don't show vocab tags mixed in with normal 'free' tags # (e.g. on dataset pages, or on the search page) schema['tags']['__extras'].append(toolkit.get_converter('free_tags_only')) check_not_empty = toolkit.get_validator('not_empty') ignore_missing = toolkit.get_validator('ignore_missing') convert_from_extras = toolkit.get_converter('convert_from_extras') schema['dataset_type'] = [convert_from_extras, check_not_empty] # Add package field-level converters get_field_processor = ext_validators.get_field_read_processor for dtype in self._dataset_types: cls1 = ext_metadata.class_for_metadata(dtype) opts1 = {'serialize-keys': True, 'key-prefix': dtype} for field_name, field in cls1.get_flattened_fields(opts=opts1).items(): schema[field_name] = [ convert_from_extras, ignore_missing, get_field_processor(field)] # Add before/after package-level processors preprocess_dataset = ext_validators.preprocess_dataset_for_read postprocess_dataset = ext_validators.postprocess_dataset_for_read schema['__before'].insert(-1, preprocess_dataset) if not '__after' in schema: schema['__after'] = [] schema['__after'].append(postprocess_dataset) # Add extra top-level fields (i.e. not under a schema) for field_name in self._extra_fields: schema[field_name] = [convert_from_extras, ignore_missing] # Done, return updated schema return schema
def _check_result_for_edit(self, data, result): key_prefix = dtype = result.get("dataset_type") obj_cls = class_for_metadata(dtype) keys = data.keys() core_keys = set(keys) & self.core_keys for key in core_keys: assert data[key] == result[key] if data.get("tags") and result.get("tags"): tags = set(map(lambda t: t["name"], data["tags"])) result_tags = set(map(lambda t: t["name"], result["tags"])) assert tags == result_tags dt_keys = filter(lambda t: t.startswith(key_prefix + "."), keys) # Note The input data may be in either flat or nested format expected_obj = obj_cls() if key_prefix in data: # Load from nested input data expected_obj.from_dict(data[key_prefix], is_flat=0, opts={"unserialize-values": "default"}) else: # Load from flattened input data expected_obj.from_dict( data, is_flat=1, opts={"unserialize-keys": True, "key-prefix": key_prefix, "unserialize-values": "default"}, ) result_obj = result[key_prefix] # Check if expected and result objects differ. The only acceptaple # changes are empty fields linked to (non-empty) core CKAN metadata. linked_fields = {".".join(map(str, k)): k1 for k, f, k1 in obj_cls.iter_linked_fields()} for change, key, (initial_value, result_value) in dictdiffer.diff(expected_obj.to_dict(), result_obj.to_dict()): if (key in linked_fields) and (not initial_value) and (result_value == result[linked_fields[key]]): pass # is a linked field else: assert False, "Unexpected change for key %s" % (key) pass
def dataset_translation_update(context, data_dict): '''Translate dataset for the active language. The accepted format data_dict is as the one passed to core `package_update`. An additional parameter is `translate_to_language` which determines the target language. If not supplied, the active language (from Pylons request) will be used. All non-translatable fields will be ignored. All fields that are not present (or are empty) in source package, will also be ignored. :param id: the name or id of the package. :type id: string :param translate_to_language: the target language :type translate_to_language: string rtype: dict ''' # Determine target language lang = _target_language(data_dict) # Fetch package in source language context.update({'translate': False, 'return_json': True}) pkg = _get_action('package_show')(context, {'id': data_dict['id']}) dtype = pkg['dataset_type'] source_lang = pkg['language'] if lang == source_lang: msg = 'The target language same as source language (%s)' % (lang) raise Invalid({'translate_to_language': msg}) md = class_for_metadata(dtype)() md.from_json(pkg[dtype]) # Check authorization _check_access('package_translation_update', context, {'org': pkg['owner_org']}) # Translate structured metadata translator = translator_for(md, source_lang) md = translator.translate(lang, data_dict[dtype]) pkg[dtype] = md.to_json(return_string=False) # Translate core CKAN metadata field_translator = translator.get_field_translator uf = fields.TextField() for k in ('title', 'notes'): v = data_dict.get(k) if not (v and pkg.get(k)): continue # nothing to translate tr = field_translator(bound_field(uf, (k, ), pkg[k])) if not tr: continue yf = tr.translate(lang, v) pkg[k] = v # Return translated view of this package pkg['translated_to_language'] = lang return pkg
assert set(data) == {'notes'} assert data['notes'] == foo.description def test_deduce_fields_foo(): yield _test_deduce_fields_foo, 'foo1' yield _test_deduce_fields_foo, 'foo2' if __name__ == '__main__': x = fixtures.foo1 #field1 = x.get_schema().get('contact_info') #fc1 = x.get_field_factory(key='contact_info') #fc2 = x.get_field_factory(field=field1) #fc3 = x.get_field_factory('contact_info') #fc4 = x.get_field_factory(field=field1) #_test_schema_for_metadata('foo1') #_test_equality('foo1') #_test_inequality('foo1', 'foo2') test_field_accessors_with_ifoo() #_test_deduce_fields_foo('foo1') from ckanext.publicamundi.lib.metadata import (class_for, class_for_object, class_for_metadata) cls1 = class_for_metadata('foo')
def postprocess_dataset_for_edit(key, data, errors, context): #logger.debug('\nPOSTPROCESS EDIT START IS %s\n', data ) assert key[0] == '__after', ( 'This validator can only be invoked in the __after stage') def debug(msg): logger.debug('Post-processing dataset for editing: %s' %(msg)) # The state we are moving to state = data.get(('state',), '') # The previous state (if exists) pkg = context.get('package') prev_state = pkg.state if pkg else '' requested_with_api = 'api_version' in context is_new = not pkg #if is_new and not requested_with_api: # return # only core metadata are expected key_prefix = dtype = data[('dataset_type',)] if not dtype in ext_metadata.dataset_types: raise Invalid('Unknown dataset-type: %s' %(dtype)) #logger.debug('DATA IS [%s]' % ', '.join(map(str, data)) ) # 1. Build metadata object cls = ext_metadata.class_for_metadata(dtype) md = cls.from_converted_data(data, for_edit=True) if not md: return # failed to create (in resources form ?) data[(key_prefix,)] = md #logger.debug('MD IS %s', md ) # 2. Validate as an object if not 'skip_validation' in context: validation_errors = md.validate(dictize_errors=True) #logger.debug("\n\n VALIDATION ERRORS IS: %s ,errors is %s, type of errors is %s \n\n", validation_errors, errors, type(validation_errors) ) #errors[('datacite.related_publication',)] = 'Missing Value' #logger.debug("\n\n validation 4 is:: %s type is %s \n\n", validation_errors['creator'], type(validation_errors['creator']), ) # Map validation_errors to errors for key, value in validation_errors.items(): #logger.debug("\n\n validation is %s, type is %s, value is %s, type is %s\n", key, type(key), value, type(value) ) #key = "('datacite.{}',)".format(key) if not isinstance(value, list) : # fix key-value for classes like Creator (contains multiple fields) k = key + '.' + next(iter(value)) # make key compatible with errors dict (tuple) k = tuple([str.encode("('datacite.%s',)" % k)]) v = value[next(iter(value))] #logger.debug("\n\n key[0] is %s, value[0] is %s \n", k, v) if v[0][0] == 'R': #RequiredMissing errors[k] = u'Missing value' else: # make key compatible with errors dict (tuple) #key = tuple([str.encode("('datacite.%s',)" % key)]) # fix error message displayed #logger.debug("\n\n value in validation is value[0] %s, type is %s, key is %s\n", value[0], type(value[0]), key ) if value[0][:8] == 'Required': key = tuple([str.encode("('datacite.%s',)" % key)]) errors[key] = u'Missing value' elif value[0][:7] == 'related': #remove duplicate error (for wrong value) key_to_remove = tuple([str.encode('datacite.%s' % key)]) key = tuple([str.encode("('datacite.%s',)" % key)]) errors[key] = u'Invalid DOI value' errors[key_to_remove] = [] #for k, v in errors.items(): # logger.debug("K: %s,type: %s v: %s,type %s ", k,type(k), v, type(v)) # Fixme Map validation_errors to errors ! ! ! ! #assert not validation_errors # 3. Convert fields to extras #logger.debug("\n MD IS: %s \n", md ) # add datacite fields after ckan extras index = 7 for k, v in md.to_extras(): #logger.debug("\n Key is: %s, value is %s \n", k, v ) data[('extras', index, 'key')] = k data[('extras', index, 'value')] = v index = index + 1 # 4. Compute next state if 'skip_validation' in context: state = data[('state',)] = 'invalid' #data[('private',)] = True #add extra value manually #data['extras', 6, 'key'] = 'datacite.funder' #data['extras', 6, 'value'] = 'NIK' if not state: if prev_state == 'invalid': state = data[('state',)] = 'active' #logger.debug('\nPOSTPROCESS EDIT END Data IS %s', data ) return
def test_factories(): # Test with IFooMetadata f1 = factory_for(schemata.IFooMetadata) o1 = f1() verifyObject(schemata.IFooMetadata, o1) assert isinstance(o1, types.FooMetadata) f1 = factory_for_metadata('foo') o1 = f1() verifyObject(schemata.IFooMetadata, o1) assert isinstance(o1, types.FooMetadata) c1 = class_for(schemata.IFooMetadata) verifyObject(IIntrospective, c1, tentative=1) o1 = c1() verifyObject(schemata.IFooMetadata, o1) assert c1 is types.FooMetadata c1 = class_for_metadata('foo') verifyObject(IIntrospective, c1, tentative=1) o1 = c1() verifyObject(schemata.IFooMetadata, o1) assert c1 is types.FooMetadata # Test with AnotherFoo def check_foo1_after_init(o): return ( o.baz == u'Baobab' and o.temporal_extent is None and isinstance(o.contact_info, types.ContactInfo)) f2 = factory_for(schemata.IFooMetadata, name='foo-1') o2 = f2() verifyObject(schemata.IFooMetadata, o2) assert isinstance(o2, AnotherFoo) assert check_foo1_after_init(o2) f2 = factory_for_metadata('foo.1') o2 = f2() verifyObject(schemata.IFooMetadata, o2) assert isinstance(o2, AnotherFoo) assert check_foo1_after_init(o2) c2 = class_for(schemata.IFooMetadata, 'foo-1') verifyObject(IIntrospective, c2, tentative=1) o2 = c2() verifyObject(schemata.IFooMetadata, o2) assert c2 is AnotherFoo assert check_foo1_after_init(o2) c2 = class_for_metadata('foo.1') verifyObject(IIntrospective, c2, tentative=1) o2 = c2() verifyObject(schemata.IFooMetadata, o2) assert c2 is AnotherFoo assert check_foo1_after_init(o2) # Test with non-registered names try: f3 = factory_for(schemata.IFooMetadata, name='a-non-existing-name') except (ValueError, LookupError) as ex: pass else: assert False, 'This should have failed' try: f3 = factory_for_metadata('') except ValueError as ex: pass else: assert False, 'This should have failed (a name is required!)' try: f3 = factory_for_metadata('foo.9') except (ValueError, LookupError) as ex: pass else: assert False, 'This should have failed'
def test_factories(): # Test with IFooMetadata f1 = factory_for(schemata.IFooMetadata) o1 = f1() verifyObject(schemata.IFooMetadata, o1) assert isinstance(o1, types.FooMetadata) f1 = factory_for_metadata('foo') o1 = f1() verifyObject(schemata.IFooMetadata, o1) assert isinstance(o1, types.FooMetadata) c1 = class_for(schemata.IFooMetadata) verifyObject(IIntrospective, c1, tentative=1) o1 = c1() verifyObject(schemata.IFooMetadata, o1) assert c1 is types.FooMetadata c1 = class_for_metadata('foo') verifyObject(IIntrospective, c1, tentative=1) o1 = c1() verifyObject(schemata.IFooMetadata, o1) assert c1 is types.FooMetadata # Test with AnotherFoo def check_foo1_after_init(o): return (o.baz == u'Baobab' and o.temporal_extent is None and isinstance(o.contact_info, types.ContactInfo)) f2 = factory_for(schemata.IFooMetadata, name='foo-1') o2 = f2() verifyObject(schemata.IFooMetadata, o2) assert isinstance(o2, AnotherFoo) assert check_foo1_after_init(o2) f2 = factory_for_metadata('foo.1') o2 = f2() verifyObject(schemata.IFooMetadata, o2) assert isinstance(o2, AnotherFoo) assert check_foo1_after_init(o2) c2 = class_for(schemata.IFooMetadata, 'foo-1') verifyObject(IIntrospective, c2, tentative=1) o2 = c2() verifyObject(schemata.IFooMetadata, o2) assert c2 is AnotherFoo assert check_foo1_after_init(o2) c2 = class_for_metadata('foo.1') verifyObject(IIntrospective, c2, tentative=1) o2 = c2() verifyObject(schemata.IFooMetadata, o2) assert c2 is AnotherFoo assert check_foo1_after_init(o2) # Test with non-registered names try: f3 = factory_for(schemata.IFooMetadata, name='a-non-existing-name') except (ValueError, LookupError) as ex: pass else: assert False, 'This should have failed' try: f3 = factory_for_metadata('') except ValueError as ex: pass else: assert False, 'This should have failed (a name is required!)' try: f3 = factory_for_metadata('foo.9') except (ValueError, LookupError) as ex: pass else: assert False, 'This should have failed'
def __modify_package_schema(self, schema): '''Define modify schema for both create/update operations. ''' check_not_empty = toolkit.get_validator('not_empty') ignore_missing = toolkit.get_validator('ignore_missing') ignore_empty = toolkit.get_validator('ignore_empty') convert_to_extras = toolkit.get_converter('convert_to_extras') default = toolkit.get_validator('default') # Add dataset-type, the field that distinguishes metadata formats is_dataset_type = ext_validators.is_dataset_type schema['dataset_type'] = [ default('ckan'), convert_to_extras, is_dataset_type, ] # Add package field-level validators/converters # Note We provide a union of fields for all supported schemata. # Of course, not all of them will be present in a specific dataset, # so any "required" constraint cannot be applied here. get_field_processor = ext_validators.get_field_edit_processor for dtype in self._dataset_types: cls1 = ext_metadata.class_for_metadata(dtype) opts1 = {'serialize-keys': True, 'key-prefix': dtype} for field_name, field in cls1.get_flattened_fields(opts=opts1).items(): # Build chain of processors for field schema[field_name] = [ ignore_missing, get_field_processor(field)] # Add before/after package-level processors preprocess_dataset = ext_validators.preprocess_dataset_for_edit postprocess_dataset = ext_validators.postprocess_dataset_for_edit schema['__before'].insert(-1, preprocess_dataset) if not '__after' in schema: schema['__after'] = [] schema['__after'].append(postprocess_dataset) # Add extra top-level fields (i.e. not bound to a schema) for field_name in self._extra_fields: schema[field_name] = [ignore_empty, convert_to_extras] # Add or replace resource field-level validators/converters guess_resource_type = ext_validators.guess_resource_type_if_empty schema['resources'].update({ 'resource_type': [ guess_resource_type, string.lower, unicode], 'format': [ check_not_empty, string.lower, unicode], }) # Done, return updated schema return schema
data = foo.deduce_fields('notes') assert set(data) == {'notes'} assert data['notes'] == foo.description def test_deduce_fields_foo(): yield _test_deduce_fields_foo, 'foo1' yield _test_deduce_fields_foo, 'foo2' if __name__ == '__main__': x = fixtures.foo1 #field1 = x.get_schema().get('contact_info') #fc1 = x.get_field_factory(key='contact_info') #fc2 = x.get_field_factory(field=field1) #fc3 = x.get_field_factory('contact_info') #fc4 = x.get_field_factory(field=field1) #_test_schema_for_metadata('foo1') #_test_equality('foo1') #_test_inequality('foo1', 'foo2') test_field_accessors_with_ifoo() #_test_deduce_fields_foo('foo1') from ckanext.publicamundi.lib.metadata import ( class_for, class_for_object, class_for_metadata) cls1 = class_for_metadata('foo')
def after_show(self, context, pkg_dict, view=None): '''Hook into the validated data dict after the package is ready for display. The main tasks here are: * Convert dataset_type-related parts of pkg_dict to a nested dict or an object. This hook is for reading purposes only, i.e for template variables, api results, form initial values etc. It should *not* affect the way the read schema is used: schema items declared at read_package_schema() should not be removed (though their values can be changed!). ''' c = toolkit.c rr = c.environ['pylons.routes_dict'] if c.environ else {} is_validated = context.get('validate', True) if not is_validated: return # noop: extras are not yet promoted to 1st-level fields for_view = context.get('for_view', False) for_edit = ( # is this package prepared for edit ? (rr.get('controller') == 'package' and rr.get('action') == 'edit') or (rr.get('controller') == 'api' and rr.get('action') == 'action' and rr.get('logic_function') in DatasetForm.after_show._api_edit_actions)) return_json = ( # do we need to return a json-friendly result ? context.get('return_json', False) or (rr.get('controller') == 'api' and rr.get('action') == 'action' and rr.get('logic_function') in DatasetForm.after_show._api_actions)) log1.info( 'Package %s is shown: for-view=%s for-edit=%s api=%s', pkg_dict.get('name'), for_view, for_edit, context.get('api_version')) # Determine dataset_type-related parameters for this package key_prefix = dtype = pkg_dict.get('dataset_type') if not dtype: return # noop: unknown dataset-type (pkg_dict has raw extras?) # Note Do not attempt to pop() flat keys here (e.g. to replace them by a # nested structure), because resource forms will clear all extra fields !! # Turn to an object md = class_for_metadata(dtype).from_converted_data(pkg_dict) # Provide a different view, if not editing if (not for_edit) and view and callable(view): try: md = view(md) except Exception as ex: log1.warn('Cannot build view %r for package %r: %s', view, pkg_dict.get('name'), str(ex)) pass # noop: keep the original view pkg_dict[key_prefix] = md # Fix for json-friendly results (so json.dumps can handle them) if return_json: # Remove flat field values (won't be needed anymore) key_prefix_1 = key_prefix + '.' for k in (y for y in pkg_dict.keys() if y.startswith(key_prefix_1)): pkg_dict.pop(k) pkg_dict[key_prefix] = md.to_json(return_string=False) return pkg_dict
def __modify_package_schema(self, schema): '''Define modify schema for both create/update operations. ''' check_not_empty = toolkit.get_validator('not_empty') ignore_missing = toolkit.get_validator('ignore_missing') ignore_empty = toolkit.get_validator('ignore_empty') convert_to_extras = toolkit.get_converter('convert_to_extras') default = toolkit.get_validator('default') # Add dataset-type, the field that distinguishes metadata formats is_dataset_type = ext_validators.is_dataset_type schema['dataset_type'] = [ default('ckan'), convert_to_extras, is_dataset_type, ] # Add package field-level validators/converters # Note We provide a union of fields for all supported schemata. # Of course, not all of them will be present in a specific dataset, # so any "required" constraint cannot be applied here. get_field_processor = ext_validators.get_field_edit_processor for dtype in self._dataset_types: cls1 = ext_metadata.class_for_metadata(dtype) opts1 = {'serialize-keys': True, 'key-prefix': dtype} for field_name, field in cls1.get_flattened_fields( opts=opts1).items(): # Build chain of processors for field schema[field_name] = [ ignore_missing, get_field_processor(field) ] # Add before/after package-level processors preprocess_dataset = ext_validators.preprocess_dataset_for_edit postprocess_dataset = ext_validators.postprocess_dataset_for_edit schema['__before'].insert(-1, preprocess_dataset) if not '__after' in schema: schema['__after'] = [] schema['__after'].append(postprocess_dataset) # Add extra top-level fields (i.e. not bound to a schema) for field_name in self._extra_fields: schema[field_name] = [ignore_empty, convert_to_extras] # Add or replace resource field-level validators/converters guess_resource_type = ext_validators.guess_resource_type_if_empty schema['resources'].update({ 'resource_type': [guess_resource_type, string.lower, unicode], 'format': [check_not_empty, string.lower, unicode], }) # Done, return updated schema return schema
def dataset_translation_update(context, data_dict): '''Translate dataset for the active language. The accepted format data_dict is as the one passed to core `package_update`. An additional parameter is `translate_to_language` which determines the target language. If not supplied, the active language (from Pylons request) will be used. All non-translatable fields will be ignored. All fields that are not present (or are empty) in source package, will also be ignored. :param id: the name or id of the package. :type id: string :param translate_to_language: the target language :type translate_to_language: string rtype: dict ''' # Determine target language lang = _target_language(data_dict) # Fetch package in source language context.update({ 'translate': False, 'return_json': True }) pkg = _get_action('package_show')(context, {'id': data_dict['id']}) dtype = pkg['dataset_type'] source_lang = pkg['language'] if lang == source_lang: msg = 'The target language same as source language (%s)' % (lang) raise Invalid({'translate_to_language': msg}) md = class_for_metadata(dtype)() md.from_json(pkg[dtype]) # Check authorization _check_access( 'package_translation_update', context, {'org': pkg['owner_org']}) # Translate structured metadata translator = translator_for(md, source_lang) md = translator.translate(lang, data_dict[dtype]) pkg[dtype] = md.to_json(return_string=False) # Translate core CKAN metadata field_translator = translator.get_field_translator uf = fields.TextField() for k in ('title', 'notes'): v = data_dict.get(k) if not (v and pkg.get(k)): continue # nothing to translate tr = field_translator(bound_field(uf, (k,), pkg[k])) if not tr: continue yf = tr.translate(lang, v) pkg[k] = v # Return translated view of this package pkg['translated_to_language'] = lang return pkg
def after_show(self, context, pkg_dict, view=None): '''Hook into the validated data dict after the package is ready for display. The main tasks here are: * Convert dataset_type-related parts of pkg_dict to a nested dict or an object. This hook is for reading purposes only, i.e for template variables, api results, form initial values etc. It should *not* affect the way the read schema is used: schema items declared at read_package_schema() should not be removed (though their values can be changed!). ''' c = toolkit.c rr = c.environ['pylons.routes_dict'] if c.environ else {} is_validated = context.get('validate', True) if not is_validated: return # noop: extras are not yet promoted to 1st-level fields for_view = context.get('for_view', False) for_edit = ( # is this package prepared for edit ? (rr.get('controller') == 'package' and rr.get('action') == 'edit') or (rr.get('controller') == 'api' and rr.get('action') == 'action' and rr.get('logic_function') in DatasetForm.after_show._api_edit_actions)) return_json = ( # do we need to return a json-friendly result ? context.get('return_json', False) or (rr.get('controller') == 'api' and rr.get('action') == 'action' and rr.get('logic_function') in DatasetForm.after_show._api_actions)) log1.info('Package %s is shown: for-view=%s for-edit=%s api=%s', pkg_dict.get('name'), for_view, for_edit, context.get('api_version')) # Determine dataset_type-related parameters for this package key_prefix = dtype = pkg_dict.get('dataset_type') if not dtype: return # noop: unknown dataset-type (pkg_dict has raw extras?) # Note Do not attempt to pop() flat keys here (e.g. to replace them by a # nested structure), because resource forms will clear all extra fields !! # Turn to an object md = class_for_metadata(dtype).from_converted_data(pkg_dict) # Provide a different view, if not editing if (not for_edit) and view and callable(view): try: md = view(md) except Exception as ex: log1.warn('Cannot build view %r for package %r: %s', view, pkg_dict.get('name'), str(ex)) pass # noop: keep the original view pkg_dict[key_prefix] = md # Fix for json-friendly results (so json.dumps can handle them) if return_json: # Remove flat field values (won't be needed anymore) key_prefix_1 = key_prefix + '.' for k in (y for y in pkg_dict.keys() if y.startswith(key_prefix_1)): pkg_dict.pop(k) pkg_dict[key_prefix] = md.to_json(return_string=False) return pkg_dict