def find_translatable_fields(cls):
for dtype in cls.packages:
md_cls = class_for_metadata(dtype)
flattened_fields = md_cls.get_flattened_fields()
for kp, uf in flattened_fields.items():
if uf.queryTaggedValue('translatable'):
cls.translatable_fields[dtype].append(kp)
def after_search(self, search_results, search_params):
'''Try to replace displayed fields with their translations (if any).
'''
from ckanext.publicamundi.lib.metadata import fields, bound_field
from ckanext.publicamundi.lib.metadata import class_for_metadata, translator_for
uf = fields.TextField()
lang = self.target_language()
for pkg in search_results['results']:
source_lang = pkg.get('language')
if not source_lang or (source_lang == lang):
continue # no need to translate
dtype = pkg['dataset_type']
md = class_for_metadata(dtype)(identifier=pkg['id'])
translator = translator_for(md, source_lang)
# Lookup translations in the context of this package
translated = False
for k in ('title', 'notes'):
tr = translator.get_field_translator(bound_field(uf, (k,), pkg[k]))
yf = tr.get(lang) if tr else None
if yf:
pkg[k] = yf.context.value
translated = True
# If at least one translation was found, mark as translated
if translated:
pkg['translated_to_language'] = lang
return search_results
def find_translatable_fields(cls):
for dtype in cls.packages:
md_cls = class_for_metadata(dtype)
flattened_fields = md_cls.get_flattened_fields()
for kp, uf in flattened_fields.items():
if uf.queryTaggedValue('translatable'):
cls.translatable_fields[dtype].append(kp)
def after_search(self, search_results, search_params):
'''Try to replace displayed fields with their translations (if any).
'''
from ckanext.publicamundi.lib.metadata import fields, bound_field
from ckanext.publicamundi.lib.metadata import class_for_metadata, translator_for
uf = fields.TextField()
lang = self.target_language()
for pkg in search_results['results']:
source_lang = pkg.get('language')
if not source_lang or (source_lang == lang):
continue # no need to translate
dtype = pkg['dataset_type']
md = class_for_metadata(dtype)(identifier=pkg['id'])
translator = translator_for(md, source_lang)
# Lookup translations in the context of this package
translated = False
for k in ('title', 'notes'):
tr = translator.get_field_translator(
bound_field(uf, (k, ), pkg[k]))
yf = tr.get(lang) if tr else None
if yf:
pkg[k] = yf.context.value
translated = True
# If at least one translation was found, mark as translated
if translated:
pkg['translated_to_language'] = lang
return search_results
def postprocess_dataset_for_edit(key, data, errors, context):
assert key[0] == '__after', (
'This validator can only be invoked in the __after stage')
def debug(msg):
logger.debug('Post-processing dataset for editing: %s' % (msg))
# The state we are moving to
state = data.get(('state', ), '')
# The previous state (if exists)
pkg = context.get('package')
prev_state = pkg.state if pkg else ''
requested_with_api = 'api_version' in context
is_new = not pkg
if is_new and not requested_with_api:
return # only core metadata are expected
key_prefix = dtype = data[('dataset_type', )]
if not dtype in ext_metadata.dataset_types:
raise Invalid('Unknown dataset-type: %s' % (dtype))
# 1. Build metadata object
cls = ext_metadata.class_for_metadata(dtype)
md = cls.from_converted_data(data, for_edit=True)
if not md:
return # failed to create (in resources form ?)
data[(key_prefix, )] = md
# 2. Validate as an object
if not 'skip_validation' in context:
validation_errors = md.validate(dictize_errors=True)
# Fixme Map validation_errors to errors
#assert not validation_errors
# 3. Convert fields to extras
extras_list = data[('extras', )]
extras_list.extend(({'key': k, 'value': v} for k, v in md.to_extras()))
# 4. Compute next state
if 'skip_validation' in context:
state = data[('state', )] = 'invalid'
#data[('private',)] = True
if not state:
if prev_state == 'invalid':
state = data[('state', )] = 'active'
return
def postprocess_dataset_for_edit(key, data, errors, context):
assert key[0] == '__after', (
'This validator can only be invoked in the __after stage')
def debug(msg):
logger.debug('Post-processing dataset for editing: %s' %(msg))
# The state we are moving to
state = data.get(('state',), '')
# The previous state (if exists)
pkg = context.get('package')
prev_state = pkg.state if pkg else ''
requested_with_api = 'api_version' in context
is_new = not pkg
if is_new and not requested_with_api:
return # only core metadata are expected
key_prefix = dtype = data[('dataset_type',)]
if not dtype in ext_metadata.dataset_types:
raise Invalid('Unknown dataset-type: %s' %(dtype))
# 1. Build metadata object
cls = ext_metadata.class_for_metadata(dtype)
md = cls.from_converted_data(data, for_edit=True)
if not md:
return # failed to create (in resources form ?)
data[(key_prefix,)] = md
# 2. Validate as an object
if not 'skip_validation' in context:
validation_errors = md.validate(dictize_errors=True)
# Fixme Map validation_errors to errors
#assert not validation_errors
# 3. Convert fields to extras
extras_list = data[('extras',)]
extras_list.extend(({'key': k, 'value': v} for k, v in md.to_extras()))
# 4. Compute next state
if 'skip_validation' in context:
state = data[('state',)] = 'invalid'
#data[('private',)] = True
if not state:
if prev_state == 'invalid':
state = data[('state',)] = 'active'
return
def _check_result_for_edit(self, data, result):
key_prefix = dtype = result.get('dataset_type')
obj_cls = class_for_metadata(dtype)
keys = data.keys()
core_keys = set(keys) & self.core_keys
for key in core_keys:
assert data[key] == result[key]
if data.get('tags') and result.get('tags'):
tags = set(map(lambda t: t['name'], data['tags']))
result_tags = set(map(lambda t: t['name'], result['tags']))
assert tags == result_tags
dt_keys = filter(lambda t: t.startswith(key_prefix + '.'), keys)
# Note The input data may be in either flat or nested format
expected_obj = obj_cls()
if key_prefix in data:
# Load from nested input data
expected_obj.from_dict(data[key_prefix],
is_flat=0,
opts={'unserialize-values': 'default'})
else:
# Load from flattened input data
expected_obj.from_dict(data,
is_flat=1,
opts={
'unserialize-keys': True,
'key-prefix': key_prefix,
'unserialize-values': 'default'
})
result_obj = result[key_prefix]
# Check if expected and result objects differ. The only acceptaple
# changes are empty fields linked to (non-empty) core CKAN metadata.
linked_fields = {
'.'.join(map(str, k)): k1
for k, f, k1 in obj_cls.iter_linked_fields()
}
for change, key, (initial_value, result_value) in dictdiffer.diff(
expected_obj.to_dict(), result_obj.to_dict()):
if (key in linked_fields) and (not initial_value) and \
(result_value == result[linked_fields[key]]):
pass # is a linked field
else:
assert False, 'Unexpected change for key %s' % (key)
pass
def show_package_schema(self):
schema = super(DatasetForm, self).show_package_schema()
# Don't show vocab tags mixed in with normal 'free' tags
# (e.g. on dataset pages, or on the search page)
schema['tags']['__extras'].append(
toolkit.get_converter('free_tags_only'))
check_not_empty = toolkit.get_validator('not_empty')
ignore_missing = toolkit.get_validator('ignore_missing')
convert_from_extras = toolkit.get_converter('convert_from_extras')
schema['dataset_type'] = [convert_from_extras, check_not_empty]
# Add package field-level converters
get_field_processor = ext_validators.get_field_read_processor
for dtype in self._dataset_types:
cls1 = ext_metadata.class_for_metadata(dtype)
opts1 = {'serialize-keys': True, 'key-prefix': dtype}
for field_name, field in cls1.get_flattened_fields(
opts=opts1).items():
schema[field_name] = [
convert_from_extras, ignore_missing,
get_field_processor(field)
]
# Add before/after package-level processors
preprocess_dataset = ext_validators.preprocess_dataset_for_read
postprocess_dataset = ext_validators.postprocess_dataset_for_read
schema['__before'].insert(-1, preprocess_dataset)
if not '__after' in schema:
schema['__after'] = []
schema['__after'].append(postprocess_dataset)
# Add extra top-level fields (i.e. not under a schema)
for field_name in self._extra_fields:
schema[field_name] = [convert_from_extras, ignore_missing]
# Done, return updated schema
return schema
def show_package_schema(self):
schema = super(DatasetForm, self).show_package_schema()
# Don't show vocab tags mixed in with normal 'free' tags
# (e.g. on dataset pages, or on the search page)
schema['tags']['__extras'].append(toolkit.get_converter('free_tags_only'))
check_not_empty = toolkit.get_validator('not_empty')
ignore_missing = toolkit.get_validator('ignore_missing')
convert_from_extras = toolkit.get_converter('convert_from_extras')
schema['dataset_type'] = [convert_from_extras, check_not_empty]
# Add package field-level converters
get_field_processor = ext_validators.get_field_read_processor
for dtype in self._dataset_types:
cls1 = ext_metadata.class_for_metadata(dtype)
opts1 = {'serialize-keys': True, 'key-prefix': dtype}
for field_name, field in cls1.get_flattened_fields(opts=opts1).items():
schema[field_name] = [
convert_from_extras, ignore_missing, get_field_processor(field)]
# Add before/after package-level processors
preprocess_dataset = ext_validators.preprocess_dataset_for_read
postprocess_dataset = ext_validators.postprocess_dataset_for_read
schema['__before'].insert(-1, preprocess_dataset)
if not '__after' in schema:
schema['__after'] = []
schema['__after'].append(postprocess_dataset)
# Add extra top-level fields (i.e. not under a schema)
for field_name in self._extra_fields:
schema[field_name] = [convert_from_extras, ignore_missing]
# Done, return updated schema
return schema
def _check_result_for_edit(self, data, result):
key_prefix = dtype = result.get("dataset_type")
obj_cls = class_for_metadata(dtype)
keys = data.keys()
core_keys = set(keys) & self.core_keys
for key in core_keys:
assert data[key] == result[key]
if data.get("tags") and result.get("tags"):
tags = set(map(lambda t: t["name"], data["tags"]))
result_tags = set(map(lambda t: t["name"], result["tags"]))
assert tags == result_tags
dt_keys = filter(lambda t: t.startswith(key_prefix + "."), keys)
# Note The input data may be in either flat or nested format
expected_obj = obj_cls()
if key_prefix in data:
# Load from nested input data
expected_obj.from_dict(data[key_prefix], is_flat=0, opts={"unserialize-values": "default"})
else:
# Load from flattened input data
expected_obj.from_dict(
data,
is_flat=1,
opts={"unserialize-keys": True, "key-prefix": key_prefix, "unserialize-values": "default"},
)
result_obj = result[key_prefix]
# Check if expected and result objects differ. The only acceptaple
# changes are empty fields linked to (non-empty) core CKAN metadata.
linked_fields = {".".join(map(str, k)): k1 for k, f, k1 in obj_cls.iter_linked_fields()}
for change, key, (initial_value, result_value) in dictdiffer.diff(expected_obj.to_dict(), result_obj.to_dict()):
if (key in linked_fields) and (not initial_value) and (result_value == result[linked_fields[key]]):
pass # is a linked field
else:
assert False, "Unexpected change for key %s" % (key)
pass
def dataset_translation_update(context, data_dict):
'''Translate dataset for the active language.
The accepted format data_dict is as the one passed to core `package_update`.
An additional parameter is `translate_to_language` which determines the target
language. If not supplied, the active language (from Pylons request) will be used.
All non-translatable fields will be ignored.
All fields that are not present (or are empty) in source package, will also be
ignored.
:param id: the name or id of the package.
:type id: string
:param translate_to_language: the target language
:type translate_to_language: string
rtype: dict
'''
# Determine target language
lang = _target_language(data_dict)
# Fetch package in source language
context.update({'translate': False, 'return_json': True})
pkg = _get_action('package_show')(context, {'id': data_dict['id']})
dtype = pkg['dataset_type']
source_lang = pkg['language']
if lang == source_lang:
msg = 'The target language same as source language (%s)' % (lang)
raise Invalid({'translate_to_language': msg})
md = class_for_metadata(dtype)()
md.from_json(pkg[dtype])
# Check authorization
_check_access('package_translation_update', context,
{'org': pkg['owner_org']})
# Translate structured metadata
translator = translator_for(md, source_lang)
md = translator.translate(lang, data_dict[dtype])
pkg[dtype] = md.to_json(return_string=False)
# Translate core CKAN metadata
field_translator = translator.get_field_translator
uf = fields.TextField()
for k in ('title', 'notes'):
v = data_dict.get(k)
if not (v and pkg.get(k)):
continue # nothing to translate
tr = field_translator(bound_field(uf, (k, ), pkg[k]))
if not tr:
continue
yf = tr.translate(lang, v)
pkg[k] = v
# Return translated view of this package
pkg['translated_to_language'] = lang
return pkg
assert set(data) == {'notes'}
assert data['notes'] == foo.description
def test_deduce_fields_foo():
yield _test_deduce_fields_foo, 'foo1'
yield _test_deduce_fields_foo, 'foo2'
if __name__ == '__main__':
x = fixtures.foo1
#field1 = x.get_schema().get('contact_info')
#fc1 = x.get_field_factory(key='contact_info')
#fc2 = x.get_field_factory(field=field1)
#fc3 = x.get_field_factory('contact_info')
#fc4 = x.get_field_factory(field=field1)
#_test_schema_for_metadata('foo1')
#_test_equality('foo1')
#_test_inequality('foo1', 'foo2')
test_field_accessors_with_ifoo()
#_test_deduce_fields_foo('foo1')
from ckanext.publicamundi.lib.metadata import (class_for, class_for_object,
class_for_metadata)
cls1 = class_for_metadata('foo')
def postprocess_dataset_for_edit(key, data, errors, context):
#logger.debug('\nPOSTPROCESS EDIT START IS %s\n', data )
assert key[0] == '__after', (
'This validator can only be invoked in the __after stage')
def debug(msg):
logger.debug('Post-processing dataset for editing: %s' %(msg))
# The state we are moving to
state = data.get(('state',), '')
# The previous state (if exists)
pkg = context.get('package')
prev_state = pkg.state if pkg else ''
requested_with_api = 'api_version' in context
is_new = not pkg
#if is_new and not requested_with_api:
# return # only core metadata are expected
key_prefix = dtype = data[('dataset_type',)]
if not dtype in ext_metadata.dataset_types:
raise Invalid('Unknown dataset-type: %s' %(dtype))
#logger.debug('DATA IS [%s]' % ', '.join(map(str, data)) )
# 1. Build metadata object
cls = ext_metadata.class_for_metadata(dtype)
md = cls.from_converted_data(data, for_edit=True)
if not md:
return # failed to create (in resources form ?)
data[(key_prefix,)] = md
#logger.debug('MD IS %s', md )
# 2. Validate as an object
if not 'skip_validation' in context:
validation_errors = md.validate(dictize_errors=True)
#logger.debug("\n\n VALIDATION ERRORS IS: %s ,errors is %s, type of errors is %s \n\n", validation_errors, errors, type(validation_errors) )
#errors[('datacite.related_publication',)] = 'Missing Value'
#logger.debug("\n\n validation 4 is:: %s type is %s \n\n", validation_errors['creator'], type(validation_errors['creator']), )
# Map validation_errors to errors
for key, value in validation_errors.items():
#logger.debug("\n\n validation is %s, type is %s, value is %s, type is %s\n", key, type(key), value, type(value) )
#key = "('datacite.{}',)".format(key)
if not isinstance(value, list) :
# fix key-value for classes like Creator (contains multiple fields)
k = key + '.' + next(iter(value))
# make key compatible with errors dict (tuple)
k = tuple([str.encode("('datacite.%s',)" % k)])
v = value[next(iter(value))]
#logger.debug("\n\n key[0] is %s, value[0] is %s \n", k, v)
if v[0][0] == 'R': #RequiredMissing
errors[k] = u'Missing value'
else:
# make key compatible with errors dict (tuple)
#key = tuple([str.encode("('datacite.%s',)" % key)])
# fix error message displayed
#logger.debug("\n\n value in validation is value[0] %s, type is %s, key is %s\n", value[0], type(value[0]), key )
if value[0][:8] == 'Required':
key = tuple([str.encode("('datacite.%s',)" % key)])
errors[key] = u'Missing value'
elif value[0][:7] == 'related':
#remove duplicate error (for wrong value)
key_to_remove = tuple([str.encode('datacite.%s' % key)])
key = tuple([str.encode("('datacite.%s',)" % key)])
errors[key] = u'Invalid DOI value'
errors[key_to_remove] = []
#for k, v in errors.items():
# logger.debug("K: %s,type: %s v: %s,type %s ", k,type(k), v, type(v))
# Fixme Map validation_errors to errors ! ! ! !
#assert not validation_errors
# 3. Convert fields to extras
#logger.debug("\n MD IS: %s \n", md )
# add datacite fields after ckan extras
index = 7
for k, v in md.to_extras():
#logger.debug("\n Key is: %s, value is %s \n", k, v )
data[('extras', index, 'key')] = k
data[('extras', index, 'value')] = v
index = index + 1
# 4. Compute next state
if 'skip_validation' in context:
state = data[('state',)] = 'invalid'
#data[('private',)] = True
#add extra value manually
#data['extras', 6, 'key'] = 'datacite.funder'
#data['extras', 6, 'value'] = 'NIK'
if not state:
if prev_state == 'invalid':
state = data[('state',)] = 'active'
#logger.debug('\nPOSTPROCESS EDIT END Data IS %s', data )
return
def test_factories():
# Test with IFooMetadata
f1 = factory_for(schemata.IFooMetadata)
o1 = f1()
verifyObject(schemata.IFooMetadata, o1)
assert isinstance(o1, types.FooMetadata)
f1 = factory_for_metadata('foo')
o1 = f1()
verifyObject(schemata.IFooMetadata, o1)
assert isinstance(o1, types.FooMetadata)
c1 = class_for(schemata.IFooMetadata)
verifyObject(IIntrospective, c1, tentative=1)
o1 = c1()
verifyObject(schemata.IFooMetadata, o1)
assert c1 is types.FooMetadata
c1 = class_for_metadata('foo')
verifyObject(IIntrospective, c1, tentative=1)
o1 = c1()
verifyObject(schemata.IFooMetadata, o1)
assert c1 is types.FooMetadata
# Test with AnotherFoo
def check_foo1_after_init(o):
return (
o.baz == u'Baobab' and
o.temporal_extent is None and
isinstance(o.contact_info, types.ContactInfo))
f2 = factory_for(schemata.IFooMetadata, name='foo-1')
o2 = f2()
verifyObject(schemata.IFooMetadata, o2)
assert isinstance(o2, AnotherFoo)
assert check_foo1_after_init(o2)
f2 = factory_for_metadata('foo.1')
o2 = f2()
verifyObject(schemata.IFooMetadata, o2)
assert isinstance(o2, AnotherFoo)
assert check_foo1_after_init(o2)
c2 = class_for(schemata.IFooMetadata, 'foo-1')
verifyObject(IIntrospective, c2, tentative=1)
o2 = c2()
verifyObject(schemata.IFooMetadata, o2)
assert c2 is AnotherFoo
assert check_foo1_after_init(o2)
c2 = class_for_metadata('foo.1')
verifyObject(IIntrospective, c2, tentative=1)
o2 = c2()
verifyObject(schemata.IFooMetadata, o2)
assert c2 is AnotherFoo
assert check_foo1_after_init(o2)
# Test with non-registered names
try:
f3 = factory_for(schemata.IFooMetadata, name='a-non-existing-name')
except (ValueError, LookupError) as ex:
pass
else:
assert False, 'This should have failed'
try:
f3 = factory_for_metadata('')
except ValueError as ex:
pass
else:
assert False, 'This should have failed (a name is required!)'
try:
f3 = factory_for_metadata('foo.9')
except (ValueError, LookupError) as ex:
pass
else:
assert False, 'This should have failed'
def test_factories():
# Test with IFooMetadata
f1 = factory_for(schemata.IFooMetadata)
o1 = f1()
verifyObject(schemata.IFooMetadata, o1)
assert isinstance(o1, types.FooMetadata)
f1 = factory_for_metadata('foo')
o1 = f1()
verifyObject(schemata.IFooMetadata, o1)
assert isinstance(o1, types.FooMetadata)
c1 = class_for(schemata.IFooMetadata)
verifyObject(IIntrospective, c1, tentative=1)
o1 = c1()
verifyObject(schemata.IFooMetadata, o1)
assert c1 is types.FooMetadata
c1 = class_for_metadata('foo')
verifyObject(IIntrospective, c1, tentative=1)
o1 = c1()
verifyObject(schemata.IFooMetadata, o1)
assert c1 is types.FooMetadata
# Test with AnotherFoo
def check_foo1_after_init(o):
return (o.baz == u'Baobab' and o.temporal_extent is None
and isinstance(o.contact_info, types.ContactInfo))
f2 = factory_for(schemata.IFooMetadata, name='foo-1')
o2 = f2()
verifyObject(schemata.IFooMetadata, o2)
assert isinstance(o2, AnotherFoo)
assert check_foo1_after_init(o2)
f2 = factory_for_metadata('foo.1')
o2 = f2()
verifyObject(schemata.IFooMetadata, o2)
assert isinstance(o2, AnotherFoo)
assert check_foo1_after_init(o2)
c2 = class_for(schemata.IFooMetadata, 'foo-1')
verifyObject(IIntrospective, c2, tentative=1)
o2 = c2()
verifyObject(schemata.IFooMetadata, o2)
assert c2 is AnotherFoo
assert check_foo1_after_init(o2)
c2 = class_for_metadata('foo.1')
verifyObject(IIntrospective, c2, tentative=1)
o2 = c2()
verifyObject(schemata.IFooMetadata, o2)
assert c2 is AnotherFoo
assert check_foo1_after_init(o2)
# Test with non-registered names
try:
f3 = factory_for(schemata.IFooMetadata, name='a-non-existing-name')
except (ValueError, LookupError) as ex:
pass
else:
assert False, 'This should have failed'
try:
f3 = factory_for_metadata('')
except ValueError as ex:
pass
else:
assert False, 'This should have failed (a name is required!)'
try:
f3 = factory_for_metadata('foo.9')
except (ValueError, LookupError) as ex:
pass
else:
assert False, 'This should have failed'
def __modify_package_schema(self, schema):
'''Define modify schema for both create/update operations.
'''
check_not_empty = toolkit.get_validator('not_empty')
ignore_missing = toolkit.get_validator('ignore_missing')
ignore_empty = toolkit.get_validator('ignore_empty')
convert_to_extras = toolkit.get_converter('convert_to_extras')
default = toolkit.get_validator('default')
# Add dataset-type, the field that distinguishes metadata formats
is_dataset_type = ext_validators.is_dataset_type
schema['dataset_type'] = [
default('ckan'), convert_to_extras, is_dataset_type,
]
# Add package field-level validators/converters
# Note We provide a union of fields for all supported schemata.
# Of course, not all of them will be present in a specific dataset,
# so any "required" constraint cannot be applied here.
get_field_processor = ext_validators.get_field_edit_processor
for dtype in self._dataset_types:
cls1 = ext_metadata.class_for_metadata(dtype)
opts1 = {'serialize-keys': True, 'key-prefix': dtype}
for field_name, field in cls1.get_flattened_fields(opts=opts1).items():
# Build chain of processors for field
schema[field_name] = [
ignore_missing, get_field_processor(field)]
# Add before/after package-level processors
preprocess_dataset = ext_validators.preprocess_dataset_for_edit
postprocess_dataset = ext_validators.postprocess_dataset_for_edit
schema['__before'].insert(-1, preprocess_dataset)
if not '__after' in schema:
schema['__after'] = []
schema['__after'].append(postprocess_dataset)
# Add extra top-level fields (i.e. not bound to a schema)
for field_name in self._extra_fields:
schema[field_name] = [ignore_empty, convert_to_extras]
# Add or replace resource field-level validators/converters
guess_resource_type = ext_validators.guess_resource_type_if_empty
schema['resources'].update({
'resource_type': [
guess_resource_type, string.lower, unicode],
'format': [
check_not_empty, string.lower, unicode],
})
# Done, return updated schema
return schema
data = foo.deduce_fields('notes')
assert set(data) == {'notes'}
assert data['notes'] == foo.description
def test_deduce_fields_foo():
yield _test_deduce_fields_foo, 'foo1'
yield _test_deduce_fields_foo, 'foo2'
if __name__ == '__main__':
x = fixtures.foo1
#field1 = x.get_schema().get('contact_info')
#fc1 = x.get_field_factory(key='contact_info')
#fc2 = x.get_field_factory(field=field1)
#fc3 = x.get_field_factory('contact_info')
#fc4 = x.get_field_factory(field=field1)
#_test_schema_for_metadata('foo1')
#_test_equality('foo1')
#_test_inequality('foo1', 'foo2')
test_field_accessors_with_ifoo()
#_test_deduce_fields_foo('foo1')
from ckanext.publicamundi.lib.metadata import (
class_for, class_for_object, class_for_metadata)
cls1 = class_for_metadata('foo')
def after_show(self, context, pkg_dict, view=None):
'''Hook into the validated data dict after the package is ready for display.
The main tasks here are:
* Convert dataset_type-related parts of pkg_dict to a nested dict or an object.
This hook is for reading purposes only, i.e for template variables, api results,
form initial values etc. It should *not* affect the way the read schema is used:
schema items declared at read_package_schema() should not be removed (though their
values can be changed!).
'''
c = toolkit.c
rr = c.environ['pylons.routes_dict'] if c.environ else {}
is_validated = context.get('validate', True)
if not is_validated:
return # noop: extras are not yet promoted to 1st-level fields
for_view = context.get('for_view', False)
for_edit = ( # is this package prepared for edit ?
(rr.get('controller') == 'package' and rr.get('action') == 'edit') or
(rr.get('controller') == 'api' and rr.get('action') == 'action' and
rr.get('logic_function') in DatasetForm.after_show._api_edit_actions))
return_json = ( # do we need to return a json-friendly result ?
context.get('return_json', False) or
(rr.get('controller') == 'api' and rr.get('action') == 'action' and
rr.get('logic_function') in DatasetForm.after_show._api_actions))
log1.info(
'Package %s is shown: for-view=%s for-edit=%s api=%s',
pkg_dict.get('name'), for_view, for_edit, context.get('api_version'))
# Determine dataset_type-related parameters for this package
key_prefix = dtype = pkg_dict.get('dataset_type')
if not dtype:
return # noop: unknown dataset-type (pkg_dict has raw extras?)
# Note Do not attempt to pop() flat keys here (e.g. to replace them by a
# nested structure), because resource forms will clear all extra fields !!
# Turn to an object
md = class_for_metadata(dtype).from_converted_data(pkg_dict)
# Provide a different view, if not editing
if (not for_edit) and view and callable(view):
try:
md = view(md)
except Exception as ex:
log1.warn('Cannot build view %r for package %r: %s',
view, pkg_dict.get('name'), str(ex))
pass # noop: keep the original view
pkg_dict[key_prefix] = md
# Fix for json-friendly results (so json.dumps can handle them)
if return_json:
# Remove flat field values (won't be needed anymore)
key_prefix_1 = key_prefix + '.'
for k in (y for y in pkg_dict.keys() if y.startswith(key_prefix_1)):
pkg_dict.pop(k)
pkg_dict[key_prefix] = md.to_json(return_string=False)
return pkg_dict
def __modify_package_schema(self, schema):
'''Define modify schema for both create/update operations.
'''
check_not_empty = toolkit.get_validator('not_empty')
ignore_missing = toolkit.get_validator('ignore_missing')
ignore_empty = toolkit.get_validator('ignore_empty')
convert_to_extras = toolkit.get_converter('convert_to_extras')
default = toolkit.get_validator('default')
# Add dataset-type, the field that distinguishes metadata formats
is_dataset_type = ext_validators.is_dataset_type
schema['dataset_type'] = [
default('ckan'),
convert_to_extras,
is_dataset_type,
]
# Add package field-level validators/converters
# Note We provide a union of fields for all supported schemata.
# Of course, not all of them will be present in a specific dataset,
# so any "required" constraint cannot be applied here.
get_field_processor = ext_validators.get_field_edit_processor
for dtype in self._dataset_types:
cls1 = ext_metadata.class_for_metadata(dtype)
opts1 = {'serialize-keys': True, 'key-prefix': dtype}
for field_name, field in cls1.get_flattened_fields(
opts=opts1).items():
# Build chain of processors for field
schema[field_name] = [
ignore_missing, get_field_processor(field)
]
# Add before/after package-level processors
preprocess_dataset = ext_validators.preprocess_dataset_for_edit
postprocess_dataset = ext_validators.postprocess_dataset_for_edit
schema['__before'].insert(-1, preprocess_dataset)
if not '__after' in schema:
schema['__after'] = []
schema['__after'].append(postprocess_dataset)
# Add extra top-level fields (i.e. not bound to a schema)
for field_name in self._extra_fields:
schema[field_name] = [ignore_empty, convert_to_extras]
# Add or replace resource field-level validators/converters
guess_resource_type = ext_validators.guess_resource_type_if_empty
schema['resources'].update({
'resource_type': [guess_resource_type, string.lower, unicode],
'format': [check_not_empty, string.lower, unicode],
})
# Done, return updated schema
return schema
def dataset_translation_update(context, data_dict):
'''Translate dataset for the active language.
The accepted format data_dict is as the one passed to core `package_update`.
An additional parameter is `translate_to_language` which determines the target
language. If not supplied, the active language (from Pylons request) will be used.
All non-translatable fields will be ignored.
All fields that are not present (or are empty) in source package, will also be
ignored.
:param id: the name or id of the package.
:type id: string
:param translate_to_language: the target language
:type translate_to_language: string
rtype: dict
'''
# Determine target language
lang = _target_language(data_dict)
# Fetch package in source language
context.update({
'translate': False,
'return_json': True
})
pkg = _get_action('package_show')(context, {'id': data_dict['id']})
dtype = pkg['dataset_type']
source_lang = pkg['language']
if lang == source_lang:
msg = 'The target language same as source language (%s)' % (lang)
raise Invalid({'translate_to_language': msg})
md = class_for_metadata(dtype)()
md.from_json(pkg[dtype])
# Check authorization
_check_access(
'package_translation_update', context, {'org': pkg['owner_org']})
# Translate structured metadata
translator = translator_for(md, source_lang)
md = translator.translate(lang, data_dict[dtype])
pkg[dtype] = md.to_json(return_string=False)
# Translate core CKAN metadata
field_translator = translator.get_field_translator
uf = fields.TextField()
for k in ('title', 'notes'):
v = data_dict.get(k)
if not (v and pkg.get(k)):
continue # nothing to translate
tr = field_translator(bound_field(uf, (k,), pkg[k]))
if not tr:
continue
yf = tr.translate(lang, v)
pkg[k] = v
# Return translated view of this package
pkg['translated_to_language'] = lang
return pkg
def after_show(self, context, pkg_dict, view=None):
'''Hook into the validated data dict after the package is ready for display.
The main tasks here are:
* Convert dataset_type-related parts of pkg_dict to a nested dict or an object.
This hook is for reading purposes only, i.e for template variables, api results,
form initial values etc. It should *not* affect the way the read schema is used:
schema items declared at read_package_schema() should not be removed (though their
values can be changed!).
'''
c = toolkit.c
rr = c.environ['pylons.routes_dict'] if c.environ else {}
is_validated = context.get('validate', True)
if not is_validated:
return # noop: extras are not yet promoted to 1st-level fields
for_view = context.get('for_view', False)
for_edit = ( # is this package prepared for edit ?
(rr.get('controller') == 'package' and rr.get('action') == 'edit')
or (rr.get('controller') == 'api' and rr.get('action') == 'action'
and rr.get('logic_function')
in DatasetForm.after_show._api_edit_actions))
return_json = ( # do we need to return a json-friendly result ?
context.get('return_json', False) or
(rr.get('controller') == 'api' and rr.get('action') == 'action' and
rr.get('logic_function') in DatasetForm.after_show._api_actions))
log1.info('Package %s is shown: for-view=%s for-edit=%s api=%s',
pkg_dict.get('name'), for_view, for_edit,
context.get('api_version'))
# Determine dataset_type-related parameters for this package
key_prefix = dtype = pkg_dict.get('dataset_type')
if not dtype:
return # noop: unknown dataset-type (pkg_dict has raw extras?)
# Note Do not attempt to pop() flat keys here (e.g. to replace them by a
# nested structure), because resource forms will clear all extra fields !!
# Turn to an object
md = class_for_metadata(dtype).from_converted_data(pkg_dict)
# Provide a different view, if not editing
if (not for_edit) and view and callable(view):
try:
md = view(md)
except Exception as ex:
log1.warn('Cannot build view %r for package %r: %s', view,
pkg_dict.get('name'), str(ex))
pass # noop: keep the original view
pkg_dict[key_prefix] = md
# Fix for json-friendly results (so json.dumps can handle them)
if return_json:
# Remove flat field values (won't be needed anymore)
key_prefix_1 = key_prefix + '.'
for k in (y for y in pkg_dict.keys()
if y.startswith(key_prefix_1)):
pkg_dict.pop(k)
pkg_dict[key_prefix] = md.to_json(return_string=False)
return pkg_dict