def after_search(self, search_results, search_params):
'''Try to replace displayed fields with their translations (if any).
'''
from ckanext.publicamundi.lib.metadata import fields, bound_field
from ckanext.publicamundi.lib.metadata import class_for_metadata, translator_for
uf = fields.TextField()
lang = self.target_language()
for pkg in search_results['results']:
source_lang = pkg.get('language')
if not source_lang or (source_lang == lang):
continue # no need to translate
dtype = pkg['dataset_type']
md = class_for_metadata(dtype)(identifier=pkg['id'])
translator = translator_for(md, source_lang)
# Lookup translations in the context of this package
translated = False
for k in ('title', 'notes'):
tr = translator.get_field_translator(
bound_field(uf, (k, ), pkg[k]))
yf = tr.get(lang) if tr else None
if yf:
pkg[k] = yf.context.value
translated = True
# If at least one translation was found, mark as translated
if translated:
pkg['translated_to_language'] = lang
return search_results
def after_search(self, search_results, search_params):
'''Try to replace displayed fields with their translations (if any).
'''
from ckanext.publicamundi.lib.metadata import fields, bound_field
from ckanext.publicamundi.lib.metadata import class_for_metadata, translator_for
uf = fields.TextField()
lang = self.target_language()
for pkg in search_results['results']:
source_lang = pkg.get('language')
if not source_lang or (source_lang == lang):
continue # no need to translate
dtype = pkg['dataset_type']
md = class_for_metadata(dtype)(identifier=pkg['id'])
translator = translator_for(md, source_lang)
# Lookup translations in the context of this package
translated = False
for k in ('title', 'notes'):
tr = translator.get_field_translator(bound_field(uf, (k,), pkg[k]))
yf = tr.get(lang) if tr else None
if yf:
pkg[k] = yf.context.value
translated = True
# If at least one translation was found, mark as translated
if translated:
pkg['translated_to_language'] = lang
return search_results
def show_metadata(self, id):
'''Show dataset's metadata formatted as a table'''
from ckanext.publicamundi.lib.metadata import formatter_for_field
from ckanext.publicamundi.lib.metadata import fields, bound_field
context = {'model': model, 'session': model.Session}
try:
pkg_dict = toolkit.get_action('package_show')(context, {'id': id})
except toolkit.ObjectNotFound as ex:
abort(404)
dtype = pkg_dict['dataset_type']
obj = pkg_dict[dtype]
q = str(toolkit.request.params.get('q', ''))
title_field = bound_field(fields.TextField(), 'title',
pkg_dict['title'])
data = {
'title':
u'%s: Metadata' % (pkg_dict['title']),
'max_depth':
1,
'extras': [{
'key': 'language',
'title': _('Source Language'),
'value': Language(pkg_dict['language']).name,
}, {
'key':
'title',
'title':
_('Title'),
'value':
markup_for_field('read:dd%s' % (('.' + q) if q else ''),
title_field,
name_prefix='',
data={'translatable': True})
}],
}
qa = 'read:table.%s' % (q) if q else 'read:table'
c.markup = markup_for(qa, obj, name_prefix=dtype, data=data)
return render('tests/page.html')
def show_metadata(self, id):
'''Show dataset's metadata formatted as a table'''
from ckanext.publicamundi.lib.metadata import formatter_for_field
from ckanext.publicamundi.lib.metadata import fields, bound_field
context = { 'model': model, 'session': model.Session }
try:
pkg_dict = toolkit.get_action('package_show')(context, { 'id': id })
except toolkit.ObjectNotFound as ex:
abort(404)
dtype = pkg_dict['dataset_type']
obj = pkg_dict[dtype]
q = str(toolkit.request.params.get('q', ''))
title_field = bound_field(fields.TextField(), 'title', pkg_dict['title'])
data = {
'title': u'%s: Metadata' % (pkg_dict['title']),
'max_depth': 1,
'extras': [
{
'key': 'language',
'title': _('Source Language'),
'value': Language(pkg_dict['language']).name,
},
{
'key': 'title',
'title': _('Title'),
'value': markup_for_field('read:dd%s' % (('.' + q) if q else ''),
title_field, name_prefix='', data={'translatable': True})
}
],
}
qa = 'read:table.%s' %(q) if q else 'read:table'
c.markup = markup_for(qa, obj, name_prefix=dtype, data=data)
return render('tests/page.html')
def markup_for_translatable_text(key, value):
uf = markup_for_translatable_text._text_field
yf = bound_field(uf, key, value)
qa = 'read:dd.translatable'
return markup_for_field(qa, yf, name_prefix='')
def dataset_translation_update(context, data_dict):
'''Translate dataset for the active language.
The accepted format data_dict is as the one passed to core `package_update`.
An additional parameter is `translate_to_language` which determines the target
language. If not supplied, the active language (from Pylons request) will be used.
All non-translatable fields will be ignored.
All fields that are not present (or are empty) in source package, will also be
ignored.
:param id: the name or id of the package.
:type id: string
:param translate_to_language: the target language
:type translate_to_language: string
rtype: dict
'''
# Determine target language
lang = _target_language(data_dict)
# Fetch package in source language
context.update({
'translate': False,
'return_json': True
})
pkg = _get_action('package_show')(context, {'id': data_dict['id']})
dtype = pkg['dataset_type']
source_lang = pkg['language']
if lang == source_lang:
msg = 'The target language same as source language (%s)' % (lang)
raise Invalid({'translate_to_language': msg})
md = class_for_metadata(dtype)()
md.from_json(pkg[dtype])
# Check authorization
_check_access(
'package_translation_update', context, {'org': pkg['owner_org']})
# Translate structured metadata
translator = translator_for(md, source_lang)
md = translator.translate(lang, data_dict[dtype])
pkg[dtype] = md.to_json(return_string=False)
# Translate core CKAN metadata
field_translator = translator.get_field_translator
uf = fields.TextField()
for k in ('title', 'notes'):
v = data_dict.get(k)
if not (v and pkg.get(k)):
continue # nothing to translate
tr = field_translator(bound_field(uf, (k,), pkg[k]))
if not tr:
continue
yf = tr.translate(lang, v)
pkg[k] = v
# Return translated view of this package
pkg['translated_to_language'] = lang
return pkg
def dataset_translation_update_field(context, data_dict):
'''Translate a dataset field for the active language.
This is similar to `dataset_translation_update` but only updates a field per
call. It's purpose is to be used when fields are updated individually.
:param id: the name or id of the package.
:type id: string
:param translate_to_language: the target language
:type translate_to_language: string
:param key: the field's key path (as dotted path or as a tuple)
:type : string or tuple
:param value: the translated text value
:type value: string
'''
# Determine target language
lang = _target_language(data_dict)
# Fetch package in source language
context.update({'translate': False})
pkg = _get_action('package_show')(context, {'id': data_dict['id']})
dtype = pkg['dataset_type']
source_lang = pkg['language']
if lang == source_lang:
msg = 'The target language same as source language (%s)' % (lang)
raise Invalid({'translate_to_language': msg})
key = data_dict.get('key')
if not key:
raise Invalid({'key': 'Missing'})
if isinstance(key, basestring):
key = tuple(key.split('.'))
else:
key = tuple(key)
value = data_dict.get('value')
if not value:
raise Invalid({'value': 'Missing'})
value = unicode(value)
# Check authorization
_check_access(
'package_translation_update', context, {'org': pkg['owner_org']})
# Update translation for field
md = pkg[dtype]
translator = translator_for(md, source_lang)
msg = None
yf = None
if len(key) < 2:
# Translate a top-level field
if key[0] in ['title', 'notes']:
# A core CKAN translatable field
uf = fields.TextField()
yf = bound_field(uf, key, pkg[key[0]])
else:
# Translate a field from structured metadata
if key[0] == dtype:
try:
yf = md.get_field(key[1:])
except:
yf = None
msg = 'No such field'
if yf and not yf.queryTaggedValue('translatable'):
yf = None
msg = 'Not translatable'
if yf:
yf.context.key = key
tr = None
if yf and yf.context.value:
tr = translator.get_field_translator(yf)
if tr:
tr.translate(lang, value)
res = {'updated': bool(tr)}
if msg:
res['message'] = msg;
return res
def dataset_translation_update(context, data_dict):
'''Translate dataset for the active language.
The accepted format data_dict is as the one passed to core `package_update`.
An additional parameter is `translate_to_language` which determines the target
language. If not supplied, the active language (from Pylons request) will be used.
All non-translatable fields will be ignored.
All fields that are not present (or are empty) in source package, will also be
ignored.
:param id: the name or id of the package.
:type id: string
:param translate_to_language: the target language
:type translate_to_language: string
rtype: dict
'''
# Determine target language
lang = _target_language(data_dict)
# Fetch package in source language
context.update({'translate': False, 'return_json': True})
pkg = _get_action('package_show')(context, {'id': data_dict['id']})
dtype = pkg['dataset_type']
source_lang = pkg['language']
if lang == source_lang:
msg = 'The target language same as source language (%s)' % (lang)
raise Invalid({'translate_to_language': msg})
md = class_for_metadata(dtype)()
md.from_json(pkg[dtype])
# Check authorization
_check_access('package_translation_update', context,
{'org': pkg['owner_org']})
# Translate structured metadata
translator = translator_for(md, source_lang)
md = translator.translate(lang, data_dict[dtype])
pkg[dtype] = md.to_json(return_string=False)
# Translate core CKAN metadata
field_translator = translator.get_field_translator
uf = fields.TextField()
for k in ('title', 'notes'):
v = data_dict.get(k)
if not (v and pkg.get(k)):
continue # nothing to translate
tr = field_translator(bound_field(uf, (k, ), pkg[k]))
if not tr:
continue
yf = tr.translate(lang, v)
pkg[k] = v
# Return translated view of this package
pkg['translated_to_language'] = lang
return pkg
def dataset_translation_update_field(context, data_dict):
'''Translate a dataset field for the active language.
This is similar to `dataset_translation_update` but only updates a field per
call. It's purpose is to be used when fields are updated individually.
:param id: the name or id of the package.
:type id: string
:param translate_to_language: the target language
:type translate_to_language: string
:param key: the field's key path (as dotted path or as a tuple)
:type : string or tuple
:param value: the translated text value
:type value: string
'''
# Determine target language
lang = _target_language(data_dict)
# Fetch package in source language
context.update({'translate': False})
pkg = _get_action('package_show')(context, {'id': data_dict['id']})
dtype = pkg['dataset_type']
source_lang = pkg['language']
if lang == source_lang:
msg = 'The target language same as source language (%s)' % (lang)
raise Invalid({'translate_to_language': msg})
key = data_dict.get('key')
if not key:
raise Invalid({'key': 'Missing'})
if isinstance(key, basestring):
key = tuple(key.split('.'))
else:
key = tuple(key)
value = data_dict.get('value')
if not value:
raise Invalid({'value': 'Missing'})
value = unicode(value)
# Check authorization
_check_access('package_translation_update', context,
{'org': pkg['owner_org']})
# Update translation for field
md = pkg[dtype]
translator = translator_for(md, source_lang)
msg = None
yf = None
if len(key) < 2:
# Translate a top-level field
if key[0] in ['title', 'notes']:
# A core CKAN translatable field
uf = fields.TextField()
yf = bound_field(uf, key, pkg[key[0]])
else:
# Translate a field from structured metadata
if key[0] == dtype:
try:
yf = md.get_field(key[1:])
except:
yf = None
msg = 'No such field'
if yf and not yf.queryTaggedValue('translatable'):
yf = None
msg = 'Not translatable'
if yf:
yf.context.key = key
tr = None
if yf and yf.context.value:
tr = translator.get_field_translator(yf)
if tr:
tr.translate(lang, value)
res = {'updated': bool(tr)}
if msg:
res['message'] = msg
return res
def import_package_translation(self, opts, *args):
'''Import (key-based) package translations from a CSV file.
Note that the importer will only add/update translations for existing packages.
The CSV input is expected to contain lines of:
(package_id, source_language, language, key, value, state)
'''
from ckan.plugins import toolkit
from ckanext.publicamundi.lib.languages import Language
from ckanext.publicamundi.lib.metadata.i18n import package_translation
from ckanext.publicamundi.lib.metadata import fields, bound_field
infile = args[0]
if not os.access(infile, os.R_OK):
raise ValueError('The input (%s) is not readable', infile)
def get_package(pkg_id):
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'api_version': '3',
'validate': False,
'translate': False,
}
return toolkit.get_action('package_show')(context, {'id': pkg_id})
uf = fields.TextField()
cnt_processed_packages, cnt_skipped_packages = 0, 0
with open(infile, 'r') as ifp:
reader = csv.DictReader(ifp)
for pkg_id, records in groupby(reader, itemgetter('package_id')):
try:
pkg = get_package(pkg_id)
except toolkit.ObjectNotFound:
pkg = None
if not pkg:
cnt_skipped_packages += 1
continue
cnt_processed_packages += 1
cnt_failed_fields, cnt_updated_fields = 0, 0
for r in records:
tr = package_translation.FieldTranslation(
pkg_id, r['source_language'])
yf = bound_field(uf, r['key'], '')
try:
tr.translate(yf, r['language'],
r['value'].decode('utf-8'))
except Exception as ex:
cnt_failed_fields += 1
self.logger.warn(
'Failed to update key "%s" for package %s: %s',
r['key'], pkg_id, str(ex))
else:
cnt_updated_fields += 1
self.logger.info(
'Updated translations for %d fields for package: %s',
cnt_updated_fields, pkg_id)
self.logger.info(
'Imported translations for %d packages. Skipped %d non-existing packages',
cnt_processed_packages, cnt_skipped_packages)
return
def import_package_translation(self, opts, *args):
'''Import (key-based) package translations from a CSV file.
Note that the importer will only add/update translations for existing packages.
The CSV input is expected to contain lines of:
(package_id, source_language, language, key, value, state)
'''
from ckan.plugins import toolkit
from ckanext.publicamundi.lib.languages import Language
from ckanext.publicamundi.lib.metadata.i18n import package_translation
from ckanext.publicamundi.lib.metadata import fields, bound_field
infile = args[0]
if not os.access(infile, os.R_OK):
raise ValueError('The input (%s) is not readable', infile)
def get_package(pkg_id):
context = {
'model': model,
'session': model.Session,
'ignore_auth': True,
'api_version': '3',
'validate': False,
'translate': False,
}
return toolkit.get_action('package_show')(context, {'id': pkg_id})
uf = fields.TextField()
cnt_processed_packages, cnt_skipped_packages = 0, 0
with open(infile, 'r') as ifp:
reader = csv.DictReader(ifp)
for pkg_id, records in groupby(reader, itemgetter('package_id')):
try:
pkg = get_package(pkg_id)
except toolkit.ObjectNotFound:
pkg = None
if not pkg:
cnt_skipped_packages += 1
continue
cnt_processed_packages += 1
cnt_failed_fields, cnt_updated_fields = 0, 0;
for r in records:
tr = package_translation.FieldTranslation(pkg_id, r['source_language'])
yf = bound_field(uf, r['key'], '')
try:
tr.translate(yf, r['language'], r['value'].decode('utf-8'))
except Exception as ex:
cnt_failed_fields += 1
self.logger.warn('Failed to update key "%s" for package %s: %s',
r['key'], pkg_id, str(ex))
else:
cnt_updated_fields += 1
self.logger.info('Updated translations for %d fields for package: %s',
cnt_updated_fields, pkg_id)
self.logger.info(
'Imported translations for %d packages. Skipped %d non-existing packages',
cnt_processed_packages, cnt_skipped_packages);
return
def markup_for_translatable_text(key, value):
uf = markup_for_translatable_text._text_field
yf = bound_field(uf, key, value)
qa = 'read:dd.translatable'
return markup_for_field(qa, yf, name_prefix='')
def after_show(self, context, pkg_dict):
'''Hook into the validated data dict after the package is ready for display.
'''
from ckanext.publicamundi.lib.metadata import fields, bound_field
try:
req_params = toolkit.request.params
except:
req_params = {} # not a web request
dtype = pkg_dict.get('dataset_type')
# Determine language context
source_language = pkg_dict.get('language')
language = self.target_language()
# Decide if a translation must take place
should_translate = None
if 'translate' in context:
should_translate = asbool(context['translate'])
elif 'translate' in req_params:
should_translate = asbool(req_params['translate'])
else:
should_translate = bool(source_language)
# Build metadata object, translate if needed
translated = None
if should_translate and (source_language != language):
translated = self.TranslatedView(source_language, language)
parent = super(MultilingualDatasetForm, self)
pkg_dict = parent.after_show(context, pkg_dict, view=translated)
if not pkg_dict:
return # noop: super method returned prematurely
md = pkg_dict[dtype]
if not translated or not translated.translator:
# Nothing more to do (either translation not needed or not working)
return pkg_dict
pkg_dict['translated_to_language'] = language
# Apart from structured package metadata, try to translate:
# * core (CKAN) package metadata
# * core (CKAN) resource metadata
uf = fields.TextField()
field_translator = translated.translator.get_field_translator
# Translate core package metadata
for k in ('title', 'notes'):
v = pkg_dict.get(k)
if not v:
continue # nothing to translate
tr = field_translator(bound_field(uf, (k, ), v))
yf = tr.get(language) if tr else None
if yf:
pkg_dict[k] = yf.context.value
# Translate core resource metadata
# Todo
return pkg_dict
def after_show(self, context, pkg_dict):
'''Hook into the validated data dict after the package is ready for display.
'''
from ckanext.publicamundi.lib.metadata import fields, bound_field
try:
req_params = toolkit.request.params
except:
req_params = {} # not a web request
dtype = pkg_dict.get('dataset_type')
# Determine language context
source_language = pkg_dict.get('language')
language = self.target_language()
# Decide if a translation must take place
should_translate = None
if 'translate' in context:
should_translate = asbool(context['translate'])
elif 'translate' in req_params:
should_translate = asbool(req_params['translate'])
else:
should_translate = bool(source_language)
# Build metadata object, translate if needed
translated = None
if should_translate and (source_language != language):
translated = self.TranslatedView(source_language, language)
parent = super(MultilingualDatasetForm, self)
pkg_dict = parent.after_show(context, pkg_dict, view=translated)
if not pkg_dict:
return # noop: super method returned prematurely
md = pkg_dict[dtype]
if not translated or not translated.translator:
# Nothing more to do (either translation not needed or not working)
return pkg_dict
pkg_dict['translated_to_language'] = language
# Apart from structured package metadata, try to translate:
# * core (CKAN) package metadata
# * core (CKAN) resource metadata
uf = fields.TextField()
field_translator = translated.translator.get_field_translator
# Translate core package metadata
for k in ('title', 'notes'):
v = pkg_dict.get(k)
if not v:
continue # nothing to translate
tr = field_translator(bound_field(uf, (k,), v))
yf = tr.get(language) if tr else None
if yf:
pkg_dict[k] = yf.context.value
# Translate core resource metadata
# Todo
return pkg_dict
