def after_search(self, search_results, search_params): '''Try to replace displayed fields with their translations (if any). ''' from ckanext.publicamundi.lib.metadata import fields, bound_field from ckanext.publicamundi.lib.metadata import class_for_metadata, translator_for uf = fields.TextField() lang = self.target_language() for pkg in search_results['results']: source_lang = pkg.get('language') if not source_lang or (source_lang == lang): continue # no need to translate dtype = pkg['dataset_type'] md = class_for_metadata(dtype)(identifier=pkg['id']) translator = translator_for(md, source_lang) # Lookup translations in the context of this package translated = False for k in ('title', 'notes'): tr = translator.get_field_translator( bound_field(uf, (k, ), pkg[k])) yf = tr.get(lang) if tr else None if yf: pkg[k] = yf.context.value translated = True # If at least one translation was found, mark as translated if translated: pkg['translated_to_language'] = lang return search_results
def after_search(self, search_results, search_params): '''Try to replace displayed fields with their translations (if any). ''' from ckanext.publicamundi.lib.metadata import fields, bound_field from ckanext.publicamundi.lib.metadata import class_for_metadata, translator_for uf = fields.TextField() lang = self.target_language() for pkg in search_results['results']: source_lang = pkg.get('language') if not source_lang or (source_lang == lang): continue # no need to translate dtype = pkg['dataset_type'] md = class_for_metadata(dtype)(identifier=pkg['id']) translator = translator_for(md, source_lang) # Lookup translations in the context of this package translated = False for k in ('title', 'notes'): tr = translator.get_field_translator(bound_field(uf, (k,), pkg[k])) yf = tr.get(lang) if tr else None if yf: pkg[k] = yf.context.value translated = True # If at least one translation was found, mark as translated if translated: pkg['translated_to_language'] = lang return search_results
def show_metadata(self, id): '''Show dataset's metadata formatted as a table''' from ckanext.publicamundi.lib.metadata import formatter_for_field from ckanext.publicamundi.lib.metadata import fields, bound_field context = {'model': model, 'session': model.Session} try: pkg_dict = toolkit.get_action('package_show')(context, {'id': id}) except toolkit.ObjectNotFound as ex: abort(404) dtype = pkg_dict['dataset_type'] obj = pkg_dict[dtype] q = str(toolkit.request.params.get('q', '')) title_field = bound_field(fields.TextField(), 'title', pkg_dict['title']) data = { 'title': u'%s: Metadata' % (pkg_dict['title']), 'max_depth': 1, 'extras': [{ 'key': 'language', 'title': _('Source Language'), 'value': Language(pkg_dict['language']).name, }, { 'key': 'title', 'title': _('Title'), 'value': markup_for_field('read:dd%s' % (('.' + q) if q else ''), title_field, name_prefix='', data={'translatable': True}) }], } qa = 'read:table.%s' % (q) if q else 'read:table' c.markup = markup_for(qa, obj, name_prefix=dtype, data=data) return render('tests/page.html')
def show_metadata(self, id): '''Show dataset's metadata formatted as a table''' from ckanext.publicamundi.lib.metadata import formatter_for_field from ckanext.publicamundi.lib.metadata import fields, bound_field context = { 'model': model, 'session': model.Session } try: pkg_dict = toolkit.get_action('package_show')(context, { 'id': id }) except toolkit.ObjectNotFound as ex: abort(404) dtype = pkg_dict['dataset_type'] obj = pkg_dict[dtype] q = str(toolkit.request.params.get('q', '')) title_field = bound_field(fields.TextField(), 'title', pkg_dict['title']) data = { 'title': u'%s: Metadata' % (pkg_dict['title']), 'max_depth': 1, 'extras': [ { 'key': 'language', 'title': _('Source Language'), 'value': Language(pkg_dict['language']).name, }, { 'key': 'title', 'title': _('Title'), 'value': markup_for_field('read:dd%s' % (('.' + q) if q else ''), title_field, name_prefix='', data={'translatable': True}) } ], } qa = 'read:table.%s' %(q) if q else 'read:table' c.markup = markup_for(qa, obj, name_prefix=dtype, data=data) return render('tests/page.html')
def markup_for_translatable_text(key, value): uf = markup_for_translatable_text._text_field yf = bound_field(uf, key, value) qa = 'read:dd.translatable' return markup_for_field(qa, yf, name_prefix='')
def dataset_translation_update(context, data_dict): '''Translate dataset for the active language. The accepted format data_dict is as the one passed to core `package_update`. An additional parameter is `translate_to_language` which determines the target language. If not supplied, the active language (from Pylons request) will be used. All non-translatable fields will be ignored. All fields that are not present (or are empty) in source package, will also be ignored. :param id: the name or id of the package. :type id: string :param translate_to_language: the target language :type translate_to_language: string rtype: dict ''' # Determine target language lang = _target_language(data_dict) # Fetch package in source language context.update({ 'translate': False, 'return_json': True }) pkg = _get_action('package_show')(context, {'id': data_dict['id']}) dtype = pkg['dataset_type'] source_lang = pkg['language'] if lang == source_lang: msg = 'The target language same as source language (%s)' % (lang) raise Invalid({'translate_to_language': msg}) md = class_for_metadata(dtype)() md.from_json(pkg[dtype]) # Check authorization _check_access( 'package_translation_update', context, {'org': pkg['owner_org']}) # Translate structured metadata translator = translator_for(md, source_lang) md = translator.translate(lang, data_dict[dtype]) pkg[dtype] = md.to_json(return_string=False) # Translate core CKAN metadata field_translator = translator.get_field_translator uf = fields.TextField() for k in ('title', 'notes'): v = data_dict.get(k) if not (v and pkg.get(k)): continue # nothing to translate tr = field_translator(bound_field(uf, (k,), pkg[k])) if not tr: continue yf = tr.translate(lang, v) pkg[k] = v # Return translated view of this package pkg['translated_to_language'] = lang return pkg
def dataset_translation_update_field(context, data_dict): '''Translate a dataset field for the active language. This is similar to `dataset_translation_update` but only updates a field per call. It's purpose is to be used when fields are updated individually. :param id: the name or id of the package. :type id: string :param translate_to_language: the target language :type translate_to_language: string :param key: the field's key path (as dotted path or as a tuple) :type : string or tuple :param value: the translated text value :type value: string ''' # Determine target language lang = _target_language(data_dict) # Fetch package in source language context.update({'translate': False}) pkg = _get_action('package_show')(context, {'id': data_dict['id']}) dtype = pkg['dataset_type'] source_lang = pkg['language'] if lang == source_lang: msg = 'The target language same as source language (%s)' % (lang) raise Invalid({'translate_to_language': msg}) key = data_dict.get('key') if not key: raise Invalid({'key': 'Missing'}) if isinstance(key, basestring): key = tuple(key.split('.')) else: key = tuple(key) value = data_dict.get('value') if not value: raise Invalid({'value': 'Missing'}) value = unicode(value) # Check authorization _check_access( 'package_translation_update', context, {'org': pkg['owner_org']}) # Update translation for field md = pkg[dtype] translator = translator_for(md, source_lang) msg = None yf = None if len(key) < 2: # Translate a top-level field if key[0] in ['title', 'notes']: # A core CKAN translatable field uf = fields.TextField() yf = bound_field(uf, key, pkg[key[0]]) else: # Translate a field from structured metadata if key[0] == dtype: try: yf = md.get_field(key[1:]) except: yf = None msg = 'No such field' if yf and not yf.queryTaggedValue('translatable'): yf = None msg = 'Not translatable' if yf: yf.context.key = key tr = None if yf and yf.context.value: tr = translator.get_field_translator(yf) if tr: tr.translate(lang, value) res = {'updated': bool(tr)} if msg: res['message'] = msg; return res
def dataset_translation_update(context, data_dict): '''Translate dataset for the active language. The accepted format data_dict is as the one passed to core `package_update`. An additional parameter is `translate_to_language` which determines the target language. If not supplied, the active language (from Pylons request) will be used. All non-translatable fields will be ignored. All fields that are not present (or are empty) in source package, will also be ignored. :param id: the name or id of the package. :type id: string :param translate_to_language: the target language :type translate_to_language: string rtype: dict ''' # Determine target language lang = _target_language(data_dict) # Fetch package in source language context.update({'translate': False, 'return_json': True}) pkg = _get_action('package_show')(context, {'id': data_dict['id']}) dtype = pkg['dataset_type'] source_lang = pkg['language'] if lang == source_lang: msg = 'The target language same as source language (%s)' % (lang) raise Invalid({'translate_to_language': msg}) md = class_for_metadata(dtype)() md.from_json(pkg[dtype]) # Check authorization _check_access('package_translation_update', context, {'org': pkg['owner_org']}) # Translate structured metadata translator = translator_for(md, source_lang) md = translator.translate(lang, data_dict[dtype]) pkg[dtype] = md.to_json(return_string=False) # Translate core CKAN metadata field_translator = translator.get_field_translator uf = fields.TextField() for k in ('title', 'notes'): v = data_dict.get(k) if not (v and pkg.get(k)): continue # nothing to translate tr = field_translator(bound_field(uf, (k, ), pkg[k])) if not tr: continue yf = tr.translate(lang, v) pkg[k] = v # Return translated view of this package pkg['translated_to_language'] = lang return pkg
def dataset_translation_update_field(context, data_dict): '''Translate a dataset field for the active language. This is similar to `dataset_translation_update` but only updates a field per call. It's purpose is to be used when fields are updated individually. :param id: the name or id of the package. :type id: string :param translate_to_language: the target language :type translate_to_language: string :param key: the field's key path (as dotted path or as a tuple) :type : string or tuple :param value: the translated text value :type value: string ''' # Determine target language lang = _target_language(data_dict) # Fetch package in source language context.update({'translate': False}) pkg = _get_action('package_show')(context, {'id': data_dict['id']}) dtype = pkg['dataset_type'] source_lang = pkg['language'] if lang == source_lang: msg = 'The target language same as source language (%s)' % (lang) raise Invalid({'translate_to_language': msg}) key = data_dict.get('key') if not key: raise Invalid({'key': 'Missing'}) if isinstance(key, basestring): key = tuple(key.split('.')) else: key = tuple(key) value = data_dict.get('value') if not value: raise Invalid({'value': 'Missing'}) value = unicode(value) # Check authorization _check_access('package_translation_update', context, {'org': pkg['owner_org']}) # Update translation for field md = pkg[dtype] translator = translator_for(md, source_lang) msg = None yf = None if len(key) < 2: # Translate a top-level field if key[0] in ['title', 'notes']: # A core CKAN translatable field uf = fields.TextField() yf = bound_field(uf, key, pkg[key[0]]) else: # Translate a field from structured metadata if key[0] == dtype: try: yf = md.get_field(key[1:]) except: yf = None msg = 'No such field' if yf and not yf.queryTaggedValue('translatable'): yf = None msg = 'Not translatable' if yf: yf.context.key = key tr = None if yf and yf.context.value: tr = translator.get_field_translator(yf) if tr: tr.translate(lang, value) res = {'updated': bool(tr)} if msg: res['message'] = msg return res
def import_package_translation(self, opts, *args): '''Import (key-based) package translations from a CSV file. Note that the importer will only add/update translations for existing packages. The CSV input is expected to contain lines of: (package_id, source_language, language, key, value, state) ''' from ckan.plugins import toolkit from ckanext.publicamundi.lib.languages import Language from ckanext.publicamundi.lib.metadata.i18n import package_translation from ckanext.publicamundi.lib.metadata import fields, bound_field infile = args[0] if not os.access(infile, os.R_OK): raise ValueError('The input (%s) is not readable', infile) def get_package(pkg_id): context = { 'model': model, 'session': model.Session, 'ignore_auth': True, 'api_version': '3', 'validate': False, 'translate': False, } return toolkit.get_action('package_show')(context, {'id': pkg_id}) uf = fields.TextField() cnt_processed_packages, cnt_skipped_packages = 0, 0 with open(infile, 'r') as ifp: reader = csv.DictReader(ifp) for pkg_id, records in groupby(reader, itemgetter('package_id')): try: pkg = get_package(pkg_id) except toolkit.ObjectNotFound: pkg = None if not pkg: cnt_skipped_packages += 1 continue cnt_processed_packages += 1 cnt_failed_fields, cnt_updated_fields = 0, 0 for r in records: tr = package_translation.FieldTranslation( pkg_id, r['source_language']) yf = bound_field(uf, r['key'], '') try: tr.translate(yf, r['language'], r['value'].decode('utf-8')) except Exception as ex: cnt_failed_fields += 1 self.logger.warn( 'Failed to update key "%s" for package %s: %s', r['key'], pkg_id, str(ex)) else: cnt_updated_fields += 1 self.logger.info( 'Updated translations for %d fields for package: %s', cnt_updated_fields, pkg_id) self.logger.info( 'Imported translations for %d packages. Skipped %d non-existing packages', cnt_processed_packages, cnt_skipped_packages) return
def import_package_translation(self, opts, *args): '''Import (key-based) package translations from a CSV file. Note that the importer will only add/update translations for existing packages. The CSV input is expected to contain lines of: (package_id, source_language, language, key, value, state) ''' from ckan.plugins import toolkit from ckanext.publicamundi.lib.languages import Language from ckanext.publicamundi.lib.metadata.i18n import package_translation from ckanext.publicamundi.lib.metadata import fields, bound_field infile = args[0] if not os.access(infile, os.R_OK): raise ValueError('The input (%s) is not readable', infile) def get_package(pkg_id): context = { 'model': model, 'session': model.Session, 'ignore_auth': True, 'api_version': '3', 'validate': False, 'translate': False, } return toolkit.get_action('package_show')(context, {'id': pkg_id}) uf = fields.TextField() cnt_processed_packages, cnt_skipped_packages = 0, 0 with open(infile, 'r') as ifp: reader = csv.DictReader(ifp) for pkg_id, records in groupby(reader, itemgetter('package_id')): try: pkg = get_package(pkg_id) except toolkit.ObjectNotFound: pkg = None if not pkg: cnt_skipped_packages += 1 continue cnt_processed_packages += 1 cnt_failed_fields, cnt_updated_fields = 0, 0; for r in records: tr = package_translation.FieldTranslation(pkg_id, r['source_language']) yf = bound_field(uf, r['key'], '') try: tr.translate(yf, r['language'], r['value'].decode('utf-8')) except Exception as ex: cnt_failed_fields += 1 self.logger.warn('Failed to update key "%s" for package %s: %s', r['key'], pkg_id, str(ex)) else: cnt_updated_fields += 1 self.logger.info('Updated translations for %d fields for package: %s', cnt_updated_fields, pkg_id) self.logger.info( 'Imported translations for %d packages. Skipped %d non-existing packages', cnt_processed_packages, cnt_skipped_packages); return
def after_show(self, context, pkg_dict): '''Hook into the validated data dict after the package is ready for display. ''' from ckanext.publicamundi.lib.metadata import fields, bound_field try: req_params = toolkit.request.params except: req_params = {} # not a web request dtype = pkg_dict.get('dataset_type') # Determine language context source_language = pkg_dict.get('language') language = self.target_language() # Decide if a translation must take place should_translate = None if 'translate' in context: should_translate = asbool(context['translate']) elif 'translate' in req_params: should_translate = asbool(req_params['translate']) else: should_translate = bool(source_language) # Build metadata object, translate if needed translated = None if should_translate and (source_language != language): translated = self.TranslatedView(source_language, language) parent = super(MultilingualDatasetForm, self) pkg_dict = parent.after_show(context, pkg_dict, view=translated) if not pkg_dict: return # noop: super method returned prematurely md = pkg_dict[dtype] if not translated or not translated.translator: # Nothing more to do (either translation not needed or not working) return pkg_dict pkg_dict['translated_to_language'] = language # Apart from structured package metadata, try to translate: # * core (CKAN) package metadata # * core (CKAN) resource metadata uf = fields.TextField() field_translator = translated.translator.get_field_translator # Translate core package metadata for k in ('title', 'notes'): v = pkg_dict.get(k) if not v: continue # nothing to translate tr = field_translator(bound_field(uf, (k, ), v)) yf = tr.get(language) if tr else None if yf: pkg_dict[k] = yf.context.value # Translate core resource metadata # Todo return pkg_dict
def after_show(self, context, pkg_dict): '''Hook into the validated data dict after the package is ready for display. ''' from ckanext.publicamundi.lib.metadata import fields, bound_field try: req_params = toolkit.request.params except: req_params = {} # not a web request dtype = pkg_dict.get('dataset_type') # Determine language context source_language = pkg_dict.get('language') language = self.target_language() # Decide if a translation must take place should_translate = None if 'translate' in context: should_translate = asbool(context['translate']) elif 'translate' in req_params: should_translate = asbool(req_params['translate']) else: should_translate = bool(source_language) # Build metadata object, translate if needed translated = None if should_translate and (source_language != language): translated = self.TranslatedView(source_language, language) parent = super(MultilingualDatasetForm, self) pkg_dict = parent.after_show(context, pkg_dict, view=translated) if not pkg_dict: return # noop: super method returned prematurely md = pkg_dict[dtype] if not translated or not translated.translator: # Nothing more to do (either translation not needed or not working) return pkg_dict pkg_dict['translated_to_language'] = language # Apart from structured package metadata, try to translate: # * core (CKAN) package metadata # * core (CKAN) resource metadata uf = fields.TextField() field_translator = translated.translator.get_field_translator # Translate core package metadata for k in ('title', 'notes'): v = pkg_dict.get(k) if not v: continue # nothing to translate tr = field_translator(bound_field(uf, (k,), v)) yf = tr.get(language) if tr else None if yf: pkg_dict[k] = yf.context.value # Translate core resource metadata # Todo return pkg_dict