def save_extra_package_multilang(pkg, lang, field_type):
try:
from ckanext.multilang.model import PackageMultilang
except ImportError:
log.warn('DCAT-AP_IT: multilang extension not available.')
return
log.debug('Creating create_loc_field for package ID: %r', str(pkg.get('id')))
PackageMultilang.persist(pkg, lang, field_type)
log.info('Localized field created successfully')
def persist_package_multilang_field(self, package_id, field_name, text, locale, field_type):
record = PackageMultilang.get(package_id, field_name, locale, field_type)
if record:
log.info('::::::::: Updating the localized {0} package field in the package_multilang table :::::::::'.format(field_name))
record.text = text
record.save()
log.info('::::::::: PACKAGE MULTILANG FIELD UPDATED SUCCESSFULLY :::::::::')
else:
log.info('::::::::: Adding new localized {0} package field in the package_multilang table :::::::::'.format(field_name))
PackageMultilang.persist({'id': package_id, 'text': text, 'field': field_name}, locale, field_type)
log.info('::::::::: PACKAGE MULTILANG FIELD PERSISTED SUCCESSFULLY :::::::::')
def upsert_package_multilang(pkg_id, field_name, field_type, lang, text):
try:
from ckanext.multilang.model import PackageMultilang
except ImportError:
log.warn('DCAT-AP_IT: multilang extension not available.')
return
pml = PackageMultilang.get(pkg_id, field_name, lang, field_type)
if not pml and text:
PackageMultilang.persist({'id':pkg_id, 'field':field_name, 'text':text}, lang, field_type)
elif pml and not text:
pml.purge()
elif pml and not pml.text == text:
pml.text = text
pml.save()
def update_extra_package_multilang(extra,
pkg_id,
field,
lang,
field_type='extra'):
try:
from ckanext.multilang.model import PackageMultilang
except ImportError:
log.warn('DCAT-AP_IT: multilang extension not available.')
return
if extra.get('key') == field.get('name', None) and field.get(
'localized', False) == True:
log.debug(':::::::::::::::Localizing schema field: %r', field['name'])
f = PackageMultilang.get(pkg_id, field['name'], lang, field_type)
if f:
if extra.get('value') == '':
f.purge()
elif f.text != extra.get('value'):
# Update the localized field value for the current language
f.text = extra.get('value')
f.save()
log.info('Localized field updated successfully')
elif extra.get('value') != '':
# Create the localized field record
save_extra_package_multilang(
{
'id': pkg_id,
'text': extra.get('value'),
'field': extra.get('key')
}, lang, 'extra')
def before_index(self, pkg_dict):
from ckanext.multilang.model import PackageMultilang
multilang_localized = PackageMultilang.get_for_package(pkg_dict['id'])
for package in multilang_localized:
log.debug('...Creating index for Package localized field: ' + package.field + ' - ' + package.lang)
pkg_dict['package_multilang_localized_' + package.field + '_' + package.lang] = package.text
log.debug('Index successfully created for Package: %r', pkg_dict.get('package_multilang_localized_' + package.field + '_' + package.lang))
'''from ckanext.multilang.model import GroupMultilang
multilang_localized = GroupMultilang.get_for_group_name(str(pkg_dict['organization']))
for organization in multilang_localized:
log.debug('...Creating index for Organization localized field: ' + organization.field + ' - ' + organization.lang)
pkg_dict['organization_multilang_localized_' + organization.field + '_' + organization.lang] = organization.text
log.debug('Index successfully created for Organization: %r', pkg_dict.get('organization_multilang_localized_' + organization.field + '_' + organization.lang))
for group in pkg_dict['groups']:
multilang_localized = GroupMultilang.get_for_group_name(str(group))
for record in multilang_localized:
log.debug('...Creating index for Group localized field: ' + organization.field + ' - ' + organization.lang)
pkg_dict['group_multilang_localized_' + record.field + '_' + record.lang] = record.text
log.debug('Index successfully created for Group: %r', pkg_dict.get('group_multilang_localized_' + organization.field + '_' + organization.lang))'''
return pkg_dict
def get_localized_pkg(pkg_dict):
if pkg_dict != '' and 'type' in pkg_dict:
# MULTILANG - Localizing package dict
lang = getLanguage()
# MULTILANG - Localizing Tags display names in Facet list
tags = pkg_dict.get('tags')
if tags:
for tag in tags:
localized_tag = TagMultilang.by_name(tag.get('name'), lang)
if localized_tag:
tag['display_name'] = localized_tag.text
# q_results = model.Session.query(PackageMultilang).filter(PackageMultilang.package_id == pkg_dict.get('id'), PackageMultilang.lang == lang).all()
q_results = PackageMultilang.get_for_package_id_and_lang(pkg_dict.get('id'), lang)
if q_results:
for result in q_results:
pkg_dict[result.field] = result.text
# MULTILANG - Localizing organization sub dict for the dataset edit page
organization = pkg_dict.get('organization')
if organization:
# q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.group_id == organization.get('id'), GroupMultilang.lang == lang).all()
q_results = GroupMultilang.get_for_group_id_and_lang(organization.get('id'), lang)
if q_results:
for result in q_results:
organization[result.field] = result.text
pkg_dict['organization'] = organization
return pkg_dict
def before_index(self, pkg_dict):
from ckanext.multilang.model import PackageMultilang
multilang_localized = PackageMultilang.get_for_package(pkg_dict['id'])
for package in multilang_localized:
log.debug('...Creating index for Package localized field: ' + package.field + ' - ' + package.lang)
pkg_dict['package_multilang_localized_' + package.field + '_' + package.lang] = package.text
log.debug('Index successfully created for Package: %r', pkg_dict.get('package_multilang_localized_' + package.field + '_' + package.lang))
return pkg_dict
def get_for_package(pkg_id):
'''
Returns all the localized fields of a dataset, in a dict of dicts, i.e.:
{FIELDNAME:{LANG:label,...},...}
Returns None if multilang extension not loaded.
'''
try:
from ckanext.multilang.model import PackageMultilang
except ImportError:
log.warn('DCAT-AP_IT: multilang extension not available.')
# TODO: if no multilang, return the dataset in a single language in the same format of the multilang data
return None
records = PackageMultilang.get_for_package(pkg_id)
return _multilang_to_dict(records)
def get_localized_field_value(field=None, pkg_id=None, field_type='extra'):
try:
from ckanext.multilang.model import PackageMultilang
except ImportError:
log.warn('DCAT-AP_IT: multilang extension not available.')
return
if field and pkg_id:
lang = get_language()
if lang:
localized_value = PackageMultilang.get(pkg_id, field, lang, field_type)
if localized_value:
return localized_value.text
else:
return None
else:
return None
else:
return None
def after_import_stage(self, package_dict):
log.info(
'::::::::: Performing after_import_stage persist operation for localised dataset content :::::::::'
)
if bool(self._package_dict):
session = Session
package_id = package_dict.get('id')
# Persisting localized packages
try:
# rows = session.query(PackageMultilang).filter(PackageMultilang.package_id == package_id).all()
rows = PackageMultilang.get_for_package(package_id)
if not rows:
log.info(
'::::::::: Adding new localised object to the package_multilang table :::::::::'
)
log.debug('::::: Persisting default metadata locale :::::')
loc_titles = self._package_dict.get('localised_titles')
if loc_titles:
log.debug('::::: Persisting title locales :::::')
for title in loc_titles:
PackageMultilang.persist(
{
'id': package_id,
'text': title.get('text'),
'field': 'title'
}, title.get('locale'))
loc_abstracts = self._package_dict.get(
'localised_abstracts')
if loc_abstracts:
log.debug('::::: Persisting abstract locales :::::')
for abstract in loc_abstracts:
PackageMultilang.persist(
{
'id': package_id,
'text': abstract.get('text'),
'field': 'notes'
}, abstract.get('locale'))
log.info(
'::::::::: OBJECT PERSISTED SUCCESSFULLY :::::::::')
else:
log.info(
'::::::::: Updating localised object in the package_multilang table :::::::::'
)
for row in rows:
if row.field == 'title':
titles = self._package_dict.get('localised_titles')
if titles:
for title in titles:
if title.get('locale') == row.lang:
row.text = title.get('text')
elif row.field == 'notes':
abstracts = self._package_dict.get(
'localised_abstracts')
if abstracts:
for abstract in abstracts:
if abstract.get('locale') == row.lang:
row.text = abstract.get('text')
row.save()
log.info('::::::::: OBJECT UPDATED SUCCESSFULLY :::::::::')
pass
except Exception, e:
# on rollback, the same closure of state
# as that of commit proceeds.
session.rollback()
log.error('Exception occurred while persisting DB objects: %s',
e)
raise
# Persisting localized Tags
loc_tags = self._package_dict.get('localized_tags')
if loc_tags:
log.debug('::::: Persisting tag locales :::::')
for tag in loc_tags:
tag_name = tag.get('text')
tag_lang = tag.get('locale')
tag_localized_name = tag.get('localized_text')
tag = TagMultilang.by_name(tag_name, tag_lang)
if tag:
# Update the existing record
if tag_localized_name and tag_localized_name != tag.text:
tag.text = tag_localized_name
try:
tag.save()
log.info(
'::::::::: OBJECT TAG UPDATED SUCCESSFULLY :::::::::'
)
pass
except Exception, e:
# on rollback, the same closure of state
# as that of commit proceeds.
session.rollback()
log.error(
'Exception occurred while persisting DB objects: %s',
e)
raise
else:
# Create a new localized record
existing_tag = model.Tag.by_name(tag_name)
if existing_tag:
TagMultilang.persist(
{
'id': existing_tag.id,
'name': tag_name,
'text': tag_localized_name
}, tag_lang)
log.info(
'::::::::: OBJECT TAG PERSISTED SUCCESSFULLY :::::::::'
)
def after_import_stage(self, package_dict):
for cls in PBZHarvester.__bases__:
c = cls()
if hasattr(c, '_package_dict'):
c._package_dict = self._package_dict
c.after_import_stage(package_dict)
package_id = package_dict.get('id')
log.debug(
"::::::::::::: Persisting localized Author package field :::::::::::::"
)
try:
## PERSISTING the 'author' standard field localized
## ------------------------------------------------
session = Session
rows = session.query(PackageMultilang).filter(
PackageMultilang.package_id == package_id,
PackageMultilang.field == 'author').all()
if not rows:
log.info(
'::::::::: Adding new localized organization author in the package_multilang table :::::::::'
)
log.debug(
'::::: Persisting organization author metadata locale :::::'
)
for org in self.localized_org:
session.add_all([
PackageMultilang(package_id=package_id,
field='author',
field_type='localized',
lang=org.get('locale'),
text=org.get('text')),
])
session.commit()
log.info('::::::::: OBJECT PERSISTED SUCCESSFULLY :::::::::')
else:
log.info(
'::::::::: Updating localized organization author in the package_multilang table :::::::::'
)
for row in rows:
if row.field == 'author':
for org in self.localized_org:
if org.get('locale') == row.lang:
row.text = org.get('text')
row.save()
log.info('::::::::: OBJECT UPDATED SUCCESSFULLY :::::::::')
## PERSISTING the 'holder' custom field localized
## ----------------------------------------------
for org in self.localized_org:
record = custom.get_field('holder', package_id,
org.get('locale'))
if record:
log.info(
'::::::::: Updating the localized holder custom field in the custom_field table :::::::::'
)
record.text = org.get('text')
record.save()
log.info(
'::::::::: CUSTOM OBJECT UPDATED SUCCESSFULLY :::::::::'
)
else:
log.info(
'::::::::: Adding new localized holder custom field in the custom_field table :::::::::'
)
# This for the holder custom field
new_holder_loc_field = custom.CustomFieldMultilang(
package_id, 'holder', org.get('locale'),
org.get('text'))
custom.CustomFieldMultilang.save(new_holder_loc_field)
log.info(
'::::::::: CUSTOM OBJECT PERSISTED SUCCESSFULLY :::::::::'
)
pass
except Exception, e:
# on rollback, the same closure of state
# as that of commit proceeds.
session.rollback()
log.error('Exception occurred while persisting DB objects: %s', e)
raise
def _save_new(self, context, package_type=None):
# The staged add dataset used the new functionality when the dataset is
# partially created so we need to know if we actually are updating or
# this is a real new.
is_an_update = False
ckan_phase = request.params.get('_ckan_phase')
from ckan.lib.search import SearchIndexError
try:
data_dict = clean_dict(
dict_fns.unflatten(tuplize_dict(parse_params(request.POST))))
if ckan_phase:
# prevent clearing of groups etc
context['allow_partial_update'] = True
# sort the tags
if 'tag_string' in data_dict:
data_dict['tags'] = self._tag_string_to_list(
data_dict['tag_string'])
if data_dict.get('pkg_name'):
is_an_update = True
# This is actually an update not a save
data_dict['id'] = data_dict['pkg_name']
del data_dict['pkg_name']
# don't change the dataset state
data_dict['state'] = 'draft'
# this is actually an edit not a save
pkg_dict = get_action('package_update')(context, data_dict)
if request.params['save'] == 'go-metadata':
# redirect to add metadata
url = h.url_for(controller='package',
action='new_metadata',
id=pkg_dict['name'])
else:
# redirect to add dataset resources
url = h.url_for(controller='package',
action='new_resource',
id=pkg_dict['name'])
redirect(url)
# Make sure we don't index this dataset
if request.params['save'] not in [
'go-resource', 'go-metadata'
]:
data_dict['state'] = 'draft'
# allow the state to be changed
context['allow_state_change'] = True
data_dict['type'] = package_type
context['message'] = data_dict.get('log_message', '')
# MULTILANG - retrieving dict for localized tag's strings
extra_tag = None
if data_dict.get('extra_tag'):
extra_tag = data_dict.get('extra_tag')
# After saving in memory the extra_tag dict this must be removed because not present in the schema
del data_dict['extra_tag']
pkg_dict = get_action('package_create')(context, data_dict)
lang = helpers.getLanguage()
# MULTILANG - persisting tags
self.localized_tags_persist(extra_tag, pkg_dict, lang)
# MULTILANG - persisting the localized package dict
log.info('::::: Persisting localised metadata locale :::::')
for field in self.pkg_localized_fields:
if pkg_dict.get(field):
PackageMultilang.persist(
{
'id': pkg_dict.get('id'),
'field': field,
'text': pkg_dict.get(field)
}, lang)
if ckan_phase:
# redirect to add dataset resources
url = h.url_for(controller='package',
action='new_resource',
id=pkg_dict['name'])
redirect(url)
self._form_save_redirect(pkg_dict['name'],
'new',
package_type=package_type)
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % '')
except NotFound, e:
abort(404, _('Dataset not found'))
def _save_edit(self, name_or_id, context, package_type=None):
from ckan.lib.search import SearchIndexError
log.debug('Package save request name: %s POST: %r', name_or_id,
request.POST)
try:
data_dict = clean_dict(
dict_fns.unflatten(tuplize_dict(parse_params(request.POST))))
if '_ckan_phase' in data_dict:
# we allow partial updates to not destroy existing resources
context['allow_partial_update'] = True
if 'tag_string' in data_dict:
data_dict['tags'] = self._tag_string_to_list(
data_dict['tag_string'])
del data_dict['_ckan_phase']
del data_dict['save']
context['message'] = data_dict.get('log_message', '')
data_dict['id'] = name_or_id
# MULTILANG - retrieving dict for localized tag's strings
extra_tag = None
if data_dict.get('extra_tag'):
extra_tag = data_dict.get('extra_tag')
# After saving in memory the extra_tag dict this must be removed because not present in the schema
del data_dict['extra_tag']
pkg = get_action('package_update')(context, data_dict)
c.pkg = context['package']
c.pkg_dict = pkg
lang = helpers.getLanguage()
# MULTILANG - persisting tags
self.localized_tags_persist(extra_tag, c.pkg_dict, lang)
# MULTILANG - persisting package dict
log.info(
':::::::::::: Saving the corresponding localized title and abstract :::::::::::::::'
)
# q_results = model.Session.query(PackageMultilang).filter(PackageMultilang.package_id == c.pkg_dict.get('id'), PackageMultilang.lang == lang).all()
q_results = PackageMultilang.get_for_package_id_and_lang(
c.pkg_dict.get('id'), lang)
if q_results:
pkg_processed_field = []
for result in q_results:
# do not update multilang field if original pkg dict doesn't
# have this field anymore. otherwise IntegrityError will raise
# because text will be None
if c.pkg_dict.has_key(result.field):
pkg_processed_field.append(result.field)
log.debug('::::::::::::::: value before %r',
result.text)
result.text = c.pkg_dict.get(result.field)
log.debug('::::::::::::::: value after %r',
result.text)
result.save()
# Check for missing localized fields in DB
for field in self.pkg_localized_fields:
if field not in pkg_processed_field:
if c.pkg_dict.get(field):
PackageMultilang.persist(
{
'id': c.pkg_dict.get('id'),
'field': field,
'text': c.pkg_dict.get(field)
}, lang)
else:
log.info(
':::::::::::: Localised fields are missing in package_multilang table, persisting defaults using values in the table package :::::::::::::::'
)
for field in self.pkg_localized_fields:
if c.pkg_dict.get(field):
PackageMultilang.persist(
{
'id': c.pkg_dict.get('id'),
'field': field,
'text': c.pkg_dict.get(field)
}, lang)
self._form_save_redirect(pkg['name'],
'edit',
package_type=package_type)
except NotAuthorized:
abort(401, _('Unauthorized to read package %s') % id)
except NotFound, e:
abort(404, _('Dataset not found'))
def before_view(self, odict):
otype = odict.get('type')
lang = helpers.getLanguage()
if lang:
if otype == 'group' or otype == 'organization':
# MULTILANG - Localizing Group/Organizzation names and descriptions in search list
# q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.group_id == odict.get('id'), GroupMultilang.lang == lang).all()
q_results = GroupMultilang.get_for_group_id_and_lang(
odict.get('id'), lang)
if q_results:
for result in q_results:
odict[result.field] = result.text
if result.field == 'title':
odict['display_name'] = result.text
elif otype == 'dataset':
# MULTILANG - Localizing Datasets names and descriptions in search list
# MULTILANG - Localizing Tags display names in Facet list
tags = odict.get('tags')
if tags:
for tag in tags:
localized_tag = TagMultilang.by_tag_id(
tag.get('id'), lang)
if localized_tag:
tag['display_name'] = localized_tag.text
# MULTILANG - Localizing package sub dict for the dataset read page
# q_results = model.Session.query(PackageMultilang).filter(PackageMultilang.package_id == odict['id'], PackageMultilang.lang == lang).all()
q_results = PackageMultilang.get_for_package_id_and_lang(
odict.get('id'), lang)
if q_results:
for result in q_results:
if odict.get(result.field, None):
odict[result.field] = result.text
else:
extras = odict.get('extras', None)
if extras and len(extras) > 0:
for extra in extras:
extra_key = extra.get('key', None)
if extra_key and extra_key == result.field:
extra['value'] = result.text
#if result.field == 'notes':
# odict['notes'] = result.text
# MULTILANG - Localizing organization sub dict for the dataset read page
organization = odict.get('organization')
if organization:
# q_results = model.Session.query(GroupMultilang).filter(GroupMultilang.group_id == organization.get('id'), GroupMultilang.lang == lang).all()
q_results = GroupMultilang.get_for_group_id_and_lang(
organization.get('id'), lang)
if q_results:
for result in q_results:
organization[result.field] = result.text
odict['organization'] = organization
# MULTILANG - Localizing resources dict
resources = odict.get('resources')
if resources:
for resource in resources:
# q_results = model.Session.query(ResourceMultilang).filter(ResourceMultilang.resource_id == resource.get('id'), ResourceMultilang.lang == lang).all()
q_results = ResourceMultilang.get_for_resource_id_and_lang(
resource.get('id'), lang)
if q_results:
for result in q_results:
resource[result.field] = result.text
return odict