def _modify_package_schema(self, schema):
##
# Getting custom package schema
##
for field in dcatapit_schema.get_custom_package_schema():
if 'ignore' in field and field['ignore'] == True:
continue
if 'couples' in field:
for couple in field['couples']:
self.update_schema_field(schema, couple)
else:
self.update_schema_field(schema, field)
schema.update({'notes': [toolkit.get_validator('not_empty')]})
##
# Getting custom resource schema
##
for field in dcatapit_schema.get_custom_resource_schema():
if 'ignore' in field and field['ignore'] == True:
continue
validators = []
for validator in field['validator']:
validators.append(toolkit.get_validator(validator))
schema['resources'].update({field['name']: validators})
log.debug("Schema updated for DCAT_AP-TI: %r", schema)
return schema
def after_create(self, context, pkg_dict):
# During the harvest the get_lang() is not defined
lang = interfaces.get_language()
otype = pkg_dict.get('type')
if lang and otype == 'dataset':
for extra in pkg_dict.get('extras'):
for field in dcatapit_schema.get_custom_package_schema():
couples = field.get('couples', [])
if couples and len(couples) > 0:
for couple in couples:
if extra.get('key') == couple.get(
'name', None) and couple.get(
'localized', False) == True:
log.debug(
':::::::::::::::Localizing custom schema field: %r',
couple['name'])
# Create the localized field record
self.create_loc_field(extra, lang,
pkg_dict.get('id'))
else:
if extra.get('key') == field.get(
'name', None) and field.get(
'localized', False) == True:
log.debug(
':::::::::::::::Localizing custom schema field: %r',
field['name'])
# Create the localized field record
self.create_loc_field(extra, lang,
pkg_dict.get('id'))
def after_show(self, context, pkg_dict):
schema = dcatapit_schema.get_custom_package_schema()
# quick hack on date fields that are in wrong format
for fdef in schema:
if fdef.get('type') != 'date':
continue
fname = fdef['name']
df_value = pkg_dict.get(fname)
if df_value:
tmp_value = validators.parse_date(df_value, df_value)
if isinstance(tmp_value, datetime.date):
try:
tmp_value = tmp_value.strftime(
fdef.get('format') or '%d-%m-%Y')
except ValueError, err:
log.warning(
"dataset %s, field %s: cannot reformat date for %s (from input %s): %s",
pkg_dict['name'],
fname,
tmp_value,
df_value,
err,
exc_info=err)
tmp_value = df_value
pkg_dict[fname] = tmp_value
def after_search(self, search_results, search_params):
## #####################################################################
# This method moves the dcatapit fields into the extras array (needed for
# the CKAN harvester).
# Basically it dynamically reverts what is done by the
# 'convert_from_extras' to allow harvesting this plugin's custom fields.
## #####################################################################
search_dicts = search_results.get('results', [])
dcatapit_schema_fields = dcatapit_schema.get_custom_package_schema()
for _dict in search_dicts:
_dict_extras = _dict.get('extras', None)
if not _dict_extras:
_dict_extras = []
_dict['extras'] = _dict_extras
for field in dcatapit_schema_fields:
field_couple = field.get('couples', [])
if len(field_couple) > 0:
for couple in field_couple:
self.manage_extras_for_search(couple, _dict, _dict_extras)
else:
self.manage_extras_for_search(field, _dict, _dict_extras)
return search_results
def after_update(self, context, pkg_dict):
# During the harvest the get_lang() is not defined
lang = interfaces.get_language()
otype = pkg_dict.get('type')
if lang and otype == 'dataset':
for extra in pkg_dict.get('extras'):
for field in dcatapit_schema.get_custom_package_schema():
couples = field.get('couples', [])
if couples and len(couples) > 0:
for couple in couples:
self.update_loc_field(extra, pkg_dict.get('id'), couple, lang)
else:
self.update_loc_field(extra, pkg_dict.get('id'), field, lang)
def show_package_schema(self):
schema = super(DCATAPITPackagePlugin, self).show_package_schema()
##
# Getting custom package schema
##
for field in dcatapit_schema.get_custom_package_schema():
if 'ignore' in field and field['ignore'] == True:
continue
if 'couples' in field:
for couple in field['couples']:
self.update_show_schema_field(schema, couple)
else:
self.update_show_schema_field(schema, field)
schema.update({
'notes': [
toolkit.get_validator('not_empty')
]
})
##
# Getting custom resource schema
##
for field in dcatapit_schema.get_custom_resource_schema():
if 'ignore' in field and field['ignore'] == True:
continue
validators = []
for validator in field['validator']:
validators.append(toolkit.get_validator(validator))
schema['resources'].update({
field['name']: validators
})
# conditionally include schema fields from MultilangResourcesPlugin
if MLR:
schema = MLR.update_schema(schema)
log.debug("Schema updated for DCAT_AP-TI: %r", schema)
return schema
def after_search(self, search_results, search_params):
## #####################################################################
# This method moves the dcatapit fields into the extras array (needed for
# the CKAN harvester).
# Basically it dynamically reverts what is done by the
# 'convert_from_extras' to allow harvesting this plugin's custom fields.
## #####################################################################
search_dicts = search_results.get('results', [])
dcatapit_schema_fields = dcatapit_schema.get_custom_package_schema()
for _dict in search_dicts:
_dict_extras = _dict.get('extras', None)
if not _dict_extras:
_dict_extras = []
_dict['extras'] = _dict_extras
for field in dcatapit_schema_fields:
field_couple = field.get('couples', [])
if len(field_couple) > 0:
for couple in field_couple:
self.manage_extras_for_search(couple, _dict, _dict_extras)
else:
self.manage_extras_for_search(field, _dict, _dict_extras)
# remove holder info if pkg is local, use org as a source
# see https://github.com/geosolutions-it/ckanext-dcatapit/pull/213#issuecomment-410668740
_dict['dataset_is_local'] = helpers.dataset_is_local(_dict['id'])
if _dict['dataset_is_local']:
_dict.pop('holder_identifier', None)
_dict.pop('holder_name', None)
self._update_pkg_rights_holder(_dict)
lang = interfaces.get_language()
facets = search_results['search_facets']
if 'dcat_theme' in facets:
themes = facets['dcat_theme']
for item in themes['items']:
name = item['name']
label = interfaces.get_localized_tag_name(tag_name=name, lang=lang)
item['display_name'] = label
return search_results
def _modify_package_schema(self, schema):
# Package schema
for field in dcatapit_schema.get_custom_package_schema():
if field.get('ignore'):
continue
if 'couples' in field:
for couple in field['couples']:
self._update_schema_field(schema, couple)
else:
self._update_schema_field(schema, field)
schema['notes'] = [toolkit.get_validator('not_empty')]
# ignore theme extra fields
junk = schema.get('__junk', [])
junk.append(toolkit.get_validator('dcatapit_remove_theme'))
schema['__junk'] = junk
# Resource schema
for field in dcatapit_schema.get_custom_resource_schema():
if field.get('ignore'):
continue
validators = []
for validator in field['validator']:
validators.append(toolkit.get_validator(validator))
schema['resources'].update({
field['name']: validators
})
# conditionally include schema fields from MultilangResourcesPlugin
if MLR:
schema = MLR.update_schema(schema)
return schema
def after_search(self, search_results, search_params):
## ##################################################################### ##
# This method move the dcatapit fields into the extras array (needed for #
# the Ckan base harevsting). #
# Basically dynamically rever what did by the 'convert_from_extras' to #
# allow harvesting the plugin's custom fields. #
## ##################################################################### ##
search_dicts = search_results.get('results', [])
dcatapit_schema_fields = dcatapit_schema.get_custom_package_schema()
for _dict in search_dicts:
_dict_extras = _dict.get('extras', [])
for field in dcatapit_schema_fields:
field_couple = field.get('couples', [])
if len(field_couple) > 0:
for couple in field_couple:
self.manage_extras_for_search(couple, _dict,
_dict_extras)
else:
self.manage_extras_for_search(field, _dict, _dict_extras)
return search_results
def get_dcatapit_package_schema():
log.debug('Retrieving DCAT-AP_IT package schema fields...')
return dcatapit_schema.get_custom_package_schema()
def after_show(self, context, pkg_dict):
schema = dcatapit_schema.get_custom_package_schema()
# quick hack on date fields that are in wrong format
for fdef in schema:
if fdef.get('type') != 'date':
continue
fname = fdef['name']
df_value = pkg_dict.get(fname)
if df_value:
tmp_value = validators.parse_date(df_value, df_value)
if isinstance(tmp_value, datetime.date):
try:
tmp_value = tmp_value.strftime(fdef.get('format') or '%d-%m-%Y')
except ValueError as err:
log.warning('dataset %s, field %s: cannot reformat date for %s (from input %s): %s',
pkg_dict['name'], fname, tmp_value, df_value, err, exc_info=err)
tmp_value = df_value
pkg_dict[fname] = tmp_value
# themes are parsed by dcat, which requires a list of URI
# we have the format like this:
# [{"theme": "AGRI", "subthemes": ["http://eurovoc.europa.eu/100253", "http://eurovoc.europa.eu/100258"]},
# {"theme": "ENVI", "subthemes": []}]
# We need to fix this.
if not context.get('for_view'):
if not any(x['key'] == 'theme' for x in pkg_dict.get('extras', [])):
# there's no theme, add the list from the aggreagate
aggr_raw = pkg_dict.get(FIELD_THEMES_AGGREGATE)
if aggr_raw is None:
# let's try and find it in extras:
aggr_raw = next((x['value'] for x in pkg_dict.get('extras', [])
if x['key'] == FIELD_THEMES_AGGREGATE), None)
if aggr_raw is None:
log.error(f'No Aggregates in dataset {pkg_dict.get("id", "_")}')
aggr_raw = json.dumps([{'theme': 'OP_DATPRO', 'subthemes':[]}])
pkg_dict[FIELD_THEMES_AGGREGATE] = aggr_raw
themes = []
for aggr in json.loads(aggr_raw):
themes.append(theme_name_to_uri(aggr['theme']))
extras = pkg_dict.get('extras', [])
extras.append({'key': 'theme', 'value': json.dumps(themes)})
pkg_dict['extras'] = extras
# in some cases (automatic solr indexing after update)
# pkg_dict may come without validation and thus
# without extras converted to main dict.
# this will ensure that holder keys are extracted to main dict
pkg_update = {}
to_remove = []
for eidx, ex in enumerate(pkg_dict.get('extras') or []):
if ex['key'].startswith('holder_'):
to_remove.append(eidx)
pkg_update[ex['key']] = ex['value']
for k in pkg_update.keys():
if k in pkg_dict:
if pkg_update[k] == pkg_dict[k]:
log.warning(f'Ignoring duplicated key {k} with same value {pkg_update[k]}')
else:
raise KeyError(f'Duplicated key in pkg_dict: {k}: {pkg_update[k]} in extras'
f' vs {pkg_dict[k]} in pkg')
for tr in reversed(to_remove):
val = pkg_dict['extras'].pop(tr)
assert val['key'].startswith('holder_'), val
pkg_dict.update(pkg_update)
# remove holder info if pkg is local, use org as a source
# see https://github.com/geosolutions-it/ckanext-dcatapit/pull/213#issuecomment-410668740
pkg_dict['dataset_is_local'] = helpers.dataset_is_local(pkg_dict['id'])
if pkg_dict['dataset_is_local']:
pkg_dict.pop('holder_identifier', None)
pkg_dict.pop('holder_name', None)
return self._update_pkg_rights_holder(pkg_dict)
