def validator(key, data, errors, context):
# if there was an error before calling our validator
# don't bother with our validation
if errors[key]:
return
value = data[key]
if value is not missing:
if isinstance(value, basestring):
value = [value]
elif not isinstance(value, list):
errors[key].append(_('expecting list of strings'))
return
else:
value = []
choice_values = static_choice_values
if not choice_values:
choice_order = [c['value'] for c in sh.scheming_field_choices(field)]
choice_values = set(choice_order)
selected = set()
for element in value:
if element in choice_values:
selected.add(element)
continue
errors[key].append(_('unexpected choice "%s"') % element)
if not errors[key]:
data[key] = json.dumps([v for v in
(static_choice_order if static_choice_values else choice_order)
if v in selected])
if field.get('required') and not selected:
errors[key].append(_('Select at least one'))
def validator(value):
if value is missing or not value:
return value
choices = sh.scheming_field_choices(field)
for c in choices:
if value == c['value']:
return value
raise Invalid(_('unexpected choice "%s"') % value)
def validator(value):
if value is missing or not value:
return value
choices = sh.scheming_field_choices(field)
for choice in choices:
if value == choice['value']:
return value
raise Invalid(_('unexpected choice "%s"') % value)
def validator(value):
if value is missing or not value:
return value
choices = scheming_field_choices(field)
for c in choices:
if value == c['value']:
return value
log.info(_('unexpected choice "%s"') % value)
return value
def validator(value):
if value is missing or not value:
return value
choices = sh.scheming_field_choices(field)
for c in choices:
# We want the value check to be case insensitive
if value.upper() == c['value'].upper():
return value
raise Invalid(_('unexpected choice "%s"') % value)
def after_search(self, search_results, search_params):
facets = search_results.get('search_facets')
results = search_results.get('results')
if not facets or not results:
return search_results
schema = scheming_helpers.scheming_get_dataset_schema(results[0]['type'])
for facet in facets.values():
for item in facet['items']:
field_name = facet['title'].replace('_facet', '')
field = scheming_helpers.scheming_field_by_name( \
schema['dataset_fields'], field_name)
if field and (field.get('choices') or \
field.get('choices_helper')):
choices = scheming_helpers.scheming_field_choices(field)
item['display_name'] = scheming_helpers. \
scheming_choices_label(choices, item['name'])
return search_results
def validator(value):
if value is missing or not value:
return value
choices = sh.scheming_field_choices(field)
for c in choices:
if value == c['value']:
return value
msg = '''
Unexpected choice "{value}". It must be either the label or the code in
brackets of one of the following: {allowed}. If you want to add another
value please contact the Site Administrators.
'''
raise Invalid(
_(
msg.format(value=value,
allowed=', '.join([
c['label'] + ' [' + c['value'] + ']'
for c in choices
]))))
def _extract_additional_fields(self, content, package_dict):
package_dict['thematic_area_string'] = self.topic
if not package_dict.get('license_id'):
package_dict['license_id'] = 'notspecified'
skip_keys = {'set_spec', 'description'}
for key, value in content.items():
if key in package_dict or key in skip_keys:
continue
if key == 'type':
key = 'publication_type'
package_dict[key] = value
package_dict.pop('extras', None)
package_dict['type'] = 'publications'
package_dict.pop('maintainer_email', None)
coverage = package_dict.pop('coverage', None)
if coverage:
schema = scheming_get_dataset_schema('publications')
field = scheming_field_by_name(schema['dataset_fields'],
'member_countries')
choices = scheming_field_choices(field)
package_dict['member_countries'] = [
choice['value']
for choice in choices if choice['label'] in coverage
] or ['other']
polygons = [
t['geometry'] for t in eez.collection
if any(country in t['properties']['GeoName']
for country in coverage)
]
# TODO: for now we are taking first polygon from possible
# list because of SOLR restriction of spatial field
# size. In future we may add additional logic here
if polygons:
package_dict['coverage'] = json.dumps(polygons[0])
return package_dict
def validator(key, data, errors, context):
# if there was an error before calling our validator
# don't bother with our validation
if errors[key]:
return
value = data[key]
if value is not missing:
if isinstance(value, basestring):
try:
value = json.loads(value)
except:
value = [value]
elif not isinstance(value, list):
errors[key].append(_('expecting list of strings'))
return
else:
value = []
choice_values = static_choice_values
if not choice_values:
choice_order = [choice['value'] for choice in sh.scheming_field_choices(field)]
choice_values = set(choice_order)
selected = set()
for element in value:
if element in choice_values:
selected.add(element)
continue
if element is not '':
errors[key].append(_('unexpected choice "%s"') % element)
if not errors[key]:
data[key] = json.dumps([v for v in
(static_choice_order if static_choice_values else choice_order)
if v in selected])
if field.get('required') and not selected:
errors[key].append(_('Select at least one'))
def import_stage(self, harvest_object):
log.debug('In PRDREngergyResourcesHarvester import_stage')
if not harvest_object:
log.error('No harvest object received')
return False
self._set_config(harvest_object.job.source.config)
if self.force_import:
status = 'change'
else:
status = self._get_object_extra(harvest_object, 'status')
if status == 'delete':
context = {
'model': model,
'session': model.Session,
'user': self._get_user_name()
}
p.toolkit.get_action('package_delete')(
context, {
'id': harvest_object.package_id
})
log.info('Deleted package {0} with guid {1}'.format(
harvest_object.package_id, harvest_object.guid))
return True
if harvest_object.content is None:
self._save_object_error(
'Empty content for object %s' % harvest_object.id,
harvest_object, 'Import')
return False
# Get the last harvested object (if any)
previous_object = model.Session.query(HarvestObject) \
.filter(HarvestObject.guid == harvest_object.guid) \
.filter(HarvestObject.current == True) \
.first()
# Flag previous object as not current anymore
if previous_object and not self.force_import:
previous_object.current = False
previous_object.add()
package_dict = self._get_package_dict(harvest_object)
if not package_dict:
return False
if not package_dict.get('name'):
package_dict['name'] = \
self._get_package_name(harvest_object, package_dict['title'])
# copy across resource ids from the existing dataset, otherwise they'll
# be recreated with new ids
if status == 'change':
existing_dataset = self._get_existing_dataset(harvest_object.guid)
if existing_dataset:
copy_across_resource_ids(existing_dataset, package_dict)
# Allow custom harvesters to modify the package dict before creating
# or updating the package
package_dict = self.modify_package_dict(package_dict, harvest_object)
# Unless already set by an extension, get the owner organization (if
# any) from the harvest source dataset
if not package_dict.get('owner_org'):
source_dataset = model.Package.get(harvest_object.source.id)
if source_dataset.owner_org:
package_dict['owner_org'] = source_dataset.owner_org
if not package_dict.get('license_id'):
package_dict['license_id'] = 'notspecified'
# Flag this object as the current one
harvest_object.current = True
harvest_object.add()
context = {
'user': self._get_user_name(),
'return_id_only': True,
'ignore_auth': True,
}
package_schema = scheming_get_dataset_schema('dataset')
field = scheming_field_by_name(package_schema['dataset_fields'],
'member_countries')
choices = scheming_field_choices(field)
mem_temp_list = [
x for x in package_dict['member_countries'] if x is not None
]
package_dict['member_countries'] = [
choice['value']
for choice in choices if choice['label'] in mem_temp_list
] or ['other']
polygons = [
t['geometry'] for t in eez.collection
if any(country in t['properties']['GeoName']
for country in mem_temp_list)
]
# TODO: for now we are taking first polygon from possible
# list because of SOLR restriction of spatial field
# size. In future we may add additional logic here
if polygons:
package_dict['coverage'] = json.dumps(polygons[0])
if status == 'new':
# context['schema'] = package_schema
# We need to explicitly provide a package ID
package_dict['id'] = unicode(uuid.uuid4())
# package_schema['id'] = [unicode]
# Save reference to the package on the object
harvest_object.package_id = package_dict['id']
harvest_object.add()
# Defer constraints and flush so the dataset can be indexed with
# the harvest object id (on the after_show hook from the harvester
# plugin)
model.Session.execute(
'SET CONSTRAINTS harvest_object_package_id_fkey DEFERRED')
model.Session.flush()
package_id = \
p.toolkit.get_action('package_create')(context, package_dict)
log.info('Created dataset with id %s', package_id)
elif status == 'change':
package_dict['id'] = harvest_object.package_id
try:
package_id = \
p.toolkit.get_action('package_update')(context, package_dict)
log.info('Updated dataset with id %s', package_id)
except NotFound:
log.info(
'Update returned NotFound, trying to create new Dataset.')
if not harvest_object.package_id:
package_dict['id'] = unicode(uuid.uuid4())
harvest_object.package_id = package_dict['id']
harvest_object.add()
else:
package_dict['id'] = harvest_object.package_id
package_id = \
p.toolkit.get_action('package_create')(context, package_dict)
log.info('Created dataset with id %s', package_id)
model.Session.commit()
stored_package = p.toolkit.get_action('package_show')(context.copy(), {
'id': package_id
})
for res in stored_package.get('resources', []):
p.toolkit.get_action('resource_create_default_resource_views')(
context.copy(), {
'package': stored_package,
'resource': res
})
return True
