def lookup(cls, name=None, title=None, for_keywords=False):
"""Lookup by name or title and return a Thesaurus instance.
This is a factory method that tries to instantiate a Thesaurus object
from a collection of well-known (mostly related to INSPIRE) vocabularies.
"""
vocab = None
if (name is None) and title:
name = vocabularies.normalize_thesaurus_title(title, for_keywords)
if name:
vocab = vocabularies.by_name(name)
else:
raise ValueError("Expected a name/title lookup")
if vocab:
kwargs = {
"title": vocab.get("title"),
"name": vocab.get("name"),
"reference_date": vocab.get("reference_date"),
"version": vocab.get("version"),
"date_type": vocab.get("date_type"),
}
return cls(**kwargs)
else:
raise ValueError('Cannot find a thesaurus named "%s"' % (name))
class IFundingReference(IObject):
funder_name = zope.schema.TextLine(title=_(u'Funder name'),
description=_(u'funder'),
required=False)
funder_identifier = zope.schema.TextLine(
title=_(u'Funder identifier'),
description=_(u'e.g. https://doi.org/10.13039/100000936'),
required=False)
funder_identifier_type = zope.schema.Choice(
title=_(u'Funder identifier type'),
vocabulary=vocabularies.by_name('funder-identifier-type').get(
'vocabulary'),
#default = 'crossrefFunder',
description=_(u'funder identifier type'),
required=False)
award_number = zope.schema.TextLine(title=_(u'Award number'),
description=_(u'e.g. GBMF3859.01'),
required=False)
award_uri = zope.schema.TextLine(
title=_(u'Award uri'),
description=_(u'e.g. https://www.moore.org/grants/list/GBMF3859.01'),
required=False)
award_title = zope.schema.TextLine(title=_(u'Award title'),
description=_(u'award title'),
required=False)
class IConformity(IObject):
title = zope.schema.Text(title=_(u'Specification'), required=True)
title.setTaggedValue('translatable', True)
date = zope.schema.Date(title=_(u'Date'), required=True)
date_type = zope.schema.Choice(
title=_(u'Date Type'),
vocabulary=vocabularies.by_name('date-types').get('vocabulary'),
required=True)
degree = zope.schema.Choice(
title=_(u'Degree'),
vocabulary=vocabularies.by_name('degrees').get('vocabulary'),
description=
_(u'This is the degree of conformity of the resource to the implementing rules adopted under Article 7(1) of Directive 2007/2/EC or other specification.'
),
default="not-evaluated",
required=True)
class IRelatedIdentifier(IObject):
related_identifier = zope.schema.TextLine(
title=_(u'Related identifier'),
description=_(u'related identifier'),
required=False)
related_identifier_type = zope.schema.Choice(
title=_(u'Related identifier type'),
vocabulary=vocabularies.by_name('related-identifier-type').get(
'vocabulary'),
#default = 'issn',
description=_(u'related identifier type'),
required=False)
relation_type = zope.schema.Choice(
title=_(u'Relation type'),
vocabulary=vocabularies.by_name('relation-type').get('vocabulary'),
#default = 'IsSupplementTo',
description=_(u'relation type'),
required=False)
related_metadata_scheme = zope.schema.TextLine(
title=_(u'Related metadata scheme'),
description=_(u'related metadat scheme'),
required=False)
related_metadata_scheme_uri = zope.schema.TextLine(
title=_(u'Related metadata scheme uri'),
description=_(u'related metadat scheme uri'),
required=False)
related_metadata_scheme_type = zope.schema.TextLine(
title=_(u'Related metadata scheme type'),
description=_(u'related metadat scheme type'),
required=False)
class IResponsibleParty(IObject):
organization = zope.schema.TextLine(title=_(u'Organization Name'),
required=True,
min_length=2)
organization.setTaggedValue('translatable', True)
email = z3c.schema.email.RFC822MailAddress(title=_(u'Email'),
required=True)
role = zope.schema.Choice(
title=_(u'Responsible Party Role'),
vocabulary=vocabularies.by_name('party-roles').get('vocabulary'),
description=_(u'This is the role of the responsible organisation.'),
default='pointOfContact',
required=True)
def vocabulary_get(self, name):
name = str(name)
r = None
vocab = vocabularies.by_name(name)
if vocab:
r = {
'date_type': vocab.get('date_type'),
'reference_date': vocab.get('reference_date'),
'title': vocab.get('title'),
'name': vocab.get('name'),
'terms': [{'token': t.token, 'value': t.value, 'title': t.title}
for t in vocab['vocabulary']],
}
response.headers['Content-Type'] = content_types['json']
return [to_json(r)]
class IReferenceSystem(IObject):
code = zope.schema.Choice(
title=_(u'System'),
description=_(u'Coordinate Reference System'),
vocabulary=vocabularies.by_name('reference-systems').get('vocabulary'),
default='http://www.opengis.net/def/crs/EPSG/0/2100',
required=True)
code_space = zope.schema.NativeStringLine(
title=_(u'Code-Space'),
description=_(u'Reference System Code-Space'),
#default = 'urn:ogc:def:crs:EPSG',
required=False)
version = zope.schema.NativeStringLine(
title=_(u'Version'),
description=_(u'Reference System version'),
#default = '6.11.2',
required=False)
class IInspireMetadata(IMetadata):
zope.interface.taggedValue('recurse-on-invariants', True)
# Metadata on metadata
contact = zope.schema.List(
title=_(u'Metadata Point of Contact'),
description=
_(u'The organisations responsible for the creation and maintenance of the metadata.'
),
required=True,
min_length=1,
max_length=4,
value_type=zope.schema.Object(IResponsibleParty,
title=_(u'Contact'),
required=True))
datestamp = zope.schema.Date(
title=_(u'Metadata Date'),
description=
_(u'The date which specifies when the metadata record was created or updated. This date shall be expressed in conformity with ISO 8601.'
),
required=False,
defaultFactory=datetime.date.today)
languagecode = zope.schema.Choice(
title=_(u'Metadata Language'),
vocabulary=vocabularies.by_name('languages-iso-639-2').get(
'vocabulary'),
description=
_(u'This is the language in which the metadata elements are expressed. The value domain of this metadata element is limited to the official languages of the Community expressed in conformity with ISO 639-2.'
),
required=True,
default='eng')
# Identification
identtype = zope.schema.DottedName(
title=_(u'Type'),
required=True,
default='dataset',
)
title = zope.schema.TextLine(
title=_(u'Resource Title'),
description=
_(u'This a characteristic (and often unique) name by which the resource is known.'
),
required=True)
title.setTaggedValue('translatable', True)
identifier = zope.schema.NativeStringLine(
title=_(u'Identifier'),
description=
_(u'A value uniquely identifying the dataset. The value domain of this metadata element is a mandatory character string code, generally assigned by the data owner, and a character string namespace uniquely identifying the context of the identifier code (for example, the data owner).'
),
required=True)
identifier.setTaggedValue('links-to', 'id')
abstract = zope.schema.Text(
title=_(u'Resource Abstract'),
description=
_(u'This is a brief narrative summary of the contents of this dataset.'
),
required=True)
abstract.setTaggedValue('translatable', True)
abstract.setTaggedValue('links-to', 'notes')
locator = zope.schema.List(
title=_(u'Resource Locator'),
description=
_(u'The resource locator defines the link(s) to the resource and/or the link to additional information about the resource. The value domain of this metadata element is a character string, commonly expressed as uniform resource locator (URL).'
),
required=True,
min_length=1,
max_length=6,
value_type=zope.schema.URI(title=_(u'Linkage'), required=True))
resource_language = zope.schema.List(
title=_(u'Resource Languages'),
description=
_(u'The language(s) used within the resource. The value domain of this metadata element is limited to the languages defined in ISO 639-2.'
),
required=False,
max_length=5,
value_type=zope.schema.Choice(
title=_(u'Resource Language'),
vocabulary=vocabularies.by_name('languages-iso-639-2').get(
'vocabulary'),
))
# Classification
topic_category = zope.schema.List(
title=_(u'Topic Categories'),
description=
_(u'The topic category is a high-level classification scheme to assist in the grouping and topic-based search of available spatial data resources. The value domain of this metadata element is defined in Part D.2.'
),
required=True,
min_length=1,
max_length=6,
value_type=zope.schema.Choice(
title=_(u'Topic Category'),
vocabulary=vocabularies.by_name('topic-category').get(
'vocabulary'),
))
topic_category.setTaggedValue('format:markup',
{'descend-if-dictized': False})
# Keywords
keywords = zope.schema.Dict(
title=_(u'Keywords'),
description=
_(u'The keyword value is a commonly used word, formalised word or phrase used to describe the subject. While the topic category is too coarse for detailed queries, keywords help narrowing a full text search and they allow for structured keyword search.'
),
required=True,
min_length=1,
key_type=zope.schema.Choice(vocabulary=SimpleVocabulary(
tuple(
SimpleTerm(k, k,
vocabularies.by_name(k).get('title'))
for k in keyword_thesaurus_names)),
title=_(u'Keyword Thesaurus')),
value_type=zope.schema.Object(IThesaurusTerms, title=_(u'Keywords')))
keywords.setTaggedValue('format:markup', {'descend-if-dictized': False})
@zope.interface.invariant
def check_keywords(obj):
if obj.keywords:
if not ('keywords-gemet-inspire-themes' in obj.keywords):
raise zope.interface.Invalid(
_(u'You need to select at least one keyword from INSPIRE data themes'
))
free_keywords = zope.schema.List(
title=_(u'Free Keywords'),
description=
_(u'The keyword value is a commonly used word, formalised word or phrase used to describe the subject. While the topic category is too coarse for detailed queries, keywords help narrowing a full text search and they allow for structured keyword search.'
),
required=False,
max_length=20,
value_type=zope.schema.Object(IFreeKeyword, title=_(u'Free Keyword')))
free_keywords.setTaggedValue('format:markup',
{'descend-if-dictized': False})
# Geographic
bounding_box = zope.schema.List(
title=_(u'Geographic Bounding Box'),
description=
_(u'This is the extent of the resource in the geographic space, given as a bounding box. The bounding box shall be expressed with westbound and eastbound longitudes, and southbound and northbound latitudes in decimal degrees, with a precision of at least two decimals.'
),
required=True,
min_length=1,
max_length=4,
value_type=zope.schema.Object(IGeographicBoundingBox,
title=_(u'Bounding Box')))
bounding_box.setTaggedValue('format:markup', {'descend-if-dictized': True})
# Temporal
temporal_extent = zope.schema.List(
title=_(u'Temporal Extents'),
description=
_(u'The temporal extent defines the time period covered by the content of the resource. This time period may be expressed as any of the following: - an individual date, - an interval of dates expressed through the starting date and end date of the interval,- a mix of individual dates and intervals of dates.'
),
required=False,
max_length=3,
value_type=zope.schema.Object(ITemporalExtent, title=_(u'Extent')))
creation_date = zope.schema.Date(
title=_(u'Creation Date'),
description=
_(u'This is the date of creation of the resource. There shall not be more than one date of creation.'
),
required=False)
publication_date = zope.schema.Date(
title=_(u'Publication Date'),
description=
_(u'This is the date of publication of the resource when available, or the date of entry into force. There may be more than one date of publication.'
),
required=False)
revision_date = zope.schema.Date(
title=_(u'Revision Date'),
description=
_(u'This is the date of last revision of the resource, if the resource has been revised. There shall not be more than one date of last revision.'
),
required=False)
@zope.interface.invariant
def check_creation_publication_order(obj):
errs = []
if obj.creation_date and obj.publication_date:
if obj.creation_date > obj.publication_date:
errs.append(
'Creation date (%s) is later than publication date (%s)' %
(obj.creation_date, obj.publication_date))
if obj.publication_date and obj.revision_date:
if obj.publication_date > obj.revision_date:
errs.append(
'Publication date (%s) is later than last revision date (%s)'
% (obj.publication_date, obj.revision_date))
if obj.creation_date and obj.revision_date:
if obj.creation_date > obj.revision_date:
errs.append(
'Creation date (%s) is later than last revision date (%s)'
% (obj.creation_date, obj.revision_date))
if errs:
raise zope.interface.Invalid(errs)
# Quality & Validity
lineage = zope.schema.Text(
title=_(u'Lineage'),
description=
_(u'This is a statement on process history and/or overall quality of the spatial data set. Where appropriate it may include a statement whether the data set has been validated or quality assured, whether it is the official version (if multiple versions exist), and whether it has legal validity. The value domain of this metadata element is free text.'
),
required=False)
lineage.setTaggedValue('translatable', True)
spatial_resolution = zope.schema.List(
title=_(u'Spatial Resolution'),
description=
_(u'''Spatial resolution refers to the level of detail of the data set. It shall be expressed as a set of zero to many resolution distances (typically for gridded data and imagery-derived products) or equivalent scales (typically for maps or map-derived products). An equivalent scale is generally expressed as an integer value expressing the scale denominator. A resolution distance shall be expressed as a numerical value associated with a unit of length.'''
),
required=False,
max_length=6,
value_type=zope.schema.Object(ISpatialResolution,
title=_(u'Resolution')))
spatial_resolution.value_type.setTaggedValue('allow-partial-update', False)
reference_system = zope.schema.Object(
IReferenceSystem,
title=_(u'Coordinate Reference System'),
description=_(u'Coordinate Reference System'),
required=False)
reference_system.setTaggedValue('format:markup',
{'descend-if-dictized': False})
# Conformity
conformity = zope.schema.List(
title=_(u'Conformity'),
description=
_(u'''This is a citation of the implementing rules adopted under Article 7(1) of Directive 2007/2/EC or other specification to which a particular resource conforms. A resource may conform to more than one implementing rules adopted under Article 7(1) of Directive 2007/2/EC or other specification. This citation shall include at least the title and a reference date (date of publication, date of last revision or of creation) of the implementing rules adopted under Article 7(1) of Directive 2007/2/EC or of the specification.'''
),
required=False,
max_length=4,
value_type=zope.schema.Object(IConformity, title=_(u'Specification')))
# Constraints
# Todo: The following fields should have suggestions (autocompleted?) values.
access_constraints = zope.schema.List(
title=_(u'Access Constraints'),
description=
_(u'Define the conditions for access and use of spatial data sets and services, and where applicable, corresponding fees as required by Article 5(2)(b) and Article 11(2)(f) of Directive 2007/2/EC. The value domain of this metadata element is free text. The element must have values. If no conditions apply to the access and use of the resource, "no conditions apply" shall be used. If conditions are unknown, "conditions unknown" shall be used. This element shall also provide information on any fees necessary to access and use the resource, if applicable, or refer to a uniform resource locator (URL) where information on fees is available.'
),
required=True,
min_length=1,
max_length=4,
value_type=zope.schema.TextLine(title=_(u'Condition')))
limitations = zope.schema.List(
title=_(u'Limitations'),
description=
_(u'When member states limit public access to spatial data sets and spatial data services under Article 13 of Directive 2007/2/EC, this metadata element shall provide information on the limitations and the reasons for them. If there are no limitations on public access, this metadata element shall indicate that fact. The value domain of this metadata element is free text.'
),
required=True,
min_length=1,
max_length=4,
value_type=zope.schema.TextLine(title=_(u'Limitation')))
# Responsible Party
responsible_party = zope.schema.List(
title=_(u'Responsible Party'),
description=
_(u'The responsible party names the organisation responsible for the establishment, management, maintenance and distribution of resources contained in this dataset.'
),
required=True,
min_length=1,
max_length=3,
value_type=zope.schema.Object(IResponsibleParty, title=_(u'Party')))
def from_xml(self, e):
'''Build and return an InspireMetadata object from a (serialized) etree Element e.
'''
def to_date(s):
return strptime(s, '%Y-%m-%d').date() if isinstance(s, str) else None
def to_responsible_party(alist):
result = []
for it in alist:
result.append(ResponsibleParty(
organization = unicode(it.organization),
email = unicode(it.email),
role = it.role))
return result
# Parse object
md = MD_Metadata(e)
datestamp = to_date(md.datestamp)
id_list = md.identification.uricode
url_list = []
if md.distribution:
for it in md.distribution.online:
url_list.append(it.url)
topic_list = []
for topic in md.identification.topiccategory:
topic_list.append(topic)
free_keywords = []
keywords = {}
for it in md.identification.keywords:
thes_title = it['thesaurus']['title']
# Lookup and instantiate a named thesaurus
thes = None
if thes_title:
try:
thes_title, thes_version = thes_title.split(',')
except:
thes_version = None
else:
thes_version = re.sub(r'^[ ]*version[ ]+(\d\.\d)$', r'\1', thes_version)
# Note thes_version can be used to enforce a specific thesaurus version
try:
thes = Thesaurus.lookup(title=thes_title, for_keywords=True)
except ValueError:
thes = None
# Treat present keywords depending on if they belong to a thesaurus
if thes:
# Treat as thesaurus terms; discard unknown terms
terms = []
for keyword in it['keywords']:
term = thes.vocabulary.by_value.get(keyword)
if not term:
term = thes.vocabulary.by_token.get(keyword)
if term:
terms.append(term.value)
keywords[thes.name] = ThesaurusTerms(thesaurus=thes, terms=terms)
else:
# Treat as free keywords (not really a thesaurus)
vocab_date = to_date(it['thesaurus']['date'])
vocab_datetype = it['thesaurus']['datetype']
if thes_title:
thes_title = unicode(thes_title)
for keyword in it['keywords']:
free_keywords.append(FreeKeyword(
value = keyword,
reference_date = vocab_date,
date_type = vocab_datetype,
originating_vocabulary = thes_title))
temporal_extent = []
if md.identification.temporalextent_start or md.identification.temporalextent_end:
temporal_extent = [TemporalExtent(
start = to_date(md.identification.temporalextent_start),
end = to_date(md.identification.temporalextent_end))]
bbox = []
if md.identification.extent:
if md.identification.extent.boundingBox:
bbox = [GeographicBoundingBox(
nblat = float(md.identification.extent.boundingBox.maxy),
sblat = float(md.identification.extent.boundingBox.miny),
eblng = float(md.identification.extent.boundingBox.maxx),
wblng = float(md.identification.extent.boundingBox.minx))]
creation_date = None
publication_date = None
revision_date = None
for it in md.identification.date:
if it.type == 'creation':
creation_date = to_date(it.date)
elif it.type == 'publication':
publication_date = to_date(it.date)
elif it.type == 'revision':
revision_date = to_date(it.date)
spatial_list = []
if len(md.identification.distance) != len(md.identification.uom):
raise Exception(_('Found unequal list lengths distance,uom (%s, %s)' % (
md.identification.distance,md.identification.uom)))
else:
for i in range(0,len(md.identification.distance)):
spatial_list.append(SpatialResolution(
distance = int(md.identification.distance[i]),
uom = unicode(md.identification.uom[i])))
for i in range(0, len(md.identification.denominators)):
spatial_list.append(SpatialResolution(
denominator = int(md.identification.denominators[i])))
conf_list = []
invalid_degree = False
#if md.referencesystem.codeSpace:
# code_space = md.referenceSystem.codeSpace
reference_system = None
if md.referencesystem:
code = md.referencesystem.code
reference_systems = vocabularies.by_name('reference-systems').get('vocabulary')
if code in reference_systems:
# Check whether the URI is provided
reference_system = ReferenceSystem(code = code)
else:
# Check whether just the EPSG code suffix is provided
code_full = 'http://www.opengis.net/def/crs/EPSG/0/{code}'.format(code=code)
if code_full in reference_systems:
reference_system = ReferenceSystem(code = code_full)
else:
raise Exception(_('Reference system not recognizable'))
if md.referencesystem.codeSpace:
reference_system.code_space = md.referencesystem.codeSpace
if md.referencesystem.version:
reference_system.version = md.referencesystem.version
if len(md.dataquality.conformancedate) != len(md.dataquality.conformancedatetype):
# Date list is unequal to datetype list, this means wrong XML so exception is thrown
raise Exception(_('Found unequal list lengths: conformance date, conformancedatetype'))
if len(md.dataquality.conformancedegree) != len(md.dataquality.conformancedate):
# Degree list is unequal to date/datetype lists, so we are unable to conclude
# to which conformity item each degree value corresponds, so all are set to
# not-evaluated (Todo: MD_Metadata bug #63)
invalid_degree = True
if md.dataquality.conformancedate:
#and len(md.dataquality.conformancedate) == len(md.dataquality.degree):
for i in range(0,len(md.dataquality.conformancedate)):
date = to_date(md.dataquality.conformancedate[i])
date_type = md.dataquality.conformancedatetype[i]
# TODO md.dataquality.conformancedatetype returns empty
if invalid_degree:
degree = 'not-evaluated'
else:
try:
if md.dataquality.conformancedegree[i] == 'true':
degree = 'conformant'
elif md.dataquality.conformancedegree[i] == 'false':
degree = 'not-conformant'
except:
degree = "not-evaluated"
title = unicode(md.dataquality.conformancetitle[i])
if title != 'None':
conf_list.append(Conformity(title=title, date=date, date_type=date_type, degree=degree))
# TODO: is title required fields? If so the following is unnecessary
else:
conf_list.append(Conformity(date=date, date_type=date_type, degree=degree))
limit_list = []
for it in md.identification.uselimitation:
limit_list.append(unicode(it))
constr_list = []
for it in md.identification.otherconstraints:
constr_list.append(unicode(it))
obj = InspireMetadata()
obj.contact = to_responsible_party(md.contact)
obj.datestamp = datestamp
obj.languagecode = md.languagecode
obj.title = unicode(md.identification.title)
obj.abstract = unicode(md.identification.abstract)
obj.identifier = id_list[0]
obj.locator = url_list
#obj.resource_language = md.identification.resourcelanguage
obj.topic_category = topic_list
obj.keywords = keywords
obj.free_keywords = free_keywords
obj.bounding_box = bbox
obj.temporal_extent = temporal_extent
obj.creation_date = creation_date
obj.publication_date = publication_date
obj.revision_date = revision_date
obj.lineage = unicode(md.dataquality.lineage)
obj.spatial_resolution = spatial_list
obj.reference_system = reference_system
obj.conformity = conf_list
obj.access_constraints = limit_list
obj.limitations = constr_list
obj.responsible_party = to_responsible_party(md.identification.contact)
return obj
def vocabulary(self):
vocab = vocabularies.by_name(self.name)
return vocab.get("vocabulary") if vocab else None
import zope.interface
import zope.schema
from zope.interface import Interface
from ckanext.publicamundi.lib import vocabularies
from ckanext.publicamundi.lib.util import check_uuid
from ckanext.publicamundi.lib.metadata import IMetadata
from ckanext.publicamundi.lib.metadata.ibase import IFieldWithContext
language_vocabulary = vocabularies.by_name('languages-iso-639-1').get('vocabulary')
__all__ = [
'IFieldTranslation',
'IKeyBasedFieldTranslation',
'IValueBasedFieldTranslation',
'ITranslatedField',
'ITranslatedMetadata',
'ITranslator',
'IMetadataTranslator',
]
class IFieldTranslation(Interface):
def get(field, language, state='active'):
'''Return a bound field translated for the given pair (field, language).
This method should always return a bound field, or None if no translation exists.
The translated value (found at .context.value) should be unicode text.
*Note* Maybe return just unicode text (avoid wrapping to a field) ?