def _get_envelope(self):
"""Parse gml:Envelope"""
tmp = self._exml.find(util.nspath_eval('gml:Envelope/gml:lowerCorner',
self.nsmap))
if tmp is None:
raise RuntimeError('Invalid gml:Envelope geometry.\
Missing gml:lowerCorner')
else:
lower_left = tmp.text
tmp = self._exml.find(util.nspath_eval('gml:Envelope/gml:upperCorner',
self.nsmap))
if tmp is None:
raise RuntimeError('Invalid gml:Envelope geometry.\
Missing gml:upperCorner')
else:
upper_right = tmp.text
llmin = lower_left.split()
urmax = upper_right.split()
if len(llmin) < 2 or len(urmax) < 2:
raise RuntimeError('Invalid gml:Envelope geometry. \
gml:lowerCorner and gml:upperCorner must hold at least x and y')
if self.crs.axisorder == 'yx':
self.wkt = util.bbox2wktpolygon('%s,%s,%s,%s' % (llmin[1],
llmin[0], urmax[1], urmax[0]))
else:
self.wkt = util.bbox2wktpolygon('%s,%s,%s,%s' % (llmin[0],
llmin[1], urmax[0], urmax[1]))
def create(self, request):
try:
styles_data = None
ows_data = None
http_status = status.HTTP_200_OK
if "styles" in request.data:
styles_data = request.data.pop("styles")
if "ows_resource" in request.data:
ows_data = request.data.pop("ows_resource")
# parse and valid record data
serializer = RecordSerializer(data=request.data, ows='post')
serializer.is_valid(raise_exception=True)
# parse and valid styles data
style_serializers = [StyleSerializer(data=style) for style in styles_data] if styles_data else []
if style_serializers:
for style_serializer in style_serializers:
style_serializer.is_valid(raise_exception=True)
# parse and vlaidate ows data
ows_serializer = OwsResourceSerializer(data=ows_data)
ows_serializer.is_valid(raise_exception=True)
# save record data.
identifier = "{}:{}".format(serializer.validated_data['workspace'], serializer.validated_data['name'])
# transform the bbox data format
if serializer.validated_data.get('bounding_box'):
bounding_box = json.loads(serializer.validated_data['bounding_box'])
bounding_box = ','.join([str(o) for o in bounding_box])
try:
serializer.validated_data['bounding_box'] = util.bbox2wktpolygon(bounding_box)
except:
traceback.print_exc()
raise serializers.ValidationError("Incorrect bounding box dataformat.")
if serializer.validated_data.get("source_legend"):
serializer.validated_data["source_legend"]["content"] = ContentFile(serializer.validated_data["source_legend"]["content"].decode("base64"))
try:
serializer.instance = Record.objects.get(identifier=identifier)
serializer.instance.active = True
for key in ["title", "abstract", "modified", "insert_date"]:
if key in serializer.validated_data:
serializer.validated_data.pop(key)
# remove fake fields
if "workspace" in serializer.validated_data:
serializer.validated_data.pop("workspace")
if "name" in serializer.validated_data:
serializer.validated_data.pop("name")
record = serializer.save()
except Record.DoesNotExist:
#record does not exist, create it
serializer.validated_data['identifier'] = identifier
# remove fake fields
if "workspace" in serializer.validated_data:
serializer.validated_data.pop("workspace")
if "name" in serializer.validated_data:
serializer.validated_data.pop("name")
record = serializer.save()
http_status = status.HTTP_201_CREATED
# set the missing data and transform the content
for style_serializer in style_serializers:
uploaded_style = style_serializer.validated_data
uploaded_style["record"] = record
uploaded_style["content"] = ContentFile(uploaded_style["content"].decode("base64"))
# set default style
origin_default_style = {
"sld": record.sld.name if record.sld else None,
"qml": record.qml.name if record.qml else None,
"lyr": record.lyr.name if record.lyr else None
}
default_style = {}
for style_serializer in style_serializers:
uploaded_style = style_serializer.validated_data
if uploaded_style.get("default", False):
# user set this style as default style, use the user's setting
default_style[uploaded_style["format"]] = uploaded_style
elif origin_default_style.get(uploaded_style["format"].lower(), None) == uploaded_style["name"]:
# the current style is configured default style.
default_style[uploaded_style["format"]] = uploaded_style
elif not origin_default_style.get(uploaded_style["format"].lower(), None) and uploaded_style["format"] not in default_style:
# no default style has been set, set the current style as the default style
default_style[uploaded_style["format"]] = uploaded_style
# clear the default flag
uploaded_style["default"] = False
# set the default style
for uploaded_style in default_style.itervalues():
uploaded_style["default"] = True
#save style
for style_serializer in style_serializers:
style_serializer.save()
ows_serializer.save(record)
record.styles = list(Style.objects.filter(record=record))
style_content = bool(request.GET.get("style_content", False))
serializer = RecordSerializer(record, style_content=style_content, ows='get')
return Response(serializer.data, status=http_status)
except serializers.ValidationError:
raise
except Exception as e:
raise serializers.ValidationError(str(e))
def _bbox(lhs, minx, miny, maxx, maxy, crs=4326):
wkt = bbox2wktpolygon(f'{minx},{miny},{maxx},{maxy}')
return text(
f"query_spatial(records_wkt_geometry, '{wkt}', 'bbox', 'false') = 'true'"
)
def test_bbox2wktpolygon(bbox, expected):
result = util.bbox2wktpolygon(bbox)
assert result == expected
def create(self, request):
try:
styles_data = None
ows_data = None
http_status = status.HTTP_200_OK
if "styles" in request.data:
styles_data = request.data.pop("styles")
if "ows_resource" in request.data:
ows_data = request.data.pop("ows_resource")
# parse and valid record data
serializer = RecordSerializer(data=request.data, ows='post')
serializer.is_valid(raise_exception=True)
# parse and valid styles data
style_serializers = [StyleSerializer(data=style) for style in styles_data] if styles_data else []
if style_serializers:
for style_serializer in style_serializers:
style_serializer.is_valid(raise_exception=True)
# parse and vlaidate ows data
ows_serializer = OwsResourceSerializer(data=ows_data)
ows_serializer.is_valid(raise_exception=True)
# save record data.
identifier = "{}:{}".format(serializer.validated_data['workspace'], serializer.validated_data['name'])
# transform the bbox data format
if serializer.validated_data.get('bounding_box'):
bounding_box = json.loads(serializer.validated_data['bounding_box'])
bounding_box = ','.join([str(o) for o in bounding_box])
try:
serializer.validated_data['bounding_box'] = util.bbox2wktpolygon(bounding_box)
except:
traceback.print_exc()
raise serializers.ValidationError("Incorrect bounding box dataformat.")
try:
serializer.instance = Record.objects.get(identifier=identifier)
serializer.instance.active = True
for key in ["title", "abstract", "modified", "insert_date"]:
if key in serializer.validated_data:
serializer.validated_data.pop(key)
# remove fake fields
if "workspace" in serializer.validated_data:
serializer.validated_data.pop("workspace")
if "name" in serializer.validated_data:
serializer.validated_data.pop("name")
record = serializer.save()
except Record.DoesNotExist:
#record does not exist, create it
serializer.validated_data['identifier'] = identifier
# remove fake fields
if "workspace" in serializer.validated_data:
serializer.validated_data.pop("workspace")
if "name" in serializer.validated_data:
serializer.validated_data.pop("name")
record = serializer.save()
http_status = status.HTTP_201_CREATED
# set the missing data and transform the content
for style_serializer in style_serializers:
uploaded_style = style_serializer.validated_data
uploaded_style["record"] = record
uploaded_style["content"] = self.createStyle(uploaded_style["content"].decode("base64"))
# set default style
origin_default_style = {
"sld": record.sld.name if record.sld else None,
"qml": record.qml.name if record.qml else None,
"lyr": record.lyr.name if record.lyr else None
}
default_style = {}
for style_serializer in style_serializers:
uploaded_style = style_serializer.validated_data
if uploaded_style.get("default", False):
# user set this style as default style, use the user's setting
default_style[uploaded_style["format"]] = uploaded_style
elif origin_default_style.get(uploaded_style["format"].lower(), None) == uploaded_style["name"]:
# the current style is configured default style.
default_style[uploaded_style["format"]] = uploaded_style
elif not origin_default_style.get(uploaded_style["format"].lower(), None) and uploaded_style["format"] not in default_style:
# no default style has been set, set the current style as the default style
default_style[uploaded_style["format"]] = uploaded_style
# clear the default flag
uploaded_style["default"] = False
# set the default style
for uploaded_style in default_style.itervalues():
uploaded_style["default"] = True
#save style
for style_serializer in style_serializers:
style_serializer.save()
ows_serializer.save(record)
record.styles = list(Style.objects.filter(record=record))
style_content = bool(request.GET.get("style_content", False))
serializer = RecordSerializer(record, style_content=style_content, ows='get')
return Response(serializer.data, status=http_status)
except serializers.ValidationError:
raise
except Exception as e:
raise serializers.ValidationError(str(e))
def _parse_iso(context, repos, exml):
from owslib.iso import MD_Metadata
recobj = repos.dataset()
links = []
md = MD_Metadata(exml)
_set(context, recobj, 'pycsw:Identifier', md.identifier)
_set(context, recobj, 'pycsw:Typename', 'gmd:MD_Metadata')
_set(context, recobj, 'pycsw:Schema', context.namespaces['gmd'])
_set(context, recobj, 'pycsw:MdSource', 'local')
_set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, recobj, 'pycsw:XML', md.xml)
_set(context, recobj, 'pycsw:AnyText', util.get_anytext(exml))
_set(context, recobj, 'pycsw:Language', md.language)
_set(context, recobj, 'pycsw:Type', md.hierarchy)
_set(context, recobj, 'pycsw:ParentIdentifier', md.parentidentifier)
_set(context, recobj, 'pycsw:Date', md.datestamp)
_set(context, recobj, 'pycsw:Modified', md.datestamp)
_set(context, recobj, 'pycsw:Source', md.dataseturi)
if md.referencesystem is not None:
_set(context, recobj, 'pycsw:CRS','urn:ogc:def:crs:EPSG:6.11:%s' %
md.referencesystem.code)
if hasattr(md, 'identification'):
_set(context, recobj, 'pycsw:Title', md.identification.title)
_set(context, recobj, 'pycsw:AlternateTitle', md.identification.alternatetitle)
_set(context, recobj, 'pycsw:Abstract', md.identification.abstract)
_set(context, recobj, 'pycsw:Relation', md.identification.aggregationinfo)
if hasattr(md.identification, 'temporalextent_start'):
_set(context, recobj, 'pycsw:TempExtent_begin', md.identification.temporalextent_start)
if hasattr(md.identification, 'temporalextent_end'):
_set(context, recobj, 'pycsw:TempExtent_end', md.identification.temporalextent_end)
if len(md.identification.topiccategory) > 0:
_set(context, recobj, 'pycsw:TopicCategory', md.identification.topiccategory[0])
if len(md.identification.resourcelanguage) > 0:
_set(context, recobj, 'pycsw:ResourceLanguage', md.identification.resourcelanguage[0])
if hasattr(md.identification, 'bbox'):
bbox = md.identification.bbox
else:
bbox = None
if (hasattr(md.identification, 'keywords') and
len(md.identification.keywords) > 0):
all_keywords = [item for sublist in md.identification.keywords for item in sublist['keywords'] if item is not None]
_set(context, recobj, 'pycsw:Keywords', ','.join(all_keywords))
_set(context, recobj, 'pycsw:KeywordType', md.identification.keywords[0]['type'])
if (hasattr(md.identification, 'creator') and
len(md.identification.creator) > 0):
all_orgs = set([item.organization for item in md.identification.creator if hasattr(item, 'organization') and item.organization is not None])
_set(context, recobj, 'pycsw:Creator', ';'.join(all_orgs))
if (hasattr(md.identification, 'publisher') and
len(md.identification.publisher) > 0):
all_orgs = set([item.organization for item in md.identification.publisher if hasattr(item, 'organization') and item.organization is not None])
_set(context, recobj, 'pycsw:Publisher', ';'.join(all_orgs))
if (hasattr(md.identification, 'contributor') and
len(md.identification.contributor) > 0):
all_orgs = set([item.organization for item in md.identification.contributor if hasattr(item, 'organization') and item.organization is not None])
_set(context, recobj, 'pycsw:Contributor', ';'.join(all_orgs))
if (hasattr(md.identification, 'contact') and
len(md.identification.contact) > 0):
all_orgs = set([item.organization for item in md.identification.contact if hasattr(item, 'organization') and item.organization is not None])
_set(context, recobj, 'pycsw:OrganizationName', ';'.join(all_orgs))
if len(md.identification.securityconstraints) > 0:
_set(context, recobj, 'pycsw:SecurityConstraints',
md.identification.securityconstraints[0])
if len(md.identification.accessconstraints) > 0:
_set(context, recobj, 'pycsw:AccessConstraints',
md.identification.accessconstraints[0])
if len(md.identification.otherconstraints) > 0:
_set(context, recobj, 'pycsw:OtherConstraints', md.identification.otherconstraints[0])
if hasattr(md.identification, 'date'):
for datenode in md.identification.date:
if datenode.type == 'revision':
_set(context, recobj, 'pycsw:RevisionDate', datenode.date)
elif datenode.type == 'creation':
_set(context, recobj, 'pycsw:CreationDate', datenode.date)
elif datenode.type == 'publication':
_set(context, recobj, 'pycsw:PublicationDate', datenode.date)
if hasattr(md.identification, 'extent') and hasattr(md.identification.extent, 'description_code'):
_set(context, recobj, 'pycsw:GeographicDescriptionCode', md.identification.extent.description_code)
if len(md.identification.denominators) > 0:
_set(context, recobj, 'pycsw:Denominator', md.identification.denominators[0])
if len(md.identification.distance) > 0:
_set(context, recobj, 'pycsw:DistanceValue', md.identification.distance[0])
if len(md.identification.uom) > 0:
_set(context, recobj, 'pycsw:DistanceUOM', md.identification.uom[0])
if len(md.identification.classification) > 0:
_set(context, recobj, 'pycsw:Classification', md.identification.classification[0])
if len(md.identification.uselimitation) > 0:
_set(context, recobj, 'pycsw:ConditionApplyingToAccessAndUse',
md.identification.uselimitation[0])
if hasattr(md.identification, 'format'):
_set(context, recobj, 'pycsw:Format', md.distribution.format)
if md.serviceidentification is not None:
_set(context, recobj, 'pycsw:ServiceType', md.serviceidentification.type)
_set(context, recobj, 'pycsw:ServiceTypeVersion', md.serviceidentification.version)
_set(context, recobj, 'pycsw:CouplingType', md.serviceidentification.couplingtype)
service_types = []
for smd in md.identificationinfo:
if smd.identtype == 'service' and smd.type is not None:
service_types.append(smd.type)
_set(context, recobj, 'pycsw:ServiceType', ','.join(service_types))
#if len(md.serviceidentification.operateson) > 0:
# _set(context, recobj, 'pycsw:operateson = VARCHAR(32),
#_set(context, recobj, 'pycsw:operation VARCHAR(32),
#_set(context, recobj, 'pycsw:operatesonidentifier VARCHAR(32),
#_set(context, recobj, 'pycsw:operatesoname VARCHAR(32),
if hasattr(md.identification, 'dataquality'):
_set(context, recobj, 'pycsw:Degree', md.dataquality.conformancedegree)
_set(context, recobj, 'pycsw:Lineage', md.dataquality.lineage)
_set(context, recobj, 'pycsw:SpecificationTitle', md.dataquality.specificationtitle)
if hasattr(md.dataquality, 'specificationdate'):
_set(context, recobj, 'pycsw:specificationDate',
md.dataquality.specificationdate[0].date)
_set(context, recobj, 'pycsw:SpecificationDateType',
md.dataquality.specificationdate[0].datetype)
if hasattr(md, 'contact') and len(md.contact) > 0:
_set(context, recobj, 'pycsw:ResponsiblePartyRole', md.contact[0].role)
LOGGER.info('Scanning for links')
if hasattr(md, 'distribution'):
dist_links = []
if hasattr(md.distribution, 'online'):
LOGGER.debug('Scanning for gmd:transferOptions element(s)')
dist_links.extend(md.distribution.online)
if hasattr(md.distribution, 'distributor'):
LOGGER.debug('Scanning for gmd:distributorTransferOptions element(s)')
for dist_member in md.distribution.distributor:
dist_links.extend(dist_member.online)
for link in dist_links:
if link.url is not None and link.protocol is None: # take a best guess
link.protocol = sniff_link(link.url)
linkstr = '%s,%s,%s,%s' % \
(link.name, link.description, link.protocol, link.url)
links.append(linkstr)
try:
LOGGER.debug('Scanning for srv:SV_ServiceIdentification links')
for sident in md.identificationinfo:
if hasattr(sident, 'operations'):
for sops in sident.operations:
for scpt in sops['connectpoint']:
LOGGER.debug('adding srv link %s', scpt.url)
linkstr = '%s,%s,%s,%s' % \
(scpt.name, scpt.description, scpt.protocol, scpt.url)
links.append(linkstr)
except Exception as err: # srv: identification does not exist
LOGGER.debug('no srv:SV_ServiceIdentification links found')
if len(links) > 0:
_set(context, recobj, 'pycsw:Links', '^'.join(links))
if bbox is not None:
try:
tmp = '%s,%s,%s,%s' % (bbox.minx, bbox.miny, bbox.maxx, bbox.maxy)
_set(context, recobj, 'pycsw:BoundingBox', util.bbox2wktpolygon(tmp))
except: # coordinates are corrupted, do not include
_set(context, recobj, 'pycsw:BoundingBox', None)
else:
_set(context, recobj, 'pycsw:BoundingBox', None)
return recobj
def test_bbox2wktpolygon(bbox, expected):
result = util.bbox2wktpolygon(bbox)
assert result == expected
def _parse_fgdc(context, repos, exml):
from owslib.fgdc import Metadata
recobj = repos.dataset()
links = []
md = Metadata(exml)
if md.idinfo.datasetid is not None: # we need an identifier
_set(context, recobj, 'pycsw:Identifier', md.idinfo.datasetid)
else: # generate one ourselves
_set(context, recobj, 'pycsw:Identifier', uuid.uuid1().get_urn())
_set(context, recobj, 'pycsw:Typename', 'fgdc:metadata')
_set(context, recobj, 'pycsw:Schema', context.namespaces['fgdc'])
_set(context, recobj, 'pycsw:MdSource', 'local')
_set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, recobj, 'pycsw:XML', md.xml)
_set(context, recobj, 'pycsw:AnyText', util.get_anytext(exml))
_set(context, recobj, 'pycsw:Language', 'en-US')
if hasattr(md.idinfo, 'descript'):
_set(context, recobj, 'pycsw:Abstract', md.idinfo.descript.abstract)
if hasattr(md.idinfo, 'keywords'):
if md.idinfo.keywords.theme:
_set(context, recobj, 'pycsw:Keywords', \
','.join(md.idinfo.keywords.theme[0]['themekey']))
if hasattr(md.idinfo.timeperd, 'timeinfo'):
if hasattr(md.idinfo.timeperd.timeinfo, 'rngdates'):
_set(context, recobj, 'pycsw:TempExtent_begin',
md.idinfo.timeperd.timeinfo.rngdates.begdate)
_set(context, recobj, 'pycsw:TempExtent_end',
md.idinfo.timeperd.timeinfo.rngdates.enddate)
if hasattr(md.idinfo, 'origin'):
_set(context, recobj, 'pycsw:Creator', md.idinfo.origin)
_set(context, recobj, 'pycsw:Publisher', md.idinfo.origin)
_set(context, recobj, 'pycsw:Contributor', md.idinfo.origin)
if hasattr(md.idinfo, 'ptcontac'):
_set(context, recobj, 'pycsw:OrganizationName', md.idinfo.ptcontac.cntorg)
_set(context, recobj, 'pycsw:AccessConstraints', md.idinfo.accconst)
_set(context, recobj, 'pycsw:OtherConstraints', md.idinfo.useconst)
_set(context, recobj, 'pycsw:Date', md.metainfo.metd)
if hasattr(md.idinfo, 'spdom') and hasattr(md.idinfo.spdom, 'bbox'):
bbox = md.idinfo.spdom.bbox
else:
bbox = None
if hasattr(md.idinfo, 'citation'):
if hasattr(md.idinfo.citation, 'citeinfo'):
_set(context, recobj, 'pycsw:Type', md.idinfo.citation.citeinfo['geoform'])
_set(context, recobj, 'pycsw:Title', md.idinfo.citation.citeinfo['title'])
_set(context, recobj,
'pycsw:PublicationDate', md.idinfo.citation.citeinfo['pubdate'])
_set(context, recobj, 'pycsw:Format', md.idinfo.citation.citeinfo['geoform'])
if md.idinfo.citation.citeinfo['onlink']:
for link in md.idinfo.citation.citeinfo['onlink']:
tmp = ',,,%s' % link
links.append(tmp)
if hasattr(md, 'distinfo') and hasattr(md.distinfo, 'stdorder'):
for link in md.distinfo.stdorder['digform']:
tmp = ',%s,,%s' % (link['name'], link['url'])
links.append(tmp)
if len(links) > 0:
_set(context, recobj, 'pycsw:Links', '^'.join(links))
if bbox is not None:
try:
tmp = '%s,%s,%s,%s' % (bbox.minx, bbox.miny, bbox.maxx, bbox.maxy)
_set(context, recobj, 'pycsw:BoundingBox', util.bbox2wktpolygon(tmp))
except: # coordinates are corrupted, do not include
_set(context, recobj, 'pycsw:BoundingBox', None)
else:
_set(context, recobj, 'pycsw:BoundingBox', None)
return recobj
def _parse_sos(context, repos, record, identifier, version):
from owslib.sos import SensorObservationService
bboxs = []
recobjs = []
serviceobj = repos.dataset()
if version == '1.0.0':
schema = 'http://www.opengis.net/sos/1.0'
else:
schema = 'http://www.opengis.net/sos/2.0'
md = SensorObservationService(record, version=version)
# generate record of service instance
_set(context, serviceobj, 'pycsw:Identifier', identifier)
_set(context, serviceobj, 'pycsw:Typename', 'csw:Record')
_set(context, serviceobj, 'pycsw:Schema', schema)
_set(context, serviceobj, 'pycsw:MdSource', record)
_set(context, serviceobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, serviceobj, 'pycsw:AnyText', util.get_anytext(etree.tostring(md._capabilities)))
_set(context, serviceobj, 'pycsw:Type', 'service')
_set(context, serviceobj, 'pycsw:Title', md.identification.title)
_set(context, serviceobj, 'pycsw:Abstract', md.identification.abstract)
_set(context, serviceobj, 'pycsw:Keywords', ','.join(md.identification.keywords))
_set(context, serviceobj, 'pycsw:Creator', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:Publisher', md.provider.name)
_set(context, serviceobj, 'pycsw:Contributor', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:OrganizationName', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:AccessConstraints', md.identification.accessconstraints)
_set(context, serviceobj, 'pycsw:OtherConstraints', md.identification.fees)
_set(context, serviceobj, 'pycsw:Source', record)
_set(context, serviceobj, 'pycsw:Format', md.identification.type)
_set(context, serviceobj, 'pycsw:CRS', 'urn:ogc:def:crs:EPSG:6.11:4326')
_set(context, serviceobj, 'pycsw:DistanceUOM', 'degrees')
_set(context, serviceobj, 'pycsw:ServiceType', 'OGC:SOS')
_set(context, serviceobj, 'pycsw:ServiceTypeVersion', md.identification.version)
_set(context, serviceobj, 'pycsw:Operation', ','.join([d.name for d in md.operations]))
_set(context, serviceobj, 'pycsw:OperatesOn', ','.join(list(md.contents)))
_set(context, serviceobj, 'pycsw:CouplingType', 'tight')
links = [
'%s,OGC-SOS Sensor Observation Service,OGC:SOS,%s' % (identifier, md.url),
]
_set(context, serviceobj, 'pycsw:Links', '^'.join(links))
# generate record foreach offering
LOGGER.debug('Harvesting %d SOS ObservationOffering\'s ', len(md.contents))
for offering in md.contents:
recobj = repos.dataset()
identifier2 = '%s-%s' % (identifier, md.contents[offering].id)
_set(context, recobj, 'pycsw:Identifier', identifier2)
_set(context, recobj, 'pycsw:Typename', 'csw:Record')
_set(context, recobj, 'pycsw:Schema', schema)
_set(context, recobj, 'pycsw:MdSource', record)
_set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, recobj, 'pycsw:Type', 'dataset')
_set(context, recobj, 'pycsw:ParentIdentifier', identifier)
_set(context, recobj, 'pycsw:Title', md.contents[offering].description)
_set(context, recobj, 'pycsw:Abstract', md.contents[offering].description)
_set(context, recobj, 'pycsw:TempExtent_begin', util.datetime2iso8601(md.contents[offering].begin_position))
_set(context, recobj, 'pycsw:TempExtent_end', util.datetime2iso8601(md.contents[offering].end_position))
#For observed_properties that have mmi url or urn, we simply want the observation name.
observed_properties = []
for obs in md.contents[offering].observed_properties:
#Observation is stored as urn representation: urn:ogc:def:phenomenon:mmisw.org:cf:sea_water_salinity
if obs.lower().startswith(('urn:', 'x-urn')):
observed_properties.append(obs.rsplit(':', 1)[-1])
#Observation is stored as uri representation: http://mmisw.org/ont/cf/parameter/sea_floor_depth_below_sea_surface
elif obs.lower().startswith(('http://', 'https://')):
observed_properties.append(obs.rsplit('/', 1)[-1])
else:
observed_properties.append(obs)
#Build anytext from description and the observed_properties.
anytext = []
anytext.append(md.contents[offering].description)
anytext.extend(observed_properties)
_set(context, recobj, 'pycsw:AnyText', util.get_anytext(anytext))
_set(context, recobj, 'pycsw:Keywords', ','.join(observed_properties))
bbox = md.contents[offering].bbox
if bbox is not None:
tmp = '%s,%s,%s,%s' % (bbox[0], bbox[1], bbox[2], bbox[3])
wkt_polygon = util.bbox2wktpolygon(tmp)
_set(context, recobj, 'pycsw:BoundingBox', wkt_polygon)
_set(context, recobj, 'pycsw:CRS', md.contents[offering].bbox_srs.id)
_set(context, recobj, 'pycsw:DistanceUOM', 'degrees')
bboxs.append(wkt_polygon)
_set(context, recobj, 'pycsw:XML', caps2iso(recobj, md, context))
recobjs.append(recobj)
# Derive a bbox based on aggregated featuretype bbox values
bbox_agg = util.bbox_from_polygons(bboxs)
if bbox_agg is not None:
_set(context, serviceobj, 'pycsw:BoundingBox', bbox_agg)
_set(context, serviceobj, 'pycsw:XML', caps2iso(serviceobj, md, context))
recobjs.insert(0, serviceobj)
return recobjs
def _parse_wcs(context, repos, record, identifier):
from owslib.wcs import WebCoverageService
bboxs = []
recobjs = []
serviceobj = repos.dataset()
md = WebCoverageService(record, '1.0.0')
# generate record of service instance
_set(context, serviceobj, 'pycsw:Identifier', identifier)
_set(context, serviceobj, 'pycsw:Typename', 'csw:Record')
_set(context, serviceobj, 'pycsw:Schema', 'http://www.opengis.net/wcs')
_set(context, serviceobj, 'pycsw:MdSource', record)
_set(context, serviceobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, serviceobj, 'pycsw:AnyText', util.get_anytext(etree.tostring(md._capabilities)))
_set(context, serviceobj, 'pycsw:Type', 'service')
_set(context, serviceobj, 'pycsw:Title', md.identification.title)
_set(context, serviceobj, 'pycsw:Abstract', md.identification.abstract)
_set(context, serviceobj, 'pycsw:Keywords', ','.join(md.identification.keywords))
_set(context, serviceobj, 'pycsw:Creator', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:Publisher', md.provider.name)
_set(context, serviceobj, 'pycsw:Contributor', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:OrganizationName', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:AccessConstraints', md.identification.accessConstraints)
_set(context, serviceobj, 'pycsw:OtherConstraints', md.identification.fees)
_set(context, serviceobj, 'pycsw:Source', record)
_set(context, serviceobj, 'pycsw:Format', md.identification.type)
_set(context, serviceobj, 'pycsw:CRS', 'urn:ogc:def:crs:EPSG:6.11:4326')
_set(context, serviceobj, 'pycsw:DistanceUOM', 'degrees')
_set(context, serviceobj, 'pycsw:ServiceType', 'OGC:WCS')
_set(context, serviceobj, 'pycsw:ServiceTypeVersion', md.identification.version)
_set(context, serviceobj, 'pycsw:Operation', ','.join([d.name for d in md.operations]))
_set(context, serviceobj, 'pycsw:OperatesOn', ','.join(list(md.contents)))
_set(context, serviceobj, 'pycsw:CouplingType', 'tight')
links = [
'%s,OGC-WCS Web Coverage Service,OGC:WCS,%s' % (identifier, md.url)
]
_set(context, serviceobj, 'pycsw:Links', '^'.join(links))
# generate record foreach coverage
LOGGER.debug('Harvesting %d WCS coverages ' % len(md.contents))
for coverage in md.contents:
recobj = repos.dataset()
identifier2 = '%s-%s' % (identifier, md.contents[coverage].id)
_set(context, recobj, 'pycsw:Identifier', identifier2)
_set(context, recobj, 'pycsw:Typename', 'csw:Record')
_set(context, recobj, 'pycsw:Schema', 'http://www.opengis.net/wcs')
_set(context, recobj, 'pycsw:MdSource', record)
_set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, recobj, 'pycsw:Type', 'dataset')
_set(context, recobj, 'pycsw:ParentIdentifier', identifier)
_set(context, recobj, 'pycsw:Title', md.contents[coverage].title)
_set(context, recobj, 'pycsw:Abstract', md.contents[coverage].abstract)
_set(context, recobj, 'pycsw:Keywords', ','.join(md.contents[coverage].keywords))
_set(context, recobj, 'pycsw:AnyText',
util.get_anytext([md.contents[coverage].title,
md.contents[coverage].abstract,
','.join(md.contents[coverage].keywords)]))
bbox = md.contents[coverage].boundingBoxWGS84
if bbox is not None:
tmp = '%s,%s,%s,%s' % (bbox[0], bbox[1], bbox[2], bbox[3])
wkt_polygon = util.bbox2wktpolygon(tmp)
_set(context, recobj, 'pycsw:BoundingBox', wkt_polygon)
_set(context, recobj, 'pycsw:CRS', 'urn:ogc:def:crs:EPSG:6.11:4326')
_set(context, recobj, 'pycsw:DistanceUOM', 'degrees')
bboxs.append(wkt_polygon)
links = [
'%s,OGC-Web Coverage Service,OGC:WCS,%s' % (md.contents[coverage].id, md.url)
]
_set(context, recobj, 'pycsw:Links', '^'.join(links))
_set(context, recobj, 'pycsw:XML', caps2iso(recobj, md, context))
recobjs.append(recobj)
# Derive a bbox based on aggregated coverage bbox values
bbox_agg = util.bbox_from_polygons(bboxs)
if bbox_agg is not None:
_set(context, serviceobj, 'pycsw:BoundingBox', bbox_agg)
_set(context, serviceobj, 'pycsw:XML', caps2iso(serviceobj, md, context))
recobjs.insert(0, serviceobj)
return recobjs
def _parse_wms(context, repos, record, identifier):
from owslib.wms import WebMapService
recobjs = []
serviceobj = repos.dataset()
md = WebMapService(record)
# generate record of service instance
_set(context, serviceobj, 'pycsw:Identifier', identifier)
_set(context, serviceobj, 'pycsw:Typename', 'csw:Record')
_set(context, serviceobj, 'pycsw:Schema', 'http://www.opengis.net/wms')
_set(context, serviceobj, 'pycsw:MdSource', record)
_set(context, serviceobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, serviceobj, 'pycsw:AnyText', util.get_anytext(md.getServiceXML()))
_set(context, serviceobj, 'pycsw:Type', 'service')
_set(context, serviceobj, 'pycsw:Title', md.identification.title)
_set(context, serviceobj, 'pycsw:Abstract', md.identification.abstract)
_set(context, serviceobj, 'pycsw:Keywords', ','.join(md.identification.keywords))
_set(context, serviceobj, 'pycsw:Creator', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:Publisher', md.provider.name)
_set(context, serviceobj, 'pycsw:Contributor', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:OrganizationName', md.provider.contact.name)
_set(context, serviceobj, 'pycsw:AccessConstraints', md.identification.accessconstraints)
_set(context, serviceobj, 'pycsw:OtherConstraints', md.identification.fees)
_set(context, serviceobj, 'pycsw:Source', record)
_set(context, serviceobj, 'pycsw:Format', md.identification.type)
for c in md.contents:
if md.contents[c].parent is None:
bbox = md.contents[c].boundingBoxWGS84
tmp = '%s,%s,%s,%s' % (bbox[0], bbox[1], bbox[2], bbox[3])
_set(context, serviceobj, 'pycsw:BoundingBox', util.bbox2wktpolygon(tmp))
break
_set(context, serviceobj, 'pycsw:CRS', 'urn:ogc:def:crs:EPSG:6.11:4326')
_set(context, serviceobj, 'pycsw:DistanceUOM', 'degrees')
_set(context, serviceobj, 'pycsw:ServiceType', 'OGC:WMS')
_set(context, serviceobj, 'pycsw:ServiceTypeVersion', md.identification.version)
_set(context, serviceobj, 'pycsw:Operation', ','.join([d.name for d in md.operations]))
_set(context, serviceobj, 'pycsw:OperatesOn', ','.join(list(md.contents)))
_set(context, serviceobj, 'pycsw:CouplingType', 'tight')
links = [
'%s,OGC-WMS Web Map Service,OGC:WMS,%s' % (identifier, md.url),
]
_set(context, serviceobj, 'pycsw:Links', '^'.join(links))
_set(context, serviceobj, 'pycsw:XML', caps2iso(serviceobj, md, context))
recobjs.append(serviceobj)
# generate record foreach layer
LOGGER.debug('Harvesting %d WMS layers' % len(md.contents))
for layer in md.contents:
recobj = repos.dataset()
identifier2 = '%s-%s' % (identifier, md.contents[layer].name)
_set(context, recobj, 'pycsw:Identifier', identifier2)
_set(context, recobj, 'pycsw:Typename', 'csw:Record')
_set(context, recobj, 'pycsw:Schema', 'http://www.opengis.net/wms')
_set(context, recobj, 'pycsw:MdSource', record)
_set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, recobj, 'pycsw:Type', 'dataset')
_set(context, recobj, 'pycsw:ParentIdentifier', identifier)
_set(context, recobj, 'pycsw:Title', md.contents[layer].title)
_set(context, recobj, 'pycsw:Abstract', md.contents[layer].abstract)
_set(context, recobj, 'pycsw:Keywords', ','.join(md.contents[layer].keywords))
_set(context, recobj, 'pycsw:AnyText',
util.get_anytext([md.contents[layer].title,
md.contents[layer].abstract,
','.join(md.contents[layer].keywords)]))
bbox = md.contents[layer].boundingBoxWGS84
if bbox is not None:
tmp = '%s,%s,%s,%s' % (bbox[0], bbox[1], bbox[2], bbox[3])
_set(context, recobj, 'pycsw:BoundingBox', util.bbox2wktpolygon(tmp))
_set(context, recobj, 'pycsw:CRS', 'urn:ogc:def:crs:EPSG:6.11:4326')
_set(context, recobj, 'pycsw:DistanceUOM', 'degrees')
else:
bbox = md.contents[layer].boundingBox
if bbox:
tmp = '%s,%s,%s,%s' % (bbox[0], bbox[1], bbox[2], bbox[3])
_set(context, recobj, 'pycsw:BoundingBox', util.bbox2wktpolygon(tmp))
_set(context, recobj, 'pycsw:CRS', 'urn:ogc:def:crs:EPSG:6.11:%s' % \
bbox[-1].split(':')[1])
params = {
'service': 'WMS',
'version': '1.1.1',
'request': 'GetMap',
'layers': md.contents[layer].name,
'format': 'image/png',
'height': '200',
'width': '200',
'srs': 'EPSG:4326',
'bbox': '%s,%s,%s,%s' % (bbox[0], bbox[1], bbox[2], bbox[3]),
'styles': ''
}
links = [
'%s,OGC-Web Map Service,OGC:WMS,%s' % (md.contents[layer].name, md.url),
'%s,Web image thumbnail (URL),WWW:LINK-1.0-http--image-thumbnail,%s' % (md.contents[layer].name, build_get_url(md.url, params))
]
_set(context, recobj, 'pycsw:Links', '^'.join(links))
_set(context, recobj, 'pycsw:XML', caps2iso(recobj, md, context))
recobjs.append(recobj)
return recobjs
def _parse_dc(context, repos, exml):
from owslib.csw import CswRecord
recobj = repos.dataset()
links = []
md = CswRecord(exml)
if md.bbox is None:
bbox = None
else:
bbox = md.bbox
_set(context, recobj, 'pycsw:Identifier', md.identifier)
_set(context, recobj, 'pycsw:Typename', 'csw:Record')
_set(context, recobj, 'pycsw:Schema', context.namespaces['csw'])
_set(context, recobj, 'pycsw:MdSource', 'local')
_set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now())
_set(context, recobj, 'pycsw:XML', md.xml)
_set(context, recobj, 'pycsw:AnyText', util.get_anytext(exml))
_set(context, recobj, 'pycsw:Language', md.language)
_set(context, recobj, 'pycsw:Type', md.type)
_set(context, recobj, 'pycsw:Title', md.title)
_set(context, recobj, 'pycsw:AlternateTitle', md.alternative)
_set(context, recobj, 'pycsw:Abstract', md.abstract)
if len(md.subjects) > 0 and None not in md.subjects:
_set(context, recobj, 'pycsw:Keywords', ','.join(md.subjects))
_set(context, recobj, 'pycsw:ParentIdentifier', md.ispartof)
_set(context, recobj, 'pycsw:Relation', md.relation)
_set(context, recobj, 'pycsw:TempExtent_begin', md.temporal)
_set(context, recobj, 'pycsw:TempExtent_end', md.temporal)
_set(context, recobj, 'pycsw:ResourceLanguage', md.language)
_set(context, recobj, 'pycsw:Creator', md.creator)
_set(context, recobj, 'pycsw:Publisher', md.publisher)
_set(context, recobj, 'pycsw:Contributor', md.contributor)
_set(context, recobj, 'pycsw:OrganizationName', md.rightsholder)
_set(context, recobj, 'pycsw:AccessConstraints', md.accessrights)
_set(context, recobj, 'pycsw:OtherConstraints', md.license)
_set(context, recobj, 'pycsw:Date', md.date)
_set(context, recobj, 'pycsw:CreationDate', md.created)
_set(context, recobj, 'pycsw:PublicationDate', md.issued)
_set(context, recobj, 'pycsw:Modified', md.modified)
_set(context, recobj, 'pycsw:Format', md.format)
_set(context, recobj, 'pycsw:Source', md.source)
for ref in md.references:
tmp = ',,%s,%s' % (ref['scheme'], ref['url'])
links.append(tmp)
for uri in md.uris:
tmp = '%s,%s,%s,%s' % \
(uri['name'], uri['description'], uri['protocol'], uri['url'])
links.append(tmp)
if len(links) > 0:
_set(context, recobj, 'pycsw:Links', '^'.join(links))
if bbox is not None:
try:
tmp = '%s,%s,%s,%s' % (bbox.minx, bbox.miny, bbox.maxx, bbox.maxy)
_set(context, recobj, 'pycsw:BoundingBox', util.bbox2wktpolygon(tmp))
except: # coordinates are corrupted, do not include
_set(context, recobj, 'pycsw:BoundingBox', None)
else:
_set(context, recobj, 'pycsw:BoundingBox', None)
return recobj