def __init__(self, element, swe_ns):
self._root = element
self._fields = []
self.definition = testXMLAttribute(self._root, "definition")
self.srs = testXMLAttribute(self._root, "referenceFrame")
for coord in self._root.findall(nsp("coordinate", swe_ns)):
self._fields.append(SweCoordinate(coord, swe_ns, self.srs, self.definition))
def __init__(self, element, ows_version='1.0.0'):
self._root = element
global _ows_version
_ows_version = ows_version
self.operations = {}
# There can be parent parameters and constraints
parameters = []
for parameter in self._root.findall(ns_ows('ows:Parameter')):
parameters.append((testXMLAttribute(parameter,'name'), {'values': [i.text for i in parameter.findall(ns_ows('.//ows:Value'))]}))
parameters = dict(parameters)
constraints = []
for constraint in self._root.findall(ns_ows('ows:Constraint')):
constraints.append((testXMLAttribute(constraint,'name'), {'values': [i.text for i in constraint.findall(ns_ows('.//ows:Value'))]}))
constraints = dict(constraints)
for op in self._root.findall(ns_ows('ows:Operation')):
co = Operation(op, ows_version)
# Parent objects get overriden by children elements
co.parameters = dict_union(parameters, co.parameters)
co.constraints = dict_union(constraints, co.constraints)
self.operations[co.name] = co
for parameter in elem.findall(util.nspath('Parameter', namespace)):
if namespace == OWS_NAMESPACE_1_1_0:
parameters.append((parameter.attrib['name'], {'values': [i.text for i in parameter.findall(util.nspath('AllowedValues/Value', namespace))]}))
else:
parameters.append((parameter.attrib['name'], {'values': [i.text for i in parameter.findall(util.nspath('Value', namespace))]}))
self.parameters = dict(parameters)
def __init__(self, element):
self.type = testXMLAttribute(element.find(nspv(
"om20:type")), nspv("xlink:href"))
self.featureOfInterest = testXMLAttribute(element.find(nspv(
"om20:featureOfInterest")), nspv("xlink:href"))
self.observedProperty = testXMLAttribute(element.find(nspv(
"om20:observedProperty")), nspv("xlink:href"))
self.procedure = testXMLAttribute(element.find(nspv(
"om20:procedure")), nspv("xlink:href"))
''' Determine if phenom time is instant or a period. This
depend on the type of observation -- this could be split out '''
instant_element = element.find(nspv(
"om20:phenomenonTime/gml32:TimeInstant"))
if instant_element is not None:
self.phenomenonTime = extract_time(instant_element)
else:
start = extract_time(element.find(nspv(
"om20:phenomenonTime/gml32:TimePeriod/gml32:beginPosition")))
end = extract_time(element.find(nspv(
"om20:phenomenonTime/gml32:TimePeriod/gml32:endPosition")))
self.phenomenonTime = TimePeriod(start, end)
self.resultTime = extract_time(element.find(nspv(
"om20:resultTime/gml32:TimeInstant/gml32:timePosition")))
self.result = element.find(nspv("om20:result"))
def __init__(self, md=None):
if md is None:
self.keywords = []
self.type = None
self.thesaurus = None
self.kwdtype_codeList = 'http://standards.iso.org/ittf/PubliclyAvailableStandards/ISO_19139_Schemas/resources/codelist/gmxCodelists.xml#MD_KeywordTypeCode'
else:
self.keywords = []
val = md.findall(util.nspath_eval('gmd:keyword/gco:CharacterString', namespaces))
for word in val:
self.keywords.append(util.testXMLValue(word))
self.type = None
val = md.find(util.nspath_eval('gmd:type/gmd:MD_KeywordTypeCode', namespaces))
self.type = util.testXMLAttribute(val, 'codeListValue')
self.thesaurus = None
val = md.find(util.nspath_eval('gmd:thesaurusName/gmd:CI_Citation', namespaces))
if val is not None:
self.thesaurus = {}
thesaurus = val.find(util.nspath_eval('gmd:title/gco:CharacterString', namespaces))
self.thesaurus['title'] = util.testXMLValue(thesaurus)
thesaurus = val.find(util.nspath_eval('gmd:date/gmd:CI_Date/gmd:date/gco:Date', namespaces))
self.thesaurus['date'] = util.testXMLValue(thesaurus)
thesaurus = val.find(util.nspath_eval('gmd:date/gmd:CI_Date/gmd:dateType/gmd:CI_DateTypeCode', namespaces))
self.thesaurus['datetype'] = util.testXMLAttribute(thesaurus, 'codeListValue')
def __init__(self, element):
super(Vector, self).__init__(element)
# Elements
self.coordinate = [Coordinate(x) for x in element.findall(nspv("swe20:coordinate"))]
# Attributes
self.referenceFrame = testXMLAttribute(element,"referenceFrame") # anyURI, required
self.localFrame = testXMLAttribute(element,"localFrame") # anyURI, optional
def __init__(self, element):
super(Matrix, self).__init__(element)
self.elementCount = element.find(nspv("swe20:elementCount/swe20:Count")) # required
self.elementType = ElementType(element.find(nspv("swe20:elementType"))) # required
self.encoding = AbstractEncoding(element.find(nspv("swe20:encoding")))
self.values = testXMLValue(element.find(nspv("swe20:values")))
self.referenceFrame = testXMLAttribute(element, "referenceFrame") # anyURI, required
self.localFrame = testXMLAttribute(element, "localFrame") # anyURI, optional
def __init__(self, element):
self._root = element
OM_NS = ns.get_versioned_namespace('om','1.0')
GML_NS = ns.get_versioned_namespace('gml','3.1.1')
SWE_NS = ns.get_versioned_namespace('swe','1.0')
XLINK_NS = ns.get_namespace("xlink")
self.name = testXMLValue(self._root.find(nsp("name", GML_NS)))
self.description = testXMLValue(self._root.find(nsp("description", GML_NS)))
self.observedProperties = []
for op in self._root.findall(nsp('observedProperty', OM_NS)):
self.observedProperties.append(testXMLAttribute(op,nsp('href', XLINK_NS)))
# BBOX
try:
envelope = self._root.find(nsp('boundedBy/Envelope', GML_NS))
lower_left_corner = testXMLValue(envelope.find(nsp('lowerCorner', GML_NS))).split(" ")
upper_right_corner = testXMLValue(envelope.find(nsp('upperCorner', GML_NS))).split(" ")
self.bbox_srs = Crs(testXMLAttribute(envelope,'srsName'))
# Always keep the BBOX as minx, miny, maxx, maxy
if self.bbox_srs.axisorder == "yx":
self.bbox = box(float(lower_left_corner[1]), float(lower_left_corner[0]), float(upper_right_corner[1]), float(upper_right_corner[0]))
else:
self.bbox = box(float(lower_left_corner[0]), float(lower_left_corner[1]), float(upper_right_corner[0]), float(upper_right_corner[1]))
except Exception:
self.bbox = None
self.bbox_srs = None
# Time range
fp = nsp('samplingTime', OM_NS)
mp = nsp('TimePeriod', GML_NS)
lp = nsp('beginPosition', GML_NS)
begin_position_element = self._root.find('%s/%s/%s' % (fp, mp, lp))
self.begin_position = extract_time(begin_position_element)
ep = nsp('endPosition', GML_NS)
end_position_element = self._root.find('%s/%s/%s' % (fp, mp, ep))
self.end_position = extract_time(end_position_element)
feature_of_interest_element = self._root.find(nsp("featureOfInterest", OM_NS))
self.feature_type = testXMLValue(feature_of_interest_element.find(nsp("FeatureCollection/metaDataProperty/name", GML_NS)))
# Now the fields change depending on the FeatureType
result_element = self._root.find(nsp("result", OM_NS))
#TODO: This should be implemented as a Factory
self.feature = None
if self.feature_type == 'timeSeries':
self.feature = SweTimeSeries(feature_of_interest=feature_of_interest_element, result=result_element, GML_NS=GML_NS, OM_NS=OM_NS, SWE_NS=SWE_NS)
elif self.feature_type == 'trajectoryProfile':
self.feature = SweTrajectoryProfile(feature_of_interest=feature_of_interest_element, result=result_element, GML_NS=GML_NS, OM_NS=OM_NS, SWE_NS=SWE_NS)
elif self.feature_type == 'timeSeriesProfile':
self.feature = SweTimeseriesProfile(feature_of_interest=feature_of_interest_element, result=result_element, GML_NS=GML_NS, OM_NS=OM_NS, SWE_NS=SWE_NS)
def __init__(self, element):
self.id = testXMLAttribute(element, nsp("gml:id"))
self.version = testXMLValue(element.find(nsp("sml:version")))
self.description = testXMLValue(element.find(nsp("gml:description")))
self.date = testXMLValue(element.find(nsp("sml:date")))
try:
self.contact = Contact(element.find(nsp("sml:contact")))
except AttributeError:
self.contact = None
self.format = testXMLValue(element.find(nsp('sml:format')))
self.url = testXMLAttribute(element.find(nsp('sml:onlineResource')), nsp('xlink:href'))
def __init__(self, element):
# Both capabilities and characteristics contain a single swe:DataRecord element
self.capabilities = {}
for cap in element.findall(nsp("sml:capabilities")):
name = testXMLAttribute(cap, "name")
self.capabilities[name] = cap[0]
self.characteristics = {}
for cha in element.findall(nsp("sml:characteristics")):
name = testXMLAttribute(cha, "name")
self.characteristics[name] = cha[0]
def __init__(self, element, swe_ns):
self._root = element
# Only supporting Text Encoding
tx = self._root.find(nsp("TextEncoding", swe_ns))
if tx is not None:
self.type = "text"
self.decimal_separator = testXMLAttribute(tx,'decimalSeparator') or "."
self.token_separator = testXMLAttribute(tx,'tokenSeparator') or ","
self.block_separator = testXMLAttribute(tx,'blockSeparator') # ignored, we use newlines
else:
raise TypeError("Only swe:TextEncoding is supported at this time")
def __init__(self, element):
super(AbstractSimpleComponent, self).__init__(element)
# Attributes
self.referenceFrame = testXMLAttribute(element,"referenceFrame") # anyURI, optional
self.axisID = testXMLAttribute(element,"axisID") # string, optional
# Elements
self.quality = [_f for _f in [Quality(q) for q in [e.find('*') for e in element.findall(nspv("swe20:quality"))] if q is not None] if _f]
try:
self.nilValues = NilValues(element.find(nspv("swe20:nilValues")))
except:
self.nilValues = None
def __init__(self, element):
super(AbstractSimpleComponent, self).__init__(element)
# Attributes
self.referenceFrame = testXMLAttribute(element,"referenceFrame") # anyURI, optional
self.axisID = testXMLAttribute(element,"axisID") # string, optional
# Elements
self.quality = filter(None, [Quality(e) for e in element.findall(nspv("swe20:quality"))])
try:
self.nilValues = NilValues(element.find(nspv("swe20:nilValues")))
except:
self.nilValues = None
def __init__(self, element):
self._root = element
self.id = testXMLAttribute(self._root,nsp('gml:id'))
self.description = testXMLValue(self._root.find(nsp('gml:description')))
self.name = testXMLValue(self._root.find(nsp('gml:name')))
self.srs = Crs(testXMLValue(self._root.find(nsp('gml:srsName'))))
# LOOK: Check on GML boundedBy to make sure we handle all of the cases
# gml:boundedBy
try:
envelope = self._root.find(nsp('gml:boundedBy/gml:Envelope'))
lower_left_corner = testXMLValue(envelope.find(nsp('gml:lowerCorner'))).split(" ")
upper_right_corner = testXMLValue(envelope.find(nsp('gml:upperCorner'))).split(" ")
# (left, bottom, right, top) in self.bbox_srs units
self.bbox = (float(lower_left_corner[1]), float(lower_left_corner[0]), float(upper_right_corner[1]), float(upper_right_corner[0]))
self.bbox_srs = Crs(testXMLAttribute(envelope,'srsName'))
except Exception:
self.bbox = None
self.bbox_srs = None
# LOOK: Support all gml:TimeGeometricPrimitivePropertyType
# Right now we are just supporting gml:TimePeriod
# sos:Time
begin_position_element = self._root.find(nsp('sos:time/gml:TimePeriod/gml:beginPosition'))
self.begin_position = extract_time(begin_position_element)
end_position_element = self._root.find(nsp('sos:time/gml:TimePeriod/gml:endPosition'))
self.end_position = extract_time(end_position_element)
self.result_model = testXMLValue(self._root.find(nsp('sos:resultModel')))
self.procedures = []
for proc in self._root.findall(nsp('sos:procedure')):
self.procedures.append(testXMLAttribute(proc,nsp('xlink:href')))
# LOOK: Support swe:Phenomenon here
# this includes compound properties
self.observed_properties = []
for op in self._root.findall(nsp('sos:observedProperty')):
self.observed_properties.append(testXMLAttribute(op,nsp('xlink:href')))
self.features_of_interest = []
for fot in self._root.findall(nsp('sos:featureOfInterest')):
self.features_of_interest.append(testXMLAttribute(fot,nsp('xlink:href')))
self.response_formats = []
for rf in self._root.findall(nsp('sos:responseFormat')):
self.response_formats.append(testXMLValue(rf))
self.response_modes = []
for rm in self._root.findall(nsp('sos:responseMode')):
self.response_modes.append(testXMLValue(rm))
def __init__(self, element):
super(TVPMeasurementMetadata, self).__init__(element)
self.uom = testXMLAttribute(element.find(nspv(
"wml2:uom")), "code")
self.interpolationType = testXMLAttribute(element.find(nspv(
"wml2:interpolationType")), nspv("xlink:href"))
self.censoredReason = testXMLAttribute(element.find(nspv(
"wml2:censoredReason")), "xlink:href")
accuracy = testXMLValue(element.find(nspv("wml2:accuracy")))
if accuracy is not None:
self.accuracy = Quantity(element)
def __init__(self, element, swe_ns):
self._root = element
# Gets whatever the first child is.
# swe:Quantity = value
ob = self._root[0]
self.name = testXMLAttribute(self._root, "name")
self.definition = testXMLAttribute(ob, "definition")
self.units = testXMLAttribute(ob.find(nsp("uom", swe_ns)), "code")
self.units_url = testXMLAttribute(ob.find(nsp("uom", swe_ns)), nspv("xlink:href"))
self.value = testXMLValue(ob.find(nsp("value", swe_ns)))
self.axis = testXMLAttribute(ob, "axisID")
def __init__(self, element):
# TODO: This only supports the sml:contact/sml:ResponsibleParty elements, but there are numerous ways to store
# contact information here.
self.role = testXMLAttribute(element, nsp("xlink:role"))
self.href = testXMLAttribute(element, nsp("xlink:href"))
self.organization = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:organizationName')))
self.phone = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:phone/sml:voice')))
self.address = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:deliveryPoint')))
self.city = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:city')))
self.region = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:administrativeArea')))
self.postcode = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:postalCode')))
self.country = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:country')))
self.email = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:electronicMailAddress')))
self.url = testXMLAttribute(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:onlineResource')), nsp("xlink:href"))
def __init__(self, element):
super(StationDS, self).__init__(element=element)
self.id = get_named_by_definition(self.system.get_identifiers_by_name("stationID"), "http://mmisw.org/ont/ioos/definition/stationID")
self.platformType = get_named_by_definition(self.system.get_classifiers_by_name("platformType"), "http://mmisw.org/ont/ioos/definition/platformType")
# Variables
self.variables = []
try:
# Using xlink:href to point to individual member sensors
self.variables = sorted(list(set([testXMLAttribute(comp, nsp("xlink:title")) for comp in self.system.components])))
except AttributeError:
# Verbose method of describing members. Pull out each individual variable definition
self.variables = sorted(list(set(itertools.chain.from_iterable([[testXMLAttribute(quan, "definition") for quan in comp.findall(".//" + nsp("swe101:Quantity"))] for comp in self.system.components]))))
def __init__(self, element):
''' Base time-value pair metadata. Still to do:
- relatedObservation
'''
self.quality = testXMLAttribute(element.find(nspv(
"wml2:quality")), nspv("xlink:href"))
self.nilReason = testXMLAttribute(element.find(nspv(
"wml2:nilReason")), nspv("xlink:href"))
self.comment = testXMLValue(element.find(nspv(
"wml2:comment")))
self.qualifier = testXMLAttribute(element.find(nspv(
"wml2:qualifier")), nspv("xlink:href"))
self.processing = testXMLValue(element.find(nspv(
"wml2:processing")), nspv("xlink:href"))
self.source = testXMLValue(element.find(nspv(
"wml2:source")), nspv("xlink:href"))
def parse_owc_content(content_node):
mimetype = util.testXMLAttribute(content_node, 'type')
url = util.testXMLAttribute(content_node, 'href')
title = util.testXMLAttribute(content_node, 'title')
child_elem = None
if len(list(content_node)) > 0:
child_elem = element_to_string(
list(content_node)[0],False)
content_dict = {
"type": mimetype,
"url": url,
"content": str(child_elem),
"title": title
}
return content_dict
def __init__(self, element):
super(StationDS, self).__init__(element=element)
self.id = self.get_ioos_def("stationID", "identifier", ont)
self.platformType = self.get_ioos_def(
"platformType", "classifier", ont
)
# Verbose method of describing members. Pull out each individual
# variable definition
self.variables = sorted(
list(
set(
itertools.chain.from_iterable(
[
[
testXMLAttribute(quan, "definition")
for quan in comp.findall(
".//" + nsp("swe101:Quantity")
)
]
for comp in self.system.components
]
)
)
)
)
# if no variables were picked up, fall back to using original SML
# components instead
if not self.variables:
self.variables = sorted(
(comp.values()[0] for comp in self.system.components)
)
def __init__(self, element):
self._root = element
self.id = testXMLAttribute(self._root, nsp('gml:id'))
self.description = testXMLValue(self._root.find(nsp('gml:description')))
self.keywords = extract_xml_list(self._root.findall(nsp('sml:keywords/sml:KeywordList/sml:keyword')))
self.documents = []
for document in self._root.findall(nsp('sml:documentation')):
self.documents.append(DocumentationMetadata(document))
self.identifiers = {}
for identifier in self._root.findall(nsp('sml:identification/sml:IdentifierList/sml:identifier')):
ident = IdentifierMetadata(identifier)
self.identifiers[ident.name] = ident
self.contacts = {}
for contact in self._root.findall(nsp('sml:contact')):
cont = ContactMetadata(contact)
self.contacts[cont.role] = cont
self.classifiers = {}
for classifier in self._root.findall(nsp('sml:classification/sml:ClassifierList/sml:classifier')):
classi = ClassifierMetadata(classifier)
self.classifiers[classi.name] = classi
self.history = {}
for event_member in self._root.findall(nsp('sml:history/sml:EventList/sml:member')):
emm = EventMemberMetadata(event_member)
if self.history.get(emm.name) is None:
self.history[emm.name] = []
self.history[emm.name] += emm.events
self.components = {}
def __init__(self, element, ows_version='1.0.0'):
self._root = element
global _ows_version
_ows_version = ows_version
self.name = testXMLValue(self._root.find(ns_ows('ows:ProviderName')))
self.url = testXMLAttribute(self._root.find(ns_ows("ows:ProviderSite")),nspath_eval('xlink:href'))
self.contact = ServiceContact(self._root.find(ns_ows("ows:ServiceContact")),ows_version)
def __init__(self, element):
self._root = element
self.name = testXMLAttribute(self._root, 'name')
self.events = []
for event in self._root.findall(nsp('sml:Event')):
self.events.append(EventMetadata(event, self.name))
def __init__(self, element):
self.history = {}
for event_member in element.findall(nsp('sml:history/sml:EventList/sml:member')):
name = testXMLAttribute(event_member, "name")
if self.history.get(name) is None:
self.history[name] = []
for e in event_member.findall(nsp("sml:Event")):
self.history[name].append(Event(e))
def __init__(self, element):
self._root = element
self.role = testXMLAttribute(self._root, nsp('xlink:role'))
self.url = testXMLAttribute(self._root, nsp('xlink:href'))
if self.url is None:
self.url = testXMLAttribute(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:onlineResource')), nsp('xlink:href'))
self.organization = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:organizationName')))
self.phone = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:phone/sml:voice')))
self.address = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:deliveryPoint')))
self.city = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:city')))
self.region = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:administrativeArea')))
self.postcode = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:postalCode')))
self.country = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:country')))
self.email = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:electronicMailAddress')))
"""
def __init__(self, element, name):
self._root = element
self.name = name
# All dates are UTC
self.date = testXMLValue(self._root.find(nsp('sml:date')))
self.description = testXMLValue(self._root.find(nsp('gml:description')))
self.documentation_url = testXMLAttribute(self._root.find(nsp('sml:documentation')), nsp('xlink:href'))
def __init__(self, element):
# No call to super(), the type object will process that.
self.name = testXMLAttribute(element, "name")
try:
self.content = eval(element[-1].tag.split("}")[-1])(element[-1])
except IndexError:
self.content = None
except BaseException:
raise
def __init__(self, **kwargs):
super(SweTimeSeries,self).__init__(**kwargs)
# Parse out GML point. Defaults to 0 depth if none specified
self.geo_srs = Crs(testXMLAttribute(self._location.find(nsp("Point", self.GML_NS)), 'srsName'))
geo = [float(v) for v in testXMLValue(self._location.find(nsp("Point/pos", self.GML_NS))).split(" ")]
if self.geo_srs.axisorder == "yx":
self.geo = sPoint(geo[1], geo[0])
else:
self.geo = sPoint(geo[0], geo[1])
try:
self.geo.z = geo[2]
except:
pass
pc = PointCollection()
for row in self.results.data:
p = Point()
time = None
z = None
lat = None
lon = None
for field in row:
if field.axis == "time":
t = dateutil.parser.parse(field.value)
p.time = t.astimezone(dateutil.tz.tzutc())
elif field.axis == "Long":
lon = field
elif field.axis == "Lat":
lat = field
elif field.axis == "h":
z = field
else:
m = Member(value=field.value,
unit=field.units,
units_definition=field.units_url,
name=field.name,
definition=field.definition,
standard=field.definition)
p.add_member(m)
# Set the spatial point
if lon.srs != lat.srs:
raise ValueError("Longitude and Latitude need to have the same SRS/CRS!")
p.location = sPoint(float(lon.value), float(lat.value), float(z.value))
pc.add_element(p)
self.data = pc
def __init__(self, element):
super(TimeRange, self).__init__(element)
# Elements
self.uom = get_uom(element.find(nspv("swe20:uom")))
try:
self.constraint = AllowedTimes(element.find(nspv("swe20:constraint/swe20:AllowedTimes"))) # AllowedTimes, min=0, max=1
except:
self.constraint = None
# Attributes
self.localFrame = testXMLAttribute(element,"localFrame") # anyURI, optional
try:
self.referenceTime = parser.parse(testXMLAttribute(element,"referenceTime")) # dateTime, optional
except (AttributeError, ValueError):
self.referenceTime = None
values = make_pair(testXMLValue(element.find(nspv("swe20:value")))) # TimePosition, min=0, max=1
self.value = [get_time(t, self.referenceTime, self.uom) for t in values]
def __init__(self, element):
super(Category, self).__init__(element)
# Elements
self.codeSpace = testXMLAttribute(element.find(nspv("swe20:codeSpace")), nspv("xlink:href")) # Reference, min=0, max=1
self.value = testXMLValue(element.find(nspv("swe20:value"))) # string, min=0, max=1
try:
self.constraint = AllowedTokens(element.find(nspv("swe20:constraint/swe20:AllowedTokens"))) # AllowedTokens, min=0, max=1
except:
self.constraint = None
def __init__(self, element):
super(Category, self).__init__(element)
# Elements
self.codeSpace = testXMLAttribute(
element.find(nspv("swe20:codeSpace")),
nspv("xlink:href")) # Reference, min=0, max=1
self.value = testXMLValue(element.find(
nspv("swe20:value"))) # string, min=0, max=1
try:
self.constraint = AllowedTokens(
element.find(nspv("swe20:constraint/swe20:AllowedTokens"))
) # AllowedTokens, min=0, max=1
except:
self.constraint = None
def __init__(self, element):
# No call to super(), the type object will process that.
self.name = testXMLAttribute(element, "name")
try:
# attempt to find a class with the same name as the XML tag parsed
# which is also contained within this module.
# Ideally the classes should be explicitly whitelisted, but I
# don't know what the set of possible classes to dispatch to should
# be
self.content = obj_mapping[element[-1].tag.split("}")[-1]](
element[-1])
except (IndexError, KeyError):
self.content = None
except BaseException:
raise
def get_field_object(element, swe_ns):
field_tag = nsp("field", swe_ns)
data_choice_tag = nsp("DataChoice", swe_ns)
if element.tag == field_tag:
name = testXMLAttribute(element, "name")
if name == "location":
return SweVector(element.find(nsp("Vector", swe_ns)), swe_ns)
elif name == "time":
return SweTime(element.find(nsp("Time", swe_ns)), swe_ns)
else:
return SweField(element, swe_ns)
elif element.tag == data_choice_tag:
return SweDataChoice(element, swe_ns)
else:
return None
def __init__(self, element):
super(StationDS, self).__init__(element=element)
self.id = get_named_by_definition(
self.system.get_identifiers_by_name("stationID"),
"http://mmisw.org/ont/ioos/definition/stationID")
self.platformType = get_named_by_definition(
self.system.get_classifiers_by_name("platformType"),
"http://mmisw.org/ont/ioos/definition/platformType")
# Verbose method of describing members. Pull out each individual variable definition
self.variables = sorted(
list(
set(
itertools.chain.from_iterable([[
testXMLAttribute(quan, "definition")
for quan in comp.findall(".//" +
nsp("swe101:Quantity"))
] for comp in self.system.components]))))
def __init__(self, element):
super(StationDS, self).__init__(element=element)
self.id = self.get_ioos_def("stationID", "identifier", ont)
self.platformType = self.get_ioos_def("platformType", "classifier", ont)
# Verbose method of describing members. Pull out each individual
# variable definition
self.variables = sorted(list(set(
itertools.chain.from_iterable(
[[testXMLAttribute(quan, "definition")
for quan in
comp.findall(".//" +
nsp("swe101:Quantity"))]
for comp in self.system.components]))))
# if no variables were picked up, fall back to using original SML
# components instead
if not self.variables:
self.variables = sorted([comp.values()[0] for
comp in self.system.components])
def __init__(self, elem, namespace=DEFAULT_OWS_NAMESPACE):
if 'name' not in elem.attrib: # This is not a valid element
return
self.name = elem.attrib['name']
self.formatOptions = ['text/xml']
parameters = []
self.methods = []
self.constraints = []
for verb in elem.findall(util.nspath('DCP/HTTP/*', namespace)):
url = util.testXMLAttribute(verb,
util.nspath('href', XLINK_NAMESPACE))
if url is not None:
verb_constraints = [
Constraint(conts, namespace) for conts in verb.findall(
util.nspath('Constraint', namespace))
]
self.methods.append({
'constraints': verb_constraints,
'type': util.xmltag_split(verb.tag),
'url': url
})
for parameter in elem.findall(util.nspath('Parameter', namespace)):
if namespace == OWS_NAMESPACE_1_1_0:
parameters.append((parameter.attrib['name'], {
'values': [
i.text for i in parameter.findall(
util.nspath('AllowedValues/Value', namespace))
]
}))
else:
parameters.append((parameter.attrib['name'], {
'values': [
i.text for i in parameter.findall(
util.nspath('Value', namespace))
]
}))
self.parameters = dict(parameters)
for constraint in elem.findall(util.nspath('Constraint', namespace)):
self.constraints.append(Constraint(constraint, namespace))
def decode_observation(self, element):
""" Returns a parsed observation of the appropriate type,
by inspecting the result element.
'element' input is the XML tree of the OM_Observation object
"""
result_element = element.find(nspath_eval("om20:result", namespaces))
if len(result_element) == 0:
result_type = testXMLAttribute(
result_element, nspath_eval("xsi:type", namespaces))
else:
result_type = list(result_element)[0].tag
if result_type.find('MeasureType') != -1:
return MeasurementObservation(element)
elif (result_type ==
'{http://www.opengis.net/waterml/2.0}MeasurementTimeseries'):
return MeasurementTimeseriesObservation(element)
else:
raise NotImplementedError('Result type {} not supported'.format(result_type))
def process_station(self, uid, offering):
""" Makes a DescribeSensor request based on a 'uid' parameter being a
station procedure. Also pass along an offering with
getCapabilities information for items such as temporal extent"""
GML_NS = "http://www.opengis.net/gml"
XLINK_NS = "http://www.w3.org/1999/xlink"
with app.app_context():
app.logger.info("process_station: %s", uid)
desc_sens = self._describe_sensor(uid, timeout=1200)
# FIXME: add some kind of notice saying the station failed
if desc_sens is None:
app.logger.warn(
"Could not get a valid describeSensor response")
return
metadata_value = etree.fromstring(desc_sens)
sensor_ml = SensorML(metadata_value)
try:
station_ds = IoosDescribeSensor(metadata_value)
# if this doesn't conform to IOOS SensorML sub, fall back to
# manually picking apart the SensorML
except ows.ExceptionReport:
station_ds = process_sensorml(sensor_ml.members[0])
unique_id = station_ds.id
if unique_id is None:
app.logger.warn(
"Could not get a 'stationID' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/stationID'"
)
return
dataset = db.Dataset.find_one({'uid': unicode(unique_id)})
if dataset is None:
dataset = db.Dataset()
dataset.uid = unicode(unique_id)
dataset['active'] = True
# Find service reference in Dataset.services and remove (to replace it)
tmp = dataset.services[:]
for d in tmp:
if d['service_id'] == self.service.get('_id'):
dataset.services.remove(d)
# Parsing messages
messages = []
# NAME
name = unicode_or_none(station_ds.shortName)
if name is None:
messages.append(
u"Could not get a 'shortName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/shortName'"
)
# DESCRIPTION
description = unicode_or_none(station_ds.longName)
if description is None:
messages.append(
u"Could not get a 'longName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/longName'"
)
# PLATFORM TYPE
asset_type = unicode_or_none(
getattr(station_ds, 'platformType', None))
if asset_type is None:
messages.append(
u"Could not get a 'platformType' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/platformType'"
)
# LOCATION is in GML
gj = None
loc = station_ds.location
if loc is not None and loc.tag == "{%s}Point" % GML_NS:
pos_element = loc.find("{%s}pos" % GML_NS)
# some older responses may uses the deprecated coordinates
# element
if pos_element is None:
# if pos not found use deprecated coordinates element
pos_element = loc.find("{%s}coordinates" % GML_NS)
# strip out points
positions = map(float, pos_element.text.split(" "))
for el in [pos_element, loc]:
srs_name = testXMLAttribute(el, "srsName")
if srs_name:
crs = Crs(srs_name)
if crs.axisorder == "yx":
gj = json.loads(
geojson.dumps(
geojson.Point([positions[1],
positions[0]])))
else:
gj = json.loads(
geojson.dumps(
geojson.Point([positions[0],
positions[1]])))
break
else:
if positions:
messages.append(
u"Position(s) found but could not parse SRS: %s, %s"
% (positions, srs_name))
else:
messages.append(
u"Found an unrecognized child of the sml:location element and did not attempt to process it: %s"
% loc)
meta_str = unicode(etree.tostring(metadata_value)).strip()
if len(meta_str) > 4000000:
messages.append(
u'Metadata document was too large to store (len: %s)' %
len(meta_str))
meta_str = u''
service = {
# Reset service
'name': name,
'description': description,
'service_type': self.service.get('service_type'),
'service_id': ObjectId(self.service.get('_id')),
'data_provider': self.service.get('data_provider'),
'metadata_type': u'sensorml',
'metadata_value': u'',
'time_min': getattr(offering, 'begin_position', None),
'time_max': getattr(offering, 'end_position', None),
'messages': map(unicode, messages),
'keywords': map(unicode, sorted(station_ds.keywords)),
'variables': map(unicode, sorted(station_ds.variables)),
'asset_type': get_common_name(asset_type),
'geojson': gj,
'updated': datetime.utcnow()
}
dataset.services.append(service)
dataset.updated = datetime.utcnow()
dataset.save()
# do compliance checker / metadata now
scores = self.ccheck_station(sensor_ml)
metamap = self.metamap_station(sensor_ml)
try:
self.save_ccheck_station('ioos', dataset._id, scores, metamap)
except Exception as e:
app.logger.warn(
"could not save compliancecheck/metamap information: %s",
e)
return "Harvest Successful"
def parseGDAReferencedElement(gdaMembers, elementName):
"""Function to parse an element of a "GetDataAvailability" member"""
element = testXMLAttribute(gdaMembers.find(nspv(elementName)),
nspv("xlink:href"))
return (element)
def decode_atomxml(xml_string):
"""
here parse atom xml to a dict for instanciating of OWC:Context
:param xmlstring:
:return: OwcContext-ready dict
"""
context_base_dict = {
"type": "FeatureCollection",
"id": None,
"bbox": None,
"properties": {
"lang": None,
"links": {
"profiles": [],
"via": [],
},
'title': None,
'abstract': None,
'updated': None,
'authors': [],
'publisher': None,
'generator': None,
'display': None,
'rights': None,
'date': None,
'categories': [],
},
'features': []
}
feed_root = etree.fromstring(xml_string)
# feed_root = etree.parse(xml_bytes)
# feed_root xml lang use?
# # log.debug(feed_root)
# feed xml:lang=en
# lang = feed_root.get('{http://www.w3.org/XML/1998/namespace}lang')
lang = util.testXMLAttribute(feed_root,
'{http://www.w3.org/XML/1998/namespace}lang')
# log.debug("lang %s ", lang)
context_base_dict['properties'].update({"lang": lang})
# <id>
val = feed_root.find(util.nspath_eval('atom:id', ns))
id = util.testXMLValue(val)
# log.debug("id %s :: %s", id, val)
context_base_dict.update({"id": id})
# <link rel="profile"
# href="http://www.opengis.net/spec/owc-atom/1.0/req/core"
# title="compliant bla bla"
# < link rel = "via" type = "application/xml" href = "..." title = "..."
vals = feed_root.findall(util.nspath_eval('atom:link', ns))
links_profile = []
links_via = []
for val in vals:
rel = util.testXMLAttribute(val, 'rel')
href = util.testXMLAttribute(val, 'href')
mimetype = util.testXMLAttribute(val, 'type')
lang = util.testXMLAttribute(val, 'lang')
title = util.testXMLAttribute(val, 'title')
length = util.testXMLAttribute(val, 'length')
link = {
"href": href,
"type": mimetype,
"length": length,
"lang": lang,
"title": title,
"rel": rel
}
# log.debug("link %s :: %s", link, vals)
if link.get("rel") == "profile" and not is_empty(link):
links_profile.append(link)
elif link.get("rel") == "via" and not is_empty(link):
links_via.append(link)
else:
log.warning("unknown link type in Ows Context section: %r", link)
context_base_dict['properties']['links'].update(
{"profiles": links_profile})
context_base_dict['properties']['links'].update({"via": links_via})
# <title>
val = feed_root.find(util.nspath_eval('atom:title', ns))
title = util.testXMLValue(val)
# log.debug("title %s :: %s", title, val)
context_base_dict['properties'].update({"title": title})
# <subtitle type = "html"
val = feed_root.find(util.nspath_eval('atom:subtitle', ns))
subtitle = util.testXMLValue(val)
# log.debug("subtitle %s :: %s", subtitle, val)
context_base_dict['properties'].update({"abstract": subtitle})
# <author> ..
# <name>
# <email>
vals = feed_root.findall(util.nspath_eval('atom:author', ns))
authors = []
for val in vals:
val_name = val.find(util.nspath_eval('atom:name', ns))
val_email = val.find(util.nspath_eval('atom:email', ns))
val_uri = val.find(util.nspath_eval('atom:uri', ns))
name = util.testXMLValue(val_name)
email = util.testXMLValue(val_email)
uri = util.testXMLValue(val_uri)
author = {"name": name, "email": email, "uri": uri}
# log.debug("author %s :: %s", author, vals)
if not is_empty(author):
authors.append(author)
context_base_dict['properties'].update({"authors": authors})
# <georss:where>
val = feed_root.find(util.nspath_eval('georss:where', ns))
if val is not None:
if len(list(val)) > 0:
xmltxt = element_to_string(list(val)[0], False)
# log.debug("geometry %s :: %s", xmltxt, val)
context_base_dict['properties'].update(
{"bbox": xmltxt.decode('utf-8')})
# <updated>2012-11-04T17:26:23Z</updated>
val = feed_root.find(util.nspath_eval('atom:updated', ns))
update_date = util.testXMLValue(val)
# log.debug("updated %s :: %s", update_date, val)
context_base_dict['properties'].update({"updated": update_date})
# <dc:date>2009-01-23T09:08:56.000Z/2009-01-23T09:14:08.000Z</dc:date>
val = feed_root.find(util.nspath_eval('dc:date', ns))
time_interval_of_interest = util.testXMLValue(val)
# log.debug("dc:date %s :: %s", time_interval_of_interest, val)
context_base_dict['properties'].update({"date": time_interval_of_interest})
# <rights>
val = feed_root.find(util.nspath_eval('atom:rights', ns))
rights = util.testXMLValue(val)
# log.debug("rights %s :: %s", rights, val)
context_base_dict['properties'].update({"rights": rights})
# <dc:publisher>
val = feed_root.find(util.nspath_eval('dc:publisher', ns))
publisher = util.testXMLValue(val)
# log.debug("dc:publisher %s :: %s", publisher, val)
context_base_dict['properties'].update({"publisher": publisher})
# <owc:display>
# <owc:pixelWidth>
val_display = feed_root.find(util.nspath_eval('owc:display', ns))
val_pixel_width = None if val_display is None \
else val_display.find(util.nspath_eval('owc:pixelWidth', ns))
val_pixel_height = None if val_display is None \
else val_display.find(util.nspath_eval('owc:pixelHeight', ns))
val_mm_per_pixel = None if val_display is None \
else val_display.find(util.nspath_eval('owc:mmPerPixel', ns))
pixel_width = util.testXMLValue(val_pixel_width)
pixel_height = util.testXMLValue(val_pixel_height)
mm_per_pixel = util.testXMLValue(val_mm_per_pixel)
owc_display = {
"pixelWidth": pixel_width,
"pixelHeight": pixel_height,
"mmPerPixel": mm_per_pixel
}
# log.debug("display %s :: %s", owc_display, val_display)
if not is_empty(owc_display):
context_base_dict['properties'].update({"display": owc_display})
# <generator uri="http://w.." version="1.0">MiraMon</generator>
val = feed_root.find(util.nspath_eval('atom:generator', ns))
name = util.testXMLValue(val)
version = util.testXMLAttribute(val, 'version')
uri = util.testXMLAttribute(val, 'uri')
owc_generator = {"name": name, "version": version, "uri": uri}
# log.debug("generator %s :: %s", owc_generator, val)
if not is_empty(owc_generator):
context_base_dict['properties'].update({"generator": owc_generator})
# <category term="maps" label="This file contains maps"/>
vals = feed_root.findall(util.nspath_eval('atom:category', ns))
categories = []
for val in vals:
term = util.testXMLAttribute(val, 'term')
scheme = util.testXMLAttribute(val, 'scheme')
label = util.testXMLAttribute(val, 'label')
category = {"term": term, "scheme": scheme, "label": label}
# log.debug("category %s :: %s", category, vals)
if not is_empty(category):
categories.append(category)
context_base_dict['properties'].update({"categories": categories})
# <entry> ...
entries = feed_root.findall(util.nspath_eval('atom:entry', ns))
resources = []
for entry in entries:
entry_dict = parse_entry(entry)
if entry_dict.get("id") is not None:
resources.append(entry_dict)
else:
log.warning("feature entry has no id, not allowed: skipping!")
context_base_dict.update({"features": resources})
return context_base_dict
def parse_entry(entry_node):
"""
parse an aotm entry into a feature/resource dict to build OwcResource from
:param entry_node: xml element root node of the atom:entry
:return: dictionary for OwcResource.from_dict()
"""
resource_base_dict = {
"type": "Feature",
"id": None,
"geometry": None,
"properties": {
'title': None,
'abstract': None,
'updated': None,
'date': None,
'authors': [],
'publisher': None,
'rights': None,
'categories': [],
"links": {
"alternates": [],
"previews": [],
"data": [],
"via": [],
},
'offerings': [],
'active': None,
'minscaledenominator': None,
'maxscaledenominator': None,
'folder': None
}
}
# <id>ftp://ftp.remotesensing.org/pub/geotiff/samples/gdal_eg/cea.txt</id>
val = entry_node.find(util.nspath_eval('atom:id', ns))
id = util.testXMLValue(val)
# log.debug("entry :id %s :: %s", id, val)
resource_base_dict.update({"id": id})
# <title>GeoTIFF Example</title>
val = entry_node.find(util.nspath_eval('atom:title', ns))
title = util.testXMLValue(val)
# log.debug("entry: title %s :: %s", id, val)
resource_base_dict['properties'].update({"title": title})
# <updated>2011-11-01T00:00:00Z</updated>
val = entry_node.find(util.nspath_eval('atom:updated', ns))
update_date = util.testXMLValue(val)
# log.debug("entry: updated %s :: %s", update_date, val)
resource_base_dict['properties'].update({"updated": update_date})
# <dc:publisher>
val = entry_node.find(util.nspath_eval('dc:publisher', ns))
publisher = util.testXMLValue(val)
# log.debug("entry: dc:publisher %s :: %s", publisher, val)
resource_base_dict['properties'].update({"publisher": publisher})
# <dc:rights>
val = entry_node.find(util.nspath_eval('dc:rights', ns))
rights = util.testXMLValue(val)
# log.debug("entry: rights %s :: %s", rights, val)
resource_base_dict['properties'].update({"rights": rights})
# <georss:where>
val = entry_node.find(util.nspath_eval('georss:where', ns))
if val is not None:
if len(list(val)) > 0:
# xmltxt = etree.tostring(
# list(val)[0], encoding='utf8', method='xml')
xmltxt = element_to_string(list(val)[0], False)
# TODO here parse geometry??
# log.debug("entry: geometry %s :: %s", xmltxt, val)
resource_base_dict.update({"geometry": xmltxt.decode('utf-8')})
# <content type = "text" > aka subtitle, aka abstract
val = entry_node.find(util.nspath_eval('atom:content', ns))
subtitle = util.testXMLValue(val)
# log.debug("entry: subtitle %s :: %s", subtitle, val)
resource_base_dict['properties'].update({"abstract": subtitle})
# <author> ..
# <name>
# <email>
vals = entry_node.findall(util.nspath_eval('atom:author', ns))
authors = []
for val in vals:
val_name = val.find(util.nspath_eval('atom:name', ns))
val_email = val.find(util.nspath_eval('atom:email', ns))
val_uri = val.find(util.nspath_eval('atom:uri', ns))
name = util.testXMLValue(val_name)
email = util.testXMLValue(val_email)
uri = util.testXMLValue(val_uri)
author = {"name": name, "email": email, "uri": uri}
# log.debug("entry: author %s :: %s", author, vals)
if not is_empty(author):
authors.append(author)
resource_base_dict['properties'].update({"authors": authors})
# <link rel="enclosure" type="image/png"
# length="12345" title="..." href="http://o..."/>
# <link rel="icon" type="image/png" title="Preview f..."
# href="http://..."/>
# <link rel="via" type="application/vnd.ogc.wms_xml"
# title="Original .." href="...."/>
vals = entry_node.findall(util.nspath_eval('atom:link', ns))
links_alternates = []
links_previews = []
links_data = []
links_via = []
for val in vals:
rel = util.testXMLAttribute(val, 'rel')
href = util.testXMLAttribute(val, 'href')
mimetype = util.testXMLAttribute(val, 'type')
lang = util.testXMLAttribute(val, 'lang')
title = util.testXMLAttribute(val, 'title')
length = util.testXMLAttribute(val, 'length')
link = {
"href": href,
"type": mimetype,
"length": length,
"lang": lang,
"title": title,
"rel": rel
}
# log.debug("entry: link %s :: %s", link, vals)
if link.get("rel") == "alternate" and not is_empty(link):
links_alternates.append(link)
elif link.get("rel") == "icon" and not is_empty(link):
links_previews.append(link)
elif link.get("rel") == "enclosure" and not is_empty(link):
links_data.append(link)
elif link.get("rel") == "via" and not is_empty(link):
links_via.append(link)
else:
log.warning("unknown link type in Ows Resource entry section: %r",
link)
resource_base_dict['properties']['links'].update(
{"alternates": links_alternates})
resource_base_dict['properties']['links'].update(
{"previews": links_previews})
resource_base_dict['properties']['links'].update({"data": links_data})
resource_base_dict['properties']['links'].update({"via": links_via})
# <owc:offering code="http://www.opengis.net/spec/owc-at...">
# <owc:content type="image/tiff" href=".."
# <owc:offering code="http://www.opengis.net/spec....l">
# <owc:content type="application/gml+xml">
# <owc:operation code="GetCapabilities" method="GET"
# type="applica..." href="..."
# <owc:request type="application/xml"> ..
# <owc:styleSet>
# <owc:name>raster</owc:name>
# <owc:title>Default Raster</owc:title>
# <owc:abstract>A sample style that draws a </owc:abstract>
# <owc:legendURL href="h...." type="image/png"/>
# </owc:styleSet>
offering_nodes = entry_node.findall(util.nspath_eval('owc:offering', ns))
offerings = []
for offering_node in offering_nodes:
offering_code = util.testXMLAttribute(offering_node, 'code')
operations = []
contents = []
styles = []
operation_nodes = offering_node.findall(
util.nspath_eval('owc:operation', ns))
for op_val in operation_nodes:
operations_code = util.testXMLAttribute(op_val, 'code')
http_method = util.testXMLAttribute(op_val, 'method')
mimetype = util.testXMLAttribute(op_val, 'type')
request_url = util.testXMLAttribute(op_val, 'href')
req_content_val = val.find(util.nspath_eval('owc:request', ns))
req_content = None
if req_content_val is not None:
req_content = parse_owc_content(req_content_val)
# TODO no example for result/response
op_dict = {
"code": operations_code,
"method": http_method,
"type": mimetype,
"href": request_url,
"request": None if is_empty(req_content) else req_content,
"result": None
}
# log.debug("entry: operation %s :: %s", op_dict, vals)
if not is_empty(op_dict):
operations.append(op_dict)
content_nodes = offering_node.findall(
util.nspath_eval('owc:content', ns))
for cont_val in content_nodes:
content_dict = parse_owc_content(cont_val)
# log.debug("entry: content_dict %s :: %s", content_dict, vals)
if not is_empty(content_dict):
contents.append(content_dict)
style_nodes = offering_node.findall(
util.nspath_eval('owc:styleSet', ns))
for style_val in style_nodes:
val_name = style_val.find(util.nspath_eval('owc:name', ns))
val_title = style_val.find(util.nspath_eval('owc:title', ns))
val_abstr = style_val.find(util.nspath_eval('owc:abstract', ns))
val_uri = style_val.find(util.nspath_eval('owc:legendURL', ns))
name = util.testXMLValue(val_name)
title = util.testXMLValue(val_title)
abstr = util.testXMLValue(val_abstr)
legend_url = util.testXMLAttribute(val_uri, 'href')
style_set = {
"name": name,
"title": title,
"abstract": abstr,
"default": None,
"legendURL": legend_url,
"content": None
}
# log.debug("entry: style_set %s :: %s", style_set, vals)
if not is_empty(style_set):
styles.append(style_set)
offering_dict = {
"code": offering_code,
"operations": operations,
"contents": contents,
"styles": styles
}
if offering_code is not None:
offerings.append(offering_dict)
resource_base_dict['properties'].update({"offerings": offerings})
# TODO no examples for active attribute
# <owc:minScaleDenominator>2500</owc:minScaleDenominator>
val = entry_node.find(util.nspath_eval('owc:minScaleDenominator', ns))
min_scale_denominator = util.testXMLValue(val)
# log.debug("entry: min-scale-... %s :: %s", min_scale_denominator, val)
resource_base_dict['properties'].update(
{"minscaledenominator": min_scale_denominator})
# <owc:maxScaleDenominator>25000</owc:maxScaleDenominator>
val = entry_node.find(util.nspath_eval('owc:maxScaleDenominator', ns))
max_scale_denominator = util.testXMLValue(val)
# log.debug("entry: max_scale_... %s :: %s", max_scale_denominator, val)
resource_base_dict['properties'].update(
{"maxscaledenominator": max_scale_denominator})
# TODO no examples for folder attribute
return resource_base_dict
def __init__(self, element):
self.id = testXMLAttribute(element, nsp('gml:id'))
self.privacyAct = testXMLAttribute(element, nsp('sml:privacyAct'))
self.intellectualPropertyRights = testXMLAttribute(element, nsp('sml:intellectualPropertyRights'))
self.copyRights = testXMLAttribute(element, nsp('sml:copyRights'))
self.documentation = [Documentation(x) for x in element.findall(nsp("sml:documentation"))]
def __init__(self, md=None):
if md is None:
self.xml = None
self.identifier = None
self.parentidentifier = None
self.language = None
self.dataseturi = None
self.languagecode = None
self.datestamp = None
self.charset = None
self.hierarchy = None
self.contact = []
self.datetimestamp = None
self.stdname = None
self.stdver = None
self.referencesystem = None
self.identification = None
self.serviceidentification = None
self.identificationinfo = []
self.distribution = None
self.dataquality = None
else:
if hasattr(md, 'getroot'): # standalone document
self.xml = etree.tostring(md.getroot())
else: # part of a larger document
self.xml = etree.tostring(md)
val = md.find(
util.nspath_eval('gmd:fileIdentifier/gco:CharacterString',
namespaces))
self.identifier = util.testXMLValue(val)
val = md.find(
util.nspath_eval('gmd:parentIdentifier/gco:CharacterString',
namespaces))
self.parentidentifier = util.testXMLValue(val)
val = md.find(
util.nspath_eval('gmd:language/gco:CharacterString',
namespaces))
self.language = util.testXMLValue(val)
val = md.find(
util.nspath_eval('gmd:dataSetURI/gco:CharacterString',
namespaces))
self.dataseturi = util.testXMLValue(val)
val = md.find(
util.nspath_eval('gmd:language/gmd:LanguageCode', namespaces))
self.languagecode = util.testXMLAttribute(val, 'codeListValue')
val = md.find(
util.nspath_eval('gmd:dateStamp/gco:Date', namespaces))
self.datestamp = util.testXMLValue(val)
if not self.datestamp:
val = md.find(
util.nspath_eval('gmd:dateStamp/gco:DateTime', namespaces))
self.datestamp = util.testXMLValue(val)
self.charset = _testCodeListValue(
md.find(
util.nspath_eval(
'gmd:characterSet/gmd:MD_CharacterSetCode',
namespaces)))
self.hierarchy = _testCodeListValue(
md.find(
util.nspath_eval('gmd:hierarchyLevel/gmd:MD_ScopeCode',
namespaces)))
self.contact = []
for i in md.findall(
util.nspath_eval('gmd:contact/gmd:CI_ResponsibleParty',
namespaces)):
o = CI_ResponsibleParty(i)
self.contact.append(o)
val = md.find(
util.nspath_eval('gmd:dateStamp/gco:DateTime', namespaces))
self.datetimestamp = util.testXMLValue(val)
val = md.find(
util.nspath_eval(
'gmd:metadataStandardName/gco:CharacterString',
namespaces))
self.stdname = util.testXMLValue(val)
val = md.find(
util.nspath_eval(
'gmd:metadataStandardVersion/gco:CharacterString',
namespaces))
self.stdver = util.testXMLValue(val)
val = md.find(
util.nspath_eval(
'gmd:referenceSystemInfo/gmd:MD_ReferenceSystem',
namespaces))
if val is not None:
self.referencesystem = MD_ReferenceSystem(val)
else:
self.referencesystem = None
# TODO: merge .identificationinfo into .identification
warnings.warn(
'the .identification and .serviceidentification properties will merge into '
'.identification being a list of properties. This is currently implemented '
'in .identificationinfo. '
'Please see https://github.com/geopython/OWSLib/issues/38 for more information',
FutureWarning)
val = md.find(
util.nspath_eval(
'gmd:identificationInfo/gmd:MD_DataIdentification',
namespaces))
val2 = md.find(
util.nspath_eval(
'gmd:identificationInfo/srv:SV_ServiceIdentification',
namespaces))
if val is not None:
self.identification = MD_DataIdentification(val, 'dataset')
self.serviceidentification = None
elif val2 is not None:
self.identification = MD_DataIdentification(val2, 'service')
self.serviceidentification = SV_ServiceIdentification(val2)
else:
self.identification = None
self.serviceidentification = None
self.identificationinfo = []
for idinfo in md.findall(
util.nspath_eval('gmd:identificationInfo', namespaces)):
if len(idinfo) > 0:
val = list(idinfo)[0]
tagval = util.xmltag_split(val.tag)
if tagval == 'MD_DataIdentification':
self.identificationinfo.append(
MD_DataIdentification(val, 'dataset'))
elif tagval == 'MD_ServiceIdentification':
self.identificationinfo.append(
MD_DataIdentification(val, 'service'))
elif tagval == 'SV_ServiceIdentification':
self.identificationinfo.append(
SV_ServiceIdentification(val))
val = md.find(
util.nspath_eval('gmd:distributionInfo/gmd:MD_Distribution',
namespaces))
if val is not None:
self.distribution = MD_Distribution(val)
else:
self.distribution = None
val = md.find(
util.nspath_eval('gmd:dataQualityInfo/gmd:DQ_DataQuality',
namespaces))
if val is not None:
self.dataquality = DQ_DataQuality(val)
else:
self.dataquality = None
def __init__(self, element):
self.name = testXMLAttribute(element, "name")
self.interface_definition = InterfaceDefinition(element.find(nsp("sml:InterfaceDefinition")))
def __init__(self, element):
super(AbstractDataComponent, self).__init__(element)
# Attributes
self.definition = testXMLAttribute(element,"definition") # anyURI, required
self.updatable = get_boolean(testXMLAttribute(element,"updatable")) # boolean, optional
self.optional = get_boolean(testXMLAttribute(element,"optional")) or False # boolean, default=False
def __init__(self, element):
self.codeSpace = testXMLAttribute(element.find(nsp('sml:Term/sml:codeSpace')), nsp("xlink:href"))
self.definition = testXMLAttribute(element.find(nsp('sml:Term')), "definition")
self.value = testXMLValue(element.find(nsp('sml:Term/sml:value')))
def __init__(self, element):
Term.__init__(self, element)
self.name = testXMLAttribute(element, "name")
def get_stations_df(self, sos_url, station_urns_sel=None):
""" Returns a Pandas Dataframe
"""
# oFrmts: IOOS SOS OutputFormat strings (first is compliant to the IOOS SOS spec,
# second is to accommodate NDBC). More info here:
# http://ioos.github.io/sos-guidelines/doc/wsdd/sos_wsdd_github_notoc/#describesensor-request:638e0b263020c13a76a55332bd966dbe
oFrmts = [
'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"',
'text/xml;subtype="sensorML/1.0.1"'
]
params = {
'service': 'SOS',
'request': 'GetCapabilities',
'acceptVersions': '1.0.0'
}
sos_url_params = sos_url + '?' + urlencode(params)
# sos_url_params_quoted = quote(sos_url_params,"/=:")
# sos_url_params_unquoted = unquote(sos_url_params)
try:
sosgc = SensorObservationService(sos_url_params)
except (ConnectionError, ReadTimeout) as e:
self.log.write(
u"\nError: unable to connect to SOS service: {url} due to HTTP connection error."
.format(url=sos_url_params))
self.log.write(
u"\nHTTP connection error: {err}.".format(err=str(e)))
sys.exit(
"\nError: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}"
.format(url=sos_url_params, err=str(e)))
# vars to store returns from sos_collector.metadata_plus_exceptions function:
sml_recs = {}
sml_errors = {}
describe_sensor_url = {}
# leverage Pyoos Collector to query for all available stations and obtain SensorML
# (if station subset not passed in --stations param)
if station_urns_sel is not None:
station_urns = station_urns_sel
else:
sos_collector = IoosSweSos(sos_url)
station_urns = [
urn.name for urn in sos_collector.server.offerings
if 'network' not in urn.name.split(':')
]
sos_collector.features = station_urns
# write out stations in SOS that will be handled:
if self.verbose:
self.log.write(u"\nStations to process for SOS: {sos}".format(
sos=sos_url_params))
print("Stations to process for SOS: {sos}".format(
sos=sos_url_params))
for feature in sos_collector.features:
self.log.write(u"\n - {feature}".format(feature=feature))
print(" - {feature}".format(feature=feature))
# iterate over possible oFrmts expected of the various SOS services (IOOS SOS 1.0, NDBC):
# for fmt in reversed(oFrmts):
for fmt in oFrmts:
try:
sml_recs, sml_errors = sos_collector.metadata_plus_exceptions(
output_format=fmt, timeout=200)
# if no valid SensorML docs returned, try next oFrmt:
if not sml_recs:
continue
else:
# assign correct DescribeSensor url (use sos_collector.feature rather than sml_recs.keys() to
# create DescribeSensor URLs for failures to record in logs):
for station in sos_collector.features:
describe_sensor_url[
station] = self.generate_describe_sensor_url(
sosgc, procedure=station, oFrmt=fmt)
# report on errors returned from metadata_plus_exceptions:
if sml_errors:
if self.verbose:
for station, msg in iteritems(sml_errors):
self.log.write(
u"\nSOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}"
.format(station=station, msg=msg))
print(
"SOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}"
.format(station=station, msg=msg))
else:
self.log.write(
u"\nSuccess, no errors returned from DescribeSensor requests in service: {sos}"
.format(sos=sos_url_params))
print(
"Success, no errors returned from DescribeSensor requests in service: {sos}"
.format(sos=sos_url_params))
break
# ServiceException shouldn't be thrown by metadata_plus_exceptions function, but handle regardless by attempting next oFrmt:
except ServiceException as e:
continue
station_recs = []
failures = []
# generate Pandas DataFrame by populating 'station_recs' list by parsing SensorML strings:
for station_idx, station_urn in enumerate(station_urns):
if station_urns_sel is not None:
# iterate oFrmts items for describe_sensor request (first is IOOS SOS spec-compliant, second is for NDBC SOS):
for fmt in oFrmts:
try:
describe_sensor_url[
station_urn] = self.generate_describe_sensor_url(
sosgc, procedure=station_urn, oFrmt=fmt)
sml_str = sosgc.describe_sensor(procedure=station_urn,
outputFormat=fmt,
timeout=200)
break
except ServiceException as e:
sml_errors[station_urn] = str(e)
continue
sml = SensorML(sml_str)
else:
# process valid SensorML responses, quietly pass on invalid stations (add to failures list for verbose reporting):
try:
sml = sml_recs[station_urn]
except KeyError:
self.log.write(
u"\n\nStation: {station} failed (no SensorML in sml_recs dict). URL: {ds}"
.format(station=station_urn,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
print(
"Station: {station} failed (no SensorML in sml_recs dict). URL: {ds}"
.format(station=station_urn,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
failures.append(station_urn)
continue
if self.sos_type.lower() == 'ndbc':
# later: add an error check
sosgc_station_offering = sosgc.contents[
'station-' + station_urn.split(':')[-1]]
else:
sosgc_station_offering = None
try:
ds = IoosDescribeSensor(sml._root)
except AttributeError:
self.log.write(
u"\nInvalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: "
.format(station=station,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
print(
"Invalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: "
.format(station=station,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
station = OrderedDict()
# debug:
if self.verbose:
self.log.write(u"\n\nProcessing station: {station}".format(
station=station_urn))
print("Processing station: {station}".format(
station=station_urn))
self.log.write("\n" +
etree.tostring(sml._root).decode('utf-8'))
# assign 'pos' to GML point location (accommodate 'gml:coordinates' as used by NDBC if gml:Point not found):
try:
pos = testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) \
if testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) is not None \
else testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:coordinates')))
station['lon'] = float(pos.split()[1])
station['lat'] = float(pos.split()[0])
except AttributeError as e:
station['lon'] = None
station['lat'] = None
system_el = sml._root.findall(self.nsp('sml:member'))[0].find(
self.nsp('sml:System'))
# Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication
# errors but there is not enough information in SensorML for alternatives)
# Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage"
documents = system_el.findall(
self.nsp('sml:documentation/sml:DocumentList/sml:member'))
documents_dct = {}
for d in documents:
document = Documentation(d)
name = testXMLAttribute(d, "name")
# url = document.documents[0].url
documents_dct[name] = document
# obtain list of contacts (accommodate 'sml:contact' element repetition used by NDBC insead of ContactList):
contacts = system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \
if system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \
else system_el.findall(self.nsp('sml:contact'))
contacts_dct = {}
for c in contacts:
contact = Contact(c)
role = contact.role.split('/')[-1]
contacts_dct[role] = contact
# verify a 'publisher' Contact exists (template expects one):
if "publisher" not in contacts_dct.keys():
self.log.write(
u"\n\nStation: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]"
.format(station=station_urn,
roles=", ".join(contacts_dct.keys())))
print(
"Station: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]"
.format(station=station_urn,
roles=", ".join(contacts_dct.keys())))
failures.append(station_urn)
continue
sweQuants = system_el.findall(
self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity'))
quant_lst = [
sweQuant.attrib['definition'] for sweQuant in sweQuants
]
parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst]
# attempt to read beginPosition, if available, otherwise use current date
# bc ISO requires date value in output location in template:
beginPosition = testXMLValue(
system_el.find(
self.nsp(
'sml:validTime/gml:TimePeriod/gml:beginPosition')))
try:
begin_service_date = parser.parse(beginPosition)
except (AttributeError, TypeError) as e:
begin_service_date = datetime.now(pytz.utc)
station['station_urn'] = station_urn
station['sos_url'] = sos_url_params
station['describesensor_url'] = describe_sensor_url[station_urn]
station['shortName'] = ds.shortName
station['longName'] = ds.longName
if self.sos_type.lower() == 'ndbc':
station['wmoID'] = station_urn.split(':')[-1]
else:
station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont)
station['serverName'] = self.server_name
# Some capabilities-level metadata:
station['title'] = sosgc.identification.title
station['abstract'] = sosgc.identification.abstract
station['keywords'] = sosgc.identification.keywords
station['begin_service_date'] = begin_service_date
# Beware that a station can have >1 classifier of the same type
# This code does not accommodate that possibility
station['platformType'] = ds.platformType
station['parentNetwork'] = ds.get_ioos_def('parentNetwork',
'classifier', ont)
station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont)
# store some nested dictionaries in 'station' for appropriate SensorML sources:
station['contacts_dct'] = contacts_dct
station['documents_dct'] = documents_dct
if self.sos_type.lower(
) == 'ndbc' and sosgc_station_offering is not None:
station['starting'] = sosgc_station_offering.begin_position
station['ending'] = sosgc_station_offering.end_position
else:
station['starting'] = ds.starting
station['ending'] = ds.ending
if self.sos_type.lower(
) == 'ndbc' and sosgc_station_offering is not None:
station[
'variable_uris'] = sosgc_station_offering.observed_properties
station['variables'] = [
var.split('/')[-1]
for var in sosgc_station_offering.observed_properties
]
station['parameter_uris'] = ','.join(station['variable_uris'])
station['parameters'] = ','.join(station['variables'])
else:
station['variable_uris'] = ds.variables
station['variables'] = [
var.split('/')[-1] for var in ds.variables
]
station['parameter_uris'] = ','.join(quant_lst)
station['parameters'] = ','.join(parameter_lst)
if self.verbose:
for var in station['variable_uris']:
self.log.write(u"\nvariable: {var}".format(var=var))
print("variable: {var}".format(var=var))
# print(sosgc.contents)
# for id, offering in sosgc.contents.iteritems():
# print("sosgc.contents: {item}".format(item=id))
# parse 'responseFormat' vals from SensorML:
# response_formats = sosgc.contents[station_urn].response_formats
response_formats = []
for id, sosgc.content in sosgc.contents.items():
if sosgc.content.name == station_urn:
response_formats = sosgc.content.response_formats
# response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items()
# if sosgc.content.name == station_urn ]
# match responseFormats from SensorML (response_formats) against those passed in --response_formats parameter to
# populate 'download_formats' list, that is then used to generate GetObservation requests for the template:
# (default --response_formats values are: 'application/json,application/zip; subtype=x-netcdf' )
download_formats = [
response_format for response_format in response_formats
if response_format in self.response_formats
]
station['response_formats'] = response_formats
station['download_formats'] = download_formats
if self.verbose:
for format in response_formats:
self.log.write(
u"\nresponseFormat: {format}".format(format=format))
print("responseFormat: {format}".format(format=format))
for format in download_formats:
self.log.write(
u"\ndownloadFormats: {format}".format(format=format))
print("downloadFormats: {format}".format(format=format))
# calculate event_time using self.getobs_req_hours:
event_time_formatstr = "{begin:%Y-%m-%dT%H:%M:%S}{utc_code}/{end:%Y-%m-%dT%H:%M:%S}{utc_code}"
utc_code = 'Z' if self.sos_type.lower() == 'ndbc' else ''
if station['starting'] is not None and station[
'ending'] is not None:
event_time = event_time_formatstr.format(
begin=station['ending'] -
timedelta(hours=self.getobs_req_hours),
end=station['ending'],
utc_code=utc_code)
if self.verbose:
self.log.write(
u"\nUsing starting/ending times from SensorML for eventTime"
)
print(
"Using starting/ending times from SensorML for eventTime"
)
self.log.write(
u"\nobservationTimeRange: starting: {start}, ending: {end}"
.format(start=station['starting'],
end=station['ending']))
print(
"observationTimeRange: starting: {start}, ending: {end}"
.format(start=station['starting'],
end=station['ending']))
else:
now = datetime.now(pytz.utc)
then = now - timedelta(hours=self.getobs_req_hours)
event_time = event_time_formatstr.format(begin=then,
end=now,
utc_code=utc_code)
if self.verbose:
self.log.write(
u"\nNo 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
print(
"No 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
if self.verbose:
self.log.write(u"\neventTime: {time}".format(time=event_time))
print("eventTime: {time}".format(time=event_time))
# create a dict to store parameters for valid example GetObservation requests for station:
getobs_req_dct = {}
# populate a parameters dictionary for download links for each 'observedProperty' type
# and secondly for each 'responseFormat' per observedProperty:
getobs_params_base = {
'service': 'SOS',
'request': 'GetObservation',
'version': '1.0.0',
'offering': station_urn,
'eventTime': event_time
}
for variable in station['variable_uris']:
getobs_params = getobs_params_base.copy()
getobs_params['observedProperty'] = variable
variable = variable.split('/')[-1]
for format in download_formats:
getobs_params['responseFormat'] = format
getobs_request_url_encoded = sos_url + '?' + urlencode(
getobs_params)
getobs_request_url = unquote(getobs_request_url_encoded)
getobs_req_dct[variable + '-' + format] = {
'variable':
variable,
'url':
getobs_request_url,
'format_type':
self.RESPONSE_FORMAT_TYPE_MAP.get(format, format),
'format_name':
self.RESPONSE_FORMAT_NAME_MAP.get(format, format)
}
if self.verbose:
self.log.write(
u"\ngetobs_request_url (var: {variable}): {getobs_request_url}"
.format(variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url))
print(
"getobs_request_url (var: {variable}): {getobs_request_url}"
.format(variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url))
# ToDo: finish adding the 'getobs_req_dct' to the output template
station['getobs_req_dct'] = getobs_req_dct
station_recs.append(station)
# extra debug for failed stations in verbose mode:
if self.verbose:
self.log.write(
u"\n\n\nSOS DescribeSensor request errors recap. Failed requests:"
)
print("SOS DescribeSensor request errors recap. Failed requests:")
for station_fail, msg in iteritems(sml_errors):
self.log.write(
u"\n{station} - {msg}. DescribeSensor URL: {ds}".format(
station=station_fail,
msg=msg,
ds=describe_sensor_url[station_fail].replace(
"&", "&")))
print("{station} - {msg}. DescribeSensor URL: {ds}".format(
station=station_fail,
msg=msg,
ds=describe_sensor_url[station_fail].replace("&",
"&")))
if failures:
self.log.write(
u"\nStations in 'failures' list (should match DescribeSensor errors):"
)
print(
"Stations in 'failures' list (should match DescribeSensor errors):"
)
for station_fail in failures:
self.log.write(u"\n{station}".format(station=station_fail))
print("{station}".format(station=station_fail))
if station_recs:
stations_df = pd.DataFrame.from_records(station_recs,
columns=station.keys())
stations_df.index = stations_df['station_urn']
return stations_df
else:
return None
def __init__(self, element):
self.reason = testXMLAttribute(element, "reason")
self.value = testXMLValue(element)
def __init__(self, element):
self.tokenSeparator = testXMLAttribute(element[-1], "tokenSeparator") # string, required
self.blockSeparator = testXMLAttribute(element[-1], "blockSeparator") # string, required
self.decimalSeparator = testXMLAttribute(element[-1], "decimalSeparator") or "." # string, optional, default="."
self.collapseWhiteSpaces = get_boolean(testXMLAttribute(element[-1], "collapseWhiteSpaces")) or True # boolean, optional, default=True
def __init__(self, element):
self._root = element
self.description = testXMLValue(
self._root.find(nspv("gml311:description")))
self.begin_position = extract_time(
self._root.find(
nspv('om10:samplingTime/gml311:TimePeriod/gml311:beginPosition'
)))
self.end_position = extract_time(
self._root.find(
nspv(
'om10:samplingTime/gml311:TimePeriod/gml311:endPosition')))
self.procedures = [
testXMLAttribute(e, nspv("xlink:href"))
for e in self._root.findall(
nspv("om10:procedure/om10:Process/gml311:member"))
]
self.observedProperties = [
testXMLAttribute(e, nspv("xlink:href"))
for e in self._root.findall(
nspv(
"om10:observedProperty/swe101:CompositePhenomenon/swe101:component"
))
]
# Can't use a full Xpath expression, so iterate over all metaDataProperties to find the IOOS FeatureType
self.feature_type = None
ft = self._root.findall(
nspv(
"om10:featureOfInterest/gml311:FeatureCollection/gml311:metaDataProperty/gml311:GenericMetaData/gml311:name"
))
ft.extend(
self._root.findall(
nspv(
"om10:featureOfInterest/gml311:FeatureCollection/gml311:metaDataProperty/gml311:name"
)))
ft_def = "http://cf-pcmdi.llnl.gov/documents/cf-conventions/1.6/cf-conventions.html#discrete-sampling-geometries"
for f in ft:
if testXMLAttribute(f, "codeSpace") == ft_def:
self.feature_type = testXMLValue(f)
# BBOX
envelope = self._root.find(
nspv(
"om10:featureOfInterest/gml311:FeatureCollection/gml311:boundedBy/gml311:Envelope"
))
self.bbox_srs = Crs(testXMLAttribute(envelope, 'srsName'))
lower_left_corner = testXMLValue(
envelope.find(nspv('gml311:lowerCorner'))).split(" ")
upper_right_corner = testXMLValue(
envelope.find(nspv('gml311:upperCorner'))).split(" ")
if self.bbox_srs.axisorder == "yx":
self.bbox = box(float(lower_left_corner[1]),
float(lower_left_corner[0]),
float(upper_right_corner[1]),
float(upper_right_corner[0]))
else:
self.bbox = box(float(lower_left_corner[0]),
float(lower_left_corner[1]),
float(upper_right_corner[0]),
float(upper_right_corner[1]))
# LOCATION
location = self._root.find(
nspv(
"om10:featureOfInterest/gml311:FeatureCollection/gml311:location"
))
# Should only have one child
geo = list(location)[-1]
self.location = {}
def get_point(element, srs):
name = testXMLValue(element.find(nspv("gml311:name")))
point = testXMLValue(element.find(nspv("gml311:pos"))).split(" ")
if srs.axisorder == "yx":
point = Point(float(point[1]), float(point[0]))
else:
point = Point(float(point[0]), float(point[1]))
return name, point
self.location_srs = Crs(testXMLAttribute(geo, "srsName"))
if geo.tag == nspv("gml311:Point"):
n, p = get_point(geo, self.location_srs)
self.location[n] = p
elif geo.tag == nspv("gml311:MultiPoint"):
for point in geo.findall(nspv("gml311:pointMembers/gml311:Point")):
n, p = get_point(point, self.location_srs)
self.location[n] = p
# Now the fields change depending on the FeatureType
self.results = self._root.find(nspv("om10:result"))
# TODO: This should be implemented as a Factory
self.feature = None
data = self.results.find(nspv("swe20:DataRecord"))
if data is not None:
if self.feature_type == 'timeSeries':
self.feature = TimeSeries(data).feature
elif self.feature_type == 'timeSeriesProfile':
self.feature = TimeSeriesProfile(data).feature
else:
print("No feature type found.")
def __init__(self, element):
self.arcrole = testXMLAttribute(element, nsp("xlink:arcrole"))
self.url = testXMLAttribute(element, nsp("xlink:href"))
self.documents = [Document(d) for d in element.findall(nsp("sml:Document"))]
def process_station(self, uid):
""" Makes a DescribeSensor request based on a 'uid' parameter being a station procedure """
GML_NS = "http://www.opengis.net/gml"
XLINK_NS = "http://www.w3.org/1999/xlink"
with app.app_context():
metadata_value = etree.fromstring(
self.sos.describe_sensor(
outputFormat=
'text/xml;subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"',
procedure=uid))
station_ds = IoosDescribeSensor(metadata_value)
unique_id = station_ds.id
if unique_id is None:
app.logger.warn(
"Could not get a 'stationID' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/stationID'"
)
return
dataset = db.Dataset.find_one({'uid': unicode(unique_id)})
if dataset is None:
dataset = db.Dataset()
dataset.uid = unicode(unique_id)
# Find service reference in Dataset.services and remove (to replace it)
tmp = dataset.services[:]
for d in tmp:
if d['service_id'] == self.service.get('_id'):
dataset.services.remove(d)
# Parsing messages
messages = []
# NAME
name = unicode_or_none(station_ds.shortName)
if name is None:
messages.append(
u"Could not get a 'shortName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/shortName'"
)
# DESCRIPTION
description = unicode_or_none(station_ds.longName)
if description is None:
messages.append(
u"Could not get a 'longName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/longName'"
)
# PLATFORM TYPE
asset_type = unicode_or_none(station_ds.platformType)
if asset_type is None:
messages.append(
u"Could not get a 'platformType' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/platformType'"
)
# LOCATION is in GML
gj = None
loc = station_ds.location
if loc is not None and loc.tag == "{%s}Point" % GML_NS:
pos_element = loc.find("{%s}pos" % GML_NS)
# strip out points
positions = map(float, testXMLValue(pos_element).split(" "))
crs = Crs(testXMLAttribute(pos_element, "srsName"))
if crs.axisorder == "yx":
gj = json.loads(
geojson.dumps(
geojson.Point([positions[1], positions[0]])))
else:
gj = json.loads(
geojson.dumps(
geojson.Point([positions[0], positions[1]])))
else:
messages.append(
u"Found an unrecognized child of the sml:location element and did not attempt to process it: %s"
% etree.tostring(loc).strip())
service = {
# Reset service
'name': name,
'description': description,
'service_type': self.service.get('service_type'),
'service_id': ObjectId(self.service.get('_id')),
'data_provider': self.service.get('data_provider'),
'metadata_type': u'sensorml',
'metadata_value':
unicode(etree.tostring(metadata_value)).strip(),
'messages': map(unicode, messages),
'keywords': map(unicode, sorted(station_ds.keywords)),
'variables': map(unicode, sorted(station_ds.variables)),
'asset_type': asset_type,
'geojson': gj,
'updated': datetime.utcnow()
}
dataset.services.append(service)
dataset.updated = datetime.utcnow()
dataset.save()
return "Harvested"
def get_uom(element):
uom = testXMLAttribute(element, "code")
if uom is None:
uom = testXMLAttribute(element, nspv("xlink:href"))
return uom
def __init__(self, **kwargs):
super(SweTimeSeries, self).__init__(**kwargs)
# Parse out GML point. Defaults to 0 depth if none specified
self.geo_srs = Crs(
testXMLAttribute(self._location.find(nsp("Point", self.GML_NS)),
'srsName'))
geo = [
float(v) for v in testXMLValue(
self._location.find(nsp("Point/pos", self.GML_NS))).split(" ")
]
if self.geo_srs.axisorder == "yx":
self.geo = sPoint(geo[1], geo[0])
else:
self.geo = sPoint(geo[0], geo[1])
try:
self.geo.z = geo[2]
except:
pass
pc = PointCollection()
for row in self.results.data:
p = Point()
time = None
z = None
lat = None
lon = None
for field in row:
if field.axis == "time":
t = dateutil.parser.parse(field.value)
p.time = t.astimezone(dateutil.tz.tzutc())
elif field.axis == "Long":
lon = field
elif field.axis == "Lat":
lat = field
elif field.axis == "h":
z = field
else:
m = Member(value=field.value,
unit=field.units,
units_definition=field.units_url,
name=field.name,
definition=field.definition,
standard=field.definition)
p.add_member(m)
# Set the spatial point
if lon.srs != lat.srs:
raise ValueError(
"Longitude and Latitude need to have the same SRS/CRS!")
p.location = sPoint(float(lon.value), float(lat.value),
float(z.value))
pc.add_element(p)
self.data = pc
def __init__(self, element):
ReferenceGroup.__init__(self, element)
GeneralInfoGroup.__init__(self, element)
self.id = testXMLAttribute(element, nsp("gml:id"))
self.date = testXMLValue(element.find(nsp('sml:date')))
self.description = testXMLValue(element.find(nsp('gml:description')))
def __init__(self, md=None, identtype=None):
if md is None:
self.identtype = None
self.title = None
self.alternatetitle = None
self.aggregationinfo = None
self.uricode = []
self.uricodespace = []
self.date = []
self.datetype = []
self.uselimitation = []
self.uselimitation_url = []
self.accessconstraints = []
self.classification = []
self.otherconstraints = []
self.securityconstraints = []
self.useconstraints = []
self.denominators = []
self.distance = []
self.uom = []
self.resourcelanguage = []
self.creator = []
self.publisher = []
self.contributor = []
self.edition = None
self.abstract = None
self.abstract_url = None
self.purpose = None
self.status = None
self.contact = []
self.keywords = []
self.keywords2 = []
self.topiccategory = []
self.supplementalinformation = None
self.extent = None
self.bbox = None
self.temporalextent_start = None
self.temporalextent_end = None
self.spatialrepresentationtype = []
else:
self.identtype = identtype
val = md.find(
util.nspath_eval(
'gmd:citation/gmd:CI_Citation/gmd:title/gco:CharacterString',
namespaces))
self.title = util.testXMLValue(val)
val = md.find(
util.nspath_eval(
'gmd:citation/gmd:CI_Citation/gmd:alternateTitle/gco:CharacterString',
namespaces))
self.alternatetitle = util.testXMLValue(val)
val = md.find(util.nspath_eval('gmd:aggregationInfo', namespaces))
self.aggregationinfo = util.testXMLValue(val)
self.uricode = []
for i in md.findall(
util.nspath_eval(
'gmd:citation/gmd:CI_Citation/gmd:identifier/gmd:RS_Identifier/gmd:code/gco:CharacterString',
namespaces)):
val = util.testXMLValue(i)
if val is not None:
self.uricode.append(val)
self.uricodespace = []
for i in md.findall(
util.nspath_eval(
'gmd:citation/gmd:CI_Citation/gmd:identifier/gmd:RS_Identifier/gmd:codeSpace/gco:CharacterString',
namespaces)):
val = util.testXMLValue(i)
if val is not None:
self.uricodespace.append(val)
self.date = []
self.datetype = []
for i in md.findall(
util.nspath_eval(
'gmd:citation/gmd:CI_Citation/gmd:date/gmd:CI_Date',
namespaces)):
self.date.append(CI_Date(i))
self.uselimitation = []
self.uselimitation_url = []
for i in md.findall(
util.nspath_eval(
'gmd:resourceConstraints/gmd:MD_Constraints/gmd:useLimitation/gco:CharacterString',
namespaces)):
val = util.testXMLValue(i)
if val is not None:
self.uselimitation.append(val)
for i in md.findall(
util.nspath_eval(
'gmd:resourceConstraints/gmd:MD_Constraints/gmd:useLimitation/gmx:Anchor',
namespaces)):
val = util.testXMLValue(i)
val1 = i.attrib.get(util.nspath_eval('xlink:href', namespaces))
if val is not None:
self.uselimitation.append(val)
self.uselimitation_url.append(val1)
self.accessconstraints = []
for i in md.findall(
util.nspath_eval(
'gmd:resourceConstraints/gmd:MD_LegalConstraints/gmd:accessConstraints/gmd:MD_RestrictionCode',
namespaces)):
val = _testCodeListValue(i)
if val is not None:
self.accessconstraints.append(val)
self.classification = []
for i in md.findall(
util.nspath_eval(
'gmd:resourceConstraints/gmd:MD_LegalConstraints/gmd:accessConstraints/gmd:MD_ClassificationCode',
namespaces)):
val = _testCodeListValue(i)
if val is not None:
self.classification.append(val)
self.otherconstraints = []
for i in md.findall(
util.nspath_eval(
'gmd:resourceConstraints/gmd:MD_LegalConstraints/gmd:otherConstraints/gco:CharacterString',
namespaces)):
val = util.testXMLValue(i)
if val is not None:
self.otherconstraints.append(val)
self.securityconstraints = []
for i in md.findall(
util.nspath_eval(
'gmd:resourceConstraints/gmd:MD_SecurityConstraints/gmd:classification/gmd:MD_ClassificationCode',
namespaces)):
val = util.testXMLValue(i)
if val is not None:
self.securityconstraints.append(val)
self.useconstraints = []
for i in md.findall(
util.nspath_eval(
'gmd:resourceConstraints/gmd:MD_LegalConstraints/gmd:useConstraints/gmd:MD_RestrictionCode',
namespaces)):
val = _testCodeListValue(i)
if val is not None:
self.useconstraints.append(val)
self.denominators = []
for i in md.findall(
util.nspath_eval(
'gmd:spatialResolution/gmd:MD_Resolution/gmd:equivalentScale/gmd:MD_RepresentativeFraction/gmd:denominator/gco:Integer',
namespaces)):
val = util.testXMLValue(i)
if val is not None:
self.denominators.append(val)
self.distance = []
self.uom = []
for i in md.findall(
util.nspath_eval(
'gmd:spatialResolution/gmd:MD_Resolution/gmd:distance/gco:Distance',
namespaces)):
val = util.testXMLValue(i)
if val is not None:
self.distance.append(val)
self.uom.append(i.get("uom"))
self.resourcelanguage = []
for i in md.findall(
util.nspath_eval('gmd:language/gmd:LanguageCode',
namespaces)):
val = _testCodeListValue(i)
if val is not None:
self.resourcelanguage.append(val)
self.creator = []
self.publisher = []
self.contributor = []
for val in md.findall(
util.nspath_eval(
'gmd:pointOfContact/gmd:CI_ResponsibleParty',
namespaces)):
role = val.find(
util.nspath_eval('gmd:role/gmd:CI_RoleCode', namespaces))
if role is not None:
clv = _testCodeListValue(role)
rp = CI_ResponsibleParty(val)
if clv == 'originator':
self.creator.append(rp)
elif clv == 'publisher':
self.publisher.append(rp)
elif clv == 'author':
self.contributor.append(rp)
val = md.find(
util.nspath_eval('gmd:edition/gco:CharacterString',
namespaces))
self.edition = util.testXMLValue(val)
val = md.find(
util.nspath_eval('gmd:abstract/gco:CharacterString',
namespaces))
self.abstract = util.testXMLValue(val)
val = md.find(
util.nspath_eval('gmd:abstract/gmx:Anchor', namespaces))
if val is not None:
self.abstract = util.testXMLValue(val)
self.abstract_url = val.attrib.get(
util.nspath_eval('xlink:href', namespaces))
val = md.find(
util.nspath_eval('gmd:purpose/gco:CharacterString',
namespaces))
self.purpose = util.testXMLValue(val)
self.status = _testCodeListValue(
md.find(
util.nspath_eval('gmd:status/gmd:MD_ProgressCode',
namespaces)))
self.contact = []
for i in md.findall(
util.nspath_eval(
'gmd:pointOfContact/gmd:CI_ResponsibleParty',
namespaces)):
o = CI_ResponsibleParty(i)
self.contact.append(o)
self.spatialrepresentationtype = []
for val in md.findall(
util.nspath_eval(
'gmd:spatialRepresentationType/gmd:MD_SpatialRepresentationTypeCode',
namespaces)):
val = util.testXMLAttribute(val, 'codeListValue')
if val:
self.spatialrepresentationtype.append(val)
warnings.warn(
'The .keywords and .keywords2 properties will merge into the '
'.keywords property in the future, with .keywords becoming a list '
'of MD_Keywords instances. This is currently implemented in .keywords2. '
'Please see https://github.com/geopython/OWSLib/issues/301 for more information',
FutureWarning)
self.keywords = []
for i in md.findall(
util.nspath_eval('gmd:descriptiveKeywords', namespaces)):
mdkw = {}
mdkw['type'] = _testCodeListValue(
i.find(
util.nspath_eval(
'gmd:MD_Keywords/gmd:type/gmd:MD_KeywordTypeCode',
namespaces)))
mdkw['thesaurus'] = {}
val = i.find(
util.nspath_eval(
'gmd:MD_Keywords/gmd:thesaurusName/gmd:CI_Citation/gmd:title/gco:CharacterString',
namespaces))
mdkw['thesaurus']['title'] = util.testXMLValue(val)
val = i.find(
util.nspath_eval(
'gmd:MD_Keywords/gmd:thesaurusName/gmd:CI_Citation/gmd:date/gmd:CI_Date/gmd:date/gco:Date',
namespaces))
mdkw['thesaurus']['date'] = util.testXMLValue(val)
val = i.find(
util.nspath_eval(
'gmd:MD_Keywords/gmd:thesaurusName/gmd:CI_Citation/gmd:date/gmd:CI_Date/gmd:dateType/gmd:CI_DateTypeCode',
namespaces))
mdkw['thesaurus']['datetype'] = util.testXMLValue(val)
mdkw['keywords'] = []
for k in i.findall(
util.nspath_eval('gmd:MD_Keywords/gmd:keyword',
namespaces)):
val = k.find(
util.nspath_eval('gco:CharacterString', namespaces))
if val is not None:
val2 = util.testXMLValue(val)
if val2 is not None:
mdkw['keywords'].append(val2)
self.keywords.append(mdkw)
self.keywords2 = []
for mdkw in md.findall(
util.nspath_eval('gmd:descriptiveKeywords/gmd:MD_Keywords',
namespaces)):
self.keywords2.append(MD_Keywords(mdkw))
self.topiccategory = []
for i in md.findall(
util.nspath_eval(
'gmd:topicCategory/gmd:MD_TopicCategoryCode',
namespaces)):
val = util.testXMLValue(i)
if val is not None:
self.topiccategory.append(val)
val = md.find(
util.nspath_eval(
'gmd:supplementalInformation/gco:CharacterString',
namespaces))
self.supplementalinformation = util.testXMLValue(val)
# There may be multiple geographicElement, create an extent
# from the one containing either an EX_GeographicBoundingBox or EX_BoundingPolygon.
# The schema also specifies an EX_GeographicDescription. This is not implemented yet.
val = None
val2 = None
val3 = None
extents = md.findall(util.nspath_eval('gmd:extent', namespaces))
extents.extend(
md.findall(util.nspath_eval('srv:extent', namespaces)))
for extent in extents:
if val is None:
for e in extent.findall(
util.nspath_eval(
'gmd:EX_Extent/gmd:geographicElement',
namespaces)):
if e.find(
util.nspath_eval(
'gmd:EX_GeographicBoundingBox',
namespaces)) is not None or e.find(
util.nspath_eval(
'gmd:EX_BoundingPolygon',
namespaces)) is not None:
val = e
break
self.extent = EX_Extent(val)
self.bbox = self.extent.boundingBox # for backwards compatibility
if val2 is None:
val2 = extent.find(
util.nspath_eval(
'gmd:EX_Extent/gmd:temporalElement/gmd:EX_TemporalExtent/gmd:extent/gml:TimePeriod/gml:beginPosition',
namespaces))
if val2 is None:
val2 = extent.find(
util.nspath_eval(
'gmd:EX_Extent/gmd:temporalElement/gmd:EX_TemporalExtent/gmd:extent/gml32:TimePeriod/gml32:beginPosition',
namespaces))
self.temporalextent_start = util.testXMLValue(val2)
if val3 is None:
val3 = extent.find(
util.nspath_eval(
'gmd:EX_Extent/gmd:temporalElement/gmd:EX_TemporalExtent/gmd:extent/gml:TimePeriod/gml:endPosition',
namespaces))
if val3 is None:
val3 = extent.find(
util.nspath_eval(
'gmd:EX_Extent/gmd:temporalElement/gmd:EX_TemporalExtent/gmd:extent/gml32:TimePeriod/gml32:endPosition',
namespaces))
self.temporalextent_end = util.testXMLValue(val3)
def __init__(self, element):
# Attributes
self.id = testXMLAttribute(element, "id") # string, optional
# Elements
self.extention = [] # anyType, min=0, max=X
def __init__(self, element):
self._root = element
OM_NS = ns.get_versioned_namespace('om', '1.0')
GML_NS = ns.get_versioned_namespace('gml', '3.1.1')
SWE_NS = ns.get_versioned_namespace('swe', '1.0')
self.name = testXMLValue(self._root.find(nsp("name", GML_NS)))
self.description = testXMLValue(
self._root.find(nsp("description", GML_NS)))
self.observedProperties = []
for op in self._root.findall(nsp('observedProperty', OM_NS)):
self.observedProperties.append(
testXMLAttribute(op, nspv('xlink:href')))
# BBOX
try:
envelope = self._root.find(nsp('boundedBy/Envelope', GML_NS))
lower_left_corner = testXMLValue(
envelope.find(nsp('lowerCorner', GML_NS))).split(" ")
upper_right_corner = testXMLValue(
envelope.find(nsp('upperCorner', GML_NS))).split(" ")
self.bbox_srs = Crs(testXMLAttribute(envelope, 'srsName'))
# Always keep the BBOX as minx, miny, maxx, maxy
if self.bbox_srs.axisorder == "yx":
self.bbox = box(float(lower_left_corner[1]),
float(lower_left_corner[0]),
float(upper_right_corner[1]),
float(upper_right_corner[0]))
else:
self.bbox = box(float(lower_left_corner[0]),
float(lower_left_corner[1]),
float(upper_right_corner[0]),
float(upper_right_corner[1]))
except Exception:
self.bbox = None
self.bbox_srs = None
# Time range
fp = nsp('samplingTime', OM_NS)
mp = nsp('TimePeriod', GML_NS)
lp = nsp('beginPosition', GML_NS)
begin_position_element = self._root.find('%s/%s/%s' % (fp, mp, lp))
self.begin_position = extract_time(begin_position_element)
ep = nsp('endPosition', GML_NS)
end_position_element = self._root.find('%s/%s/%s' % (fp, mp, ep))
self.end_position = extract_time(end_position_element)
feature_of_interest_element = self._root.find(
nsp("featureOfInterest", OM_NS))
self.feature_type = testXMLValue(
feature_of_interest_element.find(
nsp("FeatureCollection/metaDataProperty/name", GML_NS)))
# Now the fields change depending on the FeatureType
result_element = self._root.find(nsp("result", OM_NS))
#TODO: This should be implemented as a Factory
self.feature = None
if self.feature_type == 'timeSeries':
self.feature = SweTimeSeries(
feature_of_interest=feature_of_interest_element,
result=result_element,
GML_NS=GML_NS,
OM_NS=OM_NS,
SWE_NS=SWE_NS)
elif self.feature_type == 'trajectoryProfile':
self.feature = SweTrajectoryProfile(
feature_of_interest=feature_of_interest_element,
result=result_element,
GML_NS=GML_NS,
OM_NS=OM_NS,
SWE_NS=SWE_NS)
elif self.feature_type == 'timeSeriesProfile':
self.feature = SweTimeseriesProfile(
feature_of_interest=feature_of_interest_element,
result=result_element,
GML_NS=GML_NS,
OM_NS=OM_NS,
SWE_NS=SWE_NS)
def get_stations_df(self, sos_url, station_urns_sel=None):
""" Returns a GeoDataFrame
"""
# LATER: ADD ERROR TEST/CATCH AFTER EACH WEB REQUEST
oFrmt = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
params = {
'service': 'SOS',
'request': 'GetCapabilities',
'acceptVersions': '1.0.0'
}
sos_url_params = sos_url + '?' + urlencode(params)
sos_url_params_esc = sos_url_params.replace("&", "&")
# sos_url_params_quoted = quote(sos_url_params,"/=:")
# sos_url_params_unquoted = unquote(sos_url_params)
try:
sosgc = SensorObservationService(sos_url_params)
except (ConnectionError, ReadTimeout) as e:
self.log.write(
u"Error: unable to connect to SOS service: {url} due to HTTP connection error.\n"
.format(url=sos_url_params))
self.log.write(
u"HTTP connection error: {err}.\n".format(err=e.message))
sys.exit(
"Error: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}"
.format(url=sos_url_params, err=e.message))
if station_urns_sel is not None:
station_urns = station_urns_sel
else:
sos_collector = IoosSweSos(sos_url)
station_urns = [
urn.name for urn in sos_collector.server.offerings
if 'network' not in urn.name.split(':')
]
sos_collector.features = station_urns
sml_lst = sos_collector.metadata(timeout=200)
station_recs = []
for station_idx, station_urn in enumerate(station_urns):
if station_urns_sel is not None:
sml_str = sosgc.describe_sensor(procedure=station_urn,
outputFormat=oFrmt,
timeout=200)
sml = SensorML(sml_str)
else:
sml = sml_lst[station_idx]
# debug:
# if self.verbose:
# self.log.write(unicode(etree.tostring(sml._root)))
ds = IoosDescribeSensor(sml._root)
pos = testXMLValue(
ds.system.location.find(self.nsp('gml:Point/gml:pos')))
system_el = sml._root.findall(self.nsp('sml:member'))[0].find(
self.nsp('sml:System'))
# Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication
# errors but there is not enough information in SensorML for alternatives)
# Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage"
documents = system_el.findall(
self.nsp('sml:documentation/sml:DocumentList/sml:member'))
documents_dct = {}
for d in documents:
document = Documentation(d)
name = testXMLAttribute(d, "name")
# url = document.documents[0].url
documents_dct[name] = document
contacts = system_el.findall(
self.nsp('sml:contact/sml:ContactList/sml:member'))
contacts_dct = {}
for c in contacts:
contact = Contact(c)
role = contact.role.split('/')[-1]
contacts_dct[role] = contact
sweQuants = system_el.findall(
self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity'))
quant_lst = [
sweQuant.attrib['definition'] for sweQuant in sweQuants
]
parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst]
# attempt to read beginPosition, if available:
beginPosition = testXMLValue(
system_el.find(
self.nsp(
'sml:validTime/gml:TimePeriod/gml:beginPosition')))
try:
begin_service_date = parser.parse(beginPosition)
except AttributeError as e:
begin_service_date = None
station = OrderedDict()
station['station_urn'] = station_urn
station['sos_url'] = sos_url_params_esc
station['lon'] = float(pos.split()[1])
station['lat'] = float(pos.split()[0])
station['shortName'] = ds.shortName
station['longName'] = ds.longName
station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont)
station['serverName'] = self.server_name
# Some capabilities-level metadata:
station['title'] = sosgc.identification.title
station['abstract'] = sosgc.identification.abstract
station['keywords'] = sosgc.identification.keywords
station['begin_service_date'] = begin_service_date
# Beware that a station can have >1 classifier of the same type
# This code does not accommodate that possibility
station['platformType'] = ds.platformType
station['parentNetwork'] = ds.get_ioos_def('parentNetwork',
'classifier', ont)
station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont)
# store some nested dictionaries in 'station' for appopriate SensorML sources:
station['contacts_dct'] = contacts_dct
station['documents_dct'] = documents_dct
# MW: the 'operator_' and 'publisher_' attributes can be removed bc they are not used
# in the template code currently in favor of 'contacts_dct'
# station['operatorSector'] = ds.get_ioos_def('operatorSector', 'classifier', ont)
# station['operator_org'] = contacts_dct['operator'].organization
# station['operator_country'] = contacts_dct['operator'].country
# station['operator_url'] = contacts_dct['operator'].url
# station['operator_email'] = contacts_dct['operator'].email
# station['publisher'] = ds.get_ioos_def('publisher', 'classifier', ont)
# station['publisher_org'] = contacts_dct['publisher'].organization
# station['publisher_url'] = contacts_dct['publisher'].url
# station_dct['publisher_email'] = contacts_dct['publisher'].electronicMailAddress
station['starting'] = ds.starting
station['ending'] = ds.ending
# station['starting_isostr'] = datetime.isoformat(ds.starting)
# station['ending_isostr'] = datetime.isoformat(ds.ending)
station['parameter_uris'] = ','.join(quant_lst)
station['parameters'] = ','.join(parameter_lst)
station['variables'] = [var.split('/')[-1] for var in ds.variables]
# debug:
if self.verbose:
self.log.write(u"\nProcessing station: {station}\n".format(
station=station_urn))
print("\nProcessing station: {station}".format(
station=station_urn))
for var in ds.variables:
self.log.write(u"variable: {var}\n".format(var=var))
print("variable: {var}".format(var=var))
# print(sosgc.contents)
# for id, offering in sosgc.contents.iteritems():
# print("sosgc.contents: {item}".format(item=id))
# parse 'responseFormat' values and populate list:
# response_formats = sosgc.contents[station_urn].response_formats
response_formats = []
for id, sosgc.content in sosgc.contents.items():
if sosgc.content.name == station_urn:
response_formats = sosgc.content.response_formats
# response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() if sosgc.content.name == station_urn ]
# subset responseFormats (response_formats) for download links matching those passed in --response_formats parameter
# (or 'application/json,application/zip; subtype=x-netcdf' by default):
download_formats = [
response_format for response_format in response_formats
if response_format in self.response_formats
]
station['response_formats'] = response_formats
station['download_formats'] = download_formats
if self.verbose:
for format in response_formats:
self.log.write(
u"responseFormat: {format}\n".format(format=format))
print("responseFormat: {format}".format(format=format))
for format in download_formats:
self.log.write(
u"downloadFormats: {format}\n".format(format=format))
print("downloadFormats: {format}".format(format=format))
# calculate event_time using self.getobs_req_hours:
if ds.starting is not None and ds.ending is not None:
event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format(
begin=ds.ending - timedelta(hours=self.getobs_req_hours),
end=ds.ending)
else:
now = datetime.now(pytz.utc)
then = now - timedelta(hours=self.getobs_req_hours)
event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format(
begin=then, end=now)
if self.verbose:
self.log.write(
u"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}\n"
.format(then=then, now=now))
print(
"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
if self.verbose:
self.log.write(u"eventTime: {time}\n".format(time=event_time))
print("eventTime: {time}".format(time=event_time))
# create a dict to store parameters for valid example GetObservation requests for station:
getobs_req_dct = {}
# populate a parameters dictionary for download links for each 'observedProperty' type and secondly for each 'responseFormat' per observedProperty:
getobs_params_base = {
'service': 'SOS',
'request': 'GetObservation',
'version': '1.0.0',
'offering': station_urn,
'eventTime': event_time
}
for variable in ds.variables:
getobs_params = getobs_params_base.copy()
getobs_params['observedProperty'] = variable
variable = variable.split('/')[-1]
for format in download_formats:
getobs_params['responseFormat'] = format
getobs_request_url_encoded = sos_url + '?' + urlencode(
getobs_params)
getobs_request_url = unquote(getobs_request_url_encoded)
getobs_request_url_esc = getobs_request_url.replace(
"&", "&")
getobs_req_dct[variable + '-' + format] = {
'variable': variable,
'url': getobs_request_url_esc,
'format_type': self.RESPONSE_FORMAT_TYPE_MAP[format],
'format_name': self.RESPONSE_FORMAT_NAME_MAP[format]
}
if self.verbose:
self.log.write(
u"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}\n"
.format(
variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url,
getobs_request_url_esc=getobs_request_url_esc))
print(
"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}"
.format(
variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url,
getobs_request_url_esc=getobs_request_url_esc))
# ToDo: finish adding the 'getobs_req_dct' to the output template
station['getobs_req_dct'] = getobs_req_dct
station_recs.append(station)
stations_df = pd.DataFrame.from_records(station_recs,
columns=station.keys())
stations_df.index = stations_df['station_urn']
return stations_df
