def __init__(self, element, swe_ns): self._root = element self._fields = [] self.definition = testXMLAttribute(self._root, "definition") self.srs = testXMLAttribute(self._root, "referenceFrame") for coord in self._root.findall(nsp("coordinate", swe_ns)): self._fields.append(SweCoordinate(coord, swe_ns, self.srs, self.definition))
def __init__(self, element, ows_version='1.0.0'): self._root = element global _ows_version _ows_version = ows_version self.operations = {} # There can be parent parameters and constraints parameters = [] for parameter in self._root.findall(ns_ows('ows:Parameter')): parameters.append((testXMLAttribute(parameter,'name'), {'values': [i.text for i in parameter.findall(ns_ows('.//ows:Value'))]})) parameters = dict(parameters) constraints = [] for constraint in self._root.findall(ns_ows('ows:Constraint')): constraints.append((testXMLAttribute(constraint,'name'), {'values': [i.text for i in constraint.findall(ns_ows('.//ows:Value'))]})) constraints = dict(constraints) for op in self._root.findall(ns_ows('ows:Operation')): co = Operation(op, ows_version) # Parent objects get overriden by children elements co.parameters = dict_union(parameters, co.parameters) co.constraints = dict_union(constraints, co.constraints) self.operations[co.name] = co for parameter in elem.findall(util.nspath('Parameter', namespace)): if namespace == OWS_NAMESPACE_1_1_0: parameters.append((parameter.attrib['name'], {'values': [i.text for i in parameter.findall(util.nspath('AllowedValues/Value', namespace))]})) else: parameters.append((parameter.attrib['name'], {'values': [i.text for i in parameter.findall(util.nspath('Value', namespace))]})) self.parameters = dict(parameters)
def __init__(self, element): self.type = testXMLAttribute(element.find(nspv( "om20:type")), nspv("xlink:href")) self.featureOfInterest = testXMLAttribute(element.find(nspv( "om20:featureOfInterest")), nspv("xlink:href")) self.observedProperty = testXMLAttribute(element.find(nspv( "om20:observedProperty")), nspv("xlink:href")) self.procedure = testXMLAttribute(element.find(nspv( "om20:procedure")), nspv("xlink:href")) ''' Determine if phenom time is instant or a period. This depend on the type of observation -- this could be split out ''' instant_element = element.find(nspv( "om20:phenomenonTime/gml32:TimeInstant")) if instant_element is not None: self.phenomenonTime = extract_time(instant_element) else: start = extract_time(element.find(nspv( "om20:phenomenonTime/gml32:TimePeriod/gml32:beginPosition"))) end = extract_time(element.find(nspv( "om20:phenomenonTime/gml32:TimePeriod/gml32:endPosition"))) self.phenomenonTime = TimePeriod(start, end) self.resultTime = extract_time(element.find(nspv( "om20:resultTime/gml32:TimeInstant/gml32:timePosition"))) self.result = element.find(nspv("om20:result"))
def __init__(self, md=None): if md is None: self.keywords = [] self.type = None self.thesaurus = None self.kwdtype_codeList = 'http://standards.iso.org/ittf/PubliclyAvailableStandards/ISO_19139_Schemas/resources/codelist/gmxCodelists.xml#MD_KeywordTypeCode' else: self.keywords = [] val = md.findall(util.nspath_eval('gmd:keyword/gco:CharacterString', namespaces)) for word in val: self.keywords.append(util.testXMLValue(word)) self.type = None val = md.find(util.nspath_eval('gmd:type/gmd:MD_KeywordTypeCode', namespaces)) self.type = util.testXMLAttribute(val, 'codeListValue') self.thesaurus = None val = md.find(util.nspath_eval('gmd:thesaurusName/gmd:CI_Citation', namespaces)) if val is not None: self.thesaurus = {} thesaurus = val.find(util.nspath_eval('gmd:title/gco:CharacterString', namespaces)) self.thesaurus['title'] = util.testXMLValue(thesaurus) thesaurus = val.find(util.nspath_eval('gmd:date/gmd:CI_Date/gmd:date/gco:Date', namespaces)) self.thesaurus['date'] = util.testXMLValue(thesaurus) thesaurus = val.find(util.nspath_eval('gmd:date/gmd:CI_Date/gmd:dateType/gmd:CI_DateTypeCode', namespaces)) self.thesaurus['datetype'] = util.testXMLAttribute(thesaurus, 'codeListValue')
def __init__(self, element): super(Vector, self).__init__(element) # Elements self.coordinate = [Coordinate(x) for x in element.findall(nspv("swe20:coordinate"))] # Attributes self.referenceFrame = testXMLAttribute(element,"referenceFrame") # anyURI, required self.localFrame = testXMLAttribute(element,"localFrame") # anyURI, optional
def __init__(self, element): super(Matrix, self).__init__(element) self.elementCount = element.find(nspv("swe20:elementCount/swe20:Count")) # required self.elementType = ElementType(element.find(nspv("swe20:elementType"))) # required self.encoding = AbstractEncoding(element.find(nspv("swe20:encoding"))) self.values = testXMLValue(element.find(nspv("swe20:values"))) self.referenceFrame = testXMLAttribute(element, "referenceFrame") # anyURI, required self.localFrame = testXMLAttribute(element, "localFrame") # anyURI, optional
def __init__(self, element): self._root = element OM_NS = ns.get_versioned_namespace('om','1.0') GML_NS = ns.get_versioned_namespace('gml','3.1.1') SWE_NS = ns.get_versioned_namespace('swe','1.0') XLINK_NS = ns.get_namespace("xlink") self.name = testXMLValue(self._root.find(nsp("name", GML_NS))) self.description = testXMLValue(self._root.find(nsp("description", GML_NS))) self.observedProperties = [] for op in self._root.findall(nsp('observedProperty', OM_NS)): self.observedProperties.append(testXMLAttribute(op,nsp('href', XLINK_NS))) # BBOX try: envelope = self._root.find(nsp('boundedBy/Envelope', GML_NS)) lower_left_corner = testXMLValue(envelope.find(nsp('lowerCorner', GML_NS))).split(" ") upper_right_corner = testXMLValue(envelope.find(nsp('upperCorner', GML_NS))).split(" ") self.bbox_srs = Crs(testXMLAttribute(envelope,'srsName')) # Always keep the BBOX as minx, miny, maxx, maxy if self.bbox_srs.axisorder == "yx": self.bbox = box(float(lower_left_corner[1]), float(lower_left_corner[0]), float(upper_right_corner[1]), float(upper_right_corner[0])) else: self.bbox = box(float(lower_left_corner[0]), float(lower_left_corner[1]), float(upper_right_corner[0]), float(upper_right_corner[1])) except Exception: self.bbox = None self.bbox_srs = None # Time range fp = nsp('samplingTime', OM_NS) mp = nsp('TimePeriod', GML_NS) lp = nsp('beginPosition', GML_NS) begin_position_element = self._root.find('%s/%s/%s' % (fp, mp, lp)) self.begin_position = extract_time(begin_position_element) ep = nsp('endPosition', GML_NS) end_position_element = self._root.find('%s/%s/%s' % (fp, mp, ep)) self.end_position = extract_time(end_position_element) feature_of_interest_element = self._root.find(nsp("featureOfInterest", OM_NS)) self.feature_type = testXMLValue(feature_of_interest_element.find(nsp("FeatureCollection/metaDataProperty/name", GML_NS))) # Now the fields change depending on the FeatureType result_element = self._root.find(nsp("result", OM_NS)) #TODO: This should be implemented as a Factory self.feature = None if self.feature_type == 'timeSeries': self.feature = SweTimeSeries(feature_of_interest=feature_of_interest_element, result=result_element, GML_NS=GML_NS, OM_NS=OM_NS, SWE_NS=SWE_NS) elif self.feature_type == 'trajectoryProfile': self.feature = SweTrajectoryProfile(feature_of_interest=feature_of_interest_element, result=result_element, GML_NS=GML_NS, OM_NS=OM_NS, SWE_NS=SWE_NS) elif self.feature_type == 'timeSeriesProfile': self.feature = SweTimeseriesProfile(feature_of_interest=feature_of_interest_element, result=result_element, GML_NS=GML_NS, OM_NS=OM_NS, SWE_NS=SWE_NS)
def __init__(self, element): self.id = testXMLAttribute(element, nsp("gml:id")) self.version = testXMLValue(element.find(nsp("sml:version"))) self.description = testXMLValue(element.find(nsp("gml:description"))) self.date = testXMLValue(element.find(nsp("sml:date"))) try: self.contact = Contact(element.find(nsp("sml:contact"))) except AttributeError: self.contact = None self.format = testXMLValue(element.find(nsp('sml:format'))) self.url = testXMLAttribute(element.find(nsp('sml:onlineResource')), nsp('xlink:href'))
def __init__(self, element): # Both capabilities and characteristics contain a single swe:DataRecord element self.capabilities = {} for cap in element.findall(nsp("sml:capabilities")): name = testXMLAttribute(cap, "name") self.capabilities[name] = cap[0] self.characteristics = {} for cha in element.findall(nsp("sml:characteristics")): name = testXMLAttribute(cha, "name") self.characteristics[name] = cha[0]
def __init__(self, element, swe_ns): self._root = element # Only supporting Text Encoding tx = self._root.find(nsp("TextEncoding", swe_ns)) if tx is not None: self.type = "text" self.decimal_separator = testXMLAttribute(tx,'decimalSeparator') or "." self.token_separator = testXMLAttribute(tx,'tokenSeparator') or "," self.block_separator = testXMLAttribute(tx,'blockSeparator') # ignored, we use newlines else: raise TypeError("Only swe:TextEncoding is supported at this time")
def __init__(self, element): super(AbstractSimpleComponent, self).__init__(element) # Attributes self.referenceFrame = testXMLAttribute(element,"referenceFrame") # anyURI, optional self.axisID = testXMLAttribute(element,"axisID") # string, optional # Elements self.quality = [_f for _f in [Quality(q) for q in [e.find('*') for e in element.findall(nspv("swe20:quality"))] if q is not None] if _f] try: self.nilValues = NilValues(element.find(nspv("swe20:nilValues"))) except: self.nilValues = None
def __init__(self, element): super(AbstractSimpleComponent, self).__init__(element) # Attributes self.referenceFrame = testXMLAttribute(element,"referenceFrame") # anyURI, optional self.axisID = testXMLAttribute(element,"axisID") # string, optional # Elements self.quality = filter(None, [Quality(e) for e in element.findall(nspv("swe20:quality"))]) try: self.nilValues = NilValues(element.find(nspv("swe20:nilValues"))) except: self.nilValues = None
def __init__(self, element): self._root = element self.id = testXMLAttribute(self._root,nsp('gml:id')) self.description = testXMLValue(self._root.find(nsp('gml:description'))) self.name = testXMLValue(self._root.find(nsp('gml:name'))) self.srs = Crs(testXMLValue(self._root.find(nsp('gml:srsName')))) # LOOK: Check on GML boundedBy to make sure we handle all of the cases # gml:boundedBy try: envelope = self._root.find(nsp('gml:boundedBy/gml:Envelope')) lower_left_corner = testXMLValue(envelope.find(nsp('gml:lowerCorner'))).split(" ") upper_right_corner = testXMLValue(envelope.find(nsp('gml:upperCorner'))).split(" ") # (left, bottom, right, top) in self.bbox_srs units self.bbox = (float(lower_left_corner[1]), float(lower_left_corner[0]), float(upper_right_corner[1]), float(upper_right_corner[0])) self.bbox_srs = Crs(testXMLAttribute(envelope,'srsName')) except Exception: self.bbox = None self.bbox_srs = None # LOOK: Support all gml:TimeGeometricPrimitivePropertyType # Right now we are just supporting gml:TimePeriod # sos:Time begin_position_element = self._root.find(nsp('sos:time/gml:TimePeriod/gml:beginPosition')) self.begin_position = extract_time(begin_position_element) end_position_element = self._root.find(nsp('sos:time/gml:TimePeriod/gml:endPosition')) self.end_position = extract_time(end_position_element) self.result_model = testXMLValue(self._root.find(nsp('sos:resultModel'))) self.procedures = [] for proc in self._root.findall(nsp('sos:procedure')): self.procedures.append(testXMLAttribute(proc,nsp('xlink:href'))) # LOOK: Support swe:Phenomenon here # this includes compound properties self.observed_properties = [] for op in self._root.findall(nsp('sos:observedProperty')): self.observed_properties.append(testXMLAttribute(op,nsp('xlink:href'))) self.features_of_interest = [] for fot in self._root.findall(nsp('sos:featureOfInterest')): self.features_of_interest.append(testXMLAttribute(fot,nsp('xlink:href'))) self.response_formats = [] for rf in self._root.findall(nsp('sos:responseFormat')): self.response_formats.append(testXMLValue(rf)) self.response_modes = [] for rm in self._root.findall(nsp('sos:responseMode')): self.response_modes.append(testXMLValue(rm))
def __init__(self, element): super(TVPMeasurementMetadata, self).__init__(element) self.uom = testXMLAttribute(element.find(nspv( "wml2:uom")), "code") self.interpolationType = testXMLAttribute(element.find(nspv( "wml2:interpolationType")), nspv("xlink:href")) self.censoredReason = testXMLAttribute(element.find(nspv( "wml2:censoredReason")), "xlink:href") accuracy = testXMLValue(element.find(nspv("wml2:accuracy"))) if accuracy is not None: self.accuracy = Quantity(element)
def __init__(self, element, swe_ns): self._root = element # Gets whatever the first child is. # swe:Quantity = value ob = self._root[0] self.name = testXMLAttribute(self._root, "name") self.definition = testXMLAttribute(ob, "definition") self.units = testXMLAttribute(ob.find(nsp("uom", swe_ns)), "code") self.units_url = testXMLAttribute(ob.find(nsp("uom", swe_ns)), nspv("xlink:href")) self.value = testXMLValue(ob.find(nsp("value", swe_ns))) self.axis = testXMLAttribute(ob, "axisID")
def __init__(self, element): # TODO: This only supports the sml:contact/sml:ResponsibleParty elements, but there are numerous ways to store # contact information here. self.role = testXMLAttribute(element, nsp("xlink:role")) self.href = testXMLAttribute(element, nsp("xlink:href")) self.organization = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:organizationName'))) self.phone = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:phone/sml:voice'))) self.address = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:deliveryPoint'))) self.city = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:city'))) self.region = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:administrativeArea'))) self.postcode = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:postalCode'))) self.country = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:country'))) self.email = testXMLValue(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:electronicMailAddress'))) self.url = testXMLAttribute(element.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:onlineResource')), nsp("xlink:href"))
def __init__(self, element): super(StationDS, self).__init__(element=element) self.id = get_named_by_definition(self.system.get_identifiers_by_name("stationID"), "http://mmisw.org/ont/ioos/definition/stationID") self.platformType = get_named_by_definition(self.system.get_classifiers_by_name("platformType"), "http://mmisw.org/ont/ioos/definition/platformType") # Variables self.variables = [] try: # Using xlink:href to point to individual member sensors self.variables = sorted(list(set([testXMLAttribute(comp, nsp("xlink:title")) for comp in self.system.components]))) except AttributeError: # Verbose method of describing members. Pull out each individual variable definition self.variables = sorted(list(set(itertools.chain.from_iterable([[testXMLAttribute(quan, "definition") for quan in comp.findall(".//" + nsp("swe101:Quantity"))] for comp in self.system.components]))))
def __init__(self, element): ''' Base time-value pair metadata. Still to do: - relatedObservation ''' self.quality = testXMLAttribute(element.find(nspv( "wml2:quality")), nspv("xlink:href")) self.nilReason = testXMLAttribute(element.find(nspv( "wml2:nilReason")), nspv("xlink:href")) self.comment = testXMLValue(element.find(nspv( "wml2:comment"))) self.qualifier = testXMLAttribute(element.find(nspv( "wml2:qualifier")), nspv("xlink:href")) self.processing = testXMLValue(element.find(nspv( "wml2:processing")), nspv("xlink:href")) self.source = testXMLValue(element.find(nspv( "wml2:source")), nspv("xlink:href"))
def parse_owc_content(content_node): mimetype = util.testXMLAttribute(content_node, 'type') url = util.testXMLAttribute(content_node, 'href') title = util.testXMLAttribute(content_node, 'title') child_elem = None if len(list(content_node)) > 0: child_elem = element_to_string( list(content_node)[0],False) content_dict = { "type": mimetype, "url": url, "content": str(child_elem), "title": title } return content_dict
def __init__(self, element): super(StationDS, self).__init__(element=element) self.id = self.get_ioos_def("stationID", "identifier", ont) self.platformType = self.get_ioos_def( "platformType", "classifier", ont ) # Verbose method of describing members. Pull out each individual # variable definition self.variables = sorted( list( set( itertools.chain.from_iterable( [ [ testXMLAttribute(quan, "definition") for quan in comp.findall( ".//" + nsp("swe101:Quantity") ) ] for comp in self.system.components ] ) ) ) ) # if no variables were picked up, fall back to using original SML # components instead if not self.variables: self.variables = sorted( (comp.values()[0] for comp in self.system.components) )
def __init__(self, element): self._root = element self.id = testXMLAttribute(self._root, nsp('gml:id')) self.description = testXMLValue(self._root.find(nsp('gml:description'))) self.keywords = extract_xml_list(self._root.findall(nsp('sml:keywords/sml:KeywordList/sml:keyword'))) self.documents = [] for document in self._root.findall(nsp('sml:documentation')): self.documents.append(DocumentationMetadata(document)) self.identifiers = {} for identifier in self._root.findall(nsp('sml:identification/sml:IdentifierList/sml:identifier')): ident = IdentifierMetadata(identifier) self.identifiers[ident.name] = ident self.contacts = {} for contact in self._root.findall(nsp('sml:contact')): cont = ContactMetadata(contact) self.contacts[cont.role] = cont self.classifiers = {} for classifier in self._root.findall(nsp('sml:classification/sml:ClassifierList/sml:classifier')): classi = ClassifierMetadata(classifier) self.classifiers[classi.name] = classi self.history = {} for event_member in self._root.findall(nsp('sml:history/sml:EventList/sml:member')): emm = EventMemberMetadata(event_member) if self.history.get(emm.name) is None: self.history[emm.name] = [] self.history[emm.name] += emm.events self.components = {}
def __init__(self, element, ows_version='1.0.0'): self._root = element global _ows_version _ows_version = ows_version self.name = testXMLValue(self._root.find(ns_ows('ows:ProviderName'))) self.url = testXMLAttribute(self._root.find(ns_ows("ows:ProviderSite")),nspath_eval('xlink:href')) self.contact = ServiceContact(self._root.find(ns_ows("ows:ServiceContact")),ows_version)
def __init__(self, element): self._root = element self.name = testXMLAttribute(self._root, 'name') self.events = [] for event in self._root.findall(nsp('sml:Event')): self.events.append(EventMetadata(event, self.name))
def __init__(self, element): self.history = {} for event_member in element.findall(nsp('sml:history/sml:EventList/sml:member')): name = testXMLAttribute(event_member, "name") if self.history.get(name) is None: self.history[name] = [] for e in event_member.findall(nsp("sml:Event")): self.history[name].append(Event(e))
def __init__(self, element): self._root = element self.role = testXMLAttribute(self._root, nsp('xlink:role')) self.url = testXMLAttribute(self._root, nsp('xlink:href')) if self.url is None: self.url = testXMLAttribute(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:onlineResource')), nsp('xlink:href')) self.organization = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:organizationName'))) self.phone = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:phone/sml:voice'))) self.address = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:deliveryPoint'))) self.city = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:city'))) self.region = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:administrativeArea'))) self.postcode = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:postalCode'))) self.country = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:country'))) self.email = testXMLValue(self._root.find(nsp('sml:ResponsibleParty/sml:contactInfo/sml:address/sml:electronicMailAddress'))) """
def __init__(self, element, name): self._root = element self.name = name # All dates are UTC self.date = testXMLValue(self._root.find(nsp('sml:date'))) self.description = testXMLValue(self._root.find(nsp('gml:description'))) self.documentation_url = testXMLAttribute(self._root.find(nsp('sml:documentation')), nsp('xlink:href'))
def __init__(self, element): # No call to super(), the type object will process that. self.name = testXMLAttribute(element, "name") try: self.content = eval(element[-1].tag.split("}")[-1])(element[-1]) except IndexError: self.content = None except BaseException: raise
def __init__(self, **kwargs): super(SweTimeSeries,self).__init__(**kwargs) # Parse out GML point. Defaults to 0 depth if none specified self.geo_srs = Crs(testXMLAttribute(self._location.find(nsp("Point", self.GML_NS)), 'srsName')) geo = [float(v) for v in testXMLValue(self._location.find(nsp("Point/pos", self.GML_NS))).split(" ")] if self.geo_srs.axisorder == "yx": self.geo = sPoint(geo[1], geo[0]) else: self.geo = sPoint(geo[0], geo[1]) try: self.geo.z = geo[2] except: pass pc = PointCollection() for row in self.results.data: p = Point() time = None z = None lat = None lon = None for field in row: if field.axis == "time": t = dateutil.parser.parse(field.value) p.time = t.astimezone(dateutil.tz.tzutc()) elif field.axis == "Long": lon = field elif field.axis == "Lat": lat = field elif field.axis == "h": z = field else: m = Member(value=field.value, unit=field.units, units_definition=field.units_url, name=field.name, definition=field.definition, standard=field.definition) p.add_member(m) # Set the spatial point if lon.srs != lat.srs: raise ValueError("Longitude and Latitude need to have the same SRS/CRS!") p.location = sPoint(float(lon.value), float(lat.value), float(z.value)) pc.add_element(p) self.data = pc
def __init__(self, element): super(TimeRange, self).__init__(element) # Elements self.uom = get_uom(element.find(nspv("swe20:uom"))) try: self.constraint = AllowedTimes(element.find(nspv("swe20:constraint/swe20:AllowedTimes"))) # AllowedTimes, min=0, max=1 except: self.constraint = None # Attributes self.localFrame = testXMLAttribute(element,"localFrame") # anyURI, optional try: self.referenceTime = parser.parse(testXMLAttribute(element,"referenceTime")) # dateTime, optional except (AttributeError, ValueError): self.referenceTime = None values = make_pair(testXMLValue(element.find(nspv("swe20:value")))) # TimePosition, min=0, max=1 self.value = [get_time(t, self.referenceTime, self.uom) for t in values]
def __init__(self, element): super(Category, self).__init__(element) # Elements self.codeSpace = testXMLAttribute(element.find(nspv("swe20:codeSpace")), nspv("xlink:href")) # Reference, min=0, max=1 self.value = testXMLValue(element.find(nspv("swe20:value"))) # string, min=0, max=1 try: self.constraint = AllowedTokens(element.find(nspv("swe20:constraint/swe20:AllowedTokens"))) # AllowedTokens, min=0, max=1 except: self.constraint = None
def __init__(self, element): super(Category, self).__init__(element) # Elements self.codeSpace = testXMLAttribute( element.find(nspv("swe20:codeSpace")), nspv("xlink:href")) # Reference, min=0, max=1 self.value = testXMLValue(element.find( nspv("swe20:value"))) # string, min=0, max=1 try: self.constraint = AllowedTokens( element.find(nspv("swe20:constraint/swe20:AllowedTokens")) ) # AllowedTokens, min=0, max=1 except: self.constraint = None
def __init__(self, element): # No call to super(), the type object will process that. self.name = testXMLAttribute(element, "name") try: # attempt to find a class with the same name as the XML tag parsed # which is also contained within this module. # Ideally the classes should be explicitly whitelisted, but I # don't know what the set of possible classes to dispatch to should # be self.content = obj_mapping[element[-1].tag.split("}")[-1]]( element[-1]) except (IndexError, KeyError): self.content = None except BaseException: raise
def get_field_object(element, swe_ns): field_tag = nsp("field", swe_ns) data_choice_tag = nsp("DataChoice", swe_ns) if element.tag == field_tag: name = testXMLAttribute(element, "name") if name == "location": return SweVector(element.find(nsp("Vector", swe_ns)), swe_ns) elif name == "time": return SweTime(element.find(nsp("Time", swe_ns)), swe_ns) else: return SweField(element, swe_ns) elif element.tag == data_choice_tag: return SweDataChoice(element, swe_ns) else: return None
def __init__(self, element): super(StationDS, self).__init__(element=element) self.id = get_named_by_definition( self.system.get_identifiers_by_name("stationID"), "http://mmisw.org/ont/ioos/definition/stationID") self.platformType = get_named_by_definition( self.system.get_classifiers_by_name("platformType"), "http://mmisw.org/ont/ioos/definition/platformType") # Verbose method of describing members. Pull out each individual variable definition self.variables = sorted( list( set( itertools.chain.from_iterable([[ testXMLAttribute(quan, "definition") for quan in comp.findall(".//" + nsp("swe101:Quantity")) ] for comp in self.system.components]))))
def __init__(self, element): super(StationDS, self).__init__(element=element) self.id = self.get_ioos_def("stationID", "identifier", ont) self.platformType = self.get_ioos_def("platformType", "classifier", ont) # Verbose method of describing members. Pull out each individual # variable definition self.variables = sorted(list(set( itertools.chain.from_iterable( [[testXMLAttribute(quan, "definition") for quan in comp.findall(".//" + nsp("swe101:Quantity"))] for comp in self.system.components])))) # if no variables were picked up, fall back to using original SML # components instead if not self.variables: self.variables = sorted([comp.values()[0] for comp in self.system.components])
def __init__(self, elem, namespace=DEFAULT_OWS_NAMESPACE): if 'name' not in elem.attrib: # This is not a valid element return self.name = elem.attrib['name'] self.formatOptions = ['text/xml'] parameters = [] self.methods = [] self.constraints = [] for verb in elem.findall(util.nspath('DCP/HTTP/*', namespace)): url = util.testXMLAttribute(verb, util.nspath('href', XLINK_NAMESPACE)) if url is not None: verb_constraints = [ Constraint(conts, namespace) for conts in verb.findall( util.nspath('Constraint', namespace)) ] self.methods.append({ 'constraints': verb_constraints, 'type': util.xmltag_split(verb.tag), 'url': url }) for parameter in elem.findall(util.nspath('Parameter', namespace)): if namespace == OWS_NAMESPACE_1_1_0: parameters.append((parameter.attrib['name'], { 'values': [ i.text for i in parameter.findall( util.nspath('AllowedValues/Value', namespace)) ] })) else: parameters.append((parameter.attrib['name'], { 'values': [ i.text for i in parameter.findall( util.nspath('Value', namespace)) ] })) self.parameters = dict(parameters) for constraint in elem.findall(util.nspath('Constraint', namespace)): self.constraints.append(Constraint(constraint, namespace))
def decode_observation(self, element): """ Returns a parsed observation of the appropriate type, by inspecting the result element. 'element' input is the XML tree of the OM_Observation object """ result_element = element.find(nspath_eval("om20:result", namespaces)) if len(result_element) == 0: result_type = testXMLAttribute( result_element, nspath_eval("xsi:type", namespaces)) else: result_type = list(result_element)[0].tag if result_type.find('MeasureType') != -1: return MeasurementObservation(element) elif (result_type == '{http://www.opengis.net/waterml/2.0}MeasurementTimeseries'): return MeasurementTimeseriesObservation(element) else: raise NotImplementedError('Result type {} not supported'.format(result_type))
def process_station(self, uid, offering): """ Makes a DescribeSensor request based on a 'uid' parameter being a station procedure. Also pass along an offering with getCapabilities information for items such as temporal extent""" GML_NS = "http://www.opengis.net/gml" XLINK_NS = "http://www.w3.org/1999/xlink" with app.app_context(): app.logger.info("process_station: %s", uid) desc_sens = self._describe_sensor(uid, timeout=1200) # FIXME: add some kind of notice saying the station failed if desc_sens is None: app.logger.warn( "Could not get a valid describeSensor response") return metadata_value = etree.fromstring(desc_sens) sensor_ml = SensorML(metadata_value) try: station_ds = IoosDescribeSensor(metadata_value) # if this doesn't conform to IOOS SensorML sub, fall back to # manually picking apart the SensorML except ows.ExceptionReport: station_ds = process_sensorml(sensor_ml.members[0]) unique_id = station_ds.id if unique_id is None: app.logger.warn( "Could not get a 'stationID' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/stationID'" ) return dataset = db.Dataset.find_one({'uid': unicode(unique_id)}) if dataset is None: dataset = db.Dataset() dataset.uid = unicode(unique_id) dataset['active'] = True # Find service reference in Dataset.services and remove (to replace it) tmp = dataset.services[:] for d in tmp: if d['service_id'] == self.service.get('_id'): dataset.services.remove(d) # Parsing messages messages = [] # NAME name = unicode_or_none(station_ds.shortName) if name is None: messages.append( u"Could not get a 'shortName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/shortName'" ) # DESCRIPTION description = unicode_or_none(station_ds.longName) if description is None: messages.append( u"Could not get a 'longName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/longName'" ) # PLATFORM TYPE asset_type = unicode_or_none( getattr(station_ds, 'platformType', None)) if asset_type is None: messages.append( u"Could not get a 'platformType' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/platformType'" ) # LOCATION is in GML gj = None loc = station_ds.location if loc is not None and loc.tag == "{%s}Point" % GML_NS: pos_element = loc.find("{%s}pos" % GML_NS) # some older responses may uses the deprecated coordinates # element if pos_element is None: # if pos not found use deprecated coordinates element pos_element = loc.find("{%s}coordinates" % GML_NS) # strip out points positions = map(float, pos_element.text.split(" ")) for el in [pos_element, loc]: srs_name = testXMLAttribute(el, "srsName") if srs_name: crs = Crs(srs_name) if crs.axisorder == "yx": gj = json.loads( geojson.dumps( geojson.Point([positions[1], positions[0]]))) else: gj = json.loads( geojson.dumps( geojson.Point([positions[0], positions[1]]))) break else: if positions: messages.append( u"Position(s) found but could not parse SRS: %s, %s" % (positions, srs_name)) else: messages.append( u"Found an unrecognized child of the sml:location element and did not attempt to process it: %s" % loc) meta_str = unicode(etree.tostring(metadata_value)).strip() if len(meta_str) > 4000000: messages.append( u'Metadata document was too large to store (len: %s)' % len(meta_str)) meta_str = u'' service = { # Reset service 'name': name, 'description': description, 'service_type': self.service.get('service_type'), 'service_id': ObjectId(self.service.get('_id')), 'data_provider': self.service.get('data_provider'), 'metadata_type': u'sensorml', 'metadata_value': u'', 'time_min': getattr(offering, 'begin_position', None), 'time_max': getattr(offering, 'end_position', None), 'messages': map(unicode, messages), 'keywords': map(unicode, sorted(station_ds.keywords)), 'variables': map(unicode, sorted(station_ds.variables)), 'asset_type': get_common_name(asset_type), 'geojson': gj, 'updated': datetime.utcnow() } dataset.services.append(service) dataset.updated = datetime.utcnow() dataset.save() # do compliance checker / metadata now scores = self.ccheck_station(sensor_ml) metamap = self.metamap_station(sensor_ml) try: self.save_ccheck_station('ioos', dataset._id, scores, metamap) except Exception as e: app.logger.warn( "could not save compliancecheck/metamap information: %s", e) return "Harvest Successful"
def parseGDAReferencedElement(gdaMembers, elementName): """Function to parse an element of a "GetDataAvailability" member""" element = testXMLAttribute(gdaMembers.find(nspv(elementName)), nspv("xlink:href")) return (element)
def decode_atomxml(xml_string): """ here parse atom xml to a dict for instanciating of OWC:Context :param xmlstring: :return: OwcContext-ready dict """ context_base_dict = { "type": "FeatureCollection", "id": None, "bbox": None, "properties": { "lang": None, "links": { "profiles": [], "via": [], }, 'title': None, 'abstract': None, 'updated': None, 'authors': [], 'publisher': None, 'generator': None, 'display': None, 'rights': None, 'date': None, 'categories': [], }, 'features': [] } feed_root = etree.fromstring(xml_string) # feed_root = etree.parse(xml_bytes) # feed_root xml lang use? # # log.debug(feed_root) # feed xml:lang=en # lang = feed_root.get('{http://www.w3.org/XML/1998/namespace}lang') lang = util.testXMLAttribute(feed_root, '{http://www.w3.org/XML/1998/namespace}lang') # log.debug("lang %s ", lang) context_base_dict['properties'].update({"lang": lang}) # <id> val = feed_root.find(util.nspath_eval('atom:id', ns)) id = util.testXMLValue(val) # log.debug("id %s :: %s", id, val) context_base_dict.update({"id": id}) # <link rel="profile" # href="http://www.opengis.net/spec/owc-atom/1.0/req/core" # title="compliant bla bla" # < link rel = "via" type = "application/xml" href = "..." title = "..." vals = feed_root.findall(util.nspath_eval('atom:link', ns)) links_profile = [] links_via = [] for val in vals: rel = util.testXMLAttribute(val, 'rel') href = util.testXMLAttribute(val, 'href') mimetype = util.testXMLAttribute(val, 'type') lang = util.testXMLAttribute(val, 'lang') title = util.testXMLAttribute(val, 'title') length = util.testXMLAttribute(val, 'length') link = { "href": href, "type": mimetype, "length": length, "lang": lang, "title": title, "rel": rel } # log.debug("link %s :: %s", link, vals) if link.get("rel") == "profile" and not is_empty(link): links_profile.append(link) elif link.get("rel") == "via" and not is_empty(link): links_via.append(link) else: log.warning("unknown link type in Ows Context section: %r", link) context_base_dict['properties']['links'].update( {"profiles": links_profile}) context_base_dict['properties']['links'].update({"via": links_via}) # <title> val = feed_root.find(util.nspath_eval('atom:title', ns)) title = util.testXMLValue(val) # log.debug("title %s :: %s", title, val) context_base_dict['properties'].update({"title": title}) # <subtitle type = "html" val = feed_root.find(util.nspath_eval('atom:subtitle', ns)) subtitle = util.testXMLValue(val) # log.debug("subtitle %s :: %s", subtitle, val) context_base_dict['properties'].update({"abstract": subtitle}) # <author> .. # <name> # <email> vals = feed_root.findall(util.nspath_eval('atom:author', ns)) authors = [] for val in vals: val_name = val.find(util.nspath_eval('atom:name', ns)) val_email = val.find(util.nspath_eval('atom:email', ns)) val_uri = val.find(util.nspath_eval('atom:uri', ns)) name = util.testXMLValue(val_name) email = util.testXMLValue(val_email) uri = util.testXMLValue(val_uri) author = {"name": name, "email": email, "uri": uri} # log.debug("author %s :: %s", author, vals) if not is_empty(author): authors.append(author) context_base_dict['properties'].update({"authors": authors}) # <georss:where> val = feed_root.find(util.nspath_eval('georss:where', ns)) if val is not None: if len(list(val)) > 0: xmltxt = element_to_string(list(val)[0], False) # log.debug("geometry %s :: %s", xmltxt, val) context_base_dict['properties'].update( {"bbox": xmltxt.decode('utf-8')}) # <updated>2012-11-04T17:26:23Z</updated> val = feed_root.find(util.nspath_eval('atom:updated', ns)) update_date = util.testXMLValue(val) # log.debug("updated %s :: %s", update_date, val) context_base_dict['properties'].update({"updated": update_date}) # <dc:date>2009-01-23T09:08:56.000Z/2009-01-23T09:14:08.000Z</dc:date> val = feed_root.find(util.nspath_eval('dc:date', ns)) time_interval_of_interest = util.testXMLValue(val) # log.debug("dc:date %s :: %s", time_interval_of_interest, val) context_base_dict['properties'].update({"date": time_interval_of_interest}) # <rights> val = feed_root.find(util.nspath_eval('atom:rights', ns)) rights = util.testXMLValue(val) # log.debug("rights %s :: %s", rights, val) context_base_dict['properties'].update({"rights": rights}) # <dc:publisher> val = feed_root.find(util.nspath_eval('dc:publisher', ns)) publisher = util.testXMLValue(val) # log.debug("dc:publisher %s :: %s", publisher, val) context_base_dict['properties'].update({"publisher": publisher}) # <owc:display> # <owc:pixelWidth> val_display = feed_root.find(util.nspath_eval('owc:display', ns)) val_pixel_width = None if val_display is None \ else val_display.find(util.nspath_eval('owc:pixelWidth', ns)) val_pixel_height = None if val_display is None \ else val_display.find(util.nspath_eval('owc:pixelHeight', ns)) val_mm_per_pixel = None if val_display is None \ else val_display.find(util.nspath_eval('owc:mmPerPixel', ns)) pixel_width = util.testXMLValue(val_pixel_width) pixel_height = util.testXMLValue(val_pixel_height) mm_per_pixel = util.testXMLValue(val_mm_per_pixel) owc_display = { "pixelWidth": pixel_width, "pixelHeight": pixel_height, "mmPerPixel": mm_per_pixel } # log.debug("display %s :: %s", owc_display, val_display) if not is_empty(owc_display): context_base_dict['properties'].update({"display": owc_display}) # <generator uri="http://w.." version="1.0">MiraMon</generator> val = feed_root.find(util.nspath_eval('atom:generator', ns)) name = util.testXMLValue(val) version = util.testXMLAttribute(val, 'version') uri = util.testXMLAttribute(val, 'uri') owc_generator = {"name": name, "version": version, "uri": uri} # log.debug("generator %s :: %s", owc_generator, val) if not is_empty(owc_generator): context_base_dict['properties'].update({"generator": owc_generator}) # <category term="maps" label="This file contains maps"/> vals = feed_root.findall(util.nspath_eval('atom:category', ns)) categories = [] for val in vals: term = util.testXMLAttribute(val, 'term') scheme = util.testXMLAttribute(val, 'scheme') label = util.testXMLAttribute(val, 'label') category = {"term": term, "scheme": scheme, "label": label} # log.debug("category %s :: %s", category, vals) if not is_empty(category): categories.append(category) context_base_dict['properties'].update({"categories": categories}) # <entry> ... entries = feed_root.findall(util.nspath_eval('atom:entry', ns)) resources = [] for entry in entries: entry_dict = parse_entry(entry) if entry_dict.get("id") is not None: resources.append(entry_dict) else: log.warning("feature entry has no id, not allowed: skipping!") context_base_dict.update({"features": resources}) return context_base_dict
def parse_entry(entry_node): """ parse an aotm entry into a feature/resource dict to build OwcResource from :param entry_node: xml element root node of the atom:entry :return: dictionary for OwcResource.from_dict() """ resource_base_dict = { "type": "Feature", "id": None, "geometry": None, "properties": { 'title': None, 'abstract': None, 'updated': None, 'date': None, 'authors': [], 'publisher': None, 'rights': None, 'categories': [], "links": { "alternates": [], "previews": [], "data": [], "via": [], }, 'offerings': [], 'active': None, 'minscaledenominator': None, 'maxscaledenominator': None, 'folder': None } } # <id>ftp://ftp.remotesensing.org/pub/geotiff/samples/gdal_eg/cea.txt</id> val = entry_node.find(util.nspath_eval('atom:id', ns)) id = util.testXMLValue(val) # log.debug("entry :id %s :: %s", id, val) resource_base_dict.update({"id": id}) # <title>GeoTIFF Example</title> val = entry_node.find(util.nspath_eval('atom:title', ns)) title = util.testXMLValue(val) # log.debug("entry: title %s :: %s", id, val) resource_base_dict['properties'].update({"title": title}) # <updated>2011-11-01T00:00:00Z</updated> val = entry_node.find(util.nspath_eval('atom:updated', ns)) update_date = util.testXMLValue(val) # log.debug("entry: updated %s :: %s", update_date, val) resource_base_dict['properties'].update({"updated": update_date}) # <dc:publisher> val = entry_node.find(util.nspath_eval('dc:publisher', ns)) publisher = util.testXMLValue(val) # log.debug("entry: dc:publisher %s :: %s", publisher, val) resource_base_dict['properties'].update({"publisher": publisher}) # <dc:rights> val = entry_node.find(util.nspath_eval('dc:rights', ns)) rights = util.testXMLValue(val) # log.debug("entry: rights %s :: %s", rights, val) resource_base_dict['properties'].update({"rights": rights}) # <georss:where> val = entry_node.find(util.nspath_eval('georss:where', ns)) if val is not None: if len(list(val)) > 0: # xmltxt = etree.tostring( # list(val)[0], encoding='utf8', method='xml') xmltxt = element_to_string(list(val)[0], False) # TODO here parse geometry?? # log.debug("entry: geometry %s :: %s", xmltxt, val) resource_base_dict.update({"geometry": xmltxt.decode('utf-8')}) # <content type = "text" > aka subtitle, aka abstract val = entry_node.find(util.nspath_eval('atom:content', ns)) subtitle = util.testXMLValue(val) # log.debug("entry: subtitle %s :: %s", subtitle, val) resource_base_dict['properties'].update({"abstract": subtitle}) # <author> .. # <name> # <email> vals = entry_node.findall(util.nspath_eval('atom:author', ns)) authors = [] for val in vals: val_name = val.find(util.nspath_eval('atom:name', ns)) val_email = val.find(util.nspath_eval('atom:email', ns)) val_uri = val.find(util.nspath_eval('atom:uri', ns)) name = util.testXMLValue(val_name) email = util.testXMLValue(val_email) uri = util.testXMLValue(val_uri) author = {"name": name, "email": email, "uri": uri} # log.debug("entry: author %s :: %s", author, vals) if not is_empty(author): authors.append(author) resource_base_dict['properties'].update({"authors": authors}) # <link rel="enclosure" type="image/png" # length="12345" title="..." href="http://o..."/> # <link rel="icon" type="image/png" title="Preview f..." # href="http://..."/> # <link rel="via" type="application/vnd.ogc.wms_xml" # title="Original .." href="...."/> vals = entry_node.findall(util.nspath_eval('atom:link', ns)) links_alternates = [] links_previews = [] links_data = [] links_via = [] for val in vals: rel = util.testXMLAttribute(val, 'rel') href = util.testXMLAttribute(val, 'href') mimetype = util.testXMLAttribute(val, 'type') lang = util.testXMLAttribute(val, 'lang') title = util.testXMLAttribute(val, 'title') length = util.testXMLAttribute(val, 'length') link = { "href": href, "type": mimetype, "length": length, "lang": lang, "title": title, "rel": rel } # log.debug("entry: link %s :: %s", link, vals) if link.get("rel") == "alternate" and not is_empty(link): links_alternates.append(link) elif link.get("rel") == "icon" and not is_empty(link): links_previews.append(link) elif link.get("rel") == "enclosure" and not is_empty(link): links_data.append(link) elif link.get("rel") == "via" and not is_empty(link): links_via.append(link) else: log.warning("unknown link type in Ows Resource entry section: %r", link) resource_base_dict['properties']['links'].update( {"alternates": links_alternates}) resource_base_dict['properties']['links'].update( {"previews": links_previews}) resource_base_dict['properties']['links'].update({"data": links_data}) resource_base_dict['properties']['links'].update({"via": links_via}) # <owc:offering code="http://www.opengis.net/spec/owc-at..."> # <owc:content type="image/tiff" href=".." # <owc:offering code="http://www.opengis.net/spec....l"> # <owc:content type="application/gml+xml"> # <owc:operation code="GetCapabilities" method="GET" # type="applica..." href="..." # <owc:request type="application/xml"> .. # <owc:styleSet> # <owc:name>raster</owc:name> # <owc:title>Default Raster</owc:title> # <owc:abstract>A sample style that draws a </owc:abstract> # <owc:legendURL href="h...." type="image/png"/> # </owc:styleSet> offering_nodes = entry_node.findall(util.nspath_eval('owc:offering', ns)) offerings = [] for offering_node in offering_nodes: offering_code = util.testXMLAttribute(offering_node, 'code') operations = [] contents = [] styles = [] operation_nodes = offering_node.findall( util.nspath_eval('owc:operation', ns)) for op_val in operation_nodes: operations_code = util.testXMLAttribute(op_val, 'code') http_method = util.testXMLAttribute(op_val, 'method') mimetype = util.testXMLAttribute(op_val, 'type') request_url = util.testXMLAttribute(op_val, 'href') req_content_val = val.find(util.nspath_eval('owc:request', ns)) req_content = None if req_content_val is not None: req_content = parse_owc_content(req_content_val) # TODO no example for result/response op_dict = { "code": operations_code, "method": http_method, "type": mimetype, "href": request_url, "request": None if is_empty(req_content) else req_content, "result": None } # log.debug("entry: operation %s :: %s", op_dict, vals) if not is_empty(op_dict): operations.append(op_dict) content_nodes = offering_node.findall( util.nspath_eval('owc:content', ns)) for cont_val in content_nodes: content_dict = parse_owc_content(cont_val) # log.debug("entry: content_dict %s :: %s", content_dict, vals) if not is_empty(content_dict): contents.append(content_dict) style_nodes = offering_node.findall( util.nspath_eval('owc:styleSet', ns)) for style_val in style_nodes: val_name = style_val.find(util.nspath_eval('owc:name', ns)) val_title = style_val.find(util.nspath_eval('owc:title', ns)) val_abstr = style_val.find(util.nspath_eval('owc:abstract', ns)) val_uri = style_val.find(util.nspath_eval('owc:legendURL', ns)) name = util.testXMLValue(val_name) title = util.testXMLValue(val_title) abstr = util.testXMLValue(val_abstr) legend_url = util.testXMLAttribute(val_uri, 'href') style_set = { "name": name, "title": title, "abstract": abstr, "default": None, "legendURL": legend_url, "content": None } # log.debug("entry: style_set %s :: %s", style_set, vals) if not is_empty(style_set): styles.append(style_set) offering_dict = { "code": offering_code, "operations": operations, "contents": contents, "styles": styles } if offering_code is not None: offerings.append(offering_dict) resource_base_dict['properties'].update({"offerings": offerings}) # TODO no examples for active attribute # <owc:minScaleDenominator>2500</owc:minScaleDenominator> val = entry_node.find(util.nspath_eval('owc:minScaleDenominator', ns)) min_scale_denominator = util.testXMLValue(val) # log.debug("entry: min-scale-... %s :: %s", min_scale_denominator, val) resource_base_dict['properties'].update( {"minscaledenominator": min_scale_denominator}) # <owc:maxScaleDenominator>25000</owc:maxScaleDenominator> val = entry_node.find(util.nspath_eval('owc:maxScaleDenominator', ns)) max_scale_denominator = util.testXMLValue(val) # log.debug("entry: max_scale_... %s :: %s", max_scale_denominator, val) resource_base_dict['properties'].update( {"maxscaledenominator": max_scale_denominator}) # TODO no examples for folder attribute return resource_base_dict
def __init__(self, element): self.id = testXMLAttribute(element, nsp('gml:id')) self.privacyAct = testXMLAttribute(element, nsp('sml:privacyAct')) self.intellectualPropertyRights = testXMLAttribute(element, nsp('sml:intellectualPropertyRights')) self.copyRights = testXMLAttribute(element, nsp('sml:copyRights')) self.documentation = [Documentation(x) for x in element.findall(nsp("sml:documentation"))]
def __init__(self, md=None): if md is None: self.xml = None self.identifier = None self.parentidentifier = None self.language = None self.dataseturi = None self.languagecode = None self.datestamp = None self.charset = None self.hierarchy = None self.contact = [] self.datetimestamp = None self.stdname = None self.stdver = None self.referencesystem = None self.identification = None self.serviceidentification = None self.identificationinfo = [] self.distribution = None self.dataquality = None else: if hasattr(md, 'getroot'): # standalone document self.xml = etree.tostring(md.getroot()) else: # part of a larger document self.xml = etree.tostring(md) val = md.find( util.nspath_eval('gmd:fileIdentifier/gco:CharacterString', namespaces)) self.identifier = util.testXMLValue(val) val = md.find( util.nspath_eval('gmd:parentIdentifier/gco:CharacterString', namespaces)) self.parentidentifier = util.testXMLValue(val) val = md.find( util.nspath_eval('gmd:language/gco:CharacterString', namespaces)) self.language = util.testXMLValue(val) val = md.find( util.nspath_eval('gmd:dataSetURI/gco:CharacterString', namespaces)) self.dataseturi = util.testXMLValue(val) val = md.find( util.nspath_eval('gmd:language/gmd:LanguageCode', namespaces)) self.languagecode = util.testXMLAttribute(val, 'codeListValue') val = md.find( util.nspath_eval('gmd:dateStamp/gco:Date', namespaces)) self.datestamp = util.testXMLValue(val) if not self.datestamp: val = md.find( util.nspath_eval('gmd:dateStamp/gco:DateTime', namespaces)) self.datestamp = util.testXMLValue(val) self.charset = _testCodeListValue( md.find( util.nspath_eval( 'gmd:characterSet/gmd:MD_CharacterSetCode', namespaces))) self.hierarchy = _testCodeListValue( md.find( util.nspath_eval('gmd:hierarchyLevel/gmd:MD_ScopeCode', namespaces))) self.contact = [] for i in md.findall( util.nspath_eval('gmd:contact/gmd:CI_ResponsibleParty', namespaces)): o = CI_ResponsibleParty(i) self.contact.append(o) val = md.find( util.nspath_eval('gmd:dateStamp/gco:DateTime', namespaces)) self.datetimestamp = util.testXMLValue(val) val = md.find( util.nspath_eval( 'gmd:metadataStandardName/gco:CharacterString', namespaces)) self.stdname = util.testXMLValue(val) val = md.find( util.nspath_eval( 'gmd:metadataStandardVersion/gco:CharacterString', namespaces)) self.stdver = util.testXMLValue(val) val = md.find( util.nspath_eval( 'gmd:referenceSystemInfo/gmd:MD_ReferenceSystem', namespaces)) if val is not None: self.referencesystem = MD_ReferenceSystem(val) else: self.referencesystem = None # TODO: merge .identificationinfo into .identification warnings.warn( 'the .identification and .serviceidentification properties will merge into ' '.identification being a list of properties. This is currently implemented ' 'in .identificationinfo. ' 'Please see https://github.com/geopython/OWSLib/issues/38 for more information', FutureWarning) val = md.find( util.nspath_eval( 'gmd:identificationInfo/gmd:MD_DataIdentification', namespaces)) val2 = md.find( util.nspath_eval( 'gmd:identificationInfo/srv:SV_ServiceIdentification', namespaces)) if val is not None: self.identification = MD_DataIdentification(val, 'dataset') self.serviceidentification = None elif val2 is not None: self.identification = MD_DataIdentification(val2, 'service') self.serviceidentification = SV_ServiceIdentification(val2) else: self.identification = None self.serviceidentification = None self.identificationinfo = [] for idinfo in md.findall( util.nspath_eval('gmd:identificationInfo', namespaces)): if len(idinfo) > 0: val = list(idinfo)[0] tagval = util.xmltag_split(val.tag) if tagval == 'MD_DataIdentification': self.identificationinfo.append( MD_DataIdentification(val, 'dataset')) elif tagval == 'MD_ServiceIdentification': self.identificationinfo.append( MD_DataIdentification(val, 'service')) elif tagval == 'SV_ServiceIdentification': self.identificationinfo.append( SV_ServiceIdentification(val)) val = md.find( util.nspath_eval('gmd:distributionInfo/gmd:MD_Distribution', namespaces)) if val is not None: self.distribution = MD_Distribution(val) else: self.distribution = None val = md.find( util.nspath_eval('gmd:dataQualityInfo/gmd:DQ_DataQuality', namespaces)) if val is not None: self.dataquality = DQ_DataQuality(val) else: self.dataquality = None
def __init__(self, element): self.name = testXMLAttribute(element, "name") self.interface_definition = InterfaceDefinition(element.find(nsp("sml:InterfaceDefinition")))
def __init__(self, element): super(AbstractDataComponent, self).__init__(element) # Attributes self.definition = testXMLAttribute(element,"definition") # anyURI, required self.updatable = get_boolean(testXMLAttribute(element,"updatable")) # boolean, optional self.optional = get_boolean(testXMLAttribute(element,"optional")) or False # boolean, default=False
def __init__(self, element): self.codeSpace = testXMLAttribute(element.find(nsp('sml:Term/sml:codeSpace')), nsp("xlink:href")) self.definition = testXMLAttribute(element.find(nsp('sml:Term')), "definition") self.value = testXMLValue(element.find(nsp('sml:Term/sml:value')))
def __init__(self, element): Term.__init__(self, element) self.name = testXMLAttribute(element, "name")
def get_stations_df(self, sos_url, station_urns_sel=None): """ Returns a Pandas Dataframe """ # oFrmts: IOOS SOS OutputFormat strings (first is compliant to the IOOS SOS spec, # second is to accommodate NDBC). More info here: # http://ioos.github.io/sos-guidelines/doc/wsdd/sos_wsdd_github_notoc/#describesensor-request:638e0b263020c13a76a55332bd966dbe oFrmts = [ 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"', 'text/xml;subtype="sensorML/1.0.1"' ] params = { 'service': 'SOS', 'request': 'GetCapabilities', 'acceptVersions': '1.0.0' } sos_url_params = sos_url + '?' + urlencode(params) # sos_url_params_quoted = quote(sos_url_params,"/=:") # sos_url_params_unquoted = unquote(sos_url_params) try: sosgc = SensorObservationService(sos_url_params) except (ConnectionError, ReadTimeout) as e: self.log.write( u"\nError: unable to connect to SOS service: {url} due to HTTP connection error." .format(url=sos_url_params)) self.log.write( u"\nHTTP connection error: {err}.".format(err=str(e))) sys.exit( "\nError: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}" .format(url=sos_url_params, err=str(e))) # vars to store returns from sos_collector.metadata_plus_exceptions function: sml_recs = {} sml_errors = {} describe_sensor_url = {} # leverage Pyoos Collector to query for all available stations and obtain SensorML # (if station subset not passed in --stations param) if station_urns_sel is not None: station_urns = station_urns_sel else: sos_collector = IoosSweSos(sos_url) station_urns = [ urn.name for urn in sos_collector.server.offerings if 'network' not in urn.name.split(':') ] sos_collector.features = station_urns # write out stations in SOS that will be handled: if self.verbose: self.log.write(u"\nStations to process for SOS: {sos}".format( sos=sos_url_params)) print("Stations to process for SOS: {sos}".format( sos=sos_url_params)) for feature in sos_collector.features: self.log.write(u"\n - {feature}".format(feature=feature)) print(" - {feature}".format(feature=feature)) # iterate over possible oFrmts expected of the various SOS services (IOOS SOS 1.0, NDBC): # for fmt in reversed(oFrmts): for fmt in oFrmts: try: sml_recs, sml_errors = sos_collector.metadata_plus_exceptions( output_format=fmt, timeout=200) # if no valid SensorML docs returned, try next oFrmt: if not sml_recs: continue else: # assign correct DescribeSensor url (use sos_collector.feature rather than sml_recs.keys() to # create DescribeSensor URLs for failures to record in logs): for station in sos_collector.features: describe_sensor_url[ station] = self.generate_describe_sensor_url( sosgc, procedure=station, oFrmt=fmt) # report on errors returned from metadata_plus_exceptions: if sml_errors: if self.verbose: for station, msg in iteritems(sml_errors): self.log.write( u"\nSOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}" .format(station=station, msg=msg)) print( "SOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}" .format(station=station, msg=msg)) else: self.log.write( u"\nSuccess, no errors returned from DescribeSensor requests in service: {sos}" .format(sos=sos_url_params)) print( "Success, no errors returned from DescribeSensor requests in service: {sos}" .format(sos=sos_url_params)) break # ServiceException shouldn't be thrown by metadata_plus_exceptions function, but handle regardless by attempting next oFrmt: except ServiceException as e: continue station_recs = [] failures = [] # generate Pandas DataFrame by populating 'station_recs' list by parsing SensorML strings: for station_idx, station_urn in enumerate(station_urns): if station_urns_sel is not None: # iterate oFrmts items for describe_sensor request (first is IOOS SOS spec-compliant, second is for NDBC SOS): for fmt in oFrmts: try: describe_sensor_url[ station_urn] = self.generate_describe_sensor_url( sosgc, procedure=station_urn, oFrmt=fmt) sml_str = sosgc.describe_sensor(procedure=station_urn, outputFormat=fmt, timeout=200) break except ServiceException as e: sml_errors[station_urn] = str(e) continue sml = SensorML(sml_str) else: # process valid SensorML responses, quietly pass on invalid stations (add to failures list for verbose reporting): try: sml = sml_recs[station_urn] except KeyError: self.log.write( u"\n\nStation: {station} failed (no SensorML in sml_recs dict). URL: {ds}" .format(station=station_urn, ds=describe_sensor_url[station_urn].replace( "&", "&"))) print( "Station: {station} failed (no SensorML in sml_recs dict). URL: {ds}" .format(station=station_urn, ds=describe_sensor_url[station_urn].replace( "&", "&"))) failures.append(station_urn) continue if self.sos_type.lower() == 'ndbc': # later: add an error check sosgc_station_offering = sosgc.contents[ 'station-' + station_urn.split(':')[-1]] else: sosgc_station_offering = None try: ds = IoosDescribeSensor(sml._root) except AttributeError: self.log.write( u"\nInvalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: " .format(station=station, ds=describe_sensor_url[station_urn].replace( "&", "&"))) print( "Invalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: " .format(station=station, ds=describe_sensor_url[station_urn].replace( "&", "&"))) station = OrderedDict() # debug: if self.verbose: self.log.write(u"\n\nProcessing station: {station}".format( station=station_urn)) print("Processing station: {station}".format( station=station_urn)) self.log.write("\n" + etree.tostring(sml._root).decode('utf-8')) # assign 'pos' to GML point location (accommodate 'gml:coordinates' as used by NDBC if gml:Point not found): try: pos = testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) \ if testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) is not None \ else testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:coordinates'))) station['lon'] = float(pos.split()[1]) station['lat'] = float(pos.split()[0]) except AttributeError as e: station['lon'] = None station['lat'] = None system_el = sml._root.findall(self.nsp('sml:member'))[0].find( self.nsp('sml:System')) # Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication # errors but there is not enough information in SensorML for alternatives) # Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage" documents = system_el.findall( self.nsp('sml:documentation/sml:DocumentList/sml:member')) documents_dct = {} for d in documents: document = Documentation(d) name = testXMLAttribute(d, "name") # url = document.documents[0].url documents_dct[name] = document # obtain list of contacts (accommodate 'sml:contact' element repetition used by NDBC insead of ContactList): contacts = system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \ if system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \ else system_el.findall(self.nsp('sml:contact')) contacts_dct = {} for c in contacts: contact = Contact(c) role = contact.role.split('/')[-1] contacts_dct[role] = contact # verify a 'publisher' Contact exists (template expects one): if "publisher" not in contacts_dct.keys(): self.log.write( u"\n\nStation: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]" .format(station=station_urn, roles=", ".join(contacts_dct.keys()))) print( "Station: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]" .format(station=station_urn, roles=", ".join(contacts_dct.keys()))) failures.append(station_urn) continue sweQuants = system_el.findall( self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity')) quant_lst = [ sweQuant.attrib['definition'] for sweQuant in sweQuants ] parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst] # attempt to read beginPosition, if available, otherwise use current date # bc ISO requires date value in output location in template: beginPosition = testXMLValue( system_el.find( self.nsp( 'sml:validTime/gml:TimePeriod/gml:beginPosition'))) try: begin_service_date = parser.parse(beginPosition) except (AttributeError, TypeError) as e: begin_service_date = datetime.now(pytz.utc) station['station_urn'] = station_urn station['sos_url'] = sos_url_params station['describesensor_url'] = describe_sensor_url[station_urn] station['shortName'] = ds.shortName station['longName'] = ds.longName if self.sos_type.lower() == 'ndbc': station['wmoID'] = station_urn.split(':')[-1] else: station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont) station['serverName'] = self.server_name # Some capabilities-level metadata: station['title'] = sosgc.identification.title station['abstract'] = sosgc.identification.abstract station['keywords'] = sosgc.identification.keywords station['begin_service_date'] = begin_service_date # Beware that a station can have >1 classifier of the same type # This code does not accommodate that possibility station['platformType'] = ds.platformType station['parentNetwork'] = ds.get_ioos_def('parentNetwork', 'classifier', ont) station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont) # store some nested dictionaries in 'station' for appropriate SensorML sources: station['contacts_dct'] = contacts_dct station['documents_dct'] = documents_dct if self.sos_type.lower( ) == 'ndbc' and sosgc_station_offering is not None: station['starting'] = sosgc_station_offering.begin_position station['ending'] = sosgc_station_offering.end_position else: station['starting'] = ds.starting station['ending'] = ds.ending if self.sos_type.lower( ) == 'ndbc' and sosgc_station_offering is not None: station[ 'variable_uris'] = sosgc_station_offering.observed_properties station['variables'] = [ var.split('/')[-1] for var in sosgc_station_offering.observed_properties ] station['parameter_uris'] = ','.join(station['variable_uris']) station['parameters'] = ','.join(station['variables']) else: station['variable_uris'] = ds.variables station['variables'] = [ var.split('/')[-1] for var in ds.variables ] station['parameter_uris'] = ','.join(quant_lst) station['parameters'] = ','.join(parameter_lst) if self.verbose: for var in station['variable_uris']: self.log.write(u"\nvariable: {var}".format(var=var)) print("variable: {var}".format(var=var)) # print(sosgc.contents) # for id, offering in sosgc.contents.iteritems(): # print("sosgc.contents: {item}".format(item=id)) # parse 'responseFormat' vals from SensorML: # response_formats = sosgc.contents[station_urn].response_formats response_formats = [] for id, sosgc.content in sosgc.contents.items(): if sosgc.content.name == station_urn: response_formats = sosgc.content.response_formats # response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() # if sosgc.content.name == station_urn ] # match responseFormats from SensorML (response_formats) against those passed in --response_formats parameter to # populate 'download_formats' list, that is then used to generate GetObservation requests for the template: # (default --response_formats values are: 'application/json,application/zip; subtype=x-netcdf' ) download_formats = [ response_format for response_format in response_formats if response_format in self.response_formats ] station['response_formats'] = response_formats station['download_formats'] = download_formats if self.verbose: for format in response_formats: self.log.write( u"\nresponseFormat: {format}".format(format=format)) print("responseFormat: {format}".format(format=format)) for format in download_formats: self.log.write( u"\ndownloadFormats: {format}".format(format=format)) print("downloadFormats: {format}".format(format=format)) # calculate event_time using self.getobs_req_hours: event_time_formatstr = "{begin:%Y-%m-%dT%H:%M:%S}{utc_code}/{end:%Y-%m-%dT%H:%M:%S}{utc_code}" utc_code = 'Z' if self.sos_type.lower() == 'ndbc' else '' if station['starting'] is not None and station[ 'ending'] is not None: event_time = event_time_formatstr.format( begin=station['ending'] - timedelta(hours=self.getobs_req_hours), end=station['ending'], utc_code=utc_code) if self.verbose: self.log.write( u"\nUsing starting/ending times from SensorML for eventTime" ) print( "Using starting/ending times from SensorML for eventTime" ) self.log.write( u"\nobservationTimeRange: starting: {start}, ending: {end}" .format(start=station['starting'], end=station['ending'])) print( "observationTimeRange: starting: {start}, ending: {end}" .format(start=station['starting'], end=station['ending'])) else: now = datetime.now(pytz.utc) then = now - timedelta(hours=self.getobs_req_hours) event_time = event_time_formatstr.format(begin=then, end=now, utc_code=utc_code) if self.verbose: self.log.write( u"\nNo 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) print( "No 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) if self.verbose: self.log.write(u"\neventTime: {time}".format(time=event_time)) print("eventTime: {time}".format(time=event_time)) # create a dict to store parameters for valid example GetObservation requests for station: getobs_req_dct = {} # populate a parameters dictionary for download links for each 'observedProperty' type # and secondly for each 'responseFormat' per observedProperty: getobs_params_base = { 'service': 'SOS', 'request': 'GetObservation', 'version': '1.0.0', 'offering': station_urn, 'eventTime': event_time } for variable in station['variable_uris']: getobs_params = getobs_params_base.copy() getobs_params['observedProperty'] = variable variable = variable.split('/')[-1] for format in download_formats: getobs_params['responseFormat'] = format getobs_request_url_encoded = sos_url + '?' + urlencode( getobs_params) getobs_request_url = unquote(getobs_request_url_encoded) getobs_req_dct[variable + '-' + format] = { 'variable': variable, 'url': getobs_request_url, 'format_type': self.RESPONSE_FORMAT_TYPE_MAP.get(format, format), 'format_name': self.RESPONSE_FORMAT_NAME_MAP.get(format, format) } if self.verbose: self.log.write( u"\ngetobs_request_url (var: {variable}): {getobs_request_url}" .format(variable=variable.split("/")[-1], getobs_request_url=getobs_request_url)) print( "getobs_request_url (var: {variable}): {getobs_request_url}" .format(variable=variable.split("/")[-1], getobs_request_url=getobs_request_url)) # ToDo: finish adding the 'getobs_req_dct' to the output template station['getobs_req_dct'] = getobs_req_dct station_recs.append(station) # extra debug for failed stations in verbose mode: if self.verbose: self.log.write( u"\n\n\nSOS DescribeSensor request errors recap. Failed requests:" ) print("SOS DescribeSensor request errors recap. Failed requests:") for station_fail, msg in iteritems(sml_errors): self.log.write( u"\n{station} - {msg}. DescribeSensor URL: {ds}".format( station=station_fail, msg=msg, ds=describe_sensor_url[station_fail].replace( "&", "&"))) print("{station} - {msg}. DescribeSensor URL: {ds}".format( station=station_fail, msg=msg, ds=describe_sensor_url[station_fail].replace("&", "&"))) if failures: self.log.write( u"\nStations in 'failures' list (should match DescribeSensor errors):" ) print( "Stations in 'failures' list (should match DescribeSensor errors):" ) for station_fail in failures: self.log.write(u"\n{station}".format(station=station_fail)) print("{station}".format(station=station_fail)) if station_recs: stations_df = pd.DataFrame.from_records(station_recs, columns=station.keys()) stations_df.index = stations_df['station_urn'] return stations_df else: return None
def __init__(self, element): self.reason = testXMLAttribute(element, "reason") self.value = testXMLValue(element)
def __init__(self, element): self.tokenSeparator = testXMLAttribute(element[-1], "tokenSeparator") # string, required self.blockSeparator = testXMLAttribute(element[-1], "blockSeparator") # string, required self.decimalSeparator = testXMLAttribute(element[-1], "decimalSeparator") or "." # string, optional, default="." self.collapseWhiteSpaces = get_boolean(testXMLAttribute(element[-1], "collapseWhiteSpaces")) or True # boolean, optional, default=True
def __init__(self, element): self._root = element self.description = testXMLValue( self._root.find(nspv("gml311:description"))) self.begin_position = extract_time( self._root.find( nspv('om10:samplingTime/gml311:TimePeriod/gml311:beginPosition' ))) self.end_position = extract_time( self._root.find( nspv( 'om10:samplingTime/gml311:TimePeriod/gml311:endPosition'))) self.procedures = [ testXMLAttribute(e, nspv("xlink:href")) for e in self._root.findall( nspv("om10:procedure/om10:Process/gml311:member")) ] self.observedProperties = [ testXMLAttribute(e, nspv("xlink:href")) for e in self._root.findall( nspv( "om10:observedProperty/swe101:CompositePhenomenon/swe101:component" )) ] # Can't use a full Xpath expression, so iterate over all metaDataProperties to find the IOOS FeatureType self.feature_type = None ft = self._root.findall( nspv( "om10:featureOfInterest/gml311:FeatureCollection/gml311:metaDataProperty/gml311:GenericMetaData/gml311:name" )) ft.extend( self._root.findall( nspv( "om10:featureOfInterest/gml311:FeatureCollection/gml311:metaDataProperty/gml311:name" ))) ft_def = "http://cf-pcmdi.llnl.gov/documents/cf-conventions/1.6/cf-conventions.html#discrete-sampling-geometries" for f in ft: if testXMLAttribute(f, "codeSpace") == ft_def: self.feature_type = testXMLValue(f) # BBOX envelope = self._root.find( nspv( "om10:featureOfInterest/gml311:FeatureCollection/gml311:boundedBy/gml311:Envelope" )) self.bbox_srs = Crs(testXMLAttribute(envelope, 'srsName')) lower_left_corner = testXMLValue( envelope.find(nspv('gml311:lowerCorner'))).split(" ") upper_right_corner = testXMLValue( envelope.find(nspv('gml311:upperCorner'))).split(" ") if self.bbox_srs.axisorder == "yx": self.bbox = box(float(lower_left_corner[1]), float(lower_left_corner[0]), float(upper_right_corner[1]), float(upper_right_corner[0])) else: self.bbox = box(float(lower_left_corner[0]), float(lower_left_corner[1]), float(upper_right_corner[0]), float(upper_right_corner[1])) # LOCATION location = self._root.find( nspv( "om10:featureOfInterest/gml311:FeatureCollection/gml311:location" )) # Should only have one child geo = list(location)[-1] self.location = {} def get_point(element, srs): name = testXMLValue(element.find(nspv("gml311:name"))) point = testXMLValue(element.find(nspv("gml311:pos"))).split(" ") if srs.axisorder == "yx": point = Point(float(point[1]), float(point[0])) else: point = Point(float(point[0]), float(point[1])) return name, point self.location_srs = Crs(testXMLAttribute(geo, "srsName")) if geo.tag == nspv("gml311:Point"): n, p = get_point(geo, self.location_srs) self.location[n] = p elif geo.tag == nspv("gml311:MultiPoint"): for point in geo.findall(nspv("gml311:pointMembers/gml311:Point")): n, p = get_point(point, self.location_srs) self.location[n] = p # Now the fields change depending on the FeatureType self.results = self._root.find(nspv("om10:result")) # TODO: This should be implemented as a Factory self.feature = None data = self.results.find(nspv("swe20:DataRecord")) if data is not None: if self.feature_type == 'timeSeries': self.feature = TimeSeries(data).feature elif self.feature_type == 'timeSeriesProfile': self.feature = TimeSeriesProfile(data).feature else: print("No feature type found.")
def __init__(self, element): self.arcrole = testXMLAttribute(element, nsp("xlink:arcrole")) self.url = testXMLAttribute(element, nsp("xlink:href")) self.documents = [Document(d) for d in element.findall(nsp("sml:Document"))]
def process_station(self, uid): """ Makes a DescribeSensor request based on a 'uid' parameter being a station procedure """ GML_NS = "http://www.opengis.net/gml" XLINK_NS = "http://www.w3.org/1999/xlink" with app.app_context(): metadata_value = etree.fromstring( self.sos.describe_sensor( outputFormat= 'text/xml;subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"', procedure=uid)) station_ds = IoosDescribeSensor(metadata_value) unique_id = station_ds.id if unique_id is None: app.logger.warn( "Could not get a 'stationID' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/stationID'" ) return dataset = db.Dataset.find_one({'uid': unicode(unique_id)}) if dataset is None: dataset = db.Dataset() dataset.uid = unicode(unique_id) # Find service reference in Dataset.services and remove (to replace it) tmp = dataset.services[:] for d in tmp: if d['service_id'] == self.service.get('_id'): dataset.services.remove(d) # Parsing messages messages = [] # NAME name = unicode_or_none(station_ds.shortName) if name is None: messages.append( u"Could not get a 'shortName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/shortName'" ) # DESCRIPTION description = unicode_or_none(station_ds.longName) if description is None: messages.append( u"Could not get a 'longName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/longName'" ) # PLATFORM TYPE asset_type = unicode_or_none(station_ds.platformType) if asset_type is None: messages.append( u"Could not get a 'platformType' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/platformType'" ) # LOCATION is in GML gj = None loc = station_ds.location if loc is not None and loc.tag == "{%s}Point" % GML_NS: pos_element = loc.find("{%s}pos" % GML_NS) # strip out points positions = map(float, testXMLValue(pos_element).split(" ")) crs = Crs(testXMLAttribute(pos_element, "srsName")) if crs.axisorder == "yx": gj = json.loads( geojson.dumps( geojson.Point([positions[1], positions[0]]))) else: gj = json.loads( geojson.dumps( geojson.Point([positions[0], positions[1]]))) else: messages.append( u"Found an unrecognized child of the sml:location element and did not attempt to process it: %s" % etree.tostring(loc).strip()) service = { # Reset service 'name': name, 'description': description, 'service_type': self.service.get('service_type'), 'service_id': ObjectId(self.service.get('_id')), 'data_provider': self.service.get('data_provider'), 'metadata_type': u'sensorml', 'metadata_value': unicode(etree.tostring(metadata_value)).strip(), 'messages': map(unicode, messages), 'keywords': map(unicode, sorted(station_ds.keywords)), 'variables': map(unicode, sorted(station_ds.variables)), 'asset_type': asset_type, 'geojson': gj, 'updated': datetime.utcnow() } dataset.services.append(service) dataset.updated = datetime.utcnow() dataset.save() return "Harvested"
def get_uom(element): uom = testXMLAttribute(element, "code") if uom is None: uom = testXMLAttribute(element, nspv("xlink:href")) return uom
def __init__(self, **kwargs): super(SweTimeSeries, self).__init__(**kwargs) # Parse out GML point. Defaults to 0 depth if none specified self.geo_srs = Crs( testXMLAttribute(self._location.find(nsp("Point", self.GML_NS)), 'srsName')) geo = [ float(v) for v in testXMLValue( self._location.find(nsp("Point/pos", self.GML_NS))).split(" ") ] if self.geo_srs.axisorder == "yx": self.geo = sPoint(geo[1], geo[0]) else: self.geo = sPoint(geo[0], geo[1]) try: self.geo.z = geo[2] except: pass pc = PointCollection() for row in self.results.data: p = Point() time = None z = None lat = None lon = None for field in row: if field.axis == "time": t = dateutil.parser.parse(field.value) p.time = t.astimezone(dateutil.tz.tzutc()) elif field.axis == "Long": lon = field elif field.axis == "Lat": lat = field elif field.axis == "h": z = field else: m = Member(value=field.value, unit=field.units, units_definition=field.units_url, name=field.name, definition=field.definition, standard=field.definition) p.add_member(m) # Set the spatial point if lon.srs != lat.srs: raise ValueError( "Longitude and Latitude need to have the same SRS/CRS!") p.location = sPoint(float(lon.value), float(lat.value), float(z.value)) pc.add_element(p) self.data = pc
def __init__(self, element): ReferenceGroup.__init__(self, element) GeneralInfoGroup.__init__(self, element) self.id = testXMLAttribute(element, nsp("gml:id")) self.date = testXMLValue(element.find(nsp('sml:date'))) self.description = testXMLValue(element.find(nsp('gml:description')))
def __init__(self, md=None, identtype=None): if md is None: self.identtype = None self.title = None self.alternatetitle = None self.aggregationinfo = None self.uricode = [] self.uricodespace = [] self.date = [] self.datetype = [] self.uselimitation = [] self.uselimitation_url = [] self.accessconstraints = [] self.classification = [] self.otherconstraints = [] self.securityconstraints = [] self.useconstraints = [] self.denominators = [] self.distance = [] self.uom = [] self.resourcelanguage = [] self.creator = [] self.publisher = [] self.contributor = [] self.edition = None self.abstract = None self.abstract_url = None self.purpose = None self.status = None self.contact = [] self.keywords = [] self.keywords2 = [] self.topiccategory = [] self.supplementalinformation = None self.extent = None self.bbox = None self.temporalextent_start = None self.temporalextent_end = None self.spatialrepresentationtype = [] else: self.identtype = identtype val = md.find( util.nspath_eval( 'gmd:citation/gmd:CI_Citation/gmd:title/gco:CharacterString', namespaces)) self.title = util.testXMLValue(val) val = md.find( util.nspath_eval( 'gmd:citation/gmd:CI_Citation/gmd:alternateTitle/gco:CharacterString', namespaces)) self.alternatetitle = util.testXMLValue(val) val = md.find(util.nspath_eval('gmd:aggregationInfo', namespaces)) self.aggregationinfo = util.testXMLValue(val) self.uricode = [] for i in md.findall( util.nspath_eval( 'gmd:citation/gmd:CI_Citation/gmd:identifier/gmd:RS_Identifier/gmd:code/gco:CharacterString', namespaces)): val = util.testXMLValue(i) if val is not None: self.uricode.append(val) self.uricodespace = [] for i in md.findall( util.nspath_eval( 'gmd:citation/gmd:CI_Citation/gmd:identifier/gmd:RS_Identifier/gmd:codeSpace/gco:CharacterString', namespaces)): val = util.testXMLValue(i) if val is not None: self.uricodespace.append(val) self.date = [] self.datetype = [] for i in md.findall( util.nspath_eval( 'gmd:citation/gmd:CI_Citation/gmd:date/gmd:CI_Date', namespaces)): self.date.append(CI_Date(i)) self.uselimitation = [] self.uselimitation_url = [] for i in md.findall( util.nspath_eval( 'gmd:resourceConstraints/gmd:MD_Constraints/gmd:useLimitation/gco:CharacterString', namespaces)): val = util.testXMLValue(i) if val is not None: self.uselimitation.append(val) for i in md.findall( util.nspath_eval( 'gmd:resourceConstraints/gmd:MD_Constraints/gmd:useLimitation/gmx:Anchor', namespaces)): val = util.testXMLValue(i) val1 = i.attrib.get(util.nspath_eval('xlink:href', namespaces)) if val is not None: self.uselimitation.append(val) self.uselimitation_url.append(val1) self.accessconstraints = [] for i in md.findall( util.nspath_eval( 'gmd:resourceConstraints/gmd:MD_LegalConstraints/gmd:accessConstraints/gmd:MD_RestrictionCode', namespaces)): val = _testCodeListValue(i) if val is not None: self.accessconstraints.append(val) self.classification = [] for i in md.findall( util.nspath_eval( 'gmd:resourceConstraints/gmd:MD_LegalConstraints/gmd:accessConstraints/gmd:MD_ClassificationCode', namespaces)): val = _testCodeListValue(i) if val is not None: self.classification.append(val) self.otherconstraints = [] for i in md.findall( util.nspath_eval( 'gmd:resourceConstraints/gmd:MD_LegalConstraints/gmd:otherConstraints/gco:CharacterString', namespaces)): val = util.testXMLValue(i) if val is not None: self.otherconstraints.append(val) self.securityconstraints = [] for i in md.findall( util.nspath_eval( 'gmd:resourceConstraints/gmd:MD_SecurityConstraints/gmd:classification/gmd:MD_ClassificationCode', namespaces)): val = util.testXMLValue(i) if val is not None: self.securityconstraints.append(val) self.useconstraints = [] for i in md.findall( util.nspath_eval( 'gmd:resourceConstraints/gmd:MD_LegalConstraints/gmd:useConstraints/gmd:MD_RestrictionCode', namespaces)): val = _testCodeListValue(i) if val is not None: self.useconstraints.append(val) self.denominators = [] for i in md.findall( util.nspath_eval( 'gmd:spatialResolution/gmd:MD_Resolution/gmd:equivalentScale/gmd:MD_RepresentativeFraction/gmd:denominator/gco:Integer', namespaces)): val = util.testXMLValue(i) if val is not None: self.denominators.append(val) self.distance = [] self.uom = [] for i in md.findall( util.nspath_eval( 'gmd:spatialResolution/gmd:MD_Resolution/gmd:distance/gco:Distance', namespaces)): val = util.testXMLValue(i) if val is not None: self.distance.append(val) self.uom.append(i.get("uom")) self.resourcelanguage = [] for i in md.findall( util.nspath_eval('gmd:language/gmd:LanguageCode', namespaces)): val = _testCodeListValue(i) if val is not None: self.resourcelanguage.append(val) self.creator = [] self.publisher = [] self.contributor = [] for val in md.findall( util.nspath_eval( 'gmd:pointOfContact/gmd:CI_ResponsibleParty', namespaces)): role = val.find( util.nspath_eval('gmd:role/gmd:CI_RoleCode', namespaces)) if role is not None: clv = _testCodeListValue(role) rp = CI_ResponsibleParty(val) if clv == 'originator': self.creator.append(rp) elif clv == 'publisher': self.publisher.append(rp) elif clv == 'author': self.contributor.append(rp) val = md.find( util.nspath_eval('gmd:edition/gco:CharacterString', namespaces)) self.edition = util.testXMLValue(val) val = md.find( util.nspath_eval('gmd:abstract/gco:CharacterString', namespaces)) self.abstract = util.testXMLValue(val) val = md.find( util.nspath_eval('gmd:abstract/gmx:Anchor', namespaces)) if val is not None: self.abstract = util.testXMLValue(val) self.abstract_url = val.attrib.get( util.nspath_eval('xlink:href', namespaces)) val = md.find( util.nspath_eval('gmd:purpose/gco:CharacterString', namespaces)) self.purpose = util.testXMLValue(val) self.status = _testCodeListValue( md.find( util.nspath_eval('gmd:status/gmd:MD_ProgressCode', namespaces))) self.contact = [] for i in md.findall( util.nspath_eval( 'gmd:pointOfContact/gmd:CI_ResponsibleParty', namespaces)): o = CI_ResponsibleParty(i) self.contact.append(o) self.spatialrepresentationtype = [] for val in md.findall( util.nspath_eval( 'gmd:spatialRepresentationType/gmd:MD_SpatialRepresentationTypeCode', namespaces)): val = util.testXMLAttribute(val, 'codeListValue') if val: self.spatialrepresentationtype.append(val) warnings.warn( 'The .keywords and .keywords2 properties will merge into the ' '.keywords property in the future, with .keywords becoming a list ' 'of MD_Keywords instances. This is currently implemented in .keywords2. ' 'Please see https://github.com/geopython/OWSLib/issues/301 for more information', FutureWarning) self.keywords = [] for i in md.findall( util.nspath_eval('gmd:descriptiveKeywords', namespaces)): mdkw = {} mdkw['type'] = _testCodeListValue( i.find( util.nspath_eval( 'gmd:MD_Keywords/gmd:type/gmd:MD_KeywordTypeCode', namespaces))) mdkw['thesaurus'] = {} val = i.find( util.nspath_eval( 'gmd:MD_Keywords/gmd:thesaurusName/gmd:CI_Citation/gmd:title/gco:CharacterString', namespaces)) mdkw['thesaurus']['title'] = util.testXMLValue(val) val = i.find( util.nspath_eval( 'gmd:MD_Keywords/gmd:thesaurusName/gmd:CI_Citation/gmd:date/gmd:CI_Date/gmd:date/gco:Date', namespaces)) mdkw['thesaurus']['date'] = util.testXMLValue(val) val = i.find( util.nspath_eval( 'gmd:MD_Keywords/gmd:thesaurusName/gmd:CI_Citation/gmd:date/gmd:CI_Date/gmd:dateType/gmd:CI_DateTypeCode', namespaces)) mdkw['thesaurus']['datetype'] = util.testXMLValue(val) mdkw['keywords'] = [] for k in i.findall( util.nspath_eval('gmd:MD_Keywords/gmd:keyword', namespaces)): val = k.find( util.nspath_eval('gco:CharacterString', namespaces)) if val is not None: val2 = util.testXMLValue(val) if val2 is not None: mdkw['keywords'].append(val2) self.keywords.append(mdkw) self.keywords2 = [] for mdkw in md.findall( util.nspath_eval('gmd:descriptiveKeywords/gmd:MD_Keywords', namespaces)): self.keywords2.append(MD_Keywords(mdkw)) self.topiccategory = [] for i in md.findall( util.nspath_eval( 'gmd:topicCategory/gmd:MD_TopicCategoryCode', namespaces)): val = util.testXMLValue(i) if val is not None: self.topiccategory.append(val) val = md.find( util.nspath_eval( 'gmd:supplementalInformation/gco:CharacterString', namespaces)) self.supplementalinformation = util.testXMLValue(val) # There may be multiple geographicElement, create an extent # from the one containing either an EX_GeographicBoundingBox or EX_BoundingPolygon. # The schema also specifies an EX_GeographicDescription. This is not implemented yet. val = None val2 = None val3 = None extents = md.findall(util.nspath_eval('gmd:extent', namespaces)) extents.extend( md.findall(util.nspath_eval('srv:extent', namespaces))) for extent in extents: if val is None: for e in extent.findall( util.nspath_eval( 'gmd:EX_Extent/gmd:geographicElement', namespaces)): if e.find( util.nspath_eval( 'gmd:EX_GeographicBoundingBox', namespaces)) is not None or e.find( util.nspath_eval( 'gmd:EX_BoundingPolygon', namespaces)) is not None: val = e break self.extent = EX_Extent(val) self.bbox = self.extent.boundingBox # for backwards compatibility if val2 is None: val2 = extent.find( util.nspath_eval( 'gmd:EX_Extent/gmd:temporalElement/gmd:EX_TemporalExtent/gmd:extent/gml:TimePeriod/gml:beginPosition', namespaces)) if val2 is None: val2 = extent.find( util.nspath_eval( 'gmd:EX_Extent/gmd:temporalElement/gmd:EX_TemporalExtent/gmd:extent/gml32:TimePeriod/gml32:beginPosition', namespaces)) self.temporalextent_start = util.testXMLValue(val2) if val3 is None: val3 = extent.find( util.nspath_eval( 'gmd:EX_Extent/gmd:temporalElement/gmd:EX_TemporalExtent/gmd:extent/gml:TimePeriod/gml:endPosition', namespaces)) if val3 is None: val3 = extent.find( util.nspath_eval( 'gmd:EX_Extent/gmd:temporalElement/gmd:EX_TemporalExtent/gmd:extent/gml32:TimePeriod/gml32:endPosition', namespaces)) self.temporalextent_end = util.testXMLValue(val3)
def __init__(self, element): # Attributes self.id = testXMLAttribute(element, "id") # string, optional # Elements self.extention = [] # anyType, min=0, max=X
def __init__(self, element): self._root = element OM_NS = ns.get_versioned_namespace('om', '1.0') GML_NS = ns.get_versioned_namespace('gml', '3.1.1') SWE_NS = ns.get_versioned_namespace('swe', '1.0') self.name = testXMLValue(self._root.find(nsp("name", GML_NS))) self.description = testXMLValue( self._root.find(nsp("description", GML_NS))) self.observedProperties = [] for op in self._root.findall(nsp('observedProperty', OM_NS)): self.observedProperties.append( testXMLAttribute(op, nspv('xlink:href'))) # BBOX try: envelope = self._root.find(nsp('boundedBy/Envelope', GML_NS)) lower_left_corner = testXMLValue( envelope.find(nsp('lowerCorner', GML_NS))).split(" ") upper_right_corner = testXMLValue( envelope.find(nsp('upperCorner', GML_NS))).split(" ") self.bbox_srs = Crs(testXMLAttribute(envelope, 'srsName')) # Always keep the BBOX as minx, miny, maxx, maxy if self.bbox_srs.axisorder == "yx": self.bbox = box(float(lower_left_corner[1]), float(lower_left_corner[0]), float(upper_right_corner[1]), float(upper_right_corner[0])) else: self.bbox = box(float(lower_left_corner[0]), float(lower_left_corner[1]), float(upper_right_corner[0]), float(upper_right_corner[1])) except Exception: self.bbox = None self.bbox_srs = None # Time range fp = nsp('samplingTime', OM_NS) mp = nsp('TimePeriod', GML_NS) lp = nsp('beginPosition', GML_NS) begin_position_element = self._root.find('%s/%s/%s' % (fp, mp, lp)) self.begin_position = extract_time(begin_position_element) ep = nsp('endPosition', GML_NS) end_position_element = self._root.find('%s/%s/%s' % (fp, mp, ep)) self.end_position = extract_time(end_position_element) feature_of_interest_element = self._root.find( nsp("featureOfInterest", OM_NS)) self.feature_type = testXMLValue( feature_of_interest_element.find( nsp("FeatureCollection/metaDataProperty/name", GML_NS))) # Now the fields change depending on the FeatureType result_element = self._root.find(nsp("result", OM_NS)) #TODO: This should be implemented as a Factory self.feature = None if self.feature_type == 'timeSeries': self.feature = SweTimeSeries( feature_of_interest=feature_of_interest_element, result=result_element, GML_NS=GML_NS, OM_NS=OM_NS, SWE_NS=SWE_NS) elif self.feature_type == 'trajectoryProfile': self.feature = SweTrajectoryProfile( feature_of_interest=feature_of_interest_element, result=result_element, GML_NS=GML_NS, OM_NS=OM_NS, SWE_NS=SWE_NS) elif self.feature_type == 'timeSeriesProfile': self.feature = SweTimeseriesProfile( feature_of_interest=feature_of_interest_element, result=result_element, GML_NS=GML_NS, OM_NS=OM_NS, SWE_NS=SWE_NS)
def get_stations_df(self, sos_url, station_urns_sel=None): """ Returns a GeoDataFrame """ # LATER: ADD ERROR TEST/CATCH AFTER EACH WEB REQUEST oFrmt = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"' params = { 'service': 'SOS', 'request': 'GetCapabilities', 'acceptVersions': '1.0.0' } sos_url_params = sos_url + '?' + urlencode(params) sos_url_params_esc = sos_url_params.replace("&", "&") # sos_url_params_quoted = quote(sos_url_params,"/=:") # sos_url_params_unquoted = unquote(sos_url_params) try: sosgc = SensorObservationService(sos_url_params) except (ConnectionError, ReadTimeout) as e: self.log.write( u"Error: unable to connect to SOS service: {url} due to HTTP connection error.\n" .format(url=sos_url_params)) self.log.write( u"HTTP connection error: {err}.\n".format(err=e.message)) sys.exit( "Error: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}" .format(url=sos_url_params, err=e.message)) if station_urns_sel is not None: station_urns = station_urns_sel else: sos_collector = IoosSweSos(sos_url) station_urns = [ urn.name for urn in sos_collector.server.offerings if 'network' not in urn.name.split(':') ] sos_collector.features = station_urns sml_lst = sos_collector.metadata(timeout=200) station_recs = [] for station_idx, station_urn in enumerate(station_urns): if station_urns_sel is not None: sml_str = sosgc.describe_sensor(procedure=station_urn, outputFormat=oFrmt, timeout=200) sml = SensorML(sml_str) else: sml = sml_lst[station_idx] # debug: # if self.verbose: # self.log.write(unicode(etree.tostring(sml._root))) ds = IoosDescribeSensor(sml._root) pos = testXMLValue( ds.system.location.find(self.nsp('gml:Point/gml:pos'))) system_el = sml._root.findall(self.nsp('sml:member'))[0].find( self.nsp('sml:System')) # Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication # errors but there is not enough information in SensorML for alternatives) # Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage" documents = system_el.findall( self.nsp('sml:documentation/sml:DocumentList/sml:member')) documents_dct = {} for d in documents: document = Documentation(d) name = testXMLAttribute(d, "name") # url = document.documents[0].url documents_dct[name] = document contacts = system_el.findall( self.nsp('sml:contact/sml:ContactList/sml:member')) contacts_dct = {} for c in contacts: contact = Contact(c) role = contact.role.split('/')[-1] contacts_dct[role] = contact sweQuants = system_el.findall( self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity')) quant_lst = [ sweQuant.attrib['definition'] for sweQuant in sweQuants ] parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst] # attempt to read beginPosition, if available: beginPosition = testXMLValue( system_el.find( self.nsp( 'sml:validTime/gml:TimePeriod/gml:beginPosition'))) try: begin_service_date = parser.parse(beginPosition) except AttributeError as e: begin_service_date = None station = OrderedDict() station['station_urn'] = station_urn station['sos_url'] = sos_url_params_esc station['lon'] = float(pos.split()[1]) station['lat'] = float(pos.split()[0]) station['shortName'] = ds.shortName station['longName'] = ds.longName station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont) station['serverName'] = self.server_name # Some capabilities-level metadata: station['title'] = sosgc.identification.title station['abstract'] = sosgc.identification.abstract station['keywords'] = sosgc.identification.keywords station['begin_service_date'] = begin_service_date # Beware that a station can have >1 classifier of the same type # This code does not accommodate that possibility station['platformType'] = ds.platformType station['parentNetwork'] = ds.get_ioos_def('parentNetwork', 'classifier', ont) station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont) # store some nested dictionaries in 'station' for appopriate SensorML sources: station['contacts_dct'] = contacts_dct station['documents_dct'] = documents_dct # MW: the 'operator_' and 'publisher_' attributes can be removed bc they are not used # in the template code currently in favor of 'contacts_dct' # station['operatorSector'] = ds.get_ioos_def('operatorSector', 'classifier', ont) # station['operator_org'] = contacts_dct['operator'].organization # station['operator_country'] = contacts_dct['operator'].country # station['operator_url'] = contacts_dct['operator'].url # station['operator_email'] = contacts_dct['operator'].email # station['publisher'] = ds.get_ioos_def('publisher', 'classifier', ont) # station['publisher_org'] = contacts_dct['publisher'].organization # station['publisher_url'] = contacts_dct['publisher'].url # station_dct['publisher_email'] = contacts_dct['publisher'].electronicMailAddress station['starting'] = ds.starting station['ending'] = ds.ending # station['starting_isostr'] = datetime.isoformat(ds.starting) # station['ending_isostr'] = datetime.isoformat(ds.ending) station['parameter_uris'] = ','.join(quant_lst) station['parameters'] = ','.join(parameter_lst) station['variables'] = [var.split('/')[-1] for var in ds.variables] # debug: if self.verbose: self.log.write(u"\nProcessing station: {station}\n".format( station=station_urn)) print("\nProcessing station: {station}".format( station=station_urn)) for var in ds.variables: self.log.write(u"variable: {var}\n".format(var=var)) print("variable: {var}".format(var=var)) # print(sosgc.contents) # for id, offering in sosgc.contents.iteritems(): # print("sosgc.contents: {item}".format(item=id)) # parse 'responseFormat' values and populate list: # response_formats = sosgc.contents[station_urn].response_formats response_formats = [] for id, sosgc.content in sosgc.contents.items(): if sosgc.content.name == station_urn: response_formats = sosgc.content.response_formats # response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() if sosgc.content.name == station_urn ] # subset responseFormats (response_formats) for download links matching those passed in --response_formats parameter # (or 'application/json,application/zip; subtype=x-netcdf' by default): download_formats = [ response_format for response_format in response_formats if response_format in self.response_formats ] station['response_formats'] = response_formats station['download_formats'] = download_formats if self.verbose: for format in response_formats: self.log.write( u"responseFormat: {format}\n".format(format=format)) print("responseFormat: {format}".format(format=format)) for format in download_formats: self.log.write( u"downloadFormats: {format}\n".format(format=format)) print("downloadFormats: {format}".format(format=format)) # calculate event_time using self.getobs_req_hours: if ds.starting is not None and ds.ending is not None: event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format( begin=ds.ending - timedelta(hours=self.getobs_req_hours), end=ds.ending) else: now = datetime.now(pytz.utc) then = now - timedelta(hours=self.getobs_req_hours) event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format( begin=then, end=now) if self.verbose: self.log.write( u"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}\n" .format(then=then, now=now)) print( "then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) if self.verbose: self.log.write(u"eventTime: {time}\n".format(time=event_time)) print("eventTime: {time}".format(time=event_time)) # create a dict to store parameters for valid example GetObservation requests for station: getobs_req_dct = {} # populate a parameters dictionary for download links for each 'observedProperty' type and secondly for each 'responseFormat' per observedProperty: getobs_params_base = { 'service': 'SOS', 'request': 'GetObservation', 'version': '1.0.0', 'offering': station_urn, 'eventTime': event_time } for variable in ds.variables: getobs_params = getobs_params_base.copy() getobs_params['observedProperty'] = variable variable = variable.split('/')[-1] for format in download_formats: getobs_params['responseFormat'] = format getobs_request_url_encoded = sos_url + '?' + urlencode( getobs_params) getobs_request_url = unquote(getobs_request_url_encoded) getobs_request_url_esc = getobs_request_url.replace( "&", "&") getobs_req_dct[variable + '-' + format] = { 'variable': variable, 'url': getobs_request_url_esc, 'format_type': self.RESPONSE_FORMAT_TYPE_MAP[format], 'format_name': self.RESPONSE_FORMAT_NAME_MAP[format] } if self.verbose: self.log.write( u"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}\n" .format( variable=variable.split("/")[-1], getobs_request_url=getobs_request_url, getobs_request_url_esc=getobs_request_url_esc)) print( "getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}" .format( variable=variable.split("/")[-1], getobs_request_url=getobs_request_url, getobs_request_url_esc=getobs_request_url_esc)) # ToDo: finish adding the 'getobs_req_dct' to the output template station['getobs_req_dct'] = getobs_req_dct station_recs.append(station) stations_df = pd.DataFrame.from_records(station_recs, columns=station.keys()) stations_df.index = stations_df['station_urn'] return stations_df