def __init__(self, element): super(DescribeSensor, self).__init__(element=element) """ Common things between all describe sensor requests """ self.ioos_version = "1.0" self.system = SensorML(element).members[0] self.shortName = get_named_by_definition(self.system.get_identifiers_by_name("shortName"), "http://mmisw.org/ont/ioos/definition/shortName") self.longName = get_named_by_definition(self.system.get_identifiers_by_name("longName"), "http://mmisw.org/ont/ioos/definition/longName") self.keywords = map(unicode, self.system.keywords) # Location try: self.location = self.system.location[0] except (TypeError, IndexError): # No location exists self.location = None # Timerange try: timerange = testXMLValue(self.system.get_capabilities_by_name("observationTimeRange")[0].find(".//" + nsp("swe101:TimeRange/swe101:value"))).split(" ") self.starting = parser.parse(timerange[0]) self.ending = parser.parse(timerange[1]) except (AttributeError, TypeError, ValueError): self.starting = None self.ending = None
class DescribeSensor(IoosDescribeSensor): def __init__(self, element): super(DescribeSensor, self).__init__(element=element) """ Common things between all describe sensor requests """ self.ioos_version = "1.0" self.system = SensorML(element).members[0] self.shortName = get_named_by_definition( self.system.get_identifiers_by_name("shortName"), "http://mmisw.org/ont/ioos/definition/shortName" ) self.longName = get_named_by_definition( self.system.get_identifiers_by_name("longName"), "http://mmisw.org/ont/ioos/definition/longName" ) self.keywords = map(unicode, self.system.keywords) # Location try: self.location = self.system.location[0] except (TypeError, IndexError): # No location exists self.location = None # Timerange try: timerange = testXMLValue( self.system.get_capabilities_by_name("observationTimeRange")[0].find( ".//" + nsp("swe101:TimeRange/swe101:value") ) ).split(" ") self.starting = parser.parse(timerange[0]) self.ending = parser.parse(timerange[1]) except (AttributeError, TypeError, ValueError): self.starting = None self.ending = None
def metadata(self, output_format=None, feature_name_callback=None, **kwargs): """ Gets SensorML objects for all procedures in your filtered features. You should override the default output_format for servers that do not respond properly. """ callback = feature_name_callback or str if output_format is None: output_format = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"' responses = [] if self.features is not None: for feature in self.features: ds_kwargs = kwargs.copy() ds_kwargs.update({ 'outputFormat': output_format, 'procedure': callback(feature) }) responses.append( SensorML(self.server.describe_sensor(**ds_kwargs))) return responses
def metadata_plus_exceptions(self, output_format=None, feature_name_callback=None, **kwargs): """ Gets SensorML objects for all procedures in your filtered features. Return two dictionaries for service responses keyed by 'feature': responses: values are SOS DescribeSensor response text response_failures: values are exception text content furnished from ServiceException, ExceptionReport You should override the default output_format for servers that do not respond properly. """ callback = feature_name_callback or str if output_format is None: output_format = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"' responses = {} response_failures = {} if self.features is not None: for feature in self.features: ds_kwargs = kwargs.copy() ds_kwargs.update({ 'outputFormat': output_format, 'procedure': callback(feature) }) try: responses[feature] = (SensorML( self.server.describe_sensor(**ds_kwargs))) except (ServiceException, ExceptionReport) as e: response_failures[feature] = str(e) return (responses, response_failures)
def load_dataset(self, ds_str): """ Helper method to load a dataset or SOS GC/DS url. """ ds = None # try to figure out if this is a local NetCDF Dataset, a remote one, or an SOS GC/DS url doc = None pr = urlparse(ds_str) if pr.netloc: # looks like a remote url rhead = requests.head(ds_str) # if we get a 400 here, it's likely a Dataset openable OpenDAP url if rhead.status_code == 400: pass elif rhead.status_code == 200 and rhead.headers[ 'content-type'] == 'text/xml': # probably interesting, grab it r = requests.get(ds_str) r.raise_for_status() doc = r.text else: raise StandardError( "Could not understand response code %s and content-type %s" % (rhead.status_code, rhead.headers.get('content-type', 'none'))) else: # do a cheap imitation of libmagic # http://stackoverflow.com/a/7392391/84732 textchars = ''.join( map(chr, [7, 8, 9, 10, 12, 13, 27] + range(0x20, 0x100))) is_binary_string = lambda bytes: bool( bytes.translate(None, textchars)) with open(ds_str, 'rb') as f: first_chunk = f.read(1024) if is_binary_string(first_chunk): # likely netcdf file pass else: f.seek(0) doc = "".join(f.readlines()) if doc is not None: xml_doc = ET.fromstring(str(doc)) if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities": ds = SensorObservationService(ds_str, xml=str(doc)) elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML": ds = SensorML(xml_doc) else: raise StandardError("Unrecognized XML root element: %s" % xml_doc.tag) else: # no doc? try the dataset constructor ds = Dataset(ds_str) return ds
def __init__(self, element, nsmap=extension_sos_utils.nsmap): self._root = element element = element.find(f".//{nspath_eval('sml:SensorML', nsmap)}") self.sensor_ml = SensorML(element) if hasattr(self.sensor_ml, 'systems') and self.sensor_ml.systems: self.sensor = self.sensor_ml.systems[0] self.id = self.sensor.identifiers['uniqueID'].value self.name = self.sensor.identifiers['longName'].value
class DescribeSensor(IoosDescribeSensor): @classmethod def get_named_by_definition(cls, element_list, string_def): """Attempts to get an IOOS definition from a list of xml elements""" try: return next((st.value for st in element_list if st.definition == string_def)) except: return None def get_ioos_def(self, ident, elem_type, ont): """Gets a definition given an identifier and where to search for it""" if elem_type == 'identifier': getter_fn = self.system.get_identifiers_by_name elif elem_type == 'classifier': getter_fn = self.system.get_classifiers_by_name else: raise ValueError("Unknown element type '{}'".format(elem_type)) return DescribeSensor.get_named_by_definition(getter_fn(ident), urljoin(ont, ident)) def __init__(self, element): """ Common things between all describe sensor requests """ if isinstance(element, ElementType): root = element else: root = etree.fromstring(element) sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='http://mmisw.org/ont/ioos/definition/%s']".format(SML_NS) # TODO: make this cleaner if hasattr(root, 'getroot'): root = root.getroot() self.system = SensorML(element).members[0] self.ioos_version = testXMLValue(root.find(".//{%s}field[@name='ioosTemplateVersion']/{%s}Text/{%s}value" % (SWE_NS, SWE_NS, SWE_NS))) if self.ioos_version != "1.0": warnings.warn("Warning: Unsupported IOOS version (%s). Supported: [1.0]" % self.ioos_version) self.shortName = self.get_ioos_def('shortName', 'identifier', ont) self.longName = self.get_ioos_def('longName', 'identifier', ont) self.keywords = list(map(str, self.system.keywords)) # Location try: self.location = self.system.location[0] except (TypeError, IndexError): # No location exists self.location = None # Timerange try: timerange = testXMLValue(self.system.get_capabilities_by_name("observationTimeRange")[0].find(".//" + nsp("swe101:TimeRange/swe101:value"))).split(" ") self.starting = parser.parse(timerange[0]) self.ending = parser.parse(timerange[1]) except (AttributeError, TypeError, ValueError, IndexError): self.starting = None self.ending = None
def __init__(self, element, nsmap=extension_sos_utils.nsmap): self._root = element # self.procedure_description_format = testXMLValue(self._root.find(nspath_eval('swes:procedureDescriptionFormat', nsmap))) # sensor_ml = SensorML(self._root.find(nspath_eval('sml:SensorML', nsmap))) self.sensor_ml = SensorML(element) if hasattr(self.sensor_ml, 'systems') and self.sensor_ml.systems: self.sensor = self.sensor_ml.systems[0] self.id = self.sensor.identifiers['uniqueID'].value self.name = self.sensor.identifiers['longName'].value
class DescribeSensor(IoosDescribeSensor): @classmethod def get_named_by_definition(cls, element_list, string_def): """Attempts to get an IOOS definition from a list of xml elements""" try: return next((st.value for st in element_list if st.definition == string_def)) except: return None def get_ioos_def(self, ident, elem_type, ont): """Gets a definition given an identifier and where to search for it""" if elem_type == 'identifier': getter_fn = self.system.get_identifiers_by_name elif elem_type == 'classifier': getter_fn = self.system.get_classifiers_by_name else: raise ValueError("Unknown element type '{}'".format(elem_type)) return DescribeSensor.get_named_by_definition(getter_fn(ident), urljoin(ont, ident)) def __init__(self, element): """ Common things between all describe sensor requests """ if isinstance(element, str): root = etree.fromstring(element) else: root = element sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='http://mmisw.org/ont/ioos/definition/%s']".format(SML_NS) # TODO: make this cleaner if hasattr(root, 'getroot'): root = root.getroot() self.system = SensorML(element).members[0] self.ioos_version = testXMLValue(root.find(".//{%s}field[@name='ioosTemplateVersion']/{%s}Text/{%s}value" % (SWE_NS, SWE_NS, SWE_NS))) if self.ioos_version != "1.0": warnings.warn("Warning: Unsupported IOOS version (%s). Supported: [1.0]" % self.ioos_version) self.shortName = self.get_ioos_def('shortName', 'identifier', ont) self.longName = self.get_ioos_def('longName', 'identifier', ont) self.keywords = map(unicode, self.system.keywords) # Location try: self.location = self.system.location[0] except (TypeError, IndexError): # No location exists self.location = None # Timerange try: timerange = testXMLValue(self.system.get_capabilities_by_name("observationTimeRange")[0].find(".//" + nsp("swe101:TimeRange/swe101:value"))).split(" ") self.starting = parser.parse(timerange[0]) self.ending = parser.parse(timerange[1]) except (AttributeError, TypeError, ValueError, IndexError): self.starting = None self.ending = None
def __init__(self, element): """ Common things between all describe sensor requests """ if isinstance(element, ElementType): root = element else: root = etree.fromstring(element) # sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='http://mmisw.org/ont/ioos/definition/%s']".format(SML_NS) # TODO: make this cleaner if hasattr(root, "getroot"): root = root.getroot() self.system = SensorML(element).members[0] self.ioos_version = testXMLValue( root.find( ".//{%s}field[@name='ioosTemplateVersion']/{%s}Text/{%s}value" % (SWE_NS, SWE_NS, SWE_NS))) if self.ioos_version != "1.0": warnings.warn( "Warning: Unsupported IOOS version (%s). Supported: [1.0]" % self.ioos_version) self.shortName = self.get_ioos_def("shortName", "identifier", ont) self.longName = self.get_ioos_def("longName", "identifier", ont) self.keywords = list(map(str, self.system.keywords)) # Location try: self.location = self.system.location[0] except (TypeError, IndexError): # No location exists self.location = None # Timerange try: timerange = testXMLValue( self.system.get_capabilities_by_name("observationTimeRange") [0].find(".//" + nsp("swe101:TimeRange/swe101:value"))).split(" ") self.starting = parser.parse(timerange[0]) self.ending = parser.parse(timerange[1]) except (AttributeError, TypeError, ValueError, IndexError): self.starting = None self.ending = None
def syncvesk(): sensors = {} service = SensorObservationService(SOSURL, version=SOSVERSION) get_obs = service.get_operation_by_name('GetObservation') describe_sensor = service.get_operation_by_name('DescribeSensor') sensor_ids = describe_sensor.parameters['procedure']['values'] for sensor_id in sensor_ids: response = service.describe_sensor( outputFormat='http://www.opengis.net/sensorML/1.0.1', procedure=sensor_id) tr = etree.fromstring(response) element = tr.find('.//' + nspath_eval('sml:SensorML', namespaces)) ds = SensorML(element) name = ds.members[0].name sensors[sensor_id] = name offerings = [off.id for off in service.offerings] _observed_properties = list( set(op for off in service.offerings for op in off.observed_properties)) observed_properties = SOS_PARAMS.keys() end_time = datetime.now() # - timedelta(hours=120*10) start_time = end_time - timedelta(hours=12) temporalFilter = "om:phenomenonTime,{}/{}".format(start_time.isoformat(), end_time.isoformat()) response_format = 'application/json' _namespaces = "xmlns(om,http://www.opengis.net/om/2.0)" _response = service.get_observation(offerings=offerings, responseFormat=response_format, observedProperties=observed_properties, namespaces=_namespaces, temporalFilter=temporalFilter) response = json.loads(_response) for obs in response['observations']: location = obs['featureOfInterest']['name']['value'] _parameter = obs['observableProperty'] parameter = SOS_PARAMS.get(_parameter) timestamp = obs['phenomenonTime'] network = 'CNR' value = float(obs['result']['value']) rjson = { 'location': location, 'parameter': parameter, "timestamp": timestamp, 'network': network, "value": value } _write([rjson]) pass pass
def process_doc(doc): """ Attempt to parse an xml string conforming to either an SOS or SensorML dataset and return the results """ xml_doc = ET.fromstring(str(doc)) if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities": ds = SensorObservationService(ds_str, xml=str(doc)) elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML": ds = SensorML(xml_doc) else: raise ValueError("Unrecognized XML root element: %s" % xml_doc.tag) return ds
def __init__(self, element): """ Common things between all describe sensor requests """ if isinstance(element, ElementType): root = element else: root = etree.fromstring(element) # sml_str = ".//{{{0}}}identifier/{{{0}}}Term[@definition='http://mmisw.org/ont/ioos/definition/%s']".format(SML_NS) # TODO: make this cleaner if hasattr(root, "getroot"): root = root.getroot() self.system = SensorML(element).members[0] self.ioos_version = testXMLValue( root.find( ".//{%s}field[@name='ioosTemplateVersion']/{%s}Text/{%s}value" % (SWE_NS, SWE_NS, SWE_NS) ) ) if self.ioos_version != "1.0": warnings.warn( "Warning: Unsupported IOOS version (%s). Supported: [1.0]" % self.ioos_version ) self.shortName = self.get_ioos_def("shortName", "identifier", ont) self.longName = self.get_ioos_def("longName", "identifier", ont) self.keywords = list(map(str, self.system.keywords)) # Location try: self.location = self.system.location[0] except (TypeError, IndexError): # No location exists self.location = None # Timerange try: timerange = testXMLValue( self.system.get_capabilities_by_name("observationTimeRange")[ 0 ].find(".//" + nsp("swe101:TimeRange/swe101:value")) ).split(" ") self.starting = parser.parse(timerange[0]) self.ending = parser.parse(timerange[1]) except (AttributeError, TypeError, ValueError, IndexError): self.starting = None self.ending = None
def process_doc(self, doc): """ Attempt to parse an xml string conforming to either an SOS or SensorML dataset and return the results """ xml_doc = ET.fromstring(doc) if xml_doc.tag == "{http://www.opengis.net/sos/1.0}Capabilities": ds = SensorObservationService(None, xml=doc) # SensorObservationService does not store the etree doc root, # so maybe use monkey patching here for now? ds._root = xml_doc elif xml_doc.tag == "{http://www.opengis.net/sensorML/1.0.1}SensorML": ds = SensorML(xml_doc) else: raise ValueError("Unrecognized XML root element: %s" % xml_doc.tag) return ds
def sensors(self): if self._sensors is None: sensors = {} get_obs = self.service.get_operation_by_name('GetObservation') describe_sensor = self.service.get_operation_by_name( 'DescribeSensor') sensor_ids = describe_sensor.parameters['procedure']['values'] for sensor_id in sensor_ids: response = self.service.describe_sensor( outputFormat='http://www.opengis.net/sensorML/1.0.1', procedure=sensor_id) tr = etree.fromstring(response) element = tr.find('.//' + nspath_eval('sml:SensorML', namespaces)) ds = SensorML(element) name = ds.members[0].name sensors[sensor_id] = name self._sensors = sensors return self._sensors
def get_coops_metadata(station): """ Get longName and sensorName for specific station from COOPS SOS using DescribeSensor and owslib.swe.sensor.sml.SensorML. Examples -------- >>> long_name, station_id = get_coops_metadata(8651370) >>> long_name 'Duck, NC' >>> station_id 'urn:ioos:station:NOAA.NOS.CO-OPS:8651370' """ from owslib.swe.sensor.sml import SensorML url = ('opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?' 'service=SOS&' 'request=DescribeSensor&version=1.0.0&' 'outputFormat=text/xml;' 'subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"&' 'procedure=urn:ioos:station:NOAA.NOS.CO-OPS:%s') % station url = parse_url(url) xml = etree.parse(urlopen(url)) root = SensorML(xml) if not root.members or len(root.members) > 1: msg = "Expected 1 member, got {}".format raise ValueError(msg(len(root.members))) system = root.members[0] # NOTE: Some metadata of interest. # system.description # short_name = _get_value(system.identifiers, name='shortName') # [c.values() for c in system.components] long_name = _get_value(system.identifiers, name='longName') long_name = long_name.split('station, ')[-1].strip() station_id = _get_value(system.identifiers, name='stationID') return long_name, station_id
def create_maps(parsed_obs, offering, layer, new, secondsGranularity, event_time, service): """ Add layers representing offerings and observed properties to the vector map :param parsed_obs: Observations for a given offering in geoJSON format :param offering: A collection of sensors used to conveniently group them up :param layer: Count of yet existing layers in vector map :param new: Given vector map which should be updated with new layers :param secondsGranularity: Granularity in seconds """ i = layer + 1 points = dict() freeCat = 1 if flags['s']: # Set target projection of current LOCATION target_crs = grass.read_command('g.proj', flags='fj').rstrip(os.linesep) target = osr.SpatialReference(target_crs) target.ImportFromProj4(target_crs) if target == 'XY location (unprojected)': grass.fatal("Sorry, XY locations are not supported!") # The following is work in progress cols = [(u'cat', 'INTEGER PRIMARY KEY'), (u'name', 'varchar'), (u'description', 'varchar'), (u'keywords', 'varchar'), (u'sensor_type', 'varchar'), (u'system_type', 'varchar'), (u'crs', 'INTEGER'), (u'x', 'DOUBLE'), (u'y', 'DOUBLE'), (u'z', 'DOUBLE')] # new = Vector(new) if new.is_open() is False: new.open('w', tab_name=options['output'], tab_cols=cols) offs = [o.id for o in service.offerings] off_idx = offs.index(offering) outputFormat = service.get_operation_by_name( 'DescribeSensor').parameters['outputFormat']['values'][0] procedures = service.offerings[off_idx].procedures for proc in procedures: response = service.describe_sensor(procedure=proc, outputFormat=outputFormat) root = SensorML(response) system = root.members[0] name = system.name desc = system.description keywords = ','.join(system.keywords) sensType = system.classifiers['Sensor Type'].value sysType = system.classifiers['System Type'].value crs = int(system.location[0].attrib['srsName'].split(':')[1]) coords = system.location[0][0].text.replace('\n', '') sx = float(coords.split(',')[0]) sy = float(coords.split(',')[1]) sz = float(coords.split(',')[2]) # Set source projection from SOS source = osr.SpatialReference() source.ImportFromEPSG(crs) transform = osr.CoordinateTransformation(source, target) point = ogr.CreateGeometryFromWkt('POINT ({} {} {})'.format( sx, sy, sz)) point.Transform(transform) x = point.GetX() y = point.GetY() z = point.GetZ() if name not in points.keys(): points.update({name: freeCat}) point = Point(x, y, z) new.write(point, cat=freeCat, attrs=( u'{}'.format(system.name.decode('utf-8')), system.description, ','.join(system.keywords), system.classifiers['Sensor Type'].value, system.classifiers['System Type'].value, crs, float(coords.split(',')[0]), float(coords.split(',')[1]), float(coords.split(',')[2]), )) freeCat += 1 new.table.conn.commit() new.close(build=True) else: timestampPattern = '%Y-%m-%dT%H:%M:%S' # TODO: Timezone startTime = event_time.split('+')[0] epochS = int(time.mktime(time.strptime(startTime, timestampPattern))) endTime = event_time.split('+')[1].split('/')[1] epochE = int(time.mktime(time.strptime(endTime, timestampPattern))) for key, observation in parsed_obs.iteritems(): tableName = '{}_{}_{}'.format(options['output'], offering, key) if ':' in tableName: tableName = '_'.join(tableName.split(':')) if '-' in tableName: tableName = '_'.join(tableName.split('-')) if '.' in tableName: tableName = '_'.join(tableName.split('.')) data = json.loads(observation) intervals = {} for secondsStamp in range(epochS, epochE + 1, secondsGranularity): intervals.update({secondsStamp: dict()}) timestampPattern = 't%Y%m%dT%H%M%S' # TODO: Timezone cols = [(u'cat', 'INTEGER PRIMARY KEY'), (u'name', 'VARCHAR')] for a in data['features']: name = a['properties']['name'] if name not in points.keys(): if new.is_open() is False: new.open('w') points.update({name: freeCat}) new.write(Point(*a['geometry']['coordinates']), cat=freeCat) freeCat += 1 for timestamp, value in a['properties'].iteritems(): if timestamp != 'name': observationStartTime = timestamp[:-4] secondsTimestamp = int( time.mktime( time.strptime(observationStartTime, timestampPattern))) for interval in intervals.keys(): if secondsTimestamp >= interval \ and secondsTimestamp < ( interval + secondsGranularity): if name in intervals[interval].keys(): intervals[interval][name].append( float(value)) else: timestamp2 = datetime.datetime.fromtimestamp( interval).strftime('t%Y%m%dT%H%M%S') intervals[interval].update( {name: [float(value)]}) if (u'%s' % timestamp2, 'DOUBLE') not in cols: cols.append( (u'%s' % timestamp2, 'DOUBLE')) break if len(cols) > 2000: grass.warning( 'Recommended number of columns is less than 2000, you have ' 'reached {}\nYou should set an event_time with a smaller range' ' or recompile SQLite limits as described at ' 'https://sqlite.org/limits.html'.format(len(cols))) link = Link( layer=i, name=tableName, table=tableName, key='cat', database='$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db', driver='sqlite') if new.is_open(): new.close() new.open('rw') new.dblinks.add(link) new.table = new.dblinks[i - 1].table() new.table.create(cols) inserts = dict() for interval in intervals.keys(): if len(intervals[interval]) != 0: timestamp = datetime.datetime.fromtimestamp( interval).strftime('t%Y%m%dT%H%M%S') for name, values in intervals[interval].iteritems(): if options['method'] == 'average': aggregatedValue = sum(values) / len(values) elif options['method'] == 'sum': aggregatedValue = sum(values) if name not in inserts.keys(): insert = [None] * len(cols) insert[0] = points[name] insert[1] = name insert[cols.index( (timestamp, 'DOUBLE'))] = aggregatedValue inserts.update({name: insert}) else: inserts[name][cols.index( (timestamp, 'DOUBLE'))] = aggregatedValue for insert in inserts.values(): new.table.insert(tuple(insert)) new.table.conn.commit() new.close(build=False) run_command('v.build', quiet=True, map=options['output']) i += 1
def maps_without_observations(offering, resolution, service, procedures, target): """Import just pixels/sensors without their observations. :param offering: A collection of sensors used to conveniently group them up :param resolution: 2D grid resolution for rasterization :param service: SensorObservationService() type object of request :param procedures: List of queried procedures (observation providors) :param target: """ offs = [o.id for o in service.offerings] off_idx = offs.index(offering) output_format = service.get_operation_by_name( 'DescribeSensor').parameters['outputFormat']['values'][0] if procedures: procedures = procedures.split(',') else: procedures = service.offerings[off_idx].procedures tempfile_path = grass.tempfile() n = None s = None e = None w = None with open(tempfile_path, 'w') as tempFile: for proc in procedures: response = service.describe_sensor(procedure=proc, output_format=output_format) root = SensorML(response) system = root.members[0] crs = int(system.location[0].attrib['srsName'].split(':')[-1]) coords = system.location[0][0].text.replace('\n', '') sx = float(coords.split(',')[0]) sy = float(coords.split(',')[1]) sz = float(coords.split(',')[2]) transform = soslib.get_transformation(crs, target) point = ogr.CreateGeometryFromWkt('POINT ({} {} {})'.format( sx, sy, sz)) point.Transform(transform) x = point.GetX() y = point.GetY() z = point.GetZ() tempFile.write('{} {} {}\n'.format(x, y, z)) if not n: n = y + resolution / 2 s = y - resolution / 2 e = x + resolution / 2 w = x - resolution / 2 else: if y >= n: n = y + resolution / 2 if y <= s: s = y - resolution / 2 if x >= e: e = x + resolution / 2 if x <= w: w = x - resolution / 2 run_command('g.region', n=n, s=s, w=w, e=e, res=resolution) run_command('r.in.xyz', input=tempfile_path, separator='space', output='{}_{}'.format(options['output'], offering))
def process_station(self, uid, offering): """ Makes a DescribeSensor request based on a 'uid' parameter being a station procedure. Also pass along an offering with getCapabilities information for items such as temporal extent""" GML_NS = "http://www.opengis.net/gml" XLINK_NS = "http://www.w3.org/1999/xlink" with app.app_context(): app.logger.info("process_station: %s", uid) desc_sens = self._describe_sensor(uid, timeout=1200) # FIXME: add some kind of notice saying the station failed if desc_sens is None: app.logger.warn( "Could not get a valid describeSensor response") return metadata_value = etree.fromstring(desc_sens) sensor_ml = SensorML(metadata_value) try: station_ds = IoosDescribeSensor(metadata_value) # if this doesn't conform to IOOS SensorML sub, fall back to # manually picking apart the SensorML except ows.ExceptionReport: station_ds = process_sensorml(sensor_ml.members[0]) unique_id = station_ds.id if unique_id is None: app.logger.warn( "Could not get a 'stationID' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/stationID'" ) return dataset = db.Dataset.find_one({'uid': unicode(unique_id)}) if dataset is None: dataset = db.Dataset() dataset.uid = unicode(unique_id) dataset['active'] = True # Find service reference in Dataset.services and remove (to replace it) tmp = dataset.services[:] for d in tmp: if d['service_id'] == self.service.get('_id'): dataset.services.remove(d) # Parsing messages messages = [] # NAME name = unicode_or_none(station_ds.shortName) if name is None: messages.append( u"Could not get a 'shortName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/shortName'" ) # DESCRIPTION description = unicode_or_none(station_ds.longName) if description is None: messages.append( u"Could not get a 'longName' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/longName'" ) # PLATFORM TYPE asset_type = unicode_or_none( getattr(station_ds, 'platformType', None)) if asset_type is None: messages.append( u"Could not get a 'platformType' from the SensorML identifiers. Looking for a definition of 'http://mmisw.org/ont/ioos/definition/platformType'" ) # LOCATION is in GML gj = None loc = station_ds.location if loc is not None and loc.tag == "{%s}Point" % GML_NS: pos_element = loc.find("{%s}pos" % GML_NS) # some older responses may uses the deprecated coordinates # element if pos_element is None: # if pos not found use deprecated coordinates element pos_element = loc.find("{%s}coordinates" % GML_NS) # strip out points positions = map(float, pos_element.text.split(" ")) for el in [pos_element, loc]: srs_name = testXMLAttribute(el, "srsName") if srs_name: crs = Crs(srs_name) if crs.axisorder == "yx": gj = json.loads( geojson.dumps( geojson.Point([positions[1], positions[0]]))) else: gj = json.loads( geojson.dumps( geojson.Point([positions[0], positions[1]]))) break else: if positions: messages.append( u"Position(s) found but could not parse SRS: %s, %s" % (positions, srs_name)) else: messages.append( u"Found an unrecognized child of the sml:location element and did not attempt to process it: %s" % loc) meta_str = unicode(etree.tostring(metadata_value)).strip() if len(meta_str) > 4000000: messages.append( u'Metadata document was too large to store (len: %s)' % len(meta_str)) meta_str = u'' service = { # Reset service 'name': name, 'description': description, 'service_type': self.service.get('service_type'), 'service_id': ObjectId(self.service.get('_id')), 'data_provider': self.service.get('data_provider'), 'metadata_type': u'sensorml', 'metadata_value': u'', 'time_min': getattr(offering, 'begin_position', None), 'time_max': getattr(offering, 'end_position', None), 'messages': map(unicode, messages), 'keywords': map(unicode, sorted(station_ds.keywords)), 'variables': map(unicode, sorted(station_ds.variables)), 'asset_type': get_common_name(asset_type), 'geojson': gj, 'updated': datetime.utcnow() } dataset.services.append(service) dataset.updated = datetime.utcnow() dataset.save() # do compliance checker / metadata now scores = self.ccheck_station(sensor_ml) metamap = self.metamap_station(sensor_ml) try: self.save_ccheck_station('ioos', dataset._id, scores, metamap) except Exception as e: app.logger.warn( "could not save compliancecheck/metamap information: %s", e) return "Harvest Successful"
def maps_without_observations(offering, new, service, procedures, target): """Import just vector points/sensors without their observations. :param offering: A collection of sensors used to conveniently group them up :param new: Given vector map which should be updated with new layers :param service: SensorObservationService() type object of request :param procedures: List of queried procedures (observation providors) :param target: """ points = dict() free_cat = 1 cols = [(u'cat', 'INTEGER PRIMARY KEY'), (u'name', 'varchar'), (u'description', 'varchar'), (u'keywords', 'varchar'), (u'sensor_type', 'varchar'), (u'system_type', 'varchar'), (u'crs', 'INTEGER'), (u'x', 'DOUBLE'), (u'y', 'DOUBLE'), (u'z', 'DOUBLE')] # new = Vector(new) if new.is_open() is False: new.open('w', tab_name=options['output'], tab_cols=cols) offs = [o.id for o in service.offerings] off_idx = offs.index(offering) output_format = service.get_operation_by_name( 'DescribeSensor').parameters['outputFormat']['values'][0] if procedures: procedures = procedures.split(',') else: procedures = service.offerings[off_idx].procedures for proc in procedures: response = service.describe_sensor(procedure=proc, outputFormat=output_format) root = SensorML(response) system = root.members[0] name = system.name desc = system.description keywords = ','.join(system.keywords) sens_type = system.classifiers['Sensor Type'].value sys_type = system.classifiers['System Type'].value crs = int(system.location[0].attrib['srsName'].split(':')[-1]) coords = system.location[0][0].text.replace('\n', '') sx = float(coords.split(',')[0]) sy = float(coords.split(',')[1]) sz = float(coords.split(',')[2]) # Set source projection from SOS transform = soslib.get_transformation(crs, target) point = ogr.CreateGeometryFromWkt('POINT ({} {} {})'.format( sx, sy, sz)) point.Transform(transform) x = point.GetX() y = point.GetY() z = point.GetZ() if name not in points.keys(): points.update({name: free_cat}) point = Point(x, y, z) new.write(point, cat=free_cat, attrs=( u'{}'.format(system.name), system.description, ','.join(system.keywords), system.classifiers['Sensor Type'].value, system.classifiers['System Type'].value, crs, float(coords.split(',')[0]), float(coords.split(',')[1]), float(coords.split(',')[2]), )) free_cat += 1 new.table.conn.commit() new.close(build=True)
def get_stations_df(self, sos_url, station_urns_sel=None): """ Returns a Pandas Dataframe """ # oFrmts: IOOS SOS OutputFormat strings (first is compliant to the IOOS SOS spec, # second is to accommodate NDBC). More info here: # http://ioos.github.io/sos-guidelines/doc/wsdd/sos_wsdd_github_notoc/#describesensor-request:638e0b263020c13a76a55332bd966dbe oFrmts = [ 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"', 'text/xml;subtype="sensorML/1.0.1"' ] params = { 'service': 'SOS', 'request': 'GetCapabilities', 'acceptVersions': '1.0.0' } sos_url_params = sos_url + '?' + urlencode(params) # sos_url_params_quoted = quote(sos_url_params,"/=:") # sos_url_params_unquoted = unquote(sos_url_params) try: sosgc = SensorObservationService(sos_url_params) except (ConnectionError, ReadTimeout) as e: self.log.write( u"\nError: unable to connect to SOS service: {url} due to HTTP connection error." .format(url=sos_url_params)) self.log.write( u"\nHTTP connection error: {err}.".format(err=str(e))) sys.exit( "\nError: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}" .format(url=sos_url_params, err=str(e))) # vars to store returns from sos_collector.metadata_plus_exceptions function: sml_recs = {} sml_errors = {} describe_sensor_url = {} # leverage Pyoos Collector to query for all available stations and obtain SensorML # (if station subset not passed in --stations param) if station_urns_sel is not None: station_urns = station_urns_sel else: sos_collector = IoosSweSos(sos_url) station_urns = [ urn.name for urn in sos_collector.server.offerings if 'network' not in urn.name.split(':') ] sos_collector.features = station_urns # write out stations in SOS that will be handled: if self.verbose: self.log.write(u"\nStations to process for SOS: {sos}".format( sos=sos_url_params)) print("Stations to process for SOS: {sos}".format( sos=sos_url_params)) for feature in sos_collector.features: self.log.write(u"\n - {feature}".format(feature=feature)) print(" - {feature}".format(feature=feature)) # iterate over possible oFrmts expected of the various SOS services (IOOS SOS 1.0, NDBC): # for fmt in reversed(oFrmts): for fmt in oFrmts: try: sml_recs, sml_errors = sos_collector.metadata_plus_exceptions( output_format=fmt, timeout=200) # if no valid SensorML docs returned, try next oFrmt: if not sml_recs: continue else: # assign correct DescribeSensor url (use sos_collector.feature rather than sml_recs.keys() to # create DescribeSensor URLs for failures to record in logs): for station in sos_collector.features: describe_sensor_url[ station] = self.generate_describe_sensor_url( sosgc, procedure=station, oFrmt=fmt) # report on errors returned from metadata_plus_exceptions: if sml_errors: if self.verbose: for station, msg in iteritems(sml_errors): self.log.write( u"\nSOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}" .format(station=station, msg=msg)) print( "SOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}" .format(station=station, msg=msg)) else: self.log.write( u"\nSuccess, no errors returned from DescribeSensor requests in service: {sos}" .format(sos=sos_url_params)) print( "Success, no errors returned from DescribeSensor requests in service: {sos}" .format(sos=sos_url_params)) break # ServiceException shouldn't be thrown by metadata_plus_exceptions function, but handle regardless by attempting next oFrmt: except ServiceException as e: continue station_recs = [] failures = [] # generate Pandas DataFrame by populating 'station_recs' list by parsing SensorML strings: for station_idx, station_urn in enumerate(station_urns): if station_urns_sel is not None: # iterate oFrmts items for describe_sensor request (first is IOOS SOS spec-compliant, second is for NDBC SOS): for fmt in oFrmts: try: describe_sensor_url[ station_urn] = self.generate_describe_sensor_url( sosgc, procedure=station_urn, oFrmt=fmt) sml_str = sosgc.describe_sensor(procedure=station_urn, outputFormat=fmt, timeout=200) break except ServiceException as e: sml_errors[station_urn] = str(e) continue sml = SensorML(sml_str) else: # process valid SensorML responses, quietly pass on invalid stations (add to failures list for verbose reporting): try: sml = sml_recs[station_urn] except KeyError: self.log.write( u"\n\nStation: {station} failed (no SensorML in sml_recs dict). URL: {ds}" .format(station=station_urn, ds=describe_sensor_url[station_urn].replace( "&", "&"))) print( "Station: {station} failed (no SensorML in sml_recs dict). URL: {ds}" .format(station=station_urn, ds=describe_sensor_url[station_urn].replace( "&", "&"))) failures.append(station_urn) continue if self.sos_type.lower() == 'ndbc': # later: add an error check sosgc_station_offering = sosgc.contents[ 'station-' + station_urn.split(':')[-1]] else: sosgc_station_offering = None try: ds = IoosDescribeSensor(sml._root) except AttributeError: self.log.write( u"\nInvalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: " .format(station=station, ds=describe_sensor_url[station_urn].replace( "&", "&"))) print( "Invalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: " .format(station=station, ds=describe_sensor_url[station_urn].replace( "&", "&"))) station = OrderedDict() # debug: if self.verbose: self.log.write(u"\n\nProcessing station: {station}".format( station=station_urn)) print("Processing station: {station}".format( station=station_urn)) self.log.write("\n" + etree.tostring(sml._root).decode('utf-8')) # assign 'pos' to GML point location (accommodate 'gml:coordinates' as used by NDBC if gml:Point not found): try: pos = testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) \ if testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) is not None \ else testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:coordinates'))) station['lon'] = float(pos.split()[1]) station['lat'] = float(pos.split()[0]) except AttributeError as e: station['lon'] = None station['lat'] = None system_el = sml._root.findall(self.nsp('sml:member'))[0].find( self.nsp('sml:System')) # Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication # errors but there is not enough information in SensorML for alternatives) # Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage" documents = system_el.findall( self.nsp('sml:documentation/sml:DocumentList/sml:member')) documents_dct = {} for d in documents: document = Documentation(d) name = testXMLAttribute(d, "name") # url = document.documents[0].url documents_dct[name] = document # obtain list of contacts (accommodate 'sml:contact' element repetition used by NDBC insead of ContactList): contacts = system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \ if system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \ else system_el.findall(self.nsp('sml:contact')) contacts_dct = {} for c in contacts: contact = Contact(c) role = contact.role.split('/')[-1] contacts_dct[role] = contact # verify a 'publisher' Contact exists (template expects one): if "publisher" not in contacts_dct.keys(): self.log.write( u"\n\nStation: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]" .format(station=station_urn, roles=", ".join(contacts_dct.keys()))) print( "Station: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]" .format(station=station_urn, roles=", ".join(contacts_dct.keys()))) failures.append(station_urn) continue sweQuants = system_el.findall( self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity')) quant_lst = [ sweQuant.attrib['definition'] for sweQuant in sweQuants ] parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst] # attempt to read beginPosition, if available, otherwise use current date # bc ISO requires date value in output location in template: beginPosition = testXMLValue( system_el.find( self.nsp( 'sml:validTime/gml:TimePeriod/gml:beginPosition'))) try: begin_service_date = parser.parse(beginPosition) except (AttributeError, TypeError) as e: begin_service_date = datetime.now(pytz.utc) station['station_urn'] = station_urn station['sos_url'] = sos_url_params station['describesensor_url'] = describe_sensor_url[station_urn] station['shortName'] = ds.shortName station['longName'] = ds.longName if self.sos_type.lower() == 'ndbc': station['wmoID'] = station_urn.split(':')[-1] else: station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont) station['serverName'] = self.server_name # Some capabilities-level metadata: station['title'] = sosgc.identification.title station['abstract'] = sosgc.identification.abstract station['keywords'] = sosgc.identification.keywords station['begin_service_date'] = begin_service_date # Beware that a station can have >1 classifier of the same type # This code does not accommodate that possibility station['platformType'] = ds.platformType station['parentNetwork'] = ds.get_ioos_def('parentNetwork', 'classifier', ont) station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont) # store some nested dictionaries in 'station' for appropriate SensorML sources: station['contacts_dct'] = contacts_dct station['documents_dct'] = documents_dct if self.sos_type.lower( ) == 'ndbc' and sosgc_station_offering is not None: station['starting'] = sosgc_station_offering.begin_position station['ending'] = sosgc_station_offering.end_position else: station['starting'] = ds.starting station['ending'] = ds.ending if self.sos_type.lower( ) == 'ndbc' and sosgc_station_offering is not None: station[ 'variable_uris'] = sosgc_station_offering.observed_properties station['variables'] = [ var.split('/')[-1] for var in sosgc_station_offering.observed_properties ] station['parameter_uris'] = ','.join(station['variable_uris']) station['parameters'] = ','.join(station['variables']) else: station['variable_uris'] = ds.variables station['variables'] = [ var.split('/')[-1] for var in ds.variables ] station['parameter_uris'] = ','.join(quant_lst) station['parameters'] = ','.join(parameter_lst) if self.verbose: for var in station['variable_uris']: self.log.write(u"\nvariable: {var}".format(var=var)) print("variable: {var}".format(var=var)) # print(sosgc.contents) # for id, offering in sosgc.contents.iteritems(): # print("sosgc.contents: {item}".format(item=id)) # parse 'responseFormat' vals from SensorML: # response_formats = sosgc.contents[station_urn].response_formats response_formats = [] for id, sosgc.content in sosgc.contents.items(): if sosgc.content.name == station_urn: response_formats = sosgc.content.response_formats # response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() # if sosgc.content.name == station_urn ] # match responseFormats from SensorML (response_formats) against those passed in --response_formats parameter to # populate 'download_formats' list, that is then used to generate GetObservation requests for the template: # (default --response_formats values are: 'application/json,application/zip; subtype=x-netcdf' ) download_formats = [ response_format for response_format in response_formats if response_format in self.response_formats ] station['response_formats'] = response_formats station['download_formats'] = download_formats if self.verbose: for format in response_formats: self.log.write( u"\nresponseFormat: {format}".format(format=format)) print("responseFormat: {format}".format(format=format)) for format in download_formats: self.log.write( u"\ndownloadFormats: {format}".format(format=format)) print("downloadFormats: {format}".format(format=format)) # calculate event_time using self.getobs_req_hours: event_time_formatstr = "{begin:%Y-%m-%dT%H:%M:%S}{utc_code}/{end:%Y-%m-%dT%H:%M:%S}{utc_code}" utc_code = 'Z' if self.sos_type.lower() == 'ndbc' else '' if station['starting'] is not None and station[ 'ending'] is not None: event_time = event_time_formatstr.format( begin=station['ending'] - timedelta(hours=self.getobs_req_hours), end=station['ending'], utc_code=utc_code) if self.verbose: self.log.write( u"\nUsing starting/ending times from SensorML for eventTime" ) print( "Using starting/ending times from SensorML for eventTime" ) self.log.write( u"\nobservationTimeRange: starting: {start}, ending: {end}" .format(start=station['starting'], end=station['ending'])) print( "observationTimeRange: starting: {start}, ending: {end}" .format(start=station['starting'], end=station['ending'])) else: now = datetime.now(pytz.utc) then = now - timedelta(hours=self.getobs_req_hours) event_time = event_time_formatstr.format(begin=then, end=now, utc_code=utc_code) if self.verbose: self.log.write( u"\nNo 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) print( "No 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) if self.verbose: self.log.write(u"\neventTime: {time}".format(time=event_time)) print("eventTime: {time}".format(time=event_time)) # create a dict to store parameters for valid example GetObservation requests for station: getobs_req_dct = {} # populate a parameters dictionary for download links for each 'observedProperty' type # and secondly for each 'responseFormat' per observedProperty: getobs_params_base = { 'service': 'SOS', 'request': 'GetObservation', 'version': '1.0.0', 'offering': station_urn, 'eventTime': event_time } for variable in station['variable_uris']: getobs_params = getobs_params_base.copy() getobs_params['observedProperty'] = variable variable = variable.split('/')[-1] for format in download_formats: getobs_params['responseFormat'] = format getobs_request_url_encoded = sos_url + '?' + urlencode( getobs_params) getobs_request_url = unquote(getobs_request_url_encoded) getobs_req_dct[variable + '-' + format] = { 'variable': variable, 'url': getobs_request_url, 'format_type': self.RESPONSE_FORMAT_TYPE_MAP.get(format, format), 'format_name': self.RESPONSE_FORMAT_NAME_MAP.get(format, format) } if self.verbose: self.log.write( u"\ngetobs_request_url (var: {variable}): {getobs_request_url}" .format(variable=variable.split("/")[-1], getobs_request_url=getobs_request_url)) print( "getobs_request_url (var: {variable}): {getobs_request_url}" .format(variable=variable.split("/")[-1], getobs_request_url=getobs_request_url)) # ToDo: finish adding the 'getobs_req_dct' to the output template station['getobs_req_dct'] = getobs_req_dct station_recs.append(station) # extra debug for failed stations in verbose mode: if self.verbose: self.log.write( u"\n\n\nSOS DescribeSensor request errors recap. Failed requests:" ) print("SOS DescribeSensor request errors recap. Failed requests:") for station_fail, msg in iteritems(sml_errors): self.log.write( u"\n{station} - {msg}. DescribeSensor URL: {ds}".format( station=station_fail, msg=msg, ds=describe_sensor_url[station_fail].replace( "&", "&"))) print("{station} - {msg}. DescribeSensor URL: {ds}".format( station=station_fail, msg=msg, ds=describe_sensor_url[station_fail].replace("&", "&"))) if failures: self.log.write( u"\nStations in 'failures' list (should match DescribeSensor errors):" ) print( "Stations in 'failures' list (should match DescribeSensor errors):" ) for station_fail in failures: self.log.write(u"\n{station}".format(station=station_fail)) print("{station}".format(station=station_fail)) if station_recs: stations_df = pd.DataFrame.from_records(station_recs, columns=station.keys()) stations_df.index = stations_df['station_urn'] return stations_df else: return None
def get_stations_df(self, sos_url, station_urns_sel=None): """ Returns a GeoDataFrame """ # LATER: ADD ERROR TEST/CATCH AFTER EACH WEB REQUEST oFrmt = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"' params = { 'service': 'SOS', 'request': 'GetCapabilities', 'acceptVersions': '1.0.0' } sos_url_params = sos_url + '?' + urlencode(params) sos_url_params_esc = sos_url_params.replace("&", "&") # sos_url_params_quoted = quote(sos_url_params,"/=:") # sos_url_params_unquoted = unquote(sos_url_params) try: sosgc = SensorObservationService(sos_url_params) except (ConnectionError, ReadTimeout) as e: self.log.write( u"Error: unable to connect to SOS service: {url} due to HTTP connection error.\n" .format(url=sos_url_params)) self.log.write( u"HTTP connection error: {err}.\n".format(err=e.message)) sys.exit( "Error: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}" .format(url=sos_url_params, err=e.message)) if station_urns_sel is not None: station_urns = station_urns_sel else: sos_collector = IoosSweSos(sos_url) station_urns = [ urn.name for urn in sos_collector.server.offerings if 'network' not in urn.name.split(':') ] sos_collector.features = station_urns sml_lst = sos_collector.metadata(timeout=200) station_recs = [] for station_idx, station_urn in enumerate(station_urns): if station_urns_sel is not None: sml_str = sosgc.describe_sensor(procedure=station_urn, outputFormat=oFrmt, timeout=200) sml = SensorML(sml_str) else: sml = sml_lst[station_idx] # debug: # if self.verbose: # self.log.write(unicode(etree.tostring(sml._root))) ds = IoosDescribeSensor(sml._root) pos = testXMLValue( ds.system.location.find(self.nsp('gml:Point/gml:pos'))) system_el = sml._root.findall(self.nsp('sml:member'))[0].find( self.nsp('sml:System')) # Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication # errors but there is not enough information in SensorML for alternatives) # Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage" documents = system_el.findall( self.nsp('sml:documentation/sml:DocumentList/sml:member')) documents_dct = {} for d in documents: document = Documentation(d) name = testXMLAttribute(d, "name") # url = document.documents[0].url documents_dct[name] = document contacts = system_el.findall( self.nsp('sml:contact/sml:ContactList/sml:member')) contacts_dct = {} for c in contacts: contact = Contact(c) role = contact.role.split('/')[-1] contacts_dct[role] = contact sweQuants = system_el.findall( self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity')) quant_lst = [ sweQuant.attrib['definition'] for sweQuant in sweQuants ] parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst] # attempt to read beginPosition, if available: beginPosition = testXMLValue( system_el.find( self.nsp( 'sml:validTime/gml:TimePeriod/gml:beginPosition'))) try: begin_service_date = parser.parse(beginPosition) except AttributeError as e: begin_service_date = None station = OrderedDict() station['station_urn'] = station_urn station['sos_url'] = sos_url_params_esc station['lon'] = float(pos.split()[1]) station['lat'] = float(pos.split()[0]) station['shortName'] = ds.shortName station['longName'] = ds.longName station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont) station['serverName'] = self.server_name # Some capabilities-level metadata: station['title'] = sosgc.identification.title station['abstract'] = sosgc.identification.abstract station['keywords'] = sosgc.identification.keywords station['begin_service_date'] = begin_service_date # Beware that a station can have >1 classifier of the same type # This code does not accommodate that possibility station['platformType'] = ds.platformType station['parentNetwork'] = ds.get_ioos_def('parentNetwork', 'classifier', ont) station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont) # store some nested dictionaries in 'station' for appopriate SensorML sources: station['contacts_dct'] = contacts_dct station['documents_dct'] = documents_dct # MW: the 'operator_' and 'publisher_' attributes can be removed bc they are not used # in the template code currently in favor of 'contacts_dct' # station['operatorSector'] = ds.get_ioos_def('operatorSector', 'classifier', ont) # station['operator_org'] = contacts_dct['operator'].organization # station['operator_country'] = contacts_dct['operator'].country # station['operator_url'] = contacts_dct['operator'].url # station['operator_email'] = contacts_dct['operator'].email # station['publisher'] = ds.get_ioos_def('publisher', 'classifier', ont) # station['publisher_org'] = contacts_dct['publisher'].organization # station['publisher_url'] = contacts_dct['publisher'].url # station_dct['publisher_email'] = contacts_dct['publisher'].electronicMailAddress station['starting'] = ds.starting station['ending'] = ds.ending # station['starting_isostr'] = datetime.isoformat(ds.starting) # station['ending_isostr'] = datetime.isoformat(ds.ending) station['parameter_uris'] = ','.join(quant_lst) station['parameters'] = ','.join(parameter_lst) station['variables'] = [var.split('/')[-1] for var in ds.variables] # debug: if self.verbose: self.log.write(u"\nProcessing station: {station}\n".format( station=station_urn)) print("\nProcessing station: {station}".format( station=station_urn)) for var in ds.variables: self.log.write(u"variable: {var}\n".format(var=var)) print("variable: {var}".format(var=var)) # print(sosgc.contents) # for id, offering in sosgc.contents.iteritems(): # print("sosgc.contents: {item}".format(item=id)) # parse 'responseFormat' values and populate list: # response_formats = sosgc.contents[station_urn].response_formats response_formats = [] for id, sosgc.content in sosgc.contents.items(): if sosgc.content.name == station_urn: response_formats = sosgc.content.response_formats # response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() if sosgc.content.name == station_urn ] # subset responseFormats (response_formats) for download links matching those passed in --response_formats parameter # (or 'application/json,application/zip; subtype=x-netcdf' by default): download_formats = [ response_format for response_format in response_formats if response_format in self.response_formats ] station['response_formats'] = response_formats station['download_formats'] = download_formats if self.verbose: for format in response_formats: self.log.write( u"responseFormat: {format}\n".format(format=format)) print("responseFormat: {format}".format(format=format)) for format in download_formats: self.log.write( u"downloadFormats: {format}\n".format(format=format)) print("downloadFormats: {format}".format(format=format)) # calculate event_time using self.getobs_req_hours: if ds.starting is not None and ds.ending is not None: event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format( begin=ds.ending - timedelta(hours=self.getobs_req_hours), end=ds.ending) else: now = datetime.now(pytz.utc) then = now - timedelta(hours=self.getobs_req_hours) event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format( begin=then, end=now) if self.verbose: self.log.write( u"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}\n" .format(then=then, now=now)) print( "then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) if self.verbose: self.log.write(u"eventTime: {time}\n".format(time=event_time)) print("eventTime: {time}".format(time=event_time)) # create a dict to store parameters for valid example GetObservation requests for station: getobs_req_dct = {} # populate a parameters dictionary for download links for each 'observedProperty' type and secondly for each 'responseFormat' per observedProperty: getobs_params_base = { 'service': 'SOS', 'request': 'GetObservation', 'version': '1.0.0', 'offering': station_urn, 'eventTime': event_time } for variable in ds.variables: getobs_params = getobs_params_base.copy() getobs_params['observedProperty'] = variable variable = variable.split('/')[-1] for format in download_formats: getobs_params['responseFormat'] = format getobs_request_url_encoded = sos_url + '?' + urlencode( getobs_params) getobs_request_url = unquote(getobs_request_url_encoded) getobs_request_url_esc = getobs_request_url.replace( "&", "&") getobs_req_dct[variable + '-' + format] = { 'variable': variable, 'url': getobs_request_url_esc, 'format_type': self.RESPONSE_FORMAT_TYPE_MAP[format], 'format_name': self.RESPONSE_FORMAT_NAME_MAP[format] } if self.verbose: self.log.write( u"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}\n" .format( variable=variable.split("/")[-1], getobs_request_url=getobs_request_url, getobs_request_url_esc=getobs_request_url_esc)) print( "getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}" .format( variable=variable.split("/")[-1], getobs_request_url=getobs_request_url, getobs_request_url_esc=getobs_request_url_esc)) # ToDo: finish adding the 'getobs_req_dct' to the output template station['getobs_req_dct'] = getobs_req_dct station_recs.append(station) stations_df = pd.DataFrame.from_records(station_recs, columns=station.keys()) stations_df.index = stations_df['station_urn'] return stations_df