class IoosSweSosTest(unittest.TestCase): def setUp(self): # we have to pick SOS to test against, NANOOS it is: self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp") # self.c.features = [urn.name for urn in self.c.server.offerings] @pytest.mark.xfail def test_server_id(self): assert ( self.c.server.identification.title == "NANOOS Sensor Observation Service (SOS), a 52North IOOS SOS server" ) assert self.c.server.identification.service == "OGC:SOS" assert self.c.server.identification.version == "1.0.0" assert self.c.server.identification.fees == "NONE" assert self.c.server.identification.accessconstraints == "NONE" @pytest.mark.xfail def test_metdata(self): self.c.features = ["urn:ioos:station:nanoos:apl_nemo"] response = self.c.metadata( output_format='text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"' ) assert isinstance(response, list) assert isinstance(response[0], SensorML) @pytest.mark.xfail def test_metadata_plus_exceptions(self): self.c.features = ["urn:ioos:station:nanoos:apl_nemo"] response, failures = self.c.metadata_plus_exceptions( output_format='text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"' ) assert isinstance(response, dict) assert isinstance(failures, dict) assert isinstance(response[self.c.features[0]], SensorML)
class IoosSweSosTest(unittest.TestCase): def setUp(self): # we have to pick SOS to test against, NANOOS it is: self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp") # self.c.features = [urn.name for urn in self.c.server.offerings] @pytest.mark.xfail def test_server_id(self): assert ( self.c.server.identification.title == "NANOOS Sensor Observation Service (SOS), a 52North IOOS SOS server" ) assert self.c.server.identification.service == "OGC:SOS" assert self.c.server.identification.version == "1.0.0" assert self.c.server.identification.fees == "NONE" assert self.c.server.identification.accessconstraints == "NONE" @pytest.mark.xfail def test_metdata(self): self.c.features = ["urn:ioos:station:nanoos:apl_nemo"] response = self.c.metadata( output_format= 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"') assert isinstance(response, list) assert isinstance(response[0], SensorML) @pytest.mark.xfail def test_metadata_plus_exceptions(self): self.c.features = ["urn:ioos:station:nanoos:apl_nemo"] response, failures = self.c.metadata_plus_exceptions( output_format= 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"') assert isinstance(response, dict) assert isinstance(failures, dict) assert isinstance(response[self.c.features[0]], SensorML)
def setUp(self): # we have to pick SOS to test against, NANOOS it is: self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp")
else: v_frame = pd.concat([v_frame, new_frame]) v_frame = v_frame.drop_duplicates() except ExceptionReport as e: print " [-] Error obtaining {!s} from {!s} - Message from server: {!s}".format(v, sos, e) continue finally: new_start = new_end if v_frame is not None: sos_dfs.append(v_frame) else: # Use the generic IOOS SWE collector try: collector = IoosSweSos(sos) except BaseException as e: print "[-] Could not process {!s}. Reason: {!s}".format(sos, e) continue for v in variables_to_query: collector.filter(variables=[v]) collector.filter(bbox=bounding_box) collector.filter(start=start_date) collector.filter(end=end_date) collector.filter(variables=[v]) try: data = collector.collect() print data except BaseException as e: print "[-] Could not process {!s}. Reason: {!s}".format(sos, e)
def main(): #The observations we're interested in. THis is another gotcha, is it water_temperature or sea_water_temperature... obsList = ['water_temperature','salinity'] """ Create a data collection object. Contructor parameters are: url - THe SWE endpoint we're interested in version - Optional default is '1.0.0' The SWE version the endpoint. xml - Optional default is None - The XML response from a GetCapabilities query of the server. """ dataCollector = IoosSweSos(url='http://129.252.139.124/thredds/sos/carocoops.cap2.buoy.nc', xml=None) #Loop through the offerings from the server. #Offerings are the stations/platforms/data device that the server house data from. offerings = dataCollector.server.offerings for offer in offerings: #We skip over the 'all' offering. #all "*network*" offerings represent aggregations of stations; # unlike all other offerings we've dealt with basically correspond to one station. if(offer.name.split(':')[-1] != 'all'): print "Offering: %s %s Obs: %s"\ % (offer.name, offer.description, offer.observed_properties) #Check to see if the station offered has the observation we're interested in. #Loop through the observed_properties attribute and compare. obsFilterList = [] #We want to query just the observations n the obsOfInterest list. #The observer_properties should be coded with an MMI href, however due #to ncSOS bug: https://github.com/asascience-open/ncSOS/issues/107 it adds #a URN pattern. Regardless, an end user should just need to know the observation #name and not the MMI url, so this tries to do a simple #string find of our desired observation in the observed_property. for obsOfInterest in obsList: for observedProperty in offer.observed_properties: if(observedProperty.find(obsOfInterest) != -1): #Because of the ncSOS bug 107, we don't add the observedProperty. When #that bug gets addressed, we would add the observedProperty. #obsFilterList.append(observedProperty) obsFilterList.append(obsOfInterest) break if(len(obsFilterList)): #We create a filter based on the observations of interest, # and a time frame of the 12 hours of data from yesterday. #There is an ncSOS bug that affects getting the current data #Issue: https://github.com/asascience-open/ncSOS/issues/112 """ Filter function params: bbox - Bounding box, not implemented for SOS calls currently. start - start date for records end - end date for records features - I assume it's a list of the stations of interest. Not sure why this is here and then the offerings parameter in the collect() function. variables - Poorly named parameter to house the observed_properties """ try: dataCollector.filter( variables=obsFilterList, start=datetime.utcnow() - timedelta(hours=24), end=datetime.utcnow() - timedelta(hours=12)) """ collect() params: offering - Station list responseFormat - The desired return format for the SOS call. eventTime - If the start/end times are not provided in the filter(), this param can be passed in. Has to be properly formatted: %Y-%m-%dT%H:%M:%SZ/%Y-%m-%dT%H:%M:%SZ for a start/end time/date. returns a list of OmObservation objects. OmObservation object has no member functions. """ response = dataCollector.collect(offerings=[offer.name]) for obsRec in response: stationRec = obsRec.feature print "Station: %s Location: %s" % (stationRec.name, stationRec.get_location()) #The elements are a list of the observed_properties returned wrapped in a Point object. for obsProp in stationRec.get_elements(): print "Observation Date/Time: %s" % (obsProp.get_time()) #print "Member names: %s" % (obsProp.get_member_names()) for member in obsProp.get_members(): for key,value in member.iteritems(): print "%s = %s" % (key, value) except ows.ExceptionReport,e: traceback.print_exc(e) except Exception,e: traceback.print_exc(e)
def get_stations_df(self, sos_url, station_urns_sel=None): """ Returns a Pandas Dataframe """ # oFrmts: IOOS SOS OutputFormat strings (first is compliant to the IOOS SOS spec, # second is to accommodate NDBC). More info here: # http://ioos.github.io/sos-guidelines/doc/wsdd/sos_wsdd_github_notoc/#describesensor-request:638e0b263020c13a76a55332bd966dbe oFrmts = [ 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"', 'text/xml;subtype="sensorML/1.0.1"' ] params = { 'service': 'SOS', 'request': 'GetCapabilities', 'acceptVersions': '1.0.0' } sos_url_params = sos_url + '?' + urlencode(params) # sos_url_params_quoted = quote(sos_url_params,"/=:") # sos_url_params_unquoted = unquote(sos_url_params) try: sosgc = SensorObservationService(sos_url_params) except (ConnectionError, ReadTimeout) as e: self.log.write( u"\nError: unable to connect to SOS service: {url} due to HTTP connection error." .format(url=sos_url_params)) self.log.write( u"\nHTTP connection error: {err}.".format(err=str(e))) sys.exit( "\nError: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}" .format(url=sos_url_params, err=str(e))) # vars to store returns from sos_collector.metadata_plus_exceptions function: sml_recs = {} sml_errors = {} describe_sensor_url = {} # leverage Pyoos Collector to query for all available stations and obtain SensorML # (if station subset not passed in --stations param) if station_urns_sel is not None: station_urns = station_urns_sel else: sos_collector = IoosSweSos(sos_url) station_urns = [ urn.name for urn in sos_collector.server.offerings if 'network' not in urn.name.split(':') ] sos_collector.features = station_urns # write out stations in SOS that will be handled: if self.verbose: self.log.write(u"\nStations to process for SOS: {sos}".format( sos=sos_url_params)) print("Stations to process for SOS: {sos}".format( sos=sos_url_params)) for feature in sos_collector.features: self.log.write(u"\n - {feature}".format(feature=feature)) print(" - {feature}".format(feature=feature)) # iterate over possible oFrmts expected of the various SOS services (IOOS SOS 1.0, NDBC): # for fmt in reversed(oFrmts): for fmt in oFrmts: try: sml_recs, sml_errors = sos_collector.metadata_plus_exceptions( output_format=fmt, timeout=200) # if no valid SensorML docs returned, try next oFrmt: if not sml_recs: continue else: # assign correct DescribeSensor url (use sos_collector.feature rather than sml_recs.keys() to # create DescribeSensor URLs for failures to record in logs): for station in sos_collector.features: describe_sensor_url[ station] = self.generate_describe_sensor_url( sosgc, procedure=station, oFrmt=fmt) # report on errors returned from metadata_plus_exceptions: if sml_errors: if self.verbose: for station, msg in iteritems(sml_errors): self.log.write( u"\nSOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}" .format(station=station, msg=msg)) print( "SOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}" .format(station=station, msg=msg)) else: self.log.write( u"\nSuccess, no errors returned from DescribeSensor requests in service: {sos}" .format(sos=sos_url_params)) print( "Success, no errors returned from DescribeSensor requests in service: {sos}" .format(sos=sos_url_params)) break # ServiceException shouldn't be thrown by metadata_plus_exceptions function, but handle regardless by attempting next oFrmt: except ServiceException as e: continue station_recs = [] failures = [] # generate Pandas DataFrame by populating 'station_recs' list by parsing SensorML strings: for station_idx, station_urn in enumerate(station_urns): if station_urns_sel is not None: # iterate oFrmts items for describe_sensor request (first is IOOS SOS spec-compliant, second is for NDBC SOS): for fmt in oFrmts: try: describe_sensor_url[ station_urn] = self.generate_describe_sensor_url( sosgc, procedure=station_urn, oFrmt=fmt) sml_str = sosgc.describe_sensor(procedure=station_urn, outputFormat=fmt, timeout=200) break except ServiceException as e: sml_errors[station_urn] = str(e) continue sml = SensorML(sml_str) else: # process valid SensorML responses, quietly pass on invalid stations (add to failures list for verbose reporting): try: sml = sml_recs[station_urn] except KeyError: self.log.write( u"\n\nStation: {station} failed (no SensorML in sml_recs dict). URL: {ds}" .format(station=station_urn, ds=describe_sensor_url[station_urn].replace( "&", "&"))) print( "Station: {station} failed (no SensorML in sml_recs dict). URL: {ds}" .format(station=station_urn, ds=describe_sensor_url[station_urn].replace( "&", "&"))) failures.append(station_urn) continue if self.sos_type.lower() == 'ndbc': # later: add an error check sosgc_station_offering = sosgc.contents[ 'station-' + station_urn.split(':')[-1]] else: sosgc_station_offering = None try: ds = IoosDescribeSensor(sml._root) except AttributeError: self.log.write( u"\nInvalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: " .format(station=station, ds=describe_sensor_url[station_urn].replace( "&", "&"))) print( "Invalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: " .format(station=station, ds=describe_sensor_url[station_urn].replace( "&", "&"))) station = OrderedDict() # debug: if self.verbose: self.log.write(u"\n\nProcessing station: {station}".format( station=station_urn)) print("Processing station: {station}".format( station=station_urn)) self.log.write("\n" + etree.tostring(sml._root).decode('utf-8')) # assign 'pos' to GML point location (accommodate 'gml:coordinates' as used by NDBC if gml:Point not found): try: pos = testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) \ if testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) is not None \ else testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:coordinates'))) station['lon'] = float(pos.split()[1]) station['lat'] = float(pos.split()[0]) except AttributeError as e: station['lon'] = None station['lat'] = None system_el = sml._root.findall(self.nsp('sml:member'))[0].find( self.nsp('sml:System')) # Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication # errors but there is not enough information in SensorML for alternatives) # Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage" documents = system_el.findall( self.nsp('sml:documentation/sml:DocumentList/sml:member')) documents_dct = {} for d in documents: document = Documentation(d) name = testXMLAttribute(d, "name") # url = document.documents[0].url documents_dct[name] = document # obtain list of contacts (accommodate 'sml:contact' element repetition used by NDBC insead of ContactList): contacts = system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \ if system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \ else system_el.findall(self.nsp('sml:contact')) contacts_dct = {} for c in contacts: contact = Contact(c) role = contact.role.split('/')[-1] contacts_dct[role] = contact # verify a 'publisher' Contact exists (template expects one): if "publisher" not in contacts_dct.keys(): self.log.write( u"\n\nStation: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]" .format(station=station_urn, roles=", ".join(contacts_dct.keys()))) print( "Station: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]" .format(station=station_urn, roles=", ".join(contacts_dct.keys()))) failures.append(station_urn) continue sweQuants = system_el.findall( self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity')) quant_lst = [ sweQuant.attrib['definition'] for sweQuant in sweQuants ] parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst] # attempt to read beginPosition, if available, otherwise use current date # bc ISO requires date value in output location in template: beginPosition = testXMLValue( system_el.find( self.nsp( 'sml:validTime/gml:TimePeriod/gml:beginPosition'))) try: begin_service_date = parser.parse(beginPosition) except (AttributeError, TypeError) as e: begin_service_date = datetime.now(pytz.utc) station['station_urn'] = station_urn station['sos_url'] = sos_url_params station['describesensor_url'] = describe_sensor_url[station_urn] station['shortName'] = ds.shortName station['longName'] = ds.longName if self.sos_type.lower() == 'ndbc': station['wmoID'] = station_urn.split(':')[-1] else: station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont) station['serverName'] = self.server_name # Some capabilities-level metadata: station['title'] = sosgc.identification.title station['abstract'] = sosgc.identification.abstract station['keywords'] = sosgc.identification.keywords station['begin_service_date'] = begin_service_date # Beware that a station can have >1 classifier of the same type # This code does not accommodate that possibility station['platformType'] = ds.platformType station['parentNetwork'] = ds.get_ioos_def('parentNetwork', 'classifier', ont) station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont) # store some nested dictionaries in 'station' for appropriate SensorML sources: station['contacts_dct'] = contacts_dct station['documents_dct'] = documents_dct if self.sos_type.lower( ) == 'ndbc' and sosgc_station_offering is not None: station['starting'] = sosgc_station_offering.begin_position station['ending'] = sosgc_station_offering.end_position else: station['starting'] = ds.starting station['ending'] = ds.ending if self.sos_type.lower( ) == 'ndbc' and sosgc_station_offering is not None: station[ 'variable_uris'] = sosgc_station_offering.observed_properties station['variables'] = [ var.split('/')[-1] for var in sosgc_station_offering.observed_properties ] station['parameter_uris'] = ','.join(station['variable_uris']) station['parameters'] = ','.join(station['variables']) else: station['variable_uris'] = ds.variables station['variables'] = [ var.split('/')[-1] for var in ds.variables ] station['parameter_uris'] = ','.join(quant_lst) station['parameters'] = ','.join(parameter_lst) if self.verbose: for var in station['variable_uris']: self.log.write(u"\nvariable: {var}".format(var=var)) print("variable: {var}".format(var=var)) # print(sosgc.contents) # for id, offering in sosgc.contents.iteritems(): # print("sosgc.contents: {item}".format(item=id)) # parse 'responseFormat' vals from SensorML: # response_formats = sosgc.contents[station_urn].response_formats response_formats = [] for id, sosgc.content in sosgc.contents.items(): if sosgc.content.name == station_urn: response_formats = sosgc.content.response_formats # response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() # if sosgc.content.name == station_urn ] # match responseFormats from SensorML (response_formats) against those passed in --response_formats parameter to # populate 'download_formats' list, that is then used to generate GetObservation requests for the template: # (default --response_formats values are: 'application/json,application/zip; subtype=x-netcdf' ) download_formats = [ response_format for response_format in response_formats if response_format in self.response_formats ] station['response_formats'] = response_formats station['download_formats'] = download_formats if self.verbose: for format in response_formats: self.log.write( u"\nresponseFormat: {format}".format(format=format)) print("responseFormat: {format}".format(format=format)) for format in download_formats: self.log.write( u"\ndownloadFormats: {format}".format(format=format)) print("downloadFormats: {format}".format(format=format)) # calculate event_time using self.getobs_req_hours: event_time_formatstr = "{begin:%Y-%m-%dT%H:%M:%S}{utc_code}/{end:%Y-%m-%dT%H:%M:%S}{utc_code}" utc_code = 'Z' if self.sos_type.lower() == 'ndbc' else '' if station['starting'] is not None and station[ 'ending'] is not None: event_time = event_time_formatstr.format( begin=station['ending'] - timedelta(hours=self.getobs_req_hours), end=station['ending'], utc_code=utc_code) if self.verbose: self.log.write( u"\nUsing starting/ending times from SensorML for eventTime" ) print( "Using starting/ending times from SensorML for eventTime" ) self.log.write( u"\nobservationTimeRange: starting: {start}, ending: {end}" .format(start=station['starting'], end=station['ending'])) print( "observationTimeRange: starting: {start}, ending: {end}" .format(start=station['starting'], end=station['ending'])) else: now = datetime.now(pytz.utc) then = now - timedelta(hours=self.getobs_req_hours) event_time = event_time_formatstr.format(begin=then, end=now, utc_code=utc_code) if self.verbose: self.log.write( u"\nNo 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) print( "No 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) if self.verbose: self.log.write(u"\neventTime: {time}".format(time=event_time)) print("eventTime: {time}".format(time=event_time)) # create a dict to store parameters for valid example GetObservation requests for station: getobs_req_dct = {} # populate a parameters dictionary for download links for each 'observedProperty' type # and secondly for each 'responseFormat' per observedProperty: getobs_params_base = { 'service': 'SOS', 'request': 'GetObservation', 'version': '1.0.0', 'offering': station_urn, 'eventTime': event_time } for variable in station['variable_uris']: getobs_params = getobs_params_base.copy() getobs_params['observedProperty'] = variable variable = variable.split('/')[-1] for format in download_formats: getobs_params['responseFormat'] = format getobs_request_url_encoded = sos_url + '?' + urlencode( getobs_params) getobs_request_url = unquote(getobs_request_url_encoded) getobs_req_dct[variable + '-' + format] = { 'variable': variable, 'url': getobs_request_url, 'format_type': self.RESPONSE_FORMAT_TYPE_MAP.get(format, format), 'format_name': self.RESPONSE_FORMAT_NAME_MAP.get(format, format) } if self.verbose: self.log.write( u"\ngetobs_request_url (var: {variable}): {getobs_request_url}" .format(variable=variable.split("/")[-1], getobs_request_url=getobs_request_url)) print( "getobs_request_url (var: {variable}): {getobs_request_url}" .format(variable=variable.split("/")[-1], getobs_request_url=getobs_request_url)) # ToDo: finish adding the 'getobs_req_dct' to the output template station['getobs_req_dct'] = getobs_req_dct station_recs.append(station) # extra debug for failed stations in verbose mode: if self.verbose: self.log.write( u"\n\n\nSOS DescribeSensor request errors recap. Failed requests:" ) print("SOS DescribeSensor request errors recap. Failed requests:") for station_fail, msg in iteritems(sml_errors): self.log.write( u"\n{station} - {msg}. DescribeSensor URL: {ds}".format( station=station_fail, msg=msg, ds=describe_sensor_url[station_fail].replace( "&", "&"))) print("{station} - {msg}. DescribeSensor URL: {ds}".format( station=station_fail, msg=msg, ds=describe_sensor_url[station_fail].replace("&", "&"))) if failures: self.log.write( u"\nStations in 'failures' list (should match DescribeSensor errors):" ) print( "Stations in 'failures' list (should match DescribeSensor errors):" ) for station_fail in failures: self.log.write(u"\n{station}".format(station=station_fail)) print("{station}".format(station=station_fail)) if station_recs: stations_df = pd.DataFrame.from_records(station_recs, columns=station.keys()) stations_df.index = stations_df['station_urn'] return stations_df else: return None
def main(): #The observations we're interested in. THis is another gotcha, is it water_temperature or sea_water_temperature... obsList = ['water_temperature', 'salinity'] """ Create a data collection object. Contructor parameters are: url - THe SWE endpoint we're interested in version - Optional default is '1.0.0' The SWE version the endpoint. xml - Optional default is None - The XML response from a GetCapabilities query of the server. """ dataCollector = IoosSweSos( url='http://129.252.139.124/thredds/sos/carocoops.cap2.buoy.nc', xml=None) #Loop through the offerings from the server. #Offerings are the stations/platforms/data device that the server house data from. offerings = dataCollector.server.offerings for offer in offerings: #We skip over the 'all' offering. #all "*network*" offerings represent aggregations of stations; # unlike all other offerings we've dealt with basically correspond to one station. if (offer.name.split(':')[-1] != 'all'): print "Offering: %s %s Obs: %s"\ % (offer.name, offer.description, offer.observed_properties) #Check to see if the station offered has the observation we're interested in. #Loop through the observed_properties attribute and compare. obsFilterList = [] #We want to query just the observations n the obsOfInterest list. #The observer_properties should be coded with an MMI href, however due #to ncSOS bug: https://github.com/asascience-open/ncSOS/issues/107 it adds #a URN pattern. Regardless, an end user should just need to know the observation #name and not the MMI url, so this tries to do a simple #string find of our desired observation in the observed_property. for obsOfInterest in obsList: for observedProperty in offer.observed_properties: if (observedProperty.find(obsOfInterest) != -1): #Because of the ncSOS bug 107, we don't add the observedProperty. When #that bug gets addressed, we would add the observedProperty. #obsFilterList.append(observedProperty) obsFilterList.append(obsOfInterest) break if (len(obsFilterList)): #We create a filter based on the observations of interest, # and a time frame of the 12 hours of data from yesterday. #There is an ncSOS bug that affects getting the current data #Issue: https://github.com/asascience-open/ncSOS/issues/112 """ Filter function params: bbox - Bounding box, not implemented for SOS calls currently. start - start date for records end - end date for records features - I assume it's a list of the stations of interest. Not sure why this is here and then the offerings parameter in the collect() function. variables - Poorly named parameter to house the observed_properties """ try: dataCollector.filter( variables=obsFilterList, start=datetime.utcnow() - timedelta(hours=24), end=datetime.utcnow() - timedelta(hours=12)) """ collect() params: offering - Station list responseFormat - The desired return format for the SOS call. eventTime - If the start/end times are not provided in the filter(), this param can be passed in. Has to be properly formatted: %Y-%m-%dT%H:%M:%SZ/%Y-%m-%dT%H:%M:%SZ for a start/end time/date. returns a list of OmObservation objects. OmObservation object has no member functions. """ response = dataCollector.collect(offerings=[offer.name]) for obsRec in response: stationRec = obsRec.feature print "Station: %s Location: %s" % ( stationRec.name, stationRec.get_location()) #The elements are a list of the observed_properties returned wrapped in a Point object. for obsProp in stationRec.get_elements(): print "Observation Date/Time: %s" % ( obsProp.get_time()) #print "Member names: %s" % (obsProp.get_member_names()) for member in obsProp.get_members(): for key, value in member.iteritems(): print "%s = %s" % (key, value) except ows.ExceptionReport, e: traceback.print_exc(e) except Exception, e: traceback.print_exc(e)
def get_stations_df(self, sos_url, station_urns_sel=None): """ Returns a GeoDataFrame """ # LATER: ADD ERROR TEST/CATCH AFTER EACH WEB REQUEST oFrmt = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"' params = { 'service': 'SOS', 'request': 'GetCapabilities', 'acceptVersions': '1.0.0' } sos_url_params = sos_url + '?' + urlencode(params) sos_url_params_esc = sos_url_params.replace("&", "&") # sos_url_params_quoted = quote(sos_url_params,"/=:") # sos_url_params_unquoted = unquote(sos_url_params) try: sosgc = SensorObservationService(sos_url_params) except (ConnectionError, ReadTimeout) as e: self.log.write( u"Error: unable to connect to SOS service: {url} due to HTTP connection error.\n" .format(url=sos_url_params)) self.log.write( u"HTTP connection error: {err}.\n".format(err=e.message)) sys.exit( "Error: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}" .format(url=sos_url_params, err=e.message)) if station_urns_sel is not None: station_urns = station_urns_sel else: sos_collector = IoosSweSos(sos_url) station_urns = [ urn.name for urn in sos_collector.server.offerings if 'network' not in urn.name.split(':') ] sos_collector.features = station_urns sml_lst = sos_collector.metadata(timeout=200) station_recs = [] for station_idx, station_urn in enumerate(station_urns): if station_urns_sel is not None: sml_str = sosgc.describe_sensor(procedure=station_urn, outputFormat=oFrmt, timeout=200) sml = SensorML(sml_str) else: sml = sml_lst[station_idx] # debug: # if self.verbose: # self.log.write(unicode(etree.tostring(sml._root))) ds = IoosDescribeSensor(sml._root) pos = testXMLValue( ds.system.location.find(self.nsp('gml:Point/gml:pos'))) system_el = sml._root.findall(self.nsp('sml:member'))[0].find( self.nsp('sml:System')) # Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication # errors but there is not enough information in SensorML for alternatives) # Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage" documents = system_el.findall( self.nsp('sml:documentation/sml:DocumentList/sml:member')) documents_dct = {} for d in documents: document = Documentation(d) name = testXMLAttribute(d, "name") # url = document.documents[0].url documents_dct[name] = document contacts = system_el.findall( self.nsp('sml:contact/sml:ContactList/sml:member')) contacts_dct = {} for c in contacts: contact = Contact(c) role = contact.role.split('/')[-1] contacts_dct[role] = contact sweQuants = system_el.findall( self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity')) quant_lst = [ sweQuant.attrib['definition'] for sweQuant in sweQuants ] parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst] # attempt to read beginPosition, if available: beginPosition = testXMLValue( system_el.find( self.nsp( 'sml:validTime/gml:TimePeriod/gml:beginPosition'))) try: begin_service_date = parser.parse(beginPosition) except AttributeError as e: begin_service_date = None station = OrderedDict() station['station_urn'] = station_urn station['sos_url'] = sos_url_params_esc station['lon'] = float(pos.split()[1]) station['lat'] = float(pos.split()[0]) station['shortName'] = ds.shortName station['longName'] = ds.longName station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont) station['serverName'] = self.server_name # Some capabilities-level metadata: station['title'] = sosgc.identification.title station['abstract'] = sosgc.identification.abstract station['keywords'] = sosgc.identification.keywords station['begin_service_date'] = begin_service_date # Beware that a station can have >1 classifier of the same type # This code does not accommodate that possibility station['platformType'] = ds.platformType station['parentNetwork'] = ds.get_ioos_def('parentNetwork', 'classifier', ont) station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont) # store some nested dictionaries in 'station' for appopriate SensorML sources: station['contacts_dct'] = contacts_dct station['documents_dct'] = documents_dct # MW: the 'operator_' and 'publisher_' attributes can be removed bc they are not used # in the template code currently in favor of 'contacts_dct' # station['operatorSector'] = ds.get_ioos_def('operatorSector', 'classifier', ont) # station['operator_org'] = contacts_dct['operator'].organization # station['operator_country'] = contacts_dct['operator'].country # station['operator_url'] = contacts_dct['operator'].url # station['operator_email'] = contacts_dct['operator'].email # station['publisher'] = ds.get_ioos_def('publisher', 'classifier', ont) # station['publisher_org'] = contacts_dct['publisher'].organization # station['publisher_url'] = contacts_dct['publisher'].url # station_dct['publisher_email'] = contacts_dct['publisher'].electronicMailAddress station['starting'] = ds.starting station['ending'] = ds.ending # station['starting_isostr'] = datetime.isoformat(ds.starting) # station['ending_isostr'] = datetime.isoformat(ds.ending) station['parameter_uris'] = ','.join(quant_lst) station['parameters'] = ','.join(parameter_lst) station['variables'] = [var.split('/')[-1] for var in ds.variables] # debug: if self.verbose: self.log.write(u"\nProcessing station: {station}\n".format( station=station_urn)) print("\nProcessing station: {station}".format( station=station_urn)) for var in ds.variables: self.log.write(u"variable: {var}\n".format(var=var)) print("variable: {var}".format(var=var)) # print(sosgc.contents) # for id, offering in sosgc.contents.iteritems(): # print("sosgc.contents: {item}".format(item=id)) # parse 'responseFormat' values and populate list: # response_formats = sosgc.contents[station_urn].response_formats response_formats = [] for id, sosgc.content in sosgc.contents.items(): if sosgc.content.name == station_urn: response_formats = sosgc.content.response_formats # response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() if sosgc.content.name == station_urn ] # subset responseFormats (response_formats) for download links matching those passed in --response_formats parameter # (or 'application/json,application/zip; subtype=x-netcdf' by default): download_formats = [ response_format for response_format in response_formats if response_format in self.response_formats ] station['response_formats'] = response_formats station['download_formats'] = download_formats if self.verbose: for format in response_formats: self.log.write( u"responseFormat: {format}\n".format(format=format)) print("responseFormat: {format}".format(format=format)) for format in download_formats: self.log.write( u"downloadFormats: {format}\n".format(format=format)) print("downloadFormats: {format}".format(format=format)) # calculate event_time using self.getobs_req_hours: if ds.starting is not None and ds.ending is not None: event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format( begin=ds.ending - timedelta(hours=self.getobs_req_hours), end=ds.ending) else: now = datetime.now(pytz.utc) then = now - timedelta(hours=self.getobs_req_hours) event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format( begin=then, end=now) if self.verbose: self.log.write( u"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}\n" .format(then=then, now=now)) print( "then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}" .format(then=then, now=now)) if self.verbose: self.log.write(u"eventTime: {time}\n".format(time=event_time)) print("eventTime: {time}".format(time=event_time)) # create a dict to store parameters for valid example GetObservation requests for station: getobs_req_dct = {} # populate a parameters dictionary for download links for each 'observedProperty' type and secondly for each 'responseFormat' per observedProperty: getobs_params_base = { 'service': 'SOS', 'request': 'GetObservation', 'version': '1.0.0', 'offering': station_urn, 'eventTime': event_time } for variable in ds.variables: getobs_params = getobs_params_base.copy() getobs_params['observedProperty'] = variable variable = variable.split('/')[-1] for format in download_formats: getobs_params['responseFormat'] = format getobs_request_url_encoded = sos_url + '?' + urlencode( getobs_params) getobs_request_url = unquote(getobs_request_url_encoded) getobs_request_url_esc = getobs_request_url.replace( "&", "&") getobs_req_dct[variable + '-' + format] = { 'variable': variable, 'url': getobs_request_url_esc, 'format_type': self.RESPONSE_FORMAT_TYPE_MAP[format], 'format_name': self.RESPONSE_FORMAT_NAME_MAP[format] } if self.verbose: self.log.write( u"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}\n" .format( variable=variable.split("/")[-1], getobs_request_url=getobs_request_url, getobs_request_url_esc=getobs_request_url_esc)) print( "getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}" .format( variable=variable.split("/")[-1], getobs_request_url=getobs_request_url, getobs_request_url_esc=getobs_request_url_esc)) # ToDo: finish adding the 'getobs_req_dct' to the output template station['getobs_req_dct'] = getobs_req_dct station_recs.append(station) stations_df = pd.DataFrame.from_records(station_recs, columns=station.keys()) stations_df.index = stations_df['station_urn'] return stations_df