class IoosSweSosTest(unittest.TestCase):
def setUp(self):
# we have to pick SOS to test against, NANOOS it is:
self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp")
# self.c.features = [urn.name for urn in self.c.server.offerings]
@pytest.mark.xfail
def test_server_id(self):
assert (
self.c.server.identification.title
== "NANOOS Sensor Observation Service (SOS), a 52North IOOS SOS server"
)
assert self.c.server.identification.service == "OGC:SOS"
assert self.c.server.identification.version == "1.0.0"
assert self.c.server.identification.fees == "NONE"
assert self.c.server.identification.accessconstraints == "NONE"
@pytest.mark.xfail
def test_metdata(self):
self.c.features = ["urn:ioos:station:nanoos:apl_nemo"]
response = self.c.metadata(
output_format='text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
)
assert isinstance(response, list)
assert isinstance(response[0], SensorML)
@pytest.mark.xfail
def test_metadata_plus_exceptions(self):
self.c.features = ["urn:ioos:station:nanoos:apl_nemo"]
response, failures = self.c.metadata_plus_exceptions(
output_format='text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
)
assert isinstance(response, dict)
assert isinstance(failures, dict)
assert isinstance(response[self.c.features[0]], SensorML)
class IoosSweSosTest(unittest.TestCase):
def setUp(self):
# we have to pick SOS to test against, NANOOS it is:
self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp")
# self.c.features = [urn.name for urn in self.c.server.offerings]
@pytest.mark.xfail
def test_server_id(self):
assert (
self.c.server.identification.title ==
"NANOOS Sensor Observation Service (SOS), a 52North IOOS SOS server"
)
assert self.c.server.identification.service == "OGC:SOS"
assert self.c.server.identification.version == "1.0.0"
assert self.c.server.identification.fees == "NONE"
assert self.c.server.identification.accessconstraints == "NONE"
@pytest.mark.xfail
def test_metdata(self):
self.c.features = ["urn:ioos:station:nanoos:apl_nemo"]
response = self.c.metadata(
output_format=
'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"')
assert isinstance(response, list)
assert isinstance(response[0], SensorML)
@pytest.mark.xfail
def test_metadata_plus_exceptions(self):
self.c.features = ["urn:ioos:station:nanoos:apl_nemo"]
response, failures = self.c.metadata_plus_exceptions(
output_format=
'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"')
assert isinstance(response, dict)
assert isinstance(failures, dict)
assert isinstance(response[self.c.features[0]], SensorML)
def setUp(self):
# we have to pick SOS to test against, NANOOS it is:
self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp")
else:
v_frame = pd.concat([v_frame, new_frame])
v_frame = v_frame.drop_duplicates()
except ExceptionReport as e:
print " [-] Error obtaining {!s} from {!s} - Message from server: {!s}".format(v, sos, e)
continue
finally:
new_start = new_end
if v_frame is not None:
sos_dfs.append(v_frame)
else:
# Use the generic IOOS SWE collector
try:
collector = IoosSweSos(sos)
except BaseException as e:
print "[-] Could not process {!s}. Reason: {!s}".format(sos, e)
continue
for v in variables_to_query:
collector.filter(variables=[v])
collector.filter(bbox=bounding_box)
collector.filter(start=start_date)
collector.filter(end=end_date)
collector.filter(variables=[v])
try:
data = collector.collect()
print data
except BaseException as e:
print "[-] Could not process {!s}. Reason: {!s}".format(sos, e)
def setUp(self):
# we have to pick SOS to test against, NANOOS it is:
self.c = IoosSweSos("http://data.nanoos.org/52nsos/sos/kvp")
def main():
#The observations we're interested in. THis is another gotcha, is it water_temperature or sea_water_temperature...
obsList = ['water_temperature','salinity']
"""
Create a data collection object.
Contructor parameters are:
url - THe SWE endpoint we're interested in
version - Optional default is '1.0.0' The SWE version the endpoint.
xml - Optional default is None - The XML response from a GetCapabilities query of the server.
"""
dataCollector = IoosSweSos(url='http://129.252.139.124/thredds/sos/carocoops.cap2.buoy.nc', xml=None)
#Loop through the offerings from the server.
#Offerings are the stations/platforms/data device that the server house data from.
offerings = dataCollector.server.offerings
for offer in offerings:
#We skip over the 'all' offering.
#all "*network*" offerings represent aggregations of stations;
# unlike all other offerings we've dealt with basically correspond to one station.
if(offer.name.split(':')[-1] != 'all'):
print "Offering: %s %s Obs: %s"\
% (offer.name, offer.description, offer.observed_properties)
#Check to see if the station offered has the observation we're interested in.
#Loop through the observed_properties attribute and compare.
obsFilterList = []
#We want to query just the observations n the obsOfInterest list.
#The observer_properties should be coded with an MMI href, however due
#to ncSOS bug: https://github.com/asascience-open/ncSOS/issues/107 it adds
#a URN pattern. Regardless, an end user should just need to know the observation
#name and not the MMI url, so this tries to do a simple
#string find of our desired observation in the observed_property.
for obsOfInterest in obsList:
for observedProperty in offer.observed_properties:
if(observedProperty.find(obsOfInterest) != -1):
#Because of the ncSOS bug 107, we don't add the observedProperty. When
#that bug gets addressed, we would add the observedProperty.
#obsFilterList.append(observedProperty)
obsFilterList.append(obsOfInterest)
break
if(len(obsFilterList)):
#We create a filter based on the observations of interest,
# and a time frame of the 12 hours of data from yesterday.
#There is an ncSOS bug that affects getting the current data
#Issue: https://github.com/asascience-open/ncSOS/issues/112
"""
Filter function params:
bbox - Bounding box, not implemented for SOS calls currently.
start - start date for records
end - end date for records
features - I assume it's a list of the stations of interest. Not sure why this is here
and then the offerings parameter in the collect() function.
variables - Poorly named parameter to house the observed_properties
"""
try:
dataCollector.filter(
variables=obsFilterList,
start=datetime.utcnow() - timedelta(hours=24),
end=datetime.utcnow() - timedelta(hours=12))
"""
collect() params:
offering - Station list
responseFormat - The desired return format for the SOS call.
eventTime - If the start/end times are not provided in the filter(), this param
can be passed in. Has to be properly formatted: %Y-%m-%dT%H:%M:%SZ/%Y-%m-%dT%H:%M:%SZ for a start/end time/date.
returns a list of OmObservation objects. OmObservation object has no member functions.
"""
response = dataCollector.collect(offerings=[offer.name])
for obsRec in response:
stationRec = obsRec.feature
print "Station: %s Location: %s" % (stationRec.name, stationRec.get_location())
#The elements are a list of the observed_properties returned wrapped in a Point object.
for obsProp in stationRec.get_elements():
print "Observation Date/Time: %s" % (obsProp.get_time())
#print "Member names: %s" % (obsProp.get_member_names())
for member in obsProp.get_members():
for key,value in member.iteritems():
print "%s = %s" % (key, value)
except ows.ExceptionReport,e:
traceback.print_exc(e)
except Exception,e:
traceback.print_exc(e)
def get_stations_df(self, sos_url, station_urns_sel=None):
""" Returns a Pandas Dataframe
"""
# oFrmts: IOOS SOS OutputFormat strings (first is compliant to the IOOS SOS spec,
# second is to accommodate NDBC). More info here:
# http://ioos.github.io/sos-guidelines/doc/wsdd/sos_wsdd_github_notoc/#describesensor-request:638e0b263020c13a76a55332bd966dbe
oFrmts = [
'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"',
'text/xml;subtype="sensorML/1.0.1"'
]
params = {
'service': 'SOS',
'request': 'GetCapabilities',
'acceptVersions': '1.0.0'
}
sos_url_params = sos_url + '?' + urlencode(params)
# sos_url_params_quoted = quote(sos_url_params,"/=:")
# sos_url_params_unquoted = unquote(sos_url_params)
try:
sosgc = SensorObservationService(sos_url_params)
except (ConnectionError, ReadTimeout) as e:
self.log.write(
u"\nError: unable to connect to SOS service: {url} due to HTTP connection error."
.format(url=sos_url_params))
self.log.write(
u"\nHTTP connection error: {err}.".format(err=str(e)))
sys.exit(
"\nError: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}"
.format(url=sos_url_params, err=str(e)))
# vars to store returns from sos_collector.metadata_plus_exceptions function:
sml_recs = {}
sml_errors = {}
describe_sensor_url = {}
# leverage Pyoos Collector to query for all available stations and obtain SensorML
# (if station subset not passed in --stations param)
if station_urns_sel is not None:
station_urns = station_urns_sel
else:
sos_collector = IoosSweSos(sos_url)
station_urns = [
urn.name for urn in sos_collector.server.offerings
if 'network' not in urn.name.split(':')
]
sos_collector.features = station_urns
# write out stations in SOS that will be handled:
if self.verbose:
self.log.write(u"\nStations to process for SOS: {sos}".format(
sos=sos_url_params))
print("Stations to process for SOS: {sos}".format(
sos=sos_url_params))
for feature in sos_collector.features:
self.log.write(u"\n - {feature}".format(feature=feature))
print(" - {feature}".format(feature=feature))
# iterate over possible oFrmts expected of the various SOS services (IOOS SOS 1.0, NDBC):
# for fmt in reversed(oFrmts):
for fmt in oFrmts:
try:
sml_recs, sml_errors = sos_collector.metadata_plus_exceptions(
output_format=fmt, timeout=200)
# if no valid SensorML docs returned, try next oFrmt:
if not sml_recs:
continue
else:
# assign correct DescribeSensor url (use sos_collector.feature rather than sml_recs.keys() to
# create DescribeSensor URLs for failures to record in logs):
for station in sos_collector.features:
describe_sensor_url[
station] = self.generate_describe_sensor_url(
sosgc, procedure=station, oFrmt=fmt)
# report on errors returned from metadata_plus_exceptions:
if sml_errors:
if self.verbose:
for station, msg in iteritems(sml_errors):
self.log.write(
u"\nSOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}"
.format(station=station, msg=msg))
print(
"SOS DescribeSensor error returned for: {station}, skipping. Error msg: {msg}"
.format(station=station, msg=msg))
else:
self.log.write(
u"\nSuccess, no errors returned from DescribeSensor requests in service: {sos}"
.format(sos=sos_url_params))
print(
"Success, no errors returned from DescribeSensor requests in service: {sos}"
.format(sos=sos_url_params))
break
# ServiceException shouldn't be thrown by metadata_plus_exceptions function, but handle regardless by attempting next oFrmt:
except ServiceException as e:
continue
station_recs = []
failures = []
# generate Pandas DataFrame by populating 'station_recs' list by parsing SensorML strings:
for station_idx, station_urn in enumerate(station_urns):
if station_urns_sel is not None:
# iterate oFrmts items for describe_sensor request (first is IOOS SOS spec-compliant, second is for NDBC SOS):
for fmt in oFrmts:
try:
describe_sensor_url[
station_urn] = self.generate_describe_sensor_url(
sosgc, procedure=station_urn, oFrmt=fmt)
sml_str = sosgc.describe_sensor(procedure=station_urn,
outputFormat=fmt,
timeout=200)
break
except ServiceException as e:
sml_errors[station_urn] = str(e)
continue
sml = SensorML(sml_str)
else:
# process valid SensorML responses, quietly pass on invalid stations (add to failures list for verbose reporting):
try:
sml = sml_recs[station_urn]
except KeyError:
self.log.write(
u"\n\nStation: {station} failed (no SensorML in sml_recs dict). URL: {ds}"
.format(station=station_urn,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
print(
"Station: {station} failed (no SensorML in sml_recs dict). URL: {ds}"
.format(station=station_urn,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
failures.append(station_urn)
continue
if self.sos_type.lower() == 'ndbc':
# later: add an error check
sosgc_station_offering = sosgc.contents[
'station-' + station_urn.split(':')[-1]]
else:
sosgc_station_offering = None
try:
ds = IoosDescribeSensor(sml._root)
except AttributeError:
self.log.write(
u"\nInvalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: "
.format(station=station,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
print(
"Invalid SensorML passed to IoosDescribeSensor. Check DescribeSensor request for : {station}, URL: "
.format(station=station,
ds=describe_sensor_url[station_urn].replace(
"&", "&")))
station = OrderedDict()
# debug:
if self.verbose:
self.log.write(u"\n\nProcessing station: {station}".format(
station=station_urn))
print("Processing station: {station}".format(
station=station_urn))
self.log.write("\n" +
etree.tostring(sml._root).decode('utf-8'))
# assign 'pos' to GML point location (accommodate 'gml:coordinates' as used by NDBC if gml:Point not found):
try:
pos = testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) \
if testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:pos'))) is not None \
else testXMLValue(ds.system.location.find(self.nsp('gml:Point/gml:coordinates')))
station['lon'] = float(pos.split()[1])
station['lat'] = float(pos.split()[0])
except AttributeError as e:
station['lon'] = None
station['lat'] = None
system_el = sml._root.findall(self.nsp('sml:member'))[0].find(
self.nsp('sml:System'))
# Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication
# errors but there is not enough information in SensorML for alternatives)
# Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage"
documents = system_el.findall(
self.nsp('sml:documentation/sml:DocumentList/sml:member'))
documents_dct = {}
for d in documents:
document = Documentation(d)
name = testXMLAttribute(d, "name")
# url = document.documents[0].url
documents_dct[name] = document
# obtain list of contacts (accommodate 'sml:contact' element repetition used by NDBC insead of ContactList):
contacts = system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \
if system_el.findall(self.nsp('sml:contact/sml:ContactList/sml:member')) \
else system_el.findall(self.nsp('sml:contact'))
contacts_dct = {}
for c in contacts:
contact = Contact(c)
role = contact.role.split('/')[-1]
contacts_dct[role] = contact
# verify a 'publisher' Contact exists (template expects one):
if "publisher" not in contacts_dct.keys():
self.log.write(
u"\n\nStation: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]"
.format(station=station_urn,
roles=", ".join(contacts_dct.keys())))
print(
"Station: {station} skipped. No \'http://mmisw.org/ont/ioos/definition/publisher\' Contact role defined in SensorML as required. Roles defined: [{roles}]"
.format(station=station_urn,
roles=", ".join(contacts_dct.keys())))
failures.append(station_urn)
continue
sweQuants = system_el.findall(
self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity'))
quant_lst = [
sweQuant.attrib['definition'] for sweQuant in sweQuants
]
parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst]
# attempt to read beginPosition, if available, otherwise use current date
# bc ISO requires date value in output location in template:
beginPosition = testXMLValue(
system_el.find(
self.nsp(
'sml:validTime/gml:TimePeriod/gml:beginPosition')))
try:
begin_service_date = parser.parse(beginPosition)
except (AttributeError, TypeError) as e:
begin_service_date = datetime.now(pytz.utc)
station['station_urn'] = station_urn
station['sos_url'] = sos_url_params
station['describesensor_url'] = describe_sensor_url[station_urn]
station['shortName'] = ds.shortName
station['longName'] = ds.longName
if self.sos_type.lower() == 'ndbc':
station['wmoID'] = station_urn.split(':')[-1]
else:
station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont)
station['serverName'] = self.server_name
# Some capabilities-level metadata:
station['title'] = sosgc.identification.title
station['abstract'] = sosgc.identification.abstract
station['keywords'] = sosgc.identification.keywords
station['begin_service_date'] = begin_service_date
# Beware that a station can have >1 classifier of the same type
# This code does not accommodate that possibility
station['platformType'] = ds.platformType
station['parentNetwork'] = ds.get_ioos_def('parentNetwork',
'classifier', ont)
station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont)
# store some nested dictionaries in 'station' for appropriate SensorML sources:
station['contacts_dct'] = contacts_dct
station['documents_dct'] = documents_dct
if self.sos_type.lower(
) == 'ndbc' and sosgc_station_offering is not None:
station['starting'] = sosgc_station_offering.begin_position
station['ending'] = sosgc_station_offering.end_position
else:
station['starting'] = ds.starting
station['ending'] = ds.ending
if self.sos_type.lower(
) == 'ndbc' and sosgc_station_offering is not None:
station[
'variable_uris'] = sosgc_station_offering.observed_properties
station['variables'] = [
var.split('/')[-1]
for var in sosgc_station_offering.observed_properties
]
station['parameter_uris'] = ','.join(station['variable_uris'])
station['parameters'] = ','.join(station['variables'])
else:
station['variable_uris'] = ds.variables
station['variables'] = [
var.split('/')[-1] for var in ds.variables
]
station['parameter_uris'] = ','.join(quant_lst)
station['parameters'] = ','.join(parameter_lst)
if self.verbose:
for var in station['variable_uris']:
self.log.write(u"\nvariable: {var}".format(var=var))
print("variable: {var}".format(var=var))
# print(sosgc.contents)
# for id, offering in sosgc.contents.iteritems():
# print("sosgc.contents: {item}".format(item=id))
# parse 'responseFormat' vals from SensorML:
# response_formats = sosgc.contents[station_urn].response_formats
response_formats = []
for id, sosgc.content in sosgc.contents.items():
if sosgc.content.name == station_urn:
response_formats = sosgc.content.response_formats
# response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items()
# if sosgc.content.name == station_urn ]
# match responseFormats from SensorML (response_formats) against those passed in --response_formats parameter to
# populate 'download_formats' list, that is then used to generate GetObservation requests for the template:
# (default --response_formats values are: 'application/json,application/zip; subtype=x-netcdf' )
download_formats = [
response_format for response_format in response_formats
if response_format in self.response_formats
]
station['response_formats'] = response_formats
station['download_formats'] = download_formats
if self.verbose:
for format in response_formats:
self.log.write(
u"\nresponseFormat: {format}".format(format=format))
print("responseFormat: {format}".format(format=format))
for format in download_formats:
self.log.write(
u"\ndownloadFormats: {format}".format(format=format))
print("downloadFormats: {format}".format(format=format))
# calculate event_time using self.getobs_req_hours:
event_time_formatstr = "{begin:%Y-%m-%dT%H:%M:%S}{utc_code}/{end:%Y-%m-%dT%H:%M:%S}{utc_code}"
utc_code = 'Z' if self.sos_type.lower() == 'ndbc' else ''
if station['starting'] is not None and station[
'ending'] is not None:
event_time = event_time_formatstr.format(
begin=station['ending'] -
timedelta(hours=self.getobs_req_hours),
end=station['ending'],
utc_code=utc_code)
if self.verbose:
self.log.write(
u"\nUsing starting/ending times from SensorML for eventTime"
)
print(
"Using starting/ending times from SensorML for eventTime"
)
self.log.write(
u"\nobservationTimeRange: starting: {start}, ending: {end}"
.format(start=station['starting'],
end=station['ending']))
print(
"observationTimeRange: starting: {start}, ending: {end}"
.format(start=station['starting'],
end=station['ending']))
else:
now = datetime.now(pytz.utc)
then = now - timedelta(hours=self.getobs_req_hours)
event_time = event_time_formatstr.format(begin=then,
end=now,
utc_code=utc_code)
if self.verbose:
self.log.write(
u"\nNo 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
print(
"No 'observationTimeRange' present in SensorML. Using present time for eventTime: then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
if self.verbose:
self.log.write(u"\neventTime: {time}".format(time=event_time))
print("eventTime: {time}".format(time=event_time))
# create a dict to store parameters for valid example GetObservation requests for station:
getobs_req_dct = {}
# populate a parameters dictionary for download links for each 'observedProperty' type
# and secondly for each 'responseFormat' per observedProperty:
getobs_params_base = {
'service': 'SOS',
'request': 'GetObservation',
'version': '1.0.0',
'offering': station_urn,
'eventTime': event_time
}
for variable in station['variable_uris']:
getobs_params = getobs_params_base.copy()
getobs_params['observedProperty'] = variable
variable = variable.split('/')[-1]
for format in download_formats:
getobs_params['responseFormat'] = format
getobs_request_url_encoded = sos_url + '?' + urlencode(
getobs_params)
getobs_request_url = unquote(getobs_request_url_encoded)
getobs_req_dct[variable + '-' + format] = {
'variable':
variable,
'url':
getobs_request_url,
'format_type':
self.RESPONSE_FORMAT_TYPE_MAP.get(format, format),
'format_name':
self.RESPONSE_FORMAT_NAME_MAP.get(format, format)
}
if self.verbose:
self.log.write(
u"\ngetobs_request_url (var: {variable}): {getobs_request_url}"
.format(variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url))
print(
"getobs_request_url (var: {variable}): {getobs_request_url}"
.format(variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url))
# ToDo: finish adding the 'getobs_req_dct' to the output template
station['getobs_req_dct'] = getobs_req_dct
station_recs.append(station)
# extra debug for failed stations in verbose mode:
if self.verbose:
self.log.write(
u"\n\n\nSOS DescribeSensor request errors recap. Failed requests:"
)
print("SOS DescribeSensor request errors recap. Failed requests:")
for station_fail, msg in iteritems(sml_errors):
self.log.write(
u"\n{station} - {msg}. DescribeSensor URL: {ds}".format(
station=station_fail,
msg=msg,
ds=describe_sensor_url[station_fail].replace(
"&", "&")))
print("{station} - {msg}. DescribeSensor URL: {ds}".format(
station=station_fail,
msg=msg,
ds=describe_sensor_url[station_fail].replace("&",
"&")))
if failures:
self.log.write(
u"\nStations in 'failures' list (should match DescribeSensor errors):"
)
print(
"Stations in 'failures' list (should match DescribeSensor errors):"
)
for station_fail in failures:
self.log.write(u"\n{station}".format(station=station_fail))
print("{station}".format(station=station_fail))
if station_recs:
stations_df = pd.DataFrame.from_records(station_recs,
columns=station.keys())
stations_df.index = stations_df['station_urn']
return stations_df
else:
return None
def main():
#The observations we're interested in. THis is another gotcha, is it water_temperature or sea_water_temperature...
obsList = ['water_temperature', 'salinity']
"""
Create a data collection object.
Contructor parameters are:
url - THe SWE endpoint we're interested in
version - Optional default is '1.0.0' The SWE version the endpoint.
xml - Optional default is None - The XML response from a GetCapabilities query of the server.
"""
dataCollector = IoosSweSos(
url='http://129.252.139.124/thredds/sos/carocoops.cap2.buoy.nc',
xml=None)
#Loop through the offerings from the server.
#Offerings are the stations/platforms/data device that the server house data from.
offerings = dataCollector.server.offerings
for offer in offerings:
#We skip over the 'all' offering.
#all "*network*" offerings represent aggregations of stations;
# unlike all other offerings we've dealt with basically correspond to one station.
if (offer.name.split(':')[-1] != 'all'):
print "Offering: %s %s Obs: %s"\
% (offer.name, offer.description, offer.observed_properties)
#Check to see if the station offered has the observation we're interested in.
#Loop through the observed_properties attribute and compare.
obsFilterList = []
#We want to query just the observations n the obsOfInterest list.
#The observer_properties should be coded with an MMI href, however due
#to ncSOS bug: https://github.com/asascience-open/ncSOS/issues/107 it adds
#a URN pattern. Regardless, an end user should just need to know the observation
#name and not the MMI url, so this tries to do a simple
#string find of our desired observation in the observed_property.
for obsOfInterest in obsList:
for observedProperty in offer.observed_properties:
if (observedProperty.find(obsOfInterest) != -1):
#Because of the ncSOS bug 107, we don't add the observedProperty. When
#that bug gets addressed, we would add the observedProperty.
#obsFilterList.append(observedProperty)
obsFilterList.append(obsOfInterest)
break
if (len(obsFilterList)):
#We create a filter based on the observations of interest,
# and a time frame of the 12 hours of data from yesterday.
#There is an ncSOS bug that affects getting the current data
#Issue: https://github.com/asascience-open/ncSOS/issues/112
"""
Filter function params:
bbox - Bounding box, not implemented for SOS calls currently.
start - start date for records
end - end date for records
features - I assume it's a list of the stations of interest. Not sure why this is here
and then the offerings parameter in the collect() function.
variables - Poorly named parameter to house the observed_properties
"""
try:
dataCollector.filter(
variables=obsFilterList,
start=datetime.utcnow() - timedelta(hours=24),
end=datetime.utcnow() - timedelta(hours=12))
"""
collect() params:
offering - Station list
responseFormat - The desired return format for the SOS call.
eventTime - If the start/end times are not provided in the filter(), this param
can be passed in. Has to be properly formatted: %Y-%m-%dT%H:%M:%SZ/%Y-%m-%dT%H:%M:%SZ for a start/end time/date.
returns a list of OmObservation objects. OmObservation object has no member functions.
"""
response = dataCollector.collect(offerings=[offer.name])
for obsRec in response:
stationRec = obsRec.feature
print "Station: %s Location: %s" % (
stationRec.name, stationRec.get_location())
#The elements are a list of the observed_properties returned wrapped in a Point object.
for obsProp in stationRec.get_elements():
print "Observation Date/Time: %s" % (
obsProp.get_time())
#print "Member names: %s" % (obsProp.get_member_names())
for member in obsProp.get_members():
for key, value in member.iteritems():
print "%s = %s" % (key, value)
except ows.ExceptionReport, e:
traceback.print_exc(e)
except Exception, e:
traceback.print_exc(e)
def get_stations_df(self, sos_url, station_urns_sel=None):
""" Returns a GeoDataFrame
"""
# LATER: ADD ERROR TEST/CATCH AFTER EACH WEB REQUEST
oFrmt = 'text/xml; subtype="sensorML/1.0.1/profiles/ioos_sos/1.0"'
params = {
'service': 'SOS',
'request': 'GetCapabilities',
'acceptVersions': '1.0.0'
}
sos_url_params = sos_url + '?' + urlencode(params)
sos_url_params_esc = sos_url_params.replace("&", "&")
# sos_url_params_quoted = quote(sos_url_params,"/=:")
# sos_url_params_unquoted = unquote(sos_url_params)
try:
sosgc = SensorObservationService(sos_url_params)
except (ConnectionError, ReadTimeout) as e:
self.log.write(
u"Error: unable to connect to SOS service: {url} due to HTTP connection error.\n"
.format(url=sos_url_params))
self.log.write(
u"HTTP connection error: {err}.\n".format(err=e.message))
sys.exit(
"Error: unable to connect to SOS service: {url}. \nUnderlying HTTP connection error: {err}"
.format(url=sos_url_params, err=e.message))
if station_urns_sel is not None:
station_urns = station_urns_sel
else:
sos_collector = IoosSweSos(sos_url)
station_urns = [
urn.name for urn in sos_collector.server.offerings
if 'network' not in urn.name.split(':')
]
sos_collector.features = station_urns
sml_lst = sos_collector.metadata(timeout=200)
station_recs = []
for station_idx, station_urn in enumerate(station_urns):
if station_urns_sel is not None:
sml_str = sosgc.describe_sensor(procedure=station_urn,
outputFormat=oFrmt,
timeout=200)
sml = SensorML(sml_str)
else:
sml = sml_lst[station_idx]
# debug:
# if self.verbose:
# self.log.write(unicode(etree.tostring(sml._root)))
ds = IoosDescribeSensor(sml._root)
pos = testXMLValue(
ds.system.location.find(self.nsp('gml:Point/gml:pos')))
system_el = sml._root.findall(self.nsp('sml:member'))[0].find(
self.nsp('sml:System'))
# Parse the DocumentList into a dict storing documents by index value 'name' (may cause index duplication
# errors but there is not enough information in SensorML for alternatives)
# Assume that member corresponds to xlink:arcrole="urn:ogc:def:role:webPage"
documents = system_el.findall(
self.nsp('sml:documentation/sml:DocumentList/sml:member'))
documents_dct = {}
for d in documents:
document = Documentation(d)
name = testXMLAttribute(d, "name")
# url = document.documents[0].url
documents_dct[name] = document
contacts = system_el.findall(
self.nsp('sml:contact/sml:ContactList/sml:member'))
contacts_dct = {}
for c in contacts:
contact = Contact(c)
role = contact.role.split('/')[-1]
contacts_dct[role] = contact
sweQuants = system_el.findall(
self.nsp('sml:outputs/sml:OutputList/sml:output/swe:Quantity'))
quant_lst = [
sweQuant.attrib['definition'] for sweQuant in sweQuants
]
parameter_lst = [sweQuant.split('/')[-1] for sweQuant in quant_lst]
# attempt to read beginPosition, if available:
beginPosition = testXMLValue(
system_el.find(
self.nsp(
'sml:validTime/gml:TimePeriod/gml:beginPosition')))
try:
begin_service_date = parser.parse(beginPosition)
except AttributeError as e:
begin_service_date = None
station = OrderedDict()
station['station_urn'] = station_urn
station['sos_url'] = sos_url_params_esc
station['lon'] = float(pos.split()[1])
station['lat'] = float(pos.split()[0])
station['shortName'] = ds.shortName
station['longName'] = ds.longName
station['wmoID'] = ds.get_ioos_def('wmoID', 'identifier', ont)
station['serverName'] = self.server_name
# Some capabilities-level metadata:
station['title'] = sosgc.identification.title
station['abstract'] = sosgc.identification.abstract
station['keywords'] = sosgc.identification.keywords
station['begin_service_date'] = begin_service_date
# Beware that a station can have >1 classifier of the same type
# This code does not accommodate that possibility
station['platformType'] = ds.platformType
station['parentNetwork'] = ds.get_ioos_def('parentNetwork',
'classifier', ont)
station['sponsor'] = ds.get_ioos_def('sponsor', 'classifier', ont)
# store some nested dictionaries in 'station' for appopriate SensorML sources:
station['contacts_dct'] = contacts_dct
station['documents_dct'] = documents_dct
# MW: the 'operator_' and 'publisher_' attributes can be removed bc they are not used
# in the template code currently in favor of 'contacts_dct'
# station['operatorSector'] = ds.get_ioos_def('operatorSector', 'classifier', ont)
# station['operator_org'] = contacts_dct['operator'].organization
# station['operator_country'] = contacts_dct['operator'].country
# station['operator_url'] = contacts_dct['operator'].url
# station['operator_email'] = contacts_dct['operator'].email
# station['publisher'] = ds.get_ioos_def('publisher', 'classifier', ont)
# station['publisher_org'] = contacts_dct['publisher'].organization
# station['publisher_url'] = contacts_dct['publisher'].url
# station_dct['publisher_email'] = contacts_dct['publisher'].electronicMailAddress
station['starting'] = ds.starting
station['ending'] = ds.ending
# station['starting_isostr'] = datetime.isoformat(ds.starting)
# station['ending_isostr'] = datetime.isoformat(ds.ending)
station['parameter_uris'] = ','.join(quant_lst)
station['parameters'] = ','.join(parameter_lst)
station['variables'] = [var.split('/')[-1] for var in ds.variables]
# debug:
if self.verbose:
self.log.write(u"\nProcessing station: {station}\n".format(
station=station_urn))
print("\nProcessing station: {station}".format(
station=station_urn))
for var in ds.variables:
self.log.write(u"variable: {var}\n".format(var=var))
print("variable: {var}".format(var=var))
# print(sosgc.contents)
# for id, offering in sosgc.contents.iteritems():
# print("sosgc.contents: {item}".format(item=id))
# parse 'responseFormat' values and populate list:
# response_formats = sosgc.contents[station_urn].response_formats
response_formats = []
for id, sosgc.content in sosgc.contents.items():
if sosgc.content.name == station_urn:
response_formats = sosgc.content.response_formats
# response_formats = [ sosgc.content.response_formats for id, sosgc.content in sosgc.contents.items() if sosgc.content.name == station_urn ]
# subset responseFormats (response_formats) for download links matching those passed in --response_formats parameter
# (or 'application/json,application/zip; subtype=x-netcdf' by default):
download_formats = [
response_format for response_format in response_formats
if response_format in self.response_formats
]
station['response_formats'] = response_formats
station['download_formats'] = download_formats
if self.verbose:
for format in response_formats:
self.log.write(
u"responseFormat: {format}\n".format(format=format))
print("responseFormat: {format}".format(format=format))
for format in download_formats:
self.log.write(
u"downloadFormats: {format}\n".format(format=format))
print("downloadFormats: {format}".format(format=format))
# calculate event_time using self.getobs_req_hours:
if ds.starting is not None and ds.ending is not None:
event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format(
begin=ds.ending - timedelta(hours=self.getobs_req_hours),
end=ds.ending)
else:
now = datetime.now(pytz.utc)
then = now - timedelta(hours=self.getobs_req_hours)
event_time = "{begin:%Y-%m-%dT%H:%M:%S}/{end:%Y-%m-%dT%H:%M:%S}\n".format(
begin=then, end=now)
if self.verbose:
self.log.write(
u"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}\n"
.format(then=then, now=now))
print(
"then: {then:%Y-%m-%dT%H:%M:%S%z}, now: {now:%Y-%m-%dT%H:%M:%S%z}"
.format(then=then, now=now))
if self.verbose:
self.log.write(u"eventTime: {time}\n".format(time=event_time))
print("eventTime: {time}".format(time=event_time))
# create a dict to store parameters for valid example GetObservation requests for station:
getobs_req_dct = {}
# populate a parameters dictionary for download links for each 'observedProperty' type and secondly for each 'responseFormat' per observedProperty:
getobs_params_base = {
'service': 'SOS',
'request': 'GetObservation',
'version': '1.0.0',
'offering': station_urn,
'eventTime': event_time
}
for variable in ds.variables:
getobs_params = getobs_params_base.copy()
getobs_params['observedProperty'] = variable
variable = variable.split('/')[-1]
for format in download_formats:
getobs_params['responseFormat'] = format
getobs_request_url_encoded = sos_url + '?' + urlencode(
getobs_params)
getobs_request_url = unquote(getobs_request_url_encoded)
getobs_request_url_esc = getobs_request_url.replace(
"&", "&")
getobs_req_dct[variable + '-' + format] = {
'variable': variable,
'url': getobs_request_url_esc,
'format_type': self.RESPONSE_FORMAT_TYPE_MAP[format],
'format_name': self.RESPONSE_FORMAT_NAME_MAP[format]
}
if self.verbose:
self.log.write(
u"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}\n"
.format(
variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url,
getobs_request_url_esc=getobs_request_url_esc))
print(
"getobs_request_url (var: {variable}): {getobs_request_url}\ngetobs_request_url_esc (var: {variable}): {getobs_request_url_esc}"
.format(
variable=variable.split("/")[-1],
getobs_request_url=getobs_request_url,
getobs_request_url_esc=getobs_request_url_esc))
# ToDo: finish adding the 'getobs_req_dct' to the output template
station['getobs_req_dct'] = getobs_req_dct
station_recs.append(station)
stations_df = pd.DataFrame.from_records(station_recs,
columns=station.keys())
stations_df.index = stations_df['station_urn']
return stations_df