def get_coops(start, end, sos_name, units, bbox, verbose=False):
collector = CoopsSos()
collector.set_bbox(bbox)
collector.end_time = end
collector.start_time = start
collector.variables = [sos_name]
ofrs = collector.server.offerings
title = collector.server.identification.title
config = dict(units=units, sos_name=sos_name,)
data = collector2table(
collector=collector,
config=config,
col="{} ({})".format(sos_name, units.format(cf_units.UT_ISO_8859_1)),
)
# Clean the table.
table = dict(
station_name=[s._metadata.get("station_name") for s in data],
station_code=[s._metadata.get("station_code") for s in data],
sensor=[s._metadata.get("sensor") for s in data],
lon=[s._metadata.get("lon") for s in data],
lat=[s._metadata.get("lat") for s in data],
depth=[s._metadata.get("depth", "NA") for s in data],
)
table = pd.DataFrame(table).set_index("station_name")
if verbose:
print("Collector offerings")
print("{}: {} offerings".format(title, len(ofrs)))
return data, table
def get_coops(start, end, sos_name, units, bbox, verbose=False):
"""
function to read COOPS data
We need to retry in case of failure b/c the server cannot handle
the high traffic during hurricane season.
"""
print(' >> Get CO-OPS information')
collector = CoopsSos()
collector.set_bbox(bbox)
collector.end_time = end
collector.start_time = start
collector.variables = [sos_name]
ofrs = collector.server.offerings
title = collector.server.identification.title
config = dict(
units=units,
sos_name=sos_name,
datum='MSL', ###Saeed added
)
data = collector2table(
collector=collector,
config=config,
col='{} ({})'.format(sos_name, units.format(cf_units.UT_ISO_8859_1))
)
# Clean the table.
table = dict(
station_name = [s._metadata.get('station_name') for s in data],
station_code = [s._metadata.get('station_code') for s in data],
sensor = [s._metadata.get('sensor') for s in data],
lon = [s._metadata.get('lon') for s in data],
lat = [s._metadata.get('lat') for s in data],
depth = [s._metadata.get('depth', 'NA') for s in data],
)
table = pd.DataFrame(table).set_index('station_name')
if verbose:
print('Collector offerings')
print('{}: {} offerings'.format(title, len(ofrs)))
return data, table
print station_name
print "number of stations in bbox", len(st_list.keys())
return st_list
# <markdowncell>
# #COOPS Station Locations
# <codecell>
coops_collector = CoopsSos()
coops_collector.start_time = start_time
coops_collector.end_time = end_time
coops_collector.variables = data_dict["currents"]["sos_name"]
coops_collector.server.identification.title
print coops_collector.start_time, ":", coops_collector.end_time
ofrs = coops_collector.server.offerings
print(len(ofrs))
# <codecell>
print "Date: ", iso_start, " to ", iso_end
box_str = ','.join(str(e) for e in bounding_box)
print "Lat/Lon Box: ", box_str
url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?'
'service=SOS&request=GetObservation&version=1.0.0&'
'observedProperty=%s&bin=1&'
'offering=urn:ioos:network:NOAA.NOS.CO-OPS:CurrentsActive&'
st_list[station_name]["lon"] = lon_data[i]
print station_name
print "number of stations in bbox",len(st_list.keys())
return st_list
# <markdowncell>
# #COOPS Station Locations
# <codecell>
coops_collector = CoopsSos()
coops_collector.start_time = start_time
coops_collector.end_time = end_time
coops_collector.variables = data_dict["currents"]["sos_name"]
coops_collector.server.identification.title
print coops_collector.start_time,":", coops_collector.end_time
ofrs = coops_collector.server.offerings
print(len(ofrs))
# <codecell>
print "Date: ",iso_start," to ", iso_end
box_str=','.join(str(e) for e in bounding_box)
print "Lat/Lon Box: ",box_str
url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?'
'service=SOS&request=GetObservation&version=1.0.0&'
'observedProperty=%s&bin=1&'
'offering=urn:ioos:network:NOAA.NOS.CO-OPS:CurrentsActive&'
for url in sos_urls:
log.info('{}'.format(url))
# In[ ]:
from pyoos.collectors.coops.coops_sos import CoopsSos
collector = CoopsSos()
sos_name = 'water_surface_height_above_reference_datum'
datum = 'NAVD'
collector.set_datum(datum)
collector.end_time = stop
collector.start_time = start
collector.variables = [sos_name]
ofrs = collector.server.offerings
title = collector.server.identification.title
log.info(fmt(' Collector offerings '))
log.info('{}: {} offerings'.format(title, len(ofrs)))
# In[ ]:
from pandas import read_csv
from utilities import sos_request
params = dict(observedProperty=sos_name,
eventTime=start.strftime('%Y-%m-%dT%H:%M:%SZ'),
featureOfInterest='BBOX:{0},{1},{2},{3}'.format(*bbox),
# <codecell>
start_time = datetime.strptime(start_date, '%Y-%m-%d %H:%M')
end_time = datetime.strptime(end_date, '%Y-%m-%d %H:%M')
# <codecell>
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
collector = CoopsSos()
collector.set_datum('NAVD')
collector.server.identification.title
collector.start_time = start_time
collector.end_time = end_time
collector.variables = [data_dict["water"]["sos_name"]]
# <codecell>
print("Date: %s to %s" % (iso_start, iso_end))
box_str = ','.join(str(e) for e in box)
print("Lat/Lon Box: %s" % box_str)
# <codecell>
# Grab the sos url and use it for the service.
url = (sos_urls[0].split("?")[0] + '?'
'service=SOS&request=GetObservation&version=1.0.0&'
'observedProperty=%s&'
'offering=urn:ioos:network:NOAA.NOS.CO-OPS:WaterLevelActive&'
'featureOfInterest=BBOX:%s&responseFormat=text/tab-separated-values&'
ndbc.set_index('name', inplace=True)
to_html(ndbc.head())
# ### CoopsSoS
# In[9]:
from pyoos.collectors.coops.coops_sos import CoopsSos
collector_coops = CoopsSos()
collector_coops.set_bbox(bbox)
collector_coops.end_time = stop
collector_coops.start_time = start
collector_coops.variables = [sos_name]
ofrs = collector_coops.server.offerings
title = collector_coops.server.identification.title
log.info(fmt(' Collector offerings '))
log.info('{}: {} offerings'.format(title, len(ofrs)))
# In[10]:
from utilities import get_coops_metadata
coops = collector2table(collector=collector_coops)
names = []
for s in coops['station']:
# ## 1. Get observations from SOS # <codecell> collector = CoopsSos() # <codecell> collector.server.identification.title # <codecell> collector.start_time = jd_start collector.end_time = jd_stop collector.variables = [sos_name] # <codecell> ofrs = collector.server.offerings # <codecell> print len(ofrs) for p in ofrs[700:710]: print p # <markdowncell> # ### Find the SOS stations within our bounding box and time extent # We would like to just use a filter on a collection to get a new collection, but PYOOS doesn't do that yet. So we do a GetObservation request for a collection, including a bounding box, and asking for one value at the start of the time period of interest. We use that to do a bounding box filter on the SOS server, which returns 1 point for each station found. So for 3 stations, we get back 3 records, in CSV format. We can strip the station ids from the CSV, and then we have a list of stations we can use with pyoos. The template for the GetObservation query for the bounding box filtered collection was generated using the GUI at http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/
# <codecell>
start_time = datetime.strptime(start_date, '%Y-%m-%d %H:%M')
end_time = datetime.strptime(end_date, '%Y-%m-%d %H:%M')
# <codecell>
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
collector = CoopsSos()
collector.set_datum('NAVD')
collector.server.identification.title
collector.start_time = start_time
collector.end_time = end_time
collector.variables = [data_dict["water"]["sos_name"]]
# <codecell>
print("Date: %s to %s" % (iso_start, iso_end))
box_str = ','.join(str(e) for e in box)
print("Lat/Lon Box: %s" % box_str)
# <codecell>
# Grab the opendap sos url and use it for the service.
for sos_url in sos_urls:
if 'opendap' in sos_url:
break
url = (sos_url.split("?")[0] + '?'
# <markdowncell>
# ###Get most recent observations from NOAA COOPS stations in bounding box
# <codecell>
start_time = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M')
end_time = dt.datetime.strptime(end_date,'%Y-%m-%d %H:%M')
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
# Define the Coops collector
collector = CoopsSos()
print collector.server.identification.title
collector.variables = data_dict["temp"]["sos_name"]
collector.server.identification.title
# Don't specify start and end date in the filter and the most recent observation will be returned
collector.filter(bbox=bounding_box,
variables=data_dict["temp"]["sos_name"])
response = collector.raw(responseFormat="text/csv")
obs_loc_df = pd.read_csv(BytesIO(response.encode('utf-8')),
parse_dates=True,
index_col='date_time')
# Now let's specify start and end times
collector.start_time = start_time
collector.end_time = end_time
# <markdowncell>
# ###Get most recent observations from NOAA-COOPS stations in bounding box
# <codecell>
start_time = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M')
end_time = dt.datetime.strptime(end_date,'%Y-%m-%d %H:%M')
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
# Define the Coops collector
collector = CoopsSos()
print collector.server.identification.title
collector.variables = data_dict["temp"]["sos_name"]
collector.server.identification.title
# Don't specify start and end date in the filter and the most recent observation will be returned
collector.filter(bbox=bounding_box,
variables=data_dict["temp"]["sos_name"])
response = collector.raw(responseFormat="text/csv")
obs_loc_df = pd.read_csv(BytesIO(response.encode('utf-8')),
parse_dates=True,
index_col='date_time')
# Now let's specify start and end times
collector.start_time = start_time
collector.end_time = end_time
# ### SOS Requirements
# <codecell>
start_time = dt.datetime.strptime(start_date, '%Y-%m-%d %H:%M')
end_time = dt.datetime.strptime(stop_date, '%Y-%m-%d %H:%M')
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
# <codecell>
collector = CoopsSos()
collector.start_time = start_time
collector.end_time = end_time
collector.variables = data_dict["winds"]["sos_name"]
collector.server.identification.title
print collector.start_time, ":", collector.end_time
ofrs = collector.server.offerings
# <markdowncell>
# ###Find all SOS stations within the bounding box and time extent
# <codecell>
print "Date: ", iso_start, " to ", iso_end
box_str = ','.join(str(e) for e in bounding_box)
print "Lat/Lon Box: ", box_str
url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?'
print(fmt(" Filtered DAP "))
for url in dap_urls:
print("{}.html".format(url))
Now we have a nice list of all the models available in the catalog for the domain we specified.
We still need to find the observations for the same domain.
To accomplish that we will use the `pyoos` library and search the [SOS CO-OPS](https://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/) services using the virtually the same configuration options from the catalog search.
from pyoos.collectors.coops.coops_sos import CoopsSos
collector_coops = CoopsSos()
collector_coops.set_bbox(config["region"]["bbox"])
collector_coops.end_time = config["date"]["stop"]
collector_coops.start_time = config["date"]["start"]
collector_coops.variables = [config["sos_name"]]
ofrs = collector_coops.server.offerings
title = collector_coops.server.identification.title
print(fmt(" Collector offerings "))
print("{}: {} offerings".format(title, len(ofrs)))
To make it easier to work with the data we extract the time-series as pandas tables and interpolate them to a common 1-hour interval index.
import pandas as pd
from ioos_tools.ioos import collector2table
data = collector2table(
collector=collector_coops,
config=config,
col="water_surface_height_above_reference_datum (m)",
# ### SOS Requirements
# <codecell>
start_time = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M')
end_time = dt.datetime.strptime(stop_date,'%Y-%m-%d %H:%M')
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')
# <codecell>
collector = CoopsSos()
collector.start_time = start_time
collector.end_time = end_time
collector.variables = data_dict["winds"]["sos_name"]
collector.server.identification.title
print collector.start_time,":", collector.end_time
ofrs = collector.server.offerings
# <markdowncell>
# ###Find all SOS stations within the bounding box and time extent
# <codecell>
print "Date: ",iso_start," to ", iso_end
box_str=','.join(str(e) for e in bounding_box)
print "Lat/Lon Box: ",box_str
url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?'
