def get_coops(start, end, sos_name, units, bbox, verbose=False): collector = CoopsSos() collector.set_bbox(bbox) collector.end_time = end collector.start_time = start collector.variables = [sos_name] ofrs = collector.server.offerings title = collector.server.identification.title config = dict(units=units, sos_name=sos_name,) data = collector2table( collector=collector, config=config, col="{} ({})".format(sos_name, units.format(cf_units.UT_ISO_8859_1)), ) # Clean the table. table = dict( station_name=[s._metadata.get("station_name") for s in data], station_code=[s._metadata.get("station_code") for s in data], sensor=[s._metadata.get("sensor") for s in data], lon=[s._metadata.get("lon") for s in data], lat=[s._metadata.get("lat") for s in data], depth=[s._metadata.get("depth", "NA") for s in data], ) table = pd.DataFrame(table).set_index("station_name") if verbose: print("Collector offerings") print("{}: {} offerings".format(title, len(ofrs))) return data, table
def get_coops(start, end, sos_name, units, bbox, verbose=False): """ function to read COOPS data We need to retry in case of failure b/c the server cannot handle the high traffic during hurricane season. """ print(' >> Get CO-OPS information') collector = CoopsSos() collector.set_bbox(bbox) collector.end_time = end collector.start_time = start collector.variables = [sos_name] ofrs = collector.server.offerings title = collector.server.identification.title config = dict( units=units, sos_name=sos_name, datum='MSL', ###Saeed added ) data = collector2table( collector=collector, config=config, col='{} ({})'.format(sos_name, units.format(cf_units.UT_ISO_8859_1)) ) # Clean the table. table = dict( station_name = [s._metadata.get('station_name') for s in data], station_code = [s._metadata.get('station_code') for s in data], sensor = [s._metadata.get('sensor') for s in data], lon = [s._metadata.get('lon') for s in data], lat = [s._metadata.get('lat') for s in data], depth = [s._metadata.get('depth', 'NA') for s in data], ) table = pd.DataFrame(table).set_index('station_name') if verbose: print('Collector offerings') print('{}: {} offerings'.format(title, len(ofrs))) return data, table
print station_name print "number of stations in bbox", len(st_list.keys()) return st_list # <markdowncell> # #COOPS Station Locations # <codecell> coops_collector = CoopsSos() coops_collector.start_time = start_time coops_collector.end_time = end_time coops_collector.variables = data_dict["currents"]["sos_name"] coops_collector.server.identification.title print coops_collector.start_time, ":", coops_collector.end_time ofrs = coops_collector.server.offerings print(len(ofrs)) # <codecell> print "Date: ", iso_start, " to ", iso_end box_str = ','.join(str(e) for e in bounding_box) print "Lat/Lon Box: ", box_str url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?' 'service=SOS&request=GetObservation&version=1.0.0&' 'observedProperty=%s&bin=1&' 'offering=urn:ioos:network:NOAA.NOS.CO-OPS:CurrentsActive&'
st_list[station_name]["lon"] = lon_data[i] print station_name print "number of stations in bbox",len(st_list.keys()) return st_list # <markdowncell> # #COOPS Station Locations # <codecell> coops_collector = CoopsSos() coops_collector.start_time = start_time coops_collector.end_time = end_time coops_collector.variables = data_dict["currents"]["sos_name"] coops_collector.server.identification.title print coops_collector.start_time,":", coops_collector.end_time ofrs = coops_collector.server.offerings print(len(ofrs)) # <codecell> print "Date: ",iso_start," to ", iso_end box_str=','.join(str(e) for e in bounding_box) print "Lat/Lon Box: ",box_str url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?' 'service=SOS&request=GetObservation&version=1.0.0&' 'observedProperty=%s&bin=1&' 'offering=urn:ioos:network:NOAA.NOS.CO-OPS:CurrentsActive&'
for url in sos_urls: log.info('{}'.format(url)) # In[ ]: from pyoos.collectors.coops.coops_sos import CoopsSos collector = CoopsSos() sos_name = 'water_surface_height_above_reference_datum' datum = 'NAVD' collector.set_datum(datum) collector.end_time = stop collector.start_time = start collector.variables = [sos_name] ofrs = collector.server.offerings title = collector.server.identification.title log.info(fmt(' Collector offerings ')) log.info('{}: {} offerings'.format(title, len(ofrs))) # In[ ]: from pandas import read_csv from utilities import sos_request params = dict(observedProperty=sos_name, eventTime=start.strftime('%Y-%m-%dT%H:%M:%SZ'), featureOfInterest='BBOX:{0},{1},{2},{3}'.format(*bbox),
# <codecell> start_time = datetime.strptime(start_date, '%Y-%m-%d %H:%M') end_time = datetime.strptime(end_date, '%Y-%m-%d %H:%M') # <codecell> iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ') iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ') collector = CoopsSos() collector.set_datum('NAVD') collector.server.identification.title collector.start_time = start_time collector.end_time = end_time collector.variables = [data_dict["water"]["sos_name"]] # <codecell> print("Date: %s to %s" % (iso_start, iso_end)) box_str = ','.join(str(e) for e in box) print("Lat/Lon Box: %s" % box_str) # <codecell> # Grab the sos url and use it for the service. url = (sos_urls[0].split("?")[0] + '?' 'service=SOS&request=GetObservation&version=1.0.0&' 'observedProperty=%s&' 'offering=urn:ioos:network:NOAA.NOS.CO-OPS:WaterLevelActive&' 'featureOfInterest=BBOX:%s&responseFormat=text/tab-separated-values&'
ndbc.set_index('name', inplace=True) to_html(ndbc.head()) # ### CoopsSoS # In[9]: from pyoos.collectors.coops.coops_sos import CoopsSos collector_coops = CoopsSos() collector_coops.set_bbox(bbox) collector_coops.end_time = stop collector_coops.start_time = start collector_coops.variables = [sos_name] ofrs = collector_coops.server.offerings title = collector_coops.server.identification.title log.info(fmt(' Collector offerings ')) log.info('{}: {} offerings'.format(title, len(ofrs))) # In[10]: from utilities import get_coops_metadata coops = collector2table(collector=collector_coops) names = [] for s in coops['station']:
# ## 1. Get observations from SOS # <codecell> collector = CoopsSos() # <codecell> collector.server.identification.title # <codecell> collector.start_time = jd_start collector.end_time = jd_stop collector.variables = [sos_name] # <codecell> ofrs = collector.server.offerings # <codecell> print len(ofrs) for p in ofrs[700:710]: print p # <markdowncell> # ### Find the SOS stations within our bounding box and time extent # We would like to just use a filter on a collection to get a new collection, but PYOOS doesn't do that yet. So we do a GetObservation request for a collection, including a bounding box, and asking for one value at the start of the time period of interest. We use that to do a bounding box filter on the SOS server, which returns 1 point for each station found. So for 3 stations, we get back 3 records, in CSV format. We can strip the station ids from the CSV, and then we have a list of stations we can use with pyoos. The template for the GetObservation query for the bounding box filtered collection was generated using the GUI at http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/
# <codecell> start_time = datetime.strptime(start_date, '%Y-%m-%d %H:%M') end_time = datetime.strptime(end_date, '%Y-%m-%d %H:%M') # <codecell> iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ') iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ') collector = CoopsSos() collector.set_datum('NAVD') collector.server.identification.title collector.start_time = start_time collector.end_time = end_time collector.variables = [data_dict["water"]["sos_name"]] # <codecell> print("Date: %s to %s" % (iso_start, iso_end)) box_str = ','.join(str(e) for e in box) print("Lat/Lon Box: %s" % box_str) # <codecell> # Grab the opendap sos url and use it for the service. for sos_url in sos_urls: if 'opendap' in sos_url: break url = (sos_url.split("?")[0] + '?'
# <markdowncell> # ###Get most recent observations from NOAA COOPS stations in bounding box # <codecell> start_time = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M') end_time = dt.datetime.strptime(end_date,'%Y-%m-%d %H:%M') iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ') iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ') # Define the Coops collector collector = CoopsSos() print collector.server.identification.title collector.variables = data_dict["temp"]["sos_name"] collector.server.identification.title # Don't specify start and end date in the filter and the most recent observation will be returned collector.filter(bbox=bounding_box, variables=data_dict["temp"]["sos_name"]) response = collector.raw(responseFormat="text/csv") obs_loc_df = pd.read_csv(BytesIO(response.encode('utf-8')), parse_dates=True, index_col='date_time') # Now let's specify start and end times collector.start_time = start_time collector.end_time = end_time
# <markdowncell> # ###Get most recent observations from NOAA-COOPS stations in bounding box # <codecell> start_time = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M') end_time = dt.datetime.strptime(end_date,'%Y-%m-%d %H:%M') iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ') iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ') # Define the Coops collector collector = CoopsSos() print collector.server.identification.title collector.variables = data_dict["temp"]["sos_name"] collector.server.identification.title # Don't specify start and end date in the filter and the most recent observation will be returned collector.filter(bbox=bounding_box, variables=data_dict["temp"]["sos_name"]) response = collector.raw(responseFormat="text/csv") obs_loc_df = pd.read_csv(BytesIO(response.encode('utf-8')), parse_dates=True, index_col='date_time') # Now let's specify start and end times collector.start_time = start_time collector.end_time = end_time
# ### SOS Requirements # <codecell> start_time = dt.datetime.strptime(start_date, '%Y-%m-%d %H:%M') end_time = dt.datetime.strptime(stop_date, '%Y-%m-%d %H:%M') iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ') iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ') # <codecell> collector = CoopsSos() collector.start_time = start_time collector.end_time = end_time collector.variables = data_dict["winds"]["sos_name"] collector.server.identification.title print collector.start_time, ":", collector.end_time ofrs = collector.server.offerings # <markdowncell> # ###Find all SOS stations within the bounding box and time extent # <codecell> print "Date: ", iso_start, " to ", iso_end box_str = ','.join(str(e) for e in bounding_box) print "Lat/Lon Box: ", box_str url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?'
print(fmt(" Filtered DAP ")) for url in dap_urls: print("{}.html".format(url)) Now we have a nice list of all the models available in the catalog for the domain we specified. We still need to find the observations for the same domain. To accomplish that we will use the `pyoos` library and search the [SOS CO-OPS](https://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/) services using the virtually the same configuration options from the catalog search. from pyoos.collectors.coops.coops_sos import CoopsSos collector_coops = CoopsSos() collector_coops.set_bbox(config["region"]["bbox"]) collector_coops.end_time = config["date"]["stop"] collector_coops.start_time = config["date"]["start"] collector_coops.variables = [config["sos_name"]] ofrs = collector_coops.server.offerings title = collector_coops.server.identification.title print(fmt(" Collector offerings ")) print("{}: {} offerings".format(title, len(ofrs))) To make it easier to work with the data we extract the time-series as pandas tables and interpolate them to a common 1-hour interval index. import pandas as pd from ioos_tools.ioos import collector2table data = collector2table( collector=collector_coops, config=config, col="water_surface_height_above_reference_datum (m)",
# ### SOS Requirements # <codecell> start_time = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M') end_time = dt.datetime.strptime(stop_date,'%Y-%m-%d %H:%M') iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ') iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ') # <codecell> collector = CoopsSos() collector.start_time = start_time collector.end_time = end_time collector.variables = data_dict["winds"]["sos_name"] collector.server.identification.title print collector.start_time,":", collector.end_time ofrs = collector.server.offerings # <markdowncell> # ###Find all SOS stations within the bounding box and time extent # <codecell> print "Date: ",iso_start," to ", iso_end box_str=','.join(str(e) for e in bounding_box) print "Lat/Lon Box: ",box_str url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?'