print('{}: {} offerings'.format(title, len(ofrs)))
return data, table
@retry(stop_max_attempt_number=5, wait_fixed=3000)
def get_ndbc(start, end, sos_name, units, bbox, verbose=False):
"""
function to read NDBC data
We need to retry in case of failure b/c the server cannot handle
the high traffic during hurricane season.
"""
print(' >> Get NDBC information')
collector = NdbcSos()
collector.set_bbox(bbox)
collector.end_time = end
collector.start_time = start
collector.variables = [sos_name]
ofrs = collector.server.offerings
title = collector.server.identification.title
fmt = '{:*^64}'.format
print(fmt(' NDBC Collector offerings '))
print('{}: {} offerings'.format(title, len(ofrs)))
https://ioos.github.io/system-test/blog/2015/10/12/fetching_data/
def get_ndbc(start, end, bbox , sos_name='waves',datum='MSL', verbose=True):
"""
function to read NBDC data
###################
sos_name = waves
all_col = (['station_id', 'sensor_id', 'latitude (degree)', 'longitude (degree)',
'date_time', 'sea_surface_wave_significant_height (m)',
'sea_surface_wave_peak_period (s)', 'sea_surface_wave_mean_period (s)',
'sea_surface_swell_wave_significant_height (m)',
'sea_surface_swell_wave_period (s)',
'sea_surface_wind_wave_significant_height (m)',
'sea_surface_wind_wave_period (s)', 'sea_water_temperature (c)',
'sea_surface_wave_to_direction (degree)',
'sea_surface_swell_wave_to_direction (degree)',
'sea_surface_wind_wave_to_direction (degree)',
'number_of_frequencies (count)', 'center_frequencies (Hz)',
'bandwidths (Hz)', 'spectral_energy (m**2/Hz)',
'mean_wave_direction (degree)', 'principal_wave_direction (degree)',
'polar_coordinate_r1 (1)', 'polar_coordinate_r2 (1)',
'calculation_method', 'sampling_rate (Hz)', 'name'])
sos_name = winds
all_col = (['station_id', 'sensor_id', 'latitude (degree)', 'longitude (degree)',
'date_time', 'depth (m)', 'wind_from_direction (degree)',
'wind_speed (m/s)', 'wind_speed_of_gust (m/s)',
'upward_air_velocity (m/s)', 'name'])
"""
#add remove from above
if sos_name == 'waves':
col = ['sea_surface_wave_significant_height (m)','sea_surface_wave_peak_period (s)',
'sea_surface_wave_mean_period (s)','sea_water_temperature (c)',
'sea_surface_wave_to_direction (degree)']
elif sos_name == 'winds':
col = ['wind_from_direction (degree)','wind_speed (m/s)',
'wind_speed_of_gust (m/s)','upward_air_velocity (m/s)']
collector = NdbcSos()
collector.set_bbox(bbox)
collector.start_time = start
collector.variables = [sos_name]
ofrs = collector.server.offerings
title = collector.server.identification.title
collector.features = None
collector.end_time = start + datetime.timedelta(1)
response = collector.raw(responseFormat='text/csv')
df = pd.read_csv(BytesIO(response), parse_dates=True)
g = df.groupby('station_id')
df = dict()
for station in g.groups.keys():
df.update({station: g.get_group(station).iloc[0]})
df = pd.DataFrame.from_dict(df).T
station_dict = {}
for offering in collector.server.offerings:
station_dict.update({offering.name: offering.description})
names = []
for sta in df.index:
names.append(station_dict.get(sta, sta))
df['name'] = names
#override short time
collector.end_time = end
data = []
for k, row in df.iterrows():
station_id = row['station_id'].split(':')[-1]
collector.features = [station_id]
response = collector.raw(responseFormat='text/csv')
kw = dict(parse_dates=True, index_col='date_time')
obs = pd.read_csv(BytesIO(response), **kw).reset_index()
obs = obs.drop_duplicates(subset='date_time').set_index('date_time')
series = obs[col]
series._metadata = dict(
station=row.get('station_id'),
station_name=row.get('name'),
station_code=str(row.get('station_id').split(':')[-1]),
sensor=row.get('sensor_id'),
lon=row.get('longitude (degree)'),
lat=row.get('latitude (degree)'),
depth=row.get('depth (m)'),
)
data.append(series)
# Clean the table.
table = dict(
station_name = [s._metadata.get('station_name') for s in data],
station_code = [s._metadata.get('station_code') for s in data],
sensor = [s._metadata.get('sensor') for s in data],
lon = [s._metadata.get('lon') for s in data],
lat = [s._metadata.get('lat') for s in data],
depth = [s._metadata.get('depth', 'NA') for s in data],
)
table = pd.DataFrame(table).set_index('station_name')
if verbose:
print('Collector offerings')
print('{}: {} offerings'.format(title, len(ofrs)))
return data, table
dap_urls = non_stations
log.info(fmt(' Filtered DAP '))
for url in dap_urls:
log.info('{}.html'.format(url))
# ### NdbcSos
# In[7]:
from pyoos.collectors.ndbc.ndbc_sos import NdbcSos
collector_ndbc = NdbcSos()
collector_ndbc.set_bbox(bbox)
collector_ndbc.end_time = stop
collector_ndbc.start_time = start
collector_ndbc.variables = [sos_name]
ofrs = collector_ndbc.server.offerings
title = collector_ndbc.server.identification.title
log.info(fmt(' NDBC Collector offerings '))
log.info('{}: {} offerings'.format(title, len(ofrs)))
# In[8]:
from utilities import collector2table, to_html, get_ndbc_longname
ndbc = collector2table(collector=collector_ndbc)
except (IOError, OSError, RuntimeError, TimeoutError) as e:
print("Could not access URL {}.html\n{!r}".format(url, e))
dap_urls = non_stations
print(fmt(" Filtered DAP "))
for url in dap_urls:
print("{}.html".format(url))
Now we can use `pyoos` collectors for `NdbcSos`,
from pyoos.collectors.ndbc.ndbc_sos import NdbcSos
collector_ndbc = NdbcSos()
collector_ndbc.set_bbox(config["region"]["bbox"])
collector_ndbc.end_time = config["date"]["stop"]
collector_ndbc.start_time = config["date"]["start"]
collector_ndbc.variables = [config["sos_name"]]
ofrs = collector_ndbc.server.offerings
title = collector_ndbc.server.identification.title
print(fmt(" NDBC Collector offerings "))
print("{}: {} offerings".format(title, len(ofrs)))
import pandas as pd
from ioos_tools.ioos import collector2table
ndbc = collector2table(
collector=collector_ndbc, config=config, col="sea_water_temperature (C)"
)