def extractSFOModelData(lat_lon_list, name_list):
print "Extract SFOModelData from http://opendap.co-ops.nos.noaa.gov/thredds/ "
urls = buildSFOUrls(jd_start, jd_stop)
index = -1
#setup struct
df_list = {}
for n in name_list:
df_list[n] = {}
data_dates = []
for i, url in enumerate(urls):
try:
#print url
nc = netCDF4.Dataset(url, 'r')
except:
#print "\tNot Available"
break
if i == 0:
lats = nc.variables['lat'][:]
lons = nc.variables['lon'][:]
lons = lons - 360
index_list, dist_list = findSFOIndexs(lats, lons, lat_lon_list)
#Extract the model data using and MF dataset
time_dim = nc.variables['time']
u_dim = nc.variables['u']
v_dim = nc.variables['v']
u_var = u_dim[:, 0, index_list]
v_var = v_dim[:, 0, index_list]
#create the dates
dates = num2date(time_dim[:],
units=time_dim.units,
calendar='gregorian')
#data_dates.append(dates)
for i, n in enumerate(name_list):
#get lat and lon
df_list[n]['lat'] = lats[index_list[i]]
df_list[n]['lon'] = lons[index_list[i]]
#create speed and direction, convert
ws = uv2ws(u_var[:, i] * 100, v_var[:, i] * 100)
wd = uv2wd(u_var[:, i] * 100, v_var[:, i] * 100)
data = {}
data['sea_water_speed (cm/s)'] = ws
data['direction_of_sea_water_velocity (degree)'] = wd
columns = [
'sea_water_speed (cm/s)',
'direction_of_sea_water_velocity (degree)'
]
df = pd.DataFrame(data=data, index=dates, columns=columns)
#create struct
if 'data' in df_list[n]:
df_list[n]['data'] = df_list[n]['data'].append(df)
else:
df_list[n]['data'] = df
return df_list
def get_hr_radar_dap_data(dap_urls, st_list, jd_start, jd_stop):
# Use only data within 1.00 degrees
obs_df = []
obs_or_model = False
max_dist = 1.0
# Use only data where the standard deviation of the time series exceeds 0.01 m (1 cm).
# This eliminates flat line model time series that come from land points that should have had missing values.
min_var = 0.1
data_idx = []
df_list = []
for url in dap_urls:
#only look at 6km hf radar
if 'http://hfrnet.ucsd.edu/thredds/dodsC/HFRNet/USWC/' in url and "6km" in url and "GNOME" in url:
print url
#get url
nc = netCDF4.Dataset(url, 'r')
lat_dim = nc.variables['lat']
lon_dim = nc.variables['lon']
time_dim = nc.variables['time']
u_var = None
v_var = None
for key in nc.variables.iterkeys():
key_dim = nc.variables[key]
try:
if key_dim.standard_name == "surface_eastward_sea_water_velocity":
u_var = key_dim
elif key_dim.standard_name == "surface_northward_sea_water_velocity":
v_var = key_dim
elif key_dim.standard_name == "time":
time = key_dim
except:
#only if the standard name is not available
pass
#manage dates
dates = num2date(time_dim[:],
units=time_dim.units,
calendar='gregorian')
date_idx = []
date_list = []
for i, date in enumerate(dates):
if jd_start < date < jd_stop:
date_idx.append(i)
date_list.append(date)
#manage location
for st in st_list:
station = st_list[st]
f_lat = station['lat']
f_lon = station['lon']
ret = find_nearest(f_lat, f_lon, lat_dim[:], lon_dim[:])
lat_idx = ret[0]
lon_idx = ret[1]
dist_deg = ret[2]
#print "lat,lon,dist=",ret
if len(u_var.dimensions) == 3:
#3dimensions
ret = cycleAndGetData(u_var, v_var, date_idx, lat_idx,
lon_idx)
u_vals = ret[0]
v_vals = ret[1]
lat_idx = ret[2]
lon_idx = ret[3]
print "lat,lon,dist=", ret[2], ret[3]
try:
#turn vectors in the speed and direction
ws = uv2ws(u_vals, v_vals)
wd = uv2wd(u_vals, v_vals)
data_spd = []
data_dir = []
data = {}
data['sea_water_speed (cm/s)'] = np.array(ws)
data[
'direction_of_sea_water_velocity (degree)'] = np.array(
wd)
time = np.array(date_list)
df = pd.DataFrame(
data=data,
index=time,
columns=[
'sea_water_speed (cm/s)',
'direction_of_sea_water_velocity (degree)'
])
df_list.append({
"name": st,
"data": df,
"lat": lat_dim[lat_idx],
"lon": lon_dim[lon_idx],
"ws_pts": np.count_nonzero(~np.isnan(ws)),
"wd_pts": np.count_nonzero(~np.isnan(wd)),
"dist": dist_deg,
'from': url
})
except Exception, e:
print "\t\terror:", e
else:
pass
