def query_point(self):
print('Retrieving selected variables...')
ds = self.connect()
ncss = NCSS(ds.access_urls['NetcdfSubset'])
query = ncss.query()
now = datetime.utcnow()
timestamp = self.end
#converts time from datetime to standard time
self.init_time = now.strftime('%Hz-%d-%Y')
self.end_time = (now +
pd.Timedelta(hours=timestamp)).strftime('%Hz-%d-%Y')
query.time_range(now + pd.Timedelta(hours=self.start),
now + pd.Timedelta(hours=timestamp))
query.accept('netcdf4')
query.lonlat_point(self.lon, self.lat)
#pull temperature, cloud cover, and precip data
query.variables(self.variables)
data = ncss.get_data(query)
return data
def get_data(self):
# Request the GFS data from the thredds server
gfs_url = f"https://thredds.ucar.edu/thredds/catalog/grib/NCEP/GFS/Global_0p25deg/GFS_Global_0p25deg_{self.now.year}{self.now.month:02d}{self.now.day:02d}_0000.grib2/catalog.xml"
gfs_cat = TDSCatalog(gfs_url)
#https://thredds.ucar.edu/thredds/catalog/grib/NCEP/GFS/Global_0p25deg/catalog.xml'
dataset = list(gfs_cat.datasets.values())[0]
#print(dataset.access_urls)
# Create NCSS object to access the NetcdfSubset
ncss = NCSS(dataset.access_urls['NetcdfSubset'])
# query the data from the server
query = ncss.query()
query.time_range(self.start, self.end)
query.lonlat_box(north=80, south=0, east=310, west=200)
query.accept('netcdf4')
print("-----------------------------------------\n"\
+"Sit back....\nOr get your coffee....\nOr do a Sudoku....\n"\
+"-----------------------------------------\n")
print("qeueing data...")
#query.variables(str(self.query_list)).add_lonlat(True)
for i in self.query_list:
query.variables(i)
#query.variables(vort_name,hgt_name,pv_press_name,mslp_name,upflux_rad_name,u_name,v_name,
# u_src_name,v_src_name,sfc_gust_name).add_lonlat(True)
print("\ndone qeueing data.\n\ngrabbing data...\n")
# Request data for the variables you want to use
self.data = ncss.get_data(query)
print("done grabbing data!!\n-_-_-_-_-_-_-_-_-_-_-_\n")
return self.data
def get_obs(ts, mybb):
# copied from the browser url box
metar_cat_url = 'http://thredds.ucar.edu/thredds/catalog/nws/metar/ncdecoded/catalog.xml?dataset=nws/metar/ncdecoded/Metar_Station_Data_fc.cdmr'
# parse the xml
metar_cat = TDSCatalog(metar_cat_url)
# what datasets are here? only one "dataset" in this catalog
dataset = list(metar_cat.datasets.values())[0]
ncss_url = dataset.access_urls["NetcdfSubset"]
ncss = NCSS(ncss_url)
query = ncss.query().accept('csv').time(ts - datetime.timedelta(minutes=1))
query.lonlat_box(**mybb)
query.variables('air_temperature', 'dew_point_temperature', 'inches_ALTIM',
'wind_speed', 'wind_from_direction', 'cloud_area_fraction', 'weather')
try:
data = ncss.get_data(query)
lats = data['latitude'][:]
lons = data['longitude'][:]
tair = data['air_temperature'][:]
dewp = data['dew_point_temperature'][:]
slp = (data['inches_ALTIM'][:] * units('inHg')).to('mbar')
# Convert wind to components
u, v = mpcalc.get_wind_components(data['wind_speed'] * units.knot,
data['wind_from_direction'] * units.deg)
# Need to handle missing (NaN) and convert to proper code
cloud_cover = 8 * data['cloud_area_fraction']
cloud_cover[np.isnan(cloud_cover)] = 9
cloud_cover = cloud_cover.astype(np.int)
# For some reason these come back as bytes instead of strings
stid = [s.decode() for s in data['station']]
# Convert the text weather observations to WMO codes we can map to symbols
if data['weather'].dtype != bool:
wx_text = [s.decode('ascii') for s in data['weather']]
wx_codes = np.array(list(to_code(wx_text)))
else:
wx_codes = np.array([0] * len(data['weather']))
sfc_data = {'latitude': lats, 'longitude': lons,
'air_temperature': tair, 'dew_point_temperature': dewp, 'eastward_wind': u,
'northward_wind': v, 'cloud_coverage': cloud_cover,
'air_pressure_at_sea_level': slp, 'present_weather': wx_codes}
have_obs = True
except:
have_obs = False
sfc_data = {}
return sfc_data, have_obs
def get_sounding(source, lat, long):
# source unused for now bc testing only on ncss
source_place_holder = source
#print(source_place_holder)
best_gfs = TDSCatalog('http://thredds.ucar.edu/thredds/catalog/grib/NCEP/GFS/Global_0p5deg/' +
'catalog.xml?dataset=grib/NCEP/GFS/Global_0p5deg/Best')
best_ds = list(best_gfs.datasets.values())[0]
ncss = NCSS(best_ds.access_urls['NetcdfSubset'])
query = ncss.query()
query.lonlat_point(long, lat).time(datetime.utcnow())
query.accept('netcdf4')
query.variables('Temperature_isobaric', 'Relative_humidity_isobaric', 'u-component_of_wind_isobaric',
'v-component_of_wind_isobaric')
data = ncss.get_data(query)
temp = data.variables['Temperature_isobaric']
temp_vals = temp[:].squeeze() * units.kelvin
relh = data.variables['Relative_humidity_isobaric']
relh_values = relh[:] / 100
td = dewpoint_rh(temp_vals, relh_values)
td_vals = td[:].squeeze()
press = data.variables['isobaric3']
press_vals = press[:].squeeze()
u_wind = data.variables['u-component_of_wind_isobaric']
u_wind_vals = u_wind[:].squeeze()
v_wind = data.variables['v-component_of_wind_isobaric']
v_wind_vals = v_wind[:].squeeze()
# Put temp, dewpoint, pressure, u/v winds into numpy arrays and reorder
t = np.array(temp_vals)[::-1]
td = np.array(td_vals)[::-1]
p = np.array(press_vals)[::-1]
u = np.array(u_wind_vals)[::-1]
v = np.array(v_wind_vals)[::-1]
# Change units for proper skew-T
p = (p * units.pascals).to('mbar')
t = (t * units.kelvin).to('degC')
td = td * units.degC
u = (u * units('m/s')).to('knot')
v = (v * units('m/s')).to('knot')
# spd = spd * units.knot
# direc = direc * units.deg
# u, v = get_wind_components(spd, direc)
return t, td, p, u, v, lat, long, str(datetime.utcnow())[:-7]
def retrieve_gfs_analysis(
time,
lat=50,
variables=['Geopotential_height_isobaric',
'u-component_of_wind_isobaric']):
url = time.strftime(
'https://www.ncei.noaa.gov/thredds/ncss/grid/gfs-g3-anl-files/%Y%m/%Y%m%d/gfsanl_3_%Y%m%d_%H%M_000.grb2/'
)
ncss = NCSS(url)
query = ncss.query()
query.all_times().variables(*variables)
query.lonlat_box(north=lat, south=lat, east=360., west=0.)
nc_north = ncss.get_data(query)
query.lonlat_box(north=-lat, south=-lat, east=360., west=0.)
nc_south = ncss.get_data(query)
data_north = xr.open_dataset(xr.backends.NetCDF4DataStore(nc_north))
data_south = xr.open_dataset(xr.backends.NetCDF4DataStore(nc_south))
return xr.concat([data_north, data_south], dim='lat')
def get_closest_gfs(time, level, field):
"""
Retreive the current best 0.25 deg GFS model for a given field, level, time.
time : datetime object
level : level of results (in hPa)
field : CF field to retrieve
"""
# Get the catalog and best GFS entry
catalog = TDSCatalog(
'http://thredds.ucar.edu/thredds/catalog/grib/NCEP/GFS/Global_0p25deg/catalog.xml'
)
best_gfs = list(catalog.datasets.values())[1]
# Using NCSS, build a query and getch the data
ncss = NCSS(best_gfs.access_urls['NetcdfSubset'])
query = ncss.query()
query.lonlat_box(north=90, south=10, east=360, west=160)
query.vertical_level(level)
query.time(time)
query.accept('netcdf4')
query.variables(field)
data = ncss.get_data(query)
# Pull out the variables we will use
lat_var = data.variables['lat']
lon_var = data.variables['lon']
data_var = data.variables[field]
# Find the correct time dimension name
for coord in data_var.coordinates.split():
if 'time' in coord:
time_var = data.variables[coord]
break
# Convert number of hours since the reference time into an actual date
time_vals = netCDF4.num2date(time_var[:].squeeze(), time_var.units)
# Combine 1D latitude and longitudes into a 2D grid of locations
lon_2d, lat_2d = np.meshgrid(lon_var[:], lat_var[:])
# Filter the data to smooth it out a bit
data_var = ndimage.gaussian_filter(data_var[:][0][0], sigma=1.5, order=0)
return time_vals, lat_2d, lon_2d, data_var
def get_data(lon_w, lon_e, lat_s, lat_n, variable):
"""TODO
Add reset, change colors of map, variable selection, model selection, lat/long validator
"""
cat_url = 'http://thredds-jumbo.unidata.ucar.edu/thredds/catalog/grib/NCEP/GFS/Global_0p25deg/catalog.xml'
latest_gfs = get_latest_access_url(cat_url, 'NetcdfSubset')
ncss = NCSS(latest_gfs)
query = ncss.query()
query.lonlat_box(west=lon_w, east=lon_e, south=lat_s, north=lat_n).all_times()
query.accept('netcdf4')
query.variables(variable_dict(variable))
data = ncss.get_data(query)
list(data.variables.keys())
var1 = data.variables[variable_dict(variable)]
# only works if has name time+... or only has 1 dimension
for dim in var1.dimensions:
if 'time' in dim:
time_name = dim
if time_name is None:
raise ValueError("Couldn't find a time dimension for " + var1.name)
time_1d = data.variables[time_name]
lat_1d = data.variables['lat']
lon_1d = data.variables['lon']
# Reduce the dimensions of the data
lat_1d = lat_1d[:].squeeze()
lon_1d = lon_1d[:].squeeze()
# Convert the number of hours since the reference time to an actual date
time_val = num2date(time_1d[:].squeeze(), time_1d.units)
# Combine latitude and longitudes
lon_2d, lat_2d = np.meshgrid(lon_1d, lat_1d)
# Flatten() combines all the lists from meshgrid into one list
full_lat_1d = lat_2d.flatten()
full_lon_1d = lon_2d.flatten()
# Create one list that pairs longs and lats
lonlat_list = zip(full_lon_1d, full_lat_1d)
return lon_2d, lat_2d, var1, time_val, lonlat_list
def get_closest_gfs(time, level, field):
"""
Retreive the current best 0.25 deg GFS model for a given field, level, time.
time : datetime object
level : level of results (in hPa)
field : CF field to retrieve
"""
# Get the catalog and best GFS entry
catalog = TDSCatalog('http://thredds.ucar.edu/thredds/catalog/grib/NCEP/GFS/Global_0p25deg/catalog.xml')
best_gfs = list(catalog.datasets.values())[1]
# Using NCSS, build a query and getch the data
ncss = NCSS(best_gfs.access_urls['NetcdfSubset'])
query = ncss.query()
query.lonlat_box(north=90, south=10, east=360, west=160)
query.vertical_level(level)
query.time(time)
query.accept('netcdf4')
query.variables(field)
data = ncss.get_data(query)
# Pull out the variables we will use
lat_var = data.variables['lat']
lon_var = data.variables['lon']
data_var = data.variables[field]
# Find the correct time dimension name
for coord in data_var.coordinates.split():
if 'time' in coord:
time_var = data.variables[coord]
break
# Convert number of hours since the reference time into an actual date
time_vals = netCDF4.num2date(time_var[:].squeeze(), time_var.units)
# Combine 1D latitude and longitudes into a 2D grid of locations
lon_2d, lat_2d = np.meshgrid(lon_var[:], lat_var[:])
# Filter the data to smooth it out a bit
data_var = ndimage.gaussian_filter(data_var[:][0][0], sigma=1.5, order=0)
return time_vals, lat_2d, lon_2d, data_var
def retrieve_point_forecast(ds, lat, lon, var, ensemble):
'''
ds: a siphon dataset object
lat:
lon:
var: model variable name to extract
ensemble: True/False indicator if the ds object contains ensemble data
Given a siphon dataset object, retrieve the forecast variable for the
given coordinates.
If the object is from an ensemble, the variables object has an
additional dimension.
'''
ncss = NCSS(ds.access_urls['NetcdfSubset'])
query = ncss.query()
query.lonlat_point(lon, lat)
query.all_times()
query.variables(var).accept('netcdf')
data = ncss.get_data(query)
temps = data.variables[var]
time = data.variables['time']
time_vals = num2date(time[:].squeeze(), time.units)
ureg = UnitRegistry()
if ensemble:
ensemble_temp_series = []
num_ens = temps.shape[2]
for i in range(num_ens):
temp_vals = ((temps[:, :, i, :].squeeze() * ureg.kelvin)
.to(ureg.degF))
temp_series = pd.Series(temp_vals, index=time_vals)
ensemble_temp_series.append(temp_series)
return ensemble_temp_series
else:
temp_vals = (temps[:, :, 0].squeeze() * ureg.kelvin).to(ureg.degF)
temp_series = pd.Series(temp_vals, index=time_vals)
return temp_series
class TestNCSS(object):
server = 'http://thredds.ucar.edu/thredds/ncss/'
urlPath = 'grib/NCEP/GFS/Global_0p5deg/GFS_Global_0p5deg_20150612_1200.grib2'
@recorder.use_cassette('ncss_test_metadata')
def setup(self):
dt = datetime(2015, 6, 12, 15, 0, 0)
self.ncss = NCSS(self.server + self.urlPath)
self.nq = self.ncss.query().lonlat_point(-105, 40).time(dt)
self.nq.variables('Temperature_isobaric', 'Relative_humidity_isobaric')
def test_good_query(self):
assert self.ncss.validate_query(self.nq)
def test_bad_query(self):
self.nq.variables('foo')
assert not self.ncss.validate_query(self.nq)
def test_bad_query_no_vars(self):
self.nq.var.clear()
assert not self.ncss.validate_query(self.nq)
@recorder.use_cassette('ncss_gfs_xml_point')
def test_xml_point(self):
self.nq.accept('xml')
xml_data = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in xml_data
assert 'Relative_humidity_isobaric' in xml_data
assert xml_data['lat'][0] == 40
assert xml_data['lon'][0] == -105
@recorder.use_cassette('ncss_gfs_csv_point')
def test_csv_point(self):
self.nq.accept('csv')
csv_data = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in csv_data
assert 'Relative_humidity_isobaric' in csv_data
assert csv_data['lat'][0] == 40
assert csv_data['lon'][0] == -105
@recorder.use_cassette('ncss_gfs_csv_point')
def test_unit_handler_csv(self):
self.nq.accept('csv')
self.ncss.unit_handler = tuple_unit_handler
csv_data = self.ncss.get_data(self.nq)
temp = csv_data['Temperature_isobaric']
assert len(temp) == 2
assert temp[1] == 'K'
relh = csv_data['Relative_humidity_isobaric']
assert len(relh) == 2
assert relh[1] == '%'
@recorder.use_cassette('ncss_gfs_xml_point')
def test_unit_handler_xml(self):
self.nq.accept('xml')
self.ncss.unit_handler = tuple_unit_handler
xml_data = self.ncss.get_data(self.nq)
temp = xml_data['Temperature_isobaric']
assert len(temp) == 2
assert temp[1] == 'K'
relh = xml_data['Relative_humidity_isobaric']
assert len(relh) == 2
assert relh[1] == '%'
@recorder.use_cassette('ncss_gfs_netcdf_point')
def test_netcdf_point(self):
self.nq.accept('netcdf')
nc = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in nc.variables
assert 'Relative_humidity_isobaric' in nc.variables
assert nc.variables['latitude'][0] == 40
assert nc.variables['longitude'][0] == -105
@recorder.use_cassette('ncss_gfs_netcdf4_point')
def test_netcdf4_point(self):
self.nq.accept('netcdf4')
nc = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in nc.variables
assert 'Relative_humidity_isobaric' in nc.variables
assert nc.variables['latitude'][0] == 40
assert nc.variables['longitude'][0] == -105
@recorder.use_cassette('ncss_gfs_vertical_level')
def test_vertical_level(self):
self.nq.accept('csv').vertical_level(50000)
csv_data = self.ncss.get_data(self.nq)
assert str(csv_data['Temperature_isobaric'])[:6] == '263.39'
@recorder.use_cassette('ncss_gfs_csv_point')
def test_raw_csv(self):
self.nq.accept('csv')
csv_data = self.ncss.get_data_raw(self.nq)
assert csv_data.startswith(b'date,lat')
@recorder.use_cassette('ncss_gfs_csv_point')
def test_unknown_mime(self):
self.nq.accept('csv')
with response_context():
csv_data = self.ncss.get_data(self.nq)
assert csv_data.startswith(b'date,lat')
class TestNCSS(object):
"""Test NCSS queries and response parsing."""
server = 'http://thredds.ucar.edu/thredds/ncss/'
urlPath = 'grib/NCEP/GFS/Global_0p5deg/GFS_Global_0p5deg_20150612_1200.grib2'
@recorder.use_cassette('ncss_test_metadata')
def setup(self):
"""Set up for tests with a default valid query."""
dt = datetime(2015, 6, 12, 15, 0, 0)
self.ncss = NCSS(self.server + self.urlPath)
self.nq = self.ncss.query().lonlat_point(-105, 40).time(dt)
self.nq.variables('Temperature_isobaric', 'Relative_humidity_isobaric')
def test_good_query(self):
"""Test that a good query is properly validated."""
assert self.ncss.validate_query(self.nq)
def test_bad_query(self):
"""Test that a query with an unknown variable is invalid."""
self.nq.variables('foo')
assert not self.ncss.validate_query(self.nq)
def test_empty_query(self):
"""Test that an empty query is invalid."""
query = self.ncss.query()
res = self.ncss.validate_query(query)
assert not res
assert not isinstance(res, set)
def test_bad_query_no_vars(self):
"""Test that a query without variables is invalid."""
self.nq.var.clear()
assert not self.ncss.validate_query(self.nq)
@recorder.use_cassette('ncss_gfs_xml_point')
def test_xml_point(self):
"""Test parsing XML point returns."""
self.nq.accept('xml')
xml_data = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in xml_data
assert 'Relative_humidity_isobaric' in xml_data
assert xml_data['lat'][0] == 40
assert xml_data['lon'][0] == -105
@recorder.use_cassette('ncss_gfs_csv_point')
def test_csv_point(self):
"""Test parsing CSV point returns."""
self.nq.accept('csv')
csv_data = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in csv_data
assert 'Relative_humidity_isobaric' in csv_data
assert csv_data['lat'][0] == 40
assert csv_data['lon'][0] == -105
@recorder.use_cassette('ncss_gfs_csv_point')
def test_unit_handler_csv(self):
"""Test unit-handling from CSV returns."""
self.nq.accept('csv')
self.ncss.unit_handler = tuple_unit_handler
csv_data = self.ncss.get_data(self.nq)
temp = csv_data['Temperature_isobaric']
assert len(temp) == 2
assert temp[1] == 'K'
relh = csv_data['Relative_humidity_isobaric']
assert len(relh) == 2
assert relh[1] == '%'
@recorder.use_cassette('ncss_gfs_xml_point')
def test_unit_handler_xml(self):
"""Test unit-handling from XML returns."""
self.nq.accept('xml')
self.ncss.unit_handler = tuple_unit_handler
xml_data = self.ncss.get_data(self.nq)
temp = xml_data['Temperature_isobaric']
assert len(temp) == 2
assert temp[1] == 'K'
relh = xml_data['Relative_humidity_isobaric']
assert len(relh) == 2
assert relh[1] == '%'
@recorder.use_cassette('ncss_gfs_netcdf_point')
def test_netcdf_point(self):
"""Test handling of netCDF point returns."""
self.nq.accept('netcdf')
nc = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in nc.variables
assert 'Relative_humidity_isobaric' in nc.variables
assert nc.variables['latitude'][0] == 40
assert nc.variables['longitude'][0] == -105
@recorder.use_cassette('ncss_gfs_netcdf4_point')
def test_netcdf4_point(self):
"""Test handling of netCDF4 point returns."""
self.nq.accept('netcdf4')
nc = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in nc.variables
assert 'Relative_humidity_isobaric' in nc.variables
assert nc.variables['latitude'][0] == 40
assert nc.variables['longitude'][0] == -105
@recorder.use_cassette('ncss_gfs_vertical_level')
def test_vertical_level(self):
"""Test data return from a single vertical level is correct."""
self.nq.accept('csv').vertical_level(50000)
csv_data = self.ncss.get_data(self.nq)
np.testing.assert_almost_equal(csv_data['Temperature_isobaric'], np.array([263.40]), 2)
@recorder.use_cassette('ncss_gfs_csv_point')
def test_raw_csv(self):
"""Test CSV point return from a GFS request."""
self.nq.accept('csv')
csv_data = self.ncss.get_data_raw(self.nq)
assert csv_data.startswith(b'date,lat')
@recorder.use_cassette('ncss_gfs_csv_point')
def test_unknown_mime(self):
"""Test handling of unknown mimetypes."""
self.nq.accept('csv')
with response_context():
csv_data = self.ncss.get_data(self.nq)
assert csv_data.startswith(b'date,lat')
def get_ensemble_point(point,
variables=['Temperature_height_above_ground_ens'],
start=datetime.utcnow() - timedelta(hours=12),
end=datetime.utcnow() + timedelta(hours=48)):
"""
Retrieves the latest ("best") ensemble forecast valid at a single point from the Unidata THREDDS server using
the Unidata siphon library.
Requires:
point -> A tuple of (lat, lon) of the point we are trying to retrieve
variables -> A list of variables we want to retrieve. Check this page for a full list:
http://thredds.ucar.edu/thredds/metadata/grib/NCEP/GEFS/Global_1p0deg_Ensemble/members/Best?metadata=variableMap
start -> A datetime object of the earliest time to look for an ensemble initialization,
default is current time minus 12 hours
end -> The last time for which we want ensemble forecast output. Default is current time plus 48 hours.
Returns:
A dictionary with one item being the list of valid times in the data ('times') and the rest of the items
being numpy arrays of nTimes x nEnsmems for each variable requested
"""
# Import the Siphon utilities
from siphon.catalog import TDSCatalog
from siphon.ncss import NCSS
# In Siphon, we connect to a thredds catalog. Here's the address for the GEFS
catalog = 'http://thredds.ucar.edu/thredds/catalog/grib/NCEP/GEFS/Global_1p0deg_Ensemble/members/catalog.xml'
best_model = TDSCatalog(catalog)
# We select a specific dataset in this catalog, in this case the "best" (most recent) ensemble run
best_ds = list(best_model.datasets.values())[2]
ncss = NCSS(best_ds.access_urls['NetcdfSubset'])
# Here we format our subsetting query. We specify the exact point we want,
# the time range, and the variables we are requesting. We're also going
# to retrieve the data in a netcdf-like format
query = ncss.query()
query.lonlat_point(point[1], point[0])
query.time_range(start, end)
query.variables(*variables)
query.accept('netcdf')
# Actually get the data
data = ncss.get_data(query)
# Format our output into a dictionary
output = {}
for v in variables:
# After the squeeze, this is a nTimes x nEns array
output[v] = np.squeeze(data.variables[v][:])
#print output[v].shape
# Also, add times
# The 'time' variable is hours since "time_coverage_start"
# Get this in datetime format
raw_hours = list(np.squeeze(data.variables['time'][:]))
init_time = datetime.strptime(str(data.time_coverage_start),
'%Y-%m-%dT%H:%M:%SZ')
output['times'] = [init_time + timedelta(hours=int(x)) for x in raw_hours]
# Return a dictionary
return output
class TestNCSS(object):
"""Test NCSS queries and response parsing."""
server = 'http://thredds.ucar.edu/thredds/ncss/'
urlPath = 'grib/NCEP/GFS/Global_0p5deg/GFS_Global_0p5deg_20150612_1200.grib2'
@recorder.use_cassette('ncss_test_metadata')
def setup(self):
"""Set up for tests with a default valid query."""
dt = datetime(2015, 6, 12, 15, 0, 0)
self.ncss = NCSS(self.server + self.urlPath)
self.nq = self.ncss.query().lonlat_point(-105, 40).time(dt)
self.nq.variables('Temperature_isobaric', 'Relative_humidity_isobaric')
def test_good_query(self):
"""Test that a good query is properly validated."""
assert self.ncss.validate_query(self.nq)
def test_bad_query(self):
"""Test that a query with an unknown variable is invalid."""
self.nq.variables('foo')
assert not self.ncss.validate_query(self.nq)
def test_empty_query(self):
"""Test that an empty query is invalid."""
query = self.ncss.query()
res = self.ncss.validate_query(query)
assert not res
assert not isinstance(res, set)
def test_bad_query_no_vars(self):
"""Test that a query without variables is invalid."""
self.nq.var.clear()
assert not self.ncss.validate_query(self.nq)
@recorder.use_cassette('ncss_gfs_xml_point')
def test_xml_point(self):
"""Test parsing XML point returns."""
self.nq.accept('xml')
xml_data = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in xml_data
assert 'Relative_humidity_isobaric' in xml_data
assert xml_data['lat'][0] == 40
assert xml_data['lon'][0] == -105
@recorder.use_cassette('ncss_gfs_csv_point')
def test_csv_point(self):
"""Test parsing CSV point returns."""
self.nq.accept('csv')
csv_data = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in csv_data
assert 'Relative_humidity_isobaric' in csv_data
assert csv_data['lat'][0] == 40
assert csv_data['lon'][0] == -105
@recorder.use_cassette('ncss_gfs_csv_point')
def test_unit_handler_csv(self):
"""Test unit-handling from CSV returns."""
self.nq.accept('csv')
self.ncss.unit_handler = tuple_unit_handler
csv_data = self.ncss.get_data(self.nq)
temp = csv_data['Temperature_isobaric']
assert len(temp) == 2
assert temp[1] == 'K'
relh = csv_data['Relative_humidity_isobaric']
assert len(relh) == 2
assert relh[1] == '%'
@recorder.use_cassette('ncss_gfs_xml_point')
def test_unit_handler_xml(self):
"""Test unit-handling from XML returns."""
self.nq.accept('xml')
self.ncss.unit_handler = tuple_unit_handler
xml_data = self.ncss.get_data(self.nq)
temp = xml_data['Temperature_isobaric']
assert len(temp) == 2
assert temp[1] == 'K'
relh = xml_data['Relative_humidity_isobaric']
assert len(relh) == 2
assert relh[1] == '%'
@recorder.use_cassette('ncss_gfs_netcdf_point')
def test_netcdf_point(self):
"""Test handling of netCDF point returns."""
self.nq.accept('netcdf')
nc = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in nc.variables
assert 'Relative_humidity_isobaric' in nc.variables
assert nc.variables['latitude'][0] == 40
assert nc.variables['longitude'][0] == -105
@recorder.use_cassette('ncss_gfs_netcdf4_point')
def test_netcdf4_point(self):
"""Test handling of netCDF4 point returns."""
self.nq.accept('netcdf4')
nc = self.ncss.get_data(self.nq)
assert 'Temperature_isobaric' in nc.variables
assert 'Relative_humidity_isobaric' in nc.variables
assert nc.variables['latitude'][0] == 40
assert nc.variables['longitude'][0] == -105
@recorder.use_cassette('ncss_gfs_vertical_level')
def test_vertical_level(self):
"""Test data return from a single vertical level is correct."""
self.nq.accept('csv').vertical_level(50000)
csv_data = self.ncss.get_data(self.nq)
np.testing.assert_almost_equal(csv_data['Temperature_isobaric'],
np.array([263.40]), 2)
@recorder.use_cassette('ncss_gfs_csv_point')
def test_raw_csv(self):
"""Test CSV point return from a GFS request."""
self.nq.accept('csv')
csv_data = self.ncss.get_data_raw(self.nq)
assert csv_data.startswith(b'date,lat')
@recorder.use_cassette('ncss_gfs_csv_point')
def test_unknown_mime(self):
"""Test handling of unknown mimetypes."""
self.nq.accept('csv')
with response_context():
csv_data = self.ncss.get_data(self.nq)
assert csv_data.startswith(b'date,lat')
class ForecastModel(object):
"""
An object for querying and holding forecast model information for
use within the pvlib library.
Simplifies use of siphon library on a THREDDS server.
Parameters
----------
model_type: string
UNIDATA category in which the model is located.
model_name: string
Name of the UNIDATA forecast model.
set_type: string
Model dataset type.
Attributes
----------
access_url: string
URL specifying the dataset from data will be retrieved.
base_tds_url : string
The top level server address
catalog_url : string
The url path of the catalog to parse.
data: pd.DataFrame
Data returned from the query.
data_format: string
Format of the forecast data being requested from UNIDATA.
dataset: Dataset
Object containing information used to access forecast data.
dataframe_variables: list
Model variables that are present in the data.
datasets_list: list
List of all available datasets.
fm_models: Dataset
TDSCatalog object containing all available
forecast models from UNIDATA.
fm_models_list: list
List of all available forecast models from UNIDATA.
latitude: list
A list of floats containing latitude values.
location: Location
A pvlib Location object containing geographic quantities.
longitude: list
A list of floats containing longitude values.
lbox: boolean
Indicates the use of a location bounding box.
ncss: NCSS object
NCSS
model_name: string
Name of the UNIDATA forecast model.
model: Dataset
A dictionary of Dataset object, whose keys are the name of the
dataset's name.
model_url: string
The url path of the dataset to parse.
modelvariables: list
Common variable names that correspond to queryvariables.
query: NCSS query object
NCSS object used to complete the forecast data retrival.
queryvariables: list
Variables that are used to query the THREDDS Data Server.
time: DatetimeIndex
Time range.
variables: dict
Defines the variables to obtain from the weather
model and how they should be renamed to common variable names.
units: dict
Dictionary containing the units of the standard variables
and the model specific variables.
vert_level: float or integer
Vertical altitude for query data.
"""
access_url_key = 'NetcdfSubset'
catalog_url = 'http://thredds.ucar.edu/thredds/catalog.xml'
base_tds_url = catalog_url.split('/thredds/')[0]
data_format = 'netcdf'
vert_level = 100000
units = {
'temp_air': 'C',
'wind_speed': 'm/s',
'ghi': 'W/m^2',
'ghi_raw': 'W/m^2',
'dni': 'W/m^2',
'dhi': 'W/m^2',
'total_clouds': '%',
'low_clouds': '%',
'mid_clouds': '%',
'high_clouds': '%'}
def __init__(self, model_type, model_name, set_type):
self.model_type = model_type
self.model_name = model_name
self.set_type = set_type
self.catalog = TDSCatalog(self.catalog_url)
self.fm_models = TDSCatalog(self.catalog.catalog_refs[model_type].href)
self.fm_models_list = sorted(list(self.fm_models.catalog_refs.keys()))
try:
model_url = self.fm_models.catalog_refs[model_name].href
except ParseError:
raise ParseError(self.model_name + ' model may be unavailable.')
try:
self.model = TDSCatalog(model_url)
except HTTPError:
try:
self.model = TDSCatalog(model_url)
except HTTPError:
raise HTTPError(self.model_name + ' model may be unavailable.')
self.datasets_list = list(self.model.datasets.keys())
self.set_dataset()
def __repr__(self):
return '{}, {}'.format(self.model_name, self.set_type)
def set_dataset(self):
'''
Retrieves the designated dataset, creates NCSS object, and
creates a NCSS query object.
'''
keys = list(self.model.datasets.keys())
labels = [item.split()[0].lower() for item in keys]
if self.set_type == 'best':
self.dataset = self.model.datasets[keys[labels.index('best')]]
elif self.set_type == 'latest':
self.dataset = self.model.datasets[keys[labels.index('latest')]]
elif self.set_type == 'full':
self.dataset = self.model.datasets[keys[labels.index('full')]]
self.access_url = self.dataset.access_urls[self.access_url_key]
self.ncss = NCSS(self.access_url)
self.query = self.ncss.query()
def set_query_latlon(self):
'''
Sets the NCSS query location latitude and longitude.
'''
if (isinstance(self.longitude, list) and
isinstance(self.latitude, list)):
self.lbox = True
# west, east, south, north
self.query.lonlat_box(self.latitude[0], self.latitude[1],
self.longitude[0], self.longitude[1])
else:
self.lbox = False
self.query.lonlat_point(self.longitude, self.latitude)
def set_location(self, time, latitude, longitude):
'''
Sets the location for the query.
Parameters
----------
time: datetime or DatetimeIndex
Time range of the query.
'''
if isinstance(time, datetime.datetime):
tzinfo = time.tzinfo
else:
tzinfo = time.tz
if tzinfo is None:
self.location = Location(latitude, longitude)
else:
self.location = Location(latitude, longitude, tz=tzinfo)
def get_data(self, latitude, longitude, start, end,
vert_level=None, query_variables=None,
close_netcdf_data=True):
"""
Submits a query to the UNIDATA servers using Siphon NCSS and
converts the netcdf data to a pandas DataFrame.
Parameters
----------
latitude: float
The latitude value.
longitude: float
The longitude value.
start: datetime or timestamp
The start time.
end: datetime or timestamp
The end time.
vert_level: None, float or integer
Vertical altitude of interest.
variables: None or list
If None, uses self.variables.
close_netcdf_data: bool
Controls if the temporary netcdf data file should be closed.
Set to False to access the raw data.
Returns
-------
forecast_data : DataFrame
column names are the weather model's variable names.
"""
if vert_level is not None:
self.vert_level = vert_level
if query_variables is None:
self.query_variables = list(self.variables.values())
else:
self.query_variables = query_variables
self.latitude = latitude
self.longitude = longitude
self.set_query_latlon() # modifies self.query
self.set_location(start, latitude, longitude)
self.start = start
self.end = end
self.query.time_range(self.start, self.end)
self.query.vertical_level(self.vert_level)
self.query.variables(*self.query_variables)
self.query.accept(self.data_format)
self.netcdf_data = self.ncss.get_data(self.query)
# might be better to go to xarray here so that we can handle
# higher dimensional data for more advanced applications
self.data = self._netcdf2pandas(self.netcdf_data, self.query_variables)
if close_netcdf_data:
self.netcdf_data.close()
return self.data
def process_data(self, data, **kwargs):
"""
Defines the steps needed to convert raw forecast data
into processed forecast data. Most forecast models implement
their own version of this method which also call this one.
Parameters
----------
data: DataFrame
Raw forecast data
Returns
-------
data: DataFrame
Processed forecast data.
"""
data = self.rename(data)
return data
def get_processed_data(self, *args, **kwargs):
"""
Get and process forecast data.
Parameters
----------
*args: positional arguments
Passed to get_data
**kwargs: keyword arguments
Passed to get_data and process_data
Returns
-------
data: DataFrame
Processed forecast data
"""
return self.process_data(self.get_data(*args, **kwargs), **kwargs)
def rename(self, data, variables=None):
"""
Renames the columns according the variable mapping.
Parameters
----------
data: DataFrame
variables: None or dict
If None, uses self.variables
Returns
-------
data: DataFrame
Renamed data.
"""
if variables is None:
variables = self.variables
return data.rename(columns={y: x for x, y in variables.items()})
def _netcdf2pandas(self, netcdf_data, query_variables):
"""
Transforms data from netcdf to pandas DataFrame.
Parameters
----------
data: netcdf
Data returned from UNIDATA NCSS query.
query_variables: list
The variables requested.
Returns
-------
pd.DataFrame
"""
# set self.time
try:
time_var = 'time'
self.set_time(netcdf_data.variables[time_var])
except KeyError:
# which model does this dumb thing?
time_var = 'time1'
self.set_time(netcdf_data.variables[time_var])
data_dict = {key: data[:].squeeze() for key, data in
netcdf_data.variables.items() if key in query_variables}
return pd.DataFrame(data_dict, index=self.time)
def set_time(self, time):
'''
Converts time data into a pandas date object.
Parameters
----------
time: netcdf
Contains time information.
Returns
-------
pandas.DatetimeIndex
'''
times = num2date(time[:].squeeze(), time.units)
self.time = pd.DatetimeIndex(pd.Series(times), tz=self.location.tz)
def cloud_cover_to_ghi_linear(self, cloud_cover, ghi_clear, offset=35,
**kwargs):
"""
Convert cloud cover to GHI using a linear relationship.
0% cloud cover returns ghi_clear.
100% cloud cover returns offset*ghi_clear.
Parameters
----------
cloud_cover: numeric
Cloud cover in %.
ghi_clear: numeric
GHI under clear sky conditions.
offset: numeric
Determines the minimum GHI.
kwargs
Not used.
Returns
-------
ghi: numeric
Estimated GHI.
References
----------
Larson et. al. "Day-ahead forecasting of solar power output from
photovoltaic plants in the American Southwest" Renewable Energy
91, 11-20 (2016).
"""
offset = offset / 100.
cloud_cover = cloud_cover / 100.
ghi = (offset + (1 - offset) * (1 - cloud_cover)) * ghi_clear
return ghi
def cloud_cover_to_irradiance_clearsky_scaling(self, cloud_cover,
method='linear',
**kwargs):
"""
Estimates irradiance from cloud cover in the following steps:
1. Determine clear sky GHI using Ineichen model and
climatological turbidity.
2. Estimate cloudy sky GHI using a function of
cloud_cover e.g.
:py:meth:`~ForecastModel.cloud_cover_to_ghi_linear`
3. Estimate cloudy sky DNI using the DISC model.
4. Calculate DHI from DNI and DHI.
Parameters
----------
cloud_cover : Series
Cloud cover in %.
method : str
Method for converting cloud cover to GHI.
'linear' is currently the only option.
**kwargs
Passed to the method that does the conversion
Returns
-------
irrads : DataFrame
Estimated GHI, DNI, and DHI.
"""
solpos = self.location.get_solarposition(cloud_cover.index)
cs = self.location.get_clearsky(cloud_cover.index, model='ineichen',
solar_position=solpos)
method = method.lower()
if method == 'linear':
ghi = self.cloud_cover_to_ghi_linear(cloud_cover, cs['ghi'],
**kwargs)
else:
raise ValueError('invalid method argument')
dni = disc(ghi, solpos['zenith'], cloud_cover.index)['dni']
dhi = ghi - dni * np.cos(np.radians(solpos['zenith']))
irrads = pd.DataFrame({'ghi': ghi, 'dni': dni, 'dhi': dhi}).fillna(0)
return irrads
def cloud_cover_to_transmittance_linear(self, cloud_cover, offset=0.75,
**kwargs):
"""
Convert cloud cover to atmospheric transmittance using a linear
model.
0% cloud cover returns offset.
100% cloud cover returns 0.
Parameters
----------
cloud_cover : numeric
Cloud cover in %.
offset : numeric
Determines the maximum transmittance.
kwargs
Not used.
Returns
-------
ghi : numeric
Estimated GHI.
"""
transmittance = ((100.0 - cloud_cover) / 100.0) * 0.75
return transmittance
def cloud_cover_to_irradiance_liujordan(self, cloud_cover, **kwargs):
"""
Estimates irradiance from cloud cover in the following steps:
1. Determine transmittance using a function of cloud cover e.g.
:py:meth:`~ForecastModel.cloud_cover_to_transmittance_linear`
2. Calculate GHI, DNI, DHI using the
:py:func:`pvlib.irradiance.liujordan` model
Parameters
----------
cloud_cover : Series
Returns
-------
irradiance : DataFrame
Columns include ghi, dni, dhi
"""
# in principle, get_solarposition could use the forecast
# pressure, temp, etc., but the cloud cover forecast is not
# accurate enough to justify using these minor corrections
solar_position = self.location.get_solarposition(cloud_cover.index)
dni_extra = extraradiation(cloud_cover.index)
airmass = self.location.get_airmass(cloud_cover.index)
transmittance = self.cloud_cover_to_transmittance_linear(cloud_cover,
**kwargs)
irrads = liujordan(solar_position['apparent_zenith'],
transmittance, airmass['airmass_absolute'],
dni_extra=dni_extra)
irrads = irrads.fillna(0)
return irrads
def cloud_cover_to_irradiance(self, cloud_cover, how='clearsky_scaling',
**kwargs):
"""
Convert cloud cover to irradiance. A wrapper method.
Parameters
----------
cloud_cover : Series
how : str
Selects the method for conversion. Can be one of
clearsky_scaling or liujordan.
**kwargs
Passed to the selected method.
Returns
-------
irradiance : DataFrame
Columns include ghi, dni, dhi
"""
how = how.lower()
if how == 'clearsky_scaling':
irrads = self.cloud_cover_to_irradiance_clearsky_scaling(
cloud_cover, **kwargs)
elif how == 'liujordan':
irrads = self.cloud_cover_to_irradiance_liujordan(
cloud_cover, **kwargs)
else:
raise ValueError('invalid how argument')
return irrads
def kelvin_to_celsius(self, temperature):
"""
Converts Kelvin to celsius.
Parameters
----------
temperature: numeric
Returns
-------
temperature: numeric
"""
return temperature - 273.15
def isobaric_to_ambient_temperature(self, data):
"""
Calculates temperature from isobaric temperature.
Parameters
----------
data: DataFrame
Must contain columns pressure, temperature_iso,
temperature_dew_iso. Input temperature in K.
Returns
-------
temperature : Series
Temperature in K
"""
P = data['pressure'] / 100.0
Tiso = data['temperature_iso']
Td = data['temperature_dew_iso'] - 273.15
# saturation water vapor pressure
e = 6.11 * 10**((7.5 * Td) / (Td + 273.3))
# saturation water vapor mixing ratio
w = 0.622 * (e / (P - e))
T = Tiso - ((2.501 * 10.**6) / 1005.7) * w
return T
def uv_to_speed(self, data):
"""
Computes wind speed from wind components.
Parameters
----------
data : DataFrame
Must contain the columns 'wind_speed_u' and 'wind_speed_v'.
Returns
-------
wind_speed : Series
"""
wind_speed = np.sqrt(data['wind_speed_u']**2 + data['wind_speed_v']**2)
return wind_speed
def gust_to_speed(self, data, scaling=1/1.4):
"""
Computes standard wind speed from gust.
Very approximate and location dependent.
Parameters
----------
data : DataFrame
Must contain the column 'wind_speed_gust'.
Returns
-------
wind_speed : Series
"""
wind_speed = data['wind_speed_gust'] * scaling
return wind_speed
class ForecastModel(object):
'''
An object for holding forecast model information for use within the
pvlib library.
Simplifies use of siphon library on a THREDDS server.
Parameters
----------
model_type: string
UNIDATA category in which the model is located.
model_name: string
Name of the UNIDATA forecast model.
set_type: string
Model dataset type.
Attributes
----------
access_url: string
URL specifying the dataset from data will be retrieved.
base_tds_url : string
The top level server address
catalog_url : string
The url path of the catalog to parse.
columns: list
List of headers used to create the data DataFrame.
data: pd.DataFrame
Data returned from the query.
data_format: string
Format of the forecast data being requested from UNIDATA.
dataset: Dataset
Object containing information used to access forecast data.
dataframe_variables: list
Model variables that are present in the data.
datasets_list: list
List of all available datasets.
fm_models: Dataset
Object containing all available foreast models.
fm_models_list: list
List of all available forecast models from UNIDATA.
latitude: list
A list of floats containing latitude values.
location: Location
A pvlib Location object containing geographic quantities.
longitude: list
A list of floats containing longitude values.
lbox: boolean
Indicates the use of a location bounding box.
ncss: NCSS object
NCSS model_name: string
Name of the UNIDATA forecast model.
model: Dataset
A dictionary of Dataset object, whose keys are the name of the
dataset's name.
model_url: string
The url path of the dataset to parse.
modelvariables: list
Common variable names that correspond to queryvariables.
query: NCSS query object
NCSS object used to complete the forecast data retrival.
queryvariables: list
Variables that are used to query the THREDDS Data Server.
rad_type: dictionary
Dictionary labeling the method used for calculating radiation values.
time: datetime
Time range specified for the NCSS query.
utctime: DatetimeIndex
Time range in UTC.
var_stdnames: dictionary
Dictionary containing the standard names of the variables in the
query, where the keys are the common names.
var_units: dictionary
Dictionary containing the unites of the variables in the query,
where the keys are the common names.
variables: dictionary
Dictionary that translates model specific variables to
common named variables.
vert_level: float or integer
Vertical altitude for query data.
wind_type: string
Quantity that was used to calculate wind_speed.
zenith: numpy.array
Solar zenith angles for the given time range.
'''
access_url_key = 'NetcdfSubset'
catalog_url = 'http://thredds.ucar.edu/thredds/catalog.xml'
base_tds_url = catalog_url.split('/thredds/')[0]
data_format = 'netcdf'
vert_level = 100000
columns = np.array([
'temperature',
'wind_speed',
'total_clouds',
'low_clouds',
'mid_clouds',
'high_clouds',
'dni',
'dhi',
'ghi',
])
def __init__(self, model_type, model_name, set_type):
self.model_type = model_type
self.model_name = model_name
self.set_type = set_type
self.catalog = TDSCatalog(self.catalog_url)
self.fm_models = TDSCatalog(self.catalog.catalog_refs[model_type].href)
self.fm_models_list = sorted(list(self.fm_models.catalog_refs.keys()))
try:
model_url = self.fm_models.catalog_refs[model_name].href
except ParseError:
raise ParseError(self.model_name + ' model may be unavailable.')
try:
self.model = TDSCatalog(model_url)
except HTTPError:
raise HTTPError(self.model_name + ' model may be unavailable.')
self.datasets_list = list(self.model.datasets.keys())
self.set_dataset()
def set_dataset(self):
'''
Retreives the designated dataset, creates NCSS object, and
initiates a NCSS query.
'''
keys = list(self.model.datasets.keys())
labels = [item.split()[0].lower() for item in keys]
if self.set_type == 'best':
self.dataset = self.model.datasets[keys[labels.index('best')]]
elif self.set_type == 'latest':
self.dataset = self.model.datasets[keys[labels.index('latest')]]
elif self.set_type == 'full':
self.dataset = self.model.datasets[keys[labels.index('full')]]
self.access_url = self.dataset.access_urls[self.access_url_key]
self.ncss = NCSS(self.access_url)
self.query = self.ncss.query()
def set_query_latlon(self):
'''
Sets the NCSS query location latitude and longitude.
'''
if isinstance(self.longitude, list):
self.lbox = True
# west, east, south, north
self.query.lonlat_box(self.latitude[0], self.latitude[1],
self.longitude[0], self.longitude[1])
else:
self.lbox = False
self.query.lonlat_point(self.longitude, self.latitude)
def set_query_time(self):
'''
Sets the NCSS query time range.
as: single or range
'''
if len(self.utctime) == 1:
self.query.time(pd.to_datetime(self.utctime)[0])
else:
self.query.time_range(
pd.to_datetime(self.utctime)[0],
pd.to_datetime(self.utctime)[-1])
def set_location(self, time):
'''
Sets the location for
Parameters
----------
time: datetime or DatetimeIndex
Time range of the query.
'''
if isinstance(time, datetime.datetime):
tzinfo = time.tzinfo
else:
tzinfo = time.tz
if tzinfo is None:
self.location = Location(self.latitude, self.longitude)
else:
self.location = Location(self.latitude, self.longitude, tz=tzinfo)
def get_query_data(self,
latitude,
longitude,
time,
vert_level=None,
variables=None):
'''
Submits a query to the UNIDATA servers using siphon NCSS and
converts the netcdf data to a pandas DataFrame.
Parameters
----------
latitude: list
A list of floats containing latitude values.
longitude: list
A list of floats containing longitude values.
time: pd.datetimeindex
Time range of interest.
vert_level: float or integer
Vertical altitude of interest.
variables: dictionary
Variables and common names being queried.
Returns
-------
pd.DataFrame
'''
if vert_level is not None:
self.vert_level = vert_level
if variables is not None:
self.variables = variables
self.modelvariables = list(self.variables.keys())
self.queryvariables = [self.variables[key] for key in \
self.modelvariables]
self.columns = self.modelvariables
self.dataframe_variables = self.modelvariables
self.latitude = latitude
self.longitude = longitude
self.set_query_latlon()
self.set_location(time)
self.utctime = localize_to_utc(time, self.location)
self.set_query_time()
self.query.vertical_level(self.vert_level)
self.query.variables(*self.queryvariables)
self.query.accept(self.data_format)
netcdf_data = self.ncss.get_data(self.query)
try:
time_var = 'time'
self.set_time(netcdf_data.variables[time_var])
except KeyError:
time_var = 'time1'
self.set_time(netcdf_data.variables[time_var])
self.data = self.netcdf2pandas(netcdf_data)
self.set_variable_units(netcdf_data)
self.set_variable_stdnames(netcdf_data)
if self.__class__.__name__ is 'HRRR':
self.calc_temperature(netcdf_data)
self.convert_temperature()
self.calc_wind(netcdf_data)
self.calc_radiation(netcdf_data)
self.data = self.data.tz_convert(self.location.tz)
netcdf_data.close()
return self.data
def netcdf2pandas(self, data):
'''
Transforms data from netcdf to pandas DataFrame.
Currently only supports one-dimensional netcdf data.
Parameters
----------
data: netcdf
Data returned from UNIDATA NCSS query.
Returns
-------
pd.DataFrame
'''
if not self.lbox:
''' one-dimensional data '''
data_dict = {}
for var in self.dataframe_variables:
data_dict[var] = pd.Series(
data[self.variables[var]][:].squeeze(), index=self.utctime)
return pd.DataFrame(data_dict, columns=self.columns)
else:
return pd.DataFrame(columns=self.columns, index=self.utctime)
def set_time(self, time):
'''
Converts time data into a pandas date object.
Parameters
----------
time: netcdf
Contains time information.
Returns
-------
pandas.DatetimeIndex
'''
times = num2date(time[:].squeeze(), time.units)
self.time = pd.DatetimeIndex(pd.Series(times), tz='UTC')
self.time = self.time.tz_convert(self.location.tz)
self.utctime = localize_to_utc(self.time, self.location.tz)
def set_variable_units(self, data):
'''
Extracts variable unit information from netcdf data.
Parameters
----------
data: netcdf
Contains queried variable information.
'''
self.var_units = {}
for var in self.variables:
self.var_units[var] = data[self.variables[var]].units
def set_variable_stdnames(self, data):
'''
Extracts standard names from netcdf data.
Parameters
----------
data: netcdf
Contains queried variable information.
'''
self.var_stdnames = {}
for var in self.variables:
try:
self.var_stdnames[var] = \
data[self.variables[var]].standard_name
except AttributeError:
self.var_stdnames[var] = var
def calc_radiation(self, data, cloud_type='total_clouds'):
'''
Determines shortwave radiation values if they are missing from
the model data.
Parameters
----------
data: netcdf
Query data formatted in netcdf format.
cloud_type: string
Type of cloud cover to use for calculating radiation values.
'''
self.rad_type = {}
if not self.lbox and cloud_type in self.modelvariables:
cloud_prct = self.data[cloud_type]
solpos = get_solarposition(self.time, self.location)
self.zenith = np.array(solpos.zenith.tz_convert('UTC'))
for rad in ['dni', 'dhi', 'ghi']:
if self.model_name is 'HRRR_ESRL':
# HRRR_ESRL is the only model with the
# correct equation of time.
if rad in self.modelvariables:
self.data[rad] = pd.Series(
data[self.variables[rad]][:].squeeze(),
index=self.time)
self.rad_type[rad] = 'forecast'
self.data[rad].fillna(0, inplace=True)
else:
for rad in ['dni', 'dhi', 'ghi']:
self.rad_type[rad] = 'liujordan'
self.data[rad] = liujordan(self.zenith,
cloud_prct)[rad]
self.data[rad].fillna(0, inplace=True)
for var in ['dni', 'dhi', 'ghi']:
self.data[var].fillna(0, inplace=True)
self.var_units[var] = '$W m^{-2}$'
def convert_temperature(self):
'''
Converts Kelvin to celsius.
'''
if 'Temperature_surface' in self.queryvariables or 'Temperature_isobaric' in self.queryvariables:
self.data['temperature'] -= 273.15
self.var_units['temperature'] = 'C'
def calc_temperature(self, data):
'''
Calculates temperature (in degrees C) from isobaric temperature.
Parameters
----------
data: netcdf
Query data in netcdf format.
'''
P = data['Pressure_surface'][:].squeeze() / 100.0
Tiso = data['Temperature_isobaric'][:].squeeze()
Td = data['Dewpoint_temperature_isobaric'][:].squeeze() - 273.15
e = 6.11 * 10**((7.5 * Td) / (Td + 273.3))
w = 0.622 * (e / (P - e))
T = Tiso - ((2.501 * 10.**6) / 1005.7) * w
self.data['temperature'] = T
def calc_wind(self, data):
'''
Computes wind speed.
In some cases only gust wind speed is available. The wind_type
attribute will indicate the type of wind speed that is present.
Parameters
----------
data: netcdf
Query data in netcdf format.
'''
if not self.lbox:
if 'u-component_of_wind_isobaric' in self.queryvariables and \
'v-component_of_wind_isobaric' in self.queryvariables:
wind_data = np.sqrt(\
data['u-component_of_wind_isobaric'][:].squeeze()**2 +
data['v-component_of_wind_isobaric'][:].squeeze()**2)
self.wind_type = 'component'
elif 'Wind_speed_gust_surface' in self.queryvariables:
wind_data = data['Wind_speed_gust_surface'][:].squeeze()
self.wind_type = 'gust'
if 'wind_speed' in self.data:
self.data['wind_speed'] = pd.Series(wind_data, index=self.time)
self.var_units['wind_speed'] = 'm/s'
class ForecastModel(object):
'''
An object for holding forecast model information for use within the
pvlib library.
Simplifies use of siphon library on a THREDDS server.
Parameters
----------
model_type: string
UNIDATA category in which the model is located.
model_name: string
Name of the UNIDATA forecast model.
set_type: string
Model dataset type.
Attributes
----------
access_url: string
URL specifying the dataset from data will be retrieved.
base_tds_url : string
The top level server address
catalog_url : string
The url path of the catalog to parse.
columns: list
List of headers used to create the data DataFrame.
data: pd.DataFrame
Data returned from the query.
data_format: string
Format of the forecast data being requested from UNIDATA.
dataset: Dataset
Object containing information used to access forecast data.
dataframe_variables: list
Model variables that are present in the data.
datasets_list: list
List of all available datasets.
fm_models: Dataset
Object containing all available foreast models.
fm_models_list: list
List of all available forecast models from UNIDATA.
latitude: list
A list of floats containing latitude values.
location: Location
A pvlib Location object containing geographic quantities.
longitude: list
A list of floats containing longitude values.
lbox: boolean
Indicates the use of a location bounding box.
ncss: NCSS object
NCSS model_name: string
Name of the UNIDATA forecast model.
model: Dataset
A dictionary of Dataset object, whose keys are the name of the
dataset's name.
model_url: string
The url path of the dataset to parse.
modelvariables: list
Common variable names that correspond to queryvariables.
query: NCSS query object
NCSS object used to complete the forecast data retrival.
queryvariables: list
Variables that are used to query the THREDDS Data Server.
rad_type: dictionary
Dictionary labeling the method used for calculating radiation values.
time: datetime
Time range specified for the NCSS query.
utctime: DatetimeIndex
Time range in UTC.
var_stdnames: dictionary
Dictionary containing the standard names of the variables in the
query, where the keys are the common names.
var_units: dictionary
Dictionary containing the unites of the variables in the query,
where the keys are the common names.
variables: dictionary
Dictionary that translates model specific variables to
common named variables.
vert_level: float or integer
Vertical altitude for query data.
wind_type: string
Quantity that was used to calculate wind_speed.
zenith: numpy.array
Solar zenith angles for the given time range.
'''
access_url_key = 'NetcdfSubset'
catalog_url = 'http://thredds.ucar.edu/thredds/catalog.xml'
base_tds_url = catalog_url.split('/thredds/')[0]
data_format = 'netcdf'
vert_level = 100000
columns = np.array(['temperature',
'wind_speed',
'total_clouds',
'low_clouds',
'mid_clouds',
'high_clouds',
'dni',
'dhi',
'ghi', ])
def __init__(self, model_type, model_name, set_type):
self.model_type = model_type
self.model_name = model_name
self.set_type = set_type
self.catalog = TDSCatalog(self.catalog_url)
self.fm_models = TDSCatalog(self.catalog.catalog_refs[model_type].href)
self.fm_models_list = sorted(list(self.fm_models.catalog_refs.keys()))
try:
model_url = self.fm_models.catalog_refs[model_name].href
except ParseError:
raise ParseError(self.model_name + ' model may be unavailable.')
try:
self.model = TDSCatalog(model_url)
except HTTPError:
raise HTTPError(self.model_name + ' model may be unavailable.')
self.datasets_list = list(self.model.datasets.keys())
self.set_dataset()
def set_dataset(self):
'''
Retreives the designated dataset, creates NCSS object, and
initiates a NCSS query.
'''
keys = list(self.model.datasets.keys())
labels = [item.split()[0].lower() for item in keys]
if self.set_type == 'best':
self.dataset = self.model.datasets[keys[labels.index('best')]]
elif self.set_type == 'latest':
self.dataset = self.model.datasets[keys[labels.index('latest')]]
elif self.set_type == 'full':
self.dataset = self.model.datasets[keys[labels.index('full')]]
self.access_url = self.dataset.access_urls[self.access_url_key]
self.ncss = NCSS(self.access_url)
self.query = self.ncss.query()
def set_query_latlon(self):
'''
Sets the NCSS query location latitude and longitude.
'''
if isinstance(self.longitude, list):
self.lbox = True
# west, east, south, north
self.query.lonlat_box(self.latitude[0], self.latitude[1],
self.longitude[0], self.longitude[1])
else:
self.lbox = False
self.query.lonlat_point(self.longitude, self.latitude)
def set_query_time(self):
'''
Sets the NCSS query time range.
as: single or range
'''
if len(self.utctime) == 1:
self.query.time(pd.to_datetime(self.utctime)[0])
else:
self.query.time_range(pd.to_datetime(self.utctime)[0],
pd.to_datetime(self.utctime)[-1])
def set_location(self, time):
'''
Sets the location for
Parameters
----------
time: datetime or DatetimeIndex
Time range of the query.
'''
if isinstance(time, datetime.datetime):
tzinfo = time.tzinfo
else:
tzinfo = time.tz
if tzinfo is None:
self.location = Location(self.latitude, self.longitude)
else:
self.location = Location(self.latitude, self.longitude, tz=tzinfo)
def get_query_data(self, latitude, longitude, time, vert_level=None,
variables=None):
'''
Submits a query to the UNIDATA servers using siphon NCSS and
converts the netcdf data to a pandas DataFrame.
Parameters
----------
latitude: list
A list of floats containing latitude values.
longitude: list
A list of floats containing longitude values.
time: pd.datetimeindex
Time range of interest.
vert_level: float or integer
Vertical altitude of interest.
variables: dictionary
Variables and common names being queried.
Returns
-------
pd.DataFrame
'''
if vert_level is not None:
self.vert_level = vert_level
if variables is not None:
self.variables = variables
self.modelvariables = list(self.variables.keys())
self.queryvariables = [self.variables[key] for key in \
self.modelvariables]
self.columns = self.modelvariables
self.dataframe_variables = self.modelvariables
self.latitude = latitude
self.longitude = longitude
self.set_query_latlon()
self.set_location(time)
self.utctime = localize_to_utc(time, self.location)
self.set_query_time()
self.query.vertical_level(self.vert_level)
self.query.variables(*self.queryvariables)
self.query.accept(self.data_format)
netcdf_data = self.ncss.get_data(self.query)
try:
time_var = 'time'
self.set_time(netcdf_data.variables[time_var])
except KeyError:
time_var = 'time1'
self.set_time(netcdf_data.variables[time_var])
self.data = self.netcdf2pandas(netcdf_data)
self.set_variable_units(netcdf_data)
self.set_variable_stdnames(netcdf_data)
if self.__class__.__name__ is 'HRRR':
self.calc_temperature(netcdf_data)
self.convert_temperature()
self.calc_wind(netcdf_data)
self.calc_radiation(netcdf_data)
self.data = self.data.tz_convert(self.location.tz)
netcdf_data.close()
return self.data
def netcdf2pandas(self, data):
'''
Transforms data from netcdf to pandas DataFrame.
Currently only supports one-dimensional netcdf data.
Parameters
----------
data: netcdf
Data returned from UNIDATA NCSS query.
Returns
-------
pd.DataFrame
'''
if not self.lbox:
''' one-dimensional data '''
data_dict = {}
for var in self.dataframe_variables:
data_dict[var] = pd.Series(
data[self.variables[var]][:].squeeze(), index=self.utctime)
return pd.DataFrame(data_dict, columns=self.columns)
else:
return pd.DataFrame(columns=self.columns, index=self.utctime)
def set_time(self, time):
'''
Converts time data into a pandas date object.
Parameters
----------
time: netcdf
Contains time information.
Returns
-------
pandas.DatetimeIndex
'''
times = num2date(time[:].squeeze(), time.units)
self.time = pd.DatetimeIndex(pd.Series(times), tz='UTC')
self.time = self.time.tz_convert(self.location.tz)
self.utctime = localize_to_utc(self.time, self.location.tz)
def set_variable_units(self, data):
'''
Extracts variable unit information from netcdf data.
Parameters
----------
data: netcdf
Contains queried variable information.
'''
self.var_units = {}
for var in self.variables:
self.var_units[var] = data[self.variables[var]].units
def set_variable_stdnames(self, data):
'''
Extracts standard names from netcdf data.
Parameters
----------
data: netcdf
Contains queried variable information.
'''
self.var_stdnames = {}
for var in self.variables:
try:
self.var_stdnames[var] = \
data[self.variables[var]].standard_name
except AttributeError:
self.var_stdnames[var] = var
def calc_radiation(self, data, cloud_type='total_clouds'):
'''
Determines shortwave radiation values if they are missing from
the model data.
Parameters
----------
data: netcdf
Query data formatted in netcdf format.
cloud_type: string
Type of cloud cover to use for calculating radiation values.
'''
self.rad_type = {}
if not self.lbox and cloud_type in self.modelvariables:
cloud_prct = self.data[cloud_type]
solpos = get_solarposition(self.time, self.location)
self.zenith = np.array(solpos.zenith.tz_convert('UTC'))
for rad in ['dni','dhi','ghi']:
if self.model_name is 'HRRR_ESRL':
# HRRR_ESRL is the only model with the
# correct equation of time.
if rad in self.modelvariables:
self.data[rad] = pd.Series(
data[self.variables[rad]][:].squeeze(),
index=self.time)
self.rad_type[rad] = 'forecast'
self.data[rad].fillna(0, inplace=True)
else:
for rad in ['dni','dhi','ghi']:
self.rad_type[rad] = 'liujordan'
self.data[rad] = liujordan(self.zenith, cloud_prct)[rad]
self.data[rad].fillna(0, inplace=True)
for var in ['dni', 'dhi', 'ghi']:
self.data[var].fillna(0, inplace=True)
self.var_units[var] = '$W m^{-2}$'
def convert_temperature(self):
'''
Converts Kelvin to celsius.
'''
if 'Temperature_surface' in self.queryvariables or 'Temperature_isobaric' in self.queryvariables:
self.data['temperature'] -= 273.15
self.var_units['temperature'] = 'C'
def calc_temperature(self, data):
'''
Calculates temperature (in degrees C) from isobaric temperature.
Parameters
----------
data: netcdf
Query data in netcdf format.
'''
P = data['Pressure_surface'][:].squeeze() / 100.0
Tiso = data['Temperature_isobaric'][:].squeeze()
Td = data['Dewpoint_temperature_isobaric'][:].squeeze() - 273.15
e = 6.11 * 10**((7.5 * Td) / (Td + 273.3))
w = 0.622 * (e / (P - e))
T = Tiso - ((2.501 * 10.**6) / 1005.7) * w
self.data['temperature'] = T
def calc_wind(self, data):
'''
Computes wind speed.
In some cases only gust wind speed is available. The wind_type
attribute will indicate the type of wind speed that is present.
Parameters
----------
data: netcdf
Query data in netcdf format.
'''
if not self.lbox:
if 'u-component_of_wind_isobaric' in self.queryvariables and \
'v-component_of_wind_isobaric' in self.queryvariables:
wind_data = np.sqrt(\
data['u-component_of_wind_isobaric'][:].squeeze()**2 +
data['v-component_of_wind_isobaric'][:].squeeze()**2)
self.wind_type = 'component'
elif 'Wind_speed_gust_surface' in self.queryvariables:
wind_data = data['Wind_speed_gust_surface'][:].squeeze()
self.wind_type = 'gust'
if 'wind_speed' in self.data:
self.data['wind_speed'] = pd.Series(wind_data, index=self.time)
self.var_units['wind_speed'] = 'm/s'
import scipy.ndimage as ndimage
from siphon.ncss import NCSS
##################################
# Set up netCDF Subset Service link
dt = datetime(2016, 4, 16, 18)
ncss = NCSS(
'http://nomads.ncdc.noaa.gov/thredds/ncss/grid/namanl/'
'{0:%Y%m}/{0:%Y%m%d}/namanl_218_{0:%Y%m%d}_{0:%H}00_000.grb'.format(dt))
# Data Query
hgt = ncss.query().time(dt)
hgt.variables('Geopotential_height', 'u_wind', 'v_wind').add_lonlat()
# Actually getting the data
data = ncss.get_data(hgt)
##################################
# Pull apart the data
# Get dimension names to pull appropriate variables
dtime = data.variables['Geopotential_height'].dimensions[0]
dlev = data.variables['Geopotential_height'].dimensions[1]
dlat = data.variables['Geopotential_height'].dimensions[2]
dlon = data.variables['Geopotential_height'].dimensions[3]
# Get lat and lon data, as well as time data and metadata
lats = data.variables['lat'][:]
lons = data.variables['lon'][:]
lons[lons > 180] = lons[lons > 180] - 360
# Query for required variables
gfsdata = ncss.query().all_times()
gfsdata.variables(
'Geopotential_height_isobaric', 'u-component_of_wind_isobaric',
'v-component_of_wind_isobaric', 'Temperature_isobaric',
'Relative_humidity_isobaric',
'Best_4_layer_lifted_index_layer_between_two_pressure_'
'difference_from_ground_layer', 'Absolute_vorticity_isobaric',
'Pressure_reduced_to_MSL_msl',
'Dew_point_temperature_height_above_ground').add_lonlat()
# Set the lat/lon box for the data to pull in.
gfsdata.lonlat_box(-135, -60, 15, 65)
# Actually getting the data
data = ncss.get_data(gfsdata)
# Assign variable names to collected data
dtime = data.variables['Geopotential_height_isobaric'].dimensions[0]
dlev = data.variables['Geopotential_height_isobaric'].dimensions[1]
lat = data.variables['lat'][:]
lon = data.variables['lon'][:]
lev = units.hPa * data.variables[dlev][:]
times = data.variables[dtime]
vtimes = num2date(times[:], times.units)
temps = data.variables['Temperature_isobaric']
tmp = units.kelvin * temps[0, :]
uwnd = (units.meter /
units.second) * data.variables['u-component_of_wind_isobaric'][0, :]
vwnd = (units.meter /
units.second) * data.variables['v-component_of_wind_isobaric'][0, :]
class ForecastModel(object):
"""
An object for querying and holding forecast model information for
use within the pvlib library.
Simplifies use of siphon library on a THREDDS server.
Parameters
----------
model_type: string
UNIDATA category in which the model is located.
model_name: string
Name of the UNIDATA forecast model.
set_type: string
Model dataset type.
Attributes
----------
access_url: string
URL specifying the dataset from data will be retrieved.
base_tds_url : string
The top level server address
catalog_url : string
The url path of the catalog to parse.
data: pd.DataFrame
Data returned from the query.
data_format: string
Format of the forecast data being requested from UNIDATA.
dataset: Dataset
Object containing information used to access forecast data.
dataframe_variables: list
Model variables that are present in the data.
datasets_list: list
List of all available datasets.
fm_models: Dataset
TDSCatalog object containing all available
forecast models from UNIDATA.
fm_models_list: list
List of all available forecast models from UNIDATA.
latitude: list
A list of floats containing latitude values.
location: Location
A pvlib Location object containing geographic quantities.
longitude: list
A list of floats containing longitude values.
lbox: boolean
Indicates the use of a location bounding box.
ncss: NCSS object
NCSS
model_name: string
Name of the UNIDATA forecast model.
model: Dataset
A dictionary of Dataset object, whose keys are the name of the
dataset's name.
model_url: string
The url path of the dataset to parse.
modelvariables: list
Common variable names that correspond to queryvariables.
query: NCSS query object
NCSS object used to complete the forecast data retrival.
queryvariables: list
Variables that are used to query the THREDDS Data Server.
time: DatetimeIndex
Time range.
variables: dict
Defines the variables to obtain from the weather
model and how they should be renamed to common variable names.
units: dict
Dictionary containing the units of the standard variables
and the model specific variables.
vert_level: float or integer
Vertical altitude for query data.
"""
access_url_key = 'NetcdfSubset'
catalog_url = 'https://thredds.ucar.edu/thredds/catalog.xml'
base_tds_url = catalog_url.split('/thredds/')[0]
data_format = 'netcdf'
units = {
'temp_air': 'C',
'wind_speed': 'm/s',
'ghi': 'W/m^2',
'ghi_raw': 'W/m^2',
'dni': 'W/m^2',
'dhi': 'W/m^2',
'total_clouds': '%',
'low_clouds': '%',
'mid_clouds': '%',
'high_clouds': '%'
}
def __init__(self, model_type, model_name, set_type, vert_level=None):
self.model_type = model_type
self.model_name = model_name
self.set_type = set_type
self.connected = False
self.vert_level = vert_level
def connect_to_catalog(self):
self.catalog = TDSCatalog(self.catalog_url)
self.fm_models = TDSCatalog(
self.catalog.catalog_refs[self.model_type].href)
self.fm_models_list = sorted(list(self.fm_models.catalog_refs.keys()))
try:
model_url = self.fm_models.catalog_refs[self.model_name].href
except ParseError:
raise ParseError(self.model_name + ' model may be unavailable.')
try:
self.model = TDSCatalog(model_url)
except HTTPError:
try:
self.model = TDSCatalog(model_url)
except HTTPError:
raise HTTPError(self.model_name + ' model may be unavailable.')
self.datasets_list = list(self.model.datasets.keys())
self.set_dataset()
self.connected = True
def __repr__(self):
return '{}, {}'.format(self.model_name, self.set_type)
def set_dataset(self):
'''
Retrieves the designated dataset, creates NCSS object, and
creates a NCSS query object.
'''
keys = list(self.model.datasets.keys())
labels = [item.split()[0].lower() for item in keys]
if self.set_type == 'best':
self.dataset = self.model.datasets[keys[labels.index('best')]]
elif self.set_type == 'latest':
self.dataset = self.model.datasets[keys[labels.index('latest')]]
elif self.set_type == 'full':
self.dataset = self.model.datasets[keys[labels.index('full')]]
self.access_url = self.dataset.access_urls[self.access_url_key]
self.ncss = NCSS(self.access_url)
self.query = self.ncss.query()
def set_query_time_range(self, start, end):
"""
Parameters
----------
start : datetime.datetime, pandas.Timestamp
Must be tz-localized.
end : datetime.datetime, pandas.Timestamp
Must be tz-localized.
Notes
-----
Assigns ``self.start``, ``self.end``. Modifies ``self.query``
"""
self.start = pd.Timestamp(start)
self.end = pd.Timestamp(end)
if self.start.tz is None or self.end.tz is None:
raise TypeError('start and end must be tz-localized')
self.query.time_range(self.start, self.end)
def set_query_latlon(self):
'''
Sets the NCSS query location latitude and longitude.
'''
if (isinstance(self.longitude, list)
and isinstance(self.latitude, list)):
self.lbox = True
# west, east, south, north
self.query.lonlat_box(self.longitude[0], self.longitude[1],
self.latitude[0], self.latitude[1])
else:
self.lbox = False
self.query.lonlat_point(self.longitude, self.latitude)
def set_location(self, tz, latitude, longitude):
'''
Sets the location for the query.
Parameters
----------
tz: tzinfo
Timezone of the query
latitude: float
Latitude of the query
longitude: float
Longitude of the query
Notes
-----
Assigns ``self.location``.
'''
self.location = Location(latitude, longitude, tz=tz)
def get_data(self,
latitude,
longitude,
start,
end,
vert_level=None,
query_variables=None,
close_netcdf_data=True,
**kwargs):
"""
Submits a query to the UNIDATA servers using Siphon NCSS and
converts the netcdf data to a pandas DataFrame.
Parameters
----------
latitude: float
The latitude value.
longitude: float
The longitude value.
start: datetime or timestamp
The start time.
end: datetime or timestamp
The end time.
vert_level: None, float or integer, default None
Vertical altitude of interest.
query_variables: None or list, default None
If None, uses self.variables.
close_netcdf_data: bool, default True
Controls if the temporary netcdf data file should be closed.
Set to False to access the raw data.
**kwargs:
Additional keyword arguments are silently ignored.
Returns
-------
forecast_data : DataFrame
column names are the weather model's variable names.
"""
if not self.connected:
self.connect_to_catalog()
if vert_level is not None:
self.vert_level = vert_level
if query_variables is None:
self.query_variables = list(self.variables.values())
else:
self.query_variables = query_variables
self.set_query_time_range(start, end)
self.latitude = latitude
self.longitude = longitude
self.set_query_latlon() # modifies self.query
self.set_location(self.start.tz, latitude, longitude)
if self.vert_level is not None:
self.query.vertical_level(self.vert_level)
self.query.variables(*self.query_variables)
self.query.accept(self.data_format)
self.netcdf_data = self.ncss.get_data(self.query)
# might be better to go to xarray here so that we can handle
# higher dimensional data for more advanced applications
self.data = self._netcdf2pandas(self.netcdf_data, self.query_variables,
self.start, self.end)
if close_netcdf_data:
self.netcdf_data.close()
return self.data
def process_data(self, data, **kwargs):
"""
Defines the steps needed to convert raw forecast data
into processed forecast data. Most forecast models implement
their own version of this method which also call this one.
Parameters
----------
data: DataFrame
Raw forecast data
Returns
-------
data: DataFrame
Processed forecast data.
"""
data = self.rename(data)
return data
def get_processed_data(self, *args, **kwargs):
"""
Get and process forecast data.
Parameters
----------
*args: positional arguments
Passed to get_data
**kwargs: keyword arguments
Passed to get_data and process_data
Returns
-------
data: DataFrame
Processed forecast data
"""
return self.process_data(self.get_data(*args, **kwargs), **kwargs)
def rename(self, data, variables=None):
"""
Renames the columns according the variable mapping.
Parameters
----------
data: DataFrame
variables: None or dict, default None
If None, uses self.variables
Returns
-------
data: DataFrame
Renamed data.
"""
if variables is None:
variables = self.variables
return data.rename(columns={y: x for x, y in variables.items()})
def _netcdf2pandas(self, netcdf_data, query_variables, start, end):
"""
Transforms data from netcdf to pandas DataFrame.
Parameters
----------
data: netcdf
Data returned from UNIDATA NCSS query.
query_variables: list
The variables requested.
start: Timestamp
The start time
end: Timestamp
The end time
Returns
-------
pd.DataFrame
"""
# set self.time
try:
time_var = 'time'
self.set_time(netcdf_data.variables[time_var])
except KeyError:
# which model does this dumb thing?
time_var = 'time1'
self.set_time(netcdf_data.variables[time_var])
data_dict = {}
for key, data in netcdf_data.variables.items():
# if accounts for possibility of extra variable returned
if key not in query_variables:
continue
squeezed = data[:].squeeze()
if squeezed.ndim == 1:
data_dict[key] = squeezed
elif squeezed.ndim == 2:
for num, data_level in enumerate(squeezed.T):
data_dict[key + '_' + str(num)] = data_level
else:
raise ValueError('cannot parse ndim > 2')
data = pd.DataFrame(data_dict, index=self.time)
# sometimes data is returned as hours since T0
# where T0 is before start. Then the hours between
# T0 and start are added *after* end. So sort and slice
# to remove the garbage
data = data.sort_index().loc[start:end]
return data
def set_time(self, time):
'''
Converts time data into a pandas date object.
Parameters
----------
time: netcdf
Contains time information.
Returns
-------
pandas.DatetimeIndex
'''
times = num2date(time[:].squeeze(),
time.units,
only_use_cftime_datetimes=False,
only_use_python_datetimes=True)
self.time = pd.DatetimeIndex(pd.Series(times), tz=self.location.tz)
def cloud_cover_to_ghi_linear(self,
cloud_cover,
ghi_clear,
offset=35,
**kwargs):
"""
Convert cloud cover to GHI using a linear relationship.
0% cloud cover returns ghi_clear.
100% cloud cover returns offset*ghi_clear.
Parameters
----------
cloud_cover: numeric
Cloud cover in %.
ghi_clear: numeric
GHI under clear sky conditions.
offset: numeric, default 35
Determines the minimum GHI.
kwargs
Not used.
Returns
-------
ghi: numeric
Estimated GHI.
References
----------
Larson et. al. "Day-ahead forecasting of solar power output from
photovoltaic plants in the American Southwest" Renewable Energy
91, 11-20 (2016).
"""
offset = offset / 100.
cloud_cover = cloud_cover / 100.
ghi = (offset + (1 - offset) * (1 - cloud_cover)) * ghi_clear
return ghi
def cloud_cover_to_irradiance_clearsky_scaling(self,
cloud_cover,
method='linear',
**kwargs):
"""
Estimates irradiance from cloud cover in the following steps:
1. Determine clear sky GHI using Ineichen model and
climatological turbidity.
2. Estimate cloudy sky GHI using a function of
cloud_cover e.g.
:py:meth:`~ForecastModel.cloud_cover_to_ghi_linear`
3. Estimate cloudy sky DNI using the DISC model.
4. Calculate DHI from DNI and GHI.
Parameters
----------
cloud_cover : Series
Cloud cover in %.
method : str, default 'linear'
Method for converting cloud cover to GHI.
'linear' is currently the only option.
**kwargs
Passed to the method that does the conversion
Returns
-------
irrads : DataFrame
Estimated GHI, DNI, and DHI.
"""
solpos = self.location.get_solarposition(cloud_cover.index)
cs = self.location.get_clearsky(cloud_cover.index,
model='ineichen',
solar_position=solpos)
method = method.lower()
if method == 'linear':
ghi = self.cloud_cover_to_ghi_linear(cloud_cover, cs['ghi'],
**kwargs)
else:
raise ValueError('invalid method argument')
dni = disc(ghi, solpos['zenith'], cloud_cover.index)['dni']
dhi = ghi - dni * np.cos(np.radians(solpos['zenith']))
irrads = pd.DataFrame({'ghi': ghi, 'dni': dni, 'dhi': dhi}).fillna(0)
return irrads
def cloud_cover_to_transmittance_linear(self,
cloud_cover,
offset=0.75,
**kwargs):
"""
Convert cloud cover to atmospheric transmittance using a linear
model.
0% cloud cover returns offset.
100% cloud cover returns 0.
Parameters
----------
cloud_cover : numeric
Cloud cover in %.
offset : numeric, default 0.75
Determines the maximum transmittance.
kwargs
Not used.
Returns
-------
ghi : numeric
Estimated GHI.
"""
transmittance = ((100.0 - cloud_cover) / 100.0) * offset
return transmittance
def cloud_cover_to_irradiance_liujordan(self, cloud_cover, **kwargs):
"""
Estimates irradiance from cloud cover in the following steps:
1. Determine transmittance using a function of cloud cover e.g.
:py:meth:`~ForecastModel.cloud_cover_to_transmittance_linear`
2. Calculate GHI, DNI, DHI using the
:py:func:`pvlib.irradiance.liujordan` model
Parameters
----------
cloud_cover : Series
Returns
-------
irradiance : DataFrame
Columns include ghi, dni, dhi
"""
# in principle, get_solarposition could use the forecast
# pressure, temp, etc., but the cloud cover forecast is not
# accurate enough to justify using these minor corrections
solar_position = self.location.get_solarposition(cloud_cover.index)
dni_extra = get_extra_radiation(cloud_cover.index)
airmass = self.location.get_airmass(cloud_cover.index)
transmittance = self.cloud_cover_to_transmittance_linear(
cloud_cover, **kwargs)
irrads = liujordan(solar_position['apparent_zenith'],
transmittance,
airmass['airmass_absolute'],
dni_extra=dni_extra)
irrads = irrads.fillna(0)
return irrads
def cloud_cover_to_irradiance(self,
cloud_cover,
how='clearsky_scaling',
**kwargs):
"""
Convert cloud cover to irradiance. A wrapper method.
Parameters
----------
cloud_cover : Series
how : str, default 'clearsky_scaling'
Selects the method for conversion. Can be one of
clearsky_scaling or liujordan.
**kwargs
Passed to the selected method.
Returns
-------
irradiance : DataFrame
Columns include ghi, dni, dhi
"""
how = how.lower()
if how == 'clearsky_scaling':
irrads = self.cloud_cover_to_irradiance_clearsky_scaling(
cloud_cover, **kwargs)
elif how == 'liujordan':
irrads = self.cloud_cover_to_irradiance_liujordan(
cloud_cover, **kwargs)
else:
raise ValueError('invalid how argument')
return irrads
def kelvin_to_celsius(self, temperature):
"""
Converts Kelvin to celsius.
Parameters
----------
temperature: numeric
Returns
-------
temperature: numeric
"""
return temperature - 273.15
def isobaric_to_ambient_temperature(self, data):
"""
Calculates temperature from isobaric temperature.
Parameters
----------
data: DataFrame
Must contain columns pressure, temperature_iso,
temperature_dew_iso. Input temperature in K.
Returns
-------
temperature : Series
Temperature in K
"""
P = data['pressure'] / 100.0 # noqa: N806
Tiso = data['temperature_iso'] # noqa: N806
Td = data['temperature_dew_iso'] - 273.15 # noqa: N806
# saturation water vapor pressure
e = 6.11 * 10**((7.5 * Td) / (Td + 273.3))
# saturation water vapor mixing ratio
w = 0.622 * (e / (P - e))
temperature = Tiso - ((2.501 * 10.**6) / 1005.7) * w
return temperature
def uv_to_speed(self, data):
"""
Computes wind speed from wind components.
Parameters
----------
data : DataFrame
Must contain the columns 'wind_speed_u' and 'wind_speed_v'.
Returns
-------
wind_speed : Series
"""
wind_speed = np.sqrt(data['wind_speed_u']**2 + data['wind_speed_v']**2)
return wind_speed
def gust_to_speed(self, data, scaling=1 / 1.4):
"""
Computes standard wind speed from gust.
Very approximate and location dependent.
Parameters
----------
data : DataFrame
Must contain the column 'wind_speed_gust'.
Returns
-------
wind_speed : Series
"""
wind_speed = data['wind_speed_gust'] * scaling
return wind_speed
ncss = NCSS('https://nomads.ncdc.noaa.gov/thredds/ncss/grid/narr-a/198704/19870404/'
'narr-a_221_19870404_1800_000.grb')
# Bring in needed data
modeldata = ncss.query().all_times()
modeldata.variables('Geopotential_height',
'u_wind',
'v_wind',
'Temperature',
'Specific_humidity').add_lonlat()
# Set the lat/lon box for the data you want to pull in.
# lonlat_box(north_lat,south_lat,east_lon,west_lon)
modeldata.lonlat_box(-140, -60, 16, 60)
# Actually getting the data
data = ncss.get_data(modeldata)
##########################
print(list(data.variables))
#############################
# We will reduce the dimensionality of the data as it is pulled in to remove an empty time
# dimension. Additionally, units are required for input data, so the proper units will also
# be attached.
# Assign data to variable names
dtime = data.variables['Geopotential_height'].dimensions[0]
dlev = data.variables['Geopotential_height'].dimensions[1]
lat = data.variables['lat'][:]
lon = data.variables['lon'][:]
# define time range you want the data for
start = now - timedelta(days=1)
end = now
# build the query
query = ncss.query()
query.lonlat_point(-90.08, 32.32)
query.time_range(start, end)
query.variables('air_temperature', 'dew_point_temperature', 'wind_speed',
'precipitation_amount_hourly', 'inches_ALTIM',
'air_pressure_at_sea_level', 'wind_from_direction')
query.accept('netcdf')
# Get the netcdf dataset
data = ncss.get_data(query)
print(list(data.variables))
########################################
# Begin parsing out information from file including data
# Get the station ID
station_id = data['station_id'][:].tostring()
station_id = station_id.decode('utf-8')
print(station_id)
# Get time into a datetime object
time = [datetime.fromtimestamp(t) for t in data['time']]
time = sorted(time)
print(time)
# Query for Latest GFS Run
gfsdata = ncss.query().time(now).accept('netcdf4')
gfsdata.variables('Geopotential_height_isobaric',
'u-component_of_wind_isobaric',
'v-component_of_wind_isobaric').add_lonlat()
# Set the lat/lon box for the data you want to pull in.
# lonlat_box(north_lat,south_lat,east_lon,west_lon)
gfsdata.lonlat_box(0, 360, 0, 90)
# Set desired level 50000 = 50000 Pa = 500 hPa
gfsdata.vertical_level(25000)
# Actually getting the data
data = ncss.get_data(gfsdata)
#################################################
# The next cell will take the downloaded data and parse it to different variables
# for use later on. Add a cyclic point using the cartopy utility add_cyclic_point
# to the longitudes (the cyclic dimension) as well as any data that is being
# contoured or filled.
dtime = data.variables['Geopotential_height_isobaric'].dimensions[0]
dlat = data.variables['Geopotential_height_isobaric'].dimensions[2]
dlon = data.variables['Geopotential_height_isobaric'].dimensions[3]
lat = data.variables[dlat][:]
lon = data.variables[dlon][:]
# Converting times using the num2date function available through netCDF4
times = data.variables[dtime]
def radar_plus_obs(station, my_datetime, radar_title=None, bb=None,
station_radius=75000., station_layout=simple_layout,
field='reflectivity', vmin=None, vmax=None,
sweep=0):
if radar_title is None:
radar_title = 'Area '
radar = get_radar_from_aws(station, my_datetime)
# Lets get some geographical context
if bb is None:
lats = radar.gate_latitude
lons = radar.gate_longitude
min_lon = lons['data'].min()
min_lat = lats['data'].min()
max_lat = lats['data'].max()
max_lon = lons['data'].max()
bb = {'north' : max_lat,
'south' : min_lat,
'east' : max_lon,
'west' : min_lon}
else:
min_lon = bb['west']
min_lat = bb['south']
max_lon = bb['east']
max_lat = bb['north']
print('min_lat:', min_lat, ' min_lon:', min_lon,
' max_lat:', max_lat, ' max_lon:', max_lon)
index_at_start = radar.sweep_start_ray_index['data'][sweep]
time_at_start_of_radar = num2date(radar.time['data'][index_at_start],
radar.time['units'])
pacific = pytz.timezone('US/Central')
local_time = pacific.fromutc(time_at_start_of_radar)
fancy_date_string = local_time.strftime('%A %B %d at %I:%M %p %Z')
print(fancy_date_string)
metar_cat = TDSCatalog('http://thredds.ucar.edu/thredds/catalog/nws/metar/ncdecoded/catalog.xml?'
'dataset=nws/metar/ncdecoded/Metar_Station_Data_fc.cdmr')
dataset = list(metar_cat.datasets.values())[0]
ncss = NCSS(dataset.access_urls["NetcdfSubset"])
query = ncss.query().accept('csv').time(time_at_start_of_radar)
query.lonlat_box(north=max_lat, south=min_lat, east=max_lon, west=min_lon)
query.variables('air_temperature', 'dew_point_temperature', 'inches_ALTIM',
'wind_speed', 'wind_from_direction', 'cloud_area_fraction', 'weather')
data = ncss.get_data(query)
lats = data['latitude'][:]
lons = data['longitude'][:]
tair = data['air_temperature'][:]
dewp = data['dew_point_temperature'][:]
slp = (data['inches_ALTIM'][:] * units('inHg')).to('mbar')
# Convert wind to components
u, v = mpcalc.get_wind_components(data['wind_speed'] * units.knot,
data['wind_from_direction'] * units.deg)
# Need to handle missing (NaN) and convert to proper code
cloud_cover = 8 * data['cloud_area_fraction']
cloud_cover[np.isnan(cloud_cover)] = 9
cloud_cover = cloud_cover.astype(np.int)
# For some reason these come back as bytes instead of strings
stid = [s.decode() for s in data['station']]
# Convert the text weather observations to WMO codes we can map to symbols
#wx_text = [s.decode('ascii') for s in data['weather']]
#wx_codes = np.array(list(to_code(wx_text)))
sfc_data = {'latitude': lats, 'longitude': lons,
'air_temperature': tair, 'dew_point_temperature': dewp, 'eastward_wind': u,
'northward_wind': v, 'cloud_coverage': cloud_cover,
'air_pressure_at_sea_level': slp}#, 'present_weather': wx_codes}
fig = plt.figure(figsize=(10, 8))
display = pyart.graph.RadarMapDisplayCartopy(radar)
lat_0 = display.loc[0]
lon_0 = display.loc[1]
# Set our Projection
projection = cartopy.crs.Mercator(central_longitude=lon_0,
min_latitude=min_lat, max_latitude=max_lat)
# Call our function to reduce data
filter_data(sfc_data, projection, radius=station_radius, sort_key='present_weather')
print(sweep)
display.plot_ppi_map(
field, sweep, colorbar_flag=True,
title=radar_title +' area ' + field + ' \n' + fancy_date_string,
projection=projection,
min_lon=min_lon, max_lon=max_lon, min_lat=min_lat, max_lat=max_lat,
vmin=vmin, vmax=vmax)
# Mark the radar
display.plot_point(lon_0, lat_0, label_text='Radar')
# Get the current axes and plot some lat and lon lines
gl = display.ax.gridlines(draw_labels=True,
linewidth=2, color='gray', alpha=0.5, linestyle='--')
gl.xlabels_top = False
gl.ylabels_right = False
# Make the station plot
stationplot = StationPlot(display.ax, sfc_data['longitude'], sfc_data['latitude'],
transform=cartopy.crs.PlateCarree(),
fontsize=12)
station_layout.plot(stationplot, sfc_data)
return display, time_at_start_of_radar
# Set the time to current
now = datetime.utcnow()
# Query for Latest GFS Run
gfsdata_hght = ncss.query().time(now).accept('netcdf4')
gfsdata_hght.variables('Geopotential_height_isobaric').add_lonlat()
# Set the lat/lon box for the data you want to pull in.
# lonlat_box(north_lat,south_lat,east_lon,west_lon)
gfsdata_hght.lonlat_box(0, 360, 0, 90)
# Set desired level 50000 = 50000 Pa = 500 hPa
gfsdata_hght.vertical_level(25000)
# Actually getting the data
data_hght = ncss.get_data(gfsdata_hght)
# Query for Latest GFS Run
gfsdata_wind = ncss.query().time(now).accept('netcdf4')
gfsdata_wind.variables('u-component_of_wind_isobaric',
'v-component_of_wind_isobaric').add_lonlat()
# Set the lat/lon box for the data you want to pull in.
# lonlat_box(north_lat,south_lat,east_lon,west_lon)
gfsdata_wind.lonlat_box(0, 360, 0, 90)
# Set desired level 50000 = 50000 Pa = 500 hPa
gfsdata_wind.vertical_level(25000)
# Actually getting the data
data_wind = ncss.get_data(gfsdata_wind)
