def testDirection(self):
# Single scalars
self.assertAlmostEqual(util.u_flow(4.24, 45), 3, 2)
self.assertAlmostEqual(util.u_met(4.24, 45), -3, 2)
self.assertAlmostEqual(util.v_flow(4.24, 45), 3, 2)
self.assertAlmostEqual(util.v_met(4.24, 45), -3, 2)
# Numpy arrays
numpy.testing.assert_almost_equal(util.speed([2, 3, 4], [5, 6, 7]),
numpy.array([5.385, 6.708, 8.062]),
3)
numpy.testing.assert_almost_equal(util.u_flow([3, 3, 2], [0, 90, 60]),
[0, 3, numpy.sqrt(3)],
decimal=2)
numpy.testing.assert_almost_equal(util.u_met([3, 3, 2], [0, 90, 60]),
[0, -3, -numpy.sqrt(3)],
decimal=2)
numpy.testing.assert_almost_equal(util.v_flow([3, 3, 2], [0, 90, 60]),
[3, 0, 1],
decimal=2)
numpy.testing.assert_almost_equal(util.v_met([3, 3, 2], [0, 90, 60]),
[-3, 0, -1],
decimal=2)
# Test broadcasting one direction to multiple speeds
numpy.testing.assert_almost_equal(
util.u_flow([2, 2, 2], [60]),
[numpy.sqrt(3), numpy.sqrt(3),
numpy.sqrt(3)],
decimal=2)
numpy.testing.assert_almost_equal(util.v_flow([2, 2, 2], [60]),
[1, 1, 1],
decimal=2)
def uv_column_interp(u_mass_column, v_mass_column, heights_above_ground, heights_to_interp, z_o=None):
"""Calculate the wind speeds at a given height (in meters) using the two adjacent pressure levels u and v
wind components. Uses a log-law approximation from the ground to 100 m for points that are lower than the
lowest eta level in the model. Direction interpolation is not performed for levels where the log-law
approximation is used.
:param u_mass_column: WRF U wind speed already interpolated to the mass coordinate system (native is flux coordinate)
:param v_mass_column: WRF V wind speed already interpolated to the mass coordinate system (native is flux coordinate)
:param heights_above_ground:
:param heights_to_interp:
:returns 2, 1D arrays u, v of the height interpolated wind speed and direction in the column.
"""
# Make sure array lengths match
assert(u_mass_column.shape == v_mass_column.shape)
assert(u_mass_column.shape == heights_above_ground.shape)
# Make sure we've only been given a column
assert(u_mass_column.ndim == 1)
assert(v_mass_column.ndim == 1)
assert(heights_above_ground.ndim == 1)
# Convert heights_to_interp to an numpy array so we can mask it
heights_to_interp = numpy.array(heights_to_interp)
# Find the heights that need log-law interpolation i.e. below 100 m and lower than the lowest eta-half level
mask_heights_to_interp_log_law = heights_to_interp < numpy.min(heights_above_ground)
# Linear interp where we have model data
if numpy.size(heights_to_interp[~mask_heights_to_interp_log_law]) > 0:
u_mass_coord_interp_linear = numpy.interp(heights_to_interp[~mask_heights_to_interp_log_law],
heights_above_ground, u_mass_column)
v_mass_coord_interp_linear = numpy.interp(heights_to_interp[~mask_heights_to_interp_log_law],
heights_above_ground, v_mass_column)
else:
logger.info('No eta-level heights were in the range of LINEAR interpolation.')
u_mass_coord_interp_linear = []
v_mass_coord_interp_linear = []
# Log-law interp for speed using heights 100 m and below, if WRF ZNT was NOT provided
speed = util.speed(u_mass_column, v_mass_column)
if numpy.size(heights_to_interp[mask_heights_to_interp_log_law]) > 0 and z_o is None:
# Make sure there is more than one height to linearly interpolate with below 100 m
if heights_above_ground[1] < 100:
speed_interp_log_law = log_law_interp(speed, heights_above_ground,
heights_to_interp[mask_heights_to_interp_log_law], 100)
# Otherwise, issue a warning and include the second height above ground
# This situation can sometimes occur at the edges of nested domains when the PSFC value
# is lower than the lowest level of (P + PB). Speculate this is a bug in WRF where the PSFC is being taken
# from the lower resolution domain rather than being calculated with the new topo height in the higher
# resolution domain.
else:
speed_interp_log_law = log_law_interp(speed, heights_above_ground,
heights_to_interp[mask_heights_to_interp_log_law], heights_above_ground[1])
logger.info('Had to use height above 100 m for log-linear log-law interp')
logger.info('Heights used for log-linear log-law interp: {} m and {} m'
.format(heights_above_ground[0], heights_above_ground[1]))
# Use the log-law with the WRF ZNT (surface roughness) to diagnose U,V
elif numpy.size(heights_to_interp[mask_heights_to_interp_log_law]) > 0 and z_o is not None:
speed_interp_log_law = speed[0] * \
numpy.log(heights_to_interp[mask_heights_to_interp_log_law] / z_o) / \
numpy.log(numpy.min(heights_above_ground) / z_o)
else:
logger.info('No eta-level heights were in the range of LOG-LAW interpolation.')
speed_interp_log_law = []
# For log-law interpolation, use the same wind direction as the bottom level where we have WRF data for the
# direction. Assumes these heights are sorted from bottom to top, which is the WRF default.
u_bottom = u_mass_column[0]
v_bottom = v_mass_column[0]
dir_interp_log_law = util.calc_dir_deg(u_bottom, v_bottom)
u_mass_coord_interp_log_law = util.u_flow(speed_interp_log_law, dir_interp_log_law)
v_mass_coord_interp_log_law = util.v_flow(speed_interp_log_law, dir_interp_log_law)
return numpy.hstack((u_mass_coord_interp_log_law, u_mass_coord_interp_linear)), \
numpy.hstack((v_mass_coord_interp_log_law, v_mass_coord_interp_linear))
def testSpeed(self):
self.assertAlmostEqual(util.speed(8.3, 6.3), 10.42, 2)
numpy.testing.assert_almost_equal(util.speed([2, 3, 4], [5, 6, 7]),
numpy.array([5.385, 6.708, 8.062]),
3)
def insert_variable(self,
ncfile,
var_name,
domain_key=None,
replace_data=False,
sql_where="true",
file_type='windb2',
mask=None,
zero_seconds=False):
"""Inserts a netCDF file with WinDB2 or WRF output into a WinDB2 database.
*
* windb2Conn - Connection to a WinDB2 database.
* ncfile - Either an open file or a string name of a file to open.
* var_name - Name of WinDB2 supported variable or a WRF 3D variable (currently WIND, THETA, RHO).
* domain_key - Existing domain key in the database. If left blank, a new domain will be created.
* replace_data - Deletes data for the same time in the database if True. Useful for freshening data.
* file_type - Type of netCDF file to insert: {'windb2' (default), or 'wrf'}
* mask - String name of a mask in the WinDB2 database. Only relevant when creating a new domain (the mask is
* applied automatically thereafter).
*
* returns timesInsertedList, domain_key - A list of times inserted in ISO time format, and the
domain_key where the data was inserted.
:param file_type:
"""
# Make sure this the file_type of file is support
if file_type != 'windb2' and file_type != 'wrf':
raise TypeError('Unsupported file file_type: {}'.format(file_type))
# Open the WinDB netCDF file
logger.debug(
'netCDF file file_type passed to wrf.insertNcFile={}'.format(
type(ncfile)))
if type(ncfile) != Dataset:
ncfile = Dataset(ncfile, 'r')
# Get the grid dimensions and coordinates
if file_type == 'windb2':
nlong = len(ncfile.dimensions['x'])
nlat = len(ncfile.dimensions['y'])
x_coord_array = ncfile.groups['WRF']['XLONG'][:]
y_coord_array = ncfile.groups['WRF']['XLAT'][:]
if self.config['vars'][var_name]['dims'] == 3:
height_array = ncfile.variables['height'][:]
elif self.config['vars'][var_name]['dims'] == 2:
height_array = [self.config['vars'][var_name]['insert'][0]]
else:
raise Exception('Number of dimensions must either be 2 or 3')
init_t = datetime.strptime(
ncfile.groups['WRF'].SIMULATION_START_DATE,
'%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
elif file_type == 'wrf':
nlong = len(ncfile.dimensions['west_east'])
nlat = len(ncfile.dimensions['south_north'])
x_coord_array = ncfile['XLONG'][:]
y_coord_array = ncfile['XLAT'][:]
height_array = [self.config[var_name]['insert'][0]]
init_t = datetime.strptime(
ncfile.SIMULATION_START_DATE,
'%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
# Read in the vars to insert
wrf_copied_var = False
if file_type == 'windb2' and var_name.lower() == 'WIND'.lower():
u = ncfile.variables['eastward_wind'][:]
v = ncfile.variables['northward_wind'][:]
elif file_type == 'windb2' and var_name.lower() == 'DPT'.lower():
ncVariable = ncfile.variables['dew_point_temperature'][:]
# Otherwise try find the WinDB2 interp or WRF var
else:
try:
ncVariable = ncfile.variables[var_name][:]
except KeyError as e:
wrf_copied_var = True
ncVariable = ncfile.groups['WRF'][var_name][:]
# Create a new and/or domain if necessary
if domain_key is None:
if file_type == 'windb2':
domain_key = str(
self.create_new_domain(ncfile.groups['WRF'].TITLE, "WRF",
ncfile.groups['WRF'].DX, 'm', mask))
elif file_type == 'wrf':
domain_key = str(
self.create_new_domain(ncfile.TITLE, "WRF", ncfile.DX, 'm',
mask))
self.insert_horiz_geom(domain_key, x_coord_array, y_coord_array,
create_wrf_srid(self.windb2, ncfile))
# Mask the domain if necessary
if mask is not None:
self.mask_domain(domain_key, mask)
# Create a new table if necessary and add an initialization time column
if file_type == 'windb2' and var_name.lower() == 'wind'.lower():
if not self.windb2.table_exists('wind' + '_' + domain_key):
self.create_new_table(domain_key, var_name,
('speed', 'direction'),
('real', 'smallint'))
self._create_initialization_time_column(var_name, domain_key)
else:
if not self.windb2.table_exists('{}_{}'.format(
var_name.lower(), domain_key)):
self.create_new_table(domain_key, var_name, ('value', ),
('real', ))
self._create_initialization_time_column(
var_name.lower(), domain_key)
# Make sure it's a string so that we don't have concatenation problems later
domain_key = str(domain_key)
# Get the geomkeys associated with the WRF coordinates
horizGeomKey = self.calculateHorizWindGeomKeys(domain_key, nlong, nlat)
# Create a counter to execute every so often
counter = 0
startTime = datetime.now()
# Get the time array to iterate through
if file_type == 'windb2':
time_char_array = chartostring(ncfile.variables['Time'][:])
elif file_type == 'wrf':
time_char_array = chartostring(ncfile.variables['Times'][:])
# Create a statement to use
tCount = 0
timeValuesToReturn = []
for t in time_char_array:
# Create a datetime from the WRF string
t = datetime.strptime(t,
'%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
# Zero the seconds if asked to
if zero_seconds:
t = t.replace(second=0)
# Create the time in GeoServer/GeoWebCache format
timeValuesToReturn.append(t.strftime('%Y-%m-%dT%H:%M:%S.000Z'))
# Info
print('Processing time for {}: {}'.format(var_name,
timeValuesToReturn[-1]))
# Iterate through the x,y, and timearr and insert the WRF variable
for h in height_array:
# We actually need the index of the height, not the actual height itself
height = None
if file_type == 'windb2' and wrf_copied_var is False:
try:
z = numpy.argwhere(
ncfile.variables['height'][:] == h)[0, 0]
height = height_array[z]
except IndexError:
logger.error(
'Height {} to insert does not exist in WinDB2 file'
.format(h))
sys.exit(-1)
elif wrf_copied_var is True:
height = height_array[0]
else:
height = 0
counter = 0
# Open a temporary file to COPY from
tempFile = tempfile.NamedTemporaryFile(mode='w')
for x in range(horizGeomKey.shape[0]):
for y in range(horizGeomKey.shape[1]):
# Make sure that this is actually a x,y point we want to insert
# In order to create a mask of selective insert points, all
# a horizGeomKey of zero means we don't want to insert this one
if horizGeomKey[x, y] == 0:
continue
# Write the data string to the temp file
if file_type == 'windb2' and var_name.lower(
) == 'wind'.lower():
if not (numpy.isnan(u[tCount, z, y, x])
or numpy.isnan(v[tCount, z, y, x])):
# Add this row to be inserted into the database
# Note that we negate U and V so they exist in WinDB2 as the vernacular "coming from" wind direction
print('{}, {}, {}, {}, {}, {}, {}'.format(
domain_key, horizGeomKey[x, y],
t.strftime('%Y-%m-%d %H:%M:%S %Z'),
util.speed(u[tCount, z, y, x], v[tCount, z,
y, x]),
int(
util.calc_dir_deg(
-u[tCount, z, y, x],
-v[tCount, z, y, x])), height,
init_t.strftime('%Y-%m-%d %H:%M:%S %Z')),
file=tempFile)
counter += 1
elif file_type == 'windb2':
# Add this row to be inserted into the database
if self.config['vars'][var_name]['dims'] == 2:
val = ncVariable[tCount, y, x]
elif self.config['vars'][var_name]['dims'] == 3:
val = ncVariable[tCount, z, y, x]
if not numpy.isnan(val):
print('{}, {}, {}, {}, {}, {}'.format(
domain_key, horizGeomKey[x, y],
t.strftime('%Y-%m-%d %H:%M:%S %Z'), val,
height,
init_t.strftime('%Y-%m-%d %H:%M:%S %Z')),
file=tempFile)
counter += 1
elif file_type == 'wrf':
if not numpy.isnan(ncVariable[tCount, y, x]):
# Add this row to be inserted into the database
print('{}, {}, {}, {}, {}'.format(
domain_key, horizGeomKey[x, y],
t.strftime('%Y-%m-%d %H:%M:%S %Z'),
ncVariable[tCount, y, x],
init_t.strftime('%Y-%m-%d %H:%M:%S %Z')),
file=tempFile)
counter += 1
# Insert the data
tempFile.flush()
if file_type == 'windb2' and var_name.lower() == 'wind'.lower(
):
insertColumns = columns = ('domainkey', 'geomkey', 't',
'speed', 'direction', 'height',
'init')
else:
insertColumns = columns = ('domainkey', 'geomkey', 't',
'value', 'height', 'init')
try:
self.windb2.curs.copy_from(open(tempFile.name, 'r'),
var_name + '_' + domain_key,
sep=',',
columns=insertColumns)
except psycopg2.IntegrityError as e:
# Delete the duplicate data
errorTest = 'duplicate key value violates unique constraint "' + var_name.lower(
) + "_" + domain_key + '_domainkey_geomkey_t_height_init_key"'
if re.search(errorTest, str(e.pgerror)):
# Delete the data and retry the insert if asked to replace data in the function call
if replace_data:
# Rollback to the last commit (necessary to reset the database connection)
self.windb2.conn.rollback()
# Delete that timearr (assumes UTC timearr zone)
sql = 'DELETE FROM ' + var_name + '_' + domain_key + \
' WHERE t = timestamp with time zone\'' + t.strftime('%Y-%m-%d %H:%M:%S %Z') + '\' ' + \
'AND height=' + str(h)
print("Deleting conflicting times: " + sql)
self.windb2.curs.execute(sql)
self.windb2.conn.commit()
# Reinsert that timearr
self.windb2.curs.copy_from(
open(tempFile.name, 'r'),
var_name + '_' + domain_key,
sep=',',
columns=insertColumns)
# Commit again or the reinserts won't stick
self.windb2.conn.commit()
continue
# Otherwise, just notify that the insert failed because of duplicate data. We do re-raise this error
# because it's assumed that we want to suplement the WinDB with other data-heights if available.
else:
logging.warning('ERROR ON INSERT: {}'.format(
e.pgerror))
logging.warning(
'Use \'replace_data=True\' if you want the data to be reinserted.'
)
self.windb2.conn.rollback()
continue
# Commit the changes
self.windb2.conn.commit()
# Calaculate the insert rate
elapsedTime = (datetime.now() - startTime).seconds
try:
print('Inserted {}, {}-m height x,y wind points at {} I/s'.
format(counter, height_array[z],
counter / elapsedTime))
except ZeroDivisionError:
print('Inserted {}, {}-m height x,y wind points'.format(
counter, height_array[z]))
except UnboundLocalError:
print('Inserted {}, {}-m height x,y wind points'.format(
counter, height_array[0]))
# Close the tempfile so it is deleted
tempFile.close()
# Increment the time
tCount += 1
return timeValuesToReturn, domain_key
def plotBuoyWRFWindSpeedPerMonth(yearNum, monthNum, timeDeltaMinutes, wrfDomain, wrfGeomKey, wrfHeight, buoyDomain, curs):
"""Creates a hourly wind speed average and power curve plot for a particular wind farm"""
import matplotlib.pyplot as plt
import numpy
# Execute the statement to get the avg winds
windBuoyDataSql = """SELECT t, m_u, m_v, b_u, b_v
FROM (SELECT t, U(speed,direction) as m_u, V(speed,direction) as m_v
FROM wind_""" + str(wrfDomain) + """
WHERE geomkey=""" + str(wrfGeomKey) + """ AND
height=""" + str(wrfHeight) + """ AND
date_part('month', t)=""" + str(monthNum) + """ AND
date_part('year', t)=""" + str(yearNum) + """) m
LEFT JOIN
(SELECT t, U(speed,direction) as b_u, V(speed,direction) as b_v
FROM wind_""" + str(buoyDomain) + """
WHERE date_part('month', t)=""" + str(monthNum) + """ AND
date_part('year', t)=""" + str(yearNum) + """) b
USING (t)
ORDER BY t;"""
logging.debug("Executing the statement: {}".format(windBuoyDataSql))
curs.execute(windBuoyDataSql)
queryResult = curs.fetchall()
queryResult = numpy.array(queryResult)
# Divide up the data
time = queryResult[:,0]
wrfBuoyWind = numpy.array(queryResult[:,1:],dtype=numpy.float)
# Get rid of the None because they cannot be used in masked arrays, convert to NaNs
for i in range(wrfBuoyWind.shape[1]):
wrfBuoyWind[:,i] = none2NaN(wrfBuoyWind[:, i])
# Get the height of the buoy data (and assume they are all the same from this location)
buoyHeightSql = "SELECT DISTINCT(height) FROM wind_" + str(buoyDomain)
curs.execute(buoyHeightSql)
buoyHeight = curs.fetchone()[0]
# Debug
print("wrfBuoyWind=", wrfBuoyWind)
#
# Set up the resources of the wind speed plots
#
fig = plt.figure()
ax = fig.add_subplot(411)
#
# Get the name of the buoy
#
curs.execute("SELECT name FROM domain WHERE key=" + str(buoyDomain))
buoyName = str(curs.fetchone()[0]).upper()
plt.title(buoyName + ", " + str(yearNum) + "-" + str(monthNum))
# Set the output filename
filename = "wrf-wind-domain-" + str(wrfDomain) + "-buoy-" + buoyName + "-" + str(yearNum) + "-" + str(monthNum)
# Data reduction on the plots in case we have a lot of data
obsMarkEvery = int(60/timeDeltaMinutes)
# MAKE THE PLOTS FOR THE U-DIRECTION
ax.set_ylabel("u-wind (m/s)",size='small')
xVals = numpy.arange(wrfBuoyWind.shape[0])
ax.axhline(y=0,linewidth=0.5,linestyle='--',color='b')
ax.plot_date(matplotlib.dates.date2num(time), wrfBuoyWind[:,0],
linewidth=1,linestyle='-',marker=None,color='black',label='WRF(' + str(wrfHeight) + ' m)',
markevery=2)
ax.plot_date(matplotlib.dates.date2num(time)[numpy.isfinite(wrfBuoyWind[:,2])], wrfBuoyWind[:,2][numpy.isfinite(wrfBuoyWind[:,2])],
linewidth=1,linestyle='-',marker=None,color='orange',label='Obs.(' + str(buoyHeight) + ' m)',
markevery=obsMarkEvery)
ax.set_ylim((-10,10))
ax.legend(loc='upper left')
ax.legend(loc=(0,0.5))
# Set the legend font to small
leg = plt.gca().get_legend()
leg.draw_frame(False)
ltext = leg.get_texts()
llines = leg.get_lines()
plt.setp(ltext, fontsize='xx-small')
plt.setp(llines, linewidth=2.0)
# MAKE THE PLOTS FOR THE V-DIRECTION
ax = fig.add_subplot(412,sharex=ax)
ax.set_ylabel("v-wind (m/s)",size='small')
ax.axhline(y=0,linewidth=0.5,linestyle='--',color='b')
ax.plot_date(matplotlib.dates.date2num(time), wrfBuoyWind[:,1],
linewidth=1,linestyle='-',marker=None,color='black',label='WRF(' + str(wrfHeight) + ' m)',
markevery=2)
ax.plot_date(matplotlib.dates.date2num(time)[numpy.isfinite(wrfBuoyWind[:,3])], wrfBuoyWind[:,3][numpy.isfinite(wrfBuoyWind[:,3])],
linewidth=1,linestyle='-',marker=None,color='green',label='Obs.(' + str(buoyHeight) + ' m)',
markevery=obsMarkEvery)
ax.set_ylim((-10,10))
ax.legend(loc='upper left')
ax.legend(loc=(0,0.5))
# Set the legend font to small
leg = plt.gca().get_legend()
leg.draw_frame(False)
ltext = leg.get_texts()
llines = leg.get_lines()
plt.setp(ltext, fontsize='xx-small')
plt.setp(llines, linewidth=2.0)
# MAKE THE PLOTS FOR THE SPEED
speedWrf = util.speed(wrfBuoyWind[:,0], wrfBuoyWind[:,1])
speedObs = util.speed(wrfBuoyWind[:,2], wrfBuoyWind[:,3])
ax = fig.add_subplot(413,sharex=ax)
ax.set_ylabel("wind (m/s)",size='small')
ax.plot_date(matplotlib.dates.date2num(time),speedWrf,
linewidth=1,linestyle='-',marker=None,color='black',label='WRF(' + str(wrfHeight) + ' m)')
ax.plot_date(matplotlib.dates.date2num(time)[numpy.isfinite(speedObs)],speedObs[numpy.isfinite(speedObs)],
linewidth=1,linestyle='-',marker=None,color='red',label='Obs.(' + str(buoyHeight) + ' m)',
markevery=obsMarkEvery)
ax.set_ylim((0,15))
ax.legend(loc='upper left')
ax.legend(loc=(0,0.5))
# Set the legend font to small
leg = plt.gca().get_legend()
leg.draw_frame(False)
ltext = leg.get_texts() # Set the location of the tick marks
llines = leg.get_lines()
plt.setp(ltext, fontsize='xx-small')
plt.setp(llines, linewidth=2.0)
# MAKE THE PLOTS FOR ABS ERROR - masked where the obs are NaN
ax = fig.add_subplot(414,sharex=ax)
ax.set_ylabel("abs. err. (m/s)",size='small')
ax.plot_date(matplotlib.dates.date2num(time)[numpy.isfinite(speedObs)],numpy.abs(speedWrf[numpy.isfinite(speedObs)] -
speedObs[numpy.isfinite(speedObs)]),
linestyle='-',marker=None,color='red',label='speed-error',alpha=0.6)
ax.set_ylim((0,12))
ax.legend(loc='upper left')
# Set the legend font to small
leg = plt.gca().get_legend()
leg.draw_frame(False)
ltext = leg.get_texts()
llines = leg.get_lines()
plt.setp(ltext, fontsize='xx-small')
plt.setp(llines, linewidth=2.0)
# # Add an x-axis to the bottom
# xLabels = [0,7,14,21,28]
# xLabelsMinor = numpy.arange(31)*24*60./timeDeltaMinutes
# ax.set_xticks(numpy.array(xLabels)*24*60/timeDeltaMinutes)
# ax.set_xticks(xLabelsMinor,minor=True)
# ax.set_xlabel("day of month")
# ax.set_xticklabels(xLabels)
# ax.set_xlim(0,numpy.max(xVals))
fig.autofmt_xdate()
# Save the plot to a file
plt.savefig(filename + '.png',bbox_inches='tight')
plt.savefig(filename + '.eps',bbox_inches='tight')