def vaporPressureFromTemp(temp, temp_units, debug=False):
# need temperature in degrees Kelvin
k_temp = convertUnits(temp, temp_units, 'K')
# calculate saturation vapor pressure (sat_vp)
vp = 6.11 * N.exp(5420. * ((k_temp - 273.15) / (273.15 * k_temp)))
if debug: print 'vaporPressureFromTemp', vp, k_temp, temp
return vp
def _postUnpack(self, dataset_path, data, **kwargs):
to_units = kwargs.get('units', None)
if to_units is not None:
from_units = self.getDatasetUnits(dataset_path)
if from_units is not None:
data = convertUnits(data, from_units, to_units)
return data
def _prePack(self, dataset_path, data, **kwargs):
from_units = kwargs.get('units', None)
if from_units is not None:
to_units = self.getDatasetUnits(dataset_path)
if to_units is not None:
if '*' in to_units: to_units = to_units.split('*')[0]
data = convertUnits(data, from_units, to_units)
return data
def _postUnpack(self, dataset_path, data, **kwargs):
# check for units conversion
out_units = kwargs.get('units', None)
if out_units is None: return data
units = self.datasetAttribute(dataset_path, 'units', None)
if units is not None:
if out_units != units:
return convertUnits(data, units, out_units)
else:
errmsg = '"%s" dataset has no attribute named "units"'
raise AttributeError, errmsg % dataset_path
def _serialize(self, dataset_key, data, data_attrs, **kwargs):
data_units = self._dataUnits(dataset_key, data_attrs)
serial_units = kwargs.get('units', data_units)
if data_units == serial_units:
return data, data_attrs
if data_units is None:
errmsg = "Cannot convert '%s' dataset to '%s' units."
raise KeyError, errmsg % (dataset_key.name, serial_units)
per_degree = kwargs.get('per_degree',False)
try:
if not per_degree:
data = convertUnits(data, data_units, serial_units)
else:
data = convertUnits(data, 'd'+data_units, 'd'+serial_units)
except:
errmsg = "Cannot convert '%s' from '%s' to '%s'."
raise KeyError, errmsg % (dataset_key.name, data_units, serial_units)
data_attrs['units'] = serial_units
return data, data_attrs
def dewpointFromHumidityAndTemp(relative_humidity,
temp,
temp_units,
debug=False):
# saturation vapor pressure (sat_vp)
sat_vp = vaporPressureFromTemp(temp, temp_units, debug)
if debug:
print 'dewpointFromHumidityAndTemp', relative_humidity, temp, temp_units
print ' saturation vapor pressure', sat_vp
# actual vapor pressure (vp)
if isinstance(relative_humidity, N.ndarray):
relative_humidity[N.where(relative_humidity == 0)] = N.nan
vp = (relative_humidity * sat_vp) / 100.
else:
if relative_humidity == 0: vp = N.nan
else: vp = (relative_humidity * sat_vp) / 100.
if debug: print ' actual vapor pressure', vp
# dewpoint temperature in degrees Kelvin
k_dew_point = 1. / ((1. / 273.15) - (N.log(vp / 6.11) / 5420.))
# convert back to units of input temperature
dew_point = convertUnits(k_dew_point, 'K', temp_units)
if debug: print ' dew_point', k_dew_point, dew_point
return dew_point
analysis_date = message.analDate
print ' "analDate" =', analysis_date
fcast_time = message.forecastTime
if verbose: print ' "forecastTime" =', fcast_time
if fcast_time > 158: # forecastTime is MINUTES
fcast_time = analysis_date + relativedelta(minutes=fcast_time)
else: # forecastTime is hoors
fcast_time = analysis_date + relativedelta(hours=fcast_time)
if verbose: print ' forecast datetime =', fcast_time
fcast_date = fcast_time.date()
if fcast_date > last_obs_date:
data = message.values[ndfd_indexes].data
data = data.reshape(source_shape)
data[N.where(data == 9999)] = N.nan
data = convertUnits(data, 'K', 'F')
print ' forecast date =', fcast_date
daily.append((fcast_date, data))
print ' '
gribs.close()
temps[temp_var] = tuple(sorted(daily, key=lambda x: x[0]))
#print '\n\n', temps['mint']
#print '\n\n', temps['maxt']
target_year = fcast_date.year
manager = \
factory.tempextsFileManager(source, target_year, region, 'temps', mode='a')
print '\nsaving forecast to', manager.filepath
def applyBias(self,
dem_lons,
dem_lats,
dem_data,
dem_data_units,
stn_lons,
stn_lats,
stn_bias,
stn_bias_units,
report_rate=1000,
debug=False,
performance=False):
""" Apply the calculated station temperature bias to the grid nodes.
"""
PERF_MSG = 'processed %d grid nodes in'
PERF_MSG_SUFFIX = ' ... total = %d of %d'
reporter = self.reporter
search_radius = self.search_radius
c_parm = self.c_parm
vicinity = self.vicinity
min_count = report_rate - 1
dem_grid_shape = dem_lons.shape
dem_grid_size = dem_lons.size
# create empty in-memory arrays for calculated grids
biased_data = N.empty(shape=dem_grid_shape, dtype=float)
dem_data_bias = N.empty(shape=dem_grid_shape, dtype=float)
num_nodes_processed = 0
no_change = 0
start_count = datetime.now()
# make sure station and dem data are in the same units
if stn_bias_units != dem_data_units:
stn_bias = convertUnits(stn_bias, stn_bias_units, dem_data_units)
# loop thru the nodes of the raw grid and apply the station bias
for x in range(dem_grid_shape[0]):
for y in range(dem_grid_shape[1]):
if performance:
# report performance every 'report_rate' passes thru loop
if num_nodes_processed > min_count and\
num_nodes_processed % report_rate == 0:
msg = PERF_MSG % (report_rate)
sfx = PERF_MSG_SUFFIX % (num_nodes_processed,
dem_grid_size)
reporter.logPerformance(start_count, msg, sfx)
start_count = datetime.now()
node_lon = dem_lons[x, y]
node_lat = dem_lats[x, y]
node_value = dem_data[x, y]
if not self._passesApplyBiasTest(node_value, node_lon,
node_lat, stn_bias, stn_lons,
stn_lats):
biased_data[x, y] = dem_data[x, y]
dem_data_bias[x, y] = 0.
num_nodes_processed += 1
no_change += 1
continue
# get indexes of all stations within search radius of grid node
# bbox will be different for each grid node
bbox = (node_lon - search_radius, node_lon + search_radius,
node_lat - search_radius, node_lat + search_radius)
indexes = N.where((stn_lons >= bbox[0]) & (stn_lons <= bbox[1])
& (stn_lats >= bbox[2])
& (stn_lats <= bbox[3]))
# no stations within search radius
if len(indexes[0]) < 1:
# NO ADJUSTMENT CAN BE MADE
biased_data[x, y] = dem_data[x, y]
dem_data_bias[x, y] = 0.
num_nodes_processed += 1
no_change += 1
continue
# coordinates of all station in search area
area_lons = stn_lons[indexes]
area_lats = stn_lats[indexes]
# test stations for 'nearness' to the grid node
bbox = (node_lon - vicinity, node_lon + vicinity,
node_lat - vicinity, node_lat + vicinity)
nearby = N.where((area_lons >= bbox[0])
& (area_lons <= bbox[1])
& (area_lats >= bbox[2])
& (area_lats <= bbox[3]))
# in order to use MQ we must have either 2 'nearby' stations
# or 2 in each quadrant surrounding the node
if (len(nearby[0]) < 1 and not allQuadrants(
node_lon, node_lat, area_lons, area_lats)):
# NO ADJUSTMENT CAN BE MADE
biased_data[x, y] = dem_data[x, y]
dem_data_bias[x, y] = 0.
num_nodes_processed += 1
no_change += 1
continue
# run multiquadric interpolation on BIAS
data_bias = interp.mq(node_lat, node_lon, area_lats, area_lons,
stn_bias[indexes], c_parm)
if N.isfinite(data_bias):
# apply valid bias
value = dem_data[x, y] - data_bias
else:
# invalid bias ... NO ADJUSTMENT CAN BE MADE
value = dem_data[x, y]
data_bias = 0.
no_change += 1
if N.isfinite(value):
biased_data[x, y] = value
dem_data_bias[x, y] = data_bias
else:
biased_data[x, y] = dem_data[x, y]
dem_data_bias[x, y] = 0.
no_change += 1
num_nodes_processed += 1
# log performance for nodes not yet reported
unreported = num_nodes_processed % report_rate
if performance and unreported > 0:
msg = PERF_MSG % (unreported)
sfx = PERF_MSG_SUFFIX % (num_nodes_processed, dem_grid_size)
reporter.logPerformance(start_count, msg, sfx)
return biased_data, dem_data_bias, (num_nodes_processed, no_change)
def calculateBias(self,
algorithm,
stn_uids,
stn_lons,
stn_lats,
stn_data,
stn_data_units,
raw_lons,
raw_lats,
raw_data,
raw_data_units,
report_rate=100,
debug=False,
performance=False):
""" Calculate the weighted difference between the data value at
each station and the nearby grid nodes. It will use multiquadric
interpolation except when there are an insufficient number of grid
nodes nearby, then it will use a simple inverse distance weighted
average.
"""
# local refernces to instance attributes
reporter = self.reporter
vicinity = self.vicinity
min_count = report_rate - 1
PERF_MSG = 'processed %d stations (%d total) in'
# initialize station temperature bias arrays
stn_interp_data = []
stn_data_bias = []
num_stations = len(stn_uids)
# initialize tracking variables
algorithm_counts = [0, 0, 0]
station_count = 0
stations_bad_data = 0
stations_outside = 0
insufficient_coverage = 0
bias_not_calculated = 0
start_report = datetime.now()
# make sure station and dem data are in the same units
if raw_data_units != stn_data_units:
raw_data = convertUnits(raw_data, raw_data_units, stn_data_units)
# loop though list of stations making adjustments to both station and
# grid node temperature extremes
for indx in range(num_stations):
# the following is good for a limited test loop
#for indx in (84,85,278,330,337,345,360,368,444,476):
# report performance every 'report_rate' passes thru the loop
if performance and (station_count > min_count
and station_count % report_rate == 0):
reporter.logPerformance(
start_report, PERF_MSG % (report_rate, station_count))
start_report = datetime.now()
# extract observation data for this station
stn_id = stn_uids[indx]
stn_lon = stn_lons[indx]
stn_lat = stn_lats[indx]
stn_info = 'station %d (%s) at [%-9.5f, %-9.5f]' % (
indx, stn_id, stn_lon, stn_lat)
# station is not within the bounding boxx for this run
if not self._pointInBounds(stn_lon, stn_lat):
stn_interp_data.append(N.inf)
stn_data_bias.append(N.inf)
stations_outside += 1
station_count += 1
continue
stn_value = stn_data[indx]
# check for invalid data value for this station
# this shouldn't happen if station data prep is done right !!!
if not N.isfinite(stn_value):
# set missing values and skip to next iteration
stn_interp_data.append(N.inf)
stn_data_bias.append(N.inf)
stations_bad_data += 1
station_count += 1
if debug:
print 'skipped ', stn_info
print '... bad data value', stn_values
continue
# additional check that may be required by sub-classed data types
if not self._passesCalcBiasTest(stn_value, stn_lon, stn_lat,
raw_data, raw_lons, raw_lats):
stn_interp_data.append(stn_value)
stn_data_bias.append(0.)
station_count += 1
bias_not_calculated += 1
continue
# apply appripriate bias calculation algorithm
if algorithm == 'mq':
result = self.doMQInterp(stn_lon, stn_lat, stn_info, raw_lons,
raw_lats, raw_data, debug)
else:
result = self.doIDWInterp(stn_lon, stn_lat, stn_info, raw_lons,
raw_lats, raw_data, debug)
if result is None:
# set missing values and skip to next iteration
stn_interp_data.append(N.inf)
stn_data_bias.append(N.inf)
insufficient_coverage += 1
station_count += 1
continue
interpolated_value = result[1]
data_bias = interpolated_value - stn_value
estimated_value = interpolated_value - data_bias
stn_data_bias.append(data_bias)
stn_interp_data.append(estimated_value)
station_count += 1
algorithm_counts[result[0]] += 1
if performance:
unreported = station_count % report_rate
if unreported > 0:
reporter.logPerformance(start_report,
PERF_MSG % (unreported, station_count))
# convert the interpolated precip and bias to numpy arrays
stn_interp_data = N.array(stn_interp_data, dtype=float)
stn_data_bias = N.array(stn_data_bias, dtype=float)
indexes = N.where(N.isnan(stn_data_bias) | N.isinf(stn_data_bias))
bad_bias_count = len(indexes[0])
statistics = (station_count, algorithm_counts[2], algorithm_counts[1],
bad_bias_count, stations_bad_data, stations_outside,
insufficient_coverage, bias_not_calculated)
return stn_interp_data, stn_data_bias, statistics
