def qualityControl(self, file_names, out_file_names=None):
"""
qualityControl() [public]
Purpose: Main interface for the quality control utility. Will quality-control a list of files. If out_file_names is specified, each element
in the list will be matched pairwise with those in file_names. So the output from the quality control of file_names[0] will be put
in out_file_names[0], and likewise for each index. If it is not specified, each file in file_names will be overwritten with quality
controlled data.
Parameters: file_names [type=list,tuple]
A list or tuple of strings giving file names to read in and quality control.
out_file_names [type=list,tuple]
An optional argument for specifying where to write output from the quality control to.
Returns: [nothing]
"""
if type(file_names) not in [ list, tuple ]:
file_names = [ file_names ]
if out_file_names is None:
out_file_names = [ '' for f in file_names ]
if len(file_names) != len(out_file_names):
fatalError("The number of file names must be the same as the number of output file names.")
for file_name, out_file_name in zip(file_names, out_file_names):
self._qc(file_name, out_file_name)
return
def _finalizeProduct(self, plot_time, is_forecast, plot_names=[]):
"""
_finalizeProduct() [protected]
Purpose: Add final things to the product, such as the title, valid time, and border, and then save.
Parameters: forecast_hour [type=int]
Forecast hour for model products (pass in None for an observed product).
Returns: [nothing]
"""
plot_names = uniquify(plot_names)
# Modify the last plot name for joining for the title string
if len(plot_names) > 1:
plot_names[-1] = "and " + plot_names[-1]
# Create the forecast hour string according to whether or not we're passed a forecast hour.
plot_time_delta = plot_time - self._valid_time
hour = Units.convert(plot_time_delta.microseconds, 'us', 'hr') + Units.convert(plot_time_delta.seconds, 's', 'hr') + Units.convert(plot_time_delta.days, 'dy', 'hr')
file_name = self._image_file_name % { 'plot_time':hour }
if is_forecast:
fh_string = " (F%03d)" % hour
else:
fh_string = ""
if self._vertical_level in ['surface', 'sfc', "None"]:
vert_level_str = ""
else:
vert_level_str = " %s" % self._vertical_level
# Create the valid time string and assemble the title string
valid_time_string = plot_time.strftime(self._product_time_format)
title_string = "%s%s %s Valid: %s%s" % (self._product_title, vert_level_str, ", ".join(plot_names), valid_time_string, fh_string)
# Put the title on the image
pylab.title(title_string, weight="bold", size="x-small", bbox=dict(facecolor="#ffffff", alpha=0.7),x=0.5,y=0.95)
# Save the figure
try:
pylab.savefig(file_name)
except IOError:
fatalError("Couldn't save image to %s" % file_name)
print "Saved product '%s', valid at %s%s, to file %s" % (self._product_title,
valid_time_string, fh_string, file_name)
pylab.close()
return
def _sanitizeDict(self, dictionary, variables, dtype, src_name, constraint=None, required=True, private=True):
if constraint is not None:
func, error = constraint
if type(variables) not in [ list, tuple ]:
variables = [ variables ]
if type(dtype) not in [ list, tuple ]:
dtype = [ dtype ]
values = []
for var in variables:
try:
if private: key = "_%s" % var
else: key = var
value = dictionary[key]
except KeyError:
if required:
fatalError("Parameter '%s' must be specified in %s." % (var, src_name))
else:
return False
if type(value) not in dtype:
dtype_strings = [ str(t)[1:-1] for t in dtype ]
if len(dtype) > 1:
dtype_strings[-1] = "or %s" % dtype_strings[-1]
if len(dtype) > 2:
dtype_string = ", ".join(dtype_strings)
else:
dtype_string = " ".join(dtype_strings)
fatalError("Parameter '%s' in %s must have %s" % (var, src_name, dtype_string))
values.append(value)
if constraint is not None:
if not func(*values):
fatalError(error % dict(zip(variables, values)))
return True
def _loadFunctionString(self, nc, plot_dict, data_scheme, data_index=None, scratch=False):
"""
_loadFunctionString() [protected]
Purpose: Parses a function string from the input file, loads the data from the file, and converts the data to the proper units for plotting.
Parameters: nc [type=DataIO]
DataIO object that loads in the data.
plot_dict [type=dictionary]
Dictionary containing the plot attributes and their values.
data_scheme [type=dictionary]
Dictionary containing the mapping of internal variable names to the variable names in the data file.
data_index [type=np.array]
An array containing the indexes into the data array to return [not yet implemented].
Returns: [nothing]
"""
if type(plot_dict['function']) not in [ list, tuple ]:
plot_dict['function'] = [ plot_dict['function'] ]
plot_dict['data'] = {}
for function in plot_dict['function']:
parse_function = False
try:
default_units = data_scheme[self._unit_map[function]]
except KeyError:
parse_function = True
if parse_function or default_units is not None:
units_function = function
var_list, const_list = self._parseFunctionConstituents(function)
parsed_function = function
for const in const_list:
# Replace each constant in the function and units function with the proper source code to get the value
parsed_function = re.sub("(?:^|(?<=[\\W]))(%s)(?:(?=[\\W])|$)" % const, "derived.\\1", parsed_function)
for nc_variable in var_list:
if scratch:
plot_dict['scratch'] = nc.get_variable(data_scheme[self._var_map[nc_variable]], data_index)
return
if self._inCache(nc_variable):
# Check the cache to make sure the units in the cache are what we think they are (they might have been converted before putting them in).
file_units = data_scheme[self._unit_map[nc_variable]]
cache_units = self._getFromCache(nc_variable, 'units')
if file_units != cache_units:
nc_data = self._getFromCache(nc_variable)
self._updateCache(nc_variable, Units.convert(nc_data, cache_units, file_units), file_units)
else:
# Put data in the global namespace for easy access (will be deleted later)
self._updateCache(nc_variable, nc.get_variable(data_scheme[self._var_map[nc_variable]], data_index),
data_scheme[self._unit_map[nc_variable]])
for const in const_list:
# Find each constant and HootPy function in the string
match = re.search("(?:^|(?<=[\\W]))%s\\(([\\w\\, ]+)\\)?(?:(?=[\\W])|$)" % const, units_function)
# Parse out the arguments to each function. If it's not a function (and really a constant, such as g) give it an empty list for arguments.
if match is not None and match.group(1) is not None:
args = re.split("\,[\s]*", match.group(1))
else:
args = []
# Determine what the units of the data for the arguments are. If the argument's variable name is not in the cache,
# then that probably means it's the units being output from another HootPy function that's already been subbed
# into the string. Put None in its place.
arg_units = [ self._getFromCache(a, 'units') if self._inCache(a) else None for a in args ]
# Determine what the function is expecting
func_units = derived._units(const, *arg_units)
# A bit of idiot-proofing on the arguments
if len(arg_units) != len(func_units['args']): fatalError("Incorrect number of arguments for function %s." % const)
for idx in xrange(len(args)):
# Convert all argument data to the units the function is expecting (only do it if we actually have units there, and they don't need to be converted.
if arg_units[idx] is not None and arg_units[idx] != func_units['args'][idx]:
self._updateCache(args[idx], Units.convert(self._getFromCache(args[idx], 'value'), arg_units[idx], func_units['args'][idx]),
func_units['args'][idx])
# Substitute the units output from this function back into the units string
units_function = re.sub("(?:^|(?<=[\\W]))((?:%s)(?:\\([\w\\, ]+\\))?)(?:(?=[\\W])|$)" % const,
func_units['return'], units_function)
for nc_variable in var_list:
# Sub individual variables' units into the units string
if units_function.find(nc_variable) > -1:
units_function = re.sub("(?:^|(?<=[\\W]))(%s)(?:(?=[\\W])|$)" % nc_variable,
data_scheme[self._unit_map[nc_variable]], units_function)
plot_dict['default_units'] = Units.evaluateFunction(units_function)
else:
parsed_function = function
self._updateCache(parsed_function, nc.get_variable(data_scheme[self._var_map[parsed_function]], data_index), None)
plot_dict['default_units'] = None
# Load data
if len(plot_dict['function']) == 1:
if not scratch:
if type(plot_dict['plot_name']) == dict:
print "Loading/computing data for %s ..." % plot_dict['plot_name'][function]
else:
print "Loading/computing data for %s ..." % plot_dict['plot_name']
exec "plot_dict['data'] = %s " % parsed_function in globals(), locals()
# Do units conversion
if plot_dict['element_config']['units'] is not None:
if type(plot_dict['data']) in [ list, tuple ]:
plot_dict['data'] = tuple([ Units.convert(d, plot_dict['default_units'], plot_dict['element_config']['units']) for d in plot_dict['data'] ])
else:
plot_dict['data'] = Units.convert(plot_dict['data'], plot_dict['default_units'], plot_dict['element_config']['units'])
else:
if not scratch:
if type(plot_dict['plot_name']) == dict:
print "Loading/computing data for %s (%s) ..." % (plot_dict['plot_name'][function], function)
else:
print "Loading/computing data for %s (%s) ..." % (plot_dict['plot_name'], function)
exec "plot_dict['data']['%s'] = %s " % (function, parsed_function) in globals(), locals()
# Do units conversion
if plot_dict['element_config']['units'][function] is not None:
if type(plot_dict['data'][function]) in [ list, tuple ]:
plot_dict['data'][function] = tuple([ Units.convert(d, plot_dict['default_units'], plot_dict['element_config']['units'][function]) for d in plot_dict['data'][function] ])
else:
plot_dict['data'][function] = Units.convert(plot_dict['data'][function], plot_dict['default_units'], plot_dict['element_config']['units'][function])
if scratch:
for nc_variable in var_list:
self._clearCache(nc_variable)
return
