def qualityControl(self, file_names, out_file_names=None):
        """
        qualityControl() [public]
        Purpose:    Main interface for the quality control utility.  Will quality-control a list of files.  If out_file_names is specified, each element 
                        in the list will be matched pairwise with those in file_names.  So the output from the quality control of file_names[0] will be put
                        in out_file_names[0], and likewise for each index.  If it is not specified, each file in file_names will be overwritten with quality
                        controlled data.
        Parameters: file_names [type=list,tuple]
                        A list or tuple of strings giving file names to read in and quality control.
                    out_file_names [type=list,tuple]
                        An optional argument for specifying where to write output from the quality control to.
        Returns:    [nothing]
        """
        if type(file_names) not in [ list, tuple ]:
            file_names = [ file_names ]

        if out_file_names is None:
            out_file_names = [ '' for f in file_names ]

        if len(file_names) != len(out_file_names):
            fatalError("The number of file names must be the same as the number of output file names.")

        for file_name, out_file_name in zip(file_names, out_file_names):
            self._qc(file_name, out_file_name)
        return
Exemple #2
0
    def _finalizeProduct(self, plot_time, is_forecast, plot_names=[]):
        """
        _finalizeProduct() [protected]
        Purpose:    Add final things to the product, such as the title, valid time, and border, and then save.
        Parameters: forecast_hour [type=int]
                        Forecast hour for model products (pass in None for an observed product).
        Returns:    [nothing]
        """

        plot_names = uniquify(plot_names)

        # Modify the last plot name for joining for the title string
        if len(plot_names) > 1:
            plot_names[-1] = "and " + plot_names[-1]

        # Create the forecast hour string according to whether or not we're passed a forecast hour.
        plot_time_delta = plot_time - self._valid_time
        hour = Units.convert(plot_time_delta.microseconds, 'us', 'hr') + Units.convert(plot_time_delta.seconds, 's', 'hr') + Units.convert(plot_time_delta.days, 'dy', 'hr')
        file_name = self._image_file_name % { 'plot_time':hour }

        if is_forecast:
            fh_string = " (F%03d)" % hour
        else:
            fh_string = ""

        if self._vertical_level in ['surface', 'sfc', "None"]:
            vert_level_str = ""
        else:
            vert_level_str = " %s" % self._vertical_level

        # Create the valid time string and assemble the title string
        valid_time_string = plot_time.strftime(self._product_time_format)
        title_string = "%s%s %s Valid: %s%s" % (self._product_title, vert_level_str, ", ".join(plot_names), valid_time_string, fh_string)

        # Put the title on the image
        pylab.title(title_string, weight="bold", size="x-small", bbox=dict(facecolor="#ffffff", alpha=0.7),x=0.5,y=0.95)

        # Save the figure
        try:
            pylab.savefig(file_name)
        except IOError:
            fatalError("Couldn't save image to %s" % file_name)

        print "Saved product '%s', valid at %s%s, to file %s" % (self._product_title, 
            valid_time_string, fh_string, file_name)

        pylab.close()
        return
Exemple #3
0
    def _sanitizeDict(self, dictionary, variables, dtype, src_name, constraint=None, required=True, private=True):
        if constraint is not None:
            func, error = constraint

        if type(variables) not in [ list, tuple ]:
            variables = [ variables ]

        if type(dtype) not in [ list, tuple ]:
            dtype = [ dtype ]

        values = []
        for var in variables:
            try:
                if private: key = "_%s" % var
                else:       key = var

                value = dictionary[key]
            except KeyError:
                if required:
                    fatalError("Parameter '%s' must be specified in %s." % (var, src_name))
                else:
                    return False

            if type(value) not in dtype:
                dtype_strings = [ str(t)[1:-1] for t in dtype ]
                if len(dtype) > 1:
                    dtype_strings[-1] = "or %s" % dtype_strings[-1]

                if len(dtype) > 2:
                    dtype_string = ", ".join(dtype_strings)
                else:
                    dtype_string = " ".join(dtype_strings)

                fatalError("Parameter '%s' in %s must have %s" % (var, src_name, dtype_string))

            values.append(value)

        if constraint is not None:
            if not func(*values):
                fatalError(error % dict(zip(variables, values)))
        return True
Exemple #4
0
    def _loadFunctionString(self, nc, plot_dict, data_scheme, data_index=None, scratch=False):
        """
        _loadFunctionString() [protected]
        Purpose:    Parses a function string from the input file, loads the data from the file, and converts the data to the proper units for plotting.
        Parameters: nc [type=DataIO]
                        DataIO object that loads in the data. 
                    plot_dict [type=dictionary]
                        Dictionary containing the plot attributes and their values.
                    data_scheme [type=dictionary]
                        Dictionary containing the mapping of internal variable names to the variable names in the data file.
                    data_index [type=np.array]
                        An array containing the indexes into the data array to return [not yet implemented].
        Returns:    [nothing]
        """

        if type(plot_dict['function']) not in [ list, tuple ]:
            plot_dict['function'] = [ plot_dict['function'] ]

        plot_dict['data'] = {}        

        for function in plot_dict['function']:
            parse_function = False
            try:
                default_units = data_scheme[self._unit_map[function]]
            except KeyError:
                parse_function = True

            if parse_function or default_units is not None:
                units_function = function

                var_list, const_list = self._parseFunctionConstituents(function)

                parsed_function = function
                for const in const_list:
                    # Replace each constant in the function and units function with the proper source code to get the value
                    parsed_function = re.sub("(?:^|(?<=[\\W]))(%s)(?:(?=[\\W])|$)" % const, "derived.\\1", parsed_function)

                for nc_variable in var_list:
                    if scratch:
                        plot_dict['scratch'] = nc.get_variable(data_scheme[self._var_map[nc_variable]], data_index)
                        return

                    if self._inCache(nc_variable):
                        # Check the cache to make sure the units in the cache are what we think they are (they might have been converted before putting them in).
                        file_units = data_scheme[self._unit_map[nc_variable]]
                        cache_units = self._getFromCache(nc_variable, 'units')
                        if file_units != cache_units:
                            nc_data = self._getFromCache(nc_variable)
                            self._updateCache(nc_variable, Units.convert(nc_data, cache_units, file_units), file_units)
                    else:
                        # Put data in the global namespace for easy access (will be deleted later)
                        self._updateCache(nc_variable, nc.get_variable(data_scheme[self._var_map[nc_variable]], data_index),
                            data_scheme[self._unit_map[nc_variable]])

                for const in const_list:
                    # Find each constant and HootPy function in the string
                    match = re.search("(?:^|(?<=[\\W]))%s\\(([\\w\\, ]+)\\)?(?:(?=[\\W])|$)" % const, units_function)
                    # Parse out the arguments to each function.  If it's not a function (and really a constant, such as g) give it an empty list for arguments.
                    if match is not None and match.group(1) is not None:
                        args = re.split("\,[\s]*", match.group(1))
                    else:
                        args = []

                    # Determine what the units of the data for the arguments are.  If the argument's variable name is not in the cache, 
                    #   then that probably means it's the units being output from another HootPy function that's already been subbed 
                    #   into the string.  Put None in its place.
                    arg_units = [ self._getFromCache(a, 'units') if self._inCache(a) else None for a in args ]

                    # Determine what the function is expecting
                    func_units = derived._units(const, *arg_units)

                    # A bit of idiot-proofing on the arguments
                    if len(arg_units) != len(func_units['args']): fatalError("Incorrect number of arguments for function %s." % const)

                    for idx in xrange(len(args)):
                        # Convert all argument data to the units the function is expecting (only do it if we actually have units there, and they don't need to be converted.
                        if arg_units[idx] is not None and arg_units[idx] != func_units['args'][idx]:
                            self._updateCache(args[idx], Units.convert(self._getFromCache(args[idx], 'value'), arg_units[idx], func_units['args'][idx]), 
                                func_units['args'][idx])

                    # Substitute the units output from this function back into the units string
                    units_function = re.sub("(?:^|(?<=[\\W]))((?:%s)(?:\\([\w\\, ]+\\))?)(?:(?=[\\W])|$)" % const, 
                        func_units['return'], units_function)

                for nc_variable in var_list:
                    # Sub individual variables' units into the units string
                    if units_function.find(nc_variable) > -1:
                        units_function = re.sub("(?:^|(?<=[\\W]))(%s)(?:(?=[\\W])|$)" % nc_variable, 
                            data_scheme[self._unit_map[nc_variable]], units_function)
    
                plot_dict['default_units'] = Units.evaluateFunction(units_function)
            else:
                parsed_function = function
                self._updateCache(parsed_function, nc.get_variable(data_scheme[self._var_map[parsed_function]], data_index), None)
                plot_dict['default_units'] = None
    
            # Load data
            if len(plot_dict['function']) == 1:
                if not scratch:
                    if type(plot_dict['plot_name']) == dict:
                        print "Loading/computing data for %s ..." % plot_dict['plot_name'][function]
                    else:
                        print "Loading/computing data for %s ..." % plot_dict['plot_name']

                exec "plot_dict['data'] = %s " % parsed_function in globals(), locals()

                # Do units conversion
                if plot_dict['element_config']['units'] is not None:
                    if type(plot_dict['data']) in [ list, tuple ]:
                        plot_dict['data'] = tuple([ Units.convert(d, plot_dict['default_units'], plot_dict['element_config']['units']) for d in plot_dict['data'] ])
                    else:
                        plot_dict['data'] = Units.convert(plot_dict['data'], plot_dict['default_units'], plot_dict['element_config']['units'])
            else:
                if not scratch:
                    if type(plot_dict['plot_name']) == dict:
                        print "Loading/computing data for %s (%s) ..." % (plot_dict['plot_name'][function], function)
                    else:
                        print "Loading/computing data for %s (%s) ..." % (plot_dict['plot_name'], function)

                exec "plot_dict['data']['%s'] = %s " % (function, parsed_function) in globals(), locals()

                # Do units conversion
                if plot_dict['element_config']['units'][function] is not None:
                    if type(plot_dict['data'][function]) in [ list, tuple ]:
                        plot_dict['data'][function] = tuple([ Units.convert(d, plot_dict['default_units'], plot_dict['element_config']['units'][function]) for d in plot_dict['data'][function] ])
                    else:
                        plot_dict['data'][function] = Units.convert(plot_dict['data'][function], plot_dict['default_units'], plot_dict['element_config']['units'][function])

        if scratch:
            for nc_variable in var_list:
                self._clearCache(nc_variable)

        return