Пример #1
0
class LinearAnalysis(object):
    """The super class for linear analysis.  Can be used directly, but
    for prior uncertainty analyses only.  The derived types
    (pyemu.Schur, pyemu.ErrVar, pyemu.MonteCarlo, pyemu.EnsembleSmoother)
    are for different forms of posterior uncertainty analyses. This class
    tries hard to not load items until they are needed; all arguments are
    optional

    Parameters
    ----------
    jco : (varies)
        something that can be cast or loaded into a pyemu.Jco.  Can be a
        str for a filename or pyemu.Matrix/pyemu.Jco object.
    pst : (varies)
        something that can be cast into a pyemu.Pst.  Can be an str for a
        filename or an existing pyemu.Pst.  If None, a pst filename is sought
        with the same base name as the jco argument (if passed)
    parcov : (varies)
        prior parameter covariance matrix.  If str, a filename is assumed and
        the prior parameter covariance matrix is loaded from a file using
        the file extension.  If None, the prior parameter covariance matrix is
        constructed from the parameter bounds in the control file represented
        by the pst argument. Can also be a pyemu.Cov instance
    obscov : (varies)
        observation noise covariance matrix. If str, a filename is assumed and
        the observation noise covariance matrix is loaded from a file using
        the file extension.  If None, the observation noise covariance matrix is
        constructed from the weights in the control file represented
        by the pst argument. Can also be a pyemu.Cov instance
    predictions : (varies)
        prediction (aka forecast) sensitivity vectors.  If str, a filename
        is assumed and predictions are loaded from a file using the file extension.
        Can also be a pyemu.Matrix instance, a numpy.ndarray or a collection
        of pyemu.Matrix or numpy.ndarray.
    ref_var : (float)
        reference variance
    verbose : (either bool or string)
        controls log file / screen output.  If str, a filename is assumed and
            and log file is written to verbose
    
    Note
    ----
    the class makes heavy use of property decorator to encapsulate
    private attributes
    
    """
    def __init__(self,
                 jco=None,
                 pst=None,
                 parcov=None,
                 obscov=None,
                 predictions=None,
                 ref_var=1.0,
                 verbose=False,
                 resfile=False,
                 forecasts=None,
                 **kwargs):
        self.logger = Logger(verbose)
        self.log = self.logger.log
        self.jco_arg = jco
        #if jco is None:
        self.__jco = jco
        if pst is None:
            if isinstance(jco, str):
                pst_case = jco.replace(".jco", ".pst").replace(".jcb", ".pst")
                if os.path.exists(pst_case):
                    pst = pst_case
        self.pst_arg = pst
        if parcov is None and pst is not None:
            parcov = pst
        self.parcov_arg = parcov
        if obscov is None and pst is not None:
            obscov = pst
        self.obscov_arg = obscov
        self.ref_var = ref_var
        if forecasts is not None and predictions is not None:
            raise Exception("can't pass both forecasts and predictions")

        #private attributes - access is through @decorated functions
        self.__pst = None
        self.__parcov = None
        self.__obscov = None
        self.__predictions = None
        self.__qhalf = None
        self.__qhalfx = None
        self.__xtqx = None
        self.__fehalf = None
        self.__prior_prediction = None
        self.prediction_extract = None

        self.log("pre-loading base components")
        if jco is not None:
            self.__load_jco()
        if pst is not None:
            self.__load_pst()
        if parcov is not None:
            self.__load_parcov()
        if obscov is not None:
            self.__load_obscov()

        self.prediction_arg = None
        if predictions is not None:
            self.prediction_arg = predictions
        elif forecasts is not None:
            self.prediction_arg = forecasts
        elif self.pst is not None and self.jco is not None:
            if self.pst.forecast_names is not None:
                self.prediction_arg = self.pst.forecast_names
        if self.prediction_arg:
            self.__load_predictions()

        self.log("pre-loading base components")
        if len(kwargs.keys()) > 0:
            self.logger.warn("unused kwargs in type " +
                             str(self.__class__.__name__) + " : " +
                             str(kwargs))
            raise Exception("unused kwargs" + " : " + str(kwargs))
        # automatically do some things that should be done
        self.log("dropping prior information")
        pi = None
        try:
            pi = self.pst.prior_information
        except:
            self.logger.warn("unable to access self.pst: can't tell if " +
                             " any prior information needs to be dropped.")
        if pi is not None:
            self.drop_prior_information()
        self.log("dropping prior information")

        if resfile != False:
            self.log("scaling obscov by residual phi components")
            try:
                self.adjust_obscov_resfile(resfile=resfile)
            except:
                self.logger.warn("unable to a find a residuals file for " +\
                                " scaling obscov")
                self.resfile = None
                self.res = None
            self.log("scaling obscov by residual phi components")
        assert type(self.parcov) == Cov
        assert type(self.obscov) == Cov

    def __fromfile(self, filename, astype=None):
        """a private method to deduce and load a filename into a matrix object.
        Uses extension: 'jco' or 'jcb': binary, 'mat','vec' or 'cov': ASCII,
        'unc': pest uncertainty file.

        Parameters
        ----------
        filename : str
            the name of the file

        Returns
        -------
        m : pyemu.Matrix

        Raises
        ------
        exception for unrecognized extension
        
        """
        assert os.path.exists(filename),"LinearAnalysis.__fromfile(): " +\
                                        "file not found:" + filename
        ext = filename.split('.')[-1].lower()
        if ext in ["jco", "jcb"]:
            self.log("loading jco: " + filename)
            if astype is None:
                astype = Jco
            m = astype.from_binary(filename)
            self.log("loading jco: " + filename)
        elif ext in ["mat", "vec"]:
            self.log("loading ascii: " + filename)
            if astype is None:
                astype = Matrix
            m = astype.from_ascii(filename)
            self.log("loading ascii: " + filename)
        elif ext in ["cov"]:
            self.log("loading cov: " + filename)
            if astype is None:
                astype = Cov
            m = astype.from_ascii(filename)
            self.log("loading cov: " + filename)
        elif ext in ["unc"]:
            self.log("loading unc: " + filename)
            if astype is None:
                astype = Cov
            m = astype.from_uncfile(filename)
            self.log("loading unc: " + filename)
        else:
            raise Exception("linear_analysis.__fromfile(): unrecognized" +
                            " filename extension:" + str(ext))
        return m

    def __load_pst(self):
        """private method set the pst attribute
        """
        if self.pst_arg is None:
            return None
        if isinstance(self.pst_arg, Pst):
            self.__pst = self.pst_arg
            return self.pst
        else:
            try:
                self.log("loading pst: " + str(self.pst_arg))
                self.__pst = Pst(self.pst_arg)
                self.log("loading pst: " + str(self.pst_arg))
                return self.pst
            except Exception as e:
                raise Exception("linear_analysis.__load_pst(): error loading"+\
                                " pest control from argument: " +
                                str(self.pst_arg) + '\n->' + str(e))

    def __load_jco(self):
        """private method to set the jco attribute from a file or a matrix object
        """
        if self.jco_arg is None:
            return None
            #raise Exception("linear_analysis.__load_jco(): jco_arg is None")
        if isinstance(self.jco_arg, Matrix):
            self.__jco = self.jco_arg
        elif isinstance(self.jco_arg, str):
            self.__jco = self.__fromfile(self.jco_arg, astype=Jco)
        else:
            raise Exception("linear_analysis.__load_jco(): jco_arg must " +
                            "be a matrix object or a file name: " +
                            str(self.jco_arg))

    def __load_parcov(self):
        """private method to set the parcov attribute from:
                a pest control file (parameter bounds)
                a pst object
                a matrix object
                an uncert file
                an ascii matrix file
        """
        # if the parcov arg was not passed but the pst arg was,
        # reset and use parbounds to build parcov
        if not self.parcov_arg:
            if self.pst_arg:
                self.parcov_arg = self.pst_arg
            else:
                raise Exception("linear_analysis.__load_parcov(): " +
                                "parcov_arg is None")
        if isinstance(self.parcov_arg, Matrix):
            self.__parcov = self.parcov_arg
            return
        if isinstance(self.parcov_arg, np.ndarray):
            # if the passed array is a vector,
            # then assume it is the diagonal of the parcov matrix
            if len(self.parcov_arg.shape) == 1:
                assert self.parcov_arg.shape[0] == self.jco.shape[1]
                isdiagonal = True
            else:
                assert self.parcov_arg.shape[0] == self.jco.shape[1]
                assert self.parcov_arg.shape[1] == self.jco.shape[1]
                isdiagonal = False
            self.logger.warn("linear_analysis.__load_parcov(): " +
                             "instantiating parcov from ndarray, can't " +
                             "verify parameters alignment with jco")
            self.__parcov = Matrix(x=self.parcov_arg,
                                   isdiagonal=isdiagonal,
                                   row_names=self.jco.col_names,
                                   col_names=self.jco.col_names)
        self.log("loading parcov")
        if isinstance(self.parcov_arg, str):
            # if the arg is a string ending with "pst"
            # then load parcov from parbounds
            if self.parcov_arg.lower().endswith(".pst"):
                self.__parcov = Cov.from_parbounds(self.parcov_arg)
            else:
                self.__parcov = self.__fromfile(self.parcov_arg, astype=Cov)
        # if the arg is a pst object
        elif isinstance(self.parcov_arg, Pst):
            self.__parcov = Cov.from_parameter_data(self.parcov_arg)
        else:
            raise Exception("linear_analysis.__load_parcov(): " +
                            "parcov_arg must be a " +
                            "matrix object or a file name: " +
                            str(self.parcov_arg))
        self.log("loading parcov")

    def __load_obscov(self):
        """private method to set the obscov attribute from:
                a pest control file (observation weights)
                a pst object
                a matrix object
                an uncert file
                an ascii matrix file
        """
        # if the obscov arg is None, but the pst arg is not None,
        # reset and load from obs weights
        if not self.obscov_arg:
            if self.pst_arg:
                self.obscov_arg = self.pst_arg
            else:
                raise Exception("linear_analysis.__load_obscov(): " +
                                "obscov_arg is None")
        if isinstance(self.obscov_arg, Matrix):
            self.__obscov = self.obscov_arg
            return
        if isinstance(self.obscov_arg, np.ndarray):
            # if the ndarray arg is a vector,
            # assume it is the diagonal of the obscov matrix
            if len(self.obscov_arg.shape) == 1:
                assert self.obscov_arg.shape[0] == self.jco.shape[1]
                isdiagonal = True
            else:
                assert self.obscov_arg.shape[0] == self.jco.shape[0]
                assert self.obscov_arg.shape[1] == self.jco.shape[0]
                isdiagonal = False
            self.logger.warn("linear_analysis.__load_obscov(): " +
                             "instantiating obscov from ndarray,  " +
                             "can't verify observation alignment with jco")
            self.__obscov = Matrix(x=self.obscov_arg,
                                   isdiagonal=isdiagonal,
                                   row_names=self.jco.row_names,
                                   col_names=self.jco.row_names)
        self.log("loading obscov")
        if isinstance(self.obscov_arg, str):
            if self.obscov_arg.lower().endswith(".pst"):
                self.__obscov = Cov.from_obsweights(self.obscov_arg)
            else:
                self.__obscov = self.__fromfile(self.obscov_arg, astype=Cov)
        elif isinstance(self.obscov_arg, Pst):
            self.__obscov = Cov.from_observation_data(self.obscov_arg)
        else:
            raise Exception("linear_analysis.__load_obscov(): " +
                            "obscov_arg must be a " +
                            "matrix object or a file name: " +
                            str(self.obscov_arg))
        self.log("loading obscov")

    def __load_predictions(self):
        """private method set the predictions attribute from:
                mixed list of row names, matrix files and ndarrays
                a single row name
                an ascii file
            can be none if only interested in parameters.

        """
        if self.prediction_arg is None:
            self.__predictions = None
            return
        self.log("loading forecasts")
        if not isinstance(self.prediction_arg, list):
            self.prediction_arg = [self.prediction_arg]

        row_names = []
        vecs = []
        mat = None
        for arg in self.prediction_arg:
            if isinstance(arg, Matrix):
                # a vector
                if arg.shape[1] == 1:
                    vecs.append(arg)
                else:
                    assert arg.shape[0] == self.jco.shape[1],\
                    "linear_analysis.__load_predictions(): " +\
                    "multi-prediction matrix(npar,npred) not aligned " +\
                    "with jco(nobs,npar): " + str(arg.shape) +\
                    ' ' + str(self.jco.shape)
                    #for pred_name in arg.row_names:
                    #    vecs.append(arg.extract(row_names=pred_name).T)
                    mat = arg
            elif isinstance(arg, str):
                if arg.lower() in self.jco.row_names:
                    row_names.append(arg.lower())
                else:
                    try:
                        pred_mat = self.__fromfile(arg, astype=Matrix)
                    except Exception as e:
                        raise Exception("forecast argument: "+arg+" not found in " +\
                                        "jco row names and could not be " +\
                                        "loaded from a file.")
                    # vector
                    if pred_mat.shape[1] == 1:
                        vecs.append(pred_mat)
                    else:
                        #for pred_name in pred_mat.row_names:
                        #    vecs.append(pred_mat.get(row_names=pred_name))
                        if mat is None:
                            mat = pred_mat
                        else:
                            mat = mat.extend((pred_mat))
            elif isinstance(arg, np.ndarray):
                self.logger.warn("linear_analysis.__load_predictions(): " +
                                 "instantiating prediction matrix from " +
                                 "ndarray, can't verify alignment")
                self.logger.warn(
                    "linear_analysis.__load_predictions(): " +
                    "instantiating prediction matrix from " +
                    "ndarray, generating generic prediction names")

                pred_names = [
                    "pred_{0}".format(i + 1) for i in range(arg.shape[0])
                ]

                if self.jco:
                    names = self.jco.col_names
                elif self.parcov:
                    names = self.parcov.col_names
                else:
                    raise Exception("linear_analysis.__load_predictions(): " +
                                    "ndarray passed for predicitons " +
                                    "requires jco or parcov to get " +
                                    "parameter names")
                if mat is None:
                    mat = Matrix(x=arg, row_names=pred_names,
                                 col_names=names).T
                else:
                    mat = mat.extend(
                        Matrix(x=arg, row_names=pred_names, col_names=names).T)
                #for pred_name in pred_names:
                #    vecs.append(pred_matrix.get(row_names=pred_name).T)
            else:
                raise Exception("unrecognized predictions argument: " +
                                str(arg))
        # turn vecs into a pyemu.Matrix

        if len(vecs) > 0:
            xs = vecs[0].x
            for vec in vecs[1:]:
                xs = xs.extend(vec.x)
            names = [vec.col_names[0] for vec in vecs]
            if mat is None:
                mat = Matrix(x=xs,
                             row_names=vecs[0].row_names,
                             col_names=names)
            else:
                mat = mat.extend(
                    Matrix(x=np.array(xs),
                           row_names=vecs[0].row_names,
                           col_names=names))

        if len(row_names) > 0:
            extract = self.jco.extract(row_names=row_names).T
            if mat is None:
                mat = extract
            else:
                mat = mat.extend(extract)
            #for row_name in row_names:
            #    vecs.append(extract.get(row_names=row_name).T)
            # call obscov to load __obscov so that __obscov
            # (priavte) can be manipulated
            self.obscov
            self.__obscov.drop(row_names, axis=0)
        self.__predictions = mat
        try:
            fnames = [
                fname for fname in self.forecast_names
                if fname in self.pst.nnz_obs_names
            ]
        except:
            fnames = []
        if len(fnames) > 0:
            raise Exception(
                "forecasts with non-zero weight in pst: {0}".format(
                    ','.join(fnames)))
        self.log("loading forecasts")
        self.logger.statement("forecast names: {0}".format(','.join(
            mat.col_names)))
        return self.__predictions

    # these property decorators help keep from loading potentially
    # unneeded items until they are called
    # returns a reference - cheap, but can be dangerous

    @property
    def forecast_names(self):
        """ get the forecast (aka prediction) names

        Returns
        -------
        forecast_names : list
            list of forecast names
        
        """
        if self.forecasts is None:
            return []
        #return [fore.col_names[0] for fore in self.forecasts]
        return list(self.predictions.col_names)

    @property
    def parcov(self):
        """ get the prior parameter covariance matrix attribute

        Return
        ------
        parcov : pyemu.Cov
            a reference to the parcov attribute

        Note
        ----
        returns a reference
        
        if LinearAnalysis.__parcov is not set, the dynamically load the
        parcov attribute before returning
        
        """
        if not self.__parcov:
            self.__load_parcov()
        return self.__parcov

    @property
    def obscov(self):
        """ get the observation noise covariance matrix attribute

        Returns
        -------
        obscov : pyemu.Cov
            a reference to the obscov attribute

        Note
        ----
        returns a reference
        
        if LinearAnalysis.__obscov is not set, the dynamically load the
        obscov attribute before returning

        """
        if not self.__obscov:
            self.__load_obscov()
        return self.__obscov

    @property
    def nnz_obs_names(self):
        """ wrapper around pyemu.Pst.nnz_obs_names for listing non-zero
        observation names

        Returns
        -------
        nnz_obs_names : list
            pyemu.Pst.nnz_obs_names
        
        """
        if self.__pst is not None:
            return self.pst.nnz_obs_names
        else:
            return self.jco.obs_names

    def adj_par_names(self):
        """ wrapper around pyemu.Pst.adj_par_names for list adjustable parameter
        names

        Returns
        -------
        adj_par_names : list
            pyemu.Pst.adj_par_names
        
        """
        if self.__pst is not None:
            return self.pst.adj_par_names
        else:
            return self.jco.par_names

    @property
    def jco(self):
        """ get the jacobian matrix attribute

        Returns
        -------
        jco : pyemu.Jco
            the jacobian matrix attribute

        Note
        ----
        returns a reference
        
        if LinearAnalysis.__jco is not set, the dynamically load the
        jco attribute before returning
        
        """
        if not self.__jco:
            self.__load_jco()
        return self.__jco

    @property
    def predictions(self):
        """ get the predictions (aka forecasts) attribute

        Returns
        -------
        predictions : pyemu.Matrix
            a matrix of prediction sensitivity vectors (column wise)

        Note
        ----
        returns a reference
        
        if LinearAnalysis.__predictions is not set, the dynamically load the
            predictions attribute before returning

        """
        if not self.__predictions:
            self.__load_predictions()
        return self.__predictions

    @property
    def predictions_iter(self):
        """ property decorated prediction iterator

        Returns
        -------
        iterator : iterator
            iterator on prediction sensitivity vectors (matrix)

        """
        for fname in self.forecast_names:
            yield self.predictions.get(col_names=fname)

    @property
    def forecasts_iter(self):
        """synonym for LinearAnalysis.predictions_iter()
        """
        return self.predictions_iter

    @property
    def forecasts(self):
        """synonym for LinearAnalysis.predictions
        """
        return self.predictions

    @property
    def pst(self):
        """ get the pyemu.Pst attribute

        Returns
        -------
        pst : pyemu.Pst

        Note
        ----
        returns a references
        
        If LinearAnalysis.__pst is None, then the pst attribute is
        dynamically loaded before returning
        """
        if self.__pst is None and self.pst_arg is None:
            raise Exception("linear_analysis.pst: can't access self.pst:" +
                            "no pest control argument passed")
        elif self.__pst:
            return self.__pst
        else:
            self.__load_pst()
            return self.__pst

    @property
    def fehalf(self):
        """get the KL parcov scaling matrix attribute.  Create the attribute if
        it has not yet been created

        Returns
        -------
        fehalf : pyemu.Matrix

        """
        if self.__fehalf != None:
            return self.__fehalf
        self.log("fehalf")
        self.__fehalf = self.parcov.u * (self.parcov.s**(0.5))
        self.log("fehalf")
        return self.__fehalf

    @property
    def qhalf(self):
        """get the square root of the cofactor matrix attribute. Create the attribute if
        it has not yet been created

        Returns
        -------
        qhalf : pyemu.Matrix

        """
        if self.__qhalf != None:
            return self.__qhalf
        self.log("qhalf")
        self.__qhalf = self.obscov**(-0.5)
        self.log("qhalf")
        return self.__qhalf

    @property
    def qhalfx(self):
        """get the half normal matrix attribute.  Create the attribute if
        it has not yet been created

        Returns
        -------
        qhalfx : pyemu.Matrix

        """
        if self.__qhalfx is None:
            self.log("qhalfx")
            self.__qhalfx = self.qhalf * self.jco
            self.log("qhalfx")
        return self.__qhalfx

    @property
    def xtqx(self):
        """get the normal matrix attribute. Create the attribute if
        it has not yet been created

        Returns
        -------
        xtqx : pyemu.Matrix

        """
        if self.__xtqx is None:
            self.log("xtqx")
            self.__xtqx = self.jco.T * (self.obscov**-1) * self.jco
            self.log("xtqx")
        return self.__xtqx

    @property
    def mle_covariance(self):
        """ get the maximum likelihood parameter covariance matrix.


        Returns
        -------
        pyemu.Matrix : pyemu.Matrix
        
        """
        return self.xtqx.inv

    @property
    def prior_parameter(self):
        """the prior parameter covariance matrix.  Just a wrapper around
        LinearAnalysis.parcov

        Returns
        -------
        prior_parameter : pyemu.Cov

        """
        return self.parcov

    @property
    def prior_forecast(self):
        """thin wrapper for prior_prediction

        Returns
        -------
        prior_forecast : dict
            a dictionary of forecast name, prior variance pairs

        """
        return self.prior_prediction

    @property
    def mle_parameter_estimate(self):
        """ get the maximum likelihood parameter estimate.

        Returns
        -------
         post_expt : pandas.Series
            the maximum likelihood parameter estimates
        
        """
        res = self.pst.res
        assert res is not None
        # build the prior expectation parameter vector
        prior_expt = self.pst.parameter_data.loc[:, ["parval1"]].copy()
        islog = self.pst.parameter_data.partrans == "log"
        prior_expt.loc[islog] = prior_expt.loc[islog].apply(np.log10)
        prior_expt = Matrix.from_dataframe(prior_expt)
        prior_expt.col_names = ["prior_expt"]
        # build the residual vector
        res_vec = Matrix.from_dataframe(res.loc[:, ["residual"]])

        # calc posterior expectation
        upgrade = self.mle_covariance * self.jco.T * res_vec
        upgrade.col_names = ["prior_expt"]
        post_expt = prior_expt + upgrade

        # post processing - back log transform
        post_expt = pd.DataFrame(data=post_expt.x,
                                 index=post_expt.row_names,
                                 columns=["post_expt"])
        post_expt.loc[islog, :] = 10.0**post_expt.loc[islog, :]
        return post_expt

    @property
    def prior_prediction(self):
        """get a dict of prior prediction variances

        Returns
        -------
        prior_prediction : dict
            dictionary of prediction name, prior variance pairs

        """
        if self.__prior_prediction is not None:
            return self.__prior_prediction
        else:
            if self.predictions is not None:
                self.log("propagating prior to predictions")
                prior_cov = self.predictions.T *\
                                self.parcov * self.predictions
                self.__prior_prediction = {
                    n: v
                    for n, v in zip(prior_cov.row_names, np.diag(prior_cov.x))
                }
                self.log("propagating prior to predictions")
            else:
                self.__prior_prediction = {}
            return self.__prior_prediction

    def apply_karhunen_loeve_scaling(self):
        """apply karhuene-loeve scaling to the jacobian matrix.

        Note
        ----
        This scaling is not necessary for analyses using Schur's
        complement, but can be very important for error variance
        analyses.  This operation effectively transfers prior knowledge
        specified in the parcov to the jacobian and reset parcov to the
        identity matrix.

        """
        cnames = copy.deepcopy(self.jco.col_names)
        self.__jco *= self.fehalf
        self.__jco.col_names = cnames
        self.__parcov = self.parcov.identity

    def clean(self):
        """drop regularization and prior information observation from the jco
        """
        if self.pst_arg is None:
            self.logger.warn("linear_analysis.clean(): not pst object")
            return
        if not self.pst.estimation and self.pst.nprior > 0:
            self.drop_prior_information()

    def reset_pst(self, arg):
        """ reset the LinearAnalysis.pst attribute

        Parameters
        ----------
        arg : (str or matrix)
            the value to assign to the pst attribute

        """
        self.logger.warn("resetting pst")
        self.__pst = None
        self.pst_arg = arg

    def reset_parcov(self, arg=None):
        """reset the parcov attribute to None

        Parameters
        ----------
        arg : str or pyemu.Matrix
            the value to assign to the parcov attribute.  If None,
            the private __parcov attribute is cleared but not reset
        """
        self.logger.warn("resetting parcov")
        self.__parcov = None
        if arg is not None:
            self.parcov_arg = arg

    def reset_obscov(self, arg=None):
        """reset the obscov attribute to None

        Parameters
        ----------
        arg : str or pyemu.Matrix
            the value to assign to the obscov attribute.  If None,
            the private __obscov attribute is cleared but not reset
        """
        self.logger.warn("resetting obscov")
        self.__obscov = None
        if arg is not None:
            self.obscov_arg = arg

    def drop_prior_information(self):
        """drop the prior information from the jco and pst attributes
        """
        if self.jco is None:
            self.logger.warn(
                "can't drop prior info, LinearAnalysis.jco is None")
            return
        nprior_str = str(self.pst.nprior)
        self.log("removing " + nprior_str + " prior info from jco, pst, and " +
                 "obs cov")
        #pi_names = list(self.pst.prior_information.pilbl.values)
        pi_names = list(self.pst.prior_names)
        missing = [name for name in pi_names if name not in self.jco.obs_names]
        if len(missing) > 0:
            raise Exception("LinearAnalysis.drop_prior_information(): " +
                            " prior info not found: {0}".format(missing))
        if self.jco is not None:
            self.__jco.drop(pi_names, axis=0)
        self.__pst.prior_information = self.pst.null_prior
        self.__pst.control_data.pestmode = "estimation"
        #self.__obscov.drop(pi_names,axis=0)
        self.log("removing " + nprior_str + " prior info from jco, pst, and " +
                 "obs cov")

    def get(self, par_names=None, obs_names=None, astype=None):
        """method to get a new LinearAnalysis class using a
        subset of parameters and/or observations

        Parameters
        ----------
        par_names : list
            par names for new object
        obs_names : list
            obs names for new object
        astype : pyemu.Schur or pyemu.ErrVar
            type to cast the new object.  If None, return type is
            same as self

        Returns
        -------
        new : LinearAnalysis

        """
        # make sure we aren't fooling with unwanted prior information
        self.clean()
        # if there is nothing to do but copy
        if par_names is None and obs_names is None:
            if astype is not None:
                self.logger.warn("LinearAnalysis.get(): astype is not None, " +
                                 "but par_names and obs_names are None so" +
                                 "\n  ->Omitted attributes will not be " +
                                 "propagated to new instance")
            else:
                return copy.deepcopy(self)
        # make sure the args are lists
        if par_names is not None and not isinstance(par_names, list):
            par_names = [par_names]
        if obs_names is not None and not isinstance(obs_names, list):
            obs_names = [obs_names]

        if par_names is None:
            par_names = self.jco.col_names
        if obs_names is None:
            obs_names = self.jco.row_names
        # if possible, get a new parcov
        if self.parcov:
            new_parcov = self.parcov.get(col_names=[pname for pname in\
                                                    par_names if pname in\
                                                    self.parcov.col_names])
        else:
            new_parcov = None
        # if possible, get a new obscov
        if self.obscov_arg is not None:
            new_obscov = self.obscov.get(row_names=obs_names)
        else:
            new_obscov = None
        # if possible, get a new pst
        if self.pst_arg is not None:
            new_pst = self.pst.get(par_names=par_names, obs_names=obs_names)
        else:
            new_pst = None
        new_extract = None
        if self.predictions:
            # new_preds = []
            # for prediction in self.predictions:
            #     new_preds.append(prediction.get(row_names=par_names))
            new_preds = self.predictions.get(row_names=par_names)

        else:
            new_preds = None
        if self.jco_arg is not None:
            new_jco = self.jco.get(row_names=obs_names, col_names=par_names)
        else:
            new_jco = None
        if astype is not None:
            new = astype(jco=new_jco,
                         pst=new_pst,
                         parcov=new_parcov,
                         obscov=new_obscov,
                         predictions=new_preds,
                         verbose=False)
        else:
            # return a new object of the same type
            new = type(self)(jco=new_jco,
                             pst=new_pst,
                             parcov=new_parcov,
                             obscov=new_obscov,
                             predictions=new_preds,
                             verbose=False)
        return new

    def adjust_obscov_resfile(self, resfile=None):
        """reset the elements of obscov by scaling the implied weights
        based on the phi components in res_file so that the total phi
        is equal to the number of non-zero weights.

        Parameters
        ----------
        resfile : str
            residual file to use.  If None, residual file with case name is
            sought. default is None

        Note
        ----
        calls pyemu.Pst.adjust_weights_resfile()

        """
        self.pst.adjust_weights_resfile(resfile)
        self.__obscov.from_observation_data(self.pst)

    def get_par_css_dataframe(self):
        """ get a dataframe of composite scaled sensitivities.  Includes both
        PEST-style and Hill-style.

        Returns
        -------
        css : pandas.DataFrame

        """

        assert self.jco is not None
        assert self.pst is not None
        jco = self.jco.to_dataframe()
        weights = self.pst.observation_data.loc[jco.index,
                                                "weight"].copy().values
        jco = (jco.T * weights).T
        dss_sum = jco.apply(np.linalg.norm)
        css = (dss_sum / float(self.pst.nnz_obs)).to_frame()
        css.columns = ["pest_css"]
        # log transform stuff
        self.pst.add_transform_columns()
        parval1 = self.pst.parameter_data.loc[dss_sum.index,
                                              "parval1_trans"].values
        css.loc[:, "hill_css"] = (dss_sum * parval1) / (float(self.pst.nnz_obs)
                                                        **2)
        return css
Пример #2
0
class LinearAnalysis(object):
    """The base/parent class for linear analysis.

    Args:
        jco (varies, optional): something that can be cast or loaded into a `pyemu.Jco`.  Can be a
            str for a filename or `pyemu.Matrix`/`pyemu.Jco` object.
        pst (varies, optional): something that can be cast into a `pyemu.Pst`.  Can be an `str` for a
            filename or an existing `pyemu.Pst`.  If `None`, a pst filename is sought
            with the same base name as the jco argument (if passed)
        parcov (varies, optional): prior parameter covariance matrix.  If `str`, a filename is assumed and
            the prior parameter covariance matrix is loaded from a file using
            the file extension (".jcb"/".jco" for binary, ".cov"/".mat" for PEST-style ASCII matrix,
            or ".unc" for uncertainty files).  If `None`, the prior parameter covariance matrix is
            constructed from the parameter bounds in `LinearAnalysis.pst`.  Can also be a `pyemu.Cov` instance
        obscov (varies, optional): observation noise covariance matrix.  If `str`, a filename is assumed and
            the noise covariance matrix is loaded from a file using
            the file extension (".jcb"/".jco" for binary, ".cov"/".mat" for PEST-style ASCII matrix,
            or ".unc" for uncertainty files).  If `None`, the noise covariance matrix is
            constructed from the obsevation weights in `LinearAnalysis.pst`.  Can also be a `pyemu.Cov` instance
        forecasts (varies, optional): forecast sensitivity vectors.  If `str`, first an observation name is assumed (a row
            in `LinearAnalysis.jco`).  If that is not found, a filename is assumed and predictions are
            loaded from a file using the file extension.  If [`str`], a list of observation names is assumed.
            Can also be a `pyemu.Matrix` instance, a `numpy.ndarray` or a collection
            of `pyemu.Matrix` or `numpy.ndarray`.
        ref_var (float, optional): reference variance.  Default is 1.0
        verbose (`bool`): controls screen output.  If `str`, a filename is assumed and
                and log file is written.
        sigma_range (`float`, optional): defines range of upper bound - lower bound in terms of standard
            deviation (sigma). For example, if sigma_range = 4, the bounds represent 4 * sigma.
            Default is 4.0, representing approximately 95% confidence of implied normal distribution.
            This arg is only used if constructing parcov from parameter bounds.
        scale_offset (`bool`, optional): flag to apply parameter scale and offset to parameter bounds
            when calculating prior parameter covariance matrix from bounds.  This arg is onlyused if
            constructing parcov from parameter bounds.Default is True.

    Note:

        Can be used directly, but for prior uncertainty analyses only.

        The derived types (`pyemu.Schur`, `pyemu.ErrVar`) are for different
        forms of FOMS-based posterior uncertainty analyses.

        This class tries hard to not load items until they are needed; all arguments are optional.

        The class makes heavy use of property decorator to encapsulated private attributes

    Example::

        #assumes "my.pst" exists
        la = pyemu.LinearAnalysis(jco="my.jco",forecasts=["fore1","fore2"])
        print(la.prior_forecast)

    
    """
    def __init__(self,
                 jco=None,
                 pst=None,
                 parcov=None,
                 obscov=None,
                 predictions=None,
                 ref_var=1.0,
                 verbose=False,
                 resfile=False,
                 forecasts=None,
                 sigma_range=4.0,
                 scale_offset=True,
                 **kwargs):
        self.logger = Logger(verbose)
        self.log = self.logger.log
        self.jco_arg = jco
        # if jco is None:
        self.__jco = jco
        if pst is None:
            if isinstance(jco, str):
                pst_case = jco.replace(".jco", ".pst").replace(".jcb", ".pst")
                if os.path.exists(pst_case):
                    pst = pst_case
        self.pst_arg = pst
        if parcov is None and pst is not None:
            parcov = pst
        self.parcov_arg = parcov
        if obscov is None and pst is not None:
            obscov = pst
        self.obscov_arg = obscov
        self.ref_var = ref_var
        if forecasts is not None and predictions is not None:
            raise Exception("can't pass both forecasts and predictions")

        self.sigma_range = sigma_range
        self.scale_offset = scale_offset

        # private attributes - access is through @decorated functions
        self.__pst = None
        self.__parcov = None
        self.__obscov = None
        self.__predictions = None
        self.__qhalf = None
        self.__qhalfx = None
        self.__xtqx = None
        self.__fehalf = None
        self.__prior_prediction = None
        self.prediction_extract = None

        self.log("pre-loading base components")
        if jco is not None:
            self.__load_jco()
        if pst is not None:
            self.__load_pst()
        if parcov is not None:
            self.__load_parcov()
        if obscov is not None:
            self.__load_obscov()

        self.prediction_arg = None
        if predictions is not None:
            self.prediction_arg = predictions
        elif forecasts is not None:
            self.prediction_arg = forecasts
        elif self.pst is not None and self.jco is not None:
            if self.pst.forecast_names is not None:
                self.prediction_arg = self.pst.forecast_names
        if self.prediction_arg:
            self.__load_predictions()

        self.log("pre-loading base components")
        if len(kwargs.keys()) > 0:
            self.logger.warn("unused kwargs in type " +
                             str(self.__class__.__name__) + " : " +
                             str(kwargs))
            raise Exception("unused kwargs" + " : " + str(kwargs))
        # automatically do some things that should be done
        self.log("dropping prior information")
        pi = None
        try:
            pi = self.pst.prior_information
        except:
            self.logger.statement(
                "unable to access self.pst: can't tell if " +
                " any prior information needs to be dropped.")
        if pi is not None:
            self.drop_prior_information()
        self.log("dropping prior information")

        if resfile != False:
            self.log("scaling obscov by residual phi components")
            try:
                self.adjust_obscov_resfile(resfile=resfile)
            except:
                self.logger.statement(
                    "unable to a find a residuals file for " +
                    " scaling obscov")
                self.resfile = None
                self.res = None
            self.log("scaling obscov by residual phi components")
        assert type(self.parcov) == Cov
        assert type(self.obscov) == Cov

    def __fromfile(self, filename, astype=None):
        """a private method to deduce and load a filename into a matrix object.
        Uses extension: 'jco' or 'jcb': binary, 'mat','vec' or 'cov': ASCII,
        'unc': pest uncertainty file.
        
        """
        assert os.path.exists(filename), ("LinearAnalysis.__fromfile(): " +
                                          "file not found:" + filename)
        ext = filename.split(".")[-1].lower()
        if ext in ["jco", "jcb"]:
            self.log("loading jco: " + filename)
            if astype is None:
                astype = Jco
            m = astype.from_binary(filename)
            self.log("loading jco: " + filename)
        elif ext in ["mat", "vec"]:
            self.log("loading ascii: " + filename)
            if astype is None:
                astype = Matrix
            m = astype.from_ascii(filename)
            self.log("loading ascii: " + filename)
        elif ext in ["cov"]:
            self.log("loading cov: " + filename)
            if astype is None:
                astype = Cov
            if _istextfile(filename):
                m = astype.from_ascii(filename)
            else:
                m = astype.from_binary(filename)
            self.log("loading cov: " + filename)
        elif ext in ["unc"]:
            self.log("loading unc: " + filename)
            if astype is None:
                astype = Cov
            m = astype.from_uncfile(filename)
            self.log("loading unc: " + filename)
        else:
            raise Exception("linear_analysis.__fromfile(): unrecognized" +
                            " filename extension:" + str(ext))
        return m

    def __load_pst(self):
        """private method set the pst attribute
        """
        if self.pst_arg is None:
            return None
        if isinstance(self.pst_arg, Pst):
            self.__pst = self.pst_arg
            return self.pst
        else:
            try:
                self.log("loading pst: " + str(self.pst_arg))
                self.__pst = Pst(self.pst_arg)
                self.log("loading pst: " + str(self.pst_arg))
                return self.pst
            except Exception as e:
                raise Exception("linear_analysis.__load_pst(): error loading" +
                                " pest control from argument: " +
                                str(self.pst_arg) + "\n->" + str(e))

    def __load_jco(self):
        """private method to set the jco attribute from a file or a matrix object
        """
        if self.jco_arg is None:
            return None
            # raise Exception("linear_analysis.__load_jco(): jco_arg is None")
        if isinstance(self.jco_arg, Matrix):
            self.__jco = self.jco_arg
        elif isinstance(self.jco_arg, str):
            self.__jco = self.__fromfile(self.jco_arg, astype=Jco)
        else:
            raise Exception("linear_analysis.__load_jco(): jco_arg must " +
                            "be a matrix object or a file name: " +
                            str(self.jco_arg))

    def __load_parcov(self):
        """private method to set the parcov attribute from:
                a pest control file (parameter bounds)
                a pst object
                a matrix object
                an uncert file
                an ascii matrix file
        """
        # if the parcov arg was not passed but the pst arg was,
        # reset and use parbounds to build parcov
        if not self.parcov_arg:
            if self.pst_arg:
                self.parcov_arg = self.pst_arg
            else:
                raise Exception("linear_analysis.__load_parcov(): " +
                                "parcov_arg is None")
        if isinstance(self.parcov_arg, Matrix):
            self.__parcov = self.parcov_arg
            return
        if isinstance(self.parcov_arg, np.ndarray):
            # if the passed array is a vector,
            # then assume it is the diagonal of the parcov matrix
            if len(self.parcov_arg.shape) == 1:
                assert self.parcov_arg.shape[0] == self.jco.shape[1]
                isdiagonal = True
            else:
                assert self.parcov_arg.shape[0] == self.jco.shape[1]
                assert self.parcov_arg.shape[1] == self.jco.shape[1]
                isdiagonal = False
            self.logger.warn("linear_analysis.__load_parcov(): " +
                             "instantiating parcov from ndarray, can't " +
                             "verify parameters alignment with jco")
            self.__parcov = Matrix(
                x=self.parcov_arg,
                isdiagonal=isdiagonal,
                row_names=self.jco.col_names,
                col_names=self.jco.col_names,
            )
        self.log("loading parcov")
        if isinstance(self.parcov_arg, str):
            # if the arg is a string ending with "pst"
            # then load parcov from parbounds
            if self.parcov_arg.lower().endswith(".pst"):
                self.__parcov = Cov.from_parbounds(
                    self.parcov_arg,
                    sigma_range=self.sigma_range,
                    scale_offset=self.scale_offset,
                )
            else:
                self.__parcov = self.__fromfile(self.parcov_arg, astype=Cov)
        # if the arg is a pst object
        elif isinstance(self.parcov_arg, Pst):
            self.__parcov = Cov.from_parameter_data(
                self.parcov_arg,
                sigma_range=self.sigma_range,
                scale_offset=self.scale_offset,
            )
        else:
            raise Exception("linear_analysis.__load_parcov(): " +
                            "parcov_arg must be a " +
                            "matrix object or a file name: " +
                            str(self.parcov_arg))
        self.log("loading parcov")

    def __load_obscov(self):
        """private method to set the obscov attribute from:
                a pest control file (observation weights)
                a pst object
                a matrix object
                an uncert file
                an ascii matrix file
        """
        # if the obscov arg is None, but the pst arg is not None,
        # reset and load from obs weights
        if not self.obscov_arg:
            if self.pst_arg:
                self.obscov_arg = self.pst_arg
            else:
                raise Exception("linear_analysis.__load_obscov(): " +
                                "obscov_arg is None")
        if isinstance(self.obscov_arg, Matrix):
            self.__obscov = self.obscov_arg
            return
        if isinstance(self.obscov_arg, np.ndarray):
            # if the ndarray arg is a vector,
            # assume it is the diagonal of the obscov matrix
            if len(self.obscov_arg.shape) == 1:
                assert self.obscov_arg.shape[0] == self.jco.shape[1]
                isdiagonal = True
            else:
                assert self.obscov_arg.shape[0] == self.jco.shape[0]
                assert self.obscov_arg.shape[1] == self.jco.shape[0]
                isdiagonal = False
            self.logger.warn("linear_analysis.__load_obscov(): " +
                             "instantiating obscov from ndarray,  " +
                             "can't verify observation alignment with jco")
            self.__obscov = Matrix(
                x=self.obscov_arg,
                isdiagonal=isdiagonal,
                row_names=self.jco.row_names,
                col_names=self.jco.row_names,
            )
        self.log("loading obscov")
        if isinstance(self.obscov_arg, str):
            if self.obscov_arg.lower().endswith(".pst"):
                self.__obscov = Cov.from_obsweights(self.obscov_arg)
            else:
                self.__obscov = self.__fromfile(self.obscov_arg, astype=Cov)
        elif isinstance(self.obscov_arg, Pst):
            self.__obscov = Cov.from_observation_data(self.obscov_arg)
        else:
            raise Exception("linear_analysis.__load_obscov(): " +
                            "obscov_arg must be a " +
                            "matrix object or a file name: " +
                            str(self.obscov_arg))
        self.log("loading obscov")

    def __load_predictions(self):
        """private method set the predictions attribute from:
                mixed list of row names, matrix files and ndarrays
                a single row name
                an ascii file
            can be none if only interested in parameters.

        """
        if self.prediction_arg is None:
            self.__predictions = None
            return
        self.log("loading forecasts")
        if not isinstance(self.prediction_arg, list):
            self.prediction_arg = [self.prediction_arg]

        row_names = []
        vecs = []
        mat = None
        for arg in self.prediction_arg:
            if isinstance(arg, Matrix):
                # a vector
                if arg.shape[1] == 1:
                    vecs.append(arg)
                else:
                    if self.jco is not None:
                        assert arg.shape[0] == self.jco.shape[1], (
                            "linear_analysis.__load_predictions(): " +
                            "multi-prediction matrix(npar,npred) not aligned "
                            + "with jco(nobs,npar): " + str(arg.shape) + " " +
                            str(self.jco.shape))
                        # for pred_name in arg.row_names:
                        #    vecs.append(arg.extract(row_names=pred_name).T)
                    mat = arg
            elif isinstance(arg, str):
                if arg.lower() in self.jco.row_names:
                    row_names.append(arg.lower())
                else:
                    try:
                        pred_mat = self.__fromfile(arg, astype=Matrix)
                    except Exception as e:
                        raise Exception("forecast argument: " + arg +
                                        " not found in " +
                                        "jco row names and could not be " +
                                        "loaded from a file.")
                    # vector
                    if pred_mat.shape[1] == 1:
                        vecs.append(pred_mat)
                    else:
                        # for pred_name in pred_mat.row_names:
                        #    vecs.append(pred_mat.get(row_names=pred_name))
                        if mat is None:
                            mat = pred_mat
                        else:
                            mat = mat.extend((pred_mat))
            elif isinstance(arg, np.ndarray):
                self.logger.warn("linear_analysis.__load_predictions(): " +
                                 "instantiating prediction matrix from " +
                                 "ndarray, can't verify alignment")
                self.logger.warn(
                    "linear_analysis.__load_predictions(): " +
                    "instantiating prediction matrix from " +
                    "ndarray, generating generic prediction names")

                pred_names = [
                    "pred_{0}".format(i + 1) for i in range(arg.shape[0])
                ]

                if self.jco:
                    names = self.jco.col_names
                elif self.parcov:
                    names = self.parcov.col_names
                else:
                    raise Exception("linear_analysis.__load_predictions(): " +
                                    "ndarray passed for predicitons " +
                                    "requires jco or parcov to get " +
                                    "parameter names")
                if mat is None:
                    mat = Matrix(x=arg, row_names=pred_names,
                                 col_names=names).T
                else:
                    mat = mat.extend(
                        Matrix(x=arg, row_names=pred_names, col_names=names).T)
                # for pred_name in pred_names:
                #    vecs.append(pred_matrix.get(row_names=pred_name).T)
            else:
                raise Exception("unrecognized predictions argument: " +
                                str(arg))
        # turn vecs into a pyemu.Matrix

        if len(vecs) > 0:
            xs = vecs[0].x
            for vec in vecs[1:]:
                xs = xs.extend(vec.x)
            names = [vec.col_names[0] for vec in vecs]
            if mat is None:
                mat = Matrix(x=xs,
                             row_names=vecs[0].row_names,
                             col_names=names)
            else:
                mat = mat.extend(
                    Matrix(x=np.array(xs),
                           row_names=vecs[0].row_names,
                           col_names=names))

        if len(row_names) > 0:
            extract = self.jco.extract(row_names=row_names).T
            if mat is None:
                mat = extract
            else:
                mat = mat.extend(extract)
            # for row_name in row_names:
            #    vecs.append(extract.get(row_names=row_name).T)
            # call obscov to load __obscov so that __obscov
            # (priavte) can be manipulated
            # check if any forecasts are in the obscov
            so_names = set(self.__obscov.row_names)
            drop_names = [r for r in row_names if r in so_names]
            if len(drop_names) > 0:
                self.__obscov.drop(drop_names, axis=0)
        self.__predictions = mat
        try:
            nz_names = set(self.pst.nnz_obs_names)
            fnames = [
                fname for fname in self.forecast_names if fname in nz_names
            ]
            if len(row_names) > 0:
                srow_names = set(row_names)
                fnames = [
                    fname for fname in self.forecast_names
                    if fname in srow_names
                ]
            else:
                fnames = []
        except:
            fnames = []
        if len(fnames) > 0:
            self.logger.warn(
                "forecasts with non-zero weight in pst: {0}...".format(
                    ",".join(fnames)) +
                "\n -> re-setting these forecast weights to zero...")
            self.pst.observation_data.loc[fnames, "weight"] = 0.0
        self.log("loading forecasts")
        self.logger.statement("forecast names: {0}".format(",".join(
            mat.col_names)))
        return self.__predictions

    # these property decorators help keep from loading potentially
    # unneeded items until they are called
    # returns a reference - cheap, but can be dangerous

    @property
    def forecast_names(self):
        """ get the forecast (aka prediction) names

        Returns:
            ([`str`]): list of forecast names
        
        """
        if self.forecasts is None:
            return []
        # return [fore.col_names[0] for fore in self.forecasts]
        return list(self.predictions.col_names)

    @property
    def parcov(self):
        """ get the prior parameter covariance matrix attribute

        Returns:
            `pyemu.Cov`: a reference to the `LinearAnalysis.parcov` attribute
        
        """
        if not self.__parcov:
            self.__load_parcov()
        return self.__parcov

    @property
    def obscov(self):
        """ get the observation noise covariance matrix attribute

        Returns:
            `pyemu.Cov`: a reference to the `LinearAnalysis.obscov` attribute

        """
        if not self.__obscov:
            self.__load_obscov()
        return self.__obscov

    @property
    def nnz_obs_names(self):
        """non-zero-weighted observation names

        Returns:
            ['str`]: list of non-zero-weighted observation names

        Note:
            if `LinearAnalysis.pst` is `None`, returns `LinearAnalysis.jco.row_names`
        
        """
        if self.__pst is not None:
            return self.pst.nnz_obs_names
        else:
            return self.jco.obs_names

    @property
    def adj_par_names(self):
        """ adjustable parameter names

        Returns:
            ['str`]: list of adjustable parameter names

        Note:
            if `LinearAnalysis.pst` is `None`, returns `LinearAnalysis.jco.col_names`
        
        """
        if self.__pst is not None:
            return self.pst.adj_par_names
        else:
            return self.jco.par_names

    @property
    def jco(self):
        """ the jacobian matrix attribute

        Returns:
            `pyemu.Jco`: the jacobian matrix attribute

        """
        if not self.__jco:
            self.__load_jco()
        return self.__jco

    @property
    def predictions(self):
        """the prediction (aka forecast) sentivity vectors attribute

        Returns:
            `pyemu.Matrix`: a matrix of prediction sensitivity vectors (column wise)

        """
        if not self.__predictions:
            self.__load_predictions()
        return self.__predictions

    @property
    def predictions_iter(self):
        """prediction sensitivity vectors iterator

        Returns:
            `iterator`: iterator on prediction sensitivity vectors (matrix)

        Note:
            this is used for processing huge numbers of predictions
        """
        for fname in self.forecast_names:
            yield self.predictions.get(col_names=fname)

    @property
    def forecasts_iter(self):
        """forecast (e.g. prediction) sensitivity vectors iterator

        Returns:
            `iterator`: iterator on forecasts (e.g. predictions) sensitivity vectors (matrix)

        Note:
            This is used for processing huge numbers of predictions

            This is a synonym for LinearAnalysis.predictions_iter()
        """
        return self.predictions_iter

    @property
    def forecasts(self):
        """the forecast sentivity vectors attribute

        Returns:
            `pyemu.Matrix`: a matrix of forecast (prediction) sensitivity vectors (column wise)

        """
        return self.predictions

    @property
    def pst(self):
        """the pst attribute

        Returns:
            `pyemu.Pst`: the pst attribute

        """
        if self.__pst is None and self.pst_arg is None:
            raise Exception("linear_analysis.pst: can't access self.pst:" +
                            "no pest control argument passed")
        elif self.__pst:
            return self.__pst
        else:
            self.__load_pst()
            return self.__pst

    @property
    def fehalf(self):
        """Karhunen-Loeve scaling matrix attribute.

        Returns:
            `pyemu.Matrix`: the Karhunen-Loeve scaling matrix based on the prior
            parameter covariance matrix

        """
        if self.__fehalf != None:
            return self.__fehalf
        self.log("fehalf")
        self.__fehalf = self.parcov.u * (self.parcov.s**(0.5))
        self.log("fehalf")
        return self.__fehalf

    @property
    def qhalf(self):
        """square root of the cofactor matrix attribute. Create the attribute if
        it has not yet been created

        Returns:
            `pyemu.Matrix`: square root of the cofactor matrix

        """
        if self.__qhalf != None:
            return self.__qhalf
        self.log("qhalf")
        self.__qhalf = self.obscov**(-0.5)
        self.log("qhalf")
        return self.__qhalf

    @property
    def qhalfx(self):
        """half normal matrix attribute.

        Returns:
            `pyemu.Matrix`: half normal matrix attribute

        """
        if self.__qhalfx is None:
            self.log("qhalfx")
            self.__qhalfx = self.qhalf * self.jco
            self.log("qhalfx")
        return self.__qhalfx

    @property
    def xtqx(self):
        """normal matrix attribute.

        Returns:
            `pyemu.Matrix`: normal matrix attribute

        """
        if self.__xtqx is None:
            self.log("xtqx")
            self.__xtqx = self.jco.T * (self.obscov**-1) * self.jco
            self.log("xtqx")
        return self.__xtqx

    @property
    def mle_covariance(self):
        """maximum likelihood parameter covariance matrix.

        Returns:
            `pyemu.Matrix`: maximum likelihood parameter covariance matrix
        
        """
        return self.xtqx.inv

    @property
    def prior_parameter(self):
        """prior parameter covariance matrix

        Returns:
            `pyemu.Cov`: prior parameter covariance matrix

        """
        return self.parcov

    @property
    def prior_forecast(self):
        """prior forecast (e.g. prediction) variances

        Returns:
            `dict`: a dictionary of forecast name, prior variance pairs

        """
        return self.prior_prediction

    @property
    def mle_parameter_estimate(self):
        """ maximum likelihood parameter estimate.

        Returns:
            `pandas.Series`: the maximum likelihood parameter estimates
        
        """
        res = self.pst.res
        assert res is not None
        # build the prior expectation parameter vector
        prior_expt = self.pst.parameter_data.loc[:, ["parval1"]].copy()
        islog = self.pst.parameter_data.partrans == "log"
        prior_expt.loc[islog] = prior_expt.loc[islog].apply(np.log10)
        prior_expt = Matrix.from_dataframe(prior_expt)
        prior_expt.col_names = ["prior_expt"]
        # build the residual vector
        res_vec = Matrix.from_dataframe(res.loc[:, ["residual"]])

        # calc posterior expectation
        upgrade = self.mle_covariance * self.jco.T * res_vec
        upgrade.col_names = ["prior_expt"]
        post_expt = prior_expt + upgrade

        # post processing - back log transform
        post_expt = pd.DataFrame(data=post_expt.x,
                                 index=post_expt.row_names,
                                 columns=["post_expt"])
        post_expt.loc[islog, :] = 10.0**post_expt.loc[islog, :]
        return post_expt

    @property
    def prior_prediction(self):
        """prior prediction (e.g. forecast) variances

        Returns:
            `dict`: a dictionary of prediction name, prior variance pairs

        """
        if self.__prior_prediction is not None:
            return self.__prior_prediction
        else:
            if self.predictions is not None:
                self.log("propagating prior to predictions")
                prior_cov = self.predictions.T * self.parcov * self.predictions
                self.__prior_prediction = {
                    n: v
                    for n, v in zip(prior_cov.row_names, np.diag(prior_cov.x))
                }
                self.log("propagating prior to predictions")
            else:
                self.__prior_prediction = {}
            return self.__prior_prediction

    def apply_karhunen_loeve_scaling(self):
        """apply karhuene-loeve scaling to the jacobian matrix.

        Note:

            This scaling is not necessary for analyses using Schur's
            complement, but can be very important for error variance
            analyses.  This operation effectively transfers prior knowledge
            specified in the parcov to the jacobian and reset parcov to the
            identity matrix.

        """
        cnames = copy.deepcopy(self.jco.col_names)
        self.__jco *= self.fehalf
        self.__jco.col_names = cnames
        self.__parcov = self.parcov.identity

    def clean(self):
        """drop regularization and prior information observation from the jco
        """
        if self.pst_arg is None:
            self.logger.statement("linear_analysis.clean(): not pst object")
            return
        if not self.pst.estimation and self.pst.nprior > 0:
            self.drop_prior_information()

    def reset_pst(self, arg):
        """ reset the LinearAnalysis.pst attribute

        Args:
            arg (`str` or `pyemu.Pst`): the value to assign to the pst attribute

        """
        self.logger.statement("resetting pst")
        self.__pst = None
        self.pst_arg = arg

    def reset_parcov(self, arg=None):
        """reset the parcov attribute to None

        Args:
            arg (`str` or `pyemu.Matrix`): the value to assign to the parcov
                attribute.  If None, the private __parcov attribute is cleared
                but not reset
        """
        self.logger.statement("resetting parcov")
        self.__parcov = None
        if arg is not None:
            self.parcov_arg = arg

    def reset_obscov(self, arg=None):
        """reset the obscov attribute to None

        Args:
            arg (`str` or `pyemu.Matrix`): the value to assign to the obscov
                attribute.  If None, the private __obscov attribute is cleared
                but not reset
        """
        self.logger.statement("resetting obscov")
        self.__obscov = None
        if arg is not None:
            self.obscov_arg = arg

    def drop_prior_information(self):
        """drop the prior information from the jco and pst attributes
        """
        if self.jco is None:
            self.logger.statement(
                "can't drop prior info, LinearAnalysis.jco is None")
            return
        nprior_str = str(self.pst.nprior)
        self.log("removing " + nprior_str + " prior info from jco, pst, and " +
                 "obs cov")
        # pi_names = list(self.pst.prior_information.pilbl.values)
        pi_names = list(self.pst.prior_names)
        missing = [name for name in pi_names if name not in self.jco.obs_names]
        if len(missing) > 0:
            raise Exception("LinearAnalysis.drop_prior_information(): " +
                            " prior info not found: {0}".format(missing))
        if self.jco is not None:
            self.__jco.drop(pi_names, axis=0)
        self.__pst.prior_information = self.pst.null_prior
        self.__pst.control_data.pestmode = "estimation"
        # self.__obscov.drop(pi_names,axis=0)
        self.log("removing " + nprior_str + " prior info from jco, pst, and " +
                 "obs cov")

    def get(self, par_names=None, obs_names=None, astype=None):
        """method to get a new LinearAnalysis class using a
        subset of parameters and/or observations

        Args:
            par_names ([`'str`]): par names for new object
            obs_names ([`'str`]): obs names for new object
            astype (`pyemu.Schur` or `pyemu.ErrVar`): type to
                cast the new object.  If None, return type is
                same as self

        Returns:
            `LinearAnalysis`: new instance

        """
        # make sure we aren't fooling with unwanted prior information
        self.clean()
        # if there is nothing to do but copy
        if par_names is None and obs_names is None:
            if astype is not None:
                self.logger.warn("LinearAnalysis.get(): astype is not None, " +
                                 "but par_names and obs_names are None so" +
                                 "\n  ->Omitted attributes will not be " +
                                 "propagated to new instance")
            else:
                return copy.deepcopy(self)
        # make sure the args are lists
        if par_names is not None and not isinstance(par_names, list):
            par_names = [par_names]
        if obs_names is not None and not isinstance(obs_names, list):
            obs_names = [obs_names]

        if par_names is None:
            par_names = self.jco.col_names
        if obs_names is None:
            obs_names = self.jco.row_names
        # if possible, get a new parcov
        if self.parcov:
            new_parcov = self.parcov.get(col_names=[
                pname for pname in par_names if pname in self.parcov.col_names
            ])
        else:
            new_parcov = None
        # if possible, get a new obscov
        if self.obscov_arg is not None:
            new_obscov = self.obscov.get(row_names=obs_names)
        else:
            new_obscov = None
        # if possible, get a new pst
        if self.pst_arg is not None:
            new_pst = self.pst.get(par_names=par_names, obs_names=obs_names)
        else:
            new_pst = None
        new_extract = None
        if self.predictions:
            # new_preds = []
            # for prediction in self.predictions:
            #     new_preds.append(prediction.get(row_names=par_names))
            new_preds = self.predictions.get(row_names=par_names)

        else:
            new_preds = None
        if self.jco_arg is not None:
            new_jco = self.jco.get(row_names=obs_names, col_names=par_names)
        else:
            new_jco = None
        if astype is not None:
            new = astype(
                jco=new_jco,
                pst=new_pst,
                parcov=new_parcov,
                obscov=new_obscov,
                predictions=new_preds,
                verbose=False,
            )
        else:
            # return a new object of the same type
            new = type(self)(
                jco=new_jco,
                pst=new_pst,
                parcov=new_parcov,
                obscov=new_obscov,
                predictions=new_preds,
                verbose=False,
            )
        return new

    def adjust_obscov_resfile(self, resfile=None):
        """reset the elements of obscov by scaling the implied weights
        based on the phi components in res_file so that the total phi
        is equal to the number of non-zero weights.

        Args:
            resfile (`str`): residual file to use.  If None, residual
                file with case name is sought. default is None

        Note:
            calls `pyemu.Pst.adjust_weights_resfile()`

        """
        self.pst.adjust_weights_resfile(resfile)
        self.__obscov.from_observation_data(self.pst)

    def get_par_css_dataframe(self):
        """ get a dataframe of composite scaled sensitivities.  Includes both
        PEST-style and Hill-style.

        Returns:
            `pandas.DataFrame`: a dataframe of parameter names, PEST-style and
            Hill-style composite scaled sensitivity

        """

        if self.jco is None:
            raise Exception("jco is None")
        if self.pst is None:
            raise Exception("pst is None")
        jco = self.jco.to_dataframe()
        weights = self.pst.observation_data.loc[jco.index,
                                                "weight"].copy().values
        jco = (jco.T * weights).T

        dss_sum = jco.apply(np.linalg.norm)
        css = (dss_sum / float(self.pst.nnz_obs)).to_frame()
        css.columns = ["pest_css"]
        # log transform stuff
        self.pst.add_transform_columns()
        parval1 = self.pst.parameter_data.loc[dss_sum.index,
                                              "parval1_trans"].values
        css.loc[:, "hill_css"] = (dss_sum * parval1) / (float(self.pst.nnz_obs)
                                                        **2)
        return css

    def get_cso_dataframe(self):
        """get a dataframe of composite observation sensitivity, as returned by PEST in the
        seo file.

        Returns:
            `pandas.DataFrame`: dataframe of observation names and composite observation
            sensitivity

        Note:
             That this formulation deviates slightly from the PEST documentation in that the
             values are divided by (npar-1) rather than by (npar).

             The equation is cso_j = ((Q^1/2*J*J^T*Q^1/2)^1/2)_jj/(NPAR-1)


        """
        if self.jco is None:
            raise Exception("jco is None")
        if self.pst is None:
            raise Exception("pst is None")
        weights = (self.pst.observation_data.loc[self.jco.to_dataframe().index,
                                                 "weight"].copy().values)
        cso = np.diag(np.sqrt(
            (self.qhalfx.x.dot(self.qhalfx.x.T)))) / (float(self.pst.npar - 1))
        cso_df = pd.DataFrame.from_dict({
            "obnme": self.jco.to_dataframe().index,
            "cso": cso
        })
        cso_df.index = cso_df["obnme"]
        cso_df.drop("obnme", axis=1, inplace=True)
        return cso_df

    def get_obs_competition_dataframe(self):
        """get the observation competition stat a la PEST utility

        Returns:
            `pandas.DataFrame`: a dataframe of observation names by
            observation names with values equal to the PEST
            competition statistic

        """
        if self.jco is None:
            raise Exception("jco is None")
        if self.pst is None:
            raise Exception("pst is None")
        if self.pst.res is None:
            raise Exception("res is None")
        onames = self.pst.nnz_obs_names
        weights = self.pst.observation_data.loc[onames, "weight"].to_dict()
        residuals = self.pst.res.loc[onames, "residual"].to_dict()
        jco = self.jco.to_dataframe()
        df = pd.DataFrame(columns=onames, index=onames)
        for i, oname in enumerate(onames):
            df.loc[oname, oname] = 0.0
            for ooname in onames[i + 1:]:
                oc = (weights[oname] * weights[ooname] *
                      np.dot(jco.loc[oname, :].values,
                             jco.loc[ooname, :].values.transpose()))
                df.loc[oname, ooname] = oc
                df.loc[ooname, oname] = oc
        return df
Пример #3
0
Файл: la.py Проект: wk1984/pyemu
class LinearAnalysis(object):
    """ the super class for linear analysis.  Can be used for prior analyses
        only.  The derived types (schur and errvar) are for posterior analyses
        this class tries hard to not load items until they are needed
        all arguments are optional

    Parameters:
    ----------
        jco ([enumerable of] [string,ndarray,matrix objects]) : jacobian
        pst (pst object) : the pest control file object
        parcov ([enumerable of] [string,ndarray,matrix objects]) :
            parameter covariance matrix
        obscov ([enumerable of] [string,ndarray,matrix objects]):
            observation noise covariance matrix
        predictions ([enumerable of] [string,ndarray,matrix objects]) :
            prediction sensitivity vectors
        ref_var (float) : reference variance
        verbose (either bool or string) : controls log file / screen output
    Notes:
        the class makes heavy use of property decorator to encapsulate
        private attributes
    """
    def __init__(self, jco=None, pst=None, parcov=None, obscov=None,
                 predictions=None, ref_var=1.0, verbose=False,
                 resfile=False, forecasts=None,**kwargs):
        self.logger = logger(verbose)
        self.log = self.logger.log
        self.jco_arg = jco
        #if jco is None:
        self.__jco = jco
        if pst is None:
            if isinstance(jco, str):
                pst_case = jco.replace(".jco", ".pst").replace(".jcb",".pst")
                if os.path.exists(pst_case):
                    pst = pst_case
        self.pst_arg = pst
        if parcov is None and pst is not None:
            parcov = pst
        self.parcov_arg = parcov
        if obscov is None and pst is not None:
            obscov = pst
        self.obscov_arg = obscov
        self.ref_var = ref_var
        if forecasts is not None and predictions is not None:
            raise Exception("can't pass both forecasts and predictions")


        #private attributes - access is through @decorated functions
        self.__pst = None
        self.__parcov = None
        self.__obscov = None
        self.__predictions = None
        self.__qhalf = None
        self.__qhalfx = None
        self.__xtqx = None
        self.__fehalf = None
        self.__prior_prediction = None

        self.log("pre-loading base components")
        if jco is not None:
            self.__load_jco()
        if pst is not None:
            self.__load_pst()
        if parcov is not None:
            self.__load_parcov()
        if obscov is not None:
            self.__load_obscov()

        self.prediction_arg = None
        if predictions is not None:
            self.prediction_arg = predictions
        elif forecasts is not None:
            self.prediction_arg = forecasts
        elif self.pst is not None and self.jco is not None:
            if "forecasts" in self.pst.pestpp_options:
                self.prediction_arg = [i.strip() for i in self.pst.pestpp_options["forecasts"].\
                    lower().split(',')]
            elif "predictions" in self.pst.pestpp_options:
                self.prediction_arg = [i.strip() for i in self.pst.pestpp_options["predictions"].\
                    lower().split(',')]
        if self.prediction_arg:
            self.__load_predictions()


        self.log("pre-loading base components")
        if len(kwargs.keys()) > 0:
            self.logger.warn("unused kwargs in type " +
                             str(self.__class__.__name__) +
                             " : " + str(kwargs))
            raise Exception("unused kwargs" +
                             " : " + str(kwargs))
        # automatically do some things that should be done
        self.log("dropping prior information")
        pi = None
        try:
            pi = self.pst.prior_information
        except:
            self.logger.warn("unable to access self.pst: can't tell if " +
                             " any prior information needs to be dropped.")
        if pi is not None:
            self.drop_prior_information()
        self.log("dropping prior information")


        if resfile != False:
            self.log("scaling obscov by residual phi components")
            try:
                self.adjust_obscov_resfile(resfile=resfile)
            except:
                self.logger.warn("unable to a find a residuals file for " +\
                                " scaling obscov")
                self.resfile = None
                self.res = None
            self.log("scaling obscov by residual phi components")

    def __fromfile(self, filename):
        """a private method to deduce and load a filename into a matrix object

        Parameters:
        ----------
            filename (str) : the name of the file
        Returns:
        -------
            mat (or cov) object
        """
        assert os.path.exists(filename),"LinearAnalysis.__fromfile(): " +\
                                        "file not found:" + filename
        ext = filename.split('.')[-1].lower()
        if ext in ["jco", "jcb"]:
            self.log("loading jco: "+filename)
            m = Jco.from_binary(filename)
            self.log("loading jco: "+filename)
        elif ext in ["mat","vec"]:
            self.log("loading ascii: "+filename)
            m = Matrix.from_ascii(filename)
            self.log("loading ascii: "+filename)
        elif ext in ["cov"]:
            self.log("loading cov: "+filename)
            m = Cov.from_ascii(filename)
            self.log("loading cov: "+filename)
        elif ext in["unc"]:
            self.log("loading unc: "+filename)
            m = Cov.from_uncfile(filename)
            self.log("loading unc: "+filename)
        else:
            raise Exception("linear_analysis.__fromfile(): unrecognized" +
                            " filename extension:" + str(ext))
        return m


    def __load_pst(self):
        """private: set the pst attribute
        Parameters:
        ----------
            None
        Returns:
        -------
            None
        """
        if self.pst_arg is None:
            return None
        if isinstance(self.pst_arg, Pst):
            self.__pst = self.pst_arg
            return self.pst
        else:
            try:
                self.log("loading pst: " + str(self.pst_arg))
                self.__pst = Pst(self.pst_arg)
                self.log("loading pst: " + str(self.pst_arg))
                return self.pst
            except Exception as e:
                raise Exception("linear_analysis.__load_pst(): error loading"+\
                                " pest control from argument: " +
                                str(self.pst_arg) + '\n->' + str(e))


    def __load_jco(self):
        """private :set the jco attribute from a file or a matrix object
        Parameters:
        ----------
            None
        Returns:
        -------
            None
        """
        if self.jco_arg is None:
            return None
            #raise Exception("linear_analysis.__load_jco(): jco_arg is None")
        if isinstance(self.jco_arg, Matrix):
            self.__jco = self.jco_arg
        elif isinstance(self.jco_arg, str):
            self.__jco = self.__fromfile(self.jco_arg)
        else:
            raise Exception("linear_analysis.__load_jco(): jco_arg must " +
                            "be a matrix object or a file name: " +
                            str(self.jco_arg))

    def __load_parcov(self):
        """private: set the parcov attribute from:
                a pest control file (parameter bounds)
                a pst object
                a matrix object
                an uncert file
                an ascii matrix file
        """
        # if the parcov arg was not passed but the pst arg was,
        # reset and use parbounds to build parcov
        if not self.parcov_arg:
            if self.pst_arg:
                self.parcov_arg = self.pst_arg
            else:
                raise Exception("linear_analysis.__load_parcov(): " +
                                "parcov_arg is None")
        if isinstance(self.parcov_arg, Matrix):
            self.__parcov = self.parcov_arg
            return
        if isinstance(self.parcov_arg, np.ndarray):
            # if the passed array is a vector,
            # then assume it is the diagonal of the parcov matrix
            if len(self.parcov_arg.shape) == 1:
                assert self.parcov_arg.shape[0] == self.jco.shape[1]
                isdiagonal = True
            else:
                assert self.parcov_arg.shape[0] == self.jco.shape[1]
                assert self.parcov_arg.shape[1] == self.jco.shape[1]
                isdiagonal = False
            self.logger.warn("linear_analysis.__load_parcov(): " +
                             "instantiating parcov from ndarray, can't " +
                             "verify parameters alignment with jco")
            self.__parcov = Matrix(x=self.parcov_arg,
                                         isdiagonal=isdiagonal,
                                         row_names=self.jco.col_names,
                                         col_names=self.jco.col_names)
        self.log("loading parcov")
        if isinstance(self.parcov_arg,str):
            # if the arg is a string ending with "pst"
            # then load parcov from parbounds
            if self.parcov_arg.lower().endswith(".pst"):
                self.__parcov = Cov.from_parbounds(self.parcov_arg)
            else:
                self.__parcov = self.__fromfile(self.parcov_arg)
        # if the arg is a pst object
        elif isinstance(self.parcov_arg,Pst):
            self.__parcov = Cov.from_parameter_data(self.parcov_arg)
        else:
            raise Exception("linear_analysis.__load_parcov(): " +
                            "parcov_arg must be a " +
                            "matrix object or a file name: " +
                            str(self.parcov_arg))
        self.log("loading parcov")


    def __load_obscov(self):
        """private: method to set the obscov attribute from:
                a pest control file (observation weights)
                a pst object
                a matrix object
                an uncert file
                an ascii matrix file
        """
        # if the obscov arg is None, but the pst arg is not None,
        # reset and load from obs weights
        if not self.obscov_arg:
            if self.pst_arg:
                self.obscov_arg = self.pst_arg
            else:
                raise Exception("linear_analysis.__load_obscov(): " +
                                "obscov_arg is None")
        if isinstance(self.obscov_arg, Matrix):
            self.__obscov = self.obscov_arg
            return
        if isinstance(self.obscov_arg,np.ndarray):
            # if the ndarray arg is a vector,
            # assume it is the diagonal of the obscov matrix
            if len(self.obscov_arg.shape) == 1:
                assert self.obscov_arg.shape[0] == self.jco.shape[1]
                isdiagonal = True
            else:
                assert self.obscov_arg.shape[0] == self.jco.shape[0]
                assert self.obscov_arg.shape[1] == self.jco.shape[0]
                isdiagonal = False
            self.logger.warn("linear_analysis.__load_obscov(): " +
                             "instantiating obscov from ndarray,  " +
                             "can't verify observation alignment with jco")
            self.__obscov = Matrix(x=self.obscov_arg,
                                         isdiagonal=isdiagonal,
                                         row_names=self.jco.row_names,
                                         col_names=self.jco.row_names)
        self.log("loading obscov")
        if isinstance(self.obscov_arg, str):
            if self.obscov_arg.lower().endswith(".pst"):
                self.__obscov = Cov.from_obsweights(self.obscov_arg)
            else:
                self.__obscov = self.__fromfile(self.obscov_arg)
        elif isinstance(self.obscov_arg, Pst):
            self.__obscov = Cov.from_observation_data(self.obscov_arg)
        else:
            raise Exception("linear_analysis.__load_obscov(): " +
                            "obscov_arg must be a " +
                            "matrix object or a file name: " +
                            str(self.obscov_arg))
        self.log("loading obscov")


    def __load_predictions(self):
        """private: set the predictions attribute from:
                mixed list of row names, matrix files and ndarrays
                a single row name
                an ascii file
            can be none if only interested in parameters.

            linear_analysis.__predictions is stored as a list of column vectors

        """
        if self.prediction_arg is None:
            self.__predictions = None
            return
        self.log("loading forecasts")
        if not isinstance(self.prediction_arg, list):
            self.prediction_arg = [self.prediction_arg]

        row_names = []
        vecs = []
        for arg in self.prediction_arg:
            if isinstance(arg, Matrix):
                # a vector
                if arg.shape[1] == 1:
                    vecs.append(arg)
                else:
                    assert arg.shape[1] == self.jco.shape[1],\
                    "linear_analysis.__load_predictions(): " +\
                    "multi-prediction matrix(npred,npar) not aligned " +\
                    "with jco(nobs,npar): " + str(arg.shape) +\
                    ' ' + str(self.jco.shape)
                    for pred_name in arg.row_names:
                        vecs.append(arg.extract(row_names=pred_name).T)
            elif isinstance(arg, str):
                if arg.lower() in self.jco.row_names:
                    row_names.append(arg.lower())
                else:
                    pred_mat = self.__fromfile(arg)
                    # vector
                    if pred_mat.shape[1] == 1:
                        vecs.append(pred_mat)
                    else:
                        for pred_name in pred_mat.row_names:
                            vecs.append(pred_mat.get(row_names=pred_name))
            elif isinstance(arg, np.ndarray):
                self.logger.warn("linear_analysis.__load_predictions(): " +
                                "instantiating prediction matrix from " +
                                "ndarray, can't verify alignment")
                self.logger.warn("linear_analysis.__load_predictions(): " +
                                 "instantiating prediction matrix from " +
                                 "ndarray, generating generic prediction names")

                pred_names = ["pred_{0}".format(i+1) for i in range(arg.shape[0])]

                if self.jco:
                    names = self.jco.col_names
                elif self.parcov:
                    names = self.parcov.col_names
                else:
                    raise Exception("linear_analysis.__load_predictions(): " +
                                    "ndarray passed for predicitons " +
                                    "requires jco or parcov to get " +
                                    "parameter names")
                pred_matrix = Matrix(x=arg,row_names=pred_names,col_names=names)
                for pred_name in pred_names:
                    vecs.append(pred_matrix.get(row_names=pred_name).T)
            else:
                raise Exception("unrecognized predictions argument: " +
                                str(arg))
        if len(row_names) > 0:
            extract = self.jco.extract(row_names=row_names)
            for row_name in row_names:
                vecs.append(extract.get(row_names=row_name).T)

            # call obscov to load __obscov so that __obscov
            # (priavte) can be manipulated
            self.obscov
            self.__obscov.drop(row_names, axis=0)
        self.__predictions = vecs
        self.log("loading forecasts")
        return self.__predictions

    # these property decorators help keep from loading potentially
    # unneeded items until they are called
    # returns a reference - cheap, but can be dangerous

    @property
    def forecast_names(self):
        if self.forecasts is None:
            return []
        return [fore.col_names[0] for fore in self.forecasts]

    @property
    def parcov(self):
        if not self.__parcov:
            self.__load_parcov()
        return self.__parcov


    @property
    def obscov(self):
        if not self.__obscov:
            self.__load_obscov()
        return self.__obscov


    @property
    def jco(self):
        if not self.__jco:
            self.__load_jco()
        return self.__jco


    @property
    def predictions(self):
        if not self.__predictions:
            self.__load_predictions()
        return self.__predictions

    @property
    def forecasts(self):
        return self.predictions

    @property
    def pst(self):
        if self.__pst is None and self.pst_arg is None:
            raise Exception("linear_analysis.pst: can't access self.pst:" +
                            "no pest control argument passed")
        elif self.__pst:
            return self.__pst
        else:
            self.__load_pst()
            return self.__pst


    @property
    def fehalf(self):
        """set the KL parcov scaling matrix attribute
        """
        if self.__fehalf != None:
            return self.__fehalf
        self.log("fehalf")
        self.__fehalf = self.parcov.u * (self.parcov.s ** (0.5))
        self.log("fehalf")
        return self.__fehalf


    @property
    def qhalf(self):
        """set the square root of the cofactor matrix attribute
        """
        if self.__qhalf != None:
            return self.__qhalf
        self.log("qhalf")
        self.__qhalf = self.obscov ** (-0.5)
        self.log("qhalf")
        return self.__qhalf

    @property
    def qhalfx(self):
        """set the half normal matrix attribute
        """
        if self.__qhalfx is None:
            self.log("qhalfx")
            self.__qhalfx = self.qhalf * self.jco
            self.log("qhalfx")
        return self.__qhalfx

    @property
    def xtqx(self):
        """set the normal matrix attribute
        """
        if self.__xtqx is None:
            self.log("xtqx")
            self.__xtqx = self.jco.T * (self.obscov ** -1) * self.jco
            self.log("xtqx")
        return self.__xtqx

    @property
    def mle_covariance(self):
        return self.xtqx.inv

    @property
    def prior_parameter(self):
        """the prior parameter covariance matrix
        """
        return self.parcov

    @property
    def prior_forecast(self):
        """thin wrapper for prior_prediction
        """
        return self.prior_prediction


    @property
    def mle_parameter_estimate(self):
        res = self.pst.res
        assert res is not None
        # build the prior expectation parameter vector
        prior_expt = self.pst.parameter_data.loc[:,["parval1"]].copy()
        islog = self.pst.parameter_data.partrans == "log"
        prior_expt.loc[islog] = prior_expt.loc[islog].apply(np.log10)
        prior_expt = Matrix.from_dataframe(prior_expt)
        prior_expt.col_names = ["prior_expt"]
        # build the residual vector
        res_vec = Matrix.from_dataframe(res.loc[:,["residual"]])

        # calc posterior expectation
        upgrade = self.mle_covariance * self.jco.T * res_vec
        upgrade.col_names = ["prior_expt"]
        post_expt = prior_expt + upgrade

        # post processing - back log transform
        post_expt = pd.DataFrame(data=post_expt.x,index=post_expt.row_names,
                                 columns=["post_expt"])
        post_expt.loc[islog,:] = 10.0**post_expt.loc[islog,:]
        return post_expt

    @property
    def prior_prediction(self):
        """get a dict of prior prediction variances
        Parameters:
        ----------
            None
        Returns:
        -------
            dict{prediction name(str):prior variance(float)}
        """
        if self.__prior_prediction is not None:
            return self.__prior_prediction
        else:
            if self.predictions is not None:
                self.log("propagating prior to predictions")
                pred_dict = {}
                for prediction in self.predictions:
                    var = (prediction.T * self.parcov * prediction).x[0, 0]
                    pred_dict[prediction.col_names[0]] = var
                self.__prior_prediction = pred_dict
                self.log("propagating prior to predictions")
            else:
                self.__prior_prediction = {}
            return self.__prior_prediction


    def apply_karhunen_loeve_scaling(self):
        """apply karhuene-loeve scaling to the jacobian matrix.

            This scaling is not necessary for analyses using Schur's
            complement, but can be very important for error variance
            analyses.  This operation effectively transfers prior knowledge
            specified in the parcov to the jacobian and reset parcov to the
            identity matrix.
        """
        cnames = copy.deepcopy(self.jco.col_names)
        self.__jco *= self.fehalf
        self.__jco.col_names = cnames
        self.__parcov = self.parcov.identity


    def clean(self):
        """drop regularization and prior information observation from the jco
        """
        if self.pst_arg is None:
            self.logger.warn("linear_analysis.clean(): not pst object")
            return
        if not self.pst.estimation and self.pst.nprior > 0:
            self.drop_prior_information()

    def reset_pst(self,arg):
        self.logger.warn("resetting pst")
        self.__pst = None
        self.pst_arg = arg

    def reset_parcov(self,arg=None):
        """reset the parcov attribute to None
        Parameters:
        ----------
            arg (str or matrix) : the value to assign to the parcov_arg attrib
        Returns:
        -------
            None
        """
        self.logger.warn("resetting parcov")
        self.__parcov = None
        if arg is not None:
            self.parcov_arg = arg


    def reset_obscov(self,arg=None):
        """reset the obscov attribute to None
        Parameters:
        ----------
            arg (str or matrix) : the value to assign to the obscov_arg attrib
        Returns:
        -------
            None
        """
        self.logger.warn("resetting obscov")
        self.__obscov = None
        if arg is not None:
            self.obscov_arg = arg


    def drop_prior_information(self):
        """drop the prior information from the jco and pst attributes
        """
        if self.jco is None:
            self.logger.warn("can't drop prior info, LinearAnalysis.jco is None")
            return
        nprior_str = str(self.pst.nprior)
        self.log("removing " + nprior_str + " prior info from jco, pst, and " +
                                            "obs cov")
        #pi_names = list(self.pst.prior_information.pilbl.values)
        pi_names = list(self.pst.prior_names)
        missing = [name for name in pi_names if name not in self.jco.obs_names]
        if len(missing) > 0:
            raise Exception("LinearAnalysis.drop_prior_information(): "+
                            " prior info not found: {0}".format(missing))
        if self.jco is not None:
            self.__jco.drop(pi_names, axis=0)
        self.__pst.prior_information = self.pst.null_prior
        self.__pst.control_data.pestmode = "estimation"
        #self.__obscov.drop(pi_names,axis=0)
        self.log("removing " + nprior_str + " prior info from jco and pst")


    def get(self,par_names=None,obs_names=None,astype=None):
        """method to get a new LinearAnalysis class using a
             subset of parameters and/or observations
         Parameters:
         ----------
            par_names (enumerable of str) : par names for new object
            obs_names (enumerable of str) : obs names for new object
            astype (either schur or errvar type) : type to cast the new object
        Returns:
        -------
            LinearAnalysis object
        """
        # make sure we aren't fooling with unwanted prior information
        self.clean()
        # if there is nothing to do but copy
        if par_names is None and obs_names is None:
            if astype is not None:
                self.logger.warn("LinearAnalysis.get(): astype is not None, " +
                                 "but par_names and obs_names are None so" +
                                 "\n  ->Omitted attributes will not be " +
                                 "propagated to new instance")
            else:
                return copy.deepcopy(self)
        # make sure the args are lists
        if par_names is not None and not isinstance(par_names, list):
            par_names = [par_names]
        if obs_names is not None and not isinstance(obs_names, list):
            obs_names = [obs_names]

        if par_names is None:
            par_names = self.jco.col_names
        if obs_names is None:
            obs_names = self.jco.row_names
        # if possible, get a new parcov
        if self.parcov:
            new_parcov = self.parcov.get(col_names=par_names)
        else:
            new_parcov = None
        # if possible, get a new obscov
        if self.obscov_arg is not None:
            new_obscov = self.obscov.get(row_names=obs_names)
        else:
            new_obscov = None
        # if possible, get a new pst
        if self.pst_arg is not None:
            new_pst = self.pst.get(par_names=par_names,obs_names=obs_names)
        else:
            new_pst = None
        if self.predictions:
            new_preds = []
            for prediction in self.predictions:
                new_preds.append(prediction.get(row_names=par_names))
        else:
            new_preds = None
        if self.jco_arg is not None:
            new_jco = self.jco.get(row_names=obs_names, col_names=par_names)
        else:
            new_jco = None
        if astype is not None:
            return astype(jco=new_jco, pst=new_pst, parcov=new_parcov,
                          obscov=new_obscov, predictions=new_preds,
                          verbose=False)
        else:
            # return a new object of the same type
            return type(self)(jco=new_jco, pst=new_pst, parcov=new_parcov,
                              obscov=new_obscov, predictions=new_preds,
                              verbose=False)


    def adjust_obscov_resfile(self, resfile=None):
        """reset the elements of obscov by scaling the implied weights
        based on the phi components in res_file
        """
        self.pst.adjust_weights_resfile(resfile)
        self.__obscov.from_observation_data(self.pst)
Пример #4
0
class LinearAnalysis(object):
    """ the super class for linear analysis.  Can be used for prior analyses
        only.  The derived types (schur and errvar) are for posterior analyses
        this class tries hard to not load items until they are needed
        all arguments are optional

    Parameters:
    ----------
        jco ([enumerable of] [string,ndarray,matrix objects]) : jacobian
        pst (pst object) : the pest control file object
        parcov ([enumerable of] [string,ndarray,matrix objects]) :
            parameter covariance matrix
        obscov ([enumerable of] [string,ndarray,matrix objects]):
            observation noise covariance matrix
        predictions ([enumerable of] [string,ndarray,matrix objects]) :
            prediction sensitivity vectors
        ref_var (float) : reference variance
        verbose (either bool or string) : controls log file / screen output
    Notes:
        the class makes heavy use of property decorator to encapsulate
        private attributes
    """
    def __init__(self,
                 jco=None,
                 pst=None,
                 parcov=None,
                 obscov=None,
                 predictions=None,
                 ref_var=1.0,
                 verbose=False,
                 resfile=False,
                 forecasts=None,
                 **kwargs):
        self.logger = logger(verbose)
        self.log = self.logger.log
        self.jco_arg = jco
        #if jco is None:
        self.__jco = jco
        if pst is None:
            if isinstance(jco, str):
                pst_case = jco.replace(".jco", ".pst").replace(".jcb", ".pst")
                if os.path.exists(pst_case):
                    pst = pst_case
        self.pst_arg = pst
        if parcov is None and pst is not None:
            parcov = pst
        self.parcov_arg = parcov
        if obscov is None and pst is not None:
            obscov = pst
        self.obscov_arg = obscov
        self.ref_var = ref_var
        if forecasts is not None and predictions is not None:
            raise Exception("can't pass both forecasts and predictions")

        #private attributes - access is through @decorated functions
        self.__pst = None
        self.__parcov = None
        self.__obscov = None
        self.__predictions = None
        self.__qhalf = None
        self.__qhalfx = None
        self.__xtqx = None
        self.__fehalf = None
        self.__prior_prediction = None

        self.log("pre-loading base components")
        if jco is not None:
            self.__load_jco()
        if pst is not None:
            self.__load_pst()
        if parcov is not None:
            self.__load_parcov()
        if obscov is not None:
            self.__load_obscov()

        self.prediction_arg = None
        if predictions is not None:
            self.prediction_arg = predictions
        elif forecasts is not None:
            self.prediction_arg = forecasts
        elif self.pst is not None and self.jco is not None:
            if "forecasts" in self.pst.pestpp_options:
                self.prediction_arg = [i.strip() for i in self.pst.pestpp_options["forecasts"].\
                    lower().split(',')]
            elif "predictions" in self.pst.pestpp_options:
                self.prediction_arg = [i.strip() for i in self.pst.pestpp_options["predictions"].\
                    lower().split(',')]
        if self.prediction_arg:
            self.__load_predictions()

        self.log("pre-loading base components")
        if len(kwargs.keys()) > 0:
            self.logger.warn("unused kwargs in type " +
                             str(self.__class__.__name__) + " : " +
                             str(kwargs))
            raise Exception("unused kwargs" + " : " + str(kwargs))
        # automatically do some things that should be done
        self.log("dropping prior information")
        pi = None
        try:
            pi = self.pst.prior_information
        except:
            self.logger.warn("unable to access self.pst: can't tell if " +
                             " any prior information needs to be dropped.")
        if pi is not None:
            self.drop_prior_information()
        self.log("dropping prior information")

        if resfile != False:
            self.log("scaling obscov by residual phi components")
            try:
                self.adjust_obscov_resfile(resfile=resfile)
            except:
                self.logger.warn("unable to a find a residuals file for " +\
                                " scaling obscov")
                self.resfile = None
                self.res = None
            self.log("scaling obscov by residual phi components")

    def __fromfile(self, filename):
        """a private method to deduce and load a filename into a matrix object

        Parameters:
        ----------
            filename (str) : the name of the file
        Returns:
        -------
            mat (or cov) object
        """
        assert os.path.exists(filename),"LinearAnalysis.__fromfile(): " +\
                                        "file not found:" + filename
        ext = filename.split('.')[-1].lower()
        if ext in ["jco", "jcb"]:
            self.log("loading jco: " + filename)
            m = Jco.from_binary(filename)
            self.log("loading jco: " + filename)
        elif ext in ["mat", "vec"]:
            self.log("loading ascii: " + filename)
            m = Matrix.from_ascii(filename)
            self.log("loading ascii: " + filename)
        elif ext in ["cov"]:
            self.log("loading cov: " + filename)
            m = Cov.from_ascii(filename)
            self.log("loading cov: " + filename)
        elif ext in ["unc"]:
            self.log("loading unc: " + filename)
            m = Cov.from_uncfile(filename)
            self.log("loading unc: " + filename)
        else:
            raise Exception("linear_analysis.__fromfile(): unrecognized" +
                            " filename extension:" + str(ext))
        return m

    def __load_pst(self):
        """private: set the pst attribute
        Parameters:
        ----------
            None
        Returns:
        -------
            None
        """
        if self.pst_arg is None:
            return None
        if isinstance(self.pst_arg, Pst):
            self.__pst = self.pst_arg
            return self.pst
        else:
            try:
                self.log("loading pst: " + str(self.pst_arg))
                self.__pst = Pst(self.pst_arg)
                self.log("loading pst: " + str(self.pst_arg))
                return self.pst
            except Exception as e:
                raise Exception("linear_analysis.__load_pst(): error loading"+\
                                " pest control from argument: " +
                                str(self.pst_arg) + '\n->' + str(e))

    def __load_jco(self):
        """private :set the jco attribute from a file or a matrix object
        Parameters:
        ----------
            None
        Returns:
        -------
            None
        """
        if self.jco_arg is None:
            return None
            #raise Exception("linear_analysis.__load_jco(): jco_arg is None")
        if isinstance(self.jco_arg, Matrix):
            self.__jco = self.jco_arg
        elif isinstance(self.jco_arg, str):
            self.__jco = self.__fromfile(self.jco_arg)
        else:
            raise Exception("linear_analysis.__load_jco(): jco_arg must " +
                            "be a matrix object or a file name: " +
                            str(self.jco_arg))

    def __load_parcov(self):
        """private: set the parcov attribute from:
                a pest control file (parameter bounds)
                a pst object
                a matrix object
                an uncert file
                an ascii matrix file
        """
        # if the parcov arg was not passed but the pst arg was,
        # reset and use parbounds to build parcov
        if not self.parcov_arg:
            if self.pst_arg:
                self.parcov_arg = self.pst_arg
            else:
                raise Exception("linear_analysis.__load_parcov(): " +
                                "parcov_arg is None")
        if isinstance(self.parcov_arg, Matrix):
            self.__parcov = self.parcov_arg
            return
        if isinstance(self.parcov_arg, np.ndarray):
            # if the passed array is a vector,
            # then assume it is the diagonal of the parcov matrix
            if len(self.parcov_arg.shape) == 1:
                assert self.parcov_arg.shape[0] == self.jco.shape[1]
                isdiagonal = True
            else:
                assert self.parcov_arg.shape[0] == self.jco.shape[1]
                assert self.parcov_arg.shape[1] == self.jco.shape[1]
                isdiagonal = False
            self.logger.warn("linear_analysis.__load_parcov(): " +
                             "instantiating parcov from ndarray, can't " +
                             "verify parameters alignment with jco")
            self.__parcov = Matrix(x=self.parcov_arg,
                                   isdiagonal=isdiagonal,
                                   row_names=self.jco.col_names,
                                   col_names=self.jco.col_names)
        self.log("loading parcov")
        if isinstance(self.parcov_arg, str):
            # if the arg is a string ending with "pst"
            # then load parcov from parbounds
            if self.parcov_arg.lower().endswith(".pst"):
                self.__parcov = Cov.from_parbounds(self.parcov_arg)
            else:
                self.__parcov = self.__fromfile(self.parcov_arg)
        # if the arg is a pst object
        elif isinstance(self.parcov_arg, Pst):
            self.__parcov = Cov.from_parameter_data(self.parcov_arg)
        else:
            raise Exception("linear_analysis.__load_parcov(): " +
                            "parcov_arg must be a " +
                            "matrix object or a file name: " +
                            str(self.parcov_arg))
        self.log("loading parcov")

    def __load_obscov(self):
        """private: method to set the obscov attribute from:
                a pest control file (observation weights)
                a pst object
                a matrix object
                an uncert file
                an ascii matrix file
        """
        # if the obscov arg is None, but the pst arg is not None,
        # reset and load from obs weights
        if not self.obscov_arg:
            if self.pst_arg:
                self.obscov_arg = self.pst_arg
            else:
                raise Exception("linear_analysis.__load_obscov(): " +
                                "obscov_arg is None")
        if isinstance(self.obscov_arg, Matrix):
            self.__obscov = self.obscov_arg
            return
        if isinstance(self.obscov_arg, np.ndarray):
            # if the ndarray arg is a vector,
            # assume it is the diagonal of the obscov matrix
            if len(self.obscov_arg.shape) == 1:
                assert self.obscov_arg.shape[0] == self.jco.shape[1]
                isdiagonal = True
            else:
                assert self.obscov_arg.shape[0] == self.jco.shape[0]
                assert self.obscov_arg.shape[1] == self.jco.shape[0]
                isdiagonal = False
            self.logger.warn("linear_analysis.__load_obscov(): " +
                             "instantiating obscov from ndarray,  " +
                             "can't verify observation alignment with jco")
            self.__obscov = Matrix(x=self.obscov_arg,
                                   isdiagonal=isdiagonal,
                                   row_names=self.jco.row_names,
                                   col_names=self.jco.row_names)
        self.log("loading obscov")
        if isinstance(self.obscov_arg, str):
            if self.obscov_arg.lower().endswith(".pst"):
                self.__obscov = Cov.from_obsweights(self.obscov_arg)
            else:
                self.__obscov = self.__fromfile(self.obscov_arg)
        elif isinstance(self.obscov_arg, Pst):
            self.__obscov = Cov.from_observation_data(self.obscov_arg)
        else:
            raise Exception("linear_analysis.__load_obscov(): " +
                            "obscov_arg must be a " +
                            "matrix object or a file name: " +
                            str(self.obscov_arg))
        self.log("loading obscov")

    def __load_predictions(self):
        """private: set the predictions attribute from:
                mixed list of row names, matrix files and ndarrays
                a single row name
                an ascii file
            can be none if only interested in parameters.

            linear_analysis.__predictions is stored as a list of column vectors

        """
        if self.prediction_arg is None:
            self.__predictions = None
            return
        self.log("loading forecasts")
        if not isinstance(self.prediction_arg, list):
            self.prediction_arg = [self.prediction_arg]

        row_names = []
        vecs = []
        for arg in self.prediction_arg:
            if isinstance(arg, Matrix):
                # a vector
                if arg.shape[1] == 1:
                    vecs.append(arg)
                else:
                    assert arg.shape[1] == self.jco.shape[1],\
                    "linear_analysis.__load_predictions(): " +\
                    "multi-prediction matrix(npred,npar) not aligned " +\
                    "with jco(nobs,npar): " + str(arg.shape) +\
                    ' ' + str(self.jco.shape)
                    for pred_name in arg.row_names:
                        vecs.append(arg.extract(row_names=pred_name).T)
            elif isinstance(arg, str):
                if arg.lower() in self.jco.row_names:
                    row_names.append(arg.lower())
                else:
                    pred_mat = self.__fromfile(arg)
                    # vector
                    if pred_mat.shape[1] == 1:
                        vecs.append(pred_mat)
                    else:
                        for pred_name in pred_mat.row_names:
                            vecs.append(pred_mat.get(row_names=pred_name))
            elif isinstance(arg, np.ndarray):
                self.logger.warn("linear_analysis.__load_predictions(): " +
                                 "instantiating prediction matrix from " +
                                 "ndarray, can't verify alignment")
                self.logger.warn(
                    "linear_analysis.__load_predictions(): " +
                    "instantiating prediction matrix from " +
                    "ndarray, generating generic prediction names")

                pred_names = [
                    "pred_{0}".format(i + 1) for i in range(arg.shape[0])
                ]

                if self.jco:
                    names = self.jco.col_names
                elif self.parcov:
                    names = self.parcov.col_names
                else:
                    raise Exception("linear_analysis.__load_predictions(): " +
                                    "ndarray passed for predicitons " +
                                    "requires jco or parcov to get " +
                                    "parameter names")
                pred_matrix = Matrix(x=arg,
                                     row_names=pred_names,
                                     col_names=names)
                for pred_name in pred_names:
                    vecs.append(pred_matrix.get(row_names=pred_name).T)
            else:
                raise Exception("unrecognized predictions argument: " +
                                str(arg))
        if len(row_names) > 0:
            extract = self.jco.extract(row_names=row_names)
            for row_name in row_names:
                vecs.append(extract.get(row_names=row_name).T)

            # call obscov to load __obscov so that __obscov
            # (priavte) can be manipulated
            self.obscov
            self.__obscov.drop(row_names, axis=0)
        self.__predictions = vecs
        self.log("loading forecasts")
        return self.__predictions

    # these property decorators help keep from loading potentially
    # unneeded items until they are called
    # returns a reference - cheap, but can be dangerous

    @property
    def parcov(self):
        if not self.__parcov:
            self.__load_parcov()
        return self.__parcov

    @property
    def obscov(self):
        if not self.__obscov:
            self.__load_obscov()
        return self.__obscov

    @property
    def jco(self):
        if not self.__jco:
            self.__load_jco()
        return self.__jco

    @property
    def predictions(self):
        if not self.__predictions:
            self.__load_predictions()
        return self.__predictions

    @property
    def forecasts(self):
        return self.predictions

    @property
    def pst(self):
        if self.__pst is None and self.pst_arg is None:
            raise Exception("linear_analysis.pst: can't access self.pst:" +
                            "no pest control argument passed")
        elif self.__pst:
            return self.__pst
        else:
            self.__load_pst()
            return self.__pst

    @property
    def fehalf(self):
        """set the KL parcov scaling matrix attribute
        """
        if self.__fehalf != None:
            return self.__fehalf
        self.log("fehalf")
        self.__fehalf = self.parcov.u * (self.parcov.s**(0.5))
        self.log("fehalf")
        return self.__fehalf

    @property
    def qhalf(self):
        """set the square root of the cofactor matrix attribute
        """
        if self.__qhalf != None:
            return self.__qhalf
        self.log("qhalf")
        self.__qhalf = self.obscov**(-0.5)
        self.log("qhalf")
        return self.__qhalf

    @property
    def qhalfx(self):
        """set the half normal matrix attribute
        """
        if self.__qhalfx is None:
            self.log("qhalfx")
            self.__qhalfx = self.qhalf * self.jco
            self.log("qhalfx")
        return self.__qhalfx

    @property
    def xtqx(self):
        """set the normal matrix attribute
        """
        if self.__xtqx is None:
            self.log("xtqx")
            self.__xtqx = self.jco.T * (self.obscov**-1) * self.jco
            self.log("xtqx")
        return self.__xtqx

    @property
    def prior_parameter(self):
        """the prior parameter covariance matrix
        """
        return self.parcov

    @property
    def prior_forecast(self):
        """thin wrapper for prior_prediction
        """
        return self.prior_prediction

    @property
    def prior_prediction(self):
        """get a dict of prior prediction variances
        Parameters:
        ----------
            None
        Returns:
        -------
            dict{prediction name(str):prior variance(float)}
        """
        if self.__prior_prediction is not None:
            return self.__prior_prediction
        else:
            if self.predictions is not None:
                self.log("propagating prior to predictions")
                pred_dict = {}
                for prediction in self.predictions:
                    var = (prediction.T * self.parcov * prediction).x[0, 0]
                    pred_dict[prediction.col_names[0]] = var
                self.__prior_prediction = pred_dict
                self.log("propagating prior to predictions")
            else:
                self.__prior_prediction = {}
            return self.__prior_prediction

    def apply_karhunen_loeve_scaling(self):
        """apply karhuene-loeve scaling to the jacobian matrix.

            This scaling is not necessary for analyses using Schur's
            complement, but can be very important for error variance
            analyses.  This operation effectively transfers prior knowledge
            specified in the parcov to the jacobian and reset parcov to the
            identity matrix.
        """
        cnames = copy.deepcopy(self.jco.col_names)
        self.__jco *= self.fehalf
        self.__jco.col_names = cnames
        self.__parcov = self.parcov.identity

    def clean(self):
        """drop regularization and prior information observation from the jco
        """
        if self.pst_arg is None:
            self.logger.warn("linear_analysis.clean(): not pst object")
            return
        if not self.pst.estimation and self.pst.nprior > 0:
            self.drop_prior_information()

    def reset_pst(self, arg):
        self.logger.warn("resetting pst")
        self.__pst = None
        self.pst_arg = arg

    def reset_parcov(self, arg=None):
        """reset the parcov attribute to None
        Parameters:
        ----------
            arg (str or matrix) : the value to assign to the parcov_arg attrib
        Returns:
        -------
            None
        """
        self.logger.warn("resetting parcov")
        self.__parcov = None
        if arg is not None:
            self.parcov_arg = arg

    def reset_obscov(self, arg=None):
        """reset the obscov attribute to None
        Parameters:
        ----------
            arg (str or matrix) : the value to assign to the obscov_arg attrib
        Returns:
        -------
            None
        """
        self.logger.warn("resetting obscov")
        self.__obscov = None
        if arg is not None:
            self.obscov_arg = arg

    def drop_prior_information(self):
        """drop the prior information from the jco and pst attributes
        """
        if self.jco is None:
            self.logger.warn(
                "can't drop prior info, LinearAnalysis.jco is None")
            return
        nprior_str = str(self.pst.nprior)
        self.log("removing " + nprior_str + " prior info from jco, pst, and " +
                 "obs cov")
        #pi_names = list(self.pst.prior_information.pilbl.values)
        pi_names = list(self.pst.prior_names)
        missing = [name for name in pi_names if name not in self.jco.obs_names]
        if len(missing) > 0:
            raise Exception("LinearAnalysis.drop_prior_information(): " +
                            " prior info not found: {0}".format(missing))
        if self.jco is not None:
            self.__jco.drop(pi_names, axis=0)
        self.__pst.prior_information = self.pst.null_prior
        self.__pst.control_data.pestmode = "estimation"
        #self.__obscov.drop(pi_names,axis=0)
        self.log("removing " + nprior_str + " prior info from jco and pst")

    def get(self, par_names=None, obs_names=None, astype=None):
        """method to get a new LinearAnalysis class using a
             subset of parameters and/or observations
         Parameters:
         ----------
            par_names (enumerable of str) : par names for new object
            obs_names (enumerable of str) : obs names for new object
            astype (either schur or errvar type) : type to cast the new object
        Returns:
        -------
            LinearAnalysis object
        """
        # make sure we aren't fooling with unwanted prior information
        self.clean()
        # if there is nothing to do but copy
        if par_names is None and obs_names is None:
            if astype is not None:
                self.logger.warn("LinearAnalysis.get(): astype is not None, " +
                                 "but par_names and obs_names are None so" +
                                 "\n  ->Omitted attributes will not be " +
                                 "propagated to new instance")
            else:
                return copy.deepcopy(self)
        # make sure the args are lists
        if par_names is not None and not isinstance(par_names, list):
            par_names = [par_names]
        if obs_names is not None and not isinstance(obs_names, list):
            obs_names = [obs_names]

        if par_names is None:
            par_names = self.jco.col_names
        if obs_names is None:
            obs_names = self.jco.row_names
        # if possible, get a new parcov
        if self.parcov:
            new_parcov = self.parcov.get(col_names=par_names)
        else:
            new_parcov = None
        # if possible, get a new obscov
        if self.obscov_arg is not None:
            new_obscov = self.obscov.get(row_names=obs_names)
        else:
            new_obscov = None
        # if possible, get a new pst
        if self.pst_arg is not None:
            new_pst = self.pst.get(par_names=par_names, obs_names=obs_names)
        else:
            new_pst = None
        if self.predictions:
            new_preds = []
            for prediction in self.predictions:
                new_preds.append(prediction.get(row_names=par_names))
        else:
            new_preds = None
        if self.jco_arg is not None:
            new_jco = self.jco.get(row_names=obs_names, col_names=par_names)
        else:
            new_jco = None
        if astype is not None:
            return astype(jco=new_jco,
                          pst=new_pst,
                          parcov=new_parcov,
                          obscov=new_obscov,
                          predictions=new_preds,
                          verbose=False)
        else:
            # return a new object of the same type
            return type(self)(jco=new_jco,
                              pst=new_pst,
                              parcov=new_parcov,
                              obscov=new_obscov,
                              predictions=new_preds,
                              verbose=False)

    def adjust_obscov_resfile(self, resfile=None):
        """reset the elements of obscov by scaling the implied weights
        based on the phi components in res_file
        """
        self.pst.adjust_weights_resfile(resfile)
        self.__obscov.from_observation_data(self.pst)