class LinearAnalysis(object): """The super class for linear analysis. Can be used directly, but for prior uncertainty analyses only. The derived types (pyemu.Schur, pyemu.ErrVar, pyemu.MonteCarlo, pyemu.EnsembleSmoother) are for different forms of posterior uncertainty analyses. This class tries hard to not load items until they are needed; all arguments are optional Parameters ---------- jco : (varies) something that can be cast or loaded into a pyemu.Jco. Can be a str for a filename or pyemu.Matrix/pyemu.Jco object. pst : (varies) something that can be cast into a pyemu.Pst. Can be an str for a filename or an existing pyemu.Pst. If None, a pst filename is sought with the same base name as the jco argument (if passed) parcov : (varies) prior parameter covariance matrix. If str, a filename is assumed and the prior parameter covariance matrix is loaded from a file using the file extension. If None, the prior parameter covariance matrix is constructed from the parameter bounds in the control file represented by the pst argument. Can also be a pyemu.Cov instance obscov : (varies) observation noise covariance matrix. If str, a filename is assumed and the observation noise covariance matrix is loaded from a file using the file extension. If None, the observation noise covariance matrix is constructed from the weights in the control file represented by the pst argument. Can also be a pyemu.Cov instance predictions : (varies) prediction (aka forecast) sensitivity vectors. If str, a filename is assumed and predictions are loaded from a file using the file extension. Can also be a pyemu.Matrix instance, a numpy.ndarray or a collection of pyemu.Matrix or numpy.ndarray. ref_var : (float) reference variance verbose : (either bool or string) controls log file / screen output. If str, a filename is assumed and and log file is written to verbose Note ---- the class makes heavy use of property decorator to encapsulate private attributes """ def __init__(self, jco=None, pst=None, parcov=None, obscov=None, predictions=None, ref_var=1.0, verbose=False, resfile=False, forecasts=None, **kwargs): self.logger = Logger(verbose) self.log = self.logger.log self.jco_arg = jco #if jco is None: self.__jco = jco if pst is None: if isinstance(jco, str): pst_case = jco.replace(".jco", ".pst").replace(".jcb", ".pst") if os.path.exists(pst_case): pst = pst_case self.pst_arg = pst if parcov is None and pst is not None: parcov = pst self.parcov_arg = parcov if obscov is None and pst is not None: obscov = pst self.obscov_arg = obscov self.ref_var = ref_var if forecasts is not None and predictions is not None: raise Exception("can't pass both forecasts and predictions") #private attributes - access is through @decorated functions self.__pst = None self.__parcov = None self.__obscov = None self.__predictions = None self.__qhalf = None self.__qhalfx = None self.__xtqx = None self.__fehalf = None self.__prior_prediction = None self.prediction_extract = None self.log("pre-loading base components") if jco is not None: self.__load_jco() if pst is not None: self.__load_pst() if parcov is not None: self.__load_parcov() if obscov is not None: self.__load_obscov() self.prediction_arg = None if predictions is not None: self.prediction_arg = predictions elif forecasts is not None: self.prediction_arg = forecasts elif self.pst is not None and self.jco is not None: if self.pst.forecast_names is not None: self.prediction_arg = self.pst.forecast_names if self.prediction_arg: self.__load_predictions() self.log("pre-loading base components") if len(kwargs.keys()) > 0: self.logger.warn("unused kwargs in type " + str(self.__class__.__name__) + " : " + str(kwargs)) raise Exception("unused kwargs" + " : " + str(kwargs)) # automatically do some things that should be done self.log("dropping prior information") pi = None try: pi = self.pst.prior_information except: self.logger.warn("unable to access self.pst: can't tell if " + " any prior information needs to be dropped.") if pi is not None: self.drop_prior_information() self.log("dropping prior information") if resfile != False: self.log("scaling obscov by residual phi components") try: self.adjust_obscov_resfile(resfile=resfile) except: self.logger.warn("unable to a find a residuals file for " +\ " scaling obscov") self.resfile = None self.res = None self.log("scaling obscov by residual phi components") assert type(self.parcov) == Cov assert type(self.obscov) == Cov def __fromfile(self, filename, astype=None): """a private method to deduce and load a filename into a matrix object. Uses extension: 'jco' or 'jcb': binary, 'mat','vec' or 'cov': ASCII, 'unc': pest uncertainty file. Parameters ---------- filename : str the name of the file Returns ------- m : pyemu.Matrix Raises ------ exception for unrecognized extension """ assert os.path.exists(filename),"LinearAnalysis.__fromfile(): " +\ "file not found:" + filename ext = filename.split('.')[-1].lower() if ext in ["jco", "jcb"]: self.log("loading jco: " + filename) if astype is None: astype = Jco m = astype.from_binary(filename) self.log("loading jco: " + filename) elif ext in ["mat", "vec"]: self.log("loading ascii: " + filename) if astype is None: astype = Matrix m = astype.from_ascii(filename) self.log("loading ascii: " + filename) elif ext in ["cov"]: self.log("loading cov: " + filename) if astype is None: astype = Cov m = astype.from_ascii(filename) self.log("loading cov: " + filename) elif ext in ["unc"]: self.log("loading unc: " + filename) if astype is None: astype = Cov m = astype.from_uncfile(filename) self.log("loading unc: " + filename) else: raise Exception("linear_analysis.__fromfile(): unrecognized" + " filename extension:" + str(ext)) return m def __load_pst(self): """private method set the pst attribute """ if self.pst_arg is None: return None if isinstance(self.pst_arg, Pst): self.__pst = self.pst_arg return self.pst else: try: self.log("loading pst: " + str(self.pst_arg)) self.__pst = Pst(self.pst_arg) self.log("loading pst: " + str(self.pst_arg)) return self.pst except Exception as e: raise Exception("linear_analysis.__load_pst(): error loading"+\ " pest control from argument: " + str(self.pst_arg) + '\n->' + str(e)) def __load_jco(self): """private method to set the jco attribute from a file or a matrix object """ if self.jco_arg is None: return None #raise Exception("linear_analysis.__load_jco(): jco_arg is None") if isinstance(self.jco_arg, Matrix): self.__jco = self.jco_arg elif isinstance(self.jco_arg, str): self.__jco = self.__fromfile(self.jco_arg, astype=Jco) else: raise Exception("linear_analysis.__load_jco(): jco_arg must " + "be a matrix object or a file name: " + str(self.jco_arg)) def __load_parcov(self): """private method to set the parcov attribute from: a pest control file (parameter bounds) a pst object a matrix object an uncert file an ascii matrix file """ # if the parcov arg was not passed but the pst arg was, # reset and use parbounds to build parcov if not self.parcov_arg: if self.pst_arg: self.parcov_arg = self.pst_arg else: raise Exception("linear_analysis.__load_parcov(): " + "parcov_arg is None") if isinstance(self.parcov_arg, Matrix): self.__parcov = self.parcov_arg return if isinstance(self.parcov_arg, np.ndarray): # if the passed array is a vector, # then assume it is the diagonal of the parcov matrix if len(self.parcov_arg.shape) == 1: assert self.parcov_arg.shape[0] == self.jco.shape[1] isdiagonal = True else: assert self.parcov_arg.shape[0] == self.jco.shape[1] assert self.parcov_arg.shape[1] == self.jco.shape[1] isdiagonal = False self.logger.warn("linear_analysis.__load_parcov(): " + "instantiating parcov from ndarray, can't " + "verify parameters alignment with jco") self.__parcov = Matrix(x=self.parcov_arg, isdiagonal=isdiagonal, row_names=self.jco.col_names, col_names=self.jco.col_names) self.log("loading parcov") if isinstance(self.parcov_arg, str): # if the arg is a string ending with "pst" # then load parcov from parbounds if self.parcov_arg.lower().endswith(".pst"): self.__parcov = Cov.from_parbounds(self.parcov_arg) else: self.__parcov = self.__fromfile(self.parcov_arg, astype=Cov) # if the arg is a pst object elif isinstance(self.parcov_arg, Pst): self.__parcov = Cov.from_parameter_data(self.parcov_arg) else: raise Exception("linear_analysis.__load_parcov(): " + "parcov_arg must be a " + "matrix object or a file name: " + str(self.parcov_arg)) self.log("loading parcov") def __load_obscov(self): """private method to set the obscov attribute from: a pest control file (observation weights) a pst object a matrix object an uncert file an ascii matrix file """ # if the obscov arg is None, but the pst arg is not None, # reset and load from obs weights if not self.obscov_arg: if self.pst_arg: self.obscov_arg = self.pst_arg else: raise Exception("linear_analysis.__load_obscov(): " + "obscov_arg is None") if isinstance(self.obscov_arg, Matrix): self.__obscov = self.obscov_arg return if isinstance(self.obscov_arg, np.ndarray): # if the ndarray arg is a vector, # assume it is the diagonal of the obscov matrix if len(self.obscov_arg.shape) == 1: assert self.obscov_arg.shape[0] == self.jco.shape[1] isdiagonal = True else: assert self.obscov_arg.shape[0] == self.jco.shape[0] assert self.obscov_arg.shape[1] == self.jco.shape[0] isdiagonal = False self.logger.warn("linear_analysis.__load_obscov(): " + "instantiating obscov from ndarray, " + "can't verify observation alignment with jco") self.__obscov = Matrix(x=self.obscov_arg, isdiagonal=isdiagonal, row_names=self.jco.row_names, col_names=self.jco.row_names) self.log("loading obscov") if isinstance(self.obscov_arg, str): if self.obscov_arg.lower().endswith(".pst"): self.__obscov = Cov.from_obsweights(self.obscov_arg) else: self.__obscov = self.__fromfile(self.obscov_arg, astype=Cov) elif isinstance(self.obscov_arg, Pst): self.__obscov = Cov.from_observation_data(self.obscov_arg) else: raise Exception("linear_analysis.__load_obscov(): " + "obscov_arg must be a " + "matrix object or a file name: " + str(self.obscov_arg)) self.log("loading obscov") def __load_predictions(self): """private method set the predictions attribute from: mixed list of row names, matrix files and ndarrays a single row name an ascii file can be none if only interested in parameters. """ if self.prediction_arg is None: self.__predictions = None return self.log("loading forecasts") if not isinstance(self.prediction_arg, list): self.prediction_arg = [self.prediction_arg] row_names = [] vecs = [] mat = None for arg in self.prediction_arg: if isinstance(arg, Matrix): # a vector if arg.shape[1] == 1: vecs.append(arg) else: assert arg.shape[0] == self.jco.shape[1],\ "linear_analysis.__load_predictions(): " +\ "multi-prediction matrix(npar,npred) not aligned " +\ "with jco(nobs,npar): " + str(arg.shape) +\ ' ' + str(self.jco.shape) #for pred_name in arg.row_names: # vecs.append(arg.extract(row_names=pred_name).T) mat = arg elif isinstance(arg, str): if arg.lower() in self.jco.row_names: row_names.append(arg.lower()) else: try: pred_mat = self.__fromfile(arg, astype=Matrix) except Exception as e: raise Exception("forecast argument: "+arg+" not found in " +\ "jco row names and could not be " +\ "loaded from a file.") # vector if pred_mat.shape[1] == 1: vecs.append(pred_mat) else: #for pred_name in pred_mat.row_names: # vecs.append(pred_mat.get(row_names=pred_name)) if mat is None: mat = pred_mat else: mat = mat.extend((pred_mat)) elif isinstance(arg, np.ndarray): self.logger.warn("linear_analysis.__load_predictions(): " + "instantiating prediction matrix from " + "ndarray, can't verify alignment") self.logger.warn( "linear_analysis.__load_predictions(): " + "instantiating prediction matrix from " + "ndarray, generating generic prediction names") pred_names = [ "pred_{0}".format(i + 1) for i in range(arg.shape[0]) ] if self.jco: names = self.jco.col_names elif self.parcov: names = self.parcov.col_names else: raise Exception("linear_analysis.__load_predictions(): " + "ndarray passed for predicitons " + "requires jco or parcov to get " + "parameter names") if mat is None: mat = Matrix(x=arg, row_names=pred_names, col_names=names).T else: mat = mat.extend( Matrix(x=arg, row_names=pred_names, col_names=names).T) #for pred_name in pred_names: # vecs.append(pred_matrix.get(row_names=pred_name).T) else: raise Exception("unrecognized predictions argument: " + str(arg)) # turn vecs into a pyemu.Matrix if len(vecs) > 0: xs = vecs[0].x for vec in vecs[1:]: xs = xs.extend(vec.x) names = [vec.col_names[0] for vec in vecs] if mat is None: mat = Matrix(x=xs, row_names=vecs[0].row_names, col_names=names) else: mat = mat.extend( Matrix(x=np.array(xs), row_names=vecs[0].row_names, col_names=names)) if len(row_names) > 0: extract = self.jco.extract(row_names=row_names).T if mat is None: mat = extract else: mat = mat.extend(extract) #for row_name in row_names: # vecs.append(extract.get(row_names=row_name).T) # call obscov to load __obscov so that __obscov # (priavte) can be manipulated self.obscov self.__obscov.drop(row_names, axis=0) self.__predictions = mat try: fnames = [ fname for fname in self.forecast_names if fname in self.pst.nnz_obs_names ] except: fnames = [] if len(fnames) > 0: raise Exception( "forecasts with non-zero weight in pst: {0}".format( ','.join(fnames))) self.log("loading forecasts") self.logger.statement("forecast names: {0}".format(','.join( mat.col_names))) return self.__predictions # these property decorators help keep from loading potentially # unneeded items until they are called # returns a reference - cheap, but can be dangerous @property def forecast_names(self): """ get the forecast (aka prediction) names Returns ------- forecast_names : list list of forecast names """ if self.forecasts is None: return [] #return [fore.col_names[0] for fore in self.forecasts] return list(self.predictions.col_names) @property def parcov(self): """ get the prior parameter covariance matrix attribute Return ------ parcov : pyemu.Cov a reference to the parcov attribute Note ---- returns a reference if LinearAnalysis.__parcov is not set, the dynamically load the parcov attribute before returning """ if not self.__parcov: self.__load_parcov() return self.__parcov @property def obscov(self): """ get the observation noise covariance matrix attribute Returns ------- obscov : pyemu.Cov a reference to the obscov attribute Note ---- returns a reference if LinearAnalysis.__obscov is not set, the dynamically load the obscov attribute before returning """ if not self.__obscov: self.__load_obscov() return self.__obscov @property def nnz_obs_names(self): """ wrapper around pyemu.Pst.nnz_obs_names for listing non-zero observation names Returns ------- nnz_obs_names : list pyemu.Pst.nnz_obs_names """ if self.__pst is not None: return self.pst.nnz_obs_names else: return self.jco.obs_names def adj_par_names(self): """ wrapper around pyemu.Pst.adj_par_names for list adjustable parameter names Returns ------- adj_par_names : list pyemu.Pst.adj_par_names """ if self.__pst is not None: return self.pst.adj_par_names else: return self.jco.par_names @property def jco(self): """ get the jacobian matrix attribute Returns ------- jco : pyemu.Jco the jacobian matrix attribute Note ---- returns a reference if LinearAnalysis.__jco is not set, the dynamically load the jco attribute before returning """ if not self.__jco: self.__load_jco() return self.__jco @property def predictions(self): """ get the predictions (aka forecasts) attribute Returns ------- predictions : pyemu.Matrix a matrix of prediction sensitivity vectors (column wise) Note ---- returns a reference if LinearAnalysis.__predictions is not set, the dynamically load the predictions attribute before returning """ if not self.__predictions: self.__load_predictions() return self.__predictions @property def predictions_iter(self): """ property decorated prediction iterator Returns ------- iterator : iterator iterator on prediction sensitivity vectors (matrix) """ for fname in self.forecast_names: yield self.predictions.get(col_names=fname) @property def forecasts_iter(self): """synonym for LinearAnalysis.predictions_iter() """ return self.predictions_iter @property def forecasts(self): """synonym for LinearAnalysis.predictions """ return self.predictions @property def pst(self): """ get the pyemu.Pst attribute Returns ------- pst : pyemu.Pst Note ---- returns a references If LinearAnalysis.__pst is None, then the pst attribute is dynamically loaded before returning """ if self.__pst is None and self.pst_arg is None: raise Exception("linear_analysis.pst: can't access self.pst:" + "no pest control argument passed") elif self.__pst: return self.__pst else: self.__load_pst() return self.__pst @property def fehalf(self): """get the KL parcov scaling matrix attribute. Create the attribute if it has not yet been created Returns ------- fehalf : pyemu.Matrix """ if self.__fehalf != None: return self.__fehalf self.log("fehalf") self.__fehalf = self.parcov.u * (self.parcov.s**(0.5)) self.log("fehalf") return self.__fehalf @property def qhalf(self): """get the square root of the cofactor matrix attribute. Create the attribute if it has not yet been created Returns ------- qhalf : pyemu.Matrix """ if self.__qhalf != None: return self.__qhalf self.log("qhalf") self.__qhalf = self.obscov**(-0.5) self.log("qhalf") return self.__qhalf @property def qhalfx(self): """get the half normal matrix attribute. Create the attribute if it has not yet been created Returns ------- qhalfx : pyemu.Matrix """ if self.__qhalfx is None: self.log("qhalfx") self.__qhalfx = self.qhalf * self.jco self.log("qhalfx") return self.__qhalfx @property def xtqx(self): """get the normal matrix attribute. Create the attribute if it has not yet been created Returns ------- xtqx : pyemu.Matrix """ if self.__xtqx is None: self.log("xtqx") self.__xtqx = self.jco.T * (self.obscov**-1) * self.jco self.log("xtqx") return self.__xtqx @property def mle_covariance(self): """ get the maximum likelihood parameter covariance matrix. Returns ------- pyemu.Matrix : pyemu.Matrix """ return self.xtqx.inv @property def prior_parameter(self): """the prior parameter covariance matrix. Just a wrapper around LinearAnalysis.parcov Returns ------- prior_parameter : pyemu.Cov """ return self.parcov @property def prior_forecast(self): """thin wrapper for prior_prediction Returns ------- prior_forecast : dict a dictionary of forecast name, prior variance pairs """ return self.prior_prediction @property def mle_parameter_estimate(self): """ get the maximum likelihood parameter estimate. Returns ------- post_expt : pandas.Series the maximum likelihood parameter estimates """ res = self.pst.res assert res is not None # build the prior expectation parameter vector prior_expt = self.pst.parameter_data.loc[:, ["parval1"]].copy() islog = self.pst.parameter_data.partrans == "log" prior_expt.loc[islog] = prior_expt.loc[islog].apply(np.log10) prior_expt = Matrix.from_dataframe(prior_expt) prior_expt.col_names = ["prior_expt"] # build the residual vector res_vec = Matrix.from_dataframe(res.loc[:, ["residual"]]) # calc posterior expectation upgrade = self.mle_covariance * self.jco.T * res_vec upgrade.col_names = ["prior_expt"] post_expt = prior_expt + upgrade # post processing - back log transform post_expt = pd.DataFrame(data=post_expt.x, index=post_expt.row_names, columns=["post_expt"]) post_expt.loc[islog, :] = 10.0**post_expt.loc[islog, :] return post_expt @property def prior_prediction(self): """get a dict of prior prediction variances Returns ------- prior_prediction : dict dictionary of prediction name, prior variance pairs """ if self.__prior_prediction is not None: return self.__prior_prediction else: if self.predictions is not None: self.log("propagating prior to predictions") prior_cov = self.predictions.T *\ self.parcov * self.predictions self.__prior_prediction = { n: v for n, v in zip(prior_cov.row_names, np.diag(prior_cov.x)) } self.log("propagating prior to predictions") else: self.__prior_prediction = {} return self.__prior_prediction def apply_karhunen_loeve_scaling(self): """apply karhuene-loeve scaling to the jacobian matrix. Note ---- This scaling is not necessary for analyses using Schur's complement, but can be very important for error variance analyses. This operation effectively transfers prior knowledge specified in the parcov to the jacobian and reset parcov to the identity matrix. """ cnames = copy.deepcopy(self.jco.col_names) self.__jco *= self.fehalf self.__jco.col_names = cnames self.__parcov = self.parcov.identity def clean(self): """drop regularization and prior information observation from the jco """ if self.pst_arg is None: self.logger.warn("linear_analysis.clean(): not pst object") return if not self.pst.estimation and self.pst.nprior > 0: self.drop_prior_information() def reset_pst(self, arg): """ reset the LinearAnalysis.pst attribute Parameters ---------- arg : (str or matrix) the value to assign to the pst attribute """ self.logger.warn("resetting pst") self.__pst = None self.pst_arg = arg def reset_parcov(self, arg=None): """reset the parcov attribute to None Parameters ---------- arg : str or pyemu.Matrix the value to assign to the parcov attribute. If None, the private __parcov attribute is cleared but not reset """ self.logger.warn("resetting parcov") self.__parcov = None if arg is not None: self.parcov_arg = arg def reset_obscov(self, arg=None): """reset the obscov attribute to None Parameters ---------- arg : str or pyemu.Matrix the value to assign to the obscov attribute. If None, the private __obscov attribute is cleared but not reset """ self.logger.warn("resetting obscov") self.__obscov = None if arg is not None: self.obscov_arg = arg def drop_prior_information(self): """drop the prior information from the jco and pst attributes """ if self.jco is None: self.logger.warn( "can't drop prior info, LinearAnalysis.jco is None") return nprior_str = str(self.pst.nprior) self.log("removing " + nprior_str + " prior info from jco, pst, and " + "obs cov") #pi_names = list(self.pst.prior_information.pilbl.values) pi_names = list(self.pst.prior_names) missing = [name for name in pi_names if name not in self.jco.obs_names] if len(missing) > 0: raise Exception("LinearAnalysis.drop_prior_information(): " + " prior info not found: {0}".format(missing)) if self.jco is not None: self.__jco.drop(pi_names, axis=0) self.__pst.prior_information = self.pst.null_prior self.__pst.control_data.pestmode = "estimation" #self.__obscov.drop(pi_names,axis=0) self.log("removing " + nprior_str + " prior info from jco, pst, and " + "obs cov") def get(self, par_names=None, obs_names=None, astype=None): """method to get a new LinearAnalysis class using a subset of parameters and/or observations Parameters ---------- par_names : list par names for new object obs_names : list obs names for new object astype : pyemu.Schur or pyemu.ErrVar type to cast the new object. If None, return type is same as self Returns ------- new : LinearAnalysis """ # make sure we aren't fooling with unwanted prior information self.clean() # if there is nothing to do but copy if par_names is None and obs_names is None: if astype is not None: self.logger.warn("LinearAnalysis.get(): astype is not None, " + "but par_names and obs_names are None so" + "\n ->Omitted attributes will not be " + "propagated to new instance") else: return copy.deepcopy(self) # make sure the args are lists if par_names is not None and not isinstance(par_names, list): par_names = [par_names] if obs_names is not None and not isinstance(obs_names, list): obs_names = [obs_names] if par_names is None: par_names = self.jco.col_names if obs_names is None: obs_names = self.jco.row_names # if possible, get a new parcov if self.parcov: new_parcov = self.parcov.get(col_names=[pname for pname in\ par_names if pname in\ self.parcov.col_names]) else: new_parcov = None # if possible, get a new obscov if self.obscov_arg is not None: new_obscov = self.obscov.get(row_names=obs_names) else: new_obscov = None # if possible, get a new pst if self.pst_arg is not None: new_pst = self.pst.get(par_names=par_names, obs_names=obs_names) else: new_pst = None new_extract = None if self.predictions: # new_preds = [] # for prediction in self.predictions: # new_preds.append(prediction.get(row_names=par_names)) new_preds = self.predictions.get(row_names=par_names) else: new_preds = None if self.jco_arg is not None: new_jco = self.jco.get(row_names=obs_names, col_names=par_names) else: new_jco = None if astype is not None: new = astype(jco=new_jco, pst=new_pst, parcov=new_parcov, obscov=new_obscov, predictions=new_preds, verbose=False) else: # return a new object of the same type new = type(self)(jco=new_jco, pst=new_pst, parcov=new_parcov, obscov=new_obscov, predictions=new_preds, verbose=False) return new def adjust_obscov_resfile(self, resfile=None): """reset the elements of obscov by scaling the implied weights based on the phi components in res_file so that the total phi is equal to the number of non-zero weights. Parameters ---------- resfile : str residual file to use. If None, residual file with case name is sought. default is None Note ---- calls pyemu.Pst.adjust_weights_resfile() """ self.pst.adjust_weights_resfile(resfile) self.__obscov.from_observation_data(self.pst) def get_par_css_dataframe(self): """ get a dataframe of composite scaled sensitivities. Includes both PEST-style and Hill-style. Returns ------- css : pandas.DataFrame """ assert self.jco is not None assert self.pst is not None jco = self.jco.to_dataframe() weights = self.pst.observation_data.loc[jco.index, "weight"].copy().values jco = (jco.T * weights).T dss_sum = jco.apply(np.linalg.norm) css = (dss_sum / float(self.pst.nnz_obs)).to_frame() css.columns = ["pest_css"] # log transform stuff self.pst.add_transform_columns() parval1 = self.pst.parameter_data.loc[dss_sum.index, "parval1_trans"].values css.loc[:, "hill_css"] = (dss_sum * parval1) / (float(self.pst.nnz_obs) **2) return css
class LinearAnalysis(object): """The base/parent class for linear analysis. Args: jco (varies, optional): something that can be cast or loaded into a `pyemu.Jco`. Can be a str for a filename or `pyemu.Matrix`/`pyemu.Jco` object. pst (varies, optional): something that can be cast into a `pyemu.Pst`. Can be an `str` for a filename or an existing `pyemu.Pst`. If `None`, a pst filename is sought with the same base name as the jco argument (if passed) parcov (varies, optional): prior parameter covariance matrix. If `str`, a filename is assumed and the prior parameter covariance matrix is loaded from a file using the file extension (".jcb"/".jco" for binary, ".cov"/".mat" for PEST-style ASCII matrix, or ".unc" for uncertainty files). If `None`, the prior parameter covariance matrix is constructed from the parameter bounds in `LinearAnalysis.pst`. Can also be a `pyemu.Cov` instance obscov (varies, optional): observation noise covariance matrix. If `str`, a filename is assumed and the noise covariance matrix is loaded from a file using the file extension (".jcb"/".jco" for binary, ".cov"/".mat" for PEST-style ASCII matrix, or ".unc" for uncertainty files). If `None`, the noise covariance matrix is constructed from the obsevation weights in `LinearAnalysis.pst`. Can also be a `pyemu.Cov` instance forecasts (varies, optional): forecast sensitivity vectors. If `str`, first an observation name is assumed (a row in `LinearAnalysis.jco`). If that is not found, a filename is assumed and predictions are loaded from a file using the file extension. If [`str`], a list of observation names is assumed. Can also be a `pyemu.Matrix` instance, a `numpy.ndarray` or a collection of `pyemu.Matrix` or `numpy.ndarray`. ref_var (float, optional): reference variance. Default is 1.0 verbose (`bool`): controls screen output. If `str`, a filename is assumed and and log file is written. sigma_range (`float`, optional): defines range of upper bound - lower bound in terms of standard deviation (sigma). For example, if sigma_range = 4, the bounds represent 4 * sigma. Default is 4.0, representing approximately 95% confidence of implied normal distribution. This arg is only used if constructing parcov from parameter bounds. scale_offset (`bool`, optional): flag to apply parameter scale and offset to parameter bounds when calculating prior parameter covariance matrix from bounds. This arg is onlyused if constructing parcov from parameter bounds.Default is True. Note: Can be used directly, but for prior uncertainty analyses only. The derived types (`pyemu.Schur`, `pyemu.ErrVar`) are for different forms of FOMS-based posterior uncertainty analyses. This class tries hard to not load items until they are needed; all arguments are optional. The class makes heavy use of property decorator to encapsulated private attributes Example:: #assumes "my.pst" exists la = pyemu.LinearAnalysis(jco="my.jco",forecasts=["fore1","fore2"]) print(la.prior_forecast) """ def __init__(self, jco=None, pst=None, parcov=None, obscov=None, predictions=None, ref_var=1.0, verbose=False, resfile=False, forecasts=None, sigma_range=4.0, scale_offset=True, **kwargs): self.logger = Logger(verbose) self.log = self.logger.log self.jco_arg = jco # if jco is None: self.__jco = jco if pst is None: if isinstance(jco, str): pst_case = jco.replace(".jco", ".pst").replace(".jcb", ".pst") if os.path.exists(pst_case): pst = pst_case self.pst_arg = pst if parcov is None and pst is not None: parcov = pst self.parcov_arg = parcov if obscov is None and pst is not None: obscov = pst self.obscov_arg = obscov self.ref_var = ref_var if forecasts is not None and predictions is not None: raise Exception("can't pass both forecasts and predictions") self.sigma_range = sigma_range self.scale_offset = scale_offset # private attributes - access is through @decorated functions self.__pst = None self.__parcov = None self.__obscov = None self.__predictions = None self.__qhalf = None self.__qhalfx = None self.__xtqx = None self.__fehalf = None self.__prior_prediction = None self.prediction_extract = None self.log("pre-loading base components") if jco is not None: self.__load_jco() if pst is not None: self.__load_pst() if parcov is not None: self.__load_parcov() if obscov is not None: self.__load_obscov() self.prediction_arg = None if predictions is not None: self.prediction_arg = predictions elif forecasts is not None: self.prediction_arg = forecasts elif self.pst is not None and self.jco is not None: if self.pst.forecast_names is not None: self.prediction_arg = self.pst.forecast_names if self.prediction_arg: self.__load_predictions() self.log("pre-loading base components") if len(kwargs.keys()) > 0: self.logger.warn("unused kwargs in type " + str(self.__class__.__name__) + " : " + str(kwargs)) raise Exception("unused kwargs" + " : " + str(kwargs)) # automatically do some things that should be done self.log("dropping prior information") pi = None try: pi = self.pst.prior_information except: self.logger.statement( "unable to access self.pst: can't tell if " + " any prior information needs to be dropped.") if pi is not None: self.drop_prior_information() self.log("dropping prior information") if resfile != False: self.log("scaling obscov by residual phi components") try: self.adjust_obscov_resfile(resfile=resfile) except: self.logger.statement( "unable to a find a residuals file for " + " scaling obscov") self.resfile = None self.res = None self.log("scaling obscov by residual phi components") assert type(self.parcov) == Cov assert type(self.obscov) == Cov def __fromfile(self, filename, astype=None): """a private method to deduce and load a filename into a matrix object. Uses extension: 'jco' or 'jcb': binary, 'mat','vec' or 'cov': ASCII, 'unc': pest uncertainty file. """ assert os.path.exists(filename), ("LinearAnalysis.__fromfile(): " + "file not found:" + filename) ext = filename.split(".")[-1].lower() if ext in ["jco", "jcb"]: self.log("loading jco: " + filename) if astype is None: astype = Jco m = astype.from_binary(filename) self.log("loading jco: " + filename) elif ext in ["mat", "vec"]: self.log("loading ascii: " + filename) if astype is None: astype = Matrix m = astype.from_ascii(filename) self.log("loading ascii: " + filename) elif ext in ["cov"]: self.log("loading cov: " + filename) if astype is None: astype = Cov if _istextfile(filename): m = astype.from_ascii(filename) else: m = astype.from_binary(filename) self.log("loading cov: " + filename) elif ext in ["unc"]: self.log("loading unc: " + filename) if astype is None: astype = Cov m = astype.from_uncfile(filename) self.log("loading unc: " + filename) else: raise Exception("linear_analysis.__fromfile(): unrecognized" + " filename extension:" + str(ext)) return m def __load_pst(self): """private method set the pst attribute """ if self.pst_arg is None: return None if isinstance(self.pst_arg, Pst): self.__pst = self.pst_arg return self.pst else: try: self.log("loading pst: " + str(self.pst_arg)) self.__pst = Pst(self.pst_arg) self.log("loading pst: " + str(self.pst_arg)) return self.pst except Exception as e: raise Exception("linear_analysis.__load_pst(): error loading" + " pest control from argument: " + str(self.pst_arg) + "\n->" + str(e)) def __load_jco(self): """private method to set the jco attribute from a file or a matrix object """ if self.jco_arg is None: return None # raise Exception("linear_analysis.__load_jco(): jco_arg is None") if isinstance(self.jco_arg, Matrix): self.__jco = self.jco_arg elif isinstance(self.jco_arg, str): self.__jco = self.__fromfile(self.jco_arg, astype=Jco) else: raise Exception("linear_analysis.__load_jco(): jco_arg must " + "be a matrix object or a file name: " + str(self.jco_arg)) def __load_parcov(self): """private method to set the parcov attribute from: a pest control file (parameter bounds) a pst object a matrix object an uncert file an ascii matrix file """ # if the parcov arg was not passed but the pst arg was, # reset and use parbounds to build parcov if not self.parcov_arg: if self.pst_arg: self.parcov_arg = self.pst_arg else: raise Exception("linear_analysis.__load_parcov(): " + "parcov_arg is None") if isinstance(self.parcov_arg, Matrix): self.__parcov = self.parcov_arg return if isinstance(self.parcov_arg, np.ndarray): # if the passed array is a vector, # then assume it is the diagonal of the parcov matrix if len(self.parcov_arg.shape) == 1: assert self.parcov_arg.shape[0] == self.jco.shape[1] isdiagonal = True else: assert self.parcov_arg.shape[0] == self.jco.shape[1] assert self.parcov_arg.shape[1] == self.jco.shape[1] isdiagonal = False self.logger.warn("linear_analysis.__load_parcov(): " + "instantiating parcov from ndarray, can't " + "verify parameters alignment with jco") self.__parcov = Matrix( x=self.parcov_arg, isdiagonal=isdiagonal, row_names=self.jco.col_names, col_names=self.jco.col_names, ) self.log("loading parcov") if isinstance(self.parcov_arg, str): # if the arg is a string ending with "pst" # then load parcov from parbounds if self.parcov_arg.lower().endswith(".pst"): self.__parcov = Cov.from_parbounds( self.parcov_arg, sigma_range=self.sigma_range, scale_offset=self.scale_offset, ) else: self.__parcov = self.__fromfile(self.parcov_arg, astype=Cov) # if the arg is a pst object elif isinstance(self.parcov_arg, Pst): self.__parcov = Cov.from_parameter_data( self.parcov_arg, sigma_range=self.sigma_range, scale_offset=self.scale_offset, ) else: raise Exception("linear_analysis.__load_parcov(): " + "parcov_arg must be a " + "matrix object or a file name: " + str(self.parcov_arg)) self.log("loading parcov") def __load_obscov(self): """private method to set the obscov attribute from: a pest control file (observation weights) a pst object a matrix object an uncert file an ascii matrix file """ # if the obscov arg is None, but the pst arg is not None, # reset and load from obs weights if not self.obscov_arg: if self.pst_arg: self.obscov_arg = self.pst_arg else: raise Exception("linear_analysis.__load_obscov(): " + "obscov_arg is None") if isinstance(self.obscov_arg, Matrix): self.__obscov = self.obscov_arg return if isinstance(self.obscov_arg, np.ndarray): # if the ndarray arg is a vector, # assume it is the diagonal of the obscov matrix if len(self.obscov_arg.shape) == 1: assert self.obscov_arg.shape[0] == self.jco.shape[1] isdiagonal = True else: assert self.obscov_arg.shape[0] == self.jco.shape[0] assert self.obscov_arg.shape[1] == self.jco.shape[0] isdiagonal = False self.logger.warn("linear_analysis.__load_obscov(): " + "instantiating obscov from ndarray, " + "can't verify observation alignment with jco") self.__obscov = Matrix( x=self.obscov_arg, isdiagonal=isdiagonal, row_names=self.jco.row_names, col_names=self.jco.row_names, ) self.log("loading obscov") if isinstance(self.obscov_arg, str): if self.obscov_arg.lower().endswith(".pst"): self.__obscov = Cov.from_obsweights(self.obscov_arg) else: self.__obscov = self.__fromfile(self.obscov_arg, astype=Cov) elif isinstance(self.obscov_arg, Pst): self.__obscov = Cov.from_observation_data(self.obscov_arg) else: raise Exception("linear_analysis.__load_obscov(): " + "obscov_arg must be a " + "matrix object or a file name: " + str(self.obscov_arg)) self.log("loading obscov") def __load_predictions(self): """private method set the predictions attribute from: mixed list of row names, matrix files and ndarrays a single row name an ascii file can be none if only interested in parameters. """ if self.prediction_arg is None: self.__predictions = None return self.log("loading forecasts") if not isinstance(self.prediction_arg, list): self.prediction_arg = [self.prediction_arg] row_names = [] vecs = [] mat = None for arg in self.prediction_arg: if isinstance(arg, Matrix): # a vector if arg.shape[1] == 1: vecs.append(arg) else: if self.jco is not None: assert arg.shape[0] == self.jco.shape[1], ( "linear_analysis.__load_predictions(): " + "multi-prediction matrix(npar,npred) not aligned " + "with jco(nobs,npar): " + str(arg.shape) + " " + str(self.jco.shape)) # for pred_name in arg.row_names: # vecs.append(arg.extract(row_names=pred_name).T) mat = arg elif isinstance(arg, str): if arg.lower() in self.jco.row_names: row_names.append(arg.lower()) else: try: pred_mat = self.__fromfile(arg, astype=Matrix) except Exception as e: raise Exception("forecast argument: " + arg + " not found in " + "jco row names and could not be " + "loaded from a file.") # vector if pred_mat.shape[1] == 1: vecs.append(pred_mat) else: # for pred_name in pred_mat.row_names: # vecs.append(pred_mat.get(row_names=pred_name)) if mat is None: mat = pred_mat else: mat = mat.extend((pred_mat)) elif isinstance(arg, np.ndarray): self.logger.warn("linear_analysis.__load_predictions(): " + "instantiating prediction matrix from " + "ndarray, can't verify alignment") self.logger.warn( "linear_analysis.__load_predictions(): " + "instantiating prediction matrix from " + "ndarray, generating generic prediction names") pred_names = [ "pred_{0}".format(i + 1) for i in range(arg.shape[0]) ] if self.jco: names = self.jco.col_names elif self.parcov: names = self.parcov.col_names else: raise Exception("linear_analysis.__load_predictions(): " + "ndarray passed for predicitons " + "requires jco or parcov to get " + "parameter names") if mat is None: mat = Matrix(x=arg, row_names=pred_names, col_names=names).T else: mat = mat.extend( Matrix(x=arg, row_names=pred_names, col_names=names).T) # for pred_name in pred_names: # vecs.append(pred_matrix.get(row_names=pred_name).T) else: raise Exception("unrecognized predictions argument: " + str(arg)) # turn vecs into a pyemu.Matrix if len(vecs) > 0: xs = vecs[0].x for vec in vecs[1:]: xs = xs.extend(vec.x) names = [vec.col_names[0] for vec in vecs] if mat is None: mat = Matrix(x=xs, row_names=vecs[0].row_names, col_names=names) else: mat = mat.extend( Matrix(x=np.array(xs), row_names=vecs[0].row_names, col_names=names)) if len(row_names) > 0: extract = self.jco.extract(row_names=row_names).T if mat is None: mat = extract else: mat = mat.extend(extract) # for row_name in row_names: # vecs.append(extract.get(row_names=row_name).T) # call obscov to load __obscov so that __obscov # (priavte) can be manipulated # check if any forecasts are in the obscov so_names = set(self.__obscov.row_names) drop_names = [r for r in row_names if r in so_names] if len(drop_names) > 0: self.__obscov.drop(drop_names, axis=0) self.__predictions = mat try: nz_names = set(self.pst.nnz_obs_names) fnames = [ fname for fname in self.forecast_names if fname in nz_names ] if len(row_names) > 0: srow_names = set(row_names) fnames = [ fname for fname in self.forecast_names if fname in srow_names ] else: fnames = [] except: fnames = [] if len(fnames) > 0: self.logger.warn( "forecasts with non-zero weight in pst: {0}...".format( ",".join(fnames)) + "\n -> re-setting these forecast weights to zero...") self.pst.observation_data.loc[fnames, "weight"] = 0.0 self.log("loading forecasts") self.logger.statement("forecast names: {0}".format(",".join( mat.col_names))) return self.__predictions # these property decorators help keep from loading potentially # unneeded items until they are called # returns a reference - cheap, but can be dangerous @property def forecast_names(self): """ get the forecast (aka prediction) names Returns: ([`str`]): list of forecast names """ if self.forecasts is None: return [] # return [fore.col_names[0] for fore in self.forecasts] return list(self.predictions.col_names) @property def parcov(self): """ get the prior parameter covariance matrix attribute Returns: `pyemu.Cov`: a reference to the `LinearAnalysis.parcov` attribute """ if not self.__parcov: self.__load_parcov() return self.__parcov @property def obscov(self): """ get the observation noise covariance matrix attribute Returns: `pyemu.Cov`: a reference to the `LinearAnalysis.obscov` attribute """ if not self.__obscov: self.__load_obscov() return self.__obscov @property def nnz_obs_names(self): """non-zero-weighted observation names Returns: ['str`]: list of non-zero-weighted observation names Note: if `LinearAnalysis.pst` is `None`, returns `LinearAnalysis.jco.row_names` """ if self.__pst is not None: return self.pst.nnz_obs_names else: return self.jco.obs_names @property def adj_par_names(self): """ adjustable parameter names Returns: ['str`]: list of adjustable parameter names Note: if `LinearAnalysis.pst` is `None`, returns `LinearAnalysis.jco.col_names` """ if self.__pst is not None: return self.pst.adj_par_names else: return self.jco.par_names @property def jco(self): """ the jacobian matrix attribute Returns: `pyemu.Jco`: the jacobian matrix attribute """ if not self.__jco: self.__load_jco() return self.__jco @property def predictions(self): """the prediction (aka forecast) sentivity vectors attribute Returns: `pyemu.Matrix`: a matrix of prediction sensitivity vectors (column wise) """ if not self.__predictions: self.__load_predictions() return self.__predictions @property def predictions_iter(self): """prediction sensitivity vectors iterator Returns: `iterator`: iterator on prediction sensitivity vectors (matrix) Note: this is used for processing huge numbers of predictions """ for fname in self.forecast_names: yield self.predictions.get(col_names=fname) @property def forecasts_iter(self): """forecast (e.g. prediction) sensitivity vectors iterator Returns: `iterator`: iterator on forecasts (e.g. predictions) sensitivity vectors (matrix) Note: This is used for processing huge numbers of predictions This is a synonym for LinearAnalysis.predictions_iter() """ return self.predictions_iter @property def forecasts(self): """the forecast sentivity vectors attribute Returns: `pyemu.Matrix`: a matrix of forecast (prediction) sensitivity vectors (column wise) """ return self.predictions @property def pst(self): """the pst attribute Returns: `pyemu.Pst`: the pst attribute """ if self.__pst is None and self.pst_arg is None: raise Exception("linear_analysis.pst: can't access self.pst:" + "no pest control argument passed") elif self.__pst: return self.__pst else: self.__load_pst() return self.__pst @property def fehalf(self): """Karhunen-Loeve scaling matrix attribute. Returns: `pyemu.Matrix`: the Karhunen-Loeve scaling matrix based on the prior parameter covariance matrix """ if self.__fehalf != None: return self.__fehalf self.log("fehalf") self.__fehalf = self.parcov.u * (self.parcov.s**(0.5)) self.log("fehalf") return self.__fehalf @property def qhalf(self): """square root of the cofactor matrix attribute. Create the attribute if it has not yet been created Returns: `pyemu.Matrix`: square root of the cofactor matrix """ if self.__qhalf != None: return self.__qhalf self.log("qhalf") self.__qhalf = self.obscov**(-0.5) self.log("qhalf") return self.__qhalf @property def qhalfx(self): """half normal matrix attribute. Returns: `pyemu.Matrix`: half normal matrix attribute """ if self.__qhalfx is None: self.log("qhalfx") self.__qhalfx = self.qhalf * self.jco self.log("qhalfx") return self.__qhalfx @property def xtqx(self): """normal matrix attribute. Returns: `pyemu.Matrix`: normal matrix attribute """ if self.__xtqx is None: self.log("xtqx") self.__xtqx = self.jco.T * (self.obscov**-1) * self.jco self.log("xtqx") return self.__xtqx @property def mle_covariance(self): """maximum likelihood parameter covariance matrix. Returns: `pyemu.Matrix`: maximum likelihood parameter covariance matrix """ return self.xtqx.inv @property def prior_parameter(self): """prior parameter covariance matrix Returns: `pyemu.Cov`: prior parameter covariance matrix """ return self.parcov @property def prior_forecast(self): """prior forecast (e.g. prediction) variances Returns: `dict`: a dictionary of forecast name, prior variance pairs """ return self.prior_prediction @property def mle_parameter_estimate(self): """ maximum likelihood parameter estimate. Returns: `pandas.Series`: the maximum likelihood parameter estimates """ res = self.pst.res assert res is not None # build the prior expectation parameter vector prior_expt = self.pst.parameter_data.loc[:, ["parval1"]].copy() islog = self.pst.parameter_data.partrans == "log" prior_expt.loc[islog] = prior_expt.loc[islog].apply(np.log10) prior_expt = Matrix.from_dataframe(prior_expt) prior_expt.col_names = ["prior_expt"] # build the residual vector res_vec = Matrix.from_dataframe(res.loc[:, ["residual"]]) # calc posterior expectation upgrade = self.mle_covariance * self.jco.T * res_vec upgrade.col_names = ["prior_expt"] post_expt = prior_expt + upgrade # post processing - back log transform post_expt = pd.DataFrame(data=post_expt.x, index=post_expt.row_names, columns=["post_expt"]) post_expt.loc[islog, :] = 10.0**post_expt.loc[islog, :] return post_expt @property def prior_prediction(self): """prior prediction (e.g. forecast) variances Returns: `dict`: a dictionary of prediction name, prior variance pairs """ if self.__prior_prediction is not None: return self.__prior_prediction else: if self.predictions is not None: self.log("propagating prior to predictions") prior_cov = self.predictions.T * self.parcov * self.predictions self.__prior_prediction = { n: v for n, v in zip(prior_cov.row_names, np.diag(prior_cov.x)) } self.log("propagating prior to predictions") else: self.__prior_prediction = {} return self.__prior_prediction def apply_karhunen_loeve_scaling(self): """apply karhuene-loeve scaling to the jacobian matrix. Note: This scaling is not necessary for analyses using Schur's complement, but can be very important for error variance analyses. This operation effectively transfers prior knowledge specified in the parcov to the jacobian and reset parcov to the identity matrix. """ cnames = copy.deepcopy(self.jco.col_names) self.__jco *= self.fehalf self.__jco.col_names = cnames self.__parcov = self.parcov.identity def clean(self): """drop regularization and prior information observation from the jco """ if self.pst_arg is None: self.logger.statement("linear_analysis.clean(): not pst object") return if not self.pst.estimation and self.pst.nprior > 0: self.drop_prior_information() def reset_pst(self, arg): """ reset the LinearAnalysis.pst attribute Args: arg (`str` or `pyemu.Pst`): the value to assign to the pst attribute """ self.logger.statement("resetting pst") self.__pst = None self.pst_arg = arg def reset_parcov(self, arg=None): """reset the parcov attribute to None Args: arg (`str` or `pyemu.Matrix`): the value to assign to the parcov attribute. If None, the private __parcov attribute is cleared but not reset """ self.logger.statement("resetting parcov") self.__parcov = None if arg is not None: self.parcov_arg = arg def reset_obscov(self, arg=None): """reset the obscov attribute to None Args: arg (`str` or `pyemu.Matrix`): the value to assign to the obscov attribute. If None, the private __obscov attribute is cleared but not reset """ self.logger.statement("resetting obscov") self.__obscov = None if arg is not None: self.obscov_arg = arg def drop_prior_information(self): """drop the prior information from the jco and pst attributes """ if self.jco is None: self.logger.statement( "can't drop prior info, LinearAnalysis.jco is None") return nprior_str = str(self.pst.nprior) self.log("removing " + nprior_str + " prior info from jco, pst, and " + "obs cov") # pi_names = list(self.pst.prior_information.pilbl.values) pi_names = list(self.pst.prior_names) missing = [name for name in pi_names if name not in self.jco.obs_names] if len(missing) > 0: raise Exception("LinearAnalysis.drop_prior_information(): " + " prior info not found: {0}".format(missing)) if self.jco is not None: self.__jco.drop(pi_names, axis=0) self.__pst.prior_information = self.pst.null_prior self.__pst.control_data.pestmode = "estimation" # self.__obscov.drop(pi_names,axis=0) self.log("removing " + nprior_str + " prior info from jco, pst, and " + "obs cov") def get(self, par_names=None, obs_names=None, astype=None): """method to get a new LinearAnalysis class using a subset of parameters and/or observations Args: par_names ([`'str`]): par names for new object obs_names ([`'str`]): obs names for new object astype (`pyemu.Schur` or `pyemu.ErrVar`): type to cast the new object. If None, return type is same as self Returns: `LinearAnalysis`: new instance """ # make sure we aren't fooling with unwanted prior information self.clean() # if there is nothing to do but copy if par_names is None and obs_names is None: if astype is not None: self.logger.warn("LinearAnalysis.get(): astype is not None, " + "but par_names and obs_names are None so" + "\n ->Omitted attributes will not be " + "propagated to new instance") else: return copy.deepcopy(self) # make sure the args are lists if par_names is not None and not isinstance(par_names, list): par_names = [par_names] if obs_names is not None and not isinstance(obs_names, list): obs_names = [obs_names] if par_names is None: par_names = self.jco.col_names if obs_names is None: obs_names = self.jco.row_names # if possible, get a new parcov if self.parcov: new_parcov = self.parcov.get(col_names=[ pname for pname in par_names if pname in self.parcov.col_names ]) else: new_parcov = None # if possible, get a new obscov if self.obscov_arg is not None: new_obscov = self.obscov.get(row_names=obs_names) else: new_obscov = None # if possible, get a new pst if self.pst_arg is not None: new_pst = self.pst.get(par_names=par_names, obs_names=obs_names) else: new_pst = None new_extract = None if self.predictions: # new_preds = [] # for prediction in self.predictions: # new_preds.append(prediction.get(row_names=par_names)) new_preds = self.predictions.get(row_names=par_names) else: new_preds = None if self.jco_arg is not None: new_jco = self.jco.get(row_names=obs_names, col_names=par_names) else: new_jco = None if astype is not None: new = astype( jco=new_jco, pst=new_pst, parcov=new_parcov, obscov=new_obscov, predictions=new_preds, verbose=False, ) else: # return a new object of the same type new = type(self)( jco=new_jco, pst=new_pst, parcov=new_parcov, obscov=new_obscov, predictions=new_preds, verbose=False, ) return new def adjust_obscov_resfile(self, resfile=None): """reset the elements of obscov by scaling the implied weights based on the phi components in res_file so that the total phi is equal to the number of non-zero weights. Args: resfile (`str`): residual file to use. If None, residual file with case name is sought. default is None Note: calls `pyemu.Pst.adjust_weights_resfile()` """ self.pst.adjust_weights_resfile(resfile) self.__obscov.from_observation_data(self.pst) def get_par_css_dataframe(self): """ get a dataframe of composite scaled sensitivities. Includes both PEST-style and Hill-style. Returns: `pandas.DataFrame`: a dataframe of parameter names, PEST-style and Hill-style composite scaled sensitivity """ if self.jco is None: raise Exception("jco is None") if self.pst is None: raise Exception("pst is None") jco = self.jco.to_dataframe() weights = self.pst.observation_data.loc[jco.index, "weight"].copy().values jco = (jco.T * weights).T dss_sum = jco.apply(np.linalg.norm) css = (dss_sum / float(self.pst.nnz_obs)).to_frame() css.columns = ["pest_css"] # log transform stuff self.pst.add_transform_columns() parval1 = self.pst.parameter_data.loc[dss_sum.index, "parval1_trans"].values css.loc[:, "hill_css"] = (dss_sum * parval1) / (float(self.pst.nnz_obs) **2) return css def get_cso_dataframe(self): """get a dataframe of composite observation sensitivity, as returned by PEST in the seo file. Returns: `pandas.DataFrame`: dataframe of observation names and composite observation sensitivity Note: That this formulation deviates slightly from the PEST documentation in that the values are divided by (npar-1) rather than by (npar). The equation is cso_j = ((Q^1/2*J*J^T*Q^1/2)^1/2)_jj/(NPAR-1) """ if self.jco is None: raise Exception("jco is None") if self.pst is None: raise Exception("pst is None") weights = (self.pst.observation_data.loc[self.jco.to_dataframe().index, "weight"].copy().values) cso = np.diag(np.sqrt( (self.qhalfx.x.dot(self.qhalfx.x.T)))) / (float(self.pst.npar - 1)) cso_df = pd.DataFrame.from_dict({ "obnme": self.jco.to_dataframe().index, "cso": cso }) cso_df.index = cso_df["obnme"] cso_df.drop("obnme", axis=1, inplace=True) return cso_df def get_obs_competition_dataframe(self): """get the observation competition stat a la PEST utility Returns: `pandas.DataFrame`: a dataframe of observation names by observation names with values equal to the PEST competition statistic """ if self.jco is None: raise Exception("jco is None") if self.pst is None: raise Exception("pst is None") if self.pst.res is None: raise Exception("res is None") onames = self.pst.nnz_obs_names weights = self.pst.observation_data.loc[onames, "weight"].to_dict() residuals = self.pst.res.loc[onames, "residual"].to_dict() jco = self.jco.to_dataframe() df = pd.DataFrame(columns=onames, index=onames) for i, oname in enumerate(onames): df.loc[oname, oname] = 0.0 for ooname in onames[i + 1:]: oc = (weights[oname] * weights[ooname] * np.dot(jco.loc[oname, :].values, jco.loc[ooname, :].values.transpose())) df.loc[oname, ooname] = oc df.loc[ooname, oname] = oc return df
class LinearAnalysis(object): """ the super class for linear analysis. Can be used for prior analyses only. The derived types (schur and errvar) are for posterior analyses this class tries hard to not load items until they are needed all arguments are optional Parameters: ---------- jco ([enumerable of] [string,ndarray,matrix objects]) : jacobian pst (pst object) : the pest control file object parcov ([enumerable of] [string,ndarray,matrix objects]) : parameter covariance matrix obscov ([enumerable of] [string,ndarray,matrix objects]): observation noise covariance matrix predictions ([enumerable of] [string,ndarray,matrix objects]) : prediction sensitivity vectors ref_var (float) : reference variance verbose (either bool or string) : controls log file / screen output Notes: the class makes heavy use of property decorator to encapsulate private attributes """ def __init__(self, jco=None, pst=None, parcov=None, obscov=None, predictions=None, ref_var=1.0, verbose=False, resfile=False, forecasts=None,**kwargs): self.logger = logger(verbose) self.log = self.logger.log self.jco_arg = jco #if jco is None: self.__jco = jco if pst is None: if isinstance(jco, str): pst_case = jco.replace(".jco", ".pst").replace(".jcb",".pst") if os.path.exists(pst_case): pst = pst_case self.pst_arg = pst if parcov is None and pst is not None: parcov = pst self.parcov_arg = parcov if obscov is None and pst is not None: obscov = pst self.obscov_arg = obscov self.ref_var = ref_var if forecasts is not None and predictions is not None: raise Exception("can't pass both forecasts and predictions") #private attributes - access is through @decorated functions self.__pst = None self.__parcov = None self.__obscov = None self.__predictions = None self.__qhalf = None self.__qhalfx = None self.__xtqx = None self.__fehalf = None self.__prior_prediction = None self.log("pre-loading base components") if jco is not None: self.__load_jco() if pst is not None: self.__load_pst() if parcov is not None: self.__load_parcov() if obscov is not None: self.__load_obscov() self.prediction_arg = None if predictions is not None: self.prediction_arg = predictions elif forecasts is not None: self.prediction_arg = forecasts elif self.pst is not None and self.jco is not None: if "forecasts" in self.pst.pestpp_options: self.prediction_arg = [i.strip() for i in self.pst.pestpp_options["forecasts"].\ lower().split(',')] elif "predictions" in self.pst.pestpp_options: self.prediction_arg = [i.strip() for i in self.pst.pestpp_options["predictions"].\ lower().split(',')] if self.prediction_arg: self.__load_predictions() self.log("pre-loading base components") if len(kwargs.keys()) > 0: self.logger.warn("unused kwargs in type " + str(self.__class__.__name__) + " : " + str(kwargs)) raise Exception("unused kwargs" + " : " + str(kwargs)) # automatically do some things that should be done self.log("dropping prior information") pi = None try: pi = self.pst.prior_information except: self.logger.warn("unable to access self.pst: can't tell if " + " any prior information needs to be dropped.") if pi is not None: self.drop_prior_information() self.log("dropping prior information") if resfile != False: self.log("scaling obscov by residual phi components") try: self.adjust_obscov_resfile(resfile=resfile) except: self.logger.warn("unable to a find a residuals file for " +\ " scaling obscov") self.resfile = None self.res = None self.log("scaling obscov by residual phi components") def __fromfile(self, filename): """a private method to deduce and load a filename into a matrix object Parameters: ---------- filename (str) : the name of the file Returns: ------- mat (or cov) object """ assert os.path.exists(filename),"LinearAnalysis.__fromfile(): " +\ "file not found:" + filename ext = filename.split('.')[-1].lower() if ext in ["jco", "jcb"]: self.log("loading jco: "+filename) m = Jco.from_binary(filename) self.log("loading jco: "+filename) elif ext in ["mat","vec"]: self.log("loading ascii: "+filename) m = Matrix.from_ascii(filename) self.log("loading ascii: "+filename) elif ext in ["cov"]: self.log("loading cov: "+filename) m = Cov.from_ascii(filename) self.log("loading cov: "+filename) elif ext in["unc"]: self.log("loading unc: "+filename) m = Cov.from_uncfile(filename) self.log("loading unc: "+filename) else: raise Exception("linear_analysis.__fromfile(): unrecognized" + " filename extension:" + str(ext)) return m def __load_pst(self): """private: set the pst attribute Parameters: ---------- None Returns: ------- None """ if self.pst_arg is None: return None if isinstance(self.pst_arg, Pst): self.__pst = self.pst_arg return self.pst else: try: self.log("loading pst: " + str(self.pst_arg)) self.__pst = Pst(self.pst_arg) self.log("loading pst: " + str(self.pst_arg)) return self.pst except Exception as e: raise Exception("linear_analysis.__load_pst(): error loading"+\ " pest control from argument: " + str(self.pst_arg) + '\n->' + str(e)) def __load_jco(self): """private :set the jco attribute from a file or a matrix object Parameters: ---------- None Returns: ------- None """ if self.jco_arg is None: return None #raise Exception("linear_analysis.__load_jco(): jco_arg is None") if isinstance(self.jco_arg, Matrix): self.__jco = self.jco_arg elif isinstance(self.jco_arg, str): self.__jco = self.__fromfile(self.jco_arg) else: raise Exception("linear_analysis.__load_jco(): jco_arg must " + "be a matrix object or a file name: " + str(self.jco_arg)) def __load_parcov(self): """private: set the parcov attribute from: a pest control file (parameter bounds) a pst object a matrix object an uncert file an ascii matrix file """ # if the parcov arg was not passed but the pst arg was, # reset and use parbounds to build parcov if not self.parcov_arg: if self.pst_arg: self.parcov_arg = self.pst_arg else: raise Exception("linear_analysis.__load_parcov(): " + "parcov_arg is None") if isinstance(self.parcov_arg, Matrix): self.__parcov = self.parcov_arg return if isinstance(self.parcov_arg, np.ndarray): # if the passed array is a vector, # then assume it is the diagonal of the parcov matrix if len(self.parcov_arg.shape) == 1: assert self.parcov_arg.shape[0] == self.jco.shape[1] isdiagonal = True else: assert self.parcov_arg.shape[0] == self.jco.shape[1] assert self.parcov_arg.shape[1] == self.jco.shape[1] isdiagonal = False self.logger.warn("linear_analysis.__load_parcov(): " + "instantiating parcov from ndarray, can't " + "verify parameters alignment with jco") self.__parcov = Matrix(x=self.parcov_arg, isdiagonal=isdiagonal, row_names=self.jco.col_names, col_names=self.jco.col_names) self.log("loading parcov") if isinstance(self.parcov_arg,str): # if the arg is a string ending with "pst" # then load parcov from parbounds if self.parcov_arg.lower().endswith(".pst"): self.__parcov = Cov.from_parbounds(self.parcov_arg) else: self.__parcov = self.__fromfile(self.parcov_arg) # if the arg is a pst object elif isinstance(self.parcov_arg,Pst): self.__parcov = Cov.from_parameter_data(self.parcov_arg) else: raise Exception("linear_analysis.__load_parcov(): " + "parcov_arg must be a " + "matrix object or a file name: " + str(self.parcov_arg)) self.log("loading parcov") def __load_obscov(self): """private: method to set the obscov attribute from: a pest control file (observation weights) a pst object a matrix object an uncert file an ascii matrix file """ # if the obscov arg is None, but the pst arg is not None, # reset and load from obs weights if not self.obscov_arg: if self.pst_arg: self.obscov_arg = self.pst_arg else: raise Exception("linear_analysis.__load_obscov(): " + "obscov_arg is None") if isinstance(self.obscov_arg, Matrix): self.__obscov = self.obscov_arg return if isinstance(self.obscov_arg,np.ndarray): # if the ndarray arg is a vector, # assume it is the diagonal of the obscov matrix if len(self.obscov_arg.shape) == 1: assert self.obscov_arg.shape[0] == self.jco.shape[1] isdiagonal = True else: assert self.obscov_arg.shape[0] == self.jco.shape[0] assert self.obscov_arg.shape[1] == self.jco.shape[0] isdiagonal = False self.logger.warn("linear_analysis.__load_obscov(): " + "instantiating obscov from ndarray, " + "can't verify observation alignment with jco") self.__obscov = Matrix(x=self.obscov_arg, isdiagonal=isdiagonal, row_names=self.jco.row_names, col_names=self.jco.row_names) self.log("loading obscov") if isinstance(self.obscov_arg, str): if self.obscov_arg.lower().endswith(".pst"): self.__obscov = Cov.from_obsweights(self.obscov_arg) else: self.__obscov = self.__fromfile(self.obscov_arg) elif isinstance(self.obscov_arg, Pst): self.__obscov = Cov.from_observation_data(self.obscov_arg) else: raise Exception("linear_analysis.__load_obscov(): " + "obscov_arg must be a " + "matrix object or a file name: " + str(self.obscov_arg)) self.log("loading obscov") def __load_predictions(self): """private: set the predictions attribute from: mixed list of row names, matrix files and ndarrays a single row name an ascii file can be none if only interested in parameters. linear_analysis.__predictions is stored as a list of column vectors """ if self.prediction_arg is None: self.__predictions = None return self.log("loading forecasts") if not isinstance(self.prediction_arg, list): self.prediction_arg = [self.prediction_arg] row_names = [] vecs = [] for arg in self.prediction_arg: if isinstance(arg, Matrix): # a vector if arg.shape[1] == 1: vecs.append(arg) else: assert arg.shape[1] == self.jco.shape[1],\ "linear_analysis.__load_predictions(): " +\ "multi-prediction matrix(npred,npar) not aligned " +\ "with jco(nobs,npar): " + str(arg.shape) +\ ' ' + str(self.jco.shape) for pred_name in arg.row_names: vecs.append(arg.extract(row_names=pred_name).T) elif isinstance(arg, str): if arg.lower() in self.jco.row_names: row_names.append(arg.lower()) else: pred_mat = self.__fromfile(arg) # vector if pred_mat.shape[1] == 1: vecs.append(pred_mat) else: for pred_name in pred_mat.row_names: vecs.append(pred_mat.get(row_names=pred_name)) elif isinstance(arg, np.ndarray): self.logger.warn("linear_analysis.__load_predictions(): " + "instantiating prediction matrix from " + "ndarray, can't verify alignment") self.logger.warn("linear_analysis.__load_predictions(): " + "instantiating prediction matrix from " + "ndarray, generating generic prediction names") pred_names = ["pred_{0}".format(i+1) for i in range(arg.shape[0])] if self.jco: names = self.jco.col_names elif self.parcov: names = self.parcov.col_names else: raise Exception("linear_analysis.__load_predictions(): " + "ndarray passed for predicitons " + "requires jco or parcov to get " + "parameter names") pred_matrix = Matrix(x=arg,row_names=pred_names,col_names=names) for pred_name in pred_names: vecs.append(pred_matrix.get(row_names=pred_name).T) else: raise Exception("unrecognized predictions argument: " + str(arg)) if len(row_names) > 0: extract = self.jco.extract(row_names=row_names) for row_name in row_names: vecs.append(extract.get(row_names=row_name).T) # call obscov to load __obscov so that __obscov # (priavte) can be manipulated self.obscov self.__obscov.drop(row_names, axis=0) self.__predictions = vecs self.log("loading forecasts") return self.__predictions # these property decorators help keep from loading potentially # unneeded items until they are called # returns a reference - cheap, but can be dangerous @property def forecast_names(self): if self.forecasts is None: return [] return [fore.col_names[0] for fore in self.forecasts] @property def parcov(self): if not self.__parcov: self.__load_parcov() return self.__parcov @property def obscov(self): if not self.__obscov: self.__load_obscov() return self.__obscov @property def jco(self): if not self.__jco: self.__load_jco() return self.__jco @property def predictions(self): if not self.__predictions: self.__load_predictions() return self.__predictions @property def forecasts(self): return self.predictions @property def pst(self): if self.__pst is None and self.pst_arg is None: raise Exception("linear_analysis.pst: can't access self.pst:" + "no pest control argument passed") elif self.__pst: return self.__pst else: self.__load_pst() return self.__pst @property def fehalf(self): """set the KL parcov scaling matrix attribute """ if self.__fehalf != None: return self.__fehalf self.log("fehalf") self.__fehalf = self.parcov.u * (self.parcov.s ** (0.5)) self.log("fehalf") return self.__fehalf @property def qhalf(self): """set the square root of the cofactor matrix attribute """ if self.__qhalf != None: return self.__qhalf self.log("qhalf") self.__qhalf = self.obscov ** (-0.5) self.log("qhalf") return self.__qhalf @property def qhalfx(self): """set the half normal matrix attribute """ if self.__qhalfx is None: self.log("qhalfx") self.__qhalfx = self.qhalf * self.jco self.log("qhalfx") return self.__qhalfx @property def xtqx(self): """set the normal matrix attribute """ if self.__xtqx is None: self.log("xtqx") self.__xtqx = self.jco.T * (self.obscov ** -1) * self.jco self.log("xtqx") return self.__xtqx @property def mle_covariance(self): return self.xtqx.inv @property def prior_parameter(self): """the prior parameter covariance matrix """ return self.parcov @property def prior_forecast(self): """thin wrapper for prior_prediction """ return self.prior_prediction @property def mle_parameter_estimate(self): res = self.pst.res assert res is not None # build the prior expectation parameter vector prior_expt = self.pst.parameter_data.loc[:,["parval1"]].copy() islog = self.pst.parameter_data.partrans == "log" prior_expt.loc[islog] = prior_expt.loc[islog].apply(np.log10) prior_expt = Matrix.from_dataframe(prior_expt) prior_expt.col_names = ["prior_expt"] # build the residual vector res_vec = Matrix.from_dataframe(res.loc[:,["residual"]]) # calc posterior expectation upgrade = self.mle_covariance * self.jco.T * res_vec upgrade.col_names = ["prior_expt"] post_expt = prior_expt + upgrade # post processing - back log transform post_expt = pd.DataFrame(data=post_expt.x,index=post_expt.row_names, columns=["post_expt"]) post_expt.loc[islog,:] = 10.0**post_expt.loc[islog,:] return post_expt @property def prior_prediction(self): """get a dict of prior prediction variances Parameters: ---------- None Returns: ------- dict{prediction name(str):prior variance(float)} """ if self.__prior_prediction is not None: return self.__prior_prediction else: if self.predictions is not None: self.log("propagating prior to predictions") pred_dict = {} for prediction in self.predictions: var = (prediction.T * self.parcov * prediction).x[0, 0] pred_dict[prediction.col_names[0]] = var self.__prior_prediction = pred_dict self.log("propagating prior to predictions") else: self.__prior_prediction = {} return self.__prior_prediction def apply_karhunen_loeve_scaling(self): """apply karhuene-loeve scaling to the jacobian matrix. This scaling is not necessary for analyses using Schur's complement, but can be very important for error variance analyses. This operation effectively transfers prior knowledge specified in the parcov to the jacobian and reset parcov to the identity matrix. """ cnames = copy.deepcopy(self.jco.col_names) self.__jco *= self.fehalf self.__jco.col_names = cnames self.__parcov = self.parcov.identity def clean(self): """drop regularization and prior information observation from the jco """ if self.pst_arg is None: self.logger.warn("linear_analysis.clean(): not pst object") return if not self.pst.estimation and self.pst.nprior > 0: self.drop_prior_information() def reset_pst(self,arg): self.logger.warn("resetting pst") self.__pst = None self.pst_arg = arg def reset_parcov(self,arg=None): """reset the parcov attribute to None Parameters: ---------- arg (str or matrix) : the value to assign to the parcov_arg attrib Returns: ------- None """ self.logger.warn("resetting parcov") self.__parcov = None if arg is not None: self.parcov_arg = arg def reset_obscov(self,arg=None): """reset the obscov attribute to None Parameters: ---------- arg (str or matrix) : the value to assign to the obscov_arg attrib Returns: ------- None """ self.logger.warn("resetting obscov") self.__obscov = None if arg is not None: self.obscov_arg = arg def drop_prior_information(self): """drop the prior information from the jco and pst attributes """ if self.jco is None: self.logger.warn("can't drop prior info, LinearAnalysis.jco is None") return nprior_str = str(self.pst.nprior) self.log("removing " + nprior_str + " prior info from jco, pst, and " + "obs cov") #pi_names = list(self.pst.prior_information.pilbl.values) pi_names = list(self.pst.prior_names) missing = [name for name in pi_names if name not in self.jco.obs_names] if len(missing) > 0: raise Exception("LinearAnalysis.drop_prior_information(): "+ " prior info not found: {0}".format(missing)) if self.jco is not None: self.__jco.drop(pi_names, axis=0) self.__pst.prior_information = self.pst.null_prior self.__pst.control_data.pestmode = "estimation" #self.__obscov.drop(pi_names,axis=0) self.log("removing " + nprior_str + " prior info from jco and pst") def get(self,par_names=None,obs_names=None,astype=None): """method to get a new LinearAnalysis class using a subset of parameters and/or observations Parameters: ---------- par_names (enumerable of str) : par names for new object obs_names (enumerable of str) : obs names for new object astype (either schur or errvar type) : type to cast the new object Returns: ------- LinearAnalysis object """ # make sure we aren't fooling with unwanted prior information self.clean() # if there is nothing to do but copy if par_names is None and obs_names is None: if astype is not None: self.logger.warn("LinearAnalysis.get(): astype is not None, " + "but par_names and obs_names are None so" + "\n ->Omitted attributes will not be " + "propagated to new instance") else: return copy.deepcopy(self) # make sure the args are lists if par_names is not None and not isinstance(par_names, list): par_names = [par_names] if obs_names is not None and not isinstance(obs_names, list): obs_names = [obs_names] if par_names is None: par_names = self.jco.col_names if obs_names is None: obs_names = self.jco.row_names # if possible, get a new parcov if self.parcov: new_parcov = self.parcov.get(col_names=par_names) else: new_parcov = None # if possible, get a new obscov if self.obscov_arg is not None: new_obscov = self.obscov.get(row_names=obs_names) else: new_obscov = None # if possible, get a new pst if self.pst_arg is not None: new_pst = self.pst.get(par_names=par_names,obs_names=obs_names) else: new_pst = None if self.predictions: new_preds = [] for prediction in self.predictions: new_preds.append(prediction.get(row_names=par_names)) else: new_preds = None if self.jco_arg is not None: new_jco = self.jco.get(row_names=obs_names, col_names=par_names) else: new_jco = None if astype is not None: return astype(jco=new_jco, pst=new_pst, parcov=new_parcov, obscov=new_obscov, predictions=new_preds, verbose=False) else: # return a new object of the same type return type(self)(jco=new_jco, pst=new_pst, parcov=new_parcov, obscov=new_obscov, predictions=new_preds, verbose=False) def adjust_obscov_resfile(self, resfile=None): """reset the elements of obscov by scaling the implied weights based on the phi components in res_file """ self.pst.adjust_weights_resfile(resfile) self.__obscov.from_observation_data(self.pst)
class LinearAnalysis(object): """ the super class for linear analysis. Can be used for prior analyses only. The derived types (schur and errvar) are for posterior analyses this class tries hard to not load items until they are needed all arguments are optional Parameters: ---------- jco ([enumerable of] [string,ndarray,matrix objects]) : jacobian pst (pst object) : the pest control file object parcov ([enumerable of] [string,ndarray,matrix objects]) : parameter covariance matrix obscov ([enumerable of] [string,ndarray,matrix objects]): observation noise covariance matrix predictions ([enumerable of] [string,ndarray,matrix objects]) : prediction sensitivity vectors ref_var (float) : reference variance verbose (either bool or string) : controls log file / screen output Notes: the class makes heavy use of property decorator to encapsulate private attributes """ def __init__(self, jco=None, pst=None, parcov=None, obscov=None, predictions=None, ref_var=1.0, verbose=False, resfile=False, forecasts=None, **kwargs): self.logger = logger(verbose) self.log = self.logger.log self.jco_arg = jco #if jco is None: self.__jco = jco if pst is None: if isinstance(jco, str): pst_case = jco.replace(".jco", ".pst").replace(".jcb", ".pst") if os.path.exists(pst_case): pst = pst_case self.pst_arg = pst if parcov is None and pst is not None: parcov = pst self.parcov_arg = parcov if obscov is None and pst is not None: obscov = pst self.obscov_arg = obscov self.ref_var = ref_var if forecasts is not None and predictions is not None: raise Exception("can't pass both forecasts and predictions") #private attributes - access is through @decorated functions self.__pst = None self.__parcov = None self.__obscov = None self.__predictions = None self.__qhalf = None self.__qhalfx = None self.__xtqx = None self.__fehalf = None self.__prior_prediction = None self.log("pre-loading base components") if jco is not None: self.__load_jco() if pst is not None: self.__load_pst() if parcov is not None: self.__load_parcov() if obscov is not None: self.__load_obscov() self.prediction_arg = None if predictions is not None: self.prediction_arg = predictions elif forecasts is not None: self.prediction_arg = forecasts elif self.pst is not None and self.jco is not None: if "forecasts" in self.pst.pestpp_options: self.prediction_arg = [i.strip() for i in self.pst.pestpp_options["forecasts"].\ lower().split(',')] elif "predictions" in self.pst.pestpp_options: self.prediction_arg = [i.strip() for i in self.pst.pestpp_options["predictions"].\ lower().split(',')] if self.prediction_arg: self.__load_predictions() self.log("pre-loading base components") if len(kwargs.keys()) > 0: self.logger.warn("unused kwargs in type " + str(self.__class__.__name__) + " : " + str(kwargs)) raise Exception("unused kwargs" + " : " + str(kwargs)) # automatically do some things that should be done self.log("dropping prior information") pi = None try: pi = self.pst.prior_information except: self.logger.warn("unable to access self.pst: can't tell if " + " any prior information needs to be dropped.") if pi is not None: self.drop_prior_information() self.log("dropping prior information") if resfile != False: self.log("scaling obscov by residual phi components") try: self.adjust_obscov_resfile(resfile=resfile) except: self.logger.warn("unable to a find a residuals file for " +\ " scaling obscov") self.resfile = None self.res = None self.log("scaling obscov by residual phi components") def __fromfile(self, filename): """a private method to deduce and load a filename into a matrix object Parameters: ---------- filename (str) : the name of the file Returns: ------- mat (or cov) object """ assert os.path.exists(filename),"LinearAnalysis.__fromfile(): " +\ "file not found:" + filename ext = filename.split('.')[-1].lower() if ext in ["jco", "jcb"]: self.log("loading jco: " + filename) m = Jco.from_binary(filename) self.log("loading jco: " + filename) elif ext in ["mat", "vec"]: self.log("loading ascii: " + filename) m = Matrix.from_ascii(filename) self.log("loading ascii: " + filename) elif ext in ["cov"]: self.log("loading cov: " + filename) m = Cov.from_ascii(filename) self.log("loading cov: " + filename) elif ext in ["unc"]: self.log("loading unc: " + filename) m = Cov.from_uncfile(filename) self.log("loading unc: " + filename) else: raise Exception("linear_analysis.__fromfile(): unrecognized" + " filename extension:" + str(ext)) return m def __load_pst(self): """private: set the pst attribute Parameters: ---------- None Returns: ------- None """ if self.pst_arg is None: return None if isinstance(self.pst_arg, Pst): self.__pst = self.pst_arg return self.pst else: try: self.log("loading pst: " + str(self.pst_arg)) self.__pst = Pst(self.pst_arg) self.log("loading pst: " + str(self.pst_arg)) return self.pst except Exception as e: raise Exception("linear_analysis.__load_pst(): error loading"+\ " pest control from argument: " + str(self.pst_arg) + '\n->' + str(e)) def __load_jco(self): """private :set the jco attribute from a file or a matrix object Parameters: ---------- None Returns: ------- None """ if self.jco_arg is None: return None #raise Exception("linear_analysis.__load_jco(): jco_arg is None") if isinstance(self.jco_arg, Matrix): self.__jco = self.jco_arg elif isinstance(self.jco_arg, str): self.__jco = self.__fromfile(self.jco_arg) else: raise Exception("linear_analysis.__load_jco(): jco_arg must " + "be a matrix object or a file name: " + str(self.jco_arg)) def __load_parcov(self): """private: set the parcov attribute from: a pest control file (parameter bounds) a pst object a matrix object an uncert file an ascii matrix file """ # if the parcov arg was not passed but the pst arg was, # reset and use parbounds to build parcov if not self.parcov_arg: if self.pst_arg: self.parcov_arg = self.pst_arg else: raise Exception("linear_analysis.__load_parcov(): " + "parcov_arg is None") if isinstance(self.parcov_arg, Matrix): self.__parcov = self.parcov_arg return if isinstance(self.parcov_arg, np.ndarray): # if the passed array is a vector, # then assume it is the diagonal of the parcov matrix if len(self.parcov_arg.shape) == 1: assert self.parcov_arg.shape[0] == self.jco.shape[1] isdiagonal = True else: assert self.parcov_arg.shape[0] == self.jco.shape[1] assert self.parcov_arg.shape[1] == self.jco.shape[1] isdiagonal = False self.logger.warn("linear_analysis.__load_parcov(): " + "instantiating parcov from ndarray, can't " + "verify parameters alignment with jco") self.__parcov = Matrix(x=self.parcov_arg, isdiagonal=isdiagonal, row_names=self.jco.col_names, col_names=self.jco.col_names) self.log("loading parcov") if isinstance(self.parcov_arg, str): # if the arg is a string ending with "pst" # then load parcov from parbounds if self.parcov_arg.lower().endswith(".pst"): self.__parcov = Cov.from_parbounds(self.parcov_arg) else: self.__parcov = self.__fromfile(self.parcov_arg) # if the arg is a pst object elif isinstance(self.parcov_arg, Pst): self.__parcov = Cov.from_parameter_data(self.parcov_arg) else: raise Exception("linear_analysis.__load_parcov(): " + "parcov_arg must be a " + "matrix object or a file name: " + str(self.parcov_arg)) self.log("loading parcov") def __load_obscov(self): """private: method to set the obscov attribute from: a pest control file (observation weights) a pst object a matrix object an uncert file an ascii matrix file """ # if the obscov arg is None, but the pst arg is not None, # reset and load from obs weights if not self.obscov_arg: if self.pst_arg: self.obscov_arg = self.pst_arg else: raise Exception("linear_analysis.__load_obscov(): " + "obscov_arg is None") if isinstance(self.obscov_arg, Matrix): self.__obscov = self.obscov_arg return if isinstance(self.obscov_arg, np.ndarray): # if the ndarray arg is a vector, # assume it is the diagonal of the obscov matrix if len(self.obscov_arg.shape) == 1: assert self.obscov_arg.shape[0] == self.jco.shape[1] isdiagonal = True else: assert self.obscov_arg.shape[0] == self.jco.shape[0] assert self.obscov_arg.shape[1] == self.jco.shape[0] isdiagonal = False self.logger.warn("linear_analysis.__load_obscov(): " + "instantiating obscov from ndarray, " + "can't verify observation alignment with jco") self.__obscov = Matrix(x=self.obscov_arg, isdiagonal=isdiagonal, row_names=self.jco.row_names, col_names=self.jco.row_names) self.log("loading obscov") if isinstance(self.obscov_arg, str): if self.obscov_arg.lower().endswith(".pst"): self.__obscov = Cov.from_obsweights(self.obscov_arg) else: self.__obscov = self.__fromfile(self.obscov_arg) elif isinstance(self.obscov_arg, Pst): self.__obscov = Cov.from_observation_data(self.obscov_arg) else: raise Exception("linear_analysis.__load_obscov(): " + "obscov_arg must be a " + "matrix object or a file name: " + str(self.obscov_arg)) self.log("loading obscov") def __load_predictions(self): """private: set the predictions attribute from: mixed list of row names, matrix files and ndarrays a single row name an ascii file can be none if only interested in parameters. linear_analysis.__predictions is stored as a list of column vectors """ if self.prediction_arg is None: self.__predictions = None return self.log("loading forecasts") if not isinstance(self.prediction_arg, list): self.prediction_arg = [self.prediction_arg] row_names = [] vecs = [] for arg in self.prediction_arg: if isinstance(arg, Matrix): # a vector if arg.shape[1] == 1: vecs.append(arg) else: assert arg.shape[1] == self.jco.shape[1],\ "linear_analysis.__load_predictions(): " +\ "multi-prediction matrix(npred,npar) not aligned " +\ "with jco(nobs,npar): " + str(arg.shape) +\ ' ' + str(self.jco.shape) for pred_name in arg.row_names: vecs.append(arg.extract(row_names=pred_name).T) elif isinstance(arg, str): if arg.lower() in self.jco.row_names: row_names.append(arg.lower()) else: pred_mat = self.__fromfile(arg) # vector if pred_mat.shape[1] == 1: vecs.append(pred_mat) else: for pred_name in pred_mat.row_names: vecs.append(pred_mat.get(row_names=pred_name)) elif isinstance(arg, np.ndarray): self.logger.warn("linear_analysis.__load_predictions(): " + "instantiating prediction matrix from " + "ndarray, can't verify alignment") self.logger.warn( "linear_analysis.__load_predictions(): " + "instantiating prediction matrix from " + "ndarray, generating generic prediction names") pred_names = [ "pred_{0}".format(i + 1) for i in range(arg.shape[0]) ] if self.jco: names = self.jco.col_names elif self.parcov: names = self.parcov.col_names else: raise Exception("linear_analysis.__load_predictions(): " + "ndarray passed for predicitons " + "requires jco or parcov to get " + "parameter names") pred_matrix = Matrix(x=arg, row_names=pred_names, col_names=names) for pred_name in pred_names: vecs.append(pred_matrix.get(row_names=pred_name).T) else: raise Exception("unrecognized predictions argument: " + str(arg)) if len(row_names) > 0: extract = self.jco.extract(row_names=row_names) for row_name in row_names: vecs.append(extract.get(row_names=row_name).T) # call obscov to load __obscov so that __obscov # (priavte) can be manipulated self.obscov self.__obscov.drop(row_names, axis=0) self.__predictions = vecs self.log("loading forecasts") return self.__predictions # these property decorators help keep from loading potentially # unneeded items until they are called # returns a reference - cheap, but can be dangerous @property def parcov(self): if not self.__parcov: self.__load_parcov() return self.__parcov @property def obscov(self): if not self.__obscov: self.__load_obscov() return self.__obscov @property def jco(self): if not self.__jco: self.__load_jco() return self.__jco @property def predictions(self): if not self.__predictions: self.__load_predictions() return self.__predictions @property def forecasts(self): return self.predictions @property def pst(self): if self.__pst is None and self.pst_arg is None: raise Exception("linear_analysis.pst: can't access self.pst:" + "no pest control argument passed") elif self.__pst: return self.__pst else: self.__load_pst() return self.__pst @property def fehalf(self): """set the KL parcov scaling matrix attribute """ if self.__fehalf != None: return self.__fehalf self.log("fehalf") self.__fehalf = self.parcov.u * (self.parcov.s**(0.5)) self.log("fehalf") return self.__fehalf @property def qhalf(self): """set the square root of the cofactor matrix attribute """ if self.__qhalf != None: return self.__qhalf self.log("qhalf") self.__qhalf = self.obscov**(-0.5) self.log("qhalf") return self.__qhalf @property def qhalfx(self): """set the half normal matrix attribute """ if self.__qhalfx is None: self.log("qhalfx") self.__qhalfx = self.qhalf * self.jco self.log("qhalfx") return self.__qhalfx @property def xtqx(self): """set the normal matrix attribute """ if self.__xtqx is None: self.log("xtqx") self.__xtqx = self.jco.T * (self.obscov**-1) * self.jco self.log("xtqx") return self.__xtqx @property def prior_parameter(self): """the prior parameter covariance matrix """ return self.parcov @property def prior_forecast(self): """thin wrapper for prior_prediction """ return self.prior_prediction @property def prior_prediction(self): """get a dict of prior prediction variances Parameters: ---------- None Returns: ------- dict{prediction name(str):prior variance(float)} """ if self.__prior_prediction is not None: return self.__prior_prediction else: if self.predictions is not None: self.log("propagating prior to predictions") pred_dict = {} for prediction in self.predictions: var = (prediction.T * self.parcov * prediction).x[0, 0] pred_dict[prediction.col_names[0]] = var self.__prior_prediction = pred_dict self.log("propagating prior to predictions") else: self.__prior_prediction = {} return self.__prior_prediction def apply_karhunen_loeve_scaling(self): """apply karhuene-loeve scaling to the jacobian matrix. This scaling is not necessary for analyses using Schur's complement, but can be very important for error variance analyses. This operation effectively transfers prior knowledge specified in the parcov to the jacobian and reset parcov to the identity matrix. """ cnames = copy.deepcopy(self.jco.col_names) self.__jco *= self.fehalf self.__jco.col_names = cnames self.__parcov = self.parcov.identity def clean(self): """drop regularization and prior information observation from the jco """ if self.pst_arg is None: self.logger.warn("linear_analysis.clean(): not pst object") return if not self.pst.estimation and self.pst.nprior > 0: self.drop_prior_information() def reset_pst(self, arg): self.logger.warn("resetting pst") self.__pst = None self.pst_arg = arg def reset_parcov(self, arg=None): """reset the parcov attribute to None Parameters: ---------- arg (str or matrix) : the value to assign to the parcov_arg attrib Returns: ------- None """ self.logger.warn("resetting parcov") self.__parcov = None if arg is not None: self.parcov_arg = arg def reset_obscov(self, arg=None): """reset the obscov attribute to None Parameters: ---------- arg (str or matrix) : the value to assign to the obscov_arg attrib Returns: ------- None """ self.logger.warn("resetting obscov") self.__obscov = None if arg is not None: self.obscov_arg = arg def drop_prior_information(self): """drop the prior information from the jco and pst attributes """ if self.jco is None: self.logger.warn( "can't drop prior info, LinearAnalysis.jco is None") return nprior_str = str(self.pst.nprior) self.log("removing " + nprior_str + " prior info from jco, pst, and " + "obs cov") #pi_names = list(self.pst.prior_information.pilbl.values) pi_names = list(self.pst.prior_names) missing = [name for name in pi_names if name not in self.jco.obs_names] if len(missing) > 0: raise Exception("LinearAnalysis.drop_prior_information(): " + " prior info not found: {0}".format(missing)) if self.jco is not None: self.__jco.drop(pi_names, axis=0) self.__pst.prior_information = self.pst.null_prior self.__pst.control_data.pestmode = "estimation" #self.__obscov.drop(pi_names,axis=0) self.log("removing " + nprior_str + " prior info from jco and pst") def get(self, par_names=None, obs_names=None, astype=None): """method to get a new LinearAnalysis class using a subset of parameters and/or observations Parameters: ---------- par_names (enumerable of str) : par names for new object obs_names (enumerable of str) : obs names for new object astype (either schur or errvar type) : type to cast the new object Returns: ------- LinearAnalysis object """ # make sure we aren't fooling with unwanted prior information self.clean() # if there is nothing to do but copy if par_names is None and obs_names is None: if astype is not None: self.logger.warn("LinearAnalysis.get(): astype is not None, " + "but par_names and obs_names are None so" + "\n ->Omitted attributes will not be " + "propagated to new instance") else: return copy.deepcopy(self) # make sure the args are lists if par_names is not None and not isinstance(par_names, list): par_names = [par_names] if obs_names is not None and not isinstance(obs_names, list): obs_names = [obs_names] if par_names is None: par_names = self.jco.col_names if obs_names is None: obs_names = self.jco.row_names # if possible, get a new parcov if self.parcov: new_parcov = self.parcov.get(col_names=par_names) else: new_parcov = None # if possible, get a new obscov if self.obscov_arg is not None: new_obscov = self.obscov.get(row_names=obs_names) else: new_obscov = None # if possible, get a new pst if self.pst_arg is not None: new_pst = self.pst.get(par_names=par_names, obs_names=obs_names) else: new_pst = None if self.predictions: new_preds = [] for prediction in self.predictions: new_preds.append(prediction.get(row_names=par_names)) else: new_preds = None if self.jco_arg is not None: new_jco = self.jco.get(row_names=obs_names, col_names=par_names) else: new_jco = None if astype is not None: return astype(jco=new_jco, pst=new_pst, parcov=new_parcov, obscov=new_obscov, predictions=new_preds, verbose=False) else: # return a new object of the same type return type(self)(jco=new_jco, pst=new_pst, parcov=new_parcov, obscov=new_obscov, predictions=new_preds, verbose=False) def adjust_obscov_resfile(self, resfile=None): """reset the elements of obscov by scaling the implied weights based on the phi components in res_file """ self.pst.adjust_weights_resfile(resfile) self.__obscov.from_observation_data(self.pst)