def _process_cov_constraint(constraint, params): """Process covariance constraints. Args: constraint (dict) params (pd.DataFrame): see :ref:`params_df`. Returns: new_constr (dict): copy of *constraint* with a new entry called 'case', which can take the values 'all_fixed', 'uncorrelated' and 'all_free'. """ new_constr = constraint.copy() params_subset = params.loc[constraint["index"]] cov = cov_params_to_matrix(params_subset["value"].to_numpy()) dim = len(cov) off_diagonal_zero = bool((cov[np.tril_indices(dim, k=-1)] == 0).all()) fixed_helper = cov_params_to_matrix( params_subset["fixed"].to_numpy()).astype(bool) off_diagonal_fixed = bool(fixed_helper[np.tril_indices(dim, k=-1)].all()) all_fixed = bool(params_subset["fixed"].all()) if all_fixed is True: case = "all_fixed"
def _parse_shocks(optim_paras, params): """Parse the shock parameters and create the Cholesky factor.""" if sum(f"shocks_{i}" in params.index for i in ["sdcorr", "cov", "chol"]) >= 2: raise ValueError("It is not allowed to define multiple shock matrices.") elif "shocks_sdcorr" in params.index: sorted_shocks = _sort_shocks_sdcorr(optim_paras, params.loc["shocks_sdcorr"]) cov = sdcorr_params_to_matrix(sorted_shocks) optim_paras["shocks_cholesky"] = robust_cholesky(cov) elif "shocks_cov" in params.index: sorted_shocks = _sort_shocks_cov_chol( optim_paras, params.loc["shocks_cov"], "cov" ) cov = cov_params_to_matrix(sorted_shocks) optim_paras["shocks_cholesky"] = robust_cholesky(cov) elif "shocks_chol" in params.index: sorted_shocks = _sort_shocks_cov_chol( optim_paras, params.loc["shocks_chol"], "chol" ) optim_paras["shocks_cholesky"] = chol_params_to_lower_triangular_matrix( sorted_shocks ) else: raise NotImplementedError return optim_paras
def covariance_to_internal_jacobian(external_values, constr): r"""Jacobian of ``covariance_to_internal``. For reference see docstring of ``jacobian_covariance_from_internal``. In comparison to that function, however, here we want to differentiate the reverse graph external --> cov --> cholesky --> internal Again use the vectors :math:`c` and :math:`x` to denote the external and internal values, respectively. To solve for the jacobian we make use of the identity .. math:: \frac{\mathrm{d}x}{\mathrm{d}c} = (\frac{\mathrm{d}c}{\mathrm{d}x})^{-1} Args: external_values (np.ndarray): Row-wise half-vectorized covariance matrix Returns: deriv: The Jacobian matrix. """ cov = cov_params_to_matrix(external_values) chol = robust_cholesky(cov) internal = chol[np.tril_indices(len(chol))] deriv = covariance_from_internal_jacobian(internal, constr=None) deriv = np.linalg.pinv(deriv) return deriv
def _process_cov_constraint(constraint, params, fixed): """Process covariance constraints. Args: constraint (dict) params (pd.DataFrame): see :ref:`params`. Returns: new_constr (dict): copy of *constraint* with a new entry called 'case', which can take the values 'all_fixed', 'uncorrelated' and 'all_free'. """ new_constr = constraint.copy() params_subset = params.loc[constraint["index"]] fixed_subset = fixed.loc[constraint["index"]] value_mat = cov_params_to_matrix(params_subset["value"].to_numpy()) fixed_mat = cov_params_to_matrix( fixed_subset["_fixed"].to_numpy()).astype(bool) new_constr["case"] = _determine_cov_case(value_mat, fixed_mat, params_subset) return new_constr
def _map_params_to_p(params, initial, square_root_filters): nobs, nemf, nfac, _ = initial.shape nfac = nfac - 1 if square_root_filters is True else nfac filler = np.zeros((nemf, nfac, nfac)) for emf in range(nemf): filler[emf] = cov_params_to_matrix(params.loc["p", 0, emf].to_numpy()) if square_root_filters is True: filler = np.transpose(np.linalg.cholesky(filler), axes=(0, 2, 1)) initial[:, :, 1:, 1:] = filler else: initial[:] = filler
def _parse_shocks(optim_paras, params): """Parse the shock parameters and create the Cholesky factor.""" if sum(f"shocks_{i}" in params.index for i in ["sdcorr", "cov", "chol"]) >= 2: raise ValueError("It is not allowed to define multiple shock matrices.") elif "shocks_sdcorr" in params.index: cov = sdcorr_params_to_matrix(params.loc["shocks_sdcorr"]) optim_paras["shocks_cholesky"] = robust_cholesky(cov) elif "shocks_cov" in params.index: cov = cov_params_to_matrix(params.loc["shocks_cov"]) optim_paras["shocks_cholesky"] = robust_cholesky(cov) elif "shocks_chol" in params.index: optim_paras["shocks_cholesky"] = chol_params_to_lower_triangular_matrix( params.loc["shocks_chol"] ) else: raise KeyError("No shock matrix is specified.") return optim_paras
def check_constraints_are_satisfied(pc, params): """Check that params satisfies all constraints. This should be called before the more specialized constraints are rewritten to linear constraints in order to get better error messages! We let the checks pass if all "values" are np.nan. This way `process_constraints` can be used on empty params DataFrames which is useful to construct templates for start parameters that can be filled out by the user. Args: pc (list): List of constraints with processed selectors. params (pd.DataFrame): See :ref:`params` Raises: ValueError if constraints are not satisfied. """ if params["value"].notnull().any(): for constr in pc: typ = constr["type"] subset = params.iloc[constr["index"]]["value"] msg = f"{{}}:\n{subset.to_frame()}" if typ == "covariance": cov = cov_params_to_matrix(subset) e, v = np.linalg.eigh(cov) if not np.all(e > -1e-8): raise ValueError(msg.format("Invalid covariance parameters.")) elif typ == "sdcorr": cov = sdcorr_params_to_matrix(subset) dim = len(cov) if (subset.iloc[:dim] < 0).any(): raise ValueError(msg.format("Invalid standard deviations.")) if ((subset.iloc[dim:] < -1) | (subset.iloc[dim:] > 1)).any(): raise ValueError(msg.format("Invalid correlations.")) e, v = np.linalg.eigh(cov) if not np.all(e > -1e-8): raise ValueError(msg.format("Invalid sdcorr parameters.")) elif typ == "probability": if not np.isclose(subset.sum(), 1, rtol=0.01): raise ValueError(msg.format("Probabilities do not sum to 1")) if np.any(subset < 0): raise ValueError(msg.format("Negative Probability.")) if np.any(subset > 1): raise ValueError(msg.format("Probability larger than 1.")) elif typ == "increasing": if np.any(np.diff(subset) < 0): raise ValueError(msg.format("Increasing constraint violated.")) elif typ == "decreasing": if np.any(np.diff(subset) > 0): raise ValueError(msg.format("Decreasing constraint violated")) elif typ == "linear": # using sr.dot is important in case weights are a series in wrong order wsum = subset.dot(constr["weights"]) if "lower_bound" in constr and wsum < constr["lower_bound"]: raise ValueError( msg.format("Lower bound of linear constraint is violated") ) elif "upper_bound" in constr and wsum > constr["upper_bound"]: raise ValueError( msg.format("Upper bound of linear constraint violated") ) elif "value" in constr and not np.isclose(wsum, constr["value"]): raise ValueError( msg.format("Equality condition of linear constraint violated") ) elif typ == "equality": if len(subset.unique()) != 1: raise ValueError(msg.format("Equality constraint violated."))
def _covariance_to_internal(params_subset, case): """Reparametrize parameters that describe a covariance matrix to internal. The parameters in params_subset are assumed to be the lower triangular elements of a covariance matrix. If all parameters are fixed, nothing has to be done. If all off-diagonal elements are fixed to zero, it is only necessary to set the lower bounds of the off-diagonals to 0, unless already stricter. Otherwise, we do a (lower triangular) Cholesky reparametrization and restrict diagonal elements to be positive (see: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.31.494&rep=rep1&type=pdf) Note that the cholesky reparametrization is not compatible with any other constraints on the involved parameters. Moreover, it requires the covariance matrix described by the start values to be positive definite as opposed to positive semi-definite. Args: params_subset (DataFrame): relevant subset of non-internal params. case (str): can take the values 'all_free', 'uncorrelated' or 'all_fixed'. Returns: res (DataFrame): copy of params_subset with adjusted 'value' and 'lower' columns """ res = params_subset.copy() cov = cov_params_to_matrix(params_subset["value"].to_numpy()) dim = len(cov) e, v = np.linalg.eigh(cov) assert np.all(e > -1e-8), "Invalid covariance matrix." if case == "uncorrelated": lower_bound_helper = cov_params_to_matrix(params_subset["lower"]) diag_lower = np.maximum(np.diagonal(lower_bound_helper), np.zeros(dim)) lower_bound_helper[np.diag_indices(dim)] = diag_lower lower_bounds = lower_bound_helper[np.tril_indices(dim)] res["lower"] = lower_bounds assert (res["upper"] >= res["lower"]).all(), "Invalid upper bound for variance." else: chol = np.linalg.cholesky(cov) chol_coeffs = chol[np.tril_indices(dim)] res["value"] = chol_coeffs lower_bound_helper = np.full((dim, dim), -np.inf) lower_bound_helper[np.diag_indices(dim)] = 0 res["lower"] = lower_bound_helper[np.tril_indices(dim)] res["upper"] = np.inf res["fixed"] = False if params_subset["fixed"].any(): warnings.warn("Covariance parameters are unfixed.", UserWarning) for bound in ["lower", "upper"]: if np.isfinite(params_subset[bound]).any(): warnings.warn("Bounds are ignored for covariance parameters.", UserWarning) return res
Note that the cholesky reparametrization is not compatible with any other constraints on the involved parameters. Moreover, it requires the covariance matrix described by the start values to be positive definite as opposed to positive semi-definite. Args: params_subset (DataFrame): relevant subset of non-internal params. case (str): can take the values 'free', 'uncorrelated' or 'all_fixed'. Returns: res (DataFrame): copy of params_subset with adjusted 'value' and 'lower' columns """ res = params_subset.copy() if type_ == "covariance": cov = cov_params_to_matrix(params_subset["value"].to_numpy()) elif type_ == "sdcorr": cov = sdcorr_params_to_matrix(params_subset["value"].to_numpy()) else: raise ValueError("Invalid type_: {}".format(type_)) dim = len(cov) e, v = np.linalg.eigh(cov) assert np.all(e > -1e-8), "Invalid covariance matrix." if case == "uncorrelated": res["lower"] = np.maximum(res["lower"], np.zeros(len(res))) assert (res["upper"] >= res["lower"]).all(), "Invalid upper bound for variance."
def test_cov_params_to_matrix(): params = np.array([1, 0.1, 2, 0.2, 0.22, 3]) expected = np.array([[1, 0.1, 0.2], [0.1, 2, 0.22], [0.2, 0.22, 3]]) calculated = cov_params_to_matrix(params) aaae(calculated, expected)
def covariance_to_internal(external_values, constr): """Do a cholesky reparametrization.""" cov = cov_params_to_matrix(external_values) chol = robust_cholesky(cov) return chol[np.tril_indices(len(cov))]