def _process_cov_constraint(constraint, params):
"""Process covariance constraints.
Args:
constraint (dict)
params (pd.DataFrame): see :ref:`params_df`.
Returns:
new_constr (dict): copy of *constraint* with a new entry called 'case',
which can take the values 'all_fixed', 'uncorrelated' and 'all_free'.
"""
new_constr = constraint.copy()
params_subset = params.loc[constraint["index"]]
cov = cov_params_to_matrix(params_subset["value"].to_numpy())
dim = len(cov)
off_diagonal_zero = bool((cov[np.tril_indices(dim, k=-1)] == 0).all())
fixed_helper = cov_params_to_matrix(
params_subset["fixed"].to_numpy()).astype(bool)
off_diagonal_fixed = bool(fixed_helper[np.tril_indices(dim, k=-1)].all())
all_fixed = bool(params_subset["fixed"].all())
if all_fixed is True:
case = "all_fixed"
def _parse_shocks(optim_paras, params):
"""Parse the shock parameters and create the Cholesky factor."""
if sum(f"shocks_{i}" in params.index for i in ["sdcorr", "cov", "chol"]) >= 2:
raise ValueError("It is not allowed to define multiple shock matrices.")
elif "shocks_sdcorr" in params.index:
sorted_shocks = _sort_shocks_sdcorr(optim_paras, params.loc["shocks_sdcorr"])
cov = sdcorr_params_to_matrix(sorted_shocks)
optim_paras["shocks_cholesky"] = robust_cholesky(cov)
elif "shocks_cov" in params.index:
sorted_shocks = _sort_shocks_cov_chol(
optim_paras, params.loc["shocks_cov"], "cov"
)
cov = cov_params_to_matrix(sorted_shocks)
optim_paras["shocks_cholesky"] = robust_cholesky(cov)
elif "shocks_chol" in params.index:
sorted_shocks = _sort_shocks_cov_chol(
optim_paras, params.loc["shocks_chol"], "chol"
)
optim_paras["shocks_cholesky"] = chol_params_to_lower_triangular_matrix(
sorted_shocks
)
else:
raise NotImplementedError
return optim_paras
def covariance_to_internal_jacobian(external_values, constr):
r"""Jacobian of ``covariance_to_internal``.
For reference see docstring of ``jacobian_covariance_from_internal``. In
comparison to that function, however, here we want to differentiate the
reverse graph
external --> cov --> cholesky --> internal
Again use the vectors :math:`c` and :math:`x` to denote the external and
internal values, respectively. To solve for the jacobian we make use of the
identity
.. math::
\frac{\mathrm{d}x}{\mathrm{d}c} = (\frac{\mathrm{d}c}{\mathrm{d}x})^{-1}
Args:
external_values (np.ndarray): Row-wise half-vectorized covariance matrix
Returns:
deriv: The Jacobian matrix.
"""
cov = cov_params_to_matrix(external_values)
chol = robust_cholesky(cov)
internal = chol[np.tril_indices(len(chol))]
deriv = covariance_from_internal_jacobian(internal, constr=None)
deriv = np.linalg.pinv(deriv)
return deriv
def _process_cov_constraint(constraint, params, fixed):
"""Process covariance constraints.
Args:
constraint (dict)
params (pd.DataFrame): see :ref:`params`.
Returns:
new_constr (dict): copy of *constraint* with a new entry called 'case',
which can take the values 'all_fixed', 'uncorrelated' and 'all_free'.
"""
new_constr = constraint.copy()
params_subset = params.loc[constraint["index"]]
fixed_subset = fixed.loc[constraint["index"]]
value_mat = cov_params_to_matrix(params_subset["value"].to_numpy())
fixed_mat = cov_params_to_matrix(
fixed_subset["_fixed"].to_numpy()).astype(bool)
new_constr["case"] = _determine_cov_case(value_mat, fixed_mat,
params_subset)
return new_constr
def _map_params_to_p(params, initial, square_root_filters):
nobs, nemf, nfac, _ = initial.shape
nfac = nfac - 1 if square_root_filters is True else nfac
filler = np.zeros((nemf, nfac, nfac))
for emf in range(nemf):
filler[emf] = cov_params_to_matrix(params.loc["p", 0, emf].to_numpy())
if square_root_filters is True:
filler = np.transpose(np.linalg.cholesky(filler), axes=(0, 2, 1))
initial[:, :, 1:, 1:] = filler
else:
initial[:] = filler
def _parse_shocks(optim_paras, params):
"""Parse the shock parameters and create the Cholesky factor."""
if sum(f"shocks_{i}" in params.index for i in ["sdcorr", "cov", "chol"]) >= 2:
raise ValueError("It is not allowed to define multiple shock matrices.")
elif "shocks_sdcorr" in params.index:
cov = sdcorr_params_to_matrix(params.loc["shocks_sdcorr"])
optim_paras["shocks_cholesky"] = robust_cholesky(cov)
elif "shocks_cov" in params.index:
cov = cov_params_to_matrix(params.loc["shocks_cov"])
optim_paras["shocks_cholesky"] = robust_cholesky(cov)
elif "shocks_chol" in params.index:
optim_paras["shocks_cholesky"] = chol_params_to_lower_triangular_matrix(
params.loc["shocks_chol"]
)
else:
raise KeyError("No shock matrix is specified.")
return optim_paras
def check_constraints_are_satisfied(pc, params):
"""Check that params satisfies all constraints.
This should be called before the more specialized constraints are rewritten to
linear constraints in order to get better error messages!
We let the checks pass if all "values" are np.nan. This way `process_constraints`
can be used on empty params DataFrames which is useful to construct templates for
start parameters that can be filled out by the user.
Args:
pc (list): List of constraints with processed selectors.
params (pd.DataFrame): See :ref:`params`
Raises:
ValueError if constraints are not satisfied.
"""
if params["value"].notnull().any():
for constr in pc:
typ = constr["type"]
subset = params.iloc[constr["index"]]["value"]
msg = f"{{}}:\n{subset.to_frame()}"
if typ == "covariance":
cov = cov_params_to_matrix(subset)
e, v = np.linalg.eigh(cov)
if not np.all(e > -1e-8):
raise ValueError(msg.format("Invalid covariance parameters."))
elif typ == "sdcorr":
cov = sdcorr_params_to_matrix(subset)
dim = len(cov)
if (subset.iloc[:dim] < 0).any():
raise ValueError(msg.format("Invalid standard deviations."))
if ((subset.iloc[dim:] < -1) | (subset.iloc[dim:] > 1)).any():
raise ValueError(msg.format("Invalid correlations."))
e, v = np.linalg.eigh(cov)
if not np.all(e > -1e-8):
raise ValueError(msg.format("Invalid sdcorr parameters."))
elif typ == "probability":
if not np.isclose(subset.sum(), 1, rtol=0.01):
raise ValueError(msg.format("Probabilities do not sum to 1"))
if np.any(subset < 0):
raise ValueError(msg.format("Negative Probability."))
if np.any(subset > 1):
raise ValueError(msg.format("Probability larger than 1."))
elif typ == "increasing":
if np.any(np.diff(subset) < 0):
raise ValueError(msg.format("Increasing constraint violated."))
elif typ == "decreasing":
if np.any(np.diff(subset) > 0):
raise ValueError(msg.format("Decreasing constraint violated"))
elif typ == "linear":
# using sr.dot is important in case weights are a series in wrong order
wsum = subset.dot(constr["weights"])
if "lower_bound" in constr and wsum < constr["lower_bound"]:
raise ValueError(
msg.format("Lower bound of linear constraint is violated")
)
elif "upper_bound" in constr and wsum > constr["upper_bound"]:
raise ValueError(
msg.format("Upper bound of linear constraint violated")
)
elif "value" in constr and not np.isclose(wsum, constr["value"]):
raise ValueError(
msg.format("Equality condition of linear constraint violated")
)
elif typ == "equality":
if len(subset.unique()) != 1:
raise ValueError(msg.format("Equality constraint violated."))
def _covariance_to_internal(params_subset, case):
"""Reparametrize parameters that describe a covariance matrix to internal.
The parameters in params_subset are assumed to be the lower triangular elements of
a covariance matrix.
If all parameters are fixed, nothing has to be done.
If all off-diagonal elements are fixed to zero, it is only necessary to set the
lower bounds of the off-diagonals to 0, unless already stricter.
Otherwise, we do a (lower triangular) Cholesky reparametrization and restrict
diagonal elements to be positive (see:
http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.31.494&rep=rep1&type=pdf)
Note that the cholesky reparametrization is not compatible with any other
constraints on the involved parameters. Moreover, it requires the covariance matrix
described by the start values to be positive definite as opposed to positive
semi-definite.
Args:
params_subset (DataFrame): relevant subset of non-internal params.
case (str): can take the values 'all_free', 'uncorrelated' or 'all_fixed'.
Returns:
res (DataFrame): copy of params_subset with adjusted 'value' and 'lower' columns
"""
res = params_subset.copy()
cov = cov_params_to_matrix(params_subset["value"].to_numpy())
dim = len(cov)
e, v = np.linalg.eigh(cov)
assert np.all(e > -1e-8), "Invalid covariance matrix."
if case == "uncorrelated":
lower_bound_helper = cov_params_to_matrix(params_subset["lower"])
diag_lower = np.maximum(np.diagonal(lower_bound_helper), np.zeros(dim))
lower_bound_helper[np.diag_indices(dim)] = diag_lower
lower_bounds = lower_bound_helper[np.tril_indices(dim)]
res["lower"] = lower_bounds
assert (res["upper"] >=
res["lower"]).all(), "Invalid upper bound for variance."
else:
chol = np.linalg.cholesky(cov)
chol_coeffs = chol[np.tril_indices(dim)]
res["value"] = chol_coeffs
lower_bound_helper = np.full((dim, dim), -np.inf)
lower_bound_helper[np.diag_indices(dim)] = 0
res["lower"] = lower_bound_helper[np.tril_indices(dim)]
res["upper"] = np.inf
res["fixed"] = False
if params_subset["fixed"].any():
warnings.warn("Covariance parameters are unfixed.", UserWarning)
for bound in ["lower", "upper"]:
if np.isfinite(params_subset[bound]).any():
warnings.warn("Bounds are ignored for covariance parameters.",
UserWarning)
return res
Note that the cholesky reparametrization is not compatible with any other
constraints on the involved parameters. Moreover, it requires the covariance matrix
described by the start values to be positive definite as opposed to positive
semi-definite.
Args:
params_subset (DataFrame): relevant subset of non-internal params.
case (str): can take the values 'free', 'uncorrelated' or 'all_fixed'.
Returns:
res (DataFrame): copy of params_subset with adjusted 'value' and 'lower' columns
"""
res = params_subset.copy()
if type_ == "covariance":
cov = cov_params_to_matrix(params_subset["value"].to_numpy())
elif type_ == "sdcorr":
cov = sdcorr_params_to_matrix(params_subset["value"].to_numpy())
else:
raise ValueError("Invalid type_: {}".format(type_))
dim = len(cov)
e, v = np.linalg.eigh(cov)
assert np.all(e > -1e-8), "Invalid covariance matrix."
if case == "uncorrelated":
res["lower"] = np.maximum(res["lower"], np.zeros(len(res)))
assert (res["upper"] >=
res["lower"]).all(), "Invalid upper bound for variance."
def test_cov_params_to_matrix():
params = np.array([1, 0.1, 2, 0.2, 0.22, 3])
expected = np.array([[1, 0.1, 0.2], [0.1, 2, 0.22], [0.2, 0.22, 3]])
calculated = cov_params_to_matrix(params)
aaae(calculated, expected)
def covariance_to_internal(external_values, constr):
"""Do a cholesky reparametrization."""
cov = cov_params_to_matrix(external_values)
chol = robust_cholesky(cov)
return chol[np.tril_indices(len(cov))]