def k_2_p_2_sphere_line_bundle():
Psi = [(2, 1)]
k_zero = 1
k = sum([a[1] for a in Psi]) + k_zero
p = Psi[0][0]
T = 1000
n = T + p
for ii in range(10):
D = np.arange(k) * 0.2 + 1
OO = random_orthogonal(k)
try:
es = varx_minimal_estimator(Psi, m=k)
stable, H, F, G, Phi = gen_random_stable(es, k)
Y, e = VAR_sim(Phi, n, D, OO)
Y = Y.T
e = e.T
X = Y.copy()
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, p)
except Exception:
continue
es_G = varx_minimal_estimator(Psi, m=k)
es_G.set_covs(cov_res, cov_xlag)
es_G.calc_states(G)
opt = es.simple_fit(Y, X)
if not opt['success']:
print("failed with opt=")
print(opt)
else:
print('orig_llk %s est_llk=%s' %
(es_G.neg_log_llk, es.neg_log_llk))
print(Phi.PolynomialMatrix_to_3darray())
print(es.Phi)
def gradient_fit(self, Y, X):
"""simple fitting
If success, self.Phi, self.H are set to optimal
values. Returning the optimizer values
"""
from scipy.optimize import minimize
from minimal_varx import make_normalized_G
from numpy import eye
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, self.p)
self.set_covs(cov_res, cov_xlag)
# k = Y.shape[0]
m = X.shape[0]
def f_ratio(x):
# x is a vector
if self.neg_log_llk is None:
self.calc_states(G=x.reshape(-1, m))
val = self.neg_log_llk
self.neg_log_llk = None
return val
def f_gradient(x):
try:
if self._gradient_tensor is not None:
grd = self._gradient_tensor.copy()
else:
self.calc_states(G=x.reshape(-1, m))
grd = self._gradient_tensor.copy()
except Exception:
self.calc_states(G=x.reshape(-1, m))
grd = self._gradient_tensor.copy()
self._gradient_tensor = None
return grd
c = np.random.randn(self.mm_degree - self.agg_rnk, m - self.agg_rnk)
G_init = make_normalized_G(self, m, eye(m), c)
x0 = G_init.reshape(-1)
opt = minimize(f_ratio, x0, jac=f_gradient)
if opt['success']:
G_opt = opt['x'].reshape(-1, m)
self.calc_H_F_Phi(G_opt, cov_y_xlag)
return opt
def k_8_p3_generic():
Psi = [(3, 1), (2, 1), (1, 2)]
k_zero = 4
k = sum([a[1] for a in Psi]) + k_zero
p = Psi[0][0]
T = 1000
n = T + p
for ii in range(10):
D = np.arange(k) * 0.2 + 1
OO = random_orthogonal(k)
try:
es = varx_minimal_estimator(Psi, m=k)
stable, H, F, G, Phi = gen_random_stable(es,
k,
scale_range=(0.3, .6))
Y, e = VAR_sim(Phi, n, D, OO)
Y = Y.T
e = e.T
X = Y.copy()
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, p)
except Exception:
pass
es_G = varx_minimal_estimator(Psi, m=k)
es_G.set_covs(cov_res, cov_xlag)
es_G.calc_states(G)
# simple fit does not work
found = False
cnt = 0
while (not found) and (cnt < 11):
opt = es.simple_fit(Y, X)
if opt['success']:
print(Phi.PolynomialMatrix_to_3darray())
print(es.Phi)
print(es_G.neg_log_llk)
print(es.neg_log_llk)
found = True
else:
print(opt)
cnt += 1
def test_manifold_fit():
from numpy.linalg import solve
Psi = [(2, 2), (1, 1)]
k_zero = 0
k = sum([a[1] for a in Psi]) + k_zero
p = Psi[0][0]
T = 1000
n = T + p
for ii in range(1):
D = np.arange(k) * 0.2 + 1
OO = random_orthogonal(k)
try:
es = varx_minimal_estimator(Psi, m=k)
stable, H, F, G, Phi = gen_random_stable(es, k)
Y, e = VAR_sim(Phi, n, D, OO)
Y = Y.T
e = e.T
X = Y.copy()
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, p)
except Exception:
continue
es_G = varx_minimal_estimator(Psi, m=k)
es_G.set_covs(cov_res, cov_xlag)
es_G.calc_states(G)
es1 = varx_minimal_estimator(Psi, m=k)
es1.set_covs(cov_res, cov_xlag)
es2 = varx_minimal_estimator(Psi, m=k)
es2.set_covs(cov_res, cov_xlag)
h_opt = es2.hessian_fit(Y, X)
print(h_opt)
# opt = es.gradient_fit(Y, X)
# self = es1
opt = es1.manifold_fit(Y, X)
print es1.neg_log_llk
"""
def test_hessian_fit():
from numpy.linalg import solve
Psi = [(3, 1), (2, 2), (1, 1)]
k_zero = 0
k = sum([a[1] for a in Psi]) + k_zero
p = Psi[0][0]
T = 1000
n = T + p
for ii in range(1):
D = np.arange(k) * 0.2 + 1
OO = random_orthogonal(k)
try:
es = varx_minimal_estimator(Psi, m=k)
stable, H, F, G, Phi = gen_random_stable(es, k)
Y, e = VAR_sim(Phi, n, D, OO)
Y = Y.T
e = e.T
X = Y.copy()
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, p)
except Exception:
continue
es_G = varx_minimal_estimator(Psi, m=k)
es_G.set_covs(cov_res, cov_xlag)
es_G.calc_states(G)
es1 = varx_minimal_estimator(Psi, m=k)
es1.set_covs(cov_res, cov_xlag)
opt = es.gradient_fit(Y, X)
opt = es1.hessian_fit(Y, X)
if not opt['success']:
print("failed with opt=")
print(opt)
else:
print('orig_llk %s est_llk=%s' % (es_G.neg_log_llk, opt['fun']))
print(Phi.PolynomialMatrix_to_3darray())
print(es1.Phi)
def k_5_p_2_all_psi():
k = 5
m = k
p = 2
Psi0 = [(2, 2), (1, 2)]
T = 1000
n = T + p
D = np.arange(k) * 0.2 + 1
OO = random_orthogonal(k)
try:
es = varx_minimal_estimator(Psi0, m=k)
stable, H, F, G, Phi = gen_random_stable(es, k)
Y, e = VAR_sim(Phi, n, D, OO)
Y = Y.T
e = e.T
X = Y.copy()
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, p)
es_G = varx_minimal_estimator(Psi0, m=k)
es_G.set_covs(cov_res, cov_xlag)
es_G.calc_states(G)
except Exception:
pass
all_psi = list_all_psi_hat(m=m, p=p)
for Psi_ in all_psi:
Psi = psi_hat_to_psi(Psi_)
es = varx_minimal_estimator(Psi, m=k)
opt = es.simple_fit(Y, X)
print('Psi=%s' % Psi)
if not opt['success']:
print("failed with opt=")
print(opt)
else:
print('orig_llk %s est_llk=%s' %
(es_G.neg_log_llk, es.neg_log_llk))
def manifold_fit(self, Y, X):
from pymanopt.manifolds import Rotations, Euclidean, Product
from pymanopt import Problem
from pymanopt.solvers import TrustRegions
from minimal_varx import make_normalized_G
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, self.p)
self.set_covs(cov_res, cov_xlag)
m = X.shape[0]
with_c = (self.mm_degree > self.agg_rnk) and (self.m - self.agg_rnk)
C = Euclidean(self.mm_degree - self.agg_rnk, self.m - self.agg_rnk)
RG = Rotations(self.m)
if with_c:
manifold = Product([RG, C])
else:
manifold = RG
if not with_c:
c_null = np.zeros(
(self.mm_degree - self.agg_rnk, self.m - self.agg_rnk))
else:
c_null = None
if with_c:
def cost(x):
o, c = x
G = make_normalized_G(self, self.m, o, c)
self.calc_states(G)
return self.neg_log_llk
else:
def cost(x):
G = make_normalized_G(self, self.m, x, c_null)
self.calc_states(G)
return self.neg_log_llk
if with_c:
def egrad(x):
o, c = x
grad_o, grad_c = map_o_c_grad(self, self._gradient_tensor, o, c)
return [grad_o, grad_c]
else:
def egrad(x):
grad_o, grad_c = map_o_c_grad(self, self._gradient_tensor, x,
c_null)
return grad_o
if with_c:
def ehess(x, Heta):
o, c = x
eta = make_normalized_G(self, self.m, Heta[0], Heta[1])
hess_raw = self.hessian_prod(eta)
hess_o, hess_c = map_o_c_grad(self, hess_raw, o, c)
return [hess_o, hess_c]
else:
def ehess(x, Heta):
eta = make_normalized_G(self, self.m, Heta, c_null)
hess_raw = self.hessian_prod(eta)
hess_o, hess_c = map_o_c_grad(self, hess_raw, x, c_null)
return hess_o
if with_c:
min_mle = Problem(manifold, cost, egrad=egrad, ehess=ehess)
else:
min_mle = Problem(manifold, cost, egrad=egrad, ehess=ehess)
solver = TrustRegions()
opt = solver.solve(min_mle)
if with_c:
G_opt = make_normalized_G(self, self.m, opt[0], opt[1])
else:
G_opt = make_normalized_G(self, self.m, opt, c_null)
self.calc_H_F_Phi(G_opt, cov_y_xlag)
return opt
def test_hessian_calc():
from numpy.linalg import solve
Psi = [(3, 1), (2, 1), (1, 1)]
k_zero = 1
k = sum([a[1] for a in Psi]) + k_zero
p = Psi[0][0]
T = 1000
n = T + p
for ii in range(1):
D = np.arange(k) * 0.2 + 1
OO = random_orthogonal(k)
try:
es = varx_minimal_estimator(Psi, m=k)
stable, H, F, G, Phi = gen_random_stable(es, k)
Y, e = VAR_sim(Phi, n, D, OO)
Y = Y.T
e = e.T
X = Y.copy()
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, p)
except Exception:
continue
"""
es_G = varx_minimal_estimator(Psi, m=k)
es_G.set_covs(cov_res, cov_xlag)
es_G.calc_states(G)
"""
es1 = varx_minimal_estimator(Psi, m=k)
es1.set_covs(cov_res, cov_xlag)
m = k
c = np.random.randn(self.mm_degree - self.agg_rnk, m - self.agg_rnk)
OO = random_orthogonal(m)
from minimal_varx import make_normalized_G
G_test = make_normalized_G(es1, m, OO, c)
es1.calc_states(G_test)
# h = 1e-6
# self = es1
es2 = varx_minimal_estimator(Psi, m=k)
es2.set_covs(cov_res, cov_xlag)
h = 1e-6
self = es1
# eta = np.random.randn(*G.shape)
# G_1 = G_test.reshape(-1).copy()
# G_1 += h * eta.reshape(-1)
# es2.calc_states(G_1.reshape(self.G.shape))
# diff = (es2._gradient_tensor - es1._gradient_tensor) / h
# hs = hessian_prod(self, eta)
# print(diff)
g_size = self._gradient_tensor.shape[0]
appx_Hess_matrix = np.zeros((g_size, g_size))
for i in range(g_size):
G_1 = es1.G.reshape(-1).copy()
G_1[i] += h
es2.calc_states(G_1.reshape(self.G.shape))
appx_Hess_matrix[i, :] = (es2._gradient_tensor -
es1._gradient_tensor) / h
print(appx_Hess_matrix)
def hessian_prod(self, eta):
hessp = np.zeros(eta.reshape(-1).shape[0])
for i in range(hessp.shape[0]):
a_i = self.kappa_tensor[:, i].reshape(-1, self.p * self.m)
# numerator_mat
kappa_num_a_i = self._kappa_cov_numerator @ a_i.T
a_i_num = a_i @ self._cov_numerator
kappa_eta_T = self.calc_kappa(eta).T
a_i_num_eta = a_i_num @ kappa_eta_T
s1 = solve(self._numerator_mat,
kappa_num_a_i + kappa_num_a_i.T)
s2 = solve(self._numerator_mat,
self._kappa_cov_numerator @ kappa_eta_T)
first_part_num = -s1 @ s2
second_part_num = solve(self._numerator_mat, a_i_num_eta)
hess_num = first_part_num + second_part_num
# denominator
kappa_denom_a_i = self._kappa_cov_denominator @ a_i.T
a_i_denom = a_i @ self._cov_denominator
a_i_denom_eta = a_i_denom @ kappa_eta_T
sd1 = solve(self._denominator_mat,
kappa_denom_a_i + kappa_denom_a_i.T)
sd2 = solve(self._denominator_mat,
self._kappa_cov_denominator @ kappa_eta_T)
first_part_denom = -sd1 @ sd2
second_part_denom = solve(self._denominator_mat, a_i_denom_eta)
hess_denom = first_part_denom + second_part_denom
hessp[i] = 2 * np.sum(np.diagonal(hess_num - hess_denom))
return hessp
exact_Hess = np.zeros((g_size, g_size))
for jj in range(g_size):
eta_ = np.zeros(g_size)
eta_[jj] += 1
eta = eta_.reshape(G_test.shape)
exact_Hess[jj, :] = hessian_prod(self, eta)
print(exact_Hess)
def hessian_fit(self, Y, X):
"""hessian fitting
If success, self.Phi, self.H are set to optimal
values. Returning the optimizer values
"""
from scipy.optimize import minimize
from minimal_varx import make_normalized_G
from numpy import eye
cov_res, cov_xlag, cov_y_xlag = calc_extended_covariance(Y, X, self.p)
self.set_covs(cov_res, cov_xlag)
# k = Y.shape[0]
m = X.shape[0]
def f_ratio(x):
# x is a vector
if self.neg_log_llk is None:
self.calc_states(G=x.reshape(-1, m))
val = self.neg_log_llk
self.neg_log_llk = None
return val
def f_gradient(x):
try:
if self._gradient_tensor is not None:
grd = self._gradient_tensor.copy()
else:
self.calc_states(G=x.reshape(-1, m))
grd = self._gradient_tensor.copy()
except Exception:
self.calc_states(G=x.reshape(-1, m))
grd = self._gradient_tensor.copy()
self._gradient_tensor = None
return grd
def hessian_prod(x, eta):
return self.hessian_prod(eta.reshape(self.mm_degree, self.m))
def hessian_prod_(x, eta):
hessp = np.zeros(eta.shape[0])
for i in range(hessp.shape[0]):
a_i = self.kappa_tensor[:, i].reshape(-1, self.p * self.m)
# numerator_mat
kappa_num_a_i = self._kappa_cov_numerator @ a_i.T
a_i_num = a_i @ self._cov_numerator
kappa_eta_T = self.calc_kappa(eta.reshape(self.mm_degree,
self.m)).T
a_i_num_eta = a_i_num @ kappa_eta_T
s1 = solve(self._numerator_mat, kappa_num_a_i + kappa_num_a_i.T)
s2 = solve(self._numerator_mat,
self._kappa_cov_numerator @ kappa_eta_T)
first_part_num = -s1 @ s2
second_part_num = solve(self._numerator_mat, a_i_num_eta)
hess_num = first_part_num + second_part_num
# denominator
kappa_denom_a_i = self._kappa_cov_denominator @ a_i.T
a_i_denom = a_i @ self._cov_denominator
a_i_denom_eta = a_i_denom @ kappa_eta_T
sd1 = solve(self._denominator_mat,
kappa_denom_a_i + kappa_denom_a_i.T)
sd2 = solve(self._denominator_mat,
self._kappa_cov_denominator @ kappa_eta_T)
first_part_denom = -sd1 @ sd2
second_part_denom = solve(self._denominator_mat, a_i_denom_eta)
hess_denom = first_part_denom + second_part_denom
hessp[i] = 2 * np.sum(np.diagonal(hess_num - hess_denom))
return hessp
c = np.random.randn(self.mm_degree - self.agg_rnk, m - self.agg_rnk)
OO = random_orthogonal(m)
G_init = make_normalized_G(self, m, OO, c)
x0 = G_init.reshape(-1)
opt = scipy.optimize.minimize(f_ratio,
x0,
method='trust-ncg',
jac=f_gradient,
hessp=hessian_prod)
if opt['success']:
G_opt = opt['x'].reshape(-1, m)
self.calc_H_F_Phi(G_opt, cov_y_xlag)
return opt