def test_matrix_frac(self) -> None:
"""Test gradient for matrix_frac
"""
expr = cp.matrix_frac(self.A, self.B)
self.A.value = np.eye(2)
self.B.value = np.eye(2)
self.assertItemsAlmostEqual(expr.grad[self.A].toarray(), [2, 0, 0, 2])
self.assertItemsAlmostEqual(expr.grad[self.B].toarray(),
[-1, 0, 0, -1])
self.B.value = np.zeros((2, 2))
self.assertAlmostEqual(expr.grad[self.A], None)
self.assertAlmostEqual(expr.grad[self.B], None)
expr = cp.matrix_frac(self.x[:, None], self.A)
self.x.value = [2, 3]
self.A.value = np.eye(2)
self.assertItemsAlmostEqual(expr.grad[self.x].toarray(), [4, 6])
self.assertItemsAlmostEqual(expr.grad[self.A].toarray(),
[-4, -6, -6, -9])
expr = cp.matrix_frac(self.x, self.A)
self.x.value = [2, 3]
self.A.value = np.eye(2)
self.assertItemsAlmostEqual(expr.grad[self.x].toarray(), [4, 6])
self.assertItemsAlmostEqual(expr.grad[self.A].toarray(),
[-4, -6, -6, -9])
def mean_cov_prox_cvxpy(Y, eta, theta, t): if Y is None: return eta Y = Y[0] n,N = Y.shape ybar = cp.Parameter((n,1)) ybar.value = np.mean(Y,1).reshape(-1,1) Yemp = cp.Parameter((n,n)) Yemp.value = Y @ Y.T / N S = cp.Variable((n,n)) nu = cp.Variable((n,1)) eps = (1./(2*t)) main_part = -cp.log_det(S) main_part += cp.trace(S@Yemp) main_part += - 2*ybar.T@nu main_part += cp.matrix_frac(nu, S) prox_part = eps * cp.sum_squares(nu-eta[:, -1].reshape(-1,1)) prox_part += eps * cp.norm(S-eta[:, :-1], "fro")**2 prob = cp.Problem(cp.Minimize(N*main_part+prox_part)) prob.solve(verbose=False, warm_start=True) return np.hstack((S.value, nu.value))
def get_robust_hinf_policy(A, B, G, Q, R, alpha, gamma):
n, m = B.shape
wq = G.shape[1]
S = cp.Variable((n, n), symmetric=True)
Y = cp.Variable((m, n))
mu = cp.Variable()
Q_sqrt = la.sqrtm(Q)
R_sqrt = la.sqrtm(R)
f = cp.trace(S @ Q) + cp.matrix_frac(Y.T @ R_sqrt, S)
cons_mat = cp.bmat([[S @ A.T + A @ S + Y.T @ B.T + B @ Y + alpha * S + (mu / gamma ** 2) * G @ G.T,
cp.bmat([[S @ Q_sqrt, Y.T @ R_sqrt]])],
[cp.bmat([[Q_sqrt @ S], [R_sqrt @ Y]]), -mu * np.eye(m + n)]])
cons = [S >> np.eye(n), mu >= 0] + [cons_mat << -1e-3 * np.eye(n+m+n)]
try:
prob = cp.Problem(cp.Minimize(f), cons)
prob.solve(solver=cp.SCS) #cp.MOSEK)
except cp.error.SolverError as e:
raise ValueError('Solver failed with error: %s \n Try another environment seed' % e)
K = np.linalg.solve(S.value, Y.value.T).T
assert np.all(np.linalg.eigvals(S.value) > 0)
assert np.all(mu.value > 0)
assert np.all(np.linalg.eigvals(cons_mat.value) <= 0)
return K, S.value, mu.value
def rcwc(A, b, is_verbose=True, weights=None, solveroptions={}):
"""
A: m x n matrix with A_ij being an indicator for whether element j appears in sample set i
b: m x n matrix with b_ij being the weights for element j in target set i corresponding to target set mean
"""
m, n = A.shape
V = cp.Variable(shape=(n,n), PSD=True)
constraints = [V[j,j] <= 1 for j in range(n)]
objective = 0
for i in range(m):
row_mask = np.nonzero(A[i,:])[0]
rowcol_mask = np.ix_(row_mask, row_mask)
objterm = cp.quad_form(b[i,:], V) - cp.matrix_frac(V[row_mask,:] @ b[i,:], V[rowcol_mask])
if weights is not None:
objterm *= weights[i]
objective += objterm
if weights is None:
objective /= m
prob = cp.Problem(cp.Maximize(objective), constraints)
prob.solve(verbose=is_verbose, solver=cp.MOSEK, **solveroptions)
print(prob.value)
Vhat = V.value
a = np.zeros((m,n))
for i in range(m):
row_mask = np.nonzero(A[i,:])[0]
rowcol_mask = np.ix_(row_mask, row_mask)
a[i,row_mask] = np.linalg.inv(Vhat[rowcol_mask]) @ Vhat[row_mask,:] @ b[i,:]
a[A == 0] = 0
return a
def test_matrix_frac(self) -> None:
"""Test domain for matrix_frac.
"""
dom = cp.matrix_frac(self.x, self.A + np.eye(2)).domain
prob = Problem(Minimize(cp.sum(cp.diag(self.A))), dom)
prob.solve(solver=cp.SCS)
self.assertAlmostEqual(prob.value, -2, places=3)
def test_matrix_frac(self):
"""Test for the matrix_frac atom.
"""
atom = cp.matrix_frac(self.x, self.A)
self.assertEqual(atom.shape, tuple())
self.assertEqual(atom.curvature, s.CONVEX)
# Test matrix_frac shape validation.
with self.assertRaises(Exception) as cm:
cp.matrix_frac(self.x, self.C)
self.assertEqual(str(cm.exception),
"The second argument to matrix_frac must be a square matrix.")
with self.assertRaises(Exception) as cm:
cp.matrix_frac(Variable(3), self.A)
self.assertEqual(str(cm.exception),
"The arguments to matrix_frac have incompatible dimensions.")
def test_key_error(self):
"""Test examples that caused key error.
"""
import cvxpy as cvx
x = cvx.Variable()
u = -cvx.exp(x)
prob = cvx.Problem(cvx.Maximize(u), [x == 1])
prob.solve(verbose=True, solver=cvx.CVXOPT)
prob.solve(verbose=True, solver=cvx.CVXOPT)
###########################################
import numpy as np
import cvxopt
import cvxpy as cp
kD=2
Sk=cp.semidefinite(kD)
Rsk=cp.Parameter(kD,kD)
mk=cp.Variable(kD,1)
musk=cp.Parameter(kD,1)
logpart=-0.5*cp.log_det(Sk)+0.5*cp.matrix_frac(mk,Sk)+(kD/2.)*np.log(2*np.pi)
linpart=mk.T*musk-0.5*cp.trace(Sk*Rsk)
obj=logpart-linpart
prob=cp.Problem(cp.Minimize(obj))
musk.value=np.ones((2,1))
covsk=np.diag([0.3,0.5])
Rsk.value=covsk+(musk.value*musk.value.T)
prob.solve(verbose=True,solver=cp.CVXOPT)
print "second solve"
prob.solve(verbose=False, solver=cp.CVXOPT)
def test_key_error(self):
"""Test examples that caused key error.
"""
if cvx.CVXOPT in cvx.installed_solvers():
x = cvx.Variable()
u = -cvx.exp(x)
prob = cvx.Problem(cvx.Maximize(u), [x == 1])
prob.solve(verbose=True, solver=cvx.CVXOPT)
prob.solve(verbose=True, solver=cvx.CVXOPT)
###########################################
import numpy as np
kD = 2
Sk = cvx.Variable((kD, kD), PSD=True)
Rsk = cvx.Parameter((kD, kD))
mk = cvx.Variable((kD, 1))
musk = cvx.Parameter((kD, 1))
logpart = -0.5 * cvx.log_det(Sk) + 0.5 * cvx.matrix_frac(
mk, Sk) + (kD / 2.) * np.log(2 * np.pi)
linpart = mk.T * musk - 0.5 * cvx.trace(Sk * Rsk)
obj = logpart - linpart
prob = cvx.Problem(cvx.Minimize(obj), [Sk == Sk.T])
musk.value = np.ones((2, 1))
covsk = np.diag([0.3, 0.5])
Rsk.value = covsk + (musk.value * musk.value.T)
prob.solve(verbose=True, solver=cvx.CVXOPT)
print("second solve")
prob.solve(verbose=False, solver=cvx.CVXOPT)
def get_robust_lqr_sol(A, B, G, C, D, Q, R, alpha):
n, m = B.shape
wq = C.shape[0]
S = cp.Variable((n, n), symmetric=True)
Y = cp.Variable((m, n))
mu = cp.Variable()
R_sqrt = la.sqrtm(R)
f = cp.trace(S @ Q) + cp.matrix_frac(Y.T @ R_sqrt, S)
cons_mat = cp.bmat((
(A @ S + S @ A.T + cp.multiply(mu, G @ G.T) + B @ Y + Y.T @ B.T + alpha * S, S @ C.T + Y.T @ D.T),
(C @ S + D @ Y, -cp.multiply(mu, np.eye(wq)))
))
cons = [S >> 0, mu >= 1e-2] + [cons_mat << 0]
try:
prob = cp.Problem(cp.Minimize(f), cons)
prob.solve(solver=cp.SCS)
except cp.error.SolverError as e:
raise ValueError('Solver failed with error: %s \n Try another environment seed' % e)
K = np.linalg.solve(S.value, Y.value.T).T
return K, S.value
def test_matrix_frac(self) -> None:
x = Variable(5)
M = np.eye(5)
P = M.T @ M
s = cp.matrix_frac(x, P)
self.assertFalse(s.is_constant())
self.assertFalse(s.is_affine())
self.assertTrue(s.is_quadratic())
self.assertTrue(s.is_dcp())
def main():
print("What is N ... ", end='')
N = int(input())
print("\nWhat is a ... ", end='')
a = int(input())
print("\nWhat is b ... ", end='')
b = int(input())
print("\nWhat is p ... ", end='')
p = int(input())
print('\n( N, a, b, p) = (', N, ',', a, ',', b, ',', p, ')')
c1 = np.array([[1], [1], [1], [1]])
c2 = np.array([[1], [-1], [1], [-1]])
C = [c1, c2]
#Generate A
A = []
for i in range(1, N + 1):
u_i = a + (b - a) * (i - 1) / (N - 1)
tmp_a = np.array([[u_i**i for i in range(p + 1)]]) # 1 x (p+1)
tmp_b = np.transpose(tmp_a) # (p+1) x 1
A_i = np.matmul(tmp_b, tmp_a)
A.append(A_i)
# Initializing Variables, Objective, and Constraints
W = cp.Variable(N)
obj = cp.Minimize(
cp.sum([
cp.matrix_frac(c, sum([W[i] * A[i] for i in range(N)])) for c in C
]))
const = [0 <= W, cp.sum(W) == 1]
# Creating Problem & Solving
start = timeit.default_timer()
prob = cp.Problem(obj, const)
prob.solve()
stop = timeit.default_timer()
print('Index\tValue')
eps = 0.0001
for i in range(N):
if W.value[i] > eps:
print(f"{a + (b - a) * ((i+1)-1) / (N-1)}\t{W[i].value}")
print("Sum of W = ", sum(W.value))
print("\nMethod used: ", prob.solver_stats.solver_name)
print('Time to Compute: ', stop - start)
def test_matrix_frac(self):
"""Test matrix_frac atom.
"""
P = np.array([[10, 1j], [-1j, 10]])
Y = Variable((2, 2), complex=True)
b = np.arange(2)
x = Variable(2, complex=False)
value = cvx.matrix_frac(b, P).value
expr = cvx.matrix_frac(x, Y)
prob = Problem(cvx.Minimize(expr), [x == b, Y == P])
result = prob.solve(solver=cvx.SCS, eps=1e-6, max_iters=7500, verbose=True)
self.assertAlmostEqual(result, value, places=3)
b = (np.arange(2) + 3j*(np.arange(2) + 10))
x = Variable(2, complex=True)
value = cvx.matrix_frac(b, P).value
expr = cvx.matrix_frac(x, Y)
prob = Problem(cvx.Minimize(expr), [x == b, Y == P])
result = prob.solve(solver=cvx.SCS, eps=1e-6)
self.assertAlmostEqual(result, value, places=3)
b = (np.arange(2) + 10)/10j
x = Variable(2, imag=True)
value = cvx.matrix_frac(b, P).value
expr = cvx.matrix_frac(x, Y)
prob = Problem(cvx.Minimize(expr), [x == b, Y == P])
result = prob.solve(solver=cvx.SCS, eps=1e-5, max_iters=7500)
self.assertAlmostEqual(result, value, places=3)
def get_robust_pldi_policy(A, B, Q, R, alpha):
L, n, m = B.shape
S = cp.Variable((n, n), symmetric=True)
Y = cp.Variable((m, n))
R_sqrt = la.sqrtm(R)
f = cp.trace(S @ Q) + cp.matrix_frac(Y.T @ R_sqrt, S)
cons = [S >> np.eye(n)] + [A[i, :, :] @ S + B[i, :, :] @ Y + S @ A[i, :, :].T + Y.T @ B[i, :, :].T << -alpha * S for i in range(A.shape[0])]
prob = cp.Problem(cp.Minimize(f), cons)
prob.solve(solver=cp.MOSEK)
K = np.linalg.solve(S.value, Y.value.T).T
return K, S.value
def c_optimal(A, N, c):
# Initializing Variables, Objective, and Constraints
p = A[1].shape[0] - 1
W = cp.Variable(N)
obj = cp.Minimize(cp.matrix_frac(c, sum([W[i] * A[i] for i in range(N)])))
const = [0 <= W, cp.sum(W) == 1]
# Creating Problem & Solving
prob = cp.Problem(obj, const)
prob.solve()
return prob
def _construct_objective(covariance_matrices, lambdas):
''' Constructs the CVX objective function. '''
num_points = len(covariance_matrices)
num_dim = int(covariance_matrices[0].shape[0])
objective = 0
matrix_part = np.zeros([num_dim, num_dim])
for j in range(0, num_points):
matrix_part = matrix_part + covariance_matrices[j] * lambdas[j]
for i in range(0, num_dim):
k_vec = np.zeros(num_dim)
k_vec[i] = 1.0
objective = objective + cvx.matrix_frac(k_vec, matrix_part)
return objective
def get_custom_lqr_sol(A, B, Q, R, alpha):
n, m = B.shape
S = cp.Variable((n, n), symmetric=True)
Y = cp.Variable((m, n))
R_sqrt = la.sqrtm(R)
f = cp.trace(S @ Q) + cp.matrix_frac(Y.T @ R_sqrt, S)
# Exponential stability constraints from LMI book
cons = [S >> np.eye(n)] # make LMI non-homogeneous
cons += [A @ S + S @ A.T + B @ Y + Y.T @ B.T << -alpha * S]
cp.Problem(cp.Minimize(f), cons).solve()
K = np.linalg.solve(S.value, Y.value.T).T
S = S.value
return np.array(K), np.array(S)
def mode_solver(self):
"""
mode = arg min x^T*Sigma^-1*x
"""
if self.f is None:
return self.mu
m = len(self.mu)
xi = cp.Variable(m)
obj = cp.Minimize(cp.matrix_frac(xi, self.Sigma))
constraints = [self.f * (xi + self.mu) + self.g >= 0]
prob = cp.Problem(obj, constraints)
prob.solve()
# print("status:", prob.status)
if prob.status != "optimal":
raise ValueError('cannot compute the mode')
return xi.value
def mode(self, x, y):
"""
:param x: (n,) or (n,2), the design of experiment [x1,x2,...,xn]
:param y: (n,), array_like, the true value at x
:return : (in 2D, m = m1*m2)
mu: (m,) the posterior mean with interpolation condition only, Gamma*Phi^T*[Phi*Gamma*Phi^T]^-1*y
Sigma: (m,m), the covariance matrix of the posterior with interpolation condition only,
Sigma = Gamma-Gamma*Phi^T*[Phi*Gamma*Phi^T]^-1*Phi*Gamma
mode: (m,) the posterior mode which is given by the maximum of the PDF of the posterior
"""
l, Lambda, u = self.inequality_constraints()
Phi = self.interpolation_constraints(x)
Gamma = self.covariance()
Gamma = Gamma + self.alpha * np.eye(len(Gamma))
mu = Gamma @ Phi.T @ np.linalg.solve(Phi @ Gamma @ Phi.T, y)
Sigma = Gamma - Gamma @ Phi.T @ np.linalg.solve(
Phi @ Gamma @ Phi.T, Phi) @ Gamma
Sigma = Sigma + self.alpha * np.eye(len(Sigma))
if Lambda is None:
# no inequality constraints, so mode is the posterior mean
mode = mu
else:
xi = cp.Variable(np.prod(self.m))
obj = cp.Minimize(cp.matrix_frac(xi, Gamma))
constraints = [Phi @ xi == y]
if not np.all(l == -np.inf):
constraints.append(
Lambda[l != -np.inf] * xi >= l[l != -np.inf])
if not np.all(u == np.inf):
constraints.append(Lambda[u != np.inf] * xi <= u[u != np.inf])
prob = cp.Problem(obj, constraints)
# print("Problem is DCP: ", prob.is_dcp())
prob.solve()
# print("status:", prob.status)
if prob.status != "optimal":
raise ValueError('cannot compute the mode')
mode = xi.value
return mu, Sigma, mode
def get_objective(self, covariance_matrices, lambdas):
num_points = len(covariance_matrices)
num_dim = int(covariance_matrices[0].shape[0])
#print num_points
#print num_dim
#print int(covariance_matrices[0].shape[1])
objective = 0
matrix_part = np.zeros([num_dim, num_dim])
for j in xrange(0, num_points):
matrix_part = matrix_part + covariance_matrices[j] * lambdas[j]
# return np.matrix.trace(np.linalg.inv(matrix_part))
for i in xrange(0, num_dim):
k_vec = np.zeros(num_dim)
k_vec[i] = 1.0
objective = objective + cvx.matrix_frac(k_vec, matrix_part)
return objective
def small_cone(x, verbose=False):
"""
Parameters
----------
x : numpy.array
An N x d array, where d is the dimension and N is the number of points
"""
N, d = x.shape
A = cvx.Variable((d, d), PSD=True)
b = cvx.Variable(d)
cst = [cvx.norm(A @ x[i]) <= b @ x[i]
for i in range(N)] + [cvx.matrix_frac(b, A) <= 1]
obj = cvx.Minimize(-cvx.log_det(A))
prob = cvx.Problem(obj, cst)
prob.solve(verbose=verbose, eps=1e-8)
A = A.value
b = b.value
return A, b, prob
print tau_2, b
print b.shape, tau_2.shape
#inv_alpha_1 = cvx.Variable(1)
inv_alpha_1 = 0.5
# specify objective function
#obj = cvx.Minimize(-cvx.sum_entries(cvx.log(tau_1)) - cvx.log_det(A - cvx.diag(tau_1)))
#obj = cvx.Minimize(-cvx.sum_entries(cvx.log(tau_1)) - cvx.log_det(A - cvx.diag(tau_1)))
# original
#obj = cvx.Minimize( 0.5*N*(inv_alpha_1-1)*log_2_pi - 0.5*inv_alpha_1*(N*cvx.log(1/inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5*cvx.sum_entries(cvx.square(tau_2)/tau_1) + inv_alpha_1*cvx.sum_entries(log_normcdf(tau_2*cvx.sqrt(1/(inv_alpha_1*tau_1)))) +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) )
# modifications
#obj = cvx.Minimize( 0.5*N*(inv_alpha_1-1)*log_2_pi - 0.5*inv_alpha_1*(-N*cvx.log(inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5*cvx.matrix_frac(tau_2, cvx.diag(tau_1)) + inv_alpha_1*cvx.sum_entries(log_normcdf(tau_2.T*cvx.inv_pos(cvx.sqrt(inv_alpha_1*tau_1)))) +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) )
obj = cvx.Minimize(
0.5 * N * (inv_alpha_1 - 1) * log_2_pi - 0.5 * inv_alpha_1 *
(-N * cvx.log(inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) +
0.5 * cvx.matrix_frac(tau_2, cvx.diag(tau_1)) + cvx.sum_entries(
inv_alpha_1 * log_normcdf(cvx.inv_pos(cvx.sqrt(inv_alpha_1 * tau_1))))
) # +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) )
#def upper_bound_logpartition(tau, inv_alpha_1):
# tau_1, tau_2 = tau[:D+N], tau[D+N:]
# tau_1_N, tau_2_N = tau_1[D:], tau_2[D:] # first D values correspond to w
# alpha_1 = 1.0 / inv_alpha_1
# inv_alpha_2 = 1 - inv_alpha_1
# if np.any(tau_1 <= 0):
# integral_1 = INF2
# else:
# integral_1 = inv_alpha_1 * (-0.5 * ((D+N)*np.log(alpha_1) + np.sum(np.log(tau_1)) ) \
# + np.sum(norm.logcdf(np.sqrt(alpha_1)*tau_2_N/np.sqrt(tau_1_N)))) \
# + 0.5 * np.sum(np.power(tau_2, 2) / tau_1)
# mat = A - np.diag(tau_1)
tau_1 = cvx.Variable(N) #tau_2 = cvx.Variable(N) tau_2 = np.zeros(N) print tau_2, b print b.shape, tau_2.shape #inv_alpha_1 = cvx.Variable(1) inv_alpha_1 = 0.5 # specify objective function #obj = cvx.Minimize(-cvx.sum_entries(cvx.log(tau_1)) - cvx.log_det(A - cvx.diag(tau_1))) #obj = cvx.Minimize(-cvx.sum_entries(cvx.log(tau_1)) - cvx.log_det(A - cvx.diag(tau_1))) # original #obj = cvx.Minimize( 0.5*N*(inv_alpha_1-1)*log_2_pi - 0.5*inv_alpha_1*(N*cvx.log(1/inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5*cvx.sum_entries(cvx.square(tau_2)/tau_1) + inv_alpha_1*cvx.sum_entries(log_normcdf(tau_2*cvx.sqrt(1/(inv_alpha_1*tau_1)))) +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) ) # modifications #obj = cvx.Minimize( 0.5*N*(inv_alpha_1-1)*log_2_pi - 0.5*inv_alpha_1*(-N*cvx.log(inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5*cvx.matrix_frac(tau_2, cvx.diag(tau_1)) + inv_alpha_1*cvx.sum_entries(log_normcdf(tau_2.T*cvx.inv_pos(cvx.sqrt(inv_alpha_1*tau_1)))) +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) ) obj = cvx.Minimize( 0.5*N*(inv_alpha_1-1)*log_2_pi - 0.5*inv_alpha_1*(-N*cvx.log(inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5*cvx.matrix_frac(tau_2, cvx.diag(tau_1)) + cvx.sum_entries(inv_alpha_1*log_normcdf(cvx.inv_pos(cvx.sqrt(inv_alpha_1*tau_1)))) )# +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) ) #def upper_bound_logpartition(tau, inv_alpha_1): # tau_1, tau_2 = tau[:D+N], tau[D+N:] # tau_1_N, tau_2_N = tau_1[D:], tau_2[D:] # first D values correspond to w # alpha_1 = 1.0 / inv_alpha_1 # inv_alpha_2 = 1 - inv_alpha_1 # if np.any(tau_1 <= 0): # integral_1 = INF2 # else: # integral_1 = inv_alpha_1 * (-0.5 * ((D+N)*np.log(alpha_1) + np.sum(np.log(tau_1)) ) \ # + np.sum(norm.logcdf(np.sqrt(alpha_1)*tau_2_N/np.sqrt(tau_1_N)))) \ # + 0.5 * np.sum(np.power(tau_2, 2) / tau_1) # mat = A - np.diag(tau_1) # sign, logdet = np.linalg.slogdet(mat)
def matrix_frac(b, A):
return cvx.matrix_frac(b, A).value
def fit(self, X, y, groups):
"""
Fit a hierarchical linear regression to data
Parameters
----------
X : pandas data frame or numpy array
Table with predictors/covariates. Categorical variables must be properly encoded beforehand.
y : pandas data frame or numpy array
Values to predict.
groups : pandas data frame or numpy array
Groups to which each observation belongs.
Can have more than one column if there are multiple group categories (e.g. 'province' and 'city').
Values can be strings or numbers, will be one-hot-encoded internally.
Observations not belonging to any group must have NA values.
"""
## checking input
if type(X) == pd.core.frame.DataFrame:
x = X.as_matrix()
xnames = X.columns.values
elif (type(X) == np.ndarray) or (type(X)
== np.matrixlib.defmatrix.matrix):
x = X.copy()
xnames = None
else:
raise ValueError("'X' must be a pandas data frame or numpy array")
if (type(y) == pd.core.frame.DataFrame) or (type(y)
== pd.core.series.Series):
yval = y.as_matrix()
elif (type(y) == np.ndarray) or (type(y)
== np.matrixlib.defmatrix.matrix):
yval = y.copy()
else:
raise ValueError(
"'y' must be a pandas data frame, pandas series, or numpy array"
)
if self.problem == 'classification':
classes = set(yval)
if len(classes) != 2:
raise ValueError(
"Only binary classification is supported. 'y' contains " +
str(len(classes)) + " values")
if (1 in classes) and ((-1) in classes):
self.class1 = 1
self.class2 = -1
elif (1 in classes) and (0 in classes):
yval[yval == 0] = -1.0
self.class1 = 1
self.class2 = 0
else:
self.class1, self.class2 = tuple(classes)
yval[yval == class1] = 1.0
yval[yval == class2] = -1.0
if type(groups) == pd.core.series.Series:
gbin = pd.get_dummies(np.array(groups.astype('str')))
self._gdim = 1
self._gnames = ''
elif type(groups) == pd.core.frame.DataFrame:
self._gdim = groups.shape[1]
self._gnames = list(groups.columns.values)
gbin = pd.get_dummies(groups.astype('str'))
elif (type(groups)
== np.ndarray) or (type(groups)
== np.matrixlib.defmatrix.matrix):
try:
self._gdim = groups.shape[1]
except:
self._gdim = 1
gr = pd.DataFrame(groups,
columns=[
'X' + str(i) for i in range(self._gdim)
]).astype('str')
self._gnames = gr.columns.values
gbin = pd.get_dummies(gr.astype('str'))
del gr
else:
raise ValueError(
"'groups' must be a pandas data frame, pandas series, or numpy array"
)
## processing X as specified
if self.standardize:
xmeans = x.mean(axis=0)
xsd = x.std(axis=0)
x = (x - xmeans) / xsd
if self.fit_intercept or self.standardize:
x = np.hstack([np.ones((x.shape[0], 1)), x])
if xnames is not None:
xnames = ['Intercept'] + list(xnames)
nobs = x.shape[0]
nvar = x.shape[1]
assert gbin.shape[0] == nobs
assert yval.shape[0] == nobs
## putting one-hot-encd' groups into a wide numpy array
self._gbin_names = list(gbin.columns.values)
ngroupvar = len(self._gbin_names)
nweight = 1 / gbin.sum(axis=0)
ntot = np.sum(nweight)
nweight = nweight / ntot
nweight = np.hstack([nweight for g in range(nvar)]) / nvar
gbin = csc_matrix(gbin.as_matrix())
gbin = hstack(
[gbin.multiply(x[:, g].reshape(-1, 1)) for g in range(nvar)])
gbin = csr_matrix(gbin)
## modeling the problem
if self.solver_interface == 'cvxpy':
w = cvx.Variable(nvar)
v = cvx.Variable(ngroupvar * nvar)
if self.problem == 'regression':
obj = cvx.norm(yval - x * w - gbin * v) / np.sqrt(nobs)
else:
obj = cvx.sum_entries(
cvx.logistic(cvx.mul_elemwise(-yval,
x * w + gbin * v))) / nobs
if self.reweight_deviations:
if self.main_l2_reg > 0:
obj += self.main_l2_reg * cvx.norm(w)
D = cvx.Variable(nvar, nvar)
for g in range(ngroupvar):
if self.weight_by_nobs:
obj += l2_reg * cvx.matrix_frac(
cvx.mul_elemwise(
nweight[ngroupvar], v[[
i
for i in range(g, gbin.shape[1], ngroupvar)
]]), D)
else:
obj += l2_reg * cvx.matrix_frac(
v[[i for i in range(g, gbin.shape[1], ngroupvar)]],
D)
prob = cvx.Problem(cvx.Minimize(obj), [cvx.trace(D) == 1])
else:
if self.main_l2_reg > 0:
obj += self.main_l2_reg * cvx.norm(w)
if self.l1_reg > 0:
if self.weight_by_nobs:
obj += self.l1_reg * cvx.norm(
cvx.mul_elemwise(nweight, v), 1)
else:
obj += self.l1_reg * cvx.norm(v, 1)
if self.l2_reg > 0:
if self.weight_by_nobs:
obj += self.l2_reg * cvx.norm(
cvx.mul_elemwise(nweight, v), 2)
else:
obj += self.l2_reg * cvx.norm(v, 2)
if self.linf_reg > 0:
obj += self.linf_reg * cvx.norm(v, 'inf')
prob = cvx.Problem(cvx.Minimize(obj))
prob.solve(**self.cvxpy_opts)
## saving results
self.coef_ = np.array(w.value)
self.coef2_ = np.array(v.value)
if self.solver_interface == 'casadi':
xvars = MX.sym('Vars', nvar * (1 + ngroupvar))
w = xvars[:nvar]
v = xvars[nvar:]
pred = mtimes(x, w) + mtimes(gbin, v)
if self.problem == 'regression':
err = yval - pred
obj = dot(err, err) / nobs
else:
obj = sum1(log(1 + np.exp(-yval * pred))) / nobs
if self.weight_by_nobs:
regw = nweight * v
else:
regw = v
obj += self.l2_reg * dot(regw, regw)
if self.main_l2_reg > 0:
obj += self.main_l2_reg * dot(w, w)
solver = nlpsol("solver", "ipopt", {
'x': xvars,
'f': obj
}, {
'print_time': False,
'ipopt': self.ipopt_options
})
x0 = np.zeros(shape=nvar * (1 + ngroupvar))
res = solver(x0=x0)
## saving results
self.coef_ = np.array(res['x'][:nvar])
self.coef2_ = np.array(res['x'][nvar:])
if self.standardize:
div = np.array([1] + list(xsd))
self.coef_ = self.coef_.reshape(-1) / div
self.coef2_ = self.coef2_.reshape(-1) / np.hstack(
[[div[i]] * ngroupvar for i in range(nvar)])
self.coef_[0] -= np.sum(self.coef_[1:] * xmeans)
xmeans = [0] + list(xmeans)
self.coef_[0] -= np.sum(self.coef2_ *
np.hstack([[xmeans[i]] * ngroupvar
for i in range(nvar)]))
self.group_effects_ = pd.DataFrame(self.coef2_.reshape(
nvar, ngroupvar),
columns=self._gbin_names)
if xnames is not None:
self.group_effects_.index = xnames
