def test_matrix_frac(self) -> None: """Test gradient for matrix_frac """ expr = cp.matrix_frac(self.A, self.B) self.A.value = np.eye(2) self.B.value = np.eye(2) self.assertItemsAlmostEqual(expr.grad[self.A].toarray(), [2, 0, 0, 2]) self.assertItemsAlmostEqual(expr.grad[self.B].toarray(), [-1, 0, 0, -1]) self.B.value = np.zeros((2, 2)) self.assertAlmostEqual(expr.grad[self.A], None) self.assertAlmostEqual(expr.grad[self.B], None) expr = cp.matrix_frac(self.x[:, None], self.A) self.x.value = [2, 3] self.A.value = np.eye(2) self.assertItemsAlmostEqual(expr.grad[self.x].toarray(), [4, 6]) self.assertItemsAlmostEqual(expr.grad[self.A].toarray(), [-4, -6, -6, -9]) expr = cp.matrix_frac(self.x, self.A) self.x.value = [2, 3] self.A.value = np.eye(2) self.assertItemsAlmostEqual(expr.grad[self.x].toarray(), [4, 6]) self.assertItemsAlmostEqual(expr.grad[self.A].toarray(), [-4, -6, -6, -9])
def mean_cov_prox_cvxpy(Y, eta, theta, t): if Y is None: return eta Y = Y[0] n,N = Y.shape ybar = cp.Parameter((n,1)) ybar.value = np.mean(Y,1).reshape(-1,1) Yemp = cp.Parameter((n,n)) Yemp.value = Y @ Y.T / N S = cp.Variable((n,n)) nu = cp.Variable((n,1)) eps = (1./(2*t)) main_part = -cp.log_det(S) main_part += cp.trace(S@Yemp) main_part += - 2*ybar.T@nu main_part += cp.matrix_frac(nu, S) prox_part = eps * cp.sum_squares(nu-eta[:, -1].reshape(-1,1)) prox_part += eps * cp.norm(S-eta[:, :-1], "fro")**2 prob = cp.Problem(cp.Minimize(N*main_part+prox_part)) prob.solve(verbose=False, warm_start=True) return np.hstack((S.value, nu.value))
def get_robust_hinf_policy(A, B, G, Q, R, alpha, gamma): n, m = B.shape wq = G.shape[1] S = cp.Variable((n, n), symmetric=True) Y = cp.Variable((m, n)) mu = cp.Variable() Q_sqrt = la.sqrtm(Q) R_sqrt = la.sqrtm(R) f = cp.trace(S @ Q) + cp.matrix_frac(Y.T @ R_sqrt, S) cons_mat = cp.bmat([[S @ A.T + A @ S + Y.T @ B.T + B @ Y + alpha * S + (mu / gamma ** 2) * G @ G.T, cp.bmat([[S @ Q_sqrt, Y.T @ R_sqrt]])], [cp.bmat([[Q_sqrt @ S], [R_sqrt @ Y]]), -mu * np.eye(m + n)]]) cons = [S >> np.eye(n), mu >= 0] + [cons_mat << -1e-3 * np.eye(n+m+n)] try: prob = cp.Problem(cp.Minimize(f), cons) prob.solve(solver=cp.SCS) #cp.MOSEK) except cp.error.SolverError as e: raise ValueError('Solver failed with error: %s \n Try another environment seed' % e) K = np.linalg.solve(S.value, Y.value.T).T assert np.all(np.linalg.eigvals(S.value) > 0) assert np.all(mu.value > 0) assert np.all(np.linalg.eigvals(cons_mat.value) <= 0) return K, S.value, mu.value
def rcwc(A, b, is_verbose=True, weights=None, solveroptions={}): """ A: m x n matrix with A_ij being an indicator for whether element j appears in sample set i b: m x n matrix with b_ij being the weights for element j in target set i corresponding to target set mean """ m, n = A.shape V = cp.Variable(shape=(n,n), PSD=True) constraints = [V[j,j] <= 1 for j in range(n)] objective = 0 for i in range(m): row_mask = np.nonzero(A[i,:])[0] rowcol_mask = np.ix_(row_mask, row_mask) objterm = cp.quad_form(b[i,:], V) - cp.matrix_frac(V[row_mask,:] @ b[i,:], V[rowcol_mask]) if weights is not None: objterm *= weights[i] objective += objterm if weights is None: objective /= m prob = cp.Problem(cp.Maximize(objective), constraints) prob.solve(verbose=is_verbose, solver=cp.MOSEK, **solveroptions) print(prob.value) Vhat = V.value a = np.zeros((m,n)) for i in range(m): row_mask = np.nonzero(A[i,:])[0] rowcol_mask = np.ix_(row_mask, row_mask) a[i,row_mask] = np.linalg.inv(Vhat[rowcol_mask]) @ Vhat[row_mask,:] @ b[i,:] a[A == 0] = 0 return a
def test_matrix_frac(self) -> None: """Test domain for matrix_frac. """ dom = cp.matrix_frac(self.x, self.A + np.eye(2)).domain prob = Problem(Minimize(cp.sum(cp.diag(self.A))), dom) prob.solve(solver=cp.SCS) self.assertAlmostEqual(prob.value, -2, places=3)
def test_matrix_frac(self): """Test for the matrix_frac atom. """ atom = cp.matrix_frac(self.x, self.A) self.assertEqual(atom.shape, tuple()) self.assertEqual(atom.curvature, s.CONVEX) # Test matrix_frac shape validation. with self.assertRaises(Exception) as cm: cp.matrix_frac(self.x, self.C) self.assertEqual(str(cm.exception), "The second argument to matrix_frac must be a square matrix.") with self.assertRaises(Exception) as cm: cp.matrix_frac(Variable(3), self.A) self.assertEqual(str(cm.exception), "The arguments to matrix_frac have incompatible dimensions.")
def test_key_error(self): """Test examples that caused key error. """ import cvxpy as cvx x = cvx.Variable() u = -cvx.exp(x) prob = cvx.Problem(cvx.Maximize(u), [x == 1]) prob.solve(verbose=True, solver=cvx.CVXOPT) prob.solve(verbose=True, solver=cvx.CVXOPT) ########################################### import numpy as np import cvxopt import cvxpy as cp kD=2 Sk=cp.semidefinite(kD) Rsk=cp.Parameter(kD,kD) mk=cp.Variable(kD,1) musk=cp.Parameter(kD,1) logpart=-0.5*cp.log_det(Sk)+0.5*cp.matrix_frac(mk,Sk)+(kD/2.)*np.log(2*np.pi) linpart=mk.T*musk-0.5*cp.trace(Sk*Rsk) obj=logpart-linpart prob=cp.Problem(cp.Minimize(obj)) musk.value=np.ones((2,1)) covsk=np.diag([0.3,0.5]) Rsk.value=covsk+(musk.value*musk.value.T) prob.solve(verbose=True,solver=cp.CVXOPT) print "second solve" prob.solve(verbose=False, solver=cp.CVXOPT)
def test_key_error(self): """Test examples that caused key error. """ if cvx.CVXOPT in cvx.installed_solvers(): x = cvx.Variable() u = -cvx.exp(x) prob = cvx.Problem(cvx.Maximize(u), [x == 1]) prob.solve(verbose=True, solver=cvx.CVXOPT) prob.solve(verbose=True, solver=cvx.CVXOPT) ########################################### import numpy as np kD = 2 Sk = cvx.Variable((kD, kD), PSD=True) Rsk = cvx.Parameter((kD, kD)) mk = cvx.Variable((kD, 1)) musk = cvx.Parameter((kD, 1)) logpart = -0.5 * cvx.log_det(Sk) + 0.5 * cvx.matrix_frac( mk, Sk) + (kD / 2.) * np.log(2 * np.pi) linpart = mk.T * musk - 0.5 * cvx.trace(Sk * Rsk) obj = logpart - linpart prob = cvx.Problem(cvx.Minimize(obj), [Sk == Sk.T]) musk.value = np.ones((2, 1)) covsk = np.diag([0.3, 0.5]) Rsk.value = covsk + (musk.value * musk.value.T) prob.solve(verbose=True, solver=cvx.CVXOPT) print("second solve") prob.solve(verbose=False, solver=cvx.CVXOPT)
def get_robust_lqr_sol(A, B, G, C, D, Q, R, alpha): n, m = B.shape wq = C.shape[0] S = cp.Variable((n, n), symmetric=True) Y = cp.Variable((m, n)) mu = cp.Variable() R_sqrt = la.sqrtm(R) f = cp.trace(S @ Q) + cp.matrix_frac(Y.T @ R_sqrt, S) cons_mat = cp.bmat(( (A @ S + S @ A.T + cp.multiply(mu, G @ G.T) + B @ Y + Y.T @ B.T + alpha * S, S @ C.T + Y.T @ D.T), (C @ S + D @ Y, -cp.multiply(mu, np.eye(wq))) )) cons = [S >> 0, mu >= 1e-2] + [cons_mat << 0] try: prob = cp.Problem(cp.Minimize(f), cons) prob.solve(solver=cp.SCS) except cp.error.SolverError as e: raise ValueError('Solver failed with error: %s \n Try another environment seed' % e) K = np.linalg.solve(S.value, Y.value.T).T return K, S.value
def test_matrix_frac(self) -> None: x = Variable(5) M = np.eye(5) P = M.T @ M s = cp.matrix_frac(x, P) self.assertFalse(s.is_constant()) self.assertFalse(s.is_affine()) self.assertTrue(s.is_quadratic()) self.assertTrue(s.is_dcp())
def main(): print("What is N ... ", end='') N = int(input()) print("\nWhat is a ... ", end='') a = int(input()) print("\nWhat is b ... ", end='') b = int(input()) print("\nWhat is p ... ", end='') p = int(input()) print('\n( N, a, b, p) = (', N, ',', a, ',', b, ',', p, ')') c1 = np.array([[1], [1], [1], [1]]) c2 = np.array([[1], [-1], [1], [-1]]) C = [c1, c2] #Generate A A = [] for i in range(1, N + 1): u_i = a + (b - a) * (i - 1) / (N - 1) tmp_a = np.array([[u_i**i for i in range(p + 1)]]) # 1 x (p+1) tmp_b = np.transpose(tmp_a) # (p+1) x 1 A_i = np.matmul(tmp_b, tmp_a) A.append(A_i) # Initializing Variables, Objective, and Constraints W = cp.Variable(N) obj = cp.Minimize( cp.sum([ cp.matrix_frac(c, sum([W[i] * A[i] for i in range(N)])) for c in C ])) const = [0 <= W, cp.sum(W) == 1] # Creating Problem & Solving start = timeit.default_timer() prob = cp.Problem(obj, const) prob.solve() stop = timeit.default_timer() print('Index\tValue') eps = 0.0001 for i in range(N): if W.value[i] > eps: print(f"{a + (b - a) * ((i+1)-1) / (N-1)}\t{W[i].value}") print("Sum of W = ", sum(W.value)) print("\nMethod used: ", prob.solver_stats.solver_name) print('Time to Compute: ', stop - start)
def test_matrix_frac(self): """Test matrix_frac atom. """ P = np.array([[10, 1j], [-1j, 10]]) Y = Variable((2, 2), complex=True) b = np.arange(2) x = Variable(2, complex=False) value = cvx.matrix_frac(b, P).value expr = cvx.matrix_frac(x, Y) prob = Problem(cvx.Minimize(expr), [x == b, Y == P]) result = prob.solve(solver=cvx.SCS, eps=1e-6, max_iters=7500, verbose=True) self.assertAlmostEqual(result, value, places=3) b = (np.arange(2) + 3j*(np.arange(2) + 10)) x = Variable(2, complex=True) value = cvx.matrix_frac(b, P).value expr = cvx.matrix_frac(x, Y) prob = Problem(cvx.Minimize(expr), [x == b, Y == P]) result = prob.solve(solver=cvx.SCS, eps=1e-6) self.assertAlmostEqual(result, value, places=3) b = (np.arange(2) + 10)/10j x = Variable(2, imag=True) value = cvx.matrix_frac(b, P).value expr = cvx.matrix_frac(x, Y) prob = Problem(cvx.Minimize(expr), [x == b, Y == P]) result = prob.solve(solver=cvx.SCS, eps=1e-5, max_iters=7500) self.assertAlmostEqual(result, value, places=3)
def get_robust_pldi_policy(A, B, Q, R, alpha): L, n, m = B.shape S = cp.Variable((n, n), symmetric=True) Y = cp.Variable((m, n)) R_sqrt = la.sqrtm(R) f = cp.trace(S @ Q) + cp.matrix_frac(Y.T @ R_sqrt, S) cons = [S >> np.eye(n)] + [A[i, :, :] @ S + B[i, :, :] @ Y + S @ A[i, :, :].T + Y.T @ B[i, :, :].T << -alpha * S for i in range(A.shape[0])] prob = cp.Problem(cp.Minimize(f), cons) prob.solve(solver=cp.MOSEK) K = np.linalg.solve(S.value, Y.value.T).T return K, S.value
def c_optimal(A, N, c): # Initializing Variables, Objective, and Constraints p = A[1].shape[0] - 1 W = cp.Variable(N) obj = cp.Minimize(cp.matrix_frac(c, sum([W[i] * A[i] for i in range(N)]))) const = [0 <= W, cp.sum(W) == 1] # Creating Problem & Solving prob = cp.Problem(obj, const) prob.solve() return prob
def _construct_objective(covariance_matrices, lambdas): ''' Constructs the CVX objective function. ''' num_points = len(covariance_matrices) num_dim = int(covariance_matrices[0].shape[0]) objective = 0 matrix_part = np.zeros([num_dim, num_dim]) for j in range(0, num_points): matrix_part = matrix_part + covariance_matrices[j] * lambdas[j] for i in range(0, num_dim): k_vec = np.zeros(num_dim) k_vec[i] = 1.0 objective = objective + cvx.matrix_frac(k_vec, matrix_part) return objective
def get_custom_lqr_sol(A, B, Q, R, alpha): n, m = B.shape S = cp.Variable((n, n), symmetric=True) Y = cp.Variable((m, n)) R_sqrt = la.sqrtm(R) f = cp.trace(S @ Q) + cp.matrix_frac(Y.T @ R_sqrt, S) # Exponential stability constraints from LMI book cons = [S >> np.eye(n)] # make LMI non-homogeneous cons += [A @ S + S @ A.T + B @ Y + Y.T @ B.T << -alpha * S] cp.Problem(cp.Minimize(f), cons).solve() K = np.linalg.solve(S.value, Y.value.T).T S = S.value return np.array(K), np.array(S)
def mode_solver(self): """ mode = arg min x^T*Sigma^-1*x """ if self.f is None: return self.mu m = len(self.mu) xi = cp.Variable(m) obj = cp.Minimize(cp.matrix_frac(xi, self.Sigma)) constraints = [self.f * (xi + self.mu) + self.g >= 0] prob = cp.Problem(obj, constraints) prob.solve() # print("status:", prob.status) if prob.status != "optimal": raise ValueError('cannot compute the mode') return xi.value
def mode(self, x, y): """ :param x: (n,) or (n,2), the design of experiment [x1,x2,...,xn] :param y: (n,), array_like, the true value at x :return : (in 2D, m = m1*m2) mu: (m,) the posterior mean with interpolation condition only, Gamma*Phi^T*[Phi*Gamma*Phi^T]^-1*y Sigma: (m,m), the covariance matrix of the posterior with interpolation condition only, Sigma = Gamma-Gamma*Phi^T*[Phi*Gamma*Phi^T]^-1*Phi*Gamma mode: (m,) the posterior mode which is given by the maximum of the PDF of the posterior """ l, Lambda, u = self.inequality_constraints() Phi = self.interpolation_constraints(x) Gamma = self.covariance() Gamma = Gamma + self.alpha * np.eye(len(Gamma)) mu = Gamma @ Phi.T @ np.linalg.solve(Phi @ Gamma @ Phi.T, y) Sigma = Gamma - Gamma @ Phi.T @ np.linalg.solve( Phi @ Gamma @ Phi.T, Phi) @ Gamma Sigma = Sigma + self.alpha * np.eye(len(Sigma)) if Lambda is None: # no inequality constraints, so mode is the posterior mean mode = mu else: xi = cp.Variable(np.prod(self.m)) obj = cp.Minimize(cp.matrix_frac(xi, Gamma)) constraints = [Phi @ xi == y] if not np.all(l == -np.inf): constraints.append( Lambda[l != -np.inf] * xi >= l[l != -np.inf]) if not np.all(u == np.inf): constraints.append(Lambda[u != np.inf] * xi <= u[u != np.inf]) prob = cp.Problem(obj, constraints) # print("Problem is DCP: ", prob.is_dcp()) prob.solve() # print("status:", prob.status) if prob.status != "optimal": raise ValueError('cannot compute the mode') mode = xi.value return mu, Sigma, mode
def get_objective(self, covariance_matrices, lambdas): num_points = len(covariance_matrices) num_dim = int(covariance_matrices[0].shape[0]) #print num_points #print num_dim #print int(covariance_matrices[0].shape[1]) objective = 0 matrix_part = np.zeros([num_dim, num_dim]) for j in xrange(0, num_points): matrix_part = matrix_part + covariance_matrices[j] * lambdas[j] # return np.matrix.trace(np.linalg.inv(matrix_part)) for i in xrange(0, num_dim): k_vec = np.zeros(num_dim) k_vec[i] = 1.0 objective = objective + cvx.matrix_frac(k_vec, matrix_part) return objective
def small_cone(x, verbose=False): """ Parameters ---------- x : numpy.array An N x d array, where d is the dimension and N is the number of points """ N, d = x.shape A = cvx.Variable((d, d), PSD=True) b = cvx.Variable(d) cst = [cvx.norm(A @ x[i]) <= b @ x[i] for i in range(N)] + [cvx.matrix_frac(b, A) <= 1] obj = cvx.Minimize(-cvx.log_det(A)) prob = cvx.Problem(obj, cst) prob.solve(verbose=verbose, eps=1e-8) A = A.value b = b.value return A, b, prob
print tau_2, b print b.shape, tau_2.shape #inv_alpha_1 = cvx.Variable(1) inv_alpha_1 = 0.5 # specify objective function #obj = cvx.Minimize(-cvx.sum_entries(cvx.log(tau_1)) - cvx.log_det(A - cvx.diag(tau_1))) #obj = cvx.Minimize(-cvx.sum_entries(cvx.log(tau_1)) - cvx.log_det(A - cvx.diag(tau_1))) # original #obj = cvx.Minimize( 0.5*N*(inv_alpha_1-1)*log_2_pi - 0.5*inv_alpha_1*(N*cvx.log(1/inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5*cvx.sum_entries(cvx.square(tau_2)/tau_1) + inv_alpha_1*cvx.sum_entries(log_normcdf(tau_2*cvx.sqrt(1/(inv_alpha_1*tau_1)))) +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) ) # modifications #obj = cvx.Minimize( 0.5*N*(inv_alpha_1-1)*log_2_pi - 0.5*inv_alpha_1*(-N*cvx.log(inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5*cvx.matrix_frac(tau_2, cvx.diag(tau_1)) + inv_alpha_1*cvx.sum_entries(log_normcdf(tau_2.T*cvx.inv_pos(cvx.sqrt(inv_alpha_1*tau_1)))) +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) ) obj = cvx.Minimize( 0.5 * N * (inv_alpha_1 - 1) * log_2_pi - 0.5 * inv_alpha_1 * (-N * cvx.log(inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5 * cvx.matrix_frac(tau_2, cvx.diag(tau_1)) + cvx.sum_entries( inv_alpha_1 * log_normcdf(cvx.inv_pos(cvx.sqrt(inv_alpha_1 * tau_1)))) ) # +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) ) #def upper_bound_logpartition(tau, inv_alpha_1): # tau_1, tau_2 = tau[:D+N], tau[D+N:] # tau_1_N, tau_2_N = tau_1[D:], tau_2[D:] # first D values correspond to w # alpha_1 = 1.0 / inv_alpha_1 # inv_alpha_2 = 1 - inv_alpha_1 # if np.any(tau_1 <= 0): # integral_1 = INF2 # else: # integral_1 = inv_alpha_1 * (-0.5 * ((D+N)*np.log(alpha_1) + np.sum(np.log(tau_1)) ) \ # + np.sum(norm.logcdf(np.sqrt(alpha_1)*tau_2_N/np.sqrt(tau_1_N)))) \ # + 0.5 * np.sum(np.power(tau_2, 2) / tau_1) # mat = A - np.diag(tau_1)
tau_1 = cvx.Variable(N) #tau_2 = cvx.Variable(N) tau_2 = np.zeros(N) print tau_2, b print b.shape, tau_2.shape #inv_alpha_1 = cvx.Variable(1) inv_alpha_1 = 0.5 # specify objective function #obj = cvx.Minimize(-cvx.sum_entries(cvx.log(tau_1)) - cvx.log_det(A - cvx.diag(tau_1))) #obj = cvx.Minimize(-cvx.sum_entries(cvx.log(tau_1)) - cvx.log_det(A - cvx.diag(tau_1))) # original #obj = cvx.Minimize( 0.5*N*(inv_alpha_1-1)*log_2_pi - 0.5*inv_alpha_1*(N*cvx.log(1/inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5*cvx.sum_entries(cvx.square(tau_2)/tau_1) + inv_alpha_1*cvx.sum_entries(log_normcdf(tau_2*cvx.sqrt(1/(inv_alpha_1*tau_1)))) +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) ) # modifications #obj = cvx.Minimize( 0.5*N*(inv_alpha_1-1)*log_2_pi - 0.5*inv_alpha_1*(-N*cvx.log(inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5*cvx.matrix_frac(tau_2, cvx.diag(tau_1)) + inv_alpha_1*cvx.sum_entries(log_normcdf(tau_2.T*cvx.inv_pos(cvx.sqrt(inv_alpha_1*tau_1)))) +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) ) obj = cvx.Minimize( 0.5*N*(inv_alpha_1-1)*log_2_pi - 0.5*inv_alpha_1*(-N*cvx.log(inv_alpha_1) + cvx.sum_entries(cvx.log(tau_1))) + 0.5*cvx.matrix_frac(tau_2, cvx.diag(tau_1)) + cvx.sum_entries(inv_alpha_1*log_normcdf(cvx.inv_pos(cvx.sqrt(inv_alpha_1*tau_1)))) )# +0.5*N*(1-inv_alpha_1)*cvx.log(1-inv_alpha_1) -0.5*(1-inv_alpha_1)*cvx.log_det(A-cvx.diag(tau_1)) + 0.5*cvx.matrix_frac(b-tau_2, A-cvx.diag(tau_1)) ) #def upper_bound_logpartition(tau, inv_alpha_1): # tau_1, tau_2 = tau[:D+N], tau[D+N:] # tau_1_N, tau_2_N = tau_1[D:], tau_2[D:] # first D values correspond to w # alpha_1 = 1.0 / inv_alpha_1 # inv_alpha_2 = 1 - inv_alpha_1 # if np.any(tau_1 <= 0): # integral_1 = INF2 # else: # integral_1 = inv_alpha_1 * (-0.5 * ((D+N)*np.log(alpha_1) + np.sum(np.log(tau_1)) ) \ # + np.sum(norm.logcdf(np.sqrt(alpha_1)*tau_2_N/np.sqrt(tau_1_N)))) \ # + 0.5 * np.sum(np.power(tau_2, 2) / tau_1) # mat = A - np.diag(tau_1) # sign, logdet = np.linalg.slogdet(mat)
def matrix_frac(b, A): return cvx.matrix_frac(b, A).value
def fit(self, X, y, groups): """ Fit a hierarchical linear regression to data Parameters ---------- X : pandas data frame or numpy array Table with predictors/covariates. Categorical variables must be properly encoded beforehand. y : pandas data frame or numpy array Values to predict. groups : pandas data frame or numpy array Groups to which each observation belongs. Can have more than one column if there are multiple group categories (e.g. 'province' and 'city'). Values can be strings or numbers, will be one-hot-encoded internally. Observations not belonging to any group must have NA values. """ ## checking input if type(X) == pd.core.frame.DataFrame: x = X.as_matrix() xnames = X.columns.values elif (type(X) == np.ndarray) or (type(X) == np.matrixlib.defmatrix.matrix): x = X.copy() xnames = None else: raise ValueError("'X' must be a pandas data frame or numpy array") if (type(y) == pd.core.frame.DataFrame) or (type(y) == pd.core.series.Series): yval = y.as_matrix() elif (type(y) == np.ndarray) or (type(y) == np.matrixlib.defmatrix.matrix): yval = y.copy() else: raise ValueError( "'y' must be a pandas data frame, pandas series, or numpy array" ) if self.problem == 'classification': classes = set(yval) if len(classes) != 2: raise ValueError( "Only binary classification is supported. 'y' contains " + str(len(classes)) + " values") if (1 in classes) and ((-1) in classes): self.class1 = 1 self.class2 = -1 elif (1 in classes) and (0 in classes): yval[yval == 0] = -1.0 self.class1 = 1 self.class2 = 0 else: self.class1, self.class2 = tuple(classes) yval[yval == class1] = 1.0 yval[yval == class2] = -1.0 if type(groups) == pd.core.series.Series: gbin = pd.get_dummies(np.array(groups.astype('str'))) self._gdim = 1 self._gnames = '' elif type(groups) == pd.core.frame.DataFrame: self._gdim = groups.shape[1] self._gnames = list(groups.columns.values) gbin = pd.get_dummies(groups.astype('str')) elif (type(groups) == np.ndarray) or (type(groups) == np.matrixlib.defmatrix.matrix): try: self._gdim = groups.shape[1] except: self._gdim = 1 gr = pd.DataFrame(groups, columns=[ 'X' + str(i) for i in range(self._gdim) ]).astype('str') self._gnames = gr.columns.values gbin = pd.get_dummies(gr.astype('str')) del gr else: raise ValueError( "'groups' must be a pandas data frame, pandas series, or numpy array" ) ## processing X as specified if self.standardize: xmeans = x.mean(axis=0) xsd = x.std(axis=0) x = (x - xmeans) / xsd if self.fit_intercept or self.standardize: x = np.hstack([np.ones((x.shape[0], 1)), x]) if xnames is not None: xnames = ['Intercept'] + list(xnames) nobs = x.shape[0] nvar = x.shape[1] assert gbin.shape[0] == nobs assert yval.shape[0] == nobs ## putting one-hot-encd' groups into a wide numpy array self._gbin_names = list(gbin.columns.values) ngroupvar = len(self._gbin_names) nweight = 1 / gbin.sum(axis=0) ntot = np.sum(nweight) nweight = nweight / ntot nweight = np.hstack([nweight for g in range(nvar)]) / nvar gbin = csc_matrix(gbin.as_matrix()) gbin = hstack( [gbin.multiply(x[:, g].reshape(-1, 1)) for g in range(nvar)]) gbin = csr_matrix(gbin) ## modeling the problem if self.solver_interface == 'cvxpy': w = cvx.Variable(nvar) v = cvx.Variable(ngroupvar * nvar) if self.problem == 'regression': obj = cvx.norm(yval - x * w - gbin * v) / np.sqrt(nobs) else: obj = cvx.sum_entries( cvx.logistic(cvx.mul_elemwise(-yval, x * w + gbin * v))) / nobs if self.reweight_deviations: if self.main_l2_reg > 0: obj += self.main_l2_reg * cvx.norm(w) D = cvx.Variable(nvar, nvar) for g in range(ngroupvar): if self.weight_by_nobs: obj += l2_reg * cvx.matrix_frac( cvx.mul_elemwise( nweight[ngroupvar], v[[ i for i in range(g, gbin.shape[1], ngroupvar) ]]), D) else: obj += l2_reg * cvx.matrix_frac( v[[i for i in range(g, gbin.shape[1], ngroupvar)]], D) prob = cvx.Problem(cvx.Minimize(obj), [cvx.trace(D) == 1]) else: if self.main_l2_reg > 0: obj += self.main_l2_reg * cvx.norm(w) if self.l1_reg > 0: if self.weight_by_nobs: obj += self.l1_reg * cvx.norm( cvx.mul_elemwise(nweight, v), 1) else: obj += self.l1_reg * cvx.norm(v, 1) if self.l2_reg > 0: if self.weight_by_nobs: obj += self.l2_reg * cvx.norm( cvx.mul_elemwise(nweight, v), 2) else: obj += self.l2_reg * cvx.norm(v, 2) if self.linf_reg > 0: obj += self.linf_reg * cvx.norm(v, 'inf') prob = cvx.Problem(cvx.Minimize(obj)) prob.solve(**self.cvxpy_opts) ## saving results self.coef_ = np.array(w.value) self.coef2_ = np.array(v.value) if self.solver_interface == 'casadi': xvars = MX.sym('Vars', nvar * (1 + ngroupvar)) w = xvars[:nvar] v = xvars[nvar:] pred = mtimes(x, w) + mtimes(gbin, v) if self.problem == 'regression': err = yval - pred obj = dot(err, err) / nobs else: obj = sum1(log(1 + np.exp(-yval * pred))) / nobs if self.weight_by_nobs: regw = nweight * v else: regw = v obj += self.l2_reg * dot(regw, regw) if self.main_l2_reg > 0: obj += self.main_l2_reg * dot(w, w) solver = nlpsol("solver", "ipopt", { 'x': xvars, 'f': obj }, { 'print_time': False, 'ipopt': self.ipopt_options }) x0 = np.zeros(shape=nvar * (1 + ngroupvar)) res = solver(x0=x0) ## saving results self.coef_ = np.array(res['x'][:nvar]) self.coef2_ = np.array(res['x'][nvar:]) if self.standardize: div = np.array([1] + list(xsd)) self.coef_ = self.coef_.reshape(-1) / div self.coef2_ = self.coef2_.reshape(-1) / np.hstack( [[div[i]] * ngroupvar for i in range(nvar)]) self.coef_[0] -= np.sum(self.coef_[1:] * xmeans) xmeans = [0] + list(xmeans) self.coef_[0] -= np.sum(self.coef2_ * np.hstack([[xmeans[i]] * ngroupvar for i in range(nvar)])) self.group_effects_ = pd.DataFrame(self.coef2_.reshape( nvar, ngroupvar), columns=self._gbin_names) if xnames is not None: self.group_effects_.index = xnames