def test_affine_sparse(): # test using sparse matrices for affine transforms n = 100 p = 25 X1 = scipy.sparse.csr_matrix(np.random.standard_normal((n,p))) b = scipy.sparse.csr_matrix(np.random.standard_normal(n)) v = np.random.standard_normal(p) y = np.random.standard_normal(n) transform1 = rr.affine_transform(X1, b) transform1.linear_map(v) transform1.adjoint_map(y) transform1.affine_map(v) # should raise a warning about type of sparse matrix X1 = scipy.sparse.coo_matrix(np.random.standard_normal((n,p))) b = scipy.sparse.coo_matrix(np.random.standard_normal(n)) v = np.random.standard_normal(p) y = np.random.standard_normal(n) transform2 = rr.affine_transform(X1, b) transform2.linear_map(v) transform2.adjoint_map(y) transform2.affine_map(v)
def test_affine_sparse(): # test using sparse matrices for affine transforms n = 100 p = 25 X1 = scipy.sparse.csr_matrix(np.random.standard_normal((n, p))) b = scipy.sparse.csr_matrix(np.random.standard_normal(n)) v = np.random.standard_normal(p) y = np.random.standard_normal(n) transform1 = rr.affine_transform(X1, b) transform1.linear_map(v) transform1.adjoint_map(y) transform1.affine_map(v) # should raise a warning about type of sparse matrix X1 = scipy.sparse.coo_matrix(np.random.standard_normal((n, p))) b = scipy.sparse.coo_matrix(np.random.standard_normal(n)) v = np.random.standard_normal(p) y = np.random.standard_normal(n) transform2 = rr.affine_transform(X1, b) transform2.linear_map(v) transform2.adjoint_map(y) transform2.affine_map(v)
def test_affine_sum(): n = 100 p = 25 X1 = np.random.standard_normal((n,p)) X2 = np.random.standard_normal((n,p)) b = np.random.standard_normal(n) v = np.random.standard_normal(p) transform1 = rr.affine_transform(X1,b) transform2 = rr.linear_transform(X2) sum_transform = rr.affine_sum([transform1, transform2]) yield assert_array_almost_equal, np.dot(X1,v) + np.dot(X2,v) + b, sum_transform.affine_map(v) yield assert_array_almost_equal, np.dot(X1,v) + np.dot(X2,v), sum_transform.linear_map(v) yield assert_array_almost_equal, np.dot(X1.T,b) + np.dot(X2.T,b), sum_transform.adjoint_map(b) yield assert_array_almost_equal, b, sum_transform.offset_map(v) yield assert_array_almost_equal, b, sum_transform.affine_offset sum_transform = rr.affine_sum([transform1, transform2], weights=[3,4]) yield assert_array_almost_equal, 3*(np.dot(X1,v) + b) + 4*(np.dot(X2,v)), sum_transform.affine_map(v) yield assert_array_almost_equal, 3*np.dot(X1,v) + 4*np.dot(X2,v), sum_transform.linear_map(v) yield assert_array_almost_equal, 3*np.dot(X1.T,b) + 4*np.dot(X2.T,b), sum_transform.adjoint_map(b) yield assert_array_almost_equal, 3*b, sum_transform.offset_map(v) yield assert_array_almost_equal, 3*b, sum_transform.affine_offset
def sel_prob_smooth_objective(self, param, mode='both', check_feasibility=False): param = self.apply_offset(param) data = np.squeeze(self.t * self.map.A) offset_active = self.map.offset_active + data[:self.map.nactive] offset_inactive = self.map.offset_inactive + data[self.map.nactive:] active_conj_loss = rr.affine_smooth(self.active_conjugate, rr.affine_transform(self.map.B_active, offset_active)) cube_loss = neg_log_cube_probability_fs(self.q, offset_inactive, randomization_scale = self.map.randomization_scale) total_loss = rr.smooth_sum([active_conj_loss, cube_loss, self.nonnegative_barrier]) if mode == 'func': f = total_loss.smooth_objective(param, 'func') return self.scale(f) elif mode == 'grad': g = total_loss.smooth_objective(param, 'grad') return self.scale(g) elif mode == 'both': f, g = total_loss.smooth_objective(param, 'both') return self.scale(f), self.scale(g) else: raise ValueError("mode incorrectly specified")
def test_affine_sum(): n = 100 p = 25 X1 = np.random.standard_normal((n, p)) X2 = np.random.standard_normal((n, p)) b = np.random.standard_normal(n) v = np.random.standard_normal(p) transform1 = rr.affine_transform(X1, b) transform2 = rr.linear_transform(X2) sum_transform = rr.affine_sum([transform1, transform2]) yield assert_array_almost_equal, np.dot(X1, v) + np.dot( X2, v) + b, sum_transform.affine_map(v) yield assert_array_almost_equal, np.dot(X1, v) + np.dot( X2, v), sum_transform.linear_map(v) yield assert_array_almost_equal, np.dot(X1.T, b) + np.dot( X2.T, b), sum_transform.adjoint_map(b) yield assert_array_almost_equal, b, sum_transform.affine_offset sum_transform = rr.affine_sum([transform1, transform2], weights=[3, 4]) yield assert_array_almost_equal, 3 * (np.dot(X1, v) + b) + 4 * (np.dot( X2, v)), sum_transform.affine_map(v) yield assert_array_almost_equal, 3 * np.dot(X1, v) + 4 * np.dot( X2, v), sum_transform.linear_map(v) yield assert_array_almost_equal, 3 * np.dot(X1.T, b) + 4 * np.dot( X2.T, b), sum_transform.adjoint_map(b) yield assert_array_almost_equal, 3 * b, sum_transform.affine_offset
def __init__(self, map, generative_mean, coef=1., offset=None, quadratic=None): self.map = map self.q = map.p - map.nactive self.r = map.p + map.nactive self.p = map.p rr.smooth_atom.__init__(self, (2 * self.p, ), offset=offset, quadratic=quadratic, initial=self.map.feasible_point, coef=coef) self.coefs[:] = self.map.feasible_point opt_vars_0 = np.zeros(self.r, bool) opt_vars_0[self.p:] = 1 opt_vars = np.append(opt_vars_0, np.ones(self.q, bool)) opt_vars_active = np.append(opt_vars_0, np.zeros(self.q, bool)) opt_vars_inactive = np.zeros(2 * self.p, bool) opt_vars_inactive[self.r:] = 1 self._response_selector = rr.selector(~opt_vars, (2 * self.p, )) self._opt_selector_active = rr.selector(opt_vars_active, (2 * self.p, )) self._opt_selector_inactive = rr.selector(opt_vars_inactive, (2 * self.p, )) nonnegative = nonnegative_softmax_scaled(self.map.nactive) self.nonnegative_barrier = nonnegative.linear( self._opt_selector_active) cube_objective = smooth_cube_barrier(self.map.inactive_lagrange) self.cube_barrier = rr.affine_smooth(cube_objective, self._opt_selector_inactive) linear_map = np.hstack( [self.map._score_linear_term, self.map._opt_linear_term]) randomization_loss = log_likelihood(np.zeros(self.p), self.map.randomization_cov, self.p) self.randomization_loss = rr.affine_smooth( randomization_loss, rr.affine_transform(linear_map, self.map._opt_affine_term)) likelihood_loss = log_likelihood(generative_mean, self.map.score_cov, self.p) self.likelihood_loss = rr.affine_smooth(likelihood_loss, self._response_selector) self.total_loss = rr.smooth_sum([ self.randomization_loss, self.likelihood_loss, self.nonnegative_barrier, self.cube_barrier ])
def test_stack_product(): X = np.random.standard_normal((5, 30)) Y = np.random.standard_normal((5, 30)) Z = np.random.standard_normal((5, 31)) U = np.random.standard_normal((6, 30)) stack = vstack([X, Y]) assert_raises(ValueError, vstack, [X, Z]) assert_raises(ValueError, hstack, [X, U]) np.testing.assert_allclose( stack.linear_map(np.arange(30))[:5], np.dot(X, np.arange(30))) np.testing.assert_allclose( stack.linear_map(np.arange(30))[5:], np.dot(Y, np.arange(30))) np.testing.assert_allclose( stack.affine_map(np.arange(30))[:5], np.dot(X, np.arange(30))) np.testing.assert_allclose( stack.affine_map(np.arange(30))[5:], np.dot(Y, np.arange(30))) np.testing.assert_allclose( stack.adjoint_map(np.arange(10)), np.dot(X.T, np.arange(5)) + np.dot(Y.T, np.arange(5, 10))) _hstack = hstack([X, Y, Z]) _hstack.linear_map(np.arange(91)) _hstack.affine_map(np.arange(91)) _hstack.adjoint_map(np.arange(5)) b = np.random.standard_normal(5) XA = rr.affine_transform(X, b) _product = product([XA, Y]) np.testing.assert_allclose( _product.linear_map(np.arange(60))[:5], np.dot(X, np.arange(30))) np.testing.assert_allclose( _product.linear_map(np.arange(60))[5:], np.dot(Y, np.arange(30, 60))) np.testing.assert_allclose( _product.affine_map(np.arange(60))[:5], np.dot(X, np.arange(30)) + b) np.testing.assert_allclose( _product.affine_map(np.arange(60))[5:], np.dot(Y, np.arange(30, 60))) np.testing.assert_allclose( _product.adjoint_map(np.arange(10))[:30], np.dot(X.T, np.arange(5))) np.testing.assert_allclose( _product.adjoint_map(np.arange(10))[30:], np.dot(Y.T, np.arange(5, 10))) scale_prod = scalar_multiply(_product, 2) np.testing.assert_allclose(scale_prod.linear_map(np.arange(60)), 2 * _product.linear_map(np.arange(60))) np.testing.assert_allclose(scale_prod.affine_map(np.arange(60)), 2 * _product.affine_map(np.arange(60))) np.testing.assert_allclose(scale_prod.adjoint_map(np.arange(60)), 2 * _product.adjoint_map(np.arange(60)))
def __init__(self, X, initial=None, lagrange=1, rho=1): self.X = R.affine_transform(X, None) self.atom = R.l1norm(X.shape[1], l) self.rho = rho self.loss = R.quadratic.affine(X, -np.zeros(X.shape[0]), lagrange=rho/2.) self.lasso = R.container(self.loss, self.atom) self.solver = R.FISTA(self.lasso.problem()) if initial is None: self.beta[:] = np.random.standard_normal(self.atom.primal_shape) else: self.beta[:] = initial
def __init__(self, X, initial=None, lagrange=1, rho=1): self.X = R.affine_transform(X, None) self.atom = R.l1norm(X.shape[1], l) self.rho = rho self.loss = R.quadratic.affine(X, -np.zeros(X.shape[0]), lagrange=rho / 2.) self.lasso = R.container(self.loss, self.atom) self.solver = R.FISTA(self.lasso.problem()) if initial is None: self.beta[:] = np.random.standard_normal(self.atom.primal_shape) else: self.beta[:] = initial
def test_coefs_matrix(): n, p, q = 20, 10, 5 X = np.random.standard_normal((n, p)) B = np.random.standard_normal((n, q)) V = np.random.standard_normal((p, q)) Y = np.random.standard_normal((n, q)) transform1 = rr.linear_transform(X, input_shape=(p, q)) assert_equal(transform1.linear_map(V).shape, (n, q)) assert_equal(transform1.affine_map(V).shape, (n, q)) assert_equal(transform1.adjoint_map(Y).shape, (p, q)) transform2 = rr.affine_transform(X, B, input_shape=(p, q)) assert_equal(transform2.linear_map(V).shape, (n, q)) assert_equal(transform2.affine_map(V).shape, (n, q)) assert_equal(transform2.adjoint_map(Y).shape, (p, q))
def test_coefs_matrix(): n, p, q = 20, 10, 5 X = np.random.standard_normal((n, p)) B = np.random.standard_normal((n, q)) V = np.random.standard_normal((p, q)) Y = np.random.standard_normal((n, q)) transform1 = rr.linear_transform(X, input_shape=(p,q)) assert_equal(transform1.linear_map(V).shape, (n,q)) assert_equal(transform1.affine_map(V).shape, (n,q)) assert_equal(transform1.adjoint_map(Y).shape, (p,q)) transform2 = rr.affine_transform(X, B, input_shape=(p,q)) assert_equal(transform2.linear_map(V).shape, (n,q)) assert_equal(transform2.affine_map(V).shape, (n,q)) assert_equal(transform2.adjoint_map(Y).shape, (p,q))
def test_row_matrix(): # make sure we can input a vector as a transform n, p = 20, 1 x = np.random.standard_normal(n) b = np.random.standard_normal(p) v = np.random.standard_normal(n) y = np.random.standard_normal(p) transform1 = rr.linear_transform(x) transform2 = rr.affine_transform(x, b) transform1.linear_map(v) transform1.affine_map(v) transform1.adjoint_map(y) transform2.linear_map(v) transform2.affine_map(v) transform2.adjoint_map(y)
def test_class(): n, p = (10, 5) D = np.random.standard_normal((n,p)) v = np.random.standard_normal(n) pen = rr.l1norm.affine(D, v, lagrange=0.4) pen2 = rr.l1norm(n, lagrange=0.4, offset=np.random.standard_normal(n)) pen2.quadratic = None cls = type(pen) pen_aff = cls(pen2, rr.affine_transform(D, v)) for _pen in [pen, pen_aff]: # Run to ensure code gets executed in tests (smoke test) print(_pen.dual) print(_pen.latexify()) print(str(_pen)) print(repr(_pen)) print(_pen._repr_latex_()) _pen.nonsmooth_objective(np.random.standard_normal(p)) q = rr.identity_quadratic(0.5,0,0,0) smoothed_pen = _pen.smoothed(q)
def test_stack_product(): X = np.random.standard_normal((5, 30)) Y = np.random.standard_normal((5, 30)) Z = np.random.standard_normal((5, 31)) U = np.random.standard_normal((6, 30)) stack = vstack([X, Y]) assert_raises(ValueError, vstack, [X, Z]) assert_raises(ValueError, hstack, [X, U]) np.testing.assert_allclose(stack.linear_map(np.arange(30))[:5], np.dot(X, np.arange(30))) np.testing.assert_allclose(stack.linear_map(np.arange(30))[5:], np.dot(Y, np.arange(30))) np.testing.assert_allclose(stack.affine_map(np.arange(30))[:5], np.dot(X, np.arange(30))) np.testing.assert_allclose(stack.affine_map(np.arange(30))[5:], np.dot(Y, np.arange(30))) np.testing.assert_allclose(stack.adjoint_map(np.arange(10)), np.dot(X.T, np.arange(5)) + np.dot(Y.T, np.arange(5, 10))) _hstack = hstack([X, Y, Z]) _hstack.linear_map(np.arange(91)) _hstack.affine_map(np.arange(91)) _hstack.adjoint_map(np.arange(5)) b = np.random.standard_normal(5) XA = rr.affine_transform(X, b) _product = product([XA,Y]) np.testing.assert_allclose(_product.linear_map(np.arange(60))[:5], np.dot(X, np.arange(30))) np.testing.assert_allclose(_product.linear_map(np.arange(60))[5:], np.dot(Y, np.arange(30, 60))) np.testing.assert_allclose(_product.affine_map(np.arange(60))[:5], np.dot(X, np.arange(30)) + b) np.testing.assert_allclose(_product.affine_map(np.arange(60))[5:], np.dot(Y, np.arange(30, 60))) np.testing.assert_allclose(_product.adjoint_map(np.arange(10))[:30], np.dot(X.T, np.arange(5))) np.testing.assert_allclose(_product.adjoint_map(np.arange(10))[30:], np.dot(Y.T, np.arange(5, 10))) scale_prod = scalar_multiply(_product, 2) np.testing.assert_allclose(scale_prod.linear_map(np.arange(60)), 2 * _product.linear_map(np.arange(60))) np.testing.assert_allclose(scale_prod.affine_map(np.arange(60)), 2 * _product.affine_map(np.arange(60))) np.testing.assert_allclose(scale_prod.adjoint_map(np.arange(60)), 2 * _product.adjoint_map(np.arange(60)))
def __init__(self, map, generative_mean, coef=1., offset=None, quadratic=None): self.map = map self.q = map.p - map.nactive self.r = map.p + map.nactive self.p = map.p self.inactive_conjugate = self.active_conjugate = map.randomization.CGF_conjugate if self.active_conjugate is None: raise ValueError( 'randomization must know its CGF_conjugate -- currently only isotropic_gaussian and laplace are implemented and are assumed to be randomization with IID coordinates') self.inactive_lagrange = self.map.inactive_lagrange rr.smooth_atom.__init__(self, (self.r,), offset=offset, quadratic=quadratic, initial=self.map.feasible_point, coef=coef) self.coefs[:] = self.map.feasible_point nonnegative = nonnegative_softmax_scaled(self.map.nactive) opt_vars = np.zeros(self.r, bool) opt_vars[map.p:] = 1 self._opt_selector = rr.selector(opt_vars, (self.r,)) self._response_selector = rr.selector(~opt_vars, (self.r,)) self.nonnegative_barrier = nonnegative.linear(self._opt_selector) self.active_conj_loss = rr.affine_smooth(self.active_conjugate, rr.affine_transform(np.hstack([self.map.A_active, self.map.B_active]), self.map.offset_active)) cube_obj = neg_log_cube_probability(self.q, self.inactive_lagrange, randomization_scale=1.) self.cube_loss = rr.affine_smooth(cube_obj, np.hstack([self.map.A_inactive, self.map.B_inactive])) # w_1, v_1 = np.linalg.eig(self.map.score_cov) # self.score_cov_inv_half = (v_1.T.dot(np.diag(np.power(w_1, -0.5)))).dot(v_1) # likelihood_loss = rr.signal_approximator(np.squeeze(np.zeros(self.p)), coef=1.) # scaled_response_selector = rr.selector(~opt_vars, (self.r,), rr.affine_transform(self.score_cov_inv_half, # self.score_cov_inv_half. # dot(np.squeeze(generative_mean)))) #print("cov", self.map.score_cov.shape ) likelihood_loss = log_likelihood(generative_mean, self.map.score_cov, self.p) self.likelihood_loss = rr.affine_smooth(likelihood_loss, self._response_selector) self.total_loss = rr.smooth_sum([self.active_conj_loss, self.likelihood_loss, self.nonnegative_barrier, self.cube_loss])
def __init__( self, X, feasible_point, active, # the active set chosen by randomized lasso active_sign, # the set of signs of active coordinates chosen by lasso lagrange, # in R^p mean_parameter, # in R^n noise_variance, #noise_level in data randomizer, #specified randomization epsilon, # ridge penalty for randomized lasso coef=1., offset=None, quadratic=None, nstep=10): n, p = X.shape self._X = X E = active.sum() self.q = p - E self.active = active self.noise_variance = noise_variance self.randomization = randomizer self.inactive_conjugate = self.active_conjugate = randomizer.CGF_conjugate if self.active_conjugate is None: raise ValueError( 'randomization must know its CGF_conjugate -- currently only isotropic_gaussian and laplace are implemented and are assumed to be randomization with IID coordinates' ) initial = np.zeros(n + E, ) initial[n:] = feasible_point self.n = n rr.smooth_atom.__init__(self, (n + E, ), offset=offset, quadratic=quadratic, initial=initial, coef=coef) self.coefs[:] = initial opt_vars = np.zeros(n + E, bool) opt_vars[n:] = 1 nonnegative = nonnegative_softmax_scaled(E) self._opt_selector = rr.selector(opt_vars, (n + E, )) self.nonnegative_barrier = nonnegative.linear(self._opt_selector) self._response_selector = rr.selector(~opt_vars, (n + E, )) self.set_parameter(mean_parameter, noise_variance) X_E = X[:, active] B = X.T.dot(X_E) B_E = B[active] B_mE = B[~active] self.A_active = np.hstack([ -X[:, active].T, (B_E + epsilon * np.identity(E)) * active_sign[None, :] ]) self.A_inactive = np.hstack( [-X[:, ~active].T, (B_mE * active_sign[None, :])]) self.offset_active = active_sign * lagrange[active] self.offset_inactive = np.zeros(p - E) self.active_conj_loss = rr.affine_smooth( self.active_conjugate, rr.affine_transform(self.A_active, self.offset_active)) cube_obj = neg_log_cube_probability(self.q, lagrange[~active], randomization_scale=1.) self.cube_loss = rr.affine_smooth(cube_obj, self.A_inactive) self.total_loss = rr.smooth_sum([ self.active_conj_loss, self.cube_loss, self.likelihood_loss, self.nonnegative_barrier ])
def __init__(self, X, feasible_point, active, # the active set chosen by randomized marginal screening active_signs, # the set of signs of active coordinates chosen by ms threshold, # in R^p mean_parameter, noise_variance, randomizer, coef=1., offset=None, quadratic=None, nstep=10): n, p = X.shape self._X = X E = active.sum() self.q = p - E sigma = np.sqrt(noise_variance) self.active = active self.noise_variance = noise_variance self.randomization = randomizer self.inactive_conjugate = self.active_conjugate = randomizer.CGF_conjugate if self.active_conjugate is None: raise ValueError( 'randomization must know its CGF_conjugate -- currently only isotropic_gaussian and laplace are implemented and are assumed to be randomization with IID coordinates') initial = np.zeros(n + E, ) initial[n:] = feasible_point self.n = n rr.smooth_atom.__init__(self, (n + E,), offset=offset, quadratic=quadratic, initial=initial, coef=coef) self.coefs[:] = initial nonnegative = nonnegative_softmax_scaled(E) opt_vars = np.zeros(n + E, bool) opt_vars[n:] = 1 self._opt_selector = rr.selector(opt_vars, (n + E,)) self.nonnegative_barrier = nonnegative.linear(self._opt_selector) self._response_selector = rr.selector(~opt_vars, (n + E,)) self.set_parameter(mean_parameter, noise_variance) self.A_active = np.hstack([np.true_divide(-X[:, active].T, sigma), np.identity(E) * active_signs[None, :]]) self.A_inactive = np.hstack([np.true_divide(-X[:, ~active].T, sigma), np.zeros((p - E, E))]) self.offset_active = active_signs * threshold[active] self.offset_inactive = np.zeros(p - E) self.active_conj_loss = rr.affine_smooth(self.active_conjugate, rr.affine_transform(self.A_active, self.offset_active)) cube_obj = neg_log_cube_probability(self.q, threshold[~active], randomization_scale=1.) self.cube_loss = rr.affine_smooth(cube_obj, rr.affine_transform(self.A_inactive, self.offset_inactive)) self.total_loss = rr.smooth_sum([self.active_conj_loss, self.cube_loss, self.likelihood_loss, self.nonnegative_barrier])
import numpy as np import regreg.api as rr np.random.seed(400) N = 1000 P = 200 Y = 2 * np.random.binomial(1, 0.5, size=(N,)) - 1. X = np.random.standard_normal((N,P)) X[Y==1] += np.array([30,-20] + (P-2)*[0])[np.newaxis,:] X -= X.mean(0)[np.newaxis, :] X_1 = np.hstack([X, np.ones((N,1))]) transform = rr.affine_transform(-Y[:,np.newaxis] * X_1, np.ones(N)) C = 0.2 hinge = rr.positive_part(N, lagrange=C) hinge_loss = rr.linear_atom(hinge, transform) epsilon = 0.04 smoothed_hinge_loss = rr.smoothed_atom(hinge_loss, epsilon=epsilon) s = rr.selector(slice(0,P), (P+1,)) sparsity = rr.l1norm.linear(s, lagrange=3.) quadratic = rr.quadratic.linear(s, coef=0.5) from regreg.affine import power_L ltransform = rr.linear_transform(X_1) singular_value_sq = power_L(X_1) # the other smooth piece is a quadratic with identity # for quadratic form, so its lipschitz constant is 1
import numpy as np import regreg.api as rr np.random.seed(400) N = 500 P = 2 Y = 2 * np.random.binomial(1, 0.5, size=(N,)) - 1. X = np.random.standard_normal((N,P)) X[Y==1] += np.array([3,-2])[np.newaxis,:] X_1 = np.hstack([X, np.ones((N,1))]) X_1_signs = -Y[:,np.newaxis] * X_1 transform = rr.affine_transform(X_1_signs, np.ones(N)) C = 0.2 hinge = rr.positive_part(N, lagrange=C) hinge_loss = rr.linear_atom(hinge, transform) quadratic = rr.quadratic.linear(rr.selector(slice(0,P), (P+1,)), coef=0.5) problem = rr.container(quadratic, hinge_loss) solver = rr.FISTA(problem) solver.fit() import pylab pylab.clf() pylab.scatter(X[Y==1,0],X[Y==1,1], facecolor='red') pylab.scatter(X[Y==-1,0],X[Y==-1,1], facecolor='blue') fits = np.dot(X_1, problem.coefs) labels = 2 * (fits > 0) - 1
import numpy as np import regreg.api as rr np.random.seed(400) N = 1000 P = 200 Y = 2 * np.random.binomial(1, 0.5, size=(N, )) - 1. X = np.random.standard_normal((N, P)) X[Y == 1] += np.array([30, -20] + (P - 2) * [0])[np.newaxis, :] X -= X.mean(0)[np.newaxis, :] X_1 = np.hstack([X, np.ones((N, 1))]) transform = rr.affine_transform(-Y[:, np.newaxis] * X_1, np.ones(N)) C = 0.2 hinge = rr.positive_part(N, lagrange=C) hinge_loss = rr.linear_atom(hinge, transform) epsilon = 0.04 smoothed_hinge_loss = rr.smoothed_atom(hinge_loss, epsilon=epsilon) s = rr.selector(slice(0, P), (P + 1, )) sparsity = rr.l1norm.linear(s, lagrange=3.) quadratic = rr.quadratic.linear(s, coef=0.5) from regreg.affine import power_L ltransform = rr.linear_transform(X_1) singular_value_sq = power_L(X_1) # the other smooth piece is a quadratic with identity # for quadratic form, so its lipschitz constant is 1
""" Solving basis pursuit with TFOCS """ import regreg.api as rr import numpy as np import nose.tools as nt n, p = 100, 200 X = np.random.standard_normal((n, p)) beta = np.zeros(p) beta[:4] = 3 Y = np.random.standard_normal(n) + np.dot(X, beta) lscoef = np.dot(np.linalg.pinv(X), Y) minimum_l2 = np.linalg.norm(Y - np.dot(X, lscoef)) maximum_l2 = np.linalg.norm(Y) l2bound = (minimum_l2 + maximum_l2) * 0.5 l2 = rr.l2norm(n, bound=l2bound) T = rr.affine_transform(X, -Y) l1 = rr.l1norm(p, lagrange=1) primal, dual = rr.tfocs(l1, T, l2, tol=1.e-10) nt.assert_true( np.fabs(np.linalg.norm(Y - np.dot(X, primal)) - l2bound) <= l2bound * 1.e-5)
def __init__( self, X, feasible_point, #in R^{|E|_1 + |E|_2} active_1, #the active set chosen by randomized marginal screening active_2, #the active set chosen by randomized lasso active_signs_1, #the set of signs of active coordinates chosen by ms active_signs_2, #the set of signs of active coordinates chosen by lasso lagrange, #in R^p threshold, #in R^p mean_parameter, # in R^n noise_variance, randomizer, epsilon, #ridge penalty for randomized lasso coef=1., offset=None, quadratic=None, nstep=10): n, p = X.shape self._X = X E_1 = active_1.sum() E_2 = active_2.sum() sigma = np.sqrt(noise_variance) self.active_1 = active_1 self.active_2 = active_2 self.noise_variance = noise_variance self.randomization = randomizer self.inactive_conjugate = self.active_conjugate = randomizer.CGF_conjugate if self.active_conjugate is None: raise ValueError( 'randomization must know its CGF_conjugate -- currently only isotropic_gaussian and laplace are implemented and are assumed to be randomization with IID coordinates' ) initial = np.zeros(n + E_1 + E_2, ) initial[n:] = feasible_point self.n = n rr.smooth_atom.__init__(self, (n + E_1 + E_2, ), offset=offset, quadratic=quadratic, initial=initial, coef=coef) self.coefs[:] = initial nonnegative = nonnegative_softmax_scaled(E_1 + E_2) opt_vars = np.zeros(n + E_1 + E_2, bool) opt_vars[n:] = 1 self._opt_selector = rr.selector(opt_vars, (n + E_1 + E_2, )) self.nonnegative_barrier = nonnegative.linear(self._opt_selector) self._response_selector = rr.selector(~opt_vars, (n + E_1 + E_2, )) self.set_parameter(mean_parameter, noise_variance) arg_ms = np.zeros(self.n + E_1 + E_2, bool) arg_ms[:self.n + E_1] = 1 arg_lasso = np.zeros(self.n + E_1, bool) arg_lasso[:self.n] = 1 arg_lasso = np.append(arg_lasso, np.ones(E_2, bool)) self.A_active_1 = np.hstack([ np.true_divide(-X[:, active_1].T, sigma), np.identity(E_1) * active_signs_1[None, :] ]) self.A_inactive_1 = np.hstack([ np.true_divide(-X[:, ~active_1].T, sigma), np.zeros((p - E_1, E_1)) ]) self.offset_active_1 = active_signs_1 * threshold[active_1] self.offset_inactive_1 = np.zeros(p - E_1) self._active_ms = rr.selector( arg_ms, (self.n + E_1 + E_2, ), rr.affine_transform(self.A_active_1, self.offset_active_1)) self._inactive_ms = rr.selector( arg_ms, (self.n + E_1 + E_2, ), rr.affine_transform(self.A_inactive_1, self.offset_inactive_1)) self.active_conj_loss_1 = rr.affine_smooth(self.active_conjugate, self._active_ms) self.q_1 = p - E_1 cube_obj_1 = neg_log_cube_probability(self.q_1, threshold[~active_1], randomization_scale=1.) self.cube_loss_1 = rr.affine_smooth(cube_obj_1, self._inactive_ms) X_step2 = X[:, active_1] X_E_2 = X_step2[:, active_2] B = X_step2.T.dot(X_E_2) B_E = B[active_2] B_mE = B[~active_2] self.A_active_2 = np.hstack([ -X_step2[:, active_2].T, (B_E + epsilon * np.identity(E_2)) * active_signs_2[None, :] ]) self.A_inactive_2 = np.hstack( [-X_step2[:, ~active_2].T, (B_mE * active_signs_2[None, :])]) self.offset_active_2 = active_signs_2 * lagrange[active_2] self.offset_inactive_2 = np.zeros(E_1 - E_2) self._active_lasso = rr.selector( arg_lasso, (self.n + E_1 + E_2, ), rr.affine_transform(self.A_active_2, self.offset_active_2)) self._inactive_lasso = rr.selector( arg_lasso, (self.n + E_1 + E_2, ), rr.affine_transform(self.A_inactive_2, self.offset_inactive_2)) self.active_conj_loss_2 = rr.affine_smooth(self.active_conjugate, self._active_lasso) self.q_2 = E_1 - E_2 cube_obj_2 = neg_log_cube_probability(self.q_2, lagrange[~active_2], randomization_scale=1.) self.cube_loss_2 = rr.affine_smooth(cube_obj_2, self._inactive_lasso) self.total_loss = rr.smooth_sum([ self.active_conj_loss_1, self.active_conj_loss_2, self.cube_loss_1, self.cube_loss_2, self.likelihood_loss, self.nonnegative_barrier ])
""" Solving basis pursuit with TFOCS """ import regreg.api as rr import numpy as np import nose.tools as nt n, p = 100, 200 X = np.random.standard_normal((n, p)) beta = np.zeros(p) beta[:4] = 3 Y = np.random.standard_normal(n) + np.dot(X, beta) lscoef = np.dot(np.linalg.pinv(X), Y) minimum_l2 = np.linalg.norm(Y - np.dot(X, lscoef)) maximum_l2 = np.linalg.norm(Y) l2bound = (minimum_l2 + maximum_l2) * 0.5 l2 = rr.l2norm(n, bound=l2bound) T = rr.affine_transform(X, -Y) l1 = rr.l1norm(p, lagrange=1) primal, dual = rr.tfocs(l1, T, l2, tol=1.0e-10) nt.assert_true(np.fabs(np.linalg.norm(Y - np.dot(X, primal)) - l2bound) <= l2bound * 1.0e-5)