def test_affine_sparse():
# test using sparse matrices for affine transforms
n = 100
p = 25
X1 = scipy.sparse.csr_matrix(np.random.standard_normal((n,p)))
b = scipy.sparse.csr_matrix(np.random.standard_normal(n))
v = np.random.standard_normal(p)
y = np.random.standard_normal(n)
transform1 = rr.affine_transform(X1, b)
transform1.linear_map(v)
transform1.adjoint_map(y)
transform1.affine_map(v)
# should raise a warning about type of sparse matrix
X1 = scipy.sparse.coo_matrix(np.random.standard_normal((n,p)))
b = scipy.sparse.coo_matrix(np.random.standard_normal(n))
v = np.random.standard_normal(p)
y = np.random.standard_normal(n)
transform2 = rr.affine_transform(X1, b)
transform2.linear_map(v)
transform2.adjoint_map(y)
transform2.affine_map(v)
def test_affine_sparse():
# test using sparse matrices for affine transforms
n = 100
p = 25
X1 = scipy.sparse.csr_matrix(np.random.standard_normal((n, p)))
b = scipy.sparse.csr_matrix(np.random.standard_normal(n))
v = np.random.standard_normal(p)
y = np.random.standard_normal(n)
transform1 = rr.affine_transform(X1, b)
transform1.linear_map(v)
transform1.adjoint_map(y)
transform1.affine_map(v)
# should raise a warning about type of sparse matrix
X1 = scipy.sparse.coo_matrix(np.random.standard_normal((n, p)))
b = scipy.sparse.coo_matrix(np.random.standard_normal(n))
v = np.random.standard_normal(p)
y = np.random.standard_normal(n)
transform2 = rr.affine_transform(X1, b)
transform2.linear_map(v)
transform2.adjoint_map(y)
transform2.affine_map(v)
def test_affine_sum():
n = 100
p = 25
X1 = np.random.standard_normal((n,p))
X2 = np.random.standard_normal((n,p))
b = np.random.standard_normal(n)
v = np.random.standard_normal(p)
transform1 = rr.affine_transform(X1,b)
transform2 = rr.linear_transform(X2)
sum_transform = rr.affine_sum([transform1, transform2])
yield assert_array_almost_equal, np.dot(X1,v) + np.dot(X2,v) + b, sum_transform.affine_map(v)
yield assert_array_almost_equal, np.dot(X1,v) + np.dot(X2,v), sum_transform.linear_map(v)
yield assert_array_almost_equal, np.dot(X1.T,b) + np.dot(X2.T,b), sum_transform.adjoint_map(b)
yield assert_array_almost_equal, b, sum_transform.offset_map(v)
yield assert_array_almost_equal, b, sum_transform.affine_offset
sum_transform = rr.affine_sum([transform1, transform2], weights=[3,4])
yield assert_array_almost_equal, 3*(np.dot(X1,v) + b) + 4*(np.dot(X2,v)), sum_transform.affine_map(v)
yield assert_array_almost_equal, 3*np.dot(X1,v) + 4*np.dot(X2,v), sum_transform.linear_map(v)
yield assert_array_almost_equal, 3*np.dot(X1.T,b) + 4*np.dot(X2.T,b), sum_transform.adjoint_map(b)
yield assert_array_almost_equal, 3*b, sum_transform.offset_map(v)
yield assert_array_almost_equal, 3*b, sum_transform.affine_offset
def sel_prob_smooth_objective(self, param, mode='both', check_feasibility=False):
param = self.apply_offset(param)
data = np.squeeze(self.t * self.map.A)
offset_active = self.map.offset_active + data[:self.map.nactive]
offset_inactive = self.map.offset_inactive + data[self.map.nactive:]
active_conj_loss = rr.affine_smooth(self.active_conjugate,
rr.affine_transform(self.map.B_active, offset_active))
cube_loss = neg_log_cube_probability_fs(self.q, offset_inactive, randomization_scale = self.map.randomization_scale)
total_loss = rr.smooth_sum([active_conj_loss,
cube_loss,
self.nonnegative_barrier])
if mode == 'func':
f = total_loss.smooth_objective(param, 'func')
return self.scale(f)
elif mode == 'grad':
g = total_loss.smooth_objective(param, 'grad')
return self.scale(g)
elif mode == 'both':
f, g = total_loss.smooth_objective(param, 'both')
return self.scale(f), self.scale(g)
else:
raise ValueError("mode incorrectly specified")
def test_affine_sum():
n = 100
p = 25
X1 = np.random.standard_normal((n, p))
X2 = np.random.standard_normal((n, p))
b = np.random.standard_normal(n)
v = np.random.standard_normal(p)
transform1 = rr.affine_transform(X1, b)
transform2 = rr.linear_transform(X2)
sum_transform = rr.affine_sum([transform1, transform2])
yield assert_array_almost_equal, np.dot(X1, v) + np.dot(
X2, v) + b, sum_transform.affine_map(v)
yield assert_array_almost_equal, np.dot(X1, v) + np.dot(
X2, v), sum_transform.linear_map(v)
yield assert_array_almost_equal, np.dot(X1.T, b) + np.dot(
X2.T, b), sum_transform.adjoint_map(b)
yield assert_array_almost_equal, b, sum_transform.affine_offset
sum_transform = rr.affine_sum([transform1, transform2], weights=[3, 4])
yield assert_array_almost_equal, 3 * (np.dot(X1, v) + b) + 4 * (np.dot(
X2, v)), sum_transform.affine_map(v)
yield assert_array_almost_equal, 3 * np.dot(X1, v) + 4 * np.dot(
X2, v), sum_transform.linear_map(v)
yield assert_array_almost_equal, 3 * np.dot(X1.T, b) + 4 * np.dot(
X2.T, b), sum_transform.adjoint_map(b)
yield assert_array_almost_equal, 3 * b, sum_transform.affine_offset
def __init__(self,
map,
generative_mean,
coef=1.,
offset=None,
quadratic=None):
self.map = map
self.q = map.p - map.nactive
self.r = map.p + map.nactive
self.p = map.p
rr.smooth_atom.__init__(self, (2 * self.p, ),
offset=offset,
quadratic=quadratic,
initial=self.map.feasible_point,
coef=coef)
self.coefs[:] = self.map.feasible_point
opt_vars_0 = np.zeros(self.r, bool)
opt_vars_0[self.p:] = 1
opt_vars = np.append(opt_vars_0, np.ones(self.q, bool))
opt_vars_active = np.append(opt_vars_0, np.zeros(self.q, bool))
opt_vars_inactive = np.zeros(2 * self.p, bool)
opt_vars_inactive[self.r:] = 1
self._response_selector = rr.selector(~opt_vars, (2 * self.p, ))
self._opt_selector_active = rr.selector(opt_vars_active,
(2 * self.p, ))
self._opt_selector_inactive = rr.selector(opt_vars_inactive,
(2 * self.p, ))
nonnegative = nonnegative_softmax_scaled(self.map.nactive)
self.nonnegative_barrier = nonnegative.linear(
self._opt_selector_active)
cube_objective = smooth_cube_barrier(self.map.inactive_lagrange)
self.cube_barrier = rr.affine_smooth(cube_objective,
self._opt_selector_inactive)
linear_map = np.hstack(
[self.map._score_linear_term, self.map._opt_linear_term])
randomization_loss = log_likelihood(np.zeros(self.p),
self.map.randomization_cov, self.p)
self.randomization_loss = rr.affine_smooth(
randomization_loss,
rr.affine_transform(linear_map, self.map._opt_affine_term))
likelihood_loss = log_likelihood(generative_mean, self.map.score_cov,
self.p)
self.likelihood_loss = rr.affine_smooth(likelihood_loss,
self._response_selector)
self.total_loss = rr.smooth_sum([
self.randomization_loss, self.likelihood_loss,
self.nonnegative_barrier, self.cube_barrier
])
def test_stack_product():
X = np.random.standard_normal((5, 30))
Y = np.random.standard_normal((5, 30))
Z = np.random.standard_normal((5, 31))
U = np.random.standard_normal((6, 30))
stack = vstack([X, Y])
assert_raises(ValueError, vstack, [X, Z])
assert_raises(ValueError, hstack, [X, U])
np.testing.assert_allclose(
stack.linear_map(np.arange(30))[:5], np.dot(X, np.arange(30)))
np.testing.assert_allclose(
stack.linear_map(np.arange(30))[5:], np.dot(Y, np.arange(30)))
np.testing.assert_allclose(
stack.affine_map(np.arange(30))[:5], np.dot(X, np.arange(30)))
np.testing.assert_allclose(
stack.affine_map(np.arange(30))[5:], np.dot(Y, np.arange(30)))
np.testing.assert_allclose(
stack.adjoint_map(np.arange(10)),
np.dot(X.T, np.arange(5)) + np.dot(Y.T, np.arange(5, 10)))
_hstack = hstack([X, Y, Z])
_hstack.linear_map(np.arange(91))
_hstack.affine_map(np.arange(91))
_hstack.adjoint_map(np.arange(5))
b = np.random.standard_normal(5)
XA = rr.affine_transform(X, b)
_product = product([XA, Y])
np.testing.assert_allclose(
_product.linear_map(np.arange(60))[:5], np.dot(X, np.arange(30)))
np.testing.assert_allclose(
_product.linear_map(np.arange(60))[5:], np.dot(Y, np.arange(30, 60)))
np.testing.assert_allclose(
_product.affine_map(np.arange(60))[:5],
np.dot(X, np.arange(30)) + b)
np.testing.assert_allclose(
_product.affine_map(np.arange(60))[5:], np.dot(Y, np.arange(30, 60)))
np.testing.assert_allclose(
_product.adjoint_map(np.arange(10))[:30], np.dot(X.T, np.arange(5)))
np.testing.assert_allclose(
_product.adjoint_map(np.arange(10))[30:],
np.dot(Y.T, np.arange(5, 10)))
scale_prod = scalar_multiply(_product, 2)
np.testing.assert_allclose(scale_prod.linear_map(np.arange(60)),
2 * _product.linear_map(np.arange(60)))
np.testing.assert_allclose(scale_prod.affine_map(np.arange(60)),
2 * _product.affine_map(np.arange(60)))
np.testing.assert_allclose(scale_prod.adjoint_map(np.arange(60)),
2 * _product.adjoint_map(np.arange(60)))
def __init__(self, X, initial=None, lagrange=1, rho=1):
self.X = R.affine_transform(X, None)
self.atom = R.l1norm(X.shape[1], l)
self.rho = rho
self.loss = R.quadratic.affine(X, -np.zeros(X.shape[0]), lagrange=rho/2.)
self.lasso = R.container(self.loss, self.atom)
self.solver = R.FISTA(self.lasso.problem())
if initial is None:
self.beta[:] = np.random.standard_normal(self.atom.primal_shape)
else:
self.beta[:] = initial
def __init__(self, X, initial=None, lagrange=1, rho=1):
self.X = R.affine_transform(X, None)
self.atom = R.l1norm(X.shape[1], l)
self.rho = rho
self.loss = R.quadratic.affine(X,
-np.zeros(X.shape[0]),
lagrange=rho / 2.)
self.lasso = R.container(self.loss, self.atom)
self.solver = R.FISTA(self.lasso.problem())
if initial is None:
self.beta[:] = np.random.standard_normal(self.atom.primal_shape)
else:
self.beta[:] = initial
def test_coefs_matrix():
n, p, q = 20, 10, 5
X = np.random.standard_normal((n, p))
B = np.random.standard_normal((n, q))
V = np.random.standard_normal((p, q))
Y = np.random.standard_normal((n, q))
transform1 = rr.linear_transform(X, input_shape=(p, q))
assert_equal(transform1.linear_map(V).shape, (n, q))
assert_equal(transform1.affine_map(V).shape, (n, q))
assert_equal(transform1.adjoint_map(Y).shape, (p, q))
transform2 = rr.affine_transform(X, B, input_shape=(p, q))
assert_equal(transform2.linear_map(V).shape, (n, q))
assert_equal(transform2.affine_map(V).shape, (n, q))
assert_equal(transform2.adjoint_map(Y).shape, (p, q))
def test_coefs_matrix():
n, p, q = 20, 10, 5
X = np.random.standard_normal((n, p))
B = np.random.standard_normal((n, q))
V = np.random.standard_normal((p, q))
Y = np.random.standard_normal((n, q))
transform1 = rr.linear_transform(X, input_shape=(p,q))
assert_equal(transform1.linear_map(V).shape, (n,q))
assert_equal(transform1.affine_map(V).shape, (n,q))
assert_equal(transform1.adjoint_map(Y).shape, (p,q))
transform2 = rr.affine_transform(X, B, input_shape=(p,q))
assert_equal(transform2.linear_map(V).shape, (n,q))
assert_equal(transform2.affine_map(V).shape, (n,q))
assert_equal(transform2.adjoint_map(Y).shape, (p,q))
def test_row_matrix():
# make sure we can input a vector as a transform
n, p = 20, 1
x = np.random.standard_normal(n)
b = np.random.standard_normal(p)
v = np.random.standard_normal(n)
y = np.random.standard_normal(p)
transform1 = rr.linear_transform(x)
transform2 = rr.affine_transform(x, b)
transform1.linear_map(v)
transform1.affine_map(v)
transform1.adjoint_map(y)
transform2.linear_map(v)
transform2.affine_map(v)
transform2.adjoint_map(y)
def test_row_matrix():
# make sure we can input a vector as a transform
n, p = 20, 1
x = np.random.standard_normal(n)
b = np.random.standard_normal(p)
v = np.random.standard_normal(n)
y = np.random.standard_normal(p)
transform1 = rr.linear_transform(x)
transform2 = rr.affine_transform(x, b)
transform1.linear_map(v)
transform1.affine_map(v)
transform1.adjoint_map(y)
transform2.linear_map(v)
transform2.affine_map(v)
transform2.adjoint_map(y)
def test_class():
n, p = (10, 5)
D = np.random.standard_normal((n,p))
v = np.random.standard_normal(n)
pen = rr.l1norm.affine(D, v, lagrange=0.4)
pen2 = rr.l1norm(n, lagrange=0.4, offset=np.random.standard_normal(n))
pen2.quadratic = None
cls = type(pen)
pen_aff = cls(pen2, rr.affine_transform(D, v))
for _pen in [pen, pen_aff]:
# Run to ensure code gets executed in tests (smoke test)
print(_pen.dual)
print(_pen.latexify())
print(str(_pen))
print(repr(_pen))
print(_pen._repr_latex_())
_pen.nonsmooth_objective(np.random.standard_normal(p))
q = rr.identity_quadratic(0.5,0,0,0)
smoothed_pen = _pen.smoothed(q)
def test_class():
n, p = (10, 5)
D = np.random.standard_normal((n,p))
v = np.random.standard_normal(n)
pen = rr.l1norm.affine(D, v, lagrange=0.4)
pen2 = rr.l1norm(n, lagrange=0.4, offset=np.random.standard_normal(n))
pen2.quadratic = None
cls = type(pen)
pen_aff = cls(pen2, rr.affine_transform(D, v))
for _pen in [pen, pen_aff]:
# Run to ensure code gets executed in tests (smoke test)
print(_pen.dual)
print(_pen.latexify())
print(str(_pen))
print(repr(_pen))
print(_pen._repr_latex_())
_pen.nonsmooth_objective(np.random.standard_normal(p))
q = rr.identity_quadratic(0.5,0,0,0)
smoothed_pen = _pen.smoothed(q)
def test_stack_product():
X = np.random.standard_normal((5, 30))
Y = np.random.standard_normal((5, 30))
Z = np.random.standard_normal((5, 31))
U = np.random.standard_normal((6, 30))
stack = vstack([X, Y])
assert_raises(ValueError, vstack, [X, Z])
assert_raises(ValueError, hstack, [X, U])
np.testing.assert_allclose(stack.linear_map(np.arange(30))[:5], np.dot(X, np.arange(30)))
np.testing.assert_allclose(stack.linear_map(np.arange(30))[5:], np.dot(Y, np.arange(30)))
np.testing.assert_allclose(stack.affine_map(np.arange(30))[:5], np.dot(X, np.arange(30)))
np.testing.assert_allclose(stack.affine_map(np.arange(30))[5:], np.dot(Y, np.arange(30)))
np.testing.assert_allclose(stack.adjoint_map(np.arange(10)), np.dot(X.T, np.arange(5)) + np.dot(Y.T, np.arange(5, 10)))
_hstack = hstack([X, Y, Z])
_hstack.linear_map(np.arange(91))
_hstack.affine_map(np.arange(91))
_hstack.adjoint_map(np.arange(5))
b = np.random.standard_normal(5)
XA = rr.affine_transform(X, b)
_product = product([XA,Y])
np.testing.assert_allclose(_product.linear_map(np.arange(60))[:5], np.dot(X, np.arange(30)))
np.testing.assert_allclose(_product.linear_map(np.arange(60))[5:], np.dot(Y, np.arange(30, 60)))
np.testing.assert_allclose(_product.affine_map(np.arange(60))[:5], np.dot(X, np.arange(30)) + b)
np.testing.assert_allclose(_product.affine_map(np.arange(60))[5:], np.dot(Y, np.arange(30, 60)))
np.testing.assert_allclose(_product.adjoint_map(np.arange(10))[:30], np.dot(X.T, np.arange(5)))
np.testing.assert_allclose(_product.adjoint_map(np.arange(10))[30:], np.dot(Y.T, np.arange(5, 10)))
scale_prod = scalar_multiply(_product, 2)
np.testing.assert_allclose(scale_prod.linear_map(np.arange(60)), 2 * _product.linear_map(np.arange(60)))
np.testing.assert_allclose(scale_prod.affine_map(np.arange(60)), 2 * _product.affine_map(np.arange(60)))
np.testing.assert_allclose(scale_prod.adjoint_map(np.arange(60)), 2 * _product.adjoint_map(np.arange(60)))
def __init__(self,
map,
generative_mean,
coef=1.,
offset=None,
quadratic=None):
self.map = map
self.q = map.p - map.nactive
self.r = map.p + map.nactive
self.p = map.p
self.inactive_conjugate = self.active_conjugate = map.randomization.CGF_conjugate
if self.active_conjugate is None:
raise ValueError(
'randomization must know its CGF_conjugate -- currently only isotropic_gaussian and laplace are implemented and are assumed to be randomization with IID coordinates')
self.inactive_lagrange = self.map.inactive_lagrange
rr.smooth_atom.__init__(self,
(self.r,),
offset=offset,
quadratic=quadratic,
initial=self.map.feasible_point,
coef=coef)
self.coefs[:] = self.map.feasible_point
nonnegative = nonnegative_softmax_scaled(self.map.nactive)
opt_vars = np.zeros(self.r, bool)
opt_vars[map.p:] = 1
self._opt_selector = rr.selector(opt_vars, (self.r,))
self._response_selector = rr.selector(~opt_vars, (self.r,))
self.nonnegative_barrier = nonnegative.linear(self._opt_selector)
self.active_conj_loss = rr.affine_smooth(self.active_conjugate,
rr.affine_transform(np.hstack([self.map.A_active, self.map.B_active]),
self.map.offset_active))
cube_obj = neg_log_cube_probability(self.q, self.inactive_lagrange, randomization_scale=1.)
self.cube_loss = rr.affine_smooth(cube_obj, np.hstack([self.map.A_inactive, self.map.B_inactive]))
# w_1, v_1 = np.linalg.eig(self.map.score_cov)
# self.score_cov_inv_half = (v_1.T.dot(np.diag(np.power(w_1, -0.5)))).dot(v_1)
# likelihood_loss = rr.signal_approximator(np.squeeze(np.zeros(self.p)), coef=1.)
# scaled_response_selector = rr.selector(~opt_vars, (self.r,), rr.affine_transform(self.score_cov_inv_half,
# self.score_cov_inv_half.
# dot(np.squeeze(generative_mean))))
#print("cov", self.map.score_cov.shape )
likelihood_loss = log_likelihood(generative_mean, self.map.score_cov, self.p)
self.likelihood_loss = rr.affine_smooth(likelihood_loss, self._response_selector)
self.total_loss = rr.smooth_sum([self.active_conj_loss,
self.likelihood_loss,
self.nonnegative_barrier,
self.cube_loss])
def __init__(
self,
X,
feasible_point,
active, # the active set chosen by randomized lasso
active_sign, # the set of signs of active coordinates chosen by lasso
lagrange, # in R^p
mean_parameter, # in R^n
noise_variance, #noise_level in data
randomizer, #specified randomization
epsilon, # ridge penalty for randomized lasso
coef=1.,
offset=None,
quadratic=None,
nstep=10):
n, p = X.shape
self._X = X
E = active.sum()
self.q = p - E
self.active = active
self.noise_variance = noise_variance
self.randomization = randomizer
self.inactive_conjugate = self.active_conjugate = randomizer.CGF_conjugate
if self.active_conjugate is None:
raise ValueError(
'randomization must know its CGF_conjugate -- currently only isotropic_gaussian and laplace are implemented and are assumed to be randomization with IID coordinates'
)
initial = np.zeros(n + E, )
initial[n:] = feasible_point
self.n = n
rr.smooth_atom.__init__(self, (n + E, ),
offset=offset,
quadratic=quadratic,
initial=initial,
coef=coef)
self.coefs[:] = initial
opt_vars = np.zeros(n + E, bool)
opt_vars[n:] = 1
nonnegative = nonnegative_softmax_scaled(E)
self._opt_selector = rr.selector(opt_vars, (n + E, ))
self.nonnegative_barrier = nonnegative.linear(self._opt_selector)
self._response_selector = rr.selector(~opt_vars, (n + E, ))
self.set_parameter(mean_parameter, noise_variance)
X_E = X[:, active]
B = X.T.dot(X_E)
B_E = B[active]
B_mE = B[~active]
self.A_active = np.hstack([
-X[:, active].T,
(B_E + epsilon * np.identity(E)) * active_sign[None, :]
])
self.A_inactive = np.hstack(
[-X[:, ~active].T, (B_mE * active_sign[None, :])])
self.offset_active = active_sign * lagrange[active]
self.offset_inactive = np.zeros(p - E)
self.active_conj_loss = rr.affine_smooth(
self.active_conjugate,
rr.affine_transform(self.A_active, self.offset_active))
cube_obj = neg_log_cube_probability(self.q,
lagrange[~active],
randomization_scale=1.)
self.cube_loss = rr.affine_smooth(cube_obj, self.A_inactive)
self.total_loss = rr.smooth_sum([
self.active_conj_loss, self.cube_loss, self.likelihood_loss,
self.nonnegative_barrier
])
def __init__(self,
X,
feasible_point,
active, # the active set chosen by randomized marginal screening
active_signs, # the set of signs of active coordinates chosen by ms
threshold, # in R^p
mean_parameter,
noise_variance,
randomizer,
coef=1.,
offset=None,
quadratic=None,
nstep=10):
n, p = X.shape
self._X = X
E = active.sum()
self.q = p - E
sigma = np.sqrt(noise_variance)
self.active = active
self.noise_variance = noise_variance
self.randomization = randomizer
self.inactive_conjugate = self.active_conjugate = randomizer.CGF_conjugate
if self.active_conjugate is None:
raise ValueError(
'randomization must know its CGF_conjugate -- currently only isotropic_gaussian and laplace are implemented and are assumed to be randomization with IID coordinates')
initial = np.zeros(n + E, )
initial[n:] = feasible_point
self.n = n
rr.smooth_atom.__init__(self,
(n + E,),
offset=offset,
quadratic=quadratic,
initial=initial,
coef=coef)
self.coefs[:] = initial
nonnegative = nonnegative_softmax_scaled(E)
opt_vars = np.zeros(n + E, bool)
opt_vars[n:] = 1
self._opt_selector = rr.selector(opt_vars, (n + E,))
self.nonnegative_barrier = nonnegative.linear(self._opt_selector)
self._response_selector = rr.selector(~opt_vars, (n + E,))
self.set_parameter(mean_parameter, noise_variance)
self.A_active = np.hstack([np.true_divide(-X[:, active].T, sigma), np.identity(E) * active_signs[None, :]])
self.A_inactive = np.hstack([np.true_divide(-X[:, ~active].T, sigma), np.zeros((p - E, E))])
self.offset_active = active_signs * threshold[active]
self.offset_inactive = np.zeros(p - E)
self.active_conj_loss = rr.affine_smooth(self.active_conjugate,
rr.affine_transform(self.A_active, self.offset_active))
cube_obj = neg_log_cube_probability(self.q, threshold[~active], randomization_scale=1.)
self.cube_loss = rr.affine_smooth(cube_obj, rr.affine_transform(self.A_inactive, self.offset_inactive))
self.total_loss = rr.smooth_sum([self.active_conj_loss,
self.cube_loss,
self.likelihood_loss,
self.nonnegative_barrier])
import numpy as np import regreg.api as rr np.random.seed(400) N = 1000 P = 200 Y = 2 * np.random.binomial(1, 0.5, size=(N,)) - 1. X = np.random.standard_normal((N,P)) X[Y==1] += np.array([30,-20] + (P-2)*[0])[np.newaxis,:] X -= X.mean(0)[np.newaxis, :] X_1 = np.hstack([X, np.ones((N,1))]) transform = rr.affine_transform(-Y[:,np.newaxis] * X_1, np.ones(N)) C = 0.2 hinge = rr.positive_part(N, lagrange=C) hinge_loss = rr.linear_atom(hinge, transform) epsilon = 0.04 smoothed_hinge_loss = rr.smoothed_atom(hinge_loss, epsilon=epsilon) s = rr.selector(slice(0,P), (P+1,)) sparsity = rr.l1norm.linear(s, lagrange=3.) quadratic = rr.quadratic.linear(s, coef=0.5) from regreg.affine import power_L ltransform = rr.linear_transform(X_1) singular_value_sq = power_L(X_1) # the other smooth piece is a quadratic with identity # for quadratic form, so its lipschitz constant is 1
import numpy as np import regreg.api as rr np.random.seed(400) N = 500 P = 2 Y = 2 * np.random.binomial(1, 0.5, size=(N,)) - 1. X = np.random.standard_normal((N,P)) X[Y==1] += np.array([3,-2])[np.newaxis,:] X_1 = np.hstack([X, np.ones((N,1))]) X_1_signs = -Y[:,np.newaxis] * X_1 transform = rr.affine_transform(X_1_signs, np.ones(N)) C = 0.2 hinge = rr.positive_part(N, lagrange=C) hinge_loss = rr.linear_atom(hinge, transform) quadratic = rr.quadratic.linear(rr.selector(slice(0,P), (P+1,)), coef=0.5) problem = rr.container(quadratic, hinge_loss) solver = rr.FISTA(problem) solver.fit() import pylab pylab.clf() pylab.scatter(X[Y==1,0],X[Y==1,1], facecolor='red') pylab.scatter(X[Y==-1,0],X[Y==-1,1], facecolor='blue') fits = np.dot(X_1, problem.coefs) labels = 2 * (fits > 0) - 1
import numpy as np import regreg.api as rr np.random.seed(400) N = 1000 P = 200 Y = 2 * np.random.binomial(1, 0.5, size=(N, )) - 1. X = np.random.standard_normal((N, P)) X[Y == 1] += np.array([30, -20] + (P - 2) * [0])[np.newaxis, :] X -= X.mean(0)[np.newaxis, :] X_1 = np.hstack([X, np.ones((N, 1))]) transform = rr.affine_transform(-Y[:, np.newaxis] * X_1, np.ones(N)) C = 0.2 hinge = rr.positive_part(N, lagrange=C) hinge_loss = rr.linear_atom(hinge, transform) epsilon = 0.04 smoothed_hinge_loss = rr.smoothed_atom(hinge_loss, epsilon=epsilon) s = rr.selector(slice(0, P), (P + 1, )) sparsity = rr.l1norm.linear(s, lagrange=3.) quadratic = rr.quadratic.linear(s, coef=0.5) from regreg.affine import power_L ltransform = rr.linear_transform(X_1) singular_value_sq = power_L(X_1) # the other smooth piece is a quadratic with identity # for quadratic form, so its lipschitz constant is 1
"""
Solving basis pursuit with TFOCS
"""
import regreg.api as rr
import numpy as np
import nose.tools as nt
n, p = 100, 200
X = np.random.standard_normal((n, p))
beta = np.zeros(p)
beta[:4] = 3
Y = np.random.standard_normal(n) + np.dot(X, beta)
lscoef = np.dot(np.linalg.pinv(X), Y)
minimum_l2 = np.linalg.norm(Y - np.dot(X, lscoef))
maximum_l2 = np.linalg.norm(Y)
l2bound = (minimum_l2 + maximum_l2) * 0.5
l2 = rr.l2norm(n, bound=l2bound)
T = rr.affine_transform(X, -Y)
l1 = rr.l1norm(p, lagrange=1)
primal, dual = rr.tfocs(l1, T, l2, tol=1.e-10)
nt.assert_true(
np.fabs(np.linalg.norm(Y - np.dot(X, primal)) - l2bound) <= l2bound *
1.e-5)
def __init__(
self,
X,
feasible_point, #in R^{|E|_1 + |E|_2}
active_1, #the active set chosen by randomized marginal screening
active_2, #the active set chosen by randomized lasso
active_signs_1, #the set of signs of active coordinates chosen by ms
active_signs_2, #the set of signs of active coordinates chosen by lasso
lagrange, #in R^p
threshold, #in R^p
mean_parameter, # in R^n
noise_variance,
randomizer,
epsilon, #ridge penalty for randomized lasso
coef=1.,
offset=None,
quadratic=None,
nstep=10):
n, p = X.shape
self._X = X
E_1 = active_1.sum()
E_2 = active_2.sum()
sigma = np.sqrt(noise_variance)
self.active_1 = active_1
self.active_2 = active_2
self.noise_variance = noise_variance
self.randomization = randomizer
self.inactive_conjugate = self.active_conjugate = randomizer.CGF_conjugate
if self.active_conjugate is None:
raise ValueError(
'randomization must know its CGF_conjugate -- currently only isotropic_gaussian and laplace are implemented and are assumed to be randomization with IID coordinates'
)
initial = np.zeros(n + E_1 + E_2, )
initial[n:] = feasible_point
self.n = n
rr.smooth_atom.__init__(self, (n + E_1 + E_2, ),
offset=offset,
quadratic=quadratic,
initial=initial,
coef=coef)
self.coefs[:] = initial
nonnegative = nonnegative_softmax_scaled(E_1 + E_2)
opt_vars = np.zeros(n + E_1 + E_2, bool)
opt_vars[n:] = 1
self._opt_selector = rr.selector(opt_vars, (n + E_1 + E_2, ))
self.nonnegative_barrier = nonnegative.linear(self._opt_selector)
self._response_selector = rr.selector(~opt_vars, (n + E_1 + E_2, ))
self.set_parameter(mean_parameter, noise_variance)
arg_ms = np.zeros(self.n + E_1 + E_2, bool)
arg_ms[:self.n + E_1] = 1
arg_lasso = np.zeros(self.n + E_1, bool)
arg_lasso[:self.n] = 1
arg_lasso = np.append(arg_lasso, np.ones(E_2, bool))
self.A_active_1 = np.hstack([
np.true_divide(-X[:, active_1].T, sigma),
np.identity(E_1) * active_signs_1[None, :]
])
self.A_inactive_1 = np.hstack([
np.true_divide(-X[:, ~active_1].T, sigma),
np.zeros((p - E_1, E_1))
])
self.offset_active_1 = active_signs_1 * threshold[active_1]
self.offset_inactive_1 = np.zeros(p - E_1)
self._active_ms = rr.selector(
arg_ms, (self.n + E_1 + E_2, ),
rr.affine_transform(self.A_active_1, self.offset_active_1))
self._inactive_ms = rr.selector(
arg_ms, (self.n + E_1 + E_2, ),
rr.affine_transform(self.A_inactive_1, self.offset_inactive_1))
self.active_conj_loss_1 = rr.affine_smooth(self.active_conjugate,
self._active_ms)
self.q_1 = p - E_1
cube_obj_1 = neg_log_cube_probability(self.q_1,
threshold[~active_1],
randomization_scale=1.)
self.cube_loss_1 = rr.affine_smooth(cube_obj_1, self._inactive_ms)
X_step2 = X[:, active_1]
X_E_2 = X_step2[:, active_2]
B = X_step2.T.dot(X_E_2)
B_E = B[active_2]
B_mE = B[~active_2]
self.A_active_2 = np.hstack([
-X_step2[:, active_2].T,
(B_E + epsilon * np.identity(E_2)) * active_signs_2[None, :]
])
self.A_inactive_2 = np.hstack(
[-X_step2[:, ~active_2].T, (B_mE * active_signs_2[None, :])])
self.offset_active_2 = active_signs_2 * lagrange[active_2]
self.offset_inactive_2 = np.zeros(E_1 - E_2)
self._active_lasso = rr.selector(
arg_lasso, (self.n + E_1 + E_2, ),
rr.affine_transform(self.A_active_2, self.offset_active_2))
self._inactive_lasso = rr.selector(
arg_lasso, (self.n + E_1 + E_2, ),
rr.affine_transform(self.A_inactive_2, self.offset_inactive_2))
self.active_conj_loss_2 = rr.affine_smooth(self.active_conjugate,
self._active_lasso)
self.q_2 = E_1 - E_2
cube_obj_2 = neg_log_cube_probability(self.q_2,
lagrange[~active_2],
randomization_scale=1.)
self.cube_loss_2 = rr.affine_smooth(cube_obj_2, self._inactive_lasso)
self.total_loss = rr.smooth_sum([
self.active_conj_loss_1, self.active_conj_loss_2, self.cube_loss_1,
self.cube_loss_2, self.likelihood_loss, self.nonnegative_barrier
])
""" Solving basis pursuit with TFOCS """ import regreg.api as rr import numpy as np import nose.tools as nt n, p = 100, 200 X = np.random.standard_normal((n, p)) beta = np.zeros(p) beta[:4] = 3 Y = np.random.standard_normal(n) + np.dot(X, beta) lscoef = np.dot(np.linalg.pinv(X), Y) minimum_l2 = np.linalg.norm(Y - np.dot(X, lscoef)) maximum_l2 = np.linalg.norm(Y) l2bound = (minimum_l2 + maximum_l2) * 0.5 l2 = rr.l2norm(n, bound=l2bound) T = rr.affine_transform(X, -Y) l1 = rr.l1norm(p, lagrange=1) primal, dual = rr.tfocs(l1, T, l2, tol=1.0e-10) nt.assert_true(np.fabs(np.linalg.norm(Y - np.dot(X, primal)) - l2bound) <= l2bound * 1.0e-5)