Пример #1
0
def test_multinomial_vs_logistic():

    """
    Test that multinomial regression with two categories is the same as logistic regression
    """

    n = 500
    p = 10
    J = 2

    X = np.random.standard_normal(n*p).reshape((n,p))
    counts = np.random.randint(0,10,n*J).reshape((n,J)) + 2

    mult_x = rr.linear_transform(X, input_shape=(p,J-1))
    loss = rr.multinomial_deviance.linear(mult_x, counts=counts)
    problem = rr.container(loss)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs1 = solver.composite.coefs

    loss = rr.logistic_deviance.linear(X, successes=counts[:,0], trials = np.sum(counts, axis=1))
    problem = rr.container(loss)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs2 = solver.composite.coefs

    loss = rr.logistic_deviance.linear(X, successes=counts[:,1], trials = np.sum(counts, axis=1))
    problem = rr.container(loss)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs3 = solver.composite.coefs

    npt.assert_equal(coefs1.shape,(p,J-1))
    npt.assert_array_almost_equal(coefs1.flatten(), coefs2.flatten(), 5)
    npt.assert_array_almost_equal(coefs1.flatten(), -coefs3.flatten(), 5)
Пример #2
0
def test_conjugate_solver():

    # Solve Lagrange problem
    Y = np.random.standard_normal(500)
    Y[100:150] += 7
    Y[250:300] += 14
    loss = R.quadratic.shift(-Y, coef=0.5)

    sparsity = R.l1norm(len(Y), lagrange=1.4)
    D = sparse.csr_matrix((np.identity(500) + np.diag([-1] * 499, k=1))[:-1])
    fused = R.l1norm.linear(D, lagrange=25.5)
    problem = R.container(loss, sparsity, fused)

    solver = R.FISTA(problem)
    solver.fit(max_its=500, tol=1e-10)
    solution = solver.composite.coefs

    # Solve constrained version
    delta1 = np.fabs(D * solution).sum()
    delta2 = np.fabs(solution).sum()
    fused_constraint = R.l1norm.linear(D, bound=delta1)
    sparsity_constraint = R.l1norm(500, bound=delta2)
    constrained_problem = R.container(loss, fused_constraint,
                                      sparsity_constraint)
    constrained_solver = R.FISTA(constrained_problem)
    vals = constrained_solver.fit(max_its=500, tol=1e-10)
    constrained_solution = constrained_solver.composite.coefs

    npt.assert_almost_equal(np.fabs(constrained_solution).sum(), delta2, 3)
    npt.assert_almost_equal(np.fabs(D * constrained_solution).sum(), delta1, 3)

    # Solve with (shifted) conjugate function

    loss = R.quadratic.shift(-Y, coef=0.5)
    true_conjugate = R.quadratic.shift(Y, coef=0.5)
    problem = R.container(loss, fused_constraint, sparsity_constraint)
    solver = R.FISTA(problem.conjugate_composite(true_conjugate))
    solver.fit(max_its=500, tol=1e-10)
    conjugate_coefs = problem.conjugate_primal_from_dual(
        solver.composite.coefs)

    # Solve with generic conjugate function

    loss = R.quadratic.shift(-Y, coef=0.5)
    problem = R.container(loss, fused_constraint, sparsity_constraint)
    solver2 = R.FISTA(problem.conjugate_composite(conjugate_tol=1e-12))
    solver2.fit(max_its=500, tol=1e-10)
    conjugate_coefs_gen = problem.conjugate_primal_from_dual(
        solver2.composite.coefs)

    d1 = np.linalg.norm(solution -
                        constrained_solution) / np.linalg.norm(solution)
    d2 = np.linalg.norm(solution - conjugate_coefs) / np.linalg.norm(solution)
    d3 = np.linalg.norm(solution -
                        conjugate_coefs_gen) / np.linalg.norm(solution)

    npt.assert_array_less(d1, 0.01)
    npt.assert_array_less(d2, 0.01)
    npt.assert_array_less(d3, 0.01)
Пример #3
0
def test_lasso_separable():
    """
    This test verifies that the specification of a separable
    penalty yields the same results as having two linear_atoms
    with selector matrices. The penalty here is a lasso, i.e. l1
    penalty.
    """

    X = np.random.standard_normal((100,20))
    Y = np.random.standard_normal((100,)) + np.dot(X, np.random.standard_normal(20))

    penalty1 = rr.l1norm(10, lagrange=1.2)
    penalty2 = rr.l1norm(10, lagrange=1.2)
    penalty = rr.separable((20,), [penalty1, penalty2], [slice(0,10), slice(10,20)], test_for_overlap=True)

    # ensure code is tested

    print(penalty1.latexify())

    print(penalty.latexify())
    print(penalty.conjugate)
    print(penalty.dual)
    print(penalty.seminorm(np.ones(penalty.shape)))
    print(penalty.constraint(np.ones(penalty.shape), bound=2.))

    pencopy = copy(penalty)
    pencopy.set_quadratic(rr.identity_quadratic(1,0,0,0))
    pencopy.conjugate

    # solve using separable
    
    loss = rr.quadratic.affine(X, -Y, coef=0.5)
    problem = rr.separable_problem.fromatom(penalty, loss)
    solver = rr.FISTA(problem)
    solver.fit(min_its=200, tol=1.0e-12)
    coefs = solver.composite.coefs

    # solve using the usual composite

    penalty_all = rr.l1norm(20, lagrange=1.2)
    problem_all = rr.container(loss, penalty_all)
    solver_all = rr.FISTA(problem_all)
    solver_all.fit(min_its=100, tol=1.0e-12)

    coefs_all = solver_all.composite.coefs

    # solve using the selectors

    penalty_s = [rr.linear_atom(p, rr.selector(g, (20,))) for p, g in
                 zip(penalty.atoms, penalty.groups)]
    problem_s = rr.container(loss, *penalty_s)
    solver_s = rr.FISTA(problem_s)
    solver_s.fit(min_its=500, tol=1.0e-12)
    coefs_s = solver_s.composite.coefs

    np.testing.assert_almost_equal(coefs, coefs_all)
    np.testing.assert_almost_equal(coefs, coefs_s)
Пример #4
0
def test_lasso_separable():
    """
    This test verifies that the specification of a separable
    penalty yields the same results as having two linear_atoms
    with selector matrices. The penalty here is a lasso, i.e. l1
    penalty.
    """

    X = np.random.standard_normal((100,20))
    Y = np.random.standard_normal((100,)) + np.dot(X, np.random.standard_normal(20))

    penalty1 = rr.l1norm(10, lagrange=1.2)
    penalty2 = rr.l1norm(10, lagrange=1.2)
    penalty = rr.separable((20,), [penalty1, penalty2], [slice(0,10), slice(10,20)], test_for_overlap=True)

    # ensure code is tested

    print(penalty1.latexify())

    print(penalty.latexify())
    print(penalty.conjugate)
    print(penalty.dual)
    print(penalty.seminorm(np.ones(penalty.shape)))
    print(penalty.constraint(np.ones(penalty.shape), bound=2.))

    pencopy = copy(penalty)
    pencopy.set_quadratic(rr.identity_quadratic(1,0,0,0))
    pencopy.conjugate

    # solve using separable
    
    loss = rr.quadratic_loss.affine(X, -Y, coef=0.5)
    problem = rr.separable_problem.fromatom(penalty, loss)
    solver = rr.FISTA(problem)
    solver.fit(min_its=200, tol=1.0e-12)
    coefs = solver.composite.coefs

    # solve using the usual composite

    penalty_all = rr.l1norm(20, lagrange=1.2)
    problem_all = rr.container(loss, penalty_all)
    solver_all = rr.FISTA(problem_all)
    solver_all.fit(min_its=100, tol=1.0e-12)

    coefs_all = solver_all.composite.coefs

    # solve using the selectors

    penalty_s = [rr.linear_atom(p, rr.selector(g, (20,))) for p, g in
                 zip(penalty.atoms, penalty.groups)]
    problem_s = rr.container(loss, *penalty_s)
    solver_s = rr.FISTA(problem_s)
    solver_s.fit(min_its=500, tol=1.0e-12)
    coefs_s = solver_s.composite.coefs

    np.testing.assert_almost_equal(coefs, coefs_all)
    np.testing.assert_almost_equal(coefs, coefs_s)
Пример #5
0
def test_conjugate_solver():

    # Solve Lagrange problem 
    Y = np.random.standard_normal(500); Y[100:150] += 7; Y[250:300] += 14
    loss = R.quadratic.shift(-Y, coef=0.5)

    sparsity = R.l1norm(len(Y), lagrange = 1.4)
    D = sparse.csr_matrix((np.identity(500) + np.diag([-1]*499,k=1))[:-1])
    fused = R.l1norm.linear(D, lagrange = 25.5)
    problem = R.container(loss, sparsity, fused)
    
    solver = R.FISTA(problem)
    solver.fit(max_its=500, tol=1e-10)
    solution = solver.composite.coefs

    # Solve constrained version
    delta1 = np.fabs(D * solution).sum()
    delta2 = np.fabs(solution).sum()
    fused_constraint = R.l1norm.linear(D, bound = delta1)
    sparsity_constraint = R.l1norm(500, bound = delta2)
    constrained_problem = R.container(loss, fused_constraint, sparsity_constraint)
    constrained_solver = R.FISTA(constrained_problem)
    vals = constrained_solver.fit(max_its=500, tol=1e-10)
    constrained_solution = constrained_solver.composite.coefs

    npt.assert_almost_equal(np.fabs(constrained_solution).sum(), delta2, 3)
    npt.assert_almost_equal(np.fabs(D * constrained_solution).sum(), delta1, 3)


    # Solve with (shifted) conjugate function

    loss = R.quadratic.shift(-Y, coef=0.5)
    true_conjugate = R.quadratic.shift(Y, coef=0.5)
    problem = R.container(loss, fused_constraint, sparsity_constraint)
    solver = R.FISTA(problem.conjugate_composite(true_conjugate))
    solver.fit(max_its=500, tol=1e-10)
    conjugate_coefs = problem.conjugate_primal_from_dual(solver.composite.coefs)
                      

    # Solve with generic conjugate function

    loss = R.quadratic.shift(-Y, coef=0.5)
    problem = R.container(loss, fused_constraint, sparsity_constraint)
    solver2 = R.FISTA(problem.conjugate_composite(conjugate_tol=1e-12))
    solver2.fit(max_its=500, tol=1e-10)
    conjugate_coefs_gen = problem.conjugate_primal_from_dual(solver2.composite.coefs)



    d1 = np.linalg.norm(solution - constrained_solution) / np.linalg.norm(solution)
    d2 = np.linalg.norm(solution - conjugate_coefs) / np.linalg.norm(solution)
    d3 = np.linalg.norm(solution - conjugate_coefs_gen) / np.linalg.norm(solution)

    npt.assert_array_less(d1, 0.01)
    npt.assert_array_less(d2, 0.01)
    npt.assert_array_less(d3, 0.01)
Пример #6
0
def test_affine_linear_offset_l1norm():

    """
    Test linear, affine and offset with the l1norm atom
    """
    
    n = 1000
    p = 10
    
    X = np.random.standard_normal((n,p))
    Y = 10*np.random.standard_normal(n)
    
    coefs = []
    
    loss = rr.quadratic.affine(X,-Y, coef=0.5)
    sparsity = rr.l1norm(p, lagrange=5.)
    problem = rr.container(loss, sparsity)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs.append(1.*solver.composite.coefs)
    
    loss = rr.quadratic.affine(X,-Y, coef=0.5)
    sparsity = rr.l1norm.linear(np.eye(p), lagrange=5.)
    problem = rr.container(loss, sparsity)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs.append(1.*solver.composite.coefs)
    
    loss = rr.quadratic.affine(X,-Y, coef=0.5)
    sparsity = rr.l1norm.affine(np.eye(p),np.zeros(p), lagrange=5.)
    problem = rr.container(loss, sparsity)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs.append(1.*solver.composite.coefs)
    
    loss = rr.quadratic.affine(X,-Y, coef=0.5)
    sparsity = rr.l1norm.linear(np.eye(p), lagrange=5., offset=np.zeros(p))
    problem = rr.container(loss, sparsity)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs.append(1.*solver.composite.coefs)
    
    loss = rr.quadratic.affine(X,-Y, coef=0.5)
    sparsity = rr.l1norm.shift(np.zeros(p), lagrange=5.)
    problem = rr.container(loss, sparsity)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs.append(1.*solver.composite.coefs)

    for i,j in itertools.combinations(range(len(coefs)), 2):
        npt.assert_almost_equal(coefs[i], coefs[j])
def test_affine_linear_offset_l1norm():

    """
    Test linear, affine and offset with the l1norm atom
    """
    
    n = 1000
    p = 10
    
    X = np.random.standard_normal((n,p))
    Y = 10*np.random.standard_normal(n)
    
    coefs = []
    
    loss = rr.quadratic.affine(X,-Y, coef=0.5)
    sparsity = rr.l1norm(p, lagrange=5.)
    problem = rr.container(loss, sparsity)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs.append(1.*solver.composite.coefs)
    
    loss = rr.quadratic.affine(X,-Y, coef=0.5)
    sparsity = rr.l1norm.linear(np.eye(p), lagrange=5.)
    problem = rr.container(loss, sparsity)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs.append(1.*solver.composite.coefs)
    
    loss = rr.quadratic.affine(X,-Y, coef=0.5)
    sparsity = rr.l1norm.affine(np.eye(p),np.zeros(p), lagrange=5.)
    problem = rr.container(loss, sparsity)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs.append(1.*solver.composite.coefs)
    
    loss = rr.quadratic.affine(X,-Y, coef=0.5)
    sparsity = rr.l1norm.linear(np.eye(p), lagrange=5., offset=np.zeros(p))
    problem = rr.container(loss, sparsity)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs.append(1.*solver.composite.coefs)
    
    loss = rr.quadratic.affine(X,-Y, coef=0.5)
    sparsity = rr.l1norm.shift(np.zeros(p), lagrange=5.)
    problem = rr.container(loss, sparsity)
    solver = rr.FISTA(problem)
    solver.fit(debug=False, tol=1e-10)
    coefs.append(1.*solver.composite.coefs)

    for i,j in itertools.combinations(range(len(coefs)), 2):
        npt.assert_almost_equal(coefs[i], coefs[j])
Пример #8
0
    def test_container(self):
        tests = []
        atom, q, prox_center, L = self.atom, self.q, self.prox_center, self.L
        loss = self.loss

        problem = rr.container(loss, atom)
        solver = rr.FISTA(problem)
        solver.fit(tol=1.0e-12,
                   coef_stop=self.coef_stop,
                   FISTA=self.FISTA,
                   min_its=100)

        tests.append((atom.proximal(q), solver.composite.coefs,
                      'solving atom prox with container\n %s ' % str(self)))

        # write the loss in terms of a quadratic for the smooth loss and a smooth function...

        q = rr.identity_quadratic(L, prox_center, 0, 0)
        lossq = rr.quadratic.shift(prox_center.copy(), coef=0.6 * L)
        lossq.quadratic = rr.identity_quadratic(0.4 * L, prox_center.copy(), 0,
                                                0)
        problem = rr.container(lossq, atom)
        solver = rr.FISTA(problem)
        solver.fit(tol=1.0e-12, FISTA=self.FISTA, coef_stop=self.coef_stop)

        tests.append((atom.proximal(q),
                      problem.solve(tol=1.e-12,
                                    FISTA=self.FISTA,
                                    coef_stop=self.coef_stop),
                      'solving prox with container with monotonicity ' +
                      'but loss has identity_quadratic\n %s ' % str(self)))

        d = atom.conjugate
        problem = rr.container(d, loss)
        solver = rr.FISTA(problem)
        solver.fit(tol=1.0e-12,
                   coef_stop=self.coef_stop,
                   FISTA=self.FISTA,
                   min_its=100)
        tests.append((d.proximal(q), solver.composite.coefs,
                      'solving dual prox with container\n %s ' % str(self)))

        if not self.interactive:
            for test in tests:
                yield (all_close, ) + test + (self, )
        else:
            for test in tests:
                yield all_close(*((test + (self, ))))
Пример #9
0
def example4(lambda1=10):

    #Example with an initial value for backtracking

    # In the previous examples you'll see a lot of "Increasing inv_step" iterations - these are trying to find an approximate Lipschitz constant in a backtracking loop.
    # For your problem the Lipschitz constant is just the largest eigenvalue of X^TX, so you can precompute this with a few power iterations.

    n = 100
    p = 1000

    X = np.random.standard_normal(n*p).reshape((n,p))
    Y = 10*np.random.standard_normal(n)

    v = np.random.standard_normal(p)
    for i in range(10):
        v = np.dot(X.T, np.dot(X,v))
        norm = np.linalg.norm(v)
        v /= norm
    print "Approximate Lipschitz constant is", norm

    loss = rr.l2normsq.affine(X,-Y,coef=1.)
    sparsity = rr.l1norm(p, lagrange = lambda1)
    nonnegative = rr.nonnegative(p)

    problem = rr.container(loss, sparsity, nonnegative)
    solver = rr.FISTA(problem)

    #Give approximate Lipschitz constant to solver
    solver.fit(debug=True, start_inv_step=norm)

    solution = solver.composite.coefs
Пример #10
0
def test_group_lasso_separable():
    """
    This test verifies that the specification of a separable
    penalty yields the same results as having two linear_atoms
    with selector matrices. The penalty here is a group_lasso, i.e. l2
    penalty.
    """

    X = np.random.standard_normal((100,20))
    Y = np.random.standard_normal((100,)) + np.dot(X, np.random.standard_normal(20))

    penalty1 = rr.l2norm(10, lagrange=.2)
    penalty2 = rr.l2norm(10, lagrange=.2)
    penalty = rr.separable((20,), [penalty1, penalty2], [slice(0,10), slice(10,20)])

    # solve using separable
    
    loss = rr.quadratic_loss.affine(X, -Y, coef=0.5)
    problem = rr.separable_problem.fromatom(penalty, loss)
    solver = rr.FISTA(problem)
    solver.fit(min_its=200, tol=1.0e-12)
    coefs = solver.composite.coefs

    # solve using the selectors

    penalty_s = [rr.linear_atom(p, rr.selector(g, (20,))) for p, g in
                 zip(penalty.atoms, penalty.groups)]
    problem_s = rr.container(loss, *penalty_s)
    solver_s = rr.FISTA(problem_s)
    solver_s.fit(min_its=200, tol=1.0e-12)
    coefs_s = solver_s.composite.coefs

    np.testing.assert_almost_equal(coefs, coefs_s)
Пример #11
0
def test_lasso(n=100):

    l1 = 1.
    sparsity = R.l1norm(n, lagrange=l1)
    
    X = np.random.standard_normal((5000,n))
    Y = np.random.standard_normal((5000,))
    regloss = R.quadratic.affine(-X,Y)

    p=R.container(regloss, sparsity)
    solver=R.FISTA(p)
    solver.debug = True
    t1 = time.time()
    vals1 = solver.fit(max_its=800)
    t2 = time.time()
    dt1 = t2 - t1
    soln = solver.composite.coefs

    time.sleep(5)


    print soln[range(10)]

    print solver.composite.objective(soln)
    print "Times", dt1
Пример #12
0
def fused_example():

    x=np.random.standard_normal(500); x[100:150] += 7

    sparsity = R.l1norm(500, lagrange=1.3)
    D = (np.identity(500) + np.diag([-1]*499,k=1))[:-1]
    fused = R.l1norm.linear(D, lagrange=10.5)

    loss = R.quadratic.shift(-x, coef=0.5)
    pen = R.container(loss, sparsity,fused)
    solver = R.FISTA(pen)
    vals = solver.fit()
    soln = solver.composite.coefs
    
    # solution

    pylab.figure(num=1)
    pylab.clf()
    pylab.plot(soln, c='g')
    pylab.scatter(np.arange(x.shape[0]), x)

    # objective values

    pylab.figure(num=2)
    pylab.clf()
    pylab.plot(vals)
Пример #13
0
def test_group_lasso_sparse(n=100):

    def selector(p, slice):
        return np.identity(p)[slice]

    def selector_sparse(p, slice):
        return sparse.csr_matrix(np.identity(p)[slice])

    X = np.random.standard_normal((1000,500))
    Y = np.random.standard_normal((1000,))
    loss = R.quadratic.affine(X, -Y, coef=0.5)

    penalties = [R.l2norm.linear(selector(500, slice(i*100,(i+1)*100)), lagrange=.1) for i in range(5)]
    penalties[0].lagrange = 250.
    penalties[1].lagrange = 225.
    penalties[2].lagrange = 150.
    penalties[3].lagrange = 100.
    group_lasso = R.container(loss, *penalties)

    solver=R.FISTA(group_lasso)
    solver.debug = True
    t1 = time.time()
    vals = solver.fit(max_its=2000, min_its=20,tol=1e-8)
    soln1 = solver.composite.coefs
    t2 = time.time()
    dt1 = t2 - t1



    print soln1[range(10)]
Пример #14
0
def group_lasso_example():

    def selector(p, slice):
        return np.identity(p)[slice]
    penalties = [R.l2norm(selector(500, slice(i*100,(i+1)*100)), lagrange=.1) for i in range(5)]
    penalties[0].lagrange = 250.
    penalties[1].lagrange = 225.
    penalties[2].lagrange = 150.
    penalties[3].lagrange = 100.

    X = np.random.standard_normal((1000,500))
    Y = np.random.standard_normal((1000,))
    loss = R.quadratic.affine(X, -Y, coef=0.5)
    group_lasso = R.container(loss, *penalties)

    solver=R.FISTA(group_lasso)
    solver.debug = True
    vals = solver.fit(max_its=2000, min_its=20,tol=1e-10)
    soln = solver.composite.coefs

    # solution

    pylab.figure(num=1)
    pylab.clf()
    pylab.plot(soln, c='g')

    # objective values

    pylab.figure(num=2)
    pylab.clf()
    pylab.plot(vals)
Пример #15
0
def test_lasso():
    """
    this test verifies that the l1 prox can be solved
    by a primal/dual specification 

    obviously, we don't to solve the l1 prox this way,
    but it verifies that specification is working correctly

    """

    l1 = rr.l1norm(4, lagrange=2.0)

    l11 = rr.l1norm(4, lagrange=1.0)
    l12 = rr.l1norm(4, lagrange=1.0)

    X = np.random.standard_normal((10, 4))
    Y = np.random.standard_normal(10) + 3

    loss = rr.quadratic.affine(X, -Y)
    p1 = rr.container(l11, loss, l12)

    solver1 = rr.FISTA(p1)
    solver1.fit(tol=1.0e-12, min_its=500)

    p2 = rr.separable_problem.singleton(l1, loss)
    solver2 = rr.FISTA(p2)
    solver2.fit(tol=1.0e-12)

    f = p2.objective
    ans = scipy.optimize.fmin_powell(f, np.zeros(4), ftol=1.0e-12)
    print f(solver2.composite.coefs), f(ans)
    print f(solver1.composite.coefs), f(ans)

    yield ac, ans, solver2.composite.coefs, "singleton solver"
    yield ac, solver1.composite.coefs, solver2.composite.coefs, "container solver"
Пример #16
0
def test_group_lasso_separable():
    """
    This test verifies that the specification of a separable
    penalty yields the same results as having two linear_atoms
    with selector matrices. The penalty here is a group_lasso, i.e. l2
    penalty.
    """

    X = np.random.standard_normal((100,20))
    Y = np.random.standard_normal((100,)) + np.dot(X, np.random.standard_normal(20))

    penalty1 = rr.l2norm(10, lagrange=.2)
    penalty2 = rr.l2norm(10, lagrange=.2)
    penalty = rr.separable((20,), [penalty1, penalty2], [slice(0,10), slice(10,20)])

    # solve using separable
    
    loss = rr.quadratic.affine(X, -Y, coef=0.5)
    problem = rr.separable_problem.fromatom(penalty, loss)
    solver = rr.FISTA(problem)
    solver.fit(min_its=200, tol=1.0e-12)
    coefs = solver.composite.coefs

    # solve using the selectors

    penalty_s = [rr.linear_atom(p, rr.selector(g, (20,))) for p, g in
                 zip(penalty.atoms, penalty.groups)]
    problem_s = rr.container(loss, *penalty_s)
    solver_s = rr.FISTA(problem_s)
    solver_s.fit(min_its=200, tol=1.0e-12)
    coefs_s = solver_s.composite.coefs

    np.testing.assert_almost_equal(coefs, coefs_s)
Пример #17
0
def test_lasso():
    '''
    this test verifies that the l1 prox can be solved
    by a primal/dual specification 

    obviously, we don't to solve the l1 prox this way,
    but it verifies that specification is working correctly

    '''

    l1 = rr.l1norm(4, lagrange=2.)

    l11 = rr.l1norm(4, lagrange=1.)
    l12 = rr.l1norm(4, lagrange=1.)

    X = np.random.standard_normal((10, 4))
    Y = np.random.standard_normal(10) + 3

    loss = rr.quadratic.affine(X, -Y)
    p1 = rr.container(l11, loss, l12)

    solver1 = rr.FISTA(p1)
    solver1.fit(tol=1.0e-12, min_its=500)

    p2 = rr.separable_problem.singleton(l1, loss)
    solver2 = rr.FISTA(p2)
    solver2.fit(tol=1.0e-12)

    f = p2.objective
    ans = scipy.optimize.fmin_powell(f, np.zeros(4), ftol=1.0e-12)
    print(f(solver2.composite.coefs), f(ans))
    print(f(solver1.composite.coefs), f(ans))

    yield all_close, ans, solver2.composite.coefs, 'singleton solver', None
    yield all_close, solver1.composite.coefs, solver2.composite.coefs, 'container solver', None
Пример #18
0
def test_lasso_dual():

    """
    Check that the solution of the lasso signal approximator dual composite is soft-thresholding
    """

    l1 = .1
    sparsity = R.l1norm(10, lagrange=l1)
    x = np.arange(10) - 5
    loss = R.quadratic.shift(-x, coef=0.5)

    pen = R.simple_problem(loss, sparsity)
    solver = R.FISTA(pen)
    pen.lipschitz = 1
    solver.fit(backtrack=False)
    soln = solver.composite.coefs
    st = np.maximum(np.fabs(x)-l1,0) * np.sign(x) 

    np.testing.assert_almost_equal(soln,st, decimal=3)

    pen = R.simple_problem(loss, sparsity)
    solver = R.FISTA(pen)
    solver.fit(monotonicity_restart=False)
    soln = solver.composite.coefs
    st = np.maximum(np.fabs(x)-l1,0) * np.sign(x) 

    np.testing.assert_almost_equal(soln,st, decimal=3)


    pen = R.container(loss, sparsity)
    solver = R.FISTA(pen)
    solver.fit()
    soln = solver.composite.coefs

    np.testing.assert_almost_equal(soln,st, decimal=3)
Пример #19
0
def test_multiple_lasso_dual(n=500):
    """
    Check that the solution of the lasso signal approximator dual composite is soft-thresholding even when specified with multiple seminorms
    """

    l1 = 1
    sparsity1 = R.l1norm(n, lagrange=l1 * 0.75)
    sparsity2 = R.l1norm(n, lagrange=l1 * 0.25)
    x = np.random.normal(0, 1, n)
    loss = R.quadratic.shift(-x, coef=0.5)

    p = R.dual_problem.fromprimal(loss, sparsity1, sparsity2)
    t1 = time.time()
    solver = R.FISTA(p)
    solver.debug = True
    vals = solver.fit(tol=1.0e-16)
    soln = p.primal
    t2 = time.time()
    print t2 - t1
    st = np.maximum(np.fabs(x) - l1, 0) * np.sign(x)
    np.testing.assert_almost_equal(soln, st, decimal=3)

    p = R.container(loss, sparsity1, sparsity2)
    t1 = time.time()
    solver = R.FISTA(p)
    solver.debug = True
    vals = solver.fit(tol=1.0e-16)
    soln = p.primal
    t2 = time.time()
    print t2 - t1
    st = np.maximum(np.fabs(x) - l1, 0) * np.sign(x)

    print soln[range(10)]
    print st[range(10)]
    np.testing.assert_almost_equal(soln, st, decimal=3)
Пример #20
0
def test_logistic_offset():
    """
    Test the equivalence of binary/count specification in logistic_likelihood
    """

    #Form the count version of the problem
    trials = np.random.binomial(5, 0.5, 10) + 1
    successes = np.random.binomial(trials, 0.5, len(trials))
    n = len(successes)
    p = 2 * n

    X = np.hstack(
        [np.ones((n, 1)),
         np.random.normal(0, 1, n * p).reshape((n, p))])

    loss = rr.logistic_loglike.linear(X, successes=successes, trials=trials)
    weights = np.ones(p + 1)
    weights[0] = 0.
    penalty = rr.quadratic_loss.linear(weights, coef=.1, diag=True)

    prob1 = rr.container(loss, penalty)
    solver1 = rr.FISTA(prob1)
    vals = solver1.fit(tol=1e-12)
    solution1 = solver1.composite.coefs

    diff = 0.1

    loss = rr.logistic_loglike.affine(X,
                                      successes=successes,
                                      trials=trials,
                                      offset=diff * np.ones(n))
    weights = np.ones(p + 1)
    weights[0] = 0.
    penalty = rr.quadratic_loss.linear(weights, coef=.1, diag=True)

    prob2 = rr.container(loss, penalty)
    solver2 = rr.FISTA(prob2)
    vals = solver2.fit(tol=1e-12)
    solution2 = solver2.composite.coefs

    ind = np.arange(1, p + 1)

    print(solution1[np.arange(5)])
    print(solution2[np.arange(5)])

    npt.assert_array_almost_equal(solution1[ind], solution2[ind], 3)
    npt.assert_almost_equal(solution1[0] - diff, solution2[0], 2)
Пример #21
0
def example3(lambda1=10):

    #Example using a smooth approximation to the non-negativity constraint
    # On large problems this might be faster than using the actual constraint

    n = 100
    p = 1000


    X = np.random.standard_normal(n*p).reshape((n,p))
    Y = 10*np.random.standard_normal(n)

    loss = rr.l2normsq.affine(X,-Y,coef=1.)
    sparsity = rr.l1norm(p, lagrange = lambda1)
    nonnegative = rr.nonnegative(p)
    smooth_nonnegative = rr.smoothed_atom(nonnegative, epsilon = 1e-4)

    problem = rr.container(loss, sparsity, smooth_nonnegative)
    solver = rr.FISTA(problem)
    solver.fit(debug=True)

    solution1 = solver.composite.coefs



    loss = rr.l2normsq.affine(X,-Y,coef=1.)
    sparsity = rr.l1norm(p, lagrange = lambda1)
    nonnegative = rr.nonnegative(p)

    problem = rr.container(loss, sparsity, nonnegative)
    solver = rr.FISTA(problem)
    solver.fit(debug=True)

    solution2 = solver.composite.coefs


    pl.subplot(1,2,1)
    pl.hist(solution1, bins=40)

    pl.subplot(1,2,2)
    pl.scatter(solution2,solution1)
    pl.xlabel("Constraint")
    pl.ylabel("Smooth constraint")
Пример #22
0
def lasso_via_dual_split():

    def selector(p, slice):
        return np.identity(p)[slice]
    penalties = [R.l1norm(selector(500, slice(i*100,(i+1)*100)), lagrange=0.2) for i in range(5)]
    x = np.random.standard_normal(500)
    loss = R.quadratic.shift(-x, coef=0.5)
    lasso = R.container(loss,*penalties)
    solver = R.FISTA(lasso)
    np.testing.assert_almost_equal(np.maximum(np.fabs(x)-0.2, 0) * np.sign(x), solver.composite.coefs, decimal=3)
Пример #23
0
def test_logistic_counts():
    """
    Test the equivalence of binary/count specification in logistic_deviance
    """

    #Form the count version of the problem
    trials = np.random.binomial(5,0.5,100)+1
    successes = np.random.binomial(trials,0.5,len(trials)) 
    n = len(successes)
    p = 2*n
    X = np.random.normal(0,1,n*p).reshape((n,p))

    loss = rr.logistic_deviance.linear(X, successes=successes, trials=trials)
    penalty = rr.quadratic(p, coef=1.)

    prob1 = rr.container(loss, penalty)
    solver1 = rr.FISTA(prob1)
    solver1.fit()
    solution1 = solver1.composite.coefs
    
    #Form the binary version of the problem
    Ynew = []
    Xnew = []

    for i, (s,n) in enumerate(zip(successes,trials)):
        Ynew.append([1]*s + [0]*(n-s))
        for j in range(n):
            Xnew.append(X[i,:])
    Ynew = np.hstack(Ynew)
    Xnew =  np.vstack(Xnew)


    loss = rr.logistic_deviance.linear(Xnew, successes=Ynew)
    penalty = rr.quadratic(p, coef=1.)

    prob2 = rr.container(loss, penalty)
    solver2 = rr.FISTA(prob2)
    solver2.fit()
    solution2 = solver2.composite.coefs

   
    npt.assert_array_almost_equal(solution1, solution2, 3)
Пример #24
0
def test_logistic_offset():
    """
    Test the equivalence of binary/count specification in logistic_likelihood
    """

    #Form the count version of the problem
    trials = np.random.binomial(5,0.5,10)+1
    successes = np.random.binomial(trials,0.5,len(trials)) 
    n = len(successes)
    p = 2*n

    X = np.hstack([np.ones((n,1)),np.random.normal(0,1,n*p).reshape((n,p))])

    loss = rr.logistic_loglike.linear(X, successes=successes, trials=trials)
    weights = np.ones(p+1)
    weights[0] = 0.
    penalty = rr.quadratic_loss.linear(weights, coef=.1, diag=True)

    prob1 = rr.container(loss, penalty)
    solver1 = rr.FISTA(prob1)
    vals = solver1.fit(tol=1e-12)
    solution1 = solver1.composite.coefs

    diff = 0.1

    loss = rr.logistic_loglike.affine(X, successes=successes, trials=trials, offset = diff*np.ones(n))
    weights = np.ones(p+1)
    weights[0] = 0.
    penalty = rr.quadratic_loss.linear(weights, coef=.1, diag=True)

    prob2 = rr.container(loss, penalty)
    solver2 = rr.FISTA(prob2)
    vals = solver2.fit(tol=1e-12)
    solution2 = solver2.composite.coefs

    ind = range(1,p+1)

    print(solution1[range(5)])
    print(solution2[range(5)])

    npt.assert_array_almost_equal(solution1[ind], solution2[ind], 3)
    npt.assert_almost_equal(solution1[0]-diff,solution2[0], 2)
Пример #25
0
def test_logistic_counts():
    """
    Test the equivalence of binary/count specification in logistic_deviance
    """

    #Form the count version of the problem
    trials = np.random.binomial(5,0.5,100)+1
    successes = np.random.binomial(trials,0.5,len(trials)) 
    n = len(successes)
    p = 2*n
    X = np.random.normal(0,1,n*p).reshape((n,p))

    loss = rr.logistic_deviance.linear(X, successes=successes, trials=trials)
    penalty = rr.quadratic(p, coef=1.)

    prob1 = rr.container(loss, penalty)
    solver1 = rr.FISTA(prob1)
    solver1.fit()
    solution1 = solver1.composite.coefs
    
    #Form the binary version of the problem
    Ynew = []
    Xnew = []

    for i, (s,n) in enumerate(zip(successes,trials)):
        Ynew.append([1]*s + [0]*(n-s))
        for j in range(n):
            Xnew.append(X[i,:])
    Ynew = np.hstack(Ynew)
    Xnew =  np.vstack(Xnew)


    loss = rr.logistic_deviance.linear(Xnew, successes=Ynew)
    penalty = rr.quadratic(p, coef=1.)

    prob2 = rr.container(loss, penalty)
    solver2 = rr.FISTA(prob2)
    solver2.fit()
    solution2 = solver2.composite.coefs

   
    npt.assert_array_almost_equal(solution1, solution2, 3)
Пример #26
0
def group_lasso_signal_approx():

    def selector(p, slice):
        return np.identity(p)[slice]
    penalties = [R.l2norm(selector(500, slice(i*100,(i+1)*100)), lagrange=10.) for i in range(5)]
    loss = R.quadratic.shift(-x, coef=0.5)
    group_lasso = R.container(loss, **penalties)
    x = np.random.standard_normal(500)
    solver = R.FISTA(group_lasso)
    solver.fit()
    a = solver.composite.coefs
Пример #27
0
    def __init__(self, X, initial=None, lagrange=1, rho=1):
        self.X = R.affine_transform(X, None)
        self.atom = R.l1norm(X.shape[1], l)
        self.rho = rho
        self.loss = R.quadratic.affine(X, -np.zeros(X.shape[0]), lagrange=rho/2.)
        self.lasso = R.container(self.loss, self.atom)
        self.solver = R.FISTA(self.lasso.problem())

        if initial is None:
            self.beta[:] = np.random.standard_normal(self.atom.primal_shape)
        else:
            self.beta[:] = initial
Пример #28
0
    def test_container(self):
        tests = []
        atom, q, prox_center, L = self.atom, self.q, self.prox_center, self.L
        loss = self.loss

        problem = rr.container(loss, atom)
        solver = rr.FISTA(problem)
        solver.fit(tol=1.0e-12, 
                   coef_stop=self.coef_stop, FISTA=self.FISTA, min_its=100)

        tests.append((atom.proximal(q), solver.composite.coefs, 'solving atom prox with container\n %s ' % str(self)))

        # write the loss in terms of a quadratic for the smooth loss and a smooth function...

        q = rr.identity_quadratic(L, prox_center, 0, 0)
        lossq = rr.quadratic.shift(prox_center.copy(), coef=0.6*L)
        lossq.quadratic = rr.identity_quadratic(0.4*L, prox_center.copy(), 0, 0)
        problem = rr.container(lossq, atom)
        solver = rr.FISTA(problem)
        solver.fit(tol=1.0e-12, FISTA=self.FISTA, coef_stop=self.coef_stop)

        tests.append((atom.proximal(q), 
                      problem.solve(tol=1.e-12,FISTA=self.FISTA,coef_stop=self.coef_stop), 
                      'solving prox with container with monotonicity ' + 
                      'but loss has identity_quadratic\n %s ' % str(self)))

        d = atom.conjugate
        problem = rr.container(d, loss)
        solver = rr.FISTA(problem)
        solver.fit(tol=1.0e-12, 
                   coef_stop=self.coef_stop, FISTA=self.FISTA, min_its=100)
        tests.append((d.proximal(q), solver.composite.coefs, 'solving dual prox with container\n %s ' % str(self)))

        if not self.interactive:
            for test in tests:
                yield (all_close,) + test + (self,)
        else:
            for test in tests:
                yield all_close(*((test + (self,))))
Пример #29
0
    def __init__(self, X, initial=None, lagrange=1, rho=1):
        self.X = R.affine_transform(X, None)
        self.atom = R.l1norm(X.shape[1], l)
        self.rho = rho
        self.loss = R.quadratic.affine(X,
                                       -np.zeros(X.shape[0]),
                                       lagrange=rho / 2.)
        self.lasso = R.container(self.loss, self.atom)
        self.solver = R.FISTA(self.lasso.problem())

        if initial is None:
            self.beta[:] = np.random.standard_normal(self.atom.primal_shape)
        else:
            self.beta[:] = initial
Пример #30
0
def test_1d_fused_lasso():

    """
    Check the 1d fused lasso solution using an equivalent lasso formulation
    """

    n = 100
    l1 = 1.
    
    D = (np.identity(n) - np.diag(np.ones(n-1),-1))[1:]
    extra = np.zeros(n)
    extra[0] = 1.
    D = np.vstack([D,extra])
    D = sparse.csr_matrix(D)

    fused = R.l1norm.linear(D, lagrange=l1)
    
    X = np.random.standard_normal((2*n,n))
    Y = np.random.standard_normal((2*n,))
    loss = R.quadratic.affine(X, -Y, coef=0.5)
    fused_lasso = R.container(loss, fused)
    solver=R.FISTA(fused_lasso)
    vals1 = solver.fit(max_its=25000, tol=1e-10)
    soln1 = solver.composite.coefs
    
    B = np.array(sparse.tril(np.ones((n,n))).todense())
    X2 = np.dot(X,B)
    
    loss = R.quadratic.affine(X2, -Y, coef=0.5)
    sparsity = R.l1norm(n, lagrange=l1)
    lasso = R.container(loss, sparsity)
    solver = R.FISTA(lasso)
    solver.fit(tol=1e-10)

    soln2 = np.dot(B, solver.composite.coefs)

    npt.assert_array_almost_equal(soln1, soln2, 3)
Пример #31
0
def test_1d_fused_lasso():
    """
    Check the 1d fused lasso solution using an equivalent lasso formulation
    """

    n = 100
    l1 = 1.

    D = (np.identity(n) - np.diag(np.ones(n - 1), -1))[1:]
    extra = np.zeros(n)
    extra[0] = 1.
    D = np.vstack([D, extra])
    D = sparse.csr_matrix(D)

    fused = R.l1norm.linear(D, lagrange=l1)

    X = np.random.standard_normal((2 * n, n))
    Y = np.random.standard_normal((2 * n, ))
    loss = R.quadratic.affine(X, -Y, coef=0.5)
    fused_lasso = R.container(loss, fused)
    solver = R.FISTA(fused_lasso)
    vals1 = solver.fit(max_its=25000, tol=1e-10)
    soln1 = solver.composite.coefs

    B = np.array(sparse.tril(np.ones((n, n))).todense())
    X2 = np.dot(X, B)

    loss = R.quadratic.affine(X2, -Y, coef=0.5)
    sparsity = R.l1norm(n, lagrange=l1)
    lasso = R.container(loss, sparsity)
    solver = R.FISTA(lasso)
    solver.fit(tol=1e-10)

    soln2 = np.dot(B, solver.composite.coefs)

    npt.assert_array_almost_equal(soln1, soln2, 3)
Пример #32
0
def test_lasso_via_dual_split():
    """
    Test the lasso by breaking it up into multiple l1 atoms over the range of beta
    """

    def selector(p, slice):
        return np.identity(p)[slice]
    
    penalties = [R.l1norm.linear(selector(500, slice(i*100,(i+1)*100)), lagrange=0.2) for i in range(5)]
    x = np.random.standard_normal(500)
    loss = R.quadratic.shift(-x, coef=0.5)
    lasso = R.container(loss,*penalties)
    solver = R.FISTA(lasso)
    solver.fit(tol=1e-8)

    npt.assert_array_almost_equal(np.maximum(np.fabs(x)-0.2, 0) * np.sign(x), solver.composite.coefs, 3)
Пример #33
0
def test_admm_l1_seminorm():
    """
    Test ADMM using the l1norm in lagrange form
    """
    p = 1000
    Y = 10 * np.random.normal(0,1,p)

    loss = R.quadratic.shift(-Y, coef=0.5)
    sparsity = R.l1norm(p, lagrange=5.)

    prob = R.container(loss, sparsity)

    solver = R.admm_problem(prob)
    solver.fit(debug=False, tol=1e-12)
    solution = solver.beta

    npt.assert_array_almost_equal(solution, np.maximum(np.fabs(Y) - sparsity.lagrange,0.)*np.sign(Y), 3)
Пример #34
0
def test_admm_l1_constraint():
    """
    Test ADMM using the l1norm in bound form
    """

    p = 1000
    Y = 10 * np.random.normal(0, 1, p)

    loss = R.linear(Y, coef=0.5)
    sparsity = R.l1norm(p, bound=5.)
    sparsity.bound *= 1.

    prob = R.container(loss, sparsity)

    solver = R.admm_problem(prob)
    solver.fit(debug=False, tol=1e-12)
    solution = solver.beta

    npt.assert_almost_equal(np.fabs(solution).sum(), sparsity.bound, 3)
Пример #35
0
def test_multiple_lasso():
    """
    Check that the solution of the lasso signal approximator dual problem is soft-thresholding even when specified with multiple seminorms
    """

    p = 1000

    l1 = 2
    sparsity1 = R.l1norm(p, lagrange=l1 * 0.75)
    sparsity2 = R.l1norm(p, lagrange=l1 * 0.25)
    x = np.random.normal(0, 1, p)
    loss = R.quadratic.shift(-x, coef=0.5)
    p = R.container(loss, sparsity1, sparsity2)
    solver = R.FISTA(p)
    vals = solver.fit(tol=1.0e-10)
    soln = solver.composite.coefs
    st = np.maximum(np.fabs(x) - l1, 0) * np.sign(x)

    npt.assert_array_almost_equal(soln, st, 3)
Пример #36
0
def test_admm_l1_seminorm():
    """
    Test ADMM using the l1norm in lagrange form
    """
    p = 1000
    Y = 10 * np.random.normal(0, 1, p)

    loss = R.quadratic.shift(-Y, coef=0.5)
    sparsity = R.l1norm(p, lagrange=5.)

    prob = R.container(loss, sparsity)

    solver = R.admm_problem(prob)
    solver.fit(debug=False, tol=1e-12)
    solution = solver.beta

    npt.assert_array_almost_equal(
        solution,
        np.maximum(np.fabs(Y) - sparsity.lagrange, 0.) * np.sign(Y), 3)
Пример #37
0
def example2(lambda1=10):

    #Example with a non-identity X

    n = 100
    p = 1000

    X = np.random.standard_normal(n*p).reshape((n,p))
    Y = 10*np.random.standard_normal(n)

    loss = rr.l2normsq.affine(X,-Y,coef=1.)
    sparsity = rr.l1norm(p, lagrange = lambda1)
    nonnegative = rr.nonnegative(p)

    problem = rr.container(loss, sparsity, nonnegative)
    solver = rr.FISTA(problem)
    solver.fit(debug=True)

    solution = solver.composite.coefs
Пример #38
0
def test_admm_l1_constraint():
    """
    Test ADMM using the l1norm in bound form
    """
  
    p = 1000
    Y = 10 * np.random.normal(0,1,p)

    loss = R.linear(Y, coef=0.5)
    sparsity = R.l1norm(p, bound=5.)
    sparsity.bound *= 1.

    prob = R.container(loss, sparsity)

    solver = R.admm_problem(prob)
    solver.fit(debug=False, tol=1e-12)
    solution = solver.beta

    npt.assert_almost_equal(np.fabs(solution).sum(), sparsity.bound, 3)
Пример #39
0
def test_lasso_via_dual_split():
    """
    Test the lasso by breaking it up into multiple l1 atoms over the range of beta
    """
    def selector(p, slice):
        return np.identity(p)[slice]

    penalties = [
        R.l1norm.linear(selector(500, slice(i * 100, (i + 1) * 100)),
                        lagrange=0.2) for i in range(5)
    ]
    x = np.random.standard_normal(500)
    loss = R.quadratic.shift(-x, coef=0.5)
    lasso = R.container(loss, *penalties)
    solver = R.FISTA(lasso)
    solver.fit(tol=1e-8)

    npt.assert_array_almost_equal(
        np.maximum(np.fabs(x) - 0.2, 0) * np.sign(x), solver.composite.coefs,
        3)
Пример #40
0
def test_l1_seminorm():
    """
    Test using the l1norm in lagrange form
    """


    p = 1000
    Y = 10 * np.random.normal(0,1,p)

    loss = R.quadratic.shift(-Y, coef=0.5)
    sparsity = R.l1norm(p, lagrange=5.)
    sparsity.lagrange *= 1.

    prob = R.container(loss, sparsity)
    problem = prob

    solver = R.FISTA(problem)
    vals = solver.fit(tol=1e-10, max_its=500)
    solution = solver.composite.coefs

    npt.assert_array_almost_equal(solution, np.maximum(np.fabs(Y) - sparsity.lagrange,0.)*np.sign(Y), 3)
Пример #41
0
def test_multiple_lasso():

    """
    Check that the solution of the lasso signal approximator dual problem is soft-thresholding even when specified with multiple seminorms
    """
    
    
    p = 1000
    
    l1 = 2
    sparsity1 = R.l1norm(p, lagrange=l1*0.75)
    sparsity2 = R.l1norm(p, lagrange=l1*0.25)
    x = np.random.normal(0,1,p)
    loss = R.quadratic.shift(-x, coef=0.5)
    p = R.container(loss, sparsity1, sparsity2)
    solver = R.FISTA(p)
    vals = solver.fit(tol=1.0e-10)
    soln = solver.composite.coefs
    st = np.maximum(np.fabs(x)-l1,0) * np.sign(x)
    
    npt.assert_array_almost_equal(soln, st, 3)
Пример #42
0
def example1(lambda1=10):

    #Example with X = np.identity(n)
    #Try varying lambda1 to see shrinkage

    n = 100

    Y = 10*np.random.standard_normal(n)

    loss = rr.l2normsq.shift(-Y,coef=1.)
    sparsity = rr.l1norm(n, lagrange = lambda1)
    nonnegative = rr.nonnegative(n)

    problem = rr.container(loss, sparsity, nonnegative)
    solver = rr.FISTA(problem)
    solver.fit(debug=True)

    solution = solver.composite.coefs

    pl.plot(Y, color='red', label='Y')
    pl.plot(solution, color='blue', label='beta')
    pl.legend()
Пример #43
0
def test_l1_seminorm():
    """
    Test using the l1norm in lagrange form
    """

    p = 1000
    Y = 10 * np.random.normal(0, 1, p)

    loss = R.quadratic.shift(-Y, coef=0.5)
    sparsity = R.l1norm(p, lagrange=5.)
    sparsity.lagrange *= 1.

    prob = R.container(loss, sparsity)
    problem = prob

    solver = R.FISTA(problem)
    vals = solver.fit(tol=1e-10, max_its=500)
    solution = solver.composite.coefs

    npt.assert_array_almost_equal(
        solution,
        np.maximum(np.fabs(Y) - sparsity.lagrange, 0.) * np.sign(Y), 3)
Пример #44
0
def test_lasso_dual_from_primal(l1=.1, L=2.):
    """
    Check that the solution of the lasso signal approximator dual composite is soft-thresholding, when call from primal composite object
    """

    sparsity = R.l1norm(500, lagrange=l1)
    x = np.random.normal(0, 1, 500)
    y = np.random.normal(0, 1, 500)

    X = np.random.standard_normal((1000, 500))
    Y = np.random.standard_normal((1000, ))
    regloss = R.quadratic.affine(-X, Y)
    p = R.container(regloss, sparsity)

    z = x - y / L
    soln = p.proximal(R.identity_quadratic(L, z, 0, 0))
    st = np.maximum(np.fabs(z) - l1 / L, 0) * np.sign(z)

    print x[range(10)]
    print soln[range(10)]
    print st[range(10)]
    np.testing.assert_almost_equal(soln, st, decimal=3)
Пример #45
0
def test_lasso_dual_from_primal(l1 = .1, L = 2.):

    """
    Check that the solution of the lasso signal approximator dual composite is soft-thresholding, when call from primal composite object
    """

    sparsity = R.l1norm(500, lagrange=l1)
    x = np.random.normal(0,1,500)
    y = np.random.normal(0,1,500)

    X = np.random.standard_normal((1000,500))
    Y = np.random.standard_normal((1000,))
    regloss = R.quadratic.affine(-X,Y)
    p= R.container(regloss, sparsity)

    z = x - y/L
    soln = p.proximal(R.identity_quadratic(L,z,0,0))
    st = np.maximum(np.fabs(z)-l1/L,0) * np.sign(z)

    print x[range(10)]
    print soln[range(10)]
    print st[range(10)]
    np.testing.assert_almost_equal(soln,st, decimal=3)
Пример #46
0
def lasso_example():

    l1 = 20.
    sparsity = R.l1norm(500, lagrange=l1/2.)
    X = np.random.standard_normal((1000,500))
    Y = np.random.standard_normal((1000,))
    regloss = R.quadratic.affine(X,-Y, coef=0.5)
    sparsity2 = R.l1norm(500, lagrange=l1/2.)
    p=R.container(regloss, sparsity, sparsity2)
    solver=R.FISTA(p)
    solver.debug = True
    vals = solver.fit(max_its=2000, min_its = 100)
    soln = solver.composite.coefs

    # solution
    pylab.figure(num=1)
    pylab.clf()
    pylab.plot(soln, c='g')

    # objective values
    pylab.figure(num=2)
    pylab.clf()
    pylab.plot(vals)
Пример #47
0
def test_1d_fused_lasso(n=100):

    l1 = 1.

    sparsity1 = R.l1norm(n, lagrange=l1)
    D = (np.identity(n) - np.diag(np.ones(n-1),-1))[1:]
    extra = np.zeros(n)
    extra[0] = 1.
    D = np.vstack([D,extra])
    D = sparse.csr_matrix(D)

    fused = R.l1norm.linear(D, lagrange=l1)

    X = np.random.standard_normal((2*n,n))
    Y = np.random.standard_normal((2*n,))
    loss = R.quadratic.affine(X, -Y, coef=0.5)
    fused_lasso = R.container(loss, fused)
    solver=R.FISTA(fused_lasso)
    solver.debug = True
    vals1 = solver.fit(max_its=25000, tol=1e-12)
    soln1 = solver.composite.coefs

    B = np.array(sparse.tril(np.ones((n,n))).todense())
    X2 = np.dot(X,B)
Пример #48
0
def test_multiple_lasso_dual(n=500):

    """
    Check that the solution of the lasso signal approximator dual composite is soft-thresholding even when specified with multiple seminorms
    """

    l1 = 1
    sparsity1 = R.l1norm(n, lagrange=l1*0.75)
    sparsity2 = R.l1norm(n, lagrange=l1*0.25)
    x = np.random.normal(0,1,n)
    loss = R.quadratic.shift(-x, coef=0.5)

    p = R.dual_problem.fromprimal(loss, sparsity1, sparsity2)
    t1 = time.time()
    solver = R.FISTA(p)
    solver.debug = True
    vals = solver.fit(tol=1.0e-16)
    soln = p.primal
    t2 = time.time()
    print t2-t1
    st = np.maximum(np.fabs(x)-l1,0) * np.sign(x)
    np.testing.assert_almost_equal(soln,st, decimal=3)

    p = R.container(loss, sparsity1, sparsity2)
    t1 = time.time()
    solver = R.FISTA(p)
    solver.debug = True
    vals = solver.fit(tol=1.0e-16)
    soln = p.primal
    t2 = time.time()
    print t2-t1
    st = np.maximum(np.fabs(x)-l1,0) * np.sign(x)

    print soln[range(10)]
    print st[range(10)]
    np.testing.assert_almost_equal(soln,st, decimal=3)
Пример #49
0
def test_quadratic_for_smooth():
    '''
    this test is a check to ensure that the quadratic part 
    of the smooth functions are being used in the proximal step
    '''

    L = 0.45

    W = np.random.standard_normal(40)
    Z = np.random.standard_normal(40)
    U = np.random.standard_normal(40)

    atomq = rr.identity_quadratic(0.4, U, W, 0)
    atom = rr.l1norm(40, quadratic=atomq, lagrange=0.12)

    # specifying in this way should be the same as if we put 0.5*L below
    loss = rr.quadratic.shift(Z, coef=0.6 * L)
    lq = rr.identity_quadratic(0.4 * L, Z, 0, 0)
    loss.quadratic = lq

    ww = np.random.standard_normal(40)

    # specifying in this way should be the same as if we put 0.5*L below
    loss2 = rr.quadratic.shift(Z, coef=L)
    yield all_close, loss2.objective(ww), loss.objective(
        ww), 'checking objective', None

    yield all_close, lq.objective(ww, 'func'), loss.nonsmooth_objective(
        ww), 'checking nonsmooth objective', None
    yield all_close, loss2.smooth_objective(
        ww, 'func'), 0.5 / 0.3 * loss.smooth_objective(
            ww, 'func'), 'checking smooth objective func', None
    yield all_close, loss2.smooth_objective(
        ww, 'grad'), 0.5 / 0.3 * loss.smooth_objective(
            ww, 'grad'), 'checking smooth objective grad', None

    problem = rr.container(loss, atom)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-12)

    problem3 = rr.simple_problem(loss, atom)
    solver3 = rr.FISTA(problem3)
    solver3.fit(tol=1.0e-12, coef_stop=True)

    loss4 = rr.quadratic.shift(Z, coef=0.6 * L)
    problem4 = rr.simple_problem(loss4, atom)
    problem4.quadratic = lq
    solver4 = rr.FISTA(problem4)
    solver4.fit(tol=1.0e-12)

    gg_soln = rr.gengrad(problem, L)

    loss6 = rr.quadratic.shift(Z, coef=0.6 * L)
    loss6.quadratic = lq + atom.quadratic
    atomcp = copy(atom)
    atomcp.quadratic = rr.identity_quadratic(0, 0, 0, 0)
    problem6 = rr.dual_problem(loss6.conjugate, rr.identity(loss6.shape),
                               atomcp.conjugate)
    problem6.lipschitz = L + atom.quadratic.coef
    dsoln2 = problem6.solve(coef_stop=True, tol=1.e-10, max_its=100)

    problem2 = rr.container(loss2, atom)
    solver2 = rr.FISTA(problem2)
    solver2.fit(tol=1.0e-12, coef_stop=True)

    q = rr.identity_quadratic(L, Z, 0, 0)

    yield all_close, problem.objective(
        ww), atom.nonsmooth_objective(ww) + q.objective(ww, 'func'), '', None

    atom = rr.l1norm(40, quadratic=atomq, lagrange=0.12)
    aq = atom.solve(q)
    for p, msg in zip([
            solver3.composite.coefs, gg_soln, solver2.composite.coefs, dsoln2,
            solver.composite.coefs, solver4.composite.coefs
    ], [
            'simple_problem with loss having no quadratic', 'gen grad',
            'container with loss having no quadratic',
            'dual problem with loss having a quadratic',
            'container with loss having a quadratic',
            'simple_problem having a quadratic'
    ]):
        yield all_close, aq, p, msg, None
Пример #50
0
def solveit(atom, Z, W, U, linq, L, FISTA, coef_stop):

    p2 = copy(atom)
    p2.quadratic = rr.identity_quadratic(L, Z, 0, 0)

    d = atom.conjugate

    q = rr.identity_quadratic(1, Z, 0, 0)
    yield ac, Z - atom.proximal(q), d.proximal(
        q), 'testing duality of projections starting from atom %s ' % atom
    q = rr.identity_quadratic(L, Z, 0, 0)

    # use simple_problem.nonsmooth

    p2 = copy(atom)
    p2.quadratic = atom.quadratic + q
    problem = rr.simple_problem.nonsmooth(p2)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-14, FISTA=FISTA, coef_stop=coef_stop)

    yield ac, atom.proximal(
        q
    ), solver.composite.coefs, 'solving prox with simple_problem.nonsmooth with monotonicity %s ' % atom

    # use the solve method

    p2.coefs *= 0
    p2.quadratic = atom.quadratic + q
    soln = p2.solve()

    yield ac, atom.proximal(
        q), soln, 'solving prox with solve method %s ' % atom

    loss = rr.quadratic.shift(-Z, coef=L)
    problem = rr.simple_problem(loss, atom)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-12, FISTA=FISTA, coef_stop=coef_stop)

    yield ac, atom.proximal(
        q
    ), solver.composite.coefs, 'solving prox with simple_problem with monotonicity %s ' % atom

    dproblem2 = rr.dual_problem(loss.conjugate, rr.identity(loss.shape),
                                atom.conjugate)
    dcoef2 = dproblem2.solve(coef_stop=coef_stop, tol=1.e-14)
    yield ac, atom.proximal(
        q
    ), dcoef2, 'solving prox with dual_problem with monotonicity %s ' % atom

    dproblem = rr.dual_problem.fromprimal(loss, atom)
    dcoef = dproblem.solve(coef_stop=coef_stop, tol=1.0e-14)
    yield ac, atom.proximal(
        q
    ), dcoef, 'solving prox with dual_problem.fromprimal with monotonicity %s ' % atom

    # write the loss in terms of a quadratic for the smooth loss and a smooth function...

    lossq = rr.quadratic.shift(-Z, coef=0.6 * L)
    lossq.quadratic = rr.identity_quadratic(0.4 * L, Z, 0, 0)
    problem = rr.simple_problem(lossq, atom)

    yield ac, atom.proximal(q), problem.solve(
        coef_stop=coef_stop, FISTA=FISTA, tol=1.0e-12
    ), 'solving prox with simple_problem with monotonicity  but loss has identity_quadratic %s ' % atom

    problem = rr.simple_problem.nonsmooth(p2)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-14,
               monotonicity_restart=False,
               coef_stop=coef_stop,
               FISTA=FISTA)

    yield ac, atom.proximal(
        q
    ), solver.composite.coefs, 'solving prox with simple_problem.nonsmooth with no monotonocity %s ' % atom

    loss = rr.quadratic.shift(-Z, coef=L)
    problem = rr.simple_problem(loss, atom)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-12,
               monotonicity_restart=False,
               coef_stop=coef_stop,
               FISTA=FISTA)

    yield ac, atom.proximal(
        q
    ), solver.composite.coefs, 'solving prox with simple_problem %s no monotonicity_restart' % atom

    loss = rr.quadratic.shift(-Z, coef=L)
    problem = rr.separable_problem.singleton(atom, loss)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-12, coef_stop=coef_stop, FISTA=FISTA)

    yield ac, atom.proximal(
        q
    ), solver.composite.coefs, 'solving atom prox with separable_atom.singleton %s ' % atom

    loss = rr.quadratic.shift(-Z, coef=L)
    problem = rr.container(loss, atom)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-12, coef_stop=coef_stop, FISTA=FISTA)

    yield ac, atom.proximal(
        q
    ), solver.composite.coefs, 'solving atom prox with container %s ' % atom

    # write the loss in terms of a quadratic for the smooth loss and a smooth function...

    lossq = rr.quadratic.shift(-Z, coef=0.6 * L)
    lossq.quadratic = rr.identity_quadratic(0.4 * L, Z, 0, 0)
    problem = rr.container(lossq, atom)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-12, FISTA=FISTA, coef_stop=coef_stop)

    yield (
        ac, atom.proximal(q),
        problem.solve(tol=1.e-12, FISTA=FISTA, coef_stop=coef_stop),
        'solving prox with container with monotonicity  but loss has identity_quadratic %s '
        % atom)

    loss = rr.quadratic.shift(-Z, coef=L)
    problem = rr.simple_problem(loss, d)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-12,
               monotonicity_restart=False,
               coef_stop=coef_stop,
               FISTA=FISTA)
    # ac(d.proximal(q), solver.composite.coefs, 'solving dual prox with simple_problem no monotonocity %s ' % atom)
    yield (ac, d.proximal(q),
           problem.solve(tol=1.e-12,
                         FISTA=FISTA,
                         coef_stop=coef_stop,
                         monotonicity_restart=False),
           'solving dual prox with simple_problem no monotonocity %s ' % atom)

    problem = rr.container(d, loss)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-12, coef_stop=coef_stop, FISTA=FISTA)
    yield ac, d.proximal(
        q
    ), solver.composite.coefs, 'solving dual prox with container %s ' % atom

    loss = rr.quadratic.shift(-Z, coef=L)
    problem = rr.separable_problem.singleton(d, loss)
    solver = rr.FISTA(problem)
    solver.fit(tol=1.0e-12, coef_stop=coef_stop, FISTA=FISTA)

    yield ac, d.proximal(
        q
    ), solver.composite.coefs, 'solving atom prox with separable_atom.singleton %s ' % atom
Пример #51
0
import numpy as np
import pylab
from scipy import sparse
import regreg.api as R

Y = np.random.standard_normal(500)
Y[100:150] += 7
Y[250:300] += 14
loss = R.quadratic.shift(-Y, coef=0.5)

sparsity = R.l1norm(len(Y), lagrange=1.4)
# TODO should make a module to compute typical Ds
D = sparse.csr_matrix((np.identity(500) + np.diag([-1] * 499, k=1))[:-1])
fused = R.l1norm.linear(D, lagrange=25.5)
problem = R.container(loss, sparsity, fused)

solver = R.FISTA(problem)
solver.fit(max_its=100, tol=1e-10)
solution = solver.composite.coefs

delta = np.fabs(D * solution).sum()
sparsity = R.l1norm(len(Y), lagrange=1.4)
fused_constraint = R.l1norm.linear(D, bound=delta)
constrained_problem = R.container(loss, fused_constraint, sparsity)
constrained_solver = R.FISTA(constrained_problem)
constrained_solver.composite.lipschitz = 1.01
vals = constrained_solver.fit(max_its=10,
                              tol=1e-06,
                              backtrack=False,
                              monotonicity_restart=False)
constrained_solution = constrained_solver.composite.coefs