Beispiel #1
0
def test_reclasso(n=5, m=5, mu0=.5, mu1=.8, thresh=1e-4, verbose=True):  
    """
    test the reclasso algorithm using random data
    """
    
    # sample a problem
    X = np.random.randn(n+1, m)
    y = np.random.randn(n+1, 1)
    
    # solve both problems using an interior point method
    theta0 = algo.interior_point(X[:-1, :], y[:-1], mu0)
    theta1 = algo.interior_point(X, y, mu1)
    
    # prepare warm-start solution for the homotopy 
    theta0, nz0, K0, truesol0 = algo.fix_sol(X[:-1, :], y[:-1], mu0, theta0, thresh=thresh)
    theta1, nz1, K1, truesol1 = algo.fix_sol(X, y, mu1, theta1, thresh=thresh)
    
    if not truesol0 or not truesol1: raise NameError, "bad threshold for interior point solution"
    
    # solve the problem using reclasso
    theta_nz, nz, K, nbr1, nbr2 = reclasso(X, y, mu0, mu1, theta0[nz0], nz0, K0, verbose=verbose, showpath=False)
    theta = np.zeros((m, 1))
    theta[nz] = theta_nz
    
    # check the result is the same as with the interior point method
    error = np.sum((theta1 - theta)**2)/np.sum(theta1**2)
    
    assert_almost_equals(error, 0)
Beispiel #2
0
def compare_algos(n=250, m=100, d=25, N=10, mu=.1, sigma=0.01, verbose=True, showfig=False, simuname='test'):
    """
    batch experiment
    compare performance of reclasso, lars, and coordinate descent with warm start
    """
    if not showfig:
        import matplotlib
        matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    from matplotlib.patches import Polygon
    
    # sample observations
    thetaopt = np.zeros((m, 1))
    thetaopt[:d] = 1.
    
    # regularization parameter schedule
    lam = mu * np.arange(1, n+1)
    
    # create arrays to store the results
    br1 = np.empty((n-1, N))            # nb of transition points in the first step
    br2 = np.empty((n-1, N))            # nb of transition points in the second step
    brlars = np.empty((n-1, N))         # nb of transition points in lars
    err_hamming = np.empty((n-1, N))    # hamming distance between the current solution and hte optimal solution
    err_l2 = np.empty((n-1, N))         # l2 distance between the current solution and hte optimal solution
    t_h = np.empty((n-1, N))            # timing for the homorotpy
    t_lars = np.empty((n-1, N))         # timing for lars
    t_cd = np.empty((n-m/2, N))         # timing for coordinate descent + warm-start
    
    for j in range(N):
        
        # sample data
        X = np.random.randn(n, m)
        y = np.dot(X, thetaopt) + sigma*np.sqrt(d)*np.random.randn(n,1)
        
        # get first solution
        theta_nz, nz, K = algo.one_observation(X[0,:], y[0], lam[0])
        theta = np.zeros((m, 1))
        theta[nz] = theta_nz
        
        for i in range(1, n):
            
            if verbose: print '\nsimulation %d, observation %d'%(j, i)
            
            # solve using coordinate descent + warm-start
            if i >= (m/2):
                t0_cd = now()
                theta_cd, niter_cd = algo.coordinate_descent(X[:i+1, :], y[:i+1], lam[i], theta=theta.copy())
                t_cd[i-m/2, j] = now() - t0_cd    
                
            # solve recursive lasso
            t0_h = now()
            theta_nz, nz, K, nbr1, nbr2 = reclasso(X[:i+1, :], y[:i+1], lam[i-1], lam[i], theta_nz, nz, K, verbose=verbose)
            t_h[i-1, j] = now() - t0_h
            br1[i-1, j] = nbr1
            br2[i-1, j] = nbr2
            
            # update current solution using the homotopy solution
            theta = np.zeros((m, 1))
            theta[nz] = theta_nz
            
            # solve using lars
            t0_lars = now()
            thetalars_nz, nzlars, Klars, nbrlars = algo.lars(X[:i+1, :], y[:i+1], lam[i], verbose=verbose)
            t_lars[i-1, j] = now() - t0_lars
            brlars[i-1, j] = nbrlars
            thetalars = np.zeros((m, 1))
            thetalars[nzlars] = thetalars_nz
            
            # look at the error terms
            err_l2[i-1, j] = np.sqrt(((theta - thetaopt)**2).sum()) / np.sqrt((thetaopt**2).sum())
            err_hamming[i-1, j] = len(np.where((abs(thetaopt) > 0) != (abs(theta) > 0))[0])
            if i >= (m/2):
                # verify that the solution from coordinate descent is correct
                error = np.sqrt(((theta_cd - theta)**2).sum()) / np.sqrt((theta**2).sum())
                if error > 1e-3: print '\ncoordinate descent has not converged'
            
        # verify the result
        if verbose: 
            print '\ncompute solution using interior point method'
            thetaip = algo.interior_point(X, y, lam[-1])
            thetaip, nzip, Kip, truesol = algo.fix_sol(X, y, lam[-1], thetaip)
            print '\ntheta (ip) =', 
            print thetaip.T
            print '\ntheta (homotopy), error = ', np.sqrt(((theta - thetaip)**2).sum()) / np.sqrt((thetaip**2).sum())
            print theta.T
            print '\ntheta (lars), error = ', np.sqrt(((thetalars - thetaip)**2).sum()) / np.sqrt((thetaip**2).sum())
            print thetalars.T
            
    fig1 = plt.figure(1)
    plt.clf()
    ax = plt.subplot(111)
    brmean = (br1+br2).mean(axis=1)
    brstd = (br1+br2).std(axis=1)
    brlarsmean = (brlars).mean(axis=1)
    brlarsstd = (brlars).std(axis=1)
    iters = np.arange(1, n)
    p1 = plt.plot(iters, brmean)
    p2 = plt.plot(iters, brlarsmean)
    verts1 = zip(iters, brmean+brstd) + zip(iters[::-1], brmean[::-1] - brstd[::-1])
    verts2 = zip(iters, brlarsmean+brlarsstd) + zip(iters[::-1], brlarsmean[::-1] - brlarsstd[::-1])
    plt.legend((p1[0], p2[0]), ('Reclasso', 'Lars'), loc='upper right')
    poly1 = Polygon(verts1, facecolor=(.5,.5,1), edgecolor='w', alpha=.4, lw=0.)
    ax.add_patch(poly1)
    poly2 = Polygon(verts2, facecolor=(.5,1,.5), edgecolor='w', alpha=.4, lw=0.)
    ax.add_patch(poly2)
    plt.xlim(1, n-1)
    plt.xlabel('Iteration', fontsize=16)
    plt.ylabel('Number of transition points', fontsize=16)
    if showfig: plt.show()
    fig1.savefig(simuname+'_transitionpoints.pdf')
    
    fig2 = plt.figure(2)
    plt.clf()
    ax = plt.subplot(111)
    hammean = err_hamming.mean(axis=1)
    hamstd = err_hamming.std(axis=1)
    p = plt.plot(iters, hammean)
    verts = zip(iters, hammean+hamstd) + zip(iters[::-1], hammean[::-1] - hamstd[::-1])
    poly = Polygon(verts, facecolor=(.5,.5,1), edgecolor='w', alpha=.4, lw=0.)
    ax.add_patch(poly)
    plt.xlim(1, n-1)
    plt.xlabel('Iteration', fontsize=16)
    plt.ylabel('Hamming distance', fontsize=16)
    if showfig: plt.show() 
    fig2.savefig(simuname+'_hammingdistance.pdf')
    
    fig3 = plt.figure(3)
    plt.clf()
    ax = plt.subplot(111)
    l2mean = err_l2.mean(axis=1)
    l2std = err_l2.std(axis=1)
    p = plt.plot(iters, l2mean)
    verts = zip(iters, l2mean+l2std) + zip(iters[::-1], l2mean[::-1] - l2std[::-1])
    poly = Polygon(verts, facecolor=(.5,.5,1), edgecolor='w', alpha=.4, lw=0.)
    ax.add_patch(poly)
    plt.xlim(1, n-1)
    plt.xlabel('Iteration', fontsize=16)
    plt.ylabel('Relative MSE', fontsize=16)
    if showfig: plt.show() 
    fig3.savefig(simuname+'_relativel2.pdf')
    
    fig4 = plt.figure(4)
    plt.clf()
    iters_cd = np.arange(m/2, n)
    ax = plt.subplot(111)
    t_hmean = t_h.mean(axis=1)
    t_hstd = t_h.std(axis=1)
    t_larsmean = t_lars.mean(axis=1)
    t_larsstd = t_lars.std(axis=1)
    t_cdmean = t_cd.mean(axis=1)
    t_cdstd = t_cd.std(axis=1)
    p_h = plt.plot(iters, t_hmean)
    p_lars = plt.plot(iters, t_larsmean)
    p_cd = plt.plot(iters_cd, t_cdmean)
    plt.legend((p_h[0], p_lars[0], p_cd[0]), ('Reclasso', 'Lars', 'CD'), loc='best')
    verts_h = zip(iters, t_hmean+t_hstd) + zip(iters[::-1], t_hmean[::-1] - t_hstd[::-1])
    verts_lars = zip(iters, t_larsmean+t_larsstd) + zip(iters[::-1], t_larsmean[::-1] - t_larsstd[::-1])
    verts_cd = zip(iters_cd, t_cdmean+t_cdstd) + zip(iters_cd[::-1], t_cdmean[::-1] - t_cdstd[::-1])
    poly_h = Polygon(verts_h, facecolor=(.5,.5,1), edgecolor='w', alpha=.4, lw=0.)
    ax.add_patch(poly_h)
    poly_lars = Polygon(verts_lars, facecolor=(.5,1,.5), edgecolor='w', alpha=.4, lw=0.)
    ax.add_patch(poly_lars)
    poly_cd = Polygon(verts_cd, facecolor=(1,.5,.5), edgecolor='w', alpha=.4, lw=0.)
    ax.add_patch(poly_cd)
    plt.xlim(1, n-1)
    ymax = 1.2 * max((t_larsmean+t_larsstd).max(), (t_hmean+t_hstd).max())
    plt.ylim(0, ymax)
    plt.xlabel('Iteration', fontsize=16)
    plt.ylabel('Time', fontsize=16)
    if showfig: plt.show()
    fig4.savefig(simuname+'_timinginfo.pdf')