コード例 #1
0
def test1():
    X, y, W = test1_data()
    print "X shape", X.shape, " Y shape:", y.shape
    ntrain = int(y.shape[0] * 0.80)
    g, l = 1e-10, 1e10
    print "mssl"
    mssl = pMSSL(max_epochs=100,
                 quiet=False,
                 gamma=g,
                 lambd=l,
                 walgo='admm',
                 w_epochs=50,
                 omega_epochs=100,
                 ytransform="log")
    mssl.fit(X[:ntrain], y[:ntrain])
    yhat = mssl.predict(X[ntrain:])
    p = numpy.mean(
        [pearsonr(y[ntrain:, i], yhat[:, i])[0] for i in range(yhat.shape[1])])
    mse = numpy.mean((yhat - y[ntrain:])**2)
    num_omega_zeros = numpy.sum(mssl.Omega == 0)
    num_w_zeros = numpy.sum(mssl.W == 0)
    d = {
        "gamma": g,
        "lambda": l,
        "mse": mse,
        "omega_zeros": num_omega_zeros,
        "w_zeros": num_w_zeros,
        "pearson": p
    }
    #print mssl.W
    print d
コード例 #2
0
ファイル: mssl_tests.py プロジェクト: liyi-1989/pydownscale
def test_joblib():
    from sklearn.linear_model import Lasso
    X, y, W = test_log_data()
    print "X shape", X.shape, " Y shape:", y.shape
    ntrain = int(y.shape[0]*0.80)
    g, l = 1e-10, 1e10
    
    X_train = X[:ntrain]
    X_test = X[ntrain:]
    ytrain = y[:ntrain]
    ytest = y[ntrain:]

    print "MSSL"
    mssl = pMSSL(max_epochs=25, quiet=False, gamma=g, lambd=l, 
        walgo='multiprocessor', w_epochs=51, omega_epochs=100, num_proc=1,
                ytransform=StandardScaler(), rho=1., xtransform=StandardScaler())
    mssl.fit(X_train, ytrain)
    yhat = mssl.predict(X_train)
    p = numpy.mean([pearsonr(ytrain[:,i], yhat[:,i])[0] for i in range(yhat.shape[1])])
    mse = numpy.mean((yhat - ytrain)**2)
    num_omega_zeros = numpy.sum(mssl.Omega == 0)
    num_w_zeros = numpy.sum(mssl.W == 0)
    d = {"gamma": g, "lambda": l, "mse": mse, "omega_zeros": num_omega_zeros, "w_zeros": num_w_zeros, "pearson": p}
    pyplot.plot(yhat[:, 0])
    pyplot.plot(ytrain[:, 0])
    print "Trying to show plot"
    pyplot.show()
    print d
コード例 #3
0
def asd_mssl():
    model = pMSSL(max_epochs=5, quiet=False, lambd=1e10, gamma=1e-10, walgo='multiprocessor',
                  num_proc=48, w_epochs=50, omega_epochs=50)
    f = "/home/vandal.t/repos/pydownscale/pydownscale/test_data/testdata.pkl"
    lf = '/gss_gpfs_scratch/vandal.t/experiments/DownscaleData/newengland_MS_420_8835.pkl'
    data = pickle.load(open(lf, 'r'))
    asdm = ASDMultitask(data, model, season='JJA', ytransform=preprocessing.StandardScaler())
    asdm.train()
    out = asdm.predict(test_set=False)
    out.to_netcdf("test_data/ne_test_set.nc")
コード例 #4
0
ファイル: mssl_tests.py プロジェクト: liyi-1989/pydownscale
def climate():

    lambd = 1e2
    gamma = 1e2
    season = "DJF"
    data = pickle.load(open("/scratch/vandal.t/experiments/DownscaleData/monthly_804_3150.pkl", "r"))
    model = pMSSL(lambd=lambd, gamma=gamma, max_epochs=1000)
    dmodel = DownscaleModel(data, model, season=season)
    dmodel.train()
    print dmodel.model.Omega[:5, :5]
    print dmodel.model.W[:5, :5]
    print "Omega to zero", numpy.sum(dmodel.model.Omega == 0)
    print "W to zero", numpy.sum(dmodel.model.W == 0)
コード例 #5
0
def test1_mpi():
    from mpi4py import MPI

    comm = MPI.COMM_WORLD
    size = comm.Get_size()
    rank = comm.Get_rank()

    X, y, W = test1_data()
    ntrain = int(y.shape[0] * 0.80)
    gspace = numpy.linspace(1e-4, 4, 10)
    lspace = numpy.linspace(1e-4, 2, 10)

    if rank == 0:
        space = numpy.asarray([[g, l] for g in gspace for l in lspace])
        split = numpy.array_split(space, size)

    else:
        split = None

    split = comm.scatter(split, root=0)
    results = []
    for g, l in split:
        try:
            mssl = pMSSL(max_epochs=1000, quite=True, gamma=g, lambd=l)
            mssl.fit(X[:ntrain], y[:ntrain])
            yhat = mssl.predict(X[ntrain:])
            mse = numpy.mean((yhat - y[ntrain:])**2)
            num_omega_zeros = numpy.sum(mssl.Omega == 0)
            num_w_zeros = numpy.sum(mssl.W == 0)
            d = {
                "gamma": g,
                "lambda": l,
                "mse": mse,
                "omega_zeros": num_omega_zeros,
                "w_zeros": num_w_zeros
            }
        except Exception as err:
            sys.stderr.write("Gamma: %f, Lambda: %f\n%s" % (g, l, err))
            d = {"error": 1, "gamma": g, "lambda": l}
        results.append(d)
        sys.stdout.write(str(d))

    newdata = comm.gather(results)
    if rank == 0:
        newdata = [item for l in newdata
                   for item in l]  ## condense lists of lists
        data = pandas.DataFrame(newdata)
        data.to_csv("mssl-test1-mpi.csv", index=False)
コード例 #6
0
def climate():

    lambd = 1e2
    gamma = 1e2
    season = "DJF"
    data = pickle.load(
        open(
            "/scratch/vandal.t/experiments/DownscaleData/monthly_804_3150.pkl",
            "r"))
    model = pMSSL(lambd=lambd, gamma=gamma, max_epochs=1000)
    dmodel = DownscaleModel(data, model, season=season)
    dmodel.train()
    print dmodel.model.Omega[:5, :5]
    print dmodel.model.W[:5, :5]
    print "Omega to zero", numpy.sum(dmodel.model.Omega == 0)
    print "W to zero", numpy.sum(dmodel.model.W == 0)
コード例 #7
0
ファイル: mssl_tests.py プロジェクト: liyi-1989/pydownscale
def test1():
    X, y, W = test1_data()
    print "X shape", X.shape, " Y shape:", y.shape
    ntrain = int(y.shape[0]*0.80)
    g, l = 1e-10, 1e10
    print "mssl"
    mssl = pMSSL(max_epochs=100, quiet=False, gamma=g, lambd=l, 
        walgo='admm', w_epochs=50, omega_epochs=100, ytransform="log")
    mssl.fit(X[:ntrain], y[:ntrain])
    yhat = mssl.predict(X[ntrain:])
    p = numpy.mean([pearsonr(y[ntrain:,i], yhat[:, i])[0] for i in range(yhat.shape[1])])
    mse = numpy.mean((yhat - y[ntrain:])**2)
    num_omega_zeros = numpy.sum(mssl.Omega == 0)
    num_w_zeros = numpy.sum(mssl.W == 0)
    d = {"gamma": g, "lambda": l, "mse": mse, "omega_zeros": num_omega_zeros, "w_zeros": num_w_zeros, "pearson": p}
    #print mssl.W
    print d
コード例 #8
0
def test_joblib():
    from sklearn.linear_model import Lasso
    X, y, W = test_log_data()
    print "X shape", X.shape, " Y shape:", y.shape
    ntrain = int(y.shape[0] * 0.80)
    g, l = 1e-10, 1e10

    X_train = X[:ntrain]
    X_test = X[ntrain:]
    ytrain = y[:ntrain]
    ytest = y[ntrain:]

    print "MSSL"
    mssl = pMSSL(max_epochs=25,
                 quiet=False,
                 gamma=g,
                 lambd=l,
                 walgo='multiprocessor',
                 w_epochs=51,
                 omega_epochs=100,
                 num_proc=1,
                 ytransform=StandardScaler(),
                 rho=1.,
                 xtransform=StandardScaler())
    mssl.fit(X_train, ytrain)
    yhat = mssl.predict(X_train)
    p = numpy.mean(
        [pearsonr(ytrain[:, i], yhat[:, i])[0] for i in range(yhat.shape[1])])
    mse = numpy.mean((yhat - ytrain)**2)
    num_omega_zeros = numpy.sum(mssl.Omega == 0)
    num_w_zeros = numpy.sum(mssl.W == 0)
    d = {
        "gamma": g,
        "lambda": l,
        "mse": mse,
        "omega_zeros": num_omega_zeros,
        "w_zeros": num_w_zeros,
        "pearson": p
    }
    pyplot.plot(yhat[:, 0])
    pyplot.plot(ytrain[:, 0])
    print "Trying to show plot"
    pyplot.show()
    print d
コード例 #9
0
def asd_mssl():
    model = pMSSL(max_epochs=5,
                  quiet=False,
                  lambd=1e10,
                  gamma=1e-10,
                  walgo='multiprocessor',
                  num_proc=48,
                  w_epochs=50,
                  omega_epochs=50)
    f = "/home/vandal.t/repos/pydownscale/pydownscale/test_data/testdata.pkl"
    lf = '/gss_gpfs_scratch/vandal.t/experiments/DownscaleData/newengland_MS_420_8835.pkl'
    data = pickle.load(open(lf, 'r'))
    asdm = ASDMultitask(data,
                        model,
                        season='JJA',
                        ytransform=preprocessing.StandardScaler())
    asdm.train()
    out = asdm.predict(test_set=False)
    out.to_netcdf("test_data/ne_test_set.nc")
コード例 #10
0
ファイル: mssl_tests.py プロジェクト: liyi-1989/pydownscale
def test1_mpi():
    from mpi4py import MPI

    comm = MPI.COMM_WORLD
    size = comm.Get_size()
    rank = comm.Get_rank()
    
    X, y, W = test1_data()
    ntrain = int(y.shape[0]*0.80)
    gspace = numpy.linspace(1e-4, 4, 10)
    lspace = numpy.linspace(1e-4, 2, 10)

    if rank == 0:
        space = numpy.asarray([[g, l] for g in gspace for l in lspace])
        split = numpy.array_split(space, size)

    else:
        split = None

    split = comm.scatter(split, root=0)
    results = []
    for g, l in split:
        try:
            mssl = pMSSL(max_epochs=1000, quite=True, gamma=g, lambd=l)
            mssl.fit(X[:ntrain], y[:ntrain])
            yhat = mssl.predict(X[ntrain:])
            mse = numpy.mean((yhat - y[ntrain:])**2)
            num_omega_zeros = numpy.sum(mssl.Omega == 0)
            num_w_zeros = numpy.sum(mssl.W == 0)
            d = {"gamma": g, "lambda": l, "mse": mse, 
            "omega_zeros": num_omega_zeros, "w_zeros": num_w_zeros}
        except Exception as err:
            sys.stderr.write("Gamma: %f, Lambda: %f\n%s" % (g, l, err))
            d = {"error": 1, "gamma": g, "lambda": l}
        results.append(d)
        sys.stdout.write(str(d))

    newdata = comm.gather(results)
    if rank == 0:
        newdata = [item for l in newdata for item in l]   ## condense lists of lists
        data = pandas.DataFrame(newdata)
        data.to_csv("mssl-test1-mpi.csv", index=False)