Пример #1
0
def main():
    ''' main routine '''
    comm = MPI.COMM_WORLD
    rk = comm.Get_rank()
    nProc = comm.Get_size()
    
    if len(sys.argv) <= 2:
        ''' complain '''
        print 'need  arguments: plain or fft?'
        return 0
    else:
        if sys.argv[1] == 'plain':
            plain = True
        elif sys.argv[1] == 'fft':
            plain = False
        else:
            plain = True
            
    if len(sys.argv) >= 3:
        dts = sys.argv[2]
    else:
        dts = 'plmr'
            
    ''' set the parameters for how big of a problem to grab'''   
    m = 50000 # size of data
    p = 25 # number of filters
    q = 300 # length of filters
    
    if dts == 'plmr':
        rho = 1 # l1 internal parameter
        lmb = 0.005 # l1 weighting
        xi = 0.2 # weights update parameter
        ch = 1 # number of channels
    elif dts == 'mpk':
        rho = 0.1
        lmb = 4e-6
        xi = 0.2
        ch = 3

    ''' load data, given rank '''
    y = getData(m,dts,ch,rank=rk)
    
    ''' initialize weights '''    
    wt = [weightInit(p,q,comm) for ix in xrange(ch)]
    
    A = [dtm(m,p,q,fourier=not plain) for ix in xrange(ch)]
        
    for Q in A:
        print Q.n
            
    optl1 = lasso(m,A[0].n,rho,lmb,ch)
        
    newWW = [weightsUpdate.weightsUpdate(m,p,q,xi) for ix in xrange(ch)]
    print newWW[0].m
    for WWl,wl in zip(newWW,wt):
        WWl.wp = wl
    
    rrz = list()
    gap = list()
    ''' begin loop '''
    for itz in range(1000):
        ws = [WWl.wp for WWl in newWW]
        for Q,w in zip(A,wt) :
            Q.changeWeights(w)
        tm = time()
        z = optl1.solveL1(y, A)
        rrz = optl1.rrz # .append(optl1.rrz.pop())
        gap = optl1.gap #.append(optl1.gap.pop())
        print 'rank ' + repr(rk) + ' of ' + repr(nProc) +  ' solved l1 itr: ' + repr(itz) + ' time: ' + repr(time()-tm)
        tm = time()
        
        if plain:
            wmx = [WL.updatePlain(yl, wtl, zl) for WL,yl,wtl,zl in zip(newWW,y,wt,z)]
        else:
            wmx = [WL.updateFourier(yl, wtl, zl) for WL,yl,wtl,zl in zip(newWW,y,wt,z)]
            
        print 'rank ' + repr(rk) + ' of ' + repr(nProc) +  ' solved w update itr: ' + repr(itz) + ' time: ' + repr(time()-tm)
        tm = time()
        wt = [weightAgg(wmxl,p,q,comm) for wmxl in wmx]
        wp = [WL.wp for WL in newWW]
        print 'rank ' + repr(rk) + ' of ' + repr(nProc) +  ' have new weights itr: ' + repr(itz) + ' time: ' + repr(time()-tm)
        outd = {'itr':itz, 'y':y, 'z':z, 'wt':wt,'wp':wp,'m':m,'p':p,'q':q, 'rho':rho,'lmb':lmb, 'xi':xi, 'rrz':rrz,'gap':gap, 'ws':ws }
        
        if plain & (dts == 'plmr'):
            spio.savemat('miniOut_' + repr(p) + '_' + repr(nProc) + '_' + repr(rk), outd)
        elif plain & (dts == 'mpk'):
            spio.savemat('miniOutMPK_' + repr(nProc) + '_' + repr(rk), outd)
        else:
            spio.savemat('plainOut_' + repr(nProc) + '_' + repr(rk), outd) 
            
    ''' return these values if I chance to run this in an interactive Python shell'''
    return y,z,optl1,A,wt
    
    ''' for n iterations '''
    
    ''' calculate local weights '''
    
    ''' all reduce '''
Пример #2
0
def main():
    ''' main routine '''
    comm = MPI.COMM_WORLD
    rk = comm.Get_rank()
    nProc = comm.Get_size()
    
    plain = True
    if len(sys.argv) >= 2:
        if sys.argv[1] == 'plain':
            plain = False
    
    if len(sys.argv) >= 3:
        dts = sys.argv[2]
    else:
        dts = 'plmr'
        
        
        
    m = 50000 # size of data
    p = 25 # number of filters
    q = 300 # length of filters
    
    if dts == 'plmr':
        rho = 1
        lmb = 0.5
        xi = 0.2
    elif dts == 'mpk':
        rho = 0.1
        lmb = 1e-6
        xi = 0.2
        
    fac = 1.0; # np.sqrt((m/q)/2.0)
    ''' initialize MPI routine '''
    
    
    ''' load data, given rank '''
    y = getData(m,dts,rank=rk)

    
    ''' initialize weights '''
    # D = spio.loadmat('fakew.mat')
    # wt = D['wini']
#    wt = np.random.randn(q,p)/np.sqrt(q) #D['w']
    
    
    wt = weightInit(p,q,comm)
    
    if plain:
        A = convFFT(m,p,q,fct=fac)
        optl1 = lasso(m,m*(p+1),rho,lmb)
        
    else:
        print 'Doing plain!'
        A = convOperator(m,p,q)
        optl1 = lasso(m,m*(p),rho,lmb)
        
        
    newWW = weightsUpdate.weightsUpdate(m,p,q,xi,fct=fac)
    newWW.wp = wt
    
    rrz = list()
    gap = list()
    ''' begin loop '''
    for itz in range(1000):
        ws = newWW.wp
        A.changeWeights(newWW.wp)
        tm = time()
        z = optl1.solveL1(y, A)
        rrz = optl1.rrz # .append(optl1.rrz.pop())
        gap = optl1.gap #.append(optl1.gap.pop())
        print 'rank ' + repr(rk) + ' of ' + repr(nProc) +  ' solved l1 itr: ' + repr(itz) + ' time: ' + repr(time()-tm)
        tm = time()
        
        if plain:
            wmx = newWW.updateFourier(y, wt, z)
        else:
            wmx = newWW.updatePlain(y, wt, z)
            
        print 'rank ' + repr(rk) + ' of ' + repr(nProc) +  ' solved w update itr: ' + repr(itz) + ' time: ' + repr(time()-tm)
        tm = time()
        wt = weightAgg(wmx,p,q,comm)
        wp = newWW.wp
        print 'rank ' + repr(rk) + ' of ' + repr(nProc) +  ' have new weights itr: ' + repr(itz) + ' time: ' + repr(time()-tm)
        outd = {'y':y, 'z':z, 'wt':wt,'wp':wp,'m':m,'p':p,'q':q, 'rho':rho,'lmb':lmb, 'xi':xi, 'rrz':rrz,'gap':gap, 'ws':ws }
        
        if plain & (dts == 'plmr'):
            spio.savemat('miniOut_' + repr(p) + '_' + repr(nProc) + '_' + repr(rk), outd)
        elif plain & (dts == 'mpk'):
            spio.savemat('miniOutMPK_' + repr(nProc) + '_' + repr(rk), outd)
        else:
            spio.savemat('plainOut_' + repr(nProc) + '_' + repr(rk), outd) 
    
    return y,z,optl1,A,wt
    
    ''' for n iterations '''
    
    ''' calculate local weights '''
    
    ''' all reduce '''
Пример #3
0
def main():
    ''' main routine '''
    comm = MPI.COMM_WORLD
    rk = comm.Get_rank()
    nProc = comm.Get_size()

    plain = True
    if len(sys.argv) >= 2:
        if sys.argv[1] == 'plain':
            plain = False

    if len(sys.argv) >= 3:
        dts = sys.argv[2]
    else:
        dts = 'plmr'

    m = 50000  # size of data
    p = 25  # number of filters
    q = 300  # length of filters

    if dts == 'plmr':
        rho = 1
        lmb = 0.5
        xi = 0.2
    elif dts == 'mpk':
        rho = 0.1
        lmb = 1e-6
        xi = 0.2

    fac = 1.0
    # np.sqrt((m/q)/2.0)
    ''' initialize MPI routine '''
    ''' load data, given rank '''
    y = getData(m, dts, rank=rk)
    ''' initialize weights '''
    # D = spio.loadmat('fakew.mat')
    # wt = D['wini']
    #    wt = np.random.randn(q,p)/np.sqrt(q) #D['w']

    wt = weightInit(p, q, comm)

    if plain:
        A = convFFT(m, p, q, fct=fac)
        optl1 = lasso(m, m * (p + 1), rho, lmb)

    else:
        print 'Doing plain!'
        A = convOperator(m, p, q)
        optl1 = lasso(m, m * (p), rho, lmb)

    newWW = weightsUpdate.weightsUpdate(m, p, q, xi, fct=fac)
    newWW.wp = wt

    rrz = list()
    gap = list()
    ''' begin loop '''
    for itz in range(1000):
        ws = newWW.wp
        A.changeWeights(newWW.wp)
        tm = time()
        z = optl1.solveL1(y, A)
        rrz = optl1.rrz  # .append(optl1.rrz.pop())
        gap = optl1.gap  #.append(optl1.gap.pop())
        print 'rank ' + repr(rk) + ' of ' + repr(
            nProc) + ' solved l1 itr: ' + repr(itz) + ' time: ' + repr(time() -
                                                                       tm)
        tm = time()

        if plain:
            wmx = newWW.updateFourier(y, wt, z)
        else:
            wmx = newWW.updatePlain(y, wt, z)

        print 'rank ' + repr(rk) + ' of ' + repr(
            nProc) + ' solved w update itr: ' + repr(itz) + ' time: ' + repr(
                time() - tm)
        tm = time()
        wt = weightAgg(wmx, p, q, comm)
        wp = newWW.wp
        print 'rank ' + repr(rk) + ' of ' + repr(
            nProc) + ' have new weights itr: ' + repr(itz) + ' time: ' + repr(
                time() - tm)
        outd = {
            'y': y,
            'z': z,
            'wt': wt,
            'wp': wp,
            'm': m,
            'p': p,
            'q': q,
            'rho': rho,
            'lmb': lmb,
            'xi': xi,
            'rrz': rrz,
            'gap': gap,
            'ws': ws
        }

        if plain & (dts == 'plmr'):
            spio.savemat(
                'miniOut_' + repr(p) + '_' + repr(nProc) + '_' + repr(rk),
                outd)
        elif plain & (dts == 'mpk'):
            spio.savemat('miniOutMPK_' + repr(nProc) + '_' + repr(rk), outd)
        else:
            spio.savemat('plainOut_' + repr(nProc) + '_' + repr(rk), outd)

    return y, z, optl1, A, wt
    ''' for n iterations '''
    ''' calculate local weights '''
    ''' all reduce '''
Пример #4
0
def main():
    ''' main routine '''
    comm = MPI.COMM_WORLD
    rk = comm.Get_rank()
    nProc = comm.Get_size()

    if len(sys.argv) <= 2:
        ''' complain '''
        print 'need  arguments: plain or fft?'
        return 0
    else:
        if sys.argv[1] == 'plain':
            plain = True
        elif sys.argv[1] == 'fft':
            plain = False
        else:
            plain = True

    if len(sys.argv) >= 3:
        dts = sys.argv[2]
    else:
        dts = 'plmr'
    ''' set the parameters for how big of a problem to grab'''
    m = 50000  # size of data
    p = 25  # number of filters
    q = 300  # length of filters

    if dts == 'plmr':
        rho = 1  # l1 internal parameter
        lmb = 0.005  # l1 weighting
        xi = 0.2  # weights update parameter
        ch = 1  # number of channels
    elif dts == 'mpk':
        rho = 0.1
        lmb = 4e-6
        xi = 0.2
        ch = 3
    ''' load data, given rank '''
    y = getData(m, dts, ch, rank=rk)
    ''' initialize weights '''
    wt = [weightInit(p, q, comm) for ix in xrange(ch)]

    A = [dtm(m, p, q, fourier=not plain) for ix in xrange(ch)]

    for Q in A:
        print Q.n

    optl1 = lasso(m, A[0].n, rho, lmb, ch)

    newWW = [weightsUpdate.weightsUpdate(m, p, q, xi) for ix in xrange(ch)]
    print newWW[0].m
    for WWl, wl in zip(newWW, wt):
        WWl.wp = wl

    rrz = list()
    gap = list()
    ''' begin loop '''
    for itz in range(1000):
        ws = [WWl.wp for WWl in newWW]
        for Q, w in zip(A, wt):
            Q.changeWeights(w)
        tm = time()
        z = optl1.solveL1(y, A)
        rrz = optl1.rrz  # .append(optl1.rrz.pop())
        gap = optl1.gap  #.append(optl1.gap.pop())
        print 'rank ' + repr(rk) + ' of ' + repr(
            nProc) + ' solved l1 itr: ' + repr(itz) + ' time: ' + repr(time() -
                                                                       tm)
        tm = time()

        if plain:
            wmx = [
                WL.updatePlain(yl, wtl, zl)
                for WL, yl, wtl, zl in zip(newWW, y, wt, z)
            ]
        else:
            wmx = [
                WL.updateFourier(yl, wtl, zl)
                for WL, yl, wtl, zl in zip(newWW, y, wt, z)
            ]

        print 'rank ' + repr(rk) + ' of ' + repr(
            nProc) + ' solved w update itr: ' + repr(itz) + ' time: ' + repr(
                time() - tm)
        tm = time()
        wt = [weightAgg(wmxl, p, q, comm) for wmxl in wmx]
        wp = [WL.wp for WL in newWW]
        print 'rank ' + repr(rk) + ' of ' + repr(
            nProc) + ' have new weights itr: ' + repr(itz) + ' time: ' + repr(
                time() - tm)
        outd = {
            'itr': itz,
            'y': y,
            'z': z,
            'wt': wt,
            'wp': wp,
            'm': m,
            'p': p,
            'q': q,
            'rho': rho,
            'lmb': lmb,
            'xi': xi,
            'rrz': rrz,
            'gap': gap,
            'ws': ws
        }

        if plain & (dts == 'plmr'):
            spio.savemat(
                'miniOut_' + repr(p) + '_' + repr(nProc) + '_' + repr(rk),
                outd)
        elif plain & (dts == 'mpk'):
            spio.savemat('miniOutMPK_' + repr(nProc) + '_' + repr(rk), outd)
        else:
            spio.savemat('plainOut_' + repr(nProc) + '_' + repr(rk), outd)
    ''' return these values if I chance to run this in an interactive Python shell'''
    return y, z, optl1, A, wt
    ''' for n iterations '''
    ''' calculate local weights '''
    ''' all reduce '''