Python Exch_copper Examples

Example #1

File: test_copper.py Project: nathinal/Theano-MPI

from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.rank
size = comm.size
# device='gpu'+str(rank)

from test_exchanger import init_device, clean_device

drv, ctx, arr, shared_x, shared_xx = init_device(device=device)

if rank == 0: print 'original array %s' % arr

# prepare copper exchanger

from exchanger_strategy import Exch_copper
exch = Exch_copper(comm, avg=False)

exch.prepare(ctx, drv, [shared_x])
exch.exchange()

if rank == 0: print 'copper summation: %s' % shared_x.get_value()

# prepare ar exchanger

from exchanger_strategy import Exch_allreduce
exch = Exch_allreduce(comm, avg=False)

exch.prepare([shared_xx])

exch.exchange()

Example #2

File: exchanger.py Project: adeelzaman/Theano-MPI

class BSP_Exchanger(object):
    '''
    model parameter exchanger during BSP weight exchanging
    '''
    def __init__(self, config, drv, ctx, model):

        self.drv = drv
        self.ctx = ctx
        self.comm = config['comm']

        self.size = config['size']
        
        self.exch_strategy = config['exch_strategy']

        self.train_mode = config['train_mode']
        # self.cuda_aware = config['cuda_aware']
        
        # TODO make sure exchanger class doesn't keep a self copy of model, only the reference to its param list
        self.param_list = model.params
        self.vels = model.vels
        self.vels2 = model.vels2

        self.avg_func_list = []
        
        if self.train_mode == 'cdd' and self.exch_strategy == 'ar':
            
            from exchanger_strategy import Exch_allreduce
            self.exch = Exch_allreduce(self.comm, avg=False)
            self.exch.prepare(self.vels, self.vels2)
            
        elif self.train_mode == 'cdd' and self.exch_strategy == 'copper':
            
            from exchanger_strategy import Exch_copper
            self.exch = Exch_copper(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)
            
        elif self.train_mode == 'cdd' and self.exch_strategy == 'copper16':
            
            from exchanger_strategy import Exch_copper16
            self.exch = Exch_copper16(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)
            
        elif self.train_mode == 'cdd' and self.exch_strategy == 'asa32':
            
            from exchanger_strategy import Exch_asa32
            self.exch = Exch_asa32(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)
            
        elif self.train_mode == 'cdd' and self.exch_strategy == 'asa16':
            
            from exchanger_strategy import Exch_asa16
            self.exch = Exch_asa16(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)
        
        
         
        elif self.train_mode == 'avg' and self.exch_strategy == 'ar':
            
            from exchanger_strategy import Exch_allreduce
            self.exch = Exch_allreduce(self.comm)
            self.exch.prepare(self.param_list)
            
        elif self.train_mode == 'avg' and self.exch_strategy == 'copper':
            
            from exchanger_strategy import Exch_copper
            self.exch = Exch_copper(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)
            
        elif self.train_mode == 'avg' and self.exch_strategy == 'copper16':
            
            from exchanger_strategy import Exch_copper16
            self.exch = Exch_copper16(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)
            
        elif self.train_mode == 'avg' and self.exch_strategy == 'asa32':
            
            from exchanger_strategy import Exch_asa32
            self.exch = Exch_asa32(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)
            
        elif self.train_mode == 'avg' and self.exch_strategy == 'asa16':
            
            from exchanger_strategy import Exch_asa16
            self.exch = Exch_asa16(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)
                

    def exchange(self):
        
        # average w
        if self.train_mode == 'avg' and self.size > 1:
            
            if self.exch_strategy == 'ar':
                
                self.exch.exchange()

            elif self.exch_strategy == 'copper':
                
                self.exch.exchange()
                    
            elif self.exch_strategy == 'asa32': 

                self.exch.exchange()          

            elif self.exch_strategy == 'asa16':
                
                self.exch.exchange()
                
            elif self.exch_strategy == 'copper16':
            
                self.exch.exchange()

        # sum delta w
        elif self.train_mode == 'cdd' and self.size > 1:
            
            if self.exch_strategy == 'ar':
                
                self.exch.exchange()
                
            elif self.exch_strategy == 'copper':
                
                self.exch.exchange()
                
            elif self.exch_strategy == 'asa32':

                self.exch.exchange()
                
            elif self.exch_strategy == 'asa16':
                
                self.exch.exchange()
                
            elif self.exch_strategy == 'copper16':
            
                self.exch.exchange()

Example #3

File: test_copper.py Project: CityU-HAN/Theano-MPI

from mpi4py import MPI
comm=MPI.COMM_WORLD
rank=comm.rank
size=comm.size
# device='gpu'+str(rank)

from test_exchanger import init_device, clean_device

drv,ctx,arr,shared_x,shared_xx = init_device(device=device)

if rank==0: print 'original array %s' % arr

# prepare copper exchanger

from exchanger_strategy import Exch_copper
exch = Exch_copper(comm, avg=False)

exch.prepare([shared_x], ctx, drv)
exch.exchange()

if rank==0: print 'copper summation: %s' % shared_x.get_value()


# prepare ar exchanger

from exchanger_strategy import Exch_allreduce
exch = Exch_allreduce(comm, avg=False) 

exch.prepare([shared_xx])

exch.exchange()

Example #4

File: exchanger.py Project: CityU-HAN/Theano-MPI

    def __init__(self, config, drv, ctx, model):

        self.drv = drv
        self.ctx = ctx
        self.comm = config['comm']

        self.size = config['size']
        
        self.exch_strategy = config['exch_strategy']

        self.train_mode = config['train_mode']
        # self.cuda_aware = config['cuda_aware']
        
        # TODO make sure exchanger class doesn't keep a self copy of model, only the reference to its param list
        self.param_list = model.params
        self.vels = model.vels
        self.vels2 = model.vels2

        self.avg_func_list = []
        
        if self.train_mode == 'cdd' and self.exch_strategy == 'ar': #c
            
            from exchanger_strategy import Exch_allreduce
            self.exch = Exch_allreduce(self.comm, avg=False)
            self.exch.prepare(self.vels, self.vels2)
            
        elif self.train_mode == 'cdd' and self.exch_strategy == 'copper': #c
            
            from exchanger_strategy import Exch_copper
            self.exch = Exch_copper(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)
            
        elif self.train_mode == 'cdd' and self.exch_strategy == 'asa32': #c
            
            from exchanger_strategy import Exch_asa32
            self.exch = Exch_asa32(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)
            
        elif self.train_mode == 'cdd' and self.exch_strategy == 'asa16': #c
            
            from exchanger_strategy import Exch_asa16
            self.exch = Exch_asa16(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)
        
        #TODO adjust ctx, drv locations in all strategies
         
        elif self.train_mode == 'avg' and self.exch_strategy == 'ar': #c
            
            from exchanger_strategy import Exch_allreduce
            self.exch = Exch_allreduce(self.comm)
            self.exch.prepare(self.param_list)
            
        elif self.train_mode == 'avg' and self.exch_strategy == 'copper': #c
            
            from exchanger_strategy import Exch_copper
            self.exch = Exch_copper(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)
            
        elif self.train_mode == 'avg' and self.exch_strategy == 'asa32': #c
            
            from exchanger_strategy import Exch_asa32
            self.exch = Exch_asa32(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)
            
        elif self.train_mode == 'avg' and self.exch_strategy == 'asa16': #c
            
            from exchanger_strategy import Exch_asa16
            self.exch = Exch_asa16(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)

Example #5

File: test_copper.py Project: yhgon/Theano-MPI

rank = comm.rank
size = comm.size
# device='gpu'+str(rank)

from test_exchanger import init_device, clean_device

drv, ctx, arr, shared_x, shared_xx = init_device(device=device)

if rank == 0:
    print "original array %s" % arr

# prepare copper exchanger

from exchanger_strategy import Exch_copper

exch = Exch_copper(comm, avg=False)

exch.prepare(ctx, drv, [shared_x])
exch.exchange()

if rank == 0:
    print "copper summation: %s" % shared_x.get_value()


# prepare ar exchanger

from exchanger_strategy import Exch_allreduce

exch = Exch_allreduce(comm, avg=False)

exch.prepare([shared_xx])

Example #6

    def __init__(self, config, drv, ctx, model):

        self.drv = drv
        self.ctx = ctx
        self.comm = config['comm']

        self.size = config['size']

        self.exch_strategy = config['exch_strategy']

        self.train_mode = config['train_mode']
        # self.cuda_aware = config['cuda_aware']

        # TODO make sure exchanger class doesn't keep a self copy of model, only the reference to its param list
        self.param_list = model.params
        self.vels = model.vels
        self.vels2 = model.vels2

        self.avg_func_list = []

        if self.train_mode == 'cdd' and self.exch_strategy == 'ar':  #c

            from exchanger_strategy import Exch_allreduce
            self.exch = Exch_allreduce(self.comm, avg=False)
            self.exch.prepare(self.vels, self.vels2)

        elif self.train_mode == 'cdd' and self.exch_strategy == 'copper':  #c

            from exchanger_strategy import Exch_copper
            self.exch = Exch_copper(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)

        elif self.train_mode == 'cdd' and self.exch_strategy == 'asa32':  #c

            from exchanger_strategy import Exch_asa32
            self.exch = Exch_asa32(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)

        elif self.train_mode == 'cdd' and self.exch_strategy == 'asa16':  #c

            from exchanger_strategy import Exch_asa16
            self.exch = Exch_asa16(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)

        #TODO adjust ctx, drv locations in all strategies

        elif self.train_mode == 'avg' and self.exch_strategy == 'ar':  #c

            from exchanger_strategy import Exch_allreduce
            self.exch = Exch_allreduce(self.comm)
            self.exch.prepare(self.param_list)

        elif self.train_mode == 'avg' and self.exch_strategy == 'copper':  #c

            from exchanger_strategy import Exch_copper
            self.exch = Exch_copper(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)

        elif self.train_mode == 'avg' and self.exch_strategy == 'asa32':  #c

            from exchanger_strategy import Exch_asa32
            self.exch = Exch_asa32(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)

        elif self.train_mode == 'avg' and self.exch_strategy == 'asa16':  #c

            from exchanger_strategy import Exch_asa16
            self.exch = Exch_asa16(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)

Example #7

class BSP_Exchanger(object):
    '''
    model parameter exchanger during BSP weight exchanging
    '''
    def __init__(self, config, drv, ctx, model):

        self.drv = drv
        self.ctx = ctx
        self.comm = config['comm']

        self.size = config['size']

        self.exch_strategy = config['exch_strategy']

        self.worker_type = config['worker_type']
        # self.cuda_aware = config['cuda_aware']

        # TODO make sure exchanger class doesn't keep a self copy of model, only the reference to its param list
        self.param_list = model.params
        self.vels = model.vels
        self.vels2 = model.vels2

        self.avg_func_list = []

        if self.worker_type == 'cdd' and self.exch_strategy == 'ar':

            from exchanger_strategy import Exch_allreduce
            self.exch = Exch_allreduce(self.comm, avg=False)
            self.exch.prepare(self.vels, self.vels2)

        elif self.worker_type == 'cdd' and self.exch_strategy == 'copper':

            from exchanger_strategy import Exch_copper
            self.exch = Exch_copper(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)

        elif self.worker_type == 'cdd' and self.exch_strategy == 'copper16':

            from exchanger_strategy import Exch_copper16
            self.exch = Exch_copper16(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)

        elif self.worker_type == 'cdd' and self.exch_strategy == 'asa32':

            from exchanger_strategy import Exch_asa32
            self.exch = Exch_asa32(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)

        elif self.worker_type == 'cdd' and self.exch_strategy == 'asa16':

            from exchanger_strategy import Exch_asa16
            self.exch = Exch_asa16(self.comm, avg=False)
            self.exch.prepare(self.ctx, self.drv, self.vels, self.vels2)

        elif self.worker_type == 'avg' and self.exch_strategy == 'ar':

            from exchanger_strategy import Exch_allreduce
            self.exch = Exch_allreduce(self.comm)
            self.exch.prepare(self.param_list)

        elif self.worker_type == 'avg' and self.exch_strategy == 'copper':

            from exchanger_strategy import Exch_copper
            self.exch = Exch_copper(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)

        elif self.worker_type == 'avg' and self.exch_strategy == 'copper16':

            from exchanger_strategy import Exch_copper16
            self.exch = Exch_copper16(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)

        elif self.worker_type == 'avg' and self.exch_strategy == 'asa32':

            from exchanger_strategy import Exch_asa32
            self.exch = Exch_asa32(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)

        elif self.worker_type == 'avg' and self.exch_strategy == 'asa16':

            from exchanger_strategy import Exch_asa16
            self.exch = Exch_asa16(self.comm)
            self.exch.prepare(self.ctx, self.drv, self.param_list)

    def exchange(self):

        # average w
        if self.worker_type == 'avg' and self.size > 1:

            if self.exch_strategy == 'ar':

                self.exch.exchange()

            elif self.exch_strategy == 'copper':

                self.exch.exchange()

            elif self.exch_strategy == 'asa32':

                self.exch.exchange()

            elif self.exch_strategy == 'asa16':

                self.exch.exchange()

            elif self.exch_strategy == 'copper16':

                self.exch.exchange()

        # sum delta w
        elif self.worker_type == 'cdd' and self.size > 1:

            if self.exch_strategy == 'ar':

                self.exch.exchange()

            elif self.exch_strategy == 'copper':

                self.exch.exchange()

            elif self.exch_strategy == 'asa32':

                self.exch.exchange()

            elif self.exch_strategy == 'asa16':

                self.exch.exchange()

            elif self.exch_strategy == 'copper16':

                self.exch.exchange()