def save_fig(self, *args): if comm.is_mpi_used(): with Barrier(): if comm.rank() == 0: self._save_fig(*args) else: self._save_fig(*args)
def save_fig(self, *args): if comm.is_mpi_used(): with Barrier(): if comm.rank()==0: self._save_fig(*args) else: self._save_fig(*args)
def clear_paths(self, cb): for key in [ 'p_tee_out', 'p_subp_out', 'p_subp_err', 'p_bash', 'p_sbatch_out', 'p_sbatch_err' ]: if cb.get(key) and os.path.isdir(cb.get(key)): if comm.rank() == 0: os.remove(cb.get(key))
def _get_spikes_from_file_mpi(file_names): with Barrier(): if comm.rank()==0: data=_get_spikes_from_file(file_names) else: data=None with Barrier(): data=comm.bcast(data, root=0) return data
def _get_spikes_from_file_mpi(file_names): with Barrier(): if comm.rank() == 0: data = _get_spikes_from_file(file_names) else: data = None with Barrier(): data = comm.bcast(data, root=0) return data
def _text_load_mpi(f): with Barrier(): if comm.rank()==0: data=_text_load(f) else: data=None with Barrier(): data=comm.bcast(data, root=0) return data
def _text_load_mpi(f): with Barrier(): if comm.rank() == 0: data = _text_load(f) else: data = None with Barrier(): data = comm.bcast(data, root=0) return data
def collect_spikes_mpi(*args): args = list(args) for i in range(len(args)): with Barrier(): if comm.rank() == 0: for i_proc in xrange(1, comm.size()): args[i] = numpy.r_[args[i], comm.recv(source=i_proc)] else: comm.send(args[i], dest=0) args[i] = comm.bcast(args[i], root=0) return args
def collect_spikes_mpi(*args): args=list(args) for i in range(len(args)): with Barrier(): if comm.rank()==0: for i_proc in xrange(1, comm.size()): args[i] = numpy.r_[args[i], comm.recv(source=i_proc)] else: comm.send(args[i],dest=0) args[i]=comm.bcast(args[i], root=0) return args
def set(self, surfs, display_print=True): t=time.time() # Convergent connections when driver = target and pool = source. # The for each node driver not surfs from the pool is considered # Divergent connections when driver = source and pool = target. # Then for each node driver not surfs from the pool is considered driver=surfs[self.target] pool=surfs[self.source] if not (self.save['active'] and not self.save['overwrite'] and os.path.isfile(self.save['path'] +'.pkl')): self._set(driver, pool) self._save() else: d=data_to_disk.pickle_load(self.save['path'] ) self.pre=sparse.coo_matrix(d[0]) self.post=sparse.coo_matrix(d[1]) self.sets=d[2] # self.pre=numpy.array([[0]])#sparse.coo_matrix(d[0]) # self.post=numpy.array([[0]])#sparse.coo_matrix(d[1]) # self.sets=[0]#d[2] # t=time.time()-t if display_print and comm.rank()==0: s='Conn: {0:18} Connections: {1:8} Fan pool:{2:6} ({3:6}) Time:{4:5} sec Rule:{5}' a=[self.name, self.n, round(float(self.n)/driver.get_n(),0), self.fan_in, round(t,2), self.rule] print s.format(*a)
def set(self, surfs, display_print=True): t = time.time() # Convergent connections when driver = target and pool = source. # The for each node driver not surfs from the pool is considered # Divergent connections when driver = source and pool = target. # Then for each node driver not surfs from the pool is considered driver = surfs[self.target] pool = surfs[self.source] if not (self.save['active'] and not self.save['overwrite'] and os.path.isfile(self.save['path'] + '.pkl')): self._set(driver, pool) self._save() else: d = data_to_disk.pickle_load(self.save['path']) self.pre = sparse.coo_matrix(d[0]) self.post = sparse.coo_matrix(d[1]) self.sets = d[2] # self.pre=numpy.array([[0]])#sparse.coo_matrix(d[0]) # self.post=numpy.array([[0]])#sparse.coo_matrix(d[1]) # self.sets=[0]#d[2] # t = time.time() - t if display_print and comm.rank() == 0: s = 'Conn: {0:18} Connections: {1:8} Fan pool:{2:6} ({3:6}) Time:{4:5} sec Rule:{5}' a = [ self.name, self.n, round(float(self.n) / driver.get_n(), 0), self.fan_in, round(t, 2), self.rule ] print s.format(*a)
def _save_mpi(self): with Barrier(): if comm.rank()==0: self._save()
def connect_conns(params_nest, conns, popus, display_print=False): for c in conns: # if c.name[0] not in ['E','C']: # if c.name[0:5] not in ['C1_M1', 'C2_M2']: #if c.name[0:5] not in ['E']: # # continue if display_print and comm.rank() == 0: print 'Connecting ' + str(c) my_nest.MyCopyModel(params_nest[c.get_syn()], c.get_syn()) #c.copy_model( params_nest ) sr_ids = numpy.array(popus[c.get_source()].ids) tr_ids = numpy.array(popus[c.get_target()].ids) c.set_local_lockup(tr_ids) weights = list(c.get_weights_local()) delays = list(c.get_delays_local()) pre = list(sr_ids[c.get_pre_local()]) post = list(tr_ids[c.get_post_local()]) import time # time.sleep(0.1*my_nest.Rank()) # print c,'lu',c._get_local_lockup() # print c,'pre', c.get_pre_local()[0:10], len(pre) # print c,'post', c.get_post_local()[0:10], len(pre) # print sr_ids[0:10] # print tr_ids[0:10] # print c,'pre2', pre[0:10], len(pre) # print c,'pos2t', post[0:10], len(pre) # comm.barrier() # weights_input_rates=list(c.get_weights()) # delays=list(c.get_delays()) # pre=list(sr_ids[c.get_pre()]) # post=list(tr_ids[c.get_post()]) model = c.get_syn() c.clear() del c gc.collect() # delete_and_gc_collect(c) # my_nest.Connect_DC(pre, post , weights_input_rates, delays, model, only_local=True) my_nest.Connect_speed(pre, post, weights, delays, model=model) # # syn_dict={ # "model":model, # "weight":weights, # "delay":delays, # } # conn_dict={ # "rule": "one_to_one", # } # import nest # nest.Connect(pre, post, conn_dict, syn_dict) # pp(my_nest.GetConnections(list(set(pre)))[0]) # pp(my_nest.GetConnections(post)[0]) # print pre[0], post[0] # print model, len(pre), len(post), len(weights), len(delays) # print 'Connecting ' + ' my_nest.GetConnections ', len(my_nest.GetConnections(list(set(pre)))), len(pre) # print 'Connecting ' + ' my_nest.GetConnections ', len(my_nest.GetConnections(post)), len(post) # delete_and_gc_collect(weights_input_rates, delays, pre, post) # del weights del delays del pre del post gc.collect()
def _set_mpi(self, *args): # with Barrier(True): if comm.rank() == 0: self._set(*args) comm.barrier()
def _garbage_collect_mpi(self): with Barrier(): if comm.rank() == 0: self._garbage_collect()
def _text_save_mpi(*args): with Barrier(): if comm.rank()==0: _text_save(*args)
def _mkdir_mpi(*args): with Barrier(): if comm.rank()==0: _mkdir(*args)
def _set_mpi(self, *args): # with Barrier(True): if comm.rank()==0: self._set(*args) comm.barrier()
def _text_save_mpi(*args): with Barrier(): if comm.rank() == 0: _text_save(*args)
def _pickle_save_mpi(*args): with Barrier(): if comm.rank() == 0: # OBS!!! watch out for having barriers inside here. # Will stall program _pickle_save(*args)
def _mkdir_mpi(*args): with Barrier(): if comm.rank() == 0: _mkdir(*args)
def connect_conns(params_nest, conns, popus, display_print=False): for c in conns: # if c.name[0] not in ['E','C']: # if c.name[0:5] not in ['C1_M1', 'C2_M2']: #if c.name[0:5] not in ['E']: # # continue if display_print and comm.rank()==0: print 'Connecting '+str(c) my_nest.MyCopyModel( params_nest[c.get_syn()], c.get_syn()) #c.copy_model( params_nest ) sr_ids=numpy.array(popus[c.get_source()].ids) tr_ids=numpy.array(popus[c.get_target()].ids) c.set_local_lockup(tr_ids) weights=list(c.get_weights_local()) delays=list(c.get_delays_local()) pre=list(sr_ids[c.get_pre_local()]) post=list(tr_ids[c.get_post_local()]) import time # time.sleep(0.1*my_nest.Rank()) # print c,'lu',c._get_local_lockup() # print c,'pre', c.get_pre_local()[0:10], len(pre) # print c,'post', c.get_post_local()[0:10], len(pre) # print sr_ids[0:10] # print tr_ids[0:10] # print c,'pre2', pre[0:10], len(pre) # print c,'pos2t', post[0:10], len(pre) # comm.barrier() # weights_input_rates=list(c.get_weights()) # delays=list(c.get_delays()) # pre=list(sr_ids[c.get_pre()]) # post=list(tr_ids[c.get_post()]) model=c.get_syn() c.clear() del c gc.collect() # delete_and_gc_collect(c) # my_nest.Connect_DC(pre, post , weights_input_rates, delays, model, only_local=True) my_nest.Connect_speed(pre, post , weights, delays, model=model) # # syn_dict={ # "model":model, # "weight":weights, # "delay":delays, # } # conn_dict={ # "rule": "one_to_one", # } # import nest # nest.Connect(pre, post, conn_dict, syn_dict) # pp(my_nest.GetConnections(list(set(pre)))[0]) # pp(my_nest.GetConnections(post)[0]) # print pre[0], post[0] # print model, len(pre), len(post), len(weights), len(delays) # print 'Connecting ' + ' my_nest.GetConnections ', len(my_nest.GetConnections(list(set(pre)))), len(pre) # print 'Connecting ' + ' my_nest.GetConnections ', len(my_nest.GetConnections(post)), len(post) # delete_and_gc_collect(weights_input_rates, delays, pre, post) # del weights del delays del pre del post gc.collect()
def _delete_data_mpi(path, **kwargs): with Barrier(): if comm.rank()==0: _delete_data(path, **kwargs)
def _delete_data_mpi(path, **kwargs): with Barrier(): if comm.rank() == 0: _delete_data(path, **kwargs)
def _garbage_collect_mpi(self): with Barrier(): if comm.rank()==0: self._garbage_collect()
def _save_mpi(self): with Barrier(): if comm.rank() == 0: self._save()
def _pickle_save_mpi(*args): with Barrier(): if comm.rank()==0: # OBS!!! watch out for having barriers inside here. # Will stall program _pickle_save(*args)
def clear_paths(self, cb): for key in ['p_tee_out', 'p_subp_out','p_subp_err', 'p_bash','p_sbatch_out', 'p_sbatch_err']: if cb.get(key) and os.path.isdir(cb.get(key)): if comm.rank()==0: os.remove(cb.get(key))
''' Created on Jul 15, 2014 @author: mikael ''' from core.data_to_disk import pickle_load from core.parallelization import comm import sys path=sys.argv[1]+str(comm.rank()) # path='/home/mikael/git/bgmodel/core_old/misc_folder/test_subprocess/00' print path fun, args, kwargs=pickle_load(path) fun(*args, **kwargs)