def run_data_base_dump(run, net, script_name, net_name, category, file_name, **kwargs): import nest ks = nest.GetKernelStatus() db_name = kwargs.get('database_name', 'inhibition') db_table = kwargs.get('database_name', 'simulations') pp(ks['local_num_threads']) #need to supply not right picked up by nest lnt = kwargs.get('local_num_threads', ks['local_num_threads']) t = time.time() dd = run(net) par_data = cPickle.dumps(net.par.dic, -1) keys_db = [ 'computer', #varchar 'category', 'default_params', #bytea 'duration', #float 'local_num_threads', #int 'net_name', #varchar 'num_processes', #int 'script', #varchar 'simulation', 'size', #int 'simulation_time', #float 'total_num_virtual_procs' #int ] l = script_name.split('/') if len(l) == 2: simulation, script = l if len(l) == 1: simulation = l[0] script = '' values_db = [ my_socket.determine_computer(), category, # psycopg2.Binary(par_data), par_data, float(round(time.time() - t, 1)), int(lnt), net_name, int(ks['num_processes']), script, simulation, int(net.par.dic['netw']['size']), float(net.par.dic['simu']['sim_time']), int(ks['num_processes'] * lnt), ] to_binary = [False, False, True] + [False] * 9 data = [db_name, db_table, keys_db, values_db, to_binary] data_to_disk.pickle_save(data, file_name + '/' + net_name, file_extension='.db_dump') return dd
def run_data_base_dump(run, net, script_name, net_name, category, file_name, **kwargs): import nest ks=nest.GetKernelStatus() db_name=kwargs.get('database_name','inhibition') db_table=kwargs.get('database_name','simulations') pp(ks['local_num_threads']) #need to supply not right picked up by nest lnt=kwargs.get('local_num_threads',ks['local_num_threads']) t=time.time() dd=run(net) par_data = cPickle.dumps(net.par.dic, -1) keys_db=['computer',#varchar 'category', 'default_params', #bytea 'duration', #float 'local_num_threads', #int 'net_name', #varchar 'num_processes', #int 'script', #varchar 'simulation', 'size', #int 'simulation_time', #float 'total_num_virtual_procs' #int ] l=script_name.split('/') if len(l)==2: simulation, script=l if len(l)==1: simulation=l[0] script='' values_db=[my_socket.determine_computer(), category, # psycopg2.Binary(par_data), par_data, float(round(time.time()-t,1)), int(lnt), net_name, int(ks['num_processes']), script, simulation, int(net.par.dic['netw']['size']), float(net.par.dic['simu']['sim_time']), int(ks['num_processes']*lnt), ] to_binary=[False, False, True ]+[False]*9 data=[db_name, db_table, keys_db, values_db, to_binary] data_to_disk.pickle_save(data, file_name +'/'+net_name, file_extension= '.db_dump') return dd
def map_local_threads(fun, args, kwargs): local_th = int(kwargs.get('local_num_threads', 1)) if (local_th == 1) or my_socket.determine_computer() == 'milner': return map(fun, *args) n = len(args[0]) chunksize = int(math.floor(len(args[0]) / float(local_th))) jobs = [] tmp_args = [a[0] for a in args] tmp_r = fun(*tmp_args) if not type(tmp_r) in [list, tuple, numpy.ndarray]: print 'array', comm.rank() if type(tmp_r) == int: out = Array('l', numpy.zeros(n, dtype=numpy.int64)) else: out = Array('d', numpy.zeros(n)) for i in xrange(local_th): last = i == local_th - 1 a = chunkit(chunksize, i, last, *args) p = Process(target=my_map, args=(i, chunksize, last, fun, out, a)) jobs.append(p) for job in jobs: job.start() for job in jobs: job.join() r = list(out[:]) else: pool = Pool(processes=local_th) fun = Wrap(fun) args = izip(*args) f = fun.use r = pool.map(f, args) #, chunksize=len(args)/local_th) # Necessary to shut threads down. Otherwise just more and more threads # are created comm.barrier() pool.close() pool.join() return r
def map_local_threads(fun, args, kwargs): local_th=int(kwargs.get('local_num_threads', 1)) if (local_th==1) or my_socket.determine_computer()=='milner': return map(fun, *args) n=len(args[0]) chunksize=int(math.floor(len(args[0]) / float(local_th))) jobs=[] tmp_args=[a[0] for a in args ] tmp_r=fun(*tmp_args) if not type(tmp_r) in [list, tuple, numpy.ndarray]: print 'array', comm.rank() if type(tmp_r)==int: out = Array( 'l', numpy.zeros(n, dtype=numpy.int64) ) else: out = Array( 'd', numpy.zeros(n) ) for i in xrange(local_th): last=i==local_th-1 a=chunkit(chunksize, i, last, *args) p=Process(target=my_map, args=(i, chunksize, last, fun, out, a)) jobs.append(p) for job in jobs: job.start() for job in jobs: job.join() r=list(out[:]) else: pool = Pool(processes=local_th) fun = Wrap(fun) args=izip(*args) f=fun.use r = pool.map(f, args)#, chunksize=len(args)/local_th) # Necessary to shut threads down. Otherwise just more and more threads # are created comm.barrier() pool.close() pool.join() return r
'proportion_connected':proportion_connected, #related to toal number fo runs 'p_sizes':[1.]*NUM_RUNS, 'p_subsamp':[1.]*NUM_RUNS, 'STN_amp_mod':STN_amp_mod, 'threshold':14., 'tuning_freq_amp_to':'M2', 'wrapper_process':WRAPPER_PROCESS, #user defined wrapper of subprocesses } if my_socket.determine_computer()=='milner': kw_add={ 'duration':[907.,100.0], 'laptime':1007.0, 'res':7, 'rep':10, 'time_bin':100., } elif my_socket.determine_computer() in ['thalamus','supermicro']: kw_add={ 'duration':[357., 100.0], 'laptime':457., 'res':3, 'rep':5, 'time_bin':1000./256,
'labels': ['Only D1 no dop', 'D1,D2 no dop'], 'local_threads_milner': 40, 'local_threads_superm': 4, 'max_size': 20000., 'module': module, 'nets': ['Net_0', 'Net_1'], 'other_scenario': True, 'path_code': default_params.HOME_CODE, 'path_results': get_path_logs(LOAD_MILNER_ON_SUPERMICRO, FILE_NAME), 'perturbation_list': [op.get()[5]], 'proportion_connected': [0.2] * NUM_NETS, #related to toal number fo runs 'p_sizes': [1.], 'p_subsamp': [1.], 'STN_amp_mod': STN_amp_mod, } if my_socket.determine_computer() == 'milner': kw_add = { 'duration': [907., 100.0], 'laptime': 1007.0, 'res': 10, 'rep': 40, 'time_bin': 100., } elif my_socket.determine_computer() == 'supermicro': kw_add = { 'duration': [357., 100.0], 'laptime': 457., 'res': 3, 'rep': 5, 'time_bin': 1000 / 256., }
def __init__(self, name='test'): self.name = name self.computer = my_socket.determine_computer() self.conn = None self.cur = None self.spf = None #po4t forwarding object
def __init__(self, name='test'): self.name=name self.computer = my_socket.determine_computer() self.conn=None self.cur=None self.spf=None #po4t forwarding object
'proportion_connected':[0.2]*NUM_NETS, #related to toal number fo runs 'p_sizes':[ 1. ], 'p_subsamp':[ 1. ], 'STN_amp_mod':STN_amp_mod, 'tuning_freq_amp_to':'M2', 'wrapper_process':WRAPPER_PROCESS, #user defined wrapper of subprocesses } if my_socket.determine_computer()=='milner': kw_add={ 'duration':[907.,100.0], 'laptime':1007.0, 'res':10, 'rep':40, 'time_bin':100., } elif my_socket.determine_computer()=='supermicro': kw_add={ 'duration':[357., 100.0], 'laptime':457., 'res':3, 'rep':5, 'time_bin':1000/256.,