def run_data_base_dump(run, net, script_name, net_name, category, file_name,
**kwargs):
import nest
ks = nest.GetKernelStatus()
db_name = kwargs.get('database_name', 'inhibition')
db_table = kwargs.get('database_name', 'simulations')
pp(ks['local_num_threads'])
#need to supply not right picked up by nest
lnt = kwargs.get('local_num_threads', ks['local_num_threads'])
t = time.time()
dd = run(net)
par_data = cPickle.dumps(net.par.dic, -1)
keys_db = [
'computer', #varchar
'category',
'default_params', #bytea
'duration', #float
'local_num_threads', #int
'net_name', #varchar
'num_processes', #int
'script', #varchar
'simulation',
'size', #int
'simulation_time', #float
'total_num_virtual_procs' #int
]
l = script_name.split('/')
if len(l) == 2:
simulation, script = l
if len(l) == 1:
simulation = l[0]
script = ''
values_db = [
my_socket.determine_computer(),
category,
# psycopg2.Binary(par_data),
par_data,
float(round(time.time() - t, 1)),
int(lnt),
net_name,
int(ks['num_processes']),
script,
simulation,
int(net.par.dic['netw']['size']),
float(net.par.dic['simu']['sim_time']),
int(ks['num_processes'] * lnt),
]
to_binary = [False, False, True] + [False] * 9
data = [db_name, db_table, keys_db, values_db, to_binary]
data_to_disk.pickle_save(data,
file_name + '/' + net_name,
file_extension='.db_dump')
return dd
def run_data_base_dump(run, net, script_name, net_name, category, file_name, **kwargs):
import nest
ks=nest.GetKernelStatus()
db_name=kwargs.get('database_name','inhibition')
db_table=kwargs.get('database_name','simulations')
pp(ks['local_num_threads'])
#need to supply not right picked up by nest
lnt=kwargs.get('local_num_threads',ks['local_num_threads'])
t=time.time()
dd=run(net)
par_data = cPickle.dumps(net.par.dic, -1)
keys_db=['computer',#varchar
'category',
'default_params', #bytea
'duration', #float
'local_num_threads', #int
'net_name', #varchar
'num_processes', #int
'script', #varchar
'simulation',
'size', #int
'simulation_time', #float
'total_num_virtual_procs' #int
]
l=script_name.split('/')
if len(l)==2:
simulation, script=l
if len(l)==1:
simulation=l[0]
script=''
values_db=[my_socket.determine_computer(),
category,
# psycopg2.Binary(par_data),
par_data,
float(round(time.time()-t,1)),
int(lnt),
net_name,
int(ks['num_processes']),
script,
simulation,
int(net.par.dic['netw']['size']),
float(net.par.dic['simu']['sim_time']),
int(ks['num_processes']*lnt),
]
to_binary=[False,
False,
True
]+[False]*9
data=[db_name, db_table, keys_db, values_db, to_binary]
data_to_disk.pickle_save(data, file_name +'/'+net_name,
file_extension= '.db_dump')
return dd
def map_local_threads(fun, args, kwargs):
local_th = int(kwargs.get('local_num_threads', 1))
if (local_th == 1) or my_socket.determine_computer() == 'milner':
return map(fun, *args)
n = len(args[0])
chunksize = int(math.floor(len(args[0]) / float(local_th)))
jobs = []
tmp_args = [a[0] for a in args]
tmp_r = fun(*tmp_args)
if not type(tmp_r) in [list, tuple, numpy.ndarray]:
print 'array', comm.rank()
if type(tmp_r) == int:
out = Array('l', numpy.zeros(n, dtype=numpy.int64))
else:
out = Array('d', numpy.zeros(n))
for i in xrange(local_th):
last = i == local_th - 1
a = chunkit(chunksize, i, last, *args)
p = Process(target=my_map, args=(i, chunksize, last, fun, out, a))
jobs.append(p)
for job in jobs:
job.start()
for job in jobs:
job.join()
r = list(out[:])
else:
pool = Pool(processes=local_th)
fun = Wrap(fun)
args = izip(*args)
f = fun.use
r = pool.map(f, args) #, chunksize=len(args)/local_th)
# Necessary to shut threads down. Otherwise just more and more threads
# are created
comm.barrier()
pool.close()
pool.join()
return r
def map_local_threads(fun, args, kwargs):
local_th=int(kwargs.get('local_num_threads', 1))
if (local_th==1) or my_socket.determine_computer()=='milner':
return map(fun, *args)
n=len(args[0])
chunksize=int(math.floor(len(args[0]) / float(local_th)))
jobs=[]
tmp_args=[a[0] for a in args ]
tmp_r=fun(*tmp_args)
if not type(tmp_r) in [list, tuple, numpy.ndarray]:
print 'array', comm.rank()
if type(tmp_r)==int:
out = Array( 'l', numpy.zeros(n, dtype=numpy.int64) )
else:
out = Array( 'd', numpy.zeros(n) )
for i in xrange(local_th):
last=i==local_th-1
a=chunkit(chunksize, i, last, *args)
p=Process(target=my_map, args=(i, chunksize, last, fun, out, a))
jobs.append(p)
for job in jobs: job.start()
for job in jobs: job.join()
r=list(out[:])
else:
pool = Pool(processes=local_th)
fun = Wrap(fun)
args=izip(*args)
f=fun.use
r = pool.map(f, args)#, chunksize=len(args)/local_th)
# Necessary to shut threads down. Otherwise just more and more threads
# are created
comm.barrier()
pool.close()
pool.join()
return r
'proportion_connected':proportion_connected, #related to toal number fo runs
'p_sizes':[1.]*NUM_RUNS,
'p_subsamp':[1.]*NUM_RUNS,
'STN_amp_mod':STN_amp_mod,
'threshold':14.,
'tuning_freq_amp_to':'M2',
'wrapper_process':WRAPPER_PROCESS, #user defined wrapper of subprocesses
}
if my_socket.determine_computer()=='milner':
kw_add={
'duration':[907.,100.0],
'laptime':1007.0,
'res':7,
'rep':10,
'time_bin':100.,
}
elif my_socket.determine_computer() in ['thalamus','supermicro']:
kw_add={
'duration':[357., 100.0],
'laptime':457.,
'res':3,
'rep':5,
'time_bin':1000./256,
'labels': ['Only D1 no dop', 'D1,D2 no dop'],
'local_threads_milner': 40,
'local_threads_superm': 4,
'max_size': 20000.,
'module': module,
'nets': ['Net_0', 'Net_1'],
'other_scenario': True,
'path_code': default_params.HOME_CODE,
'path_results': get_path_logs(LOAD_MILNER_ON_SUPERMICRO, FILE_NAME),
'perturbation_list': [op.get()[5]],
'proportion_connected': [0.2] * NUM_NETS, #related to toal number fo runs
'p_sizes': [1.],
'p_subsamp': [1.],
'STN_amp_mod': STN_amp_mod,
}
if my_socket.determine_computer() == 'milner':
kw_add = {
'duration': [907., 100.0],
'laptime': 1007.0,
'res': 10,
'rep': 40,
'time_bin': 100.,
}
elif my_socket.determine_computer() == 'supermicro':
kw_add = {
'duration': [357., 100.0],
'laptime': 457.,
'res': 3,
'rep': 5,
'time_bin': 1000 / 256.,
}
def __init__(self, name='test'):
self.name = name
self.computer = my_socket.determine_computer()
self.conn = None
self.cur = None
self.spf = None #po4t forwarding object
def __init__(self, name='test'):
self.name=name
self.computer = my_socket.determine_computer()
self.conn=None
self.cur=None
self.spf=None #po4t forwarding object
'proportion_connected':[0.2]*NUM_NETS, #related to toal number fo runs
'p_sizes':[
1.
],
'p_subsamp':[
1.
],
'STN_amp_mod':STN_amp_mod,
'tuning_freq_amp_to':'M2',
'wrapper_process':WRAPPER_PROCESS, #user defined wrapper of subprocesses
}
if my_socket.determine_computer()=='milner':
kw_add={
'duration':[907.,100.0],
'laptime':1007.0,
'res':10,
'rep':40,
'time_bin':100.,
}
elif my_socket.determine_computer()=='supermicro':
kw_add={
'duration':[357., 100.0],
'laptime':457.,
'res':3,
'rep':5,
'time_bin':1000/256.,
