def epoch(*args):
obj, kw=args
path_results=kw.get('path_results')
if not os.path.isdir(path_results):
data_to_disk.mkdir(path_results)
ja=kw.get('job_admin')(**kw)
wp=kw.get('wrapper_process')
ja.save_obj(obj)
ja.gen_job_script()
args=ja.get_subp_args()
p=do(*args, **kw)
ja.process=p
job_id=ja.get_job_id()
script_name=obj.get_name()
p=wp(p, job_id, script_name)
return p
def do(*args, **kwargs):
path_out, path_err=args[0:2]
args_call=args[2:]
path='/'.join(path_out.split('/')[0:-1])
if not os.path.isdir(path):
data_to_disk.mkdir(path)
f_out=open(path_out, "wb", 0)
f_err=open(path_err, "wb", 0)
if kwargs.get('debug', False):
p=subprocess.Popen(args_call, stderr=subprocess.STDOUT)
else:
print args_call
p=subprocess.Popen(args_call,
stdout=f_out,
stderr=f_err,
# stdout=subprocess.PIPE,
# stderr=subprocess.PIPE,
# stderr=subprocess.STDOUT,
)
f_out.close()
f_err.close()
# Needed such that process finish before evaluation in test
# script
# out, err = p.communicate()
# print out
# print err
return p
def setUp(self):
from core import data_to_disk
import os
self.path='/home/mikael/results/papers/inhibition/network/unittest/plot_settings/'
if not os.path.isdir(self.path):
data_to_disk.mkdir(self.path)
def setUp(self):
from core import data_to_disk
import os
self.path = '/home/mikael/results/papers/inhibition/network/unittest/plot_settings/'
if not os.path.isdir(self.path):
data_to_disk.mkdir(self.path)
def setUp(self):
from core.network import default_params
from core import data_to_disk
self.path = dr.HOME + '/results/unittest/my_nest/'
self.path_nest = self.path + 'nest/'
data_to_disk.mkdir(self.path_nest)
ResetKernel()
def setUp(self):
from core.network import default_params
from core import data_to_disk
self.path=dr.HOME+'/results/unittest/my_nest/'
self.path_nest=self.path+'nest/'
data_to_disk.mkdir(self.path_nest)
ResetKernel()
def get_path_nest(script_name, keys, par=None):
if not par:
par = default_params.Inhibition()
path = par.get_path_data()
file_name = path + script_name + '/' + '_'.join(keys) + '/nest/'
# file_name = home + '/results/papers/inhibition/network/' + script_name
data_to_disk.mkdir(file_name)
return file_name
def test_41_phase_diff_mpi(self):
from core.data_to_disk import mkdir
import pickle
import os
import subprocess
fs=1000.0
kwargs = get_kwargs_phase_diff(fs)
n_pop, sim_time=10, 500.0
x=dummy_data_pop(n_pop, **{'fs':fs,
'scale':0.5,'sim_time':sim_time})
y=dummy_data_pop(n_pop, **{'fs':fs,
'scale':0.5,'sim_time':sim_time,
'shift':0.})
kwargs['inspect']=False
kwargs['local_num_threads']=2
data_path= self.home+('/results/unittest/signal_processing'
+'/signal_processing_phase_diff_mpi/')
script_name=os.getcwd()+('/test_scripts_MPI/'
+'signal_processing_phase_diff_mpi.py')
fileName=data_path+'data_in.pkl'
fileOut=data_path+'data_out.pkl'
mkdir(data_path)
f=open(fileName, 'w') #open in binary mode
pickle.dump([x,y, kwargs], f, -1)
f.close()
np=4
p=subprocess.Popen(['mpirun', '-np', str(np), 'python',
script_name, fileName, fileOut],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
# stderr=subprocess.STDOUT
)
out, err = p.communicate()
# print out
# print err
kwargs['local_num_threads']=2*4
p0=phases_diff(x, y, **kwargs)
f=open(fileOut, 'rb') #open in binary mode
p1=pickle.load(f)
f.close()
l0=numpy.round(p0.ravel(),2)
l1=numpy.round(p1.ravel(),2)
# abs since order of signal comparisons are random
# se phase_diff
l0,l1=list(numpy.abs(l0)), list(numpy.abs(l1))
self.assertListEqual(l0,l1 )
def setUp(self):
self.home = expanduser("~")
self.n = 12
self.n_sets = 3
self.args = ['unittest']
self.kwargs = default_kwargs_net(self.n, self.n_sets)
self.sim_time = 10000.
dp = self.home + '/results/unittest/my_population/nest/'
data_to_disk.mkdir(dp)
my_nest.ResetKernel(display=False, data_path=dp)
my_nest.SetKernelStatus({'overwrite_files': True})
def setUp(self):
self.home=expanduser("~")
self.n=12
self.n_sets=3
self.args=['unittest']
self.kwargs=default_kwargs_net(self.n, self.n_sets)
self.sim_time=10000.
dp=self.home+'/results/unittest/my_population/nest/'
data_to_disk.mkdir(dp)
my_nest.ResetKernel(display=False, data_path=dp)
my_nest.SetKernelStatus({'overwrite_files':True})
def save(self):
d=self.get_params_as_dic()
path=dr.HOME_DATA+'/'+self.script_name+'/'+self.date_time
if not os.path.isdir(path):
data_to_disk.mkdir(path)
l=os.listdir(path)
n=len(l)
data_to_disk.pickle_save(d, path+'/data_'+str(n)+'_'+self.data_label)
def save(self):
d = self.get_params_as_dic()
path = dr.HOME_DATA + '/' + self.script_name + '/' + self.date_time
if not os.path.isdir(path):
data_to_disk.mkdir(path)
l = os.listdir(path)
n = len(l)
data_to_disk.pickle_save(
d, path + '/data_' + str(n) + '_' + self.data_label)
def __init__(self, **kw):
index=kw.get('index') #simulation index
# path_code=kw.get('path_code')
pr=kw.get('path_results')
self.num_mpi_task=kw.get('num-mpi-task')
# self.local_threads=10
self.p_subp_out=pr+"/std/subp/out{0:0>4}".format(index)
self.p_subp_err=pr+'/std/subp/err{0:0>4}'.format(index)
self.p_par=pr+'/params/run{0:0>4}.pkl'.format(index)
self.p_script=dr.HOME_CODE+'/core_old/core/parallel_excecution/simulation.py'
data_to_disk.mkdir('/'.join(self.p_subp_out.split('/')[0:-1]))
for key, value in kw.items():
self.__dict__[key] = value
def test_5_load_from_disk(self):
from os.path import expanduser
s = expanduser("~")
s= s+'/results/unittest/my_population'
data_to_disk.mkdir(s)
my_nest.SetKernelStatus({'local_num_threads':2,
'data_path':s,
'overwrite_files': True,})
self.kwargs['sd']['params'].update({'to_memory':False,
'to_file':True})
g=self.sim_group().get_spike_signal()
g[0].firing_rate(1, display=True)
# pylab.show()
for filename in os.listdir(s):
if filename.endswith(".gdf"):
os.remove(s+'/'+filename)
def setUp(self):
global subprocess
subprocess=Mockup_subprocess
# print data0()
self.path=dr.HOME+'/results/unittest/job_handler'
if not os.path.isdir(self.path):
data_to_disk.mkdir(self.path)
if not os.path.isfile(self.path+'/data0'):
data_to_disk.txt_save(data0(), self.path+'/data0', file_extension='')
if not os.path.isfile(self.path+'/data1'):
data_to_disk.txt_save(data1(), self.path+'/data1', file_extension='')
if not os.path.isfile(self.path+'/data2'):
data_to_disk.txt_save(data2(), self.path+'/data2', file_extension='')
if not os.path.isfile(self.path+'/data3'):
data_to_disk.txt_save(data3(), self.path+'/data3', file_extension='')
p_list=[Mockup_process('1',None), Mockup_process('2',None), Mockup_process('3',None)]
jobs=[28372, 28373, 28374]
names=['Net_0', 'Net_1', 'Net_2']
wp_list=[Mockup_wrap_process_milner(p,j,n) for p,j,n in zip(p_list, jobs,names)]
kw={'p_list':wp_list,
'loop_time':1,
'log_to_file':True,
'log_file_name':os.getcwd()+'/job_handler/log'}
self.obj_milner=Handler(**kw)
p_list=[Mockup_process('1',None), Mockup_process('2',None), Mockup_process('3',None)]
wp_list=[Mockup_wrap_process_batch(p,j,n) for p,j,n in zip(p_list, jobs,names)]
kw={'p_list':wp_list,
'loop_time':1,
'log_to_file':True,
'log_file_name':os.getcwd()+'/job_handler/log'}
self.obj_super=Handler(**kw)
self.obj_empty=Handler(kw={'p_list':[],
'loop_time':1,
'log_to_file':True,
'log_file_name':os.getcwd()+'/job_handler/log'})
def test_5_load_from_disk(self):
from os.path import expanduser
s = expanduser("~")
s = s + '/results/unittest/my_population'
data_to_disk.mkdir(s)
my_nest.SetKernelStatus({
'local_num_threads': 2,
'data_path': s,
'overwrite_files': True,
})
self.kwargs['sd']['params'].update({
'to_memory': False,
'to_file': True
})
g = self.sim_group().get_spike_signal()
g[0].firing_rate(1, display=True)
# pylab.show()
for filename in os.listdir(s):
if filename.endswith(".gdf"):
os.remove(s + '/' + filename)
def loop(*args, **kwargs):
n, m_list, args_list, kwargs_list=args
db_save=kwargs_list[0].get('database_save', False)
path_results=kwargs_list[0].get('path_results')
# process_type=kwargs_list[0].get('process_type')
# read_subp_jobs=kwargs_list[0].get('read_subp_jobs')
log_file_name=path_results+'/std/job_handler_log'
data_to_disk.mkdir(path_results+'/std/')
h=job_handler.Handler(loop_time=5,
log_to_file=True,
log_file_name=log_file_name,
**kwargs)
for m in m_list:
q=Queue.Queue()
for _ in range(m):
if not kwargs_list:
continue
k=kwargs_list.pop(0)
if k['active']:
q.put([args_list.pop(0), k])
else:
args_list.pop(0)
h.loop_with_queue(n, q, epoch,
loop_print=True)
if db_save:
save_to_database(path_results)
def __init__(self,**kw):
index=kw.get('index') #simulation index
pr=kw.get('path_results')
self.p_subp_out=pr+"/std/subp/out{0:0>4}".format(index)
self.p_subp_err=pr+'/std/subp/err{0:0>4}'.format(index)
self.p_tee_out=pr+'/std/tee/out{0:0>4}'.format(index)
self.p_par=pr+'/params/run{0:0>4}.pkl'.format(index)
self.p_script=dr.HOME_CODE+'/core_old/core/parallel_excecution/simulation.py'
self.p_bash0=dr.HOME_CODE+'/core_old/core/parallel_excecution/jobb0_supermicro.sh'
self.p_bash=pr+'/jobbs/jobb_{0:0>4}.sh'.format(index)
data_to_disk.mkdir('/'.join(self.p_subp_out.split('/')[0:-1]))
data_to_disk.mkdir('/'.join(self.p_tee_out.split('/')[0:-1]))
data_to_disk.mkdir('/'.join(self.p_bash.split('/')[0:-1]))
for key, value in kw.items():
self.__dict__[key] = value
def test_41_phase_diff_mpi(self):
from core.data_to_disk import mkdir
import pickle
import os
import subprocess
fs = 1000.0
kwargs = get_kwargs_phase_diff(fs)
n_pop, sim_time = 10, 500.0
x = dummy_data_pop(n_pop, **{
'fs': fs,
'scale': 0.5,
'sim_time': sim_time
})
y = dummy_data_pop(
n_pop, **{
'fs': fs,
'scale': 0.5,
'sim_time': sim_time,
'shift': 0.
})
kwargs['inspect'] = False
kwargs['local_num_threads'] = 2
data_path = self.home + ('/results/unittest/signal_processing' +
'/signal_processing_phase_diff_mpi/')
script_name = os.getcwd() + ('/test_scripts_MPI/' +
'signal_processing_phase_diff_mpi.py')
fileName = data_path + 'data_in.pkl'
fileOut = data_path + 'data_out.pkl'
mkdir(data_path)
f = open(fileName, 'w') #open in binary mode
pickle.dump([x, y, kwargs], f, -1)
f.close()
np = 4
p = subprocess.Popen(
[
'mpirun', '-np',
str(np), 'python', script_name, fileName, fileOut
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
# stderr=subprocess.STDOUT
)
out, err = p.communicate()
# print out
# print err
kwargs['local_num_threads'] = 2 * 4
p0 = phases_diff(x, y, **kwargs)
f = open(fileOut, 'rb') #open in binary mode
p1 = pickle.load(f)
f.close()
l0 = numpy.round(p0.ravel(), 2)
l1 = numpy.round(p1.ravel(), 2)
# abs since order of signal comparisons are random
# se phase_diff
l0, l1 = list(numpy.abs(l0)), list(numpy.abs(l1))
self.assertListEqual(l0, l1)
from core.network import default_params
from core import my_nest
from core import misc
from core.my_population import MyNetworkNode
import pprint
pp = pprint.pprint
from core.network.manager import get_storage_list, save, load
from core import directories as dir
from core import data_to_disk
import os
path = dir.HOME_DATA + '/' + __file__.split('/')[-1][0:-3]
if not os.path.isdir(path):
data_to_disk.mkdir(path)
par = default_params.Inhibition()
setup = Setup(50, 20)
def gs_builder(*args, **kwargs):
import matplotlib.gridspec as gridspec
n_rows = kwargs.get('n_rows', 2)
n_cols = kwargs.get('n_cols', 1)
order = kwargs.get('order', 'col')
gs = gridspec.GridSpec(n_rows, n_cols)
gs.update(wspace=kwargs.get('wspace', 0.1),
hspace=kwargs.get('hspace', 0.1))
iterator = [
def main():
record_from_models = [['CO', 'M1', 'M2', 'FS', 'GA', 'GI', 'ST', 'SN'],
[
'CO', 'M1', 'M2', 'F1', 'F2', 'GA', 'GI', 'ST',
'SN'
]]
labels = ['Control-h0', 'Control-h1']
start = 1000.0
stop = 1000.0 + 100. * 10.
kwargs = {
'kwargs_network': {
'save_conn': False,
'verbose': True
},
'par_rep': {
'simu': {
'threads': 2,
'sd_params': {
'to_file': True,
'to_memory': False
},
'print_time': True,
'start_rec': start,
'stop_rec': stop,
'sim_time': stop
},
'netw': {
'size': 5000.0,
'tata_dop': 0.8
}
}
}
use_class = [Bcpnn_h0, Bcpnn_h1]
#Inhibition_no_parrot
#for setup in setup_list: setup.extend([10000., 1000.0, 11000.0, Inhibition_no_parrot, {}])
setup_list = []
for l, uc in zip(labels, use_class):
kwargs['class_network_construction'] = uc
setup_list.append([l, deepcopy(kwargs)])
nms = Network_models_dic(setup_list, Network_model)
nms.simulate([0] * 2, labels, record_from_models)
plot_models = [['CO', 'M1', 'M2', 'FS', 'GI', 'SN'],
['M1', 'M2', 'F1', 'F2', 'GI', 'SN']]
plot_lables_models = [[
'Cortex', '$MSN_{D1}$', '$MSN_{D2}$', 'FSN', '$GPe_{Type I}$', 'SNr'
],
[
'$MSN_{D1}$', '$MSN_{D2}$', '$FSN_{1}$',
'$FSN_{2}$', '$GPe_{Type I}$', 'SNr'
]]
plot_lables_prefix_models = [[
'State', 'Action', 'Action', 'Action', 'Action', 'Action'
], ['Action', 'Action', 'Action', 'Action', 'Action', 'Action']]
figs = nms.show_bcpnn(labels,
plot_models,
xlim=[start, stop],
plot_lables_models=plot_lables_models,
plot_lables_prefix_models=plot_lables_prefix_models)
pylab.show()
i = 0
for fig in figs:
import os
if not os.path.isdir(nms.path_pictures):
data_to_disk.mkdir(nms.path_pictures)
fig.savefig(nms.path_pictures + '-fig' + str(i) + '.svg', format='svg')
i += 1
chunked= chunks(cmd_lines,4)
from subprocess import Popen, call
from os import mkdir
for cmd_lines in chunked:
for number, line in enumerate(cmd_lines):
script_name=line.split(' ')[1]
newpath = script_name
if not os.path.isdir('./'+newpath):
data_to_disk.mkdir('./'+newpath)
cmd = ('python /home/mikael/git/bgmodel/'+
'scripts_inhibition/do.py '
+ line.strip())
print 'Running %r in %r' % (cmd, newpath)
stdout= newpath+"/stdout.txt"
stderr=newpath+'/stderr.txt'
with open(stdout,"wb") as out, open(stderr,"wb") as err:
Popen(cmd, shell=True, stdout=out, stderr=err)
def p_out_data(self, host):
path = dr.HOME + '/results/unittest/parallel_excecution/' + host
data_to_disk.mkdir(path)
return path
from core.network import default_params
from core import my_nest, my_population
from core import misc
from core.my_population import MyNetworkNode
from core.my_signals import Data_generic, Data_IF_curve, Data_scatter
import pprint
pp=pprint.pprint
from core.network.manager import get_storage_list, save, load
from core import directories as dr
from core import data_to_disk
import os
path=dr.HOME_DATA+'/'+__file__.split('/')[-1][0:-3]
if not os.path.isdir(path):
data_to_disk.mkdir(path)
par=default_params.Inhibition()
setup=Setup(50,20)
def gs_builder(*args, **kwargs):
import matplotlib.gridspec as gridspec
n_rows=kwargs.get('n_rows',2)
n_cols=kwargs.get('n_cols',1)
order=kwargs.get('order', 'col')
gs = gridspec.GridSpec(n_rows, n_cols)
gs.update(wspace=kwargs.get('wspace', 0.1 ),
hspace=kwargs.get('hspace', 0.1 ))
iterator = [[slice(0,1),slice(0,1)],
[slice(0,1),slice(1,2)],
def create_folders(l):
for e in l:
path = '/'.join(e.split('/')[0:-1])
data_to_disk.mkdir(path)
assert os.path.isdir(path), path + ' failed to be created'
def p_out_data(self, host):
path=dr.HOME+'/results/unittest/parallel_excecution/'+host
data_to_disk.mkdir(path)
return path
def setUp(self):
global subprocess
subprocess = Mockup_subprocess
# print data0()
self.path = dr.HOME + '/results/unittest/job_handler'
if not os.path.isdir(self.path):
data_to_disk.mkdir(self.path)
if not os.path.isfile(self.path + '/data0'):
data_to_disk.txt_save(data0(),
self.path + '/data0',
file_extension='')
if not os.path.isfile(self.path + '/data1'):
data_to_disk.txt_save(data1(),
self.path + '/data1',
file_extension='')
if not os.path.isfile(self.path + '/data2'):
data_to_disk.txt_save(data2(),
self.path + '/data2',
file_extension='')
if not os.path.isfile(self.path + '/data3'):
data_to_disk.txt_save(data3(),
self.path + '/data3',
file_extension='')
p_list = [
Mockup_process('1', None),
Mockup_process('2', None),
Mockup_process('3', None)
]
jobs = [28372, 28373, 28374]
names = ['Net_0', 'Net_1', 'Net_2']
wp_list = [
Mockup_wrap_process_milner(p, j, n)
for p, j, n in zip(p_list, jobs, names)
]
kw = {
'p_list': wp_list,
'loop_time': 1,
'log_to_file': True,
'log_file_name': os.getcwd() + '/job_handler/log'
}
self.obj_milner = Handler(**kw)
p_list = [
Mockup_process('1', None),
Mockup_process('2', None),
Mockup_process('3', None)
]
wp_list = [
Mockup_wrap_process_batch(p, j, n)
for p, j, n in zip(p_list, jobs, names)
]
kw = {
'p_list': wp_list,
'loop_time': 1,
'log_to_file': True,
'log_file_name': os.getcwd() + '/job_handler/log'
}
self.obj_super = Handler(**kw)
self.obj_empty = Handler(
kw={
'p_list': [],
'loop_time': 1,
'log_to_file': True,
'log_file_name': os.getcwd() + '/job_handler/log'
})
