def test_file_overwriting(self):
self.traj.f_store()
with pt.open_file(self.filename, mode='r') as file:
nchildren = len(file.root._v_children)
self.assertTrue(nchildren > 0)
env2 = Environment(filename=self.filename,
log_config=get_log_config())
traj2 = env2.v_trajectory
traj2.f_store()
self.assertTrue(os.path.exists(self.filename))
with pt.open_file(self.filename, mode='r') as file:
nchildren = len(file.root._v_children)
self.assertTrue(nchildren > 1)
env3 = Environment(filename=self.filename, overwrite_file=True,
log_config=get_log_config())
self.assertFalse(os.path.exists(self.filename))
env2.f_disable_logging()
env3.f_disable_logging()
def test_time_display_of_loading(self):
filename = make_temp_dir('sloooow.hdf5')
env = Environment(trajectory='traj', add_time=True, filename=filename,
log_stdout=False,
log_config=get_log_config(),
dynamic_imports=SlowResult,
display_time=0.1)
traj = env.v_traj
res=traj.f_add_result(SlowResult, 'iii', 42, 43, comment='llk')
traj.f_store()
service_logger = traj.v_storage_service._logger
root = logging.getLogger('pypet')
old_level = root.level
service_logger.setLevel(logging.INFO)
root.setLevel(logging.INFO)
traj.f_load(load_data=3)
service_logger.setLevel(old_level)
root.setLevel(old_level)
path = get_log_path(traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
self.assertTrue('nodes/s)' in full_text)
env.f_disable_logging()
def test_file_overwriting(self):
self.traj.f_store()
with ptcompat.open_file(self.filename, mode='r') as file:
nchildren = len(file.root._v_children)
self.assertTrue(nchildren > 0)
env2 = Environment(filename=self.filename,
log_config=get_log_config())
traj2 = env2.v_trajectory
traj2.f_store()
self.assertTrue(os.path.exists(self.filename))
with ptcompat.open_file(self.filename, mode='r') as file:
nchildren = len(file.root._v_children)
self.assertTrue(nchildren > 1)
env3 = Environment(filename=self.filename, overwrite_file=True,
log_config=get_log_config())
self.assertFalse(os.path.exists(self.filename))
env2.f_disable_logging()
env3.f_disable_logging()
def test_time_display_of_loading(self):
filename = make_temp_dir('sloooow.hdf5')
env = Environment(trajectory='traj', add_time=True, filename=filename,
log_stdout=False,
log_config=get_log_config(),
dynamic_imports=SlowResult,
display_time=0.1)
traj = env.v_traj
res=traj.f_add_result(SlowResult, 'iii', 42, 43, comment='llk')
traj.f_store()
service_logger = traj.v_storage_service._logger
root = logging.getLogger('pypet')
old_level = root.level
service_logger.setLevel(logging.INFO)
root.setLevel(logging.INFO)
traj.f_load(load_data=3)
service_logger.setLevel(old_level)
root.setLevel(old_level)
path = get_log_path(traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
self.assertTrue('nodes/s)' in full_text)
env.f_disable_logging()
def main():
# Let's be very verbose!
logging.basicConfig(level = logging.INFO)
# Let's do multiprocessing this time with a lock (which is default)
filename = os.path.join('hdf5', 'example_07.hdf5')
env = Environment(trajectory='Example_07_BRIAN',
filename=filename,
file_title='Example_07_Brian',
comment = 'Go Brian!',
dynamically_imported_classes=[BrianMonitorResult, BrianParameter],
multiproc=True,
wrap_mode='QUEUE',
ncores=2)
traj = env.v_trajectory
# 1st a) add the parameters
add_params(traj)
# 1st b) prepare, we want to explore the different network sizes and different tauw time scales
traj.f_explore(cartesian_product({traj.f_get('N').v_full_name:[50,60],
traj.f_get('tauw').v_full_name:[30*ms,40*ms]}))
# 2nd let's run our experiment
env.f_run(run_net)
# You can take a look at the results in the hdf5 file if you want!
# Finally disable logging and close all log-files
env.f_disable_logging()
def main():
filename = os.path.join('hdf5', 'Clustered_Network.hdf5')
env = Environment(trajectory='Clustered_Network',
add_time=False,
filename=filename,
continuable=False,
lazy_debug=False,
multiproc=True,
ncores=2,
use_pool=False, # We cannot use a pool, our network cannot be pickled
wrap_mode='QUEUE',
overwrite_file=True)
#Get the trajectory container
traj = env.v_trajectory
# We introduce a `meta` parameter that we can use to easily rescale our network
scale = 0.5 # To obtain the results from the paper scale this to 1.0
# Be aware that your machine will need a lot of memory then!
traj.f_add_parameter('simulation.scale', scale,
comment='Meta parameter that can scale default settings. '
'Rescales number of neurons and connections strenghts, but '
'not the clustersize.')
# We create a Manager and pass all our components to the Manager.
# Note the order, CNNeuronGroups are scheduled before CNConnections,
# and the Fano Factor computation depends on the CNMonitorAnalysis
clustered_network_manager = NetworkManager(network_runner=CNNetworkRunner(),
component_list=(CNNeuronGroup(), CNConnections()),
analyser_list=(CNMonitorAnalysis(),CNFanoFactorComputer()))
# Add original parameters (but scaled according to `scale`)
clustered_network_manager.add_parameters(traj)
# We need `tolist` here since our parameter is a python float and not a
# numpy float.
explore_list = np.arange(1.0, 2.6, 0.2).tolist()
# Explore different values of `R_ee`
traj.f_explore({'R_ee' : explore_list})
# Pre-build network components
clustered_network_manager.pre_build(traj)
# Run the network simulation
traj.f_store() # Let's store the parameters already before the run
env.f_run(clustered_network_manager.run_network)
# Finally disable logging and close all log-files
env.f_disable_logging()
def main():
filename = os.path.join('hdf5', 'Clustered_Network.hdf5')
env = Environment(trajectory='Clustered_Network',
add_time=False,
filename=filename,
continuable=False,
lazy_debug=False,
multiproc=True,
ncores=2,
use_pool=False, # We cannot use a pool, our network cannot be pickled
wrap_mode='QUEUE',
overwrite_file=True)
#Get the trajectory container
traj = env.v_trajectory
# We introduce a `meta` parameter that we can use to easily rescale our network
scale = 1.0 # To obtain the results from the paper scale this to 1.0
# Be aware that your machine will need a lot of memory then!
traj.f_add_parameter('simulation.scale', scale,
comment='Meta parameter that can scale default settings. '
'Rescales number of neurons and connections strenghts, but '
'not the clustersize.')
# We create a Manager and pass all our components to the Manager.
# Note the order, CNNeuronGroups are scheduled before CNConnections,
# and the Fano Factor computation depends on the CNMonitorAnalysis
clustered_network_manager = NetworkManager(network_runner=CNNetworkRunner(),
component_list=(CNNeuronGroup(), CNConnections()),
analyser_list=(CNMonitorAnalysis(),CNFanoFactorComputer()))
# Add original parameters (but scaled according to `scale`)
clustered_network_manager.add_parameters(traj)
# We need `tolist` here since our parameter is a python float and not a
# numpy float.
explore_list = np.arange(1.0, 2.6, 0.2).tolist()
# Explore different values of `R_ee`
traj.f_explore({'R_ee' : explore_list})
# Pre-build network components
clustered_network_manager.pre_build(traj)
# Run the network simulation
traj.f_store() # Let's store the parameters already before the run
env.f_run(clustered_network_manager.run_network)
# Finally disable logging and close all log-files
env.f_disable_logging()
class LinkMergeTest(TrajectoryComparator):
tags = 'integration', 'hdf5', 'environment', 'links', 'merge'
def test_merge_with_linked_derived_parameter(self, disable_logging=True):
logging.basicConfig(level=logging.ERROR)
self.logfolder = make_temp_dir(
os.path.join('experiments', 'tests', 'Log'))
random.seed()
self.trajname1 = 'T1' + make_trajectory_name(self)
self.trajname2 = 'T2' + make_trajectory_name(self)
self.filename = make_temp_dir(
os.path.join('experiments', 'tests', 'HDF5',
'test%s.hdf5' % self.trajname1))
self.env1 = Environment(trajectory=self.trajname1,
filename=self.filename,
file_title=self.trajname1,
log_stdout=False,
log_config=get_log_config())
self.env2 = Environment(trajectory=self.trajname2,
filename=self.filename,
file_title=self.trajname2,
log_stdout=False,
log_config=get_log_config())
self.traj1 = self.env1.v_trajectory
self.traj2 = self.env2.v_trajectory
create_link_params(self.traj1)
create_link_params(self.traj2)
explore_params(self.traj1)
explore_params2(self.traj2)
self.traj1.f_add_derived_parameter('test.$.gg', 42)
self.traj2.f_add_derived_parameter('test.$.gg', 44)
self.traj1.f_add_derived_parameter('test.hh.$', 111)
self.traj2.f_add_derived_parameter('test.hh.$', 53)
self.env1.f_run(dostuff_and_add_links)
self.env2.f_run(dostuff_and_add_links)
old_length = len(self.traj1)
self.traj1.f_merge(self.traj2, remove_duplicates=True)
self.traj1.f_load(load_data=2)
for run in self.traj1.f_get_run_names():
self.traj1.v_crun = run
idx = self.traj1.v_idx
param = self.traj1['test.crun.gg']
if idx < old_length:
self.assertTrue(param == 42)
else:
self.assertTrue(param == 44)
param = self.traj1['test.hh.crun']
if idx < old_length:
self.assertTrue(param == 111)
else:
self.assertTrue(param == 53)
self.assertTrue(len(self.traj1) > old_length)
for irun in range(len(self.traj1.f_get_run_names())):
self.assertTrue(self.traj1.res['r_%d' % irun] == self.traj1.paramB)
self.assertTrue(
self.traj1.res.runs['r_%d' % irun].paraBL == self.traj1.paramB)
if disable_logging:
self.env1.f_disable_logging()
self.env2.f_disable_logging()
return old_length
def test_remerging(self):
prev_old_length = self.test_merge_with_linked_derived_parameter(
disable_logging=False)
name = self.traj1
self.bfilename = make_temp_dir(
os.path.join('experiments', 'tests', 'HDF5',
'backup_test%s.hdf5' % self.trajname1))
self.traj1.f_load(load_data=2)
self.traj1.f_backup(backup_filename=self.bfilename)
self.traj3 = load_trajectory(index=-1,
filename=self.bfilename,
load_all=2)
old_length = len(self.traj1)
self.traj1.f_merge(self.traj3, backup=False, remove_duplicates=False)
self.assertTrue(len(self.traj1) > old_length)
self.traj1.f_load(load_data=2)
for run in self.traj1.f_get_run_names():
self.traj1.v_crun = run
idx = self.traj1.v_idx
param = self.traj1['test.crun.gg']
if idx < prev_old_length or old_length <= idx < prev_old_length + old_length:
self.assertTrue(param == 42, '%s != 42' % str(param))
else:
self.assertTrue(param == 44, '%s != 44' % str(param))
param = self.traj1['test.hh.crun']
if idx < prev_old_length or old_length <= idx < prev_old_length + old_length:
self.assertTrue(param == 111, '%s != 111' % str(param))
else:
self.assertTrue(param == 53, '%s != 53' % str(param))
self.assertTrue(len(self.traj1) > old_length)
for irun in range(len(self.traj1.f_get_run_names())):
self.assertTrue(
self.traj1.res.runs['r_%d' % irun].paraBL == self.traj1.paramB)
self.assertTrue(self.traj1.res['r_%d' % irun] == self.traj1.paramB)
self.env1.f_disable_logging()
self.env2.f_disable_logging()
class LinkMergeTest(TrajectoryComparator):
tags = 'integration', 'hdf5', 'environment', 'links', 'merge'
def test_merge_with_linked_derived_parameter(self, disable_logging = True):
logging.basicConfig(level = logging.ERROR)
self.logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
random.seed()
self.trajname1 = 'T1'+ make_trajectory_name(self)
self.trajname2 = 'T2'+make_trajectory_name(self)
self.filename = make_temp_dir(os.path.join('experiments',
'tests',
'HDF5',
'test%s.hdf5' % self.trajname1))
self.env1 = Environment(trajectory=self.trajname1, filename=self.filename,
file_title=self.trajname1,
log_stdout=False, log_config=get_log_config())
self.env2 = Environment(trajectory=self.trajname2, filename=self.filename,
file_title=self.trajname2,
log_stdout=False, log_config=get_log_config())
self.traj1 = self.env1.v_trajectory
self.traj2 = self.env2.v_trajectory
create_link_params(self.traj1)
create_link_params(self.traj2)
explore_params(self.traj1)
explore_params2(self.traj2)
self.traj1.f_add_derived_parameter('test.$.gg', 42)
self.traj2.f_add_derived_parameter('test.$.gg', 44)
self.traj1.f_add_derived_parameter('test.hh.$', 111)
self.traj2.f_add_derived_parameter('test.hh.$', 53)
self.env1.f_run(dostuff_and_add_links)
self.env2.f_run(dostuff_and_add_links)
old_length = len(self.traj1)
self.traj1.f_merge(self.traj2, remove_duplicates=True)
self.traj1.f_load(load_data=2)
for run in self.traj1.f_get_run_names():
self.traj1.v_crun = run
idx = self.traj1.v_idx
param = self.traj1['test.crun.gg']
if idx < old_length:
self.assertTrue(param == 42)
else:
self.assertTrue(param == 44)
param = self.traj1['test.hh.crun']
if idx < old_length:
self.assertTrue(param == 111)
else:
self.assertTrue(param == 53)
self.assertTrue(len(self.traj1) > old_length)
for irun in range(len(self.traj1.f_get_run_names())):
self.assertTrue(self.traj1.res['r_%d' % irun] == self.traj1.paramB)
self.assertTrue(self.traj1.res.runs['r_%d' % irun].paraBL == self.traj1.paramB)
if disable_logging:
self.env1.f_disable_logging()
self.env2.f_disable_logging()
return old_length
def test_remerging(self):
prev_old_length = self.test_merge_with_linked_derived_parameter(disable_logging=False)
name = self.traj1
self.bfilename = make_temp_dir(os.path.join('experiments',
'tests',
'HDF5',
'backup_test%s.hdf5' % self.trajname1))
self.traj1.f_load(load_data=2)
self.traj1.f_backup(backup_filename=self.bfilename)
self.traj3 = load_trajectory(index=-1, filename=self.bfilename, load_all=2)
old_length = len(self.traj1)
self.traj1.f_merge(self.traj3, backup=False, remove_duplicates=False)
self.assertTrue(len(self.traj1) > old_length)
self.traj1.f_load(load_data=2)
for run in self.traj1.f_get_run_names():
self.traj1.v_crun = run
idx = self.traj1.v_idx
param = self.traj1['test.crun.gg']
if idx < prev_old_length or old_length <= idx < prev_old_length + old_length:
self.assertTrue(param == 42, '%s != 42' % str(param))
else:
self.assertTrue(param == 44, '%s != 44' % str(param))
param = self.traj1['test.hh.crun']
if idx < prev_old_length or old_length <= idx < prev_old_length + old_length:
self.assertTrue(param == 111, '%s != 111' % str(param))
else:
self.assertTrue(param == 53, '%s != 53' % str(param))
self.assertTrue(len(self.traj1)>old_length)
for irun in range(len(self.traj1.f_get_run_names())):
self.assertTrue(self.traj1.res.runs['r_%d' % irun].paraBL == self.traj1.paramB)
self.assertTrue(self.traj1.res['r_%d' % irun] == self.traj1.paramB)
self.env1.f_disable_logging()
self.env2.f_disable_logging()
