def test_file_overwriting(self):
self.traj.f_store()
with pt.open_file(self.filename, mode='r') as file:
nchildren = len(file.root._v_children)
self.assertTrue(nchildren > 0)
env2 = Environment(filename=self.filename,
log_config=get_log_config())
traj2 = env2.v_trajectory
traj2.f_store()
self.assertTrue(os.path.exists(self.filename))
with pt.open_file(self.filename, mode='r') as file:
nchildren = len(file.root._v_children)
self.assertTrue(nchildren > 1)
env3 = Environment(filename=self.filename, overwrite_file=True,
log_config=get_log_config())
self.assertFalse(os.path.exists(self.filename))
env2.f_disable_logging()
env3.f_disable_logging()
def test_file_overwriting(self):
self.traj.f_store()
with ptcompat.open_file(self.filename, mode='r') as file:
nchildren = len(file.root._v_children)
self.assertTrue(nchildren > 0)
env2 = Environment(filename=self.filename,
log_config=get_log_config())
traj2 = env2.v_trajectory
traj2.f_store()
self.assertTrue(os.path.exists(self.filename))
with ptcompat.open_file(self.filename, mode='r') as file:
nchildren = len(file.root._v_children)
self.assertTrue(nchildren > 1)
env3 = Environment(filename=self.filename, overwrite_file=True,
log_config=get_log_config())
self.assertFalse(os.path.exists(self.filename))
env2.f_disable_logging()
env3.f_disable_logging()
def test_time_display_of_loading(self):
filename = make_temp_dir('sloooow.hdf5')
env = Environment(trajectory='traj', add_time=True, filename=filename,
log_stdout=False,
log_config=get_log_config(),
dynamic_imports=SlowResult,
display_time=0.1)
traj = env.v_traj
res=traj.f_add_result(SlowResult, 'iii', 42, 43, comment='llk')
traj.f_store()
service_logger = traj.v_storage_service._logger
root = logging.getLogger('pypet')
old_level = root.level
service_logger.setLevel(logging.INFO)
root.setLevel(logging.INFO)
traj.f_load(load_data=3)
service_logger.setLevel(old_level)
root.setLevel(old_level)
path = get_log_path(traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
self.assertTrue('nodes/s)' in full_text)
env.f_disable_logging()
def test_expand_after_reload(self):
self.traj.f_add_parameter('TEST', 'test_expand_after_reload')
###Explore
self.explore(self.traj)
self.make_run()
traj_name = self.traj.v_name
self.env = Environment(trajectory=self.traj,
log_stdout=False,
log_config=get_log_config())
self.traj = self.env.v_trajectory
self.traj.f_load(name=traj_name)
self.traj.res.f_remove()
self.traj.dpar.f_remove()
self.expand()
get_root_logger().info('\n $$$$$$$$$$$$ Second Run $$$$$$$$$$ \n')
self.make_run()
newtraj = self.load_trajectory(trajectory_name=self.traj.v_name,as_new=False)
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.compare_trajectories(self.traj, newtraj)
def test_without_pre_run(self):
runner = NetworkRunner()
components = [TheNeurons(), TheConnection(), TheSimulation()]
analyser = [TheMonitors()]
nm = NetworkManager(network_runner=runner, component_list=components,
analyser_list=analyser)
env = Environment(trajectory='Test_'+repr(time.time()).replace('.','_'),
filename=make_temp_dir(os.path.join(
'experiments',
'tests',
'briantests',
'HDF5',
'briantest.hdf5')),
file_title='test',
log_config=get_log_config(),
dynamic_imports=['pypet.brian2.parameter.BrianParameter',
BrianMonitorResult, BrianResult],
multiproc=True,
ncores=2)
traj = env.v_traj
nm.add_parameters(traj)
traj.f_explore({'v01': [11*mV, 13*mV]})
env.f_run(nm.run_network)
self.check_data(traj)
def test_time_display_of_loading(self):
filename = make_temp_dir('sloooow.hdf5')
env = Environment(trajectory='traj', add_time=True, filename=filename,
log_stdout=False,
log_config=get_log_config(),
dynamic_imports=SlowResult,
display_time=0.1)
traj = env.v_traj
res=traj.f_add_result(SlowResult, 'iii', 42, 43, comment='llk')
traj.f_store()
service_logger = traj.v_storage_service._logger
root = logging.getLogger('pypet')
old_level = root.level
service_logger.setLevel(logging.INFO)
root.setLevel(logging.INFO)
traj.f_load(load_data=3)
service_logger.setLevel(old_level)
root.setLevel(old_level)
path = get_log_path(traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
self.assertTrue('nodes/s)' in full_text)
env.f_disable_logging()
def test_logging_stdout(self):
filename = 'teststdoutlog.hdf5'
filename = make_temp_dir(filename)
folder = make_temp_dir('logs')
env = Environment(trajectory=make_trajectory_name(self),
filename=filename, log_config=get_log_config(),
# log_levels=logging.CRITICAL, # needed for the test
log_stdout=('STDOUT', 50), #log_folder=folder
)
env.f_run(log_error)
traj = env.v_traj
path = get_log_path(traj)
mainstr = 'sTdOuTLoGGinG'
print(mainstr)
env.f_disable_logging()
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
self.assertTrue(mainstr in full_text)
self.assertTrue('4444444' not in full_text)
self.assertTrue('DEBUG' not in full_text)
def make_environment_mp(self, idx, filename):
#self.filename = '../../experiments/tests/HDF5/test.hdf5'
self.logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
self.cnt_folder = make_temp_dir(os.path.join('experiments',
'tests',
'cnt'))
trajname = 'Test%d' % idx + '_' + make_trajectory_name(self)
env = Environment(trajectory=trajname,
dynamic_imports=[CustomParameter],
filename=filename,
file_title=trajname,
log_stdout=False,
purge_duplicate_comments=False,
log_config=get_log_config(),
continuable=True,
continue_folder=self.cnt_folder,
delete_continue=False,
multiproc=True,
use_pool=True,
ncores=4)
self.envs.append(env)
self.trajs.append( env.v_trajectory)
def setUp(self):
self.set_mode()
self.filename = make_temp_dir(os.path.join('experiments','tests','HDF5','sort_tests.hdf5'))
self.trajname = make_trajectory_name(self)
env = Environment(trajectory=self.trajname,filename=self.filename,
file_title=self.trajname,
log_stdout=self.log_stdout,
log_config=get_log_config() if self.log_config else None,
multiproc=self.multiproc,
wrap_mode=self.mode,
ncores=self.ncores,
use_pool=self.use_pool,
use_scoop=self.use_scoop,
port=self.port,
freeze_input=self.freeze_input,
graceful_exit=self.graceful_exit)
traj = env.v_trajectory
traj.v_standard_parameter=Parameter
traj.f_add_parameter('x',99)
traj.f_add_parameter('y',99)
self.env=env
self.traj=traj
def test_logging_stdout(self):
filename = 'teststdoutlog.hdf5'
filename = make_temp_dir(filename)
folder = make_temp_dir('logs')
env = Environment(trajectory=make_trajectory_name(self),
filename=filename, log_config=get_log_config(),
# log_levels=logging.CRITICAL, # needed for the test
log_stdout=('STDOUT', 50), #log_folder=folder
)
env.f_run(log_error)
traj = env.v_traj
path = get_log_path(traj)
mainstr = 'sTdOuTLoGGinG'
print(mainstr)
env.f_disable_logging()
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
self.assertTrue(mainstr in full_text)
self.assertTrue('4444444' not in full_text)
self.assertTrue('DEBUG' not in full_text)
def test_maximum_overview_size(self):
filename = make_temp_dir('maxisze.hdf5')
env = Environment(trajectory='Testmigrate', filename=filename,
log_config=get_log_config())
traj = env.v_trajectory
for irun in range(pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH):
traj.f_add_parameter('f%d.x' % irun, 5)
traj.f_store()
store = ptcompat.open_file(filename, mode='r+')
table = ptcompat.get_child(store.root,traj.v_name).overview.parameters_overview
self.assertEquals(table.nrows, pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH)
store.close()
for irun in range(pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH,
2*pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH):
traj.f_add_parameter('f%d.x' % irun, 5)
traj.f_store()
store = ptcompat.open_file(filename, mode='r+')
table = ptcompat.get_child(store.root,traj.v_name).overview.parameters_overview
self.assertEquals(table.nrows, pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH)
store.close()
env.f_disable_logging()
def test_hdf5_store_load_monitorresult(self):
traj_name = make_trajectory_name(self)
file_name = make_temp_dir(os.path.join('brian2', 'tests', 'hdf5', 'test_%s.hdf5' % traj_name))
env = Environment(trajectory=traj_name, filename=file_name, log_config=get_log_config(),
dynamic_imports=[Brian2MonitorResult], add_time=False, storage_service=HDF5StorageService)
traj = env.v_trajectory
traj.v_standard_result = Brian2MonitorResult
traj.f_add_result('brian2.single.millivolts_single_a', 10*mvolt, comment='single value a')
traj.f_add_result('brian2.single.millivolts_single_c', 11*mvolt, comment='single value b')
traj.f_add_result('brian2.array.millivolts_array_a', [11, 12]*mvolt, comment='array')
traj.f_add_result('mV1', 42.0*mV)
# results can hold much more than a single data item:
traj.f_add_result('ampere1', 1*mA, 44, test=300*mV, test2=[1,2,3],
test3=np.array([1,2,3])*mA, comment='Result keeping track of many things')
traj.f_add_result('integer', 16)
traj.f_add_result('kHz05', 0.5*kHz)
traj.f_add_result('nested_array', np.array([[6.,7.,8.],[9.,10.,11.]]) * ms)
traj.f_add_result('b2a', np.array([1., 2.]) * mV)
traj.f_store()
traj2 = load_trajectory(filename=file_name, name=traj_name, dynamic_imports=[Brian2MonitorResult], load_data=2)
#traj._logger.error('traj :'+str(traj))
#traj._logger.error('traj2:'+str(traj2))
self.compare_trajectories(traj, traj2)
def test_hdf5_store_load_parameter(self):
traj_name = make_trajectory_name(self)
file_name = make_temp_dir(os.path.join('brian2', 'tests', 'hdf5', 'test_%s.hdf5' % traj_name))
env = Environment(trajectory=traj_name, filename=file_name, log_config=get_log_config(),
dynamic_imports=[Brian2Parameter], add_time=False, storage_service=HDF5StorageService)
traj = env.v_trajectory
traj.v_standard_parameter = Brian2Parameter
traj.f_add_parameter('brian2.single.millivolts', 10*mvolt, comment='single value')
#traj.f_add_parameter('brian2.array.millivolts', [11, 12]*mvolt, comment='array')
#traj.f_add_parameter('mV1', 42.0*mV)
#traj.f_add_parameter('ampere1', 1*mA)
#traj.f_add_parameter('integer', 16)
#traj.f_add_parameter('kHz05', 0.5*kHz)
#traj.f_add_parameter('nested_array', np.array([[6.,7.,8.],[9.,10.,11.]]) * ms)
#traj.f_add_parameter('b2a', np.array([1., 2.]) * mV)
# We also need to check if explorations work with hdf5 store!
#explore_dict = {'ampere1': [1*mA, 2*mA, 3*mA],
# 'integer': [42,43,44],
# 'b2a': [np.array([1., 2.]) * mV, np.array([1., 4.]) * mV,
# np.array([1., 2.]) * mV]}
#traj.f_explore(explore_dict)
traj.f_store()
traj2 = load_trajectory(filename=file_name, name=traj_name, dynamic_imports=[Brian2Parameter],
load_data=2)
self.compare_trajectories(traj, traj2)
def make_environment_mp(self, idx, filename):
#self.filename = '../../experiments/tests/HDF5/test.hdf5'
self.logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
self.cnt_folder = make_temp_dir(os.path.join('experiments',
'tests',
'cnt'))
trajname = 'Test%d' % idx + '_' + make_trajectory_name(self)
env = Environment(trajectory=trajname,
dynamic_imports=[CustomParameter],
filename=filename,
file_title=trajname,
log_stdout=False,
purge_duplicate_comments=False,
log_config=get_log_config(),
continuable=True,
continue_folder=self.cnt_folder,
delete_continue=False,
multiproc=True,
use_pool=True,
ncores=4)
self.envs.append(env)
self.trajs.append( env.v_trajectory)
def setUp(self):
self.set_mode()
self.filename = make_temp_dir(os.path.join('experiments','tests','HDF5','sort_tests.hdf5'))
self.trajname = make_trajectory_name(self)
env = Environment(trajectory=self.trajname,filename=self.filename,
file_title=self.trajname,
log_stdout=self.log_stdout,
log_config=get_log_config() if self.log_config else None,
multiproc=self.multiproc,
wrap_mode=self.mode,
ncores=self.ncores,
use_pool=self.use_pool,
use_scoop=self.use_scoop,
port=self.port,
freeze_input=self.freeze_input,)
traj = env.v_trajectory
traj.v_standard_parameter=Parameter
traj.f_add_parameter('x',0)
traj.f_add_parameter('y',0)
self.env=env
self.traj=traj
def setUp(self):
env = Environment(
trajectory='Test_' + repr(time.time()).replace('.', '_'),
filename=make_temp_dir(
os.path.join('experiments', 'tests', 'briantests', 'HDF5',
'briantest.hdf5')),
file_title='test',
log_config=get_log_config(),
dynamic_imports=[
'pypet.brian.parameter.BrianParameter', BrianMonitorResult
],
multiproc=False)
traj = env.v_trajectory
#env._set_standard_storage()
#env._hdf5_queue_writer._hdf5storageservice = LazyStorageService()
traj = env.v_trajectory
#traj.set_storage_service(LazyStorageService())
add_params(traj)
#traj.mode='Parallel'
traj.f_explore(
cartesian_product({
traj.f_get('N').v_full_name: [50, 60],
traj.f_get('tauw').v_full_name: [30 * ms, 40 * ms]
}))
self.traj = traj
self.env = env
self.traj = traj
def make_environment(self,
idx,
filename,
continuable=True,
delete_continue=False):
#self.filename = '../../experiments/tests/HDF5/test.hdf5'
self.logfolder = make_temp_dir(
os.path.join('experiments', 'tests', 'Log'))
self.cnt_folder = make_temp_dir(
os.path.join('experiments', 'tests', 'cnt'))
trajname = 'Test%d' % idx + '_' + make_trajectory_name(self)
env = Environment(trajectory=trajname,
filename=filename,
file_title=trajname,
log_stdout=False,
log_config=get_log_config(),
continuable=continuable,
continue_folder=self.cnt_folder,
delete_continue=delete_continue,
large_overview_tables=True)
self.envs.append(env)
self.trajs.append(env.v_trajectory)
def setUp(self):
env = Environment(trajectory='Test_'+repr(time.time()).replace('.','_'),
filename=make_temp_dir(os.path.join(
'experiments',
'tests',
'briantests',
'HDF5',
'briantest.hdf5')),
file_title='test',
log_config=get_log_config(),
dynamic_imports=['pypet.brian.parameter.BrianParameter',
BrianMonitorResult],
multiproc=False)
traj = env.v_trajectory
#env._set_standard_storage()
#env._hdf5_queue_writer._hdf5storageservice = LazyStorageService()
traj = env.v_trajectory
#traj.set_storage_service(LazyStorageService())
add_params(traj)
#traj.mode='Parallel'
traj.f_explore(cartesian_product({traj.f_get('N').v_full_name:[50,60],
traj.f_get('tauw').v_full_name:[30*ms,40*ms]}))
self.traj = traj
self.env = env
self.traj = traj
def test_expand_after_reload(self):
self.traj.f_add_parameter('TEST', 'test_expand_after_reload')
###Explore
self.explore(self.traj)
self.make_run()
traj_name = self.traj.v_name
self.env = Environment(trajectory=self.traj,
log_stdout=False,
log_config=get_log_config())
self.traj = self.env.v_trajectory
self.traj.f_load(name=traj_name)
self.traj.res.f_remove()
self.traj.dpar.f_remove()
self.expand()
get_root_logger().info('\n $$$$$$$$$$$$ Second Run $$$$$$$$$$ \n')
self.make_run()
newtraj = self.load_trajectory(trajectory_name=self.traj.v_name,as_new=False)
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.compare_trajectories(self.traj, newtraj)
def make_environment(self, idx, filename):
#self.filename = '../../experiments/tests/HDF5/test.hdf5'
self.logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
self.cnt_folder = make_temp_dir(os.path.join('experiments','tests','cnt'))
trajname = 'Test%d' % idx + '_' + make_trajectory_name(self)
env = Environment(trajectory=trajname,
filename=filename,
file_title=trajname,
log_stdout=False,
log_config=get_log_config(),
continuable=True,
continue_folder=self.cnt_folder,
delete_continue=False,
large_overview_tables=True)
self.envs.append(env)
self.trajs.append( env.v_trajectory)
def test_maximum_overview_size(self):
filename = make_temp_dir('maxisze.hdf5')
env = Environment(trajectory='Testmigrate', filename=filename,
log_config=get_log_config(), add_time=True)
traj = env.v_trajectory
for irun in range(pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH):
traj.f_add_parameter('f%d.x' % irun, 5)
traj.f_store()
store = pt.open_file(filename, mode='r+')
table = store.root._f_get_child(traj.v_name).overview.parameters_overview
self.assertEquals(table.nrows, pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH)
store.close()
for irun in range(pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH,
2*pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH):
traj.f_add_parameter('f%d.x' % irun, 5)
traj.f_store()
store = pt.open_file(filename, mode='r+')
table = store.root._f_get_child(traj.v_name).overview.parameters_overview
self.assertEquals(table.nrows, pypetconstants.HDF5_MAX_OVERVIEW_TABLE_LENGTH)
store.close()
env.f_disable_logging()
def test_without_pre_run(self):
runner = NetworkRunner()
components = [TheNeurons(), TheConnection(), TheSimulation()]
analyser = [TheMonitors()]
nm = NetworkManager(network_runner=runner,
component_list=components,
analyser_list=analyser)
env = Environment(
trajectory='Test_' + repr(time.time()).replace('.', '_'),
filename=make_temp_dir(
os.path.join('experiments', 'tests', 'briantests', 'HDF5',
'briantest.hdf5')),
file_title='test',
log_config=get_log_config(),
dynamic_imports=[
'pypet.brian2.parameter.BrianParameter', BrianMonitorResult,
BrianResult
],
multiproc=True,
ncores=2)
traj = env.v_traj
nm.add_parameters(traj)
traj.f_explore({'v01': [11 * mV, 13 * mV]})
env.f_run(nm.run_network)
self.check_data(traj)
def _make_env(self, idx):
return Environment(trajectory=self.trajname+str(idx),filename=self.filename,
file_title=self.trajname,
log_stdout=False,
log_config=get_log_config(),
multiproc=self.multiproc,
wrap_mode=self.mode,
ncores=self.ncores)
def test_fail_on_wrong_kwarg(self):
with self.assertRaises(ValueError):
filename = 'testsfail.hdf5'
env = Environment(filename=make_temp_dir(filename),
log_stdout=True,
log_config=get_log_config(),
logger_names=('STDERROR', 'STDOUT'),
foo='bar')
def test_pickling(self):
manager = LoggingManager(log_config=get_log_config(), log_stdout=True)
manager.extract_replacements(FakeTraj())
manager.check_log_config()
manager.make_logging_handlers_and_tools()
dump = pickle.dumps(manager)
new_manager = pickle.loads(dump)
manager.finalize()
def test_fail_on_wrong_kwarg(self):
with self.assertRaises(ValueError):
filename = 'testsfail.hdf5'
env = Environment(filename=make_temp_dir(filename),
log_stdout=True,
log_config=get_log_config(),
logger_names=('STDERROR', 'STDOUT'),
foo='bar')
def test_pickling(self):
manager = LoggingManager(log_config=get_log_config(), log_stdout=True,
trajectory=FakeTraj())
manager.check_log_config()
manager.make_logging_handlers_and_tools()
dump = pickle.dumps(manager)
new_manager = pickle.loads(dump)
manager.finalize()
def _make_env(self, idx):
return Environment(trajectory=self.trajname + str(idx),
filename=self.filename,
file_title=self.trajname,
log_stdout=False,
log_config=get_log_config(),
multiproc=self.multiproc,
wrap_mode=self.mode,
ncores=self.ncores)
def setUp(self):
self.set_mode()
self.logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
random.seed()
self.trajname = make_trajectory_name(self)
self.filename = make_temp_dir(os.path.join('experiments',
'tests',
'HDF5',
'test%s.hdf5' % self.trajname))
env = Environment(trajectory=self.trajname, filename=self.filename,
file_title=self.trajname,
log_stdout=self.log_stdout,
log_config=get_log_config(),
results_per_run=5,
wildcard_functions=self.wildcard_functions,
derived_parameters_per_run=5,
multiproc=self.multiproc,
ncores=self.ncores,
wrap_mode=self.mode,
use_pool=self.use_pool,
gc_interval=self.gc_interval,
freeze_input=self.freeze_input,
fletcher32=self.fletcher32,
complevel=self.complevel,
complib=self.complib,
shuffle=self.shuffle,
pandas_append=self.pandas_append,
pandas_format=self.pandas_format,
encoding=self.encoding,
niceness=self.niceness,
use_scoop=self.use_scoop,
port=self.port,
add_time=self.add_time,
timeout=self.timeout,
graceful_exit=self.graceful_exit)
traj = env.v_trajectory
traj.v_standard_parameter=Parameter
## Create some parameters
self.param_dict={}
create_param_dict(self.param_dict)
### Add some parameter:
add_params(traj,self.param_dict)
#remember the trajectory and the environment
self.traj = traj
self.env = env
def setUp(self):
self.set_mode()
self.logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
random.seed()
self.trajname = make_trajectory_name(self)
self.filename = make_temp_dir(os.path.join('experiments',
'tests',
'HDF5',
'test%s.hdf5' % self.trajname))
env = Environment(trajectory=self.trajname, filename=self.filename,
file_title=self.trajname,
log_stdout=self.log_stdout,
log_config=get_log_config(),
results_per_run=5,
wildcard_functions=self.wildcard_functions,
derived_parameters_per_run=5,
multiproc=self.multiproc,
ncores=self.ncores,
wrap_mode=self.mode,
use_pool=self.use_pool,
gc_interval=self.gc_interval,
freeze_input=self.freeze_input,
fletcher32=self.fletcher32,
complevel=self.complevel,
complib=self.complib,
shuffle=self.shuffle,
pandas_append=self.pandas_append,
pandas_format=self.pandas_format,
encoding=self.encoding,
niceness=self.niceness,
use_scoop=self.use_scoop,
port=self.port,
add_time=self.add_time,
timeout=self.timeout,
graceful_exit=self.graceful_exit)
traj = env.v_trajectory
traj.v_standard_parameter=Parameter
## Create some parameters
self.param_dict={}
create_param_dict(self.param_dict)
### Add some parameter:
add_params(traj,self.param_dict)
#remember the trajectory and the environment
self.traj = traj
self.env = env
def make_environment(self, idx, filename, **kwargs):
#self.filename = make_temp_dir('experiments/tests/HDF5/test.hdf5')
logfolder = make_temp_dir(os.path.join('experiments','tests','Log'))
trajname = make_trajectory_name(self) + '__' +str(idx) +'_'
env = Environment(trajectory=trajname,filename=filename, file_title=trajname,
log_stdout=False,
large_overview_tables=True, log_config=get_log_config(),
**kwargs)
self.envs.append(env)
self.trajs.append( env.v_trajectory)
def make_environment(self, idx, filename, **kwargs):
#self.filename = make_temp_dir('experiments/tests/HDF5/test.hdf5')
logfolder = make_temp_dir(os.path.join('experiments','tests','Log'))
trajname = make_trajectory_name(self) + '__' +str(idx) +'_'
env = Environment(trajectory=trajname,filename=filename, file_title=trajname,
log_stdout=False,
large_overview_tables=True, log_config=get_log_config(),
**kwargs)
self.envs.append(env)
self.trajs.append( env.v_trajectory)
def set_mode(self):
self.mode = Dummy()
self.mode.wrap_mode = 'LOCK'
self.mode.multiproc = False
self.mode.ncores = 1
self.mode.use_pool=True
self.mode.pandas_format='fixed'
self.mode.pandas_append=False
self.mode.complib = 'blosc'
self.mode.complevel=9
self.mode.shuffle=True
self.mode.fletcher32 = False
self.mode.encoding = 'utf8'
self.mode.log_stdout=False
self.mode.log_config=get_log_config()
def set_mode(self):
self.mode = Dummy()
self.mode.wrap_mode = 'LOCK'
self.mode.multiproc = False
self.mode.ncores = 1
self.mode.use_pool=True
self.mode.pandas_format='fixed'
self.mode.pandas_append=False
self.mode.complib = 'blosc'
self.mode.complevel=9
self.mode.shuffle=True
self.mode.fletcher32 = False
self.mode.encoding = 'utf8'
self.mode.log_stdout=False
self.mode.log_config=get_log_config()
def test_hdf5_store_load_result(self):
traj_name = make_trajectory_name(self)
file_name = make_temp_dir(
os.path.join('brian2', 'tests', 'hdf5',
'test_%s.hdf5' % traj_name))
env = Environment(trajectory=traj_name,
filename=file_name,
log_config=get_log_config(),
dynamic_imports=[Brian2Result],
add_time=False,
storage_service=HDF5StorageService)
traj = env.v_trajectory
traj.v_standard_result = Brian2Result
traj.f_add_result('brian2.single.millivolts_single_a',
10 * mvolt,
comment='single value a')
traj.f_add_result('brian2.single.millivolts_single_c',
11 * mvolt,
comment='single value b')
traj.f_add_result('brian2.array.millivolts_array_a', [11, 12] * mvolt,
comment='array')
traj.f_add_result('mV1', 42.0 * mV)
# results can hold much more than a single data item:
traj.f_add_result('ampere1',
1 * mA,
44,
test=300 * mV,
test2=[1, 2, 3],
test3=np.array([1, 2, 3]) * mA,
comment='Result keeping track of many things')
traj.f_add_result('integer', 16)
traj.f_add_result('kHz05', 0.5 * kHz)
traj.f_add_result('nested_array',
np.array([[6., 7., 8.], [9., 10., 11.]]) * ms)
traj.f_add_result('b2a', np.array([1., 2.]) * mV)
traj.f_add_result('nounit',
Quantity(np.array([[6., 7., 8.], [9., 10., 11.]])))
traj.f_store()
traj2 = load_trajectory(filename=file_name,
name=traj_name,
dynamic_imports=[Brian2Result],
load_data=2)
self.compare_trajectories(traj, traj2)
def setUp(self):
self.set_mode()
logging.basicConfig(level=logging.ERROR)
self.logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
random.seed()
self.trajname = make_trajectory_name(self)
self.filename = make_temp_dir(os.path.join('experiments',
'tests',
'HDF5',
'test%s.hdf5' % self.trajname))
env = Environment(trajectory=self.trajname, filename=self.filename,
file_title=self.trajname,
log_stdout=self.log_stdout,
log_config=get_log_config(),
results_per_run=5,
derived_parameters_per_run=5,
multiproc=self.multiproc,
ncores=self.ncores,
wrap_mode=self.mode,
use_pool=self.use_pool,
fletcher32=self.fletcher32,
complevel=self.complevel,
complib=self.complib,
shuffle=self.shuffle,
pandas_append=self.pandas_append,
pandas_format=self.pandas_format,
encoding=self.encoding)
traj = env.v_trajectory
traj.v_standard_parameter=Parameter
## Create some parameters
create_link_params(traj)
### Add some parameter:
explore_params(traj)
#remember the trajectory and the environment
self.traj = traj
self.env = env
def test_full_store(self):
filename = make_temp_dir('full_store.hdf5')
with Environment(filename=filename,
log_config=get_log_config()) as env:
traj = env.v_trajectory
traj.par.x = Parameter('x', 3, 'jj')
traj.f_explore({'x': [1,2,3]})
env.f_run(add_one_particular_item, True)
traj = load_trajectory(index=-1, filename=filename)
self.assertTrue('hi' in traj)
def test_full_store(self):
filename = make_temp_dir('full_store.hdf5')
with Environment(filename=filename,
log_config=get_log_config()) as env:
traj = env.v_trajectory
traj.par.x = Parameter('x', 3, 'jj')
traj.f_explore({'x': [1,2,3]})
env.f_run(add_one_particular_item, True)
traj = load_trajectory(index=-1, filename=filename)
self.assertTrue('hi' in traj)
def test_expand(self):
###Explore
self.explore(self.traj)
results = self.env.f_run(multiply)
self.are_results_in_order(results)
get_root_logger().info(results)
traj = self.traj
self.assertEqual(len(traj),
len(list(compat.listvalues(self.explore_dict)[0])))
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.check_if_z_is_correct(traj)
traj_name = self.env.v_trajectory.v_name
del self.env
self.env = Environment(trajectory=self.traj,
log_stdout=False,
log_config=get_log_config())
self.traj = self.env.v_trajectory
self.traj.f_load(name=traj_name)
self.expand(self.traj)
results = self.env.f_run(multiply)
self.are_results_in_order(results)
traj = self.traj
self.assertTrue(
len(traj) == len(compat.listvalues(self.expand_dict)[0]) +
len(compat.listvalues(self.explore_dict)[0]))
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.check_if_z_is_correct(traj)
newtraj = self.load_trajectory(trajectory_name=self.traj.v_name,
as_new=False)
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.compare_trajectories(self.traj, newtraj)
def setUp(self):
self.set_mode()
logging.basicConfig(level=logging.ERROR)
self.logfolder = make_temp_dir(
os.path.join('experiments', 'tests', 'Log'))
random.seed()
self.trajname = make_trajectory_name(self)
self.filename = make_temp_dir(
os.path.join('experiments', 'tests', 'HDF5',
'test%s.hdf5' % self.trajname))
env = Environment(trajectory=self.trajname,
filename=self.filename,
file_title=self.trajname,
log_stdout=self.log_stdout,
log_config=get_log_config(),
results_per_run=5,
derived_parameters_per_run=5,
multiproc=self.multiproc,
ncores=self.ncores,
wrap_mode=self.mode,
use_pool=self.use_pool,
fletcher32=self.fletcher32,
complevel=self.complevel,
complib=self.complib,
shuffle=self.shuffle,
pandas_append=self.pandas_append,
pandas_format=self.pandas_format,
encoding=self.encoding)
traj = env.v_trajectory
traj.v_standard_parameter = Parameter
## Create some parameters
create_link_params(traj)
### Add some parameter:
explore_params(traj)
#remember the trajectory and the environment
self.traj = traj
self.env = env
def test_logging_show_progress_print(self):
self.make_env(log_config=get_log_config(), log_stdout=('prostdout', 50),
report_progress=(3, 'print'))
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
self.env.f_disable_logging()
path = get_log_path(self.env.v_traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
progress = 'prostdout CRITICAL PROGRESS: Finished'
self.assertTrue(progress in full_text)
bar = '[=='
self.assertIn(bar, full_text)
def test_logging_show_progress_print(self):
self.make_env(log_config=get_log_config(), log_stdout=('prostdout', 50),
report_progress=(3, 'print'))
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
self.env.f_disable_logging()
path = get_log_path(self.env.v_traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
progress = 'prostdout CRITICAL PROGRESS: Finished'
self.assertTrue(progress in full_text)
bar = '[=='
self.assertIn(bar, full_text)
def test_net(self):
env = Environment(trajectory='Test_'+repr(time.time()).replace('.','_'),
filename=make_temp_dir(os.path.join(
'experiments',
'tests',
'briantests',
'HDF5',
'briantest.hdf5')),
file_title='test',
log_config=get_log_config(),
dynamic_imports=['pypet.brian2.parameter.Brian2Parameter',
Brian2MonitorResult],
multiproc=False)
traj = env.v_traj
traj.f_add_parameter(Brian2Parameter, 'v0', 0.0*mV,
comment='Input bias')
traj.f_explore({'v0': [11*mV, 13*mV, 15*mV]})
env.f_run(run_network)
self.get_data(traj)
def test_overwrite_annotations_and_results(self):
filename = make_temp_dir('overwrite.hdf5')
env = Environment(trajectory='testoverwrite',
filename=filename,
log_config=get_log_config(),
overwrite_file=True)
traj = env.v_traj
traj.f_add_parameter('grp.x', 5, comment='hi')
traj.grp.v_comment = 'hi'
traj.grp.v_annotations['a'] = 'b'
traj.f_store()
traj.f_remove_child('parameters', recursive=True)
traj.f_load(load_data=2)
self.assertTrue(traj.x == 5)
self.assertTrue(traj.grp.v_comment == 'hi')
self.assertTrue(traj.grp.v_annotations['a'] == 'b')
traj.f_get('x').f_unlock()
traj.grp.x = 22
traj.f_get('x').v_comment = 'hu'
traj.grp.v_annotations['a'] = 'c'
traj.grp.v_comment = 'hu'
traj.f_store_item(traj.f_get('x'), store_data=3)
traj.f_store_item(traj.grp, store_data=3)
traj.f_remove_child('parameters', recursive=True)
traj.f_load(load_data=2)
self.assertTrue(traj.x == 22)
self.assertTrue(traj.grp.v_comment == 'hu')
self.assertTrue(traj.grp.v_annotations['a'] == 'c')
env.f_disable_logging()
def test_expand(self):
###Explore
self.explore(self.traj)
results = self.env.f_run(multiply)
self.are_results_in_order(results)
get_root_logger().info(results)
traj = self.traj
self.assertEqual(len(traj), len(list(compat.listvalues(self.explore_dict)[0])))
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.check_if_z_is_correct(traj)
traj_name = self.env.v_trajectory.v_name
del self.env
self.env = Environment(trajectory=self.traj,
log_stdout=False,
log_config=get_log_config())
self.traj = self.env.v_trajectory
self.traj.f_load(name=traj_name)
self.expand(self.traj)
results = self.env.f_run(multiply)
self.are_results_in_order(results)
traj = self.traj
self.assertTrue(len(traj) == len(compat.listvalues(self.expand_dict)[0])+ len(compat.listvalues(self.explore_dict)[0]))
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.check_if_z_is_correct(traj)
newtraj = self.load_trajectory(trajectory_name=self.traj.v_name,as_new=False)
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.compare_trajectories(self.traj,newtraj)
def test_net(self):
env = Environment(
trajectory='Test_' + repr(time.time()).replace('.', '_'),
filename=make_temp_dir(
os.path.join('experiments', 'tests', 'briantests', 'HDF5',
'briantest.hdf5')),
file_title='test',
log_config=get_log_config(),
dynamic_imports=[
'pypet.brian2.parameter.Brian2Parameter', Brian2MonitorResult
],
multiproc=False)
traj = env.v_traj
traj.f_add_parameter(Brian2Parameter,
'v0',
0.0 * mV,
comment='Input bias')
traj.f_explore({'v0': [11 * mV, 13 * mV, 15 * mV]})
env.f_run(run_network)
self.get_data(traj)
def test_overwrite_annotations_and_results(self):
filename = make_temp_dir('overwrite.hdf5')
env = Environment(trajectory='testoverwrite', filename=filename,
log_config=get_log_config())
traj = env.v_traj
traj.f_add_parameter('grp.x', 5, comment='hi')
traj.grp.v_comment='hi'
traj.grp.v_annotations['a'] = 'b'
traj.f_store()
traj.f_remove_child('parameters', recursive=True)
traj.f_load(load_data=2)
self.assertTrue(traj.x == 5)
self.assertTrue(traj.grp.v_comment == 'hi')
self.assertTrue(traj.grp.v_annotations['a'] == 'b')
traj.f_get('x').f_unlock()
traj.grp.x = 22
traj.f_get('x').v_comment='hu'
traj.grp.v_annotations['a'] = 'c'
traj.grp.v_comment = 'hu'
traj.f_store_item(traj.f_get('x'), store_data=3)
traj.f_store_item(traj.grp, store_data=3)
traj.f_remove_child('parameters', recursive=True)
traj.f_load(load_data=2)
self.assertTrue(traj.x == 22)
self.assertTrue(traj.grp.v_comment == 'hu')
self.assertTrue(traj.grp.v_annotations['a'] == 'c')
env.f_disable_logging()
def setUp(self):
super(TestMergeResultsSort,self).setUp()
env2 = Environment(trajectory=self.trajname+'2',filename=self.filename,
file_title=self.trajname,
log_stdout=False,
log_config=get_log_config(),
multiproc=self.multiproc,
wrap_mode=self.mode,
ncores=self.ncores)
traj2 = env2.v_trajectory
traj2.v_standard_parameter=Parameter
traj2.f_add_parameter('x',0)
traj2.f_add_parameter('y',0)
self.env2=env2
self.traj2=traj2
def setUp(self):
super(TestMergeResultsSort,self).setUp()
env2 = Environment(trajectory=self.trajname+'2',filename=self.filename,
file_title=self.trajname,
log_stdout=False,
log_config=get_log_config(),
multiproc=self.multiproc,
wrap_mode=self.mode,
ncores=self.ncores)
traj2 = env2.v_trajectory
traj2.v_standard_parameter=Parameter
traj2.f_add_parameter('x',0)
traj2.f_add_parameter('y',0)
self.env2=env2
self.traj2=traj2
def test_throw_warning_if_old_kw_is_used(self):
pass
filename = make_temp_dir('hdfwarning.hdf5')
with warnings.catch_warnings(record=True) as w:
env = Environment(trajectory='test', filename=filename,
dynamically_imported_classes=[],
log_config=get_log_config())
with warnings.catch_warnings(record=True) as w:
traj = Trajectory(dynamically_imported_classes=[])
traj = env.v_trajectory
traj.f_store()
with warnings.catch_warnings(record=True) as w:
traj.f_load(dynamically_imported_classes=[])
env.f_disable_logging()
def make_environment(self, filename, trajname='Test', log=True, **kwargs):
#self.filename = '../../experiments/tests/HDF5/test.hdf5'
filename = make_temp_dir(filename)
logfolder = make_temp_dir(os.path.join('experiments', 'tests', 'Log'))
cntfolder = make_temp_dir(os.path.join('experiments', 'tests', 'cnt'))
if log:
log_config = get_log_config()
else:
log_config = None
env = Environment(
trajectory=trajname,
# log_levels=logging.INFO,
# log_config=None,
log_config=log_config,
dynamic_imports=[CustomParameter],
filename=filename,
log_stdout=False,
**self.env_kwargs)
return env, filename, logfolder, cntfolder
def setUp(self):
self.set_mode()
self.logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
random.seed()
self.trajname = make_trajectory_name(self)
self.filename = make_temp_dir(os.path.join('experiments',
'tests',
'HDF5',
'test%s.hdf5' % self.trajname))
env = Environment(trajectory=self.trajname, filename=self.filename,
file_title=self.trajname,
log_stdout=False,
port=self.url,
log_config=get_log_config(),
results_per_run=5,
derived_parameters_per_run=5,
multiproc=self.multiproc,
ncores=self.ncores,
wrap_mode=self.mode,
use_pool=self.use_pool,
fletcher32=self.fletcher32,
complevel=self.complevel,
complib=self.complib,
shuffle=self.shuffle,
pandas_append=self.pandas_append,
pandas_format=self.pandas_format,
encoding=self.encoding)
traj = env.v_trajectory
self.param_dict={}
create_param_dict(self.param_dict)
add_params(traj,self.param_dict)
self.traj = traj
self.env = env
def setUp(self):
self.set_mode()
self.logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
random.seed()
self.trajname = make_trajectory_name(self)
self.filename = make_temp_dir(os.path.join('experiments',
'tests',
'HDF5',
'test%s.hdf5' % self.trajname))
env = Environment(trajectory=self.trajname, filename=self.filename,
file_title=self.trajname,
log_stdout=False,
log_config=get_log_config(),
results_per_run=5,
derived_parameters_per_run=5,
multiproc=self.multiproc,
ncores=self.ncores,
wrap_mode=self.mode,
use_pool=self.use_pool,
fletcher32=self.fletcher32,
complevel=self.complevel,
complib=self.complib,
shuffle=self.shuffle,
pandas_append=self.pandas_append,
pandas_format=self.pandas_format,
encoding=self.encoding)
traj = env.v_trajectory
self.param_dict={}
create_param_dict(self.param_dict)
add_params(traj,self.param_dict)
self.traj = traj
self.env = env
def test_logging_show_progress(self):
self.make_env(log_config=get_log_config(),
# log_folder=make_temp_dir('logs'),
# log_levels=logging.ERROR,
report_progress=(3, 'progress', 40))
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
self.env.f_disable_logging()
traj = self.env.v_traj
path = get_log_path(traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
progress = 'PROGRESS: Finished'
self.assertTrue(progress in full_text)
bar = '[=='
self.assertTrue(bar in full_text)
def test_logging_show_progress(self):
self.make_env(log_config=get_log_config(),
# log_folder=make_temp_dir('logs'),
# log_levels=logging.ERROR,
report_progress=(3, 'progress', 40))
self.add_params(self.traj)
self.explore(self.traj)
self.env.f_run(log_all_levels)
self.env.f_disable_logging()
traj = self.env.v_traj
path = get_log_path(traj)
mainfilename = os.path.join(path, 'LOG.txt')
with open(mainfilename, mode='r') as mainf:
full_text = mainf.read()
progress = 'PROGRESS: Finished'
self.assertTrue(progress in full_text)
bar = '[=='
self.assertTrue(bar in full_text)
def test_hdf5_store_load_parameter(self):
traj_name = make_trajectory_name(self)
file_name = make_temp_dir(
os.path.join('brian2', 'tests', 'hdf5',
'test_%s.hdf5' % traj_name))
env = Environment(trajectory=traj_name,
filename=file_name,
log_config=get_log_config(),
dynamic_imports=[Brian2Parameter],
add_time=False,
storage_service=HDF5StorageService)
traj = env.v_trajectory
traj.v_standard_parameter = Brian2Parameter
traj.f_add_parameter('brian2.single.millivolts',
10 * mvolt,
comment='single value')
#traj.f_add_parameter('brian2.array.millivolts', [11, 12]*mvolt, comment='array')
#traj.f_add_parameter('mV1', 42.0*mV)
#traj.f_add_parameter('ampere1', 1*mA)
#traj.f_add_parameter('integer', 16)
#traj.f_add_parameter('kHz05', 0.5*kHz)
#traj.f_add_parameter('nested_array', np.array([[6.,7.,8.],[9.,10.,11.]]) * ms)
#traj.f_add_parameter('b2a', np.array([1., 2.]) * mV)
# We also need to check if explorations work with hdf5 store!
#explore_dict = {'ampere1': [1*mA, 2*mA, 3*mA],
# 'integer': [42,43,44],
# 'b2a': [np.array([1., 2.]) * mV, np.array([1., 4.]) * mV,
# np.array([1., 2.]) * mV]}
#traj.f_explore(explore_dict)
traj.f_store()
traj2 = load_trajectory(filename=file_name,
name=traj_name,
dynamic_imports=[Brian2Parameter],
load_data=2)
self.compare_trajectories(traj, traj2)
def make_environment(self, filename, trajname='Test', log=True, **kwargs):
#self.filename = '../../experiments/tests/HDF5/test.hdf5'
filename = make_temp_dir(filename)
logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
cntfolder = make_temp_dir(os.path.join('experiments',
'tests',
'cnt'))
if log:
log_config = get_log_config()
else:
log_config = None
env = Environment(trajectory=trajname,
# log_levels=logging.INFO,
# log_config=None,
log_config=log_config,
dynamic_imports=[CustomParameter],
filename=filename, log_stdout=False,
**self.env_kwargs)
return env, filename, logfolder, cntfolder
def test_merge_with_linked_derived_parameter(self, disable_logging=True):
logging.basicConfig(level=logging.ERROR)
self.logfolder = make_temp_dir(
os.path.join('experiments', 'tests', 'Log'))
random.seed()
self.trajname1 = 'T1' + make_trajectory_name(self)
self.trajname2 = 'T2' + make_trajectory_name(self)
self.filename = make_temp_dir(
os.path.join('experiments', 'tests', 'HDF5',
'test%s.hdf5' % self.trajname1))
self.env1 = Environment(trajectory=self.trajname1,
filename=self.filename,
file_title=self.trajname1,
log_stdout=False,
log_config=get_log_config())
self.env2 = Environment(trajectory=self.trajname2,
filename=self.filename,
file_title=self.trajname2,
log_stdout=False,
log_config=get_log_config())
self.traj1 = self.env1.v_trajectory
self.traj2 = self.env2.v_trajectory
create_link_params(self.traj1)
create_link_params(self.traj2)
explore_params(self.traj1)
explore_params2(self.traj2)
self.traj1.f_add_derived_parameter('test.$.gg', 42)
self.traj2.f_add_derived_parameter('test.$.gg', 44)
self.traj1.f_add_derived_parameter('test.hh.$', 111)
self.traj2.f_add_derived_parameter('test.hh.$', 53)
self.env1.f_run(dostuff_and_add_links)
self.env2.f_run(dostuff_and_add_links)
old_length = len(self.traj1)
self.traj1.f_merge(self.traj2, remove_duplicates=True)
self.traj1.f_load(load_data=2)
for run in self.traj1.f_get_run_names():
self.traj1.v_crun = run
idx = self.traj1.v_idx
param = self.traj1['test.crun.gg']
if idx < old_length:
self.assertTrue(param == 42)
else:
self.assertTrue(param == 44)
param = self.traj1['test.hh.crun']
if idx < old_length:
self.assertTrue(param == 111)
else:
self.assertTrue(param == 53)
self.assertTrue(len(self.traj1) > old_length)
for irun in range(len(self.traj1.f_get_run_names())):
self.assertTrue(self.traj1.res['r_%d' % irun] == self.traj1.paramB)
self.assertTrue(
self.traj1.res.runs['r_%d' % irun].paraBL == self.traj1.paramB)
if disable_logging:
self.env1.f_disable_logging()
self.env2.f_disable_logging()
return old_length
def test_merge_with_linked_derived_parameter(self, disable_logging = True):
logging.basicConfig(level = logging.ERROR)
self.logfolder = make_temp_dir(os.path.join('experiments',
'tests',
'Log'))
random.seed()
self.trajname1 = 'T1'+ make_trajectory_name(self)
self.trajname2 = 'T2'+make_trajectory_name(self)
self.filename = make_temp_dir(os.path.join('experiments',
'tests',
'HDF5',
'test%s.hdf5' % self.trajname1))
self.env1 = Environment(trajectory=self.trajname1, filename=self.filename,
file_title=self.trajname1,
log_stdout=False, log_config=get_log_config())
self.env2 = Environment(trajectory=self.trajname2, filename=self.filename,
file_title=self.trajname2,
log_stdout=False, log_config=get_log_config())
self.traj1 = self.env1.v_trajectory
self.traj2 = self.env2.v_trajectory
create_link_params(self.traj1)
create_link_params(self.traj2)
explore_params(self.traj1)
explore_params2(self.traj2)
self.traj1.f_add_derived_parameter('test.$.gg', 42)
self.traj2.f_add_derived_parameter('test.$.gg', 44)
self.traj1.f_add_derived_parameter('test.hh.$', 111)
self.traj2.f_add_derived_parameter('test.hh.$', 53)
self.env1.f_run(dostuff_and_add_links)
self.env2.f_run(dostuff_and_add_links)
old_length = len(self.traj1)
self.traj1.f_merge(self.traj2, remove_duplicates=True)
self.traj1.f_load(load_data=2)
for run in self.traj1.f_get_run_names():
self.traj1.v_crun = run
idx = self.traj1.v_idx
param = self.traj1['test.crun.gg']
if idx < old_length:
self.assertTrue(param == 42)
else:
self.assertTrue(param == 44)
param = self.traj1['test.hh.crun']
if idx < old_length:
self.assertTrue(param == 111)
else:
self.assertTrue(param == 53)
self.assertTrue(len(self.traj1) > old_length)
for irun in range(len(self.traj1.f_get_run_names())):
self.assertTrue(self.traj1.res['r_%d' % irun] == self.traj1.paramB)
self.assertTrue(self.traj1.res.runs['r_%d' % irun].paraBL == self.traj1.paramB)
if disable_logging:
self.env1.f_disable_logging()
self.env2.f_disable_logging()
return old_length