def test_overwrite_stuff(self):
traj = Trajectory(name='Test', filename=make_temp_file('testowrite.hdf5'))
res = traj.f_add_result('mytest.test', a='b', c='d')
traj.f_store()
res['a'] = 333
res['c'] = 123445
traj.f_store_item(res, overwrite='a')
# Should emit a warning
traj.f_store_item(res, overwrite=['a', 'b'])
traj.f_load(load_results=3)
res = traj.test
self.assertTrue(res['a']==333)
self.assertTrue(res['c']=='d')
res['c'] = 123445
traj.f_store_item(res, overwrite=True)
res.f_empty()
traj.f_load(load_results=3)
self.assertTrue(traj.test['c']==123445)
def test_store_and_load_large_dictionary(self):
traj = Trajectory(name='Test', filename=make_temp_file('large_dict.hdf5'))
large_dict = {}
for irun in range(1025):
large_dict['item_%d' % irun] = irun
large_dict2 = {}
for irun in range(33):
large_dict2['item_%d' % irun] = irun
traj.f_add_result('large_dict', large_dict, comment='Huge_dict!')
traj.f_add_result('large_dict2', large_dict2, comment='Not so large dict!')
traj.f_store()
traj_name = traj.v_name
traj2 = Trajectory(filename=make_temp_file('large_dict.hdf5'))
traj2.f_load(name=traj_name, load_all=2)
self.compare_trajectories(traj, traj2)
def load_trajectory(self,trajectory_index=None,trajectory_name=None,as_new=False):
### Load The Trajectory and check if the values are still the same
newtraj = Trajectory()
newtraj.v_storage_service=HDF5StorageService(filename=self.filename)
newtraj.f_load(name=trajectory_name, load_derived_parameters=2,load_results=2,
index=trajectory_index, as_new=as_new)
return newtraj
def test_a_large_run(self):
get_root_logger().info('Testing large run')
self.traj.f_add_parameter('TEST', 'test_run')
###Explore
self.explore_large(self.traj)
self.make_run_large_data()
self.assertTrue(self.traj.f_is_completed())
# Check if printing and repr work
get_root_logger().info(str(self.env))
get_root_logger().info(repr(self.env))
newtraj = Trajectory()
newtraj.f_load(name=self.traj.v_name, as_new=False, load_data=2, filename=self.filename)
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.compare_trajectories(self.traj,newtraj)
size=os.path.getsize(self.filename)
size_in_mb = size/1000000.
get_root_logger().info('Size is %sMB' % str(size_in_mb))
self.assertTrue(size_in_mb < 30.0, 'Size is %sMB > 30MB' % str(size_in_mb))
def load_trajectory(self,trajectory_index=None,trajectory_name=None,as_new=False, how=2):
### Load The Trajectory and check if the values are still the same
newtraj = Trajectory()
newtraj.v_storage_service=HDF5StorageService(filename=self.filename)
newtraj.f_load(name=trajectory_name, index=trajectory_index, as_new=as_new,
load_derived_parameters=how, load_results=how)
return newtraj
def test_store_items_and_groups(self):
traj = Trajectory(name='testtraj', filename=make_temp_file('teststoreitems.hdf5'))
traj.f_store()
traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj.f_store_items(['test','testres','group1'])
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_file('teststoreitems.hdf5'))
traj2.f_load(load_results=2,load_parameters=2)
traj.f_add_result('Im.stored.along.a.path', 43)
traj.Im.stored.along.v_annotations['wtf'] =4444
traj.res.f_store_child('Im.stored.along.a.path')
traj2.res.f_load_child('Im.stored.along.a.path', load_data=2)
self.compare_trajectories(traj,traj2)
def test_partially_delete_stuff(self):
traj = Trajectory(name='TestDelete',
filename=make_temp_file('testpartiallydel.hdf5'))
res = traj.f_add_result('mytest.test', a='b', c='d')
traj.f_store()
self.assertTrue('a' in res)
traj.f_delete_item(res, delete_only=['a'], remove_from_item=True)
self.assertTrue('c' in res)
self.assertTrue('a' not in res)
res['a'] = 'offf'
self.assertTrue('a' in res)
traj.f_load(load_results=3)
self.assertTrue('a' not in res)
self.assertTrue('c' in res)
traj.f_delete_item(res, remove_from_trajectory=True, remove_empty_groups=True)
self.assertTrue('results' not in traj)
self.assertTrue(res not in traj)
def test_loading(filenames, traj_names):
loading_times = np.zeros(len(traj_names))
loading_times_wd = np.zeros(len(traj_names))
n_groups= np.zeros(len(traj_names), dtype='int')
for idx, traj_name in enumerate(traj_names):
filename = filenames[idx]
traj = Trajectory(name=traj_name, filename=filename, add_time=False)
start = time.time()
traj.f_load(load_parameters=2, load_results=1, load_derived_parameters=1)
elapsed = (time.time() - start)
loading_times[idx]=elapsed
n_groups[idx] = len([x for x in traj.f_iter_nodes(recursive=True)])
del traj
traj = Trajectory(name=traj_name, filename=filename, add_time=False)
start = time.time()
traj.f_load(load_all=2)
elapsed = (time.time() - start)
loading_times_wd[idx]=elapsed
for idx, loading_time in enumerate(loading_times):
loading_time_wd = loading_times_wd[idx]
groups = n_groups[idx]
print('Groups: %d, Loading: %.3fs, with Data: %.3fs' % (groups, loading_time, loading_time_wd))
def test_a_large_run(self):
get_root_logger().info('Testing large run')
self.traj.f_add_parameter('TEST', 'test_run')
###Explore
self.explore_large(self.traj)
self.make_run_large_data()
self.assertTrue(self.traj.f_is_completed())
# Check if printing and repr work
get_root_logger().info(str(self.env))
get_root_logger().info(repr(self.env))
newtraj = Trajectory()
newtraj.f_load(name=self.traj.v_name, as_new=False, load_data=2, filename=self.filename)
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.compare_trajectories(self.traj,newtraj)
size=os.path.getsize(self.filename)
size_in_mb = size/1000000.
get_root_logger().info('Size is %sMB' % str(size_in_mb))
self.assertTrue(size_in_mb < 30.0, 'Size is %sMB > 30MB' % str(size_in_mb))
def test_net(self):
self.env.f_run(run_net)
self.traj.f_load(load_derived_parameters=2, load_results=2)
traj2 = Trajectory(name = self.traj.v_name, add_time=False,
filename=make_temp_file('experiments/tests/briantests/HDF5/briantest.hdf5'),
dynamically_imported_classes=['pypet.brian.parameter.BrianParameter',
BrianMonitorResult])
traj2.f_load(load_parameters=2, load_derived_parameters=2, load_results=2)
self.compare_trajectories(self.traj, traj2)
def load_trajectory(fname):
tr = Trajectory(name='tr1',
add_time=False,
filename=fname,
dynamic_imports=[Brian2MonitorResult, Brian2Parameter])
# pypet.pypetconstants.LOAD_NOTHING --> 0
# pypet.pypetconstants.LOAD_SKELETON --> 1
# pypet.pypetconstants.LOAD_DATA --> 2
tr.f_load(load_parameters=2, load_derived_parameters=2, load_results=1)
tr.v_auto_load = True
return tr
def test_run(self):
self.env.f_run(dostuff_and_add_links)
self.traj.f_load(load_data=2)
traj2 = Trajectory()
traj2.f_load(name=self.traj.v_name, filename=self.filename)
traj2.f_load(load_data=2)
for run in self.traj.f_get_run_names():
self.assertTrue(self.traj.res.runs[run].paraBL is self.traj.paramB)
self.compare_trajectories(self.traj, traj2)
def test_run(self):
self.env.f_run(dostuff_and_add_links)
self.traj.f_load(load_data=2)
traj2 = Trajectory()
traj2.f_load(name=self.traj.v_name, filename=self.filename)
traj2.f_load(load_data=2)
for run in self.traj.f_get_run_names():
self.assertTrue(self.traj.res.runs[run].paraBL is self.traj.paramB)
self.compare_trajectories(self.traj, traj2)
def test_store_load_with_hdf5(self):
traj_name = 'test_%s' % self.__class__.__name__
filename = make_temp_dir(traj_name + '.hdf5')
traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename, overwrite_file=True)
for res in self.results.values():
traj.f_add_result(res)
traj.f_store()
new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename)
new_traj.f_load(load_data=2)
self.compare_trajectories(traj, new_traj)
def test_store_load_with_hdf5(self):
traj_name = 'test_%s' % self.__class__.__name__
filename = make_temp_dir(traj_name + '.hdf5')
traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename, overwrite_file=True)
for res in self.results.values():
traj.f_add_result(res)
traj.f_store()
new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename)
new_traj.f_load(load_data=2)
self.compare_trajectories(traj, new_traj)
def test_store_load_with_hdf5_no_data(self):
traj_name = 'test_%s' % self.__class__.__name__
filename = make_temp_dir(traj_name + 'nodata.hdf5')
traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename, overwrite_file=True)
for param in self.param.values():
param._data = None
traj.f_add_parameter(param)
traj.f_store()
new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename)
new_traj.f_load(load_data=2)
self.compare_trajectories(traj, new_traj)
def test_store_load_with_hdf5_no_data(self):
traj_name = 'test_%s' % self.__class__.__name__
filename = make_temp_dir(traj_name + 'nodata.hdf5')
traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename, overwrite_file=True)
for param in self.param.values():
param._data = None
traj.f_add_parameter(param)
traj.f_store()
new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename)
new_traj.f_load(load_data=2)
self.compare_trajectories(traj, new_traj)
def test_net(self):
self.env.f_run(run_net)
self.traj.f_load(load_derived_parameters=2, load_results=2)
traj2 = Trajectory(name = self.traj.v_name, add_time=False,
filename=make_temp_dir(os.path.join(
'experiments',
'tests',
'briantests',
'HDF5',
'briantest.hdf5')),
dynamic_imports=['pypet.brian.parameter.BrianParameter',
BrianMonitorResult])
traj2.f_load(load_parameters=2, load_derived_parameters=2, load_results=2)
self.compare_trajectories(self.traj, traj2)
def test_file_translation(self):
filename = make_temp_dir('to_new_tree.hdf5')
mytraj = Trajectory('SCRATCH', filename=filename)
mytraj.f_add_parameter('Test.Group.Test', 42)
mytraj.f_add_derived_parameter('trajectory.saaaa', 33)
mytraj.f_add_derived_parameter('trajectory.intraj.dpar1', 33)
mytraj.f_add_derived_parameter('run_00000008.inrun.dpar2', 33)
mytraj.f_add_derived_parameter('run_00000001.inrun.dpar3', 35)
mytraj.f_add_result('trajectory.intraj.res1', 33)
mytraj.f_add_result('run_00000008.inrun.res1', 33)
mytraj.f_store()
mytraj.f_migrate(new_name=mytraj.v_name + 'PETER', in_store=True)
mytraj.f_store()
fu = FileUpdater(filename=filename, backup=True)
fu.update_file()
mytraj = Trajectory(name=mytraj.v_name,
add_time=False,
filename=filename)
mytraj.f_load(load_parameters=2,
load_derived_parameters=2,
load_results=2)
for node in mytraj.f_iter_nodes():
self.assertTrue(node.v_name != 'trajectory')
if 'run_' in node.v_full_name:
self.assertTrue('.runs.' in node.v_full_name)
remove_data()
def test_net(self):
self.env.f_run(run_net)
self.traj.f_load(load_derived_parameters=2, load_results=2)
traj2 = Trajectory(name=self.traj.v_name,
add_time=False,
filename=make_temp_dir(
os.path.join('experiments', 'tests',
'briantests', 'HDF5',
'briantest.hdf5')),
dynamic_imports=[
'pypet.brian.parameter.BrianParameter',
BrianMonitorResult
])
traj2.f_load(load_parameters=2,
load_derived_parameters=2,
load_results=2)
self.compare_trajectories(self.traj, traj2)
def test_storage_service_errors(self):
traj = Trajectory(filename=make_temp_file('testnoservice.hdf5'))
traj_name = traj.v_name
# you cannot store stuff before the trajectory was stored once:
with self.assertRaises(ValueError):
traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)
traj.f_store()
with self.assertRaises(ValueError):
traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = 'test')
with self.assertRaises(pex.NoSuchServiceError):
traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)
with self.assertRaises(ValueError):
traj.f_load(name = 'test', index=1)
with self.assertRaises(RuntimeError):
traj.v_storage_service.store('LIST', [('LEAF',None,None,None,None)], trajectory_name = traj.v_name)
with self.assertRaises(ValueError):
traj.f_load(index=9999)
with self.assertRaises(ValueError):
traj.f_load(name='Non-Existising-Traj')
def main():
folder = 'experiments/example_11/HDF5/'
filename = 'Clustered_Network.hdf5'
filename = os.path.join(folder, filename)
# If we pass a filename to the trajectory a new HDF5StorageService will
# be automatically created
traj = Trajectory(filename=filename,
dynamically_imported_classes=[BrianDurationParameter,
BrianMonitorResult,
BrianParameter])
# Let's create and fake environment to enable logging:
Environment(traj, do_single_runs=False)
# Load the trajectory, but onyl laod the skeleton of the results
traj.f_load(index=0, # Change if you do not want to load the very first trajectory
load_parameters=2,
load_derived_parameters=2,
load_results=1)
# Find the result instances related to the fano factor
fano_dict = traj.f_get_from_runs('mean_fano_factor', fast_access=False)
# Load the data of the fano factor results
ffs = fano_dict.values()
traj.f_load_items(ffs)
# Extract all values and R_ee values for each run
ffs_values = [x.f_get() for x in ffs]
Rees = traj.f_get('R_ee').f_get_range()
# Plot average fano factor as a function of R_ee
plt.plot(Rees, ffs_values)
plt.xlabel('R_ee')
plt.ylabel('Avg. Fano Factor')
plt.show()
def test_pipeline(self):
filename = 'testpostprocpipe.hdf5'
env1, filename, _, _ = self.make_environment(filename, 'k1')
env2 = self.make_environment(filename, 'k2', log=False)[0]
traj1 = env1.v_trajectory
traj2 = env2.v_trajectory
trajs = [traj1, traj2]
traj2.f_add_parameter('x', 1, comment='1st')
traj2.f_add_parameter('y', 1, comment='1st')
exp_dict2 = {
'x': [1, 2, 3, 4, 1, 2, 2, 3],
'y': [1, 2, 3, 4, 1, 2, 0, 1]
}
traj2.f_explore(exp_dict2)
res1 = env1.pipeline(pipeline=mypipeline)
self.are_results_in_order(res1)
res2 = env2.f_run(Multiply(), 22)
self.are_results_in_order(res2)
traj_name = traj1.v_name
traj1 = Trajectory(traj_name, add_time=False, filename=filename)
traj1.f_load(load_data=2)
traj2.f_load(load_data=2)
self.compare_trajectories(traj1, traj2)
env1.f_disable_logging()
env2.f_disable_logging()
def test_pipeline(self):
filename = 'testpostprocpipe.hdf5'
env1, filename, _, _ = self.make_environment(filename, 'k1')
env2 = self.make_environment(filename, 'k2', log=False)[0]
traj1 = env1.v_trajectory
traj2 = env2.v_trajectory
trajs = [traj1, traj2]
traj2.f_add_parameter('x', 1, comment='1st')
traj2.f_add_parameter('y', 1, comment='1st')
exp_dict2 = {'x':[1, 2, 3, 4, 1, 2, 2, 3],
'y':[1, 2, 3, 4, 1, 2, 0, 1]}
traj2.f_explore(exp_dict2)
res1 = env1.f_pipeline(pipeline=mypipeline)
self.are_results_in_order(res1)
res2 = env2.f_run(Multiply(), 22)
self.are_results_in_order(res2)
traj_name = traj1.v_name
traj1 = Trajectory(traj_name, add_time=False, filename=filename)
traj1.f_load(load_data=2)
traj2.f_load(load_data=2)
self.compare_trajectories(traj1, traj2)
env1.f_disable_logging()
env2.f_disable_logging()
def test_f_iter_runs_auto_load(self):
###Explore
self.explore(self.traj)
results = self.env.f_run(multiply)
self.are_results_in_order(results)
traj = self.traj
self.assertTrue(len(traj) == len(compat.listvalues(self.explore_dict)[0]))
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.check_if_z_is_correct(traj)
newtraj = Trajectory()
newtraj.v_storage_service=HDF5StorageService(filename=self.filename)
newtraj.f_load(name=self.traj.v_name, index=None, as_new=False, load_data=0)
newtraj.v_auto_load = True
newtraj.par.f_load_child('y', load_data=1)
for idx, run_name in enumerate(self.traj.f_iter_runs()):
newtraj.v_crun=run_name
self.traj.v_idx = idx
newtraj.v_idx = idx
nameset = set((x.v_name for x in traj.f_iter_nodes(predicate=(idx,))))
self.assertTrue('run_%08d' % (idx+1) not in nameset)
self.assertTrue('run_%08d' % idx in nameset)
self.assertTrue(traj.v_crun == run_name)
self.assertTrue(newtraj.res.runs.crun.z==newtraj.par.x*newtraj.par.y,' z != x*y: %s != %s * %s' %
(str(newtraj.crun.z),str(newtraj.x),str(newtraj.y)))
traj = self.traj
self.assertTrue(traj.v_idx == -1)
self.assertTrue(traj.v_crun is None)
self.assertTrue(traj.v_crun_ == pypetconstants.RUN_NAME_DUMMY)
self.assertTrue(newtraj.v_idx == idx)
def test_file_translation(self):
filename = make_temp_dir('to_new_tree.hdf5')
mytraj = Trajectory('SCRATCH', filename=filename)
mytraj.f_add_parameter('Test.Group.Test', 42)
mytraj.f_add_derived_parameter('trajectory.saaaa',33)
mytraj.f_add_derived_parameter('trajectory.intraj.dpar1',33)
mytraj.f_add_derived_parameter('run_00000008.inrun.dpar2',33)
mytraj.f_add_derived_parameter('run_00000001.inrun.dpar3',35)
mytraj.f_add_result('trajectory.intraj.res1',33)
mytraj.f_add_result('run_00000008.inrun.res1',33)
mytraj.f_store()
mytraj.f_migrate(new_name=mytraj.v_name + 'PETER', in_store=True)
mytraj.f_store()
fu=FileUpdater(filename=filename, backup=True)
fu.update_file()
mytraj = Trajectory(name=mytraj.v_name, add_time=False, filename=filename)
mytraj.f_load(load_parameters=2, load_derived_parameters=2, load_results=2)
for node in mytraj.f_iter_nodes():
self.assertTrue(node.v_name != 'trajectory')
if 'run_' in node.v_full_name:
self.assertTrue('.runs.' in node.v_full_name)
remove_data()
def test_f_iter_runs_auto_load(self):
###Explore
self.explore(self.traj)
results = self.env.f_run(multiply)
self.are_results_in_order(results)
traj = self.traj
self.assertTrue(len(traj) == len(list(self.explore_dict.values())[0]))
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.check_if_z_is_correct(traj)
newtraj = Trajectory()
newtraj.v_storage_service=HDF5StorageService(filename=self.filename)
newtraj.f_load(name=self.traj.v_name, index=None, as_new=False, load_data=0)
newtraj.v_auto_load = True
newtraj.par.f_load_child('y', load_data=1)
for idx, run_name in enumerate(self.traj.f_iter_runs()):
newtraj.v_crun=run_name
self.traj.v_idx = idx
newtraj.v_idx = idx
nameset = set((x.v_name for x in traj.f_iter_nodes(predicate=(idx,))))
self.assertTrue('run_%08d' % (idx+1) not in nameset)
self.assertTrue('run_%08d' % idx in nameset)
self.assertTrue(traj.v_crun == run_name)
self.assertTrue(newtraj.res.runs.crun.z==newtraj.par.x*newtraj.par.y,' z != x*y: %s != %s * %s' %
(str(newtraj.crun.z),str(newtraj.x),str(newtraj.y)))
traj = self.traj
self.assertTrue(traj.v_idx == -1)
self.assertTrue(traj.v_crun is None)
self.assertTrue(traj.v_crun_ == pypetconstants.RUN_NAME_DUMMY)
self.assertTrue(newtraj.v_idx == idx)
def test_version_mismatch(self):
traj = Trajectory(name='Test', filename=make_temp_file('testversionmismatch.hdf5'))
traj.f_add_parameter('group1.test',42)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj._version='0.1a.1'
traj.f_store()
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_file('testversionmismatch.hdf5'))
with self.assertRaises(pex.VersionMismatchError):
traj2.f_load(load_results=2,load_parameters=2)
traj2.f_load(load_results=2,load_parameters=2, force=True)
self.compare_trajectories(traj,traj2)
def profile_single_storing(profile_stroing=False, profile_loading=True):
logging.basicConfig(level = logging.INFO)
logfolder = os.path.join(tempfile.gettempdir(), TEMPDIR, 'logs')
pathfolder = os.path.join(tempfile.gettempdir(), TEMPDIR, 'hdf5')
res_per_run = 100
env = Environment(log_folder=logfolder, filename=pathfolder,
ncores=2, multiproc=False,
use_pool=True,
wrap_mode='QUEUE')
traj = env.v_trajectory
traj.f_add_parameter('res_per_run', res_per_run)
traj.f_add_parameter('trial', 0)
traj.f_explore({'trial':list(range(10))})
runexp = lambda : env.f_run(add_data)
if profile_stroing:
cProfile.runctx('runexp()', {'runexp': runexp},globals(), sort=1,
filename='store_stats.profile')
else:
runexp()
print('########################################################################')
traj = Trajectory(name=traj.v_name, add_time=False, filename= traj.v_storage_service.filename)
load = lambda : traj.f_load(load_parameters=2, load_results=1)
if profile_loading:
cProfile.runctx('load()', {'load': load},globals(), filename='load_stats.profile', sort=1)
traj.huge_matrices.f_set(monty='Always look on the bright side of life!')
# Next we can store our new string called monty to disk. Since our trajectory was already
# stored to disk once, we can make use of the functionality to store individual items:
traj.f_store_item('huge_matrices')
# Neat, hu? Ok now let's load some of it back, for educational purposes let's start with a fresh
# trajectory. Let's keep the old trajectory name in mind. The current time is added to the
# trajectory name on creation (if you do not want this, just say `add_time=False`).
# Thus, the name is not `example_09_huge_data`, but `example_09_huge_data_XXXX_XX_XX_XXhXXmXXs`:
old_traj_name = traj.v_name
del traj
traj = Trajectory(filename=filename)
# We only want to load the skeleton but not the data:
traj.f_load(name=old_traj_name, load_results=pypetconstants.LOAD_SKELETON)
# Check if we only loaded the skeleton, that means the `huge_matrices` result must be empty:
if traj.huge_matrices.f_is_empty():
print('Told you!')
else:
print('Unbelievable, this sucks!')
# Now let's only load `monty` and `mat1`.
# We can do this by passing the keyword argument `load_only` to the load item function:
traj.f_load_item('huge_matrices', load_only=['monty','mat1'])
# Check if this worked:
if ('monty' in traj.huge_matrices and
'mat1' in traj.huge_matrices and
not 'mat2' in traj.huge_matrices ):
def main():
env = Environment(trajectory='Example_05_Euler_Integration',
filename='experiments/example_05/HDF5/example_05.hdf5',
file_title='Example_05_Euler_Integration',
log_folder='experiments/example_05/LOGS/',
comment = 'Go for Euler!')
traj = env.v_trajectory
trajectory_name = traj.v_name
# 1st a) phase parameter addition
add_parameters(traj)
# 1st b) phase preparation
# We will add the differential equation (well, its source code only) as a derived parameter
traj.f_add_derived_parameter(FunctionParameter,'diff_eq', diff_lorenz,
comment='Source code of our equation!')
# We want to explore some initial conditions
traj.f_explore({'initial_conditions' : [
np.array([0.01,0.01,0.01]),
np.array([2.02,0.02,0.02]),
np.array([42.0,4.2,0.42])
]})
# 3 different conditions are enough for an illustrative example
# 2nd phase let's run the experiment
# We pass `euler_scheme` as our top-level simulation function and
# the Lorenz equation 'diff_lorenz' as an additional argument
env.f_run(euler_scheme, diff_lorenz)
# We don't have a 3rd phase of post-processing here
# 4th phase analysis.
# I would recommend to do post-processing completely independent from the simulation,
# but for simplicity let's do it here.
# Let's assume that we start all over again and load the entire trajectory new.
# Yet, there is an error within this approach, do you spot it?
del traj
traj = Trajectory(filename='experiments/example_05/HDF5/example_05.hdf5')
# We will only fully load parameters and derived parameters.
# Results will be loaded manually later on.
try:
# However, this will fail because our trajectory does not know how to
# build the FunctionParameter. You have seen this coming, right?
traj.f_load(name=trajectory_name,load_parameters=2,
load_derived_parameters=2,load_results=1)
except ImportError as e:
print 'That did\'nt work, I am sorry. %s ' % e.message
# Ok, let's try again but this time with adding our parameter to the imports
traj = Trajectory(filename='experiments/example_05/HDF5/example_05.hdf5',
dynamically_imported_classes=FunctionParameter)
# Now it works:
traj.f_load(name=trajectory_name,load_parameters=2,
load_derived_parameters=2,load_results=1)
#For the fun of it, let's print the source code
print '\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq
# Let's get the exploration array:
initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range()
# Now let's plot our simulated equations for the different initial conditions:
# We will iterate through the run names
for idx, run_name in enumerate(traj.f_get_run_names()):
#Get the result of run idx from the trajectory
euler_result = traj.results.f_get(run_name).euler_evolution
# Now we manually need to load the result. Actually the results are not so large and we
# could load them all at once. But for demonstration we do as if they were huge:
traj.f_load_item(euler_result)
euler_data = euler_result.data
#Plot fancy 3d plot
fig = plt.figure(idx)
ax = fig.gca(projection='3d')
x = euler_data[:,0]
y = euler_data[:,1]
z = euler_data[:,2]
ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx]))
plt.legend()
plt.show()
# Now we free the data again (because we assume its huuuuuuge):
del euler_data
euler_result.f_empty()
# Now let's see how we can reload the stored data from above. # We do not need an environment for that, just a trajectory. from pypet.trajectory import Trajectory # So, first let's create a new trajectory and pass it the path and name of the HDF5 file. # Yet, to be very clear let's delete all the old stuff. del traj # Before deleting the environment let's disable logging and close all log-files env.disable_logging() del env traj = Trajectory(filename=filename) # Now we want to load all stored data. traj.f_load(index=-1, load_parameters=2, load_results=2) # Above `index` specifies that we want to load the trajectory with that particular index # within the HDF5 file. We could instead also specify a `name`. # Counting works also backwards, so `-1` yields the last or newest trajectory in the file. # # Next we need to specify how the data is loaded. # Therefore, we have to set the keyword arguments `load_parameters` and `load_results`, # here we chose both to be `2`. # `0` would mean we do not want to load anything at all. # `1` would mean we only want to load the empty hulls or skeletons of our parameters # or results. Accordingly, we would add parameters or results to our trajectory # but they would not contain any data. # Instead `2` means we want to load the parameters and results including the data they contain. # Finally we want to print a result of a particular run.
traj.huge_matrices.f_set(monty='Always look on the bright side of life!')
# Next we can store our new string called monty to disk. Since our trajectory was already
# stored to disk once, we can make use of the functionality to store individual items:
traj.f_store_item('huge_matrices')
# Neat, hu? Ok now let's load some of it back, for educational purposes let's start with a fresh
# trajectory. Let's keep the old trajectory name in mind. The current time is added to the
# trajectory name on creation (if you do not want this, just say `add_time=False`).
# Thus, the name is not `example_09_huge_data`, but `example_09_huge_data_XXXX_XX_XX_XXhXXmXXs`:
old_traj_name = traj.v_name
del traj
traj = Trajectory(filename=filename)
# We only want to load the skeleton but not the data:
traj.f_load(name=old_traj_name, load_results=pypetconstants.LOAD_SKELETON)
# Check if we only loaded the skeleton, that means the `huge_matrices` result must be empty:
if traj.huge_matrices.f_is_empty():
print('Told you!')
else:
print('Unbelievable, this sucks!')
# Now let's only load `monty` and `mat1`.
# We can do this by passing the keyword argument `load_only` to the load item function:
traj.f_load_item('huge_matrices', load_only=['monty', 'mat1'])
# Check if this worked:
if ('monty' in traj.huge_matrices and 'mat1' in traj.huge_matrices
and not 'mat2' in traj.huge_matrices):
class AnnotationsTest(unittest.TestCase):
def setUp(self):
self.filename = make_temp_file('experiments/tests/HDF5/annotations.hdf5')
self.traj = Trajectory(name='Annotations', filename = self.filename)
self.traj.f_add_result('testres', 42)
self.traj.f_add_parameter('testgroup.testparam', 42)
self.make_annotations()
self.add_annotations(self.traj)
self.assertTrue(len([node for node in self.traj.f_iter_nodes(recursive=True)]) == 5)
def make_annotations(self):
self.annotations={}
self.annotations['dict']={'33':12,'kkk':[1,2,'h'], 3:{'a':42.0}}
self.annotations['list']= [self.annotations['dict'],33]
self.annotations['string'] = 'string'
self.annotations['integer'] = 42
self.annotations['tuple']=(3,4,5)
self.annotations['Numpy_Data'] = np.array(['fff','ddd'])
self.annotations[0] = 7777
def add_annotations(self, traj):
funcs = 5
for idx,node in enumerate(traj.f_iter_nodes(recursive=True)):
for name in self.annotations:
anno = self.annotations[name]
if name == 0:
node.f_set_annotations(anno)
node.v_annotations.f_set(anno)
elif idx % funcs == 0:
node.f_set_annotations(**{name:anno})
elif idx % funcs == 1:
node.v_annotations.f_set(**{name:anno})
elif idx % funcs == 2:
node.v_annotations.f_set_single(name,anno)
elif idx % funcs == 3:
setattr(node.v_annotations,name, anno)
elif idx % funcs == 4:
node.v_annotations[name]=anno
def test_annotations_insert(self):
for node in self.traj.f_iter_nodes(recursive=True):
for name in self.annotations:
anno = self.annotations[name]
node_anno = node.v_annotations[name]
self.assertTrue(comp.nested_equal(anno, node_anno),
'%s != %s' % (str(anno), str(node_anno)))
def test_pickling(self):
dump = pickle.dumps(self.traj)
del self.traj
self.traj = pickle.loads(dump)
self.test_annotations_insert()
def test_storage_and_loading(self):
self.traj.f_store()
traj_name = self.traj.v_name
del self.traj
self.traj = Trajectory(filename=self.filename)
self.traj.f_load(name=traj_name, load_results=2, load_parameters=2, load_derived_parameters=2,
load_other_data=2)
self.test_annotations_insert()
def test_attribute_deletion(self):
for node in self.traj.f_iter_nodes(recursive=True):
name_list=[name for name in node.v_annotations]
for name in name_list:
delattr(node.v_annotations, name)
self.assertTrue(node.v_annotations.f_is_empty())
def test_get_item(self):
for node in self.traj.f_iter_nodes(recursive=True):
for key, val1 in node.v_annotations.f_to_dict().items():
val2 = node.v_annotations[key]
self.assertTrue(comp.nested_equal(val1,val2))
def test_get_item_no_copy(self):
for node in self.traj.f_iter_nodes(recursive=True):
for key, val1 in node.v_annotations.f_to_dict(copy=False).items():
val2 = node.v_annotations[key]
self.assertTrue(comp.nested_equal(val1,val2))
@staticmethod
def dict_to_str(dictionary):
resstr = ''
new_dict={}
for key, val in dictionary.items():
if key == 0:
key = 'annotation'
new_dict[key]=val
for key in sorted(new_dict.keys()):
resstr+='%s=%s; ' % (key,str(new_dict[key]))
return resstr[:-2]
def test_to_str(self):
dict_str = self.dict_to_str(self.annotations)
for node in self.traj.f_iter_nodes(recursive=True):
ann_str = node.f_ann_to_str()
self.assertTrue(not ann_str.endswith(' ') or not ann_str.endswith(','))
for name in self.annotations:
if name==0:
name = 'annotation'
self.assertTrue(name in ann_str)
self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))
ann_str = str(node.v_annotations)
self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))
def test_single_get_and_getattr_and_setattr(self):
self.traj.f_add_parameter('test2', 42)
self.traj.f_get('test2').v_annotations.test = 4
self.assertTrue(self.traj.f_get('test2').v_annotations.test, 4)
self.assertTrue(self.traj.f_get('test2').v_annotations.f_get(), 4)
def test_get_annotations(self):
key_list = sorted(self.annotations.keys())
for node in self.traj.f_iter_nodes(recursive=True):
for name in self.annotations:
self.assertTrue(comp.nested_equal(self.annotations[name],
node.f_get_annotations(name)))
val_list = node.f_get_annotations(*key_list)
for idx, val in enumerate(val_list):
self.assertTrue(comp.nested_equal(self.annotations[key_list[idx]], val))
def test_f_get_errors(self):
for node in self.traj.f_iter_nodes(recursive=True):
with self.assertRaises(ValueError):
node.v_annotations.f_get()
with self.assertRaises(AttributeError):
node.v_annotations.f_get('gdsdfd')
testparam = self.traj.f_add_parameter('ggg',343)
with self.assertRaises(AttributeError):
testparam.v_annotations.f_get()
def test_f_set_numbering(self):
int_list = range(10)
for node in self.traj.f_iter_nodes(recursive=True):
node.v_annotations.f_set(*int_list)
self.assertEqual(node.v_annotations.f_get(*int_list), tuple(int_list))
for integer in int_list:
if integer == 0:
name = 'annotation'
else:
name = 'annotation_%d' % integer
self.assertTrue(name in node.v_annotations)
class AnnotationsTest(unittest.TestCase):
tags = 'unittest', 'annotations'
def setUp(self):
self.filename = make_temp_dir(
os.path.join('experiments', 'tests', 'HDF5', 'annotations.hdf5'))
self.traj = Trajectory(name='Annotations', filename=self.filename)
self.traj.f_add_result('testres', 42)
self.traj.f_add_parameter('testgroup.testparam', 42)
self.make_annotations()
self.add_annotations(self.traj)
pred = lambda x: 'config' not in x.v_full_name
x = len([
node
for node in self.traj.f_iter_nodes(recursive=True, predicate=pred)
])
self.assertTrue(x == 5, '%s != %s' % (str(x), str(5)))
def tearDown(self):
remove_data()
def make_annotations(self):
self.annotations = {}
self.annotations['dict'] = {
'33': 12,
'kkk': [1, 2, 'h'],
3: {
'a': 42.0
}
}
self.annotations['list'] = [self.annotations['dict'], 33]
self.annotations['string'] = 'string'
self.annotations['integer'] = 42
self.annotations['tuple'] = (3, 4, 5)
self.annotations['Numpy_Data'] = np.array(['fff', 'ddd'])
self.annotations[0] = 7777
def add_annotations(self, traj):
funcs = 5
for idx, node in enumerate(
[traj] + [node for node in traj.f_iter_nodes(recursive=True)]):
for name in self.annotations:
anno = self.annotations[name]
if name == 0:
node.f_set_annotations(anno)
node.v_annotations.f_set(anno)
elif idx % funcs == 0:
node.f_set_annotations(**{name: anno})
elif idx % funcs == 1:
node.v_annotations.f_set(**{name: anno})
elif idx % funcs == 2:
node.v_annotations.f_set_single(name, anno)
elif idx % funcs == 3:
setattr(node.v_annotations, name, anno)
elif idx % funcs == 4:
node.v_annotations[name] = anno
def test_annotations_insert(self):
for idx,node in \
enumerate([self.traj] + [node for node in self.traj.f_iter_nodes(recursive=True)]):
for name in self.annotations:
anno = self.annotations[name]
node_anno = node.v_annotations[name]
self.assertTrue(comp.nested_equal(anno, node_anno),
'%s != %s' % (str(anno), str(node_anno)))
def test_pickling(self):
dump = pickle.dumps(self.traj)
del self.traj
self.traj = pickle.loads(dump)
self.test_annotations_insert()
def test_storage_and_loading(self):
self.traj.f_store()
traj_name = self.traj.v_name
del self.traj
self.traj = Trajectory(filename=self.filename)
self.traj.f_load(name=traj_name,
load_parameters=2,
load_derived_parameters=2,
load_results=2,
load_other_data=2)
self.test_annotations_insert()
def test_attribute_deletion(self):
for node in self.traj.f_iter_nodes(recursive=True):
name_list = [name for name in node.v_annotations]
for name in name_list:
delattr(node.v_annotations, name)
self.assertTrue(node.v_annotations.f_is_empty())
def test_item_deletion(self):
for node in self.traj.f_iter_nodes(recursive=True):
name_list = [name for name in node.v_annotations]
for name in name_list:
del node.v_annotations[name]
self.assertTrue(node.v_annotations.f_is_empty())
def test_get_item(self):
for node in self.traj.f_iter_nodes(recursive=True):
for key, val1 in node.v_annotations.f_to_dict().items():
val2 = node.v_annotations[key]
self.assertTrue(comp.nested_equal(val1, val2))
def test_get_item_no_copy(self):
for node in self.traj.f_iter_nodes(recursive=True):
for key, val1 in node.v_annotations.f_to_dict(copy=False).items():
val2 = node.v_annotations[key]
self.assertTrue(comp.nested_equal(val1, val2))
@staticmethod
def dict_to_str(dictionary):
resstr = ''
new_dict = {}
for key, val in dictionary.items():
if key == 0:
key = 'annotation'
new_dict[key] = val
for key in sorted(new_dict.keys()):
resstr += '%s=%s; ' % (key, str(new_dict[key]))
return resstr[:-2]
def test_to_str(self):
dict_str = self.dict_to_str(self.annotations)
for node in self.traj.f_iter_nodes(recursive=True):
ann_str = node.f_ann_to_str()
self.assertTrue(not ann_str.endswith(' ')
or not ann_str.endswith(','))
for name in self.annotations:
if name == 0:
name = 'annotation'
self.assertTrue(name in ann_str)
self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))
ann_str = str(node.v_annotations)
self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))
def test_single_get_and_getattr_and_setattr(self):
self.traj.f_add_parameter('test2', 42)
self.traj.f_get('test2').v_annotations.test = 4
self.assertTrue(self.traj.f_get('test2').v_annotations.test, 4)
self.assertTrue(self.traj.f_get('test2').v_annotations.f_get(), 4)
def test_get_annotations(self):
key_list = list(self.annotations.keys())
for node in self.traj.f_iter_nodes(recursive=True):
for name in self.annotations:
self.assertTrue(
comp.nested_equal(self.annotations[name],
node.f_get_annotations(name)))
val_list = node.f_get_annotations(*key_list)
for idx, val in enumerate(val_list):
self.assertTrue(
comp.nested_equal(self.annotations[key_list[idx]], val))
def test_f_get_errors(self):
for node in self.traj.f_iter_nodes(recursive=True):
with self.assertRaises(ValueError):
node.v_annotations.f_get()
with self.assertRaises(AttributeError):
node.v_annotations.f_get('gdsdfd')
testparam = self.traj.f_add_parameter('ggg', 343)
with self.assertRaises(AttributeError):
testparam.v_annotations.f_get()
def test_f_set_numbering(self):
int_list = list(range(10))
for node in self.traj.f_iter_nodes(recursive=True):
node.v_annotations.f_set(*int_list)
self.assertEqual(node.v_annotations.f_get(*int_list),
tuple(int_list))
for integer in int_list:
if integer == 0:
name = 'annotation'
else:
name = 'annotation_%d' % integer
self.assertTrue(name in node.v_annotations)
# Now let's see how we can reload the stored data from above. # We do not need an environment for that, just a trajectory. from pypet.trajectory import Trajectory # So, first let's create a new trajectory and pass it the path and name of the HDF5 file. # Yet, to be very clear let's delete all the old stuff. del traj # Before deleting the environment let's disable logging and close all log-files env.f_disable_logging() del env traj = Trajectory(filename=filename) # Now we want to load all stored data. traj.f_load(index=-1, load_parameters=2, load_results=2) # Above `index` specifies that we want to load the trajectory with that particular index # within the HDF5 file. We could instead also specify a `name`. # Counting works also backwards, so `-1` yields the last or newest trajectory in the file. # # Next we need to specify how the data is loaded. # Therefore, we have to set the keyword arguments `load_parameters` and `load_results`, # here we chose both to be `2`. # `0` would mean we do not want to load anything at all. # `1` would mean we only want to load the empty hulls or skeletons of our parameters # or results. Accordingly, we would add parameters or results to our trajectory # but they would not contain any data. # Instead `2` means we want to load the parameters and results including the data they contain. # Finally we want to print a result of a particular run.
def main():
# This time we don't need an environment since we just going to look
# at data in the trajectory
traj = Trajectory('FiringRate', add_time=False)
# Let's load the trajectory from the file
# Only load the parameters, we will load the results on the fly as we need them
traj.f_load(filename='./hdf5/FiringRate.hdf5', load_parameters=2,
load_results=0, load_derived_parameters=0)
# We'll simply use auto loading so all data will be loaded when needed.
traj.v_auto_load = True
rates_frame = traj.res.summary.firing_rates.rates_frame
# Here we load the data automatically on the fly
plt.figure()
plt.subplot(2,1,1)
#Let's iterate through the columns and plot the different firing rates :
for tau_ref, I_col in rates_frame.iteritems():
plt.plot(I_col.index, I_col, label='Avg. Rate for tau_ref=%s' % str(tau_ref))
# Label the plot
plt.xlabel('I')
plt.ylabel('f[Hz]')
plt.title('Firing as a function of input current `I`')
plt.legend()
# Also let's plot an example run, how about run 13 ?
example_run = 13
traj.v_idx = example_run # We make the trajectory behave as a single run container.
# This short statement has two major effects:
# a) all explored parameters are set to the value of run 13,
# b) if there are tree nodes with names other than the current run aka `run_00000013`
# they are simply ignored, if we use the `$` sign or the `crun` statement,
# these are translated into `run_00000013`.
# Get the example data
example_I = traj.I
example_tau_ref = traj.tau_ref
example_V = traj.results.neuron.crun.V # Here crun stands for run_00000013
# We need the time step...
dt = traj.dt
# ...to create an x-axis for the plot
dt_array = [irun * dt for irun in range(len(example_V))]
# And plot the development of V over time,
# Since this is rather repetitive, we only
# plot the first eighth of it.
plt.subplot(2,1,2)
plt.plot(dt_array, example_V)
plt.xlim((0, dt*len(example_V)/8))
# Label the axis
plt.xlabel('t[ms]')
plt.ylabel('V')
plt.title('Example of development of V for I=%s, tau_ref=%s in run %d' %
(str(example_I), str(example_tau_ref), traj.v_idx))
# And let's take a look at it
plt.show()
# Finally revoke the `traj.v_idx=13` statement and set everything back to normal.
# Since our analysis is done here, we could skip that, but it is always a good idea
# to do that.
traj.f_restore_default()