def test_store_load_with_hdf5(self):
traj_name = 'test_%s' % self.__class__.__name__
filename = make_temp_dir(traj_name + '.hdf5')
traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename, overwrite_file=True)
for res in self.results.values():
traj.f_add_result(res)
traj.f_store()
new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename)
new_traj.f_load(load_data=2)
self.compare_trajectories(traj, new_traj)
def load_trajectory(self,trajectory_index=None,trajectory_name=None,as_new=False, how=2):
### Load The Trajectory and check if the values are still the same
newtraj = Trajectory()
newtraj.v_storage_service=HDF5StorageService(filename=self.filename)
newtraj.f_load(name=trajectory_name, index=trajectory_index, as_new=as_new,
load_derived_parameters=how, load_results=how)
return newtraj
def test_a_large_run(self):
get_root_logger().info('Testing large run')
self.traj.f_add_parameter('TEST', 'test_run')
###Explore
self.explore_large(self.traj)
self.make_run_large_data()
self.assertTrue(self.traj.f_is_completed())
# Check if printing and repr work
get_root_logger().info(str(self.env))
get_root_logger().info(repr(self.env))
newtraj = Trajectory()
newtraj.f_load(name=self.traj.v_name, as_new=False, load_data=2, filename=self.filename)
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.compare_trajectories(self.traj,newtraj)
size=os.path.getsize(self.filename)
size_in_mb = size/1000000.
get_root_logger().info('Size is %sMB' % str(size_in_mb))
self.assertTrue(size_in_mb < 30.0, 'Size is %sMB > 30MB' % str(size_in_mb))
def test_store_load_with_hdf5_no_data(self):
traj_name = 'test_%s' % self.__class__.__name__
filename = make_temp_dir(traj_name + 'nodata.hdf5')
traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename, overwrite_file=True)
for param in self.param.values():
param._data = None
traj.f_add_parameter(param)
traj.f_store()
new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename)
new_traj.f_load(load_data=2)
self.compare_trajectories(traj, new_traj)
def test_file_translation(self):
filename = make_temp_dir('to_new_tree.hdf5')
mytraj = Trajectory('SCRATCH', filename=filename)
mytraj.f_add_parameter('Test.Group.Test', 42)
mytraj.f_add_derived_parameter('trajectory.saaaa', 33)
mytraj.f_add_derived_parameter('trajectory.intraj.dpar1', 33)
mytraj.f_add_derived_parameter('run_00000008.inrun.dpar2', 33)
mytraj.f_add_derived_parameter('run_00000001.inrun.dpar3', 35)
mytraj.f_add_result('trajectory.intraj.res1', 33)
mytraj.f_add_result('run_00000008.inrun.res1', 33)
mytraj.f_store()
mytraj.f_migrate(new_name=mytraj.v_name + 'PETER', in_store=True)
mytraj.f_store()
fu = FileUpdater(filename=filename, backup=True)
fu.update_file()
mytraj = Trajectory(name=mytraj.v_name,
add_time=False,
filename=filename)
mytraj.f_load(load_parameters=2,
load_derived_parameters=2,
load_results=2)
for node in mytraj.f_iter_nodes():
self.assertTrue(node.v_name != 'trajectory')
if 'run_' in node.v_full_name:
self.assertTrue('.runs.' in node.v_full_name)
remove_data()
def load_trajectory(fname):
tr = Trajectory(name='tr1',
add_time=False,
filename=fname,
dynamic_imports=[Brian2MonitorResult, Brian2Parameter])
# pypet.pypetconstants.LOAD_NOTHING --> 0
# pypet.pypetconstants.LOAD_SKELETON --> 1
# pypet.pypetconstants.LOAD_DATA --> 2
tr.f_load(load_parameters=2, load_derived_parameters=2, load_results=1)
tr.v_auto_load = True
return tr
def test_run(self):
self.env.f_run(dostuff_and_add_links)
self.traj.f_load(load_data=2)
traj2 = Trajectory()
traj2.f_load(name=self.traj.v_name, filename=self.filename)
traj2.f_load(load_data=2)
for run in self.traj.f_get_run_names():
self.assertTrue(self.traj.res.runs[run].paraBL is self.traj.paramB)
self.compare_trajectories(self.traj, traj2)
def test_storage_and_loading(self):
self.traj.f_store()
traj_name = self.traj.v_name
del self.traj
self.traj = Trajectory(filename=self.filename)
self.traj.f_load(name=traj_name,
load_parameters=2,
load_derived_parameters=2,
load_results=2,
load_other_data=2)
self.test_annotations_insert()
def test_multiple_storage_and_loading(self):
self.filenames = [
make_temp_dir(
os.path.join('experiments', 'tests', 'HDF5',
'multiple_storage_and_loading.hdf5')), 0
]
self.envs = []
self.trajs = []
for irun, filename in enumerate(self.filenames):
if isinstance(filename, int):
filename = self.filenames[filename]
self.make_environment(irun, filename)
self.param_dict = {}
create_param_dict(self.param_dict)
for irun in range(len(self.filenames)):
add_params(self.trajs[irun], self.param_dict)
self.explore(self.trajs[0])
self.explore(self.trajs[1])
for irun in range(len(self.filenames)):
self.make_run(self.envs[irun])
#self.trajs[0].f_store()
temp_sservice = self.trajs[0].v_storage_service
temp_name = self.trajs[0].v_name
self.trajs[0] = Trajectory()
self.trajs[0].v_storage_service = temp_sservice
self.trajs[0].f_load(name=temp_name,
as_new=False,
load_parameters=2,
load_derived_parameters=2,
load_results=2,
load_other_data=2)
#self.trajs[0].f_load(trajectory_name=temp_name,as_new=False, load_params=2, load_derived_params=2, load_results=2)
self.trajs[1].f_load_skeleton()
self.trajs[1].f_load_items(self.trajs[1].f_to_dict().values(),
only_empties=True)
self.compare_trajectories(self.trajs[0], self.trajs[1])
def test_net(self):
self.env.f_run(run_net)
self.traj.f_load(load_derived_parameters=2, load_results=2)
traj2 = Trajectory(name=self.traj.v_name,
add_time=False,
filename=make_temp_dir(
os.path.join('experiments', 'tests',
'briantests', 'HDF5',
'briantest.hdf5')),
dynamic_imports=[
'pypet.brian.parameter.BrianParameter',
BrianMonitorResult
])
traj2.f_load(load_parameters=2,
load_derived_parameters=2,
load_results=2)
self.compare_trajectories(self.traj, traj2)
def setUp(self):
self.filename = make_temp_dir(
os.path.join('experiments', 'tests', 'HDF5', 'annotations.hdf5'))
self.traj = Trajectory(name='Annotations', filename=self.filename)
self.traj.f_add_result('testres', 42)
self.traj.f_add_parameter('testgroup.testparam', 42)
self.make_annotations()
self.add_annotations(self.traj)
pred = lambda x: 'config' not in x.v_full_name
x = len([
node
for node in self.traj.f_iter_nodes(recursive=True, predicate=pred)
])
self.assertTrue(x == 5, '%s != %s' % (str(x), str(5)))
def test_pipeline(self):
filename = 'testpostprocpipe.hdf5'
env1, filename, _, _ = self.make_environment(filename, 'k1')
env2 = self.make_environment(filename, 'k2', log=False)[0]
traj1 = env1.v_trajectory
traj2 = env2.v_trajectory
trajs = [traj1, traj2]
traj2.f_add_parameter('x', 1, comment='1st')
traj2.f_add_parameter('y', 1, comment='1st')
exp_dict2 = {
'x': [1, 2, 3, 4, 1, 2, 2, 3],
'y': [1, 2, 3, 4, 1, 2, 0, 1]
}
traj2.f_explore(exp_dict2)
res1 = env1.pipeline(pipeline=mypipeline)
self.are_results_in_order(res1)
res2 = env2.f_run(Multiply(), 22)
self.are_results_in_order(res2)
traj_name = traj1.v_name
traj1 = Trajectory(traj_name, add_time=False, filename=filename)
traj1.f_load(load_data=2)
traj2.f_load(load_data=2)
self.compare_trajectories(traj1, traj2)
env1.f_disable_logging()
env2.f_disable_logging()
def test_f_iter_runs_auto_load(self):
###Explore
self.explore(self.traj)
results = self.env.f_run(multiply)
self.are_results_in_order(results)
traj = self.traj
self.assertTrue(len(traj) == len(list(self.explore_dict.values())[0]))
self.traj.f_load_skeleton()
self.traj.f_load_items(self.traj.f_to_dict().keys(), only_empties=True)
self.check_if_z_is_correct(traj)
newtraj = Trajectory()
newtraj.v_storage_service=HDF5StorageService(filename=self.filename)
newtraj.f_load(name=self.traj.v_name, index=None, as_new=False, load_data=0)
newtraj.v_auto_load = True
newtraj.par.f_load_child('y', load_data=1)
for idx, run_name in enumerate(self.traj.f_iter_runs()):
newtraj.v_crun=run_name
self.traj.v_idx = idx
newtraj.v_idx = idx
nameset = set((x.v_name for x in traj.f_iter_nodes(predicate=(idx,))))
self.assertTrue('run_%08d' % (idx+1) not in nameset)
self.assertTrue('run_%08d' % idx in nameset)
self.assertTrue(traj.v_crun == run_name)
self.assertTrue(newtraj.res.runs.crun.z==newtraj.par.x*newtraj.par.y,' z != x*y: %s != %s * %s' %
(str(newtraj.crun.z),str(newtraj.x),str(newtraj.y)))
traj = self.traj
self.assertTrue(traj.v_idx == -1)
self.assertTrue(traj.v_crun is None)
self.assertTrue(traj.v_crun_ == pypetconstants.RUN_NAME_DUMMY)
self.assertTrue(newtraj.v_idx == idx)
__author__ = 'Robert Meyer'
import numpy as np
import os # For path names being viable under Windows and Linux
from pypet.trajectory import Trajectory
from pypet import pypetconstants
# Here I show how to store and load results in parts if they are quite large.
# I will skip using an environment and only work with a trajectory.
# We can create a trajectory and hand it a filename directly and it will create an
# HDF5StorageService for us:
filename = os.path.join('hdf5', 'example_09.hdf5')
traj = Trajectory(name='example_09_huge_data',
filename=filename,
overwrite_file=True)
# Now we directly add a huge result. Note that we could do the exact same procedure during
# a single run, there is no syntactical difference.
# However, the sub branch now is different, the result will be found under `traj.results.trajectory`
# instead of `traj.results.run_XXXXXXXX` (where XXXXXXX is the current run index, e.g. 00000007).
# We will add two large matrices a 100 by 100 by 100 one and 1000 by 1000 one, both containing
# random numbers. They are called `mat1` and `mat2` and are handled by the same result object
# called `huge_matrices`:
traj.f_add_result('huge_matrices',
mat1=np.random.rand(100, 100, 100),
mat2=np.random.rand(1000, 1000))
# Note that the result will not support fast access since it contains more than a single
# data item. Even if there was only `mat1`, because the name is `mat1` instead of `huge_matrices`
env.run(multiply) # Now let's see how we can reload the stored data from above. # We do not need an environment for that, just a trajectory. from pypet.trajectory import Trajectory # So, first let's create a new trajectory and pass it the path and name of the HDF5 file. # Yet, to be very clear let's delete all the old stuff. del traj # Before deleting the environment let's disable logging and close all log-files env.disable_logging() del env traj = Trajectory(filename=filename) # Now we want to load all stored data. traj.f_load(index=-1, load_parameters=2, load_results=2) # Above `index` specifies that we want to load the trajectory with that particular index # within the HDF5 file. We could instead also specify a `name`. # Counting works also backwards, so `-1` yields the last or newest trajectory in the file. # # Next we need to specify how the data is loaded. # Therefore, we have to set the keyword arguments `load_parameters` and `load_results`, # here we chose both to be `2`. # `0` would mean we do not want to load anything at all. # `1` would mean we only want to load the empty hulls or skeletons of our parameters # or results. Accordingly, we would add parameters or results to our trajectory # but they would not contain any data.
