def test_find_in_all_runs_with_links(self):
traj = Trajectory()
traj.f_add_parameter('FloatParam')
traj.par.FloatParam = 4.0
self.explore_dict = {'FloatParam': [1.0, 1.1, 1.2, 1.3]}
traj.f_explore(self.explore_dict)
self.assertTrue(len(traj) == 4)
traj.f_add_result('results.runs.run_00000000.sub.resulttest', 42)
traj.f_add_result('results.runs.run_00000001.sub.resulttest', 43)
traj.f_add_result('results.runs.run_00000002.sub.resulttest', 44)
traj.f_add_result('results.runs.run_00000002.sub.resulttest2', 42)
traj.f_add_result('results.runs.run_00000003.sub.resulttest2', 43)
traj.f_add_derived_parameter(
'derived_parameters.runs.run_00000002.testing', 44)
res_dict = traj.f_get_from_runs('resulttest', fast_access=True)
self.assertTrue(len(res_dict) == 3)
self.assertTrue(res_dict['run_00000001'] == 43)
self.assertTrue('run_00000003' not in res_dict)
res_dict = traj.f_get_from_runs(name='sub.resulttest2',
use_indices=True)
self.assertTrue(len(res_dict) == 2)
self.assertTrue(res_dict[3] is traj.f_get('run_00000003.resulttest2'))
self.assertTrue(1 not in res_dict)
traj.res.runs.r_0.f_add_link('resulttest2',
traj.r_1.f_get('resulttest'))
res_dict = traj.f_get_from_runs(name='resulttest2', use_indices=True)
self.assertTrue(len(res_dict) == 3)
self.assertTrue(res_dict[0] is traj.f_get('run_00000001.resulttest'))
self.assertTrue(1 not in res_dict)
res_dict = traj.f_get_from_runs(name='resulttest2',
use_indices=True,
with_links=False)
self.assertTrue(len(res_dict) == 2)
self.assertTrue(0 not in res_dict)
self.assertTrue(1 not in res_dict)
def test_auto_load(self):
traj = Trajectory(name='Testautoload', filename=make_temp_dir('autoload.hdf5'))
traj.v_auto_load = True
traj.f_add_result('I.am.$.a.mean.resu', 42, comment='Test')
traj.f_add_derived_parameter('ffa', 42)
traj.f_store()
ffa=traj.f_get('ffa')
ffa.f_unlock()
ffa.f_empty()
self.assertTrue(ffa.f_is_empty())
traj.f_remove_child('results', recursive=True)
# check auto load
val = traj.res.I.am.crun.a.mean.resu
self.assertTrue(val==42)
val = traj.ffa
self.assertTrue(val==42)
with self.assertRaises(pex.DataNotInStorageError):
traj.kdsfdsf
def test_auto_load(self):
traj = Trajectory(name='Testautoload', filename=make_temp_dir('autoload.hdf5'),
add_time=True)
traj.v_auto_load = True
traj.f_add_result('I.am.$.a.mean.resu', 42, comment='Test')
traj.f_add_derived_parameter('ffa', 42)
traj.f_store()
ffa=traj.f_get('ffa')
ffa.f_unlock()
ffa.f_empty()
self.assertTrue(ffa.f_is_empty())
traj.f_remove_child('results', recursive=True)
# check auto load
val = traj.res.I.am.crun.a.mean.resu
self.assertTrue(val==42)
val = traj.ffa
self.assertTrue(val==42)
with self.assertRaises(pex.DataNotInStorageError):
traj.kdsfdsf
def main():
filename = os.path.join('hdf5', 'Clustered_Network.hdf5')
# If we pass a filename to the trajectory a new HDF5StorageService will
# be automatically created
traj = Trajectory(filename=filename,
dynamically_imported_classes=[BrianMonitorResult,
BrianParameter])
# Let's create and fake environment to enable logging:
env = Environment(traj, do_single_runs=False)
# Load the trajectory, but onyl laod the skeleton of the results
traj.f_load(index=-1, load_parameters=2, load_derived_parameters=2, load_results=1)
# Find the result instances related to the fano factor
fano_dict = traj.f_get_from_runs('mean_fano_factor', fast_access=False)
# Load the data of the fano factor results
ffs = fano_dict.values()
traj.f_load_items(ffs)
# Extract all values and R_ee values for each run
ffs_values = [x.f_get() for x in ffs]
Rees = traj.f_get('R_ee').f_get_range()
# Plot average fano factor as a function of R_ee
plt.plot(Rees, ffs_values)
plt.xlabel('R_ee')
plt.ylabel('Avg. Fano Factor')
plt.show()
# Finally disable logging and close all log-files
env.disable_logging()
def test_find_in_all_runs_with_links(self):
traj = Trajectory()
traj.f_add_parameter('FloatParam')
traj.par.FloatParam=4.0
self.explore_dict = {'FloatParam':[1.0,1.1,1.2,1.3]}
traj.f_explore(self.explore_dict)
self.assertTrue(len(traj) == 4)
traj.f_add_result('results.runs.run_00000000.sub.resulttest', 42)
traj.f_add_result('results.runs.run_00000001.sub.resulttest', 43)
traj.f_add_result('results.runs.run_00000002.sub.resulttest', 44)
traj.f_add_result('results.runs.run_00000002.sub.resulttest2', 42)
traj.f_add_result('results.runs.run_00000003.sub.resulttest2', 43)
traj.f_add_derived_parameter('derived_parameters.runs.run_00000002.testing', 44)
res_dict = traj.f_get_from_runs('resulttest', fast_access=True)
self.assertTrue(len(res_dict)==3)
self.assertTrue(res_dict['run_00000001']==43)
self.assertTrue('run_00000003' not in res_dict)
res_dict = traj.f_get_from_runs(name='sub.resulttest2', use_indices=True)
self.assertTrue(len(res_dict)==2)
self.assertTrue(res_dict[3] is traj.f_get('run_00000003.resulttest2'))
self.assertTrue(1 not in res_dict)
traj.res.runs.r_0.f_add_link('resulttest2', traj.r_1.f_get('resulttest'))
res_dict = traj.f_get_from_runs(name='resulttest2', use_indices=True)
self.assertTrue(len(res_dict)==3)
self.assertTrue(res_dict[0] is traj.f_get('run_00000001.resulttest'))
self.assertTrue(1 not in res_dict)
res_dict = traj.f_get_from_runs(name='resulttest2', use_indices=True, with_links=False)
self.assertTrue(len(res_dict)==2)
self.assertTrue(0 not in res_dict)
self.assertTrue(1 not in res_dict)
def test_not_getting_links(self):
traj = Trajectory()
traj.f_add_parameter_group('test.test3')
traj.f_add_parameter_group('test2')
traj.test.f_add_link('circle1', traj.test2)
traj.test2.f_add_link('circle2', traj.test)
self.assertTrue(traj.test.circle1 is traj.test2)
traj.v_with_links = False
with self.assertRaises(AttributeError):
traj.test.circle1
found = False
for item in traj.test.f_iter_nodes(recursive=True, with_links=True):
if item is traj.test2:
found = True
self.assertTrue(found)
for item in traj.test.f_iter_nodes(recursive=True, with_links=False):
if item is traj.test2:
self.assertTrue(False)
traj.v_with_links = True
self.assertTrue('circle1' in traj)
self.assertFalse(traj.f_contains('circle1', with_links=False))
self.assertTrue('circle1' in traj.test)
self.assertFalse(traj.test.f_contains('circle1', with_links=False))
self.assertTrue(traj.test2.test3 is traj.par.test.test3)
traj.v_with_links = False
with self.assertRaises(AttributeError):
traj.test2.test3
traj.v_with_links = True
self.assertTrue(traj['test2.test3'] is traj.test3)
with self.assertRaises(AttributeError):
traj.f_get('test2.test3', with_links=False)
def test_not_getting_links(self):
traj = Trajectory()
traj.f_add_parameter_group('test.test3')
traj.f_add_parameter_group('test2')
traj.test.f_add_link('circle1' , traj.test2)
traj.test2.f_add_link('circle2' , traj.test)
self.assertTrue(traj.test.circle1 is traj.test2)
traj.v_with_links = False
with self.assertRaises(AttributeError):
traj.test.circle1
found = False
for item in traj.test.f_iter_nodes(recursive=True, with_links=True):
if item is traj.test2:
found=True
self.assertTrue(found)
for item in traj.test.f_iter_nodes(recursive=True, with_links=False):
if item is traj.test2:
self.assertTrue(False)
traj.v_with_links=True
self.assertTrue('circle1' in traj)
self.assertFalse(traj.f_contains('circle1', with_links=False))
self.assertTrue('circle1' in traj.test)
self.assertFalse(traj.test.f_contains('circle1', with_links=False))
self.assertTrue(traj.test2.test3 is traj.par.test.test3)
traj.v_with_links = False
with self.assertRaises(AttributeError):
traj.test2.test3
traj.v_with_links = True
self.assertTrue(traj['test2.test3'] is traj.test3)
with self.assertRaises(AttributeError):
traj.f_get('test2.test3', with_links=False)
def test_loading_explored_parameters(self):
filename = make_temp_dir('load_explored.hdf5')
traj = Trajectory(filename=filename, overwrite_file=True, add_time=False)
traj.par.x = Parameter('x', 42, comment='answer')
traj.f_explore({'x':[1,2,3,4]})
traj.f_store()
name = traj.v_name
traj = Trajectory(filename=filename, add_time=False)
traj.f_load()
x = traj.f_get('x')
self.assertIs(x, traj._explored_parameters['parameters.x'])
def test_loading_explored_parameters(self):
filename = make_temp_dir('load_explored.hdf5')
traj = Trajectory(filename=filename, overwrite_file=True, add_time=False)
traj.par.x = Parameter('x', 42, comment='answer')
traj.f_explore({'x':[1,2,3,4]})
traj.f_store()
name = traj.v_name
traj = Trajectory(filename=filename, add_time=False)
traj.f_load()
x = traj.f_get('x')
self.assertIs(x, traj._explored_parameters['parameters.x'])
def test_shortenings_of_names(self):
traj = Trajectory(filename=make_temp_dir('testshortening.hdf5'), add_time=True)
traj.f_aconf('g', 444)
self.assertTrue(isinstance(traj.f_get('g'), Parameter))
self.assertTrue(traj.conf.g == 444)
traj.f_apar('g', 444)
self.assertTrue(isinstance(traj.par.f_get('g'), Parameter))
self.assertTrue(traj.par.g == 444)
traj.f_adpar('g', 445)
self.assertTrue(isinstance(traj.derived_parameters.f_get('g'), Parameter))
self.assertTrue(traj.dpar.g == 445)
traj.f_ares('g', 454)
self.assertTrue(isinstance(traj.res.f_get('g'), Result))
self.assertTrue(traj.res.g == 454)
def test_shortenings_of_names(self):
traj = Trajectory(filename=make_temp_dir('testshortening.hdf5'))
traj.f_aconf('g', 444)
self.assertTrue(isinstance(traj.f_get('g'), Parameter))
self.assertTrue(traj.conf.g == 444)
traj.f_apar('g', 444)
self.assertTrue(isinstance(traj.par.f_get('g'), Parameter))
self.assertTrue(traj.par.g == 444)
traj.f_adpar('g', 445)
self.assertTrue(isinstance(traj.derived_parameters.f_get('g'), Parameter))
self.assertTrue(traj.dpar.g == 445)
traj.f_ares('g', 454)
self.assertTrue(isinstance(traj.res.f_get('g'), Result))
self.assertTrue(traj.res.g == 454)
from pypet import Trajectory, Result, Parameter
traj = Trajectory()
# There are more ways to add data,
# 1st the standard way:
traj.f_add_parameter('x', 1, comment='I am the first dimension!')
# 2nd by providing a new parameter/result instance, be aware that the data is added where
# you specify it. There are no such things as shortcuts for parameter creation:
traj.parameters.y = Parameter('y', 1, comment='I am the second dimension!')
# 3rd as before, but if our new leaf has NO name it will be renamed accordingly:
traj.parameters.t = Parameter('', 1, comment='Third dimension')
# See:
print('t=' + str(traj.t))
# This also works for adding groups on the fly and with the well known *dot* notation:
traj.parameters.subgroup = Parameter('subgroup.subsubgroup.w', 2)
# See
print('w='+str(traj.par.subgroup.subsubgroup.w))
# Finally, there's one more thing. Using this notation we can also add links.
# Simply use the `=` assignment with objects that already exist in your trajectory:
traj.mylink = traj.f_get('x')
# now `mylink` links to parameter `x`, also fast access works:
print('Linking to x gives: ' + str(traj.mylink))
def test_new_assignment_method(self):
filename = make_temp_dir('newassignment.hdf5')
traj = Trajectory(filename=filename, add_time=True)
comment = 'A number'
traj.par.x = Parameter('', 44, comment)
self.assertTrue(traj.f_get('x').v_comment == comment)
traj.x = 45
self.assertTrue(traj.par.f_get('x').f_get() == 45)
self.assertTrue(isinstance(traj.f_get('x'), Parameter))
with self.assertRaises(AttributeError):
traj.f = Parameter('lll', 444, 'lll')
traj.f = Parameter('', 444, 'lll')
self.assertTrue(traj.f_get('f').v_name == 'f')
conf = traj.conf
with self.assertRaises(AttributeError):
conf = traj.conf.jjjj
traj.f_set_properties(fast_access=True)
traj.crun = Result('', k=43, m='JJJ')
self.assertTrue(traj.run_A['k'] == 43)
with self.assertRaises(AttributeError):
traj.f_set_properties(j=7)
with self.assertRaises(AttributeError):
traj.f_set_properties(depth=7)
traj.hui = Result('hui', ('444', 'kkkk',), 'l')
self.assertTrue(traj.f_get('hui')[1] == 'l')
traj.f_get('hui').f_set(('445', 'kkkk',))
self.assertTrue(traj.f_get('hui')[1] == 'l')
self.assertTrue(traj.hui[0] == ('445', 'kkkk',))
traj.f_add_link('klkikju', traj.par) # for shizzle
traj.meee = Result('meee.h', 43, hui = 3213, comment='du')
self.assertTrue(traj.meee.h.h == 43)
with self.assertRaises(TypeError):
traj.par.jj = NNGroupNode('jj', comment='mi')
with self.assertRaises(TypeError):
traj.res.jj = NNGroupNode('jj', comment='mi')
with self.assertRaises(TypeError):
traj.conf.jj = NNGroupNode('jj', comment='mi')
with self.assertRaises(TypeError):
traj.dpar.jj = NNGroupNode('jj', comment='mi')
with self.assertRaises(TypeError):
traj.par.jj = ResultGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.dpar.jj = ResultGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.conf.jj = ResultGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.jj = ResultGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.par.jj = ConfigGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.dpar.jj = ConfigGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.res.jj = ConfigGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.jj = ConfigGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.par.jj = DerivedParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.conf.jj = DerivedParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.res.jj = DerivedParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.jj = DerivedParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.dpar.jj = ParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.res.jj = ParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.conf.jj = ParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.jj = ParameterGroup('jj', comment='mi')
traj.par.jj = ParameterGroup('jj', comment='mi')
traj.res.jj = ResultGroup('jj', comment='mi')
traj.jj = NNGroupNode('jj')
cg = ConfigGroup('a.g')
traj.conf.a = cg
self.assertTrue(traj.f_get('conf.a.g', shortcuts=False) is cg)
dg = DerivedParameterGroup('ttt')
traj.dpar.ttt = dg
self.assertTrue(traj.f_get('dpar.ttt', shortcuts=False) is dg)
traj.mylink = traj.par
self.assertTrue(traj.mylink is traj.par)
traj.vvv = NNGroupNode('', comment='kkk')
self.assertTrue(traj.vvv.v_full_name == 'vvv')
self.assertTrue(traj.par.jj.v_name == 'jj')
traj.ff = MyParamGroup('ff')
traj.par.g = MyParamGroup('')
pg = traj.f_add_parameter_group(comment='gg', full_name='me')
self.assertTrue(traj.par.me is pg)
traj.f_store()
traj = load_trajectory(index=-1, filename=filename, dynamic_imports=MyParamGroup)
self.assertTrue(isinstance(traj.ff, MyParamGroup))
self.assertTrue(isinstance(traj.par.g, MyParamGroup))
traj.par.hiho = Parameter('hiho', 42, comment='you')
traj.par.g1 = Parameter('g1.g2.g3.g4.g5', 43)
self.assertTrue(traj.hiho == 42)
self.assertTrue(isinstance(traj.par.g1, ParameterGroup ))
self.assertTrue(isinstance(traj.par.g3, ParameterGroup ))
self.assertTrue(traj.g3.g5 == 43)
def main():
filename = os.path.join('hdf5', 'example_05.hdf5')
env = Environment(trajectory='Example_05_Euler_Integration',
filename=filename,
file_title='Example_05_Euler_Integration',
comment='Go for Euler!')
traj = env.v_trajectory
trajectory_name = traj.v_name
# 1st a) phase parameter addition
add_parameters(traj)
# 1st b) phase preparation
# We will add the differential equation (well, its source code only) as a derived parameter
traj.f_add_derived_parameter(FunctionParameter,'diff_eq', diff_lorenz,
comment='Source code of our equation!')
# We want to explore some initial conditions
traj.f_explore({'initial_conditions' : [
np.array([0.01,0.01,0.01]),
np.array([2.02,0.02,0.02]),
np.array([42.0,4.2,0.42])
]})
# 3 different conditions are enough for an illustrative example
# 2nd phase let's run the experiment
# We pass `euler_scheme` as our top-level simulation function and
# the Lorenz equation 'diff_lorenz' as an additional argument
env.f_run(euler_scheme, diff_lorenz)
# We don't have a 3rd phase of post-processing here
# 4th phase analysis.
# I would recommend to do post-processing completely independent from the simulation,
# but for simplicity let's do it here.
# Let's assume that we start all over again and load the entire trajectory new.
# Yet, there is an error within this approach, do you spot it?
del traj
traj = Trajectory(filename=filename)
# We will only fully load parameters and derived parameters.
# Results will be loaded manually later on.
try:
# However, this will fail because our trajectory does not know how to
# build the FunctionParameter. You have seen this coming, right?
traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2,
load_results=1)
except ImportError as e:
print('That did\'nt work, I am sorry: %s ' % str(e))
# Ok, let's try again but this time with adding our parameter to the imports
traj = Trajectory(filename=filename,
dynamically_imported_classes=FunctionParameter)
# Now it works:
traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2,
load_results=1)
#For the fun of it, let's print the source code
print('\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq)
# Let's get the exploration array:
initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range()
# Now let's plot our simulated equations for the different initial conditions:
# We will iterate through the run names
for idx, run_name in enumerate(traj.f_get_run_names()):
#Get the result of run idx from the trajectory
euler_result = traj.results.f_get(run_name).euler_evolution
# Now we manually need to load the result. Actually the results are not so large and we
# could load them all at once. But for demonstration we do as if they were huge:
traj.f_load_item(euler_result)
euler_data = euler_result.data
#Plot fancy 3d plot
fig = plt.figure(idx)
ax = fig.gca(projection='3d')
x = euler_data[:,0]
y = euler_data[:,1]
z = euler_data[:,2]
ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx]))
plt.legend()
plt.show()
# Now we free the data again (because we assume its huuuuuuge):
del euler_data
euler_result.f_empty()
# You have to click through the images to stop the example_05 module!
# Finally disable logging and close all log-files
env.f_disable_logging()
def test_conversions(self):
filename = make_temp_dir('hdf5manipulation.hdf5')
traj = Trajectory(name=make_trajectory_name(self), filename=filename)
trajname = traj.v_name
traj.v_standard_result = SharedResult
traj.f_store(only_init=True)
traj.f_add_result('shared_data')
thedata = np.zeros((1000, 1000))
myarray = SharedArray('array', traj.shared_data, trajectory=traj)
traj.shared_data['array'] = myarray
mytable = SharedTable('t1', traj.shared_data, trajectory=traj)
traj.shared_data['t1'] = mytable
dadict = {
'hi': [1, 2, 3, 4, 5],
'shu': ['bi', 'du', 'da', 'ha', 'hui']
}
dadict2 = {'answer': [42]}
res = traj.f_add_result('shared.dfs')
res['df'] = SharedPandasFrame()
res['df'].create_shared_data(data=pd.DataFrame(dadict),
trajectory=traj)
frame = SharedPandasFrame('df1',
traj.f_get('shared.dfs'),
trajectory=traj,
add_to_parent=True)
frame.create_shared_data(data=pd.DataFrame(dadict2), )
res['df1'] = frame
traj.f_add_result('mylist', [1, 2, 3])
traj.f_add_result('my.mytuple', k=(1, 2, 3), wa=42)
traj.f_add_result('my.myarray', np.zeros((50, 50)))
traj.f_add_result('my.myframe', data=pd.DataFrame(dadict2))
traj.f_add_result('my.mytable', ObjectTable(data=dadict2))
myarray.create_shared_data(data=thedata)
mytable.create_shared_data(first_row={
'hi': compat.tobytes('hi'),
'huhu': np.ones(3)
})
traj.f_store()
data = myarray.read()
myarray.get_data_node()
self.assertTrue(np.all(data == thedata))
with StorageContextManager(traj):
myarray[2, 2] = 10
data = myarray.read()
self.assertTrue(data[2, 2] == 10)
self.assertTrue(data[2, 2] == 10)
self.assertFalse(traj.v_storage_service.is_open)
traj = load_trajectory(name=trajname,
filename=filename,
load_all=2,
dynamic_imports=SharedResult)
make_ordinary_result(traj.shared_data, 'array', trajectory=traj)
array = traj.shared_data.array
self.assertTrue(isinstance(array, np.ndarray))
thedata[2, 2] = 10
self.assertTrue(np.all(array == thedata))
make_ordinary_result(
traj.shared_data,
't1',
trajectory=traj,
)
t1 = traj.shared_data.t1
self.assertTrue(isinstance(t1, ObjectTable))
self.assertTrue(np.all(t1['huhu'][0] == np.ones(3)))
dfs = traj.shared.dfs
make_ordinary_result(traj.shared.dfs, 'df', trajectory=traj)
theframe = dfs.f_get('df')
self.assertTrue(isinstance(dfs, Result))
self.assertTrue(isinstance(theframe, pd.DataFrame))
self.assertTrue(theframe['hi'][0] == 1)
listres = traj.f_get('mylist')
listres = make_shared_result(listres, 0, trajectory=traj)
with StorageContextManager(traj):
self.assertTrue(listres[0][2] == 3)
listres[0][0] = 4
self.assertTrue(listres[0][0] == 4)
listres = make_ordinary_result(listres, 0, trajectory=traj)
traj = load_trajectory(name=trajname,
filename=filename,
load_all=2,
dynamic_imports=SharedResult)
mylist = traj.mylist
self.assertTrue(isinstance(listres, Result))
self.assertTrue(mylist[0] == 4)
self.assertTrue(isinstance(mylist, list))
mytuple = traj.mytuple
with self.assertRaises(AttributeError):
mytuple = make_shared_result(mytuple,
'mylist',
traj,
new_class=SharedArray)
mytuple = make_shared_result(mytuple, 'k', traj, new_class=SharedArray)
self.assertTrue(mytuple.k[1] == 2)
mytuple = make_ordinary_result(mytuple, 'k', trajectory=traj)
self.assertTrue(isinstance(mytuple.k, tuple))
self.assertTrue(mytuple.k[2] == 3)
myframe = traj.myframe
myframe = make_shared_result(myframe, 'data', traj)
theframe = myframe.data.read()
self.assertTrue(theframe['answer'][0] == 42)
myframe = make_ordinary_result(myframe, 'data', trajectory=traj)
traj.f_load_item(myframe)
self.assertTrue(myframe.data['answer'][0] == 42)
mytable = traj.f_get('mytable')
mytable = make_shared_result(mytable, 0, traj)
self.assertTrue(isinstance(mytable[0], SharedTable))
rows = mytable.mytable.read()
self.assertTrue(rows[0][0] == 42)
mytable = make_ordinary_result(mytable, 0, trajectory=traj)
self.assertTrue(isinstance(mytable, Result))
self.assertTrue(mytable[0]['answer'][0] == 42)
traj = Trajectory('Example', filename=filename,
comment='Access and Storage!')
# We add our first parameter with the data 'Harrison Ford'
traj.f_add_parameter('starwars.characters.han_solo', 'Harrison Ford')
# This automatically added the groups 'starwars' and the subgroup 'characters'
# Let's get the characters subgroup
characters = traj.parameters.starwars.characters
# Since characters is unique we could also use shortcuts
characters = traj.characters
# Or the get method
characters = traj.f_get('characters')
# Or square brackets
characters = traj['characters']
# Lets add another character
characters.f_add_parameter('luke_skywalker', 'Mark Hamill', comment='May the force be with you!')
#The full name of luke skywalker is now `parameters.starwars.characters.luke_skywalker`:
print('The full name of the new Skywalker Parameter is %s' %
traj.f_get('luke_skywalker').v_full_name)
#Lets see what happens if we have not unique entries:
traj.f_add_parameter_group('spaceballs.characters')
# Now our shortcuts no longer work, since we have two character groups!
def main():
filename = os.path.join('hdf5', 'example_05.hdf5')
env = Environment(trajectory='Example_05_Euler_Integration',
filename=filename,
file_title='Example_05_Euler_Integration',
overwrite_file=True,
comment='Go for Euler!')
traj = env.trajectory
trajectory_name = traj.v_name
# 1st a) phase parameter addition
add_parameters(traj)
# 1st b) phase preparation
# We will add the differential equation (well, its source code only) as a derived parameter
traj.f_add_derived_parameter(FunctionParameter,'diff_eq', diff_lorenz,
comment='Source code of our equation!')
# We want to explore some initial conditions
traj.f_explore({'initial_conditions' : [
np.array([0.01,0.01,0.01]),
np.array([2.02,0.02,0.02]),
np.array([42.0,4.2,0.42])
]})
# 3 different conditions are enough for an illustrative example
# 2nd phase let's run the experiment
# We pass `euler_scheme` as our top-level simulation function and
# the Lorenz equation 'diff_lorenz' as an additional argument
env.run(euler_scheme, diff_lorenz)
# We don't have a 3rd phase of post-processing here
# 4th phase analysis.
# I would recommend to do post-processing completely independent from the simulation,
# but for simplicity let's do it here.
# Let's assume that we start all over again and load the entire trajectory new.
# Yet, there is an error within this approach, do you spot it?
del traj
traj = Trajectory(filename=filename)
# We will only fully load parameters and derived parameters.
# Results will be loaded manually later on.
try:
# However, this will fail because our trajectory does not know how to
# build the FunctionParameter. You have seen this coming, right?
traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2,
load_results=1)
except ImportError as e:
print('That did\'nt work, I am sorry: %s ' % str(e))
# Ok, let's try again but this time with adding our parameter to the imports
traj = Trajectory(filename=filename,
dynamically_imported_classes=FunctionParameter)
# Now it works:
traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2,
load_results=1)
#For the fun of it, let's print the source code
print('\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq)
# Let's get the exploration array:
initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range()
# Now let's plot our simulated equations for the different initial conditions:
# We will iterate through the run names
for idx, run_name in enumerate(traj.f_get_run_names()):
#Get the result of run idx from the trajectory
euler_result = traj.results.f_get(run_name).euler_evolution
# Now we manually need to load the result. Actually the results are not so large and we
# could load them all at once. But for demonstration we do as if they were huge:
traj.f_load_item(euler_result)
euler_data = euler_result.data
#Plot fancy 3d plot
fig = plt.figure(idx)
ax = fig.gca(projection='3d')
x = euler_data[:,0]
y = euler_data[:,1]
z = euler_data[:,2]
ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx]))
plt.legend()
plt.show()
# Now we free the data again (because we assume its huuuuuuge):
del euler_data
euler_result.f_empty()
# You have to click through the images to stop the example_05 module!
# Finally disable logging and close all log-files
env.disable_logging()
filename=filename,
overwrite_file=True,
comment='Access and Storage!')
# We add our first parameter with the data 'Harrison Ford'
traj.f_add_parameter('starwars.characters.han_solo', 'Harrison Ford')
# This automatically added the groups 'starwars' and the subgroup 'characters'
# Let's get the characters subgroup
characters = traj.parameters.starwars.characters
# Since characters is unique we could also use shortcuts
characters = traj.characters
# Or the get method
characters = traj.f_get('characters')
# Or square brackets
characters = traj['characters']
# Lets add another character
characters.f_add_parameter('luke_skywalker',
'Mark Hamill',
comment='May the force be with you!')
#The full name of luke skywalker is now `parameters.starwars.characters.luke_skywalker`:
print('The full name of the new Skywalker Parameter is %s' %
traj.f_get('luke_skywalker').v_full_name)
#Lets see what happens if we have not unique entries:
traj.f_add_parameter_group('spaceballs.characters')
def test_new_assignment_method(self):
filename = make_temp_dir('newassignment.hdf5')
traj = Trajectory(filename=filename)
traj.v_lazy_adding = True
comment = 'A number'
traj.par.x = 44, comment
self.assertTrue(traj.f_get('x').v_comment == comment)
traj.par.iamgroup = a_new_group
self.assertTrue(isinstance(traj.iamgroup, ParameterGroup))
traj.v_lazy_adding = False
traj.x = 45
self.assertTrue(traj.par.f_get('x').f_get() == 45)
self.assertTrue(isinstance(traj.f_get('x'), Parameter))
traj.f = Parameter('lll', 444, 'lll')
self.assertTrue(traj.f_get('f').v_name == 'f')
traj.v_lazy_adding = True
traj.res.k = 22, 'Hi'
self.assertTrue(isinstance(traj.f_get('k'), Result))
self.assertTrue(traj.f_get('k')[1] == 'Hi')
with self.assertRaises(AttributeError):
traj.res.k = 33, 'adsd'
conf = traj.conf
with self.assertRaises(AttributeError):
conf = traj.conf.jjjj
traj.f_set_properties(fast_access=True)
traj.crun = 43, 'JJJ'
self.assertTrue(traj.run_A[0] == 43)
with self.assertRaises(AttributeError):
traj.f_set_properties(j=7)
with self.assertRaises(AttributeError):
traj.f_set_properties(depth=7)
traj.hui = (('444', 'kkkk',), 'l')
self.assertTrue(traj.f_get('hui')[1] == 'l')
with self.assertRaises(AttributeError):
traj.hui = ('445', 'kkkk',)
traj.f_get('hui').f_set(('445', 'kkkk',))
self.assertTrue(traj.f_get('hui')[1] == 'l')
self.assertTrue(traj.hui[0] == ('445', 'kkkk',))
traj.f_add_link('klkikju', traj.par) # for shizzle
traj.meee = Result('h', 43, hui = 3213, comment='du')
self.assertTrue(traj.meee.h.h == 43)
with self.assertRaises(TypeError):
traj.par.mu = NNGroupNode('jj', comment='mi')
with self.assertRaises(TypeError):
traj.res.mu = NNGroupNode('jj', comment='mi')
with self.assertRaises(TypeError):
traj.conf.mu = NNGroupNode('jj', comment='mi')
with self.assertRaises(TypeError):
traj.dpar.mu = NNGroupNode('jj', comment='mi')
with self.assertRaises(TypeError):
traj.par.mu = ResultGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.dpar.mu = ResultGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.conf.mu = ResultGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.mu = ResultGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.par.mu = ConfigGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.dpar.mu = ConfigGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.res.mu = ConfigGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.mu = ConfigGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.par.mu = DerivedParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.conf.mu = DerivedParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.res.mu = DerivedParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.mu = DerivedParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.dpar.mu = ParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.res.mu = ParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.conf.mu = ParameterGroup('jj', comment='mi')
with self.assertRaises(TypeError):
traj.mu = ParameterGroup('jj', comment='mi')
traj.par.mu = ParameterGroup('jj', comment='mi')
traj.res.mus = ResultGroup('jj', comment='mi')
traj.mu = NNGroupNode('jj')
cg = ConfigGroup('a.g')
traj.conf.a = cg
self.assertTrue(traj.f_get('conf.a.a.g', shortcuts=False) is cg)
dg = DerivedParameterGroup('ttt')
traj.dpar.ttt = dg
self.assertTrue(traj.f_get('dpar.ttt', shortcuts=False) is dg)
traj.mylink = traj.par
self.assertTrue(traj.mylink is traj.par)
traj.vvv = NNGroupNode('', comment='kkk')
self.assertTrue(traj.vvv.v_full_name == 'vvv')
self.assertTrue(traj.par.mu.v_name == 'mu')
traj.rrr = MyParamGroup('ff')
traj.par.g = MyParamGroup('')
pg = traj.f_add_parameter_group(comment='gg', full_name='me')
self.assertTrue(traj.par.me is pg)
traj.f_store()
traj = load_trajectory(index=-1, filename=filename, dynamic_imports=MyParamGroup)
self.assertTrue(isinstance(traj.rrr, NNGroupNode))
self.assertTrue(isinstance(traj.rrr.ff, MyParamGroup))
self.assertTrue(isinstance(traj.par.g, MyParamGroup))
traj.par = Parameter('hiho', 42, comment='you')
traj.par = Parameter('g1.g2.g3.g4.g5', 43)
self.assertTrue(traj.hiho == 42)
self.assertTrue(isinstance(traj.par.g1, ParameterGroup ))
self.assertTrue(isinstance(traj.par.g3, ParameterGroup ))
self.assertTrue(traj.g3.g5 == 43)
def test_conversions(self):
filename = make_temp_dir("hdf5manipulation.hdf5")
traj = Trajectory(name=make_trajectory_name(self), filename=filename)
trajname = traj.v_name
traj.v_standard_result = SharedResult
traj.f_store(only_init=True)
traj.f_add_result("shared_data")
thedata = np.zeros((1000, 1000))
myarray = SharedArray("array", traj.shared_data, trajectory=traj)
traj.shared_data["array"] = myarray
mytable = SharedTable("t1", traj.shared_data, trajectory=traj)
traj.shared_data["t1"] = mytable
# mytable2 = SharedTableResult('h.t2', trajectory=traj)
# mytable3 = SharedTableResult('jjj.t3', trajectory=traj)
dadict = {"hi": [1, 2, 3, 4, 5], "shu": ["bi", "du", "da", "ha", "hui"]}
dadict2 = {"answer": [42]}
res = traj.f_add_result("shared.dfs")
res["df"] = SharedPandasFrame()
res["df"].create_shared_data(data=pd.DataFrame(dadict), trajectory=traj)
frame = SharedPandasFrame("df1", traj.f_get("shared.dfs"), trajectory=traj)
frame.create_shared_data(data=pd.DataFrame(dadict2))
res["df1"] = frame
traj.f_add_result("mylist", [1, 2, 3])
traj.f_add_result("my.mytuple", k=(1, 2, 3), wa=42)
traj.f_add_result("my.myarray", np.zeros((50, 50)))
traj.f_add_result("my.myframe", data=pd.DataFrame(dadict2))
traj.f_add_result("my.mytable", ObjectTable(data=dadict2))
myarray.create_shared_data(data=thedata)
mytable.create_shared_data(first_row={"hi": compat.tobytes("hi"), "huhu": np.ones(3)})
traj.f_store()
data = myarray.read()
arr = myarray.get_data_node()
self.assertTrue(np.all(data == thedata))
with StorageContextManager(traj) as cm:
myarray[2, 2] = 10
data = myarray.read()
self.assertTrue(data[2, 2] == 10)
self.assertTrue(data[2, 2] == 10)
self.assertFalse(traj.v_storage_service.is_open)
traj = load_trajectory(name=trajname, filename=filename, load_all=2, dynamic_imports=SharedResult)
make_ordinary_result(traj.shared_data, "array", trajectory=traj)
array = traj.shared_data.array
self.assertTrue(isinstance(array, np.ndarray))
thedata[2, 2] = 10
self.assertTrue(np.all(array == thedata))
make_ordinary_result(traj.shared_data, "t1", trajectory=traj)
t1 = traj.shared_data.t1
self.assertTrue(isinstance(t1, ObjectTable)) #
self.assertTrue(np.all(t1["huhu"][0] == np.ones(3)))
dfs = traj.shared.dfs
make_ordinary_result(traj.shared.dfs, "df", trajectory=traj)
theframe = dfs.f_get("df")
self.assertTrue(isinstance(dfs, Result))
self.assertTrue(isinstance(theframe, pd.DataFrame))
self.assertTrue(theframe["hi"][0] == 1)
listres = traj.f_get("mylist")
listres = make_shared_result(listres, 0, trajectory=traj)
with StorageContextManager(traj) as cm:
self.assertTrue(listres[0][2] == 3)
listres[0][0] = 4
self.assertTrue(listres[0][0] == 4)
listres = make_ordinary_result(listres, 0, trajectory=traj)
traj = load_trajectory(name=trajname, filename=filename, load_all=2, dynamic_imports=SharedResult)
mylist = traj.mylist
self.assertTrue(isinstance(listres, Result))
self.assertTrue(mylist[0] == 4)
self.assertTrue(isinstance(mylist, list))
mytuple = traj.mytuple
with self.assertRaises(AttributeError):
mytuple = make_shared_result(mytuple, "mylist", traj, new_class=SharedArray)
mytuple = make_shared_result(mytuple, "k", traj, new_class=SharedArray)
self.assertTrue(mytuple.k[1] == 2)
mytuple = make_ordinary_result(mytuple, "k", trajectory=traj)
self.assertTrue(isinstance(mytuple.k, tuple))
self.assertTrue(mytuple.k[2] == 3)
myframe = traj.myframe
myframe = make_shared_result(myframe, "data", traj)
theframe = myframe.data.read()
self.assertTrue(theframe["answer"][0] == 42)
myframe = make_ordinary_result(myframe, "data", trajectory=traj)
traj.f_load_item(myframe)
self.assertTrue(myframe.data["answer"][0] == 42)
mytable = traj.f_get("mytable")
mytable = make_shared_result(mytable, 0, traj)
self.assertTrue(isinstance(mytable[0], SharedTable))
rows = mytable.mytable.read()
self.assertTrue(rows[0][0] == 42)
mytable = make_ordinary_result(mytable, 0, trajectory=traj)
self.assertTrue(isinstance(mytable, Result))
self.assertTrue(mytable[0]["answer"][0] == 42)
# In[ ]:
traj.f_get_run_names()
# In[ ]:
def my_filter_function(location,dt):
result = location =='mars' and dt=1e-2
return result
# In[ ]:
set(traj.f_get('incline').f_get_range())
# In[ ]:
def filter_function(loc,dt,vmax,vent_radius,incline):
result = loc=='mars' and dt==1e-2 and vmax==50 and vent_radius==0.5 and incline==1.0
return result
# In[ ]:
def standard_filter(loc, dt, jitter, incline):
result = loc=='mars' and dt==1e-2 and jitter==0.1 and incline!=10.0
return result
__author__ = 'Robert Meyer'
from pypet import Trajectory, Result, Parameter
traj = Trajectory()
# There are more ways to add data,
# 1st the standard way:
traj.f_add_parameter('x', 1, comment='I am the first dimension!')
# 2nd by providing a new parameter/result instance, be aware that the data is added where
# you specify it. There are no such things as shortcuts for parameter creation:
traj.parameters.y = Parameter('y', 1, comment='I am the second dimension!')
# 3rd as before, but if our new leaf has NO name it will be renamed accordingly:
traj.parameters.t = Parameter('', 1, comment='Third dimension')
# See:
print('t=' + str(traj.t))
# This also works for adding groups on the fly and with the well known *dot* notation:
traj.parameters.subgroup = Parameter('subgroup.subsubgroup.w', 2)
# See
print('w=' + str(traj.par.subgroup.subsubgroup.w))
# Finally, there's one more thing. Using this notation we can also add links.
# Simply use the `=` assignment with objects that already exist in your trajectory:
traj.mylink = traj.f_get('x')
# now `mylink` links to parameter `x`, also fast access works:
print('Linking to x gives: ' + str(traj.mylink))
# In[ ]:
traj.f_get_run_names()
# In[ ]:
def my_filter_function(location, dt):
result = location == 'mars' and dt = 1e-2
return result
# In[ ]:
set(traj.f_get('incline').f_get_range())
# In[ ]:
def filter_function(loc, dt, vmax, vent_radius, incline):
result = loc == 'mars' and dt == 1e-2 and vmax == 50 and vent_radius == 0.5 and incline == 1.0
return result
# In[ ]:
def standard_filter(loc, dt, jitter, incline):
result = loc == 'mars' and dt == 1e-2 and jitter == 0.1 and incline != 10.0
return result
# And you can add a new parameter via
traj.parameters.u = 1, 'Fourth dimension'
print('u=' + str(traj.u))
# However, now you can no longer change values of existing parameters,
# because this is interpreted as a new parameter addition, so this fails:
try:
traj.parameters.u = 2
print('I won`t be reached')
except AttributeError as exc:
print('Told you: `%s`' % repr(exc))
# See:
print('u=' + str(traj.par.u))
# But disabling the new adding method makes this work again
traj.v_lazy_adding = False
traj.f_get('u').f_unlock()
traj.parameters.u = 3
# now we simply change `u` to be 3
# There's also a lazy version to add new group nodes:
from pypet import new_group
traj.v_lazy_adding=True
traj.im_new = new_group
# And `im_new` is a new group node:
print(traj.im_new)
# Finally, there's one more thing. Using this notation we can also add links.
# Simply use the `=` assignment with objects that already exist in your trajectory:
traj.mylink = traj.f_get('x')
# now `mylink` links to parameter `x`, also fast access works:
def test_conversions(self):
filename = make_temp_dir('hdf5manipulation.hdf5')
traj = Trajectory(name=make_trajectory_name(self), filename=filename)
trajname = traj.v_name
traj.v_standard_result = SharedResult
traj.f_store(only_init=True)
traj.f_add_result('shared_data')
thedata = np.zeros((1000, 1000))
myarray = SharedArray('array', traj.shared_data, trajectory=traj)
traj.shared_data['array'] = myarray
mytable = SharedTable('t1', traj.shared_data, trajectory=traj)
traj.shared_data['t1'] = mytable
dadict = {'hi': [1, 2, 3, 4, 5], 'shu': ['bi', 'du', 'da', 'ha', 'hui']}
dadict2 = {'answer': [42]}
res = traj.f_add_result('shared.dfs')
res['df'] = SharedPandasFrame()
res['df'].create_shared_data(data=pd.DataFrame(dadict), trajectory=traj)
frame = SharedPandasFrame('df1', traj.f_get('shared.dfs'), trajectory=traj,
add_to_parent=True)
frame.create_shared_data(data=pd.DataFrame(dadict2),)
res['df1'] = frame
traj.f_add_result('mylist', [1, 2, 3])
traj.f_add_result('my.mytuple', k=(1, 2, 3), wa=42)
traj.f_add_result('my.myarray', np.zeros((50, 50)))
traj.f_add_result('my.myframe', data=pd.DataFrame(dadict2))
traj.f_add_result('my.mytable', ObjectTable(data=dadict2))
myarray.create_shared_data(data=thedata)
mytable.create_shared_data(first_row={'hi': compat.tobytes('hi'), 'huhu': np.ones(3)})
traj.f_store()
data = myarray.read()
myarray.get_data_node()
self.assertTrue(np.all(data == thedata))
with StorageContextManager(traj):
myarray[2, 2] = 10
data = myarray.read()
self.assertTrue(data[2, 2] == 10)
self.assertTrue(data[2, 2] == 10)
self.assertFalse(traj.v_storage_service.is_open)
traj = load_trajectory(name=trajname, filename=filename, load_all=2,
dynamic_imports=SharedResult)
make_ordinary_result(traj.shared_data, 'array', trajectory=traj)
array = traj.shared_data.array
self.assertTrue(isinstance(array, np.ndarray))
thedata[2, 2] = 10
self.assertTrue(np.all(array == thedata))
make_ordinary_result(traj.shared_data, 't1', trajectory=traj,)
t1 = traj.shared_data.t1
self.assertTrue(isinstance(t1, ObjectTable))
self.assertTrue(np.all(t1['huhu'][0] == np.ones(3)))
dfs = traj.shared.dfs
make_ordinary_result(traj.shared.dfs, 'df', trajectory=traj)
theframe = dfs.f_get('df')
self.assertTrue(isinstance(dfs, Result))
self.assertTrue(isinstance(theframe, pd.DataFrame))
self.assertTrue(theframe['hi'][0] == 1)
listres = traj.f_get('mylist')
listres = make_shared_result(listres, 0, trajectory=traj)
with StorageContextManager(traj):
self.assertTrue(listres[0][2] == 3)
listres[0][0] = 4
self.assertTrue(listres[0][0] == 4)
listres = make_ordinary_result(listres, 0, trajectory=traj)
traj = load_trajectory(name=trajname, filename=filename, load_all=2,
dynamic_imports=SharedResult)
mylist = traj.mylist
self.assertTrue(isinstance(listres, Result))
self.assertTrue(mylist[0] == 4)
self.assertTrue(isinstance(mylist, list))
mytuple = traj.mytuple
with self.assertRaises(AttributeError):
mytuple = make_shared_result(mytuple, 'mylist', traj, new_class=SharedArray)
mytuple = make_shared_result(mytuple, 'k', traj, new_class=SharedArray)
self.assertTrue(mytuple.k[1] == 2)
mytuple = make_ordinary_result(mytuple, 'k', trajectory=traj)
self.assertTrue(isinstance(mytuple.k, tuple))
self.assertTrue(mytuple.k[2] == 3)
myframe = traj.myframe
myframe = make_shared_result(myframe, 'data', traj)
theframe = myframe.data.read()
self.assertTrue(theframe['answer'][0] == 42)
myframe = make_ordinary_result(myframe, 'data', trajectory=traj)
traj.f_load_item(myframe)
self.assertTrue(myframe.data['answer'][0] == 42)
mytable = traj.f_get('mytable')
mytable = make_shared_result(mytable, 0, traj)
self.assertTrue(isinstance(mytable[0], SharedTable))
rows = mytable.mytable.read()
self.assertTrue(rows[0][0] == 42)
mytable = make_ordinary_result(mytable, 0, trajectory=traj)
self.assertTrue(isinstance(mytable, Result))
self.assertTrue(mytable[0]['answer'][0] == 42)
