def test_store_items_and_groups(self):
traj = Trajectory(name='testtraj', filename=make_temp_dir('teststoreitems.hdf5'),
add_time=True)
traj.f_store()
traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj.f_store_items(['test','testres','group1'])
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_dir('teststoreitems.hdf5'))
traj2.f_load(load_parameters=2, load_results=2)
traj.f_add_result('Im.stored.along.a.path', 43)
traj.Im.stored.along.v_annotations['wtf'] =4444
traj.res.f_store_child('Im.stored.along.a.path')
traj2.res.f_load_child('Im.stored.along.a.path', load_data=2)
self.compare_trajectories(traj,traj2)
def make_plots(resultsPath):
traj = Trajectory('tone-in-noise', add_time=False)
traj.f_load(load_parameters=2,
load_derived_parameters=0,
load_results=1,
load_other_data=0,
filename=resultsPath)
traj.v_auto_load = True
with PdfPages(
path.join(path.expanduser('~'), "pypet-output",
'synaptopathy.pdf')) as pdf:
synaptopathy_effect(traj, pdf)
with PdfPages(
path.join(path.expanduser('~'), "pypet-output",
'periphery.pdf')) as pdf:
periphery_effect(traj, pdf)
with PdfPages(
path.join(path.expanduser('~'), "pypet-output",
'weighting.pdf')) as pdf:
weighting_effect(traj, pdf)
with PdfPages(
path.join(path.expanduser('~'), "pypet-output",
'brainstem.pdf')) as pdf:
brainstem_effect(traj, pdf)
return 0
def load_trajectory(self,trajectory_index=None,trajectory_name=None,as_new=False):
### Load The Trajectory and check if the values are still the same
newtraj = Trajectory(filename=self.filename)
newtraj.f_load(name=trajectory_name, index=trajectory_index, as_new=as_new,
load_parameters=2, load_derived_parameters=2, load_results=2,
load_other_data=2)
return newtraj
def test_overwrite_stuff(self):
traj = Trajectory(name='TestOverwrite', filename=make_temp_dir('testowrite.hdf5'))
res = traj.f_add_result('mytest.test', a='b', c='d')
traj.f_store()
res['a'] = np.array([1,2,3])
res['c'] = 123445
traj.f_store_item(res, overwrite='a', complevel=4)
# Should emit a warning
traj.f_store_item(res, overwrite=['a', 'b'])
traj.f_load(load_results=3)
res = traj.test
self.assertTrue((res['a']==np.array([1,2,3])).all())
self.assertTrue(res['c']=='d')
res['c'] = 123445
traj.f_store_item(res, store_data=3)
res.f_empty()
traj.f_load(load_results=3)
self.assertTrue(traj.test['c']==123445)
def test_version_mismatch(self):
traj = Trajectory(name='TestVERSION', filename=make_temp_dir('testversionmismatch.hdf5'),
add_time=True)
traj.f_add_parameter('group1.test',42)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj._version='0.1a.1'
traj.f_store()
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_dir('testversionmismatch.hdf5'))
with self.assertRaises(pex.VersionMismatchError):
traj2.f_load(load_parameters=2, load_results=2)
traj2.f_load(load_parameters=2, load_results=2, force=True)
self.compare_trajectories(traj,traj2)
get_root_logger().info('Mismatch testing done!')
def print_traj_parameters_explored(traj_dir):
# Load the trajectory from the hdf5 file
# Only load parameters, results will be loaded at runtime (auto loading)
#traj_dir = os.path.join('trajectories', '2019_03_21_22h48m29s_HCP_test')
#if not os.path.isdir(traj_dir):
# traj_dir = os.path.join('..', traj_dir)
traj_filename = 'traj.hdf5'
traj_fullpath = os.path.join(traj_dir, traj_filename)
traj = Trajectory()
traj.f_load(filename=traj_fullpath,
index=0,
load_parameters=2,
load_results=0,
load_derived_parameters=0,
force=True)
# Turn on auto loading
traj.v_auto_load = True
# Count number of runs
runs_n = len(traj.f_get_run_names())
print('number of runs = {0}'.format(runs_n))
# Get list of explored parameters
parameters_explored = [
str.split(par, '.').pop() for par in (traj.f_get_explored_parameters())
]
print(parameters_explored)
def load_trajectory(self,trajectory_index=None,trajectory_name=None,as_new=False):
### Load The Trajectory and check if the values are still the same
newtraj = Trajectory(filename=self.filename)
newtraj.f_load(name=trajectory_name, index=trajectory_index, as_new=as_new,
load_parameters=2, load_derived_parameters=2, load_results=2,
load_other_data=2)
return newtraj
def test_store_and_load_large_dictionary(self):
traj = Trajectory(name='Testlargedict', filename=make_temp_dir('large_dict.hdf5'))
large_dict = {}
for irun in range(1025):
large_dict['item_%d' % irun] = irun
large_dict2 = {}
for irun in range(33):
large_dict2['item_%d' % irun] = irun
traj.f_add_result('large_dict', large_dict, comment='Huge_dict!')
traj.f_add_result('large_dict2', large_dict2, comment='Not so large dict!')
traj.f_store()
traj_name = traj.v_name
traj2 = Trajectory(filename=make_temp_dir('large_dict.hdf5'))
traj2.f_load(name=traj_name, load_data=2)
self.compare_trajectories(traj, traj2)
def test_delete_links(self):
traj = Trajectory(name='TestDelete',
filename=make_temp_dir('testpartiallydel.hdf5'),
add_time=True)
res = traj.f_add_result('mytest.test', a='b', c='d')
traj.f_add_link('x.y', res)
traj.f_add_link('x.g.h', res)
traj.f_store()
traj.f_remove_child('x', recursive=True)
traj.f_load()
self.assertEqual(traj.x.y.a, traj.test.a)
self.assertEqual(traj.x.g.h.a, traj.test.a)
traj.f_delete_link('x.y', remove_from_trajectory=True)
traj.f_delete_link((traj.x.g, 'h'), remove_from_trajectory=True)
traj.f_load()
with self.assertRaises(AttributeError):
traj.x.g.h
def test_partially_delete_stuff(self):
traj = Trajectory(name='TestDelete',
filename=make_temp_dir('testpartiallydel.hdf5'),
add_time=True)
res = traj.f_add_result('mytest.test', a='b', c='d')
traj.f_store()
self.assertTrue('a' in res)
traj.f_delete_item(res, delete_only=['a'], remove_from_item=True)
self.assertTrue('c' in res)
self.assertTrue('a' not in res)
res['a'] = 'offf'
self.assertTrue('a' in res)
traj.f_load(load_results=3)
self.assertTrue('a' not in res)
self.assertTrue('c' in res)
traj.f_delete_item(res, remove_from_trajectory=True)
self.assertTrue('results' in traj)
self.assertTrue(res not in traj)
def test_overwrite_stuff(self):
traj = Trajectory(name='TestOverwrite', filename=make_temp_dir('testowrite.hdf5'),
add_time=True)
res = traj.f_add_result('mytest.test', a='b', c='d')
traj.f_store()
res['a'] = np.array([1,2,3])
res['c'] = 123445
traj.f_store_item(res, overwrite='a', complevel=4)
# Should emit a warning
traj.f_store_item(res, overwrite=['a', 'b'])
traj.f_load(load_results=3)
res = traj.test
self.assertTrue((res['a']==np.array([1,2,3])).all())
self.assertTrue(res['c']=='d')
res['c'] = 123445
traj.f_store_item(res, store_data=3)
res.f_empty()
traj.f_load(load_results=3)
self.assertTrue(traj.test['c']==123445)
def main():
filename = os.path.join('hdf5', 'Clustered_Network.hdf5')
# If we pass a filename to the trajectory a new HDF5StorageService will
# be automatically created
traj = Trajectory(filename=filename,
dynamically_imported_classes=[BrianMonitorResult,
BrianParameter])
# Let's create and fake environment to enable logging:
env = Environment(traj, do_single_runs=False)
# Load the trajectory, but onyl laod the skeleton of the results
traj.f_load(index=-1, load_parameters=2, load_derived_parameters=2, load_results=1)
# Find the result instances related to the fano factor
fano_dict = traj.f_get_from_runs('mean_fano_factor', fast_access=False)
# Load the data of the fano factor results
ffs = fano_dict.values()
traj.f_load_items(ffs)
# Extract all values and R_ee values for each run
ffs_values = [x.f_get() for x in ffs]
Rees = traj.f_get('R_ee').f_get_range()
# Plot average fano factor as a function of R_ee
plt.plot(Rees, ffs_values)
plt.xlabel('R_ee')
plt.ylabel('Avg. Fano Factor')
plt.show()
# Finally disable logging and close all log-files
env.disable_logging()
def test_delete_links(self):
traj = Trajectory(name='TestDelete',
filename=make_temp_dir('testpartiallydel.hdf5'),
add_time=True)
res = traj.f_add_result('mytest.test', a='b', c='d')
traj.f_add_link('x.y', res)
traj.f_add_link('x.g.h', res)
traj.f_store()
traj.f_remove_child('x', recursive=True)
traj.f_load()
self.assertEqual(traj.x.y.a, traj.test.a)
self.assertEqual(traj.x.g.h.a, traj.test.a)
traj.f_delete_link('x.y', remove_from_trajectory=True)
traj.f_delete_link((traj.x.g, 'h'), remove_from_trajectory=True)
traj.f_load()
with self.assertRaises(AttributeError):
traj.x.g.h
def test_version_mismatch(self):
traj = Trajectory(name='TestVERSION', filename=make_temp_dir('testversionmismatch.hdf5'),
add_time=True)
traj.f_add_parameter('group1.test',42)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj._version='0.1a.1'
traj.f_store()
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_dir('testversionmismatch.hdf5'))
with self.assertRaises(pex.VersionMismatchError):
traj2.f_load(load_parameters=2, load_results=2)
traj2.f_load(load_parameters=2, load_results=2, force=True)
self.compare_trajectories(traj,traj2)
get_root_logger().info('Mismatch testing done!')
def test_store_and_load_large_dictionary(self):
traj = Trajectory(name='Testlargedict', filename=make_temp_dir('large_dict.hdf5'),
add_time=True)
large_dict = {}
for irun in range(1025):
large_dict['item_%d' % irun] = irun
large_dict2 = {}
for irun in range(33):
large_dict2['item_%d' % irun] = irun
traj.f_add_result('large_dict', large_dict, comment='Huge_dict!')
traj.f_add_result('large_dict2', large_dict2, comment='Not so large dict!')
traj.f_store()
traj_name = traj.v_name
traj2 = Trajectory(filename=make_temp_dir('large_dict.hdf5'),
add_time=True)
traj2.f_load(name=traj_name, load_data=2)
self.compare_trajectories(traj, traj2)
def test_store_items_and_groups(self):
traj = Trajectory(name='testtraj', filename=make_temp_dir('teststoreitems.hdf5'))
traj.f_store()
traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj.f_store_items(['test','testres','group1'])
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_dir('teststoreitems.hdf5'))
traj2.f_load(load_parameters=2, load_results=2)
traj.f_add_result('Im.stored.along.a.path', 43)
traj.Im.stored.along.v_annotations['wtf'] =4444
traj.res.f_store_child('Im.stored.along.a.path')
traj2.res.f_load_child('Im.stored.along.a.path', load_data=2)
self.compare_trajectories(traj,traj2)
def test_partially_delete_stuff(self):
traj = Trajectory(name='TestDelete',
filename=make_temp_dir('testpartiallydel.hdf5'))
res = traj.f_add_result('mytest.test', a='b', c='d')
traj.f_store()
self.assertTrue('a' in res)
traj.f_delete_item(res, delete_only=['a'], remove_from_item=True)
self.assertTrue('c' in res)
self.assertTrue('a' not in res)
res['a'] = 'offf'
self.assertTrue('a' in res)
traj.f_load(load_results=3)
self.assertTrue('a' not in res)
self.assertTrue('c' in res)
traj.f_delete_item(res, remove_from_trajectory=True)
self.assertTrue('results' in traj)
self.assertTrue(res not in traj)
def test_storage_and_loading(self):
filename = make_temp_dir('linktest.hdf5')
traj = Trajectory(filename=filename)
traj.f_add_parameter_group('test')
traj.f_add_parameter_group('test2')
res = traj.f_add_result('kk', 42)
traj.par.f_add_link('gg', res)
traj.f_add_link('hh', res)
traj.f_add_link('jj', traj.par)
traj.f_add_link('ii', res)
traj.test.f_add_link('circle1', traj.test2)
traj.test2.f_add_link('circle2', traj.test)
traj.test.f_add_link('circle2', traj.test.circle1.circle2)
traj.f_add_parameter_group('test.ab.bc.cd')
traj.cd.f_add_link(traj.test)
traj.test.f_add_link(traj.cd)
traj.f_store()
traj2 = Trajectory(filename=filename)
traj2.f_load(name=traj.v_name, load_data=2)
self.assertTrue(traj.kk == traj2.gg, '%s != %s' % (traj.kk, traj2.gg))
self.assertTrue(traj.cd.test is traj.test)
self.assertTrue(len(traj._linked_by), len(traj2._linked_by))
self.compare_trajectories(traj, traj2)
self.assertTrue('jj' in traj2._nn_interface._links_count)
traj2.f_remove_child('jj')
self.assertTrue('jj' not in traj2._nn_interface._links_count)
traj2.f_remove_child('hh')
traj2.f_remove_child('ii')
traj2.f_remove_child('parameters', recursive=True)
traj2.v_auto_load = True
group = traj2.par.test2.circle2
self.assertTrue(group is traj2.test)
retest = traj2.test.circle1
self.assertTrue(retest is traj2.test2)
self.assertTrue(traj2.test.circle2 is traj2.test)
self.assertTrue(traj2.hh == traj2.res.kk)
traj2.v_auto_load = False
traj2.f_load_child('jj')
self.assertTrue(traj2.jj is traj2.par)
traj2.f_load(load_data=2)
self.assertTrue(traj2.ii == traj2.res.kk)
def test_storing_and_manipulating(self):
filename = make_temp_dir("hdf5manipulation.hdf5")
traj = Trajectory(name=make_trajectory_name(self), filename=filename)
trajname = traj.v_name
thedata = np.zeros((1000, 1000))
res = traj.f_add_result(SharedResult, "shared")
myarray = SharedArray("array", res, trajectory=traj, add_to_parent=True)
mytable = SharedTable("t1", res, trajectory=traj, add_to_parent=True)
mytable2 = SharedTable("t2", res, trajectory=traj, add_to_parent=True)
mytable3 = SharedTable("t3", res, trajectory=traj, add_to_parent=True)
traj.f_store(only_init=True)
myarray.create_shared_data(data=thedata)
mytable.create_shared_data(first_row={"hi": compat.tobytes("hi"), "huhu": np.ones(3)})
mytable2.create_shared_data(description={"ha": pt.StringCol(2, pos=0), "haha": pt.FloatCol(pos=1)})
mytable3.create_shared_data(description={"ha": pt.StringCol(2, pos=0), "haha": pt.FloatCol(pos=1)})
traj.f_store()
newrow = {"ha": "hu", "haha": 4.0}
with self.assertRaises(TypeError):
row = traj.shared.t2.row
with StorageContextManager(traj) as cm:
row = traj.shared.t2.row
for irun in range(11):
for key, val in newrow.items():
row[key] = val
row.append()
traj.shared.t3.flush()
data = myarray.read()
arr = myarray.get_data_node()
self.assertTrue(np.all(data == thedata))
with StorageContextManager(traj) as cm:
myarray[2, 2] = 10
data = myarray.read()
self.assertTrue(data[2, 2] == 10)
self.assertTrue(data[2, 2] == 10)
self.assertFalse(traj.v_storage_service.is_open)
traj = load_trajectory(name=trajname, filename=filename)
traj.f_load(load_data=2)
traj.shared.t2.traj = traj
traj.shared.t1.traj = traj
traj.shared.array.traj = traj
self.assertTrue(traj.shared.t2.nrows == 11, "%s != 11" % str(traj.shared.t2.nrows))
self.assertTrue(traj.shared.t2[0]["ha"] == compat.tobytes("hu"), traj.shared.t2[0]["ha"])
self.assertTrue(traj.shared.t2[1]["ha"] == compat.tobytes("hu"), traj.shared.t2[1]["ha"])
self.assertTrue("huhu" in traj.shared.t1.colnames)
self.assertTrue(traj.shared.array[2, 2] == 10)
def test_loading_explored_parameters(self):
filename = make_temp_dir('load_explored.hdf5')
traj = Trajectory(filename=filename, overwrite_file=True, add_time=False)
traj.par.x = Parameter('x', 42, comment='answer')
traj.f_explore({'x':[1,2,3,4]})
traj.f_store()
name = traj.v_name
traj = Trajectory(filename=filename, add_time=False)
traj.f_load()
x = traj.f_get('x')
self.assertIs(x, traj._explored_parameters['parameters.x'])
def test_loading_explored_parameters(self):
filename = make_temp_dir('load_explored.hdf5')
traj = Trajectory(filename=filename, overwrite_file=True, add_time=False)
traj.par.x = Parameter('x', 42, comment='answer')
traj.f_explore({'x':[1,2,3,4]})
traj.f_store()
name = traj.v_name
traj = Trajectory(filename=filename, add_time=False)
traj.f_load()
x = traj.f_get('x')
self.assertIs(x, traj._explored_parameters['parameters.x'])
def test_max_depth_loading_and_storing(self):
filename = make_temp_dir('newassignment.hdf5')
traj = Trajectory(filename=filename, overwrite_file=True)
traj.par.d1 = Parameter('d1.d2.d3.d4.d5', 55)
traj.f_store(max_depth=4)
traj = load_trajectory(index=-1, filename=filename)
traj.f_load(load_data=2)
self.assertTrue('d3' in traj)
self.assertFalse('d4' in traj)
traj = load_trajectory(index=-1, filename=filename, max_depth=3)
self.assertTrue('d2' in traj)
self.assertFalse('d3' in traj)
traj.par.f_remove(recursive=True)
traj.dpar.d1 = Parameter('d1.d2.d3.d4.d5', 123)
traj.dpar.f_store_child('d1', recursive=True, max_depth=3)
traj.dpar.f_remove_child('d1', recursive=True)
self.assertTrue('d1' not in traj)
traj.dpar.f_load_child('d1', recursive=True)
self.assertTrue('d3' in traj)
self.assertTrue('d4' not in traj)
traj.dpar.f_remove_child('d1', recursive=True)
self.assertTrue('d1' not in traj)
traj.dpar.f_load_child('d1', recursive=True, max_depth=2)
self.assertTrue('d2' in traj)
self.assertTrue('d3' not in traj)
traj.dpar.l1 = Parameter('l1.l2.l3.l4.l5', 123)
traj.dpar.f_store(recursive=True, max_depth=0)
self.assertFalse(traj.dpar.l1._stored)
traj.dpar.f_store(recursive=True, max_depth=4)
traj.dpar.f_remove()
self.assertTrue('l1' not in traj)
traj.dpar.f_load(recursive=True)
self.assertTrue('l4' in traj)
self.assertTrue('l5' not in traj)
traj.dpar.f_remove()
self.assertTrue('l1' not in traj)
traj.dpar.f_load(recursive=True, max_depth=3)
self.assertTrue('l3' in traj)
self.assertTrue('l4' not in traj)
def test_max_depth_loading_and_storing(self):
filename = make_temp_dir('newassignment.hdf5')
traj = Trajectory(filename=filename, overwrite_file=True)
traj.par = Parameter('d1.d2.d3.d4.d5', 55)
traj.f_store(max_depth=4)
traj = load_trajectory(index=-1, filename=filename)
traj.f_load(load_data=2)
self.assertTrue('d3' in traj)
self.assertFalse('d4' in traj)
traj = load_trajectory(index=-1, filename=filename, max_depth=3)
self.assertTrue('d2' in traj)
self.assertFalse('d3' in traj)
traj.par.f_remove(recursive=True)
traj.dpar = Parameter('d1.d2.d3.d4.d5', 123)
traj.dpar.f_store_child('d1', recursive=True, max_depth=3)
traj.dpar.f_remove_child('d1', recursive=True)
self.assertTrue('d1' not in traj)
traj.dpar.f_load_child('d1', recursive=True)
self.assertTrue('d3' in traj)
self.assertTrue('d4' not in traj)
traj.dpar.f_remove_child('d1', recursive=True)
self.assertTrue('d1' not in traj)
traj.dpar.f_load_child('d1', recursive=True, max_depth=2)
self.assertTrue('d2' in traj)
self.assertTrue('d3' not in traj)
traj.dpar = Parameter('l1.l2.l3.l4.l5', 123)
traj.dpar.f_store(recursive=True, max_depth=0)
self.assertFalse(traj.dpar.l1._stored)
traj.dpar.f_store(recursive=True, max_depth=4)
traj.dpar.f_remove()
self.assertTrue('l1' not in traj)
traj.dpar.f_load(recursive=True)
self.assertTrue('l4' in traj)
self.assertTrue('l5' not in traj)
traj.dpar.f_remove()
self.assertTrue('l1' not in traj)
traj.dpar.f_load(recursive=True, max_depth=3)
self.assertTrue('l3' in traj)
self.assertTrue('l4' not in traj)
def test_storing_and_loading_groups(self):
filename = make_temp_dir('grpgrp.hdf5')
traj = Trajectory(name='traj', add_time=True, filename=filename)
res = traj.f_add_result('aaa.bbb.ccc.iii', 42, 43, comment=7777 * '6')
traj.ccc.v_annotations['gg'] = 4
res = traj.f_add_result('aaa.ddd.eee.jjj', 42, 43, comment=777 * '6')
traj.ccc.v_annotations['j'] = 'osajdsojds'
traj.f_store(only_init=True)
traj.f_store_item('aaa', recursive=True)
newtraj = load_trajectory(traj.v_name, filename=filename, load_all=2)
self.compare_trajectories(traj, newtraj)
traj.iii.f_set(55)
self.assertFalse(results_equal(traj.iii, newtraj.iii))
traj.aaa.f_store(recursive=True, store_data=3)
newtraj.bbb.f_load(recursive=True, load_data=3)
self.compare_trajectories(traj, newtraj)
traj.ccc.v_annotations['gg'] = 5
traj.f_load(load_data=3)
self.assertTrue(traj.ccc.v_annotations['gg'] == 4)
traj.ccc.v_annotations['gg'] = 5
traj.f_store(store_data=3)
newtraj.f_load(load_data=2)
self.assertTrue(newtraj.ccc.v_annotations['gg'] == 4)
newtraj.f_load(load_data=3)
self.assertTrue(newtraj.ccc.v_annotations['gg'] == 5)
traj.ccc.f_add_link('link', res)
traj.f_store_item(traj.ccc, store_data=3, with_links=False)
newtraj.f_load(load_data=3)
self.assertTrue('link' not in newtraj.ccc)
traj.f_store_item(traj.ccc,
store_data=3,
with_links=True,
recursive=True)
newtraj.f_load_item(newtraj.ccc, with_links=False, recursive=True)
self.assertTrue('link' not in newtraj.ccc)
newtraj.f_load_item(newtraj.ccc, recursive=True)
self.assertTrue('link' in newtraj.ccc)
def test_backwards_compatibility(self):
# Test only makes sense with python 2.7 or lower
old_pypet_traj = Trajectory()
module_path, init_file = os.path.split(pypet.__file__)
filename= os.path.join(module_path, 'tests','testdata','pypet_v0_1b_6.hdf5')
old_pypet_traj.f_load(index=-1, load_data=2, force=True, filename=filename)
self.assertTrue(old_pypet_traj.v_version=='0.1b.6')
self.assertTrue(old_pypet_traj.par.x==0)
self.assertTrue(len(old_pypet_traj)==9)
self.assertTrue(old_pypet_traj.res.runs.r_4.z==12)
nexplored = len(old_pypet_traj._explored_parameters)
self.assertGreater(nexplored, 0)
for param in old_pypet_traj.f_get_explored_parameters():
self.assertTrue(old_pypet_traj.f_contains(param))
def test_storing_and_loading_groups(self):
filename = make_temp_dir('grpgrp.hdf5')
traj = Trajectory(name='traj', add_time=True, filename=filename)
res=traj.f_add_result('aaa.bbb.ccc.iii', 42, 43, comment=7777 * '6')
traj.ccc.v_annotations['gg']=4
res=traj.f_add_result('aaa.ddd.eee.jjj', 42, 43, comment=777 * '6')
traj.ccc.v_annotations['j'] = 'osajdsojds'
traj.f_store(only_init=True)
traj.f_store_item('aaa', recursive=True)
newtraj = load_trajectory(traj.v_name, filename=filename, load_all=2)
self.compare_trajectories(traj, newtraj)
traj.iii.f_set(55)
self.assertFalse(results_equal(traj.iii, newtraj.iii))
traj.aaa.f_store(recursive=True, store_data=3)
newtraj.bbb.f_load(recursive=True, load_data=3)
self.compare_trajectories(traj, newtraj)
traj.ccc.v_annotations['gg'] = 5
traj.f_load(load_data=3)
self.assertTrue(traj.ccc.v_annotations['gg'] == 4)
traj.ccc.v_annotations['gg'] = 5
traj.f_store(store_data=3)
newtraj.f_load(load_data=2)
self.assertTrue(newtraj.ccc.v_annotations['gg'] == 4)
newtraj.f_load(load_data=3)
self.assertTrue(newtraj.ccc.v_annotations['gg'] == 5)
traj.ccc.f_add_link('link', res)
traj.f_store_item(traj.ccc, store_data=3, with_links=False)
newtraj.f_load(load_data=3)
self.assertTrue('link' not in newtraj.ccc)
traj.f_store_item(traj.ccc, store_data=3, with_links=True, recursive=True)
newtraj.f_load_item(newtraj.ccc, with_links=False, recursive=True)
self.assertTrue('link' not in newtraj.ccc)
newtraj.f_load_item(newtraj.ccc, recursive=True)
self.assertTrue('link' in newtraj.ccc)
def test_backwards_compatibility(self):
# Test only makes sense with python 2.7 or lower
old_pypet_traj = Trajectory()
module_path, init_file = os.path.split(pypet.__file__)
filename = os.path.join(module_path, 'tests', 'testdata',
'pypet_v0_1b_6.hdf5')
old_pypet_traj.f_load(index=-1,
load_data=2,
force=True,
filename=filename)
self.assertTrue(old_pypet_traj.v_version == '0.1b.6')
self.assertTrue(old_pypet_traj.par.x == 0)
self.assertTrue(len(old_pypet_traj) == 9)
self.assertTrue(old_pypet_traj.res.runs.r_4.z == 12)
nexplored = len(old_pypet_traj._explored_parameters)
self.assertGreater(nexplored, 0)
for param in old_pypet_traj.f_get_explored_parameters():
self.assertTrue(old_pypet_traj.f_contains(param))
def test_link_deletion(self):
filename = make_temp_dir('linktest2.hdf5')
traj = Trajectory(filename=filename)
traj.f_add_parameter_group('test')
traj.f_add_parameter_group('test2')
res= traj.f_add_result('kk', 42)
traj.par.f_add_link('gg', res)
traj.test.f_add_link('circle1' , traj.test2)
traj.test2.f_add_link('circle2' , traj.test)
traj.f_store()
traj.f_delete_link('par.gg')
traj2 = Trajectory(filename=filename)
traj2.f_load(name=traj.v_name, load_data=2)
with self.assertRaises(AttributeError):
traj2.gg
def test_throw_warning_if_old_kw_is_used(self):
pass
filename = make_temp_dir('hdfwarning.hdf5')
with warnings.catch_warnings(record=True) as w:
env = Environment(trajectory='test', filename=filename,
dynamically_imported_classes=[],
log_config=get_log_config())
with warnings.catch_warnings(record=True) as w:
traj = Trajectory(dynamically_imported_classes=[])
traj = env.v_trajectory
traj.f_store()
with warnings.catch_warnings(record=True) as w:
traj.f_load(dynamically_imported_classes=[])
env.f_disable_logging()
def test_link_deletion(self):
filename = make_temp_dir('linktest2.hdf5')
traj = Trajectory(filename=filename)
traj.f_add_parameter_group('test')
traj.f_add_parameter_group('test2')
res = traj.f_add_result('kk', 42)
traj.par.f_add_link('gg', res)
traj.test.f_add_link('circle1', traj.test2)
traj.test2.f_add_link('circle2', traj.test)
traj.f_store()
traj.f_delete_link('par.gg')
traj2 = Trajectory(filename=filename)
traj2.f_load(name=traj.v_name, load_data=2)
with self.assertRaises(AttributeError):
traj2.gg
def test_storage_service_errors(self):
traj = Trajectory(filename=make_temp_dir('testnoservice.hdf5'))
traj_name = traj.v_name
# you cannot store stuff before the trajectory was stored once:
with self.assertRaises(ValueError):
traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)
traj.f_store()
with self.assertRaises(ValueError):
traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = 'test')
with self.assertRaises(pex.NoSuchServiceError):
traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)
with self.assertRaises(ValueError):
traj.f_load(name='test', index=1)
with self.assertRaises(RuntimeError):
traj.v_storage_service.store('LIST', [('LEAF',None,None,None,None)],
trajectory_name = traj.v_name)
with self.assertRaises(ValueError):
traj.f_load(index=9999)
with self.assertRaises(ValueError):
traj.f_load(name='Non-Existising-Traj')
def test_storage_service_errors(self):
traj = Trajectory(filename=make_temp_dir('testnoservice.hdf5'), add_time=True)
traj_name = traj.v_name
# you cannot store stuff before the trajectory was stored once:
with self.assertRaises(ValueError):
traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)
traj.f_store()
with self.assertRaises(ValueError):
traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = 'test')
with self.assertRaises(pex.NoSuchServiceError):
traj.v_storage_service.store('FAKESERVICE', self, trajectory_name = traj.v_name)
with self.assertRaises(ValueError):
traj.f_load(name='test', index=1)
with self.assertRaises(RuntimeError):
traj.v_storage_service.store('LIST', [('LEAF',None,None,None,None)],
trajectory_name = traj.v_name)
with self.assertRaises(ValueError):
traj.f_load(index=9999)
with self.assertRaises(ValueError):
traj.f_load(name='Non-Existising-Traj')
from pypet import Trajectory
import matplotlib.pyplot as plt
plt.style.use("seaborn")
# path
SIM_NAME = "covariate_binary"
SIM_PATH = "/home/simon/Documents/NNVI/simulations/" + SIM_NAME
RES_FILE = SIM_PATH + "/results/" + SIM_NAME + ".hdf5"
# load pypet trajectory
traj = Trajectory(SIM_NAME, add_time=False)
traj.f_load(
filename=RES_FILE,
load_data=2,
load_results=2,
load_derived_parameters=2,
load_other_data=2,
load_parameters=2,
force=True
)
traj.v_auto_load = True
# load results
seed = [r.f_get("seed").seed for r in traj.res.runs]
p_bin = [r.f_get("p_bin").p_bin for r in traj.res.runs]
N = [r.f_get("N").N for r in traj.res.runs]
mse = [r.f_get("mse").mse for r in traj.res.runs]
elbo = [r.f_get("elbo").elbo for r in traj.res.runs]
auroc = [r.f_get("auroc").auroc for r in traj.res.runs]
dist_proj = [r.f_get("dist_proj").dist_proj for r in traj.res.runs]
def test_delete_whole_subtrees(self):
filename = make_temp_dir('testdeltree.hdf5')
traj = Trajectory(name='TestDelete',
filename=filename, large_overview_tables=True,
add_time=True)
res = traj.f_add_result('mytest.yourtest.test', a='b', c='d')
dpar = traj.f_add_derived_parameter('mmm.gr.dpdp', 666)
res = traj.f_add_result('hhh.ll', a='b', c='d')
res = traj.f_add_derived_parameter('hhh.gg', 555)
traj.f_store()
with pt.open_file(filename) as fh:
daroot = fh.root._f_get_child(traj.v_name)
dpar_table = daroot.overview.derived_parameters_overview
self.assertTrue(len(dpar_table) == 2)
res_table = daroot.overview.results_overview
self.assertTrue((len(res_table)) == 2)
with self.assertRaises(TypeError):
traj.f_remove_item(traj.yourtest)
with self.assertRaises(TypeError):
traj.f_delete_item(traj.yourtest)
traj.f_remove_item(traj.yourtest, recursive=True)
self.assertTrue('mytest' in traj)
self.assertTrue('yourtest' not in traj)
traj.f_load(load_data=2)
self.assertTrue('yourtest.test' in traj)
traj.f_delete_item(traj.yourtest, recursive=True, remove_from_trajectory=True)
traj.f_delete_item(traj.mmm, recursive=True, remove_from_trajectory=True)
traj.f_load(load_data=2)
self.assertTrue('yourtest.test' not in traj)
self.assertTrue('yourtest' not in traj)
with pt.open_file(filename) as fh:
daroot = fh.root._f_get_child(traj.v_name)
dpar_table = daroot.overview.derived_parameters_overview
self.assertTrue(len(dpar_table) == 2)
res_table = daroot.overview.results_overview
self.assertTrue((len(res_table)) == 2)
traj.f_add_parameter('ggg', 43)
traj.f_add_parameter('hhh.mmm', 45)
traj.f_add_parameter('jjj', 55)
traj.f_add_parameter('hhh.nnn', 55555)
traj.f_explore({'ggg':[1,2,3]})
traj.f_store()
with pt.open_file(filename) as fh:
daroot = fh.root._f_get_child(traj.v_name)
par_table = daroot.overview.parameters_overview
self.assertTrue(len(par_table) == 4)
traj.f_delete_item('par.hhh', recursive=True, remove_from_trajectory=True)
traj.f_add_parameter('saddsdfdsfd', 111)
traj.f_store()
with pt.open_file(filename) as fh:
daroot = fh.root._f_get_child(traj.v_name)
par_table = daroot.overview.parameters_overview
self.assertTrue(len(par_table) == 5)
# Set final trajectory path
traj_final_fullpath = os.path.join(output_dir, traj_filename)
# Copy the last trajectory in the list to the output directory
# It will be used as a starting trajectory to merge the others into
shutil.copy(traj_fullpath_list[-1], traj_final_fullpath)
# Then remove last trajectory from list
trajectories.pop()
traj_fullpath_list.pop()
# Load copied trajectory from output directory.
# We'll merge the other trajectories into this one,
# which will then become the final trajectory
traj_final = Trajectory()
traj_final.f_load(filename=traj_final_fullpath,
index=0,
load_parameters=2,
load_derived_parameters=2,
load_results=2,
load_other_data=2,
force=True)
# Merge trajectories
print('Merging trajectory files...')
for traj_to_merge in trajectories:
traj_final.f_merge(
traj_to_merge,
remove_duplicates=True,
backup=False,
# backup_filename=True,
delete_other_trajectory=False,
slow_merge=True,
ignore_data=())
def test_storing_and_manipulating(self):
filename = make_temp_dir('hdf5manipulation.hdf5')
traj = Trajectory(name=make_trajectory_name(self), filename=filename)
trajname = traj.v_name
thedata = np.zeros((1000, 1000))
res = traj.f_add_result(SharedResult, 'shared')
myarray = SharedArray('array',
res,
trajectory=traj,
add_to_parent=True)
mytable = SharedTable('t1', res, trajectory=traj, add_to_parent=True)
mytable2 = SharedTable('t2', res, trajectory=traj, add_to_parent=True)
mytable3 = SharedTable('t3', res, trajectory=traj, add_to_parent=True)
traj.f_store(only_init=True)
myarray.create_shared_data(data=thedata)
mytable.create_shared_data(first_row={
'hi': compat.tobytes('hi'),
'huhu': np.ones(3)
})
mytable2.create_shared_data(description={
'ha': pt.StringCol(2, pos=0),
'haha': pt.FloatCol(pos=1)
})
mytable3.create_shared_data(description={
'ha': pt.StringCol(2, pos=0),
'haha': pt.FloatCol(pos=1)
})
traj.f_store()
newrow = {'ha': 'hu', 'haha': 4.0}
with self.assertRaises(TypeError):
traj.shared.t2.row
with StorageContextManager(traj) as cm:
row = traj.shared.t2.row
for irun in range(11):
for key, val in newrow.items():
row[key] = val
row.append()
traj.shared.t3.flush()
data = myarray.read()
myarray.get_data_node()
self.assertTrue(np.all(data == thedata))
with StorageContextManager(traj):
myarray[2, 2] = 10
data = myarray.read()
self.assertTrue(data[2, 2] == 10)
self.assertTrue(data[2, 2] == 10)
self.assertFalse(traj.v_storage_service.is_open)
traj = load_trajectory(name=trajname, filename=filename)
traj.f_load(load_data=2)
traj.shared.t2.traj = traj
traj.shared.t1.traj = traj
traj.shared.array.traj = traj
self.assertTrue(traj.shared.t2.nrows == 11,
'%s != 11' % str(traj.shared.t2.nrows))
self.assertTrue(traj.shared.t2[0]['ha'] == compat.tobytes('hu'),
traj.shared.t2[0]['ha'])
self.assertTrue(traj.shared.t2[1]['ha'] == compat.tobytes('hu'),
traj.shared.t2[1]['ha'])
self.assertTrue('huhu' in traj.shared.t1.colnames)
self.assertTrue(traj.shared.array[2, 2] == 10)
def main():
filename = os.path.join('hdf5', 'example_05.hdf5')
env = Environment(trajectory='Example_05_Euler_Integration',
filename=filename,
file_title='Example_05_Euler_Integration',
comment='Go for Euler!')
traj = env.v_trajectory
trajectory_name = traj.v_name
# 1st a) phase parameter addition
add_parameters(traj)
# 1st b) phase preparation
# We will add the differential equation (well, its source code only) as a derived parameter
traj.f_add_derived_parameter(FunctionParameter,'diff_eq', diff_lorenz,
comment='Source code of our equation!')
# We want to explore some initial conditions
traj.f_explore({'initial_conditions' : [
np.array([0.01,0.01,0.01]),
np.array([2.02,0.02,0.02]),
np.array([42.0,4.2,0.42])
]})
# 3 different conditions are enough for an illustrative example
# 2nd phase let's run the experiment
# We pass `euler_scheme` as our top-level simulation function and
# the Lorenz equation 'diff_lorenz' as an additional argument
env.f_run(euler_scheme, diff_lorenz)
# We don't have a 3rd phase of post-processing here
# 4th phase analysis.
# I would recommend to do post-processing completely independent from the simulation,
# but for simplicity let's do it here.
# Let's assume that we start all over again and load the entire trajectory new.
# Yet, there is an error within this approach, do you spot it?
del traj
traj = Trajectory(filename=filename)
# We will only fully load parameters and derived parameters.
# Results will be loaded manually later on.
try:
# However, this will fail because our trajectory does not know how to
# build the FunctionParameter. You have seen this coming, right?
traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2,
load_results=1)
except ImportError as e:
print('That did\'nt work, I am sorry: %s ' % str(e))
# Ok, let's try again but this time with adding our parameter to the imports
traj = Trajectory(filename=filename,
dynamically_imported_classes=FunctionParameter)
# Now it works:
traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2,
load_results=1)
#For the fun of it, let's print the source code
print('\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq)
# Let's get the exploration array:
initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range()
# Now let's plot our simulated equations for the different initial conditions:
# We will iterate through the run names
for idx, run_name in enumerate(traj.f_get_run_names()):
#Get the result of run idx from the trajectory
euler_result = traj.results.f_get(run_name).euler_evolution
# Now we manually need to load the result. Actually the results are not so large and we
# could load them all at once. But for demonstration we do as if they were huge:
traj.f_load_item(euler_result)
euler_data = euler_result.data
#Plot fancy 3d plot
fig = plt.figure(idx)
ax = fig.gca(projection='3d')
x = euler_data[:,0]
y = euler_data[:,1]
z = euler_data[:,2]
ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx]))
plt.legend()
plt.show()
# Now we free the data again (because we assume its huuuuuuge):
del euler_data
euler_result.f_empty()
# You have to click through the images to stop the example_05 module!
# Finally disable logging and close all log-files
env.f_disable_logging()
def main():
filename = os.path.join('hdf5', 'example_05.hdf5')
env = Environment(trajectory='Example_05_Euler_Integration',
filename=filename,
file_title='Example_05_Euler_Integration',
overwrite_file=True,
comment='Go for Euler!')
traj = env.trajectory
trajectory_name = traj.v_name
# 1st a) phase parameter addition
add_parameters(traj)
# 1st b) phase preparation
# We will add the differential equation (well, its source code only) as a derived parameter
traj.f_add_derived_parameter(FunctionParameter,'diff_eq', diff_lorenz,
comment='Source code of our equation!')
# We want to explore some initial conditions
traj.f_explore({'initial_conditions' : [
np.array([0.01,0.01,0.01]),
np.array([2.02,0.02,0.02]),
np.array([42.0,4.2,0.42])
]})
# 3 different conditions are enough for an illustrative example
# 2nd phase let's run the experiment
# We pass `euler_scheme` as our top-level simulation function and
# the Lorenz equation 'diff_lorenz' as an additional argument
env.run(euler_scheme, diff_lorenz)
# We don't have a 3rd phase of post-processing here
# 4th phase analysis.
# I would recommend to do post-processing completely independent from the simulation,
# but for simplicity let's do it here.
# Let's assume that we start all over again and load the entire trajectory new.
# Yet, there is an error within this approach, do you spot it?
del traj
traj = Trajectory(filename=filename)
# We will only fully load parameters and derived parameters.
# Results will be loaded manually later on.
try:
# However, this will fail because our trajectory does not know how to
# build the FunctionParameter. You have seen this coming, right?
traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2,
load_results=1)
except ImportError as e:
print('That did\'nt work, I am sorry: %s ' % str(e))
# Ok, let's try again but this time with adding our parameter to the imports
traj = Trajectory(filename=filename,
dynamically_imported_classes=FunctionParameter)
# Now it works:
traj.f_load(name=trajectory_name, load_parameters=2, load_derived_parameters=2,
load_results=1)
#For the fun of it, let's print the source code
print('\n ---------- The source code of your function ---------- \n %s' % traj.diff_eq)
# Let's get the exploration array:
initial_conditions_exploration_array = traj.f_get('initial_conditions').f_get_range()
# Now let's plot our simulated equations for the different initial conditions:
# We will iterate through the run names
for idx, run_name in enumerate(traj.f_get_run_names()):
#Get the result of run idx from the trajectory
euler_result = traj.results.f_get(run_name).euler_evolution
# Now we manually need to load the result. Actually the results are not so large and we
# could load them all at once. But for demonstration we do as if they were huge:
traj.f_load_item(euler_result)
euler_data = euler_result.data
#Plot fancy 3d plot
fig = plt.figure(idx)
ax = fig.gca(projection='3d')
x = euler_data[:,0]
y = euler_data[:,1]
z = euler_data[:,2]
ax.plot(x, y, z, label='Initial Conditions: %s' % str(initial_conditions_exploration_array[idx]))
plt.legend()
plt.show()
# Now we free the data again (because we assume its huuuuuuge):
del euler_data
euler_result.f_empty()
# You have to click through the images to stop the example_05 module!
# Finally disable logging and close all log-files
env.disable_logging()
# That was easy, let's assume we already completed a simulation and now we add a veeeery large
# result that we want to store to disk immediately and than empty it
traj.f_add_result('starwars.gross_income_of_film', amount=10.1 ** 11, currency='$$$',
comment='George Lucas is rich, dude!')
# This is a large number, we better store it and than free the memory:
traj.f_store_item('gross_income_of_film')
traj.gross_income_of_film.f_empty()
# Now lets reload the trajectory
del traj
traj = Trajectory(filename=filename)
# We want to load the last trajectory in the file, therefore index = -1
# We want to load the parameters, therefore load_parameters=2
# We only want to load the skeleton of the results, so load_results=1
traj.f_load(index=-1, load_parameters=2, load_results=1)
# Let's check if our result is really empty
if traj.gross_income_of_film.f_is_empty():
print('Nothing there!')
else:
print('I found something!')
# Ok, let's manually reload the result
traj.f_load_item('gross_income_of_film')
if traj.gross_income_of_film.f_is_empty():
print('Still empty :-(')
else:
print('George Lucas earned %s%s!' %(str(traj.gross_income_of_film.amount),
traj.gross_income_of_film.currency))
def main():
traj_dir = sys.argv[1]
traj_filename = sys.argv[2]
file_prefix = sys.argv[3]
run_i = 0
if len(sys.argv) > 4:
run_i = np.int(sys.argv[4])
print('run_i= {0}'.format(run_i))
print('traj_dir= {0}'.format(traj_dir))
print('traj_filename= {0}'.format(traj_filename))
print('file_prefix= {0}'.format(file_prefix))
# Change current directory to the one containing the trajectory files
os.chdir(traj_dir)
# Load the trajectory from the hdf5 file
traj_fullpath = os.path.join(traj_dir, traj_filename)
traj = Trajectory()
traj.f_load(
filename=traj_fullpath,
index=0,
as_new=False,
force=True,
load_parameters=2,
load_derived_parameters=2,
load_results=2,
load_other_data=2
)
# Set current run
traj.v_idx = run_i
# Read number of nodes
nodes_n = traj.par.topology.initial.nodes_n
# Generate initial network
G = network_dynamics.generate_network(traj.par.topology.initial)
# Get adjacency matrix
adjacency_matrix = np.array(nx.to_numpy_matrix(
G,
nodelist=np.array(range(0, traj.par.topology.initial.nodes_n)),
dtype=int
))
# Add self-loops
np.fill_diagonal(adjacency_matrix, 1)
# Generate initial node coupling
coupling_matrix = network_dynamics.generate_coupling(
traj.par.node_coupling.initial,
adjacency_matrix
)
# Generate delay
delay_matrices = network_dynamics.generate_delay(traj.par.delay.initial, adjacency_matrix)
# Generate coefficient matrices
coefficient_matrices = np.transpose(delay_matrices * coupling_matrix, (0, 2, 1))
# Run dynamics
time_series = network_dynamics.run_dynamics(
traj.par.node_dynamics,
coefficient_matrices
)
# Create settings dictionary
print('network_inference.p_value = {0}\n'.format(
traj.par.network_inference.p_value))
settings = {
'min_lag_sources': traj.par.network_inference.min_lag_sources,
'max_lag_sources': traj.par.network_inference.max_lag_sources,
'tau_sources': traj.par.network_inference.tau_sources,
'max_lag_target': traj.par.network_inference.max_lag_target,
'tau_target': traj.par.network_inference.tau_target,
'cmi_estimator': traj.par.network_inference.cmi_estimator,
'kraskov_k': traj.par.network_inference.kraskov_k,
'num_threads': traj.par.network_inference.jidt_threads_n,
'permute_in_time': traj.par.network_inference.permute_in_time,
'n_perm_max_stat': traj.par.network_inference.n_perm_max_stat,
'n_perm_min_stat': traj.par.network_inference.n_perm_min_stat,
'n_perm_omnibus': traj.par.network_inference.n_perm_omnibus,
'n_perm_max_seq': traj.par.network_inference.n_perm_max_seq,
'fdr_correction': traj.par.network_inference.fdr_correction,
'alpha_max_stat': traj.par.network_inference.p_value,
'alpha_min_stat': traj.par.network_inference.p_value,
'alpha_omnibus': traj.par.network_inference.p_value,
'alpha_max_seq': traj.par.network_inference.p_value,
'alpha_fdr': traj.par.network_inference.p_value
}
# Save objects to disk
np.save(os.path.join(traj_dir, '.'.join([traj.v_crun, 'topology.initial.adjacency_matrix', 'npy'])), adjacency_matrix)
np.save(os.path.join(traj_dir, '.'.join([traj.v_crun, 'node_coupling.initial.coupling_matrix', 'npy'])), coupling_matrix)
np.save(os.path.join(traj_dir, '.'.join([traj.v_crun, 'node_coupling.initial.coefficient_matrices', 'npy'])), coefficient_matrices)
np.save(os.path.join(traj_dir, '.'.join([traj.v_crun, 'delay.initial.delay_matrices', 'npy'])), delay_matrices)
np.save(os.path.join(traj_dir, '.'.join([traj.v_crun, 'node_dynamics.time_series', 'npy'])), time_series)
save_obj(settings, traj_dir, '.'.join([traj.v_crun, 'settings.pkl']))
# Path to PBS script
job_script_path = os.path.join(traj_dir, 'run_python_script.pbs')
# Run job array
job_walltime_hours = 6#1 + int(np.ceil((nodes_n + 20) / 30))
job_walltime_minutes = 0
job_settings = {
'N': 'run{0}'.format(run_i),
'J': '{0}-{1}'.format(0, nodes_n - 1),
'l': 'walltime={0}:{1}:00'.format(job_walltime_hours, job_walltime_minutes),
'q': 'defaultQ'
}
job_args = {
'python_script_path': '/project/RDS-FEI-InfoDynFuncStruct-RW/Leo/inference/hpc_pypet_single_target.py',
'traj_dir': traj_dir,
'traj_filename': traj_filename,
'file_prefix': traj.v_crun
}
run_job_array(job_script_path, job_settings, job_args)
for traj_dir in traj_dir_fullpaths:
if os.path.exists(traj_dir):
print('OK: Directory found: {0}'.format(traj_dir))
else:
raise ValueError('ERROR: Directory not found: {0}'.format(traj_dir))
for traj_dir in traj_dir_fullpaths:
# Change current directory to the one containing the trajectory files
os.chdir(traj_dir)
# Load the trajectory (only parameters)
traj_fullpath = os.path.join(traj_dir, traj_filename)
traj = Trajectory()
traj.f_load(filename=traj_fullpath,
index=0,
load_parameters=2,
load_results=0,
load_derived_parameters=0,
force=True)
# Turn on auto loading
traj.v_auto_load = True
# Ensure trajectory was not already assembled
if not traj.f_is_completed():
# Save a backup version of the original trajectory
traj_backup_fullpath = os.path.join(
traj_dir, traj_filename + '.backup' +
datetime.now().strftime("%Y_%m_%d_%Hh%Mm%Ss"))
shutil.copy(traj_fullpath, traj_backup_fullpath)
# Create a pypet Environment object and link it to the trajectory
env = Environment(trajectory=traj)
def main():
# This time we don't need an environment since we just going to look
# at data in the trajectory
traj = Trajectory('FiringRate', add_time=False)
# Let's load the trajectory from the file
# Only load the parameters, we will load the results on the fly as we need them
filename = os.path.join('hdf5', 'FiringRate.hdf5')
traj.f_load(load_parameters=2, load_derived_parameters=0, load_results=0,
load_other_data=0, filename=filename)
# We'll simply use auto loading so all data will be loaded when needed.
traj.v_auto_load = True
rates_frame = traj.res.summary.firing_rates.rates_frame
# Here we load the data automatically on the fly
plt.figure()
plt.subplot(2,1,1)
#Let's iterate through the columns and plot the different firing rates :
for tau_ref, I_col in rates_frame.iteritems():
plt.plot(I_col.index, I_col, label='Avg. Rate for tau_ref=%s' % str(tau_ref))
# Label the plot
plt.xlabel('I')
plt.ylabel('f[Hz]')
plt.title('Firing as a function of input current `I`')
plt.legend(loc='best')
# Also let's plot an example run, how about run 13 ?
example_run = 13
traj.v_idx = example_run # We make the trajectory behave as a single run container.
# This short statement has two major effects:
# a) all explored parameters are set to the value of run 13,
# b) if there are tree nodes with names other than the current run aka `run_00000013`
# they are simply ignored, if we use the `$` sign or the `crun` statement,
# these are translated into `run_00000013`.
# Get the example data
example_I = traj.I
example_tau_ref = traj.tau_ref
example_V = traj.results.neuron.crun.V # Here crun stands for run_00000013
# We need the time step...
dt = traj.dt
# ...to create an x-axis for the plot
dt_array = [irun * dt for irun in range(len(example_V))]
# And plot the development of V over time,
# Since this is rather repetitive, we only
# plot the first eighth of it.
plt.subplot(2,1,2)
plt.plot(dt_array, example_V)
plt.xlim((0, dt*len(example_V)/8))
# Label the axis
plt.xlabel('t[ms]')
plt.ylabel('V')
plt.title('Example of development of V for I=%s, tau_ref=%s in run %d' %
(str(example_I), str(example_tau_ref), traj.v_idx))
# And let's take a look at it
plt.show()
# Finally revoke the `traj.v_idx=13` statement and set everything back to normal.
# Since our analysis is done here, we could skip that, but it is always a good idea
# to do that.
traj.f_restore_default()
# coding: utf-8
# In[ ]:
from pypet import Trajectory
# In[ ]:
traj = Trajectory(filename='./pypet/FanSimulation_2016_02_22_14h46m09s.hdf5')
# In[ ]:
traj.f_load(index=-1, load_parameters=2, load_results=2)
# In[ ]:
traj.f_get_parameters()
# In[ ]:
traj.f_get_explored_parameters()
# In[ ]:
traj.f_get_run_names()
# In[ ]:
def my_filter_function(location, dt):
result = location == 'mars' and dt = 1e-2
def test_storage_and_loading(self):
filename = make_temp_dir('linktest.hdf5')
traj = Trajectory(filename=filename)
traj.f_add_parameter_group('test')
traj.f_add_parameter_group('test2')
res= traj.f_add_result('kk', 42)
traj.par.f_add_link('gg', res)
traj.f_add_link('hh', res)
traj.f_add_link('jj', traj.par)
traj.f_add_link('ii', res)
traj.test.f_add_link('circle1' , traj.test2)
traj.test2.f_add_link('circle2' , traj.test)
traj.test.f_add_link('circle2' , traj.test.circle1.circle2)
traj.f_add_parameter_group('test.ab.bc.cd')
traj.cd.f_add_link(traj.test)
traj.test.f_add_link(traj.cd)
traj.f_store()
traj2 = Trajectory(filename=filename)
traj2.f_load(name=traj.v_name, load_data=2)
self.assertTrue(traj.kk == traj2.gg, '%s != %s' % (traj.kk, traj2.gg))
self.assertTrue(traj.cd.test is traj.test)
self.assertTrue(len(traj._linked_by), len(traj2._linked_by))
self.compare_trajectories(traj, traj2)
self.assertTrue('jj' in traj2._nn_interface._links_count)
traj2.f_remove_child('jj')
self.assertTrue('jj' not in traj2._nn_interface._links_count)
traj2.f_remove_child('hh')
traj2.f_remove_child('ii')
traj2.f_remove_child('parameters', recursive=True)
traj2.v_auto_load = True
group = traj2.par.test2.circle2
self.assertTrue(group is traj2.test)
retest = traj2.test.circle1
self.assertTrue(retest is traj2.test2)
self.assertTrue(traj2.test.circle2 is traj2.test)
self.assertTrue(traj2.hh == traj2.res.kk)
traj2.v_auto_load = False
traj2.f_load_child('jj')
self.assertTrue(traj2.jj is traj2.par)
traj2.f_load(load_data=2)
self.assertTrue(traj2.ii == traj2.res.kk)
traj_dir = sys.argv[1]
traj_filename = sys.argv[2]
file_prefix = sys.argv[3]
target_id = np.int(sys.argv[4])
print('target_id= {0}'.format(target_id))
print('traj_dir= {0}'.format(traj_dir))
print('traj_filename= {0}'.format(traj_filename))
print('file_prefix= {0}'.format(file_prefix))
# Load the trajectory from the hdf5 file
traj = Trajectory()
traj.f_load(filename=os.path.join(traj_dir, traj_filename),
index=0,
as_new=False,
force=True,
load_parameters=2,
load_derived_parameters=2,
load_results=2,
load_other_data=2)
# Load time series
time_series = np.load(
os.path.join(traj_dir,
'.'.join([file_prefix, 'node_dynamics.time_series.npy'])))
# Load settings dictionary
settings = load_obj(
os.path.join(traj_dir, '.'.join([file_prefix, 'settings.pkl'])))
network_inference = traj.parameters.network_inference
def test_delete_whole_subtrees(self):
filename = make_temp_dir('testdeltree.hdf5')
traj = Trajectory(name='TestDelete',
filename=filename, large_overview_tables=True)
res = traj.f_add_result('mytest.yourtest.test', a='b', c='d')
dpar = traj.f_add_derived_parameter('mmm.gr.dpdp', 666)
res = traj.f_add_result('hhh.ll', a='b', c='d')
res = traj.f_add_derived_parameter('hhh.gg', 555)
traj.f_store()
with ptcompat.open_file(filename) as fh:
daroot = ptcompat.get_child(fh.root, traj.v_name)
dpar_table = daroot.overview.derived_parameters_overview
self.assertTrue(len(dpar_table) == 2)
res_table = daroot.overview.results_overview
self.assertTrue((len(res_table)) == 2)
with self.assertRaises(TypeError):
traj.f_remove_item(traj.yourtest)
with self.assertRaises(TypeError):
traj.f_delete_item(traj.yourtest)
traj.f_remove_item(traj.yourtest, recursive=True)
self.assertTrue('mytest' in traj)
self.assertTrue('yourtest' not in traj)
traj.f_load(load_data=2)
self.assertTrue('yourtest.test' in traj)
traj.f_delete_item(traj.yourtest, recursive=True, remove_from_trajectory=True)
traj.f_delete_item(traj.mmm, recursive=True, remove_from_trajectory=True)
traj.f_load(load_data=2)
self.assertTrue('yourtest.test' not in traj)
self.assertTrue('yourtest' not in traj)
with ptcompat.open_file(filename) as fh:
daroot = ptcompat.get_child(fh.root, traj.v_name)
dpar_table = daroot.overview.derived_parameters_overview
self.assertTrue(len(dpar_table) == 2)
res_table = daroot.overview.results_overview
self.assertTrue((len(res_table)) == 2)
traj.f_add_parameter('ggg', 43)
traj.f_add_parameter('hhh.mmm', 45)
traj.f_add_parameter('jjj', 55)
traj.f_add_parameter('hhh.nnn', 55555)
traj.f_explore({'ggg':[1,2,3]})
traj.f_store()
with ptcompat.open_file(filename) as fh:
daroot = ptcompat.get_child(fh.root, traj.v_name)
par_table = daroot.overview.parameters_overview
self.assertTrue(len(par_table) == 4)
traj.f_delete_item('par.hhh', recursive=True, remove_from_trajectory=True)
traj.f_add_parameter('saddsdfdsfd', 111)
traj.f_store()
with ptcompat.open_file(filename) as fh:
daroot = ptcompat.get_child(fh.root, traj.v_name)
par_table = daroot.overview.parameters_overview
self.assertTrue(len(par_table) == 5)
# with self.assertRaises(TypeError):
# # We cannot delete something containing an explored parameter
# traj.f_delete_item('par', recursive=True)
with self.assertRaises(TypeError):
traj.f_delete_item('ggg')
def main():
# This time we don't need an environment since we just going to look
# at data in the trajectory
traj = Trajectory('FiringRate', add_time=False)
# Let's load the trajectory from the file
# Only load the parameters, we will load the results on the fly as we need them
filename = os.path.join('hdf5', 'FiringRate.hdf5')
traj.f_load(load_parameters=2,
load_derived_parameters=0,
load_results=0,
load_other_data=0,
filename=filename)
# We'll simply use auto loading so all data will be loaded when needed.
traj.v_auto_load = True
rates_frame = traj.res.summary.firing_rates.rates_frame
# Here we load the data automatically on the fly
plt.figure()
plt.subplot(2, 1, 1)
#Let's iterate through the columns and plot the different firing rates :
for tau_ref, I_col in rates_frame.iteritems():
plt.plot(I_col.index,
I_col,
label='Avg. Rate for tau_ref=%s' % str(tau_ref))
# Label the plot
plt.xlabel('I')
plt.ylabel('f[Hz]')
plt.title('Firing as a function of input current `I`')
plt.legend(loc='best')
# Also let's plot an example run, how about run 13 ?
example_run = 13
traj.v_idx = example_run # We make the trajectory behave as a single run container.
# This short statement has two major effects:
# a) all explored parameters are set to the value of run 13,
# b) if there are tree nodes with names other than the current run aka `run_00000013`
# they are simply ignored, if we use the `$` sign or the `crun` statement,
# these are translated into `run_00000013`.
# Get the example data
example_I = traj.I
example_tau_ref = traj.tau_ref
example_V = traj.results.neuron.crun.V # Here crun stands for run_00000013
# We need the time step...
dt = traj.dt
# ...to create an x-axis for the plot
dt_array = [irun * dt for irun in range(len(example_V))]
# And plot the development of V over time,
# Since this is rather repetitive, we only
# plot the first eighth of it.
plt.subplot(2, 1, 2)
plt.plot(dt_array, example_V)
plt.xlim((0, dt * len(example_V) / 8))
# Label the axis
plt.xlabel('t[ms]')
plt.ylabel('V')
plt.title('Example of development of V for I=%s, tau_ref=%s in run %d' %
(str(example_I), str(example_tau_ref), traj.v_idx))
# And let's take a look at it
plt.show()
# Finally revoke the `traj.v_idx=13` statement and set everything back to normal.
# Since our analysis is done here, we could skip that, but it is always a good idea
# to do that.
traj.f_restore_default()
tr.f_explore(explore_dict)
env.run(run_net)
if not first_run:
print("\nSimulation successful. Now merging...\n")
tr1 = Trajectory(name='tr1',
add_time=False,
filename=filename,
dynamic_imports=[Brian2MonitorResult,
Brian2Parameter])
tr1.f_load(load_parameters=2, load_derived_parameters=2,
load_results=2)
tr2 = Trajectory(name='tr2',
add_time=False,
filename=filename,
dynamic_imports=[Brian2MonitorResult,
Brian2Parameter])
tr2.f_load(load_parameters=2, load_derived_parameters=2,
load_results=2)
tr1.f_merge(tr2, backup_filename=True,
move_data=True, delete_other_trajectory = True)
