def test_store_items_and_groups(self):
traj = Trajectory(name='testtraj', filename=make_temp_dir('teststoreitems.hdf5'))
traj.f_store()
traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj.f_store_items(['test','testres','group1'])
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_dir('teststoreitems.hdf5'))
traj2.f_load(load_parameters=2, load_results=2)
traj.f_add_result('Im.stored.along.a.path', 43)
traj.Im.stored.along.v_annotations['wtf'] =4444
traj.res.f_store_child('Im.stored.along.a.path')
traj2.res.f_load_child('Im.stored.along.a.path', load_data=2)
self.compare_trajectories(traj,traj2)
def test_loading_as_new(self):
filename = make_temp_dir('asnew.h5')
traj = Trajectory(name='TestPartial', filename=filename)
traj.f_add_parameter('x', 3)
traj.f_add_parameter('y', 2)
traj.f_explore({'x': [12,3,44], 'y':[1,23,4]})
traj.f_store()
traj = load_trajectory(name=traj.v_name, filename=filename)
with self.assertRaises(TypeError):
traj.f_shrink()
traj = load_trajectory(name=traj.v_name, filename=filename, as_new=True,
new_name='TestTraj', add_time=False)
self.assertTrue(traj.v_name == 'TestTraj')
self.assertTrue(len(traj) == 3)
traj.f_shrink()
self.assertTrue(len(traj) == 1)
def test_version_mismatch(self):
traj = Trajectory(name='TestVERSION', filename=make_temp_dir('testversionmismatch.hdf5'),
add_time=True)
traj.f_add_parameter('group1.test',42)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj._version='0.1a.1'
traj.f_store()
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_dir('testversionmismatch.hdf5'))
with self.assertRaises(pex.VersionMismatchError):
traj2.f_load(load_parameters=2, load_results=2)
traj2.f_load(load_parameters=2, load_results=2, force=True)
self.compare_trajectories(traj,traj2)
get_root_logger().info('Mismatch testing done!')
def test_loading_as_new(self):
filename = make_temp_dir('asnew.h5')
traj = Trajectory(name='TestPartial', filename=filename, add_time=True)
traj.f_add_parameter('x', 3)
traj.f_add_parameter('y', 2)
traj.f_explore({'x': [12,3,44], 'y':[1,23,4]})
traj.f_store()
traj = load_trajectory(name=traj.v_name, filename=filename)
with self.assertRaises(TypeError):
traj.f_shrink()
traj = load_trajectory(name=traj.v_name, filename=filename, as_new=True,
new_name='TestTraj', add_time=False)
self.assertTrue(traj.v_name == 'TestTraj')
self.assertEqual(len(traj), 3)
self.assertEqual(len(traj._explored_parameters), 2)
traj.f_shrink()
self.assertTrue(len(traj) == 1)
def test_store_items_and_groups(self):
traj = Trajectory(name='testtraj', filename=make_temp_dir('teststoreitems.hdf5'),
add_time=True)
traj.f_store()
traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj.f_store_items(['test','testres','group1'])
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_dir('teststoreitems.hdf5'))
traj2.f_load(load_parameters=2, load_results=2)
traj.f_add_result('Im.stored.along.a.path', 43)
traj.Im.stored.along.v_annotations['wtf'] =4444
traj.res.f_store_child('Im.stored.along.a.path')
traj2.res.f_load_child('Im.stored.along.a.path', load_data=2)
self.compare_trajectories(traj,traj2)
def test_version_mismatch(self):
traj = Trajectory(name='TestVERSION', filename=make_temp_dir('testversionmismatch.hdf5'),
add_time=True)
traj.f_add_parameter('group1.test',42)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj._version='0.1a.1'
traj.f_store()
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_dir('testversionmismatch.hdf5'))
with self.assertRaises(pex.VersionMismatchError):
traj2.f_load(load_parameters=2, load_results=2)
traj2.f_load(load_parameters=2, load_results=2, force=True)
self.compare_trajectories(traj,traj2)
get_root_logger().info('Mismatch testing done!')
def test_links_according_to_run(self):
traj = Trajectory()
traj.f_add_parameter('test.hi', 44)
traj.f_explore({'hi': [1,2,3]})
traj.f_add_parameter_group('test.test.test2')
traj.f_add_parameter_group('test2')
traj.test2.f_add_link('test', traj.test)
traj.v_idx = 1
def test_links_according_to_run(self):
traj = Trajectory()
traj.f_add_parameter('test.hi', 44)
traj.f_explore({'hi': [1, 2, 3]})
traj.f_add_parameter_group('test.test.test2')
traj.f_add_parameter_group('test2')
traj.test2.f_add_link('test', traj.test)
traj.v_idx = 1
def test_wildcard_search(self):
traj = Trajectory(name='Testwildcard',
filename=make_temp_dir('wilcard.hdf5'),
add_time=True)
traj.f_add_parameter('expl', 2)
traj.f_explore({'expl': [1, 2, 3, 4]})
traj.f_add_result('wc2test.$.hhh', 333)
traj.f_add_leaf('results.wctest.run_00000000.jjj', 42)
traj.f_add_result('results.wctest.run_00000001.jjj', 43)
traj.f_add_result('results.wctest.%s.jjj' % traj.f_wildcard('$', -1),
43)
traj.v_crun = 1
self.assertTrue(traj.results.wctest['$'].jjj == 43)
self.assertTrue(traj.results.wc2test.crun.hhh == 333)
traj.f_store()
get_root_logger().info('Removing child1')
traj.f_remove_child('results', recursive=True)
get_root_logger().info('Doing auto-load')
traj.v_auto_load = True
self.assertTrue(traj.results.wctest['$'].jjj == 43)
self.assertTrue(traj.results.wc2test.crun.hhh == 333)
get_root_logger().info('Removing child2')
traj.f_remove_child('results', recursive=True)
get_root_logger().info('auto-loading')
traj.v_auto_load = True
self.assertTrue(traj.results.wctest[-1].jjj == 43)
self.assertTrue(traj.results.wc2test[-1].hhh == 333)
get_root_logger().info('Removing child3')
traj.f_remove_child('results', recursive=True)
get_root_logger().info('auto-loading')
traj.v_auto_load = True
self.assertTrue(traj.results.wctest[1].jjj == 43)
self.assertTrue(traj.results.wc2test[-1].hhh == 333)
get_root_logger().info('Done with wildcard test')
def test_file_renaming(self):
traj_name = 'test'
traj = Trajectory('test', add_time=False)
traj.f_add_parameter('x', 42)
traj.f_explore({'x': [1,2,3]})
rename_string = '$traj_$set_$run'
solution_1 = 'test_run_set_ALL_run_ALL'
solution_2 = 'test_run_set_00000_run_00000000'
renaming_1 = rename_log_file(rename_string, traj)
self.assertEqual(renaming_1, solution_1)
traj.v_idx = 0
renaming_2 = rename_log_file(rename_string, traj)
self.assertEqual(renaming_2, solution_2)
def test_file_renaming(self):
traj_name = 'test'
traj = Trajectory('test', add_time=False)
traj.f_add_parameter('x', 42)
traj.f_explore({'x': [1,2,3]})
rename_string = '$traj_$set_$run'
solution_1 = 'test_run_set_ALL_run_ALL'
solution_2 = 'test_run_set_00000_run_00000000'
renaming_1 = rename_log_file(rename_string, traj)
self.assertEqual(renaming_1, solution_1)
traj.v_idx = 0
renaming_2 = rename_log_file(rename_string, traj)
self.assertEqual(renaming_2, solution_2)
def test_wildcard_search(self):
traj = Trajectory(name='Testwildcard', filename=make_temp_dir('wilcard.hdf5'),
add_time=True)
traj.f_add_parameter('expl', 2)
traj.f_explore({'expl':[1,2,3,4]})
traj.f_add_result('wc2test.$.hhh', 333)
traj.f_add_leaf('results.wctest.run_00000000.jjj', 42)
traj.f_add_result('results.wctest.run_00000001.jjj', 43)
traj.f_add_result('results.wctest.%s.jjj' % traj.f_wildcard('$', -1), 43)
traj.v_crun = 1
self.assertTrue(traj.results.wctest['$'].jjj==43)
self.assertTrue(traj.results.wc2test.crun.hhh==333)
traj.f_store()
get_root_logger().info('Removing child1')
traj.f_remove_child('results', recursive=True)
get_root_logger().info('Doing auto-load')
traj.v_auto_load = True
self.assertTrue(traj.results.wctest['$'].jjj==43)
self.assertTrue(traj.results.wc2test.crun.hhh==333)
get_root_logger().info('Removing child2')
traj.f_remove_child('results', recursive=True)
get_root_logger().info('auto-loading')
traj.v_auto_load = True
self.assertTrue(traj.results.wctest[-1].jjj==43)
self.assertTrue(traj.results.wc2test[-1].hhh==333)
get_root_logger().info('Removing child3')
traj.f_remove_child('results', recursive=True)
get_root_logger().info('auto-loading')
traj.v_auto_load = True
self.assertTrue(traj.results.wctest[1].jjj==43)
self.assertTrue(traj.results.wc2test[-1].hhh==333)
get_root_logger().info('Done with wildcard test')
def test_find_in_all_runs_with_links(self):
traj = Trajectory()
traj.f_add_parameter('FloatParam')
traj.par.FloatParam = 4.0
self.explore_dict = {'FloatParam': [1.0, 1.1, 1.2, 1.3]}
traj.f_explore(self.explore_dict)
self.assertTrue(len(traj) == 4)
traj.f_add_result('results.runs.run_00000000.sub.resulttest', 42)
traj.f_add_result('results.runs.run_00000001.sub.resulttest', 43)
traj.f_add_result('results.runs.run_00000002.sub.resulttest', 44)
traj.f_add_result('results.runs.run_00000002.sub.resulttest2', 42)
traj.f_add_result('results.runs.run_00000003.sub.resulttest2', 43)
traj.f_add_derived_parameter(
'derived_parameters.runs.run_00000002.testing', 44)
res_dict = traj.f_get_from_runs('resulttest', fast_access=True)
self.assertTrue(len(res_dict) == 3)
self.assertTrue(res_dict['run_00000001'] == 43)
self.assertTrue('run_00000003' not in res_dict)
res_dict = traj.f_get_from_runs(name='sub.resulttest2',
use_indices=True)
self.assertTrue(len(res_dict) == 2)
self.assertTrue(res_dict[3] is traj.f_get('run_00000003.resulttest2'))
self.assertTrue(1 not in res_dict)
traj.res.runs.r_0.f_add_link('resulttest2',
traj.r_1.f_get('resulttest'))
res_dict = traj.f_get_from_runs(name='resulttest2', use_indices=True)
self.assertTrue(len(res_dict) == 3)
self.assertTrue(res_dict[0] is traj.f_get('run_00000001.resulttest'))
self.assertTrue(1 not in res_dict)
res_dict = traj.f_get_from_runs(name='resulttest2',
use_indices=True,
with_links=False)
self.assertTrue(len(res_dict) == 2)
self.assertTrue(0 not in res_dict)
self.assertTrue(1 not in res_dict)
def test_loading_and_storing_empty_containers(self):
filename = make_temp_dir('empty_containers.hdf5')
traj = Trajectory(filename=filename, add_time=True)
# traj.f_add_parameter('empty.dict', {})
# traj.f_add_parameter('empty.list', [])
traj.f_add_parameter(ArrayParameter, 'empty.tuple', ())
traj.f_add_parameter(ArrayParameter, 'empty.array',
np.array([], dtype=float))
spsparse_csc = spsp.csc_matrix((2, 10))
spsparse_csr = spsp.csr_matrix((6660, 660))
spsparse_bsr = spsp.bsr_matrix((3330, 2220))
spsparse_dia = spsp.dia_matrix((1230, 1230))
traj.f_add_parameter(SparseParameter, 'empty.csc', spsparse_csc)
traj.f_add_parameter(SparseParameter, 'empty.csr', spsparse_csr)
traj.f_add_parameter(SparseParameter, 'empty.bsr', spsparse_bsr)
traj.f_add_parameter(SparseParameter, 'empty.dia', spsparse_dia)
traj.f_add_result(SparseResult,
'empty.all',
dict={},
list=[],
series=pd.Series(),
frame=pd.DataFrame(),
panel=pd.Panel(),
**traj.par.f_to_dict(short_names=True,
fast_access=True))
traj.f_store()
newtraj = load_trajectory(index=-1, filename=filename)
newtraj.f_load(load_data=2)
epg = newtraj.par.empty
self.assertTrue(type(epg.tuple) is tuple)
self.assertTrue(len(epg.tuple) == 0)
self.assertTrue(type(epg.array) is np.ndarray)
self.assertTrue(epg.array.size == 0)
self.assertTrue(spsp.isspmatrix_csr(epg.csr))
self.assertTrue(epg.csr.size == 0)
self.assertTrue(spsp.isspmatrix_csc(epg.csc))
self.assertTrue(epg.csc.size == 0)
self.assertTrue(spsp.isspmatrix_bsr(epg.bsr))
self.assertTrue(epg.bsr.size == 0)
self.assertTrue(spsp.isspmatrix_dia(epg.dia))
self.assertTrue(epg.dia.size == 0)
self.compare_trajectories(traj, newtraj)
def test_find_in_all_runs_with_links(self):
traj = Trajectory()
traj.f_add_parameter('FloatParam')
traj.par.FloatParam=4.0
self.explore_dict = {'FloatParam':[1.0,1.1,1.2,1.3]}
traj.f_explore(self.explore_dict)
self.assertTrue(len(traj) == 4)
traj.f_add_result('results.runs.run_00000000.sub.resulttest', 42)
traj.f_add_result('results.runs.run_00000001.sub.resulttest', 43)
traj.f_add_result('results.runs.run_00000002.sub.resulttest', 44)
traj.f_add_result('results.runs.run_00000002.sub.resulttest2', 42)
traj.f_add_result('results.runs.run_00000003.sub.resulttest2', 43)
traj.f_add_derived_parameter('derived_parameters.runs.run_00000002.testing', 44)
res_dict = traj.f_get_from_runs('resulttest', fast_access=True)
self.assertTrue(len(res_dict)==3)
self.assertTrue(res_dict['run_00000001']==43)
self.assertTrue('run_00000003' not in res_dict)
res_dict = traj.f_get_from_runs(name='sub.resulttest2', use_indices=True)
self.assertTrue(len(res_dict)==2)
self.assertTrue(res_dict[3] is traj.f_get('run_00000003.resulttest2'))
self.assertTrue(1 not in res_dict)
traj.res.runs.r_0.f_add_link('resulttest2', traj.r_1.f_get('resulttest'))
res_dict = traj.f_get_from_runs(name='resulttest2', use_indices=True)
self.assertTrue(len(res_dict)==3)
self.assertTrue(res_dict[0] is traj.f_get('run_00000001.resulttest'))
self.assertTrue(1 not in res_dict)
res_dict = traj.f_get_from_runs(name='resulttest2', use_indices=True, with_links=False)
self.assertTrue(len(res_dict)==2)
self.assertTrue(0 not in res_dict)
self.assertTrue(1 not in res_dict)
def test_link_creation(self):
traj = Trajectory()
traj.f_add_parameter_group('test.test3')
traj.f_add_parameter_group('test2')
with self.assertRaises(AttributeError):
traj.par.f_add_link('test', traj.test2)
with self.assertRaises(ValueError):
traj.f_add_link('parameters', traj.test)
with self.assertRaises(ValueError):
traj.f_add_link('kkkk', PickleResult('fff', 555))
traj.test.f_add_link('circle1' , traj.test2)
traj.test2.f_add_link('circle2' , traj.test)
self.assertTrue(traj.test.circle1.circle2.circle1.circle2 is traj.test)
traj.f_add_link('hh', traj.test)
traj.par.f_add_link('overview', traj.test)
with self.assertRaises(ValueError):
traj.f_add_link('overview', traj.test)
with self.assertRaises(ValueError):
traj.f_add_link('v_crun')
with self.assertRaises(ValueError):
traj.par.f_add_link('gg', traj)
with self.assertRaises(AttributeError):
traj.f_add_parameter('test.circle1.testy', 33)
traj.par.f_add_link('gg', traj.circle1)
self.assertTrue(traj.gg is traj.test2)
self.assertTrue(traj.test2.test3 is traj.par.test.test3)
traj.f_add_link(traj.test3)
self.assertTrue('test3' in traj._links)
def test_link_creation(self):
traj = Trajectory()
traj.f_add_parameter_group('test.test3')
traj.f_add_parameter_group('test2')
with self.assertRaises(AttributeError):
traj.par.f_add_link('test', traj.test2)
with self.assertRaises(ValueError):
traj.f_add_link('parameters', traj.test)
with self.assertRaises(ValueError):
traj.f_add_link('kkkk', PickleResult('fff', 555))
traj.test.f_add_link('circle1', traj.test2)
traj.test2.f_add_link('circle2', traj.test)
self.assertTrue(traj.test.circle1.circle2.circle1.circle2 is traj.test)
traj.f_add_link('hh', traj.test)
traj.par.f_add_link('overview', traj.test)
with self.assertRaises(ValueError):
traj.f_add_link('overview', traj.test)
with self.assertRaises(ValueError):
traj.f_add_link('v_crun')
with self.assertRaises(ValueError):
traj.par.f_add_link('gg', traj)
with self.assertRaises(AttributeError):
traj.f_add_parameter('test.circle1.testy', 33)
traj.par.f_add_link('gg', traj.circle1)
self.assertTrue(traj.gg is traj.test2)
self.assertTrue(traj.test2.test3 is traj.par.test.test3)
traj.f_add_link(traj.test3)
self.assertTrue('test3' in traj._links)
def test_loading_and_storing_empty_containers(self):
filename = make_temp_dir('empty_containers.hdf5')
traj = Trajectory(filename=filename)
# traj.f_add_parameter('empty.dict', {})
# traj.f_add_parameter('empty.list', [])
traj.f_add_parameter(ArrayParameter, 'empty.tuple', ())
traj.f_add_parameter(ArrayParameter, 'empty.array', np.array([], dtype=float))
spsparse_csc = spsp.csc_matrix((2,10))
spsparse_csr = spsp.csr_matrix((6660,660))
spsparse_bsr = spsp.bsr_matrix((3330,2220))
spsparse_dia = spsp.dia_matrix((1230,1230))
traj.f_add_parameter(SparseParameter, 'empty.csc', spsparse_csc)
traj.f_add_parameter(SparseParameter, 'empty.csr', spsparse_csr)
traj.f_add_parameter(SparseParameter, 'empty.bsr', spsparse_bsr)
traj.f_add_parameter(SparseParameter, 'empty.dia', spsparse_dia)
traj.f_add_result(SparseResult, 'empty.all', dict={}, list=[],
series = pd.Series(),
frame = pd.DataFrame(),
panel = pd.Panel(),
**traj.par.f_to_dict(short_names=True, fast_access=True))
traj.f_store()
newtraj = load_trajectory(index=-1, filename=filename)
newtraj.f_load(load_data=2)
epg = newtraj.par.empty
self.assertTrue(type(epg.tuple) is tuple)
self.assertTrue(len(epg.tuple) == 0)
self.assertTrue(type(epg.array) is np.ndarray)
self.assertTrue(epg.array.size == 0)
self.assertTrue(spsp.isspmatrix_csr(epg.csr))
self.assertTrue(epg.csr.size == 0)
self.assertTrue(spsp.isspmatrix_csc(epg.csc))
self.assertTrue(epg.csc.size == 0)
self.assertTrue(spsp.isspmatrix_bsr(epg.bsr))
self.assertTrue(epg.bsr.size == 0)
self.assertTrue(spsp.isspmatrix_dia(epg.dia))
self.assertTrue(epg.dia.size == 0)
self.compare_trajectories(traj, newtraj)
def test_get_all_not_links(self):
traj = Trajectory()
traj.f_add_parameter('test.hi', 44)
traj.f_explore({'hi': [1,2,3]})
traj.f_add_parameter_group('test.test.test2')
traj.f_add_parameter_group('test2')
traj.test2.f_add_link('test', traj.test)
nodes = traj.f_get_all('par.test')
self.assertTrue(len(nodes) == 2)
nodes = traj.f_get_all('par.test', shortcuts=False)
self.assertTrue(len(nodes) == 1)
traj.f_set_crun(0)
traj.f_add_group('f.$.h')
traj.f_add_group('f.$.g.h')
traj.f_add_group('f.$.i')
traj.crun.i.f_add_link('h', traj.crun.h)
nodes = traj.f_get_all('$.h')
self.assertTrue(len(nodes)==2)
nodes = traj.f_get_all('h')
self.assertTrue(len(nodes)==2)
traj.v_idx = -1
nodes = traj.f_get_all('h')
self.assertTrue(len(nodes)==2)
def test_get_all_not_links(self):
traj = Trajectory()
traj.f_add_parameter('test.hi', 44)
traj.f_explore({'hi': [1, 2, 3]})
traj.f_add_parameter_group('test.test.test2')
traj.f_add_parameter_group('test2')
traj.test2.f_add_link('test', traj.test)
nodes = traj.f_get_all('par.test')
self.assertTrue(len(nodes) == 2)
nodes = traj.f_get_all('par.test', shortcuts=False)
self.assertTrue(len(nodes) == 1)
traj.f_set_crun(0)
traj.f_add_group('f.$.h')
traj.f_add_group('f.$.g.h')
traj.f_add_group('f.$.i')
traj.crun.i.f_add_link('h', traj.crun.h)
nodes = traj.f_get_all('$.h')
self.assertTrue(len(nodes) == 2)
nodes = traj.f_get_all('h')
self.assertTrue(len(nodes) == 2)
traj.v_idx = -1
nodes = traj.f_get_all('h')
self.assertTrue(len(nodes) == 2)
def test_file_size_many_params(self):
filename = make_temp_dir('filesize.hdf5')
traj = Trajectory(filename=filename, overwrite_file=True, add_time=False)
npars = 700
traj.f_store()
for irun in range(npars):
par = traj.f_add_parameter('test.test%d' % irun, 42+irun, comment='duh!')
traj.f_store_item(par)
size = os.path.getsize(filename)
size_in_mb = size/1000000.
get_root_logger().info('Size is %sMB' % str(size_in_mb))
self.assertTrue(size_in_mb < 10.0, 'Size is %sMB > 10MB' % str(size_in_mb))
def test_file_size_many_params(self):
filename = make_temp_dir('filesize.hdf5')
traj = Trajectory(filename=filename, overwrite_file=True, add_time=False)
npars = 700
traj.f_store()
for irun in range(npars):
par = traj.f_add_parameter('test.test%d' % irun, 42+irun, comment='duh!')
traj.f_store_item(par)
size = os.path.getsize(filename)
size_in_mb = size/1000000.
get_root_logger().info('Size is %sMB' % str(size_in_mb))
self.assertTrue(size_in_mb < 10.0, 'Size is %sMB > 10MB' % str(size_in_mb))
def main():
"""Main function to protect the *entry point* of the program."""
# Load settings from file
settings_file = 'pypet_settings.pkl'
settings = load_obj(settings_file)
# Print settings dictionary
print('\nSettings dictionary:')
for key, value in settings.items():
print(key, ' : ', value)
print('\nParameters to explore:')
for key, value in settings.items():
if isinstance(value, list):
print(key, ' : ', value)
# Create new folder to store results
traj_dir = os.getcwd()
# Read output path (if provided)
if len(sys.argv) > 1:
# Add trailing slash if missing
dir_provided = os.path.join(sys.argv[1], '')
# Check if provided directory exists
if os.path.isdir(dir_provided):
# Convert to full path
traj_dir = os.path.abspath(dir_provided)
else:
print(
'WARNING: Output path not found, current directory will be used instead'
)
else:
print(
'WARNING: Output path not provided, current directory will be used instead'
)
# Add time stamp (the final '' is to make sure there is a trailing slash)
traj_dir = os.path.join(traj_dir,
datetime.now().strftime("%Y_%m_%d_%Hh%Mm%Ss"), '')
# Create directory with time stamp
os.makedirs(traj_dir)
# Change current directory to the one containing the trajectory files
os.chdir(traj_dir)
print('Trajectory and results will be stored in: {0}'.format(traj_dir))
# Create new pypet Trajectory object
traj_filename = 'traj.hdf5'
traj_fullpath = os.path.join(traj_dir, traj_filename)
traj = Trajectory(filename=traj_fullpath)
# -------------------------------------------------------------------
# Add config parameters (those that DO NOT influence the final result of the experiment)
traj.f_add_config('debug', False, comment='Activate debug mode')
# #traj.f_add_config('max_mem_frac', 0.7, comment='Fraction of global GPU memory to use')
# Set up trajectory parameters
param_to_explore = {}
for key, val in settings.items():
if isinstance(val, list):
param_to_explore[key] = val
traj.f_add_parameter(key, val[0])
else:
traj.f_add_parameter(key, val)
# Define parameter combinations to explore (a trajectory in
# the parameter space). The second argument, the tuple, specifies the order
# of the cartesian product.
# The variable on the right most side changes fastest and defines the
# 'inner for-loop' of the cartesian product
explore_dict = cartesian_product(param_to_explore,
tuple(param_to_explore.keys()))
print(explore_dict)
traj.f_explore(explore_dict)
# Store trajectory parameters to disk
pypet_utils.print_traj_leaves(traj,
'parameters',
file=os.path.join(traj_dir,
'traj_parameters.txt'))
# Store trajectory
traj.f_store()
# Define PBS script
bash_lines = '\n'.join([
'#! /bin/bash',
'#PBS -P InfoDynFuncStruct',
'#PBS -l select=1:ncpus=1:mem=1GB',
#'#PBS -l select=1:ncpus=1:ngpus=1:mem=1GB',
'#PBS -M [email protected]',
'#PBS -m abe',
'module load java',
'module load python/3.5.1',
'module load cuda/8.0.44',
'source /project/RDS-FEI-InfoDynFuncStruct-RW/Leo/idtxl_env/bin/activate',
'cd ${traj_dir}',
'python ${python_script_path} ${traj_dir} ${traj_filename} ${file_prefix} $PBS_ARRAY_INDEX'
])
# Save PBS script file (automatically generated)
bash_script_name = 'run_python_script.pbs'
job_script_path = os.path.join(traj_dir, bash_script_name)
with open(job_script_path, 'w', newline='\n') as bash_file:
bash_file.writelines(bash_lines)
# Run job array
job_walltime_hours = 0
job_walltime_minutes = 5
#after_job_array_ends = 1573895
job_settings = {
'N': 'run_traj',
'l': 'walltime={0}:{1}:00'.format(job_walltime_hours,
job_walltime_minutes),
#'W': 'depend=afteranyarray:{0}[]'.format(after_job_array_ends),
'q': 'defaultQ'
}
if len(traj.f_get_run_names()) > 1:
job_settings['J'] = '{0}-{1}'.format(0,
len(traj.f_get_run_names()) - 1)
job_args = {
'python_script_path':
'/project/RDS-FEI-InfoDynFuncStruct-RW/Leo/inference/hpc_pypet_single_run.py',
'traj_dir': traj_dir,
'traj_filename': traj_filename,
'file_prefix': 'none'
}
run_job_array(job_script_path, job_settings, job_args)
__author__ = 'Robert Meyer'
from pypet import Trajectory, Result, Parameter
traj = Trajectory()
# There are more ways to add data,
# 1st the standard way:
traj.f_add_parameter('x', 1, comment='I am the first dimension!')
# 2nd by providing a new parameter/result instance, be aware that the data is added where
# you specify it. There are no such things as shortcuts for parameter creation:
traj.parameters.y = Parameter('y', 1, comment='I am the second dimension!')
# 3rd as before, but if our new leaf has NO name it will be renamed accordingly:
traj.parameters.t = Parameter('', 1, comment='Third dimension')
# See:
print('t=' + str(traj.t))
# What happens if our new parameter's name does not match the name passed to the constructor?
traj.parameters.subgroup = Parameter('v', 2, comment='Fourth dimension')
# Well, since 'subgroup' != 'v', 'subgroup' becomes just another group node created on the fly
print(traj.parameters.subgroup)
# This even works for already existing groups and with the well known *dot* notation:
traj.parameters = Parameter('subgroup.subsubgroup.w', 2)
# See
print('w='+str(traj.par.subgroup.subsubgroup.w))
# There's a lazy version which does not require a constructor.
# This can be turned on via
traj.v_lazy_adding = True
def main():
# No environment here ;-)
filename = os.path.join('experiments', 'example_20.hdf5')
traj = Trajectory('onemax', filename=filename, overwrite_file=True)
# ------- Add parameters ------- #
traj.f_add_parameter('popsize', 100)
traj.f_add_parameter('CXPB', 0.5)
traj.f_add_parameter('MUTPB', 0.2)
traj.f_add_parameter('NGEN', 20)
traj.f_add_parameter('generation', 0)
traj.f_add_parameter('ind_idx', 0)
traj.f_add_parameter('ind_len', 50)
traj.f_add_parameter('indpb', 0.005)
traj.f_add_parameter('tournsize', 3)
traj.f_add_parameter('seed', 42)
traj.f_store(only_init=True)
# ------- Create and register functions with DEAP ------- #
creator.create("FitnessMax", base.Fitness, weights=(1.0, ))
creator.create("Individual", list, fitness=creator.FitnessMax)
toolbox = base.Toolbox()
# Attribute generator
toolbox.register("attr_bool", random.randint, 0, 1)
# Structure initializers
toolbox.register("individual", tools.initRepeat, creator.Individual,
toolbox.attr_bool, traj.ind_len)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
# Operator registering
toolbox.register("mate", tools.cxTwoPoint)
toolbox.register("mutate", tools.mutFlipBit, indpb=traj.indpb)
toolbox.register("select", tools.selTournament, tournsize=traj.tournsize)
toolbox.register("evaluate", eval_wrapper)
pool = multip.Pool(4)
toolbox.register("map", pool.map) # We use the pool's map function!
# ------- Initialize Population -------- #
random.seed(traj.seed)
pop = toolbox.population(n=traj.popsize)
CXPB, MUTPB, NGEN = traj.CXPB, traj.MUTPB, traj.NGEN
start_idx = 0 # We need to count executed runs
print("Start of evolution")
for g in range(traj.NGEN):
print("-- Generation %i --" % g)
# Determine individuals that need to be evaluated
eval_pop = [ind for ind in pop if not ind.fitness.valid]
# Add as many explored runs as individuals that need to be evaluated
traj.f_expand(
cartesian_product({
'generation': [g],
'ind_idx': range(len(eval_pop))
}))
# We need to make the storage service multiprocessing safe
mc = MultiprocContext(traj, wrap_mode='QUEUE')
mc.f_start()
# Create a single iterable to be passed to our fitness function (wrapper).
# `yields='copy'` is important, the pool's `map` function will
# go over the whole iterator at once and store it in memory.
# So for every run we need a copy of the trajectory.
# Alternatively, you could use `yields='self'` and use the pool's `imap` function.
zip_iterable = izip(traj.f_iter_runs(start_idx, yields='copy'),
eval_pop)
fitnesses = toolbox.map(eval_wrapper, zip_iterable)
# fitnesses is just a list of tuples [(fitness,), ...]
for idx, fitness in enumerate(fitnesses):
# Update fitnesses
eval_pop[idx].fitness.values = fitness
# Finalize the multiproc wrapper
mc.f_finalize()
# Update start index
start_idx += len(eval_pop)
print(" Evaluated %i individuals" % len(eval_pop))
# Gather all the fitnesses in one list and print the stats
fits = [ind.fitness.values[0] for ind in pop]
length = len(pop)
mean = sum(fits) / length
sum2 = sum(x * x for x in fits)
std = abs(sum2 / length - mean**2)**0.5
print(" Min %s" % min(fits))
print(" Max %s" % max(fits))
print(" Avg %s" % mean)
print(" Std %s" % std)
# ------- Create the next generation by crossover and mutation -------- #
if g < traj.NGEN - 1: # not necessary for the last generation
# Select the next generation individuals
offspring = toolbox.select(pop, len(pop))
# Clone the selected individuals
offspring = list(map(toolbox.clone, offspring))
# Apply crossover and mutation on the offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < MUTPB:
toolbox.mutate(mutant)
del mutant.fitness.values
# The population is entirely replaced by the offspring
pop[:] = offspring
# Stop the multiprocessing pool
pool.close()
pool.join()
print("-- End of (successful) evolution --")
best_ind = tools.selBest(pop, 1)[0]
print("Best individual is %s, %s" % (best_ind, best_ind.fitness.values))
traj.f_store() # And store all the rest of the data
def test_delete_whole_subtrees(self):
filename = make_temp_dir('testdeltree.hdf5')
traj = Trajectory(name='TestDelete',
filename=filename, large_overview_tables=True,
add_time=True)
res = traj.f_add_result('mytest.yourtest.test', a='b', c='d')
dpar = traj.f_add_derived_parameter('mmm.gr.dpdp', 666)
res = traj.f_add_result('hhh.ll', a='b', c='d')
res = traj.f_add_derived_parameter('hhh.gg', 555)
traj.f_store()
with pt.open_file(filename) as fh:
daroot = fh.root._f_get_child(traj.v_name)
dpar_table = daroot.overview.derived_parameters_overview
self.assertTrue(len(dpar_table) == 2)
res_table = daroot.overview.results_overview
self.assertTrue((len(res_table)) == 2)
with self.assertRaises(TypeError):
traj.f_remove_item(traj.yourtest)
with self.assertRaises(TypeError):
traj.f_delete_item(traj.yourtest)
traj.f_remove_item(traj.yourtest, recursive=True)
self.assertTrue('mytest' in traj)
self.assertTrue('yourtest' not in traj)
traj.f_load(load_data=2)
self.assertTrue('yourtest.test' in traj)
traj.f_delete_item(traj.yourtest, recursive=True, remove_from_trajectory=True)
traj.f_delete_item(traj.mmm, recursive=True, remove_from_trajectory=True)
traj.f_load(load_data=2)
self.assertTrue('yourtest.test' not in traj)
self.assertTrue('yourtest' not in traj)
with pt.open_file(filename) as fh:
daroot = fh.root._f_get_child(traj.v_name)
dpar_table = daroot.overview.derived_parameters_overview
self.assertTrue(len(dpar_table) == 2)
res_table = daroot.overview.results_overview
self.assertTrue((len(res_table)) == 2)
traj.f_add_parameter('ggg', 43)
traj.f_add_parameter('hhh.mmm', 45)
traj.f_add_parameter('jjj', 55)
traj.f_add_parameter('hhh.nnn', 55555)
traj.f_explore({'ggg':[1,2,3]})
traj.f_store()
with pt.open_file(filename) as fh:
daroot = fh.root._f_get_child(traj.v_name)
par_table = daroot.overview.parameters_overview
self.assertTrue(len(par_table) == 4)
traj.f_delete_item('par.hhh', recursive=True, remove_from_trajectory=True)
traj.f_add_parameter('saddsdfdsfd', 111)
traj.f_store()
with pt.open_file(filename) as fh:
daroot = fh.root._f_get_child(traj.v_name)
par_table = daroot.overview.parameters_overview
self.assertTrue(len(par_table) == 5)
__author__ = 'Robert Meyer'
import os # To allow pathnames under Windows and Linux
from pypet import Trajectory, NotUniqueNodeError
# We first generate a new Trajectory
filename = os.path.join('hdf5', 'example_02.hdf5')
traj = Trajectory('Example', filename=filename,
comment='Access and Storage!')
# We add our first parameter with the data 'Harrison Ford'
traj.f_add_parameter('starwars.characters.han_solo', 'Harrison Ford')
# This automatically added the groups 'starwars' and the subgroup 'characters'
# Let's get the characters subgroup
characters = traj.parameters.starwars.characters
# Since characters is unique we could also use shortcuts
characters = traj.characters
# Or the get method
characters = traj.f_get('characters')
# Or square brackets
characters = traj['characters']
# Lets add another character
characters.f_add_parameter('luke_skywalker', 'Mark Hamill', comment='May the force be with you!')
__author__ = 'Robert Meyer'
import os # To allow pathnames under Windows and Linux
from pypet import Trajectory, NotUniqueNodeError
# We first generate a new Trajectory
filename = os.path.join('hdf5', 'example_02.hdf5')
traj = Trajectory('Example',
filename=filename,
overwrite_file=True,
comment='Access and Storage!')
# We add our first parameter with the data 'Harrison Ford'
traj.f_add_parameter('starwars.characters.han_solo', 'Harrison Ford')
# This automatically added the groups 'starwars' and the subgroup 'characters'
# Let's get the characters subgroup
characters = traj.parameters.starwars.characters
# Since characters is unique we could also use shortcuts
characters = traj.characters
# Or the get method
characters = traj.f_get('characters')
# Or square brackets
characters = traj['characters']
# Lets add another character
characters.f_add_parameter('luke_skywalker',
__author__ = 'Robert Meyer'
from pypet import Trajectory, Result, Parameter
traj = Trajectory()
# There are more ways to add data,
# 1st the standard way:
traj.f_add_parameter('x', 1, comment='I am the first dimension!')
# 2nd by providing a new parameter/result instance, be aware that the data is added where
# you specify it. There are no such things as shortcuts for parameter creation:
traj.parameters.y = Parameter('y', 1, comment='I am the second dimension!')
# 3rd as before, but if our new leaf has NO name it will be renamed accordingly:
traj.parameters.t = Parameter('', 1, comment='Third dimension')
# See:
print('t=' + str(traj.t))
# This also works for adding groups on the fly and with the well known *dot* notation:
traj.parameters.subgroup = Parameter('subgroup.subsubgroup.w', 2)
# See
print('w=' + str(traj.par.subgroup.subsubgroup.w))
# Finally, there's one more thing. Using this notation we can also add links.
# Simply use the `=` assignment with objects that already exist in your trajectory:
traj.mylink = traj.f_get('x')
# now `mylink` links to parameter `x`, also fast access works:
print('Linking to x gives: ' + str(traj.mylink))
def test_delete_whole_subtrees(self):
filename = make_temp_dir('testdeltree.hdf5')
traj = Trajectory(name='TestDelete',
filename=filename, large_overview_tables=True)
res = traj.f_add_result('mytest.yourtest.test', a='b', c='d')
dpar = traj.f_add_derived_parameter('mmm.gr.dpdp', 666)
res = traj.f_add_result('hhh.ll', a='b', c='d')
res = traj.f_add_derived_parameter('hhh.gg', 555)
traj.f_store()
with ptcompat.open_file(filename) as fh:
daroot = ptcompat.get_child(fh.root, traj.v_name)
dpar_table = daroot.overview.derived_parameters_overview
self.assertTrue(len(dpar_table) == 2)
res_table = daroot.overview.results_overview
self.assertTrue((len(res_table)) == 2)
with self.assertRaises(TypeError):
traj.f_remove_item(traj.yourtest)
with self.assertRaises(TypeError):
traj.f_delete_item(traj.yourtest)
traj.f_remove_item(traj.yourtest, recursive=True)
self.assertTrue('mytest' in traj)
self.assertTrue('yourtest' not in traj)
traj.f_load(load_data=2)
self.assertTrue('yourtest.test' in traj)
traj.f_delete_item(traj.yourtest, recursive=True, remove_from_trajectory=True)
traj.f_delete_item(traj.mmm, recursive=True, remove_from_trajectory=True)
traj.f_load(load_data=2)
self.assertTrue('yourtest.test' not in traj)
self.assertTrue('yourtest' not in traj)
with ptcompat.open_file(filename) as fh:
daroot = ptcompat.get_child(fh.root, traj.v_name)
dpar_table = daroot.overview.derived_parameters_overview
self.assertTrue(len(dpar_table) == 2)
res_table = daroot.overview.results_overview
self.assertTrue((len(res_table)) == 2)
traj.f_add_parameter('ggg', 43)
traj.f_add_parameter('hhh.mmm', 45)
traj.f_add_parameter('jjj', 55)
traj.f_add_parameter('hhh.nnn', 55555)
traj.f_explore({'ggg':[1,2,3]})
traj.f_store()
with ptcompat.open_file(filename) as fh:
daroot = ptcompat.get_child(fh.root, traj.v_name)
par_table = daroot.overview.parameters_overview
self.assertTrue(len(par_table) == 4)
traj.f_delete_item('par.hhh', recursive=True, remove_from_trajectory=True)
traj.f_add_parameter('saddsdfdsfd', 111)
traj.f_store()
with ptcompat.open_file(filename) as fh:
daroot = ptcompat.get_child(fh.root, traj.v_name)
par_table = daroot.overview.parameters_overview
self.assertTrue(len(par_table) == 5)
# with self.assertRaises(TypeError):
# # We cannot delete something containing an explored parameter
# traj.f_delete_item('par', recursive=True)
with self.assertRaises(TypeError):
traj.f_delete_item('ggg')
def main():
# No environment here ;-)
filename = os.path.join('experiments', 'example_20.hdf5')
traj = Trajectory('onemax', filename=filename, overwrite_file=True)
# ------- Add parameters ------- #
traj.f_add_parameter('popsize', 100)
traj.f_add_parameter('CXPB', 0.5)
traj.f_add_parameter('MUTPB', 0.2)
traj.f_add_parameter('NGEN', 20)
traj.f_add_parameter('generation', 0)
traj.f_add_parameter('ind_idx', 0)
traj.f_add_parameter('ind_len', 50)
traj.f_add_parameter('indpb', 0.005)
traj.f_add_parameter('tournsize', 3)
traj.f_add_parameter('seed', 42)
traj.f_store(only_init=True)
# ------- Create and register functions with DEAP ------- #
creator.create("FitnessMax", base.Fitness, weights=(1.0,))
creator.create("Individual", list, fitness=creator.FitnessMax)
toolbox = base.Toolbox()
# Attribute generator
toolbox.register("attr_bool", random.randint, 0, 1)
# Structure initializers
toolbox.register("individual", tools.initRepeat, creator.Individual,
toolbox.attr_bool, traj.ind_len)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
# Operator registering
toolbox.register("mate", tools.cxTwoPoint)
toolbox.register("mutate", tools.mutFlipBit, indpb=traj.indpb)
toolbox.register("select", tools.selTournament, tournsize=traj.tournsize)
toolbox.register("evaluate", eval_wrapper)
pool = multip.Pool(4)
toolbox.register("map", pool.map) # We use the pool's map function!
# ------- Initialize Population -------- #
random.seed(traj.seed)
pop = toolbox.population(n=traj.popsize)
CXPB, MUTPB, NGEN = traj.CXPB, traj.MUTPB, traj.NGEN
start_idx = 0 # We need to count executed runs
print("Start of evolution")
for g in range(traj.NGEN):
print("-- Generation %i --" % g)
# Determine individuals that need to be evaluated
eval_pop = [ind for ind in pop if not ind.fitness.valid]
# Add as many explored runs as individuals that need to be evaluated
traj.f_expand(cartesian_product({'generation': [g], 'ind_idx': range(len(eval_pop))}))
# We need to make the storage service multiprocessing safe
mc = MultiprocContext(traj, wrap_mode='QUEUE')
mc.f_start()
# Create a single iterable to be passed to our fitness function (wrapper).
# `yields='copy'` is important, the pool's `map` function will
# go over the whole iterator at once and store it in memory.
# So for every run we need a copy of the trajectory.
# Alternatively, you could use `yields='self'` and use the pool's `imap` function.
zip_iterable = izip(traj.f_iter_runs(start_idx, yields='copy'), eval_pop)
fitnesses = toolbox.map(eval_wrapper, zip_iterable)
# fitnesses is just a list of tuples [(fitness,), ...]
for idx, fitness in enumerate(fitnesses):
# Update fitnesses
eval_pop[idx].fitness.values = fitness
# Finalize the multiproc wrapper
mc.f_finalize()
# Update start index
start_idx += len(eval_pop)
print(" Evaluated %i individuals" % len(eval_pop))
# Gather all the fitnesses in one list and print the stats
fits = [ind.fitness.values[0] for ind in pop]
length = len(pop)
mean = sum(fits) / length
sum2 = sum(x*x for x in fits)
std = abs(sum2 / length - mean**2)**0.5
print(" Min %s" % min(fits))
print(" Max %s" % max(fits))
print(" Avg %s" % mean)
print(" Std %s" % std)
# ------- Create the next generation by crossover and mutation -------- #
if g < traj.NGEN -1: # not necessary for the last generation
# Select the next generation individuals
offspring = toolbox.select(pop, len(pop))
# Clone the selected individuals
offspring = list(map(toolbox.clone, offspring))
# Apply crossover and mutation on the offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
if random.random() < CXPB:
toolbox.mate(child1, child2)
del child1.fitness.values
del child2.fitness.values
for mutant in offspring:
if random.random() < MUTPB:
toolbox.mutate(mutant)
del mutant.fitness.values
# The population is entirely replaced by the offspring
pop[:] = offspring
# Stop the multiprocessing pool
pool.close()
pool.join()
print("-- End of (successful) evolution --")
best_ind = tools.selBest(pop, 1)[0]
print("Best individual is %s, %s" % (best_ind, best_ind.fitness.values))
traj.f_store() # And store all the rest of the data
