def setUp(self):
logging.basicConfig(level = logging.INFO)
traj = Trajectory('Test')
traj.v_storage_service = LazyStorageService()
large_amount = 111
for irun in range(large_amount):
name = 'There.Are.Many.Parameters.Like.Me' + str(irun)
traj.f_add_parameter(name, irun)
traj.f_add_parameter('TestExplorer', 1)
traj.v_fast_access=False
traj.f_explore({traj.TestExplorer.v_full_name:[1,2,3,4,5]})
traj.v_fast_access=True
self.traj = traj
self.n = 1
self.single_run = self.traj._make_single_run(self.n)
self.assertTrue(len(self.single_run)==1)
def test_store_items_and_groups(self):
traj = Trajectory(name='testtraj', filename=make_temp_file('teststoreitems.hdf5'))
traj.f_store()
traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj.f_store_items(['test','testres','group1'])
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_file('teststoreitems.hdf5'))
traj2.f_load(load_results=2,load_parameters=2)
traj.f_add_result('Im.stored.along.a.path', 43)
traj.Im.stored.along.v_annotations['wtf'] =4444
traj.res.f_store_child('Im.stored.along.a.path')
traj2.res.f_load_child('Im.stored.along.a.path', load_data=2)
self.compare_trajectories(traj,traj2)
def test_f_is_completed(self):
traj = Trajectory()
traj.f_add_parameter('test', 42)
traj.f_explore({'test':[1,2,3,4]})
self.assertFalse(traj.f_is_completed())
for run_name in traj.f_get_run_names():
self.assertFalse(traj.f_is_completed(run_name))
traj._run_information[traj.f_idx_to_run(1)]['completed']=1
self.assertFalse(traj.f_is_completed())
self.assertTrue(traj.f_is_completed(1))
for run_name in traj.f_get_run_names():
traj._run_information[run_name]['completed']=1
self.assertTrue(traj.f_is_completed())
for run_name in traj.f_get_run_names():
self.assertTrue(traj.f_is_completed(run_name))
def setUp(self):
name = 'Traj'
traj = Trajectory(name)
traj.f_add_parameter('x',0)
traj.f_add_parameter('y',0.0)
traj.f_add_parameter('z','test')
traj.f_add_parameter('ar', np.array([1,2,3]))
traj.f_add_parameter('scalar', 42)
self.explore(traj)
self.traj=traj
def test_store_load_with_hdf5(self):
traj_name = 'test_%s' % self.__class__.__name__
filename = make_temp_dir(traj_name + '.hdf5')
traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename, overwrite_file=True)
for param in self.param.values():
traj.f_add_parameter(param)
traj.f_store()
new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename)
new_traj.f_load(load_data=2)
self.compare_trajectories(traj, new_traj)
def test_not_adding_pars_during_single_run(self):
traj = Trajectory()
traj._is_run = True
with self.assertRaises(TypeError):
traj.f_add_parameter('dd')
with self.assertRaises(TypeError):
traj.f_add_parameter_group('dd')
with self.assertRaises(TypeError):
traj.f_add_config('dd')
with self.assertRaises(TypeError):
traj.f_add_config_group('dd')
def test_store_load_with_hdf5(self):
traj_name = 'test_%s' % self.__class__.__name__
filename = make_temp_dir(traj_name + '.hdf5')
traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename, overwrite_file=True)
for param in self.param.values():
traj.f_add_parameter(param)
traj.f_store()
new_traj = Trajectory(name=traj_name, dynamic_imports=self.dynamic_imports,
filename = filename)
new_traj.f_load(load_data=2)
self.compare_trajectories(traj, new_traj)
def prepare():
logging.basicConfig(level = logging.INFO)
traj = Trajectory('Test')
traj.v_storage_service=LazyStorageService()
large_amount = 11111
for irun in range(large_amount):
name = 'Hurz.L' + str(irun)
traj.f_add_parameter(name,irun)
traj.f_add_parameter('Hurz.Test', data=1)
return traj
def test_file_translation(self):
filename = make_temp_dir('to_new_tree.hdf5')
mytraj = Trajectory('SCRATCH', filename=filename)
mytraj.f_add_parameter('Test.Group.Test', 42)
mytraj.f_add_derived_parameter('trajectory.saaaa', 33)
mytraj.f_add_derived_parameter('trajectory.intraj.dpar1', 33)
mytraj.f_add_derived_parameter('run_00000008.inrun.dpar2', 33)
mytraj.f_add_derived_parameter('run_00000001.inrun.dpar3', 35)
mytraj.f_add_result('trajectory.intraj.res1', 33)
mytraj.f_add_result('run_00000008.inrun.res1', 33)
mytraj.f_store()
mytraj.f_migrate(new_name=mytraj.v_name + 'PETER', in_store=True)
mytraj.f_store()
fu = FileUpdater(filename=filename, backup=True)
fu.update_file()
mytraj = Trajectory(name=mytraj.v_name,
add_time=False,
filename=filename)
mytraj.f_load(load_parameters=2,
load_derived_parameters=2,
load_results=2)
for node in mytraj.f_iter_nodes():
self.assertTrue(node.v_name != 'trajectory')
if 'run_' in node.v_full_name:
self.assertTrue('.runs.' in node.v_full_name)
remove_data()
def test_wildcard_search(self):
traj = Trajectory(name='Test', filename=make_temp_file('wilcard.hdf5'))
traj.f_add_parameter('expl', 2)
traj.f_explore({'expl':[1,2,3,4]})
traj.f_add_result('wc2test.$.hhh', 333)
traj.f_add_leaf('results.wctest.run_00000000.jjj', 42)
traj.f_add_result('results.wctest.run_00000001.jjj', 43)
traj.v_as_run = 1
self.assertTrue(traj.results.wctest['$'].jjj==43)
self.assertTrue(traj.results.wc2test.crun.hhh==333)
traj.f_store()
traj.f_remove_child('results', recursive=True)
traj.v_auto_load = True
self.assertTrue(traj.results.wctest['$'].jjj==43)
self.assertTrue(traj.results.wc2test.crun.hhh==333)
traj.f_remove_child('results', recursive=True)
traj.v_auto_load = True
self.assertTrue(traj.results.wctest[-2].jjj==43)
self.assertTrue(traj.results.wc2test[-2].hhh==333)
traj.f_remove_child('results', recursive=True)
traj.v_auto_load = True
self.assertTrue(traj.results.wctest[1].jjj==43)
self.assertTrue(traj.results.wc2test[-1].hhh==333)
def test_file_translation(self):
filename = make_temp_dir('to_new_tree.hdf5')
mytraj = Trajectory('SCRATCH', filename=filename)
mytraj.f_add_parameter('Test.Group.Test', 42)
mytraj.f_add_derived_parameter('trajectory.saaaa',33)
mytraj.f_add_derived_parameter('trajectory.intraj.dpar1',33)
mytraj.f_add_derived_parameter('run_00000008.inrun.dpar2',33)
mytraj.f_add_derived_parameter('run_00000001.inrun.dpar3',35)
mytraj.f_add_result('trajectory.intraj.res1',33)
mytraj.f_add_result('run_00000008.inrun.res1',33)
mytraj.f_store()
mytraj.f_migrate(new_name=mytraj.v_name + 'PETER', in_store=True)
mytraj.f_store()
fu=FileUpdater(filename=filename, backup=True)
fu.update_file()
mytraj = Trajectory(name=mytraj.v_name, add_time=False, filename=filename)
mytraj.f_load(load_parameters=2, load_derived_parameters=2, load_results=2)
for node in mytraj.f_iter_nodes():
self.assertTrue(node.v_name != 'trajectory')
if 'run_' in node.v_full_name:
self.assertTrue('.runs.' in node.v_full_name)
remove_data()
def test_version_mismatch(self):
traj = Trajectory(name='Test', filename=make_temp_file('testversionmismatch.hdf5'))
traj.f_add_parameter('group1.test',42)
traj.f_add_result('testres', 42)
traj.group1.f_set_annotations(Test=44)
traj._version='0.1a.1'
traj.f_store()
traj2 = Trajectory(name=traj.v_name, add_time=False,
filename=make_temp_file('testversionmismatch.hdf5'))
with self.assertRaises(pex.VersionMismatchError):
traj2.f_load(load_results=2,load_parameters=2)
traj2.f_load(load_results=2,load_parameters=2, force=True)
self.compare_trajectories(traj,traj2)
class AnnotationsTest(unittest.TestCase):
tags = 'unittest', 'annotations'
def setUp(self):
self.filename = make_temp_dir(
os.path.join('experiments', 'tests', 'HDF5', 'annotations.hdf5'))
self.traj = Trajectory(name='Annotations', filename=self.filename)
self.traj.f_add_result('testres', 42)
self.traj.f_add_parameter('testgroup.testparam', 42)
self.make_annotations()
self.add_annotations(self.traj)
pred = lambda x: 'config' not in x.v_full_name
x = len([
node
for node in self.traj.f_iter_nodes(recursive=True, predicate=pred)
])
self.assertTrue(x == 5, '%s != %s' % (str(x), str(5)))
def tearDown(self):
remove_data()
def make_annotations(self):
self.annotations = {}
self.annotations['dict'] = {
'33': 12,
'kkk': [1, 2, 'h'],
3: {
'a': 42.0
}
}
self.annotations['list'] = [self.annotations['dict'], 33]
self.annotations['string'] = 'string'
self.annotations['integer'] = 42
self.annotations['tuple'] = (3, 4, 5)
self.annotations['Numpy_Data'] = np.array(['fff', 'ddd'])
self.annotations[0] = 7777
def add_annotations(self, traj):
funcs = 5
for idx, node in enumerate(
[traj] + [node for node in traj.f_iter_nodes(recursive=True)]):
for name in self.annotations:
anno = self.annotations[name]
if name == 0:
node.f_set_annotations(anno)
node.v_annotations.f_set(anno)
elif idx % funcs == 0:
node.f_set_annotations(**{name: anno})
elif idx % funcs == 1:
node.v_annotations.f_set(**{name: anno})
elif idx % funcs == 2:
node.v_annotations.f_set_single(name, anno)
elif idx % funcs == 3:
setattr(node.v_annotations, name, anno)
elif idx % funcs == 4:
node.v_annotations[name] = anno
def test_annotations_insert(self):
for idx,node in \
enumerate([self.traj] + [node for node in self.traj.f_iter_nodes(recursive=True)]):
for name in self.annotations:
anno = self.annotations[name]
node_anno = node.v_annotations[name]
self.assertTrue(comp.nested_equal(anno, node_anno),
'%s != %s' % (str(anno), str(node_anno)))
def test_pickling(self):
dump = pickle.dumps(self.traj)
del self.traj
self.traj = pickle.loads(dump)
self.test_annotations_insert()
def test_storage_and_loading(self):
self.traj.f_store()
traj_name = self.traj.v_name
del self.traj
self.traj = Trajectory(filename=self.filename)
self.traj.f_load(name=traj_name,
load_parameters=2,
load_derived_parameters=2,
load_results=2,
load_other_data=2)
self.test_annotations_insert()
def test_attribute_deletion(self):
for node in self.traj.f_iter_nodes(recursive=True):
name_list = [name for name in node.v_annotations]
for name in name_list:
delattr(node.v_annotations, name)
self.assertTrue(node.v_annotations.f_is_empty())
def test_item_deletion(self):
for node in self.traj.f_iter_nodes(recursive=True):
name_list = [name for name in node.v_annotations]
for name in name_list:
del node.v_annotations[name]
self.assertTrue(node.v_annotations.f_is_empty())
def test_get_item(self):
for node in self.traj.f_iter_nodes(recursive=True):
for key, val1 in node.v_annotations.f_to_dict().items():
val2 = node.v_annotations[key]
self.assertTrue(comp.nested_equal(val1, val2))
def test_get_item_no_copy(self):
for node in self.traj.f_iter_nodes(recursive=True):
for key, val1 in node.v_annotations.f_to_dict(copy=False).items():
val2 = node.v_annotations[key]
self.assertTrue(comp.nested_equal(val1, val2))
@staticmethod
def dict_to_str(dictionary):
resstr = ''
new_dict = {}
for key, val in dictionary.items():
if key == 0:
key = 'annotation'
new_dict[key] = val
for key in sorted(new_dict.keys()):
resstr += '%s=%s; ' % (key, str(new_dict[key]))
return resstr[:-2]
def test_to_str(self):
dict_str = self.dict_to_str(self.annotations)
for node in self.traj.f_iter_nodes(recursive=True):
ann_str = node.f_ann_to_str()
self.assertTrue(not ann_str.endswith(' ')
or not ann_str.endswith(','))
for name in self.annotations:
if name == 0:
name = 'annotation'
self.assertTrue(name in ann_str)
self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))
ann_str = str(node.v_annotations)
self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))
def test_single_get_and_getattr_and_setattr(self):
self.traj.f_add_parameter('test2', 42)
self.traj.f_get('test2').v_annotations.test = 4
self.assertTrue(self.traj.f_get('test2').v_annotations.test, 4)
self.assertTrue(self.traj.f_get('test2').v_annotations.f_get(), 4)
def test_get_annotations(self):
key_list = list(self.annotations.keys())
for node in self.traj.f_iter_nodes(recursive=True):
for name in self.annotations:
self.assertTrue(
comp.nested_equal(self.annotations[name],
node.f_get_annotations(name)))
val_list = node.f_get_annotations(*key_list)
for idx, val in enumerate(val_list):
self.assertTrue(
comp.nested_equal(self.annotations[key_list[idx]], val))
def test_f_get_errors(self):
for node in self.traj.f_iter_nodes(recursive=True):
with self.assertRaises(ValueError):
node.v_annotations.f_get()
with self.assertRaises(AttributeError):
node.v_annotations.f_get('gdsdfd')
testparam = self.traj.f_add_parameter('ggg', 343)
with self.assertRaises(AttributeError):
testparam.v_annotations.f_get()
def test_f_set_numbering(self):
int_list = list(range(10))
for node in self.traj.f_iter_nodes(recursive=True):
node.v_annotations.f_set(*int_list)
self.assertEqual(node.v_annotations.f_get(*int_list),
tuple(int_list))
for integer in int_list:
if integer == 0:
name = 'annotation'
else:
name = 'annotation_%d' % integer
self.assertTrue(name in node.v_annotations)
class AnnotationsTest(unittest.TestCase):
def setUp(self):
self.filename = make_temp_file('experiments/tests/HDF5/annotations.hdf5')
self.traj = Trajectory(name='Annotations', filename = self.filename)
self.traj.f_add_result('testres', 42)
self.traj.f_add_parameter('testgroup.testparam', 42)
self.make_annotations()
self.add_annotations(self.traj)
self.assertTrue(len([node for node in self.traj.f_iter_nodes(recursive=True)]) == 5)
def make_annotations(self):
self.annotations={}
self.annotations['dict']={'33':12,'kkk':[1,2,'h'], 3:{'a':42.0}}
self.annotations['list']= [self.annotations['dict'],33]
self.annotations['string'] = 'string'
self.annotations['integer'] = 42
self.annotations['tuple']=(3,4,5)
self.annotations['Numpy_Data'] = np.array(['fff','ddd'])
self.annotations[0] = 7777
def add_annotations(self, traj):
funcs = 5
for idx,node in enumerate(traj.f_iter_nodes(recursive=True)):
for name in self.annotations:
anno = self.annotations[name]
if name == 0:
node.f_set_annotations(anno)
node.v_annotations.f_set(anno)
elif idx % funcs == 0:
node.f_set_annotations(**{name:anno})
elif idx % funcs == 1:
node.v_annotations.f_set(**{name:anno})
elif idx % funcs == 2:
node.v_annotations.f_set_single(name,anno)
elif idx % funcs == 3:
setattr(node.v_annotations,name, anno)
elif idx % funcs == 4:
node.v_annotations[name]=anno
def test_annotations_insert(self):
for node in self.traj.f_iter_nodes(recursive=True):
for name in self.annotations:
anno = self.annotations[name]
node_anno = node.v_annotations[name]
self.assertTrue(comp.nested_equal(anno, node_anno),
'%s != %s' % (str(anno), str(node_anno)))
def test_pickling(self):
dump = pickle.dumps(self.traj)
del self.traj
self.traj = pickle.loads(dump)
self.test_annotations_insert()
def test_storage_and_loading(self):
self.traj.f_store()
traj_name = self.traj.v_name
del self.traj
self.traj = Trajectory(filename=self.filename)
self.traj.f_load(name=traj_name, load_results=2, load_parameters=2, load_derived_parameters=2,
load_other_data=2)
self.test_annotations_insert()
def test_attribute_deletion(self):
for node in self.traj.f_iter_nodes(recursive=True):
name_list=[name for name in node.v_annotations]
for name in name_list:
delattr(node.v_annotations, name)
self.assertTrue(node.v_annotations.f_is_empty())
def test_get_item(self):
for node in self.traj.f_iter_nodes(recursive=True):
for key, val1 in node.v_annotations.f_to_dict().items():
val2 = node.v_annotations[key]
self.assertTrue(comp.nested_equal(val1,val2))
def test_get_item_no_copy(self):
for node in self.traj.f_iter_nodes(recursive=True):
for key, val1 in node.v_annotations.f_to_dict(copy=False).items():
val2 = node.v_annotations[key]
self.assertTrue(comp.nested_equal(val1,val2))
@staticmethod
def dict_to_str(dictionary):
resstr = ''
new_dict={}
for key, val in dictionary.items():
if key == 0:
key = 'annotation'
new_dict[key]=val
for key in sorted(new_dict.keys()):
resstr+='%s=%s; ' % (key,str(new_dict[key]))
return resstr[:-2]
def test_to_str(self):
dict_str = self.dict_to_str(self.annotations)
for node in self.traj.f_iter_nodes(recursive=True):
ann_str = node.f_ann_to_str()
self.assertTrue(not ann_str.endswith(' ') or not ann_str.endswith(','))
for name in self.annotations:
if name==0:
name = 'annotation'
self.assertTrue(name in ann_str)
self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))
ann_str = str(node.v_annotations)
self.assertEqual(dict_str, ann_str, '%s!=%s' % (dict_str, ann_str))
def test_single_get_and_getattr_and_setattr(self):
self.traj.f_add_parameter('test2', 42)
self.traj.f_get('test2').v_annotations.test = 4
self.assertTrue(self.traj.f_get('test2').v_annotations.test, 4)
self.assertTrue(self.traj.f_get('test2').v_annotations.f_get(), 4)
def test_get_annotations(self):
key_list = sorted(self.annotations.keys())
for node in self.traj.f_iter_nodes(recursive=True):
for name in self.annotations:
self.assertTrue(comp.nested_equal(self.annotations[name],
node.f_get_annotations(name)))
val_list = node.f_get_annotations(*key_list)
for idx, val in enumerate(val_list):
self.assertTrue(comp.nested_equal(self.annotations[key_list[idx]], val))
def test_f_get_errors(self):
for node in self.traj.f_iter_nodes(recursive=True):
with self.assertRaises(ValueError):
node.v_annotations.f_get()
with self.assertRaises(AttributeError):
node.v_annotations.f_get('gdsdfd')
testparam = self.traj.f_add_parameter('ggg',343)
with self.assertRaises(AttributeError):
testparam.v_annotations.f_get()
def test_f_set_numbering(self):
int_list = range(10)
for node in self.traj.f_iter_nodes(recursive=True):
node.v_annotations.f_set(*int_list)
self.assertEqual(node.v_annotations.f_get(*int_list), tuple(int_list))
for integer in int_list:
if integer == 0:
name = 'annotation'
else:
name = 'annotation_%d' % integer
self.assertTrue(name in node.v_annotations)
class TrajectoryMergeTest(unittest.TestCase):
def setUp(self):
name = 'Moop'
self.traj = Trajectory(name,[ImAParameterInDisguise])
comment = 'This is a comment'
self.traj.v_comment=comment
self.assertTrue(comment == self.traj.v_comment)
self.traj.f_add_parameter('IntParam',3)
sparsemat = spsp.csr_matrix((1000,1000))
sparsemat[1,2] = 17.777
#self.traj.f_add_parameter('SparseParam', sparsemat, param_type=PickleParameter)
self.traj.f_add_parameter('FloatParam')
self.traj.f_add_derived_parameter(Parameter('FortyTwo', 42))
self.traj.f_add_parameter('Trials',0)
self.traj.f_add_result(Result,'Im.A.Simple.Result',44444)
self.traj.FloatParam=4.0
self.traj.v_storage_service = LazyStorageService()
self.traj.f_explore({'FloatParam':[1.0,1.1,1.2,1.3],'Trials':[0,1,2,3]})
self.assertTrue(len(self.traj) == 4)
name2 = 'aaaaah'
self.traj2 = Trajectory(name2,[ImAParameterInDisguise])
comment = 'This is a comment'
self.traj2.v_comment=comment
self.assertTrue(comment == self.traj2.v_comment)
self.traj2.f_add_parameter('IntParam',3)
sparsemat = spsp.csr_matrix((1000,1000))
sparsemat[1,2] = 17.777
#self.traj2.f_add_parameter('SparseParam', sparsemat, param_type=PickleParameter)
self.traj2.f_add_parameter('Trials',0)
self.traj2.f_add_parameter('FloatParam')
self.traj2.f_add_derived_parameter(Parameter('FortyTwo', 42))
self.traj2.f_add_result(Result,'Im.A.Simple.Result',44444)
self.traj2.FloatParam=4.0
self.traj2.f_explore({'FloatParam':[42.0,43.0,1.2,1.3],'Trials':[0,1,2,3]})
self.traj2.v_storage_service = LazyStorageService()
self.assertTrue(len(self.traj2) == 4)
def test_merge_parameters_without_remove(self):
# remove_duplicates = True should be discarded by the trial parameter
self.traj._merge_parameters(self.traj2, trial_parameter_name='Trials',remove_duplicates=True)
def test_merge_parameters_with_remove(self):
self.traj._merge_parameters(self.traj2,remove_duplicates=True)
def test_merge_without_remove(self):
self.traj.f_merge(self.traj2, remove_duplicates=True,trial_parameter='Trials')
def test_merge_with_remove(self):
self.traj.f_merge(self.traj2, remove_duplicates=True)
class TrajectoryTest(unittest.TestCase):
def setUp(self):
name = 'Moop'
self.traj = Trajectory(name,dynamically_imported_classes=[ImAParameterInDisguise,
'pypet.tests.test_helpers.ImAResultInDisguise'])
self.assertTrue(self.traj.f_is_empty())
comment = 'This is a comment'
self.traj.v_comment=comment
self.assertTrue(comment == self.traj.v_comment)
self.traj.f_add_parameter('IntParam',3)
sparsemat = spsp.csr_matrix((1000,1000))
sparsemat[1,2] = 17.777
self.traj.f_add_parameter(PickleParameter,'SparseParam', sparsemat)
self.traj.f_add_parameter('FloatParam')
self.traj.f_add_derived_parameter(Parameter('FortyTwo', 42))
self.traj.f_add_result(Result,'Im.A.Simple.Result',44444)
self.traj.FloatParam=4.0
self.explore_dict = {'FloatParam':[1.0,1.1,1.2,1.3]}
self.traj.f_explore(self.explore_dict)
self.assertTrue(len(self.traj) == 4)
self.traj.f_add_parameter_group('peter.paul')
self.traj.f_add_parameter('peter.markus.yve',6)
self.traj.f_add_result('Test',42)
self.traj.peter.f_add_parameter('paul.peter')
self.traj.f_add_config('make.impossible.promises',1)
with self.assertRaises(AttributeError):
self.traj.markus.peter
with self.assertRaises(ValueError):
self.traj.f_add_parameter('Peter. h ._hurz')
def test_shrink(self):
self.assertTrue(len(self.traj)>1)
self.traj.f_shrink()
self.assertTrue(len(self.traj)==1)
self.assertTrue(len(self.traj.f_get_explored_parameters())==0)
def test_get_all(self):
all_nodes = self.traj.f_get_all('peter')
self.assertTrue(len(all_nodes)==2)
all_nodes = self.traj.f_get_all('peter.yve')
self.assertTrue(len(all_nodes)==1)
all_nodes = self.traj.f_get_all('paul.yve')
self.assertTrue(len(all_nodes)==0)
self.traj.f_add_parameter('paul.paul')
all_nodes = self.traj.peter.f_get_all('paul')
self.assertTrue(len(all_nodes)==1)
self.traj.f_add_result('results.runs.run_00000000.x.y',10)
self.traj.f_add_result('results.runs.run_00000001.x.y',10)
self.traj.f_add_derived_parameter('hfhfhf.x.y')
self.traj.f_add_result('x.y.y')
self.traj.f_as_run(1)
all_nodes=self.traj.f_get_all('x.y')
self.assertTrue(len(all_nodes)==4)
def test_backwards_search(self):
x=self.traj.peter.f_get('paul.peter', backwards_search=False)
y=self.traj.f_get('peter.peter', backwards_search=True)
self.assertTrue(x is y)
#
# def test_value_error_on_search_strategy_assignment(self):
# with self.assertRaises(ValueError):
# self.traj.v_search_strategy = 'ewnforenfre'
def test_get_data_dictionaries_directly(self):
############## Cofig ###################
config_dict_from_subtree = self.traj.config.f_to_dict()
self.assertTrue(len(config_dict_from_subtree)>0)
config_dict_directly = self.traj.f_get_config(copy=True)
self.assertTrue(comp.nested_equal(config_dict_directly,config_dict_from_subtree),
'%s!=%s' % (str(config_dict_directly),str(config_dict_directly)))
config_dict_directly = self.traj.f_get_config(copy=False)
self.assertTrue(comp.nested_equal(config_dict_directly,config_dict_from_subtree),
'%s!=%s' % (str(config_dict_directly),str(config_dict_directly)))
config_dict_from_subtree = self.traj.config.f_to_dict(fast_access=True)
with self.assertRaises(ValueError):
config_dict_directly = self.traj.f_get_config(copy=False, fast_access=True)
config_dict_directly = self.traj.f_get_config(copy=True, fast_access=True)
self.assertTrue(comp.nested_equal(config_dict_directly,config_dict_from_subtree),
'%s!=%s' % (str(config_dict_directly),str(config_dict_directly)))
############## Parameters #############################
parameters_dict_from_subtree = self.traj.parameters.f_to_dict()
self.assertTrue(len(parameters_dict_from_subtree)>0)
parameters_dict_directly = self.traj.f_get_parameters(copy=True)
self.assertTrue(comp.nested_equal(parameters_dict_directly,parameters_dict_from_subtree),
'%s!=%s' % (str(parameters_dict_directly),str(parameters_dict_directly)))
parameters_dict_directly = self.traj.f_get_parameters(copy=False)
self.assertTrue(comp.nested_equal(parameters_dict_directly,parameters_dict_from_subtree),
'%s!=%s' % (str(parameters_dict_directly),str(parameters_dict_directly)))
### Empty Parameters won't support fast access so we need to set
self.traj.paul.peter.f_set(42)
parameters_dict_from_subtree = self.traj.parameters.f_to_dict(fast_access=True)
with self.assertRaises(ValueError):
parameters_dict_directly = self.traj.f_get_parameters(copy=False, fast_access=True)
parameters_dict_directly = self.traj.f_get_parameters(copy=True, fast_access=True)
self.assertTrue(comp.nested_equal(parameters_dict_directly,parameters_dict_from_subtree),
'%s!=%s' % (str(parameters_dict_directly),str(parameters_dict_directly)))
################# Derived Parameters ############################
derived_parameters_dict_from_subtree = self.traj.derived_parameters.f_to_dict()
self.assertTrue(len(derived_parameters_dict_from_subtree)>0)
derived_parameters_dict_directly = self.traj.f_get_derived_parameters(copy=True)
self.assertTrue(comp.nested_equal(derived_parameters_dict_directly,derived_parameters_dict_from_subtree),
'%s!=%s' % (str(derived_parameters_dict_directly),str(derived_parameters_dict_directly)))
derived_parameters_dict_directly = self.traj.f_get_derived_parameters(copy=False)
self.assertTrue(comp.nested_equal(derived_parameters_dict_directly,derived_parameters_dict_from_subtree),
'%s!=%s' % (str(derived_parameters_dict_directly),str(derived_parameters_dict_directly)))
derived_parameters_dict_from_subtree = self.traj.derived_parameters.f_to_dict(fast_access=True)
with self.assertRaises(ValueError):
derived_parameters_dict_directly = self.traj.f_get_derived_parameters(copy=False, fast_access=True)
derived_parameters_dict_directly = self.traj.f_get_derived_parameters(copy=True, fast_access=True)
self.assertTrue(comp.nested_equal(derived_parameters_dict_directly,derived_parameters_dict_from_subtree),
'%s!=%s' % (str(derived_parameters_dict_directly),str(derived_parameters_dict_directly)))
############## Results #################################
results_dict_from_subtree = self.traj.results.f_to_dict()
self.assertTrue(len(results_dict_from_subtree)>0)
results_dict_directly = self.traj.f_get_results(copy=True)
self.assertTrue(comp.nested_equal(results_dict_directly,results_dict_from_subtree),
'%s!=%s' % (str(results_dict_directly),str(results_dict_directly)))
results_dict_directly = self.traj.f_get_results(copy=False)
self.assertTrue(comp.nested_equal(results_dict_directly,results_dict_from_subtree),
'%s!=%s' % (str(results_dict_directly),str(results_dict_directly)))
results_dict_from_subtree = self.traj.results.f_to_dict(fast_access=True)
with self.assertRaises(ValueError):
results_dict_directly = self.traj.f_get_results(copy=False, fast_access=True)
results_dict_directly = self.traj.f_get_results(copy=True, fast_access=True)
self.assertTrue(comp.nested_equal(results_dict_directly,results_dict_from_subtree),
'%s!=%s' % (str(results_dict_directly),str(results_dict_directly)))
################### Explored Parameters #######################
# #We need to unlock the parameter because we have accessed it above
# self.traj.f_get('yve').f_unlock()
# explore_dict = {'yve':[4,5,65,66]}
# # We can add to existing explored parameters if we match the length
# self.traj.f_expand(explore_dict)
explore_dict_directly = self.traj.f_get_explored_parameters(copy=False)
for key in self.explore_dict:
self.assertTrue(comp.nested_equal(self.traj.f_get(key),
explore_dict_directly[self.traj.f_get(key).v_full_name]))
explore_dict_directly = self.traj.f_get_explored_parameters(copy=True)
for key in self.explore_dict:
self.assertTrue(comp.nested_equal(self.traj.f_get(key),
explore_dict_directly[self.traj.f_get(key).v_full_name]))
with self.assertRaises(ValueError):
explore_dict_directly = self.traj.f_get_explored_parameters(copy=False, fast_access=True)
explore_dict_directly = self.traj.f_get_explored_parameters(copy=True, fast_access=True)
for key in self.explore_dict:
self.assertTrue(comp.nested_equal(self.traj.f_get(key,fast_access=True),
explore_dict_directly[self.traj.f_get(key).v_full_name]))
def test_increase_exploration(self):
self.explore_dict = {'IntParam':[2,1,1,3]}
self.traj.f_explore(self.explore_dict)
self.assertTrue(len(self.traj._explored_parameters)==2)
self.traj._stored=True
with self.assertRaises(TypeError):
self.traj.f_explore(self.explore_dict)
def test_f_get(self):
self.traj.v_fast_access=True
self.traj.f_get('FloatParam', fast_access=True) == self.traj.FloatParam
self.traj.v_fast_access=False
self.assertTrue(self.traj.f_get('FloatParam').f_get() == 4.0 , '%d != 4.0' %self.traj.f_get('FloatParam').f_get())
self.assertTrue(self.traj.FortyTwo.f_get() == 42)
def test_get_item(self):
self.assertEqual(self.traj.markus.yve, self.traj['markus.yve'])
self.assertEqual(self.traj.markus.yve, self.traj['markus','yve'])
# def test_round_brackets(self):
# x='markus'
#
# y='yve'
#
# self.assertEqual(self.traj(x)(y), self.traj.markus.f_get('yve'))
@staticmethod
def get_depth_dict(traj, as_run=None):
bfs_queue=[traj]
depth_dict={}
while len(bfs_queue)>0:
item = bfs_queue.pop(0)
depth = item.v_depth
if not depth in depth_dict:
depth_dict[depth]=[]
depth_dict[depth].append(item)
if not item.v_is_leaf:
if as_run in item._children:
bfs_queue.append(item._children[as_run])
for child in item._children.values():
if not (child.v_name.startswith(pypetconstants.RUN_NAME) and
child.v_name != pypetconstants.RUN_NAME_DUMMY):
bfs_queue.append(child)
else:
for child in item._children.values():
bfs_queue.append(child)
return depth_dict
def test_iter_bfs(self):
depth_dict = self.get_depth_dict(self.traj)
depth_dict[0] =[]
prev_depth = 0
for node in self.traj.f_iter_nodes(recursive=True):
if prev_depth != node.v_depth:
self.assertEqual(len(depth_dict[prev_depth]),0)
prev_depth = node.v_depth
depth_dict[node.v_depth].remove(node)
depth_dict = self.get_depth_dict(self.traj)
depth_dict[0] =[]
prev_depth = 0
self.traj.v_iter_recursive = True
for node in self.traj:
if prev_depth != node.v_depth:
self.assertEqual(len(depth_dict[prev_depth]),0)
prev_depth = node.v_depth
depth_dict[node.v_depth].remove(node)
#
#
# def test_iter_dfs(self):
#
# prev_node = None
#
# x= [x for x in self.traj.f_iter_nodes(recursive=True, search_strategy='DFS')]
#
# for node in self.traj.f_iter_nodes(recursive=True, search_strategy='DFS'):
# if not prev_node is None:
# if not prev_node.v_is_leaf and len(prev_node._children) > 0:
# self.assertTrue(node.v_name in prev_node._children)
#
# prev_node = node
#
# prev_node = None
#
# self.traj.v_iter_recursive=True
# self.traj.v_search_strategy='DFS'
#
# y = [y for y in self.traj]
#
# self.assertEqual(x,y)
#
# for node in self.traj:
# if not prev_node is None:
# if not prev_node.v_is_leaf and len(prev_node._children) > 0:
# self.assertTrue(node.v_name in prev_node._children)
#
# prev_node = node
def test_iter_bfs_as_run(self):
as_run = 1
self.traj.f_add_result('results.run_00000000.resulttest', 42)
self.traj.f_add_result('results.run_00000001.resulttest', 43)
self.traj.f_as_run(as_run)
depth_dict = self.get_depth_dict(self.traj, self.traj.f_idx_to_run(as_run))
depth_dict[0] =[]
prev_depth = 0
for node in self.traj.f_iter_nodes(recursive=True):
self.assertTrue('run_00000000' not in node.v_full_name)
if prev_depth != node.v_depth:
self.assertEqual(len(depth_dict[prev_depth]),0)
prev_depth = node.v_depth
depth_dict[node.v_depth].remove(node)
# def test_iter_dfs_as_run(self):
#
# self.traj.f_add_result('results.run_00000000.resulttest', 42)
# self.traj.f_add_result('results.run_00000001.resulttest', 43)
#
# self.traj.f_as_run('run_00000001')
#
# prev_node = None
#
# x= [x for x in self.traj.f_iter_nodes(recursive=True, search_strategy='DFS')]
#
# for node in self.traj.f_iter_nodes(recursive=True, search_strategy='DFS'):
# self.assertTrue('run_00000000' not in node.v_full_name)
#
# if not prev_node is None:
# if not prev_node.v_is_leaf and len(prev_node._children) > 0:
# self.assertTrue(node.v_name in prev_node._children)
#
# prev_node = node
def test_find_in_all_runs(self):
self.traj.f_add_result('results.runs.run_00000000.sub.resulttest', 42)
self.traj.f_add_result('results.runs.run_00000001.sub.resulttest', 43)
self.traj.f_add_result('results.runs.run_00000002.sub.resulttest', 44)
self.traj.f_add_result('results.runs.run_00000002.sub.resulttest2', 42)
self.traj.f_add_result('results.runs.run_00000003.sub.resulttest2', 43)
self.traj.f_add_derived_parameter('derived_parameters.runs.run_00000002.testing', 44)
res_dict = self.traj.f_get_from_runs('kkkkkkdjfoiuref')
self.assertTrue(len(res_dict)==0)
res_dict = self.traj.f_get_from_runs('resulttest', fast_access=True)
self.assertTrue(len(res_dict)==3)
self.assertTrue(res_dict['run_00000001']==43)
self.assertTrue('run_00000003' not in res_dict)
res_dict = self.traj.f_get_from_runs(name='sub.resulttest2', use_indices=True)
self.assertTrue(len(res_dict)==2)
self.assertTrue(res_dict[3]is self.traj.f_get('run_00000003.resulttest2'))
self.assertTrue(1 not in res_dict)
res_dict = self.traj.f_get_from_runs(name='testing', where='derived_parameters')
self.assertTrue(len(res_dict)==1)
self.traj.f_add_result('results.runs.run_00000002.sub.sub.sub.sub.resulttest2', 444)
self.traj.f_add_result('results.runs.run_00000002.sub.sub.sub.resulttest2', 444)
with self.assertRaises(pex.NotUniqueNodeError):
self.traj.f_get_from_runs('sub.sub.resulttest2', backwards_search=True)
with self.assertRaises(ValueError):
self.traj.f_get_from_runs('test', where='Portland')
def test_illegal_namings(self):
self.traj=Trajectory('resulttest2')
with self.assertRaises(ValueError):
self.traj.f_add_parameter('f_get')
with self.assertRaises(ValueError):
self.traj.f_add_result('test.$.k.$')
rg=self.traj.f_add_result_group('ggg.$')
with self.assertRaises(ValueError):
rg.f_add_result('$.fff')
self.traj.f_add_result_group('test.$.k')
with self.assertRaises(ValueError):
self.traj.res.k.f_add_result('$.jjj')
with self.assertRaises(ValueError):
self.traj.f_add_parameter('e'*129)
with self.assertRaises(ValueError):
self.traj.f_add_parameter('e'*120+'.j'*120+'.k'*40)
with self.assertRaises(ValueError):
self.traj.f_add_parameter('crun',22)
def test_max_depth(self):
self.traj.f_add_parameter('halo.this.is.a.depth.testrr')
contains = self.traj.f_contains('a.depth.testrr', max_depth=3)
self.assertFalse(contains)
contains = self.traj.par.f_contains('halo.this.is.a.depth.testrr', shortcuts=False)
self.assertTrue(contains)
contains = self.traj.par.f_contains('halo.this.depth.testrr', shortcuts=True)
self.assertTrue(contains)
contains = self.traj.par.f_contains('testrr', shortcuts=True)
self.assertTrue(contains)
contains = 'testrr' in self.traj
self.assertTrue(contains)
contains = self.traj.par.f_contains('testrr', max_depth=5)
self.assertFalse(contains)
self.traj.v_max_depth = 5
contains = 'testrr' in self.traj
self.assertFalse(contains)
def test_root_getting(self):
traj = Trajectory()
traj.f_add_config_group('ff')
root = traj.ff.f_get_root()
self.assertTrue(root is traj)
def test_not_adding_pars_during_single_run(self):
traj = Trajectory()
traj._is_run = True
with self.assertRaises(TypeError):
traj.f_add_parameter('dd')
with self.assertRaises(TypeError):
traj.f_add_parameter_group('dd')
with self.assertRaises(TypeError):
traj.f_add_config('dd')
with self.assertRaises(TypeError):
traj.f_add_config_group('dd')
def test_attribute_error_raises_when_leaf_and_group_with_same_name_are_added(self):
self.traj = Trajectory()
self.traj.f_add_parameter('test.param1')
with self.assertRaises(AttributeError):
self.traj.f_add_parameter('test.param1.param2')
with self.assertRaises(AttributeError):
self.traj.f_add_parameter('test')
def testremove(self):
self.traj.f_remove_item(self.traj.f_get('peter.markus.yve'),remove_empty_groups=True)
with self.assertRaises(AttributeError):
self.traj.peter.markus.yve
self.assertFalse('peter.markus.yve' in self.traj)
#self.assertTrue(len(self.traj)==1)
self.traj.f_remove_item('FortyTwo',remove_empty_groups=True)
self.traj.f_remove_item('SparseParam')
self.traj.f_remove_item('IntParam')
#self.assertTrue(len(self.traj)==1)
def test_changing(self):
self.traj.f_preset_config('testconf', 1)
self.traj.f_preset_parameter('testparam', 1)
self.traj.f_preset_parameter('I_do_not_exist', 2)
self.traj.f_add_parameter('testparam', 0)
self.traj.f_add_config('testconf', 0)
self.traj.v_fast_access=True
self.assertTrue(self.traj.testparam == 1)
self.assertTrue(self.traj.testconf == 1)
### should raise an error because 'I_do_not_exist', does not exist:
with self.assertRaises(pex.PresettingError):
self.traj._prepare_experiment()
def test_f_is_completed(self):
traj = Trajectory()
traj.f_add_parameter('test', 42)
traj.f_explore({'test':[1,2,3,4]})
self.assertFalse(traj.f_is_completed())
for run_name in traj.f_get_run_names():
self.assertFalse(traj.f_is_completed(run_name))
traj._run_information[traj.f_idx_to_run(1)]['completed']=1
self.assertFalse(traj.f_is_completed())
self.assertTrue(traj.f_is_completed(1))
for run_name in traj.f_get_run_names():
traj._run_information[run_name]['completed']=1
self.assertTrue(traj.f_is_completed())
for run_name in traj.f_get_run_names():
self.assertTrue(traj.f_is_completed(run_name))
def test_if_picklable(self):
self.traj.v_fast_access=True
#self.traj.v_full_copy=True
dump = pickle.dumps(self.traj)
newtraj = pickle.loads(dump)
self.assertTrue(len(newtraj) == len(self.traj))
new_items = newtraj.f_to_dict(fast_access=True)
for key, val in self.traj.f_to_dict(fast_access=True).items():
#val = newtraj.f_get(table_name)
nval = new_items[key]
if isinstance(val, BaseResult):
for ikey in val._data:
self.assertTrue(str(nval.f_get(ikey))==str(val.f_get(ikey)))
else:
self.assertTrue(str(val)==str(nval), '%s != %s' %(str(val),str(nval)))
def test_dynamic_class_loading(self):
self.traj.f_add_parameter(ImAParameterInDisguise,'Rolf', 1.8)
def test_standard_change_param_change(self):
self.traj.v_standard_parameter=ImAParameterInDisguise
self.traj.f_add_parameter('I.should_be_not.normal')
self.assertIsInstance(self.traj.f_get('normal'), ImAParameterInDisguise,'Param is %s insted of ParamInDisguise.' %str(type(self.traj.normal)))
self.traj.v_standard_result=ImAResultInDisguise
self.traj.f_add_result('Peter.Parker')
self.assertIsInstance(self.traj.Parker, ImAResultInDisguise)
def test_remove_of_explored_stuff_if_saved(self):
self.traj = Trajectory()
self.traj.f_add_parameter('test', 42)
self.traj.f_explore({'test':[1,2,3,4]})
self.traj._stored=True
self.traj.parameters.f_remove_child('test')
len(self.traj) == 4
def test_remov_of_explored_stuff_if_not_saved(self):
self.traj = Trajectory()
self.traj.f_add_parameter('test', 42)
self.traj.f_explore({'test':[1,2,3,4]})
self.traj.parameters.f_remove_child('test')
self.assertTrue(len(self.traj) == 1)
def test_not_unique_search(self):
self.traj = Trajectory()
self.traj.f_add_parameter('ghgghg.test')
self.traj.f_add_parameter('ghdsfdfdsfdsghg.test')
with self.assertRaises(pex.NotUniqueNodeError):
self.traj.test
self.traj.f_add_parameter('depth0.depth1.depth2.findme', 42)
self.traj.f_add_parameter('depth0.depth1.findme', 43)
self.assertTrue(self.traj.findme==43)
with self.assertRaises(pex.NotUniqueNodeError):
self.traj.f_get('depth0.findme', backwards_search=True)
def test_contains_item_identity(self):
peterpaul = self.traj.f_get('peter.paul')
self.assertTrue(peterpaul in self.traj)
peterpaulcopy = copy.deepcopy(peterpaul)
self.assertFalse(peterpaulcopy in self.traj)
def test_get_children(self):
for node in self.traj.f_iter_nodes():
if not node.v_is_leaf:
self.assertEqual(id(node.f_get_children(copy=False)), id(node._children))
self.assertNotEqual(id(node.f_get_children(copy=True)), id(node._children))
self.assertEqual(sorted(node.f_get_children(copy=True).keys()),
sorted(node._children.keys()))
def test_short_cuts(self):
self.traj = Trajectory()
self.traj.f_add_parameter('test', 42)
self.traj.f_add_config('tefffst', 42)
self.traj.f_add_derived_parameter('dtest', 42)
self.traj.f_add_result('safd', 42)
self.traj.f_explore({'test':[1,2,3,4]})
self.assertEqual(id(self.traj.par), id(self.traj.parameters))
#self.assertEqual(id(self.traj.param), id(self.traj.parameters))
self.assertEqual(id(self.traj.dpar), id(self.traj.derived_parameters))
#self.assertEqual(id(self.traj.dparam), id(self.traj.derived_parameters))
self.assertEqual(id(self.traj.conf), id(self.traj.config))
self.assertEqual(id(self.traj.res), id(self.traj.results))
srun = self.traj._make_single_run(3)
srun.f_add_result('sdffds',42)
self.assertEqual(id(srun.results.crun), id(srun.results.f_get(srun.v_name)))
def main():
pool = Pool(2)
lock = Manager().Lock()
logging.basicConfig(level=logging.DEBUG)
log_to_stderr()
traj = Trajectory(name='MyTrajectory',filename='../experiments/test.hdf5')
config['multiproc'] = True
#traj.load_trajectory(trajectoryname='MyTrajectory_2013_05_23_14h29m26s')
traj.f_add_parameter('test.testparam', param_type=ParameterSet, **{'Fuechse':1,'Sapiens':1,'Comment':'ladida'})
traj.last.foo = 'bar'
traj.f_add_parameter('Network.Cm')
traj.Parameters.Network.Cm.value= 1.0
traj.Parameters.Network.Cm.unit = 'pF'
print traj.Parameters.Network.Cm()
traj.last.herbert = 4.5
traj.Parameters.test.testparam.Konstantin = 'Konstantin'
traj.f_add_parameter(full_parameter_name='honky', **{'mirta':np.array([[1,2,7],[3,2,17]])})
traj.f_add_parameter('flonky',**{'val' : 10})
param1 = traj.Parameters.test.testparam
param2 = traj.Parameters.honky
param3 = traj.last
print param1()
print param3('val')
exp2_list = range(3)
exp1_list = range(3)
exp3_list = range(3)
explore_dict = { param1.gfn('Sapiens') : exp1_list,
param2.get_fullname('mirta'):exp2_list,
param3.gfn('val') : exp3_list}
cmb_list=[(param3.gfn('val'),param1.gfn('Sapiens'))]
traj.f_add_derived_parameter(full_parameter_name='foo', **{'bar' : -1}).moo = 'zip'
lilma = spsp.lil_matrix((10000,1000))
lilma[0,100] = 555
lilma[9999,999] = 11
traj.f_add_parameter('sparse', mat=lilma, param_type = SparseParameter)
traj.f_explore(ut.cartesian_product,explore_dict,cmb_list)
traj._prepare_experiment()
traj.f_store()
pathname = config['logfolder']
it = ((traj._make_single_run(n),lock, pathname) for n in xrange(len(traj)))
moo = pool.imap(do_stuff, it)
pool.close()
pool.join()
__author__ = 'Robert Meyer'
from pypet.trajectory import Trajectory
from pypet.pypetexceptions import NotUniqueNodeError
# We first generate a new Trajectory
traj = Trajectory('Example', filename='experiments/example_02/HDF5/example_02.hdf5',
comment='Access and Storage!')
# We add our first parameter with the data 'Harrison Ford'
traj.f_add_parameter('starwars.characters.han_solo', 'Harrison Ford')
# This automatically added the groups 'starwars' and the subgroup 'characters'
# Let's get the characters subgroup
characters = traj.parameters.starwars.characters
# Since characters is unique we could also use shortcuts
characters = traj.characters
# Or the get method
characters = traj.f_get('characters')
# Or square brackets
characters = traj['characters']
# Lets add another character
characters.f_add_parameter('luke_skywalker', 'Mark Hamill', comment='May the force be with you!')