def test_continueing_remove_completed(self): self.filenames = [make_temp_dir('test_continueing_remove_completed.hdf5')] self.envs=[] self.trajs = [] for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename, continuable=True, delete_continue=True) self.param_dict={} create_param_dict(self.param_dict) for irun in range(len(self.filenames)): add_params(self.trajs[irun], self.param_dict) self.explore(self.trajs[0]) for irun in range(len(self.filenames)): self.make_run(self.envs[irun]) traj_name = self.trajs[0].v_name continue_folder = os.path.join(self.cnt_folder, self.trajs[0].v_name) self.assertFalse(os.path.isdir(continue_folder))
def merge_basic_only_adding_more_trials(self, copy_nodes=False, delete_traj=False, slow_merge=False): self.envs=[] self.trajs = [] for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename) self.param_dict={} create_param_dict(self.param_dict) for irun in [0,1,2]: add_params(self.trajs[irun], self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) self.compare_explore_more_trials(self.trajs[2]) for irun in [0,1,2]: self.make_run(self.envs[irun]) for irun in [0,1,2]: self.trajs[irun].f_load_skeleton() self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.trajs[1].f_add_result('gg.rrororo33o333o3o3oo3',1234567890) self.trajs[1].f_store_item('rrororo33o333o3o3oo3') self.trajs[1].res.gg.v_annotations['lala'] = 'Sonnenschein' self.trajs[1].f_store_item('gg') self.trajs[2].f_add_result('gg.rrororo33o333o3o3oo3',1234567890) self.trajs[2].f_store_item('rrororo33o333o3o3oo3') self.trajs[2].res.gg.v_annotations['lala'] = 'Sonnenschein' self.trajs[2].f_store_item('gg') ##f_merge without destroying the original trajectory merged_traj = self.trajs[0] self.trajs[1].f_remove_child('results', recursive=True) merged_traj.f_merge(self.trajs[1], move_data=not copy_nodes, delete_other_trajectory=delete_traj, trial_parameter='trial', slow_merge=slow_merge) merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.compare_trajectories(merged_traj,self.trajs[2])
def test_removal(self): self.filenames = [ make_temp_dir( os.path.join('experiments', 'tests', 'HDF5', 'removal.hdf5')), 0 ] self.envs = [] self.trajs = [] for irun, filename in enumerate(self.filenames): if isinstance(filename, int): filename = self.filenames[filename] self.make_environment(irun, filename) self.param_dict = {} create_param_dict(self.param_dict) for irun in range(len(self.filenames)): add_params(self.trajs[irun], self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) for irun in range(len(self.filenames)): self.make_run(self.envs[irun]) self.trajs[0].f_add_parameter('Delete.Me', 'I will be deleted!') self.trajs[0].f_store_item('Delete.Me') self.trajs[0].f_remove_item(self.trajs[0].f_get('Delete.Me')) self.assertTrue('Delete.Me' not in self.trajs[0], 'Delete.Me is still in traj') self.trajs[0].f_load_skeleton() self.trajs[0].f_load_item('Delete.Me') self.trajs[0].f_delete_item(self.trajs[0].f_get('Delete.Me'), remove_from_trajectory=True) self.trajs[0].f_load_skeleton() self.assertTrue(not 'Delete.Me' in self.trajs[0], 'Delete.Me is still in traj') for irun in range(len(self.filenames)): self.trajs[irun].f_load_child( 'results', recursive=True, load_data=pypetconstants.OVERWRITE_DATA) self.trajs[irun].f_load_child( 'derived_parameters', recursive=True, load_data=pypetconstants.OVERWRITE_DATA) self.compare_trajectories(self.trajs[0], self.trajs[1])
def test_merging_errors_if_trajs_do_not_match(self): self.filenames = [make_temp_dir(os.path.join('experiments', 'tests', 'HDF5', 'merge_errors.hdf5')), 0] self.envs=[] self.trajs = [] for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename) self.param_dict={} create_param_dict(self.param_dict) for irun in [0,1]: add_params(self.trajs[irun], self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) for irun in [0,1]: self.make_run(self.envs[irun]) for irun in [0,1]: self.trajs[irun].f_load_skeleton() self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.trajs[0].f_add_parameter('Merging.Denied', 13) ##f_merge without destroying the original trajectory merged_traj = self.trajs[0] # We cannot merge trajectories which parameters differ with self.assertRaises(TypeError): merged_traj.f_merge(self.trajs[1]) self.trajs[1].f_add_parameter('Merging.Denied', 13.13) # We cannot merge trajectories where parameters differ in type with self.assertRaises(TypeError): merged_traj.f_merge(self.trajs[1]) self.trajs[1].f_get('Denied').f_unlock() self.trajs[1].f_get('Denied').f_set(15) merged_traj.f_merge(self.trajs[1])
def test_merging_trajectories_in_different_subspace(self): self.filenames = [make_temp_dir(os.path.join('experiments', 'tests', 'HDF5', 'merge_diff_subspace.hdf5')), 0, 0] self.envs=[] self.trajs = [] for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename, wildcard_functions = {('$', 'crun') : my_run_func, ('$set', 'crunset'): my_set_func}) self.param_dict={} create_param_dict(self.param_dict) for irun in [0,1,2]: add_params(self.trajs[irun], self.param_dict) self.explore(self.trajs[0]) self.explore2(self.trajs[1]) self.compare_explore_diff_subspace(self.trajs[2]) for irun in [0,1,2]: self.make_run(self.envs[irun]) for irun in [0,1,2]: self.trajs[irun].f_load_skeleton() self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890) self.trajs[1].f_store_item('rrororo33o333o3o3oo3') self.trajs[2].f_add_result('rrororo33o333o3o3oo3',1234567890) self.trajs[2].f_store_item('rrororo33o333o3o3oo3') ##f_merge without destroying the original trajectory merged_traj = self.trajs[0] merged_traj.f_merge(self.trajs[1], move_data=True, delete_other_trajectory=True) merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.compare_trajectories(merged_traj,self.trajs[2])
def setUp(self): self.multiproc = True self.mode = 'LOCK' self.trajname = make_trajectory_name(self) self.filename = make_temp_dir( os.path.join('experiments', 'tests', 'HDF5', '%s.hdf5' % self.trajname)) self.logfolder = make_temp_dir( os.path.join('experiments', 'tests', 'Log')) random.seed() cap_dicts = ( dict(cpu_cap=0.000001), # Ensure that these are triggered dict(memory_cap=(0.000001, 150.0)), dict(swap_cap=0.000001, )) cap_dict = cap_dicts[CapTest.cap_count] env = Environment(trajectory=self.trajname, filename=self.filename, file_title=self.trajname, log_folder=self.logfolder, logger_names=('pypet', 'test'), log_levels='ERROR', log_stdout=False, results_per_run=5, derived_parameters_per_run=5, multiproc=True, ncores=4, use_pool=False, niceness=check_nice(11), **cap_dict) logging.getLogger('test').error('Using Cap: %s and file: %s' % (str(cap_dict), str(self.filename))) # Loop through all possible cap configurations # and test one at a time CapTest.cap_count += 1 CapTest.cap_count = CapTest.cap_count % len(cap_dicts) traj = env.v_trajectory ## Create some parameters self.param_dict = {} create_param_dict(self.param_dict) ### Add some parameter: add_params(traj, self.param_dict) #remember the trajectory and the environment self.traj = traj self.env = env
def setUp(self): self.set_mode() self.logfolder = make_temp_dir(os.path.join('experiments', 'tests', 'Log')) random.seed() self.trajname = make_trajectory_name(self) self.filename = make_temp_dir(os.path.join('experiments', 'tests', 'HDF5', 'test%s.hdf5' % self.trajname)) env = Environment(trajectory=self.trajname, filename=self.filename, file_title=self.trajname, log_stdout=self.log_stdout, log_config=get_log_config(), results_per_run=5, wildcard_functions=self.wildcard_functions, derived_parameters_per_run=5, multiproc=self.multiproc, ncores=self.ncores, wrap_mode=self.mode, use_pool=self.use_pool, gc_interval=self.gc_interval, freeze_input=self.freeze_input, fletcher32=self.fletcher32, complevel=self.complevel, complib=self.complib, shuffle=self.shuffle, pandas_append=self.pandas_append, pandas_format=self.pandas_format, encoding=self.encoding, niceness=self.niceness, use_scoop=self.use_scoop, port=self.port, add_time=self.add_time, timeout=self.timeout, graceful_exit=self.graceful_exit) traj = env.v_trajectory traj.v_standard_parameter=Parameter ## Create some parameters self.param_dict={} create_param_dict(self.param_dict) ### Add some parameter: add_params(traj,self.param_dict) #remember the trajectory and the environment self.traj = traj self.env = env
def test_removal(self): self.filenames = [make_temp_dir(os.path.join('experiments', 'tests', 'HDF5', 'removal.hdf5')), 0] self.envs=[] self.trajs = [] for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename) self.param_dict={} create_param_dict(self.param_dict) for irun in range(len(self.filenames)): add_params(self.trajs[irun], self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) for irun in range(len(self.filenames)): self.make_run(self.envs[irun]) self.trajs[0].f_add_parameter('Delete.Me', 'I will be deleted!') self.trajs[0].f_store_item('Delete.Me') self.trajs[0].f_remove_item(self.trajs[0].f_get('Delete.Me')) self.assertTrue('Delete.Me' not in self.trajs[0],'Delete.Me is still in traj') self.trajs[0].f_load_skeleton() self.trajs[0].f_load_item('Delete.Me') self.trajs[0].f_delete_item(self.trajs[0].f_get('Delete.Me'), remove_from_trajectory=True) self.trajs[0].f_load_skeleton() self.assertTrue(not 'Delete.Me' in self.trajs[0],'Delete.Me is still in traj') for irun in range(len(self.filenames)): self.trajs[irun].f_load_skeleton() self.trajs[irun].f_load_child('results',recursive=True,load_data=pypetconstants.UPDATE_DATA) self.trajs[irun].f_load_child('derived_parameters',recursive=True,load_data=pypetconstants.UPDATE_DATA) self.compare_trajectories(self.trajs[0],self.trajs[1])
def setUp(self): self.multiproc = True self.mode = 'LOCK' self.trajname = make_trajectory_name(self) self.filename = make_temp_dir(os.path.join('experiments', 'tests', 'HDF5', '%s.hdf5' % self.trajname)) self.logfolder = make_temp_dir(os.path.join('experiments', 'tests', 'Log')) random.seed() cap_dicts = (dict(cpu_cap=0.000001), # Ensure that these are triggered dict(memory_cap=(0.000001, 150.0)), dict(swap_cap=0.000001,)) cap_dict = cap_dicts[CapTest.cap_count] env = Environment(trajectory=self.trajname,filename=self.filename, file_title=self.trajname, log_folder=self.logfolder, logger_names=('pypet', 'test'), log_levels='ERROR', log_stdout=False, results_per_run=5, derived_parameters_per_run=5, multiproc=True, ncores=4, use_pool=False, niceness = check_nice(11), **cap_dict) logging.getLogger('test').error('Using Cap: %s and file: %s' % (str(cap_dict), str(self.filename))) # Loop through all possible cap configurations # and test one at a time CapTest.cap_count += 1 CapTest.cap_count = CapTest.cap_count % len(cap_dicts) traj = env.v_trajectory ## Create some parameters self.param_dict={} create_param_dict(self.param_dict) ### Add some parameter: add_params(traj,self.param_dict) #remember the trajectory and the environment self.traj = traj self.env = env
def basic_and_skipping_duplicates_which_will_be_all(self): self.envs=[] self.trajs = [] for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename) self.param_dict={} create_param_dict(self.param_dict) for irun in [0,1]: add_params(self.trajs[irun],self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) for irun in [0,1]: self.make_run(self.envs[irun]) for irun in [0,1]: self.trajs[irun].f_load_skeleton() self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.trajs[0].f_add_result('rrororo33o333o3o3oo3',1234567890) self.trajs[0].f_store_item('rrororo33o333o3o3oo3') self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890) self.trajs[1].f_store_item('rrororo33o333o3o3oo3') ##f_merge without destroying the original trajectory merged_traj = self.trajs[0] merged_traj.f_merge(self.trajs[1], move_data=False, delete_other_trajectory=False, remove_duplicates=True) merged_traj.f_load_skeleton() merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.compare_trajectories(merged_traj,self.trajs[1])
def merge_basic_only_adding_more_trials_with_backup(self,copy_nodes): self.envs=[] self.trajs = [] for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename) self.param_dict={} create_param_dict(self.param_dict) for irun in [0,1,2]: add_params(self.trajs[irun],self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) self.compare_explore_more_trials(self.trajs[2]) for irun in [0,1,2]: self.make_run(self.envs[irun]) for irun in [0,1,2]: self.trajs[irun].f_load_skeleton() self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890) self.trajs[1].f_store_item('rrororo33o333o3o3oo3') self.trajs[2].f_add_result('rrororo33o333o3o3oo3',1234567890) self.trajs[2].f_store_item('rrororo33o333o3o3oo3') ##f_merge without destroying the original trajectory merged_traj = self.trajs[0] merged_traj.f_merge(self.trajs[1], move_data=not copy_nodes, delete_other_trajectory=False, trial_parameter='trial', backup_filename=1) merged_traj.f_load_skeleton() merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.compare_trajectories(merged_traj,self.trajs[2])
def test_multiple_storage_and_loading(self): self.filenames = [make_temp_dir(os.path.join('experiments', 'tests', 'HDF5', 'merge1.hdf5')), 0] self.envs=[] self.trajs = [] for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename) self.param_dict={} create_param_dict(self.param_dict) for irun in range(len(self.filenames)): add_params(self.trajs[irun],self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) for irun in range(len(self.filenames)): self.make_run(self.envs[irun]) #self.trajs[0].f_store() temp_sservice = self.trajs[0].v_storage_service temp_name = self.trajs[0].v_name self.trajs[0] = Trajectory() self.trajs[0].v_storage_service=temp_sservice self.trajs[0].f_load(name=temp_name, as_new=False, load_parameters=2, load_derived_parameters=2, load_results=2, load_other_data=2) #self.trajs[0].f_load(trajectory_name=temp_name,as_new=False, load_params=2, load_derived_params=2, load_results=2) self.trajs[1].f_load_skeleton() self.trajs[1].f_load_items(self.trajs[1].f_to_dict().values(),only_empties=True) self.compare_trajectories(self.trajs[0],self.trajs[1])
def test_continueing(self): self.filenames = [make_temp_dir('test_continuing.hdf5'), 0] self.envs = [] self.trajs = [] for irun, filename in enumerate(self.filenames): if isinstance(filename, int): filename = self.filenames[filename] self.make_environment(irun, filename, continuable=irun == 1, delete_continue=False) self.param_dict = {} create_param_dict(self.param_dict) for irun in range(len(self.filenames)): add_params(self.trajs[irun], self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) for irun in range(len(self.filenames)): self.make_run(self.envs[irun]) traj_name = self.trajs[1].v_name continue_folder = os.path.join(self.cnt_folder, self.trajs[1].v_name) self.trajs[1] = self.envs[1].v_trajectory self._remove_nresults(self.trajs[1], 3, continue_folder) self.envs[1].v_current_idx = 0 results = self.envs[1].resume(trajectory_name=traj_name) for irun in range(len(self.filenames)): self.trajs[irun].f_load( load_parameters=pypetconstants.OVERWRITE_DATA, load_derived_parameters=pypetconstants.OVERWRITE_DATA, load_results=pypetconstants.OVERWRITE_DATA, load_other_data=pypetconstants.OVERWRITE_DATA) self.compare_trajectories(self.trajs[0], self.trajs[1]) #shutil.rmtree(self.cnt_folder) self.assertEqual(len(self.trajs[1]), len(results))
def test_multiple_storage_and_loading(self): self.filenames = [ make_temp_dir( os.path.join('experiments', 'tests', 'HDF5', 'multiple_storage_and_loading.hdf5')), 0 ] self.envs = [] self.trajs = [] for irun, filename in enumerate(self.filenames): if isinstance(filename, int): filename = self.filenames[filename] self.make_environment(irun, filename) self.param_dict = {} create_param_dict(self.param_dict) for irun in range(len(self.filenames)): add_params(self.trajs[irun], self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) for irun in range(len(self.filenames)): self.make_run(self.envs[irun]) #self.trajs[0].f_store() temp_sservice = self.trajs[0].v_storage_service temp_name = self.trajs[0].v_name self.trajs[0] = Trajectory() self.trajs[0].v_storage_service = temp_sservice self.trajs[0].f_load(name=temp_name, as_new=False, load_parameters=2, load_derived_parameters=2, load_results=2, load_other_data=2) #self.trajs[0].f_load(trajectory_name=temp_name,as_new=False, load_params=2, load_derived_params=2, load_results=2) self.trajs[1].f_load_skeleton() self.trajs[1].f_load_items(self.trajs[1].f_to_dict().values(), only_empties=True) self.compare_trajectories(self.trajs[0], self.trajs[1])
def test_continueing(self): self.filenames = [make_temp_dir('test_continuing.hdf5'), 0] self.envs=[] self.trajs = [] for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename, continuable=irun==1, delete_continue=False) self.param_dict={} create_param_dict(self.param_dict) for irun in range(len(self.filenames)): add_params(self.trajs[irun], self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) for irun in range(len(self.filenames)): self.make_run(self.envs[irun]) traj_name = self.trajs[1].v_name continue_folder = os.path.join(self.cnt_folder, self.trajs[1].v_name) self.trajs[1]=self.envs[1].v_trajectory self._remove_nresults(self.trajs[1], 3, continue_folder) self.envs[1].v_current_idx = 0 results = self.envs[1].f_continue(trajectory_name = traj_name) for irun in range(len(self.filenames)): self.trajs[irun].f_load(load_parameters=pypetconstants.OVERWRITE_DATA, load_derived_parameters=pypetconstants.OVERWRITE_DATA, load_results=pypetconstants.OVERWRITE_DATA, load_other_data=pypetconstants.OVERWRITE_DATA) self.compare_trajectories(self.trajs[0],self.trajs[1]) #shutil.rmtree(self.cnt_folder) self.assertEqual(len(self.trajs[1]), len(results))
def test_run(): global filename np.random.seed() trajname = 'profiling' filename = make_temp_dir(os.path.join('hdf5', 'test%s.hdf5' % trajname)) env = Environment(trajectory=trajname, filename=filename, file_title=trajname, log_stdout=False, results_per_run=5, derived_parameters_per_run=5, multiproc=False, ncores=1, wrap_mode='LOCK', use_pool=False, overwrite_file=True) traj = env.v_trajectory traj.v_standard_parameter = Parameter ## Create some parameters param_dict = {} create_param_dict(param_dict) ### Add some parameter: add_params(traj, param_dict) #remember the trajectory and the environment traj = traj env = env traj.f_add_parameter('TEST', 'test_run') ###Explore explore(traj) ### Make a test run simple_arg = -13 simple_kwarg = 13.0 env.f_run(simple_calculations, simple_arg, simple_kwarg=simple_kwarg) size = os.path.getsize(filename) size_in_mb = size / 1000000. print('Size is %sMB' % str(size_in_mb))
def test_run(): global filename np.random.seed() trajname = 'profiling' filename = make_temp_dir(os.path.join('hdf5', 'test%s.hdf5' % trajname)) env = Environment(trajectory=trajname, filename=filename, file_title=trajname, log_stdout=False, results_per_run=5, derived_parameters_per_run=5, multiproc=False, ncores=1, wrap_mode='LOCK', use_pool=False, overwrite_file=True) traj = env.v_trajectory traj.v_standard_parameter=Parameter ## Create some parameters param_dict={} create_param_dict(param_dict) ### Add some parameter: add_params(traj,param_dict) #remember the trajectory and the environment traj = traj env = env traj.f_add_parameter('TEST', 'test_run') ###Explore explore(traj) ### Make a test run simple_arg = -13 simple_kwarg= 13.0 env.f_run(simple_calculations,simple_arg,simple_kwarg=simple_kwarg) size=os.path.getsize(filename) size_in_mb = size/1000000. print('Size is %sMB' % str(size_in_mb))
def test_continueing_remove_completed(self): self.filenames = [make_temp_dir('test_removal_comp.hdf5'), 0] self.envs=[] self.trajs = [] for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename) self.param_dict={} create_param_dict(self.param_dict) for irun in range(len(self.filenames)): add_params(self.trajs[irun], self.param_dict) self.explore(self.trajs[0]) self.explore(self.trajs[1]) for irun in range(len(self.filenames)): self.make_run(self.envs[irun]) traj_name = self.trajs[0].v_name continue_folder = os.path.join(self.cnt_folder, self.trajs[0].v_name) self._remove_nresults_from_traj(2) self.make_environment(0, self.filenames[0]) self.envs[-1].f_continue(trajectory_name = traj_name) self.trajs[-1]=self.envs[-1].v_trajectory for irun in range(len(self.filenames)+1): self.trajs[irun].f_load_skeleton() self.trajs[irun].f_load(load_parameters=pypetconstants.OVERWRITE_DATA, load_derived_parameters=pypetconstants.OVERWRITE_DATA, load_results=pypetconstants.OVERWRITE_DATA, load_other_data=pypetconstants.OVERWRITE_DATA) self.compare_trajectories(self.trajs[-1],self.trajs[1])
def setUp(self): self.set_mode() self.logfolder = make_temp_dir(os.path.join('experiments', 'tests', 'Log')) random.seed() self.trajname = make_trajectory_name(self) self.filename = make_temp_dir(os.path.join('experiments', 'tests', 'HDF5', 'test%s.hdf5' % self.trajname)) env = Environment(trajectory=self.trajname, filename=self.filename, file_title=self.trajname, log_stdout=False, log_config=get_log_config(), results_per_run=5, derived_parameters_per_run=5, multiproc=self.multiproc, ncores=self.ncores, wrap_mode=self.mode, use_pool=self.use_pool, fletcher32=self.fletcher32, complevel=self.complevel, complib=self.complib, shuffle=self.shuffle, pandas_append=self.pandas_append, pandas_format=self.pandas_format, encoding=self.encoding) traj = env.v_trajectory self.param_dict={} create_param_dict(self.param_dict) add_params(traj,self.param_dict) self.traj = traj self.env = env
def setUp(self): self.set_mode() self.logfolder = make_temp_dir(os.path.join('experiments', 'tests', 'Log')) random.seed() self.trajname = make_trajectory_name(self) self.filename = make_temp_dir(os.path.join('experiments', 'tests', 'HDF5', 'test%s.hdf5' % self.trajname)) env = Environment(trajectory=self.trajname, filename=self.filename, file_title=self.trajname, log_stdout=False, port=self.url, log_config=get_log_config(), results_per_run=5, derived_parameters_per_run=5, multiproc=self.multiproc, ncores=self.ncores, wrap_mode=self.mode, use_pool=self.use_pool, fletcher32=self.fletcher32, complevel=self.complevel, complib=self.complib, shuffle=self.shuffle, pandas_append=self.pandas_append, pandas_format=self.pandas_format, encoding=self.encoding) traj = env.v_trajectory self.param_dict={} create_param_dict(self.param_dict) add_params(traj,self.param_dict) self.traj = traj self.env = env
def basic_and_skipping_duplicates_which_leads_to_one_remaining(self, slow_merge=False): self.envs=[] self.trajs = [] ntrajs = len(self.filenames) for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename) self.param_dict={} create_param_dict(self.param_dict) for irun in range(ntrajs): add_params(self.trajs[irun],self.param_dict) self.explore(self.trajs[0]) self.explore_trials_differently(self.trajs[1]) self.compare_explore_more_trials_with_removing_duplicates(self.trajs[2]) for irun in range(ntrajs): self.make_run(self.envs[irun]) for irun in range(ntrajs): self.trajs[irun].f_load_skeleton() self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890) self.trajs[1].f_store_item('rrororo33o333o3o3oo3') self.trajs[2].f_add_result('rrororo33o333o3o3oo3',1234567890) self.trajs[2].f_store_item('rrororo33o333o3o3oo3') run_name = pypetconstants.FORMATTED_RUN_NAME % 1 run_name2 = pypetconstants.FORMATTED_RUN_NAME % 5 self.trajs[1].f_add_result('%s.rrr' % run_name, 123) self.trajs[1].f_store_item('%s.rrr' % run_name) self.trajs[2].f_add_result('%s.rrr' % run_name2, 123) self.trajs[2].f_store_item('%s.rrr' % run_name2) self.trajs[1].f_add_result('Ignore.Me', 42) self.trajs[1].f_apar('Ignore.Me', 42) self.trajs[1].f_adpar('Ignore.Me', 42) ##f_merge without destroying the original trajectory merged_traj = self.trajs[0] merged_traj.f_merge(self.trajs[1], move_data=False, delete_other_trajectory=False, remove_duplicates=True, ignore_data=('results.Ignore.Me', 'parameters.Ignore.Me', 'derived_parameters.Ignore.Me'), slow_merge=slow_merge) merged_traj.f_load_skeleton() merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.compare_trajectories(merged_traj,self.trajs[2])
def basic_and_skipping_duplicates_which_leads_to_one_remaining(self): self.envs=[] self.trajs = [] ntrajs = len(self.filenames) for irun,filename in enumerate(self.filenames): if isinstance(filename,int): filename = self.filenames[filename] self.make_environment( irun, filename) self.param_dict={} create_param_dict(self.param_dict) for irun in range(ntrajs): add_params(self.trajs[irun],self.param_dict) self.explore(self.trajs[0]) self.explore_trials_differently(self.trajs[1]) self.compare_explore_more_trials_with_removing_duplicates(self.trajs[2]) for irun in range(ntrajs): self.make_run(self.envs[irun]) for irun in range(ntrajs): self.trajs[irun].f_load_skeleton() self.trajs[irun].f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.trajs[1].f_add_result('rrororo33o333o3o3oo3',1234567890) self.trajs[1].f_store_item('rrororo33o333o3o3oo3') self.trajs[2].f_add_result('rrororo33o333o3o3oo3',1234567890) self.trajs[2].f_store_item('rrororo33o333o3o3oo3') run_name = pypetconstants.FORMATTED_RUN_NAME % 1 run_name2 = pypetconstants.FORMATTED_RUN_NAME % 5 self.trajs[1].f_add_result('%s.rrr' % run_name, 123) self.trajs[1].f_store_item('%s.rrr' % run_name) self.trajs[2].f_add_result('%s.rrr' % run_name2, 123) self.trajs[2].f_store_item('%s.rrr' % run_name2) self.trajs[1].f_add_result('Ignore.Me', 42) self.trajs[1].f_apar('Ignore.Me', 42) self.trajs[1].f_adpar('Ignore.Me', 42) ##f_merge without destroying the original trajectory merged_traj = self.trajs[0] merged_traj.f_merge(self.trajs[1], move_data=False, delete_other_trajectory=False, remove_duplicates=True, ignore_data=('results.Ignore.Me', 'parameters.Ignore.Me', 'derived_parameters.Ignore.Me')) merged_traj.f_load_skeleton() merged_traj.f_load(load_parameters=pypetconstants.UPDATE_DATA, load_derived_parameters=pypetconstants.UPDATE_DATA, load_results=pypetconstants.UPDATE_DATA, load_other_data=pypetconstants.UPDATE_DATA) self.compare_trajectories(merged_traj,self.trajs[2])