def test_store_items_and_groups(self): traj = Trajectory(name='testtraj', filename=make_temp_dir('teststoreitems.hdf5')) traj.f_store() traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH) traj.f_add_result('testres', 42) traj.group1.f_set_annotations(Test=44) traj.f_store_items(['test','testres','group1']) traj2 = Trajectory(name=traj.v_name, add_time=False, filename=make_temp_dir('teststoreitems.hdf5')) traj2.f_load(load_parameters=2, load_results=2) traj.f_add_result('Im.stored.along.a.path', 43) traj.Im.stored.along.v_annotations['wtf'] =4444 traj.res.f_store_child('Im.stored.along.a.path') traj2.res.f_load_child('Im.stored.along.a.path', load_data=2) self.compare_trajectories(traj,traj2)
def test_loading_as_new(self): filename = make_temp_dir('asnew.h5') traj = Trajectory(name='TestPartial', filename=filename) traj.f_add_parameter('x', 3) traj.f_add_parameter('y', 2) traj.f_explore({'x': [12,3,44], 'y':[1,23,4]}) traj.f_store() traj = load_trajectory(name=traj.v_name, filename=filename) with self.assertRaises(TypeError): traj.f_shrink() traj = load_trajectory(name=traj.v_name, filename=filename, as_new=True, new_name='TestTraj', add_time=False) self.assertTrue(traj.v_name == 'TestTraj') self.assertTrue(len(traj) == 3) traj.f_shrink() self.assertTrue(len(traj) == 1)
def test_version_mismatch(self): traj = Trajectory(name='TestVERSION', filename=make_temp_dir('testversionmismatch.hdf5'), add_time=True) traj.f_add_parameter('group1.test',42) traj.f_add_result('testres', 42) traj.group1.f_set_annotations(Test=44) traj._version='0.1a.1' traj.f_store() traj2 = Trajectory(name=traj.v_name, add_time=False, filename=make_temp_dir('testversionmismatch.hdf5')) with self.assertRaises(pex.VersionMismatchError): traj2.f_load(load_parameters=2, load_results=2) traj2.f_load(load_parameters=2, load_results=2, force=True) self.compare_trajectories(traj,traj2) get_root_logger().info('Mismatch testing done!')
def test_loading_as_new(self): filename = make_temp_dir('asnew.h5') traj = Trajectory(name='TestPartial', filename=filename, add_time=True) traj.f_add_parameter('x', 3) traj.f_add_parameter('y', 2) traj.f_explore({'x': [12,3,44], 'y':[1,23,4]}) traj.f_store() traj = load_trajectory(name=traj.v_name, filename=filename) with self.assertRaises(TypeError): traj.f_shrink() traj = load_trajectory(name=traj.v_name, filename=filename, as_new=True, new_name='TestTraj', add_time=False) self.assertTrue(traj.v_name == 'TestTraj') self.assertEqual(len(traj), 3) self.assertEqual(len(traj._explored_parameters), 2) traj.f_shrink() self.assertTrue(len(traj) == 1)
def test_store_items_and_groups(self): traj = Trajectory(name='testtraj', filename=make_temp_dir('teststoreitems.hdf5'), add_time=True) traj.f_store() traj.f_add_parameter('group1.test',42, comment= 'TooLong' * pypetconstants.HDF5_STRCOL_MAX_COMMENT_LENGTH) traj.f_add_result('testres', 42) traj.group1.f_set_annotations(Test=44) traj.f_store_items(['test','testres','group1']) traj2 = Trajectory(name=traj.v_name, add_time=False, filename=make_temp_dir('teststoreitems.hdf5')) traj2.f_load(load_parameters=2, load_results=2) traj.f_add_result('Im.stored.along.a.path', 43) traj.Im.stored.along.v_annotations['wtf'] =4444 traj.res.f_store_child('Im.stored.along.a.path') traj2.res.f_load_child('Im.stored.along.a.path', load_data=2) self.compare_trajectories(traj,traj2)
def test_links_according_to_run(self): traj = Trajectory() traj.f_add_parameter('test.hi', 44) traj.f_explore({'hi': [1,2,3]}) traj.f_add_parameter_group('test.test.test2') traj.f_add_parameter_group('test2') traj.test2.f_add_link('test', traj.test) traj.v_idx = 1
def test_links_according_to_run(self): traj = Trajectory() traj.f_add_parameter('test.hi', 44) traj.f_explore({'hi': [1, 2, 3]}) traj.f_add_parameter_group('test.test.test2') traj.f_add_parameter_group('test2') traj.test2.f_add_link('test', traj.test) traj.v_idx = 1
def test_wildcard_search(self): traj = Trajectory(name='Testwildcard', filename=make_temp_dir('wilcard.hdf5'), add_time=True) traj.f_add_parameter('expl', 2) traj.f_explore({'expl': [1, 2, 3, 4]}) traj.f_add_result('wc2test.$.hhh', 333) traj.f_add_leaf('results.wctest.run_00000000.jjj', 42) traj.f_add_result('results.wctest.run_00000001.jjj', 43) traj.f_add_result('results.wctest.%s.jjj' % traj.f_wildcard('$', -1), 43) traj.v_crun = 1 self.assertTrue(traj.results.wctest['$'].jjj == 43) self.assertTrue(traj.results.wc2test.crun.hhh == 333) traj.f_store() get_root_logger().info('Removing child1') traj.f_remove_child('results', recursive=True) get_root_logger().info('Doing auto-load') traj.v_auto_load = True self.assertTrue(traj.results.wctest['$'].jjj == 43) self.assertTrue(traj.results.wc2test.crun.hhh == 333) get_root_logger().info('Removing child2') traj.f_remove_child('results', recursive=True) get_root_logger().info('auto-loading') traj.v_auto_load = True self.assertTrue(traj.results.wctest[-1].jjj == 43) self.assertTrue(traj.results.wc2test[-1].hhh == 333) get_root_logger().info('Removing child3') traj.f_remove_child('results', recursive=True) get_root_logger().info('auto-loading') traj.v_auto_load = True self.assertTrue(traj.results.wctest[1].jjj == 43) self.assertTrue(traj.results.wc2test[-1].hhh == 333) get_root_logger().info('Done with wildcard test')
def test_file_renaming(self): traj_name = 'test' traj = Trajectory('test', add_time=False) traj.f_add_parameter('x', 42) traj.f_explore({'x': [1,2,3]}) rename_string = '$traj_$set_$run' solution_1 = 'test_run_set_ALL_run_ALL' solution_2 = 'test_run_set_00000_run_00000000' renaming_1 = rename_log_file(rename_string, traj) self.assertEqual(renaming_1, solution_1) traj.v_idx = 0 renaming_2 = rename_log_file(rename_string, traj) self.assertEqual(renaming_2, solution_2)
def test_wildcard_search(self): traj = Trajectory(name='Testwildcard', filename=make_temp_dir('wilcard.hdf5'), add_time=True) traj.f_add_parameter('expl', 2) traj.f_explore({'expl':[1,2,3,4]}) traj.f_add_result('wc2test.$.hhh', 333) traj.f_add_leaf('results.wctest.run_00000000.jjj', 42) traj.f_add_result('results.wctest.run_00000001.jjj', 43) traj.f_add_result('results.wctest.%s.jjj' % traj.f_wildcard('$', -1), 43) traj.v_crun = 1 self.assertTrue(traj.results.wctest['$'].jjj==43) self.assertTrue(traj.results.wc2test.crun.hhh==333) traj.f_store() get_root_logger().info('Removing child1') traj.f_remove_child('results', recursive=True) get_root_logger().info('Doing auto-load') traj.v_auto_load = True self.assertTrue(traj.results.wctest['$'].jjj==43) self.assertTrue(traj.results.wc2test.crun.hhh==333) get_root_logger().info('Removing child2') traj.f_remove_child('results', recursive=True) get_root_logger().info('auto-loading') traj.v_auto_load = True self.assertTrue(traj.results.wctest[-1].jjj==43) self.assertTrue(traj.results.wc2test[-1].hhh==333) get_root_logger().info('Removing child3') traj.f_remove_child('results', recursive=True) get_root_logger().info('auto-loading') traj.v_auto_load = True self.assertTrue(traj.results.wctest[1].jjj==43) self.assertTrue(traj.results.wc2test[-1].hhh==333) get_root_logger().info('Done with wildcard test')
def test_find_in_all_runs_with_links(self): traj = Trajectory() traj.f_add_parameter('FloatParam') traj.par.FloatParam = 4.0 self.explore_dict = {'FloatParam': [1.0, 1.1, 1.2, 1.3]} traj.f_explore(self.explore_dict) self.assertTrue(len(traj) == 4) traj.f_add_result('results.runs.run_00000000.sub.resulttest', 42) traj.f_add_result('results.runs.run_00000001.sub.resulttest', 43) traj.f_add_result('results.runs.run_00000002.sub.resulttest', 44) traj.f_add_result('results.runs.run_00000002.sub.resulttest2', 42) traj.f_add_result('results.runs.run_00000003.sub.resulttest2', 43) traj.f_add_derived_parameter( 'derived_parameters.runs.run_00000002.testing', 44) res_dict = traj.f_get_from_runs('resulttest', fast_access=True) self.assertTrue(len(res_dict) == 3) self.assertTrue(res_dict['run_00000001'] == 43) self.assertTrue('run_00000003' not in res_dict) res_dict = traj.f_get_from_runs(name='sub.resulttest2', use_indices=True) self.assertTrue(len(res_dict) == 2) self.assertTrue(res_dict[3] is traj.f_get('run_00000003.resulttest2')) self.assertTrue(1 not in res_dict) traj.res.runs.r_0.f_add_link('resulttest2', traj.r_1.f_get('resulttest')) res_dict = traj.f_get_from_runs(name='resulttest2', use_indices=True) self.assertTrue(len(res_dict) == 3) self.assertTrue(res_dict[0] is traj.f_get('run_00000001.resulttest')) self.assertTrue(1 not in res_dict) res_dict = traj.f_get_from_runs(name='resulttest2', use_indices=True, with_links=False) self.assertTrue(len(res_dict) == 2) self.assertTrue(0 not in res_dict) self.assertTrue(1 not in res_dict)
def test_loading_and_storing_empty_containers(self): filename = make_temp_dir('empty_containers.hdf5') traj = Trajectory(filename=filename, add_time=True) # traj.f_add_parameter('empty.dict', {}) # traj.f_add_parameter('empty.list', []) traj.f_add_parameter(ArrayParameter, 'empty.tuple', ()) traj.f_add_parameter(ArrayParameter, 'empty.array', np.array([], dtype=float)) spsparse_csc = spsp.csc_matrix((2, 10)) spsparse_csr = spsp.csr_matrix((6660, 660)) spsparse_bsr = spsp.bsr_matrix((3330, 2220)) spsparse_dia = spsp.dia_matrix((1230, 1230)) traj.f_add_parameter(SparseParameter, 'empty.csc', spsparse_csc) traj.f_add_parameter(SparseParameter, 'empty.csr', spsparse_csr) traj.f_add_parameter(SparseParameter, 'empty.bsr', spsparse_bsr) traj.f_add_parameter(SparseParameter, 'empty.dia', spsparse_dia) traj.f_add_result(SparseResult, 'empty.all', dict={}, list=[], series=pd.Series(), frame=pd.DataFrame(), panel=pd.Panel(), **traj.par.f_to_dict(short_names=True, fast_access=True)) traj.f_store() newtraj = load_trajectory(index=-1, filename=filename) newtraj.f_load(load_data=2) epg = newtraj.par.empty self.assertTrue(type(epg.tuple) is tuple) self.assertTrue(len(epg.tuple) == 0) self.assertTrue(type(epg.array) is np.ndarray) self.assertTrue(epg.array.size == 0) self.assertTrue(spsp.isspmatrix_csr(epg.csr)) self.assertTrue(epg.csr.size == 0) self.assertTrue(spsp.isspmatrix_csc(epg.csc)) self.assertTrue(epg.csc.size == 0) self.assertTrue(spsp.isspmatrix_bsr(epg.bsr)) self.assertTrue(epg.bsr.size == 0) self.assertTrue(spsp.isspmatrix_dia(epg.dia)) self.assertTrue(epg.dia.size == 0) self.compare_trajectories(traj, newtraj)
def test_find_in_all_runs_with_links(self): traj = Trajectory() traj.f_add_parameter('FloatParam') traj.par.FloatParam=4.0 self.explore_dict = {'FloatParam':[1.0,1.1,1.2,1.3]} traj.f_explore(self.explore_dict) self.assertTrue(len(traj) == 4) traj.f_add_result('results.runs.run_00000000.sub.resulttest', 42) traj.f_add_result('results.runs.run_00000001.sub.resulttest', 43) traj.f_add_result('results.runs.run_00000002.sub.resulttest', 44) traj.f_add_result('results.runs.run_00000002.sub.resulttest2', 42) traj.f_add_result('results.runs.run_00000003.sub.resulttest2', 43) traj.f_add_derived_parameter('derived_parameters.runs.run_00000002.testing', 44) res_dict = traj.f_get_from_runs('resulttest', fast_access=True) self.assertTrue(len(res_dict)==3) self.assertTrue(res_dict['run_00000001']==43) self.assertTrue('run_00000003' not in res_dict) res_dict = traj.f_get_from_runs(name='sub.resulttest2', use_indices=True) self.assertTrue(len(res_dict)==2) self.assertTrue(res_dict[3] is traj.f_get('run_00000003.resulttest2')) self.assertTrue(1 not in res_dict) traj.res.runs.r_0.f_add_link('resulttest2', traj.r_1.f_get('resulttest')) res_dict = traj.f_get_from_runs(name='resulttest2', use_indices=True) self.assertTrue(len(res_dict)==3) self.assertTrue(res_dict[0] is traj.f_get('run_00000001.resulttest')) self.assertTrue(1 not in res_dict) res_dict = traj.f_get_from_runs(name='resulttest2', use_indices=True, with_links=False) self.assertTrue(len(res_dict)==2) self.assertTrue(0 not in res_dict) self.assertTrue(1 not in res_dict)
def test_link_creation(self): traj = Trajectory() traj.f_add_parameter_group('test.test3') traj.f_add_parameter_group('test2') with self.assertRaises(AttributeError): traj.par.f_add_link('test', traj.test2) with self.assertRaises(ValueError): traj.f_add_link('parameters', traj.test) with self.assertRaises(ValueError): traj.f_add_link('kkkk', PickleResult('fff', 555)) traj.test.f_add_link('circle1' , traj.test2) traj.test2.f_add_link('circle2' , traj.test) self.assertTrue(traj.test.circle1.circle2.circle1.circle2 is traj.test) traj.f_add_link('hh', traj.test) traj.par.f_add_link('overview', traj.test) with self.assertRaises(ValueError): traj.f_add_link('overview', traj.test) with self.assertRaises(ValueError): traj.f_add_link('v_crun') with self.assertRaises(ValueError): traj.par.f_add_link('gg', traj) with self.assertRaises(AttributeError): traj.f_add_parameter('test.circle1.testy', 33) traj.par.f_add_link('gg', traj.circle1) self.assertTrue(traj.gg is traj.test2) self.assertTrue(traj.test2.test3 is traj.par.test.test3) traj.f_add_link(traj.test3) self.assertTrue('test3' in traj._links)
def test_link_creation(self): traj = Trajectory() traj.f_add_parameter_group('test.test3') traj.f_add_parameter_group('test2') with self.assertRaises(AttributeError): traj.par.f_add_link('test', traj.test2) with self.assertRaises(ValueError): traj.f_add_link('parameters', traj.test) with self.assertRaises(ValueError): traj.f_add_link('kkkk', PickleResult('fff', 555)) traj.test.f_add_link('circle1', traj.test2) traj.test2.f_add_link('circle2', traj.test) self.assertTrue(traj.test.circle1.circle2.circle1.circle2 is traj.test) traj.f_add_link('hh', traj.test) traj.par.f_add_link('overview', traj.test) with self.assertRaises(ValueError): traj.f_add_link('overview', traj.test) with self.assertRaises(ValueError): traj.f_add_link('v_crun') with self.assertRaises(ValueError): traj.par.f_add_link('gg', traj) with self.assertRaises(AttributeError): traj.f_add_parameter('test.circle1.testy', 33) traj.par.f_add_link('gg', traj.circle1) self.assertTrue(traj.gg is traj.test2) self.assertTrue(traj.test2.test3 is traj.par.test.test3) traj.f_add_link(traj.test3) self.assertTrue('test3' in traj._links)
def test_loading_and_storing_empty_containers(self): filename = make_temp_dir('empty_containers.hdf5') traj = Trajectory(filename=filename) # traj.f_add_parameter('empty.dict', {}) # traj.f_add_parameter('empty.list', []) traj.f_add_parameter(ArrayParameter, 'empty.tuple', ()) traj.f_add_parameter(ArrayParameter, 'empty.array', np.array([], dtype=float)) spsparse_csc = spsp.csc_matrix((2,10)) spsparse_csr = spsp.csr_matrix((6660,660)) spsparse_bsr = spsp.bsr_matrix((3330,2220)) spsparse_dia = spsp.dia_matrix((1230,1230)) traj.f_add_parameter(SparseParameter, 'empty.csc', spsparse_csc) traj.f_add_parameter(SparseParameter, 'empty.csr', spsparse_csr) traj.f_add_parameter(SparseParameter, 'empty.bsr', spsparse_bsr) traj.f_add_parameter(SparseParameter, 'empty.dia', spsparse_dia) traj.f_add_result(SparseResult, 'empty.all', dict={}, list=[], series = pd.Series(), frame = pd.DataFrame(), panel = pd.Panel(), **traj.par.f_to_dict(short_names=True, fast_access=True)) traj.f_store() newtraj = load_trajectory(index=-1, filename=filename) newtraj.f_load(load_data=2) epg = newtraj.par.empty self.assertTrue(type(epg.tuple) is tuple) self.assertTrue(len(epg.tuple) == 0) self.assertTrue(type(epg.array) is np.ndarray) self.assertTrue(epg.array.size == 0) self.assertTrue(spsp.isspmatrix_csr(epg.csr)) self.assertTrue(epg.csr.size == 0) self.assertTrue(spsp.isspmatrix_csc(epg.csc)) self.assertTrue(epg.csc.size == 0) self.assertTrue(spsp.isspmatrix_bsr(epg.bsr)) self.assertTrue(epg.bsr.size == 0) self.assertTrue(spsp.isspmatrix_dia(epg.dia)) self.assertTrue(epg.dia.size == 0) self.compare_trajectories(traj, newtraj)
def test_get_all_not_links(self): traj = Trajectory() traj.f_add_parameter('test.hi', 44) traj.f_explore({'hi': [1,2,3]}) traj.f_add_parameter_group('test.test.test2') traj.f_add_parameter_group('test2') traj.test2.f_add_link('test', traj.test) nodes = traj.f_get_all('par.test') self.assertTrue(len(nodes) == 2) nodes = traj.f_get_all('par.test', shortcuts=False) self.assertTrue(len(nodes) == 1) traj.f_set_crun(0) traj.f_add_group('f.$.h') traj.f_add_group('f.$.g.h') traj.f_add_group('f.$.i') traj.crun.i.f_add_link('h', traj.crun.h) nodes = traj.f_get_all('$.h') self.assertTrue(len(nodes)==2) nodes = traj.f_get_all('h') self.assertTrue(len(nodes)==2) traj.v_idx = -1 nodes = traj.f_get_all('h') self.assertTrue(len(nodes)==2)
def test_get_all_not_links(self): traj = Trajectory() traj.f_add_parameter('test.hi', 44) traj.f_explore({'hi': [1, 2, 3]}) traj.f_add_parameter_group('test.test.test2') traj.f_add_parameter_group('test2') traj.test2.f_add_link('test', traj.test) nodes = traj.f_get_all('par.test') self.assertTrue(len(nodes) == 2) nodes = traj.f_get_all('par.test', shortcuts=False) self.assertTrue(len(nodes) == 1) traj.f_set_crun(0) traj.f_add_group('f.$.h') traj.f_add_group('f.$.g.h') traj.f_add_group('f.$.i') traj.crun.i.f_add_link('h', traj.crun.h) nodes = traj.f_get_all('$.h') self.assertTrue(len(nodes) == 2) nodes = traj.f_get_all('h') self.assertTrue(len(nodes) == 2) traj.v_idx = -1 nodes = traj.f_get_all('h') self.assertTrue(len(nodes) == 2)
def test_file_size_many_params(self): filename = make_temp_dir('filesize.hdf5') traj = Trajectory(filename=filename, overwrite_file=True, add_time=False) npars = 700 traj.f_store() for irun in range(npars): par = traj.f_add_parameter('test.test%d' % irun, 42+irun, comment='duh!') traj.f_store_item(par) size = os.path.getsize(filename) size_in_mb = size/1000000. get_root_logger().info('Size is %sMB' % str(size_in_mb)) self.assertTrue(size_in_mb < 10.0, 'Size is %sMB > 10MB' % str(size_in_mb))
def main(): """Main function to protect the *entry point* of the program.""" # Load settings from file settings_file = 'pypet_settings.pkl' settings = load_obj(settings_file) # Print settings dictionary print('\nSettings dictionary:') for key, value in settings.items(): print(key, ' : ', value) print('\nParameters to explore:') for key, value in settings.items(): if isinstance(value, list): print(key, ' : ', value) # Create new folder to store results traj_dir = os.getcwd() # Read output path (if provided) if len(sys.argv) > 1: # Add trailing slash if missing dir_provided = os.path.join(sys.argv[1], '') # Check if provided directory exists if os.path.isdir(dir_provided): # Convert to full path traj_dir = os.path.abspath(dir_provided) else: print( 'WARNING: Output path not found, current directory will be used instead' ) else: print( 'WARNING: Output path not provided, current directory will be used instead' ) # Add time stamp (the final '' is to make sure there is a trailing slash) traj_dir = os.path.join(traj_dir, datetime.now().strftime("%Y_%m_%d_%Hh%Mm%Ss"), '') # Create directory with time stamp os.makedirs(traj_dir) # Change current directory to the one containing the trajectory files os.chdir(traj_dir) print('Trajectory and results will be stored in: {0}'.format(traj_dir)) # Create new pypet Trajectory object traj_filename = 'traj.hdf5' traj_fullpath = os.path.join(traj_dir, traj_filename) traj = Trajectory(filename=traj_fullpath) # ------------------------------------------------------------------- # Add config parameters (those that DO NOT influence the final result of the experiment) traj.f_add_config('debug', False, comment='Activate debug mode') # #traj.f_add_config('max_mem_frac', 0.7, comment='Fraction of global GPU memory to use') # Set up trajectory parameters param_to_explore = {} for key, val in settings.items(): if isinstance(val, list): param_to_explore[key] = val traj.f_add_parameter(key, val[0]) else: traj.f_add_parameter(key, val) # Define parameter combinations to explore (a trajectory in # the parameter space). The second argument, the tuple, specifies the order # of the cartesian product. # The variable on the right most side changes fastest and defines the # 'inner for-loop' of the cartesian product explore_dict = cartesian_product(param_to_explore, tuple(param_to_explore.keys())) print(explore_dict) traj.f_explore(explore_dict) # Store trajectory parameters to disk pypet_utils.print_traj_leaves(traj, 'parameters', file=os.path.join(traj_dir, 'traj_parameters.txt')) # Store trajectory traj.f_store() # Define PBS script bash_lines = '\n'.join([ '#! /bin/bash', '#PBS -P InfoDynFuncStruct', '#PBS -l select=1:ncpus=1:mem=1GB', #'#PBS -l select=1:ncpus=1:ngpus=1:mem=1GB', '#PBS -M [email protected]', '#PBS -m abe', 'module load java', 'module load python/3.5.1', 'module load cuda/8.0.44', 'source /project/RDS-FEI-InfoDynFuncStruct-RW/Leo/idtxl_env/bin/activate', 'cd ${traj_dir}', 'python ${python_script_path} ${traj_dir} ${traj_filename} ${file_prefix} $PBS_ARRAY_INDEX' ]) # Save PBS script file (automatically generated) bash_script_name = 'run_python_script.pbs' job_script_path = os.path.join(traj_dir, bash_script_name) with open(job_script_path, 'w', newline='\n') as bash_file: bash_file.writelines(bash_lines) # Run job array job_walltime_hours = 0 job_walltime_minutes = 5 #after_job_array_ends = 1573895 job_settings = { 'N': 'run_traj', 'l': 'walltime={0}:{1}:00'.format(job_walltime_hours, job_walltime_minutes), #'W': 'depend=afteranyarray:{0}[]'.format(after_job_array_ends), 'q': 'defaultQ' } if len(traj.f_get_run_names()) > 1: job_settings['J'] = '{0}-{1}'.format(0, len(traj.f_get_run_names()) - 1) job_args = { 'python_script_path': '/project/RDS-FEI-InfoDynFuncStruct-RW/Leo/inference/hpc_pypet_single_run.py', 'traj_dir': traj_dir, 'traj_filename': traj_filename, 'file_prefix': 'none' } run_job_array(job_script_path, job_settings, job_args)
__author__ = 'Robert Meyer' from pypet import Trajectory, Result, Parameter traj = Trajectory() # There are more ways to add data, # 1st the standard way: traj.f_add_parameter('x', 1, comment='I am the first dimension!') # 2nd by providing a new parameter/result instance, be aware that the data is added where # you specify it. There are no such things as shortcuts for parameter creation: traj.parameters.y = Parameter('y', 1, comment='I am the second dimension!') # 3rd as before, but if our new leaf has NO name it will be renamed accordingly: traj.parameters.t = Parameter('', 1, comment='Third dimension') # See: print('t=' + str(traj.t)) # What happens if our new parameter's name does not match the name passed to the constructor? traj.parameters.subgroup = Parameter('v', 2, comment='Fourth dimension') # Well, since 'subgroup' != 'v', 'subgroup' becomes just another group node created on the fly print(traj.parameters.subgroup) # This even works for already existing groups and with the well known *dot* notation: traj.parameters = Parameter('subgroup.subsubgroup.w', 2) # See print('w='+str(traj.par.subgroup.subsubgroup.w)) # There's a lazy version which does not require a constructor. # This can be turned on via traj.v_lazy_adding = True
def main(): # No environment here ;-) filename = os.path.join('experiments', 'example_20.hdf5') traj = Trajectory('onemax', filename=filename, overwrite_file=True) # ------- Add parameters ------- # traj.f_add_parameter('popsize', 100) traj.f_add_parameter('CXPB', 0.5) traj.f_add_parameter('MUTPB', 0.2) traj.f_add_parameter('NGEN', 20) traj.f_add_parameter('generation', 0) traj.f_add_parameter('ind_idx', 0) traj.f_add_parameter('ind_len', 50) traj.f_add_parameter('indpb', 0.005) traj.f_add_parameter('tournsize', 3) traj.f_add_parameter('seed', 42) traj.f_store(only_init=True) # ------- Create and register functions with DEAP ------- # creator.create("FitnessMax", base.Fitness, weights=(1.0, )) creator.create("Individual", list, fitness=creator.FitnessMax) toolbox = base.Toolbox() # Attribute generator toolbox.register("attr_bool", random.randint, 0, 1) # Structure initializers toolbox.register("individual", tools.initRepeat, creator.Individual, toolbox.attr_bool, traj.ind_len) toolbox.register("population", tools.initRepeat, list, toolbox.individual) # Operator registering toolbox.register("mate", tools.cxTwoPoint) toolbox.register("mutate", tools.mutFlipBit, indpb=traj.indpb) toolbox.register("select", tools.selTournament, tournsize=traj.tournsize) toolbox.register("evaluate", eval_wrapper) pool = multip.Pool(4) toolbox.register("map", pool.map) # We use the pool's map function! # ------- Initialize Population -------- # random.seed(traj.seed) pop = toolbox.population(n=traj.popsize) CXPB, MUTPB, NGEN = traj.CXPB, traj.MUTPB, traj.NGEN start_idx = 0 # We need to count executed runs print("Start of evolution") for g in range(traj.NGEN): print("-- Generation %i --" % g) # Determine individuals that need to be evaluated eval_pop = [ind for ind in pop if not ind.fitness.valid] # Add as many explored runs as individuals that need to be evaluated traj.f_expand( cartesian_product({ 'generation': [g], 'ind_idx': range(len(eval_pop)) })) # We need to make the storage service multiprocessing safe mc = MultiprocContext(traj, wrap_mode='QUEUE') mc.f_start() # Create a single iterable to be passed to our fitness function (wrapper). # `yields='copy'` is important, the pool's `map` function will # go over the whole iterator at once and store it in memory. # So for every run we need a copy of the trajectory. # Alternatively, you could use `yields='self'` and use the pool's `imap` function. zip_iterable = izip(traj.f_iter_runs(start_idx, yields='copy'), eval_pop) fitnesses = toolbox.map(eval_wrapper, zip_iterable) # fitnesses is just a list of tuples [(fitness,), ...] for idx, fitness in enumerate(fitnesses): # Update fitnesses eval_pop[idx].fitness.values = fitness # Finalize the multiproc wrapper mc.f_finalize() # Update start index start_idx += len(eval_pop) print(" Evaluated %i individuals" % len(eval_pop)) # Gather all the fitnesses in one list and print the stats fits = [ind.fitness.values[0] for ind in pop] length = len(pop) mean = sum(fits) / length sum2 = sum(x * x for x in fits) std = abs(sum2 / length - mean**2)**0.5 print(" Min %s" % min(fits)) print(" Max %s" % max(fits)) print(" Avg %s" % mean) print(" Std %s" % std) # ------- Create the next generation by crossover and mutation -------- # if g < traj.NGEN - 1: # not necessary for the last generation # Select the next generation individuals offspring = toolbox.select(pop, len(pop)) # Clone the selected individuals offspring = list(map(toolbox.clone, offspring)) # Apply crossover and mutation on the offspring for child1, child2 in zip(offspring[::2], offspring[1::2]): if random.random() < CXPB: toolbox.mate(child1, child2) del child1.fitness.values del child2.fitness.values for mutant in offspring: if random.random() < MUTPB: toolbox.mutate(mutant) del mutant.fitness.values # The population is entirely replaced by the offspring pop[:] = offspring # Stop the multiprocessing pool pool.close() pool.join() print("-- End of (successful) evolution --") best_ind = tools.selBest(pop, 1)[0] print("Best individual is %s, %s" % (best_ind, best_ind.fitness.values)) traj.f_store() # And store all the rest of the data
def test_delete_whole_subtrees(self): filename = make_temp_dir('testdeltree.hdf5') traj = Trajectory(name='TestDelete', filename=filename, large_overview_tables=True, add_time=True) res = traj.f_add_result('mytest.yourtest.test', a='b', c='d') dpar = traj.f_add_derived_parameter('mmm.gr.dpdp', 666) res = traj.f_add_result('hhh.ll', a='b', c='d') res = traj.f_add_derived_parameter('hhh.gg', 555) traj.f_store() with pt.open_file(filename) as fh: daroot = fh.root._f_get_child(traj.v_name) dpar_table = daroot.overview.derived_parameters_overview self.assertTrue(len(dpar_table) == 2) res_table = daroot.overview.results_overview self.assertTrue((len(res_table)) == 2) with self.assertRaises(TypeError): traj.f_remove_item(traj.yourtest) with self.assertRaises(TypeError): traj.f_delete_item(traj.yourtest) traj.f_remove_item(traj.yourtest, recursive=True) self.assertTrue('mytest' in traj) self.assertTrue('yourtest' not in traj) traj.f_load(load_data=2) self.assertTrue('yourtest.test' in traj) traj.f_delete_item(traj.yourtest, recursive=True, remove_from_trajectory=True) traj.f_delete_item(traj.mmm, recursive=True, remove_from_trajectory=True) traj.f_load(load_data=2) self.assertTrue('yourtest.test' not in traj) self.assertTrue('yourtest' not in traj) with pt.open_file(filename) as fh: daroot = fh.root._f_get_child(traj.v_name) dpar_table = daroot.overview.derived_parameters_overview self.assertTrue(len(dpar_table) == 2) res_table = daroot.overview.results_overview self.assertTrue((len(res_table)) == 2) traj.f_add_parameter('ggg', 43) traj.f_add_parameter('hhh.mmm', 45) traj.f_add_parameter('jjj', 55) traj.f_add_parameter('hhh.nnn', 55555) traj.f_explore({'ggg':[1,2,3]}) traj.f_store() with pt.open_file(filename) as fh: daroot = fh.root._f_get_child(traj.v_name) par_table = daroot.overview.parameters_overview self.assertTrue(len(par_table) == 4) traj.f_delete_item('par.hhh', recursive=True, remove_from_trajectory=True) traj.f_add_parameter('saddsdfdsfd', 111) traj.f_store() with pt.open_file(filename) as fh: daroot = fh.root._f_get_child(traj.v_name) par_table = daroot.overview.parameters_overview self.assertTrue(len(par_table) == 5)
__author__ = 'Robert Meyer' import os # To allow pathnames under Windows and Linux from pypet import Trajectory, NotUniqueNodeError # We first generate a new Trajectory filename = os.path.join('hdf5', 'example_02.hdf5') traj = Trajectory('Example', filename=filename, comment='Access and Storage!') # We add our first parameter with the data 'Harrison Ford' traj.f_add_parameter('starwars.characters.han_solo', 'Harrison Ford') # This automatically added the groups 'starwars' and the subgroup 'characters' # Let's get the characters subgroup characters = traj.parameters.starwars.characters # Since characters is unique we could also use shortcuts characters = traj.characters # Or the get method characters = traj.f_get('characters') # Or square brackets characters = traj['characters'] # Lets add another character characters.f_add_parameter('luke_skywalker', 'Mark Hamill', comment='May the force be with you!')
__author__ = 'Robert Meyer' import os # To allow pathnames under Windows and Linux from pypet import Trajectory, NotUniqueNodeError # We first generate a new Trajectory filename = os.path.join('hdf5', 'example_02.hdf5') traj = Trajectory('Example', filename=filename, overwrite_file=True, comment='Access and Storage!') # We add our first parameter with the data 'Harrison Ford' traj.f_add_parameter('starwars.characters.han_solo', 'Harrison Ford') # This automatically added the groups 'starwars' and the subgroup 'characters' # Let's get the characters subgroup characters = traj.parameters.starwars.characters # Since characters is unique we could also use shortcuts characters = traj.characters # Or the get method characters = traj.f_get('characters') # Or square brackets characters = traj['characters'] # Lets add another character characters.f_add_parameter('luke_skywalker',
__author__ = 'Robert Meyer' from pypet import Trajectory, Result, Parameter traj = Trajectory() # There are more ways to add data, # 1st the standard way: traj.f_add_parameter('x', 1, comment='I am the first dimension!') # 2nd by providing a new parameter/result instance, be aware that the data is added where # you specify it. There are no such things as shortcuts for parameter creation: traj.parameters.y = Parameter('y', 1, comment='I am the second dimension!') # 3rd as before, but if our new leaf has NO name it will be renamed accordingly: traj.parameters.t = Parameter('', 1, comment='Third dimension') # See: print('t=' + str(traj.t)) # This also works for adding groups on the fly and with the well known *dot* notation: traj.parameters.subgroup = Parameter('subgroup.subsubgroup.w', 2) # See print('w=' + str(traj.par.subgroup.subsubgroup.w)) # Finally, there's one more thing. Using this notation we can also add links. # Simply use the `=` assignment with objects that already exist in your trajectory: traj.mylink = traj.f_get('x') # now `mylink` links to parameter `x`, also fast access works: print('Linking to x gives: ' + str(traj.mylink))
def test_delete_whole_subtrees(self): filename = make_temp_dir('testdeltree.hdf5') traj = Trajectory(name='TestDelete', filename=filename, large_overview_tables=True) res = traj.f_add_result('mytest.yourtest.test', a='b', c='d') dpar = traj.f_add_derived_parameter('mmm.gr.dpdp', 666) res = traj.f_add_result('hhh.ll', a='b', c='d') res = traj.f_add_derived_parameter('hhh.gg', 555) traj.f_store() with ptcompat.open_file(filename) as fh: daroot = ptcompat.get_child(fh.root, traj.v_name) dpar_table = daroot.overview.derived_parameters_overview self.assertTrue(len(dpar_table) == 2) res_table = daroot.overview.results_overview self.assertTrue((len(res_table)) == 2) with self.assertRaises(TypeError): traj.f_remove_item(traj.yourtest) with self.assertRaises(TypeError): traj.f_delete_item(traj.yourtest) traj.f_remove_item(traj.yourtest, recursive=True) self.assertTrue('mytest' in traj) self.assertTrue('yourtest' not in traj) traj.f_load(load_data=2) self.assertTrue('yourtest.test' in traj) traj.f_delete_item(traj.yourtest, recursive=True, remove_from_trajectory=True) traj.f_delete_item(traj.mmm, recursive=True, remove_from_trajectory=True) traj.f_load(load_data=2) self.assertTrue('yourtest.test' not in traj) self.assertTrue('yourtest' not in traj) with ptcompat.open_file(filename) as fh: daroot = ptcompat.get_child(fh.root, traj.v_name) dpar_table = daroot.overview.derived_parameters_overview self.assertTrue(len(dpar_table) == 2) res_table = daroot.overview.results_overview self.assertTrue((len(res_table)) == 2) traj.f_add_parameter('ggg', 43) traj.f_add_parameter('hhh.mmm', 45) traj.f_add_parameter('jjj', 55) traj.f_add_parameter('hhh.nnn', 55555) traj.f_explore({'ggg':[1,2,3]}) traj.f_store() with ptcompat.open_file(filename) as fh: daroot = ptcompat.get_child(fh.root, traj.v_name) par_table = daroot.overview.parameters_overview self.assertTrue(len(par_table) == 4) traj.f_delete_item('par.hhh', recursive=True, remove_from_trajectory=True) traj.f_add_parameter('saddsdfdsfd', 111) traj.f_store() with ptcompat.open_file(filename) as fh: daroot = ptcompat.get_child(fh.root, traj.v_name) par_table = daroot.overview.parameters_overview self.assertTrue(len(par_table) == 5) # with self.assertRaises(TypeError): # # We cannot delete something containing an explored parameter # traj.f_delete_item('par', recursive=True) with self.assertRaises(TypeError): traj.f_delete_item('ggg')
def main(): # No environment here ;-) filename = os.path.join('experiments', 'example_20.hdf5') traj = Trajectory('onemax', filename=filename, overwrite_file=True) # ------- Add parameters ------- # traj.f_add_parameter('popsize', 100) traj.f_add_parameter('CXPB', 0.5) traj.f_add_parameter('MUTPB', 0.2) traj.f_add_parameter('NGEN', 20) traj.f_add_parameter('generation', 0) traj.f_add_parameter('ind_idx', 0) traj.f_add_parameter('ind_len', 50) traj.f_add_parameter('indpb', 0.005) traj.f_add_parameter('tournsize', 3) traj.f_add_parameter('seed', 42) traj.f_store(only_init=True) # ------- Create and register functions with DEAP ------- # creator.create("FitnessMax", base.Fitness, weights=(1.0,)) creator.create("Individual", list, fitness=creator.FitnessMax) toolbox = base.Toolbox() # Attribute generator toolbox.register("attr_bool", random.randint, 0, 1) # Structure initializers toolbox.register("individual", tools.initRepeat, creator.Individual, toolbox.attr_bool, traj.ind_len) toolbox.register("population", tools.initRepeat, list, toolbox.individual) # Operator registering toolbox.register("mate", tools.cxTwoPoint) toolbox.register("mutate", tools.mutFlipBit, indpb=traj.indpb) toolbox.register("select", tools.selTournament, tournsize=traj.tournsize) toolbox.register("evaluate", eval_wrapper) pool = multip.Pool(4) toolbox.register("map", pool.map) # We use the pool's map function! # ------- Initialize Population -------- # random.seed(traj.seed) pop = toolbox.population(n=traj.popsize) CXPB, MUTPB, NGEN = traj.CXPB, traj.MUTPB, traj.NGEN start_idx = 0 # We need to count executed runs print("Start of evolution") for g in range(traj.NGEN): print("-- Generation %i --" % g) # Determine individuals that need to be evaluated eval_pop = [ind for ind in pop if not ind.fitness.valid] # Add as many explored runs as individuals that need to be evaluated traj.f_expand(cartesian_product({'generation': [g], 'ind_idx': range(len(eval_pop))})) # We need to make the storage service multiprocessing safe mc = MultiprocContext(traj, wrap_mode='QUEUE') mc.f_start() # Create a single iterable to be passed to our fitness function (wrapper). # `yields='copy'` is important, the pool's `map` function will # go over the whole iterator at once and store it in memory. # So for every run we need a copy of the trajectory. # Alternatively, you could use `yields='self'` and use the pool's `imap` function. zip_iterable = izip(traj.f_iter_runs(start_idx, yields='copy'), eval_pop) fitnesses = toolbox.map(eval_wrapper, zip_iterable) # fitnesses is just a list of tuples [(fitness,), ...] for idx, fitness in enumerate(fitnesses): # Update fitnesses eval_pop[idx].fitness.values = fitness # Finalize the multiproc wrapper mc.f_finalize() # Update start index start_idx += len(eval_pop) print(" Evaluated %i individuals" % len(eval_pop)) # Gather all the fitnesses in one list and print the stats fits = [ind.fitness.values[0] for ind in pop] length = len(pop) mean = sum(fits) / length sum2 = sum(x*x for x in fits) std = abs(sum2 / length - mean**2)**0.5 print(" Min %s" % min(fits)) print(" Max %s" % max(fits)) print(" Avg %s" % mean) print(" Std %s" % std) # ------- Create the next generation by crossover and mutation -------- # if g < traj.NGEN -1: # not necessary for the last generation # Select the next generation individuals offspring = toolbox.select(pop, len(pop)) # Clone the selected individuals offspring = list(map(toolbox.clone, offspring)) # Apply crossover and mutation on the offspring for child1, child2 in zip(offspring[::2], offspring[1::2]): if random.random() < CXPB: toolbox.mate(child1, child2) del child1.fitness.values del child2.fitness.values for mutant in offspring: if random.random() < MUTPB: toolbox.mutate(mutant) del mutant.fitness.values # The population is entirely replaced by the offspring pop[:] = offspring # Stop the multiprocessing pool pool.close() pool.join() print("-- End of (successful) evolution --") best_ind = tools.selBest(pop, 1)[0] print("Best individual is %s, %s" % (best_ind, best_ind.fitness.values)) traj.f_store() # And store all the rest of the data