def _batch_run(dir='unnamed', batch_id='template', space=None, save_data_in_hdf5=False, single_method=single_run, process_method=null_processing, post_process_method=None, final_process_method=None, multiprocessing=True, resumable=True, overwrite=False, sim_config=None, params=None, config=None, post_kwargs={}, run_kwargs={}): saved_args = locals() # print(locals()) traj_name = f'{batch_id}_traj' parent_dir_path = f'{BatchRunFolder}/{dir}' dir_path = os.path.join(parent_dir_path, batch_id) plot_path = os.path.join(dir_path, f'{batch_id}.pdf') data_path = os.path.join(dir_path, f'{batch_id}.csv') filename = f'{dir_path}/{batch_id}.hdf5' build_new = True if os.path.exists(parent_dir_path) and os.path.exists( dir_path) and overwrite == False: build_new = False try: print('Trying to resume existing trajectory') env = Environment(continuable=True) env.resume(trajectory_name=traj_name, resume_folder=dir_path) print('Resumed existing trajectory') build_new = False except: try: print('Trying to load existing trajectory') traj = load_trajectory(filename=filename, name=traj_name, load_all=0) env = Environment(trajectory=traj) traj.f_load(index=None, load_parameters=2, load_results=0) traj.f_expand(space) print('Loaded existing trajectory') build_new = False except: print( 'Neither of resuming or expanding of existing trajectory worked' ) if build_new: if multiprocessing: multiproc = True resumable = False wrap_mode = pypetconstants.WRAP_MODE_QUEUE else: multiproc = False resumable = True wrap_mode = pypetconstants.WRAP_MODE_LOCK # try: print('Trying to create novel environment') env = Environment( trajectory=traj_name, filename=filename, file_title=batch_id, comment=f'{batch_id} batch run!', large_overview_tables=True, overwrite_file=True, resumable=False, resume_folder=dir_path, multiproc=multiproc, ncores=4, use_pool= True, # Our runs are inexpensive we can get rid of overhead by using a pool freeze_input= True, # We can avoid some overhead by freezing the input to the pool wrap_mode=wrap_mode, graceful_exit=True) traj = env.traj print('Created novel environment') fly_params, env_params, sim_params = sim_config[ 'fly_params'], sim_config['env_params'], sim_config['sim_params'] if all(v is not None for v in [sim_params, env_params, fly_params]): traj = load_default_configuration(traj, sim_params=sim_params, env_params=env_params, fly_params=fly_params) elif params is not None: for p in params: traj.f_apar(p, 0.0) if config is not None: for k, v in config.items(): traj.f_aconf(k, v) traj.f_aconf('parent_dir_path', parent_dir_path, comment='The parent directory') traj.f_aconf('dir_path', dir_path, comment='The directory path for saving data') traj.f_aconf('plot_path', plot_path, comment='The file path for saving plot') traj.f_aconf('data_path', data_path, comment='The file path for saving data') traj.f_aconf('dataset_path', f'{dir_path}/{batch_id}', comment='The directory path for saving datasets') traj.f_explore(space) # except: # raise ValueError(f'Failed to perform batch run {batch_id}') if post_process_method is not None: env.add_postprocessing(post_process_method, **post_kwargs) env.run(single_method, process_method, save_data_in_hdf5=save_data_in_hdf5, save_to=dir_path, common_folder=batch_id, **run_kwargs) env.disable_logging() print('Batch run complete') if final_process_method is not None: return final_process_method(env.traj)
def _batch_run(dir='unnamed', batch_id='template', space=None, save_data_in_hdf5=False, single_method=single_run, process_method=null_processing, post_process_method=None, final_process_method=None, multiprocessing=True, resumable=True, overwrite=False, sim_config=None, params=None, optimization=None, post_kwargs={}, run_kwargs={}): saved_args = locals() traj_name = f'{batch_id}_traj' parent_dir_path = f'{paths.BatchRunFolder}/{dir}' dir_path = os.path.join(parent_dir_path, batch_id) filename = f'{dir_path}/{batch_id}.hdf5' build_new = True if os.path.exists(parent_dir_path) and os.path.exists( dir_path) and overwrite == False: build_new = False try: # print('Trying to resume existing trajectory') env = Environment(continuable=True) env.resume(trajectory_name=traj_name, resume_folder=dir_path) print('Resumed existing trajectory') build_new = False except: try: # print('Trying to load existing trajectory') traj = load_trajectory(filename=filename, name=traj_name, load_all=0) env = Environment(trajectory=traj, multiproc=True, ncores=4) traj = config_traj(traj, optimization) traj.f_load(index=None, load_parameters=2, load_results=0) traj.f_expand(space) print('Loaded existing trajectory') build_new = False except: print( 'Neither of resuming or expanding of existing trajectory worked' ) if build_new: if multiprocessing: multiproc = True resumable = False wrap_mode = pypetconstants.WRAP_MODE_QUEUE else: multiproc = False resumable = True wrap_mode = pypetconstants.WRAP_MODE_LOCK # print('Trying to create novel environment') env = Environment( trajectory=traj_name, filename=filename, file_title=batch_id, comment=f'{batch_id} batch run!', large_overview_tables=True, overwrite_file=True, resumable=False, resume_folder=dir_path, multiproc=multiproc, ncores=4, use_pool= True, # Our runs are inexpensive we can get rid of overhead by using a pool freeze_input= True, # We can avoid some overhead by freezing the input to the pool wrap_mode=wrap_mode, graceful_exit=True) print('Created novel environment') traj = prepare_traj(env.traj, sim_config, params, batch_id, parent_dir_path, dir_path) traj = config_traj(traj, optimization) traj.f_explore(space) if post_process_method is not None: env.add_postprocessing(post_process_method, **post_kwargs) env.run(single_method, process_method, save_data_in_hdf5=save_data_in_hdf5, save_to=dir_path, **run_kwargs) env.disable_logging() print('Batch run complete') if final_process_method is not None: results = final_process_method(env.traj) # print(results) return results