def test_run_snapshot(self, function_tmp_path_factory, lj_snapshot, lj_configuration): workdir = str(function_tmp_path_factory.mktemp('test_run')) # write the snapshot to the file system serial_snap = Orchestrator.serialize(lj_snapshot) snap_path = osp.join(workdir, 'snapshot.snap.dill.pkl') with open(snap_path, 'wb') as wf: wf.write(serial_snap) # write the configuration to the file system serial_config = Orchestrator.serialize(lj_configuration) config_path = osp.join(workdir, 'config.config.dill.pkl') with open(config_path, 'wb') as wf: wf.write(serial_config) n_steps = str(100) n_seconds = str(5) runner = CliRunner() result = runner.invoke( wepy_cli, [ 'run', 'snapshot', '--job-dir', workdir, snap_path, config_path, n_seconds, n_steps ], catch_exceptions=False, ) assert result.exit_code == 0
def ls_runs(orchestrator): """ \b Parameters ---------- orchestrator : \b Returns ------- """ orch = Orchestrator(orch_path=orchestrator, mode='r') runs = orch.run_hashes() orch.close() hash_listing_str = "\n".join( ["{}, {}".format(start, end) for start, end in runs]) click.echo(hash_listing_str)
def add_config(configuration, orchestrator): orch = Orchestrator(orchestrator, mode='r+') serial_config = configuration.read() config_hash = orch.add_serial_snapshot(serial_config) orch.close() click.echo(config_hash)
def add_snapshot(snapshot, orchestrator): orch = Orchestrator(orchestrator, mode='r+') serial_snapshot = snapshot.read() snaphash = orch.add_serial_snapshot(serial_snapshot) orch.close() click.echo(snaphash)
def get_run_cycles(end_hash, start_hash, orchestrator): orch = Orchestrator(orchestrator, mode='r') start_serial_snapshot = orch.snapshot_kv[start_hash] end_serial_snapshot = orch.snapshot_kv[end_hash] # get the records values for this run rec_d = {field : value for field, value in zip(Orchestrator.RUN_SELECT_FIELDS, orch.get_run_record(start_hash, end_hash))} click.echo(rec_d['last_cycle_idx'])
def reconcile_hdf5(orchestrator, hdf5, run_ids): """ \b Parameters ---------- orchestrator : hdf5 : \b Returns ------- """ # parse the run ids run_ids = [tuple(run_id.split(',')) for run_id in run_ids] orch = Orchestrator(orchestrator, mode='r') hdf5_path = osp.realpath(hdf5) click.echo("Combining the HDF5s together, saving to:") click.echo(hdf5_path) # combine the HDF5 files from those orchestrators combine_orch_wepy_hdf5s(orch, hdf5_path, run_ids=run_ids)
def make_sim_manager(self, n_walkers, apparatus, config): walkers = self.make_initial_walkers(self.init_state, n_walkers) snapshot = SimSnapshot(walkers, apparatus) sim_manager = Orchestrator.gen_sim_manager(snapshot, config) return sim_manager
def ls_configs(orchestrator): """ Parameters ---------- orchestrator : Returns ------- """ orch = Orchestrator(orch_path=orchestrator, mode='r') message = hash_listing_formatter(orch.configuration_hashes) orch.close() click.echo(message)
def get_config(output, config_hash, orchestrator): # first check if the output is None, if it is we automatically # generate a file in the cwd that is the hash of the snapshot if output is None: output = "{}.config.dill.pkl".format(config_hash) # check that it doesn't exist, and fail if it does, since we # don't want to implicitly overwrite stuff if osp.exists(output): raise OSError("No output path was specified and default alredy exists, exiting.") orch = Orchestrator(orchestrator, mode='r') serial_snapshot = orch.configuration_kv[config_hash] with open(output, 'wb') as wf: wf.write(serial_snapshot) orch.close()
def ls_snapshots(orchestrator): """ \b Parameters ---------- orchestrator : \b Returns ------- """ orch = Orchestrator(orch_path=orchestrator, mode='r') message = hash_listing_formatter(orch.snapshot_hashes) orch.close() click.echo(message)
def reconcile_hdf5(orchestrator, hdf5, run_ids): """For an orchestrator with multiple runs combine the HDF5 results into a single one. This requires that the paths inside of the reporters for the configurations used for a run still have valid paths to the HDF5 files. \b Parameters ---------- orchestrator : Path The orchestrator to retrieve HDF5s for hdf5 : Path Path to the resultant HDF5. run_ids : str String specifying a run as start and end hash e.g. 'd0cb2e6fbcc8c2d66d67c845120c7f6b,b4b96580ae57f133d5f3b6ce25affa6d' \b Returns ------- """ # parse the run ids run_ids = [tuple(run_id.split(',')) for run_id in run_ids] orch = Orchestrator(orchestrator, mode='r') hdf5_path = osp.realpath(hdf5) click.echo("Combining the HDF5s together, saving to:") click.echo(hdf5_path) # combine the HDF5 files from those orchestrators combine_orch_wepy_hdf5s(orch, hdf5_path, run_ids=run_ids)
def run_snapshot(log, n_workers, checkpoint_freq, job_dir, job_name, narration, n_cycle_steps, run_time, configuration, snapshot): """ \b Parameters ---------- log : n_workers : checkpoint_freq : job_dir : job_name : narration : n_cycle_steps : run_time : start_hash : orchestrator : \b Returns ------- """ set_loglevel(log) logging.info("Loading the starting snapshot file") # read the config and snapshot in serial_snapshot = snapshot.read() logging.info("Creating orchestrating orch database") # make the orchestrator for this simulation in memory to start orch = Orchestrator() logging.info("Adding the starting snapshot to database") start_hash = orch.add_serial_snapshot(serial_snapshot) # settle what the defaults etc. are for the different options as they are interdependent job_dir, job_name, narration, config = settle_run_options(n_workers=n_workers, job_dir=job_dir, job_name=job_name, narration=narration, configuration=configuration, start_hash=start_hash) # add the parametrized configuration to the orchestrator # config_hash = orch.add_serial_configuration(config) logging.info("Orchestrator loaded") logging.info("Running snapshot by time") run_orch = orch.orchestrate_snapshot_run_by_time(start_hash, run_time, n_cycle_steps, checkpoint_freq=checkpoint_freq, work_dir=job_dir, config_name=job_name, narration=narration, configuration=config, ) logging.info("Finished running snapshot by time") start_hash, end_hash = run_orch.run_hashes()[0] run_orch.close() logging.info("Closed the resultant orch") # write the run tuple out to the log run_line_str = "Run start and end hashes: {}, {}".format(start_hash, end_hash) # log it logging.info(run_line_str) # also put it to the terminal click.echo(run_line_str) orch.close() logging.info("closed the orchestrating orch database")
sim_apparatus = WepySimApparatus(runner, resampler=resampler, boundary_conditions=ubc) # we also create a default configuration for the orchestrator that # will be used unless one is given at runtime for the creation of a # simulation manager configuration = Configuration(n_workers=4, reporter_classes=reporter_classes, reporter_partial_kwargs=reporter_kwargs) # we also want to set up the orchestrator with some default walkers to # use to get us started. Otherwise these could be provided from a # snapshot or on their own. Ideally we only want to have a single # script setting up an orchestrator and then manage everything else on # the command line intereactively from then on out init_weight = 1.0 / N_WALKERS init_walkers = [ Walker(OpenMMState(init_sim_state), init_weight) for i in range(N_WALKERS) ] # then create the seed/root/master orchestrator which will be used # from here on out orchestrator = Orchestrator(sim_apparatus, default_init_walkers=init_walkers, default_configuration=configuration) # save it orchestrator.dump('LJ-pair.orch', mode='wb')
def create_orch(orchestrator): orch = Orchestrator(orchestrator, mode='x') orch.close()
'max_n_regions' : RESAMPLER.max_n_regions, 'max_region_sizes' : RESAMPLER.max_region_sizes, 'bc_cutoff_distance' : BC.cutoff_distance} REPORTER_KWARGS = [hdf5_reporter_kwargs, dashboard_reporter_kwargs] N_WORKERS = 8 CONFIGURATION = Configuration(n_workers=N_WORKERS, reporter_classes=REPORTER_CLASSES, reporter_partial_kwargs=REPORTER_KWARGS) print("created configuration") ### Initial Walkers N_WALKERS = 48 INIT_WEIGHT = 1.0 / N_WALKERS INIT_WALKERS = [Walker(deepcopy(INIT_STATE), INIT_WEIGHT) for i in range(N_WALKERS)] print("created init walkers") ### Orchestrator ORCHESTRATOR = Orchestrator(APPARATUS, default_init_walkers=INIT_WALKERS, default_configuration=CONFIGURATION) print("created orchestrator, creating object file now") ORCH_NAME = "sEH-TPPU" dump_orchestrator(ORCHESTRATOR, "{}.orch".format(ORCH_NAME), mode='wb')
def get_run(output, end_hash, start_hash, orchestrator): # first check if the output is None, if it is we automatically # generate a file in the cwd that is the hash of the snapshot if output is None: output = "{}-{}.orch.sqlite".format(start_hash, end_hash) # check that it doesn't exist, and fail if it does, since we # don't want to implicitly overwrite stuff if osp.exists(output): raise OSError("No output path was specified and default alredy exists, exiting.") orch = Orchestrator(orchestrator, mode='r') start_serial_snapshot = orch.snapshot_kv[start_hash] end_serial_snapshot = orch.snapshot_kv[end_hash] # get the records values for this run rec_d = {field : value for field, value in zip(Orchestrator.RUN_SELECT_FIELDS, orch.get_run_record(start_hash, end_hash))} config = orch.configuration_kv[rec_d['config_hash']] # create a new orchestrator at the output location new_orch = Orchestrator(output, mode='w') _ = new_orch.add_serial_snapshot(start_serial_snapshot) _ = new_orch.add_serial_snapshot(end_serial_snapshot) config_hash = new_orch.add_serial_configuration(config) new_orch.register_run(start_hash, end_hash, config_hash, rec_d['last_cycle_idx']) orch.close() new_orch.close()
def test_apparatus_configuration(datadir_factory, mocker): config = Configuration() assert config.apparatus_opts == {} reparam_config = config.reparametrize(apparatus_opts={ 'runner': { 'platform': 'CPU', }, }) assert reparam_config.apparatus_opts == { 'runner': { 'platform': 'CPU', }, } ## test that we can change the apparatus parameters in the sim_manager system_mock = mocker.Mock() topology_mock = mocker.Mock() integrator_mock = mocker.Mock() runner = OpenMMRunner( system_mock, topology_mock, integrator_mock, ) resampler_mock = mocker.MagicMock() # mocker.patch('WExploreResampler') apparatus = WepySimApparatus( runner, resampler=resampler_mock, boundary_conditions=None, ) apparatus._filters = ( runner, None, resampler_mock, ) state_mock = mocker.MagicMock( ) # mocker.patch('wepy.walker.WalkerState', autospec=True) walkers = [Walker(state_mock, 0.1) for i in range(1)] snapshot = SimSnapshot( walkers, apparatus, ) snapshot._walkers = walkers snapshot._apparatus = apparatus datadir = datadir_factory.mkdatadir() orch = Orchestrator(orch_path=str(datadir / "test.orch.sqlite3")) sim_manager = orch.gen_sim_manager( snapshot, reparam_config, ) sim_manager.init() # sim_mock = mocker.patch('wepy.runners.openmm.omma.Simulation') # platform_mock = mocker.patch('wepy.runners.openmm.omm.Platform') # _ = sim_manager.run_cycle( # walkers, # 2, # 0, # runner_opts={ # 'platform' : 'CPU', # } # ) # platform_mock.getPlatformByName.assert_called_with('CPU') sim_mock = mocker.patch('wepy.runners.openmm.omma.Simulation') platform_mock = mocker.patch('wepy.runners.openmm.omm.Platform') platform_mock.getPlatformByName.\ return_value.getPropertyNames.\ return_value = ('Threads',) _ = sim_manager.run_cycle(walkers, 2, 0, runner_opts={ 'platform': 'CPU', 'platform_kwargs': { 'Threads': '3' }, }) platform_mock.getPlatformByName.assert_called_with('CPU') platform_mock.getPlatformByName.\ return_value.getPropertyNames.\ assert_called() platform_mock.getPlatformByName.\ return_value.setPropertyDefaultValue.\ assert_called_with( 'Threads', '3' )
def settle_run_options(n_workers=None, job_dir=None, job_name=None, narration=None, configuration=None, start_hash=None): """ Parameters ---------- n_workers : (Default value = None) job_dir : (Default value = None) job_name : (Default value = None) narration : (Default value = None) \b Returns ------- """ # the default for the job name is the start hash if none is given if job_name == START_HASH: job_name = start_hash # if the job_name is given and the default value for the job_dir # is given (i.e. not specified by the user) we set the job-dir as # the job_name if job_name is not None and job_dir == CURDIR: job_dir = job_name # if the special value for curdir is given we get the systems # current directory, this is the default. if job_dir == CURDIR: job_dir = osp.curdir # normalize the job_dir job_dir = osp.realpath(job_dir) # if a path for a configuration was given we want to use it so we # unpickle it and return it, otherwise return None and use the # default one in the orchestrator config = None if configuration is not None: with open(configuration, 'rb') as rf: config = Orchestrator.deserialize(rf.read()) # we need to reparametrize the configuration here since the # orchestrator API will ignore reparametrization values if a # concrete Configuration is given. if config is not None: # collect the kwargs for the work mapper to be reparametrized work_mapper_pkwargs = {'num_workers' : n_workers} config = config.reparametrize(work_dir=job_dir, config_name=job_name, narration=narration, work_mapper_partial_kwargs=work_mapper_pkwargs) return job_dir, job_name, narration, config
def run_orch(log, n_workers, checkpoint_freq, job_dir, job_name, narration, configuration, n_cycle_steps, run_time, start_hash, orchestrator): """ \b Parameters ---------- log : n_workers : checkpoint_freq : job_dir : job_name : narration : n_cycle_steps : run_time : start_hash : orchestrator : \b Returns ------- """ set_loglevel(log) # settle what the defaults etc. are for the different options as they are interdependent job_dir, job_name, narration, config = settle_run_options(n_workers=n_workers, job_dir=job_dir, job_name=job_name, narration=narration, configuration=configuration, start_hash=start_hash) # Open a wrapper around the orchestrator database that provides # the inputs for the simulation orch = Orchestrator(orchestrator, mode='r') logging.info("Orchestrator loaded") logging.info("Running snapshot by time") run_orch = orch.orchestrate_snapshot_run_by_time(start_hash, run_time, n_cycle_steps, checkpoint_freq=checkpoint_freq, work_dir=job_dir, config_name=job_name, narration=narration, configuration=config, ) logging.info("Finished running snapshot by time") start_hash, end_hash = run_orch.run_hashes()[0] logging.info("Closing the resultant orchestrator") run_orch.close() # write the run tuple out to the log run_line_str = "Run start and end hashes: {}, {}".format(start_hash, end_hash) # log it logging.info(run_line_str) # also put it to the terminal click.echo(run_line_str) logging.info("Closing the orchestrating orch") orch.close()
sim_apparatus = WepySimApparatus(runner, resampler=resampler, boundary_conditions=ubc) # we also create a default configuration for the orchestrator that # will be used unless one is given at runtime for the creation of a # simulation manager configuration = Configuration(reporter_classes=reporter_classes, reporter_partial_kwargs=reporter_kwargs) # we also want to set up the orchestrator with some default walkers to # use to get us started. Otherwise these could be provided from a # snapshot or on their own. Ideally we only want to have a single # script setting up an orchestrator and then manage everything else on # the command line intereactively from then on out init_weight = 1.0 / N_WALKERS init_walkers = [ Walker(OpenMMState(init_sim_state), init_weight) for i in range(N_WALKERS) ] # then create the seed/root/master orchestrator which will be used # from here on out orch = Orchestrator(orch_path='LJ-pair.orch.sqlite', mode='w') # set the defaults orch.set_default_sim_apparatus(sim_apparatus) orch.set_default_init_walkers(init_walkers) orch.set_default_configuration(configuration) orch.gen_default_snapshot()
def settle_run_options( n_workers=None, job_dir=None, job_name=None, narration=None, monitor_http_port=None, tag=None, configuration=None, start_hash=None, ): """ Parameters ---------- n_workers : (Default value = None) job_dir : (Default value = None) job_name : (Default value = None) narration : (Default value = None) \b Returns ------- """ # the default for the job name is the start hash if none is given if job_name == START_HASH: job_name = start_hash # if the job_name is given and the default value for the job_dir # is given (i.e. not specified by the user) we set the job-dir as # the job_name if job_name is not None and job_dir == CURDIR: job_dir = job_name # if the special value for curdir is given we get the systems # current directory, this is the default. if job_dir == CURDIR: job_dir = osp.curdir # normalize the job_dir job_dir = osp.realpath(job_dir) # if a path for a configuration was given we want to use it so we # unpickle it and return it, otherwise return None and use the # default one in the orchestrator config = None if configuration is not None: with open(configuration, 'rb') as rf: config = Orchestrator.deserialize(rf.read()) ## Monitoring # nothing to do, just use the port or tag if its given monitor_pkwargs = { 'tag': tag, 'port': monitor_http_port, } # we need to reparametrize the configuration here since the # orchestrator API will ignore reparametrization values if a # concrete Configuration is given. if config is not None: # if there is a change in the number of workers we need to # recalculate all of the partial kwargs work_mapper_pkwargs = deepcopy(config.work_mapper_partial_kwargs) # if the number of workers has changed update all the relevant # fields, otherwise leave it alone if work_mapper_pkwargs['num_workers'] != n_workers: work_mapper_pkwargs['num_workers'] = n_workers work_mapper_pkwargs['device_ids'] = [ str(i) for i in range(n_workers) ] config = config.reparametrize( work_dir=job_dir, config_name=job_name, narration=narration, work_mapper_partial_kwargs=work_mapper_pkwargs, monitor_partial_kwargs=monitor_pkwargs, ) return job_dir, job_name, narration, config