def get_run(output, end_hash, start_hash, orchestrator):
# first check if the output is None, if it is we automatically
# generate a file in the cwd that is the hash of the snapshot
if output is None:
output = "{}-{}.orch.sqlite".format(start_hash, end_hash)
# check that it doesn't exist, and fail if it does, since we
# don't want to implicitly overwrite stuff
if osp.exists(output):
raise OSError("No output path was specified and default alredy exists, exiting.")
orch = Orchestrator(orchestrator, mode='r')
start_serial_snapshot = orch.snapshot_kv[start_hash]
end_serial_snapshot = orch.snapshot_kv[end_hash]
# get the records values for this run
rec_d = {field : value for field, value in
zip(Orchestrator.RUN_SELECT_FIELDS, orch.get_run_record(start_hash, end_hash))}
config = orch.configuration_kv[rec_d['config_hash']]
# create a new orchestrator at the output location
new_orch = Orchestrator(output, mode='w')
_ = new_orch.add_serial_snapshot(start_serial_snapshot)
_ = new_orch.add_serial_snapshot(end_serial_snapshot)
config_hash = new_orch.add_serial_configuration(config)
new_orch.register_run(start_hash, end_hash, config_hash, rec_d['last_cycle_idx'])
orch.close()
new_orch.close()
def reconcile_hdf5(orchestrator, hdf5, run_ids):
"""
\b
Parameters
----------
orchestrator :
hdf5 :
\b
Returns
-------
"""
# parse the run ids
run_ids = [tuple(run_id.split(',')) for run_id in run_ids]
orch = Orchestrator(orchestrator, mode='r')
hdf5_path = osp.realpath(hdf5)
click.echo("Combining the HDF5s together, saving to:")
click.echo(hdf5_path)
# combine the HDF5 files from those orchestrators
combine_orch_wepy_hdf5s(orch, hdf5_path, run_ids=run_ids)
def ls_runs(orchestrator):
"""
\b
Parameters
----------
orchestrator :
\b
Returns
-------
"""
orch = Orchestrator(orch_path=orchestrator, mode='r')
runs = orch.run_hashes()
orch.close()
hash_listing_str = "\n".join(
["{}, {}".format(start, end) for start, end in runs])
click.echo(hash_listing_str)
def add_config(configuration, orchestrator):
orch = Orchestrator(orchestrator, mode='r+')
serial_config = configuration.read()
config_hash = orch.add_serial_snapshot(serial_config)
orch.close()
click.echo(config_hash)
def add_snapshot(snapshot, orchestrator):
orch = Orchestrator(orchestrator, mode='r+')
serial_snapshot = snapshot.read()
snaphash = orch.add_serial_snapshot(serial_snapshot)
orch.close()
click.echo(snaphash)
def get_run_cycles(end_hash, start_hash, orchestrator):
orch = Orchestrator(orchestrator, mode='r')
start_serial_snapshot = orch.snapshot_kv[start_hash]
end_serial_snapshot = orch.snapshot_kv[end_hash]
# get the records values for this run
rec_d = {field : value for field, value in
zip(Orchestrator.RUN_SELECT_FIELDS, orch.get_run_record(start_hash, end_hash))}
click.echo(rec_d['last_cycle_idx'])
def ls_configs(orchestrator):
"""
Parameters
----------
orchestrator :
Returns
-------
"""
orch = Orchestrator(orch_path=orchestrator, mode='r')
message = hash_listing_formatter(orch.configuration_hashes)
orch.close()
click.echo(message)
def get_config(output, config_hash, orchestrator):
# first check if the output is None, if it is we automatically
# generate a file in the cwd that is the hash of the snapshot
if output is None:
output = "{}.config.dill.pkl".format(config_hash)
# check that it doesn't exist, and fail if it does, since we
# don't want to implicitly overwrite stuff
if osp.exists(output):
raise OSError("No output path was specified and default alredy exists, exiting.")
orch = Orchestrator(orchestrator, mode='r')
serial_snapshot = orch.configuration_kv[config_hash]
with open(output, 'wb') as wf:
wf.write(serial_snapshot)
orch.close()
def ls_snapshots(orchestrator):
"""
\b
Parameters
----------
orchestrator :
\b
Returns
-------
"""
orch = Orchestrator(orch_path=orchestrator, mode='r')
message = hash_listing_formatter(orch.snapshot_hashes)
orch.close()
click.echo(message)
def reconcile_hdf5(orchestrator, hdf5, run_ids):
"""For an orchestrator with multiple runs combine the HDF5 results
into a single one.
This requires that the paths inside of the reporters for the
configurations used for a run still have valid paths to the HDF5
files.
\b
Parameters
----------
orchestrator : Path
The orchestrator to retrieve HDF5s for
hdf5 : Path
Path to the resultant HDF5.
run_ids : str
String specifying a run as start and end hash
e.g. 'd0cb2e6fbcc8c2d66d67c845120c7f6b,b4b96580ae57f133d5f3b6ce25affa6d'
\b
Returns
-------
"""
# parse the run ids
run_ids = [tuple(run_id.split(',')) for run_id in run_ids]
orch = Orchestrator(orchestrator, mode='r')
hdf5_path = osp.realpath(hdf5)
click.echo("Combining the HDF5s together, saving to:")
click.echo(hdf5_path)
# combine the HDF5 files from those orchestrators
combine_orch_wepy_hdf5s(orch, hdf5_path, run_ids=run_ids)
def run_orch(log, n_workers, checkpoint_freq, job_dir, job_name, narration, configuration,
n_cycle_steps, run_time, start_hash, orchestrator):
"""
\b
Parameters
----------
log :
n_workers :
checkpoint_freq :
job_dir :
job_name :
narration :
n_cycle_steps :
run_time :
start_hash :
orchestrator :
\b
Returns
-------
"""
set_loglevel(log)
# settle what the defaults etc. are for the different options as they are interdependent
job_dir, job_name, narration, config = settle_run_options(n_workers=n_workers,
job_dir=job_dir,
job_name=job_name,
narration=narration,
configuration=configuration,
start_hash=start_hash)
# Open a wrapper around the orchestrator database that provides
# the inputs for the simulation
orch = Orchestrator(orchestrator, mode='r')
logging.info("Orchestrator loaded")
logging.info("Running snapshot by time")
run_orch = orch.orchestrate_snapshot_run_by_time(start_hash,
run_time,
n_cycle_steps,
checkpoint_freq=checkpoint_freq,
work_dir=job_dir,
config_name=job_name,
narration=narration,
configuration=config,
)
logging.info("Finished running snapshot by time")
start_hash, end_hash = run_orch.run_hashes()[0]
logging.info("Closing the resultant orchestrator")
run_orch.close()
# write the run tuple out to the log
run_line_str = "Run start and end hashes: {}, {}".format(start_hash, end_hash)
# log it
logging.info(run_line_str)
# also put it to the terminal
click.echo(run_line_str)
logging.info("Closing the orchestrating orch")
orch.close()
sim_apparatus = WepySimApparatus(runner,
resampler=resampler,
boundary_conditions=ubc)
# we also create a default configuration for the orchestrator that
# will be used unless one is given at runtime for the creation of a
# simulation manager
configuration = Configuration(reporter_classes=reporter_classes,
reporter_partial_kwargs=reporter_kwargs)
# we also want to set up the orchestrator with some default walkers to
# use to get us started. Otherwise these could be provided from a
# snapshot or on their own. Ideally we only want to have a single
# script setting up an orchestrator and then manage everything else on
# the command line intereactively from then on out
init_weight = 1.0 / N_WALKERS
init_walkers = [
Walker(OpenMMState(init_sim_state), init_weight) for i in range(N_WALKERS)
]
# then create the seed/root/master orchestrator which will be used
# from here on out
orch = Orchestrator(orch_path='LJ-pair.orch.sqlite', mode='w')
# set the defaults
orch.set_default_sim_apparatus(sim_apparatus)
orch.set_default_init_walkers(init_walkers)
orch.set_default_configuration(configuration)
orch.gen_default_snapshot()
def create_orch(orchestrator):
orch = Orchestrator(orchestrator, mode='x')
orch.close()
sim_apparatus = WepySimApparatus(runner,
resampler=resampler,
boundary_conditions=ubc)
# we also create a default configuration for the orchestrator that
# will be used unless one is given at runtime for the creation of a
# simulation manager
configuration = Configuration(n_workers=4,
reporter_classes=reporter_classes,
reporter_partial_kwargs=reporter_kwargs)
# we also want to set up the orchestrator with some default walkers to
# use to get us started. Otherwise these could be provided from a
# snapshot or on their own. Ideally we only want to have a single
# script setting up an orchestrator and then manage everything else on
# the command line intereactively from then on out
init_weight = 1.0 / N_WALKERS
init_walkers = [
Walker(OpenMMState(init_sim_state), init_weight) for i in range(N_WALKERS)
]
# then create the seed/root/master orchestrator which will be used
# from here on out
orchestrator = Orchestrator(sim_apparatus,
default_init_walkers=init_walkers,
default_configuration=configuration)
# save it
orchestrator.dump('LJ-pair.orch', mode='wb')
'max_n_regions' : RESAMPLER.max_n_regions,
'max_region_sizes' : RESAMPLER.max_region_sizes,
'bc_cutoff_distance' : BC.cutoff_distance}
REPORTER_KWARGS = [hdf5_reporter_kwargs, dashboard_reporter_kwargs]
N_WORKERS = 8
CONFIGURATION = Configuration(n_workers=N_WORKERS,
reporter_classes=REPORTER_CLASSES,
reporter_partial_kwargs=REPORTER_KWARGS)
print("created configuration")
### Initial Walkers
N_WALKERS = 48
INIT_WEIGHT = 1.0 / N_WALKERS
INIT_WALKERS = [Walker(deepcopy(INIT_STATE), INIT_WEIGHT) for i in range(N_WALKERS)]
print("created init walkers")
### Orchestrator
ORCHESTRATOR = Orchestrator(APPARATUS,
default_init_walkers=INIT_WALKERS,
default_configuration=CONFIGURATION)
print("created orchestrator, creating object file now")
ORCH_NAME = "sEH-TPPU"
dump_orchestrator(ORCHESTRATOR, "{}.orch".format(ORCH_NAME), mode='wb')
def test_apparatus_configuration(datadir_factory, mocker):
config = Configuration()
assert config.apparatus_opts == {}
reparam_config = config.reparametrize(apparatus_opts={
'runner': {
'platform': 'CPU',
},
})
assert reparam_config.apparatus_opts == {
'runner': {
'platform': 'CPU',
},
}
## test that we can change the apparatus parameters in the sim_manager
system_mock = mocker.Mock()
topology_mock = mocker.Mock()
integrator_mock = mocker.Mock()
runner = OpenMMRunner(
system_mock,
topology_mock,
integrator_mock,
)
resampler_mock = mocker.MagicMock() # mocker.patch('WExploreResampler')
apparatus = WepySimApparatus(
runner,
resampler=resampler_mock,
boundary_conditions=None,
)
apparatus._filters = (
runner,
None,
resampler_mock,
)
state_mock = mocker.MagicMock(
) # mocker.patch('wepy.walker.WalkerState', autospec=True)
walkers = [Walker(state_mock, 0.1) for i in range(1)]
snapshot = SimSnapshot(
walkers,
apparatus,
)
snapshot._walkers = walkers
snapshot._apparatus = apparatus
datadir = datadir_factory.mkdatadir()
orch = Orchestrator(orch_path=str(datadir / "test.orch.sqlite3"))
sim_manager = orch.gen_sim_manager(
snapshot,
reparam_config,
)
sim_manager.init()
# sim_mock = mocker.patch('wepy.runners.openmm.omma.Simulation')
# platform_mock = mocker.patch('wepy.runners.openmm.omm.Platform')
# _ = sim_manager.run_cycle(
# walkers,
# 2,
# 0,
# runner_opts={
# 'platform' : 'CPU',
# }
# )
# platform_mock.getPlatformByName.assert_called_with('CPU')
sim_mock = mocker.patch('wepy.runners.openmm.omma.Simulation')
platform_mock = mocker.patch('wepy.runners.openmm.omm.Platform')
platform_mock.getPlatformByName.\
return_value.getPropertyNames.\
return_value = ('Threads',)
_ = sim_manager.run_cycle(walkers,
2,
0,
runner_opts={
'platform': 'CPU',
'platform_kwargs': {
'Threads': '3'
},
})
platform_mock.getPlatformByName.assert_called_with('CPU')
platform_mock.getPlatformByName.\
return_value.getPropertyNames.\
assert_called()
platform_mock.getPlatformByName.\
return_value.setPropertyDefaultValue.\
assert_called_with(
'Threads',
'3'
)
boundary_conditions=ubc)
# we also create a default configuration for the orchestrator that
# will be used unless one is given at runtime for the creation of a
# simulation manager
configuration = Configuration(reporter_classes=reporter_classes,
reporter_partial_kwargs=reporter_kwargs)
# we also want to set up the orchestrator with some default walkers to
# use to get us started. Otherwise these could be provided from a
# snapshot or on their own. Ideally we only want to have a single
# script setting up an orchestrator and then manage everything else on
# the command line intereactively from then on out
init_weight = 1.0 / N_WALKERS
init_walkers = [
Walker(OpenMMState(init_sim_state), init_weight) for i in range(N_WALKERS)
]
# then create the seed/root/master orchestrator which will be used
# from here on out
orch = Orchestrator(
orch_path=str(output_dir / 'LJ-pair.orch.sqlite'),
mode='w',
)
# set the defaults
orch.set_default_sim_apparatus(sim_apparatus)
orch.set_default_init_walkers(init_walkers)
orch.set_default_configuration(configuration)
orch.gen_default_snapshot()
def run_snapshot(log, n_workers, checkpoint_freq, job_dir, job_name, narration,
n_cycle_steps, run_time, configuration, snapshot):
"""
\b
Parameters
----------
log :
n_workers :
checkpoint_freq :
job_dir :
job_name :
narration :
n_cycle_steps :
run_time :
start_hash :
orchestrator :
\b
Returns
-------
"""
set_loglevel(log)
logging.info("Loading the starting snapshot file")
# read the config and snapshot in
serial_snapshot = snapshot.read()
logging.info("Creating orchestrating orch database")
# make the orchestrator for this simulation in memory to start
orch = Orchestrator()
logging.info("Adding the starting snapshot to database")
start_hash = orch.add_serial_snapshot(serial_snapshot)
# settle what the defaults etc. are for the different options as they are interdependent
job_dir, job_name, narration, config = settle_run_options(n_workers=n_workers,
job_dir=job_dir,
job_name=job_name,
narration=narration,
configuration=configuration,
start_hash=start_hash)
# add the parametrized configuration to the orchestrator
# config_hash = orch.add_serial_configuration(config)
logging.info("Orchestrator loaded")
logging.info("Running snapshot by time")
run_orch = orch.orchestrate_snapshot_run_by_time(start_hash,
run_time,
n_cycle_steps,
checkpoint_freq=checkpoint_freq,
work_dir=job_dir,
config_name=job_name,
narration=narration,
configuration=config,
)
logging.info("Finished running snapshot by time")
start_hash, end_hash = run_orch.run_hashes()[0]
run_orch.close()
logging.info("Closed the resultant orch")
# write the run tuple out to the log
run_line_str = "Run start and end hashes: {}, {}".format(start_hash, end_hash)
# log it
logging.info(run_line_str)
# also put it to the terminal
click.echo(run_line_str)
orch.close()
logging.info("closed the orchestrating orch database")
