def test_build_model_specification_failure():
with pytest.raises(ConfigurationError):
build_model_specification('made_up_file.yaml')
with pytest.raises(ConfigurationError):
test_dir = os.path.dirname(os.path.realpath(__file__))
model_spec = test_dir + '/../test_data/bad_model_specification.txt'
assert os.path.exists(model_spec), 'Test directory structure is broken'
build_model_specification(model_spec)
def model_specification(mocker):
test_dir = os.path.dirname(os.path.realpath(__file__))
user_config = test_dir + '/test_data/mock_user_config.yaml'
model_spec = test_dir + '/test_data/mock_model_specification.yaml'
expand_user_mock = mocker.patch(
'vivarium.framework.configuration.os.path.expanduser')
expand_user_mock.return_value = user_config
return build_model_specification(model_spec)
def parse(
command: str,
input_model_specification_path: Optional[Path],
artifact_path: Optional[Path],
model_specification_path: Path,
results_root: Path,
keyspace: "Keyspace",
) -> ConfigTree:
if command in [COMMANDS.restart, COMMANDS.expand]:
return build_model_specification(model_specification_path)
if command == COMMANDS.load_test:
return build_model_specification()
model_specification = build_model_specification(
input_model_specification_path)
model_specification.configuration.update(
{OUTPUT_DATA_KEY: {
RESULTS_DIRECTORY_KEY: str(results_root)
}},
source=__file__,
)
artifact_path_is_cli_arg = artifact_path is not None
artifact_path_in_keyspace = FULL_ARTIFACT_PATH_KEY in keyspace
if artifact_path_is_cli_arg and artifact_path_in_keyspace:
raise ConfigurationError(
"Artifact path cannot be specified both in the branch specification file"
" and as a command line argument.",
str(artifact_path),
)
elif artifact_path_is_cli_arg:
model_specification.configuration[INPUT_DATA_KEY].update(
{ARTIFACT_PATH_KEY: str(artifact_path)}, source=__file__)
else:
# Artifact path comes from the model spec.
# Parsing here ensures the key exists and the value points
# to an actual file.
parse_artifact_path_config(model_specification.configuration)
return model_specification
def initialize_simulation_from_model_specification(model_specification_file):
model_specification = build_model_specification(model_specification_file)
plugin_config = model_specification.plugins
component_config = model_specification.components
simulation_config = model_specification.configuration
plugin_manager = PluginManager(plugin_config)
component_config_parser = plugin_manager.get_plugin(
'component_configuration_parser')
components = component_config_parser.get_components(component_config)
return InteractiveContext(simulation_config, components, plugin_manager)
def __init__(self, model_specification_file: str,
branch_configuration_file: str, output_directory: Path,
logging_directories: Dict[str, Path], num_input_draws: int,
num_random_seeds: int, restart: bool, expand: Dict[str, int],
no_batch: bool):
self.number_already_completed = 0
self.output_directory = output_directory
self.no_batch = no_batch
self.sge_log_directory = logging_directories['sge']
self.worker_log_directory = logging_directories['worker']
if restart:
self.keyspace = Keyspace.from_previous_run(self.output_directory)
self.existing_outputs = pd.read_hdf(self.output_directory /
'output.hdf')
if expand:
self.keyspace.add_draws(expand['num_draws'])
self.keyspace.add_seeds(expand['num_seeds'])
self.keyspace.persist(self.output_directory)
else:
model_specification = build_model_specification(
model_specification_file)
self.keyspace = Keyspace.from_branch_configuration(
num_input_draws, num_random_seeds, branch_configuration_file)
if "input_data.artifact_path" in self.keyspace.get_data():
raise ValueError(
"An artifact path can only be supplied in the model specification file, "
"not the branches configuration.")
if "artifact_path" in model_specification.configuration.input_data:
artifact_path = parse_artifact_path_config(
model_specification.configuration)
model_specification.configuration.input_data.update(
{"artifact_path": artifact_path}, source=__file__)
model_specification_path = self.output_directory / 'model_specification.yaml'
shutil.copy(model_specification_file, model_specification_path)
self.existing_outputs = None
# Log some basic stuff about the simulation to be run.
self.keyspace.persist(self.output_directory)
self.model_specification = self.output_directory / 'model_specification.yaml'
def test_build_model_specification(mocker, test_spec, test_user_config):
expand_user_mock = mocker.patch(
'vivarium.framework.configuration.Path.expanduser')
expand_user_mock.return_value = test_user_config
loaded_model_spec = build_model_specification(test_spec)
test_data = DEFAULT_PLUGINS.copy()
with test_spec.open() as f:
model_data = yaml.full_load(f)
test_data.update(model_data)
with test_user_config.open() as f:
user_data = yaml.full_load(f)
test_data['configuration'].update(user_data)
assert loaded_model_spec.to_dict() == test_data
def test_build_model_specification_failure(mocker, test_data_dir, test_spec):
with pytest.raises(ConfigurationError):
build_model_specification('made_up_file.yaml')
with pytest.raises(ConfigurationError):
model_spec = test_data_dir / 'bad_model_specification.txt'
build_model_specification(model_spec)
with test_spec.open() as f:
spec_dict = yaml.full_load(f)
spec_dict.update({'invalid_key': 'some_value'})
load_mock = mocker.patch('vivarium.framework.configuration.yaml.full_load')
load_mock.return_value = spec_dict
with pytest.raises(ConfigurationError):
build_model_specification(str(test_spec))
def run_nth_draw(model_specification_file, draw_number):
"""
Run a model simulation for a specific draw number.
:param model_specification_file: The YAML model specification file.
:param draw_number: The draw number to select for rates and values that
have multiple draws.
"""
logger = logging.getLogger(__name__)
logger.info('{} Simulating draw #{} for {} ...'.format(
datetime.datetime.now().strftime("%H:%M:%S"),
draw_number, model_specification_file))
spec = config.build_model_specification(model_specification_file)
spec.configuration.input_data.input_draw_number = draw_number
simulation = initialise_simulation_from_specification_config(spec)
simulation.setup()
metrics, final_state = engine.run(simulation)
logger.info('{} Simulation for draw #{} complete'.format(
datetime.datetime.now().strftime("%H:%M:%S"),
draw_number))
return metrics, final_state
def test_build_model_specification(mocker):
test_dir = os.path.dirname(os.path.realpath(__file__))
user_config = test_dir + '/../test_data/mock_user_config.yaml'
model_spec = test_dir + '/../test_data/mock_model_specification.yaml'
expand_user_mock = mocker.patch(
'vivarium.framework.configuration.os.path.expanduser')
expand_user_mock.return_value = user_config
loaded_model_spec = build_model_specification(model_spec)
test_data = DEFAULT_PLUGINS
with open(model_spec) as f:
model_data = yaml.load(f)
test_data.update(model_data)
with open(user_config) as f:
user_data = yaml.load(f)
test_data['configuration'].update(user_data)
assert loaded_model_spec.to_dict() == test_data
def run_simulation(model_specification_file, results_directory):
results_writer = get_results_writer(results_directory,
model_specification_file)
model_specification = build_model_specification(model_specification_file)
model_specification.configuration.output_data.update(
{'results_directory': results_writer.results_root},
layer='override',
source='command_line')
simulation = setup_simulation(model_specification)
metrics, final_state = run(simulation)
_log.debug(pformat(metrics))
unused_config_keys = simulation.configuration.unused_keys()
if unused_config_keys:
_log.debug("Some configuration keys not used during run: %s",
unused_config_keys)
idx = pd.Index([simulation.configuration.randomness.random_seed],
name='random_seed')
metrics = pd.DataFrame(metrics, index=idx)
results_writer.write_output(metrics, 'output.hdf')
results_writer.write_output(final_state, 'final_state.hdf')
def __init__(self,
model_specification: Union[str, Path, ConfigTree] = None,
components: Union[List, Dict, ConfigTree] = None,
configuration: Union[Dict, ConfigTree] = None,
plugin_configuration: Union[Dict, ConfigTree] = None):
# Bootstrap phase: Parse arguments, make private managers
component_configuration = components if isinstance(
components, (dict, ConfigTree)) else None
self._additional_components = components if isinstance(
components, List) else []
model_specification = build_model_specification(
model_specification, component_configuration, configuration,
plugin_configuration)
self._plugin_configuration = model_specification.plugins
self._component_configuration = model_specification.components
self.configuration = model_specification.configuration
self._plugin_manager = PluginManager(model_specification.plugins)
# TODO: Setup logger here.
self._builder = Builder(self.configuration, self._plugin_manager)
# This formally starts the initialization phase (this call makes the
# life-cycle manager).
self._lifecycle = self._plugin_manager.get_plugin('lifecycle')
self._lifecycle.add_phase(
'setup', ['setup', 'post_setup', 'population_creation'])
self._lifecycle.add_phase('main_loop', [
'time_step__prepare', 'time_step', 'time_step__cleanup',
'collect_metrics'
],
loop=True)
self._lifecycle.add_phase('simulation_end',
['simulation_end', 'report'])
self._component_manager = self._plugin_manager.get_plugin(
'component_manager')
self._component_manager.setup(self.configuration, self._lifecycle)
self._clock = self._plugin_manager.get_plugin('clock')
self._values = self._plugin_manager.get_plugin('value')
self._events = self._plugin_manager.get_plugin('event')
self._population = self._plugin_manager.get_plugin('population')
self._resource = self._plugin_manager.get_plugin('resource')
self._tables = self._plugin_manager.get_plugin('lookup')
self._randomness = self._plugin_manager.get_plugin('randomness')
self._data = self._plugin_manager.get_plugin('data')
for name, controller in self._plugin_manager.get_optional_controllers(
).items():
setattr(self, f'_{name}', controller)
# The order the managers are added is important. It represents the
# order in which they will be set up. The clock is required by
# several of the other managers, including the lifecycle manager. The
# lifecycle manager is also required by most managers. The randomness
# manager requires the population manager. The remaining managers need
# no ordering.
managers = [
self._clock, self._lifecycle, self._resource, self._values,
self._population, self._randomness, self._events, self._tables,
self._data
] + list(self._plugin_manager.get_optional_controllers().values())
self._component_manager.add_managers(managers)
component_config_parser = self._plugin_manager.get_plugin(
'component_configuration_parser')
# Tack extra components onto the end of the list generated from the model specification.
components = (component_config_parser.get_components(
self._component_configuration) + self._additional_components +
[Metrics()])
self._lifecycle.add_constraint(self.add_components,
allow_during=['initialization'])
self._lifecycle.add_constraint(
self.get_population,
restrict_during=['initialization', 'setup', 'post_setup'])
self.add_components(components)
def model_specification(mocker, test_spec, test_user_config):
expand_user_mock = mocker.patch(
'vivarium.framework.configuration.Path.expanduser')
expand_user_mock.return_value = test_user_config
return build_model_specification(test_spec)