def test_simple_init(mv_kwargs): """Tests whether initialisation works for all basic cases.""" # With the full set of arguments mv = Multiverse(**mv_kwargs) # Assert some basic types assert isinstance(mv.wm, WorkerManager) assert isinstance(mv.dm, DataManager) assert isinstance(mv.pm, PlotManager) # Without the run configuration mv_kwargs.pop('run_cfg_path') mv_kwargs['paths']['model_note'] += "_wo_run_cfg" Multiverse(**mv_kwargs) # Suppressing the user config mv_kwargs['user_cfg_path'] = False mv_kwargs['paths']['model_note'] += "_wo_user_cfg" Multiverse(**mv_kwargs) # NOTE Without specifying a path, the search path will be used, which makes # the results untestable and creates spurious folders for the user. # Therefore, we cannot test for the case where no user config is given ... # No user config path given -> search at default location mv_kwargs['user_cfg_path'] = None mv_kwargs['paths']['model_note'] = "_user_cfg_path_none" Multiverse(**mv_kwargs) # No user config at default search location Multiverse.USER_CFG_SEARCH_PATH = "this_is_not_a_path" mv_kwargs['paths']['model_note'] = "_user_cfg_path_none_and_no_class_var" Multiverse(**mv_kwargs)
def test_stop_conditions(mv_kwargs): """An integration test for stop conditions""" mv_kwargs['run_cfg_path'] = STOP_COND_CFG_PATH mv = Multiverse(**mv_kwargs) mv.run_sweep() time.sleep(2) assert len(mv.wm.tasks) == 13 assert len(mv.wm.stopped_tasks) == 13 # all stopped
def test_prepare_executable(mv_kwargs): """Tests handling of the executable, i.e. copying to a temporary location and emitting helpful error messages """ mv_kwargs['executable_control'] = dict(run_from_tmpdir=False) mv = Multiverse(**mv_kwargs) # The dummy model should be available at this point, so _prepare_executable # should have correctly set a binary path, but not in a temporary directory assert mv._model_binpath is not None assert mv._tmpdir is None original_binpath = mv._model_binpath # Now, let the executable be copied to a temporary location mv._prepare_executable(run_from_tmpdir=True) assert mv._model_binpath != original_binpath assert mv._tmpdir is not None # Adjust the info bundle for this Multiverse to use the temporary location tmp_binpath = mv._model_binpath mv._info_bundle = copy.deepcopy(mv.info_bundle) mv._info_bundle.paths['binary'] = tmp_binpath # With the executable in a temporary location, we can change its access # rights to test the PermissionError os.chmod(tmp_binpath, 0o600) with pytest.raises(PermissionError, match="is not executable"): mv._prepare_executable() # Finally, remove that (temporary) file, to test the FileNotFound error os.remove(tmp_binpath) with pytest.raises(FileNotFoundError, match="Did you build it?"): mv._prepare_executable()
def test_FrozenMultiverse(mv_kwargs, cluster_env): """Test the FrozenMultiverse class""" # Need a regular Multiverse and corresponding output for that mv = Multiverse(**mv_kwargs) mv.run_single() # NOTE Need to adjust the data directory in order to not create collisions # in the eval directory due to same timestamps ... # Now create a frozen Multiverse from that one # Without run directory, the latest one should be loaded print("\nInitializing FrozenMultiverse without further kwargs") FrozenMultiverse(**mv_kwargs, data_manager=dict(out_dir="eval/{timestamp:}_1")) # With a relative path, the corresponding directory should be found print("\nInitializing FrozenMultiverse with timestamp as run_dir") FrozenMultiverse(**mv_kwargs, run_dir=os.path.basename(mv.dirs['run']), data_manager=dict(out_dir="eval/{timestamp:}_2")) # With an absolute path, that path should be used directly print("\nInitializing FrozenMultiverse with absolute path to run_dir") FrozenMultiverse(**mv_kwargs, run_dir=mv.dirs['run'], data_manager=dict(out_dir="eval/{timestamp:}_3")) # With a relative path, the path relative to the CWD should be used print("\nInitializing FrozenMultiverse with relative path to run_dir") FrozenMultiverse(**mv_kwargs, run_dir=os.path.relpath(mv.dirs['run'], start=os.getcwd()), data_manager=dict(out_dir="eval/{timestamp:}_4")) # Bad type of run directory should fail with pytest.raises(TypeError, match="Argument run_dir needs"): FrozenMultiverse(**mv_kwargs, run_dir=123, data_manager=dict(out_dir="eval/{timestamp:}_5")) # Non-existing directory should fail with pytest.raises(IOError, match="No run directory found at"): FrozenMultiverse(**mv_kwargs, run_dir="my_non-existing_directory", data_manager=dict(out_dir="eval/{timestamp:}_6")) # Cluster mode print("\nInitializing FrozenMultiverse in cluster mode") mv_kwargs['run_cfg_path'] = CLUSTER_MODE_CFG_PATH mv_kwargs['cluster_params'] = dict(env=cluster_env) FrozenMultiverse(**mv_kwargs, run_dir=os.path.relpath(mv.dirs['run'], start=os.getcwd()), data_manager=dict(out_dir="eval/{timestamp:}_7")) with pytest.raises(NotImplementedError, match="use_meta_cfg_from_run_dir"): FrozenMultiverse(**mv_kwargs, run_dir="/some/path/to/a/run_dir", use_meta_cfg_from_run_dir=True, data_manager=dict(out_dir="eval/{timestamp:}_7"))
def test_config_handling(mv_kwargs): """Tests the config handling of the Multiverse""" # Multiverse that does not load the default user config mv_kwargs['user_cfg_path'] = False Multiverse(**mv_kwargs) # Testing whether errors are raised # Multiverse with wrong run config mv_kwargs['run_cfg_path'] = 'an/invalid/run_cfg_path' with pytest.raises(FileNotFoundError): Multiverse(**mv_kwargs)
def test_shared_worker_manager(mv_kwargs): """Tests using a shared WorkerManager between multiple Multiverses, an experimental feature that allows to use the WorkerManager for running multiple Multiverses. """ mvs = list() shared_wm = None # Create a number of Multiverse, some with sweeps configured for i in range(5): _kws = copy.deepcopy(mv_kwargs) _kws['paths']['model_note'] += f"_no{i}" if i%2 == 0: _kws['run_cfg_path'] = SWEEP_CFG_PATH # Create Multiverse and manually add tasks mv = Multiverse(**_kws, _shared_worker_manager=shared_wm) mv._add_sim_tasks() # Keep track of it mvs.append(mv) # Define the shared WorkerManager instance (for the next iteration) shared_wm = mvs[0].wm # There should now be a total of 14 tasks, 4 each from Multiverses 0, 2, # and 4, and one each from Multiverses 1 and 3 assert len(mvs) == 5 assert len(shared_wm.tasks) == (4 + 1 + 4 + 1 + 4) # Let the shared WorkerManager start working shared_wm.start_working() # Check the output directories of each Multiverse were created (proxy for # the run having succeeded) for i, mv in enumerate(mvs): if i%2 == 0: uni_names = ('uni1', 'uni2', 'uni3', 'uni4') else: uni_names = ('uni0',) for uni_name in uni_names: assert os.path.isdir(os.path.join(mv.dirs['data'], uni_name)) # Report files will only be created for the first Multiverse, because # there is (and can be) only one Reporter instance. _report_file = os.path.join(mv.dirs['run'], '_report.txt') _sweep_info_file = os.path.join(mv.dirs['run'], '_sweep_info.txt') if i == 0: assert os.path.isfile(_report_file) assert os.path.isfile(_sweep_info_file) else: assert not os.path.isfile(_report_file) assert not os.path.isfile(_sweep_info_file)
def test_detect_doubled_folders(mv_kwargs): """Tests whether an existing folder will raise an exception.""" # Init Multiverse Multiverse(**mv_kwargs) # create output folders again # expect error due to existing folders with pytest.raises(RuntimeError, match="Simulation directory already"): # And another one, that will also create a directory Multiverse(**mv_kwargs) Multiverse(**mv_kwargs)
def dm_after_large_sweep(mv_kwargs) -> DataManager: """Initialises a Multiverse with a DataManager, runs a simulation with output going into a temporary directory, then returns the DataManager.""" # Initialise the Multiverse mv_kwargs['run_cfg_path'] = LARGE_SWEEP_CFG_PATH mv = Multiverse(**mv_kwargs) # Run a sweep mv.run_sweep() # Return the data manager return mv.dm
def test_prolog_and_epilog_is_run(mv_kwargs): """Test that the prolog and epilog are always run""" # NOTE Ensure that information on parallel execution is logged mv_kwargs['parameter_space'] = dict(log_levels=dict(model='debug')) # Run with default settings and check log message mv = Multiverse(**mv_kwargs) mv.run() log = mv.wm.tasks[0].streams['out']['log'] assert any(("Prolog finished." in line for line in log)) assert any(("Epilog finished." in line for line in log)) # The "Invoking epilog ..." message should _not_ be there in *this* case, # because it denotes that the simulation stopped after receiving a signal assert not any(("Invoking epilog ..." in line for line in log)) # Now perform a longer simulation with a timeout mv_kwargs['parameter_space']['num_steps'] = int(1e9) mv_kwargs['parameter_space']['write_every'] = int(1e6) mv_kwargs['run_kwargs'] = dict(timeout=1.) mv_kwargs['paths']['model_note'] = "with_timeout" mv = Multiverse(**mv_kwargs) mv.run() log = mv.wm.tasks[0].streams['out']['log'] assert any(("Prolog finished." in line for line in log)) assert any(("Invoking epilog ..." in line for line in log)) assert any(("Epilog finished." in line for line in log))
def default_mv(mv_kwargs) -> Multiverse: """Initialises a unique default configuration of the Multiverse to test everything beyond initialisation. Using the mv_kwargs fixture, it is assured that the output directory is unique. """ return Multiverse(**mv_kwargs)
def test_backup(mv_kwargs): """Tests whether the backup of all config parts and the executable works""" mv = Multiverse(**mv_kwargs) cfg_path = mv.dirs['config'] assert os.path.isfile(os.path.join(cfg_path, 'base_cfg.yml')) assert os.path.isfile(os.path.join(cfg_path, 'user_cfg.yml')) assert os.path.isfile(os.path.join(cfg_path, 'model_cfg.yml')) assert os.path.isfile(os.path.join(cfg_path, 'run_cfg.yml')) assert os.path.isfile(os.path.join(cfg_path, 'update_cfg.yml')) # And once more, now including the executable mv_kwargs['backups'] = dict(backup_executable=True) mv_kwargs['paths']['model_note'] = "with-exec-backup" mv = Multiverse(**mv_kwargs) assert os.path.isfile(os.path.join(mv.dirs['run'], 'backup', 'dummy'))
def test_bifurcation_diagram_2d(tmpdir): """Test plotting of the bifurcation diagram""" # Create and run simulation raise_exc = {'plot_manager': {'raise_exc': True}} mv = Multiverse(model_name='SavannaHomogeneous', run_cfg_path=BIFURCATION_DIAGRAM_2D_RUN, paths=dict(out_dir=str(tmpdir)), **raise_exc) mv.run_sweep() # Load mv.dm.load_from_cfg(print_tree=False) # Plot the bifurcation using the last datapoint mv.pm.plot_from_cfg(plots_cfg=BIFURCATION_DIAGRAM_2D_PLOTS, plot_only=["bifurcation_diagram_2d"]) # Plot the bifurcation using the fixpoint mv.pm.plot_from_cfg(plots_cfg=BIFURCATION_DIAGRAM_2D_PLOTS, plot_only=["bifurcation_diagram_2d_fixpoint_to_plot"])
def test_multiple_runs_not_allowed(mv_kwargs): """Assert that multiple runs are prohibited""" # Create Multiverse and run mv = Multiverse(**mv_kwargs) mv.run_single() # Another run should not be possible with pytest.raises(RuntimeError, match="Could not add simulation task"): mv.run_single()
def test_parallel_init(mv_kwargs): """Test enabling parallel execution through the config""" # NOTE Ensure that information on parallel execution is logged mv_kwargs['parameter_space'] = dict(log_levels=dict(core='debug')) # Run with default settings and check log message mv_kwargs['paths']['model_note'] = "pexec_disabled" mv = Multiverse(**mv_kwargs) mv.run() log = mv.wm.tasks[0].streams['out']['log'] assert any(("Parallel execution disabled" in line for line in log)) # Now override default setting mv_kwargs['parameter_space']['parallel_execution'] = dict(enabled=True) mv_kwargs['paths']['model_note'] = "pexec_enabled" mv = Multiverse(**mv_kwargs) mv.run() log = mv.wm.tasks[0].streams['out']['log'] assert any(("Parallel execution enabled" in line for line in log))
def test_run_sweep(mv_kwargs): """Tests a run with a single simulation""" # Adjust the defaults to use the sweep configuration for run configuration mv_kwargs['run_cfg_path'] = SWEEP_CFG_PATH mv = Multiverse(**mv_kwargs) # Run the sweep mv.run() # There should now be four directories in the data directory assert len(os.listdir(mv.dirs['data'])) == 4 # With a parameter space without volume, i.e. without any sweeps added, # the sweep should not be possible mv_kwargs['run_cfg_path'] = RUN_CFG_PATH mv_kwargs['paths']['model_note'] = "_invalid_cfg" mv = Multiverse(**mv_kwargs) with pytest.raises(ValueError, match="The parameter space has no sweeps"): mv.run_sweep()
def test_bifurcation_diagram(tmpdir): """Test plotting of the bifurcation diagram""" # Create and run simulation raise_exc = {'plot_manager': {'raise_exc': True}} mv = Multiverse(model_name='SavannaHomogeneous', run_cfg_path=BIFURCATION_DIAGRAM_RUN, paths=dict(out_dir=str(tmpdir)), **raise_exc) mv.run_sweep() # Load mv.dm.load_from_cfg(print_tree=False) # Plot the bifurcation using the last datapoint mv.pm.plot_from_cfg(plots_cfg=BIFURCATION_DIAGRAM_PLOTS, plot_only=["bifurcation_endpoint"]) # Plot the bifurcation using the fixpoint mv.pm.plot_from_cfg(plots_cfg=BIFURCATION_DIAGRAM_PLOTS, plot_only=["bifurcation_fixpoint"]) mv.pm.plot_from_cfg(plots_cfg=BIFURCATION_DIAGRAM_PLOTS, plot_only=["bifurcation_fixpoint_to_plot"]) # Plot the bifurcation using scatter mv.pm.plot_from_cfg(plots_cfg=BIFURCATION_DIAGRAM_PLOTS, plot_only=["bifurcation_scatter"]) # Plot the bifurcation using oscillation mv.pm.plot_from_cfg(plots_cfg=BIFURCATION_DIAGRAM_PLOTS, plot_only=["bifurcation_oscillation"]) # Redo simulation, but using several initial conditions mv = Multiverse( model_name='SavannaHomogeneous', run_cfg_path=BIFURCATION_DIAGRAM_RUN, paths=dict(out_dir=str(tmpdir)), **raise_exc, parameter_space=dict(seed=psp.ParamDim(default=0, range=[4]))) mv.run_sweep() mv.dm.load_from_cfg(print_tree=False) # Plot the bifurcation using multistability mv.pm.plot_from_cfg(plots_cfg=BIFURCATION_DIAGRAM_PLOTS, plot_only=["bifurcation_fixpoint"])
def test_renew_plot_manager(mv_kwargs): """Tests the renewal of PlotManager instances in the Multiverse""" mv = Multiverse(**mv_kwargs) initial_pm = mv.pm # Try to renew it (with a bad config). The old one should remain with pytest.raises(ValueError, match="Failed setting up"): mv.renew_plot_manager(foo="bar") assert mv.pm is initial_pm # Again, this time it should work mv.renew_plot_manager() assert mv.pm is not initial_pm
def test_parameter_validation(mv_kwargs): """Tests integration of the parameter validation feature""" # Works mv_kwargs['run_cfg_path'] = RUN_CFG_PATH_VALID mv_kwargs['model_name'] = "ForestFire" mv_kwargs['paths']['model_note'] = "valid" mv = Multiverse(**mv_kwargs) mv.run_single() # Fails mv_kwargs['run_cfg_path'] = RUN_CFG_PATH_INVALID mv_kwargs['model_name'] = "ForestFire" mv_kwargs['paths']['model_note'] = "invalid" with pytest.raises(ValidationError, match="Validation failed for 3 para"): mv = Multiverse(**mv_kwargs)
def test_graph_plots(tmpdir): """Tests the plot_funcs.dag.graph module""" # Create and run simulation raise_exc = {'plot_manager': {'raise_exc': True}} mv = Multiverse(model_name='CopyMeGraph', run_cfg_path=GRAPH_RUN, paths=dict(out_dir=str(tmpdir)), **raise_exc) mv.run_single() # Load mv.dm.load_from_cfg(print_tree=False) # Single graph plots mv.pm.plot_from_cfg( plots_cfg=GRAPH_PLOTS, plot_only=( "Graph", "DiGraph", "MultiGraph", "MultiDiGraph", "ExternalProperties", "Example_graph_plot", "custom_node_positioning_model", "explicit_node_positions", "custom_graph_creation", "custom_graph_arr_creation", ), ) # Animation plots mv.pm.plot_from_cfg( plots_cfg=GRAPH_PLOTS, plot_only=[ "graph_anim1", "graph_anim2", "graph_anim3", "graph_anim_external", "graph_anim4", "graph_anim_custom_graph_creation", ], ) # Test failing cases – if possible these test are done in the (faster) # GraphPlot-class test. # Providing invalid dag tag for external property with pytest.raises( PlotCreatorError, match=( "No tag 'some_state_transformed' found in the data selected by " "the DAG!"), ): mv.pm.plot_from_cfg(plots_cfg=GRAPH_PLOTS, plot_only=["invalid_ext_prop"]) # Ambiguous time specifications for animation with pytest.raises(PlotCreatorError, match="ambiguous animation time specifications"): mv.pm.plot_from_cfg(plots_cfg=GRAPH_PLOTS, plot_only=["anim_amgiguous_time_spec"]) # Trying to animate from single nx.Graph with pytest.raises(PlotCreatorError, match="due to invalid type of the 'graph'"): mv.pm.plot_from_cfg(plots_cfg=GRAPH_PLOTS, plot_only=["anim_not_dataarray"])
def test_cluster_mode_run(mv_kwargs, cluster_env_specific): cluster_env = cluster_env_specific # Define a custom test environment mv_kwargs['run_cfg_path'] = CLUSTER_MODE_CFG_PATH mv_kwargs['cluster_params'] = dict(env=cluster_env) # Parameter space has 12 points # Five nodes are being used: node002, node003, node004, node006, node011 # Test for first node, should perform 3 simulations cluster_env['TEST_NODENAME'] = "node002" mv_kwargs['paths']['model_note'] = "node002" mv = Multiverse(**mv_kwargs) mv.run_sweep() assert mv.wm.num_finished_tasks == 3 assert [t.name for t in mv.wm.tasks] == ['uni01', 'uni06', 'uni11'] # NOTE: simulated universes are uni01 ... uni12 # Test for second node, should also perform 3 simulations cluster_env['TEST_NODENAME'] = "node003" mv_kwargs['paths']['model_note'] = "node003" mv = Multiverse(**mv_kwargs) mv.run_sweep() assert mv.wm.num_finished_tasks == 3 assert [t.name for t in mv.wm.tasks] == ['uni02', 'uni07', 'uni12'] # The third node should only perform 2 simulations cluster_env['TEST_NODENAME'] = "node004" mv_kwargs['paths']['model_note'] = "node004" mv = Multiverse(**mv_kwargs) mv.run_sweep() assert mv.wm.num_finished_tasks == 2 assert [t.name for t in mv.wm.tasks] == ['uni03', 'uni08'] # The fourth and fifth node should also perform only 2 simulations cluster_env['TEST_NODENAME'] = "node006" mv_kwargs['paths']['model_note'] = "node006" mv = Multiverse(**mv_kwargs) mv.run_sweep() assert mv.wm.num_finished_tasks == 2 assert [t.name for t in mv.wm.tasks] == ['uni04', 'uni09'] cluster_env['TEST_NODENAME'] = "node011" mv_kwargs['paths']['model_note'] = "node011" mv = Multiverse(**mv_kwargs) mv.run_sweep() assert mv.wm.num_finished_tasks == 2 assert [t.name for t in mv.wm.tasks] == ['uni05', 'uni10']
def test_cluster_mode(mv_kwargs, cluster_env): """Tests cluster mode basics like: resolution of parameters, creation of the run directory, ... """ # Define a custom test environment mv_kwargs['run_cfg_path'] = CLUSTER_MODE_CFG_PATH mv_kwargs['cluster_params'] = dict(env=cluster_env) # Create the Multiverse mv = Multiverse(**mv_kwargs) rcps = mv.resolved_cluster_params assert len(rcps) == 10 + 1 # Check the custom output directory assert 'my_custom_dir' in mv.dirs['run'] # Check the job ID is part of the run directory path assert 'job123' in mv.dirs['run'] # Make sure the required keys are available assert all([k in rcps for k in ('job_id', 'num_nodes', 'node_list', 'node_name', 'timestamp')]) # Check some types assert isinstance(rcps['job_id'], int) assert isinstance(rcps['num_nodes'], int) assert isinstance(rcps['num_procs'], int) assert isinstance(rcps['node_list'], list) assert isinstance(rcps['timestamp'], int) # Check some values assert rcps['node_index'] == 3 # for node006 assert rcps['timestamp'] > 0 assert "node006" in rcps['node_list'] assert len(rcps['node_list']) == 5 # NOTE Actual parsing of node list is checked in test__cluster.py # Can add additional info to the run directory mv_kwargs['cluster_params']['additional_run_dir_fstrs'] = ["xyz{job_id:}", "N{num_nodes:}"] mv = Multiverse(**mv_kwargs) assert 'xyz123_N5' in mv.dirs['run'] # Single-node case cluster_env['TEST_JOB_NUM_NODES'] = '1' cluster_env['TEST_JOB_NODELIST'] = "node006" mv = Multiverse(**mv_kwargs) assert mv.resolved_cluster_params['node_list'] == ["node006"] # Test error messages; also see test__cluster.py for more dedicated tests # Node name not in node list cluster_env['TEST_NODENAME'] = 'node042' with pytest.raises(ValueError, match="Failed parsing node list"): Multiverse(**mv_kwargs) # Wrong number of nodes cluster_env['TEST_NODENAME'] = 'node003' cluster_env['TEST_JOB_NUM_NODES'] = '3' with pytest.raises(ValueError, match="Failed parsing node list"): Multiverse(**mv_kwargs) # Missing environment variables cluster_env.pop('TEST_NODENAME') with pytest.raises(ValueError, match="Missing required environment variable"): Multiverse(**mv_kwargs)
def test_invalid_model_name_and_operation(default_mv, mv_kwargs): """Tests for correct behaviour upon invalid model names""" # Try to instantiate with invalid model name mv_kwargs['model_name'] = "invalid_model_RandomShit_bgsbjkbkfvwuRfopiwehGP" with pytest.raises(KeyError, match="No model with name 'invalid_model_"): Multiverse(**mv_kwargs)