def test_is_new_instance(): """ Verify that every instance of RunOptions() and its subclasses is a new instance (i.e. all factory methods are used properly.) """ check_if_different(RunOptions(), RunOptions()) check_if_different(TrainerSettings(), TrainerSettings())
def test_step_takes_steps_for_all_non_waiting_envs(self, mock_create_worker): mock_create_worker.side_effect = create_worker_mock manager = SubprocessEnvManager(mock_env_factory, RunOptions(), 3) manager.step_queue = Mock() manager.step_queue.get_nowait.side_effect = [ EnvironmentResponse(EnvironmentCommand.STEP, 0, StepResponse(0, None, {})), EnvironmentResponse(EnvironmentCommand.STEP, 1, StepResponse(1, None, {})), EmptyQueue(), ] step_mock = Mock() last_steps = [Mock(), Mock(), Mock()] manager.env_workers[0].previous_step = last_steps[0] manager.env_workers[1].previous_step = last_steps[1] manager.env_workers[2].previous_step = last_steps[2] manager.env_workers[2].waiting = True manager._take_step = Mock(return_value=step_mock) res = manager._step() for i, env in enumerate(manager.env_workers): if i < 2: env.send.assert_called_with(EnvironmentCommand.STEP, step_mock) manager.step_queue.get_nowait.assert_called() # Check that the "last steps" are set to the value returned for each step self.assertEqual( manager.env_workers[i].previous_step.current_all_step_result, i ) assert res == [ manager.env_workers[0].previous_step, manager.env_workers[1].previous_step, ]
def test_reset_passes_reset_params(self, mock_create_worker): mock_create_worker.side_effect = create_worker_mock manager = SubprocessEnvManager(mock_env_factory, RunOptions(), 1) params = {"test": "params"} manager._reset_env(params) manager.env_workers[0].send.assert_called_with( EnvironmentCommand.RESET, (params))
def test_environments_are_created(self, mock_create_worker): mock_create_worker.side_effect = create_worker_mock run_options = RunOptions() env = SubprocessEnvManager(mock_env_factory, run_options, 2) # Creates two processes env.create_worker.assert_has_calls([ mock.call(0, env.step_queue, mock_env_factory, run_options), mock.call(1, env.step_queue, mock_env_factory, run_options), ]) self.assertEqual(len(env.env_workers), 2)
def test_no_configuration(): """ Verify that a new config will have a PPO trainer with extrinsic rewards. """ blank_runoptions = RunOptions() blank_runoptions.behaviors.set_config_specified(False) assert isinstance(blank_runoptions.behaviors["test"], TrainerSettings) assert isinstance(blank_runoptions.behaviors["test"].hyperparameters, PPOSettings) assert (RewardSignalType.EXTRINSIC in blank_runoptions.behaviors["test"].reward_signals)
def test_crashed_env_restarts(self, mock_create_worker): crashing_worker = MockEnvWorker( 0, EnvironmentResponse(EnvironmentCommand.RESET, 0, 0) ) restarting_worker = MockEnvWorker( 0, EnvironmentResponse(EnvironmentCommand.RESET, 0, 0) ) healthy_worker = MockEnvWorker( 1, EnvironmentResponse(EnvironmentCommand.RESET, 1, 1) ) mock_create_worker.side_effect = [ crashing_worker, healthy_worker, restarting_worker, ] manager = SubprocessEnvManager(mock_env_factory, RunOptions(), 2) manager.step_queue = Mock() manager.step_queue.get_nowait.side_effect = [ EnvironmentResponse( EnvironmentCommand.ENV_EXITED, 0, UnityCommunicationException("Test msg"), ), EnvironmentResponse(EnvironmentCommand.CLOSED, 0, None), EnvironmentResponse(EnvironmentCommand.STEP, 1, StepResponse(0, None, {})), EmptyQueue(), EnvironmentResponse(EnvironmentCommand.STEP, 0, StepResponse(1, None, {})), EnvironmentResponse(EnvironmentCommand.STEP, 1, StepResponse(2, None, {})), EmptyQueue(), ] step_mock = Mock() last_steps = [Mock(), Mock(), Mock()] assert crashing_worker is manager.env_workers[0] assert healthy_worker is manager.env_workers[1] crashing_worker.previous_step = last_steps[0] crashing_worker.waiting = True healthy_worker.previous_step = last_steps[1] healthy_worker.waiting = True manager._take_step = Mock(return_value=step_mock) manager._step() healthy_worker.send.assert_has_calls( [ call(EnvironmentCommand.ENVIRONMENT_PARAMETERS, ANY), call(EnvironmentCommand.RESET, ANY), call(EnvironmentCommand.STEP, ANY), ] ) restarting_worker.send.assert_has_calls( [ call(EnvironmentCommand.ENVIRONMENT_PARAMETERS, ANY), call(EnvironmentCommand.RESET, ANY), call(EnvironmentCommand.STEP, ANY), ] )
def test_reset_collects_results_from_all_envs(self, mock_create_worker): mock_create_worker.side_effect = create_worker_mock manager = SubprocessEnvManager(mock_env_factory, RunOptions(), 4) params = {"test": "params"} res = manager._reset_env(params) for i, env in enumerate(manager.env_workers): env.send.assert_called_with(EnvironmentCommand.RESET, (params)) env.recv.assert_called() # Check that the "last steps" are set to the value returned for each step self.assertEqual( manager.env_workers[i].previous_step.current_all_step_result, i ) assert res == list(map(lambda ew: ew.previous_step, manager.env_workers))
def test_subprocess_env_raises_errors(num_envs): def failing_env_factory(worker_id, config): import time # Sleep momentarily to allow time for the EnvManager to be waiting for the # subprocess response. We won't be able to capture failures from the subprocess # that cause it to close the pipe before we can send the first message. time.sleep(0.5) raise UnityEnvironmentException() env_manager = SubprocessEnvManager(failing_env_factory, RunOptions(), num_envs) with pytest.raises(UnityEnvironmentException): env_manager.reset() env_manager.close()
def test_handles_no_config_provided(): """ Make sure the trainer setup handles no configs provided at all. """ brain_name = "testbrain" no_default_config = RunOptions().behaviors trainer_factory = trainer_util.TrainerFactory( trainer_config=no_default_config, output_path="output_path", train_model=True, load_model=False, seed=42, ) trainer_factory.generate(brain_name)
def test_subprocess_failing_step(num_envs): def failing_step_env_factory(_worker_id, _config): env = UnexpectedExceptionEnvironment( ["1D"], use_discrete=True, to_raise=CustomTestOnlyException ) return env env_manager = SubprocessEnvManager(failing_step_env_factory, RunOptions()) # Expect the exception raised to be routed back up to the top level. with pytest.raises(CustomTestOnlyException): check_environment_trains( failing_step_env_factory(0, []), {"1D": ppo_dummy_config()}, env_manager=env_manager, success_threshold=None, ) env_manager.close()
def test_handles_no_config_provided(): """ Make sure the trainer setup handles no configs provided at all. """ brain_name = "testbrain" no_default_config = RunOptions().behaviors # Pretend this was created without a YAML file no_default_config.set_config_specified(False) trainer_factory = TrainerFactory( trainer_config=no_default_config, output_path="output_path", train_model=True, load_model=False, seed=42, param_manager=EnvironmentParameterManager(), ) trainer_factory.generate(brain_name)
def test_training_behaviors_collects_results_from_all_envs( self, mock_create_worker): def create_worker_mock(worker_id, step_queue, env_factor, engine_c): return MockEnvWorker( worker_id, EnvironmentResponse(EnvironmentCommand.RESET, worker_id, {f"key{worker_id}": worker_id}), ) mock_create_worker.side_effect = create_worker_mock manager = SubprocessEnvManager(mock_env_factory, RunOptions(), 4) res = manager.training_behaviors for env in manager.env_workers: env.send.assert_called_with(EnvironmentCommand.BEHAVIOR_SPECS) env.recv.assert_called() for worker_id in range(4): assert f"key{worker_id}" in res assert res[f"key{worker_id}"] == worker_id
def test_subprocess_env_endtoend(num_envs): def simple_env_factory(worker_id, config): env = SimpleEnvironment(["1D"], action_sizes=(0, 1)) return env env_manager = SubprocessEnvManager(simple_env_factory, RunOptions(), num_envs) # Run PPO using env_manager check_environment_trains( simple_env_factory(0, []), {"1D": ppo_dummy_config()}, env_manager=env_manager, success_threshold=None, ) # Note we can't check the env's rewards directly (since they're in separate processes) so we # check the StatsReporter's debug stat writer's last reward. assert isinstance(StatsReporter.writers[0], DebugWriter) assert all( val > 0.7 for val in StatsReporter.writers[0].get_last_rewards().values() ) env_manager.close()
def test_advance(self, mock_create_worker, training_behaviors_mock, step_mock): brain_name = "testbrain" action_info_dict = {brain_name: MagicMock()} mock_create_worker.side_effect = create_worker_mock env_manager = SubprocessEnvManager(mock_env_factory, RunOptions(), 3) training_behaviors_mock.return_value = [brain_name] agent_manager_mock = mock.Mock() mock_policy = mock.Mock() agent_manager_mock.policy_queue.get_nowait.side_effect = [ mock_policy, mock_policy, AgentManagerQueue.Empty(), ] env_manager.set_agent_manager(brain_name, agent_manager_mock) step_info_dict = {brain_name: (Mock(), Mock())} env_stats = { "averaged": (1.0, StatsAggregationMethod.AVERAGE), "most_recent": (2.0, StatsAggregationMethod.MOST_RECENT), } step_info = EnvironmentStep(step_info_dict, 0, action_info_dict, env_stats) step_mock.return_value = [step_info] env_manager.process_steps(env_manager.get_steps()) # Test add_experiences env_manager._step.assert_called_once() agent_manager_mock.add_experiences.assert_called_once_with( step_info.current_all_step_result[brain_name][0], step_info.current_all_step_result[brain_name][1], 0, step_info.brain_name_to_action_info[brain_name], ) # Test policy queue assert env_manager.policies[brain_name] == mock_policy assert agent_manager_mock.policy == mock_policy
def test_handles_no_config_provided(BrainParametersMock): """ Make sure the trainer setup handles no configs provided at all. """ brain_name = "testbrain" no_default_config = RunOptions().behaviors brain_parameters = BrainParameters( brain_name=brain_name, vector_observation_space_size=1, camera_resolutions=[], vector_action_space_size=[2], vector_action_descriptions=[], vector_action_space_type=0, ) trainer_factory = trainer_util.TrainerFactory( trainer_config=no_default_config, output_path="output_path", train_model=True, load_model=False, seed=42, ) trainer_factory.generate(brain_parameters.brain_name)
def poca_config(): return RunOptions(behaviors={"test_brain": poca_dummy_config()})
def test_pickle(): # Make sure RunOptions is pickle-able. run_options = RunOptions() p = pickle.dumps(run_options) pickle.loads(p)
def test_exportable_settings(use_defaults): """ Test that structuring and unstructuring a RunOptions object results in the same configuration representation. """ # Try to enable as many features as possible in this test YAML to hit all the # edge cases. Set as much as possible as non-default values to ensure no flukes. test_yaml = """ behaviors: 3DBall: trainer_type: sac hyperparameters: learning_rate: 0.0004 learning_rate_schedule: constant batch_size: 64 buffer_size: 200000 buffer_init_steps: 100 tau: 0.006 steps_per_update: 10.0 save_replay_buffer: true init_entcoef: 0.5 reward_signal_steps_per_update: 10.0 network_settings: normalize: false hidden_units: 256 num_layers: 3 vis_encode_type: nature_cnn memory: memory_size: 1288 sequence_length: 12 reward_signals: extrinsic: gamma: 0.999 strength: 1.0 curiosity: gamma: 0.999 strength: 1.0 keep_checkpoints: 5 max_steps: 500000 time_horizon: 1000 summary_freq: 12000 checkpoint_interval: 1 threaded: true env_settings: env_path: test_env_path env_args: - test_env_args1 - test_env_args2 base_port: 12345 num_envs: 8 seed: 12345 engine_settings: width: 12345 height: 12345 quality_level: 12345 time_scale: 12345 target_frame_rate: 12345 capture_frame_rate: 12345 no_graphics: true checkpoint_settings: run_id: test_run_id initialize_from: test_directory load_model: false resume: true force: true train_model: false inference: false debug: true environment_parameters: big_wall_height: curriculum: - name: Lesson0 completion_criteria: measure: progress behavior: BigWallJump signal_smoothing: true min_lesson_length: 100 threshold: 0.1 value: sampler_type: uniform sampler_parameters: min_value: 0.0 max_value: 4.0 - name: Lesson1 completion_criteria: measure: reward behavior: BigWallJump signal_smoothing: true min_lesson_length: 100 threshold: 0.2 value: sampler_type: gaussian sampler_parameters: mean: 4.0 st_dev: 7.0 - name: Lesson2 completion_criteria: measure: progress behavior: BigWallJump signal_smoothing: true min_lesson_length: 20 threshold: 0.3 value: sampler_type: multirangeuniform sampler_parameters: intervals: [[1.0, 2.0],[4.0, 5.0]] - name: Lesson3 value: 8.0 small_wall_height: 42.0 other_wall_height: sampler_type: multirangeuniform sampler_parameters: intervals: [[1.0, 2.0],[4.0, 5.0]] """ if not use_defaults: loaded_yaml = yaml.safe_load(test_yaml) run_options = RunOptions.from_dict(yaml.safe_load(test_yaml)) else: run_options = RunOptions() dict_export = run_options.as_dict() if not use_defaults: # Don't need to check if no yaml check_dict_is_at_least(loaded_yaml, dict_export, exceptions=["environment_parameters"]) # Re-import and verify has same elements run_options2 = RunOptions.from_dict(dict_export) second_export = run_options2.as_dict() check_dict_is_at_least(dict_export, second_export) # Should be able to use equality instead of back-and-forth once environment_parameters # is working check_dict_is_at_least(second_export, dict_export) # Check that the two exports are the same assert dict_export == second_export
def test_exportable_settings(use_defaults): """ Test that structuring and unstructuring a RunOptions object results in the same configuration representation. """ # Try to enable as many features as possible in this test YAML to hit all the # edge cases. Set as much as possible as non-default values to ensure no flukes. test_yaml = """ behaviors: 3DBall: trainer_type: sac hyperparameters: learning_rate: 0.0004 learning_rate_schedule: constant batch_size: 64 buffer_size: 200000 buffer_init_steps: 100 tau: 0.006 steps_per_update: 10.0 save_replay_buffer: true init_entcoef: 0.5 reward_signal_steps_per_update: 10.0 network_settings: normalize: false hidden_units: 256 num_layers: 3 vis_encode_type: nature_cnn memory: memory_size: 1288 sequence_length: 12 reward_signals: extrinsic: gamma: 0.999 strength: 1.0 curiosity: gamma: 0.999 strength: 1.0 keep_checkpoints: 5 max_steps: 500000 time_horizon: 1000 summary_freq: 12000 checkpoint_interval: 1 threaded: true env_settings: env_path: test_env_path env_args: - test_env_args1 - test_env_args2 base_port: 12345 num_envs: 8 seed: 12345 engine_settings: width: 12345 height: 12345 quality_level: 12345 time_scale: 12345 target_frame_rate: 12345 capture_frame_rate: 12345 no_graphics: true checkpoint_settings: run_id: test_run_id initialize_from: test_directory load_model: false resume: true force: true train_model: false inference: false debug: true """ if not use_defaults: loaded_yaml = yaml.safe_load(test_yaml) run_options = RunOptions.from_dict(yaml.safe_load(test_yaml)) else: run_options = RunOptions() dict_export = run_options.as_dict() if not use_defaults: # Don't need to check if no yaml check_dict_is_at_least(loaded_yaml, dict_export) # Re-import and verify has same elements run_options2 = RunOptions.from_dict(dict_export) second_export = run_options2.as_dict() # Check that the two exports are the same assert dict_export == second_export
def dummy_config(): return RunOptions(behaviors={"testbrain": ppo_dummy_config()})
def dummy_config(): return RunOptions(behaviors={"testbrain": PPO_CONFIG})
def test_register_stats_writers(): # Make sure that the ExampleStatsWriter gets returned from the list of all StatsWriters stats_writers = register_stats_writer_plugins(RunOptions()) assert any(isinstance(sw, ExampleStatsWriter) for sw in stats_writers)
def sac_config(): return RunOptions(behaviors={"test_brain": sac_dummy_config()})