def test_update_policy_and_value_function(self): action_space = spaces.Discrete(2) observation_space = spaces.Box(0, 1, (1, )) sut = ActorCritic('', observation_space, action_space) # Set up. self._setup_trajectory(sut) sut._process_accumulated_trajectory(True) sut._trainer = MagicMock() sut._adjust_learning_rate = MagicMock() # Call test method. sut._update_networks() # Verify value network behavior. self.assertEqual(sut._trainer.train_minibatch.call_count, 1) call_args = sut._trainer.train_minibatch.call_args np.testing.assert_array_equal( call_args[0][0][sut._input_variables], [np.array([0.1], np.float32), np.array([0.2], np.float32)]) np.testing.assert_array_almost_equal( call_args[0][0][sut._value_network_output_variables], [[2.9975], [3.05]]) np.testing.assert_array_equal( call_args[0][0][sut._policy_network_output_variables], [np.array([1, 0], np.float32), np.array([0, 1], np.float32)]) np.testing.assert_array_almost_equal( call_args[0][0][sut._policy_network_weight_variables], [[0.9975], [2.05]]) # Verify data buffer size. self.assertEqual(len(sut._input_buffer), 0)
def test_update_policy_and_value_function(self): action_space = spaces.Discrete(2) observation_space = spaces.Box(0, 1, (1,)) sut = ActorCritic('', observation_space, action_space) # Set up. self._setup_trajectory(sut) sut._process_accumulated_trajectory(True) sut._trainer = MagicMock() sut._adjust_learning_rate = MagicMock() # Call test method. sut._update_networks() # Verify value network behavior. self.assertEqual( sut._trainer.train_minibatch.call_count, 1) call_args = sut._trainer.train_minibatch.call_args np.testing.assert_array_equal( call_args[0][0][sut._input_variables], [np.array([0.1], np.float32), np.array([0.2], np.float32)]) np.testing.assert_array_almost_equal( call_args[0][0][sut._value_network_output_variables], [[2.9975], [3.05]]) np.testing.assert_array_equal( call_args[0][0][sut._policy_network_output_variables], [np.array([1, 0], np.float32), np.array([0, 1], np.float32)]) np.testing.assert_array_almost_equal( call_args[0][0][sut._policy_network_weight_variables], [[0.9975], [2.05]]) # Verify data buffer size. self.assertEqual(len(sut._input_buffer), 0)
def test_rollout_with_update(self, mock_parameters): self._setup_parameters(mock_parameters.return_value) mock_parameters.return_value.update_frequency = 2 action_space = spaces.Discrete(2) observation_space = spaces.Box(0, 1, (1, )) sut = ActorCritic('', observation_space, action_space) sut._update_networks = MagicMock() sut._choose_action = Mock( side_effect=[(0, ''), (1, ''), (1, ''), (0, ''), (1, ''), (0, '')]) sut.start(np.array([0.1], np.float32)) sut.step(0.1, np.array([0.2], np.float32)) self.assertEqual(sut._trajectory_rewards, [0.1]) self.assertEqual(sut._trajectory_actions, [0, 1]) self.assertEqual(sut._trajectory_states, [0.1, 0.2]) self.assertEqual(sut._update_networks.call_count, 0) sut.step(0.2, np.array([0.3], np.float32)) self.assertEqual(sut._trajectory_rewards, []) self.assertEqual(sut._trajectory_actions, [1]) self.assertEqual(sut._trajectory_states, [0.3]) self.assertEqual(sut._update_networks.call_count, 1) sut.step(0.3, np.array([0.4], np.float32)) self.assertEqual(sut._trajectory_rewards, [0.3]) self.assertEqual(sut._trajectory_actions, [1, 0]) self.assertEqual(sut._trajectory_states, [0.3, 0.4]) self.assertEqual(sut._update_networks.call_count, 1) sut.start(np.array([0.5], np.float32)) self.assertEqual(sut._trajectory_rewards, []) self.assertEqual(sut._trajectory_actions, [1]) self.assertEqual(sut._trajectory_states, [0.5]) self.assertEqual(sut._update_networks.call_count, 1) sut.step(0.4, np.array([0.6], np.float32)) self.assertEqual(sut._trajectory_rewards, []) self.assertEqual(sut._trajectory_actions, [0]) self.assertEqual(sut._trajectory_states, [0.6]) self.assertEqual(sut._update_networks.call_count, 2) sut.end(0.5, np.array([0.7], np.float32)) self.assertEqual(sut._trajectory_rewards, [0.5]) self.assertEqual(sut._trajectory_actions, [0]) self.assertEqual(sut._trajectory_states, [0.6]) self.assertEqual(sut._update_networks.call_count, 2)
def test_rollout_with_update(self, mock_parameters): self._setup_parameters(mock_parameters.return_value) mock_parameters.return_value.update_frequency = 2 action_space = spaces.Discrete(2) observation_space = spaces.Box(0, 1, (1,)) sut = ActorCritic('', observation_space, action_space) sut._update_networks = MagicMock() sut._choose_action = Mock(side_effect=[ (0, ''), (1, ''), (1, ''), (0, ''), (1, ''), (0, '')]) sut.start(np.array([0.1], np.float32)) sut.step(0.1, np.array([0.2], np.float32)) self.assertEqual(sut._trajectory_rewards, [0.1]) self.assertEqual(sut._trajectory_actions, [0, 1]) self.assertEqual(sut._trajectory_states, [0.1, 0.2]) self.assertEqual(sut._update_networks.call_count, 0) sut.step(0.2, np.array([0.3], np.float32)) self.assertEqual(sut._trajectory_rewards, []) self.assertEqual(sut._trajectory_actions, [1]) self.assertEqual(sut._trajectory_states, [0.3]) self.assertEqual(sut._update_networks.call_count, 1) sut.step(0.3, np.array([0.4], np.float32)) self.assertEqual(sut._trajectory_rewards, [0.3]) self.assertEqual(sut._trajectory_actions, [1, 0]) self.assertEqual(sut._trajectory_states, [0.3, 0.4]) self.assertEqual(sut._update_networks.call_count, 1) sut.start(np.array([0.5], np.float32)) self.assertEqual(sut._trajectory_rewards, []) self.assertEqual(sut._trajectory_actions, [1]) self.assertEqual(sut._trajectory_states, [0.5]) self.assertEqual(sut._update_networks.call_count, 1) sut.step(0.4, np.array([0.6], np.float32)) self.assertEqual(sut._trajectory_rewards, []) self.assertEqual(sut._trajectory_actions, [0]) self.assertEqual(sut._trajectory_states, [0.6]) self.assertEqual(sut._update_networks.call_count, 2) sut.end(0.5, np.array([0.7], np.float32)) self.assertEqual(sut._trajectory_rewards, [0.5]) self.assertEqual(sut._trajectory_actions, [0]) self.assertEqual(sut._trajectory_states, [0.6]) self.assertEqual(sut._update_networks.call_count, 2)