def test_update_policy_and_value_function(self): action_space = spaces.Discrete(2) observation_space = spaces.Box(0, 1, (1, )) sut = ActorCritic('', observation_space, action_space) # Set up. self._setup_trajectory(sut) sut._process_accumulated_trajectory(True) sut._trainer = MagicMock() sut._adjust_learning_rate = MagicMock() # Call test method. sut._update_networks() # Verify value network behavior. self.assertEqual(sut._trainer.train_minibatch.call_count, 1) call_args = sut._trainer.train_minibatch.call_args np.testing.assert_array_equal( call_args[0][0][sut._input_variables], [np.array([0.1], np.float32), np.array([0.2], np.float32)]) np.testing.assert_array_almost_equal( call_args[0][0][sut._value_network_output_variables], [[2.9975], [3.05]]) np.testing.assert_array_equal( call_args[0][0][sut._policy_network_output_variables], [np.array([1, 0], np.float32), np.array([0, 1], np.float32)]) np.testing.assert_array_almost_equal( call_args[0][0][sut._policy_network_weight_variables], [[0.9975], [2.05]]) # Verify data buffer size. self.assertEqual(len(sut._input_buffer), 0)
def test_update_policy_and_value_function(self): action_space = spaces.Discrete(2) observation_space = spaces.Box(0, 1, (1,)) sut = ActorCritic('', observation_space, action_space) # Set up. self._setup_trajectory(sut) sut._process_accumulated_trajectory(True) sut._trainer = MagicMock() sut._adjust_learning_rate = MagicMock() # Call test method. sut._update_networks() # Verify value network behavior. self.assertEqual( sut._trainer.train_minibatch.call_count, 1) call_args = sut._trainer.train_minibatch.call_args np.testing.assert_array_equal( call_args[0][0][sut._input_variables], [np.array([0.1], np.float32), np.array([0.2], np.float32)]) np.testing.assert_array_almost_equal( call_args[0][0][sut._value_network_output_variables], [[2.9975], [3.05]]) np.testing.assert_array_equal( call_args[0][0][sut._policy_network_output_variables], [np.array([1, 0], np.float32), np.array([0, 1], np.float32)]) np.testing.assert_array_almost_equal( call_args[0][0][sut._policy_network_weight_variables], [[0.9975], [2.05]]) # Verify data buffer size. self.assertEqual(len(sut._input_buffer), 0)
def test_process_accumulated_trajectory_keep_last(self): action_space = spaces.Discrete(2) observation_space = spaces.Box(0, 1, (1, )) sut = ActorCritic('', observation_space, action_space) # Set up. self._setup_trajectory(sut) # Call test method. sut._process_accumulated_trajectory(True) # Verify results. self.assertEqual(len(sut._trajectory_rewards), 0) self.assertEqual(len(sut._trajectory_actions), 0) self.assertEqual(sut._trajectory_states, [np.array([0.3], np.float32)])
def test_process_accumulated_trajectory_keep_last(self): action_space = spaces.Discrete(2) observation_space = spaces.Box(0, 1, (1,)) sut = ActorCritic('', observation_space, action_space) # Set up. self._setup_trajectory(sut) # Call test method. sut._process_accumulated_trajectory(True) # Verify results. self.assertEqual(len(sut._trajectory_rewards), 0) self.assertEqual(len(sut._trajectory_actions), 0) self.assertEqual(sut._trajectory_states, [np.array([0.3], np.float32)])
def test_process_accumulated_trajectory(self): action_space = spaces.Discrete(2) observation_space = spaces.Box(0, 1, (1,)) sut = ActorCritic('', observation_space, action_space) # Set up. self._setup_trajectory(sut) # Call test method. sut._process_accumulated_trajectory(False) # Verify results. self.assertEqual(len(sut._trajectory_rewards), 0) self.assertEqual(len(sut._trajectory_actions), 0) self.assertEqual(len(sut._trajectory_states), 0) np.testing.assert_array_equal( sut._input_buffer, [np.array([0.1], np.float32), np.array([0.2], np.float32)]) # For unknown reason, got [2.9974999999999996] instead of [2.9975] for # the following testcase, therefore use assert_array_almost_equal. np.testing.assert_array_almost_equal( sut._value_network_output_buffer, [ [2.9975], # 3.05 * 0.95 + 0.1 [3.05] # 3 (initial_r) * 0.95 + 0.2 ]) np.testing.assert_array_equal( sut._policy_network_output_buffer, [ np.array([1, 0], np.float32), np.array([0, 1], np.float32) ] ) np.testing.assert_array_almost_equal( sut._policy_network_weight_buffer, [ [0.9975], # 2.9975 - 2 [2.05] # 3.05 - 1 ])
def test_process_accumulated_trajectory(self): action_space = spaces.Discrete(2) observation_space = spaces.Box(0, 1, (1,)) sut = ActorCritic('', observation_space, action_space) # Set up. self._setup_trajectory(sut) # Call test method. sut._process_accumulated_trajectory(False) # Verify results. self.assertEqual(len(sut._trajectory_rewards), 0) self.assertEqual(len(sut._trajectory_actions), 0) self.assertEqual(len(sut._trajectory_states), 0) np.testing.assert_array_equal( sut._input_buffer, [np.array([0.1], np.float32), np.array([0.2], np.float32)]) # For unknown reason, got [2.9974999999999996] instead of [2.9975] for # the following testcase, therefore use assert_array_almost_equal. np.testing.assert_array_almost_equal( sut._value_network_output_buffer, [ [2.9975], # 3.05 * 0.95 + 0.1 [3.05] # 3 (initial_r) * 0.95 + 0.2 ]) np.testing.assert_array_equal( sut._policy_network_output_buffer, [ np.array([1, 0], np.float32), np.array([0, 1], np.float32) ] ) np.testing.assert_array_almost_equal( sut._policy_network_weight_buffer, [ [0.9975], # 2.9975 - 2 [2.05] # 3.05 - 1 ])