def test_update_policy_and_value_function(self):
action_space = spaces.Discrete(2)
observation_space = spaces.Box(0, 1, (1, ))
sut = ActorCritic('', observation_space, action_space)
# Set up.
self._setup_trajectory(sut)
sut._process_accumulated_trajectory(True)
sut._trainer = MagicMock()
sut._adjust_learning_rate = MagicMock()
# Call test method.
sut._update_networks()
# Verify value network behavior.
self.assertEqual(sut._trainer.train_minibatch.call_count, 1)
call_args = sut._trainer.train_minibatch.call_args
np.testing.assert_array_equal(
call_args[0][0][sut._input_variables],
[np.array([0.1], np.float32),
np.array([0.2], np.float32)])
np.testing.assert_array_almost_equal(
call_args[0][0][sut._value_network_output_variables],
[[2.9975], [3.05]])
np.testing.assert_array_equal(
call_args[0][0][sut._policy_network_output_variables],
[np.array([1, 0], np.float32),
np.array([0, 1], np.float32)])
np.testing.assert_array_almost_equal(
call_args[0][0][sut._policy_network_weight_variables],
[[0.9975], [2.05]])
# Verify data buffer size.
self.assertEqual(len(sut._input_buffer), 0)
def test_update_policy_and_value_function(self):
action_space = spaces.Discrete(2)
observation_space = spaces.Box(0, 1, (1,))
sut = ActorCritic('', observation_space, action_space)
# Set up.
self._setup_trajectory(sut)
sut._process_accumulated_trajectory(True)
sut._trainer = MagicMock()
sut._adjust_learning_rate = MagicMock()
# Call test method.
sut._update_networks()
# Verify value network behavior.
self.assertEqual(
sut._trainer.train_minibatch.call_count, 1)
call_args = sut._trainer.train_minibatch.call_args
np.testing.assert_array_equal(
call_args[0][0][sut._input_variables],
[np.array([0.1], np.float32), np.array([0.2], np.float32)])
np.testing.assert_array_almost_equal(
call_args[0][0][sut._value_network_output_variables],
[[2.9975], [3.05]])
np.testing.assert_array_equal(
call_args[0][0][sut._policy_network_output_variables],
[np.array([1, 0], np.float32), np.array([0, 1], np.float32)])
np.testing.assert_array_almost_equal(
call_args[0][0][sut._policy_network_weight_variables],
[[0.9975], [2.05]])
# Verify data buffer size.
self.assertEqual(len(sut._input_buffer), 0)
def test_process_accumulated_trajectory_keep_last(self):
action_space = spaces.Discrete(2)
observation_space = spaces.Box(0, 1, (1, ))
sut = ActorCritic('', observation_space, action_space)
# Set up.
self._setup_trajectory(sut)
# Call test method.
sut._process_accumulated_trajectory(True)
# Verify results.
self.assertEqual(len(sut._trajectory_rewards), 0)
self.assertEqual(len(sut._trajectory_actions), 0)
self.assertEqual(sut._trajectory_states, [np.array([0.3], np.float32)])
def test_process_accumulated_trajectory_keep_last(self):
action_space = spaces.Discrete(2)
observation_space = spaces.Box(0, 1, (1,))
sut = ActorCritic('', observation_space, action_space)
# Set up.
self._setup_trajectory(sut)
# Call test method.
sut._process_accumulated_trajectory(True)
# Verify results.
self.assertEqual(len(sut._trajectory_rewards), 0)
self.assertEqual(len(sut._trajectory_actions), 0)
self.assertEqual(sut._trajectory_states, [np.array([0.3], np.float32)])
def test_process_accumulated_trajectory(self):
action_space = spaces.Discrete(2)
observation_space = spaces.Box(0, 1, (1,))
sut = ActorCritic('', observation_space, action_space)
# Set up.
self._setup_trajectory(sut)
# Call test method.
sut._process_accumulated_trajectory(False)
# Verify results.
self.assertEqual(len(sut._trajectory_rewards), 0)
self.assertEqual(len(sut._trajectory_actions), 0)
self.assertEqual(len(sut._trajectory_states), 0)
np.testing.assert_array_equal(
sut._input_buffer,
[np.array([0.1], np.float32), np.array([0.2], np.float32)])
# For unknown reason, got [2.9974999999999996] instead of [2.9975] for
# the following testcase, therefore use assert_array_almost_equal.
np.testing.assert_array_almost_equal(
sut._value_network_output_buffer,
[
[2.9975], # 3.05 * 0.95 + 0.1
[3.05] # 3 (initial_r) * 0.95 + 0.2
])
np.testing.assert_array_equal(
sut._policy_network_output_buffer,
[
np.array([1, 0], np.float32),
np.array([0, 1], np.float32)
]
)
np.testing.assert_array_almost_equal(
sut._policy_network_weight_buffer,
[
[0.9975], # 2.9975 - 2
[2.05] # 3.05 - 1
])
def test_process_accumulated_trajectory(self):
action_space = spaces.Discrete(2)
observation_space = spaces.Box(0, 1, (1,))
sut = ActorCritic('', observation_space, action_space)
# Set up.
self._setup_trajectory(sut)
# Call test method.
sut._process_accumulated_trajectory(False)
# Verify results.
self.assertEqual(len(sut._trajectory_rewards), 0)
self.assertEqual(len(sut._trajectory_actions), 0)
self.assertEqual(len(sut._trajectory_states), 0)
np.testing.assert_array_equal(
sut._input_buffer,
[np.array([0.1], np.float32), np.array([0.2], np.float32)])
# For unknown reason, got [2.9974999999999996] instead of [2.9975] for
# the following testcase, therefore use assert_array_almost_equal.
np.testing.assert_array_almost_equal(
sut._value_network_output_buffer,
[
[2.9975], # 3.05 * 0.95 + 0.1
[3.05] # 3 (initial_r) * 0.95 + 0.2
])
np.testing.assert_array_equal(
sut._policy_network_output_buffer,
[
np.array([1, 0], np.float32),
np.array([0, 1], np.float32)
]
)
np.testing.assert_array_almost_equal(
sut._policy_network_weight_buffer,
[
[0.9975], # 2.9975 - 2
[2.05] # 3.05 - 1
])