def test_action_processing_in_single_server_queue():
"""
Tests the interpretation and application of actions in the single server queue with
deterministic dynamics. This is validated in terms of the environment state and rewards.
"""
# Set up single server queue with no arrivals and deterministic dynamics.
cost_per_buffer = np.ones((1, 1))
initial_state = (10,)
capacity = np.ones((1, 1)) * np.inf
demand_rate_val = 0.0
job_conservation_flag = True
demand_rate = np.array([demand_rate_val])[:, None]
buffer_processing_matrix = - np.ones((1, 1))
constituency_matrix = np.ones((1, 1))
list_boundary_constraint_matrices = [constituency_matrix]
# Construct environment.
job_generator = DeterministicDiscreteReviewJobGenerator(
sim_time_interval=1,
demand_rate=demand_rate,
buffer_processing_matrix=buffer_processing_matrix
)
state_initialiser = stinit.DeterministicCRWStateInitialiser(initial_state)
env = RLControlledRandomWalk(cost_per_buffer, capacity, constituency_matrix, job_generator,
state_initialiser, job_conservation_flag,
list_boundary_constraint_matrices)
new_state_idle, reward_idle, _, _ = env.step(np.array([1, 0]))
assert new_state_idle[0] == initial_state[0]
assert reward_idle == initial_state[0] * -1.
new_state_active, reward_active, _, _ = env.step(np.array([0, 1]))
assert new_state_active[0] == initial_state[0] - 1
assert reward_active == (initial_state[0] - 1) * -1.
def test_stepping_environment_with_action_tuple():
"""
Tests the RL environment wrapper for the ksrs_network_model example as per the examples file
from which the initial set up code is copied.
see snc/stochastic_network_control/environments/examples.py for the original code.
"""
# Set up the environment parameters.
alpha1, alpha3 = 2, 2
mu1, mu2, mu3, mu4 = 10, 3, 10, 3
cost_per_buffer = np.ones((4, 1))
initial_state = (0, 0, 0, 0)
capacity = np.ones((4, 1)) * np.inf
job_conservation_flag = True
seed = 72
demand_rate = np.array([alpha1, 0, alpha3, 0])[:, None]
buffer_processing_matrix = np.array([[-mu1, 0, 0, 0],
[mu1, -mu2, 0, 0],
[0, 0, -mu3, 0],
[0, 0, mu3, -mu4]])
constituency_matrix = np.array([[1, 0, 0, 1],
[0, 1, 1, 0]])
# Construct environment.
job_generator = ScaledBernoulliServicesPoissonArrivalsGenerator(
demand_rate, buffer_processing_matrix, job_gen_seed=seed)
state_initialiser = stinit.DeterministicCRWStateInitialiser(initial_state)
env = RLControlledRandomWalk(cost_per_buffer, capacity, constituency_matrix, job_generator,
state_initialiser, job_conservation_flag)
# Set up a numpy action tuple with an action for each resource and ensure that the environment
# is able to interpret it and update itself.
action_tuple = (np.array([0, 1, 0]), np.array([0, 0, 1]))
env.step(action_tuple)
def test_action_processing_with_multiple_resource_sets():
"""
Tests the interpretation and application of actions in a more complex environment with
deterministic dynamics. This is validated in terms of the environment state and rewards.
"""
# Set up the environment parameters with no arrivals and deterministic dynamics.
alpha1, alpha3 = 0, 0
mu1, mu2, mu3, mu4 = 10, 3, 10, 3
cost_per_buffer = np.ones((4, 1))
initial_state = (100, 100, 100, 100)
capacity = np.ones((4, 1)) * np.inf
job_conservation_flag = True
demand_rate = np.array([alpha1, 0, alpha3, 0])[:, None]
buffer_processing_matrix = np.array([[-mu1, 0, 0, 0],
[mu1, -mu2, 0, 0],
[0, 0, -mu3, 0],
[0, 0, mu3, -mu4]])
constituency_matrix = np.array([[1, 0, 0, 1],
[0, 1, 1, 0]])
# Construct deterministic job generator.
job_generator = DeterministicDiscreteReviewJobGenerator(
sim_time_interval=1,
demand_rate=demand_rate,
buffer_processing_matrix=buffer_processing_matrix
)
state_initialiser = stinit.DeterministicCRWStateInitialiser(initial_state)
env = RLControlledRandomWalk(cost_per_buffer, capacity, constituency_matrix, job_generator,
state_initialiser, job_conservation_flag)
# Ensure that the idling actions do not change the environment state and are rewarded
# accordingly.
idle_action = (np.array([1, 0, 0]), np.array([1, 0, 0]))
new_state_idle, reward_idle, _, _ = env.step(idle_action)
assert all((np.array(initial_state) / sum(initial_state)) == new_state_idle)
assert reward_idle == -1 * np.dot(initial_state, cost_per_buffer)
# Ensure that the action which activates activities 2 and 4 is interpreted and rewarded
# correctly.
active_action = (np.array([0, 0, 1]), np.array([0, 1, 0]))
new_state_active, reward_active, _, _ = env.step(active_action)
assert np.allclose(new_state_active, np.array([100, 97, 100, 97]) / 394)
assert reward_active == -1 * np.dot(env.state.flatten(), cost_per_buffer)
def test_state_scaling_only_for_rl():
"""
Tests that the state of the underlying ControlledRandomWalk object is not mutated as a side
effect of the RLControlledRandomWalk wrapper.
"""
# Set up the environment parameters with no arrivals and deterministic dynamics.
alpha1, alpha3 = 0, 0
mu1, mu2, mu3, mu4 = 10, 3, 10, 3
cost_per_buffer = np.ones((4, 1))
initial_state = (100, 100, 100, 100)
capacity = np.ones((4, 1)) * np.inf
job_conservation_flag = True
demand_rate = np.array([alpha1, 0, alpha3, 0])[:, None]
buffer_processing_matrix = np.array([[-mu1, 0, 0, 0],
[mu1, -mu2, 0, 0],
[0, 0, -mu3, 0],
[0, 0, mu3, -mu4]])
constituency_matrix = np.array([[1, 0, 0, 1],
[0, 1, 1, 0]])
# Construct deterministic job generator.
job_generator = DeterministicDiscreteReviewJobGenerator(
sim_time_interval=1,
demand_rate=demand_rate,
buffer_processing_matrix=buffer_processing_matrix
)
state_initialiser = stinit.DeterministicCRWStateInitialiser(initial_state)
env = RLControlledRandomWalk(cost_per_buffer, capacity, constituency_matrix, job_generator,
state_initialiser, job_conservation_flag)
# Ensure that scaling is applied to the RL state but not the underlying stated.
assert np.all(env.state == np.array([[100], [100], [100], [100]]))
active_action = (np.array([0, 0, 1]), np.array([0, 1, 0]))
new_state_active, _, _, _ = env.step(active_action)
assert np.all(env.state == np.array([[100], [97], [100], [97]]))
assert np.allclose(new_state_active, np.array([100, 97, 100, 97]) / 394)