def test_add_layer_to_simple_nn(self):
# Space must contain batch dimension (otherwise, NNlayer will complain).
space = FloatBox(shape=(3, ), add_batch_rank=True)
# Create a simple neural net from json.
neural_net = NeuralNetwork.from_spec(
config_from_path(
"configs/test_simple_nn.json")) # type: NeuralNetwork
# Add another layer to it.
neural_net.add_layer(DenseLayer(units=10, scope="last-layer"))
# Do not seed, we calculate expectations manually.
test = ComponentTest(component=neural_net,
input_spaces=dict(nn_input=space))
# Batch of size=3.
input_ = space.sample(3)
# Calculate output manually.
var_dict = test.read_variable_values(neural_net.variable_registry)
expected = dense_layer(
dense_layer(input_,
var_dict["test-network/hidden-layer/dense/kernel"],
var_dict["test-network/hidden-layer/dense/bias"]),
var_dict["test-network/last-layer/dense/kernel"],
var_dict["test-network/last-layer/dense/bias"])
test.test(("apply", input_),
expected_outputs=dict(output=expected),
decimals=5)
test.terminate()
def test_keras_style_simple_nn(self):
# Input Space of the network.
input_space = FloatBox(shape=(3,), add_batch_rank=True)
# Create a DenseLayer with a fixed `call` method input space for the arg `inputs`.
output1 = DenseLayer(units=5, activation="linear", scope="a")(input_space)
# Create a DenseLayer whose `inputs` arg is the resulting DataOpRec of output1's `call` output.
output2 = DenseLayer(units=7, activation="relu", scope="b")(output1)
# This will trace back automatically through the given output DataOpRec(s) and add all components
# on the way to the input-space to this network.
neural_net = NeuralNetwork(outputs=output2)
test = ComponentTest(component=neural_net, input_spaces=dict(inputs=input_space))
# Batch of size=n.
input_ = input_space.sample(5)
# Calculate output manually.
var_dict = neural_net.get_variables("a/dense/kernel", "a/dense/bias", "b/dense/kernel", "b/dense/bias", global_scope=False)
w1_value = test.read_variable_values(var_dict["a/dense/kernel"])
b1_value = test.read_variable_values(var_dict["a/dense/bias"])
w2_value = test.read_variable_values(var_dict["b/dense/kernel"])
b2_value = test.read_variable_values(var_dict["b/dense/bias"])
expected = relu(dense_layer(dense_layer(input_, w1_value, b1_value), w2_value, b2_value))
test.test(("call", input_), expected_outputs=expected, decimals=5)
test.terminate()
def test_keras_style_two_separate_input_spaces(self):
# Define two input Spaces first. Independently (no container).
input_space_1 = IntBox(3, add_batch_rank=True)
input_space_2 = FloatBox(shape=(4,), add_batch_rank=True)
# One-hot flatten the int tensor.
flatten_layer_out = ReShape(flatten=True, flatten_categories=True)(input_space_1)
# Run the float tensor through two dense layers.
dense_1_out = DenseLayer(units=3, scope="d1")(input_space_2)
dense_2_out = DenseLayer(units=5, scope="d2")(dense_1_out)
# Concat everything.
cat_out = ConcatLayer()(flatten_layer_out, dense_2_out)
# Use the `outputs` arg to allow your network to trace back the data flow until the input space.
neural_net = NeuralNetwork(inputs=[input_space_1, input_space_2], outputs=cat_out)
test = ComponentTest(component=neural_net, input_spaces=dict(inputs=[input_space_1, input_space_2]))
var_dict = neural_net.variable_registry
w1_value = test.read_variable_values(var_dict["neural-network/d1/dense/kernel"])
b1_value = test.read_variable_values(var_dict["neural-network/d1/dense/bias"])
w2_value = test.read_variable_values(var_dict["neural-network/d2/dense/kernel"])
b2_value = test.read_variable_values(var_dict["neural-network/d2/dense/bias"])
# Batch of size=n.
input_ = [input_space_1.sample(4), input_space_2.sample(4)]
expected = np.concatenate([ # concat everything
one_hot(input_[0]), # int flattening
dense_layer(dense_layer(input_[1], w1_value, b1_value), w2_value, b2_value) # float -> 2 x dense
], axis=-1)
out = test.test(("call", input_), expected_outputs=expected)
test.terminate()
def test_keras_style_one_container_input_space(self):
# Define one container input Space.
input_space = Tuple(IntBox(3), FloatBox(shape=(4,)), add_batch_rank=True)
# One-hot flatten the int tensor.
flatten_layer_out = ReShape(flatten=True, flatten_categories=True)(input_space[0])
# Run the float tensor through two dense layers.
dense_1_out = DenseLayer(units=3, scope="d1")(input_space[1])
dense_2_out = DenseLayer(units=5, scope="d2")(dense_1_out)
# Concat everything.
cat_out = ConcatLayer()(flatten_layer_out, dense_2_out)
# Use the `outputs` arg to allow your network to trace back the data flow until the input space.
# `inputs` is not needed here as we only have one single input (the Tuple).
neural_net = NeuralNetwork(outputs=cat_out)
test = ComponentTest(component=neural_net, input_spaces=dict(inputs=input_space))
var_dict = neural_net.variable_registry
w1_value = test.read_variable_values(var_dict["neural-network/d1/dense/kernel"])
b1_value = test.read_variable_values(var_dict["neural-network/d1/dense/bias"])
w2_value = test.read_variable_values(var_dict["neural-network/d2/dense/kernel"])
b2_value = test.read_variable_values(var_dict["neural-network/d2/dense/bias"])
# Batch of size=n.
input_ = input_space.sample(4)
expected = np.concatenate([ # concat everything
one_hot(input_[0]), # int flattening
dense_layer(dense_layer(input_[1], w1_value, b1_value), w2_value, b2_value) # float -> 2 x dense
], axis=-1)
out = test.test(("call", tuple([input_])), expected_outputs=expected)
test.terminate()
def test_environment_stepper_on_deterministic_env_with_action_probs_lstm(self):
internal_states_space = Tuple(FloatBox(shape=(3,)), FloatBox(shape=(3,)))
preprocessor_spec = [dict(type="multiply", factor=0.1)]
network_spec = config_from_path("configs/test_lstm_nn.json")
exploration_spec = None
actor_component = ActorComponent(
preprocessor_spec,
dict(network_spec=network_spec, action_space=self.deterministic_env_action_space),
exploration_spec
)
environment_stepper = EnvironmentStepper(
environment_spec=dict(type="deterministic_env", steps_to_terminal=3),
actor_component_spec=actor_component,
state_space=self.deterministic_env_state_space,
reward_space="float32",
internal_states_space=internal_states_space,
add_action_probs=True,
action_probs_space=self.deterministic_action_probs_space,
num_steps=4,
)
test = ComponentTest(
component=environment_stepper,
action_space=self.deterministic_env_action_space,
)
weights = test.read_variable_values(environment_stepper.actor_component.policy.variable_registry)
policy_scope = "environment-stepper/actor-component/policy/"
weights_lstm = weights[policy_scope+"test-lstm-network/lstm-layer/lstm-cell/kernel"]
biases_lstm = weights[policy_scope+"test-lstm-network/lstm-layer/lstm-cell/bias"]
weights_action = weights[policy_scope+"action-adapter-0/action-network/action-layer/dense/kernel"]
biases_action = weights[policy_scope+"action-adapter-0/action-network/action-layer/dense/bias"]
# Step 3 times through the Env and collect results.
lstm_1 = lstm_layer(np.array([[[0.0]]]), weights_lstm, biases_lstm)
lstm_2 = lstm_layer(np.array([[[0.1]]]), weights_lstm, biases_lstm, lstm_1[1])
lstm_3 = lstm_layer(np.array([[[0.2]]]), weights_lstm, biases_lstm, lstm_2[1])
lstm_4 = lstm_layer(np.array([[[0.0]]]), weights_lstm, biases_lstm, lstm_3[1])
expected = (
np.array([False, False, True, False]),
np.array([[0.0], [1.0], [2.0], [0.0], [1.0]]), # s' (raw)
np.array([
softmax(dense_layer(np.squeeze(lstm_1[0]), weights_action, biases_action)),
softmax(dense_layer(np.squeeze(lstm_2[0]), weights_action, biases_action)),
softmax(dense_layer(np.squeeze(lstm_3[0]), weights_action, biases_action)),
softmax(dense_layer(np.squeeze(lstm_4[0]), weights_action, biases_action)),
]), # action probs
# internal states
(
np.squeeze(np.array([[[0.0, 0.0, 0.0]], lstm_1[1][0], lstm_2[1][0], lstm_3[1][0], lstm_4[1][0]])),
np.squeeze(np.array([[[0.0, 0.0, 0.0]], lstm_1[1][1], lstm_2[1][1], lstm_3[1][1], lstm_4[1][1]]))
)
)
test.test("step", expected_outputs=expected)
# Make sure we close the session (to shut down the Env on the server).
test.terminate()
def test_simple_nn(self):
# Space must contain batch dimension (otherwise, NNlayer will complain).
space = FloatBox(shape=(3, ), add_batch_rank=True)
# Create a simple neural net from json.
neural_net = NeuralNetwork.from_spec(
config_from_path(
"configs/test_simple_nn.json")) # type: NeuralNetwork
# Do not seed, we calculate expectations manually.
test = ComponentTest(component=neural_net,
input_spaces=dict(nn_input=space))
# Batch of size=3.
input_ = np.array([[0.1, 0.2, 0.3], [1.0, 2.0, 3.0],
[10.0, 20.0, 30.0]])
# Calculate output manually.
var_dict = neural_net.get_variables("hidden-layer/dense/kernel",
"hidden-layer/dense/bias",
global_scope=False)
w1_value = test.read_variable_values(
var_dict["hidden-layer/dense/kernel"])
b1_value = test.read_variable_values(
var_dict["hidden-layer/dense/bias"])
expected = dense_layer(input_, w1_value, b1_value)
test.test(("apply", input_),
expected_outputs=dict(output=expected),
decimals=5)
test.terminate()
def test_simple_nn_using_layers(self):
# Space must contain batch dimension (otherwise, NNlayer will complain).
space = FloatBox(shape=(4, ), add_batch_rank=True)
# Create a simple neural net from json.
nn_layers = config_from_path("configs/test_simple_nn.json")
neural_net = NeuralNetwork(*nn_layers["layers"])
# Do not seed, we calculate expectations manually.
test = ComponentTest(component=neural_net,
input_spaces=dict(inputs=space))
# Batch of size=3.
input_ = space.sample(4)
# Calculate output manually.
var_dict = neural_net.get_variables("hidden-layer/dense/kernel",
"hidden-layer/dense/bias",
global_scope=False)
w1_value = test.read_variable_values(
var_dict["hidden-layer/dense/kernel"])
b1_value = test.read_variable_values(
var_dict["hidden-layer/dense/bias"])
expected = dense_layer(input_, w1_value, b1_value)
test.test(("call", input_), expected_outputs=expected, decimals=5)
test.terminate()
def test_multi_input_stream_neural_network_with_dict(self):
# Space must contain batch dimension (otherwise, NNlayer will complain).
input_space = Dict(
a=FloatBox(shape=(3,)),
b=IntBox(4, shape=()),
add_batch_rank=True
)
multi_input_nn = MultiInputStreamNeuralNetwork(
input_network_specs=dict(
a=[],
b=[{"type": "reshape", "flatten": True, "flatten_categories": True}]
),
post_network_spec=[{"type": "dense", "units": 2}],
)
test = ComponentTest(component=multi_input_nn, input_spaces=dict(inputs=input_space))
# Batch of size=n.
nn_inputs = input_space.sample(5)
global_scope = "multi-input-stream-nn/post-concat-nn/dense-layer/dense/"
# Calculate output manually.
var_dict = test.read_variable_values()
b_flat = one_hot(nn_inputs["b"], depth=4)
concat_out = np.concatenate((nn_inputs["a"], b_flat), axis=-1)
expected = dense_layer(concat_out, var_dict[global_scope+"kernel"], var_dict[global_scope+"bias"])
test.test(("call", nn_inputs), expected_outputs=expected)
test.terminate()
def test_multi_input_stream_neural_network_with_tuple(self):
# Space must contain batch dimension (otherwise, NNLayer will complain).
input_space = Tuple(
IntBox(3, shape=()),
FloatBox(shape=(8,)),
IntBox(4, shape=()),
add_batch_rank=True
)
multi_input_nn = MultiInputStreamNeuralNetwork(
input_network_specs=(
[{"type": "reshape", "flatten": True, "flatten_categories": True}], # intbox -> flatten
[{"type": "dense", "units": 2}], # floatbox -> dense
[{"type": "reshape", "flatten": True, "flatten_categories": True}] # inbox -> flatten
),
post_network_spec=[{"type": "dense", "units": 3}],
)
test = ComponentTest(component=multi_input_nn, input_spaces=dict(inputs=input_space))
# Batch of size=n.
nn_inputs = input_space.sample(3)
global_scope_pre = "multi-input-stream-nn/input-stream-nn-"
global_scope_post = "multi-input-stream-nn/post-concat-nn/dense-layer/dense/"
# Calculate output manually.
var_dict = test.read_variable_values()
flat_0 = one_hot(nn_inputs[0], depth=3)
dense_1 = dense_layer(
nn_inputs[1], var_dict[global_scope_pre+"1/dense-layer/dense/kernel"],
var_dict[global_scope_pre+"1/dense-layer/dense/bias"]
)
flat_2 = one_hot(nn_inputs[2], depth=4)
concat_out = np.concatenate((flat_0, dense_1, flat_2), axis=-1)
expected = dense_layer(concat_out, var_dict[global_scope_post+"kernel"], var_dict[global_scope_post+"bias"])
test.test(("call", tuple([nn_inputs])), expected_outputs=expected)
test.terminate()
def test_simple_variational_auto_encoder(self):
# Space must contain batch dimension (otherwise, NNlayer will complain).
input_spaces = dict(
input_=FloatBox(shape=(3,), add_batch_rank=True), z_vector=FloatBox(shape=(1,), add_batch_rank=True)
)
variational_auto_encoder = VariationalAutoEncoder(
z_units=1,
encoder_network_spec=config_from_path("configs/test_vae_encoder_network.json"),
decoder_network_spec=config_from_path("configs/test_vae_decoder_network.json")
)
# Do not seed, we calculate expectations manually.
test = ComponentTest(component=variational_auto_encoder, input_spaces=input_spaces)
# Batch of size=3.
input_ = np.array([[0.1, 0.2, 0.3], [1.0, 2.0, 3.0], [10.0, 20.0, 30.0]])
global_scope = "variational-auto-encoder/"
# Calculate output manually.
var_dict = test.read_variable_values(variational_auto_encoder.variable_registry)
encoder_network_out = dense_layer(
input_, var_dict[global_scope+"encoder-network/encoder-layer/dense/kernel"],
var_dict[global_scope+"encoder-network/encoder-layer/dense/bias"]
)
expected_mean = dense_layer(
encoder_network_out, var_dict[global_scope+"mean-layer/dense/kernel"],
var_dict[global_scope+"mean-layer/dense/bias"]
)
expected_stddev = dense_layer(
encoder_network_out, var_dict[global_scope + "stddev-layer/dense/kernel"],
var_dict[global_scope + "stddev-layer/dense/bias"]
)
out = test.test(("encode", input_), expected_outputs=None)
recursive_assert_almost_equal(out["mean"], expected_mean, decimals=5)
recursive_assert_almost_equal(out["stddev"], np.exp(expected_stddev), decimals=5)
self.assertTrue(out["z_sample"].shape == (3, 1))
test.terminate()
def test_batch_apply_component_with_simple_input_space(self):
input_space = FloatBox(shape=(3,), add_batch_rank=True, add_time_rank=True)
sub_component = DenseLayer(units=4, biases_spec=False)
batch_apply = BatchApply(sub_component=sub_component, api_method_name="call")
test = ComponentTest(component=batch_apply, input_spaces=dict(input_=input_space))
weights = test.read_variable_values(batch_apply.variable_registry["batch-apply/dense-layer/dense/kernel"])
sample = input_space.sample(size=(5, 10))
sample_folded = np.reshape(sample, newshape=(50, 3))
expected = dense_layer(sample_folded, weights)
expected = np.reshape(expected, newshape=(5, 10, 4))
test.test(("call", sample), expected_outputs=expected)
def test_lstm_nn(self):
# Space must contain batch dimension (otherwise, NNlayer will complain).
#units = 3
batch_size = 2
time_steps = 4
input_nodes = 2
input_space = FloatBox(shape=(input_nodes, ),
add_batch_rank=True,
add_time_rank=True)
#internal_states_space = Tuple(FloatBox(shape=(units,)), FloatBox(shape=(units,)), add_batch_rank=True)
neural_net = NeuralNetwork.from_spec(
config_from_path("configs/test_dense_to_lstm_nn.json"))
# Do not seed, we calculate expectations manually.
test = ComponentTest(component=neural_net,
input_spaces=dict(inputs=input_space))
# Batch of size=2, time-steps=3.
input_ = input_space.sample((batch_size, time_steps))
# Calculate output manually.
w0_value = test.read_variable_values(
neural_net.
variable_registry["test-lstm-network/dense-layer/dense/kernel"])
b0_value = test.read_variable_values(
neural_net.
variable_registry["test-lstm-network/dense-layer/dense/bias"])
lstm_w_value = test.read_variable_values(
neural_net.
variable_registry["test-lstm-network/lstm-layer/lstm-cell/kernel"])
lstm_b_value = test.read_variable_values(
neural_net.
variable_registry["test-lstm-network/lstm-layer/lstm-cell/bias"])
d0_out = dense_layer(input_, w0_value, b0_value)
lstm_out, last_internal_states = lstm_layer(d0_out,
lstm_w_value,
lstm_b_value,
time_major=False)
expected = [lstm_out, last_internal_states]
test.test(("call", input_),
expected_outputs=tuple(expected),
decimals=5)
test.terminate()
def test_lstm_nn_with_custom_apply(self):
# Space must contain batch dimension (otherwise, NNlayer will complain).
units = 3
batch_size = 2
time_steps = 4
input_nodes = 2
input_space = FloatBox(shape=(input_nodes, ),
add_batch_rank=True,
add_time_rank=True)
internal_states_space = Tuple(FloatBox(shape=(units, )),
FloatBox(shape=(units, )),
add_batch_rank=True)
def custom_apply(self, input_, internal_states=None):
d0_out = self.get_sub_component_by_name("d0").apply(input_)
lstm_out = self.get_sub_component_by_name("lstm").apply(
d0_out, internal_states)
d1_out = self.get_sub_component_by_name("d1").apply(
lstm_out["output"])
return dict(output=d1_out,
last_internal_states=lstm_out["last_internal_states"])
# Create a simple neural net with the above custom API-method.
neural_net = NeuralNetwork(DenseLayer(units, scope="d0"),
LSTMLayer(units, scope="lstm"),
DenseLayer(units, scope="d1"),
api_methods={("apply", custom_apply)})
# Do not seed, we calculate expectations manually.
test = ComponentTest(component=neural_net,
input_spaces=dict(
input_=input_space,
internal_states=internal_states_space))
# Batch of size=2, time-steps=3.
input_ = input_space.sample((batch_size, time_steps))
internal_states = internal_states_space.sample(batch_size)
# Calculate output manually.
w0_value = test.read_variable_values(
neural_net.variable_registry["neural-network/d0/dense/kernel"])
b0_value = test.read_variable_values(
neural_net.variable_registry["neural-network/d0/dense/bias"])
w1_value = test.read_variable_values(
neural_net.variable_registry["neural-network/d1/dense/kernel"])
b1_value = test.read_variable_values(
neural_net.variable_registry["neural-network/d1/dense/bias"])
lstm_w_value = test.read_variable_values(
neural_net.
variable_registry["neural-network/lstm/lstm-cell/kernel"])
lstm_b_value = test.read_variable_values(
neural_net.variable_registry["neural-network/lstm/lstm-cell/bias"])
d0_out = dense_layer(input_, w0_value, b0_value)
lstm_out, last_internal_states = lstm_layer(
d0_out,
lstm_w_value,
lstm_b_value,
initial_internal_states=internal_states,
time_major=False)
d1_out = dense_layer(lstm_out, w1_value, b1_value)
expected = dict(output=d1_out,
last_internal_states=last_internal_states)
test.test(("apply", [input_, internal_states]),
expected_outputs=expected,
decimals=5)
test.terminate()
def test_environment_stepper_on_deterministic_env_with_returning_action_probs(
self):
preprocessor_spec = [dict(type="divide", divisor=2)]
network_spec = config_from_path("configs/test_simple_nn.json")
exploration_spec = None
actor_component = ActorComponent(
preprocessor_spec,
dict(network_spec=network_spec,
action_space=self.deterministic_env_action_space),
exploration_spec)
environment_stepper = EnvironmentStepper(
environment_spec=dict(type="deterministic_env",
steps_to_terminal=6),
actor_component_spec=actor_component,
state_space=self.deterministic_env_state_space,
reward_space="float32",
add_action_probs=True,
action_probs_space=self.deterministic_action_probs_space,
num_steps=3)
test = ComponentTest(
component=environment_stepper,
action_space=self.deterministic_env_action_space,
)
weights = test.read_variable_values(
environment_stepper.actor_component.policy.variables)
weights_hid = weights[
"environment-stepper/actor-component/policy/test-network/hidden-layer/dense/kernel"]
biases_hid = weights[
"environment-stepper/actor-component/policy/test-network/hidden-layer/dense/bias"]
weights_action = weights[
"environment-stepper/actor-component/policy/action-adapter/action-layer/dense/kernel"]
biases_action = weights[
"environment-stepper/actor-component/policy/action-adapter/action-layer/dense/bias"]
# Reset the stepper.
test.test("reset")
# Step 3 times through the Env and collect results.
expected = (
None,
(
# t_
np.array([True, False, False, False]),
# s' (raw)
np.array([[0.0], [1.0], [2.0], [3.0]]),
# action probs
np.array([
[0.0, 0.0], # <- init (no input gets sent through NN).
softmax(
dense_layer(
dense_layer(np.array([0.0]), weights_hid,
biases_hid), weights_action,
biases_action)),
softmax(
dense_layer(
dense_layer(np.array([0.5]), weights_hid,
biases_hid), weights_action,
biases_action)),
softmax(
dense_layer(
dense_layer(np.array([1.0]), weights_hid,
biases_hid), weights_action,
biases_action))
])))
test.test("step", expected_outputs=expected, decimals=3)
# Step again, check whether stitching of states/etc.. works.
expected = (
None,
(
np.array([False, False, False, True]),
np.array([[3.0], [4.0], [5.0], [0.0]]), # s' (raw)
np.array([
[0.0, 0.0], # <- init (no input gets sent through NN).
softmax(
dense_layer(
dense_layer(np.array([1.5]), weights_hid,
biases_hid), weights_action,
biases_action)),
softmax(
dense_layer(
dense_layer(np.array([2.0]), weights_hid,
biases_hid), weights_action,
biases_action)),
softmax(
dense_layer(
dense_layer(np.array([2.5]), weights_hid,
biases_hid), weights_action,
biases_action))
])))
test.test("step", expected_outputs=expected, decimals=3)
# Make sure we close the session (to shut down the Env on the server).
test.terminate()
