def test_layer_forward_pass_insensitive_to_internal_state_init(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
time_steps = specs.get('time_steps', 3)
eps = specs.get('eps', 1e-8)
layer.forward_pass(layer_buffers)
# get outputs after normal forward pass
outputs = {}
for key, value in layer_buffers.outputs.items():
outputs[key] = HANDLER.get_numpy_copy(value)
# randomize internal state
for internal, shape_template in layer.internal_shapes.items():
value = layer_buffers.internals[internal]
if shape_template.scales_with_time:
# exclude context slice
HANDLER.set_from_numpy(
value[:time_steps],
np.random.randn(time_steps, *value.shape[1:]))
else:
HANDLER.set_from_numpy(value, np.random.randn(*value.shape))
# compare new output
layer.forward_pass(layer_buffers)
for key, value in layer_buffers.outputs.items():
assert np.allclose(outputs[key], HANDLER.get_numpy_copy(value),
rtol=eps, atol=eps), internal
def test_layer_backward_pass_insensitive_to_internal_state_init(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
time_steps = specs.get('time_steps', 3)
eps = specs.get('eps', 1e-8)
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# get deltas after normal backward pass
deltas = {}
for key, value in layer_buffers.input_deltas.items():
deltas[key] = HANDLER.get_numpy_copy(value)
# randomize internal state
for internal, shape_template in layer.internal_shapes.items():
value = layer_buffers.internals[internal]
if shape_template.scales_with_time:
# exclude context slice
HANDLER.set_from_numpy(
value[:time_steps],
np.random.randn(time_steps, *value.shape[1:]))
else:
HANDLER.set_from_numpy(value, np.random.randn(*value.shape))
# clear deltas
for k, v in layer_buffers.input_deltas.items():
HANDLER.fill(v, 0.0)
# compare new deltas
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
for key, value in layer_buffers.input_deltas.items():
assert np.allclose(deltas[key], HANDLER.get_numpy_copy(value),
rtol=eps, atol=eps), \
"Failed for internal.{} when inspecting {}".format(internal, key)
def test_layer_backward_pass_insensitive_to_internal_state_init(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
time_steps = specs.get('time_steps', 3)
eps = specs.get('eps', 1e-8)
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# get deltas after normal backward pass
deltas = {}
for key, value in layer_buffers.input_deltas.items():
deltas[key] = HANDLER.get_numpy_copy(value)
# randomize internal state
for internal, shape_template in layer.internal_shapes.items():
value = layer_buffers.internals[internal]
if shape_template.scales_with_time:
# exclude context slice
HANDLER.set_from_numpy(
value[:time_steps],
np.random.randn(time_steps, *value.shape[1:]))
else:
HANDLER.set_from_numpy(value, np.random.randn(*value.shape))
# clear deltas
for k, v in layer_buffers.input_deltas.items():
HANDLER.fill(v, 0.0)
# compare new deltas
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
for key, value in layer_buffers.input_deltas.items():
assert np.allclose(deltas[key], HANDLER.get_numpy_copy(value),
rtol=eps, atol=eps), \
"Failed for internal.{} when inspecting {}".format(internal, key)
def test_layer_forward_pass_insensitive_to_internal_state_init(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
time_steps = specs.get('time_steps', 3)
eps = specs.get('eps', 1e-8)
layer.forward_pass(layer_buffers)
# get outputs after normal forward pass
outputs = {}
for key, value in layer_buffers.outputs.items():
outputs[key] = HANDLER.get_numpy_copy(value)
# randomize internal state
for internal, shape_template in layer.internal_shapes.items():
value = layer_buffers.internals[internal]
if shape_template.scales_with_time:
# exclude context slice
HANDLER.set_from_numpy(
value[:time_steps],
np.random.randn(time_steps, *value.shape[1:]))
else:
HANDLER.set_from_numpy(value, np.random.randn(*value.shape))
# compare new output
layer.forward_pass(layer_buffers)
for key, value in layer_buffers.outputs.items():
assert np.allclose(outputs[key],
HANDLER.get_numpy_copy(value),
rtol=eps,
atol=eps), internal
def test_layer_add_to_deltas(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
eps = specs.get('eps', 1e-8)
for key in layer_buffers.output_deltas.keys():
HANDLER.fill(layer_buffers.output_deltas[key], 1.0)
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# get deltas
deltas = {}
for key, value in layer_buffers.input_deltas.items():
deltas[key] = HANDLER.get_numpy_copy(value)
# clear all bwd buffers except inputs and outputs
for key, value in layer_buffers.internals.items():
HANDLER.fill(value, 0)
for key, value in layer_buffers.gradients.items():
HANDLER.fill(value, 0)
# set all bwd_buffer inputs to 1.0
for key, value in layer_buffers.input_deltas.items():
HANDLER.fill(value, 1.0)
# output_deltas buffer may have changed due to inplace activation. Reset.
for key in layer_buffers.output_deltas.keys():
HANDLER.fill(layer_buffers.output_deltas[key], 1.0)
# do a second forward/backward pass
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# assert all input deltas are 1.0 bigger
for key, value in layer_buffers.input_deltas.items():
obtained = HANDLER.get_numpy_copy(value)
passed = np.allclose(deltas[key] + 1.0, obtained,
rtol=eps, atol=eps)
if not passed:
print("Adding deltas test failed for {}!".format(key))
print("Calculated Deltas:\n", obtained)
print("Expected Deltas:\n", deltas[key] + 1.0)
print("Difference:\n", deltas[key] + 1.0 - obtained)
assert passed, key
def test_layer_add_to_deltas(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
eps = specs.get('eps', 1e-8)
for key in layer_buffers.output_deltas.keys():
HANDLER.fill(layer_buffers.output_deltas[key], 1.0)
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# get deltas
deltas = {}
for key, value in layer_buffers.input_deltas.items():
deltas[key] = HANDLER.get_numpy_copy(value)
# clear all bwd buffers except inputs and outputs
for key, value in layer_buffers.internals.items():
HANDLER.fill(value, 0)
for key, value in layer_buffers.gradients.items():
HANDLER.fill(value, 0)
# set all bwd_buffer inputs to 1.0
for key, value in layer_buffers.input_deltas.items():
HANDLER.fill(value, 1.0)
# output_deltas buffer may have changed due to inplace activation. Reset.
for key in layer_buffers.output_deltas.keys():
HANDLER.fill(layer_buffers.output_deltas[key], 1.0)
# do a second forward/backward pass
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# assert all input deltas are 1.0 bigger
for key, value in layer_buffers.input_deltas.items():
obtained = HANDLER.get_numpy_copy(value)
passed = np.allclose(deltas[key] + 1.0, obtained,
rtol=eps, atol=eps)
if not passed:
print("Adding deltas test failed for {}!".format(key))
print("Calculated Deltas:\n", obtained)
print("Expected Deltas:\n", deltas[key] + 1.0)
print("Difference:\n", deltas[key] + 1.0 - obtained)
assert passed, key