def test_layer_backward_pass_insensitive_to_internal_state_init(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
time_steps = specs.get('time_steps', 3)
eps = specs.get('eps', 1e-8)
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# get deltas after normal backward pass
deltas = {}
for key, value in layer_buffers.input_deltas.items():
deltas[key] = HANDLER.get_numpy_copy(value)
# randomize internal state
for internal, shape_template in layer.internal_shapes.items():
value = layer_buffers.internals[internal]
if shape_template.scales_with_time:
# exclude context slice
HANDLER.set_from_numpy(
value[:time_steps],
np.random.randn(time_steps, *value.shape[1:]))
else:
HANDLER.set_from_numpy(value, np.random.randn(*value.shape))
# clear deltas
for k, v in layer_buffers.input_deltas.items():
HANDLER.fill(v, 0.0)
# compare new deltas
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
for key, value in layer_buffers.input_deltas.items():
assert np.allclose(deltas[key], HANDLER.get_numpy_copy(value),
rtol=eps, atol=eps), \
"Failed for internal.{} when inspecting {}".format(internal, key)
def test_layer_backward_pass_insensitive_to_internal_state_init(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
time_steps = specs.get('time_steps', 3)
eps = specs.get('eps', 1e-8)
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# get deltas after normal backward pass
deltas = {}
for key, value in layer_buffers.input_deltas.items():
deltas[key] = HANDLER.get_numpy_copy(value)
# randomize internal state
for internal, shape_template in layer.internal_shapes.items():
value = layer_buffers.internals[internal]
if shape_template.scales_with_time:
# exclude context slice
HANDLER.set_from_numpy(
value[:time_steps],
np.random.randn(time_steps, *value.shape[1:]))
else:
HANDLER.set_from_numpy(value, np.random.randn(*value.shape))
# clear deltas
for k, v in layer_buffers.input_deltas.items():
HANDLER.fill(v, 0.0)
# compare new deltas
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
for key, value in layer_buffers.input_deltas.items():
assert np.allclose(deltas[key], HANDLER.get_numpy_copy(value),
rtol=eps, atol=eps), \
"Failed for internal.{} when inspecting {}".format(internal, key)
def test_context_slice_allows_continuing_forward_pass(net_with_context):
net = net_with_context
net.set_handler(HANDLER)
net.initialize(Gaussian(0.1), seed=1234)
all_data = np.random.randn(4, 1, 2)
# First do a pass on all the data
net.provide_external_data({'default': all_data})
net.forward_pass()
final_context = [
HANDLER.get_numpy_copy(x) if x is not None else None
for x in net.get_context()
]
final_outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default)
# Pass only part of data
data_a = all_data[:2]
net.provide_external_data({'default': data_a})
net.forward_pass()
# Pass rest of data with context
data_b = all_data[2:]
net.provide_external_data({'default': data_b})
net.forward_pass(context=net.get_context())
context = [
HANDLER.get_numpy_copy(x) if x is not None else None
for x in net.get_context()
]
outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default)
# Check if outputs are the same as final_outputs
success = np.allclose(outputs[:-1], final_outputs[2:-1])
if not success:
print("Outputs:\n", outputs[:-1])
print("Should match:\n", final_outputs[2:-1])
raise AssertionError("Outputs did not match.")
# Check if context is same as final_context
assert len(context) == len(final_context), "Context list sizes mismatch!"
for (x, y) in zip(context, final_context):
if x is None:
assert y is None
else:
# print("Context:\n", x)
# print("Should match:\n", y)
assert np.allclose(x, y)
def test_context_slice_allows_continuing_forward_pass(net_with_context):
net = net_with_context
net.set_handler(HANDLER)
net.initialize(Gaussian(0.1), seed=1234)
all_data = np.random.randn(4, 1, 2)
# First do a pass on all the data
net.provide_external_data({'default': all_data})
net.forward_pass()
final_context = [HANDLER.get_numpy_copy(x) if x is not None else None
for x in net.get_context()]
final_outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default)
# Pass only part of data
data_a = all_data[:2]
net.provide_external_data({'default': data_a})
net.forward_pass()
# Pass rest of data with context
data_b = all_data[2:]
net.provide_external_data({'default': data_b})
net.forward_pass(context=net.get_context())
context = [HANDLER.get_numpy_copy(x) if x is not None else None
for x in net.get_context()]
outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default)
# Check if outputs are the same as final_outputs
success = np.allclose(outputs[:-1], final_outputs[2:-1])
if not success:
print("Outputs:\n", outputs[:-1])
print("Should match:\n", final_outputs[2:-1])
raise AssertionError("Outputs did not match.")
# Check if context is same as final_context
assert len(context) == len(final_context), "Context list sizes mismatch!"
for (x, y) in zip(context, final_context):
if x is None:
assert y is None
else:
# print("Context:\n", x)
# print("Should match:\n", y)
assert np.allclose(x, y)
def test_layer_forward_pass_insensitive_to_internal_state_init(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
time_steps = specs.get('time_steps', 3)
eps = specs.get('eps', 1e-8)
layer.forward_pass(layer_buffers)
# get outputs after normal forward pass
outputs = {}
for key, value in layer_buffers.outputs.items():
outputs[key] = HANDLER.get_numpy_copy(value)
# randomize internal state
for internal, shape_template in layer.internal_shapes.items():
value = layer_buffers.internals[internal]
if shape_template.scales_with_time:
# exclude context slice
HANDLER.set_from_numpy(
value[:time_steps],
np.random.randn(time_steps, *value.shape[1:]))
else:
HANDLER.set_from_numpy(value, np.random.randn(*value.shape))
# compare new output
layer.forward_pass(layer_buffers)
for key, value in layer_buffers.outputs.items():
assert np.allclose(outputs[key], HANDLER.get_numpy_copy(value),
rtol=eps, atol=eps), internal
def test_layer_forward_pass_insensitive_to_internal_state_init(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
time_steps = specs.get('time_steps', 3)
eps = specs.get('eps', 1e-8)
layer.forward_pass(layer_buffers)
# get outputs after normal forward pass
outputs = {}
for key, value in layer_buffers.outputs.items():
outputs[key] = HANDLER.get_numpy_copy(value)
# randomize internal state
for internal, shape_template in layer.internal_shapes.items():
value = layer_buffers.internals[internal]
if shape_template.scales_with_time:
# exclude context slice
HANDLER.set_from_numpy(
value[:time_steps],
np.random.randn(time_steps, *value.shape[1:]))
else:
HANDLER.set_from_numpy(value, np.random.randn(*value.shape))
# compare new output
layer.forward_pass(layer_buffers)
for key, value in layer_buffers.outputs.items():
assert np.allclose(outputs[key],
HANDLER.get_numpy_copy(value),
rtol=eps,
atol=eps), internal
def test_elementwise_act_func_gradients():
pairs_to_test = [
(HANDLER.sigmoid, HANDLER.sigmoid_deriv),
(HANDLER.tanh, HANDLER.tanh_deriv),
(HANDLER.rel, HANDLER.rel_deriv),
]
test_shape = (3, 2, 4)
for fwd, bwd in pairs_to_test:
inputs = HANDLER.create_from_numpy(np.random.randn(*test_shape))
outputs = HANDLER.zeros(test_shape)
doutputs = HANDLER.ones(test_shape)
dinputs = HANDLER.zeros(test_shape)
fwd(inputs, outputs)
bwd(inputs, outputs, doutputs, dinputs)
grad_calc = HANDLER.get_numpy_copy(dinputs)
size = inputs.size
x0 = HANDLER.get_numpy_copy(inputs).reshape((size,))
def f(x):
flat_inputs = inputs.reshape((size,))
HANDLER.set_from_numpy(flat_inputs, x)
HANDLER.fill(outputs, 0.0)
fwd(inputs, outputs)
return HANDLER.get_numpy_copy(outputs).sum()
grad_approx = approx_fprime(x0, f, 1e-5).reshape(grad_calc.shape)
close = np.allclose(grad_approx, grad_calc, rtol=1e-4, atol=1e-4)
if not close:
print("-----------------------------")
print("Testing", fwd.__name__)
print("-- Approximated Gradient ----")
print(grad_approx)
print("---- Calculated Gradient ----")
print(grad_calc)
print("------------- Difference ----")
print(grad_approx - grad_calc)
assert close
def test_elementwise_act_func_gradients():
pairs_to_test = [(HANDLER.sigmoid, HANDLER.sigmoid_deriv),
(HANDLER.tanh, HANDLER.tanh_deriv),
(HANDLER.rel, HANDLER.rel_deriv)]
test_shape = (3, 2, 4)
for fwd, bwd in pairs_to_test:
inputs = HANDLER.create_from_numpy(np.random.randn(*test_shape))
outputs = HANDLER.zeros(test_shape)
doutputs = HANDLER.ones(test_shape)
dinputs = HANDLER.zeros(test_shape)
fwd(inputs, outputs)
bwd(inputs, outputs, doutputs, dinputs)
grad_calc = HANDLER.get_numpy_copy(dinputs)
size = inputs.size
x0 = HANDLER.get_numpy_copy(inputs).reshape((size, ))
def f(x):
flat_inputs = inputs.reshape((size, ))
HANDLER.set_from_numpy(flat_inputs, x)
HANDLER.fill(outputs, 0.)
fwd(inputs, outputs)
return HANDLER.get_numpy_copy(outputs).sum()
grad_approx = approx_fprime(x0, f, 1e-5).reshape(grad_calc.shape)
close = np.allclose(grad_approx, grad_calc, rtol=1e-4, atol=1e-4)
if not close:
print("-----------------------------")
print("Testing", fwd.__name__)
print('-- Approximated Gradient ----')
print(grad_approx)
print('---- Calculated Gradient ----')
print(grad_calc)
print('------------- Difference ----')
print(grad_approx - grad_calc)
assert close
def f(x):
flat_inputs = inputs.reshape((size,))
HANDLER.set_from_numpy(flat_inputs, x)
HANDLER.fill(outputs, 0.)
fwd(inputs, outputs)
return HANDLER.get_numpy_copy(outputs).sum()
def test_layer_add_to_deltas(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
eps = specs.get('eps', 1e-8)
for key in layer_buffers.output_deltas.keys():
HANDLER.fill(layer_buffers.output_deltas[key], 1.0)
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# get deltas
deltas = {}
for key, value in layer_buffers.input_deltas.items():
deltas[key] = HANDLER.get_numpy_copy(value)
# clear all bwd buffers except inputs and outputs
for key, value in layer_buffers.internals.items():
HANDLER.fill(value, 0)
for key, value in layer_buffers.gradients.items():
HANDLER.fill(value, 0)
# set all bwd_buffer inputs to 1.0
for key, value in layer_buffers.input_deltas.items():
HANDLER.fill(value, 1.0)
# output_deltas buffer may have changed due to inplace activation. Reset.
for key in layer_buffers.output_deltas.keys():
HANDLER.fill(layer_buffers.output_deltas[key], 1.0)
# do a second forward/backward pass
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# assert all input deltas are 1.0 bigger
for key, value in layer_buffers.input_deltas.items():
obtained = HANDLER.get_numpy_copy(value)
passed = np.allclose(deltas[key] + 1.0, obtained,
rtol=eps, atol=eps)
if not passed:
print("Adding deltas test failed for {}!".format(key))
print("Calculated Deltas:\n", obtained)
print("Expected Deltas:\n", deltas[key] + 1.0)
print("Difference:\n", deltas[key] + 1.0 - obtained)
assert passed, key
def f(x):
flat_inputs = inputs.reshape((size, ))
HANDLER.set_from_numpy(flat_inputs, x)
HANDLER.fill(outputs, 0.)
fwd(inputs, outputs)
return HANDLER.get_numpy_copy(outputs).sum()
def test_layer_add_to_deltas(layer_specs):
layer, specs = layer_specs
layer_buffers = set_up_layer(layer, specs)
eps = specs.get('eps', 1e-8)
for key in layer_buffers.output_deltas.keys():
HANDLER.fill(layer_buffers.output_deltas[key], 1.0)
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# get deltas
deltas = {}
for key, value in layer_buffers.input_deltas.items():
deltas[key] = HANDLER.get_numpy_copy(value)
# clear all bwd buffers except inputs and outputs
for key, value in layer_buffers.internals.items():
HANDLER.fill(value, 0)
for key, value in layer_buffers.gradients.items():
HANDLER.fill(value, 0)
# set all bwd_buffer inputs to 1.0
for key, value in layer_buffers.input_deltas.items():
HANDLER.fill(value, 1.0)
# output_deltas buffer may have changed due to inplace activation. Reset.
for key in layer_buffers.output_deltas.keys():
HANDLER.fill(layer_buffers.output_deltas[key], 1.0)
# do a second forward/backward pass
layer.forward_pass(layer_buffers)
layer.backward_pass(layer_buffers)
# assert all input deltas are 1.0 bigger
for key, value in layer_buffers.input_deltas.items():
obtained = HANDLER.get_numpy_copy(value)
passed = np.allclose(deltas[key] + 1.0, obtained, rtol=eps, atol=eps)
if not passed:
print("Adding deltas test failed for {}!".format(key))
print("Calculated Deltas:\n", obtained)
print("Expected Deltas:\n", deltas[key] + 1.0)
print("Difference:\n", deltas[key] + 1.0 - obtained)
assert passed, key