def test_layer_backward_pass_insensitive_to_internal_state_init(layer_specs): layer, specs = layer_specs layer_buffers = set_up_layer(layer, specs) time_steps = specs.get('time_steps', 3) eps = specs.get('eps', 1e-8) layer.forward_pass(layer_buffers) layer.backward_pass(layer_buffers) # get deltas after normal backward pass deltas = {} for key, value in layer_buffers.input_deltas.items(): deltas[key] = HANDLER.get_numpy_copy(value) # randomize internal state for internal, shape_template in layer.internal_shapes.items(): value = layer_buffers.internals[internal] if shape_template.scales_with_time: # exclude context slice HANDLER.set_from_numpy( value[:time_steps], np.random.randn(time_steps, *value.shape[1:])) else: HANDLER.set_from_numpy(value, np.random.randn(*value.shape)) # clear deltas for k, v in layer_buffers.input_deltas.items(): HANDLER.fill(v, 0.0) # compare new deltas layer.forward_pass(layer_buffers) layer.backward_pass(layer_buffers) for key, value in layer_buffers.input_deltas.items(): assert np.allclose(deltas[key], HANDLER.get_numpy_copy(value), rtol=eps, atol=eps), \ "Failed for internal.{} when inspecting {}".format(internal, key)
def test_context_slice_allows_continuing_forward_pass(net_with_context): net = net_with_context net.set_handler(HANDLER) net.initialize(Gaussian(0.1), seed=1234) all_data = np.random.randn(4, 1, 2) # First do a pass on all the data net.provide_external_data({'default': all_data}) net.forward_pass() final_context = [ HANDLER.get_numpy_copy(x) if x is not None else None for x in net.get_context() ] final_outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default) # Pass only part of data data_a = all_data[:2] net.provide_external_data({'default': data_a}) net.forward_pass() # Pass rest of data with context data_b = all_data[2:] net.provide_external_data({'default': data_b}) net.forward_pass(context=net.get_context()) context = [ HANDLER.get_numpy_copy(x) if x is not None else None for x in net.get_context() ] outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default) # Check if outputs are the same as final_outputs success = np.allclose(outputs[:-1], final_outputs[2:-1]) if not success: print("Outputs:\n", outputs[:-1]) print("Should match:\n", final_outputs[2:-1]) raise AssertionError("Outputs did not match.") # Check if context is same as final_context assert len(context) == len(final_context), "Context list sizes mismatch!" for (x, y) in zip(context, final_context): if x is None: assert y is None else: # print("Context:\n", x) # print("Should match:\n", y) assert np.allclose(x, y)
def test_context_slice_allows_continuing_forward_pass(net_with_context): net = net_with_context net.set_handler(HANDLER) net.initialize(Gaussian(0.1), seed=1234) all_data = np.random.randn(4, 1, 2) # First do a pass on all the data net.provide_external_data({'default': all_data}) net.forward_pass() final_context = [HANDLER.get_numpy_copy(x) if x is not None else None for x in net.get_context()] final_outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default) # Pass only part of data data_a = all_data[:2] net.provide_external_data({'default': data_a}) net.forward_pass() # Pass rest of data with context data_b = all_data[2:] net.provide_external_data({'default': data_b}) net.forward_pass(context=net.get_context()) context = [HANDLER.get_numpy_copy(x) if x is not None else None for x in net.get_context()] outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default) # Check if outputs are the same as final_outputs success = np.allclose(outputs[:-1], final_outputs[2:-1]) if not success: print("Outputs:\n", outputs[:-1]) print("Should match:\n", final_outputs[2:-1]) raise AssertionError("Outputs did not match.") # Check if context is same as final_context assert len(context) == len(final_context), "Context list sizes mismatch!" for (x, y) in zip(context, final_context): if x is None: assert y is None else: # print("Context:\n", x) # print("Should match:\n", y) assert np.allclose(x, y)
def test_layer_forward_pass_insensitive_to_internal_state_init(layer_specs): layer, specs = layer_specs layer_buffers = set_up_layer(layer, specs) time_steps = specs.get('time_steps', 3) eps = specs.get('eps', 1e-8) layer.forward_pass(layer_buffers) # get outputs after normal forward pass outputs = {} for key, value in layer_buffers.outputs.items(): outputs[key] = HANDLER.get_numpy_copy(value) # randomize internal state for internal, shape_template in layer.internal_shapes.items(): value = layer_buffers.internals[internal] if shape_template.scales_with_time: # exclude context slice HANDLER.set_from_numpy( value[:time_steps], np.random.randn(time_steps, *value.shape[1:])) else: HANDLER.set_from_numpy(value, np.random.randn(*value.shape)) # compare new output layer.forward_pass(layer_buffers) for key, value in layer_buffers.outputs.items(): assert np.allclose(outputs[key], HANDLER.get_numpy_copy(value), rtol=eps, atol=eps), internal
def test_elementwise_act_func_gradients(): pairs_to_test = [ (HANDLER.sigmoid, HANDLER.sigmoid_deriv), (HANDLER.tanh, HANDLER.tanh_deriv), (HANDLER.rel, HANDLER.rel_deriv), ] test_shape = (3, 2, 4) for fwd, bwd in pairs_to_test: inputs = HANDLER.create_from_numpy(np.random.randn(*test_shape)) outputs = HANDLER.zeros(test_shape) doutputs = HANDLER.ones(test_shape) dinputs = HANDLER.zeros(test_shape) fwd(inputs, outputs) bwd(inputs, outputs, doutputs, dinputs) grad_calc = HANDLER.get_numpy_copy(dinputs) size = inputs.size x0 = HANDLER.get_numpy_copy(inputs).reshape((size,)) def f(x): flat_inputs = inputs.reshape((size,)) HANDLER.set_from_numpy(flat_inputs, x) HANDLER.fill(outputs, 0.0) fwd(inputs, outputs) return HANDLER.get_numpy_copy(outputs).sum() grad_approx = approx_fprime(x0, f, 1e-5).reshape(grad_calc.shape) close = np.allclose(grad_approx, grad_calc, rtol=1e-4, atol=1e-4) if not close: print("-----------------------------") print("Testing", fwd.__name__) print("-- Approximated Gradient ----") print(grad_approx) print("---- Calculated Gradient ----") print(grad_calc) print("------------- Difference ----") print(grad_approx - grad_calc) assert close
def test_elementwise_act_func_gradients(): pairs_to_test = [(HANDLER.sigmoid, HANDLER.sigmoid_deriv), (HANDLER.tanh, HANDLER.tanh_deriv), (HANDLER.rel, HANDLER.rel_deriv)] test_shape = (3, 2, 4) for fwd, bwd in pairs_to_test: inputs = HANDLER.create_from_numpy(np.random.randn(*test_shape)) outputs = HANDLER.zeros(test_shape) doutputs = HANDLER.ones(test_shape) dinputs = HANDLER.zeros(test_shape) fwd(inputs, outputs) bwd(inputs, outputs, doutputs, dinputs) grad_calc = HANDLER.get_numpy_copy(dinputs) size = inputs.size x0 = HANDLER.get_numpy_copy(inputs).reshape((size, )) def f(x): flat_inputs = inputs.reshape((size, )) HANDLER.set_from_numpy(flat_inputs, x) HANDLER.fill(outputs, 0.) fwd(inputs, outputs) return HANDLER.get_numpy_copy(outputs).sum() grad_approx = approx_fprime(x0, f, 1e-5).reshape(grad_calc.shape) close = np.allclose(grad_approx, grad_calc, rtol=1e-4, atol=1e-4) if not close: print("-----------------------------") print("Testing", fwd.__name__) print('-- Approximated Gradient ----') print(grad_approx) print('---- Calculated Gradient ----') print(grad_calc) print('------------- Difference ----') print(grad_approx - grad_calc) assert close
def f(x): flat_inputs = inputs.reshape((size,)) HANDLER.set_from_numpy(flat_inputs, x) HANDLER.fill(outputs, 0.) fwd(inputs, outputs) return HANDLER.get_numpy_copy(outputs).sum()
def test_layer_add_to_deltas(layer_specs): layer, specs = layer_specs layer_buffers = set_up_layer(layer, specs) eps = specs.get('eps', 1e-8) for key in layer_buffers.output_deltas.keys(): HANDLER.fill(layer_buffers.output_deltas[key], 1.0) layer.forward_pass(layer_buffers) layer.backward_pass(layer_buffers) # get deltas deltas = {} for key, value in layer_buffers.input_deltas.items(): deltas[key] = HANDLER.get_numpy_copy(value) # clear all bwd buffers except inputs and outputs for key, value in layer_buffers.internals.items(): HANDLER.fill(value, 0) for key, value in layer_buffers.gradients.items(): HANDLER.fill(value, 0) # set all bwd_buffer inputs to 1.0 for key, value in layer_buffers.input_deltas.items(): HANDLER.fill(value, 1.0) # output_deltas buffer may have changed due to inplace activation. Reset. for key in layer_buffers.output_deltas.keys(): HANDLER.fill(layer_buffers.output_deltas[key], 1.0) # do a second forward/backward pass layer.forward_pass(layer_buffers) layer.backward_pass(layer_buffers) # assert all input deltas are 1.0 bigger for key, value in layer_buffers.input_deltas.items(): obtained = HANDLER.get_numpy_copy(value) passed = np.allclose(deltas[key] + 1.0, obtained, rtol=eps, atol=eps) if not passed: print("Adding deltas test failed for {}!".format(key)) print("Calculated Deltas:\n", obtained) print("Expected Deltas:\n", deltas[key] + 1.0) print("Difference:\n", deltas[key] + 1.0 - obtained) assert passed, key
def f(x): flat_inputs = inputs.reshape((size, )) HANDLER.set_from_numpy(flat_inputs, x) HANDLER.fill(outputs, 0.) fwd(inputs, outputs) return HANDLER.get_numpy_copy(outputs).sum()