Beispiel #1
0
def test_layer_backward_pass_insensitive_to_internal_state_init(layer_specs):
    layer, specs = layer_specs
    layer_buffers = set_up_layer(layer, specs)
    time_steps = specs.get('time_steps', 3)
    eps = specs.get('eps', 1e-8)
    layer.forward_pass(layer_buffers)
    layer.backward_pass(layer_buffers)

    # get deltas after normal backward pass
    deltas = {}
    for key, value in layer_buffers.input_deltas.items():
        deltas[key] = HANDLER.get_numpy_copy(value)

    # randomize internal state
    for internal, shape_template in layer.internal_shapes.items():
        value = layer_buffers.internals[internal]
        if shape_template.scales_with_time:
            # exclude context slice
            HANDLER.set_from_numpy(
                value[:time_steps],
                np.random.randn(time_steps, *value.shape[1:]))
        else:
            HANDLER.set_from_numpy(value, np.random.randn(*value.shape))

        # clear deltas
        for k, v in layer_buffers.input_deltas.items():
            HANDLER.fill(v, 0.0)

        # compare new deltas
        layer.forward_pass(layer_buffers)
        layer.backward_pass(layer_buffers)
        for key, value in layer_buffers.input_deltas.items():
            assert np.allclose(deltas[key], HANDLER.get_numpy_copy(value),
                               rtol=eps, atol=eps), \
                "Failed for internal.{} when inspecting {}".format(internal, key)
Beispiel #2
0
def test_layer_backward_pass_insensitive_to_internal_state_init(layer_specs):
    layer, specs = layer_specs
    layer_buffers = set_up_layer(layer, specs)
    time_steps = specs.get('time_steps', 3)
    eps = specs.get('eps', 1e-8)
    layer.forward_pass(layer_buffers)
    layer.backward_pass(layer_buffers)

    # get deltas after normal backward pass
    deltas = {}
    for key, value in layer_buffers.input_deltas.items():
        deltas[key] = HANDLER.get_numpy_copy(value)

    # randomize internal state
    for internal, shape_template in layer.internal_shapes.items():
        value = layer_buffers.internals[internal]
        if shape_template.scales_with_time:
            # exclude context slice
            HANDLER.set_from_numpy(
                value[:time_steps],
                np.random.randn(time_steps, *value.shape[1:]))
        else:
            HANDLER.set_from_numpy(value, np.random.randn(*value.shape))

        # clear deltas
        for k, v in layer_buffers.input_deltas.items():
            HANDLER.fill(v, 0.0)

        # compare new deltas
        layer.forward_pass(layer_buffers)
        layer.backward_pass(layer_buffers)
        for key, value in layer_buffers.input_deltas.items():
            assert np.allclose(deltas[key], HANDLER.get_numpy_copy(value),
                               rtol=eps, atol=eps), \
                "Failed for internal.{} when inspecting {}".format(internal, key)
Beispiel #3
0
def test_context_slice_allows_continuing_forward_pass(net_with_context):
    net = net_with_context
    net.set_handler(HANDLER)
    net.initialize(Gaussian(0.1), seed=1234)
    all_data = np.random.randn(4, 1, 2)

    # First do a pass on all the data
    net.provide_external_data({'default': all_data})
    net.forward_pass()
    final_context = [
        HANDLER.get_numpy_copy(x) if x is not None else None
        for x in net.get_context()
    ]
    final_outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default)

    # Pass only part of data
    data_a = all_data[:2]
    net.provide_external_data({'default': data_a})
    net.forward_pass()

    # Pass rest of data with context
    data_b = all_data[2:]
    net.provide_external_data({'default': data_b})
    net.forward_pass(context=net.get_context())
    context = [
        HANDLER.get_numpy_copy(x) if x is not None else None
        for x in net.get_context()
    ]
    outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default)

    # Check if outputs are the same as final_outputs
    success = np.allclose(outputs[:-1], final_outputs[2:-1])
    if not success:
        print("Outputs:\n", outputs[:-1])
        print("Should match:\n", final_outputs[2:-1])
        raise AssertionError("Outputs did not match.")

    # Check if context is same as final_context
    assert len(context) == len(final_context), "Context list sizes mismatch!"

    for (x, y) in zip(context, final_context):
        if x is None:
            assert y is None
        else:
            # print("Context:\n", x)
            # print("Should match:\n", y)
            assert np.allclose(x, y)
def test_context_slice_allows_continuing_forward_pass(net_with_context):
    net = net_with_context
    net.set_handler(HANDLER)
    net.initialize(Gaussian(0.1), seed=1234)
    all_data = np.random.randn(4, 1, 2)

    # First do a pass on all the data
    net.provide_external_data({'default': all_data})
    net.forward_pass()
    final_context = [HANDLER.get_numpy_copy(x) if x is not None else None
                     for x in net.get_context()]
    final_outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default)

    # Pass only part of data
    data_a = all_data[:2]
    net.provide_external_data({'default': data_a})
    net.forward_pass()

    # Pass rest of data with context
    data_b = all_data[2:]
    net.provide_external_data({'default': data_b})
    net.forward_pass(context=net.get_context())
    context = [HANDLER.get_numpy_copy(x) if x is not None else None
               for x in net.get_context()]
    outputs = HANDLER.get_numpy_copy(net.buffer.out.outputs.default)

    # Check if outputs are the same as final_outputs
    success = np.allclose(outputs[:-1], final_outputs[2:-1])
    if not success:
        print("Outputs:\n", outputs[:-1])
        print("Should match:\n", final_outputs[2:-1])
        raise AssertionError("Outputs did not match.")

    # Check if context is same as final_context
    assert len(context) == len(final_context), "Context list sizes mismatch!"

    for (x, y) in zip(context, final_context):
        if x is None:
            assert y is None
        else:
            # print("Context:\n", x)
            # print("Should match:\n", y)
            assert np.allclose(x, y)
Beispiel #5
0
def test_layer_forward_pass_insensitive_to_internal_state_init(layer_specs):
    layer, specs = layer_specs
    layer_buffers = set_up_layer(layer, specs)
    time_steps = specs.get('time_steps', 3)

    eps = specs.get('eps', 1e-8)
    layer.forward_pass(layer_buffers)

    # get outputs after normal forward pass
    outputs = {}
    for key, value in layer_buffers.outputs.items():
        outputs[key] = HANDLER.get_numpy_copy(value)

    # randomize internal state
    for internal, shape_template in layer.internal_shapes.items():
        value = layer_buffers.internals[internal]
        if shape_template.scales_with_time:
            # exclude context slice
            HANDLER.set_from_numpy(
                value[:time_steps],
                np.random.randn(time_steps, *value.shape[1:]))
        else:
            HANDLER.set_from_numpy(value, np.random.randn(*value.shape))

        # compare new output
        layer.forward_pass(layer_buffers)
        for key, value in layer_buffers.outputs.items():
            assert np.allclose(outputs[key], HANDLER.get_numpy_copy(value),
                               rtol=eps, atol=eps), internal
Beispiel #6
0
def test_layer_forward_pass_insensitive_to_internal_state_init(layer_specs):
    layer, specs = layer_specs
    layer_buffers = set_up_layer(layer, specs)
    time_steps = specs.get('time_steps', 3)

    eps = specs.get('eps', 1e-8)
    layer.forward_pass(layer_buffers)

    # get outputs after normal forward pass
    outputs = {}
    for key, value in layer_buffers.outputs.items():
        outputs[key] = HANDLER.get_numpy_copy(value)

    # randomize internal state
    for internal, shape_template in layer.internal_shapes.items():
        value = layer_buffers.internals[internal]
        if shape_template.scales_with_time:
            # exclude context slice
            HANDLER.set_from_numpy(
                value[:time_steps],
                np.random.randn(time_steps, *value.shape[1:]))
        else:
            HANDLER.set_from_numpy(value, np.random.randn(*value.shape))

        # compare new output
        layer.forward_pass(layer_buffers)
        for key, value in layer_buffers.outputs.items():
            assert np.allclose(outputs[key],
                               HANDLER.get_numpy_copy(value),
                               rtol=eps,
                               atol=eps), internal
Beispiel #7
0
def test_elementwise_act_func_gradients():
    pairs_to_test = [
        (HANDLER.sigmoid, HANDLER.sigmoid_deriv),
        (HANDLER.tanh, HANDLER.tanh_deriv),
        (HANDLER.rel, HANDLER.rel_deriv),
    ]
    test_shape = (3, 2, 4)

    for fwd, bwd in pairs_to_test:
        inputs = HANDLER.create_from_numpy(np.random.randn(*test_shape))
        outputs = HANDLER.zeros(test_shape)
        doutputs = HANDLER.ones(test_shape)
        dinputs = HANDLER.zeros(test_shape)
        fwd(inputs, outputs)
        bwd(inputs, outputs, doutputs, dinputs)
        grad_calc = HANDLER.get_numpy_copy(dinputs)

        size = inputs.size
        x0 = HANDLER.get_numpy_copy(inputs).reshape((size,))

        def f(x):
            flat_inputs = inputs.reshape((size,))
            HANDLER.set_from_numpy(flat_inputs, x)
            HANDLER.fill(outputs, 0.0)
            fwd(inputs, outputs)
            return HANDLER.get_numpy_copy(outputs).sum()

        grad_approx = approx_fprime(x0, f, 1e-5).reshape(grad_calc.shape)

        close = np.allclose(grad_approx, grad_calc, rtol=1e-4, atol=1e-4)
        if not close:
            print("-----------------------------")
            print("Testing", fwd.__name__)
            print("-- Approximated Gradient ----")
            print(grad_approx)
            print("---- Calculated Gradient ----")
            print(grad_calc)
            print("------------- Difference ----")
            print(grad_approx - grad_calc)
        assert close
Beispiel #8
0
def test_elementwise_act_func_gradients():
    pairs_to_test = [(HANDLER.sigmoid, HANDLER.sigmoid_deriv),
                     (HANDLER.tanh, HANDLER.tanh_deriv),
                     (HANDLER.rel, HANDLER.rel_deriv)]
    test_shape = (3, 2, 4)

    for fwd, bwd in pairs_to_test:
        inputs = HANDLER.create_from_numpy(np.random.randn(*test_shape))
        outputs = HANDLER.zeros(test_shape)
        doutputs = HANDLER.ones(test_shape)
        dinputs = HANDLER.zeros(test_shape)
        fwd(inputs, outputs)
        bwd(inputs, outputs, doutputs, dinputs)
        grad_calc = HANDLER.get_numpy_copy(dinputs)

        size = inputs.size
        x0 = HANDLER.get_numpy_copy(inputs).reshape((size, ))

        def f(x):
            flat_inputs = inputs.reshape((size, ))
            HANDLER.set_from_numpy(flat_inputs, x)
            HANDLER.fill(outputs, 0.)
            fwd(inputs, outputs)
            return HANDLER.get_numpy_copy(outputs).sum()

        grad_approx = approx_fprime(x0, f, 1e-5).reshape(grad_calc.shape)

        close = np.allclose(grad_approx, grad_calc, rtol=1e-4, atol=1e-4)
        if not close:
            print("-----------------------------")
            print("Testing", fwd.__name__)
            print('-- Approximated Gradient ----')
            print(grad_approx)
            print('---- Calculated Gradient ----')
            print(grad_calc)
            print('------------- Difference ----')
            print(grad_approx - grad_calc)
        assert close
Beispiel #9
0
 def f(x):
     flat_inputs = inputs.reshape((size,))
     HANDLER.set_from_numpy(flat_inputs, x)
     HANDLER.fill(outputs, 0.)
     fwd(inputs, outputs)
     return HANDLER.get_numpy_copy(outputs).sum()
Beispiel #10
0
def test_layer_add_to_deltas(layer_specs):
    layer, specs = layer_specs
    layer_buffers = set_up_layer(layer, specs)
    eps = specs.get('eps', 1e-8)
    for key in layer_buffers.output_deltas.keys():
        HANDLER.fill(layer_buffers.output_deltas[key], 1.0)

    layer.forward_pass(layer_buffers)
    layer.backward_pass(layer_buffers)

    # get deltas
    deltas = {}
    for key, value in layer_buffers.input_deltas.items():
        deltas[key] = HANDLER.get_numpy_copy(value)

    # clear all bwd buffers except inputs and outputs
    for key, value in layer_buffers.internals.items():
        HANDLER.fill(value, 0)
    for key, value in layer_buffers.gradients.items():
        HANDLER.fill(value, 0)
    # set all bwd_buffer inputs to 1.0
    for key, value in layer_buffers.input_deltas.items():
        HANDLER.fill(value, 1.0)
    # output_deltas buffer may have changed due to inplace activation. Reset.
    for key in layer_buffers.output_deltas.keys():
        HANDLER.fill(layer_buffers.output_deltas[key], 1.0)

    # do a second forward/backward pass
    layer.forward_pass(layer_buffers)
    layer.backward_pass(layer_buffers)

    # assert all input deltas are 1.0 bigger
    for key, value in layer_buffers.input_deltas.items():
        obtained = HANDLER.get_numpy_copy(value)
        passed = np.allclose(deltas[key] + 1.0, obtained,
                             rtol=eps, atol=eps)
        if not passed:
            print("Adding deltas test failed for {}!".format(key))
            print("Calculated Deltas:\n", obtained)
            print("Expected Deltas:\n", deltas[key] + 1.0)
            print("Difference:\n", deltas[key] + 1.0 - obtained)
        assert passed, key
Beispiel #11
0
 def f(x):
     flat_inputs = inputs.reshape((size, ))
     HANDLER.set_from_numpy(flat_inputs, x)
     HANDLER.fill(outputs, 0.)
     fwd(inputs, outputs)
     return HANDLER.get_numpy_copy(outputs).sum()
Beispiel #12
0
def test_layer_add_to_deltas(layer_specs):
    layer, specs = layer_specs
    layer_buffers = set_up_layer(layer, specs)
    eps = specs.get('eps', 1e-8)
    for key in layer_buffers.output_deltas.keys():
        HANDLER.fill(layer_buffers.output_deltas[key], 1.0)

    layer.forward_pass(layer_buffers)
    layer.backward_pass(layer_buffers)

    # get deltas
    deltas = {}
    for key, value in layer_buffers.input_deltas.items():
        deltas[key] = HANDLER.get_numpy_copy(value)

    # clear all bwd buffers except inputs and outputs
    for key, value in layer_buffers.internals.items():
        HANDLER.fill(value, 0)
    for key, value in layer_buffers.gradients.items():
        HANDLER.fill(value, 0)
    # set all bwd_buffer inputs to 1.0
    for key, value in layer_buffers.input_deltas.items():
        HANDLER.fill(value, 1.0)
    # output_deltas buffer may have changed due to inplace activation. Reset.
    for key in layer_buffers.output_deltas.keys():
        HANDLER.fill(layer_buffers.output_deltas[key], 1.0)

    # do a second forward/backward pass
    layer.forward_pass(layer_buffers)
    layer.backward_pass(layer_buffers)

    # assert all input deltas are 1.0 bigger
    for key, value in layer_buffers.input_deltas.items():
        obtained = HANDLER.get_numpy_copy(value)
        passed = np.allclose(deltas[key] + 1.0, obtained, rtol=eps, atol=eps)
        if not passed:
            print("Adding deltas test failed for {}!".format(key))
            print("Calculated Deltas:\n", obtained)
            print("Expected Deltas:\n", deltas[key] + 1.0)
            print("Difference:\n", deltas[key] + 1.0 - obtained)
        assert passed, key