Example #1
0
def test_reduction_deriv(transformer_factory, reduction, sub_axes):
    if reduction in ('max', 'min'):
        pytest.skip("max/min needed to be tested differently")
    if sub_axes in (slice(0, 2, None), slice(1, None,
                                             None)) and reduction == "prod":
        pytest.config.flex_skip_now("Too big values for Flex ( > 32767 )")
    axes = ng.make_axes([
        ng.make_axis(length=4),
        ng.make_axis(length=10),
        ng.make_axis(length=10)
    ])

    delta = .001

    u = rng.discrete_uniform(1.0, 2.0, 2 * delta, axes)

    bered = getattr(ng, reduction)
    reduction_axes = axes[sub_axes]

    # Need to test max/min differently since if two elements are extremums
    # and we modify one, the derivative will change.
    p_u = ng.placeholder(axes)
    graph_reduce = bered(p_u, reduction_axes=reduction_axes)

    check_derivative(graph_reduce, p_u, delta, u, atol=1e-1, rtol=1e-1)
Example #2
0
def test_log_sigmoid_deriv(transformer_factory, input_tensor):
    """TODO."""
    p_u = input_tensor
    u = rng.uniform(-3.0, 3.0, p_u.axes)

    log_val_u = ng.log(ng.sigmoid(p_u))

    check_derivative(log_val_u, p_u, 0.001, u, atol=1e-2, rtol=1e-2)
Example #3
0
def test_sigmoid_deriv(input_tensor):
    """TODO."""
    p_u = input_tensor
    u = rng.uniform(-3.0, 3.0, p_u.axes)

    val_u = ng.sigmoid(p_u)

    check_derivative(val_u, p_u, 0.001, u, atol=1e-2, rtol=1e-2)
Example #4
0
def test_print_op_bprop():
    """
    Ensure bprop of PrintOp is correct (passes through exactly the delta)
    """
    x = ng.placeholder(ng.make_axes([ng.make_axis(length=10)]))

    # randomly initialize
    x_value = rng.uniform(-1, 1, x.axes)

    check_derivative(ng.PrintOp(x), x, 0.001, x_value, atol=1e-3, rtol=1e-3)
Example #5
0
def test_reciprocal_derivative(transformer_factory, input_tensor):
    """TODO."""
    p_u = input_tensor

    delta = .001

    u = rng.uniform(.1, 5.0, p_u.axes)

    rec_u = ng.reciprocal(p_u)

    check_derivative(rec_u, p_u, delta, u, atol=1e-2, rtol=1e-2)
Example #6
0
    def test_dimshuffle_bprop(self, x, A, B):
        """
        dimshuffle a 2d array and make sure bprop works
        """
        # randomly initialize
        x_value = rng.uniform(-1, 1, x.axes)

        check_derivative(
            ng.axes_with_order(x, [B, A]),
            x, 0.001, x_value,
            atol=1e-3, rtol=1e-3
        )
Example #7
0
def test_cross_entropy_softmax_rec_deriv(transformer_factory, recurrent_input_tensor):
    p_x = recurrent_input_tensor
    p_t = ng.placeholder(p_x.axes)

    x = rng.uniform(0, 1, p_x.axes)
    t = np_softmax(rng.uniform(0, 1, p_t.axes), 0)

    check_derivative(
        ng.cross_entropy_multi(ng.softmax(p_x), p_t),
        p_x, 0.001, x,
        parameters=[p_t],
        parameter_values=[t],
        atol=1e-2, rtol=1e-2
    )
Example #8
0
def test_prod_deriv(
        prod_deriv_arrays):  # Argon Transformer error - TODO triage
    """
    Test reduce product's gradient
    """
    def power_set(lst):
        """
        power_set([0, 1, 2]) is:
        [[], [0], [1], [0, 1], [2], [0, 2], [1, 2], [0, 1, 2]]
        """
        result = [[]]
        for x in lst:
            result.extend([subset + [x] for subset in result])
        return result

    def get_all_reduction_axes(axes):
        """
        Get all possible reduction axes
        """
        ndim = len(axes.lengths)
        if ndim == 0:
            return axes
        else:
            results = []
            all_indices = power_set(range(ndim))
            for indices in all_indices:
                if not indices:
                    results.append(ng.make_axes([]))
                else:
                    results.append(
                        ng.make_axes([axes[index] for index in indices]))
            return results

    def shape_to_axes(shape):
        """
        Convert shape to axes
        """
        if not shape:
            return ng.make_axes()
        axes = ng.make_axes([ng.make_axis(length=s) for s in shape])
        return axes

    x_val = prod_deriv_arrays
    axes = shape_to_axes(x_val.shape)
    all_reduction_axes = get_all_reduction_axes(axes)
    for reduction_axes in all_reduction_axes:
        x = ng.placeholder(axes=axes)
        x_prod = ng.prod(x, reduction_axes)
        check_derivative(x_prod, x, 0.001, x_val, atol=1e-3, rtol=1e-3)
Example #9
0
def test_flatten_deriv_simplified():
    """
    Test derivative with dot and flatten
    """
    ax_N = ng.make_axis(length=3)
    ax_Y = ng.make_axis(length=2)

    x = ng.placeholder(ng.make_axes([ax_N]))
    w = ng.constant([5, 2], axes=ng.make_axes([ax_Y]))
    logits = ng.dot(x, w)
    cost = ng.sum(logits, reduction_axes=logits.axes)

    delta = 0.001
    u = rng.uniform(.1, 5.0, x.axes)
    check_derivative(cost, x, delta, u, atol=1e-2, rtol=1e-2)
Example #10
0
def test_elementwise_binary_ops_matched_args_deriv_rhs(elementwise_binary_op, symmetric_tensor):
    """TODO."""
    be_op = getattr(ng, elementwise_binary_op)
    p_u = symmetric_tensor
    p_v = ng.placeholder(p_u.axes)

    u = rng.uniform(-1.0, 1.0, p_u.axes)
    v = rng.uniform(1.0, 2.0, p_v.axes)

    check_derivative(
        be_op(p_u, p_v), p_v, 0.001, v,
        parameters=[p_u],
        parameter_values=[u],
        atol=1e-3, rtol=1e-3,
    )
Example #11
0
def test_flatten_deriv():
    from ngraph.frontends.neon import ax
    np.random.seed(0)

    # set shape
    C, D, H, W, N = (3, 1, 28, 28, 8)  # image
    Y = 10

    ax.C.length = C
    ax.D.length = D
    ax.H.length = H
    ax.W.length = W
    ax.N.length = N
    ax.Y.length = Y

    # conv output
    conv = ng.placeholder(ng.make_axes([ax.N, ax.H, ax.W, ax.C]))

    # flatten
    flatten = ng.flatten_at(conv, idx=1)
    num_flatten = flatten.axes.lengths[1]
    flatten = ng.cast_axes(flatten,
                           ng.make_axes([ax.N, ng.make_axis(num_flatten)]))

    # fc
    fc_weights_axes = ng.make_axes([ng.make_axis(num_flatten), ax.Y])
    fc_weights = ng.constant(np.random.randn(num_flatten, Y),
                             axes=fc_weights_axes)
    flatten_casted = ng.cast_axes(
        flatten, ng.make_axes([flatten.axes[0], fc_weights_axes[0] - 1]))
    logits = ng.dot(flatten_casted, fc_weights)
    cost = ng.sum(logits, reduction_axes=logits.axes)

    delta = 0.001
    u = rng.uniform(.1, 5.0, conv.axes)
    check_derivative(cost, conv, delta, u, atol=1e-2, rtol=1e-2)
Example #12
0
def test_softmax_rec_deriv(transformer_factory, recurrent_input_tensor):
    p_x = recurrent_input_tensor
    x = rng.uniform(0, 1, p_x.axes)

    check_derivative(ng.softmax(p_x), p_x, 0.001, x, atol=1e-2, rtol=1e-2)
Example #13
0
def test_prod_deriv(transformer_factory):
    """
    Test reduce product's gradient
    """
    def power_set(lst):
        """
        power_set([0, 1, 2]) is:
        [[], [0], [1], [0, 1], [2], [0, 2], [1, 2], [0, 1, 2]]
        """
        result = [[]]
        for x in lst:
            result.extend([subset + [x] for subset in result])
        return result

    def get_all_reduction_axes(axes):
        """
        Get all possible reduction axes
        """
        ndim = len(axes.lengths)
        if ndim == 0:
            return axes
        else:
            results = []
            all_indices = power_set(range(ndim))
            for indices in all_indices:
                if not indices:
                    results.append(ng.make_axes([]))
                else:
                    results.append(
                        ng.make_axes([axes[index] for index in indices]))
            return results

    def shape_to_axes(shape):
        """
        Convert shape to axes
        """
        if not shape:
            return ng.make_axes()
        axes = ng.make_axes([ng.make_axis(length=s) for s in shape])
        return axes

    # test cases
    test_cases = [
        np.array([[[1., 2., 3.], [4., 5., 0.], [0., 6., 0.]],
                  [[1., 2., 3.], [4., 5., 6.], [7., 8., 0.]]]),
        np.array([[1., 2., 3.], [4., 5., 0.], [0., 6., 0.]]),
        np.array([1., 2., 3.]),
        np.array([0., 2., 3.]),
        np.array([0., 0., 3.]),
        np.array([0., 0., 0.]),
        np.array([0.]),
        np.array([2.]),
        np.array(0.),
        np.array(2.),
    ]

    for x_val in test_cases:
        axes = shape_to_axes(x_val.shape)
        all_reduction_axes = get_all_reduction_axes(axes)
        for reduction_axes in all_reduction_axes:
            x = ng.placeholder(axes=axes)
            x_prod = ng.prod(x, reduction_axes)
            check_derivative(x_prod, x, 0.001, x_val, atol=1e-3, rtol=1e-3)
Example #14
0
def test_softmax_deriv(input_tensor):
    p_x = input_tensor
    x = rng.uniform(0, 1, p_x.axes)

    check_derivative(ng.softmax(p_x), p_x, 0.001, x, atol=1e-2, rtol=1e-2)