def test_reduction_deriv(transformer_factory, reduction, sub_axes):
if reduction in ('max', 'min'):
pytest.skip("max/min needed to be tested differently")
if sub_axes in (slice(0, 2, None), slice(1, None,
None)) and reduction == "prod":
pytest.config.flex_skip_now("Too big values for Flex ( > 32767 )")
axes = ng.make_axes([
ng.make_axis(length=4),
ng.make_axis(length=10),
ng.make_axis(length=10)
])
delta = .001
u = rng.discrete_uniform(1.0, 2.0, 2 * delta, axes)
bered = getattr(ng, reduction)
reduction_axes = axes[sub_axes]
# Need to test max/min differently since if two elements are extremums
# and we modify one, the derivative will change.
p_u = ng.placeholder(axes)
graph_reduce = bered(p_u, reduction_axes=reduction_axes)
check_derivative(graph_reduce, p_u, delta, u, atol=1e-1, rtol=1e-1)
def test_log_sigmoid_deriv(transformer_factory, input_tensor):
"""TODO."""
p_u = input_tensor
u = rng.uniform(-3.0, 3.0, p_u.axes)
log_val_u = ng.log(ng.sigmoid(p_u))
check_derivative(log_val_u, p_u, 0.001, u, atol=1e-2, rtol=1e-2)
def test_sigmoid_deriv(input_tensor):
"""TODO."""
p_u = input_tensor
u = rng.uniform(-3.0, 3.0, p_u.axes)
val_u = ng.sigmoid(p_u)
check_derivative(val_u, p_u, 0.001, u, atol=1e-2, rtol=1e-2)
def test_print_op_bprop():
"""
Ensure bprop of PrintOp is correct (passes through exactly the delta)
"""
x = ng.placeholder(ng.make_axes([ng.make_axis(length=10)]))
# randomly initialize
x_value = rng.uniform(-1, 1, x.axes)
check_derivative(ng.PrintOp(x), x, 0.001, x_value, atol=1e-3, rtol=1e-3)
def test_reciprocal_derivative(transformer_factory, input_tensor):
"""TODO."""
p_u = input_tensor
delta = .001
u = rng.uniform(.1, 5.0, p_u.axes)
rec_u = ng.reciprocal(p_u)
check_derivative(rec_u, p_u, delta, u, atol=1e-2, rtol=1e-2)
def test_dimshuffle_bprop(self, x, A, B):
"""
dimshuffle a 2d array and make sure bprop works
"""
# randomly initialize
x_value = rng.uniform(-1, 1, x.axes)
check_derivative(
ng.axes_with_order(x, [B, A]),
x, 0.001, x_value,
atol=1e-3, rtol=1e-3
)
def test_cross_entropy_softmax_rec_deriv(transformer_factory, recurrent_input_tensor):
p_x = recurrent_input_tensor
p_t = ng.placeholder(p_x.axes)
x = rng.uniform(0, 1, p_x.axes)
t = np_softmax(rng.uniform(0, 1, p_t.axes), 0)
check_derivative(
ng.cross_entropy_multi(ng.softmax(p_x), p_t),
p_x, 0.001, x,
parameters=[p_t],
parameter_values=[t],
atol=1e-2, rtol=1e-2
)
def test_prod_deriv(
prod_deriv_arrays): # Argon Transformer error - TODO triage
"""
Test reduce product's gradient
"""
def power_set(lst):
"""
power_set([0, 1, 2]) is:
[[], [0], [1], [0, 1], [2], [0, 2], [1, 2], [0, 1, 2]]
"""
result = [[]]
for x in lst:
result.extend([subset + [x] for subset in result])
return result
def get_all_reduction_axes(axes):
"""
Get all possible reduction axes
"""
ndim = len(axes.lengths)
if ndim == 0:
return axes
else:
results = []
all_indices = power_set(range(ndim))
for indices in all_indices:
if not indices:
results.append(ng.make_axes([]))
else:
results.append(
ng.make_axes([axes[index] for index in indices]))
return results
def shape_to_axes(shape):
"""
Convert shape to axes
"""
if not shape:
return ng.make_axes()
axes = ng.make_axes([ng.make_axis(length=s) for s in shape])
return axes
x_val = prod_deriv_arrays
axes = shape_to_axes(x_val.shape)
all_reduction_axes = get_all_reduction_axes(axes)
for reduction_axes in all_reduction_axes:
x = ng.placeholder(axes=axes)
x_prod = ng.prod(x, reduction_axes)
check_derivative(x_prod, x, 0.001, x_val, atol=1e-3, rtol=1e-3)
def test_flatten_deriv_simplified():
"""
Test derivative with dot and flatten
"""
ax_N = ng.make_axis(length=3)
ax_Y = ng.make_axis(length=2)
x = ng.placeholder(ng.make_axes([ax_N]))
w = ng.constant([5, 2], axes=ng.make_axes([ax_Y]))
logits = ng.dot(x, w)
cost = ng.sum(logits, reduction_axes=logits.axes)
delta = 0.001
u = rng.uniform(.1, 5.0, x.axes)
check_derivative(cost, x, delta, u, atol=1e-2, rtol=1e-2)
def test_elementwise_binary_ops_matched_args_deriv_rhs(elementwise_binary_op, symmetric_tensor):
"""TODO."""
be_op = getattr(ng, elementwise_binary_op)
p_u = symmetric_tensor
p_v = ng.placeholder(p_u.axes)
u = rng.uniform(-1.0, 1.0, p_u.axes)
v = rng.uniform(1.0, 2.0, p_v.axes)
check_derivative(
be_op(p_u, p_v), p_v, 0.001, v,
parameters=[p_u],
parameter_values=[u],
atol=1e-3, rtol=1e-3,
)
def test_flatten_deriv():
from ngraph.frontends.neon import ax
np.random.seed(0)
# set shape
C, D, H, W, N = (3, 1, 28, 28, 8) # image
Y = 10
ax.C.length = C
ax.D.length = D
ax.H.length = H
ax.W.length = W
ax.N.length = N
ax.Y.length = Y
# conv output
conv = ng.placeholder(ng.make_axes([ax.N, ax.H, ax.W, ax.C]))
# flatten
flatten = ng.flatten_at(conv, idx=1)
num_flatten = flatten.axes.lengths[1]
flatten = ng.cast_axes(flatten,
ng.make_axes([ax.N, ng.make_axis(num_flatten)]))
# fc
fc_weights_axes = ng.make_axes([ng.make_axis(num_flatten), ax.Y])
fc_weights = ng.constant(np.random.randn(num_flatten, Y),
axes=fc_weights_axes)
flatten_casted = ng.cast_axes(
flatten, ng.make_axes([flatten.axes[0], fc_weights_axes[0] - 1]))
logits = ng.dot(flatten_casted, fc_weights)
cost = ng.sum(logits, reduction_axes=logits.axes)
delta = 0.001
u = rng.uniform(.1, 5.0, conv.axes)
check_derivative(cost, conv, delta, u, atol=1e-2, rtol=1e-2)
def test_softmax_rec_deriv(transformer_factory, recurrent_input_tensor):
p_x = recurrent_input_tensor
x = rng.uniform(0, 1, p_x.axes)
check_derivative(ng.softmax(p_x), p_x, 0.001, x, atol=1e-2, rtol=1e-2)
def test_prod_deriv(transformer_factory):
"""
Test reduce product's gradient
"""
def power_set(lst):
"""
power_set([0, 1, 2]) is:
[[], [0], [1], [0, 1], [2], [0, 2], [1, 2], [0, 1, 2]]
"""
result = [[]]
for x in lst:
result.extend([subset + [x] for subset in result])
return result
def get_all_reduction_axes(axes):
"""
Get all possible reduction axes
"""
ndim = len(axes.lengths)
if ndim == 0:
return axes
else:
results = []
all_indices = power_set(range(ndim))
for indices in all_indices:
if not indices:
results.append(ng.make_axes([]))
else:
results.append(
ng.make_axes([axes[index] for index in indices]))
return results
def shape_to_axes(shape):
"""
Convert shape to axes
"""
if not shape:
return ng.make_axes()
axes = ng.make_axes([ng.make_axis(length=s) for s in shape])
return axes
# test cases
test_cases = [
np.array([[[1., 2., 3.], [4., 5., 0.], [0., 6., 0.]],
[[1., 2., 3.], [4., 5., 6.], [7., 8., 0.]]]),
np.array([[1., 2., 3.], [4., 5., 0.], [0., 6., 0.]]),
np.array([1., 2., 3.]),
np.array([0., 2., 3.]),
np.array([0., 0., 3.]),
np.array([0., 0., 0.]),
np.array([0.]),
np.array([2.]),
np.array(0.),
np.array(2.),
]
for x_val in test_cases:
axes = shape_to_axes(x_val.shape)
all_reduction_axes = get_all_reduction_axes(axes)
for reduction_axes in all_reduction_axes:
x = ng.placeholder(axes=axes)
x_prod = ng.prod(x, reduction_axes)
check_derivative(x_prod, x, 0.001, x_val, atol=1e-3, rtol=1e-3)
def test_softmax_deriv(input_tensor):
p_x = input_tensor
x = rng.uniform(0, 1, p_x.axes)
check_derivative(ng.softmax(p_x), p_x, 0.001, x, atol=1e-2, rtol=1e-2)