def test_kernel_cache(transformer_factory):
X = ng.make_axis(32)
Y = ng.make_axis(32)
C = ng.make_axis(16384)
axes = ng.make_axes([
X,
Y
])
bcast_axes = ng.make_axes([
X,
Y,
C
])
# Limiting maximum absolute value for tensors elements to 7.9.
# See description in function test_exit_condition above
is_flex = is_flex_factory(transformer_factory)
clip_val = 7.9 if is_flex else 0
x_val = rng.randn_abs_clip(axes, clip_max=clip_val)
y_val = rng.randn_abs_clip(bcast_axes, clip_max=clip_val)
z_val = rng.randn_abs_clip(bcast_axes, clip_max=clip_val)
x = ng.constant(x_val, axes)
y = ng.constant(y_val, bcast_axes)
z = ng.constant(z_val, bcast_axes)
out = ng.add(ng.add(x, y), z)
with executor(out) as ex:
graph_val = ex()
np_val = np.add(np.add(x_val.reshape(32, 32, 1), y_val), z_val)
ng.testing.assert_allclose(graph_val, np_val, rtol=1e-4, atol_multiplier=2)
def test_broadcast():
M = ng.make_axis(length=1)
N = ng.make_axis(length=4)
np_a = np.array([[1, 2, 3, 4]], dtype=np.float32)
np_c = np.add(np_a, 2)
a = ng.constant(np_a, [M, N])
c = ng.add(a, 2)
with executor(c) as _add:
result = _add()
assert np.allclose(result, np_c)
def test_cputensor_add_constant():
"""TODO."""
M = ng.make_axis(length=1)
N = ng.make_axis(length=3)
np_a = np.array([[1, 2, 3]], dtype=np.float32)
np_c = np.add(np_a, 2)
a = ng.constant(np_a, [M, N])
b = ng.constant(2)
c = ng.add(a, b)
with executor(c) as ex:
result = ex()
assert np.array_equal(result, np_c)
def test_add_with_scalar():
H = ng.make_axis(length=1, name='height')
W = ng.make_axis(length=4, name='width')
a = ng.placeholder(axes=[H, W])
d = ng.add(a, -5)
with executor(d, a) as _add:
d_val = _add([10, 20, 30, 40])
# compute reference through numpy
d_val_ref = np.add(np.array([10, 20, 30, 40], dtype=np.float32).reshape(1, 4),
np.array([-5], dtype=np.float32))
assert np.allclose(d_val[0], d_val_ref)
def test_add_with_mul():
H = ng.make_axis(length=1, name='height')
W = ng.make_axis(length=1, name='width')
a = ng.placeholder(axes=[H, W])
b = ng.placeholder(axes=[H, W])
c = ng.placeholder(axes=[H, W])
d = ng.multiply(ng.add(a, b), c)
with executor(d, a, b, c) as _mul:
d_val = _mul([10], [20], [10])
# compute reference through numpy
_add_ref = np.add(np.full([1, 1], 10, dtype=np.float32),
np.full([1, 1], 20, dtype=np.float32))
d_val_ref = np.multiply(_add_ref, np.full([1, 1], 10, dtype=np.float32))
assert np.allclose(d_val, d_val_ref)
def test_cputensor_fusion():
"""TODO."""
M = ng.make_axis(length=1)
N = ng.make_axis(length=3)
np_a = np.array([[1, 2, 3]], dtype=np.float32)
np_b = np.array([[3, 2, 1]], dtype=np.float32)
np_d = np.multiply(np_b, np.add(np_a, 2))
a = ng.constant(np_a, [M, N])
b = ng.constant(np_b, [M, N])
c = ng.constant(2)
d = ng.multiply(b, ng.add(a, c))
with executor(d) as ex:
result = ex()
assert np.array_equal(result, np_d)
def test_4d_chained(transformer_factory, input_axes):
# Limiting maximum absolute value for tensors elements to 7.9.
# See description in function test_exit_condition above
# Limitting minimum absolute value for tensors being input to reciprocal operation to 1/7.9
#
# This is consequence of the above and flexpoint accuracy.
# Numbers very small have poor absolute accuracy. When reciprocal of them is calculated the
# results becomes very large and has even worse accuracy. When small numbers would be accepted
# as an input to reciprocal in the test the absolute maximum value of the result is undefined
# and so absolute tolerance.
# To have possibility to set atol in the test and test could pass with it minimum element of
# the tensor that is input to reciprocal operation has to be limited.
is_flex = is_flex_factory(transformer_factory)
clip_val_max = 7.9 if is_flex else 0
clip_val_min = 1.0 / 7.9 if is_flex else 0
x_val = rng.randn_abs_clip(input_axes, clip_min=clip_val_min, clip_max=clip_val_max)
y_val = rng.randn_abs_clip(input_axes, clip_max=clip_val_max)
x = ng.constant(x_val, input_axes)
y = ng.constant(y_val, input_axes)
im = ng.reciprocal(x)
out = ng.sum(ng.add(im, y), reduction_axes=input_axes[0])
with executor(out) as ex:
graph_val = ex()
np_val = np.sum(np.add(np.reciprocal(x_val), y_val), 0)
# atol_multiplier = 15 * x_val.shape[0]
#
# x_val.shape[0] is number elements added together in operation
# ng.sum(X, reduction_axes=input_axes[0])
#
# 15 is calculated the following way:
#
# Input tensor has values from the range 1/7.9 - 7.9
# For DEC=12 absolute error is equal to 0.5*2^-12 = 0.000122
# 1/7.9 = 0.126582 with this error becomes 0.126704
# Reciprocal of 1/7.9 is 7.9
# Reciprocal of 1/7.9 + err = 7.892389
# Absolute difference is 0.007611
# It is 15.2 times larger then atol limit 5e-4 from Argon transformer
ng.testing.assert_allclose(graph_val, np_val, rtol=1e-4, atol_multiplier=15 * x_val.shape[0])
def test_idempotent_axes_c():
"""
Test test axes transformations with autodiff, case c, with broadcast,
slice, cast and dim-shuffle
"""
with ExecutorFactory() as ex:
axes = ng.make_axes([ng.make_axis(3), ng.make_axis(1)])
result_axes = [ng.make_axis(length=axis.length) for axis in axes]
# variable
w = ng.variable(axes, initial_value=np.ones((3, 1)))
# broadcast l / r, introducing dummy length 1 axes
l = ng.broadcast(w, axes)
r = ng.broadcast(w, axes)
# slice
axes_slice = [slice(None, None, None), slice(None, None, None)]
l_sliced = ng.tensor_slice(l, axes_slice)
r_sliced = ng.tensor_slice(r, axes_slice)
# cast r
r_sliced_casted = ng.cast_axes(r_sliced, axes)
# perform add
result = ng.add(l_sliced, r_sliced_casted)
# cast / dimshuffle
result = ng.cast_axes(result, result_axes)
result = ng.axes_with_order(result, result_axes)
# cost and grad
cost = ng.sum(result, reduction_axes=result.axes)
grad = ng.deriv(cost, w)
grad_comp = ex.executor(grad)
cost_comp = ex.executor(cost)
cost_comp_ng = cost_comp()
grad_comp_ng = grad_comp()
grad_comp_np = np.ones((3, 1)) * 2.
assert cost_comp_ng == 6.0
assert np.array_equal(grad_comp_ng, grad_comp_np)
def test_4d_elementwise(transformer_factory, input_axes):
# Limiting maximum absolute value for tensors elements to 7.9.
# See description in function test_exit_condition above
is_flex = is_flex_factory(transformer_factory)
clip_val = 7.9 if is_flex else 0
x_val = rng.randn_abs_clip(input_axes, clip_max=clip_val)
y_val = rng.randn_abs_clip(input_axes, clip_max=clip_val)
x = ng.constant(x_val, input_axes)
y = ng.constant(y_val, input_axes)
out = ng.add(x, y)
with executor(out) as ex:
graph_val = ex()
np_val = np.add(x_val, y_val)
ng.testing.assert_allclose(graph_val, np_val, rtol=1e-4)
def test_idempotent_axes_b():
"""
Test test axes transformations with autodiff, case b, with broadcast applied
to the same tensor
"""
with ExecutorFactory() as ex:
axes = ng.make_axes([ng.make_axis(3), ng.make_axis(1)])
w = ng.variable(axes, initial_value=np.ones((3, 1)))
l = ng.broadcast(w, axes)
r = ng.broadcast(w, axes)
result = ng.add(l, r)
result = ng.cast_axes(result, axes)
cost = ng.sum(result, reduction_axes=axes)
grad = ng.deriv(cost, w)
grad_comp = ex.executor(grad)
cost_comp = ex.executor(cost)
assert cost_comp() == 6.0
assert np.array_equal(grad_comp(), np.ones((3, 1)) * 2.)
def test_multiple_computation():
H = ng.make_axis(length=1, name='height')
W = ng.make_axis(length=1, name='width')
a = ng.placeholder(axes=[H, W])
b = ng.placeholder(axes=[H, W])
_mul = ng.multiply(a, b)
_add = ng.add(a, b)
with ExecutorFactory() as ex:
# Define computations
_mul_computation = ex.executor(_mul, a, b)
_mul_val = _mul_computation([10], [20])
_add_computation = ex.executor(_add, a, b)
_add_val = _add_computation([10], [20])
# compute reference value
_mul_ref = np.multiply(np.full([1, 1], 10, dtype=np.float32),
np.full([1, 1], 20, dtype=np.float32))
_add_ref = np.add(np.full([1, 1], 10, dtype=np.float32),
np.full([1, 1], 20, dtype=np.float32))
assert np.allclose(_add_val, _add_ref)
assert np.allclose(_mul_val, _mul_ref)