def test_constant_init():
"""TODO."""
a = ng.constant(5)
with executor(a) as ex:
result = ex()
assert (result == 5)
nparray = np.array(range(5))
a = ng.constant(nparray)
with executor(a) as ex:
result = ex()
ng.testing.assert_allclose(result, nparray)
def test_kernel_cache(transformer_factory):
X = ng.make_axis(32)
Y = ng.make_axis(32)
C = ng.make_axis(16384)
axes = ng.make_axes([
X,
Y
])
bcast_axes = ng.make_axes([
X,
Y,
C
])
# Limiting maximum absolute value for tensors elements to 7.9.
# See description in function test_exit_condition above
is_flex = is_flex_factory(transformer_factory)
clip_val = 7.9 if is_flex else 0
x_val = rng.randn_abs_clip(axes, clip_max=clip_val)
y_val = rng.randn_abs_clip(bcast_axes, clip_max=clip_val)
z_val = rng.randn_abs_clip(bcast_axes, clip_max=clip_val)
x = ng.constant(x_val, axes)
y = ng.constant(y_val, bcast_axes)
z = ng.constant(z_val, bcast_axes)
out = ng.add(ng.add(x, y), z)
with executor(out) as ex:
graph_val = ex()
np_val = np.add(np.add(x_val.reshape(32, 32, 1), y_val), z_val)
ng.testing.assert_allclose(graph_val, np_val, rtol=1e-4, atol_multiplier=2)
def test_constant_multiply():
"""TODO."""
a = ng.constant(4)
b = ng.constant(2)
c = ng.multiply(a, b)
with executor(c) as ex:
result = ex()
assert result == 8
def test_constant_add():
"""TODO."""
a = ng.constant(1)
b = ng.constant(2)
c = a + b
with executor(c) as ex:
result = ex()
assert result == 3
def baseline_value(self, y, t):
'''
Use defined ngraph constructed computation to evaluate
cost function on inputs y and t
'''
N = ng.make_axis(length=y.shape[0])
Y, T = ng.placeholder([N]), ng.placeholder([N])
with executor(self.ng_computation(Y, T), Y, T) as ex:
return ex(y, t)
def test_uniform_range_posneg(input_tensor_axes):
"""TODO."""
ng_a = ng.uniform(input_tensor_axes, low=-0.5, high=0.5)
with executor(ng_a) as ex:
result = ex()
print(result)
assert np.all(result < 0.5)
assert np.all(result >= -0.5)
assert not np.all(result >= 0.0)
def test_execute_non_placeholder():
"""
Expect a failure if a non-input (Variable) is used as an argument to
executor.
"""
N = ng.make_axis(length=1)
x = ng.temporary([N])
y = ng.variable([N])
with pytest.raises(ValueError):
with executor(x + y, x, y) as ex:
ex
def test_missing_arguments_to_execute():
"""
Expect a failure if the wrong number of arguments are passed to a
computation.
"""
N = ng.make_axis(length=1)
x = ng.placeholder([N])
y = ng.placeholder([N])
with executor(x + y, x, y) as f:
with pytest.raises(ValueError):
f(1)
def test_tensor_size():
n, m = 3, 4
N = ng.make_axis(length=n)
M = ng.make_axis(length=m)
aaxes = ng.make_axes([N, M])
x = ng.placeholder(aaxes)
size_fun = ng.tensor_size(x)
nptensor = np.arange(n * m).reshape(n, m)
with executor(size_fun, x) as ex:
assert ex(nptensor) == n * m
def test_cputensor_add_constant():
"""TODO."""
M = ng.make_axis(length=1)
N = ng.make_axis(length=3)
np_a = np.array([[1, 2, 3]], dtype=np.float32)
np_c = np.add(np_a, 2)
a = ng.constant(np_a, [M, N])
b = ng.constant(2)
c = ng.add(a, b)
with executor(c) as ex:
result = ex()
assert np.array_equal(result, np_c)
def test_cputensor_add():
"""TODO."""
Y = ng.make_axis(length=2)
M = ng.make_axis(length=2)
N = ng.make_axis(length=2)
a = ng.constant(np.array([3, 5], dtype=np.float32), [Y])
b = ng.constant(np.array([3, 5], dtype=np.float32), [Y])
c = a + b
with executor(c) as ex:
result = ex()
assert np.array_equal(result, [6, 10])
np_a = np.array([[1, 2], [3, 4]], dtype=np.float32)
np_b = np.array([[1, 2], [3, 4]], dtype=np.float32)
np_c = np_a + np_b
a = ng.constant(np_a, [M, N])
b = ng.constant(np_b, [M, N])
c = a + b
with executor(c) as ex:
result = ex()
assert np.array_equal(result, np_c)
def test_broadcast():
M = ng.make_axis(length=1)
N = ng.make_axis(length=4)
np_a = np.array([[1, 2, 3, 4]], dtype=np.float32)
np_c = np.add(np_a, 2)
a = ng.constant(np_a, [M, N])
c = ng.add(a, 2)
with executor(c) as _add:
result = _add()
assert np.allclose(result, np_c)
def test_causal_convolution(conv1d_placeholder, spatial_onehot, output_size,
width):
""" Test that causal convolutions only operate on leftward inputs"""
conv_layer = Convolution((3, output_size), lambda x: 1, padding="causal")
output = conv_layer(conv1d_placeholder)
output_width = output.axes.find_by_name("W")[0].length
assert output_width == width, "Causal convolution output width != " \
"input width: {} != {}".format(output_width, width)
with executor(output, conv1d_placeholder) as comp:
output_val = comp(spatial_onehot)
# First 1 is at width // 2, so anything before that should be 0
assert (
output_val[:, :width //
2] == 0).all(), "Acausal outputs in causal convolution"
def test_add_with_scalar():
H = ng.make_axis(length=1, name='height')
W = ng.make_axis(length=4, name='width')
a = ng.placeholder(axes=[H, W])
d = ng.add(a, -5)
with executor(d, a) as _add:
d_val = _add([10, 20, 30, 40])
# compute reference through numpy
d_val_ref = np.add(np.array([10, 20, 30, 40], dtype=np.float32).reshape(1, 4),
np.array([-5], dtype=np.float32))
assert np.allclose(d_val[0], d_val_ref)
def test_dimshuffle_fprop(self, x, A, B):
"""
dimshuffle a 2d array and make sure fprop works
"""
# compute convolution with graph
output = ng.axes_with_order(x, [B, A])
assert output.axes == ng.make_axes([B, A])
# randomly initialize
x_value = rng.uniform(-1, 1, x.axes)
with executor(output, x) as ex:
result = ex(x_value)
ng.testing.assert_allclose(result, x_value.T)
def test_4d_chained(transformer_factory, input_axes):
# Limiting maximum absolute value for tensors elements to 7.9.
# See description in function test_exit_condition above
# Limitting minimum absolute value for tensors being input to reciprocal operation to 1/7.9
#
# This is consequence of the above and flexpoint accuracy.
# Numbers very small have poor absolute accuracy. When reciprocal of them is calculated the
# results becomes very large and has even worse accuracy. When small numbers would be accepted
# as an input to reciprocal in the test the absolute maximum value of the result is undefined
# and so absolute tolerance.
# To have possibility to set atol in the test and test could pass with it minimum element of
# the tensor that is input to reciprocal operation has to be limited.
is_flex = is_flex_factory(transformer_factory)
clip_val_max = 7.9 if is_flex else 0
clip_val_min = 1.0 / 7.9 if is_flex else 0
x_val = rng.randn_abs_clip(input_axes, clip_min=clip_val_min, clip_max=clip_val_max)
y_val = rng.randn_abs_clip(input_axes, clip_max=clip_val_max)
x = ng.constant(x_val, input_axes)
y = ng.constant(y_val, input_axes)
im = ng.reciprocal(x)
out = ng.sum(ng.add(im, y), reduction_axes=input_axes[0])
with executor(out) as ex:
graph_val = ex()
np_val = np.sum(np.add(np.reciprocal(x_val), y_val), 0)
# atol_multiplier = 15 * x_val.shape[0]
#
# x_val.shape[0] is number elements added together in operation
# ng.sum(X, reduction_axes=input_axes[0])
#
# 15 is calculated the following way:
#
# Input tensor has values from the range 1/7.9 - 7.9
# For DEC=12 absolute error is equal to 0.5*2^-12 = 0.000122
# 1/7.9 = 0.126582 with this error becomes 0.126704
# Reciprocal of 1/7.9 is 7.9
# Reciprocal of 1/7.9 + err = 7.892389
# Absolute difference is 0.007611
# It is 15.2 times larger then atol limit 5e-4 from Argon transformer
ng.testing.assert_allclose(graph_val, np_val, rtol=1e-4, atol_multiplier=15 * x_val.shape[0])
def test_add_with_mul():
H = ng.make_axis(length=1, name='height')
W = ng.make_axis(length=1, name='width')
a = ng.placeholder(axes=[H, W])
b = ng.placeholder(axes=[H, W])
c = ng.placeholder(axes=[H, W])
d = ng.multiply(ng.add(a, b), c)
with executor(d, a, b, c) as _mul:
d_val = _mul([10], [20], [10])
# compute reference through numpy
_add_ref = np.add(np.full([1, 1], 10, dtype=np.float32),
np.full([1, 1], 20, dtype=np.float32))
d_val_ref = np.multiply(_add_ref, np.full([1, 1], 10, dtype=np.float32))
assert np.allclose(d_val, d_val_ref)
def test_cputensor_dot():
Y = ng.make_axis(length=2)
M = ng.make_axis(length=1)
N = ng.make_axis(length=3)
np_a = np.array([[1, 2, 3]], dtype=np.float32)
np_b = np.array([[1, 2], [2, 3], [3, 4]], dtype=np.float32)
np_c = np.dot(np_a, np_b)
a = ng.constant(np_a, [M, N]).named('a')
b = ng.constant(np_b, [N, Y]).named('b')
c = ng.dot(a, b)
with executor(c) as ex:
result = ex()
assert np.array_equal(result, np_c)
def test_cputensor_fusion():
"""TODO."""
M = ng.make_axis(length=1)
N = ng.make_axis(length=3)
np_a = np.array([[1, 2, 3]], dtype=np.float32)
np_b = np.array([[3, 2, 1]], dtype=np.float32)
np_d = np.multiply(np_b, np.add(np_a, 2))
a = ng.constant(np_a, [M, N])
b = ng.constant(np_b, [M, N])
c = ng.constant(2)
d = ng.multiply(b, ng.add(a, c))
with executor(d) as ex:
result = ex()
assert np.array_equal(result, np_d)
def ngraph_l2_norm(np_array):
"""
TODO.
Arguments:
np_array: TODO
Returns:
TODO
"""
axes = ()
for i, l in enumerate(np_array.shape):
axes |= (ng.make_axis(length=l).named('axis%s' % i),)
np_tensor = ng.constant(np_array, axes)
var = ng.variable(axes, initial_value=np_tensor)
with executor(ng.sqrt(ng.squared_L2(var))) as ex:
return ex()
def test_dot():
H = ng.make_axis(length=1)
W = ng.make_axis(length=4)
np_a = np.array([[1, 2, 3, 4]], dtype=np.float32)
np_b = np.array(3, dtype=np.float32)
a = ng.constant(np_a, [H, W])
b = ng.constant(np_b, [])
c = ng.dot(a, b)
with executor(c) as _dot:
_dot_val = _dot()
# compute reference
_dot_val_ref = np.dot(np_a, np_b)
# this checks the dot product between scalar and vector, this is equivalent to
# elementwise multiplication between scalar and vector
assert np.allclose(_dot_val, _dot_val_ref)
def test_cast_axis():
"""
Test AxesCastOp
"""
H = ng.make_axis(length=1, name='height')
W = ng.make_axis(length=4, name='width')
axes_input = [H, W]
a = ng.placeholder(axes=axes_input)
axes_output = ng.make_axes([ng.make_axis(name=ax.name + 'p', length=ax.length)
for ax in axes_input])
b = ng.cast_axes(a, axes_output)
with executor(b, a) as _cast_axis:
a_val = np.array([10, 20, 30, 40], dtype=np.float32).reshape(1, 4)
b_val = _cast_axis(a_val)
b_val_ref = a_val
assert np.allclose(b_val, b_val_ref)
def test_cputensor_mlp():
"""TODO."""
D = ng.make_axis(length=3)
H = ng.make_axis(length=2)
N = ng.make_axis(length=1)
np_x = np.array([[1, 2, 3]], dtype=np.float32)
np_w = np.array([[1, 1], [1, 1], [1, 1]], dtype=np.float32)
np_b = np.array([1, 2], dtype=np.float32)
np_c = np.dot(np_x, np_w) + np_b
x = ng.constant(np_x, [N, D])
w = ng.constant(np_w, [D, H])
b = ng.constant(np_b, [H])
wx = ng.dot(x, w)
c = wx + b
with executor(c) as ex:
result = ex()
assert np.array_equal(result, np_c)
def test_4d_reduction(transformer_factory, input_axes):
# Limiting maximum absolute value for tensors elements to 7.9.
# See description in function test_exit_condition above
is_flex = is_flex_factory(transformer_factory)
clip_val = 7.9 if is_flex else 0
x_val = rng.randn_abs_clip(input_axes, clip_max=clip_val)
x = ng.constant(x_val, input_axes)
out1 = ng.sum(x, reduction_axes=input_axes[1])
out2 = ng.sum(x, reduction_axes=input_axes[3])
with executor([out1, out2]) as ex:
graph_val1, graph_val2 = ex()
np_val1 = np.sum(x_val, 1)
np_val2 = np.sum(x_val, 3)
ng.testing.assert_allclose(graph_val1, np_val1, rtol=1e-4, atol_multiplier=x_val.shape[1])
ng.testing.assert_allclose(graph_val2, np_val2, rtol=1e-4, atol_multiplier=x_val.shape[3])
def test_normal_negative_mean():
"""TODO."""
M = ng.make_axis(100).named('M')
N = ng.make_axis(100).named('N')
mean = -0.5
std = 1.0
ng_a = [M, N]
ng_a = ng.normal(ng_a, loc=mean, scale=std)
with executor(ng_a) as ex:
result = ex()
print(np.mean(result))
print(np.std(result))
assert np.allclose(np.mean(result), mean, rtol=0.1, atol=0.02)
assert np.allclose(np.std(result), std, rtol=0.1, atol=0.02)
assert not np.all(result >= 0.0)
assert not np.all(result < 0.0)
def test_4d_elementwise(transformer_factory, input_axes):
# Limiting maximum absolute value for tensors elements to 7.9.
# See description in function test_exit_condition above
is_flex = is_flex_factory(transformer_factory)
clip_val = 7.9 if is_flex else 0
x_val = rng.randn_abs_clip(input_axes, clip_max=clip_val)
y_val = rng.randn_abs_clip(input_axes, clip_max=clip_val)
x = ng.constant(x_val, input_axes)
y = ng.constant(y_val, input_axes)
out = ng.add(x, y)
with executor(out) as ex:
graph_val = ex()
np_val = np.add(x_val, y_val)
ng.testing.assert_allclose(graph_val, np_val, rtol=1e-4)
def test_evaluation_twice():
"""Test executing a computation graph twice on a one layer MLP."""
C = ng.make_axis(length=2)
D = ng.make_axis(length=2)
W = ng.make_axis(length=1)
x = ng.constant(np.array([[1, 2], [3, 4]], dtype='float32'),
ng.make_axes([C, D]))
hidden1_weights = ng.constant(np.array([[1], [1]], dtype='float32'),
ng.make_axes([C, W]))
hidden1_biases = ng.constant(np.array([[2], [2]], dtype='float32'),
ng.make_axes([D, W]))
hidden1 = ng.dot(hidden1_weights, x) + hidden1_biases
with executor(hidden1) as comp:
result_1 = comp()
result_2 = comp()
assert np.array_equal(result_1, result_2)
def test_sum(num_units, sequence_length, batch_size):
"""
This tests for a non-deterministic error that arose in ng.sum following
a dot product using the gpu transformer.
"""
shape = (num_units, sequence_length, batch_size)
np_inp = np.random.uniform(-1, 1, shape)
# Use an identity weight matrix on top of it
np_w = np.eye(shape[0])
# Create ngraph versions
inp = ng.constant(np_inp)
reduction_axes = inp.axes[:-2]
other_axes = inp.axes[-2:]
new_axis = ng.make_axis(length=shape[0])
w_axes = ng.make_axes(new_axis) | reduction_axes
w = ng.constant(np_w, axes=w_axes)
# Reshape to do similar dot in numpy
inp_reshape = np.reshape(
np_inp, (np.prod(reduction_axes.lengths), np.prod(other_axes.lengths)))
w_reshape = np.reshape(np_w, (new_axis.length, inp_reshape.shape[0]))
# Reduce dimensions with identity weight matrix
np_x = np.dot(w_reshape, inp_reshape)
x = ng.dot(w, inp)
# Sum over all but the first axis
output_axes = ng.make_axes(x.axes[0])
y = ng.sum(x, out_axes=output_axes)
np_y = np.sum(np_x, axis=1)
with executor([y, x]) as f:
y_val, x_val = f()
assert_allclose(x_val.ravel(), np_x.ravel(), atol=1e-1)
assert_allclose(y_val, np_y, atol=1e-1)
def test_tensor_dot_tensor():
"""TODO."""
C = ng.make_axis().named('C')
D = ng.make_axis().named('D')
H = ng.make_axis().named('H')
N = ng.make_axis().named('N')
tests = [
{
'tensor1': [[1, 2], [4, 5], [3, 4]],
'tensor1_axes': (C, D),
'tensor2': [2, 5],
'tensor2_axes': (D,),
'expected_output': [12, 33, 26],
'axes_lengths': {C: 3, D: 2}
},
{
'tensor1': [[1, 4, 3], [2, 5, 4]],
'tensor1_axes': (D, C),
'tensor2': [2, 5],
'tensor2_axes': (D,),
'expected_output': [12, 33, 26],
'axes_lengths': {C: 3, D: 2}
},
{
'tensor1': [[[1, 4], [2, 5]], [[7, 12], [13, 2]]],
'tensor1_axes': (N, D, C),
'tensor2': [[[3, 6], [7, 2]], [[9, 8], [10, 4]]],
'tensor2_axes': (H, D, C),
'expected_output': [[51, 81], [188, 297]],
'axes_lengths': {N: 2, D: 2, C: 2, H: 2}
},
{
'tensor1': [1, 2],
'tensor1_axes': (C,),
'tensor2': [7, 11, 13],
'tensor2_axes': (D,),
'expected_output': [[7, 11, 13], [14, 22, 26]],
'axes_lengths': {C: 2, D: 3}
},
{
'tensor1': [[1, 4], [6, 2]],
'tensor1_axes': (C, D),
'tensor2': [[1, 4], [6, 2]],
'tensor2_axes': (C, D),
'expected_output': 57,
'axes_lengths': {C: 2, D: 2}
}
]
for test in tests:
# set up axis
for axis, length in test['axes_lengths'].items():
axis.length = length
# set up tensors
tensor1 = ng.placeholder(test['tensor1_axes'])
value1 = np.array(test['tensor1'], dtype=np.float32)
tensor2 = ng.placeholder(test['tensor2_axes'])
value2 = np.array(
test['tensor2'], dtype=np.float32
)
# compute outputs
expected_output = np.array(test['expected_output'], dtype=np.float32)
dot = ng.dot(tensor1, tensor2)
with executor(dot, tensor1, tensor2) as evaluated_fun:
# assert outputs are equal
evaluated = evaluated_fun(value1, value2)
np.testing.assert_equal(evaluated, expected_output)
def test_dimshuffle_op():
A = ng.make_axis().named('A')
B = ng.make_axis().named('B')
C = ng.make_axis().named('C')
D = ng.make_axis().named('D')
tests = [
{
'input_tensor': [
[
[
[1, 2, 3, 4],
[5, 6, 7, 8],
[9, 10, 11, 12],
],
[
[13, 14, 15, 16],
[17, 18, 19, 20],
[21, 22, 23, 24],
],
],
],
'input_tensor_axes': (A, B, C, D),
'output_tensor_axes': (B, D, A, C),
'axes_lengths': {
A: 1,
B: 2,
C: 3,
D: 4
},
'expected_result': [[
[[1, 5, 9]],
[[2, 6, 10]],
[[3, 7, 11]],
[[4, 8, 12]],
], [[[13, 17, 21]], [[14, 18, 22]], [[15, 19, 23]], [[16, 20,
24]]]]
},
{
'input_tensor': [[
[
[1, 2, 3, 4],
[5, 6, 7, 8],
[9, 10, 11, 12],
],
[
[13, 14, 15, 16],
[17, 18, 19, 20],
[21, 22, 23, 24],
],
],
[[
[25, 26, 27, 28],
[29, 30, 31, 32],
[33, 34, 35, 36],
],
[
[37, 38, 39, 40],
[41, 42, 43, 44],
[45, 46, 47, 48],
]]],
'input_tensor_axes': (A, B, C, D),
'output_tensor_axes': (B, D, A, C),
'axes_lengths': {
A: 2,
B: 2,
C: 3,
D: 4
},
'expected_result': [[[
[1, 5, 9],
[25, 29, 33],
], [
[2, 6, 10],
[26, 30, 34],
], [
[3, 7, 11],
[27, 31, 35],
], [
[4, 8, 12],
[28, 32, 36],
]],
[[
[13, 17, 21],
[37, 41, 45],
], [
[14, 18, 22],
[38, 42, 46],
], [[15, 19, 23], [39, 43, 47]],
[
[16, 20, 24],
[40, 44, 48],
]]]
},
]
for test in tests:
for axis, length in test['axes_lengths'].items():
axis.length = length
input_tensor = ng.placeholder(test['input_tensor_axes'])
input_tensor_value = np.array(test['input_tensor'], dtype=np.float32)
# This list of operations should add a dimshuffle operation to the graph.
a = ng.negative(input_tensor)
b = ng.axes_with_order(a, test['output_tensor_axes'])
c = ng.negative(b)
with executor(c, input_tensor) as ex:
out = ex(input_tensor_value)
ng.testing.assert_allclose(out, test['expected_result'])
