def compare_optimizer_variable_select(opt_ng, opt_ref):
# Set up data placeholders
C = ng.make_axis(20)
N = ng.make_axis(32, name='N')
data = ng.placeholder([C, N])
target = ng.placeholder([N])
# params to be updated using optimizer to be tested
np_W1 = np.random.rand(C.length)
np_W2 = np.random.rand(C.length)
W1 = ng.variable([C], initial_value=np_W1)
W2 = ng.variable([C], initial_value=np_W2)
# Set up op graph
cost = ng.sum(target - ng.dot(W1, data) - ng.dot(W2, data), out_axis=())
updated_weights = ng.sequential([opt_ng(cost, variables=[W1]), W1])
# Set up the computation and run the "train" loop
with ExecutorFactory() as ex:
opt_ng_comp = ex.transformer.computation([updated_weights, W2], data, target)
mock_dataset = data_generator(20, C.length, N.length)
for x, y in mock_dataset:
[ng_W1, ng_W2] = opt_ng_comp(x, y) # updated weights for ngraph optimizer
np_W1 = opt_ref(x, np_W1) # updated weights for reference optimizer
ng.testing.assert_allclose(np_W1, ng_W1, rtol=1e-3)
ng.testing.assert_allclose(np_W2, ng_W2, rtol=1e-3)
def test_flat_tensor_dot_tensor():
"""
Ensure that a flattened argument axis is not unflattend in the result.
"""
H = ng.make_axis(2)
W = ng.make_axis(7)
C = ng.make_axis(3)
K = ng.make_axis(11)
axes_a = ng.make_axes([H, W, C])
a = ng.constant(np.ones(axes_a.lengths), axes=axes_a)
flat_a = ng.flatten_at(a, 2)
axes_b = ng.make_axes([C, K])
b = ng.constant(np.ones(axes_b.lengths), axes=axes_b)
result = ng.dot(b, flat_a)
with ExecutorFactory() as factory:
result_fun = factory.executor(result)
result_val = result_fun()
result_correct = np.ones_like(result_val) * C.length
ng.testing.assert_allclose(result_val, result_correct)
def test_weight_clipping(w_clip, optimizer):
opt_ng = optimizer(0.1, weight_clip_value=w_clip)
# Set up data placeholders
C = ng.make_axis(20)
N = ng.make_axis(32, name='N')
data = ng.placeholder([C, N])
target = ng.placeholder([N])
# params to be updated using optimizer to be tested
# make sure initial values are higher than clip values
np_W = 10 * w_clip * (2 * np.random.rand(C.length) - 1)
W = ng.variable([C], initial_value=np_W)
# double check generated initial W value
assert np.max(np_W) > w_clip
assert np.min(np_W) < -w_clip
# Set up op graph
cost = ng.sum(target - ng.dot(W, data), out_axis=())
updated_weights = ng.sequential([opt_ng(cost), W])
epsilon = w_clip * 1e-3
# Set up the computation and run the "train" loop
with ExecutorFactory() as ex:
opt_ng_comp = ex.transformer.computation(updated_weights, data, target)
mock_dataset = data_generator(20, C.length, N.length)
for x, y in mock_dataset:
ng_W = opt_ng_comp(x, y) # updated weights for ngraph optimizer
assert np.max(ng_W) < w_clip + epsilon
assert np.min(ng_W) > -w_clip - epsilon
def test_flatten_deriv_simplified():
"""
Test derivative with dot and flatten
"""
ax_N = ng.make_axis(length=3)
ax_Y = ng.make_axis(length=2)
x = ng.placeholder(ng.make_axes([ax_N]))
w = ng.constant([5, 2], axes=ng.make_axes([ax_Y]))
logits = ng.dot(x, w)
cost = ng.sum(logits, reduction_axes=logits.axes)
delta = 0.001
u = rng.uniform(.1, 5.0, x.axes)
check_derivative(cost, x, delta, u, atol=1e-2, rtol=1e-2)
def test_cputensor_dot():
Y = ng.make_axis(length=2)
M = ng.make_axis(length=1)
N = ng.make_axis(length=3)
np_a = np.array([[1, 2, 3]], dtype=np.float32)
np_b = np.array([[1, 2], [2, 3], [3, 4]], dtype=np.float32)
np_c = np.dot(np_a, np_b)
a = ng.constant(np_a, [M, N]).named('a')
b = ng.constant(np_b, [N, Y]).named('b')
c = ng.dot(a, b)
with executor(c) as ex:
result = ex()
assert np.array_equal(result, np_c)
def test_dot():
H = ng.make_axis(length=1)
W = ng.make_axis(length=4)
np_a = np.array([[1, 2, 3, 4]], dtype=np.float32)
np_b = np.array(3, dtype=np.float32)
a = ng.constant(np_a, [H, W])
b = ng.constant(np_b, [])
c = ng.dot(a, b)
with executor(c) as _dot:
_dot_val = _dot()
# compute reference
_dot_val_ref = np.dot(np_a, np_b)
# this checks the dot product between scalar and vector, this is equivalent to
# elementwise multiplication between scalar and vector
assert np.allclose(_dot_val, _dot_val_ref)
def test_cputensor_mlp():
"""TODO."""
D = ng.make_axis(length=3)
H = ng.make_axis(length=2)
N = ng.make_axis(length=1)
np_x = np.array([[1, 2, 3]], dtype=np.float32)
np_w = np.array([[1, 1], [1, 1], [1, 1]], dtype=np.float32)
np_b = np.array([1, 2], dtype=np.float32)
np_c = np.dot(np_x, np_w) + np_b
x = ng.constant(np_x, [N, D])
w = ng.constant(np_w, [D, H])
b = ng.constant(np_b, [H])
wx = ng.dot(x, w)
c = wx + b
with executor(c) as ex:
result = ex()
assert np.array_equal(result, np_c)
def test_logreg():
# xs: (C, N), y: (N,)
xs = np.array([[0.52, 0.88, 0.52, 0.74], [1.12, -1.08, 0.06, -2.49],
[0.77, 0.15, -1.3, 1.39]])
ys = np.array([1, 1, 0, 1])
max_iter = 10
alpha = 0.1
thetas = np.array([0., 0., 0.])
np_logreg = NumpyLogreg(xs, ys, thetas)
C, N = ng.make_axis(length=3), ng.make_axis(length=4)
# input tensors
xs_v = ng.placeholder((C, N))
ys_v = ng.placeholder([N])
alpha_v = ng.placeholder(())
thetas_var = ng.variable([C], initial_value=thetas)
# define ops
ys_pred = ng.sigmoid(ng.dot(thetas_var, xs_v))
log_likelihoods = ng.log(ys_pred) * ys_v + ng.log(1 - ys_pred) * (1 - ys_v)
loss = -ng.sum(log_likelihoods, reduction_axes=[N])
grad_comp = ng.deriv(loss, thetas_var)
weight_update = ng.sequential(
[ng.assign(thetas_var, thetas_var - alpha_v * grad_comp), thetas_var])
# transformer
with ExecutorFactory() as ex:
train_eval_func = ex.executor([grad_comp, loss, weight_update], xs_v,
ys_v, alpha_v)
# evaluate
for i in range(max_iter):
grad_np, loss_np, thetas_np = np_logreg.optimize(alpha)
grad_ng, loss_ng, thetas_ng = train_eval_func(xs, ys, alpha)
ng.testing.assert_allclose(loss_np, loss_ng, rtol=1e-05, atol=1e-05, \
transformer_overwrite=False)
ng.testing.assert_allclose(grad_np, grad_ng, rtol=1e-05, atol=1e-05, \
transformer_overwrite=False)
ng.testing.assert_allclose(thetas_np, thetas_ng, rtol=1e-05, atol=1e-05, \
transformer_overwrite=False)
def test_evaluation_twice():
"""Test executing a computation graph twice on a one layer MLP."""
C = ng.make_axis(length=2)
D = ng.make_axis(length=2)
W = ng.make_axis(length=1)
x = ng.constant(np.array([[1, 2], [3, 4]], dtype='float32'),
ng.make_axes([C, D]))
hidden1_weights = ng.constant(np.array([[1], [1]], dtype='float32'),
ng.make_axes([C, W]))
hidden1_biases = ng.constant(np.array([[2], [2]], dtype='float32'),
ng.make_axes([D, W]))
hidden1 = ng.dot(hidden1_weights, x) + hidden1_biases
with executor(hidden1) as comp:
result_1 = comp()
result_2 = comp()
assert np.array_equal(result_1, result_2)
def test_sum(num_units, sequence_length, batch_size):
"""
This tests for a non-deterministic error that arose in ng.sum following
a dot product using the gpu transformer.
"""
shape = (num_units, sequence_length, batch_size)
np_inp = np.random.uniform(-1, 1, shape)
# Use an identity weight matrix on top of it
np_w = np.eye(shape[0])
# Create ngraph versions
inp = ng.constant(np_inp)
reduction_axes = inp.axes[:-2]
other_axes = inp.axes[-2:]
new_axis = ng.make_axis(length=shape[0])
w_axes = ng.make_axes(new_axis) | reduction_axes
w = ng.constant(np_w, axes=w_axes)
# Reshape to do similar dot in numpy
inp_reshape = np.reshape(
np_inp, (np.prod(reduction_axes.lengths), np.prod(other_axes.lengths)))
w_reshape = np.reshape(np_w, (new_axis.length, inp_reshape.shape[0]))
# Reduce dimensions with identity weight matrix
np_x = np.dot(w_reshape, inp_reshape)
x = ng.dot(w, inp)
# Sum over all but the first axis
output_axes = ng.make_axes(x.axes[0])
y = ng.sum(x, out_axes=output_axes)
np_y = np.sum(np_x, axis=1)
with executor([y, x]) as f:
y_val, x_val = f()
assert_allclose(x_val.ravel(), np_x.ravel(), atol=1e-1)
assert_allclose(y_val, np_y, atol=1e-1)
def test_tensor_dot_tensor():
"""TODO."""
C = ng.make_axis().named('C')
D = ng.make_axis().named('D')
H = ng.make_axis().named('H')
N = ng.make_axis().named('N')
tests = [
{
'tensor1': [[1, 2], [4, 5], [3, 4]],
'tensor1_axes': (C, D),
'tensor2': [2, 5],
'tensor2_axes': (D,),
'expected_output': [12, 33, 26],
'axes_lengths': {C: 3, D: 2}
},
{
'tensor1': [[1, 4, 3], [2, 5, 4]],
'tensor1_axes': (D, C),
'tensor2': [2, 5],
'tensor2_axes': (D,),
'expected_output': [12, 33, 26],
'axes_lengths': {C: 3, D: 2}
},
{
'tensor1': [[[1, 4], [2, 5]], [[7, 12], [13, 2]]],
'tensor1_axes': (N, D, C),
'tensor2': [[[3, 6], [7, 2]], [[9, 8], [10, 4]]],
'tensor2_axes': (H, D, C),
'expected_output': [[51, 81], [188, 297]],
'axes_lengths': {N: 2, D: 2, C: 2, H: 2}
},
{
'tensor1': [1, 2],
'tensor1_axes': (C,),
'tensor2': [7, 11, 13],
'tensor2_axes': (D,),
'expected_output': [[7, 11, 13], [14, 22, 26]],
'axes_lengths': {C: 2, D: 3}
},
{
'tensor1': [[1, 4], [6, 2]],
'tensor1_axes': (C, D),
'tensor2': [[1, 4], [6, 2]],
'tensor2_axes': (C, D),
'expected_output': 57,
'axes_lengths': {C: 2, D: 2}
}
]
for test in tests:
# set up axis
for axis, length in test['axes_lengths'].items():
axis.length = length
# set up tensors
tensor1 = ng.placeholder(test['tensor1_axes'])
value1 = np.array(test['tensor1'], dtype=np.float32)
tensor2 = ng.placeholder(test['tensor2_axes'])
value2 = np.array(
test['tensor2'], dtype=np.float32
)
# compute outputs
expected_output = np.array(test['expected_output'], dtype=np.float32)
dot = ng.dot(tensor1, tensor2)
with executor(dot, tensor1, tensor2) as evaluated_fun:
# assert outputs are equal
evaluated = evaluated_fun(value1, value2)
np.testing.assert_equal(evaluated, expected_output)
def test_tensor_dot_tensor():
"""TODO."""
C = ng.make_axis().named('C')
D = ng.make_axis().named('D')
H = ng.make_axis().named('H')
N = ng.make_axis().named('N')
tests = [
{
'tensor1': [[1, 2], [4, 5], [3, 4]],
'tensor1_axes': (C, D),
'tensor2': [2, 5],
'tensor2_axes': (D,),
'expected_output': [12, 33, 26],
'axes_lengths': {C: 3, D: 2}
},
{
'tensor1': [[1, 4, 3], [2, 5, 4]],
'tensor1_axes': (D, C),
'tensor2': [2, 5],
'tensor2_axes': (D,),
'expected_output': [12, 33, 26],
'axes_lengths': {C: 3, D: 2}
},
{
'tensor1': [[[1, 4], [2, 5]], [[7, 12], [13, 2]]],
'tensor1_axes': (N, D, C),
'tensor2': [[[3, 6], [7, 2]], [[9, 8], [10, 4]]],
'tensor2_axes': (H, D, C),
'expected_output': [[51, 81], [188, 297]],
'axes_lengths': {N: 2, D: 2, C: 2, H: 2}
},
{
'tensor1': [1, 2],
'tensor1_axes': (C,),
'tensor2': [7, 11, 13],
'tensor2_axes': (D,),
'expected_output': [[7, 11, 13], [14, 22, 26]],
'axes_lengths': {C: 2, D: 3}
},
{
'tensor1': [[1, 4], [6, 2]],
'tensor1_axes': (C, D),
'tensor2': [[1, 4], [6, 2]],
'tensor2_axes': (C, D),
'expected_output': 57,
'axes_lengths': {C: 2, D: 2}
}
]
for test in tests:
# set up axis
for axis, length in test['axes_lengths'].items():
axis.length = length
# set up tensors
tensor1 = ng.placeholder(test['tensor1_axes'])
value1 = np.array(test['tensor1'], dtype=np.float32)
tensor2 = ng.placeholder(test['tensor2_axes'])
value2 = np.array(
test['tensor2'], dtype=np.float32
)
# compute outputs
expected_output = np.array(test['expected_output'], dtype=np.float32)
with ExecutorFactory() as ex:
dot = ng.dot(tensor1, tensor2)
evaluated_fun = ex.executor(dot, tensor1, tensor2)
deriv1_fun_num = ex.numeric_derivative(dot, tensor1, 1e-3, tensor2)
deriv1_fun_sym = ex.derivative(dot, tensor1, tensor2)
deriv2_fun_num = ex.numeric_derivative(dot, tensor2, 1e-3, tensor1)
deriv2_fun_sym = ex.derivative(dot, tensor2, tensor1)
# assert outputs are equal
evaluated = evaluated_fun(value1, value2)
np.testing.assert_equal(evaluated, expected_output)
# assert derivative wrt to both tensors is the same when computed
# symbolically by ngraph and numerically
deriv1_val_sym = deriv1_fun_sym(value1, value2)
deriv2_val_sym = deriv2_fun_sym(value2, value1)
deriv1_val_num = deriv1_fun_num(value1, value2)
deriv2_val_num = deriv2_fun_num(value2, value1)
ng.testing.assert_allclose(deriv1_val_num, deriv1_val_sym, rtol=1e-2, atol=1e-2)
ng.testing.assert_allclose(deriv2_val_num, deriv2_val_sym, rtol=1e-2, atol=1e-2)
def test_dot_sum_backprop():
delta = 1e-3
rtol = atol = 1e-2
C = ng.make_axis(length=2).named('C')
N = ng.make_axis(length=3, name='N')
x_axes = ng.make_axes((C, N))
y_axes = ng.make_axes((C,))
x_np = np.random.random(x_axes.lengths).astype('float32')
y_np = np.random.random(y_axes.lengths).astype('float32')
x_np[...] = [[1.0, 0.0, 1.0], [2.0, 0.0, 3.0]]
y_np[...] = [-1.0, 1.0]
x = ng.placeholder(x_axes).named('x')
y = ng.placeholder(y_axes).named('y')
d = ng.dot(x, y)
s = ng.sum(d, out_axes=())
with ExecutorFactory() as ex:
s_fun = ex.executor(s, x, y)
d_fun = ex.executor(d, x, y)
dd_dx_fun_num = ex.numeric_derivative(d, x, delta, y)
dd_dx_fun_sym = ex.derivative(d, x, y)
dd_dy_fun_num = ex.numeric_derivative(d, y, delta, x)
dd_dy_fun_sym = ex.derivative(d, y, x)
ds_dx_fun_num = ex.numeric_derivative(s, x, delta, y)
ds_dx_fun_sym = ex.derivative(s, x, y)
ds_dy_fun_num = ex.numeric_derivative(s, y, delta, x)
ds_dy_fun_sym = ex.derivative(s, y, x)
# assert outputs are equal
d_np = x_np.T.dot(y_np)
d_val = d_fun(x_np, y_np)
ng.testing.assert_allclose(d_np, d_val, rtol=rtol, atol=atol)
s_np = np.sum(d_np)
s_val = s_fun(x_np, y_np)
ng.testing.assert_allclose(s_val, s_np, rtol=rtol, atol=atol)
# assert derivative wrt to both tensors is the same when computed
# symbolically by ngraph and numerically
dd_dx_val_sym = dd_dx_fun_sym(x_np, y_np)
dd_dy_val_sym = dd_dy_fun_sym(y_np, x_np)
ds_dx_val_sym = ds_dx_fun_sym(x_np, y_np)
ds_dy_val_sym = ds_dy_fun_sym(y_np, x_np)
dd_dx_val_num = dd_dx_fun_num(x_np, y_np)
dd_dy_val_num = dd_dy_fun_num(y_np, x_np)
ds_dx_val_num = ds_dx_fun_num(x_np, y_np)
ds_dy_val_num = ds_dy_fun_num(y_np, x_np)
ng.testing.assert_allclose(dd_dx_val_num, dd_dx_val_sym, rtol=rtol, atol=atol)
ng.testing.assert_allclose(dd_dy_val_num, dd_dy_val_sym, rtol=rtol, atol=atol)
ng.testing.assert_allclose(ds_dx_val_num, ds_dx_val_sym, rtol=rtol, atol=atol)
ng.testing.assert_allclose(ds_dy_val_num, ds_dy_val_sym, rtol=rtol, atol=atol)
