def test_lut(lut_args):
"""
test lut fprop and bprop
"""
pad_idx = 0
with ExecutorFactory() as ex:
vocab_size, embed_dim, bsz, seq_len, mem_size = lut_args
V = ng.make_axis(vocab_size)
F = ng.make_axis(embed_dim)
M = ng.make_axis(mem_size)
ax.N.length = bsz
ax.REC.length = seq_len
# Multi-axis input to LUT
ax_idx = ng.make_axes([M, ax.REC, ax.N])
ax_lut = ng.make_axes([V, F])
lut = ng.placeholder(ax_lut)
idx = ng.placeholder(ax_idx)
idx_flat = ng.flatten(idx)
ax_out = idx_flat.axes | ng.make_axes([F])
# fprop
lut_out_ng = ng.lookuptable(lut, idx_flat, ax_out, pad_idx=pad_idx)
fprop_fun = ex.executor(lut_out_ng, lut, idx)
# bprop
update_error = ng.placeholder(ax_out)
update_out_ng = lookuptable_update(update_error, lut, idx, lut_out_ng)
update_fun = ex.executor(update_out_ng, update_error, lut, idx)
# provide actual inputs and execute the graph
lut_value = rng.uniform(-1, 1, lut.axes)
idx_value = rng.random_integers(0, vocab_size - 1, idx.axes)
fprop_lut = fprop_fun(lut_value, idx_value).copy()
# compare fprop
fprop_ref = lut_fprop_ref(lut_value, idx_value)
ng.testing.assert_allclose(fprop_lut, fprop_ref, rtol=0.0, atol=1.0e-5)
# provide actual delta and execute the update op
update_value = rng.uniform(-1, 1, update_error.axes)
update_lut = update_fun(update_value, lut_value, idx_value).copy()
# compare bprop (udpate)
update_ref = lut_update_ref(
update_value,
lut_value,
idx_value,
pad_idx=pad_idx)
ng.testing.assert_allclose(
update_lut, update_ref, rtol=0.0, atol=1.0e-5)
def make_weights(
input_placeholder,
hidden_size,
weight_initializer,
bias_initializer,
init_state=False):
gates = ['i', 'f', 'o', 'g']
# input axis + any extra axes of length 1
in_feature_axes = tuple(input_placeholder.axes)[:-2]
out_feature_axes = ng.make_axes([ng.make_axis(hidden_size)])
batch_axis = input_placeholder.axes.batch_axis()
hidden_axis = ng.make_axis(hidden_size)
w_in_axes = ng.make_axes(hidden_axis) + in_feature_axes
w_rec_axes = ng.make_axes(hidden_axis) + out_feature_axes
W_in = {gate: weight_initializer(w_in_axes) for gate in gates}
W_rec = {gate: weight_initializer(w_rec_axes) for gate in gates}
b = {gate: bias_initializer(hidden_axis) for gate in gates}
if init_state is True:
ax_s = ng.make_axes([hidden_axis, batch_axis])
init_state = {name: ng.placeholder(ax_s) for name in ['h', 'c']}
init_state_value = {
name: rng.uniform(-1, 1, ax_s) for name in ['h', 'c']}
else:
init_state = None
init_state_value = None
return W_in, W_rec, b, init_state, init_state_value
def test_distributed_graph_plus_two(transformer_factory):
H = ng.make_axis(length=4, name='height')
W = ng.make_axis(length=6, name='width')
x = ng.placeholder(axes=[H, W])
with ng.metadata(device_id=('1', '2'), parallel=W):
x_plus_one = x + 1
x_plus_two = x_plus_one + 1
np_x = np.random.randint(100, size=[H.length, W.length])
with ExecutorFactory() as ex:
computation = ex.executor(x_plus_two, x)
res = computation(np_x)
np.testing.assert_array_equal(res, np_x + 2)
def test_init_gaussian():
N = 128
C = 4
# XS, YS = g.gen_data(N, 10)
X = ng.placeholder(axes=ng.Axes([C, N]))
Y = ng.placeholder(axes=ng.Axes([N]))
alpha = ng.placeholder(axes=ng.Axes())
# W = ng.Variable(axes=ng.Axes([C]), initial_value=ng.fill(sequential))
W = ng.Variable(axes=ng.Axes([C]), initial_value=10)
L = W + 1 + alpha
transformer = ng.NumPyTransformer()
update_fun = transformer.computation([L, W], alpha, X, Y)
xs = np.zeros((C, N), dtype=np.float32)
ys = np.zeros((N, ), dtype=np.float32)
loss_val, w_val = update_fun(5.0, xs, ys)
print(loss_val)
def test_tensor_slice():
"""
slicing a tensor should work like numpy
"""
input_axes = ng.make_axes(
[ng.make_axis(10), ng.make_axis(20),
ng.make_axis(5)])
x = ng.placeholder(axes=input_axes)
assert x[:5].axes.full_lengths == (5, 20, 5)
assert x[:, 2:7].axes.full_lengths == (10, 5, 5)
assert x[:5, :, :-1].axes.full_lengths == (5, 20, 4)
def test_cross_enropy_rec(transformer_factory):
N = ng.make_axis(name='N', batch=True)
W = ng.make_axis(name='W')
T = ng.make_axis(name='T', recurrent=True)
W.length = 3
T.length = 4
N.length = 10
axes = ng.make_axes([W, T, N])
p_x = ng.placeholder(axes)
p_t = ng.placeholder(axes)
cross_entropy_sm_x_t = ng.cross_entropy_multi(ng.softmax(p_x), p_t)
x = rng.uniform(0, 1, axes)
t = np_softmax(rng.uniform(0, 1, axes), 0)
def f_np(x, t):
return np_cross_entropy_multi(np_softmax(x, 0), t, axis=0)
compare_f_at_x(cross_entropy_sm_x_t, [p_x, p_t], f_np, [x, t], rtol=1e-5)
def test_softmax_rec_deriv(transformer_factory):
N = ng.make_axis(name='N', batch=True)
W = ng.make_axis(name='W')
T = ng.make_axis(name='T', recurrent=True)
W.length = 3
T.length = 4
N.length = 10
axes = ng.make_axes([W, T, N])
x = rng.uniform(0, 1, axes)
p_x = ng.placeholder(axes)
check_derivative(ng.softmax(p_x), p_x, 0.001, x, atol=1e-2, rtol=1e-2)
def test_softmax_rec(transformer_factory):
N = ng.make_axis(name='N', batch=True)
W = ng.make_axis(name='W')
T = ng.make_axis(name='T', recurrent=True)
W.length = 3
T.length = 4
N.length = 10
axes = ng.make_axes([W, T, N])
x = rng.uniform(0, 1, axes)
p_x = ng.placeholder(axes)
compare_f_at_x(ng.softmax(p_x), p_x, lambda x: np_softmax(x, 0), x, rtol=1e-5)
def test_cross_entropy_softmax_deriv(transformer_factory):
N = ng.make_axis(name='N', batch=True)
W = ng.make_axis(name='W')
W.length = 3
N.length = 10
axes = ng.make_axes([W, N])
p_x = ng.placeholder(axes)
p_t = ng.placeholder(axes)
x = rng.uniform(0, 1, axes)
t = np_softmax(rng.uniform(0, 1, axes), 0)
check_derivative(ng.cross_entropy_multi(ng.softmax(p_x), p_t),
p_x,
0.001,
x,
parameters=[p_t],
parameter_values=[t],
atol=1e-2,
rtol=1e-2)
def test_convolution_backprop(transformer_factory, n128_hw32_c3_2x2):
"""
test convolution backprop path
"""
cf = ConvParams(**n128_hw32_c3_2x2)
inputs = ng.placeholder(axes=cf.ax_i)
filters = ng.placeholder(axes=cf.ax_f)
# randomly initialize
input_value = rng.uniform(-1, 1, cf.ax_i)
filter_value = rng.uniform(-1, 1, cf.ax_f)
output = ng.sum(ng.convolution(cf.conv_params, inputs, filters, cf.ax_o),
out_axes=())
with ExecutorFactory() as factory:
dcdf_sym_fun = factory.derivative(output, filters, inputs)
dcdf_num_fun = factory.numeric_derivative(output, filters, .01, inputs)
dcdf_sym_val = dcdf_sym_fun(filter_value, input_value)
dcdf_num_val = dcdf_num_fun(filter_value, input_value)
ng.testing.assert_allclose(dcdf_sym_val, dcdf_num_val, rtol=0.01)
def test_logreg(transformer_factory):
# xs: (C, N), y: (N,)
xs = np.array([[0.52, 0.88, 0.52, 0.74], [1.12, -1.08, 0.06, -2.49],
[0.77, 0.15, -1.3, 1.39]])
ys = np.array([1, 1, 0, 1])
max_iter = 10
alpha = 0.1
thetas = np.array([0., 0., 0.])
np_logreg = NumpyLogreg(xs, ys, thetas)
C, N = ng.make_axis(length=3), ng.make_axis(length=4)
# input tensors
xs_v = ng.placeholder((C, N))
ys_v = ng.placeholder([N])
alpha_v = ng.placeholder(())
thetas_var = ng.variable([C], initial_value=thetas)
# define ops
ys_pred = ng.sigmoid(ng.dot(thetas_var, xs_v))
log_likelihoods = ng.log(ys_pred) * ys_v + ng.log(1 - ys_pred) * (1 - ys_v)
loss = -ng.sum(log_likelihoods, reduction_axes=[N])
grad_comp = ng.deriv(loss, thetas_var)
weight_update = ng.sequential(
[ng.assign(thetas_var, thetas_var - alpha_v * grad_comp), thetas_var])
# transformer
with ExecutorFactory() as ex:
train_eval_func = ex.executor([grad_comp, loss, weight_update], xs_v,
ys_v, alpha_v)
# evaluate
for i in range(max_iter):
grad_np, loss_np, thetas_np = np_logreg.optimize(alpha)
grad_ng, loss_ng, thetas_ng = train_eval_func(xs, ys, alpha)
ng.testing.assert_allclose(loss_np, loss_ng)
ng.testing.assert_allclose(grad_np, grad_ng)
ng.testing.assert_allclose(thetas_np, thetas_ng)
def test_fill_slice(transformer_factory):
axes = ng.make_axes([ng.make_axis(length=2), ng.make_axis(length=8)])
a = ng.placeholder(axes=axes)
b = ng.sequential([ng.fill(a[:, 1], 0), ng.value_of(a)])
with ExecutorFactory() as ex:
func = ex.executor(b, a)
baseline = func(
np.array([[1, 2, 3, 4, 5, 6, 7, 8], [8, 7, 6, 5, 4, 3, 2, 1]],
dtype=np.float32))
expected = np.array([[1, 0, 3, 4, 5, 6, 7, 8],
[8, 0, 6, 5, 4, 3, 2, 1]])
ng.testing.assert_allclose(baseline, expected)
def test_cross_entropy_rec(transformer_factory, recurrent_input_tensor):
p_x = recurrent_input_tensor
p_t = ng.placeholder(p_x.axes)
cross_entropy_sm_x_t = ng.cross_entropy_multi(ng.softmax(p_x), p_t)
x = rng.uniform(0, 1, p_x.axes)
t = np_softmax(rng.uniform(0, 1, p_t.axes), 0)
def f_np(x, t):
return np_cross_entropy_multi(np_softmax(x, 0), t, axis=0)
compare_f_at_x(cross_entropy_sm_x_t, [p_x, p_t], f_np, [x, t], rtol=1e-5)
def test_conv(transformer_factory, n64_hw32_c32_3x3):
cf = ConvParams(**n64_hw32_c32_3x3)
inputs = ng.placeholder(axes=cf.ax_i)
filters = ng.placeholder(axes=cf.ax_f)
# randomly initialize
input_value = rng.uniform(-0.5, 0.5, cf.ax_i)
filter_value = rng.uniform(-0.5, 0.5, cf.ax_f)
error_value = rng.uniform(-0.5, 0.5, cf.ax_o)
inputs = ng.placeholder(cf.ax_i)
filters = ng.placeholder(cf.ax_f)
errors = ng.placeholder(cf.ax_o)
output = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o)
bprop_out = bprop_conv(errors, inputs, filters, output)
updat_out = update_conv(errors, inputs, filters, output)
with executor([output, bprop_out, updat_out], inputs, filters,
errors) as conv_executor:
result_ng, gradI_ng, gradF_ng = conv_executor(input_value,
filter_value,
error_value)
# Compute reference with NumPy
result_np, gradI_np, gradF_np = reference_conv(cf.dimI, cf.dimF, cf.dimO,
cf.conv_params, input_value,
filter_value, error_value)
# Compare fprop
assert np.allclose(result_ng, result_np, rtol=0, atol=0.5)
# Compare bprop
assert np.allclose(gradI_ng, gradI_np, rtol=0, atol=0.5)
# Compare update
assert np.allclose(gradF_ng, gradF_np, rtol=0, atol=2)
def test_cross_entropy_softmax_rec_deriv(transformer_factory, recurrent_input_tensor):
p_x = recurrent_input_tensor
p_t = ng.placeholder(p_x.axes)
x = rng.uniform(0, 1, p_x.axes)
t = np_softmax(rng.uniform(0, 1, p_t.axes), 0)
check_derivative(
ng.cross_entropy_multi(ng.softmax(p_x), p_t),
p_x, 0.001, x,
parameters=[p_t],
parameter_values=[t],
atol=1e-2, rtol=1e-2
)
def test_cross_entropy_binary_logistic_shortcut(input_tensor):
"""TODO."""
p_u = input_tensor
p_v = ng.placeholder(p_u.axes)
u = rng.uniform(-3.0, 3.0, p_u.axes)
v = np_softmax(rng.uniform(-3.0, 3.0, p_u.axes), 0)
cel = cross_entropy_binary_logistic(u, v)
cel_shortcut = cross_entropy_binary_logistic_shortcut(u, v)
ng.testing.assert_allclose(cel, cel_shortcut, rtol=1e-5)
with executor(ng.cross_entropy_binary_inner(ng.sigmoid(p_u), p_v), p_u, p_v) as ex:
cel_graph = ex(u, v)
ng.testing.assert_allclose(cel, cel_graph, rtol=1e-5)
def test_assign(transformer_factory, operands, test_name):
v = ng.variable(())
ng_placeholder = ng.placeholder(())
vset = ng.sequential([ng.assign(v, ng_placeholder), v])
iterations = len(operands) != 1
with executor(vset, ng_placeholder) as ex:
for i in operands:
flex_result = ex(i[0])
print("flex: ", flex_result)
print("expected: ", i[1])
if iterations:
assert_allclose(flex_result, i[1])
else:
assert flex_result == i[1]
def test_wrong_op_name():
"""
test wrong number of batch axes at input
"""
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N])
inputs = ng.placeholder(axes=ax_i)
pooltype = 'min'
pool_params = dict(op=pooltype)
with pytest.raises(ValueError) as exinfo:
ng.pooling(pool_params, inputs, {})
assert str(exinfo.value) == "Unsupported pooling type: {pooltype}. Only max and avg " \
"pooling currently supported. ".format(pooltype=pooltype)
def test_first_axes_not_same():
"""
test first axes are not the same
"""
padding = dict(pad_d=0, pad_h=0, pad_w=0)
strides = dict(str_d=1, str_h=1, str_w=1)
dilation = dict(dil_d=1, dil_h=1, dil_w=1)
conv_params = padding.copy()
conv_params.update(strides)
conv_params.update(dilation)
ax_i = ng.make_axes([ax.D, ax.C, ax.H, ax.W, ax.N])
ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K])
inputs = ng.placeholder(ax_i)
filters = ng.placeholder(ax_f)
with pytest.raises(ValueError) as exinfo:
ng.convolution(conv_params, inputs, filters, {})
assert str(exinfo.value) == 'the first axis in input {inputs} and filter {filters} ' \
'are not the same.'.format(
inputs=inputs.axes[0],
filters=filters.axes[0])
def make_placeholders(self):
batch_axis = ng.make_axis(length=self.batch_size, name="N")
time_axis = ng.make_axis(length=self.time_steps, name="REC")
feature_axis = ng.make_axis(length=self.nfeatures, name="feature_axis")
dict = {}
for k in self.data_arrays.keys():
if k == 'inp_txt' or k == 'teacher_tgt':
p_axes = ng.make_axes([batch_axis, time_axis, feature_axis])
else:
p_axes = ng.make_axes([batch_axis, time_axis])
dict[k] = ng.placeholder(p_axes)
return dict
def test_wrong_input_shape_length():
"""
test wrong input shape length
"""
pf = PoolParams()
ax_i = pf.ax_i[:-1]
inputs = ng.placeholder(axes=ax_i)
with pytest.raises(ValueError) as exinfo:
ng.pooling(pf.pool_params, inputs, {})
assert str(exinfo.value) == 'pooling input shape must be length 5, found {}' \
.format(len(ax_i))
def test_tensor_size(transformer_factory):
n, m = 3, 4
N = ng.make_axis(length=n)
M = ng.make_axis(length=m)
aaxes = ng.make_axes([N, M])
x = ng.placeholder(aaxes)
size_fun = ng.tensor_size(x)
nptensor = np.arange(n * m).reshape(n, m)
with executor(size_fun, x) as ex:
assert ex(nptensor) == n * m
def make_placeholder(input_size, sequence_length, batch_size, extra_axes=0):
input_axis = ng.make_axis()
recurrent_axis = ng.make_axis(name='R')
batch_axis = ng.make_axis(name='N')
input_axes = ng.make_axes([input_axis, recurrent_axis, batch_axis])
input_axes.set_shape((input_size, sequence_length, batch_size))
input_axes = ng.make_axes([ng.make_axis(length=1) for _ in range(extra_axes)]) + input_axes
input_placeholder = ng.placeholder(input_axes)
input_value = rng.uniform(-0.01, 0.01, input_axes)
return input_placeholder, input_value
def get_placeholder_from_operand(operand, axes=None):
if not isinstance(axes, ng.Axes):
if not isinstance(operand, np.ndarray):
axes = ()
else:
if len(operand.shape) > 1:
rows, columns = operand.shape
N = ng.make_axis(length=rows)
M = ng.make_axis(length=columns)
axes = ng.make_axes([N, M])
else:
O = ng.make_axis(length=operand.size)
axes = ng.make_axes([O])
return ng.placeholder(axes), axes
def test_update_comm_deps_scatter_gather():
ax_a = ng.make_axis(length=10, name='A')
ax_b = ng.make_axis(length=15, name='B')
axes = ng.make_axes([ax_a, ax_b])
parallel_metadata = dict(parallel=ax_a, device_id=(0, 1),
transformer=None, host_transformer=None, device=None)
with ng.metadata(transformer='cpu0'):
with ng.metadata(**parallel_metadata):
from_node_a = ng.placeholder(axes)
to_node_a = ng.placeholder(axes)
scatter_send_x = ScatterSendOp(from_node=from_node_a, to_node=to_node_a)
scatter_recv_a = ScatterRecvOp(to_node=to_node_a, send_node=scatter_send_x)
with ng.metadata(**parallel_metadata):
x_plus_one_a = scatter_recv_a + 1
gather_send_x_plus_one_a = GatherSendOp(from_node=x_plus_one_a)
with ng.metadata(transformer='cpu1'):
with ng.metadata(**parallel_metadata):
to_node_b = ng.placeholder(axes)
scatter_recv_b = ScatterRecvOp(to_node=to_node_b, send_node=scatter_send_x)
with ng.metadata(**parallel_metadata):
x_plus_one_b = scatter_recv_b + 1
gather_send_x_plus_one_b = GatherSendOp(from_node=x_plus_one_b)
with ng.metadata(transformer='cpu0'):
with ng.metadata(**parallel_metadata):
gather_recv_x_plus_one_a = GatherRecvOp(from_node=from_node_a, to_node=to_node_a,
send_node=gather_send_x_plus_one_a)
z_a = gather_recv_x_plus_one_a + 1
update_comm_deps((scatter_send_x, gather_send_x_plus_one_a, z_a))
update_comm_deps((gather_send_x_plus_one_b,))
assert set([scatter_send_x]) == set(scatter_recv_a.control_deps)
assert set([scatter_send_x, gather_send_x_plus_one_a]) == \
set(gather_recv_x_plus_one_a.control_deps)
def test_scatter_gather_node_axes(config):
t = config
axes = ng.make_axes([ng.make_axis(length) for length in t['axes']])
parallel_axis = axes[t['parallel_axis']]
with ng.metadata(device=None,
device_id='0',
transformer='cpu0',
host_transformer=None):
from_node = ng.placeholder(axes=axes)
to_node = ng.placeholder(axes=axes)
with ng.metadata(device=None,
device_id=t['device_id'],
transformer=None,
parallel=parallel_axis,
host_transformer=None):
par_node = ng.placeholder(axes=axes)
scatter_send_op = ScatterSendOp(from_node=from_node, to_node=par_node)
assert axes == scatter_send_op.axes
assert t['slices'] == scatter_send_op.slices
scatter_recv_op = ScatterRecvOp(to_node=par_node,
send_node=scatter_send_op)
for sct_a, a in zip(scatter_recv_op.axes, axes):
assert sct_a.length == a.length
gather_send_op = GatherSendOp(from_node=scatter_recv_op)
assert_axes_eq_len(scatter_recv_op.axes, gather_send_op.axes)
gather_recv_op = GatherRecvOp(from_node=par_node,
to_node=to_node,
send_node=gather_send_op)
assert_axes_eq_len(axes, gather_recv_op.axes)
assert t['slices'] == gather_recv_op.slices
def test_weight_clipping(w_clip, optimizer):
opt_ng = optimizer(0.1, weight_clip_value=w_clip)
if isinstance(opt_ng, Adam):
pytest.config.argon_skip_now("Argon Transformer error") # TODO triage
# Set up data placeholders
C = ng.make_axis(20)
N = ng.make_axis(32, name='N')
data = ng.placeholder([C, N])
target = ng.placeholder([N])
# params to be updated using optimizer to be tested
# make sure initial values are higher than clip values
np_W = 10 * w_clip * (2 * np.random.rand(C.length) - 1)
W = ng.variable([C], initial_value=np_W)
# double check generated initial W value
assert np.max(np_W) > w_clip
assert np.min(np_W) < -w_clip
# Set up op graph
cost = ng.sum(target - ng.dot(W, data), out_axis=())
updated_weights = ng.sequential([opt_ng(cost), W])
epsilon = w_clip * 1e-3
# Set up the computation and run the "train" loop
with ExecutorFactory() as ex:
opt_ng_comp = ex.transformer.computation(updated_weights, data, target)
mock_dataset = data_generator(20, C.length, N.length)
for x, y in mock_dataset:
ng_W = opt_ng_comp(x, y) # updated weights for ngraph optimizer
assert np.max(ng_W) < w_clip + epsilon
assert np.min(ng_W) > -w_clip - epsilon
def test_prod_deriv(
prod_deriv_arrays): # Argon Transformer error - TODO triage
"""
Test reduce product's gradient
"""
def power_set(lst):
"""
power_set([0, 1, 2]) is:
[[], [0], [1], [0, 1], [2], [0, 2], [1, 2], [0, 1, 2]]
"""
result = [[]]
for x in lst:
result.extend([subset + [x] for subset in result])
return result
def get_all_reduction_axes(axes):
"""
Get all possible reduction axes
"""
ndim = len(axes.lengths)
if ndim == 0:
return axes
else:
results = []
all_indices = power_set(range(ndim))
for indices in all_indices:
if not indices:
results.append(ng.make_axes([]))
else:
results.append(
ng.make_axes([axes[index] for index in indices]))
return results
def shape_to_axes(shape):
"""
Convert shape to axes
"""
if not shape:
return ng.make_axes()
axes = ng.make_axes([ng.make_axis(length=s) for s in shape])
return axes
x_val = prod_deriv_arrays
axes = shape_to_axes(x_val.shape)
all_reduction_axes = get_all_reduction_axes(axes)
for reduction_axes in all_reduction_axes:
x = ng.placeholder(axes=axes)
x_prod = ng.prod(x, reduction_axes)
check_derivative(x_prod, x, 0.001, x_val, atol=1e-3, rtol=1e-3)
def test_flatten_deriv_simplified():
"""
Test derivative with dot and flatten
"""
ax_N = ng.make_axis(length=3)
ax_Y = ng.make_axis(length=2)
x = ng.placeholder(ng.make_axes([ax_N]))
w = ng.constant([5, 2], axes=ng.make_axes([ax_Y]))
logits = ng.dot(x, w)
cost = ng.sum(logits, reduction_axes=logits.axes)
delta = 0.001
u = rng.uniform(.1, 5.0, x.axes)
check_derivative(cost, x, delta, u, atol=1e-2, rtol=1e-2)
def test_deconv(transformer_factory, deconv_n4_hw4_c1_5x5):
cf = ConvParams(**deconv_n4_hw4_c1_5x5)
# randomly initialize
input_value = rng.uniform(-0.5, 0.5, cf.ax_i)
filter_value = rng.uniform(-0.5, 0.5, cf.ax_f)
error_value = rng.uniform(-0.5, 0.5, cf.ax_o)
inputs = ng.placeholder(cf.ax_i)
filters = ng.placeholder(cf.ax_f)
errors = ng.placeholder(cf.ax_o)
output = ng.deconvolution(cf.conv_params, inputs, filters, axes=cf.ax_o)
bprop_out = ng.deriv(output, inputs, errors)
updat_out = ng.deriv(output, filters, errors)
with executor([output, bprop_out, updat_out], inputs, filters,
errors) as conv_executor:
result_ng, gradI_ng, gradF_ng = conv_executor(input_value,
filter_value,
error_value)
# Compute reference with NumPy
result_np = reference_deconv_fprop(cf.conv_params, input_value,
filter_value)
gradI_np, gradF_np = reference_deconv_bprop(cf.conv_params, error_value,
input_value, filter_value)
# Compare fprop
assert np.allclose(result_ng, result_np, rtol=0.1, atol=0)
# Compare bprop
assert np.allclose(gradI_ng, gradI_np, rtol=0.1, atol=0)
# Compare update
assert np.allclose(gradF_ng, gradF_np, rtol=0.1, atol=0)
def test_cross_entropy_multi_axis_order(transformer_factory, input_tensor):
"""If y and t have different axis orders, it should give the same result"""
y = input_tensor
t1 = ng.placeholder(y.axes)
# Reorder axes
feature_axis, batch_axis = y.axes
t2 = ng.placeholder(ng.make_axes([batch_axis, feature_axis]))
# Set up numpy variables
np_y = np.random.uniform(0, 1, y.axes.lengths)
if feature_axis.length > batch_axis.length:
np_t1 = np.eye(feature_axis.length)[:, :batch_axis.length]
else:
np_t1 = np.eye(batch_axis.length)[:feature_axis.length, :]
np_t2 = np_t1.T
with ExecutorFactory() as ex:
f1 = ex.executor(ng.cross_entropy_multi(ng.softmax(y), t1), y, t1)
f2 = ex.executor(ng.cross_entropy_multi(ng.softmax(y), t2), y, t2)
out1 = f1(np_y, np_t1)
out2 = f2(np_y, np_t2)
ng.testing.assert_allclose(out1.ravel(), out2.ravel(), rtol=1e-5)
def test_layer_caching():
in_obj = ng.placeholder(())
layer = SimpleLayer()
out_train = layer(in_obj)
out_train2 = layer(in_obj)
with Layer.inference_mode_on():
out_inference = layer(in_obj)
out_inference2 = layer(in_obj)
out_train3 = layer(in_obj)
assert out_train is out_train2, "Training mode call not cached"
assert out_inference is out_inference2, "Inference mode call not cached"
assert out_train is not out_inference, "Training and inference mode calls are the same"
assert out_train is out_train3, "Training mode not restored"
def make_placeholder(input_size, sequence_length, batch_size, extra_axes=0):
input_axis = ng.make_axis(name='features')
recurrent_axis = ng.make_axis(name='REC_REP')
batch_axis = ng.make_axis(name='N')
input_axes = ng.make_axes([input_axis, recurrent_axis, batch_axis])
input_axes.set_shape((input_size, sequence_length, batch_size))
input_axes = ng.make_axes([ng.make_axis(length=1, name='features_' + str(i))
for i in range(extra_axes)]) + input_axes
input_placeholder = ng.placeholder(input_axes)
rng = RandomTensorGenerator()
input_value = rng.uniform(-0.01, 0.01, input_axes)
return input_placeholder, input_value
