def test_flat_tensor_dot_tensor():
"""
Ensure that a flattened argument axis is not unflattend in the result.
"""
H = ng.make_axis(2)
W = ng.make_axis(7)
C = ng.make_axis(3)
K = ng.make_axis(11)
axes_a = ng.make_axes([H, W, C])
a = ng.constant(np.ones(axes_a.lengths), axes=axes_a)
flat_a = ng.flatten_at(a, 2)
axes_b = ng.make_axes([C, K])
b = ng.constant(np.ones(axes_b.lengths), axes=axes_b)
result = ng.dot(b, flat_a)
with ExecutorFactory() as factory:
result_fun = factory.executor(result)
result_val = result_fun()
result_correct = np.ones_like(result_val) * C.length
ng.testing.assert_allclose(result_val, result_correct)
def test_conv_flatten_deriv(n4_hw12_c3_5x5):
"""
Test deriv of conv followed by flatten
"""
cf = ConvParams(**n4_hw12_c3_5x5)
axes_rsck = ng.make_axes([cf.ax_f[2], cf.ax_f[3], cf.ax_f[0], cf.ax_f[-1]])
axes_rsck_prime = ng.make_axes([ng.make_axis(name=ax.name + 'p', length=ax.length)
for ax in axes_rsck])
axes_nmpqk = ng.make_axes([cf.ax_o[-1], cf.ax_o[1], cf.ax_o[2], cf.ax_o[3], cf.ax_o[0]])
# broadcast input / filter axes
input_var = ng.variable(cf.ax_i).named('input')
input_val = np.ones(input_var.axes.lengths)
filter_rsck_prime = ng.variable(axes_rsck_prime).named('filter')
filter_var = filter_rsck_prime
filter_rsck = ng.cast_axes(filter_rsck_prime, axes_rsck).named('frsck')
filter_trsck = ng.expand_dims(filter_rsck, cf.ax_f[1], 0).named('ftrsck')
filter_ctrsk = ng.axes_with_order(filter_trsck, axes=cf.ax_f).named('ctrsk')
# convolution
output_kmpqn = ng.convolution(cf.conv_params, input_var, filter_ctrsk, axes=cf.ax_o)
output_nmpqk = ng.axes_with_order(output_kmpqn, axes=axes_nmpqk)
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
output_npqk = ng.tensor_slice(output_nmpqk, out_slicing)
output = ng.flatten_at(output_npqk, idx=1)
# cost and grad
cost = ng.sum(output, out_axes=())
filter_val = np.ones(filter_var.axes.lengths)
with ExecutorFactory() as factory:
conv_comp = factory.executor(output, filter_var, input_var)
grad_filter_num_comp = factory.numeric_derivative(cost, filter_var, 1.0, input_var)
grad_filter_sym_comp = factory.derivative(cost, filter_var, input_var)
grad_input_num_comp = factory.numeric_derivative(cost, input_var, 1.0, filter_var)
grad_input_sym_comp = factory.derivative(cost, input_var, filter_var)
conv_val = conv_comp(filter_val, input_val)
conv_val_num = np.empty_like(conv_val)
conv_val_num.fill(np.prod(cf.ax_f.lengths[:-1]))
ng.testing.assert_allclose(conv_val, conv_val_num)
grad_filter_num_val = grad_filter_num_comp(filter_val, input_val)
grad_filter_sym_val = grad_filter_sym_comp(filter_val, input_val)
ng.testing.assert_allclose(grad_filter_num_val, grad_filter_sym_val)
grad_input_num_val = grad_input_num_comp(input_val, filter_val)
grad_input_sym_val = grad_input_sym_comp(input_val, filter_val)
ng.testing.assert_allclose(grad_input_num_val, grad_input_sym_val)
def Reshape(self, tf_node, inputs):
"""
Reshapes a tensor.
Arguments:
tf_node: NodeDef object, the tensorflow node to convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the tensorflow node.
Inputs to tf_node:
tensor, shape, name
"""
# TODO: currently only support constants and flatten to 1d and 2d
# get inputs
tensor, shape = inputs
def get_flatten_idx(shape_i, shape_o):
"""
check if flattening shape is valid
Args:
shape_i: input tensor shape
shape_o: output flattend tensor shape
Returns:
None if flatten not valid, otherwise the flatten_at index
"""
return None
# get input and output shape
shape_i = tensor.shape.lengths
shape_o = tuple(shape.const.astype(int))
if np.prod(shape_i) != np.prod(shape_o):
raise ValueError("Total size of input and output dimension "
"mismatch.")
if tensor.const is not None:
# reshape const
np_val = np.reshape(tensor.const, shape_o)
return ng.constant(np_val,
make_pos_axes(np_val.shape)).named(tf_node.name)
else:
ndims_o = len(shape_o)
if ndims_o != 1 and ndims_o != 2:
raise NotImplementedError("Reshape can only support flatten"
"to 1d or 2d.")
if ndims_o == 1:
tensor = ng.flatten(tensor)
else:
cumprods = list(np.cumprod(shape_i))
flatten_at_idx = cumprods.index(shape_o[0]) + 1
tensor = ng.flatten_at(tensor, flatten_at_idx)
res = ng.cast_axes(tensor, make_pos_axes(shape_o))
return res.named(tf_node.name)
def Flatten(onnx_node, ng_inputs): # type: (NodeWrapper, List[TensorOp]) -> Op
"""Flatten the input tensor into a 2D matrix."""
data = ng_inputs[0]
axis = onnx_node.get_attribute_value('axis', 1)
if axis < 0 or axis > len(data.axes):
raise ValueError(
'Flatten node (%s): %d is not a valid value for `axis`.',
onnx_node.name, axis)
return cast_to_pos_axes(ng.flatten_at(data, axis))
def test_conv_flatten_deriv(transformer_factory):
"""
Test deriv of conv followed by flatten
"""
# set shape
C, D, H, W, N = (3, 1, 28, 28, 8)
C, T, R, S, K = (3, 1, 5, 5, 32)
# i, f, o axes
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N])
ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K])
ax_o = ng.make_axes([
ng.make_axis(32, roles=[ar.Channel]),
ng.make_axis(1, roles=[ar.Depth]),
ng.make_axis(24, roles=[ar.Height]),
ng.make_axis(24, roles=[ar.Width]), ax.N
])
ax_i.set_shape((C, D, H, W, N))
ax_f.set_shape((C, T, R, S, K))
params = dict(pad_d=0, pad_h=0, pad_w=0, str_d=1, str_h=1, str_w=1)
axes_rsck = ng.make_axes([ax.R, ax.S, ax.C, ax.K])
axes_rsck_prime = ng.make_axes(
[ng.make_axis(l) for l in axes_rsck.lengths])
# broadcast input / filter axes
image = ng.constant(np.ones(ax_i.lengths), ax_i)
filter = ng.variable(axes_rsck_prime, initial_value=np.ones((R, S, C, K)))
filter_casted = ng.cast_axes(filter, axes_rsck)
filter_casted = ng.expand_dims(filter_casted, ax.T, 0)
filter_casted = ng.axes_with_order(filter_casted, axes=ax_f)
# convolution
output = ng.convolution(params, image, filter_casted, axes=ax_o)
oC, oD, oH, oW, oN = output.axes
output = ng.axes_with_order(output,
axes=ng.make_axes([oN, oD, oH, oW, oC]))
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
conv = ng.Slice(output, out_slicing)
flatten = ng.flatten_at(conv, idx=1)
# cost and grad
cost = ng.sum(flatten, reduction_axes=flatten.axes)
grad = ng.deriv(cost, filter)
# compute
conv_grad_comp = executor([conv, grad])
conv_val, grad_val = conv_grad_comp()
assert np.allclose(conv_val, np.zeros_like(conv_val) + 75.)
assert np.allclose(grad_val, np.zeros_like(grad_val) + 4608.)
def reshape_workaround(
data, shape_out): # type: (TensorOp, Sequence[int]) -> TensorOp
"""Limited workaround for tensor reshape operation."""
shape_in = data.shape.lengths
if np.prod(shape_in) != np.prod(shape_out):
raise ValueError(
'Total size of input (%d) and output (%d) dimension mismatch.',
np.prod(shape_in), np.prod(shape_out))
ndims_out = len(shape_out)
if ndims_out == 1:
tensor = ng.flatten(data)
elif ndims_out == 2:
cumprods = list(np.cumprod(shape_in))
flatten_at_idx = cumprods.index(shape_out[0]) + 1
tensor = ng.flatten_at(data, flatten_at_idx)
else:
raise NotImplementedError(
'Reshape can only support flatten to 1d or 2d.')
return ng.cast_axes(tensor, make_pos_axes(shape_out))
def test_flatten_deriv():
from ngraph.frontends.neon import ax
np.random.seed(0)
# set shape
C, D, H, W, N = (3, 1, 28, 28, 8) # image
Y = 10
ax.C.length = C
ax.D.length = D
ax.H.length = H
ax.W.length = W
ax.N.length = N
ax.Y.length = Y
# conv output
conv = ng.placeholder(ng.make_axes([ax.N, ax.H, ax.W, ax.C]))
# flatten
flatten = ng.flatten_at(conv, idx=1)
num_flatten = flatten.axes.lengths[1]
flatten = ng.cast_axes(flatten,
ng.make_axes([ax.N, ng.make_axis(num_flatten)]))
# fc
fc_weights_axes = ng.make_axes([ng.make_axis(num_flatten), ax.Y])
fc_weights = ng.constant(np.random.randn(num_flatten, Y),
axes=fc_weights_axes)
flatten_casted = ng.cast_axes(
flatten, ng.make_axes([flatten.axes[0], fc_weights_axes[0] - 1]))
logits = ng.dot(flatten_casted, fc_weights)
cost = ng.sum(logits, reduction_axes=logits.axes)
delta = 0.001
u = rng.uniform(.1, 5.0, conv.axes)
check_derivative(cost, conv, delta, u, atol=1e-2, rtol=1e-2)
def test_conv_flatten_deriv(transformer_factory):
"""
Test deriv of conv followed by flatten
"""
# set shape
# NOTE: N must be >= 4 for GPU, but for CPU this could be decreased to
# speed up the test
N = 4
C, D, H, W = (3, 1, 28, 28)
T, R, S, K = (1, 5, 5, 8)
params = dict(pad_d=0,
pad_h=0,
pad_w=0,
str_d=1,
str_h=1,
str_w=1,
dil_d=1,
dil_h=1,
dil_w=1)
# i, f, o axes
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N])
ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K])
ax_o = ng.make_axes([
ng.make_axis(roles=[ar.features_input]).named('C'),
ng.make_axis(roles=[ar.features_0]).named('D'),
ng.make_axis(roles=[ar.features_1]).named('H'),
ng.make_axis(roles=[ar.features_2]).named('W'), ax.N
])
ax_i.set_shape((C, D, H, W, N))
ax_f.set_shape((C, T, R, S, K))
ax_o.set_shape((K, D - T + 1, H - R + 1, W - S + 1, N))
axes_rsck = ng.make_axes([ax.R, ax.S, ax.C, ax.K])
axes_rsck_prime = ng.make_axes([
ng.make_axis(axis.length).named(axis.name + 'p') for axis in axes_rsck
])
axes_nmpqk = ng.make_axes([ax_o[-1], ax_o[1], ax_o[2], ax_o[3], ax_o[0]])
# broadcast input / filter axes
input_var = ng.variable(ax_i).named('input')
input_var.input = True
input_val = np.ones(input_var.axes.lengths)
filter_rsck_prime = ng.variable(axes_rsck_prime)
filter_var = filter_rsck_prime
filter_rsck = ng.cast_axes(filter_rsck_prime, axes_rsck)
filter_trsck = ng.expand_dims(filter_rsck, ax.T, 0)
filter_ctrsk = ng.axes_with_order(filter_trsck, axes=ax_f)
# convolution
output_kmpqn = ng.convolution(params, input_var, filter_ctrsk, axes=ax_o)
output_nmpqk = ng.axes_with_order(output_kmpqn, axes=axes_nmpqk)
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
output_npqk = ng.tensor_slice(output_nmpqk, out_slicing)
output = ng.flatten_at(output_npqk, idx=1)
# cost and grad
cost = ng.sum(output, out_axes=())
filter_var.input = True
filter_var.named('filter')
filter_val = np.ones(filter_var.axes.lengths)
with ExecutorFactory() as factory:
conv_comp = factory.executor(output, filter_var, input_var)
grad_filter_num_comp = factory.numeric_derivative(
cost, filter_var, 1.0, input_var)
grad_filter_sym_comp = factory.derivative(cost, filter_var, input_var)
grad_input_num_comp = factory.numeric_derivative(
cost, input_var, 1.0, filter_var)
grad_input_sym_comp = factory.derivative(cost, input_var, filter_var)
conv_val = conv_comp(filter_val, input_val)
conv_val_num = np.empty_like(conv_val)
conv_val_num.fill(C * T * R * S)
assert ng.testing.allclose(conv_val, conv_val_num)
grad_filter_num_val = grad_filter_num_comp(filter_val, input_val)
grad_filter_sym_val = grad_filter_sym_comp(filter_val, input_val)
assert ng.testing.allclose(grad_filter_num_val, grad_filter_sym_val)
grad_input_num_val = grad_input_num_comp(input_val, filter_val)
grad_input_sym_val = grad_input_sym_comp(input_val, filter_val)
assert ng.testing.allclose(grad_input_num_val, grad_input_sym_val)