def test_convolution(transformer_factory):
"""
test convolution forward path
"""
N = 128
C, K = 3, 8
D, T = 1, 1
H = W = 32
R = S = 2
padding = dict(pad_d=0, pad_h=0, pad_w=0)
strides = dict(str_d=1, str_h=1, str_w=1)
conv_params = padding.copy()
conv_params.update(strides)
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N])
ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K])
ax_i.set_shape((C, D, H, W, N))
ax_f.set_shape((C, T, R, S, K))
ax_o = ng.make_axes([
ng.make_axis(ax_f.role_axes(ar.Channelout)[0].length,
name='C',
roles=[ar.Channel]),
spatial_axis(ax_i,
ax_f,
padding['pad_d'],
strides['str_d'],
role=ar.Depth),
spatial_axis(ax_i,
ax_f,
padding['pad_h'],
strides['str_h'],
role=ar.Height),
spatial_axis(ax_i,
ax_f,
padding['pad_w'],
strides['str_w'],
role=ar.Width), ax.N
])
inputs = ng.placeholder(axes=ax_i)
filters = ng.placeholder(axes=ax_f)
# randomly initialize
input_value = rng.uniform(-1, 1, ax_i)
filter_value = rng.uniform(-1, 1, ax_f)
assert input_value.shape == ax_i.lengths
assert filter_value.shape == ax_f.lengths
inputs = ng.placeholder(ax_i)
filters = ng.placeholder(ax_f)
output = ng.convolution(conv_params, inputs, filters, axes=ax_o)
targets = ng.placeholder(axes=output.axes)
costs = ng.cross_entropy_binary(ng.sigmoid(output), targets)
error = ng.sum(costs, out_axes=()) / ng.batch_size(costs)
d_inputs = ng.deriv(error, inputs)
d_filters = ng.deriv(error, filters)
targets_value = rng.uniform(.1, 0.9, output.axes)
conv_executor = executor([output, error, d_inputs, d_filters], inputs,
filters, targets)
result_ng, err_ng, gradI_ng, gradF_ng = conv_executor(
input_value, filter_value, targets_value)
# Now compute reference values via NEON
NervanaObject.be.bsz = N
neon_layer = Convolution(fshape=(R, S, K),
padding=padding,
strides=strides)
inp = neon_layer.be.array(input_value.reshape(C * H * W * D, N))
neon_layer.W = neon_layer.be.array(filter_value.reshape(C * R * S * T, K))
neon_layer.dW = neon_layer.be.empty_like(neon_layer.W)
neon_layer.configure((C, H, W))
neon_layer.prev_layer = True
neon_layer.allocate()
neon_layer.set_deltas(DummyDeltaBuffers())
result_ne = neon_layer.fprop(inp).get().reshape(output.axes.lengths)
act_result_ne = 1. / (1.0 + np.exp(-result_ne))
err = neon_layer.be.array(
(act_result_ne - targets_value).reshape(-1, N) / float(N))
gradI_ne = neon_layer.bprop(err).get().reshape(ax_i.lengths)
gradF_ne = neon_layer.dW.get().reshape(ax_f.lengths)
# Compare fprop
np.testing.assert_allclose(result_ng, result_ne, rtol=0, atol=1e-6)
# Compare bprop
np.testing.assert_allclose(gradI_ng, gradI_ne, rtol=0, atol=1e-6)
# Compare update
np.testing.assert_allclose(gradF_ng, gradF_ne, rtol=0, atol=1e-4)
def Conv2D(self, tf_node, inputs):
"""
Computes a 2-D convolution given 4D input and filter tensors.
Arguments:
tf_node: NodeDef object, the tensorflow node to convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the tensorflow node.
Inputs to tf_node:
input, filter
TODO: assume default tensorflow layout NHWC, RSCK,
need to support NCHW as well
need to clean up / merge with maxpool
Notes on output shape:
https://www.tensorflow.org/api_docs/python/nn.html#convolution
"""
image, weight = inputs
# TODO: currently NHWC only
assert tf_node.attr['data_format'].s.decode("ascii") == "NHWC"
# set axes shape
ax_N = ng.make_axis(batch=True)
ax_C = ng.make_axis(roles=[ar.Channel])
ax_D = ng.make_axis(roles=[ar.Depth])
ax_H = ng.make_axis(roles=[ar.Height])
ax_W = ng.make_axis(roles=[ar.Width])
ax_T = ng.make_axis(roles=[ar.Depth])
ax_R = ng.make_axis(roles=[ar.Height])
ax_S = ng.make_axis(roles=[ar.Width])
ax_K = ng.make_axis(roles=[ar.Channelout])
ng.make_axes([ax_N, ax_H, ax_W, ax_C]).set_shape(image.axes.lengths)
ng.make_axes([ax_R, ax_S, ax_C, ax_K]).set_shape(weight.axes.lengths)
ax_D.length = 1
ax_T.length = 1
# strides params
tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)]
if len(tf_strides) != 4:
raise ValueError("Length of strides my be 4.")
if tf_strides[0] != 1:
raise NotImplementedError('Strides on batch axis (N) must be 1.')
if tf_strides[3] != 1:
raise NotImplementedError('Strides on channel axis (C) must be 1.')
str_h, str_w = tf_strides[1], tf_strides[2]
# padding params
padding = tf_node.attr['padding'].s.decode("ascii")
pad_t, pad_b, pad_l, pad_r = tf_conv2d_pool_padding(
image.axes.lengths, weight.axes.lengths, tf_strides, padding)
if pad_t != pad_b or pad_l != pad_r:
raise NotImplementedError("Requires symmetric padding in ngraph:"
"pad_t(%s) == pad_b(%s) and"
"pad_l(%s) == pad_r(%s)" %
(pad_t, pad_b, pad_l, pad_r))
# conv params
params = dict(pad_d=0,
pad_h=pad_t,
pad_w=pad_l,
str_d=1,
str_h=str_h,
str_w=str_w)
# i, f, o axes
ax_i = ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N])
ax_f = ng.make_axes([ax_C, ax_T, ax_R, ax_S, ax_K])
ax_o = ng.make_axes([
ng.make_axis(ax_K.length, name='C', roles=[ar.Channel]),
spatial_axis(ax_i, ax_f, params['pad_d'], params['str_d'],
ar.Depth),
spatial_axis(ax_i, ax_f, params['pad_h'], params['str_h'],
ar.Height),
spatial_axis(ax_i, ax_f, params['pad_w'], params['str_w'],
ar.Width), ax_N
])
# broadcast input / filter axes
image = ng.cast_axes(image, ng.make_axes([ax_N, ax_H, ax_W, ax_C]))
image = ng.expand_dims(image, ax_D, 1) # NHWC -> NDHWC
image = ng.axes_with_order(image, axes=ax_i) # NDHWC -> CDHWN
weight = ng.cast_axes(weight, ng.make_axes([ax_R, ax_S, ax_C, ax_K]))
weight = ng.expand_dims(weight, ax_T, 0) # RSCK -> TRSCK
weight = ng.axes_with_order(weight, axes=ax_f) # TRSCK -> CTRSK
# convolution
output = ng.convolution(params, image, weight, axes=ax_o)
# cast back to NHWC
oC, oD, oH, oW, oN = output.axes
output = ng.broadcast(output, ng.make_axes([oN, oD, oH, oW, oC]))
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
output = ng.Slice(output, out_slicing)
return output
def MaxPool(self, tf_node, inputs):
"""
Performs the max pooling on the input.
Arguments:
tf_node: NodeDef object, the tensorflow node tso convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the tensorflow node.
Inputs to tf_node:
input
TODO: assume default tensorflow layout NHWC, RSCK,
need to support NCHW as well
need to clean up / merge with conv2d
Notes on output shape:
https://www.tensorflow.org/api_docs/python/nn.html#convolution
"""
image = inputs[0]
# TODO: currently NHWC only
assert tf_node.attr['data_format'].s.decode("ascii") == "NHWC"
# set axes shape
ax_N = ng.make_axis(batch=True)
ax_C = ng.make_axis(roles=[ar.Channel])
ax_D = ng.make_axis(roles=[ar.Depth])
ax_H = ng.make_axis(roles=[ar.Height])
ax_W = ng.make_axis(roles=[ar.Width])
ng.make_axes([ax_N, ax_H, ax_W, ax_C]).set_shape(image.axes.lengths)
ax_D.length = 1
# ksize params
tf_ksize = [int(s) for s in list(tf_node.attr['ksize'].list.i)]
if len(tf_ksize) != 4:
raise ValueError("Length of ksize my be 4.")
if tf_ksize[0] != 1:
raise NotImplementedError('Ksize on batch axis (N) must be 1.')
if tf_ksize[3] != 1:
raise NotImplementedError('Ksize on channel axis (C) must be 1.'
'Cross map pooling to be implemented.')
R, S = tf_ksize[1:3]
T = J = 1
# strides params
tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)]
if len(tf_strides) != 4:
raise ValueError("Length of strides my be 4.")
if tf_strides[0] != 1:
raise NotImplementedError('Strides on batch axis (N) must be 1.')
if tf_strides[3] != 1:
raise NotImplementedError('Strides on channel axis (C) must be 1.')
str_h, str_w = tf_strides[1], tf_strides[2]
# padding params
padding = tf_node.attr['padding'].s.decode("ascii")
pad_t, pad_b, pad_l, pad_r = tf_conv2d_pool_padding(
image.axes.lengths, (R, S, ax_C.length, ax_C.length), tf_strides,
padding)
if pad_t != pad_b or pad_l != pad_r:
raise NotImplementedError("Requires symmetric padding in ngraph:"
"pad_t(%s) == pad_b(%s) and"
"pad_l(%s) == pad_r(%s)" %
(pad_t, pad_b, pad_l, pad_r))
# pooling params
params = dict(op='max',
pad_d=0,
pad_h=pad_t,
pad_w=pad_l,
pad_c=0,
str_d=1,
str_h=str_h,
str_w=str_w,
str_c=1,
J=J,
T=T,
R=R,
S=S)
# i, f, o axes
ax_i = ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N])
ax_o = ng.make_axes([
spatial_axis(ax_i, J, params['pad_c'], params['str_c'],
ar.Channel),
spatial_axis(ax_i, T, params['pad_d'], params['str_d'], ar.Depth),
spatial_axis(ax_i, R, params['pad_h'], params['str_h'], ar.Height),
spatial_axis(ax_i, S, params['pad_w'], params['str_w'], ar.Width),
ax_N
])
# broadcast input / filter axes
image = ng.cast_axes(image, ng.make_axes([ax_N, ax_H, ax_W, ax_C]))
image = ng.expand_dims(image, ax_D, 1) # NHWC -> NDHWC
image = ng.axes_with_order(image, axes=ax_i) # NDHWC -> CDHWN
# pooling
output = ng.pooling(params, image, axes=ax_o)
# cast back to NHWC
oC, oD, oH, oW, oN = output.axes
output = ng.broadcast(output, ng.make_axes([oN, oD, oH, oW, oC]))
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
output = ng.Slice(output, out_slicing)
return output
def test_pooling():
"""
test pooling forward and backward path
"""
N = 128
C = 3
D = 1
H = W = 32
J = T = 1
R = S = 2
ngt.make_transformer()
padding = dict(pad_d=0, pad_h=0, pad_w=0, pad_c=0)
strides = dict(str_d=1, str_h=1, str_w=1, str_c=1)
fshape = dict(J=J, T=T, R=R, S=S)
pool_params = dict(op='max')
pool_params.update(padding)
pool_params.update(strides)
pool_params.update(fshape)
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N])
ax_i.set_shape((C, D, H, W, N))
inputs = ng.placeholder(axes=ax_i)
ax_o = ng.make_axes([
spatial_axis(ax_i, J, padding['pad_c'], strides['str_c'], role=ar.Channel),
spatial_axis(ax_i, T, padding['pad_d'], strides['str_d'], role=ar.Depth),
spatial_axis(ax_i, R, padding['pad_h'], strides['str_h'], role=ar.Height),
spatial_axis(ax_i, S, padding['pad_w'], strides['str_w'], role=ar.Width),
ax.N
])
# randomly initialize
input_value = rng.uniform(-1, 1, ax_i)
assert input_value.shape == ax_i.lengths
# compute convolution with graph
output = ng.pooling(pool_params, inputs, axes=ax_o)
targets = ng.placeholder(axes=ax_o)
costs = ng.cross_entropy_binary(ng.sigmoid(output), targets)
error = ng.sum(costs, out_axes=()) / ng.batch_size(costs)
d_inputs = ng.deriv(error, inputs)
targets_value = rng.uniform(.1, 0.9, output.axes)
conv_executor = executor([output, error, d_inputs], inputs, targets)
result_ng, err_ng, gradI_ng = conv_executor(input_value, targets_value)
# Now compute reference values via NEON
NervanaObject.be.bsz = N
neon_layer = Pooling(fshape=fshape, padding=padding, strides=strides, op="max")
inp = neon_layer.be.array(input_value.reshape(C * H * W * D, N))
neon_layer.configure((C, H, W))
neon_layer.prev_layer = True
neon_layer.allocate()
neon_layer.set_deltas(DummyDeltaBuffers())
result_ne = neon_layer.fprop(inp).get().reshape(output.axes.lengths)
act_result_ne = 1. / (1.0 + np.exp(-result_ne))
err = neon_layer.be.array((act_result_ne - targets_value).reshape(-1, N) / float(N))
gradI_ne = neon_layer.bprop(err).get().reshape(ax_i.lengths)
# Compare fprop
np.testing.assert_allclose(result_ng, result_ne, rtol=0, atol=1e-6)
# Compare bprop
np.testing.assert_allclose(gradI_ng, gradI_ne, rtol=0, atol=1e-6)