def test_wrong_op_name():
"""
test wrong number of batch axes at input
"""
pf = PoolParams(op='min')
inputs = ng.placeholder(axes=pf.ax_i)
with pytest.raises(ValueError) as exinfo:
ng.pooling(pf.pool_params, inputs, {})
assert str(exinfo.value) == "Unsupported pooling type: {pooltype}. Only max and avg " \
"pooling currently supported. ".format(pooltype=pf.pool_params['op'])
def test_wrong_input_shape_length():
"""
test wrong input shape length
"""
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W])
inputs = ng.placeholder(axes=ax_i)
pool_params = dict(op='max')
with pytest.raises(ValueError) as exinfo:
ng.pooling(pool_params, inputs, {})
assert str(exinfo.value) == 'pooling input shape must be length 5, found {}' \
.format(len(ax_i))
def test_wrong_op_name():
"""
test wrong number of batch axes at input
"""
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N])
inputs = ng.placeholder(axes=ax_i)
pooltype = 'min'
pool_params = dict(op=pooltype)
with pytest.raises(ValueError) as exinfo:
ng.pooling(pool_params, inputs, {})
assert str(exinfo.value) == "Unsupported pooling type: {pooltype}. Only max and avg " \
"pooling currently supported. ".format(pooltype=pooltype)
def test_wrong_input_shape_length():
"""
test wrong input shape length
"""
pf = PoolParams()
ax_i = pf.ax_i[:-1]
inputs = ng.placeholder(axes=ax_i)
with pytest.raises(ValueError) as exinfo:
ng.pooling(pf.pool_params, inputs, {})
assert str(exinfo.value) == 'pooling input shape must be length 5, found {}' \
.format(len(ax_i))
def train_outputs(self, in_obj):
ppm = self.poolparams.copy()
in_obj = ng.axes_with_role_order(in_obj, self.role_order)
in_axes = in_obj.axes
if self.o_axes is None:
self.o_axes = ng.make_axes([
ng.make_axis(roles=a.roles).named(a.short_name)
for a in in_axes if not a.is_batch
])
# set lengths
out_shape = [
output_dim(in_axes[0].length, ppm['J'], ppm['pad_d'],
ppm['str_d']),
output_dim(in_axes[1].length, ppm['T'], ppm['pad_d'],
ppm['str_d']),
output_dim(in_axes[2].length, ppm['R'], ppm['pad_h'],
ppm['str_h']),
output_dim(in_axes[3].length, ppm['S'], ppm['pad_w'],
ppm['str_w'])
]
self.o_axes.set_shape(out_shape)
self.o_axes += in_axes.batch_axes()
return ng.pooling(ppm, in_obj, axes=self.o_axes)
def train_outputs(self, in_obj):
ppm = self.poolparams.copy()
in_axes = in_obj.axes
# TODO: clean this up
if self.o_axes is None:
self.o_axes = ng.make_axes([
ng.spatial_axis(in_axes,
ppm['J'],
ppm['pad_c'],
ppm['str_c'],
role=ar.Channel),
ng.spatial_axis(in_axes,
ppm['T'],
ppm['pad_d'],
ppm['str_d'],
role=ar.Depth),
ng.spatial_axis(in_axes,
ppm['R'],
ppm['pad_h'],
ppm['str_h'],
role=ar.Height),
ng.spatial_axis(in_axes,
ppm['S'],
ppm['pad_w'],
ppm['str_w'],
role=ar.Width), ax.N
])
return ng.pooling(ppm, in_obj, axes=self.o_axes)
def test_wrong_number_of_batch_axes_at_input():
"""
test wrong number of batch axes at input
"""
C = 3
D = 1
ax_C = ng.make_axis(name='N', length=C)
ax_D = ng.make_axis(name='N', length=D)
pool_params = dict(op='max')
ax_i = ng.make_axes([ax_C, ax_D, ax.H, ax.W, ax.N])
inputs = ng.placeholder(axes=ax_i)
with pytest.raises(ValueError) as exinfo:
ng.pooling(pool_params, inputs, {})
assert str(exinfo.value) == "Input must have one batch axis. Found {n_batch_axes} batch" \
" axes: {batch_axes} and {n_sample_axes} sample axes: {sample_axes}.".format(
n_batch_axes=len(inputs.axes.batch_axes()),
batch_axes=inputs.axes.batch_axes(),
n_sample_axes=len(inputs.axes.sample_axes()),
sample_axes=inputs.axes.sample_axes())
def get_fprop_bprop(self, input_value):
ip = ng.placeholder(axes=self.ax_i)
ep = ng.placeholder(axes=self.ax_o)
iv = np.array(input_value).astype(np.float32).reshape(self.dimI)
ev = np.ones(self.dimO) * 4
output = ng.pooling(self.pool_params, ip, axes=self.ax_o)
delta = BpropPoolOp(ep, ip, output)
with executor([output, delta], ip, ep) as pool_executor:
output_value, delta_value = pool_executor(iv, ev)
return output_value, delta_value
def _pooling_op(self, cntk_op, inputs):
"""
Computes the pooling of a tensor.
Arguments:
cntk_op: CNTK function to be imported.
inputs: List of inputs to this node.
Returns:
A ngraph Op.
"""
inputs = self._expand_input_axes(inputs[0])
C, D, H, W, N = inputs.axes
M, P, Q = self._make_out_axes(cntk_op.shape)
if cntk_op.attributes['poolingType'] == 0:
pool_type = 'max'
else:
pool_type = 'avg'
strides = self._make_strides(cntk_op.attributes['strides'])
kernel = self._make_kernel(cntk_op.attributes['poolingWindowShape'])
pad = self._make_padding('pool', cntk_op.attributes['autoPadding'],
(D.length, H.length, W.length, C.length),
kernel,
(M.length, P.length, Q.length, C.length),
strides)
params = dict(op=pool_type,
pad_d=pad[0],
pad_h=pad[1],
pad_w=pad[2],
pad_c=pad[3],
str_d=strides[0],
str_h=strides[1],
str_w=strides[2],
str_c=strides[3],
T=kernel[0],
R=kernel[1],
S=kernel[2],
J=kernel[3])
pool = ng.pooling(params, inputs, [C, M, P, Q, N])
return squeeze_axes([pool])[0]
def _pooling_op(self, cntk_op, inputs, op):
"""
Computes the pooling of a tensor.
CNTK Ngraph
in ((C, H, W), N) (C, D, H, W, N)
out (N, P, Q, K) (K, M, P, Q, N)
Arguments:
cntk_op: CNTK function to be imported.
inputs: List of inputs to this node.
op: 'max' for MaxPooling and 'avg' for AvgPooling
Returns:
A ngraph Op.
"""
inputs = self._expand_input_axes(inputs[0])
C, D, H, W, N = inputs.axes
M, P, Q = self._make_out_axes(cntk_op.shape)
strides = self._make_strides(cntk_op.attributes['strides'])
kernel = self._make_kernel(cntk_op.attributes['poolingWindowShape'])
pad = self._make_padding('pool', cntk_op.attributes['autoPadding'],
(D.length, H.length, W.length, C.length),
kernel,
(M.length, P.length, Q.length, C.length),
strides)
params = dict(op=op,
pad_d=pad[0],
pad_h=pad[1],
pad_w=pad[2],
pad_c=pad[3],
str_d=strides[0],
str_h=strides[1],
str_w=strides[2],
str_c=strides[3],
T=kernel[0],
R=kernel[1],
S=kernel[2],
J=kernel[3])
pool = ng.pooling(params, inputs, [C, M, P, Q, N])
return remove_ones_axes([pool])[0]
def make_pooling_op(onnx_node, ng_inputs, custom_pool_params=None):
# type: (NodeWrapper, List[TensorOp], Dict) -> Op
"""
Create an ngraph pooling Op based on an ONNX node.
:param onnx_node: wrapped ONNX node for a pooling op
:param ng_inputs: ngraph TensorOp input tensors
:param custom_pool_params: optional pool_params overriding values based on onnx_node
:return: ngraph pooling op
"""
x = ng_inputs[0]
if len(x.axes) == 4: # 2D pooling
# Reshape x axes from ONNX (N, C, H, W) to ngraph (C, D, H, W, N)
x = reorder_axes(x, 'NCHW', 'CDHWN')
elif len(x.axes) == 5: # 3D pooling
# Reshape x axes from ONNX (N, C, H, W, D) to ngraph (C, D, H, W, N)
x = reorder_axes(x, 'NCHWD', 'CDHWN')
else:
raise NotImplementedError(
'%s node (%s): only 2D and 3D pooling ops are supported.',
onnx_node.op_type, onnx_node.name)
pool_params = get_pool_params(onnx_node)
if custom_pool_params:
pool_params.update(custom_pool_params)
output_axes = make_pool_output_axes(x, pool_params)
ng_op = ng.pooling(pool_params, x, output_axes)
# ONNX output should have axes in the order N, C, H, W, D
ng_op = reorder_axes(ng_op, 'CDHWN', 'NCHWD')
if len(ng_inputs[0].axes
) == 4: # 2D convolution, slice away the D axis from output
ng_op = ng.tensor_slice(ng_op, [
slice(None), slice(None),
slice(None), slice(None), 0
])
return ng_op
def __call__(self, in_obj):
ppm = self.poolparams.copy()
in_obj = reorder_spatial_axes(in_obj)
in_axes = in_obj.axes
if self.o_axes is None:
self.o_axes = ng.make_axes([
ng.make_axis(name=a.name) for a in in_axes if not a.is_batch
])
# set lengths
out_shape = [
output_dim(in_axes[0].length, ppm['J'], ppm['pad_d'], ppm['str_d']),
output_dim(in_axes[1].length, ppm['T'], ppm['pad_d'], ppm['str_d']),
output_dim(in_axes[2].length, ppm['R'], ppm['pad_h'], ppm['str_h']),
output_dim(in_axes[3].length, ppm['S'], ppm['pad_w'], ppm['str_w'])
]
self.o_axes.set_shape(out_shape)
self.o_axes |= in_axes.batch_axis()
return ng.pooling(ppm, in_obj, axes=self.o_axes)
def test_pooling():
"""
test pooling forward and backward path
"""
N = 128
C = 3
D = 1
H = W = 32
J = T = 1
R = S = 2
ngt.make_transformer()
padding = dict(pad_d=0, pad_h=0, pad_w=0, pad_c=0)
strides = dict(str_d=1, str_h=1, str_w=1, str_c=1)
fshape = dict(J=J, T=T, R=R, S=S)
pool_params = dict(op='max')
pool_params.update(padding)
pool_params.update(strides)
pool_params.update(fshape)
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N])
ax_i.set_shape((C, D, H, W, N))
inputs = ng.placeholder(axes=ax_i)
ax_o = ng.make_axes([
ng.make_axis(roles=[ar.features_input]).named('C'),
ng.make_axis(roles=[ar.features_0]).named('D'),
ng.make_axis(roles=[ar.features_1]).named('H'),
ng.make_axis(roles=[ar.features_2]).named('W'), ax.N
])
ax_o[:-1].set_shape((output_dim(C, J, padding['pad_c'], strides['str_c']),
output_dim(D, T, padding['pad_d'], strides['str_d']),
output_dim(H, R, padding['pad_h'], strides['str_h']),
output_dim(W, S, padding['pad_w'], strides['str_w'])))
# randomly initialize
input_value = rng.uniform(-1, 1, ax_i)
assert input_value.shape == ax_i.lengths
# compute convolution with graph
output = ng.pooling(pool_params, inputs, axes=ax_o)
targets = ng.placeholder(axes=ax_o)
costs = ng.cross_entropy_binary(ng.sigmoid(output), targets)
error = ng.sum(costs, out_axes=()) / ng.batch_size(costs)
d_inputs = ng.deriv(error, inputs)
targets_value = rng.uniform(.1, 0.9, output.axes)
with executor([output, error, d_inputs], inputs, targets) as conv_executor:
result_ng, err_ng, gradI_ng = conv_executor(input_value, targets_value)
# Now compute reference values via NEON
NervanaObject.be.bsz = N
neon_layer = Pooling(fshape=fshape,
padding=padding,
strides=strides,
op="max")
inp = neon_layer.be.array(input_value.reshape(C * H * W * D, N))
neon_layer.configure((C, H, W))
neon_layer.prev_layer = True
neon_layer.allocate()
neon_layer.set_deltas(DummyDeltaBuffers())
result_ne = neon_layer.fprop(inp).get().reshape(output.axes.lengths)
act_result_ne = 1. / (1.0 + np.exp(-result_ne))
err = neon_layer.be.array(
(act_result_ne - targets_value).reshape(-1, N) / float(N))
gradI_ne = neon_layer.bprop(err).get().reshape(ax_i.lengths)
# Compare fprop
ng.testing.assert_allclose(result_ng, result_ne, rtol=0, atol=1e-6)
# Compare bprop
ng.testing.assert_allclose(gradI_ng, gradI_ne, rtol=0, atol=1e-6)
def MaxPool(self, tf_node, inputs):
"""
Performs the max pooling on the input.
Arguments:
tf_node: NodeDef object, the tensorflow node to convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the tensorflow node.
Inputs to tf_node:
input
TODO: assume default tensorflow layout NHWC, RSCK,
need to support NCHW as well
need to clean up / merge with conv2d
Axes:
Tensorflow Ngraph
in (N, H, W, C) (C, D, H, W, N)
out (N, P, Q, K) (K, M, P, Q, N)
Notes on output shape:
https://www.tensorflow.org/api_docs/python/nn.html#convolution
"""
image = inputs[0]
# TODO: currently NHWC only
assert tf_node.attr['data_format'].s.decode("ascii") == "NHWC"
# new axes
C, D, H, W, K, M, P, Q = [ng.make_axis() for _ in range(8)]
N = ng.make_axis(name='N')
D.length, M.length = 1, 1 # only supports 2D conv for now
# tf's input axes
ax_i_tf = ng.make_axes([N, H, W, C])
ax_i_tf.set_shape(image.axes.lengths)
# ksize params
tf_ksize = [int(s) for s in list(tf_node.attr['ksize'].list.i)]
if len(tf_ksize) != 4:
raise ValueError("Length of ksize my be 4.")
if tf_ksize[0] != 1:
raise NotImplementedError('Ksize on batch axis (N) must be 1.')
if tf_ksize[3] != 1:
raise NotImplementedError('Ksize on channel axis (C) must be 1.'
'Cross map pooling to be implemented.')
R_length, S_length = tf_ksize[1:3]
T_length = J_length = 1
# strides params
tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)]
if len(tf_strides) != 4:
raise ValueError("Length of strides my be 4.")
if tf_strides[0] != 1:
raise NotImplementedError('Strides on batch axis (N) must be 1.')
if tf_strides[3] != 1:
raise NotImplementedError('Strides on channel axis (C) must be 1.')
str_h, str_w = tf_strides[1], tf_strides[2]
# padding params
padding = tf_node.attr['padding'].s.decode("ascii")
pad_t, pad_b, pad_l, pad_r = common_conv2d_pool_padding(
image.axes.lengths, (R_length, S_length, C.length, C.length),
tf_strides, padding)
if pad_t != pad_b or pad_l != pad_r:
raise NotImplementedError("Requires symmetric padding in ngraph:"
"pad_t(%s) == pad_b(%s) and"
"pad_l(%s) == pad_r(%s)" %
(pad_t, pad_b, pad_l, pad_r))
# pooling params
params = dict(op='max',
pad_d=0, pad_h=pad_t, pad_w=pad_l, pad_c=0,
str_d=1, str_h=str_h, str_w=str_w, str_c=1,
J=J_length, T=T_length, R=R_length, S=S_length)
# tf's output axes
ax_o_tf = ng.make_axes([N, P, Q, K])
ax_o_tf.set_shape(common_conv2d_pool_output_shape(image.axes.lengths,
(R_length, S_length,
C.length, C.length),
tf_strides, padding))
# ngraph's i, f, o axes
ax_i = ng.make_axes([C, D, H, W, N])
ax_o = ng.make_axes([K, M, P, Q, N])
# image NHWC -> CDHWN
image = ng.cast_axes(image, ng.make_axes([N, H, W, C]))
image = ng.expand_dims(image, D, 1) # NHWC -> NDHWC
image = ng.axes_with_order(image, ax_i) # NDHWC -> CDHWN
# pooling
output = ng.pooling(params, image, axes=ax_o)
# output KMPQN -> NPQK
# KMPQN -> NMPQK
output = ng.axes_with_order(output, ng.make_axes(
[N, M, P, Q, K]))
# NMPQK -> NPQK
output = ng.tensor_slice(output, [slice(None), 0, slice(None),
slice(None), slice(None)])
return output
def MaxPool(self, tf_node, inputs):
"""
Performs the max pooling on the input.
Arguments:
tf_node: NodeDef object, the tensorflow node tso convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the tensorflow node.
Inputs to tf_node:
input
TODO: assume default tensorflow layout NHWC, RSCK,
need to support NCHW as well
need to clean up / merge with conv2d
Notes on output shape:
https://www.tensorflow.org/api_docs/python/nn.html#convolution
"""
image = inputs[0]
# TODO: currently NHWC only
assert tf_node.attr['data_format'].s.decode("ascii") == "NHWC"
# set axes shape
ax_N = ng.make_axis(batch=True)
ax_C = ng.make_axis(roles=[ar.Channel])
ax_D = ng.make_axis(roles=[ar.Depth])
ax_H = ng.make_axis(roles=[ar.Height])
ax_W = ng.make_axis(roles=[ar.Width])
ng.make_axes([ax_N, ax_H, ax_W, ax_C]).set_shape(image.axes.lengths)
ax_D.length = 1
# ksize params
tf_ksize = [int(s) for s in list(tf_node.attr['ksize'].list.i)]
if len(tf_ksize) != 4:
raise ValueError("Length of ksize my be 4.")
if tf_ksize[0] != 1:
raise NotImplementedError('Ksize on batch axis (N) must be 1.')
if tf_ksize[3] != 1:
raise NotImplementedError('Ksize on channel axis (C) must be 1.'
'Cross map pooling to be implemented.')
R, S = tf_ksize[1:3]
T = J = 1
# strides params
tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)]
if len(tf_strides) != 4:
raise ValueError("Length of strides my be 4.")
if tf_strides[0] != 1:
raise NotImplementedError('Strides on batch axis (N) must be 1.')
if tf_strides[3] != 1:
raise NotImplementedError('Strides on channel axis (C) must be 1.')
str_h, str_w = tf_strides[1], tf_strides[2]
# padding params
padding = tf_node.attr['padding'].s.decode("ascii")
pad_t, pad_b, pad_l, pad_r = tf_conv2d_pool_padding(
image.axes.lengths, (R, S, ax_C.length, ax_C.length), tf_strides,
padding)
if pad_t != pad_b or pad_l != pad_r:
raise NotImplementedError("Requires symmetric padding in ngraph:"
"pad_t(%s) == pad_b(%s) and"
"pad_l(%s) == pad_r(%s)" %
(pad_t, pad_b, pad_l, pad_r))
# pooling params
params = dict(op='max',
pad_d=0,
pad_h=pad_t,
pad_w=pad_l,
pad_c=0,
str_d=1,
str_h=str_h,
str_w=str_w,
str_c=1,
J=J,
T=T,
R=R,
S=S)
# i, f, o axes
ax_i = ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N])
ax_o = ng.make_axes([
spatial_axis(ax_i, J, params['pad_c'], params['str_c'],
ar.Channel),
spatial_axis(ax_i, T, params['pad_d'], params['str_d'], ar.Depth),
spatial_axis(ax_i, R, params['pad_h'], params['str_h'], ar.Height),
spatial_axis(ax_i, S, params['pad_w'], params['str_w'], ar.Width),
ax_N
])
# broadcast input / filter axes
image = ng.cast_axes(image, ng.make_axes([ax_N, ax_H, ax_W, ax_C]))
image = ng.expand_dims(image, ax_D, 1) # NHWC -> NDHWC
image = ng.axes_with_order(image, axes=ax_i) # NDHWC -> CDHWN
# pooling
output = ng.pooling(params, image, axes=ax_o)
# cast back to NHWC
oC, oD, oH, oW, oN = output.axes
output = ng.broadcast(output, ng.make_axes([oN, oD, oH, oW, oC]))
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
output = ng.Slice(output, out_slicing)
return output
def Pool(self, c2_op, inputs):
"""
Performs max or average pooling on the input.
Arguments:
c2_op: NodeDef object, the tensorflow node to convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the c2_op node.
Inputs to c2_op:
input
"""
supported_pooling = {'MaxPool': 'max', 'AveragePool': 'avg'}
image = inputs[0]
# TODO: we assume NCHW, make some assert here?
# set input axes shape
ax_N = ng.make_axis(name='N')
ax_C = ng.make_axis()
ax_D = ng.make_axis(length=1)
ax_H = ng.make_axis()
ax_W = ng.make_axis()
ng.make_axes([ax_N, ax_C, ax_H, ax_W]).set_shape(image.axes.lengths)
# create placeholders for output axes
oC = ng.make_axis(name='C')
oD = ng.make_axis(length=1, name='D')
oH = ng.make_axis(name='H')
oW = ng.make_axis(name='W')
# spatial kernel size
kernel_size = [int(val.i) for val in c2_op.arg if val.name == "kernel"]
if len(kernel_size) != 1:
raise ValueError("Kernel size must be scalar value")
# kernel is square
kernel_h = kernel_w = kernel_size[0]
kernel_d = kernel_c = 1
# strides params
stride_size = [int(val.i) for val in c2_op.arg if val.name == "stride"]
if len(stride_size) != 1:
raise ValueError("Stride size must be scalar value")
stride_h = stride_w = stride_size[0]
# padding params
pad_t, pad_b, pad_l, pad_r = \
_c2_padding(c2_op,
in_NHWC=[ax_N.length, ax_H.length, ax_W.length, ax_C.length],
kernel_HWIO=[kernel_h, kernel_w, ax_C.length, ax_C.length],
stride_NHWC=[1, stride_h, stride_w, 1])
if pad_t != pad_b or pad_l != pad_r:
raise NotImplementedError("Requires symmetric padding in ngraph:"
"pad_t(%s) == pad_b(%s) and"
"pad_l(%s) == pad_r(%s)" %
(pad_t, pad_b, pad_l, pad_r))
# pooling params
params = dict(op=supported_pooling[c2_op.type],
pad_d=0,
pad_h=pad_t,
pad_w=pad_l,
pad_c=0,
str_d=1,
str_h=stride_h,
str_w=stride_w,
str_c=1,
J=kernel_c,
T=kernel_d,
R=kernel_h,
S=kernel_w)
# i, o axes
oC.length = output_dim(ax_C.length, kernel_c, params['pad_c'],
params['str_c'])
oD.length = output_dim(ax_D.length, kernel_d, params['pad_d'],
params['str_d'])
oH.length = output_dim(ax_H.length, kernel_h, params['pad_h'],
params['str_h'])
oW.length = output_dim(ax_W.length, kernel_w, params['pad_w'],
params['str_w'])
ax_i = ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N])
ax_o = ng.make_axes([oC, oD, oH, oW, ax_N])
# broadcast input / filter axes
image = ng.cast_axes(image, ng.make_axes([ax_N, ax_C, ax_H, ax_W]))
image = ng.expand_dims(image, ax_D, 1) # NCHW -> NDCHW
image = ng.axes_with_order(image, axes=ax_i) # NDCHW -> CDHWN
# pooling
output = ng.pooling(params, image, axes=ax_o)
# cast back to NDCHW
output = ng.broadcast(output, ng.make_axes([ax_N, oD, oC, oH, oW]))
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
output = ng.tensor_slice(output, out_slicing)
return output
