def test_wrong_number_of_batch_axes_at_input():
"""
test wrong number of batch axes at input
"""
padding = dict(pad_d=0, pad_h=0, pad_w=0)
strides = dict(str_d=1, str_h=1, str_w=1)
dilation = dict(dil_d=1, dil_h=1, dil_w=1)
conv_params = padding.copy()
conv_params.update(strides)
conv_params.update(dilation)
C = 3
D = 1
ax_C = ng.make_axis(name='N', length=C)
ax_D = ng.make_axis(name='N', length=D)
ax_i = ng.make_axes([ax_C, ax_D, ax.H, ax.W, ax.N])
ax_f = ng.make_axes([ax_C, ax.T, ax.R, ax.S, ax.K])
inputs = ng.placeholder(axes=ax_i)
filters = ng.placeholder(ax_f)
with pytest.raises(ValueError) as exinfo:
ng.convolution(conv_params, inputs, filters, {})
assert str(exinfo.value) == "Input must have one batch axis. Found {n_batch_axes} " \
"batch axes: {batch_axes} Found {n_sample_axes} sample axes: {sample_axes}.".format(
n_batch_axes=len(inputs.axes.batch_axes()),
batch_axes=inputs.axes.batch_axes(),
n_sample_axes=len(inputs.axes.sample_axes()),
sample_axes=inputs.axes.sample_axes())
def test_wrong_input_shape_length():
"""
test wrong input shape length
"""
cf = ConvParams()
ax_i = cf.ax_i[:-1]
inputs = ng.placeholder(ax_i)
filters = ng.placeholder(cf.ax_f)
with pytest.raises(ValueError) as exinfo:
ng.convolution(cf.conv_params, inputs, filters, {})
assert str(exinfo.value) == 'convolution input shape must be length 5, found {}'\
.format(len(ax_i))
def make_convolution_op(onnx_node, ng_inputs):
# type: (NodeWrapper, List[NgraphNode]) -> NgraphNode
"""
Create an ngraph convolution Op based on an ONNX node.
:param onnx_node: wrapped ONNX node for Conv of ConvTranspose op
:param ng_inputs: ngraph TensorOp input tensors
:return: ngraph Op for convolution or deconvolution
"""
if len(ng_inputs) == 3:
x, weights, bias = ng_inputs
elif len(ng_inputs) == 2:
x, weights = ng_inputs
bias = ng.constant(0, dtype=get_dtype(x.get_element_type()))
else:
raise ValueError(
'Conv node (%s): unexpected number of input values: %d.',
onnx_node.name, len(ng_inputs))
groups = onnx_node.get_attribute_value('group', 1)
if groups != 1:
log.warning(
'Conv node (%s): `group` attribute value %d is not supported.',
onnx_node.name, groups)
strides = get_strides(onnx_node)
dilation = get_dilations(onnx_node)
padding_below, padding_above = get_pads(onnx_node)
conv = ng.convolution(x, weights, strides, dilation, padding_below,
padding_above)
return conv + bias
def create_encoder(input_shape, levels = 4):
import ngraph as ng
# input
input_node = ng.parameter(input_shape, np.float32, name="data")
padding_begin = padding_end = [0, 0]
strides = [1, 1]
dilations = [1, 1]
input_channels = [input_shape[1]]
last_output = input_node
# convolution layers
for i in range(levels):
input_c = input_channels[-1]
output_c = input_c * 2
conv_w = np.random.uniform(0, 1, [output_c, input_c, 5, 5]).astype(np.float32)
conv_node = ng.convolution(last_output, conv_w, strides, padding_begin, padding_end, dilations)
input_channels.append(output_c)
last_output = conv_node
# deconvolution layers
for i in range(levels):
input_c = input_channels[-2]
output_c = input_channels.pop(-1)
deconv_w = np.random.uniform(0, 1, [output_c, input_c, 5, 5]).astype(np.float32)
deconv_node = ng.convolution_backprop_data(last_output, deconv_w, strides)
last_output = deconv_node
# result
last_output.set_friendly_name("out")
result_node = ng.result(last_output)
return ng.Function(result_node, [input_node], "Encoder")
def test_first_axes_not_same():
"""
test first axes are not the same
"""
cf = ConvParams()
ax_i = cf.ax_i[1:2] + cf.ax_i[0:1] + cf.ax_i[2:] # D, C, H, W, N
inputs = ng.placeholder(ax_i)
filters = ng.placeholder(cf.ax_f)
with pytest.raises(ValueError) as exinfo:
ng.convolution(cf.conv_params, inputs, filters, {})
assert str(exinfo.value) == 'the first axis in input {inputs} and filter {filters} ' \
'are not the same.'.format(
inputs=inputs.axes[0],
filters=filters.axes[0])
def test_convolution_with_non_zero_padding():
element_type = Type.f32
image_shape = Shape([1, 1, 10, 10])
filter_shape = Shape([1, 1, 3, 3])
data = Parameter(element_type, image_shape)
filters = Parameter(element_type, filter_shape)
parameter_list = [data, filters]
image_arr = np.arange(100, dtype=np.float32).reshape(1, 1, 10, 10)
filter_arr = (np.ones(9, dtype=np.float32).reshape(1, 1, 3, 3)) * -1
filter_arr[0][0][1][1] = 1
strides = [1, 1]
dilations = [2, 2]
pads_begin = [2, 1]
pads_end = [1, 2]
model = ng.convolution(data, filters, strides, pads_begin, pads_end, dilations)
function = Function([model], parameter_list, "test")
runtime = get_runtime()
computation = runtime.computation(function, *parameter_list)
result = computation(image_arr, filter_arr)[0]
expected = convolution2d(
image_arr[0][0], filter_arr[0][0], strides, dilations, pads_begin, pads_end
).reshape([1, 1, 9, 9])
assert np.allclose(result, expected)
def train_outputs(self, in_obj):
cpm = self.convparams.copy()
in_obj = ng.axes_with_role_order(in_obj, self.role_order)
in_axes = in_obj.axes
if self.f_axes is None:
self.f_axes = ng.make_axes([in_axes[0]])
for nm, role in zip('TRSK', self.filter_roles[1:]):
self.f_axes += ng.make_axis(roles=[role],
length=cpm[nm]).named(nm)
self.W = ng.variable(axes=self.f_axes, initial_value=self.init)
if self.o_axes is None:
self.o_axes = ng.make_axes([
ng.make_axis(roles=a.roles).named(a.short_name)
for a in in_axes if not a.is_batch
])
# set lengths
out_shape = [
self.f_axes[-1].length,
output_dim(in_axes[1].length, cpm['T'], cpm['pad_d'],
cpm['str_d'], False, cpm['dil_d']),
output_dim(in_axes[2].length, cpm['R'], cpm['pad_h'],
cpm['str_h'], False, cpm['dil_h']),
output_dim(in_axes[3].length, cpm['S'], cpm['pad_w'],
cpm['str_w'], False, cpm['dil_w'])
]
self.o_axes.set_shape(out_shape)
self.o_axes += in_axes.batch_axes()
return ng.convolution(cpm, in_obj, self.W, axes=self.o_axes)
def test_conv(n64_hw32_c32_3x3):
cf = ConvParams(**n64_hw32_c32_3x3)
inputs = ng.placeholder(axes=cf.ax_i)
filters = ng.placeholder(axes=cf.ax_f)
# randomly initialize
input_value = rng.uniform(-0.5, 0.5, cf.ax_i)
filter_value = rng.uniform(-0.5, 0.5, cf.ax_f)
error_value = rng.uniform(-0.5, 0.5, cf.ax_o)
inputs = ng.placeholder(cf.ax_i)
filters = ng.placeholder(cf.ax_f)
errors = ng.placeholder(cf.ax_o)
output = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o)
bprop_out = bprop_conv(errors, inputs, filters, output)
updat_out = update_conv(errors, inputs, filters, output)
with executor([output, bprop_out, updat_out], inputs, filters, errors) as conv_executor:
result_ng, gradI_ng, gradF_ng = conv_executor(input_value, filter_value, error_value)
# Compute reference with NumPy
result_np, gradI_np, gradF_np = reference_conv(cf.dimI, cf.dimF, cf.dimO,
cf.conv_params,
input_value, filter_value, error_value)
# Compare fprop
assert np.allclose(result_ng, result_np, rtol=0, atol=0.5)
# Compare bprop
assert np.allclose(gradI_ng, gradI_np, rtol=0, atol=0.5)
# Compare update
assert np.allclose(gradF_ng, gradF_np, rtol=0, atol=2)
def test_convolution_simple():
element_type = Type.f32
image_shape = Shape([1, 1, 16, 16])
filter_shape = Shape([1, 1, 3, 3])
data = Parameter(element_type, image_shape)
filters = Parameter(element_type, filter_shape)
parameter_list = [data, filters]
image_arr = np.arange(-128, 128, 1, dtype=np.float32).reshape(1, 1, 16, 16)
filter_arr = np.ones(9, dtype=np.float32).reshape(1, 1, 3, 3)
filter_arr[0][0][0][0] = -1
filter_arr[0][0][1][1] = -1
filter_arr[0][0][2][2] = -1
filter_arr[0][0][0][2] = -1
filter_arr[0][0][2][0] = -1
strides = [1, 1]
pads_begin = [0, 0]
pads_end = [0, 0]
dilations = [1, 1]
model = ng.convolution(data, filters, strides, pads_begin, pads_end, dilations)
function = Function([model], parameter_list, "test")
runtime = get_runtime()
computation = runtime.computation(function, *parameter_list)
result = computation(image_arr, filter_arr)[0]
expected = convolution2d(image_arr[0][0], filter_arr[0][0]).reshape(1, 1, 14, 14)
assert np.allclose(result, expected)
def test_conv_flatten_deriv(n4_hw12_c3_5x5):
"""
Test deriv of conv followed by flatten
"""
cf = ConvParams(**n4_hw12_c3_5x5)
axes_rsck = ng.make_axes([cf.ax_f[2], cf.ax_f[3], cf.ax_f[0], cf.ax_f[-1]])
axes_rsck_prime = ng.make_axes([ng.make_axis(name=ax.name + 'p', length=ax.length)
for ax in axes_rsck])
axes_nmpqk = ng.make_axes([cf.ax_o[-1], cf.ax_o[1], cf.ax_o[2], cf.ax_o[3], cf.ax_o[0]])
# broadcast input / filter axes
input_var = ng.variable(cf.ax_i).named('input')
input_val = np.ones(input_var.axes.lengths)
filter_rsck_prime = ng.variable(axes_rsck_prime).named('filter')
filter_var = filter_rsck_prime
filter_rsck = ng.cast_axes(filter_rsck_prime, axes_rsck).named('frsck')
filter_trsck = ng.expand_dims(filter_rsck, cf.ax_f[1], 0).named('ftrsck')
filter_ctrsk = ng.axes_with_order(filter_trsck, axes=cf.ax_f).named('ctrsk')
# convolution
output_kmpqn = ng.convolution(cf.conv_params, input_var, filter_ctrsk, axes=cf.ax_o)
output_nmpqk = ng.axes_with_order(output_kmpqn, axes=axes_nmpqk)
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
output_npqk = ng.tensor_slice(output_nmpqk, out_slicing)
output = ng.flatten_at(output_npqk, idx=1)
# cost and grad
cost = ng.sum(output, out_axes=())
filter_val = np.ones(filter_var.axes.lengths)
with ExecutorFactory() as factory:
conv_comp = factory.executor(output, filter_var, input_var)
grad_filter_num_comp = factory.numeric_derivative(cost, filter_var, 1.0, input_var)
grad_filter_sym_comp = factory.derivative(cost, filter_var, input_var)
grad_input_num_comp = factory.numeric_derivative(cost, input_var, 1.0, filter_var)
grad_input_sym_comp = factory.derivative(cost, input_var, filter_var)
conv_val = conv_comp(filter_val, input_val)
conv_val_num = np.empty_like(conv_val)
conv_val_num.fill(np.prod(cf.ax_f.lengths[:-1]))
ng.testing.assert_allclose(conv_val, conv_val_num)
grad_filter_num_val = grad_filter_num_comp(filter_val, input_val)
grad_filter_sym_val = grad_filter_sym_comp(filter_val, input_val)
ng.testing.assert_allclose(grad_filter_num_val, grad_filter_sym_val)
grad_input_num_val = grad_input_num_comp(input_val, filter_val)
grad_input_sym_val = grad_input_sym_comp(input_val, filter_val)
ng.testing.assert_allclose(grad_input_num_val, grad_input_sym_val)
def test_wrong_input_shape_length():
"""
test wrong input shape length
"""
padding = dict(pad_d=0, pad_h=0, pad_w=0)
strides = dict(str_d=1, str_h=1, str_w=1)
conv_params = padding.copy()
conv_params.update(strides)
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W])
ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K])
inputs = ng.placeholder(ax_i)
filters = ng.placeholder(ax_f)
with pytest.raises(ValueError) as exinfo:
ng.convolution(conv_params, inputs, filters, {})
assert str(exinfo.value) == 'convolution input shape must be length 5, found {}'\
.format(len(ax_i))
def test_first_axes_not_same():
"""
test first axes are not the same
"""
padding = dict(pad_d=0, pad_h=0, pad_w=0)
strides = dict(str_d=1, str_h=1, str_w=1)
conv_params = padding.copy()
conv_params.update(strides)
ax_i = ng.make_axes([ax.D, ax.C, ax.H, ax.W, ax.N])
ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K])
inputs = ng.placeholder(ax_i)
filters = ng.placeholder(ax_f)
with pytest.raises(ValueError) as exinfo:
ng.convolution(conv_params, inputs, filters, {})
assert str(exinfo.value) == 'the first axis in input {inputs} and filter {filters} ' \
'are not the same.'.format(
inputs=inputs.axes[0],
filters=filters.axes[0])
def test_convolution_backprop(transformer_factory):
"""
test convolution backprop path
"""
N = 128
C, K = 3, 2
D, T = 1, 1
H = W = 32
R = S = 2
padding = dict(pad_d=0, pad_h=0, pad_w=0)
strides = dict(str_d=1, str_h=1, str_w=1)
dilation = dict(dil_d=1, dil_h=1, dil_w=1)
conv_params = padding.copy()
conv_params.update(strides)
conv_params.update(dilation)
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N])
ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K])
ax_i.set_shape((C, D, H, W, N))
ax_f.set_shape((C, T, R, S, K))
ax_o = ng.make_axes([
ng.make_axis(roles=[ar.features_input]).named('C'),
ng.make_axis(roles=[ar.features_0]).named('D'),
ng.make_axis(roles=[ar.features_1]).named('H'),
ng.make_axis(roles=[ar.features_2]).named('W'), ax.N
])
ax_o[:-1].set_shape((K, output_dim(D, T, padding['pad_d'],
strides['str_d']),
output_dim(H, R, padding['pad_h'], strides['str_h']),
output_dim(W, S, padding['pad_w'], strides['str_w'])))
inputs = ng.placeholder(axes=ax_i)
filters = ng.placeholder(axes=ax_f)
# randomly initialize
input_value = rng.uniform(-1, 1, ax_i)
filter_value = rng.uniform(-1, 1, ax_f)
assert input_value.shape == ax_i.lengths
assert filter_value.shape == ax_f.lengths
output = ng.sum(ng.convolution(conv_params, inputs, filters, ax_o),
out_axes=())
with ExecutorFactory() as factory:
dcdf_sym_fun = factory.derivative(output, filters, inputs)
dcdf_num_fun = factory.numeric_derivative(output, filters, .01, inputs)
dcdf_sym_val = dcdf_sym_fun(filter_value, input_value)
dcdf_num_val = dcdf_num_fun(filter_value, input_value)
ng.testing.assert_allclose(dcdf_sym_val, dcdf_num_val, rtol=1)
def execute_convolution(image_height,
image_width,
filter_height,
filter_width,
channel=16,
batch_size=32,
filter_count=8,
image_3rd_dim=1,
filter_3rd_dim=1,
padding=(0, 0, 0),
stride=(1, 1, 1),
dilation=1,
np_comparison=False):
pad_h, pad_w, pad_d = padding
str_h, str_w, str_d = stride
cf = ConvParams(C=channel,
N=batch_size,
K=filter_count,
D=image_3rd_dim,
H=image_height,
W=image_width,
T=filter_3rd_dim,
R=filter_height,
S=filter_width,
pad_d=pad_d,
pad_h=pad_h,
pad_w=pad_w,
str_d=str_d,
str_h=str_h,
str_w=str_w,
dil_d=dilation,
dil_h=dilation,
dil_w=dilation)
inputs = ng.placeholder(cf.ax_i)
filters = ng.placeholder(cf.ax_f)
rng = RandomTensorGenerator(0, np.float32)
input_value = rng.uniform(-4, 4, cf.ax_i, dtype=int)
filter_value = rng.uniform(-4, 4, cf.ax_f, dtype=int)
error_value = rng.uniform(-0.5, 0.5, cf.ax_o)
with executor(
ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o),
inputs, filters) as const_executor:
out = const_executor(input_value, filter_value)
if np_comparison:
np_out, gradInp, gradF_np = \
reference_conv(cf.dimI, cf.dimF, cf.dimO, cf.conv_params, input_value, filter_value,
error_value)
return out, np_out
return out
def test_conv_flatten_deriv(transformer_factory):
"""
Test deriv of conv followed by flatten
"""
# set shape
C, D, H, W, N = (3, 1, 28, 28, 8)
C, T, R, S, K = (3, 1, 5, 5, 32)
# i, f, o axes
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N])
ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K])
ax_o = ng.make_axes([
ng.make_axis(32, roles=[ar.Channel]),
ng.make_axis(1, roles=[ar.Depth]),
ng.make_axis(24, roles=[ar.Height]),
ng.make_axis(24, roles=[ar.Width]), ax.N
])
ax_i.set_shape((C, D, H, W, N))
ax_f.set_shape((C, T, R, S, K))
params = dict(pad_d=0, pad_h=0, pad_w=0, str_d=1, str_h=1, str_w=1)
axes_rsck = ng.make_axes([ax.R, ax.S, ax.C, ax.K])
axes_rsck_prime = ng.make_axes(
[ng.make_axis(l) for l in axes_rsck.lengths])
# broadcast input / filter axes
image = ng.constant(np.ones(ax_i.lengths), ax_i)
filter = ng.variable(axes_rsck_prime, initial_value=np.ones((R, S, C, K)))
filter_casted = ng.cast_axes(filter, axes_rsck)
filter_casted = ng.expand_dims(filter_casted, ax.T, 0)
filter_casted = ng.axes_with_order(filter_casted, axes=ax_f)
# convolution
output = ng.convolution(params, image, filter_casted, axes=ax_o)
oC, oD, oH, oW, oN = output.axes
output = ng.axes_with_order(output,
axes=ng.make_axes([oN, oD, oH, oW, oC]))
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
conv = ng.Slice(output, out_slicing)
flatten = ng.flatten_at(conv, idx=1)
# cost and grad
cost = ng.sum(flatten, reduction_axes=flatten.axes)
grad = ng.deriv(cost, filter)
# compute
conv_grad_comp = executor([conv, grad])
conv_val, grad_val = conv_grad_comp()
assert np.allclose(conv_val, np.zeros_like(conv_val) + 75.)
assert np.allclose(grad_val, np.zeros_like(grad_val) + 4608.)
def _convolution_op(self, cntk_op, inputs):
"""
Computes the convolution of a tensor with operand.
CNTK Ngraph
in ((C, H, W) N)) (C, D, H, W, N)
filter (O, C, M1, M2) (C, T, M1, M2, O)
out ((O, H, W) N) (O, M, H, W, N)
Arguments:
cntk_op: CNTK operation to be imported.
inputs: List of inputs to this node.
Returns:
A ngraph Op.
"""
filters, inputs = inputs
inputs = self._expand_input_axes(inputs)
C, D, H, W, N = inputs.axes
filters = self._expand_filters_axes(filters, C)
_, T, M1, M2, O = filters.axes
M, oH, oW = self._make_out_axes(cntk_op.shape)
out_axes = [O, M, oH, oW, N]
strides = self._make_strides(cntk_op.attributes['strides'])
pad = self._make_padding('conv', cntk_op.attributes['autoPadding'],
(D.length, H.length, W.length, C.length),
(T.length, M1.length, M2.length, C.length),
(M.length, oH.length, oW.length, O.length),
strides)
params = dict(pad_d=pad[0],
pad_h=pad[1],
pad_w=pad[2],
pad_c=pad[3],
str_d=strides[0],
str_h=strides[1],
str_w=strides[2],
str_c=strides[3],
dil_d=1,
dil_h=1,
dil_w=1)
conv = ng.convolution(params, inputs, filters, out_axes)
return remove_ones_axes([conv])[0]
def test_convolution_simple():
element_type = Type.f32
image_shape = Shape([1, 1, 16, 16])
filter_shape = Shape([1, 1, 3, 3])
data = Parameter(element_type, image_shape)
filters = Parameter(element_type, filter_shape)
parameter_list = [data, filters]
image_arr = np.arange(-128, 128, 1, dtype=np.float32).reshape(1, 1, 16, 16)
filter_arr = np.ones(9, dtype=np.float32).reshape(1, 1, 3, 3)
filter_arr[0][0][0][0] = -1
filter_arr[0][0][1][1] = -1
filter_arr[0][0][2][2] = -1
filter_arr[0][0][0][2] = -1
filter_arr[0][0][2][0] = -1
result_arr = np.zeros(196, dtype=np.float32).reshape(1, 1, 14, 14)
strides = [1, 1]
pads_begin = [0, 0]
pads_end = [0, 0]
dilations = [1, 1]
model = ng.convolution(data, filters, strides, pads_begin, pads_end,
dilations)
function = Function([model], parameter_list, "test")
backend = Backend.create(test.BACKEND_NAME)
a = backend.create_tensor(element_type, image_shape)
b = backend.create_tensor(element_type, filter_shape)
a.write(util.numpy_to_c(image_arr), 16 * 16 * 4)
b.write(util.numpy_to_c(filter_arr), 3 * 3 * 4)
result = backend.create_tensor(element_type, Shape([1, 1, 14, 14]))
result.write(util.numpy_to_c(result_arr), 14 * 14 * 4)
handle = backend.compile(function)
handle.call([result], [a, b])
result.read(util.numpy_to_c(result_arr), 14 * 14 * 4)
result_arr_ref = convolution2d(image_arr[0][0],
filter_arr[0][0]).reshape(1, 1, 14, 14)
assert np.allclose(result_arr, result_arr_ref)
def test_convolution_backprop(n128_hw32_c3_2x2):
"""
test convolution backprop path
"""
cf = ConvParams(**n128_hw32_c3_2x2)
inputs = ng.placeholder(axes=cf.ax_i)
filters = ng.placeholder(axes=cf.ax_f)
# randomly initialize
input_value = rng.uniform(-1, 1, cf.ax_i)
filter_value = rng.uniform(-1, 1, cf.ax_f)
output = ng.sum(ng.convolution(cf.conv_params, inputs, filters, cf.ax_o), out_axes=())
with ExecutorFactory() as factory:
dcdf_sym_fun = factory.derivative(output, filters, inputs)
dcdf_num_fun = factory.numeric_derivative(output, filters, .01, inputs)
dcdf_sym_val = dcdf_sym_fun(filter_value, input_value)
dcdf_num_val = dcdf_num_fun(filter_value, input_value)
ng.testing.assert_allclose(dcdf_sym_val, dcdf_num_val, rtol=0.01)
def test_convolution_with_padding():
element_type = Type.f32
image_shape = Shape([1, 1, 10, 10])
filter_shape = Shape([1, 1, 3, 3])
data = Parameter(element_type, image_shape)
filters = Parameter(element_type, filter_shape)
parameter_list = [data, filters]
image_arr = np.arange(100, dtype=np.float32).reshape(1, 1, 10, 10)
filter_arr = np.zeros(9, dtype=np.float32).reshape(1, 1, 3, 3)
filter_arr[0][0][1][1] = 1
strides = [1, 1]
dilations = [2, 2]
pads_begin = [0, 0]
pads_end = [0, 0]
model = ng.convolution(data, filters, strides, pads_begin, pads_end,
dilations)
function = Function([model], parameter_list, "test")
backend = Backend.create(test.BACKEND_NAME)
a = backend.create_tensor(element_type, image_shape)
b = backend.create_tensor(element_type, filter_shape)
a.write(util.numpy_to_c(image_arr), 10 * 10 * 4)
b.write(util.numpy_to_c(filter_arr), 3 * 3 * 4)
result_arr = np.zeros(36, dtype=np.float32).reshape(1, 1, 6, 6)
result = backend.create_tensor(element_type, Shape([1, 1, 6, 6]))
result.write(util.numpy_to_c(result_arr), 6 * 6 * 4)
handle = backend.compile(function)
handle.call([result], [a, b])
result.read(util.numpy_to_c(result_arr), 6 * 6 * 4)
result_arr_ref = convolution2d(image_arr[0][0], filter_arr[0][0], strides,
dilations, pads_begin,
pads_end).reshape(1, 1, 6, 6)
assert np.allclose(result_arr, result_arr_ref)
def train_outputs(self, in_obj):
cpm = self.convparams.copy()
in_axes = in_obj.axes
if self.f_axes is None:
self.f_axes = in_axes.role_axes(ar.Channel)
for _ax in (ax.T, ax.R, ax.S, ax.K):
self.f_axes += ng.make_axis(roles=_ax.roles).named(_ax.short_name)
self.f_axes[1:].set_shape(itemgetter(*'TRSK')(cpm))
self.W = ng.variable(axes=self.f_axes, initial_value=self.init(self.f_axes.lengths))
# TODO: clean this up
if self.o_axes is None:
self.o_axes = ng.make_axes([
ng.make_axis(self.f_axes[4].length, roles=[ar.Channel]).named('C'),
ng.spatial_axis(in_axes, self.f_axes, cpm['pad_d'], cpm['str_d'], role=ar.Depth),
ng.spatial_axis(in_axes, self.f_axes, cpm['pad_h'], cpm['str_h'], role=ar.Height),
ng.spatial_axis(in_axes, self.f_axes, cpm['pad_w'], cpm['str_w'], role=ar.Width),
ax.N
])
return ng.convolution(cpm, in_obj, self.W, axes=self.o_axes)
def __call__(self, in_obj):
cpm = self.convparams.copy()
in_obj = reorder_spatial_axes(in_obj)
in_axes = in_obj.axes
if self.f_axes is None:
self.f_axes = ng.make_axes([in_axes[0]])
for nm in 'TRSK':
self.f_axes |= ng.make_axis(length=cpm[nm], name=nm)
# mark 'K' as a shadow axis for the initializers.
self.axes_map = shadow_axes_map(self.f_axes.find_by_name('K'))
self.f_axes = ng.make_axes([
axis if axis.name != 'K' else list(self.axes_map.keys())[0]
for axis in self.f_axes
])
self.W = ng.variable(axes=self.f_axes, initial_value=self.init,
scope=self.scope).named('convwt')
if self.o_axes is None:
self.o_axes = ng.make_axes([
ng.make_axis(name=a.name) for a in in_axes if not a.is_batch
])
# set lengths
out_shape = [
self.f_axes[-1].length,
output_dim(in_axes[1].length, cpm['T'], cpm['pad_d'], cpm['str_d'], False,
cpm['dil_d']),
output_dim(in_axes[2].length, cpm['R'], cpm['pad_h'], cpm['str_h'], False,
cpm['dil_h']),
output_dim(in_axes[3].length, cpm['S'], cpm['pad_w'], cpm['str_w'], False,
cpm['dil_w'])
]
self.o_axes.set_shape(out_shape)
self.o_axes |= in_axes.batch_axis()
return ng.map_roles(ng.convolution(cpm, in_obj, self.W, axes=self.o_axes), self.axes_map)
def Conv(self, c2_op, inputs):
"""
Computes a 2-D convolution given 4D input and filter tensors.
Arguments:
c2_op: NodeDef object, the caffe2 node to convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the caffe2 node.
Inputs to c2_op:
input, wegiths, filter
Supports caffe2's layout NHWC and NCHW as well.
"""
X, W, bias = inputs
order = [val.s for val in c2_op.arg if val.name == "order"]
if 1 != len(order):
raise ValueError("Multiple order values in convolution")
order = order[0]
if order not in ("NHWC", "NCHW"):
raise NotImplementedError("Unsupported order in convolution: {}",
order)
# set input axes shape
ax_N = ng.make_axis(name='N')
ax_C = ng.make_axis()
ax_D = ng.make_axis(length=1)
ax_H = ng.make_axis()
ax_W = ng.make_axis()
# set kernel axes shape
ax_kernel_D = ng.make_axis(length=1)
ax_kernel_H = ng.make_axis()
ax_kernel_W = ng.make_axis()
ax_kernel_ofm = ng.make_axis()
# create placeholders for output axes
oC = ng.make_axis(name='C')
oD = ng.make_axis(name='D', length=1)
oH = ng.make_axis(name='H')
oW = ng.make_axis(name='W')
axes_order = {
'NCHW': {
'X': [ax_N, ax_C, ax_H, ax_W],
'W': [ax_kernel_ofm, ax_C, ax_kernel_H, ax_kernel_W]
},
'NHWC': {
'X': [ax_N, ax_H, ax_W, ax_C],
'W': [ax_kernel_ofm, ax_kernel_H, ax_kernel_W, ax_C]
},
}
ng.make_axes(axes_order[order]['X']).set_shape(X.axes.lengths)
ng.make_axes(axes_order[order]['W']).set_shape(W.axes.lengths)
if 1 != len(bias.axes):
raise ValueError("Bias's must be 1D.")
if ax_kernel_ofm.length != bias.axes.lengths[0]:
raise ValueError(
"Bias's length must equal to number of output feature maps.")
# strides params
stride_size = [int(val.i) for val in c2_op.arg if val.name == "stride"]
if len(stride_size) != 1:
raise ValueError("Stride size must be scalar value")
str_h = str_w = stride_size[0]
# padding params
pad_t, pad_b, pad_l, pad_r = \
_c2_padding(c2_op,
in_NHWC=[ax_N.length, ax_H.length, ax_W.length, ax_C.length],
kernel_HWIO=[ax_kernel_H.length, ax_kernel_W.length,
ax_C.length, ax_kernel_ofm.length],
stride_NHWC=[1, str_h, str_w, 1])
if pad_t != pad_b or pad_l != pad_r:
raise NotImplementedError("Requires symmetric padding in ngraph:"
"pad_t(%s) == pad_b(%s) and"
"pad_l(%s) == pad_r(%s)" %
(pad_t, pad_b, pad_l, pad_r))
# conv params
params = dict(pad_d=0,
pad_h=pad_t,
pad_w=pad_l,
str_d=1,
str_h=str_h,
str_w=str_w,
dil_d=1,
dil_h=1,
dil_w=1)
# input, weight, output axes
internal_ax_dict = {
'X':
ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N]),
'W':
ng.make_axes(
[ax_C, ax_kernel_D, ax_kernel_H, ax_kernel_W, ax_kernel_ofm])
}
oC.length = ax_kernel_ofm.length
oH.length = output_dim(ax_H.length, ax_kernel_H.length,
params['pad_h'], params['str_h'])
oW.length = output_dim(ax_W.length, ax_kernel_W.length,
params['pad_w'], params['str_w'])
internal_ax_dict['Y'] = ng.make_axes([oC, oD, oH, oW, ax_N])
# broadcast input / filter axes
# flow for NHWC order: | flow for NCHW order:
# input: | input:
# expand dims: NHWC -> NDHWC | expand dims: NCHW -> NDCHW
# reorder: NDHWC -> CDHWN | reorder: NDCHW -> CDHWN
# weights: | weights:
# expand dims: (ofm)HWC -> D(ofm)HWC | expand dims: (ofm)CHWC -> D(ofm)CHW
# reorder: D(ofm)HWC -> CDHW(ofm) | reorder: D(ofm)CHW -> CDHW(ofm)
X = ng.cast_axes(X, ng.make_axes(axes_order[order]['X']))
X = ng.expand_dims(X, ax_D, 1)
X = ng.axes_with_order(X, axes=internal_ax_dict['X'])
W = ng.cast_axes(W, ng.make_axes(axes_order[order]['W']))
W = ng.expand_dims(W, ax_kernel_D, 0)
W = ng.axes_with_order(W, axes=internal_ax_dict['W'])
# convolution
Y = ng.convolution(params, X, W, axes=internal_ax_dict['Y'])
# cast back to proper format
Y = ng.broadcast(Y, ng.make_axes([ax_N, oD, oH, oW, oC])) if "NHWC" == order \
else ng.broadcast(Y, ng.make_axes([ax_N, oD, oC, oH, oW])) # NCHW
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
Y = ng.tensor_slice(Y, out_slicing)
def _conv_bias_add(c2_op, inputs):
X, bias = inputs
bias = ng.cast_axes(bias,
axes=ng.make_axes(
[X.axes[1 if 'NCHW' == order else 3]]))
Y = ng.Add(X, bias)
return Y
return _conv_bias_add(c2_op, [Y, bias])
def create_ngraph_function(args) -> Function:
weights = np.fromfile(args.model, dtype=np.float32)
weights_offset = 0
padding_begin = [0, 0]
padding_end = [0, 0]
# input
input_shape = [64, 1, 28, 28]
param_node = ngraph.parameter(input_shape, np.float32, 'Parameter')
# convolution 1
conv_1_kernel_shape, conv_1_kernel_length = shape_and_length([20, 1, 5, 5])
conv_1_kernel = ngraph.constant(
weights[0:conv_1_kernel_length].reshape(conv_1_kernel_shape))
weights_offset += conv_1_kernel_length
conv_1_node = ngraph.convolution(param_node, conv_1_kernel, [1, 1],
padding_begin, padding_end, [1, 1])
# add 1
add_1_kernel_shape, add_1_kernel_length = shape_and_length([1, 20, 1, 1])
add_1_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
add_1_kernel_length].reshape(add_1_kernel_shape))
weights_offset += add_1_kernel_length
add_1_node = ngraph.add(conv_1_node, add_1_kernel)
# maxpool 1
maxpool_1_node = ngraph.max_pool(add_1_node, [2, 2], padding_begin,
padding_end, [2, 2], 'ceil', None)
# convolution 2
conv_2_kernel_shape, conv_2_kernel_length = shape_and_length(
[50, 20, 5, 5])
conv_2_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
conv_2_kernel_length].reshape(conv_2_kernel_shape))
weights_offset += conv_2_kernel_length
conv_2_node = ngraph.convolution(maxpool_1_node, conv_2_kernel, [1, 1],
padding_begin, padding_end, [1, 1])
# add 2
add_2_kernel_shape, add_2_kernel_length = shape_and_length([1, 50, 1, 1])
add_2_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
add_2_kernel_length].reshape(add_2_kernel_shape))
weights_offset += add_2_kernel_length
add_2_node = ngraph.add(conv_2_node, add_2_kernel)
# maxpool 2
maxpool_2_node = ngraph.max_pool(add_2_node, [2, 2], padding_begin,
padding_end, [2, 2], 'ceil', None)
# reshape 1
reshape_1_dims, reshape_1_length = shape_and_length([2])
# workaround to get int64 weights from float32 ndarray w/o unnecessary copying
dtype_weights = np.frombuffer(weights[weights_offset:weights_offset +
2 * reshape_1_length],
dtype=np.int64)
reshape_1_kernel = ngraph.constant(dtype_weights)
weights_offset += 2 * reshape_1_length
reshape_1_node = ngraph.reshape(maxpool_2_node, reshape_1_kernel, True)
# matmul 1
matmul_1_kernel_shape, matmul_1_kernel_length = shape_and_length(
[500, 800])
matmul_1_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
matmul_1_kernel_length].reshape(matmul_1_kernel_shape))
weights_offset += matmul_1_kernel_length
matmul_1_node = ngraph.matmul(reshape_1_node, matmul_1_kernel, False, True)
# add 3
add_3_kernel_shape, add_3_kernel_length = shape_and_length([1, 500])
add_3_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
add_3_kernel_length].reshape(add_3_kernel_shape))
weights_offset += add_3_kernel_length
add_3_node = ngraph.add(matmul_1_node, add_3_kernel)
# ReLU
relu_node = ngraph.relu(add_3_node)
# reshape 2
reshape_2_kernel = ngraph.constant(dtype_weights)
reshape_2_node = ngraph.reshape(relu_node, reshape_2_kernel, True)
# matmul 2
matmul_2_kernel_shape, matmul_2_kernel_length = shape_and_length([10, 500])
matmul_2_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
matmul_2_kernel_length].reshape(matmul_2_kernel_shape))
weights_offset += matmul_2_kernel_length
matmul_2_node = ngraph.matmul(reshape_2_node, matmul_2_kernel, False, True)
# add 4
add_4_kernel_shape, add_4_kernel_length = shape_and_length([1, 10])
add_4_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
add_4_kernel_length].reshape(add_4_kernel_shape))
weights_offset += add_4_kernel_length
add_4_node = ngraph.add(matmul_2_node, add_4_kernel)
# softmax
softmax_axis = 1
softmax_node = ngraph.softmax(add_4_node, softmax_axis)
# result
result_node = ngraph.result(softmax_node)
# nGraph function
function = Function(result_node, [param_node], 'lenet')
return function
def test_convolution(transformer_factory):
"""
test convolution forward path
"""
N = 128
C, K = 3, 8
D, T = 1, 1
H = W = 32
R = S = 2
padding = dict(pad_d=0, pad_h=0, pad_w=0)
strides = dict(str_d=1, str_h=1, str_w=1)
conv_params = padding.copy()
conv_params.update(strides)
ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N])
ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K])
ax_i.set_shape((C, D, H, W, N))
ax_f.set_shape((C, T, R, S, K))
ax_o = ng.make_axes([
ng.make_axis(ax_f.role_axes(ar.Channelout)[0].length,
name='C',
roles=[ar.Channel]),
spatial_axis(ax_i,
ax_f,
padding['pad_d'],
strides['str_d'],
role=ar.Depth),
spatial_axis(ax_i,
ax_f,
padding['pad_h'],
strides['str_h'],
role=ar.Height),
spatial_axis(ax_i,
ax_f,
padding['pad_w'],
strides['str_w'],
role=ar.Width), ax.N
])
inputs = ng.placeholder(axes=ax_i)
filters = ng.placeholder(axes=ax_f)
# randomly initialize
input_value = rng.uniform(-1, 1, ax_i)
filter_value = rng.uniform(-1, 1, ax_f)
assert input_value.shape == ax_i.lengths
assert filter_value.shape == ax_f.lengths
inputs = ng.placeholder(ax_i)
filters = ng.placeholder(ax_f)
output = ng.convolution(conv_params, inputs, filters, axes=ax_o)
targets = ng.placeholder(axes=output.axes)
costs = ng.cross_entropy_binary(ng.sigmoid(output), targets)
error = ng.sum(costs, out_axes=()) / ng.batch_size(costs)
d_inputs = ng.deriv(error, inputs)
d_filters = ng.deriv(error, filters)
targets_value = rng.uniform(.1, 0.9, output.axes)
conv_executor = executor([output, error, d_inputs, d_filters], inputs,
filters, targets)
result_ng, err_ng, gradI_ng, gradF_ng = conv_executor(
input_value, filter_value, targets_value)
# Now compute reference values via NEON
NervanaObject.be.bsz = N
neon_layer = Convolution(fshape=(R, S, K),
padding=padding,
strides=strides)
inp = neon_layer.be.array(input_value.reshape(C * H * W * D, N))
neon_layer.W = neon_layer.be.array(filter_value.reshape(C * R * S * T, K))
neon_layer.dW = neon_layer.be.empty_like(neon_layer.W)
neon_layer.configure((C, H, W))
neon_layer.prev_layer = True
neon_layer.allocate()
neon_layer.set_deltas(DummyDeltaBuffers())
result_ne = neon_layer.fprop(inp).get().reshape(output.axes.lengths)
act_result_ne = 1. / (1.0 + np.exp(-result_ne))
err = neon_layer.be.array(
(act_result_ne - targets_value).reshape(-1, N) / float(N))
gradI_ne = neon_layer.bprop(err).get().reshape(ax_i.lengths)
gradF_ne = neon_layer.dW.get().reshape(ax_f.lengths)
# Compare fprop
np.testing.assert_allclose(result_ng, result_ne, rtol=0, atol=1e-6)
# Compare bprop
np.testing.assert_allclose(gradI_ng, gradI_ne, rtol=0, atol=1e-6)
# Compare update
np.testing.assert_allclose(gradF_ng, gradF_ne, rtol=0, atol=1e-4)
def create_ngraph_function(args: argparse.Namespace) -> ngraph.impl.Function:
"""Create a network on the fly from the source code using ngraph"""
def shape_and_length(shape: list) -> typing.Tuple[list, int]:
length = reduce(lambda x, y: x * y, shape)
return shape, length
weights = np.fromfile(args.model, dtype=np.float32)
weights_offset = 0
padding_begin = padding_end = [0, 0]
# input
input_shape = [64, 1, 28, 28]
param_node = ngraph.parameter(input_shape, np.float32, 'Parameter')
# convolution 1
conv_1_kernel_shape, conv_1_kernel_length = shape_and_length([20, 1, 5, 5])
conv_1_kernel = ngraph.constant(
weights[0:conv_1_kernel_length].reshape(conv_1_kernel_shape))
weights_offset += conv_1_kernel_length
conv_1_node = ngraph.convolution(param_node, conv_1_kernel, [1, 1],
padding_begin, padding_end, [1, 1])
# add 1
add_1_kernel_shape, add_1_kernel_length = shape_and_length([1, 20, 1, 1])
add_1_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
add_1_kernel_length].reshape(add_1_kernel_shape), )
weights_offset += add_1_kernel_length
add_1_node = ngraph.add(conv_1_node, add_1_kernel)
# maxpool 1
maxpool_1_node = ngraph.max_pool(add_1_node, [2, 2], padding_begin,
padding_end, [2, 2], 'ceil', None)
# convolution 2
conv_2_kernel_shape, conv_2_kernel_length = shape_and_length(
[50, 20, 5, 5])
conv_2_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
conv_2_kernel_length].reshape(conv_2_kernel_shape), )
weights_offset += conv_2_kernel_length
conv_2_node = ngraph.convolution(maxpool_1_node, conv_2_kernel, [1, 1],
padding_begin, padding_end, [1, 1])
# add 2
add_2_kernel_shape, add_2_kernel_length = shape_and_length([1, 50, 1, 1])
add_2_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
add_2_kernel_length].reshape(add_2_kernel_shape), )
weights_offset += add_2_kernel_length
add_2_node = ngraph.add(conv_2_node, add_2_kernel)
# maxpool 2
maxpool_2_node = ngraph.max_pool(add_2_node, [2, 2], padding_begin,
padding_end, [2, 2], 'ceil', None)
# reshape 1
reshape_1_dims, reshape_1_length = shape_and_length([2])
# workaround to get int64 weights from float32 ndarray w/o unnecessary copying
dtype_weights = np.frombuffer(
weights[weights_offset:weights_offset + 2 * reshape_1_length],
dtype=np.int64,
)
reshape_1_kernel = ngraph.constant(dtype_weights)
weights_offset += 2 * reshape_1_length
reshape_1_node = ngraph.reshape(maxpool_2_node, reshape_1_kernel, True)
# matmul 1
matmul_1_kernel_shape, matmul_1_kernel_length = shape_and_length(
[500, 800])
matmul_1_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
matmul_1_kernel_length].reshape(matmul_1_kernel_shape), )
weights_offset += matmul_1_kernel_length
matmul_1_node = ngraph.matmul(reshape_1_node, matmul_1_kernel, False, True)
# add 3
add_3_kernel_shape, add_3_kernel_length = shape_and_length([1, 500])
add_3_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
add_3_kernel_length].reshape(add_3_kernel_shape), )
weights_offset += add_3_kernel_length
add_3_node = ngraph.add(matmul_1_node, add_3_kernel)
# ReLU
relu_node = ngraph.relu(add_3_node)
# reshape 2
reshape_2_kernel = ngraph.constant(dtype_weights)
reshape_2_node = ngraph.reshape(relu_node, reshape_2_kernel, True)
# matmul 2
matmul_2_kernel_shape, matmul_2_kernel_length = shape_and_length([10, 500])
matmul_2_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
matmul_2_kernel_length].reshape(matmul_2_kernel_shape), )
weights_offset += matmul_2_kernel_length
matmul_2_node = ngraph.matmul(reshape_2_node, matmul_2_kernel, False, True)
# add 4
add_4_kernel_shape, add_4_kernel_length = shape_and_length([1, 10])
add_4_kernel = ngraph.constant(
weights[weights_offset:weights_offset +
add_4_kernel_length].reshape(add_4_kernel_shape), )
weights_offset += add_4_kernel_length
add_4_node = ngraph.add(matmul_2_node, add_4_kernel)
# softmax
softmax_axis = 1
softmax_node = ngraph.softmax(add_4_node, softmax_axis)
# result
result_node = ngraph.result(softmax_node)
return ngraph.impl.Function(result_node, [param_node], 'lenet')
def test_default_arguments_convolution_2d():
manager_name = pytest.config.getoption('backend', default='CPU')
runtime = ng.runtime(manager_name=manager_name)
# input_x should have shape N(batch) x C x H x W
input_x = ng.constant(
np.array([[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.]],
dtype=np.float32).reshape(1, 1, 9, 9))
# filter weights should have shape M x C x kH x kW
input_filter = ng.constant(
np.array([[1., 0., -1.], [2., 0., -2.], [1., 0., -1.]],
dtype=np.float32).reshape(1, 1, 3, 3))
# convolution with padding=1 should produce 9 x 9 output:
model = runtime.computation(
ng.convolution(input_x,
input_filter,
padding_above=[1, 1],
padding_below=[1, 1]))
result = model()
assert np.array_equal(
result,
np.array([[[[0., -15., -15., 15., 15., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -15., -15., 15., 15., 0., 0., 0., 0.]]]],
dtype=np.float32))
# convolution with padding=0 should produce 7 x 7 output:
model = runtime.computation(ng.convolution(input_x, input_filter))
result = model()
assert np.array_equal(
result,
np.array([[[[-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0],
[-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0],
[-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0],
[-20, -20, 20, 20, 0, 0, 0]]]],
dtype=np.float32))
# convolution with strides=2 should produce 4 x 4 output:
model = runtime.computation(
ng.convolution(input_x, input_filter, strides=[2, 2]))
result = model()
assert np.array_equal(
result,
np.array([[[[-20., 20., 0., 0.], [-20., 20., 0., 0.],
[-20., 20., 0., 0.], [-20., 20., 0., 0.]]]],
dtype=np.float32))
# convolution with dilation=2 should produce 5 x 5 output:
model = runtime.computation(
ng.convolution(input_x, input_filter, dilation=(2, 2)))
result = model()
assert np.array_equal(
result,
np.array([[[[0, 0, 20, 20, 0], [0, 0, 20, 20, 0], [0, 0, 20, 20, 0],
[0, 0, 20, 20, 0], [0, 0, 20, 20, 0]]]],
dtype=np.float32))
def test_recvop_tensorupdate():
"""
The tensor (RecvOp_#_#) associated with the following conv op has two views:
1) Non-flat view (e.g. RecvOp_#_#_1_1_1_1_4.shape=(1,1,1,1,4))
2) Flat view (e.g. RecvOp_#_#_1_4.shape = (1,4))
This test ensures that inside RecvOp code generation, the generated code
should make sure both views get updated (e.g. by using update_RecvOp_#_# API)
In this test, ng.dot operation tends to use the flat view (i.e. RecvOp_#_#_1_4)
And previously RecvOp with RecvOp_#_#_1_1_1_1_4 = recv_from_send(send_id) failed
to update both two views (i.e. flat and non-flat view of the same buffer/tensor)
"""
class ConvParams(object):
def __init__(self,
C=1,
N=1,
K=1,
D=1,
H=1,
W=1,
T=1,
R=1,
S=1,
pad_d=0,
pad_h=0,
pad_w=0,
str_d=1,
str_h=1,
str_w=1):
from ngraph.frontends.common.utils import conv_output_dim
M = conv_output_dim(D, T, pad_d, str_d)
P = conv_output_dim(H, R, pad_h, str_h)
Q = conv_output_dim(W, S, pad_w, str_w)
self.dimO = (K, M, P, Q, N)
self.dimI = (C, D, H, W, N)
self.dimF = (C, T, R, S, K)
self.conv_params = dict(pad_d=pad_d,
pad_h=pad_h,
pad_w=pad_w,
str_d=str_d,
str_h=str_h,
str_w=str_w,
dil_d=1,
dil_h=1,
dil_w=1)
self.batch_axis = ng.make_axis(name='N', length=N)
self.ax_i = ng.make_axes([
ng.make_axis(name='C', length=C),
ng.make_axis(name='D', length=D),
ng.make_axis(name='H', length=H),
ng.make_axis(name='W', length=W), self.batch_axis
])
self.ax_f = ng.make_axes([
ng.make_axis(name='C', length=C),
ng.make_axis(name='D', length=T),
ng.make_axis(name='H', length=R),
ng.make_axis(name='W', length=S),
ng.make_axis(name='K', length=K),
])
self.ax_o = ng.make_axes([
ng.make_axis(name='C', length=K),
ng.make_axis(name='D', length=M),
ng.make_axis(name='H', length=P),
ng.make_axis(name='W', length=Q), self.batch_axis
])
# Layer 1, using convolutation introduces multi/flatten view of tensors
cf = ConvParams(C=2, N=4, K=1, H=2, W=2, R=2, S=2)
inputs = ng.placeholder(axes=cf.ax_i)
filters = ng.placeholder(axes=cf.ax_f)
# randomly initialize
from ngraph.testing import RandomTensorGenerator
rng = RandomTensorGenerator(0, np.float32)
# put value 1 into inputs/filters for conv
input_value = rng.uniform(1, 1, cf.ax_i)
filter_value = rng.uniform(1, 1, cf.ax_f)
conv = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o)
# Layer 2, using dot to ensure recv_op.axes == send_op.axes
from ngraph.frontends.neon import UniformInit
# put value 1 into weights for dot
init_uni = UniformInit(1, 1)
W_A = ng.make_axis(length=2)
w_axes = ng.make_axes(W_A) + conv.axes.feature_axes()
w = ng.variable(axes=w_axes, initial_value=init_uni)
with ng.metadata(device_id='1'):
dot = ng.dot(w, conv)
with ExecutorFactory() as ex:
dot_comp = ex.executor(dot, filters, inputs)
dot_val = dot_comp(filter_value, input_value)
np.testing.assert_array_equal(dot_val,
[[8., 8., 8., 8.], [8., 8., 8., 8.]])
def Conv2D(self, tf_node, inputs):
"""
Computes a 2-D convolution given 4D input and filter tensors.
Arguments:
tf_node: NodeDef object, the tensorflow node to convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the tensorflow node.
Inputs to tf_node:
input, filter
TODO: assume default tensorflow layout NHWC, RSCK,
need to support NCHW as well
need to clean up / merge with maxpool
Notes on output shape:
https://www.tensorflow.org/api_docs/python/nn.html#convolution
"""
image, weight = inputs
# TODO: currently NHWC only
assert tf_node.attr['data_format'].s.decode("ascii") == "NHWC"
# set axes shape
ax_N = ng.make_axis(batch=True)
ax_C = ng.make_axis(roles=[ar.Channel])
ax_D = ng.make_axis(roles=[ar.Depth])
ax_H = ng.make_axis(roles=[ar.Height])
ax_W = ng.make_axis(roles=[ar.Width])
ax_T = ng.make_axis(roles=[ar.Depth])
ax_R = ng.make_axis(roles=[ar.Height])
ax_S = ng.make_axis(roles=[ar.Width])
ax_K = ng.make_axis(roles=[ar.Channelout])
ng.make_axes([ax_N, ax_H, ax_W, ax_C]).set_shape(image.axes.lengths)
ng.make_axes([ax_R, ax_S, ax_C, ax_K]).set_shape(weight.axes.lengths)
ax_D.length = 1
ax_T.length = 1
# strides params
tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)]
if len(tf_strides) != 4:
raise ValueError("Length of strides my be 4.")
if tf_strides[0] != 1:
raise NotImplementedError('Strides on batch axis (N) must be 1.')
if tf_strides[3] != 1:
raise NotImplementedError('Strides on channel axis (C) must be 1.')
str_h, str_w = tf_strides[1], tf_strides[2]
# padding params
padding = tf_node.attr['padding'].s.decode("ascii")
pad_t, pad_b, pad_l, pad_r = tf_conv2d_pool_padding(
image.axes.lengths, weight.axes.lengths, tf_strides, padding)
if pad_t != pad_b or pad_l != pad_r:
raise NotImplementedError("Requires symmetric padding in ngraph:"
"pad_t(%s) == pad_b(%s) and"
"pad_l(%s) == pad_r(%s)" %
(pad_t, pad_b, pad_l, pad_r))
# conv params
params = dict(pad_d=0,
pad_h=pad_t,
pad_w=pad_l,
str_d=1,
str_h=str_h,
str_w=str_w)
# i, f, o axes
ax_i = ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N])
ax_f = ng.make_axes([ax_C, ax_T, ax_R, ax_S, ax_K])
ax_o = ng.make_axes([
ng.make_axis(ax_K.length, name='C', roles=[ar.Channel]),
spatial_axis(ax_i, ax_f, params['pad_d'], params['str_d'],
ar.Depth),
spatial_axis(ax_i, ax_f, params['pad_h'], params['str_h'],
ar.Height),
spatial_axis(ax_i, ax_f, params['pad_w'], params['str_w'],
ar.Width), ax_N
])
# broadcast input / filter axes
image = ng.cast_axes(image, ng.make_axes([ax_N, ax_H, ax_W, ax_C]))
image = ng.expand_dims(image, ax_D, 1) # NHWC -> NDHWC
image = ng.axes_with_order(image, axes=ax_i) # NDHWC -> CDHWN
weight = ng.cast_axes(weight, ng.make_axes([ax_R, ax_S, ax_C, ax_K]))
weight = ng.expand_dims(weight, ax_T, 0) # RSCK -> TRSCK
weight = ng.axes_with_order(weight, axes=ax_f) # TRSCK -> CTRSK
# convolution
output = ng.convolution(params, image, weight, axes=ax_o)
# cast back to NHWC
oC, oD, oH, oW, oN = output.axes
output = ng.broadcast(output, ng.make_axes([oN, oD, oH, oW, oC]))
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
output = ng.Slice(output, out_slicing)
return output
def Conv2D(self, tf_node, inputs):
"""
Computes a 2-D convolution given 4D input and filter tensors.
Arguments:
tf_node: NodeDef object, the tensorflow node to convert.
inputs: List of ngraph Ops as inputs to this node.
Returns:
A ngraph Op corresponding to the tensorflow node.
Inputs to tf_node:
input, filter
TODO: assume default tensorflow layout NHWC, RSCK,
need to support NCHW as well
need to clean up / merge with maxpool
Axes:
Tensorflow Ngraph
in (N, H, W, C) (C, D, H, W, N)
filter (R, S, C, K) (C, T, R, S, K)
out (N, P, Q, K) (K, M, P, Q, N)
Notes on output shape:
https://www.tensorflow.org/api_docs/python/nn.html#convolution
"""
image, weight = inputs
# TODO: currently NHWC only
if tf_node.attr['data_format'].s.decode("ascii") != "NHWC":
raise NotImplementedError("Only supports NHWC import for now.")
# check in_C == f_C
if image.axes.lengths[3] != weight.axes.lengths[2]:
raise ValueError("Image's C dimension (%s) must be equal to "
"filter's C dimension (%s)."
% (image.axes.lengths[3], weight.axes.lengths[2]))
# strides params
tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)]
if len(tf_strides) != 4:
raise ValueError("Length of strides my be 4.")
if tf_strides[0] != 1:
raise NotImplementedError('Strides on batch axis (N) must be 1.')
if tf_strides[3] != 1:
raise NotImplementedError('Strides on channel axis (C) must be 1.')
str_h, str_w = tf_strides[1], tf_strides[2]
# padding params
padding = tf_node.attr['padding'].s.decode("ascii")
pad_t, pad_b, pad_l, pad_r = common_conv2d_pool_padding(
image.axes.lengths, weight.axes.lengths, tf_strides, padding)
if pad_t != pad_b or pad_l != pad_r:
raise NotImplementedError("Requires symmetric padding in ngraph:"
"pad_t(%s) == pad_b(%s) and"
"pad_l(%s) == pad_r(%s)" %
(pad_t, pad_b, pad_l, pad_r))
# conv params
params = dict(pad_d=0, pad_h=pad_t, pad_w=pad_l,
str_d=1, str_h=str_h, str_w=str_w,
dil_d=1, dil_h=1, dil_w=1)
# new axes
C, D, H, W, T, R, S, K, M, P, Q = [ng.make_axis() for _ in range(11)]
N = ng.make_axis(name='N')
D.length, T.length, M.length = 1, 1, 1 # only supports 2D conv for now
# tf's i, f, o axes
ax_i_tf = ng.make_axes([N, H, W, C])
ax_f_tf = ng.make_axes([R, S, C, K])
ax_o_tf = ng.make_axes([N, P, Q, K])
ax_i_tf.set_shape(image.axes.lengths)
ax_f_tf.set_shape(weight.axes.lengths)
ax_o_tf.set_shape(common_conv2d_pool_output_shape(image.axes.lengths,
weight.axes.lengths,
tf_strides, padding))
# ngraph's i, f, o axes
ax_i = ng.make_axes([C, D, H, W, N])
ax_f = ng.make_axes([C, T, R, S, K])
ax_o = ng.make_axes([K, M, P, Q, N])
# image NHWC -> CDHWN
image = ng.cast_axes(image, ng.make_axes([N, H, W, C]))
image = ng.expand_dims(image, D, 1) # NHWC -> NDHWC
image = ng.axes_with_order(image, ax_i) # NDHWC -> CDHWN
# weights RSCK -> CTRSK
weight = ng.cast_axes(weight, ng.make_axes([R, S, C, K]))
weight = ng.expand_dims(weight, T, 0) # RSCK -> TRSCK
weight = ng.axes_with_order(weight, ax_f) # TRSCK -> CTRSK
# convolution
output = ng.convolution(params, image, weight, axes=ax_o)
# output KMPQN -> NPQK
# KMPQN -> NMPQK
output = ng.axes_with_order(output, ng.make_axes([N, M, P, Q, K]))
# NMPQK -> NPQK
output = ng.tensor_slice(output, [slice(None), 0, slice(None),
slice(None), slice(None)])
return output
def test_convolution_2d():
runtime = get_runtime()
# input_x should have shape N(batch) x C x H x W
input_x = ng.constant(np.array([
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.],
[0., 0., 5., 5., 0., 0., 0., 0., 0.]], dtype=np.float32).reshape(1, 1, 9, 9))
# filter weights should have shape M x C x kH x kW
input_filter = ng.constant(np.array([
[1., 0., -1.],
[2., 0., -2.],
[1., 0., -1.]], dtype=np.float32).reshape(1, 1, 3, 3))
# convolution with padding=1 should produce 9 x 9 output:
model = runtime.computation(ng.convolution(input_x, input_filter, padding_above=[1, 1],
padding_below=[1, 1]))
result = model()
assert np.allclose(result,
np.array([[[[0., -15., -15., 15., 15., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -20., -20., 20., 20., 0., 0., 0., 0.],
[0., -15., -15., 15., 15., 0., 0., 0., 0.]]]],
dtype=np.float32))
# convolution with padding=0 should produce 7 x 7 output:
model = runtime.computation(ng.convolution(input_x, input_filter))
result = model()
assert np.allclose(result,
np.array([[[[-20, -20, 20, 20, 0, 0, 0],
[-20, -20, 20, 20, 0, 0, 0],
[-20, -20, 20, 20, 0, 0, 0],
[-20, -20, 20, 20, 0, 0, 0],
[-20, -20, 20, 20, 0, 0, 0],
[-20, -20, 20, 20, 0, 0, 0],
[-20, -20, 20, 20, 0, 0, 0]]]],
dtype=np.float32))
# convolution with strides=2 should produce 4 x 4 output:
model = runtime.computation(ng.convolution(input_x, input_filter, filter_strides=[2, 2]))
result = model()
assert np.allclose(result,
np.array([[[[-20., 20., 0., 0.],
[-20., 20., 0., 0.],
[-20., 20., 0., 0.],
[-20., 20., 0., 0.]]]],
dtype=np.float32))
# convolution with dilation=2 should produce 5 x 5 output:
model = runtime.computation(ng.convolution(input_x, input_filter,
filter_dilation_strides=(2, 2)))
result = model()
assert np.allclose(result,
np.array([[[[0, 0, 20, 20, 0],
[0, 0, 20, 20, 0],
[0, 0, 20, 20, 0],
[0, 0, 20, 20, 0],
[0, 0, 20, 20, 0]]]],
dtype=np.float32))