def __call__(self, in_obj):
in_axes = in_obj.axes
if in_axes.channel_axis() is None:
red_axes = ng.make_axes(in_axes.recurrent_axis()) + in_axes.batch_axes()
else:
red_axes = in_axes - in_axes.channel_axis()
out_axes = in_axes - red_axes
in_obj = ng.flatten(in_obj, out_axes | red_axes.flatten(force=True))
if self.gamma is None:
self.gvar = self.gvar or ng.persistent_tensor(axes=out_axes, initial_value=1.0)
self.gmean = self.gmean or ng.persistent_tensor(axes=out_axes, initial_value=0.0)
self.gamma = ng.variable(axes=out_axes,
initial_value=self.init_gamma,
scope=self.scope).named('gamma')
self.beta = ng.variable(axes=out_axes,
initial_value=self.init_beta,
scope=self.scope).named('beta')
xmean = ng.mean(in_obj, out_axes=out_axes)
xvar = ng.variance(in_obj, out_axes=out_axes)
if Layer.inference_mode:
return ng.unflatten(self.gamma * ((in_obj - self.gmean) *
ng.reciprocal(ng.sqrt(self.gvar + self.eps))) + self.beta)
else:
return ng.sequential([
ng.assign(self.gmean, self.gmean * self.rho + xmean * (1.0 - self.rho)),
ng.assign(self.gvar, self.gvar * self.rho + xvar * (1.0 - self.rho)),
ng.unflatten(self.gamma * ((in_obj - xmean) *
ng.reciprocal(ng.sqrt(xvar + self.eps))) + self.beta)
])
def test_variance_sqrt_inverse(transformer_factory, input_tensor):
inputs = input_tensor
targets = ng.placeholder(inputs.axes)
epsilon = 1e-3
inp_stat = ng.reciprocal(
ng.sqrt(
ng.variance(inputs, reduction_axes=inputs.axes.batch_axes()) +
epsilon))
err = ng.sum(inp_stat - targets, out_axes=())
d_inputs = ng.deriv(err, inputs)
with executor([err, d_inputs], inputs, targets) as comp_func:
input_value = rng.uniform(-1, 1, inputs.axes)
target_value = rng.uniform(-1, 1, targets.axes)
ng_f_res, ng_b_res = comp_func(input_value, target_value)
npv = np.var(input_value, axis=1, keepdims=True) + epsilon
np_f_res = 1.0 / np.sqrt(npv)
npv_delta = 2 * (input_value -
np.mean(input_value, axis=1, keepdims=True))
np_b_res = -0.5 * np_f_res / npv * npv_delta
np_f_res = np.sum(np_f_res - target_value)
ng.testing.assert_allclose(np_f_res, ng_f_res, atol=1e-4, rtol=1e-4)
ng.testing.assert_allclose(np_b_res, ng_b_res, atol=1e-4, rtol=1e-4)
def BatchNormalization(onnx_node,
ng_inputs): # type: (NodeWrapper, List[TensorOp]) -> Op
x, scale, bias, mean, var = ng_inputs
is_test = onnx_node.get_attribute_value('is_test', 1)
spatial = onnx_node.get_attribute_value('spatial', 1)
epsilon = onnx_node.get_attribute_value('epsilon', 1e-3)
# @TODO: Implement learning mode support
# momentum = onnx_node.get_attribute_value('momentum', 0.99)
if not is_test:
raise NotImplementedError(
'BatchNormalization node (%s): only `is_test` mode is currently '
'supported.', onnx_node.name)
if not spatial:
raise NotImplementedError(
'BatchNormalization node (%s): only `spatial` mode is currently '
'supported.', onnx_node.name)
if len(x.axes) == 5:
x = rename_axes(x, 'NCHWD')
else:
x = rename_axes(x, 'NCHW')
mean = rename_axes(mean, 'C')
scale = rename_axes(scale, 'C')
bias = rename_axes(bias, 'C')
var = rename_axes(var, 'C')
ng_op = ng.unflatten(scale *
((x - mean) * ng.reciprocal(ng.sqrt(var + epsilon))) +
bias)
return cast_to_pos_axes(ng_op)
def test_reciprocal_derivative(transformer_factory, input_tensor):
"""TODO."""
p_u = input_tensor
delta = .001
u = rng.uniform(.1, 5.0, p_u.axes)
rec_u = ng.reciprocal(p_u)
check_derivative(rec_u, p_u, delta, u, atol=1e-2, rtol=1e-2)
def test_reciprocal(transformer_factory, input_tensor):
"""TODO."""
p_u = input_tensor
u = rng.uniform(.1, 5.0, p_u.axes)
rec_u_np = np.reciprocal(u)
rec_u = ng.reciprocal(p_u)
with ExecutorFactory() as ex:
rec_u_graph = ex.executor(rec_u, p_u)(u)
ng.testing.assert_allclose(rec_u_np, rec_u_graph)
def Reciprocal(self, cntk_op, inputs):
"""
Returns element-wise reciprocal of inputs[0].
Arguments:
inputs: List of inputs to this node.
Returns:
A ngraph Op.
"""
return ng.reciprocal(inputs[0]).named(cntk_op.uid)
def Reciprocal(self, cntk_op, inputs):
"""
Returns element-wise reciprocal of inputs[0].
Arguments:
cntk_op: CNTK operation to be imported.
inputs: List of inputs to this node.
Returns:
A ngraph Op.
"""
return ng.reciprocal(inputs[0]).named(cntk_op.uid)
def test_4d_chained(transformer_factory, input_axes):
x_val = np.absolute(np.random.randn(*input_axes.lengths))
y_val = np.absolute(np.random.randn(*input_axes.lengths))
x = ng.constant(x_val, input_axes)
y = ng.constant(y_val, input_axes)
im = ng.reciprocal(x)
out = ng.sum(ng.add(im, y), reduction_axes=input_axes[0])
with executor(out) as ex:
graph_val = ex()
np_val = np.sum(np.add(np.reciprocal(x_val), y_val), 0)
np.testing.assert_allclose(graph_val, np_val, rtol=1e-4)
def test_4d_chained(transformer_factory, input_axes):
# Limiting maximum absolute value for tensors elements to 7.9.
# See description in function test_exit_condition above
# Limitting minimum absolute value for tensors being input to reciprocal operation to 1/7.9
#
# This is consequence of the above and flexpoint accuracy.
# Numbers very small have poor absolute accuracy. When reciprocal of them is calculated the
# results becomes very large and has even worse accuracy. When small numbers would be accepted
# as an input to reciprocal in the test the absolute maximum value of the result is undefined
# and so absolute tolerance.
# To have possibility to set atol in the test and test could pass with it minimum element of
# the tensor that is input to reciprocal operation has to be limited.
is_flex = is_flex_factory(transformer_factory)
clip_val_max = 7.9 if is_flex else 0
clip_val_min = 1.0 / 7.9 if is_flex else 0
x_val = rng.randn_abs_clip(input_axes,
clip_min=clip_val_min,
clip_max=clip_val_max)
y_val = rng.randn_abs_clip(input_axes, clip_max=clip_val_max)
x = ng.constant(x_val, input_axes)
y = ng.constant(y_val, input_axes)
im = ng.reciprocal(x)
out = ng.sum(ng.add(im, y), reduction_axes=input_axes[0])
with executor(out) as ex:
graph_val = ex()
np_val = np.sum(np.add(np.reciprocal(x_val), y_val), 0)
# atol_multiplier = 15 * x_val.shape[0]
#
# x_val.shape[0] is number elements added together in operation
# ng.sum(X, reduction_axes=input_axes[0])
#
# 15 is calculated the following way:
#
# Input tensor has values from the range 1/7.9 - 7.9
# For DEC=12 absolute error is equal to 0.5*2^-12 = 0.000122
# 1/7.9 = 0.126582 with this error becomes 0.126704
# Reciprocal of 1/7.9 is 7.9
# Reciprocal of 1/7.9 + err = 7.892389
# Absolute difference is 0.007611
# It is 15.2 times larger then atol limit 5e-4 from Argon transformer
ng.testing.assert_allclose(graph_val,
np_val,
rtol=1e-4,
atol_multiplier=15 * x_val.shape[0])
def test_reciprocal_derivative(transformer_factory):
"""TODO."""
N = ng.make_axis(name='N')
W = ng.make_axis(name='W')
delta = .001
W.length = 20
N.length = 128
axes = ng.make_axes([W, N])
p_u = ng.placeholder(axes)
u = rng.uniform(.1, 5.0, p_u.axes)
rec_u = ng.reciprocal(p_u)
check_derivative(rec_u, p_u, delta, u, atol=1e-2, rtol=1e-2)
def construct_batchnorm_fprop_pattern(self):
"""
Generate graph op that represents a pattern for batchnorm fprop operation.
self.gamma * ((in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps))) + self.beta
Returns:
Single pattern that matches batchnorm fprop op
"""
self.batchnorm_fprop_input_tensor_label = "in_obj"
self.batchnorm_fprop_gamma_label = "gamma"
self.batchnorm_fprop_beta_label = "beta"
self.batchnorm_fprop_variance_label = "variance"
self.batchnorm_fprop_epsilon_label = "epsilon"
self.batchnorm_fprop_mean_label = "mean"
# bind the label to the op's which needed to be updated in the dict
in_obj = PatternLabelOp(self.batchnorm_fprop_input_tensor_label,
(lambda op: isinstance(op, ContiguousOp)))
flatten_tensor = PatternSkipOp(in_obj,
(lambda op: isinstance(op, Flatten)))
gamma = PatternLabelOp(self.batchnorm_fprop_gamma_label,
(lambda op: isinstance(op, BroadcastOp)))
beta = PatternLabelOp(self.batchnorm_fprop_beta_label,
(lambda op: isinstance(op, BroadcastOp)))
variance = PatternLabelOp(self.batchnorm_fprop_variance_label,
(lambda op: isinstance(op, Divide)))
epsilon = PatternLabelOp(self.batchnorm_fprop_epsilon_label,
(lambda op: isinstance(op, BroadcastOp)))
mean = PatternLabelOp(self.batchnorm_fprop_mean_label,
(lambda op: isinstance(op, Divide)))
# construct the fprop batchnorm pattern matching the computation graph
# ng.sqrt(xvar + self.eps)
SqrtofVarianceAndEps = ng.sqrt(ng.add(variance, epsilon))
# ng.reciprocal(ng.sqrt(xvar + self.eps))
reciprocal_op = ng.reciprocal(SqrtofVarianceAndEps)
reciprocal_op_w_braodcast = ng.PatternSkipOp(reciprocal_op,
lambda op: isinstance(op, BroadcastOp))
mean_bcast = ng.PatternSkipOp(mean, lambda op: isinstance(op, BroadcastOp))
# (in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps))
mul_op_1 = ng.multiply(ng.subtract(flatten_tensor, mean_bcast), reciprocal_op_w_braodcast)
# "self.gamma * ((in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps)))
MultiplyGamma = ng.multiply(mul_op_1, gamma)
# self.gamma * ((in_obj - xmean) * ng.reciprocal(ng.sqrt(xvar + self.eps))) + self.beta
AddBeta = ng.Unflatten(ng.Add(MultiplyGamma, beta))
return AddBeta
def test_reciprocal(transformer_factory):
"""TODO."""
N = ng.make_axis(name='N')
W = ng.make_axis(name='W')
W.length = 20
N.length = 128
axes = ng.make_axes([W, N])
p_u = ng.placeholder(axes)
u = rng.uniform(.1, 5.0, p_u.axes)
rec_u_np = np.reciprocal(u)
rec_u = ng.reciprocal(p_u)
ex = ExecutorFactory()
rec_u_graph = ex.executor(rec_u, p_u)(u)
np.testing.assert_allclose(rec_u_np, rec_u_graph)
def __call__(self,
in_obj,
channel_axes="C",
spatial_axes=("D", "H", "W"),
**kwargs):
"""
Arguments:
in_obj (Op): Input op
channel_axes (str): name of the expected channel axis type - defaults to "C"
spatial_axes (tuple): names of expected depth, height and width axis types - defaults
to "D", "H", and "W"
"""
if isinstance(spatial_axes, dict):
spatial_axes = tuple(
spatial_axes.get(name, name) for name in ("D", "H", "W"))
elif isinstance(spatial_axes, tuple):
if len(spatial_axes) < 3:
raise ValueError(
"spatial_axes must have length 3 (e.g. ('D', 'H', 'W'))")
spatial_axes = tuple(
name if name else default
for name, default in zip(spatial_axes, ("D", "H", "W")))
orig_axes = in_obj.axes
in_obj = reorder_spatial_axes(in_obj, channel_axes, spatial_axes)
channel_axes = in_obj.axes.get_by_names(channel_axes)
spatial_axes = in_obj.axes.get_by_names(*spatial_axes)
filter_axes = self._filter_axes(channel_axes, spatial_axes)
# mark 'K' as a shadow axis for the initializers.
axes_map = shadow_axes_map(filter_axes.find_by_name('K'))
filter_axes = ng.make_axes([
axis if axis.name != 'K' else list(axes_map.keys())[0]
for axis in filter_axes
])
if not self.initialized:
if not self.weight_norm:
self.W = ng.variable(axes=filter_axes,
initial_value=self.init,
metadata={
"label": LABELS["weight"]
}).named("W")
else:
self.v = ng.variable(axes=filter_axes,
initial_value=self.init,
metadata={
"label": LABELS["weight"]
}).named("v")
out_axes = ng.make_axes(
[filter_axes.get_by_names("K__NG_SHADOW")])
v_norm = ng.mean(ng.square(self.v), out_axes=out_axes)
self.g = ng.variable(axes=out_axes,
initial_value=self.init,
metadata={
"label": LABELS["weight"]
}).named("g")
self.W = self.g * self.v * ng.reciprocal(
ng.sqrt(v_norm + 1e-3))
else:
if filter_axes != self.W.axes:
raise ValueError(
("{layer_name} layer has already been initialized with an "
"input object which has resulted in filter axes: "
"{existing_filter_axes}. This new input object has axes: "
"{input_axes}, which implies the need for filter axes: "
"{new_filter_axes} which are different than the existing "
"filter axes.").format(
layer_name=self.name,
existing_filter_axes=self.W.axes,
input_axes=in_obj.axes,
new_filter_axes=filter_axes,
))
output = ng.map_roles(
self._conv_op(in_obj, channel_axes, spatial_axes), axes_map)
# Reorder the output to match the input order
output_axis_order = ng.make_axes(
[output.axes.find_by_name(ax.name)[0] for ax in orig_axes])
# Remove introduced axes. If their length is > 1, then perhaps they should be kept
slices = [
0 if (ax not in orig_axes) and ax.length == 1 else slice(None)
for ax in output.axes
]
output = ng.tensor_slice(output, slices)
# New axes with length > 1 may have been introduced. Add them to the end.
output_axis_order = output_axis_order | output.axes
return ng.axes_with_order(output, output_axis_order)
def Reciprocal(onnx_node,
ng_inputs): # type: (NodeWrapper, List[TensorOp]) -> Op
return ng.reciprocal(ng_inputs[0])
