def batch_normalization(x, mean, variance,
offset, scale,
decay=0.9,
variance_epsilon=1e-3,
use_global_stats=-1,
name=None):
norm_x = ops.BatchNorm([x, mean, variance], decay, variance_epsilon, use_global_stats, name=name)
return ops.Scale([norm_x, scale, offset], name=name + '_scale' if name is not None else name)
def batch_norm_with_global_normalization(t, m, v,
beta, gamma,
decay=0.9,
variance_epsilon=1e-3,
scale_after_normalization=True,
use_global_stats=-1,
name=None):
norm_x = ops.BatchNorm([t, m, v], decay, variance_epsilon, use_global_stats, name=name)
if scale_after_normalization:
return ops.Scale([norm_x, gamma, beta], name=name + '_scale' if name is not None else name)
else: return norm_x
def batch_normalization(inputs, gamma, beta, mean, var, **kwargs):
"""Batch Normalization, introduced by `[Ioffe & Szegedy, 2015] <https://arxiv.org/abs/1502.03167>`_
Parameters
----------
inputs : Tensor
The input tensor.
gamma: Tensor
The scale parameter.
beta: Tensor
The shift parameter.
mean: Tensor
The moving average of mean.
var: Tensor
The moving average of variance.
Returns
-------
Tensor
The output tensor.
"""
return ops.BatchNorm([inputs, mean, var, gamma, beta])
def call(self, inputs, training=False):
use_stats = 0 if training else 1
if self.fused:
return ops.FusedBatchNorm([
inputs, self.moving_mean, self.moving_variance, self.gamma,
self.beta
],
axis=self.axis,
momentum=self.momentum,
eps=self.epsilon,
use_stats=use_stats,
mode='DEFAULT')
x_norm = ops.BatchNorm(
[inputs, self.moving_mean, self.moving_variance],
axis=self.axis,
momentum=self.momentum,
eps=self.epsilon,
use_stats=use_stats,
mode='DEFAULT')
if self.gamma is not None:
# use scale
if self.beta is not None:
return ops.Scale([x_norm, self.gamma, self.beta],
axis=self.axis,
num_axes=1)
else:
return ops.Scale([x_norm, self.gamma],
axis=self.axis,
num_axes=1)
else:
# do not use scale
if self.beta is not None:
return ops.BiasAdd([x_norm, self.beta],
data_format=self._data_format)
else:
return x_norm
def LayerSetup(self, bottom):
inputs = [bottom] + [blob['data'] for blob in self._blobs]
return _ops.BatchNorm(inputs, **self.arguments)
def Setup(self, bottom):
super(BatchNormLayer, self).Setup(bottom)
return ops.BatchNorm(bottom + [blob['data'] for blob in self._blobs],
**self._param)
