def forward(self, inputs, **kwargs): outputs = normalization_ops.batch_norm( [inputs, self.gamma, self.beta, self.moving_mean, self.moving_var], axis=self.axis, momentum=self.decay, epsilon=self.epsilon, use_stats=0 if self.training else 1) if self.act: outputs = self.act(outputs) return outputs
def batch_normalization( x, moving_mean, moving_variance, offset, scale, axis=-1, momentum=0.9, variance_epsilon=1e-5, trainable=False, name=None, ): r"""Apply the batch normalization. `[Ioffe & Szegedy, 2015] <https://arxiv.org/abs/1502.03167>`_. The normalization is defined as: .. math:: y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta The moving average of stats are calculated as: .. math:: x_{\text{running}} = \text{momentum} * x_{\text{running}} + (1 - \text{momentum}) * x_{\text{batch}} Parameters ---------- x : dragon.Tensor The input tensor. moving_mean : dragon.Tensor The moving mean. moving_variance : dragon.Tensor The moving variance. offset : dragon.Tensor The :math:`\beta` tensor. scale : dragon.Tensor The :math:`\gamma` tensor. axis : int, optional, default=-1 The channel axis. momentum : Union[float, dragon.Tensor], optional The value to :math:`\text{momentum}`. variance_epsilon : float, optional, default=1e-5 The value to :math:`\epsilon`. trainable : bool, optional, default=False The optional training flag. name : str, optional The operation name. Returns ------- dragon.Tensor The output tensor. """ return normalization_ops.batch_norm( [x, scale, offset, moving_mean, moving_variance], axis=axis, momentum=momentum, epsilon=variance_epsilon, use_stats=not trainable, name=name, )
def fused_batch_norm( x, scale, offset, mean, variance, epsilon=0.001, data_format='NHWC', is_training=True, name=None, exponential_avg_factor=1.0, ): r"""Apply the batch normalization. `[Ioffe & Szegedy, 2015] <https://arxiv.org/abs/1502.03167>`_. The normalization is defined as: .. math:: y = \frac{x - \mathrm{E}[x]} {\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta The moving average of stats are calculated as: .. math:: x_{\text{moving}} = \text{momentum} * x_{\text{moving}} + + (1 - \text{momentum}) * x_{\text{batch}} Parameters ---------- x : dragon.Tensor The input tensor. scale : dragon.Tensor The :math:`\gamma` tensor. offset : dragon.Tensor The :math:`\beta` tensor. mean : dragon.Tensor The running mean tensor. variance : dragon.Tensor The running variance tensor. epsilon : float, optional, default=1e-3 The value to :math:`\epsilon`. data_format : str, optional, default='NHWC' ``'NCHW'`` or ``'NHWC'``. is_training : bool, optional, default=True The value to indicate training or inference. name : str, optional The operation name. exponential_avg_factor : float, optional, default=1.0 The value to :math:`1 - \text{momentum}`. Returns ------- dragon.Tensor The output tensor. """ return normalization_ops.batch_norm( [x, scale, offset, mean, variance], axis=1 if data_format.startswith('NC') else -1, momentum=1 - exponential_avg_factor, epsilon=epsilon, use_stats=not is_training, name=name, )
def __call__(self, bottom): inputs = [bottom, self._weight, self._bias] + \ [blob['data'] for blob in self._blobs[:2]] return normalization_ops.batch_norm(inputs, **self.arguments)
def call(self, inputs, training=None): return normalization_ops.batch_norm( [inputs, self.gamma, self.beta, self.moving_mean, self.moving_variance], axis=self.axis, momentum=self.momentum, epsilon=self.epsilon, use_stats=not training)
def __call__(self, bottom): if len(self.blobs) == 0: self.build(bottom) inputs = [bottom, self.weight, self.bias] + \ [blob['data'] for blob in self.blobs[:2]] return normalization_ops.batch_norm(inputs, **self.call_args)