def check_forward(self, args): y = functions.batch_normalization(*[chainer.Variable(i) for i in args], eps=self.eps) self.assertEqual(y.data.dtype, self.dtype) y_expect = _batch_normalization(self.expander, self.gamma, self.beta, self.x, self.mean, self.var) gradient_check.assert_allclose(y_expect, y.data, **self.check_forward_optionss)
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype)) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay ret = functions.batch_normalization( x, gamma, beta, eps=self.eps, running_mean=self.avg_mean, running_var=self.avg_var, decay=decay) else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = functions.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def check_forward(self, args, use_cudnn=True): y = functions.batch_normalization( *[chainer.Variable(i) for i in args], running_mean=None, running_var=None, decay=self.decay, eps=self.eps, use_cudnn=use_cudnn) self.assertEqual(y.data.dtype, self.dtype) y_expect = _batch_normalization( self.expander, self.gamma, self.beta, self.x, self.mean, self.var) testing.assert_allclose( y_expect, y.data, **self.check_forward_options)
def check_forward(self, inputs, backend_config): y_expected, = self.forward_cpu(inputs) if backend_config.use_cuda: inputs = cuda.to_gpu(inputs) if not self.c_contiguous: inputs = _to_fcontiguous(inputs) with backend_config: y = functions.batch_normalization( *inputs, running_mean=None, running_var=None, decay=self.decay, eps=self.eps) assert y.data.dtype == self.dtype testing.assert_allclose( y_expected, y.data, **self.check_forward_options)
def check_forward(self, inputs, backend_config): # TODO(niboshi): Support it if backend_config.use_chainerx and self.dtype == numpy.float16: raise unittest.SkipTest('ChainerX does not support float16') if self.running_statistics: running_mean_expected = self.running_mean.copy() running_var_expected = self.running_var.copy() else: running_mean_expected = None running_var_expected = None y_expected, = self.forward_cpu( inputs, running_mean_expected, running_var_expected) inputs = backend_config.get_array(inputs) running_mean = backend_config.get_array(self.running_mean) running_var = backend_config.get_array(self.running_var) if not self.c_contiguous: inputs = _as_noncontiguous_array(inputs) running_mean = _as_noncontiguous_array(running_mean) running_var = _as_noncontiguous_array(running_var) with backend_config: y = functions.batch_normalization( *inputs, running_mean=running_mean, running_var=running_var, **self.bn_options) assert y.data.dtype == self.dtype testing.assert_allclose( y_expected, y.data, **self.check_forward_options) if self.running_statistics: testing.assert_allclose( running_mean_expected, running_mean, **self.check_forward_options) testing.assert_allclose( running_var_expected, running_var, **self.check_forward_options)
def check_forward(self, inputs, backend_config): if self.running_statistics: running_mean_expected = self.running_mean.copy() running_var_expected = self.running_var.copy() else: running_mean_expected = None running_var_expected = None y_expected, = self.forward_cpu( inputs, running_mean_expected, running_var_expected) inputs = backend_config.get_array(inputs) running_mean = backend_config.get_array(self.running_mean) running_var = backend_config.get_array(self.running_var) if not self.c_contiguous: with backend_config: inputs = _as_noncontiguous_array(inputs) running_mean = _as_noncontiguous_array(running_mean) running_var = _as_noncontiguous_array(running_var) with backend_config: y = functions.batch_normalization( *inputs, running_mean=running_mean, running_var=running_var, **self.bn_options) assert y.data.dtype == self.dtype testing.assert_allclose( y_expected, y.data, **self.check_forward_options) if self.running_statistics: testing.assert_allclose( running_mean_expected, running_mean, **self.check_forward_options) testing.assert_allclose( running_var_expected, running_var, **self.check_forward_options)
def test_invalid_batch_no_batch_axis(self): args = self.create_batch((1, 3,), (1, 3, 1)) with testing.assert_warns(UserWarning): functions.batch_normalization(*args, axis=2)
def test_invalid(self): with self.assertRaises(RuntimeError): functions.batch_normalization(*self.args, eps=2e-6)
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype)) if configuration.config.train or self.enforce_compute: # if self.force_compute is true, then compute following... if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay ret = functions.batch_normalization( x, gamma, beta, eps=self.eps, running_mean=self.avg_mean, running_var=self.avg_var, decay=decay) else: # if is not train like test # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = functions.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ # check argument argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) original_shape = x.shape batch_size = original_shape[0] # reshape input x if batchsize > 1 if batch_size > 1: reshaped_x = functions.expand_dims(x, axis=0) else: reshaped_x = x if hasattr(self, 'gamma'): gamma = self.gamma if self.norm_grad: gamma.add_batch(batch_size) else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable( self.xp.ones(self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta if self.norm_grad: beta.add_batch(batch_size) else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable( self.xp.zeros(self.avg_mean.shape, dtype=x.dtype)) #align shapes if x was reshaped if batch_size > 1: mean = self.xp.stack((self.avg_mean, ) * batch_size) var = self.xp.stack((self.avg_var, ) * batch_size) gamma = functions.stack((gamma, ) * batch_size) beta = functions.stack((beta, ) * batch_size) else: mean = self.xp.asarray(self.avg_mean) var = self.xp.asarray(self.avg_var) chainer_version = int(chainer.__version__.split('.')[0]) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay if chainer_version < 3: func = batch_normalization.BatchNormalizationFunction( self.eps, mean, var, decay) ret = func(reshaped_x, gamma, beta) else: ret = functions.batch_normalization(reshaped_x, gamma, beta, eps=self.eps, running_mean=mean, running_var=var, decay=decay) else: head_ndim = gamma.ndim + 1 axis = (0, ) + tuple(range(head_ndim, reshaped_x.ndim)) mean = reshaped_x.data.mean(axis=axis) var = reshaped_x.data.var(axis=axis) ret = functions.fixed_batch_normalization(reshaped_x, gamma, beta, mean, var, self.eps) # ret is normalized input x if batch_size > 1: ret = functions.reshape(ret, original_shape) return ret
def f(*inputs): y = functions.batch_normalization(*inputs, decay=self.decay, eps=self.eps, axis=self.axis) return y,
def f(*inputs): y = functions.batch_normalization( *inputs, decay=self.decay, eps=self.eps) return y * y, # make nonlinear against beta
def forward(self, x, **kwargs): """forward(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ finetune, = argument.parse_kwargs( kwargs, ('finetune', False), test='test argument is not supported anymore. ' 'Use chainer.using_config') if self.avg_mean is None: param_shape = tuple([ d for i, d in enumerate(x.shape) if i not in self.axis]) self._initialize_params(param_shape) gamma = self.gamma if gamma is None: with cuda.get_device_from_id(self._device_id): gamma = self.xp.ones( self.avg_mean.shape, dtype=x.dtype) beta = self.beta if beta is None: with cuda.get_device_from_id(self._device_id): beta = self.xp.zeros( self.avg_mean.shape, dtype=x.dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay ret = functions.batch_normalization( x, gamma, beta, eps=self.eps, running_mean=self.avg_mean, running_var=self.avg_var, decay=decay, axis=self.axis) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = functions.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, axis=self.axis) return ret
def test_invalid_batch(self): args = self.create_batch((3, ), (1, 3)) with testing.assert_warns(UserWarning): functions.batch_normalization(*args)
def test_valid(self): functions.batch_normalization(*self.args, eps=1e-5)
def forward(self): return functions.batch_normalization( *[chainer.Variable(i) for i in self.args], eps=self.eps, running_mean=self.mean, running_var=self.var)
def f(*inputs): return functions.batch_normalization(*inputs, **self.bn_options)
def test_valid_batch_no_batch_axis(self): args = self.create_batch((1, 3,), (1, 3, 2)) with warnings.catch_warnings(record=True) as w: functions.batch_normalization(*args, axis=2) assert len(w) == 0
def test_invalid(self): eps = -0.1 if chainer.backends.cuda.libcudnn.get_build_version() < 7500: eps = 2e-6 with self.assertRaises(RuntimeError): functions.batch_normalization(*self.args, eps=eps)
def batch_normalization(self, *args): return functions.batch_normalization( *args, decay=self.decay, eps=self.eps)
def forward(self, x, **kwargs): """forward(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ finetune, = argument.parse_kwargs( kwargs, ('finetune', False), test='test argument is not supported anymore. ' 'Use chainer.using_config') if self.avg_mean is None: param_shape = tuple([ d for i, d in enumerate(x.shape) if i not in self.axis]) self._initialize_params(param_shape) gamma = self.gamma if gamma is None: with chainer.using_device(self.device): gamma = self.xp.ones( self.avg_mean.shape, dtype=self._highprec_dtype) beta = self.beta if beta is None: with chainer.using_device(self.device): beta = self.xp.zeros( self.avg_mean.shape, dtype=self._highprec_dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay avg_mean = self.avg_mean avg_var = self.avg_var if chainer.config.in_recomputing: # Do not update statistics when extra forward computation is # called. if finetune: self.N -= 1 # Revert the count avg_mean = None avg_var = None ret = functions.batch_normalization( x, gamma, beta, eps=self.eps, running_mean=avg_mean, running_var=avg_var, decay=decay, axis=self.axis) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = functions.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, axis=self.axis) return ret
def f(*inputs): y = functions.batch_normalization( *inputs, decay=self.decay, eps=self.eps) return y,
def test_valid_batch(self): args = self.create_batch((3, ), (1, 3, 2, 2)) with warnings.catch_warnings(record=True) as w: functions.batch_normalization(*args) assert len(w) == 0
def f(*inputs): y = functions.batch_normalization(*inputs, decay=self.decay, eps=self.eps, axis=self.axis) return y * y, # make nonlinear against beta
def f(*inputs): return functions.batch_normalization( *inputs, **self.bn_options)
def forward(self): return functions.batch_normalization( *[chainer.Variable(i) for i in self.args], eps=self.eps, running_mean=self.mean, running_var=self.var, use_cudnn=self.use_cudnn)
def f(*inputs): y = functions.batch_normalization( *inputs, **self.bn_options) return y * y, # make nonlinear against beta
def forward(self, x, **kwargs): """forward(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ finetune, = argument.parse_kwargs( kwargs, ('finetune', False), test='test argument is not supported anymore. ' 'Use chainer.using_config') if self.avg_mean is None: param_shape = tuple([ d for i, d in enumerate(x.shape) if i not in self.axis]) self._initialize_params(param_shape) gamma = self.gamma if gamma is None: with chainer.using_device(self.device): gamma = self.xp.ones( self.avg_mean.shape, dtype=x.dtype) beta = self.beta if beta is None: with chainer.using_device(self.device): beta = self.xp.zeros( self.avg_mean.shape, dtype=x.dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay avg_mean = self.avg_mean avg_var = self.avg_var if chainer.config.in_recomputing: # Do not update statistics when extra forward computation is # called. if finetune: self.N -= 1 # Revert the count avg_mean = None avg_var = None ret = functions.batch_normalization( x, gamma, beta, eps=self.eps, running_mean=avg_mean, running_var=avg_var, decay=decay, axis=self.axis) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = functions.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, axis=self.axis) return ret