def check_forward(self, x): mkld.enable_batch_normalization = True self.func1 = batch_normalization.BatchNormalizationFunction( self.eps, self.mean, self.var, self.decay, False) start = time.time() y = self.func1.forward((x, self.gamma, self.beta)) end = time.time() self.assertEqual(y[0].dtype, self.dtype) print("mkldnn timing:", end - start) mkld.enable_batch_normalization = False self.func2 = batch_normalization.BatchNormalizationFunction( self.eps, self.mean, self.var, self.decay, False) start = time.time() y_expect = self.func2.forward((x, self.gamma, self.beta)) end = time.time() print("numpy timing:", end - start) testing.assert_allclose(self.func1.running_mean, self.func2.running_mean, **self.check_forward_optionss) testing.assert_allclose(self.func1.running_var, self.func2.running_var, **self.check_forward_optionss) testing.assert_allclose(y_expect[0], y[0], **self.check_forward_optionss)
def check_backward(self, args, y_grad, use_cudnn='always'): with chainer.using_config('use_cudnn', use_cudnn), \ chainer.using_config('train', self.train): gradient_check.check_backward( batch_normalization.BatchNormalizationFunction( mean=None, var=None, decay=self.decay, eps=self.eps), args, y_grad, **self.check_backward_options)
def __call__(self, x, test=False, finetune=False): """Invokes the forward propagation of BatchNormalization. BatchNormalization accepts additional arguments, which controls three different running mode. Args: x (Variable): Input variable. test (bool): If ``True``, BatchNormalization runs in testing mode; it normalizes the input using pre-computed statistics. finetune (bool): If ``finetune`` is ``True`` and ``test`` is ``False``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. If ``test`` is ``False``, then BatchNormalization runs in training mode; it computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. """ if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device(self._device_id): gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype), volatile='auto') if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device(self._device_id): beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype), volatile='auto') # Var is always ones with cuda.get_device(self._device_id): self.one_var = self.xp.ones(self.avg_mean.shape, dtype=x.dtype) if not test: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalizationFunction( self.eps, self.avg_mean, self.avg_var, True, decay, self.use_cudnn) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean, volatile='auto') #var = variable.Variable(self.avg_var, volatile='auto') var = variable.Variable(self.one_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, self.use_cudnn) return ret
def check_backward(self, args, y_grad): gradient_check.check_backward( batch_normalization.BatchNormalizationFunction(eps=self.eps), args, y_grad, eps=1e-2, **self.check_backward_options)
def check_backward(self, args, y_grad): gradient_check.check_backward( batch_normalization.BatchNormalizationFunction(mean=None, var=None, train=self.train, decay=self.decay, eps=self.eps), args, y_grad, **self.check_backward_options)
def check_backward(self, args, y_grad): gradient_check.check_backward( batch_normalization.BatchNormalizationFunction(eps=self.eps), args, y_grad, eps=1e-2, rtol=1e-3, atol=1e-4)
def __call__(self, x, finetune=False): """Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluatino during training, and normalizes the input using batch statistics. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device(self._device_id): gamma = variable.Variable(self.xp.ones(self.avg_mean.shape, dtype=x.dtype), volatile='auto') if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device(self._device_id): beta = variable.Variable(self.xp.zeros(self.avg_mean.shape, dtype=x.dtype), volatile='auto') if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalizationFunction( self.eps, self.avg_mean, self.avg_var, decay, self.use_cudnn) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean, volatile='auto') var = variable.Variable(self.avg_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, self.use_cudnn) return ret
def __call__(self, x, test=False, finetune=False): """Invokes the forward propagation of BatchNormalization. BatchNormalization accepts additional arguments, which controls three different running mode. Args: x (Variable): An input variable. test (bool): If ``True``, BatchNormalization runs in testing mode; it normalizes the input using pre-computed statistics. finetune (bool): If ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. If ``test`` and ``finetune`` are both ``False``, then BatchNormalization runs in training mode; it computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. """ use_batch_mean = not test or finetune if use_batch_mean: func = batch_normalization.BatchNormalizationFunction(self.eps) ret = func(x, self.gamma, self.beta) if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay with cuda.get_device(x.data): m = x.data.size // self.gamma.data.size adjust = m / max(m - 1., 1.) # unbiased estimation self.avg_mean *= decay func.mean *= 1 - decay # reuse buffer as a temporary self.avg_mean += func.mean del func.mean self.avg_var *= decay func.var *= (1 - decay) * adjust # reuse buffer as a temporary self.avg_var += func.var del func.var else: mean = variable.Variable(self.avg_mean, volatile='auto') var = variable.Variable(self.avg_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, self.gamma, self.beta, mean, var, self.eps) return ret
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluatino during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', train)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable( self.xp.ones(self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable( self.xp.zeros(self.avg_mean.shape, dtype=x.dtype)) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalizationFunction( self.eps, self.avg_mean, self.avg_var, decay) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ # check argument argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) original_shape = x.shape batch_size = original_shape[0] # reshape input x if batchsize > 1 if batch_size > 1: reshaped_x = functions.expand_dims(x, axis=0) else: reshaped_x = x if hasattr(self, 'gamma'): gamma = self.gamma if self.norm_grad: # gamma.add_batch(batch_size) gamma.n_batch = batch_size else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta if self.norm_grad: # beta.add_batch(batch_size) beta.n_batch = batch_size else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype)) #align shapes if x was reshaped if batch_size > 1: mean = self.xp.stack((self.avg_mean,) * batch_size) var = self.xp.stack((self.avg_var,) * batch_size) gamma = functions.stack((gamma,) * batch_size) beta = functions.stack((beta,) * batch_size) else: mean = self.xp.asarray(self.avg_mean) var = self.xp.asarray(self.avg_var) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalizationFunction( self.eps, mean, var, decay) ret = func(reshaped_x, gamma, beta) else: head_ndim = gamma.ndim + 1 axis = (0,) + tuple(range(head_ndim, reshaped_x.ndim)) mean = reshaped_x.data.mean(axis=axis) var = reshaped_x.data.var(axis=axis) ret = functions.fixed_batch_normalization( reshaped_x, gamma, beta, mean, var, self.eps) # ret is normalized input x if batch_size > 1: ret = functions.reshape(ret, original_shape) return ret