def __call__(self, x, finetune=False): if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype)) if chainer.configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = MultiNodeBatchNormalizationFunction( self.comm, self.eps, self.avg_mean, self.avg_var, decay) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, finetune=False): if self.gamma is not None: gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = self.xp.ones(self.avg_mean.shape, dtype=x.dtype) if self.beta is not None: beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = self.xp.zeros(self.avg_mean.shape, dtype=x.dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_renormalization.BatchRenormalizationFunction( self.eps, self.avg_mean, self.avg_var, decay, self.rmax, self.dmax) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, test=False, finetune=False): """Invokes the forward propagation of BatchNormalization. BatchNormalization accepts additional arguments, which controlls three different running mode. Args: x (Variable): An input variable. test (bool): If ``True``, BatchNormalization runs in testing mode; it normalizes the input using precomputed statistics. finetune (bool): If ``True``, BatchNormalization runs in finetuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. If ``test`` and ``finetune`` are both ``False``, then BatchNormalization runs in training mode; it computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. """ use_batch_mean = not test or finetune if use_batch_mean: ret = batch_normalization.batch_normalization( x, self.gamma, self.beta, self.eps) func = ret.creator if finetune: self.N += 1 decay = 1. / self.N else: decay = self.decay m = x.data.size // self.gamma.data.size adjust = m / max(m - 1., 1.) # unbiased estimatio self.avg_mean = cuda.to_gpu(self.avg_mean) * decay func.mean = cuda.to_gpu(func.mean) * ( 1 - decay) # reuse buffer as a temporary self.avg_mean += func.mean del func.mean self.avg_var = cuda.to_gpu(self.avg_var) * decay func.var = cuda.to_gpu( func.var) * (1 - decay) * adjust # reuse buffer as a temporary self.avg_var += func.var del func.var # self.avg_mean *= decay # func.mean *= 1 - decay # reuse buffer as a temporary # self.avg_mean += func.mean # del func.mean # self.avg_var *= decay # func.var *= (1 - decay) * adjust # reuse buffer as a temporary # self.avg_var += func.var # del func.var else: mean = variable.Variable(self.avg_mean, volatile='auto') var = variable.Variable(self.avg_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, self.gamma, self.beta, mean, var, self.eps) return ret
def __call__(self, x, test=False, finetune=False): """Invokes the forward propagation of BatchNormalization. BatchNormalization accepts additional arguments, which controls three different running mode. Args: x (Variable): Input variable. test (bool): If ``True``, BatchNormalization runs in testing mode; it normalizes the input using pre-computed statistics. finetune (bool): If ``finetune`` is ``True`` and ``test`` is ``False``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. If ``test`` is ``False``, then BatchNormalization runs in training mode; it computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. """ if hasattr(self, 'gamma'): gamma = self.gamma else: gamma_ = self.xp.ones(self.avg_mean.shape, dtype=x.dtype) if self.device is None: gamma = variable.Variable(gamma_, volatile='auto') else: gamma = \ variable.Variable(cuda.to_gpu(gamma_, self.device), volatile='auto') if hasattr(self, 'beta'): beta = self.beta else: beta_ = self.xp.zeros(self.avg_mean.shape, dtype=x.dtype) if self.device is None: beta = variable.Variable(beta_, volatile='auto') else: beta = variable.Variable(cuda.to_gpu(beta_, self.device), volatile='auto') if not test: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalizationFunction( self.eps, self.avg_mean, self.avg_var, True, decay, self.use_cudnn) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean, volatile='auto') var = variable.Variable(self.avg_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, self.use_cudnn) return ret
def __call__(self, x, test=False, finetune=False): """Invokes the forward propagation of BatchNormalization. BatchNormalization accepts additional arguments, which controls three different running mode. Args: x (Variable): An input variable. test (bool): If ``True``, BatchNormalization runs in testing mode; it normalizes the input using pre-computed statistics. finetune (bool): If ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. If ``test`` and ``finetune`` are both ``False``, then BatchNormalization runs in training mode; it computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. """ # use_batch_mean = not test or finetune -------------------------------- # ----------------------------------------------------------------------------- use_batch_mean = False if hasattr(self, 'gamma'): gamma = self.gamma else: gamma = variable.Variable(self.xp.ones(self.avg_mean.shape, dtype=x.dtype), volatile='auto') if hasattr(self, 'beta'): beta = self.beta else: beta = variable.Variable(self.xp.zeros(self.avg_mean.shape, dtype=x.dtype), volatile='auto') if use_batch_mean: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = function_batch_normalization.BatchNormalizationFunction( self.eps, self.avg_mean, self.avg_var, True, decay) ret = func(x, gamma, beta) self.avg_mean = func.running_mean self.avg_var = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) #, volatile='auto') var = variable.Variable(self.avg_var) #, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def fixed_batch_renormalization(x, gamma, beta, mean, var, eps=2e-5): warnings.warn( 'fixed_batch_renormalization is deprecated. ' 'Use fixed_batch_normalization instead.', DeprecationWarning) with configuration.using_config('train', False): return batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, eps)
def __call__(self, x, finetune=False): if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable( self.xp.ones(self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable( self.xp.zeros(self.avg_mean.shape, dtype=x.dtype)) if chainer.configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = MultiNodeBatchNormalizationFunction(self.comm, self.eps, self.avg_mean, self.avg_var, decay) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def forward(self, x, finetune=False): if self.gamma is not None: gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = self.xp.ones( self.avg_mean.shape, dtype=x.dtype) if self.beta is not None: beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = self.xp.zeros( self.avg_mean.shape, dtype=x.dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay ret = batch_renormalization.batch_renormalization( x, gamma, beta, self.rmax, self.dmax, self.eps, self.avg_mean, self.avg_var, decay, update_statistics=True) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def forward(self, x, finetune=False): if self.gamma is not None: gamma = self.gamma else: with chainer.using_device(self.device): gamma = self.xp.ones( self.avg_mean.shape, dtype=x.dtype) if self.beta is not None: beta = self.beta else: with chainer.using_device(self.device): beta = self.xp.zeros( self.avg_mean.shape, dtype=x.dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay ret = batch_renormalization.batch_renormalization( x, gamma, beta, self.rmax, self.dmax, self.eps, self.avg_mean, self.avg_var, decay, update_statistics=True) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, test=False, finetune=False): """Invokes the forward propagation of BatchNormalization. BatchNormalization accepts additional arguments, which controlls three different running mode. Args: x (Variable): An input variable. test (bool): If ``True``, BatchNormalization runs in testing mode; it normalizes the input using precomputed statistics. finetune (bool): If ``True``, BatchNormalization runs in finetuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. If ``test`` and ``finetune`` are both ``False``, then BatchNormalization runs in training mode; it computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. """ use_batch_mean = not test or finetune if use_batch_mean: ret = batch_normalization.batch_normalization( x, self.gamma, self.beta, self.eps) func = ret.creator if finetune: self.N += 1 decay = 1. / self.N else: decay = self.decay m = x.data.size // self.gamma.data.size adjust = m / max(m - 1., 1.) # unbiased estimatio self.avg_mean = cuda.to_gpu(self.avg_mean) * decay func.mean = cuda.to_gpu(func.mean) * (1 - decay) # reuse buffer as a temporary self.avg_mean += func.mean del func.mean self.avg_var = cuda.to_gpu(self.avg_var) * decay func.var = cuda.to_gpu(func.var) * (1 - decay) * adjust # reuse buffer as a temporary self.avg_var += func.var del func.var # self.avg_mean *= decay # func.mean *= 1 - decay # reuse buffer as a temporary # self.avg_mean += func.mean # del func.mean # self.avg_var *= decay # func.var *= (1 - decay) * adjust # reuse buffer as a temporary # self.avg_var += func.var # del func.var else: mean = variable.Variable(self.avg_mean, volatile='auto') var = variable.Variable(self.avg_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, self.gamma, self.beta, mean, var, self.eps) return ret
def fixed_batch_renormalization(x, gamma, beta, mean, var, eps=2e-5): warnings.warn( 'fixed_batch_renormalization is deprecated. ' 'Use fixed_batch_normalization instead.', DeprecationWarning) with configuration.using_config('train', False): return batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, eps )
def __call__(self, x, gamma, beta, **kwargs): """Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluatino during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', train)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. gamma (Variable): Input variable of gamma of shape finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) with cuda.get_device_from_id(self._device_id): _gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype)) with cuda.get_device_from_id(self._device_id): _beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype)) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay if self.comm is not None: func = MultiNodeBatchNormalizationFunction( self.comm, self.eps, self.avg_mean, self.avg_var, decay) ret = func(x, _gamma, _beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: ret = chainer.functions.batch_normalization(x, _gamma, _beta, eps=self.eps, running_mean=self.avg_mean, running_var=self.avg_var, decay=decay) else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = batch_normalization.fixed_batch_normalization( x, _gamma, _beta, mean, var, self.eps) shape = ret.shape ndim = len(shape) gamma = F.broadcast_to(F.reshape(gamma, list(gamma.shape) + [1] * (ndim - len(gamma.shape))), shape) beta = F.broadcast_to(F.reshape(beta, list(beta.shape) + [1] * (ndim - len(beta.shape))), shape) return gamma * ret + beta
def forward(self, x, finetune=False): if self.gamma is not None: gamma = self.gamma else: with chainer.using_device(self.device): gamma = self.xp.ones(self.avg_mean.shape, dtype=x.dtype) if self.beta is not None: beta = self.beta else: with chainer.using_device(self.device): beta = self.xp.zeros(self.avg_mean.shape, dtype=x.dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay avg_mean = self.avg_mean avg_var = self.avg_var update_statistics = True if chainer.config.in_recomputing: # Do not update statistics when extra forward computation is # called. if finetune: self.N -= 1 # Revert the count avg_mean = self._prev_avg_mean avg_var = self._prev_avg_var update_statistics = False elif chainer.config._will_recompute: self._prev_avg_mean = avg_mean.copy() self._prev_avg_var = avg_var.copy() ret = batch_renormalization.batch_renormalization( x, gamma, beta, self.rmax, self.dmax, self.eps, avg_mean, avg_var, decay, update_statistics=update_statistics) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, finetune=False): """Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluatino during training, and normalizes the input using batch statistics. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device(self._device_id): gamma = variable.Variable(self.xp.ones(self.avg_mean.shape, dtype=x.dtype), volatile='auto') if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device(self._device_id): beta = variable.Variable(self.xp.zeros(self.avg_mean.shape, dtype=x.dtype), volatile='auto') if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalizationFunction( self.eps, self.avg_mean, self.avg_var, decay, self.use_cudnn) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean, volatile='auto') var = variable.Variable(self.avg_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, self.use_cudnn) return ret
def forward(self, x, finetune=False): if self.gamma is not None: gamma = self.gamma else: with chainer.using_device(self.device): gamma = self.xp.ones( self.avg_mean.shape, dtype=x.dtype) if self.beta is not None: beta = self.beta else: with chainer.using_device(self.device): beta = self.xp.zeros( self.avg_mean.shape, dtype=x.dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay avg_mean = self.avg_mean avg_var = self.avg_var if chainer.config.in_recomputing: # Do not update statistics when extra forward computation is # called. if finetune: self.N -= 1 # Revert the count avg_mean = self.xp.zeros_like(self.avg_mean) avg_var = self.xp.zeros_like(self.avg_var) ret = batch_renormalization.batch_renormalization( x, gamma, beta, self.rmax, self.dmax, self.eps, avg_mean, avg_var, decay, update_statistics=True) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, finetune=False): if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable( self.xp.ones(self.avg_mean.shape, dtype=self._highprec_dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable( self.xp.zeros(self.avg_mean.shape, dtype=self._highprec_dtype)) if chainer.configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalization( self.eps, self.avg_mean, self.avg_var, decay, impl_selector=( chainermn_batch_normalization.MultiNodeBNImplSelector( self.comm, self._communication_backend))) ret = func.apply((x, gamma, beta))[0] self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluatino during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', train)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype)) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalizationFunction( self.eps, self.avg_mean, self.avg_var, decay) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, test=False, finetune=False): """Invokes the forward propagation of BatchNormalization. BatchNormalization accepts additional arguments, which controls three different running mode. Args: x (Variable): An input variable. test (bool): If ``True``, BatchNormalization runs in testing mode; it normalizes the input using pre-computed statistics. finetune (bool): If ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. If ``test`` and ``finetune`` are both ``False``, then BatchNormalization runs in training mode; it computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. """ use_batch_mean = not test or finetune if hasattr(self, 'gamma'): gamma = self.gamma else: gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.data.dtype), volatile='auto') if hasattr(self, 'beta'): beta = self.beta else: beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.data.dtype), volatile='auto') if use_batch_mean: func = batch_normalization.BatchNormalizationFunction(self.eps) ret = func(x, gamma, beta) if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay with cuda.get_device(x.data): m = x.data.size // gamma.data.size adjust = m / max(m - 1., 1.) # unbiased estimation self.avg_mean *= decay func.mean *= 1 - decay # reuse buffer as a temporary self.avg_mean += func.mean del func.mean self.avg_var *= decay func.var *= (1 - decay) * adjust # reuse buffer as a temporary self.avg_var += func.var del func.var else: mean = variable.Variable(self.avg_mean, volatile='auto') var = variable.Variable(self.avg_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluatino during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', train)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable( self.xp.ones(self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable( self.xp.zeros(self.avg_mean.shape, dtype=x.dtype)) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalizationFunction( self.eps, self.avg_mean, self.avg_var, decay) ret = func(x, gamma, beta) self.avg_mean[:] = func.running_mean self.avg_var[:] = func.running_var else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def __call__(self, x, test=False, finetune=False): """Invokes the forward propagation of BatchNormalization. BatchNormalization accepts additional arguments, which controls three different running mode. Args: x (Variable): An input variable. test (bool): If ``True``, BatchNormalization runs in testing mode; it normalizes the input using pre-computed statistics. finetune (bool): If ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. If ``test`` and ``finetune`` are both ``False``, then BatchNormalization runs in training mode; it computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. """ # use_batch_mean = not test or finetune -------------------------------- # ----------------------------------------------------------------------------- use_batch_mean = False #True # Falseに設定して、後段のfixed batch normalizationをコールするように設定する. # でないと、mean, stdが実行長毎に代わる(学習時)なので、推論時に計算できましえn... if hasattr(self, 'gamma'): gamma = self.gamma else: gamma = variable.Variable(self.xp.ones(self.avg_mean.shape, dtype=x.dtype), volatile='auto') if hasattr(self, 'beta'): beta = self.beta else: beta = variable.Variable(self.xp.zeros(self.avg_mean.shape, dtype=x.dtype), volatile='auto') # デバッグ用に表示したルーチン # print("gamma in link") # print(gamma.data) # print("beta in link") # print(beta.data) if use_batch_mean: # print("use batch mean") if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay # print("self.avg_mean") # print(self.avg_mean) func = function_batch_normalization.BatchNormalizationFunction( self.eps, self.avg_mean, self.avg_var, True, decay) # func = function_batch_normalization.BatchNormalizationFunction( # self.eps, self.avg_mean, self.avg_var, False, decay) ret = func(x, gamma, beta) self.avg_mean = func.running_mean self.avg_var = func.running_var # print("func.running_mean") # print(func.running_mean) else: # print("use running average(fixed batch normalization)") # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean, volatile='auto') var = variable.Variable(self.avg_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) # print("mean") # print(self.avg_mean) # print("var") # print(self.avg_var) return ret
def __call__(self, x, test=False, finetune=False): """Invokes the forward propagation of BatchNormalization. BatchNormalization accepts additional arguments, which controls three different running mode. Args: x (Variable): An input variable. test (bool): If ``True``, BatchNormalization runs in testing mode; it normalizes the input using pre-computed statistics. finetune (bool): If ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. If ``test`` and ``finetune`` are both ``False``, then BatchNormalization runs in training mode; it computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. """ use_batch_mean = not test or finetune if hasattr(self, 'gamma'): gamma = self.gamma else: gamma = variable.Variable(self.xp.ones(self.avg_mean.shape, dtype=x.data.dtype), volatile='auto') if hasattr(self, 'beta'): beta = self.beta else: beta = variable.Variable(self.xp.zeros(self.avg_mean.shape, dtype=x.data.dtype), volatile='auto') if use_batch_mean: func = batch_normalization.BatchNormalizationFunction(self.eps) ret = func(x, gamma, beta) if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay with cuda.get_device(x.data): m = x.data.size // gamma.data.size adjust = m / max(m - 1., 1.) # unbiased estimation self.avg_mean *= decay func.mean *= 1 - decay # reuse buffer as a temporary self.avg_mean += func.mean del func.mean self.avg_var *= decay func.var *= (1 - decay) * adjust # reuse buffer as a temporary self.avg_var += func.var del func.var else: mean = variable.Variable(self.avg_mean, volatile='auto') var = variable.Variable(self.avg_var, volatile='auto') ret = batch_normalization.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret