def check_forward(self, x):
mkld.enable_batch_normalization = True
self.func1 = batch_normalization.BatchNormalizationFunction(
self.eps, self.mean, self.var, self.decay, False)
start = time.time()
y = self.func1.forward((x, self.gamma, self.beta))
end = time.time()
self.assertEqual(y[0].dtype, self.dtype)
print("mkldnn timing:", end - start)
mkld.enable_batch_normalization = False
self.func2 = batch_normalization.BatchNormalizationFunction(
self.eps, self.mean, self.var, self.decay, False)
start = time.time()
y_expect = self.func2.forward((x, self.gamma, self.beta))
end = time.time()
print("numpy timing:", end - start)
testing.assert_allclose(self.func1.running_mean,
self.func2.running_mean,
**self.check_forward_optionss)
testing.assert_allclose(self.func1.running_var, self.func2.running_var,
**self.check_forward_optionss)
testing.assert_allclose(y_expect[0], y[0],
**self.check_forward_optionss)
def check_backward(self, args, y_grad, use_cudnn='always'):
with chainer.using_config('use_cudnn', use_cudnn), \
chainer.using_config('train', self.train):
gradient_check.check_backward(
batch_normalization.BatchNormalizationFunction(
mean=None, var=None, decay=self.decay, eps=self.eps), args,
y_grad, **self.check_backward_options)
def __call__(self, x, test=False, finetune=False):
"""Invokes the forward propagation of BatchNormalization.
BatchNormalization accepts additional arguments, which controls three
different running mode.
Args:
x (Variable): Input variable.
test (bool): If ``True``, BatchNormalization runs in testing mode;
it normalizes the input using pre-computed statistics.
finetune (bool): If ``finetune`` is ``True`` and ``test`` is
``False``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
If ``test`` is ``False``, then BatchNormalization runs in training
mode; it computes moving averages of mean and variance for evaluation
during training, and normalizes the input using batch statistics.
"""
if hasattr(self, 'gamma'):
gamma = self.gamma
else:
with cuda.get_device(self._device_id):
gamma = variable.Variable(self.xp.ones(
self.avg_mean.shape, dtype=x.dtype), volatile='auto')
if hasattr(self, 'beta'):
beta = self.beta
else:
with cuda.get_device(self._device_id):
beta = variable.Variable(self.xp.zeros(
self.avg_mean.shape, dtype=x.dtype), volatile='auto')
# Var is always ones
with cuda.get_device(self._device_id):
self.one_var = self.xp.ones(self.avg_mean.shape, dtype=x.dtype)
if not test:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
func = batch_normalization.BatchNormalizationFunction(
self.eps, self.avg_mean, self.avg_var, True, decay,
self.use_cudnn)
ret = func(x, gamma, beta)
self.avg_mean[:] = func.running_mean
self.avg_var[:] = func.running_var
else:
# Use running average statistics or fine-tuned statistics.
mean = variable.Variable(self.avg_mean, volatile='auto')
#var = variable.Variable(self.avg_var, volatile='auto')
var = variable.Variable(self.one_var, volatile='auto')
ret = batch_normalization.fixed_batch_normalization(
x, gamma, beta, mean, var, self.eps, self.use_cudnn)
return ret
def check_backward(self, args, y_grad):
gradient_check.check_backward(
batch_normalization.BatchNormalizationFunction(eps=self.eps),
args,
y_grad,
eps=1e-2,
**self.check_backward_options)
def check_backward(self, args, y_grad):
gradient_check.check_backward(
batch_normalization.BatchNormalizationFunction(mean=None,
var=None,
train=self.train,
decay=self.decay,
eps=self.eps), args,
y_grad, **self.check_backward_options)
def check_backward(self, args, y_grad):
gradient_check.check_backward(
batch_normalization.BatchNormalizationFunction(eps=self.eps),
args,
y_grad,
eps=1e-2,
rtol=1e-3,
atol=1e-4)
def __call__(self, x, finetune=False):
"""Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluatino during training, and normalizes the
input using batch statistics.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
if hasattr(self, 'gamma'):
gamma = self.gamma
else:
with cuda.get_device(self._device_id):
gamma = variable.Variable(self.xp.ones(self.avg_mean.shape,
dtype=x.dtype),
volatile='auto')
if hasattr(self, 'beta'):
beta = self.beta
else:
with cuda.get_device(self._device_id):
beta = variable.Variable(self.xp.zeros(self.avg_mean.shape,
dtype=x.dtype),
volatile='auto')
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
func = batch_normalization.BatchNormalizationFunction(
self.eps, self.avg_mean, self.avg_var, decay, self.use_cudnn)
ret = func(x, gamma, beta)
self.avg_mean[:] = func.running_mean
self.avg_var[:] = func.running_var
else:
# Use running average statistics or fine-tuned statistics.
mean = variable.Variable(self.avg_mean, volatile='auto')
var = variable.Variable(self.avg_var, volatile='auto')
ret = batch_normalization.fixed_batch_normalization(
x, gamma, beta, mean, var, self.eps, self.use_cudnn)
return ret
def __call__(self, x, test=False, finetune=False):
"""Invokes the forward propagation of BatchNormalization.
BatchNormalization accepts additional arguments, which controls three
different running mode.
Args:
x (Variable): An input variable.
test (bool): If ``True``, BatchNormalization runs in testing mode;
it normalizes the input using pre-computed statistics.
finetune (bool): If ``True``, BatchNormalization runs in
fine-tuning mode; it accumulates the input array to compute
population statistics for normalization, and normalizes the
input using batch statistics.
If ``test`` and ``finetune`` are both ``False``, then
BatchNormalization runs in training mode; it computes moving averages
of mean and variance for evaluation during training, and normalizes the
input using batch statistics.
"""
use_batch_mean = not test or finetune
if use_batch_mean:
func = batch_normalization.BatchNormalizationFunction(self.eps)
ret = func(x, self.gamma, self.beta)
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
with cuda.get_device(x.data):
m = x.data.size // self.gamma.data.size
adjust = m / max(m - 1., 1.) # unbiased estimation
self.avg_mean *= decay
func.mean *= 1 - decay # reuse buffer as a temporary
self.avg_mean += func.mean
del func.mean
self.avg_var *= decay
func.var *= (1 - decay) * adjust # reuse buffer as a temporary
self.avg_var += func.var
del func.var
else:
mean = variable.Variable(self.avg_mean, volatile='auto')
var = variable.Variable(self.avg_var, volatile='auto')
ret = batch_normalization.fixed_batch_normalization(
x, self.gamma, self.beta, mean, var, self.eps)
return ret
def __call__(self, x, **kwargs):
"""__call__(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluatino during training, and normalizes the
input using batch statistics.
.. warning::
``test`` argument is not supported anymore since v2.
Instead, use ``chainer.using_config('train', train)``.
See :func:`chainer.using_config`.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
argument.check_unexpected_kwargs(
kwargs,
test='test argument is not supported anymore. '
'Use chainer.using_config')
finetune, = argument.parse_kwargs(kwargs, ('finetune', False))
if hasattr(self, 'gamma'):
gamma = self.gamma
else:
with cuda.get_device_from_id(self._device_id):
gamma = variable.Variable(
self.xp.ones(self.avg_mean.shape, dtype=x.dtype))
if hasattr(self, 'beta'):
beta = self.beta
else:
with cuda.get_device_from_id(self._device_id):
beta = variable.Variable(
self.xp.zeros(self.avg_mean.shape, dtype=x.dtype))
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
func = batch_normalization.BatchNormalizationFunction(
self.eps, self.avg_mean, self.avg_var, decay)
ret = func(x, gamma, beta)
self.avg_mean[:] = func.running_mean
self.avg_var[:] = func.running_var
else:
# Use running average statistics or fine-tuned statistics.
mean = variable.Variable(self.avg_mean)
var = variable.Variable(self.avg_var)
ret = batch_normalization.fixed_batch_normalization(
x, gamma, beta, mean, var, self.eps)
return ret
def __call__(self, x, **kwargs):
"""__call__(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluation during training, and normalizes the
input using batch statistics.
.. warning::
``test`` argument is not supported anymore since v2.
Instead, use ``chainer.using_config('train', False)``.
See :func:`chainer.using_config`.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
# check argument
argument.check_unexpected_kwargs(
kwargs, test='test argument is not supported anymore. '
'Use chainer.using_config')
finetune, = argument.parse_kwargs(kwargs, ('finetune', False))
original_shape = x.shape
batch_size = original_shape[0]
# reshape input x if batchsize > 1
if batch_size > 1:
reshaped_x = functions.expand_dims(x, axis=0)
else:
reshaped_x = x
if hasattr(self, 'gamma'):
gamma = self.gamma
if self.norm_grad:
# gamma.add_batch(batch_size)
gamma.n_batch = batch_size
else:
with cuda.get_device_from_id(self._device_id):
gamma = variable.Variable(self.xp.ones(
self.avg_mean.shape, dtype=x.dtype))
if hasattr(self, 'beta'):
beta = self.beta
if self.norm_grad:
# beta.add_batch(batch_size)
beta.n_batch = batch_size
else:
with cuda.get_device_from_id(self._device_id):
beta = variable.Variable(self.xp.zeros(
self.avg_mean.shape, dtype=x.dtype))
#align shapes if x was reshaped
if batch_size > 1:
mean = self.xp.stack((self.avg_mean,) * batch_size)
var = self.xp.stack((self.avg_var,) * batch_size)
gamma = functions.stack((gamma,) * batch_size)
beta = functions.stack((beta,) * batch_size)
else:
mean = self.xp.asarray(self.avg_mean)
var = self.xp.asarray(self.avg_var)
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
func = batch_normalization.BatchNormalizationFunction(
self.eps, mean, var, decay)
ret = func(reshaped_x, gamma, beta)
else:
head_ndim = gamma.ndim + 1
axis = (0,) + tuple(range(head_ndim, reshaped_x.ndim))
mean = reshaped_x.data.mean(axis=axis)
var = reshaped_x.data.var(axis=axis)
ret = functions.fixed_batch_normalization(
reshaped_x, gamma, beta, mean, var, self.eps)
# ret is normalized input x
if batch_size > 1:
ret = functions.reshape(ret, original_shape)
return ret
