def call_bn(bn, x, test=False, update_batch_stats=True):
if test:
return F.fixed_batch_normalization(x, bn.gamma, bn.beta, bn.avg_mean, bn.avg_var, use_cudnn=False)
elif not update_batch_stats:
return F.batch_normalization(x, bn.gamma, bn.beta, use_cudnn=False)
else:
return bn(x)
def __call__(self, input_, time, **kwargs):
argument.check_unexpected_kwargs(
kwargs,
test='test argument is not supported anymore. '
'Use chainer.using_config')
finetune, = argument.parse_kwargs(kwargs, ('finetune', False))
self._check_input_dim(input_)
if time >= self.max_length:
time = self.max_length - 1
running_mean = getattr(self, 'running_mean_{}'.format(time))
running_var = getattr(self, 'running_var_{}'.format(time))
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
return F.batch_normalization(input_,
running_mean=running_mean,
running_var=running_var,
gamma=self.gamma,
beta=self.beta,
eps=self.eps,
decay=decay)
else:
running_mean = chainer.Variable(running_mean)
running_var = chainer.Variable(running_var)
return F.fixed_batch_normalization(input_,
mean=running_mean,
var=running_var,
gamma=self.gamma,
beta=self.beta,
eps=self.eps)
def test_forward4(self):
N, C, H, W = 20, 10, 5, 5
x, gamma, beta, mean, var = get_params(N, C, H, W)
cy = CF.fixed_batch_normalization(x, gamma, beta, mean, var)
with dezero.test_mode():
y = F.batch_nrom(x, gamma, beta, mean, var)
self.assertTrue(array_allclose(y.data, cy.data))
def check_forward(self, args):
y = functions.fixed_batch_normalization(*[chainer.Variable(i) for i in args], eps=self.eps)
self.assertEqual(y.data.dtype, self.dtype)
y_expect = _batch_normalization(self.expander, self.gamma, self.beta, self.x, self.mean, self.var)
gradient_check.assert_allclose(y_expect, y.data, **self.check_forward_optionss)
def __call__(self, x, **kwargs):
"""__call__(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluation during training, and normalizes the
input using batch statistics.
.. warning::
``test`` argument is not supported anymore since v2.
Instead, use ``chainer.using_config('train', False)``.
See :func:`chainer.using_config`.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
finetune, = argument.parse_kwargs(
kwargs, ('finetune', False),
test='test argument is not supported anymore. '
'Use chainer.using_config')
if hasattr(self, 'gamma'):
gamma = self.gamma
else:
with cuda.get_device_from_id(self._device_id):
gamma = variable.Variable(self.xp.ones(
self.avg_mean.shape, dtype=x.dtype))
if hasattr(self, 'beta'):
beta = self.beta
else:
with cuda.get_device_from_id(self._device_id):
beta = variable.Variable(self.xp.zeros(
self.avg_mean.shape, dtype=x.dtype))
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
ret = functions.batch_normalization(
x, gamma, beta, eps=self.eps, running_mean=self.avg_mean,
running_var=self.avg_var, decay=decay, axis=self.axis)
else:
# Use running average statistics or fine-tuned statistics.
mean = variable.Variable(self.avg_mean)
var = variable.Variable(self.avg_var)
ret = functions.fixed_batch_normalization(
x, gamma, beta, mean, var, self.eps, axis=self.axis)
return ret
def __call__(self, x, **kwargs):
"""__call__(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluation during training, and normalizes the
input using batch statistics.
.. warning::
``test`` argument is not supported anymore since v2.
Instead, use ``chainer.using_config('train', False)``.
See :func:`chainer.using_config`.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
argument.check_unexpected_kwargs(
kwargs, test='test argument is not supported anymore. '
'Use chainer.using_config')
finetune, = argument.parse_kwargs(kwargs, ('finetune', False))
if hasattr(self, 'gamma'):
gamma = self.gamma
else:
with cuda.get_device_from_id(self._device_id):
gamma = variable.Variable(self.xp.ones(
self.avg_mean.shape, dtype=x.dtype))
if hasattr(self, 'beta'):
beta = self.beta
else:
with cuda.get_device_from_id(self._device_id):
beta = variable.Variable(self.xp.zeros(
self.avg_mean.shape, dtype=x.dtype))
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
ret = functions.batch_normalization(
x, gamma, beta, eps=self.eps, running_mean=self.avg_mean,
running_var=self.avg_var, decay=decay)
else:
# Use running average statistics or fine-tuned statistics.
mean = variable.Variable(self.avg_mean)
var = variable.Variable(self.avg_var)
ret = functions.fixed_batch_normalization(
x, gamma, beta, mean, var, self.eps)
return ret
def gen_convtranspose_bn(test_name):
gb = onnx_script.GraphBuilder(test_name)
bsize = 2
ichan = 3
ochan = 4
ksize = 3
isize = 7
x = aranges(bsize, ochan, isize, isize)
w = aranges(ochan, ichan, ksize, ksize) * 0.01
scale = aranges(ichan) * 0.1 + 1
bias = aranges(ichan) * 0.1 + 2
mean = aranges(ichan) * 0.1 + 3
var = aranges(ichan) * 0.1 + 4
conv = F.deconvolution_2d(x, w, pad=1, outsize=(isize, isize))
y = F.fixed_batch_normalization(conv, scale, bias, mean, var)
x_v = gb.input('x', x)
w_v = gb.param('w', w)
scale_v = gb.param('scale', scale)
bias_v = gb.param('bias', bias)
mean_v = gb.param('mean', mean)
var_v = gb.param('var', var)
conv_v = gb.ConvTranspose([x_v, w_v],
kernel_shape=[ksize, ksize],
pads=[1, 1, 1, 1],
output_shape=[isize, isize])
y_v = gb.BatchNormalization([conv_v, scale_v, bias_v, mean_v, var_v])
gb.output(y_v, y)
gb.gen_test()
def batchnorm():
x = rand((1, 3, 32, 32))
gamma = rand((3, ))
beta = rand((3, ))
mean = rand((3, ))
var = rand((3, ))
y = F.fixed_batch_normalization(x, gamma, beta, mean, var)
return {'input': x}, {'out': y}
def check_forward(self, args):
y = functions.fixed_batch_normalization(
*[chainer.Variable(i) for i in args], eps=self.eps)
self.assertEqual(y.data.dtype, numpy.float32)
y_expect = _batch_normalization(
self.expander, self.gamma, self.beta, self.x, self.mean, self.var)
gradient_check.assert_allclose(y_expect, y.data, rtol=1e-3, atol=1e-4)
def check_forward(self, args, use_cudnn='always'):
with chainer.using_config('use_cudnn', use_cudnn):
y = functions.fixed_batch_normalization(
*[chainer.Variable(i) for i in args], eps=self.eps)
self.assertEqual(y.data.dtype, self.dtype)
y_expect = _batch_normalization(self.expander, self.gamma, self.beta,
self.x, self.mean, self.var)
testing.assert_allclose(y_expect, y.data, **self.check_forward_options)
def check_forward(self, args, use_cudnn=True):
y = functions.fixed_batch_normalization(
*[chainer.Variable(i) for i in args],
eps=self.eps, use_cudnn=use_cudnn)
self.assertEqual(y.data.dtype, self.dtype)
y_expect = _batch_normalization(
self.expander, self.gamma, self.beta, self.x, self.mean, self.var)
testing.assert_allclose(
y_expect, y.data, **self.check_forward_options)
def check_forward(self, x):
mkld.enable_batch_normalization = True
start = time.time()
y = F.fixed_batch_normalization(x, self.gamma, self.beta, self.mean,
self.var, self.eps, False)
end = time.time()
mkldnn_timing = end - start
self.assertEqual(y[0].dtype, self.dtype)
mkld.enable_batch_normalization = False
start = time.time()
y_expect = F.fixed_batch_normalization(x, self.gamma, self.beta,
self.mean, self.var, self.eps,
False)
end = time.time()
cpu_timing = end - start
print("mkldnn timing:", mkldnn_timing)
print("cpu timing:", cpu_timing)
testing.assert_allclose(y_expect.data, y.data,
**self.check_forward_optionss)
def check_forward(self, inputs, backend_config):
y_expected, = self.forward_cpu(inputs)
if backend_config.use_cuda:
inputs = cuda.to_gpu(inputs)
if not self.c_contiguous:
inputs = _to_fcontiguous(inputs)
with backend_config:
y = functions.fixed_batch_normalization(*inputs, eps=self.eps)
assert y.data.dtype == self.dtype
testing.assert_allclose(
y_expected, y.data, **self.check_forward_options)
def check_forward(self, inputs, backend_config):
y_expected, = self.forward_cpu(inputs)
if backend_config.use_cuda:
inputs = cuda.to_gpu(inputs)
if not self.c_contiguous:
inputs = _to_fcontiguous(inputs)
with backend_config:
y = functions.fixed_batch_normalization(*inputs, eps=self.eps)
assert y.data.dtype == self.dtype
testing.assert_allclose(y_expected, y.data,
**self.check_forward_options)
def __call__(self, x):
assert x.ndim == 4 # BCHW
if self.gamma.data is None:
in_size = x[0].shape
self._initialize_params(in_size)
mean = F.broadcast_to(F.mean(x, keepdims=True, axis=(1, 2, 3)),
x.shape)
var = F.broadcast_to(
F.mean((x - mean)**2, keepdims=True, axis=(1, 2, 3)), x.shape)
mean = mean[0]
var = var[0]
return F.fixed_batch_normalization(x, self.gamma, self.beta, mean, var,
self.eps)
def check_forward(self, inputs, enable_backprop, backend_config):
y_expected, = self.forward_cpu(inputs)
inputs = backend_config.get_array(inputs)
if not self.c_contiguous:
with backend_config:
inputs = _as_noncontiguous_array(inputs)
with chainer.using_config('enable_backprop', enable_backprop):
with backend_config:
y = functions.fixed_batch_normalization(*inputs, eps=self.eps)
assert y.data.dtype == self.dtype
testing.assert_allclose(y_expected, y.data,
**self.check_forward_options)
def check_forward(self, inputs, enable_backprop, backend_config):
y_expected, = self.forward_cpu(inputs)
inputs = backend_config.get_array(inputs)
if not self.c_contiguous:
with backend_config:
inputs = _as_noncontiguous_array(inputs)
with chainer.using_config('enable_backprop', enable_backprop):
with backend_config:
y = functions.fixed_batch_normalization(*inputs, eps=self.eps)
assert y.data.dtype == self.dtype
testing.assert_allclose(
y_expected, y.data, **self.check_forward_options)
def check_forward(self, inputs, enable_backprop, backend_config):
# TODO(niboshi): Support it
if backend_config.use_chainerx and self.dtype == numpy.float16:
raise unittest.SkipTest('ChainerX does not support float16')
y_expected, = self.forward_cpu(inputs)
inputs = backend_config.get_array(inputs)
if not self.c_contiguous:
inputs = _as_noncontiguous_array(inputs)
with chainer.using_config('enable_backprop', enable_backprop):
with backend_config:
y = functions.fixed_batch_normalization(*inputs, eps=self.eps)
assert y.data.dtype == self.dtype
testing.assert_allclose(y_expected, y.data,
**self.check_forward_options)
def check_forward(self, inputs, enable_backprop, backend_config):
# TODO(niboshi): Support it
if backend_config.use_chainerx and self.dtype == numpy.float16:
raise unittest.SkipTest('ChainerX does not support float16')
y_expected, = self.forward_cpu(inputs)
inputs = backend_config.get_array(inputs)
if not self.c_contiguous:
inputs = _as_noncontiguous_array(inputs)
with chainer.using_config('enable_backprop', enable_backprop):
with backend_config:
y = functions.fixed_batch_normalization(*inputs, eps=self.eps)
assert y.data.dtype == self.dtype
testing.assert_allclose(
y_expected, y.data, **self.check_forward_options)
def forward(self, x):
with cuda.get_device_from_id(self._device_id):
gamma = avg_var = chainer.as_variable(
self.xp.ones(self.avg_mean.shape, dtype=x.dtype))
beta = chainer.as_variable(
self.xp.zeros(self.avg_mean.shape, dtype=x.dtype))
if configuration.config.train:
decay = self.decay
F.batch_normalization(x,
gamma,
beta,
eps=self.eps,
running_mean=self.avg_mean,
running_var=avg_var,
decay=decay)
mean = chainer.as_variable(self.avg_mean)
var = chainer.as_variable(avg_var)
ret = F.fixed_batch_normalization(x, gamma, beta, mean, var, self.eps)
return ret
def test_invalid(self):
with self.assertRaises(RuntimeError):
functions.fixed_batch_normalization(*self.args, eps=2e-6)
def test_invalid(self):
eps = -0.1
if chainer.backends.cuda.libcudnn.get_build_version() < 7500:
eps = 2e-6
with self.assertRaises(RuntimeError):
functions.fixed_batch_normalization(*self.args, eps=eps)
def __call__(self, x, **kwargs):
"""__call__(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluation during training, and normalizes the
input using batch statistics.
.. warning::
``test`` argument is not supported anymore since v2.
Instead, use ``chainer.using_config('train', False)``.
See :func:`chainer.using_config`.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
# check argument
argument.check_unexpected_kwargs(
kwargs,
test='test argument is not supported anymore. '
'Use chainer.using_config')
finetune, = argument.parse_kwargs(kwargs, ('finetune', False))
# reshape input x
original_shape = x.shape
batch_size, n_ch = original_shape[:2]
new_shape = (1, batch_size * n_ch) + original_shape[2:]
reshaped_x = F.reshape(x, new_shape)
if hasattr(self, 'gamma'):
gamma = self.gamma
else:
with cuda.get_device_from_id(self._device_id):
gamma = variable.Variable(
self.xp.ones(self.avg_mean.shape, dtype=x.dtype))
if hasattr(self, 'beta'):
beta = self.beta
else:
with cuda.get_device_from_id(self._device_id):
beta = variable.Variable(
self.xp.zeros(self.avg_mean.shape, dtype=x.dtype))
mean = self.xp.hstack([self.avg_mean] * batch_size)
var = self.xp.hstack([self.avg_var] * batch_size)
gamma = self.xp.hstack([gamma.array] * batch_size)
beta = self.xp.hstack([beta.array] * batch_size)
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
ret = F.batch_normalization(reshaped_x,
gamma,
beta,
eps=self.eps,
running_mean=mean,
running_var=var,
decay=decay)
else:
# Use running average statistics or fine-tuned statistics.
ret = F.fixed_batch_normalization(reshaped_x, gamma, beta, mean,
var, self.eps)
# ret is normalized input x
return F.reshape(ret, original_shape)
def forward(self, x, **kwargs):
"""forward(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluation during training, and normalizes the
input using batch statistics.
.. warning::
``test`` argument is not supported anymore since v2.
Instead, use ``chainer.using_config('train', False)``.
See :func:`chainer.using_config`.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
finetune, = argument.parse_kwargs(
kwargs, ('finetune', False),
test='test argument is not supported anymore. '
'Use chainer.using_config')
if self.avg_mean is None:
param_shape = tuple([
d
for i, d in enumerate(x.shape)
if i not in self.axis])
self._initialize_params(param_shape)
gamma = self.gamma
if gamma is None:
with cuda.get_device_from_id(self._device_id):
gamma = self.xp.ones(
self.avg_mean.shape, dtype=x.dtype)
beta = self.beta
if beta is None:
with cuda.get_device_from_id(self._device_id):
beta = self.xp.zeros(
self.avg_mean.shape, dtype=x.dtype)
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
ret = functions.batch_normalization(
x, gamma, beta, eps=self.eps, running_mean=self.avg_mean,
running_var=self.avg_var, decay=decay, axis=self.axis)
else:
# Use running average statistics or fine-tuned statistics.
mean = self.avg_mean
var = self.avg_var
ret = functions.fixed_batch_normalization(
x, gamma, beta, mean, var, self.eps, axis=self.axis)
return ret
def forward(self, x, **kwargs):
"""forward(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluation during training, and normalizes the
input using batch statistics.
.. warning::
``test`` argument is not supported anymore since v2.
Instead, use ``chainer.using_config('train', False)``.
See :func:`chainer.using_config`.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
finetune, = argument.parse_kwargs(
kwargs, ('finetune', False),
test='test argument is not supported anymore. '
'Use chainer.using_config')
if self.avg_mean is None:
param_shape = tuple(
[d for i, d in enumerate(x.shape) if i not in self.axis])
self._initialize_params(param_shape)
gamma = self.gamma
if gamma is None:
with chainer.using_device(self.device):
gamma = self.xp.ones(self.avg_mean.shape, dtype=x.dtype)
beta = self.beta
if beta is None:
with chainer.using_device(self.device):
beta = self.xp.zeros(self.avg_mean.shape, dtype=x.dtype)
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
avg_mean = self.avg_mean
avg_var = self.avg_var
if chainer.config.in_recomputing:
# Do not update statistics when extra forward computation is
# called.
if finetune:
self.N -= 1 # Revert the count
avg_mean = None
avg_var = None
ret = functions.batch_normalization(x,
gamma,
beta,
eps=self.eps,
running_mean=avg_mean,
running_var=avg_var,
decay=decay,
axis=self.axis)
else:
# Use running average statistics or fine-tuned statistics.
mean = self.avg_mean
var = self.avg_var
ret = functions.fixed_batch_normalization(x,
gamma,
beta,
mean,
var,
self.eps,
axis=self.axis)
return ret
def forward(self, x, **kwargs):
"""forward(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluation during training, and normalizes the
input using batch statistics.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
finetune, = argument.parse_kwargs(
kwargs, ('finetune', False),
test='test argument is not supported anymore. '
'Use chainer.using_config')
if self.avg_mean is None:
param_shape = tuple([
d
for i, d in enumerate(x.shape)
if i not in self.axis])
self._initialize_params(param_shape)
gamma = self.gamma
if gamma is None:
with chainer.using_device(self.device):
gamma = self.xp.ones(
self.avg_mean.shape, dtype=self._highprec_dtype)
beta = self.beta
if beta is None:
with chainer.using_device(self.device):
beta = self.xp.zeros(
self.avg_mean.shape, dtype=self._highprec_dtype)
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
avg_mean = self.avg_mean
avg_var = self.avg_var
if chainer.config.in_recomputing:
# Do not update statistics when extra forward computation is
# called.
if finetune:
self.N -= 1 # Revert the count
avg_mean = None
avg_var = None
ret = functions.batch_normalization(
x, gamma, beta, eps=self.eps, running_mean=avg_mean,
running_var=avg_var, decay=decay, axis=self.axis)
else:
# Use running average statistics or fine-tuned statistics.
mean = self.avg_mean
var = self.avg_var
ret = functions.fixed_batch_normalization(
x, gamma, beta, mean, var, self.eps, axis=self.axis)
return ret
def __call__(self, x, **kwargs):
"""__call__(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluation during training, and normalizes the
input using batch statistics.
.. warning::
``test`` argument is not supported anymore since v2.
Instead, use ``chainer.using_config('train', False)``.
See :func:`chainer.using_config`.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
argument.check_unexpected_kwargs(
kwargs, test='test argument is not supported anymore. '
'Use chainer.using_config')
finetune, = argument.parse_kwargs(kwargs, ('finetune', False))
# if hasattr(self, 'gamma_rr'):
# gamma_rr = self.gamma_rr
# gamma_ii = self.gamma_ii
# gamma_ri = self.gamma_ri
# else:
# assert False, "yikes, no gamms"
# with cuda.get_device_from_id(self._device_id):
# gamma = variable.Variable(self.xp.ones(
# self.avg_mean.shape, dtype=x.dtype))
# if hasattr(self, 'beta'):
# beta = self.beta
# else:
# with cuda.get_device_from_id(self._device_id):
# beta = variable.Variable(self.xp.zeros(
# self.avg_mean.shape, dtype=x.dtype))
# if configuration.config.train:
if True:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
# Brad says, "OK" -- LOL
mu_real = F.mean(x[:,x.shape[1]/2:,:,:], axis=(0,2,3))
mu_imag = F.mean(x[:,:x.shape[1]/2,:,:], axis=(0,2,3))
#### Do the moving average stuff~~~~~
with chainer.no_backprop_mode():
self.moving_real_mean = self.moving_real_mean*self.decay + mu_real*(1-self.decay)
self.moving_imag_mean = self.moving_imag_mean*self.decay + mu_imag*(1-self.decay)
mu_real = self.moving_real_mean
mu_imag = self.moving_imag_mean
x_real = x[:,:x.shape[1]/2,:,:]
x_imag = x[:,x.shape[1]/2:,:,:]
# x_real_center = x[:,:x.shape[1]/2,:,:] - mu_real.data
# b_x_real, b_mu = F.broadcast(x[:,:x.shape[1]/2,:,:], mu_real.reshape(1,mu_real.shape[0],1,1))
b_mu_real = F.broadcast_to(F.reshape(mu_real, (1, mu_real.shape[0],1,1)), (x.shape[0], x.shape[1]/2, x.shape[2], x.shape[3]))
x_real_center = x_real - b_mu_real
# b_x_imag, b_mu = F.broadcast(x[:,x.shape[1]/2:,:,:], mu_imag.reshape(1,mu_imag.shape[0],1,1))
b_mu_imag = F.broadcast_to(F.reshape(mu_imag, (1, mu_imag.shape[0],1,1)), (x.shape[0], x.shape[1]/2, x.shape[2], x.shape[3]))
x_imag_center = x_imag - b_mu_imag
# x_imag_center = x[:,x.shape[1]/2:,:,:] - mu_imag.data
Vrr = F.mean(x_real_center**2, axis=(0,2,3)) + self.eps
Vii = F.mean(x_imag_center**2, axis=(0,2,3)) + self.eps
Vri = F.mean(x_real_center * x_imag_center, axis=(0,2,3)) + self.eps
#### Do the moving average stuff~~~~~
with chainer.no_backprop_mode():
self.moving_Vrr = self.moving_Vrr*self.decay + Vrr*(1-self.decay)
self.moving_Vii = self.moving_Vii*self.decay + Vii*(1-self.decay)
self.moving_Vri = self.moving_Vri*self.decay + Vri*(1-self.decay)
Vrr = self.moving_Vrr
Vii = self.moving_Vii
Vri = self.moving_Vri
tau = Vrr + Vii
# delta = (Vrr * Vii) - (Vri ** 2) = Determinant. Guaranteed >= 0 because SPD
delta = (Vrr * Vii) - (Vri ** 2)
s = F.sqrt(delta) # Determinant of square root matrix
t = F.sqrt(tau + 2 * s)
inverse_st = 1.0 / (s * t)
Wrr = (Vii + s) * inverse_st
Wii = (Vrr + s) * inverse_st
Wri = -Vri * inverse_st
# Wrr_b = F.broadcast_to(Wrr.reshape(1,Wrr.shape[0], 1,1), x_real_center.shape)
# Wii_b = F.broadcast_to(Wii.reshape(1,Wii.shape[0], 1,1), x_real_center.shape)
# Wri_b = F.broadcast_to(Wri.reshape(1,Wri.shape[0], 1,1), x_real_center.shape)
Wrr_b = F.broadcast_to(F.reshape(Wrr, (1,Wrr.shape[0], 1,1)), x_real_center.shape)
Wii_b = F.broadcast_to(F.reshape(Wii, (1,Wii.shape[0], 1,1)), x_real_center.shape)
Wri_b = F.broadcast_to(F.reshape(Wri, (1,Wri.shape[0], 1,1)), x_real_center.shape)
x_tilda_real = Wrr_b.data*x_real_center + Wri_b.data*x_imag_center
x_tilda_imag = Wri_b.data*x_real_center + Wii_b.data*x_imag_center
# gamma_rr = F.broadcast_to(self.gamma_rr.reshape(1,self.gamma_rr.shape[0], 1,1), x_tilda_real.shape)
# gamma_ri = F.broadcast_to(self.gamma_ri.reshape(1,self.gamma_ri.shape[0], 1,1), x_tilda_real.shape)
# gamma_ii = F.broadcast_to(self.gamma_ii.reshape(1,self.gamma_ii.shape[0], 1,1), x_tilda_real.shape)
gamma_rr_b = F.broadcast_to(F.reshape(self.gamma_rr, (1,self.gamma_rr.shape[0], 1,1)), x_tilda_real.shape)
gamma_ri_b = F.broadcast_to(F.reshape(self.gamma_ri, (1,self.gamma_ri.shape[0], 1,1)), x_tilda_real.shape)
gamma_ii_b = F.broadcast_to(F.reshape(self.gamma_ii, (1,self.gamma_ii.shape[0], 1,1)), x_tilda_real.shape)
x_final_real = gamma_rr_b * x_tilda_real + gamma_ri_b * x_tilda_imag
x_final_imag = gamma_ri_b * x_tilda_real + gamma_ii_b * x_tilda_imag
x_final_real = x_final_real + F.broadcast_to(F.reshape(self.beta[:self.beta.shape[0]/2], (1,self.beta.shape[0]/2, 1,1)),
(x_final_real.shape))
x_final_imag = x_final_imag + F.broadcast_to(F.reshape(self.beta[self.beta.shape[0]/2:], (1,self.beta.shape[0]/2, 1,1)),
(x_final_imag.shape))
# print self.gamma_rr.debug_print()
# print x_final_real.debug_print()
return F.concat([x_final_real,x_final_imag], axis=1)
# keep to know what format exactly is being returned
# ret = functions.batch_normalization(
# x, gamma, beta, eps=self.eps, running_mean=self.avg_mean,
# running_var=self.avg_var, decay=decay)
# print type(ret), ret.shape
else:
# Use running average statistics or fine-tuned statistics.
assert False
mean = variable.Variable(self.avg_mean)
var = variable.Variable(self.avg_var)
ret = functions.fixed_batch_normalization(
x, gamma, beta, mean, var, self.eps)
return ret
def f(*inputs):
y = functions.fixed_batch_normalization(*inputs, eps=self.eps)
return y * y, # make nonlinear against beta
def f(*inputs):
y = functions.fixed_batch_normalization(*inputs, eps=self.eps)
return y,
def batch_normalization(self, *args):
return functions.fixed_batch_normalization(*args, eps=self.eps)
def f(*inputs):
return functions.fixed_batch_normalization(*inputs, eps=self.eps)
def f(*inputs):
y = functions.fixed_batch_normalization(*inputs, eps=self.eps)
return y * y, # make nonlinear against beta
def test_invalid(self):
eps = -0.1
if chainer.backends.cuda.libcudnn.get_build_version() < 7500:
eps = 2e-6
with self.assertRaises(RuntimeError):
functions.fixed_batch_normalization(*self.args, eps=eps)
def __call__(self, x, **kwargs):
"""__call__(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluation during training, and normalizes the
input using batch statistics.
.. warning::
``test`` argument is not supported anymore since v2.
Instead, use ``chainer.using_config('train', False)``.
See :func:`chainer.using_config`.
Args:
x (Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
# check argument
argument.check_unexpected_kwargs(
kwargs, test='test argument is not supported anymore. '
'Use chainer.using_config')
finetune, = argument.parse_kwargs(kwargs, ('finetune', False))
original_shape = x.shape
batch_size = original_shape[0]
# reshape input x if batchsize > 1
if batch_size > 1:
reshaped_x = functions.expand_dims(x, axis=0)
else:
reshaped_x = x
if hasattr(self, 'gamma'):
gamma = self.gamma
if self.norm_grad:
# gamma.add_batch(batch_size)
gamma.n_batch = batch_size
else:
with cuda.get_device_from_id(self._device_id):
gamma = variable.Variable(self.xp.ones(
self.avg_mean.shape, dtype=x.dtype))
if hasattr(self, 'beta'):
beta = self.beta
if self.norm_grad:
# beta.add_batch(batch_size)
beta.n_batch = batch_size
else:
with cuda.get_device_from_id(self._device_id):
beta = variable.Variable(self.xp.zeros(
self.avg_mean.shape, dtype=x.dtype))
#align shapes if x was reshaped
if batch_size > 1:
mean = self.xp.stack((self.avg_mean,) * batch_size)
var = self.xp.stack((self.avg_var,) * batch_size)
gamma = functions.stack((gamma,) * batch_size)
beta = functions.stack((beta,) * batch_size)
else:
mean = self.xp.asarray(self.avg_mean)
var = self.xp.asarray(self.avg_var)
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
func = batch_normalization.BatchNormalizationFunction(
self.eps, mean, var, decay)
ret = func(reshaped_x, gamma, beta)
else:
head_ndim = gamma.ndim + 1
axis = (0,) + tuple(range(head_ndim, reshaped_x.ndim))
mean = reshaped_x.data.mean(axis=axis)
var = reshaped_x.data.var(axis=axis)
ret = functions.fixed_batch_normalization(
reshaped_x, gamma, beta, mean, var, self.eps)
# ret is normalized input x
if batch_size > 1:
ret = functions.reshape(ret, original_shape)
return ret
def test_valid(self):
functions.fixed_batch_normalization(*self.args, eps=1e-5)
def batch_normalization(self, *args):
return functions.fixed_batch_normalization(*args, eps=self.eps)
def __call__(self, x):
mean = x.array.mean(axis=0)
var = x.array.var(axis=0)
gamma = np.ones_like(mean, dtype=x.dtype)
beta = np.zeros_like(mean, dtype=x.dtype)
return F.fixed_batch_normalization(x, gamma, beta, mean, var)
def test_valid(self):
functions.fixed_batch_normalization(*self.args, eps=1e-5)
def __call__(self, x, gamma, beta, mean, var):
return F.fixed_batch_normalization(x, gamma, beta, mean, var)
def forward(self, x, **kwargs):
"""forward(self, x, finetune=False)
Invokes the forward propagation of BatchNormalization.
In training mode, the BatchNormalization computes moving averages of
mean and variance for evaluation during training, and normalizes the
input using batch statistics.
Args:
x (~chainer.Variable): Input variable.
finetune (bool): If it is in the training mode and ``finetune`` is
``True``, BatchNormalization runs in fine-tuning mode; it
accumulates the input array to compute population statistics
for normalization, and normalizes the input using batch
statistics.
"""
finetune, = argument.parse_kwargs(
kwargs, ('finetune', False),
test='test argument is not supported anymore. '
'Use chainer.using_config')
if self.avg_mean is None:
param_shape = tuple(
[d for i, d in enumerate(x.shape) if i not in self.axis])
self._initialize_params(param_shape)
# When using static_graph optimizations beta or gamma might not be
# initialized and is not retained by the function, so the
# static forward pass will get a None instead
gamma = self.gamma
if gamma is None:
gamma, = self._get_gamma()
beta = self.beta
if beta is None:
beta, = self._get_beta()
if configuration.config.train:
if finetune:
self.N += 1
decay = 1. - 1. / self.N
else:
decay = self.decay
avg_mean = self.avg_mean
avg_var = self.avg_var
if chainer.config.in_recomputing:
# Do not update statistics when extra forward computation is
# called.
if finetune:
self.N -= 1 # Revert the count
avg_mean = None
avg_var = None
ret = functions.batch_normalization(x,
gamma,
beta,
eps=self.eps,
running_mean=avg_mean,
running_var=avg_var,
decay=decay,
axis=self.axis)
else:
# Use running average statistics or fine-tuned statistics.
mean = self.avg_mean
var = self.avg_var
ret = functions.fixed_batch_normalization(x,
gamma,
beta,
mean,
var,
self.eps,
axis=self.axis)
return ret
def test_invalid(self):
with self.assertRaises(RuntimeError):
functions.fixed_batch_normalization(*self.args, eps=2e-6)
