def call_bn(bn, x, test=False, update_batch_stats=True): if test: return F.fixed_batch_normalization(x, bn.gamma, bn.beta, bn.avg_mean, bn.avg_var, use_cudnn=False) elif not update_batch_stats: return F.batch_normalization(x, bn.gamma, bn.beta, use_cudnn=False) else: return bn(x)
def __call__(self, input_, time, **kwargs): argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) self._check_input_dim(input_) if time >= self.max_length: time = self.max_length - 1 running_mean = getattr(self, 'running_mean_{}'.format(time)) running_var = getattr(self, 'running_var_{}'.format(time)) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay return F.batch_normalization(input_, running_mean=running_mean, running_var=running_var, gamma=self.gamma, beta=self.beta, eps=self.eps, decay=decay) else: running_mean = chainer.Variable(running_mean) running_var = chainer.Variable(running_var) return F.fixed_batch_normalization(input_, mean=running_mean, var=running_var, gamma=self.gamma, beta=self.beta, eps=self.eps)
def test_forward4(self): N, C, H, W = 20, 10, 5, 5 x, gamma, beta, mean, var = get_params(N, C, H, W) cy = CF.fixed_batch_normalization(x, gamma, beta, mean, var) with dezero.test_mode(): y = F.batch_nrom(x, gamma, beta, mean, var) self.assertTrue(array_allclose(y.data, cy.data))
def check_forward(self, args): y = functions.fixed_batch_normalization(*[chainer.Variable(i) for i in args], eps=self.eps) self.assertEqual(y.data.dtype, self.dtype) y_expect = _batch_normalization(self.expander, self.gamma, self.beta, self.x, self.mean, self.var) gradient_check.assert_allclose(y_expect, y.data, **self.check_forward_optionss)
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ finetune, = argument.parse_kwargs( kwargs, ('finetune', False), test='test argument is not supported anymore. ' 'Use chainer.using_config') if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype)) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay ret = functions.batch_normalization( x, gamma, beta, eps=self.eps, running_mean=self.avg_mean, running_var=self.avg_var, decay=decay, axis=self.axis) else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = functions.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, axis=self.axis) return ret
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype)) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay ret = functions.batch_normalization( x, gamma, beta, eps=self.eps, running_mean=self.avg_mean, running_var=self.avg_var, decay=decay) else: # Use running average statistics or fine-tuned statistics. mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = functions.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def gen_convtranspose_bn(test_name): gb = onnx_script.GraphBuilder(test_name) bsize = 2 ichan = 3 ochan = 4 ksize = 3 isize = 7 x = aranges(bsize, ochan, isize, isize) w = aranges(ochan, ichan, ksize, ksize) * 0.01 scale = aranges(ichan) * 0.1 + 1 bias = aranges(ichan) * 0.1 + 2 mean = aranges(ichan) * 0.1 + 3 var = aranges(ichan) * 0.1 + 4 conv = F.deconvolution_2d(x, w, pad=1, outsize=(isize, isize)) y = F.fixed_batch_normalization(conv, scale, bias, mean, var) x_v = gb.input('x', x) w_v = gb.param('w', w) scale_v = gb.param('scale', scale) bias_v = gb.param('bias', bias) mean_v = gb.param('mean', mean) var_v = gb.param('var', var) conv_v = gb.ConvTranspose([x_v, w_v], kernel_shape=[ksize, ksize], pads=[1, 1, 1, 1], output_shape=[isize, isize]) y_v = gb.BatchNormalization([conv_v, scale_v, bias_v, mean_v, var_v]) gb.output(y_v, y) gb.gen_test()
def batchnorm(): x = rand((1, 3, 32, 32)) gamma = rand((3, )) beta = rand((3, )) mean = rand((3, )) var = rand((3, )) y = F.fixed_batch_normalization(x, gamma, beta, mean, var) return {'input': x}, {'out': y}
def check_forward(self, args): y = functions.fixed_batch_normalization( *[chainer.Variable(i) for i in args], eps=self.eps) self.assertEqual(y.data.dtype, numpy.float32) y_expect = _batch_normalization( self.expander, self.gamma, self.beta, self.x, self.mean, self.var) gradient_check.assert_allclose(y_expect, y.data, rtol=1e-3, atol=1e-4)
def check_forward(self, args, use_cudnn='always'): with chainer.using_config('use_cudnn', use_cudnn): y = functions.fixed_batch_normalization( *[chainer.Variable(i) for i in args], eps=self.eps) self.assertEqual(y.data.dtype, self.dtype) y_expect = _batch_normalization(self.expander, self.gamma, self.beta, self.x, self.mean, self.var) testing.assert_allclose(y_expect, y.data, **self.check_forward_options)
def check_forward(self, args, use_cudnn=True): y = functions.fixed_batch_normalization( *[chainer.Variable(i) for i in args], eps=self.eps, use_cudnn=use_cudnn) self.assertEqual(y.data.dtype, self.dtype) y_expect = _batch_normalization( self.expander, self.gamma, self.beta, self.x, self.mean, self.var) testing.assert_allclose( y_expect, y.data, **self.check_forward_options)
def check_forward(self, x): mkld.enable_batch_normalization = True start = time.time() y = F.fixed_batch_normalization(x, self.gamma, self.beta, self.mean, self.var, self.eps, False) end = time.time() mkldnn_timing = end - start self.assertEqual(y[0].dtype, self.dtype) mkld.enable_batch_normalization = False start = time.time() y_expect = F.fixed_batch_normalization(x, self.gamma, self.beta, self.mean, self.var, self.eps, False) end = time.time() cpu_timing = end - start print("mkldnn timing:", mkldnn_timing) print("cpu timing:", cpu_timing) testing.assert_allclose(y_expect.data, y.data, **self.check_forward_optionss)
def check_forward(self, inputs, backend_config): y_expected, = self.forward_cpu(inputs) if backend_config.use_cuda: inputs = cuda.to_gpu(inputs) if not self.c_contiguous: inputs = _to_fcontiguous(inputs) with backend_config: y = functions.fixed_batch_normalization(*inputs, eps=self.eps) assert y.data.dtype == self.dtype testing.assert_allclose( y_expected, y.data, **self.check_forward_options)
def check_forward(self, inputs, backend_config): y_expected, = self.forward_cpu(inputs) if backend_config.use_cuda: inputs = cuda.to_gpu(inputs) if not self.c_contiguous: inputs = _to_fcontiguous(inputs) with backend_config: y = functions.fixed_batch_normalization(*inputs, eps=self.eps) assert y.data.dtype == self.dtype testing.assert_allclose(y_expected, y.data, **self.check_forward_options)
def __call__(self, x): assert x.ndim == 4 # BCHW if self.gamma.data is None: in_size = x[0].shape self._initialize_params(in_size) mean = F.broadcast_to(F.mean(x, keepdims=True, axis=(1, 2, 3)), x.shape) var = F.broadcast_to( F.mean((x - mean)**2, keepdims=True, axis=(1, 2, 3)), x.shape) mean = mean[0] var = var[0] return F.fixed_batch_normalization(x, self.gamma, self.beta, mean, var, self.eps)
def check_forward(self, inputs, enable_backprop, backend_config): y_expected, = self.forward_cpu(inputs) inputs = backend_config.get_array(inputs) if not self.c_contiguous: with backend_config: inputs = _as_noncontiguous_array(inputs) with chainer.using_config('enable_backprop', enable_backprop): with backend_config: y = functions.fixed_batch_normalization(*inputs, eps=self.eps) assert y.data.dtype == self.dtype testing.assert_allclose(y_expected, y.data, **self.check_forward_options)
def check_forward(self, inputs, enable_backprop, backend_config): y_expected, = self.forward_cpu(inputs) inputs = backend_config.get_array(inputs) if not self.c_contiguous: with backend_config: inputs = _as_noncontiguous_array(inputs) with chainer.using_config('enable_backprop', enable_backprop): with backend_config: y = functions.fixed_batch_normalization(*inputs, eps=self.eps) assert y.data.dtype == self.dtype testing.assert_allclose( y_expected, y.data, **self.check_forward_options)
def check_forward(self, inputs, enable_backprop, backend_config): # TODO(niboshi): Support it if backend_config.use_chainerx and self.dtype == numpy.float16: raise unittest.SkipTest('ChainerX does not support float16') y_expected, = self.forward_cpu(inputs) inputs = backend_config.get_array(inputs) if not self.c_contiguous: inputs = _as_noncontiguous_array(inputs) with chainer.using_config('enable_backprop', enable_backprop): with backend_config: y = functions.fixed_batch_normalization(*inputs, eps=self.eps) assert y.data.dtype == self.dtype testing.assert_allclose(y_expected, y.data, **self.check_forward_options)
def check_forward(self, inputs, enable_backprop, backend_config): # TODO(niboshi): Support it if backend_config.use_chainerx and self.dtype == numpy.float16: raise unittest.SkipTest('ChainerX does not support float16') y_expected, = self.forward_cpu(inputs) inputs = backend_config.get_array(inputs) if not self.c_contiguous: inputs = _as_noncontiguous_array(inputs) with chainer.using_config('enable_backprop', enable_backprop): with backend_config: y = functions.fixed_batch_normalization(*inputs, eps=self.eps) assert y.data.dtype == self.dtype testing.assert_allclose( y_expected, y.data, **self.check_forward_options)
def forward(self, x): with cuda.get_device_from_id(self._device_id): gamma = avg_var = chainer.as_variable( self.xp.ones(self.avg_mean.shape, dtype=x.dtype)) beta = chainer.as_variable( self.xp.zeros(self.avg_mean.shape, dtype=x.dtype)) if configuration.config.train: decay = self.decay F.batch_normalization(x, gamma, beta, eps=self.eps, running_mean=self.avg_mean, running_var=avg_var, decay=decay) mean = chainer.as_variable(self.avg_mean) var = chainer.as_variable(avg_var) ret = F.fixed_batch_normalization(x, gamma, beta, mean, var, self.eps) return ret
def test_invalid(self): with self.assertRaises(RuntimeError): functions.fixed_batch_normalization(*self.args, eps=2e-6)
def test_invalid(self): eps = -0.1 if chainer.backends.cuda.libcudnn.get_build_version() < 7500: eps = 2e-6 with self.assertRaises(RuntimeError): functions.fixed_batch_normalization(*self.args, eps=eps)
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ # check argument argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) # reshape input x original_shape = x.shape batch_size, n_ch = original_shape[:2] new_shape = (1, batch_size * n_ch) + original_shape[2:] reshaped_x = F.reshape(x, new_shape) if hasattr(self, 'gamma'): gamma = self.gamma else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable( self.xp.ones(self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable( self.xp.zeros(self.avg_mean.shape, dtype=x.dtype)) mean = self.xp.hstack([self.avg_mean] * batch_size) var = self.xp.hstack([self.avg_var] * batch_size) gamma = self.xp.hstack([gamma.array] * batch_size) beta = self.xp.hstack([beta.array] * batch_size) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay ret = F.batch_normalization(reshaped_x, gamma, beta, eps=self.eps, running_mean=mean, running_var=var, decay=decay) else: # Use running average statistics or fine-tuned statistics. ret = F.fixed_batch_normalization(reshaped_x, gamma, beta, mean, var, self.eps) # ret is normalized input x return F.reshape(ret, original_shape)
def forward(self, x, **kwargs): """forward(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ finetune, = argument.parse_kwargs( kwargs, ('finetune', False), test='test argument is not supported anymore. ' 'Use chainer.using_config') if self.avg_mean is None: param_shape = tuple([ d for i, d in enumerate(x.shape) if i not in self.axis]) self._initialize_params(param_shape) gamma = self.gamma if gamma is None: with cuda.get_device_from_id(self._device_id): gamma = self.xp.ones( self.avg_mean.shape, dtype=x.dtype) beta = self.beta if beta is None: with cuda.get_device_from_id(self._device_id): beta = self.xp.zeros( self.avg_mean.shape, dtype=x.dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay ret = functions.batch_normalization( x, gamma, beta, eps=self.eps, running_mean=self.avg_mean, running_var=self.avg_var, decay=decay, axis=self.axis) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = functions.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, axis=self.axis) return ret
def forward(self, x, **kwargs): """forward(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ finetune, = argument.parse_kwargs( kwargs, ('finetune', False), test='test argument is not supported anymore. ' 'Use chainer.using_config') if self.avg_mean is None: param_shape = tuple( [d for i, d in enumerate(x.shape) if i not in self.axis]) self._initialize_params(param_shape) gamma = self.gamma if gamma is None: with chainer.using_device(self.device): gamma = self.xp.ones(self.avg_mean.shape, dtype=x.dtype) beta = self.beta if beta is None: with chainer.using_device(self.device): beta = self.xp.zeros(self.avg_mean.shape, dtype=x.dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay avg_mean = self.avg_mean avg_var = self.avg_var if chainer.config.in_recomputing: # Do not update statistics when extra forward computation is # called. if finetune: self.N -= 1 # Revert the count avg_mean = None avg_var = None ret = functions.batch_normalization(x, gamma, beta, eps=self.eps, running_mean=avg_mean, running_var=avg_var, decay=decay, axis=self.axis) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = functions.fixed_batch_normalization(x, gamma, beta, mean, var, self.eps, axis=self.axis) return ret
def forward(self, x, **kwargs): """forward(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ finetune, = argument.parse_kwargs( kwargs, ('finetune', False), test='test argument is not supported anymore. ' 'Use chainer.using_config') if self.avg_mean is None: param_shape = tuple([ d for i, d in enumerate(x.shape) if i not in self.axis]) self._initialize_params(param_shape) gamma = self.gamma if gamma is None: with chainer.using_device(self.device): gamma = self.xp.ones( self.avg_mean.shape, dtype=self._highprec_dtype) beta = self.beta if beta is None: with chainer.using_device(self.device): beta = self.xp.zeros( self.avg_mean.shape, dtype=self._highprec_dtype) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay avg_mean = self.avg_mean avg_var = self.avg_var if chainer.config.in_recomputing: # Do not update statistics when extra forward computation is # called. if finetune: self.N -= 1 # Revert the count avg_mean = None avg_var = None ret = functions.batch_normalization( x, gamma, beta, eps=self.eps, running_mean=avg_mean, running_var=avg_var, decay=decay, axis=self.axis) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = functions.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps, axis=self.axis) return ret
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) # if hasattr(self, 'gamma_rr'): # gamma_rr = self.gamma_rr # gamma_ii = self.gamma_ii # gamma_ri = self.gamma_ri # else: # assert False, "yikes, no gamms" # with cuda.get_device_from_id(self._device_id): # gamma = variable.Variable(self.xp.ones( # self.avg_mean.shape, dtype=x.dtype)) # if hasattr(self, 'beta'): # beta = self.beta # else: # with cuda.get_device_from_id(self._device_id): # beta = variable.Variable(self.xp.zeros( # self.avg_mean.shape, dtype=x.dtype)) # if configuration.config.train: if True: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay # Brad says, "OK" -- LOL mu_real = F.mean(x[:,x.shape[1]/2:,:,:], axis=(0,2,3)) mu_imag = F.mean(x[:,:x.shape[1]/2,:,:], axis=(0,2,3)) #### Do the moving average stuff~~~~~ with chainer.no_backprop_mode(): self.moving_real_mean = self.moving_real_mean*self.decay + mu_real*(1-self.decay) self.moving_imag_mean = self.moving_imag_mean*self.decay + mu_imag*(1-self.decay) mu_real = self.moving_real_mean mu_imag = self.moving_imag_mean x_real = x[:,:x.shape[1]/2,:,:] x_imag = x[:,x.shape[1]/2:,:,:] # x_real_center = x[:,:x.shape[1]/2,:,:] - mu_real.data # b_x_real, b_mu = F.broadcast(x[:,:x.shape[1]/2,:,:], mu_real.reshape(1,mu_real.shape[0],1,1)) b_mu_real = F.broadcast_to(F.reshape(mu_real, (1, mu_real.shape[0],1,1)), (x.shape[0], x.shape[1]/2, x.shape[2], x.shape[3])) x_real_center = x_real - b_mu_real # b_x_imag, b_mu = F.broadcast(x[:,x.shape[1]/2:,:,:], mu_imag.reshape(1,mu_imag.shape[0],1,1)) b_mu_imag = F.broadcast_to(F.reshape(mu_imag, (1, mu_imag.shape[0],1,1)), (x.shape[0], x.shape[1]/2, x.shape[2], x.shape[3])) x_imag_center = x_imag - b_mu_imag # x_imag_center = x[:,x.shape[1]/2:,:,:] - mu_imag.data Vrr = F.mean(x_real_center**2, axis=(0,2,3)) + self.eps Vii = F.mean(x_imag_center**2, axis=(0,2,3)) + self.eps Vri = F.mean(x_real_center * x_imag_center, axis=(0,2,3)) + self.eps #### Do the moving average stuff~~~~~ with chainer.no_backprop_mode(): self.moving_Vrr = self.moving_Vrr*self.decay + Vrr*(1-self.decay) self.moving_Vii = self.moving_Vii*self.decay + Vii*(1-self.decay) self.moving_Vri = self.moving_Vri*self.decay + Vri*(1-self.decay) Vrr = self.moving_Vrr Vii = self.moving_Vii Vri = self.moving_Vri tau = Vrr + Vii # delta = (Vrr * Vii) - (Vri ** 2) = Determinant. Guaranteed >= 0 because SPD delta = (Vrr * Vii) - (Vri ** 2) s = F.sqrt(delta) # Determinant of square root matrix t = F.sqrt(tau + 2 * s) inverse_st = 1.0 / (s * t) Wrr = (Vii + s) * inverse_st Wii = (Vrr + s) * inverse_st Wri = -Vri * inverse_st # Wrr_b = F.broadcast_to(Wrr.reshape(1,Wrr.shape[0], 1,1), x_real_center.shape) # Wii_b = F.broadcast_to(Wii.reshape(1,Wii.shape[0], 1,1), x_real_center.shape) # Wri_b = F.broadcast_to(Wri.reshape(1,Wri.shape[0], 1,1), x_real_center.shape) Wrr_b = F.broadcast_to(F.reshape(Wrr, (1,Wrr.shape[0], 1,1)), x_real_center.shape) Wii_b = F.broadcast_to(F.reshape(Wii, (1,Wii.shape[0], 1,1)), x_real_center.shape) Wri_b = F.broadcast_to(F.reshape(Wri, (1,Wri.shape[0], 1,1)), x_real_center.shape) x_tilda_real = Wrr_b.data*x_real_center + Wri_b.data*x_imag_center x_tilda_imag = Wri_b.data*x_real_center + Wii_b.data*x_imag_center # gamma_rr = F.broadcast_to(self.gamma_rr.reshape(1,self.gamma_rr.shape[0], 1,1), x_tilda_real.shape) # gamma_ri = F.broadcast_to(self.gamma_ri.reshape(1,self.gamma_ri.shape[0], 1,1), x_tilda_real.shape) # gamma_ii = F.broadcast_to(self.gamma_ii.reshape(1,self.gamma_ii.shape[0], 1,1), x_tilda_real.shape) gamma_rr_b = F.broadcast_to(F.reshape(self.gamma_rr, (1,self.gamma_rr.shape[0], 1,1)), x_tilda_real.shape) gamma_ri_b = F.broadcast_to(F.reshape(self.gamma_ri, (1,self.gamma_ri.shape[0], 1,1)), x_tilda_real.shape) gamma_ii_b = F.broadcast_to(F.reshape(self.gamma_ii, (1,self.gamma_ii.shape[0], 1,1)), x_tilda_real.shape) x_final_real = gamma_rr_b * x_tilda_real + gamma_ri_b * x_tilda_imag x_final_imag = gamma_ri_b * x_tilda_real + gamma_ii_b * x_tilda_imag x_final_real = x_final_real + F.broadcast_to(F.reshape(self.beta[:self.beta.shape[0]/2], (1,self.beta.shape[0]/2, 1,1)), (x_final_real.shape)) x_final_imag = x_final_imag + F.broadcast_to(F.reshape(self.beta[self.beta.shape[0]/2:], (1,self.beta.shape[0]/2, 1,1)), (x_final_imag.shape)) # print self.gamma_rr.debug_print() # print x_final_real.debug_print() return F.concat([x_final_real,x_final_imag], axis=1) # keep to know what format exactly is being returned # ret = functions.batch_normalization( # x, gamma, beta, eps=self.eps, running_mean=self.avg_mean, # running_var=self.avg_var, decay=decay) # print type(ret), ret.shape else: # Use running average statistics or fine-tuned statistics. assert False mean = variable.Variable(self.avg_mean) var = variable.Variable(self.avg_var) ret = functions.fixed_batch_normalization( x, gamma, beta, mean, var, self.eps) return ret
def f(*inputs): y = functions.fixed_batch_normalization(*inputs, eps=self.eps) return y * y, # make nonlinear against beta
def f(*inputs): y = functions.fixed_batch_normalization(*inputs, eps=self.eps) return y,
def batch_normalization(self, *args): return functions.fixed_batch_normalization(*args, eps=self.eps)
def f(*inputs): return functions.fixed_batch_normalization(*inputs, eps=self.eps)
def __call__(self, x, **kwargs): """__call__(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. .. warning:: ``test`` argument is not supported anymore since v2. Instead, use ``chainer.using_config('train', False)``. See :func:`chainer.using_config`. Args: x (Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ # check argument argument.check_unexpected_kwargs( kwargs, test='test argument is not supported anymore. ' 'Use chainer.using_config') finetune, = argument.parse_kwargs(kwargs, ('finetune', False)) original_shape = x.shape batch_size = original_shape[0] # reshape input x if batchsize > 1 if batch_size > 1: reshaped_x = functions.expand_dims(x, axis=0) else: reshaped_x = x if hasattr(self, 'gamma'): gamma = self.gamma if self.norm_grad: # gamma.add_batch(batch_size) gamma.n_batch = batch_size else: with cuda.get_device_from_id(self._device_id): gamma = variable.Variable(self.xp.ones( self.avg_mean.shape, dtype=x.dtype)) if hasattr(self, 'beta'): beta = self.beta if self.norm_grad: # beta.add_batch(batch_size) beta.n_batch = batch_size else: with cuda.get_device_from_id(self._device_id): beta = variable.Variable(self.xp.zeros( self.avg_mean.shape, dtype=x.dtype)) #align shapes if x was reshaped if batch_size > 1: mean = self.xp.stack((self.avg_mean,) * batch_size) var = self.xp.stack((self.avg_var,) * batch_size) gamma = functions.stack((gamma,) * batch_size) beta = functions.stack((beta,) * batch_size) else: mean = self.xp.asarray(self.avg_mean) var = self.xp.asarray(self.avg_var) if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay func = batch_normalization.BatchNormalizationFunction( self.eps, mean, var, decay) ret = func(reshaped_x, gamma, beta) else: head_ndim = gamma.ndim + 1 axis = (0,) + tuple(range(head_ndim, reshaped_x.ndim)) mean = reshaped_x.data.mean(axis=axis) var = reshaped_x.data.var(axis=axis) ret = functions.fixed_batch_normalization( reshaped_x, gamma, beta, mean, var, self.eps) # ret is normalized input x if batch_size > 1: ret = functions.reshape(ret, original_shape) return ret
def test_valid(self): functions.fixed_batch_normalization(*self.args, eps=1e-5)
def __call__(self, x): mean = x.array.mean(axis=0) var = x.array.var(axis=0) gamma = np.ones_like(mean, dtype=x.dtype) beta = np.zeros_like(mean, dtype=x.dtype) return F.fixed_batch_normalization(x, gamma, beta, mean, var)
def __call__(self, x, gamma, beta, mean, var): return F.fixed_batch_normalization(x, gamma, beta, mean, var)
def forward(self, x, **kwargs): """forward(self, x, finetune=False) Invokes the forward propagation of BatchNormalization. In training mode, the BatchNormalization computes moving averages of mean and variance for evaluation during training, and normalizes the input using batch statistics. Args: x (~chainer.Variable): Input variable. finetune (bool): If it is in the training mode and ``finetune`` is ``True``, BatchNormalization runs in fine-tuning mode; it accumulates the input array to compute population statistics for normalization, and normalizes the input using batch statistics. """ finetune, = argument.parse_kwargs( kwargs, ('finetune', False), test='test argument is not supported anymore. ' 'Use chainer.using_config') if self.avg_mean is None: param_shape = tuple( [d for i, d in enumerate(x.shape) if i not in self.axis]) self._initialize_params(param_shape) # When using static_graph optimizations beta or gamma might not be # initialized and is not retained by the function, so the # static forward pass will get a None instead gamma = self.gamma if gamma is None: gamma, = self._get_gamma() beta = self.beta if beta is None: beta, = self._get_beta() if configuration.config.train: if finetune: self.N += 1 decay = 1. - 1. / self.N else: decay = self.decay avg_mean = self.avg_mean avg_var = self.avg_var if chainer.config.in_recomputing: # Do not update statistics when extra forward computation is # called. if finetune: self.N -= 1 # Revert the count avg_mean = None avg_var = None ret = functions.batch_normalization(x, gamma, beta, eps=self.eps, running_mean=avg_mean, running_var=avg_var, decay=decay, axis=self.axis) else: # Use running average statistics or fine-tuned statistics. mean = self.avg_mean var = self.avg_var ret = functions.fixed_batch_normalization(x, gamma, beta, mean, var, self.eps, axis=self.axis) return ret