def _tfBatchNormV2(self, x, m, v, beta, gamma, epsilon,
scale_after_normalization, shift_after_normalization):
"""New implementation."""
return nn_impl.batch_normalization(x, m, v, beta if
shift_after_normalization else None,
gamma if scale_after_normalization else
None, epsilon)
def _tfBatchNormV2(self, x, m, v, beta, gamma, epsilon,
scale_after_normalization, shift_after_normalization):
"""New implementation."""
return nn_impl.batch_normalization(x, m, v, beta if
shift_after_normalization else None,
gamma if scale_after_normalization else
None, epsilon)
def _inference_ref(self, x, scale, offset, mean, var, epsilon, data_format):
if data_format not in ['NHWC', 'NCHW']:
raise ValueError('data_format must be NCHW or NHWC, '
'got %s.' % data_format)
if data_format == 'NCHW':
x = array_ops.transpose(x, [0, 2, 3, 1])
y = nn_impl.batch_normalization(x, mean, var, offset, scale, epsilon)
if data_format == 'NCHW':
y = array_ops.transpose(y, [0, 3, 1, 2])
return y.eval()
def _training_ref(self, x, scale, offset, epsilon, data_format):
if data_format not in ['NHWC', 'NCHW']:
raise ValueError('data_format must be NCHW or NHWC, '
'got %s.' % data_format)
if data_format == 'NCHW':
x = array_ops.transpose(x, [0, 2, 3, 1])
mean, var = nn_impl.moments(x, [0, 1, 2], keep_dims=False)
y = nn_impl.batch_normalization(x, mean, var, offset, scale, epsilon)
if data_format == 'NCHW':
y = array_ops.transpose(y, [0, 3, 1, 2])
return y.eval(), mean.eval(), var.eval()
def _inference_ref(self, x, scale, offset, mean, var, epsilon,
data_format):
if data_format not in ['NHWC', 'NCHW']:
raise ValueError('data_format must be NCHW or NHWC, '
'got %s.' % data_format)
if data_format == 'NCHW':
x = array_ops.transpose(x, [0, 2, 3, 1])
y = nn_impl.batch_normalization(x, mean, var, offset, scale, epsilon)
if data_format == 'NCHW':
y = array_ops.transpose(y, [0, 3, 1, 2])
return y.eval()
def _training_ref(self, x, scale, offset, epsilon, data_format):
if data_format not in ['NHWC', 'NCHW']:
raise ValueError('data_format must be NCHW or NHWC, '
'got %s.' % data_format)
if data_format == 'NCHW':
x = array_ops.transpose(x, [0, 2, 3, 1])
mean, var = nn_impl.moments(x, [0, 1, 2], keep_dims=False)
y = nn_impl.batch_normalization(x, mean, var, offset, scale, epsilon)
if data_format == 'NCHW':
y = array_ops.transpose(y, [0, 3, 1, 2])
return y.eval(), mean.eval(), var.eval()
def batch_norm_py(tensor, mean, variance, beta, gamma, scale):
"""Python implementation of batch normalization."""
return nn_impl.batch_normalization(tensor, mean, variance, beta, gamma if
scale else None, 0.001)
def _batchnorm(self, input_x, scope, \
gamma_value, beta_value,\
moving_mean_value, moving_variance_value,\
is_training):
""" Wrapper function for batch normalization. """
with variable_scope.variable_scope(scope):
gamma_initial = init_ops.constant_initializer(
gamma_value, dtypes.float32)
gamma = gap_finetune.get_variable(\
name='gamma',
shape=gamma_value.shape,
dtype=dtypes.float32,
initializer=gamma_initial,
gap=self.gap,
gap_vars=self.gap_vars)
beta_initial = init_ops.constant_initializer(
beta_value, dtypes.float32)
beta = gap_finetune.get_variable(\
name='beta',
shape=beta_value.shape,
dtype=dtypes.float32,
initializer=beta_initial,
gap=self.gap,
gap_vars=self.gap_vars)
moving_mean_initial = init_ops.constant_initializer(\
moving_mean_value,
dtypes.float32)
moving_mean = gap_finetune.get_variable(\
name='moving_mean',
shape=moving_mean_value.shape,
dtype=dtypes.float32,
initializer=moving_mean_initial,
gap=self.gap,
gap_vars=self.gap_vars)
moving_variance_initial = init_ops.constant_initializer(\
moving_variance_value,
dtypes.float32)
moving_variance = gap_finetune.get_variable(\
name='moving_variance',
shape=moving_variance_value.shape,
dtype=dtypes.float32,
initializer=moving_variance_initial,
gap=self.gap,
gap_vars=self.gap_vars)
def mean_var_with_update():
mean, variance = nn_impl.moments(input_x, [0, 1, 2],
name='moments')
with ops.control_dependencies([\
moving_averages.assign_moving_average(\
moving_mean, mean, 0.9),
moving_averages.assign_moving_average(\
moving_variance, variance, 0.9)]):
return array_ops.identity(mean), array_ops.identity(
variance)
mean, variance = control_flow_ops.cond(is_training, \
mean_var_with_update, \
lambda: (moving_mean, moving_variance))
out = nn_impl.batch_normalization(input_x, mean, variance, beta,
gamma, 0.001)
return out
def _batch_norm(_input,
trnFlag,
eps=1e-3,
name="batch_norm",
ema_decay=0.5,
dtype=dtypes.float32):
"""
A wrapped BN operation used for 2D or 3D convolution as described in:
https://gist.github.com/tomokishii/0ce3bdac1588b5cca9fa5fbdf6e1c412
https://stackoverflow.com/questions/33949786/how-could-i-use-batch-normalization-in-tensorflow?answertab=votes#tab-top
:param _input: tensor, always convolution result before Relu
:param eps: scalar,
:param trnFlag: bool, whether training or not
:param ema_decay: scalar, moving average used of BN's beta and gamma
:param dtype: tf.dtype, data type
:return:
tensor, BN reuslt which has the same shape as _input
"""
shape = _input.get_shape().as_list()
with variable_scope.variable_scope(name) as scope:
beta = variable_scope.get_variable(
"beta", [shape[-1]],
dtype=dtype,
initializer=init_ops.constant_initializer(0., dtype=dtype),
trainable=True)
gamma = variable_scope.get_variable(
"gamma", [shape[-1]],
dtype=dtype,
initializer=init_ops.random_normal_initializer(1.,
0.01,
dtype=dtype,
seed=20170705),
trainable=True)
if shape.__len__() == 2: # fc, [batch_size, num_dim]
batch_mean, batch_var = nn_impl.moments(_input, [0],
name="moments")
elif shape.__len__(
) == 4: # conv, [batch_size, width, heigth, channel]
batch_mean, batch_var = nn_impl.moments(_input, [0, 1, 2],
name="moments")
elif shape.__len__(
) == 5: # conv, [batch_size, depth, width, heigth, channel]
batch_mean, batch_var = nn_impl.moments(_input, [0, 1, 2, 3],
name="moments")
else:
raise 'wrong _input shape, it must have dim of 2 or 4 or 5'
ema = training.ExponentialMovingAverage(decay=ema_decay)
def mean_var_with_update():
ema_apply_op = ema.apply([batch_mean, batch_var])
with ops.control_dependencies([ema_apply_op]):
return array_ops.identity(batch_mean), array_ops.identity(
batch_var)
mean, var = control_flow_ops.cond(
trnFlag, mean_var_with_update, lambda:
(ema.average(batch_mean), ema.average(batch_var)))
bn_out = nn_impl.batch_normalization(_input, mean, var, beta, gamma,
eps)
return bn_out