def _batch_norm(self, x, name):
"""Batch Normalization"""
with tf.variable_scope(name):
params_shape = [x.get_shape()[-1]]
beta = tf.get_variable(
'beta', params_shape, tf.float64,
norm_init(0.0, stddev=0.1, dtype=tf.float64))
gamma = tf.get_variable('gamma', params_shape, tf.float64,
unif_init(0.1, 0.5, dtype=tf.float64))
mv_mean = tf.get_variable('moving_mean',
params_shape,
tf.float64,
const_init(0.0, tf.float64),
trainable=False)
mv_var = tf.get_variable('moving_variance',
params_shape,
tf.float64,
const_init(1.0, tf.float64),
trainable=False)
# Training Ops
mean, variance = tf.nn.moments(x, [0], name='moments')
hoge = assign_moving_average(mv_mean, mean, 0.99)
piyo = assign_moving_average(mv_var, variance, 0.99)
self._extra_train_ops.extend([hoge, piyo])
mean, variance = control_flow_ops.cond(self.is_training, lambda:
(mean, variance), lambda:
(mv_mean, mv_var))
y = tf.nn.batch_normalization(x, mean, variance, beta, gamma, 1e-6)
y.set_shape(x.get_shape())
return y
def _batch_norm(self, x, name):
with tf.variable_scope(name):
params_shape = [x.get_shape()[-1]]
beta = tf.get_variable(
'beta', params_shape, tf.float64,
norm_init(0.0, stddev=0.1, dtype=tf.float64))
gamma = tf.get_variable('gamma', params_shape, tf.float64,
unif_init(.1, .5, dtype=tf.float64))
mv_mean = tf.get_variable('moving_mean',
params_shape,
tf.float64,
const_init(0, tf.float64),
trainable=False)
mv_var = tf.get_variable('moving_variance',
params_shape,
tf.float64,
const_init(1, tf.float64),
trainable=False)
mean, variance = tf.nn.moments(x, [0])
self._extra_train_ops.append(
assign_moving_average(mv_mean, mean, .99))
self._extra_train_ops.append(
assign_moving_average(mv_var, variance, .99))
mean, variance = control_flow_ops.cond(self.is_training, lambda :(mean, variance),\
lambda : (mv_mean,mv_var))
y = tf.nn.batch_normalization(x, mean, variance, beta, gamma, 1e-6)
y.set_shape(x.get_shape())
return y
def _batch_norm(self, x, name):
""" Batch normalization """
with tf.variable_scope(name):
params_shape = [x.get_shape()[-1]]
beta = tf.get_variable(
'beta', params_shape, tf.float64,
norm_init(0.0, stddev=0.1, dtype=tf.float64))
print(self.sess.run(tf.shape(beta)))
gamma = tf.get_variable('gamma', params_shape, tf.float64,
unif_init(0.1, 0.5, dtype=tf.float64))
mv_mean = tf.get_variable('moving_mean',
params_shape,
tf.float64,
const_init(0.0, tf.float64),
trainable=False)
mv_var = tf.get_variable('moving_variance',
params_shape,
tf.float64,
const_init(1.0, tf.float64),
trainable=False)
# These ops will only be preformed when training
mean, variance = tf.nn.moments(x, [0], name='moments')
self._extra_train_ops.append(
assign_moving_average(mv_mean, mean, 0.99))
self._extra_train_ops.append(
assign_moving_average(mv_var, variance, 0.99))
mean, variance = control_flow_ops.cond(self.is_training, lambda:
(mean, variance), lambda:
(mv_mean, mv_var))
y = tf.nn.batch_normalization(x, mean, variance, beta, gamma, 1e-6)
y.set_shape(x.get_shape())
return y
def _batch_norm(self, x, name):
"""Batch normalization"""
with tf.variable_scope(name):
params_shape = [x.get_shape()[-1]]
beta = tf.get_variable(name='beta',
shape=params_shape,
dtype=tf.float64,
initializer=norm_init(mean=0.0,
stddev=0.1,
dtype=tf.float64),
trainable=True)
gamma = tf.get_variable(name='gamma',
shape=params_shape,
dtype=tf.float64,
initializer=unif_init(minval=0.1,
maxval=0.5,
dtype=tf.float64),
trainable=True)
mv_mean = tf.get_variable(name='moving_mean',
shape=params_shape,
dtype=tf.float64,
initializer=const_init(value=0.0,
dtype=tf.float64),
trainable=False)
mv_var = tf.get_variable(name='moving_variance',
shape=params_shape,
dtype=tf.float64,
initializer=const_init(value=1.0,
dtype=tf.float64),
trainable=False)
# These ops will only be performed when training:
mean, variance = tf.nn.moments(x=x, axes=[0], name='moments')
self._extra_train_ops.append(
assign_moving_average(variable=mv_mean, value=mean,
decay=0.99))
self._extra_train_ops.append(
assign_moving_average(variable=mv_var,
value=variance,
decay=0.99))
mean, variance = control_flow_ops.cond(pred=self.is_training,
true_fn=lambda:
(mean, variance),
false_fn=lambda:
(mv_mean, mv_var))
y = tf.nn.batch_normalization(x=x,
mean=mean,
variance=variance,
offset=beta,
scale=gamma,
variance_epsilon=1e-6)
y.set_shape(x.get_shape())
return y