def _batch_norm(self, x, name):
""" Batch normalization """
with tf.variable_scope(name):
params_shape = [x.get_shape()[-1]]
beta = tf.get_variable(
'beta', params_shape, tf.float64,
norm_init(0.0, stddev=0.1, dtype=tf.float64))
print(self.sess.run(tf.shape(beta)))
gamma = tf.get_variable('gamma', params_shape, tf.float64,
unif_init(0.1, 0.5, dtype=tf.float64))
mv_mean = tf.get_variable('moving_mean',
params_shape,
tf.float64,
const_init(0.0, tf.float64),
trainable=False)
mv_var = tf.get_variable('moving_variance',
params_shape,
tf.float64,
const_init(1.0, tf.float64),
trainable=False)
# These ops will only be preformed when training
mean, variance = tf.nn.moments(x, [0], name='moments')
self._extra_train_ops.append(
assign_moving_average(mv_mean, mean, 0.99))
self._extra_train_ops.append(
assign_moving_average(mv_var, variance, 0.99))
mean, variance = control_flow_ops.cond(self.is_training, lambda:
(mean, variance), lambda:
(mv_mean, mv_var))
y = tf.nn.batch_normalization(x, mean, variance, beta, gamma, 1e-6)
y.set_shape(x.get_shape())
return y
def _batch_norm(self, x, name):
"""Batch Normalization"""
with tf.variable_scope(name):
params_shape = [x.get_shape()[-1]]
beta = tf.get_variable(
'beta', params_shape, tf.float64,
norm_init(0.0, stddev=0.1, dtype=tf.float64))
gamma = tf.get_variable('gamma', params_shape, tf.float64,
unif_init(0.1, 0.5, dtype=tf.float64))
mv_mean = tf.get_variable('moving_mean',
params_shape,
tf.float64,
const_init(0.0, tf.float64),
trainable=False)
mv_var = tf.get_variable('moving_variance',
params_shape,
tf.float64,
const_init(1.0, tf.float64),
trainable=False)
# Training Ops
mean, variance = tf.nn.moments(x, [0], name='moments')
hoge = assign_moving_average(mv_mean, mean, 0.99)
piyo = assign_moving_average(mv_var, variance, 0.99)
self._extra_train_ops.extend([hoge, piyo])
mean, variance = control_flow_ops.cond(self.is_training, lambda:
(mean, variance), lambda:
(mv_mean, mv_var))
y = tf.nn.batch_normalization(x, mean, variance, beta, gamma, 1e-6)
y.set_shape(x.get_shape())
return y
def _batch_norm(self, x, name):
with tf.variable_scope(name):
params_shape = [x.get_shape()[-1]]
beta = tf.get_variable(
'beta', params_shape, tf.float64,
norm_init(0.0, stddev=0.1, dtype=tf.float64))
gamma = tf.get_variable('gamma', params_shape, tf.float64,
unif_init(.1, .5, dtype=tf.float64))
mv_mean = tf.get_variable('moving_mean',
params_shape,
tf.float64,
const_init(0, tf.float64),
trainable=False)
mv_var = tf.get_variable('moving_variance',
params_shape,
tf.float64,
const_init(1, tf.float64),
trainable=False)
mean, variance = tf.nn.moments(x, [0])
self._extra_train_ops.append(
assign_moving_average(mv_mean, mean, .99))
self._extra_train_ops.append(
assign_moving_average(mv_var, variance, .99))
mean, variance = control_flow_ops.cond(self.is_training, lambda :(mean, variance),\
lambda : (mv_mean,mv_var))
y = tf.nn.batch_normalization(x, mean, variance, beta, gamma, 1e-6)
y.set_shape(x.get_shape())
return y
def _one_layer(self, input_, out_sz,
activation_fn=tf.nn.sigmoid,
std=1.0, name='linear'):
with tf.variable_scope(name,reuse=tf.AUTO_REUSE):
shape = input_.get_shape().as_list()
w = tf.get_variable('Weights',
[shape[1], out_sz], tf.float64,
norm_init(stddev=std/np.sqrt(shape[1]+out_sz)))
b = tf.get_variable('Bias',
[1,out_sz], tf.float64,
norm_init(stddev=std/np.sqrt(1+out_sz)))
affine = tf.matmul(input_, w)+b
if activation_fn is not None:
return activation_fn(affine)
else:
return affine
def _one_layer(self,
input_,
in_sz,
out_sz,
activation_fn=tf.nn.relu,
std=5.0,
name='linear'):
with tf.variable_scope(name):
w = tf.get_variable(
'Weight', [in_sz, out_sz], tf.float64,
norm_init(stddev=std / np.sqrt(in_sz + out_sz)))
bias = tf.get_variable('Bias', [1, out_sz], tf.float64,
norm_init(stddev=std / np.sqrt(1 + out_sz)))
hidden = tf.matmul(input_, w) + tf.matmul(self.allones_m, bias)
if activation_fn is not None:
return activation_fn(hidden)
else:
return hidden
def _batch_norm(self, x, name):
"""Batch normalization"""
with tf.variable_scope(name):
params_shape = [x.get_shape()[-1]]
beta = tf.get_variable(name='beta',
shape=params_shape,
dtype=tf.float64,
initializer=norm_init(mean=0.0,
stddev=0.1,
dtype=tf.float64),
trainable=True)
gamma = tf.get_variable(name='gamma',
shape=params_shape,
dtype=tf.float64,
initializer=unif_init(minval=0.1,
maxval=0.5,
dtype=tf.float64),
trainable=True)
mv_mean = tf.get_variable(name='moving_mean',
shape=params_shape,
dtype=tf.float64,
initializer=const_init(value=0.0,
dtype=tf.float64),
trainable=False)
mv_var = tf.get_variable(name='moving_variance',
shape=params_shape,
dtype=tf.float64,
initializer=const_init(value=1.0,
dtype=tf.float64),
trainable=False)
# These ops will only be performed when training:
mean, variance = tf.nn.moments(x=x, axes=[0], name='moments')
self._extra_train_ops.append(
assign_moving_average(variable=mv_mean, value=mean,
decay=0.99))
self._extra_train_ops.append(
assign_moving_average(variable=mv_var,
value=variance,
decay=0.99))
mean, variance = control_flow_ops.cond(pred=self.is_training,
true_fn=lambda:
(mean, variance),
false_fn=lambda:
(mv_mean, mv_var))
y = tf.nn.batch_normalization(x=x,
mean=mean,
variance=variance,
offset=beta,
scale=gamma,
variance_epsilon=1e-6)
y.set_shape(x.get_shape())
return y
def _one_layer(self, input_, in_sz, out_sz, activation_fn=tf.nn.relu, std=5.0, name='linear'):
with tf.variable_scope(name):
w = tf.get_variable('Weight', [in_sz, out_sz], tf.float64,
norm_init(stddev=std / np.sqrt(in_sz + out_sz)))
hidden = tf.matmul(input_, w)
hidden_bn = self._batch_norm(hidden, out_sz, name='normal')
if activation_fn is not None:
return activation_fn(hidden_bn)
else:
return hidden_bn
def _one_layer( self , input_, out_sz, activation_fn = 'ReLU', std =5.0 , name = 'linear' ): #One layer of neural network
with tf.variable_scope( name ):
shape = input_.get_shape().as_list()
w = tf.get_variable( 'Matrix', [shape[1], out_sz], tf.float64,
norm_init(stddev= std / np.sqrt(shape[1]+out_sz)) )
hidden = tf.matmul( input_, w )
hidden_bn = self._batch_norm( hidden, name = 'normal' )
if activation_fn == 'relu': #Use leaky ReLU
return tf.nn.relu(hidden_bn) + 0.2 * tf.nn.relu(-hidden_bn)
else:
return tf.nn.relu(hidden_bn) + 0.05 * tf.nn.relu(-hidden_bn)
def _one_layer(self, input_, out_sz, activation_fn=tf.nn.relu, std=5.0, name='linear'):
with tf.variable_scope(name):
shape = input_.get_shape().as_list()
w = tf.get_variable('Matrix', [shape[1], out_sz],
tf.float64, norm_init(stddev= std/np.sqrt(shape[1]+out_sz)))
hidden = tf.matmul(input_, w)
hidden_bn = self._batch_norm(hidden, name='normal')
if activation_fn != None:
return activation_fn(hidden_bn)
else:
return hidden_bn