def __init__(
self,
layer=None,
n_units=100,
act=tf.identity,
binarized_weight=tf.identity,
W_init=tf.truncated_normal_initializer(stddev=0.1),
b_init=tf.constant_initializer(value=0.0),
W_init_args={},
b_init_args={},
name='b_dense_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
if self.inputs.get_shape().ndims != 2:
raise Exception(
"The input dimension must be rank 2, please reshape or flatten it"
)
n_in = int(self.inputs.get_shape()[-1])
self.n_units = n_units
logging.info("Binarized_DenseLayer %s: %d %s" %
(self.name, self.n_units, act.__name__))
with tf.variable_scope(name) as vs:
W = tf.get_variable(name='W',
shape=(n_in, n_units),
initializer=W_init,
**W_init_args)
bin_w = binarized_weight(W)
if b_init is not None:
try:
b = tf.get_variable(name='b',
shape=(n_units),
initializer=b_init,
**b_init_args)
except: # If initializer is a constant, do not specify shape.
b = tf.get_variable(name='b',
initializer=b_init,
**b_init_args)
self.outputs = act(tf.matmul(self.inputs, bin_w) + b)
else:
self.outputs = act(tf.matmul(self.inputs, bin_w))
# Hint : list(), dict() is pass by value (shallow), without them, it is
# pass by reference.
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
if b_init is not None:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])
def __init__(
self,
layer=None,
shape=(5, 1, 5),
stride=1,
act=tf.identity,
dilation_rate=1,
padding='SAME',
binarize_weight=tf.identity,
W_init=tf.truncated_normal_initializer(stddev=0.02),
b_init=None,
W_init_args={},
b_init_args={},
use_cudnn_on_gpu=True,
data_format='NWC',
name='cnn_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
logging.info(
"BinarizedConvolution1D %s: shape:%s strides:%s pad:%s activation:%s"
% (self.name, str(shape), str(stride), padding, act.__name__))
with tf.variable_scope(name) as vs:
W = tf.get_variable(name='W_conv1d',
shape=shape,
initializer=W_init,
**W_init_args)
bin_w = binarize_weight(W)
self.outputs = tf.nn.convolution(self.inputs,
bin_w,
strides=(stride, ),
padding=padding,
dilation_rate=(dilation_rate, ),
data_format=data_format)
if b_init:
b = tf.get_variable(name='b_conv1d',
shape=(shape[-1]),
initializer=b_init,
**b_init_args)
self.outputs = self.outputs + b
self.outputs = act(self.outputs)
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
if b_init:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])
def binary_outputs(net, name=None):
scope_name = tf.get_variable_scope().name
if scope_name:
name = scope_name + '/' + name
shape = net.outputs.get_shape()
logging.info("BinaryOutputs {:}: shape:{:}".format(name, shape))
outputs = binary_value(net.outputs)
net_new = copy.copy(net)
net_new.outputs = outputs
net_new.all_layers.extend([outputs])
return net_new
def global_max_net(net, name=None):
scope_name = tf.get_variable_scope().name
if scope_name:
name = scope_name + '/' + name
shape = net.outputs.get_shape()
logging.info("GlobalMaxPool {:}: shape:{:}".format(name, shape))
outputs = tf.reduce_max(net.outputs, 1)
net_new = copy.copy(net)
net_new.outputs = outputs
net_new.all_layers.extend([outputs])
return net_new
def __init__(
self,
prev_layer,
init_scale=0.05,
name='scale',
):
Layer.__init__(self, prev_layer=prev_layer, name=name)
self.inputs = prev_layer.outputs
logging.info("ScaleLayer %s: init_scale: %f" %
(self.name, init_scale))
with tf.variable_scope(name):
# scale = tf.get_variable(name='scale_factor', init, trainable=True, )
scale = tf.get_variable(
"scale",
shape=[1],
initializer=tf.constant_initializer(value=init_scale),
trainable=False)
self.outputs = self.inputs * scale
self.all_layers.append(self.outputs)
self.all_params.append(scale)
def __init__(
self,
prev_layer,
decay=0.9,
epsilon=0.00001,
act=tf.identity,
is_train=False,
beta_init=tf.zeros_initializer,
gamma_init=tf.random_normal_initializer(mean=1.0, stddev=0.002),
name='batchnorm_layer',
):
Layer.__init__(self, prev_layer=prev_layer, name=name)
self.inputs = prev_layer.outputs
logging.info(
"BatchNormLayer %s: decay:%f epsilon:%f act:%s is_train:%s" %
(self.name, decay, epsilon, act.__name__, is_train))
x_shape = self.inputs.get_shape()
params_shape = x_shape[-1:]
from tensorflow.python.training import moving_averages
with tf.variable_scope(name):
axis = list(range(len(x_shape) - 1))
# 1. beta, gamma
variables = []
if beta_init:
if tf.__version__ > '0.12.1' and beta_init == tf.zeros_initializer:
beta_init = beta_init()
beta = tf.get_variable('beta',
shape=params_shape,
initializer=beta_init,
dtype=LayersConfig.tf_dtype,
trainable=is_train)
variables.append(beta)
else:
beta = None
if gamma_init:
gamma = tf.get_variable(
'gamma',
shape=params_shape,
initializer=gamma_init,
dtype=LayersConfig.tf_dtype,
trainable=is_train,
)
variables.append(gamma)
else:
gamma = None
# 2.
if tf.__version__ > '0.12.1':
moving_mean_init = tf.zeros_initializer()
else:
moving_mean_init = tf.zeros_initializer
moving_mean = tf.get_variable('moving_mean',
params_shape,
initializer=moving_mean_init,
dtype=LayersConfig.tf_dtype,
trainable=False)
moving_variance = tf.get_variable(
'moving_variance',
params_shape,
initializer=tf.constant_initializer(1.),
dtype=LayersConfig.tf_dtype,
trainable=False,
)
# 3.
# These ops will only be preformed when training.
mean, variance = tf.nn.moments(self.inputs, axis)
self.mean = mean
self.variance = variance
try: # TF12
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay,
zero_debias=False) # if zero_debias=True, has bias
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay,
zero_debias=False) # if zero_debias=True, has bias
# logging.info("TF12 moving")
except Exception: # TF11
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay)
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay)
# logging.info("TF11 moving")
def mean_var_with_update():
with tf.control_dependencies(
[update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
if is_train:
mean, var = mean_var_with_update()
self.outputs = act(
tf.nn.batch_normalization(self.inputs, mean, var, beta,
gamma, epsilon))
else:
self.outputs = act(
tf.nn.batch_normalization(self.inputs, moving_mean,
moving_variance, beta, gamma,
epsilon))
variables.extend([moving_mean, moving_variance])
self.all_layers.append(self.outputs)
self.all_params.extend(variables)