def __init__(
self,
layer = None,
size = [],
is_scale = True,
method = 0,
align_corners = False,
name ='upsample3d_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
if len(self.inputs.get_shape()) == 5:
if is_scale:
size_d = size[0] * int(self.inputs.get_shape()[1])
size_h = size[1] * int(self.inputs.get_shape()[2])
size_w = size[2] * int(self.inputs.get_shape()[3])
size = [int(size_h), int(size_h), int(size_w)]
else:
raise Exception("Donot support shape %s" % self.inputs.get_shape())
print(" UpSampling3dLayer %s: is_scale:%s size:%s method:%d align_corners:%s" %
(name, is_scale, size, method, align_corners))
with tf.variable_scope(name) as vs:
self.outputs = tf.image.resize_images(self.inputs, size=size, method=method, align_corners=align_corners)
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend( [self.outputs] )
def __init__(
self,
layer = [],
combine_fn = tf.minimum,
name ='elementwise_layer',
act = None,
):
Layer.__init__(self, name=name)
if act:
print(" [TL] ElementwiseLayer %s: size:%s fn:%s, act:%s" % (
self.name, layer[0].outputs.get_shape(), combine_fn.__name__, act.__name__))
else:
print(" [TL] ElementwiseLayer %s: size:%s fn:%s" % (
self.name, layer[0].outputs.get_shape(), combine_fn.__name__))
self.outputs = layer[0].outputs
# print(self.outputs._shape, type(self.outputs._shape))
for l in layer[1:]:
# assert str(self.outputs.get_shape()) == str(l.outputs.get_shape()), "Hint: the input shapes should be the same. %s != %s" % (self.outputs.get_shape() , str(l.outputs.get_shape()))
self.outputs = combine_fn(self.outputs, l.outputs, name=name)
if act:
self.outputs = act(self.outputs)
self.all_layers = list(layer[0].all_layers)
self.all_params = list(layer[0].all_params)
self.all_drop = dict(layer[0].all_drop)
for i in range(1, len(layer)):
self.all_layers.extend(list(layer[i].all_layers))
self.all_params.extend(list(layer[i].all_params))
self.all_drop.update(dict(layer[i].all_drop))
self.all_layers = list_remove_repeat(self.all_layers)
self.all_params = list_remove_repeat(self.all_params)
def __init__(
self,
inputs=None,
vocabulary_size=80000,
embedding_size=200,
E_init=tf.random_uniform_initializer(-0.1, 0.1),
E_init_args={},
name='embedding_layer',
):
Layer.__init__(self, name=name)
self.inputs = inputs
print(" [TL] EmbeddingInputlayer %s: (%d, %d)" %
(self.name, vocabulary_size, embedding_size))
with tf.variable_scope(name, reuse=False) as vs:
tl.layers.set_name_reuse(False)
embeddings = tf.get_variable(name='embeddings',
shape=(vocabulary_size,
embedding_size),
initializer=E_init,
**E_init_args)
embed = tf.nn.embedding_lookup(embeddings, self.inputs)
self.outputs = embed
self.all_layers = [self.outputs]
self.all_params = [embeddings]
self.all_drop = {}
def __init__(
self,
layer = [],
combine_fn = tf.minimum,
name ='elementwise_layer',
act = None,
):
Layer.__init__(self, name=name)
'''
if act:
#print(" [TL] ElementwiseLayer %s: size:%s fn:%s, act:%s" % (
#self.name, layer[0].outputs.get_shape(), combine_fn.__name__, act.__name__))
else:
#print(" [TL] ElementwiseLayer %s: size:%s fn:%s" % (
#self.name, layer[0].outputs.get_shape(), combine_fn.__name__))
'''
self.outputs = layer[0].outputs
# #print(self.outputs._shape, type(self.outputs._shape))
for l in layer[1:]:
# assert str(self.outputs.get_shape()) == str(l.outputs.get_shape()), "Hint: the input shapes should be the same. %s != %s" % (self.outputs.get_shape() , str(l.outputs.get_shape()))
self.outputs = combine_fn(self.outputs, l.outputs, name=name)
if act:
self.outputs = act(self.outputs)
self.all_layers = list(layer[0].all_layers)
self.all_params = list(layer[0].all_params)
self.all_drop = dict(layer[0].all_drop)
for i in range(1, len(layer)):
self.all_layers.extend(list(layer[i].all_layers))
self.all_params.extend(list(layer[i].all_params))
self.all_drop.update(dict(layer[i].all_drop))
self.all_layers = list_remove_repeat(self.all_layers)
self.all_params = list_remove_repeat(self.all_params)
def __init__(
self,
layer=None,
act=tf.identity,
epsilon=1e-5,
scale_init=tf.constant_initializer(1.0),
offset_init=tf.constant_initializer(0.0),
G=32,
name='group_norm',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
print(" [TL] GroupNormLayer %s: epsilon:%f act:%s" % (self.name, epsilon, act.__name__))
inputs_shape = get_shape(layer.outputs)
G = tf.minimum(G, inputs_shape[-1])
# [N, H, W, C] to [N, C, H, W]
temp_input = tf.transpose(self.inputs, [0, 3, 1, 2])
temp_input = tf.reshape(temp_input, [inputs_shape[0], G, inputs_shape[-1]//G, inputs_shape[1], inputs_shape[2]],
name='group_reshape1')
with tf.variable_scope(name) as vs:
mean, var = tf.nn.moments(temp_input, [2, 3, 4], keep_dims=True)
scale = tf.get_variable('scale', shape=[1, inputs_shape[-1], 1, 1], initializer=scale_init, dtype=D_TYPE)
offset = tf.get_variable('offset', shape=[1, inputs_shape[-1], 1, 1], initializer=offset_init, dtype=D_TYPE)
temp_input = (temp_input - mean) / tf.sqrt(var + epsilon)
temp_input = tf.reshape(temp_input, shape=[inputs_shape[0], inputs_shape[-1], inputs_shape[1], inputs_shape[2]],
name='group_reshape2')
self.outputs = scale * temp_input + offset
self.outputs = tf.transpose(self.outputs, [0, 2, 3, 1])
self.outputs = act(self.outputs)
variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name)
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
self.all_params.extend(variables)
def __init__(
self,
layer=None,
channel_shared=False,
a_init=tf.constant_initializer(value=0.0),
a_init_args={},
# restore = True,
name="prelu_layer"):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
print(" [TL] PReluLayer %s: channel_shared:%s" % (self.name, channel_shared))
if channel_shared:
w_shape = (1, )
else:
w_shape = int(self.inputs.get_shape()[-1])
# with tf.name_scope(name) as scope:
with tf.variable_scope(name) as vs:
with tf.device('/cpu:0'):
alphas = tf.get_variable(name='alphas', shape=w_shape, initializer=a_init, dtype=D_TYPE, **a_init_args)
try: ## TF 1.0
self.outputs = tf.nn.relu(self.inputs) + tf.multiply(alphas, (self.inputs - tf.abs(self.inputs))) * 0.5
except: ## TF 0.12
self.outputs = tf.nn.relu(self.inputs) + tf.mul(alphas, (self.inputs - tf.abs(self.inputs))) * 0.5
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
self.all_params.extend([alphas])
def __init__(
self,
layer=None,
n_units=100,
act=tf.identity,
W_init=tf.truncated_normal_initializer(stddev=0.1),
b_init=tf.constant_initializer(value=0.0),
W_init_args={},
b_init_args={},
name='dense_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
if self.inputs.get_shape().ndims != 2:
raise Exception(
"The input dimension must be rank 2, please reshape or flatten it"
)
n_in = int(self.inputs.get_shape()[-1])
self.n_units = n_units
print(" [TL] DenseLayer %s: %d %s" %
(self.name, self.n_units, act.__name__))
with tf.variable_scope(name) as vs:
with tf.device('/cpu:0'): # MGPU 增加 '/cpu:0'，(以及dtype=D_TYPE)
W = tf.get_variable(name='W',
shape=(n_in, n_units),
initializer=W_init,
dtype=D_TYPE,
**W_init_args)
if b_init is not None:
try:
with tf.device(
'/cpu:0'): # MGPU 增加 '/cpu:0'，(以及dtype=D_TYPE)
b = tf.get_variable(name='b',
shape=(n_units),
initializer=b_init,
dtype=D_TYPE,
**b_init_args)
except: # If initializer is a constant, do not specify shape.
with tf.device(
'/cpu:0'): # MGPU 增加 '/cpu:0'，(以及dtype=D_TYPE)
b = tf.get_variable(name='b',
initializer=b_init,
dtype=D_TYPE,
**b_init_args)
self.outputs = act(tf.matmul(self.inputs, W) + b)
else:
self.outputs = act(tf.matmul(self.inputs, W))
# Hint : list(), dict() is pass by value (shallow), without them, it is
# pass by reference.
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
if b_init is not None:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])
def __init__(
self,
layer=None,
act=tf.identity,
shape=[5, 5, 1, 100],
strides=[1, 1, 1, 1],
padding='SAME',
W_init=tf.truncated_normal_initializer(stddev=0.02),
b_init=tf.constant_initializer(value=0.0),
W_init_args={},
b_init_args={},
use_cudnn_on_gpu=None,
data_format=None,
name='cnn_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
print(" [TL] Conv2dLayer %s: shape:%s strides:%s pad:%s act:%s" %
(self.name, str(shape), str(strides), padding, act.__name__))
with tf.variable_scope(name) as vs:
with tf.device('/cpu:0'):
W = tf.get_variable(name='W_conv2d',
shape=shape,
initializer=W_init,
dtype=D_TYPE,
**W_init_args)
if b_init:
with tf.device('/cpu:0'):
b = tf.get_variable(name='b_conv2d',
shape=(shape[-1]),
initializer=b_init,
dtype=D_TYPE,
**b_init_args)
self.outputs = act(
tf.nn.conv2d(self.inputs,
W,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format) + b)
else:
self.outputs = act(
tf.nn.conv2d(self.inputs,
W,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format))
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
if b_init:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])
def __init__(
self,
layer=None,
shape=[5, 5, 1, 100],
strides=[1, 1, 1, 1],
padding='SAME',
act=tf.identity,
binarize_weight=tf.identity,
W_init=tf.truncated_normal_initializer(stddev=0.02),
b_init=tf.zeros_initializer,
W_init_args={},
b_init_args={},
use_cudnn_on_gpu=True,
data_format=None,
name='cnn_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
print(
" [TL] Binarized_Convolution %s: shape:%s strides:%s pad:%s activation:%s"
% (self.name, str(shape), str(strides), padding, act.__name__))
with tf.variable_scope(name) as vs:
W = tf.get_variable(name='W_conv2d',
shape=shape,
initializer=W_init,
**W_init_args)
# print(tf.shape(W))
bin_w = binarize_weight(W)
if b_init:
b = tf.get_variable(name='b_conv2d',
shape=(shape[-1]),
initializer=b_init,
**b_init_args)
self.outputs = act(
tf.nn.conv2d(self.inputs,
bin_w,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format) + b)
else:
self.outputs = act(
tf.nn.conv2d(self.inputs,
bin_w,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format))
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
if b_init:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])
def __init__(self, layer=None, act=tf.identity, name='b_out'):
Layer.__init__(self, name=name)
self.outputs = layer.outputs
self.outputs = act(self.outputs)
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
def __init__(self, layer=None, name='relu'):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
with tf.variable_scope(name):
self.outputs = tf.nn.relu(self.inputs)
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend( [self.outputs] )
def __init__(self, layer=None, name='padding', desired_channels=0):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
self.desired_channels = desired_channels
with tf.variable_scope(name):
self.outputs = self.pad_depth(self.inputs, self.desired_channels)
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
def __init__(
self,
layer=None,
shape=(5, 1, 5),
stride=1,
act=tf.identity,
dilation_rate=1,
padding='SAME',
binarize_weight=tf.identity,
W_init=tf.truncated_normal_initializer(stddev=0.02),
b_init=None,
W_init_args={},
b_init_args={},
use_cudnn_on_gpu=True,
data_format='NWC',
name='cnn_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
logging.info(
"BinarizedConvolution1D %s: shape:%s strides:%s pad:%s activation:%s"
% (self.name, str(shape), str(stride), padding, act.__name__))
with tf.variable_scope(name) as vs:
W = tf.get_variable(name='W_conv1d',
shape=shape,
initializer=W_init,
**W_init_args)
bin_w = binarize_weight(W)
self.outputs = tf.nn.convolution(self.inputs,
bin_w,
strides=(stride, ),
padding=padding,
dilation_rate=(dilation_rate, ),
data_format=data_format)
if b_init:
b = tf.get_variable(name='b_conv1d',
shape=(shape[-1]),
initializer=b_init,
**b_init_args)
self.outputs = self.outputs + b
self.outputs = act(self.outputs)
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
if b_init:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])
def __init__(
self,
layer=None,
act=tf.identity,
epsilon=1e-5,
scale_init=tf.constant_initializer(1.0),
offset_init=tf.constant_initializer(0.0),
G=32,
name='group_norm',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
print(" [TL] GroupNormLayer %s: epsilon:%f act:%s" %
(self.name, epsilon, act.__name__))
inputs_shape = get_shape(layer.outputs)
G = tf.minimum(G, inputs_shape[-1])
# [N, H, W, C] to [N, C, H, W]
temp_input = tf.transpose(self.inputs, [0, 3, 1, 2])
temp_input = tf.reshape(temp_input, [
inputs_shape[0], G, inputs_shape[-1] // G, inputs_shape[1],
inputs_shape[2]
],
name='group_reshape1')
with tf.variable_scope(name) as vs:
mean, var = tf.nn.moments(temp_input, [2, 3, 4], keep_dims=True)
scale = tf.get_variable('scale',
shape=[1, inputs_shape[-1], 1, 1],
initializer=scale_init,
dtype=D_TYPE)
offset = tf.get_variable('offset',
shape=[1, inputs_shape[-1], 1, 1],
initializer=offset_init,
dtype=D_TYPE)
temp_input = (temp_input - mean) / tf.sqrt(var + epsilon)
temp_input = tf.reshape(temp_input,
shape=[
inputs_shape[0], inputs_shape[-1],
inputs_shape[1], inputs_shape[2]
],
name='group_reshape2')
self.outputs = scale * temp_input + offset
self.outputs = tf.transpose(self.outputs, [0, 2, 3, 1])
self.outputs = act(self.outputs)
variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES,
scope=vs.name)
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
self.all_params.extend(variables)
def __init__(
self,
layer=None,
n_units=100,
act=tf.identity,
W_init=tf.truncated_normal_initializer(stddev=0.1),
b_init=tf.constant_initializer(value=0.0),
W_init_args={},
b_init_args={},
name='dense_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
if self.inputs.get_shape().ndims != 2:
raise Exception("The input dimension must be rank 2, please reshape or flatten it")
n_in = int(self.inputs.get_shape()[-1])
self.n_units = n_units
print(" [TL] DenseLayer %s: %d %s" % (self.name, self.n_units, act.__name__))
with tf.variable_scope(name) as vs:
with tf.device('/cpu:0'):
W = tf.get_variable(name='W', shape=(n_in, n_units), initializer=W_init, dtype=D_TYPE, **W_init_args)
if b_init is not None:
try:
with tf.device('/cpu:0'):
b = tf.get_variable(name='b', shape=(n_units), initializer=b_init, dtype=D_TYPE, **b_init_args)
except: # If initializer is a constant, do not specify shape.
with tf.device('/cpu:0'):
b = tf.get_variable(name='b', initializer=b_init, dtype=D_TYPE, **b_init_args)
self.outputs = act(tf.matmul(self.inputs, W) + b)
else:
self.outputs = act(tf.matmul(self.inputs, W))
# Hint : list(), dict() is pass by value (shallow), without them, it is
# pass by reference.
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
if b_init is not None:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])
def __init__(
self,
layer=[],
combine_fn=tf.minimum, # 默认是取对应元素最小值
name='elementwise_layer',
act=None,
):
Layer.__init__(self, name=name) # [shortcut, residual]
if act:
print(" [TL] ElementwiseLayer %s: size:%s fn:%s, act:%s" %
(self.name, layer[0].outputs.get_shape(),
combine_fn.__name__, act.__name__))
else:
print(
" [TL] ElementwiseLayer %s: size:%s fn:%s" %
(self.name, layer[0].outputs.get_shape(), combine_fn.__name__)
) # ElementwiseLayer resnet_v1_50/block1/unit_1/bottleneck_v1/combine_layer: size:(?, 56, 56, 256) fn:add
self.outputs = layer[0].outputs # shortcut (?, 56, 56, 256)
# print(self.outputs._shape, type(self.outputs._shape))
for l in layer[1:]: # residual (?, 56, 56, 256)
# assert str(self.outputs.get_shape()) == str(l.outputs.get_shape()), "Hint: the input shapes should be the same. %s != %s" % (self.outputs.get_shape() , str(l.outputs.get_shape()))
self.outputs = combine_fn(
self.outputs, l.outputs, name=name
) # shortcut (?, 56, 56, 256) + residual (?, 56, 56, 256)
if act:
self.outputs = act(self.outputs)
self.all_layers = list(layer[0].all_layers)
self.all_params = list(layer[0].all_params)
self.all_drop = dict(layer[0].all_drop)
for i in range(1, len(layer)):
self.all_layers.extend(list(layer[i].all_layers))
self.all_params.extend(list(layer[i].all_params))
self.all_drop.update(dict(layer[i].all_drop))
self.all_layers = list_remove_repeat(
self.all_layers
) # 剔除掉重复的layers，resnet_v1_50/conv1，resnet_v1_50/bn0，resnet_v1_50/prelu0
self.all_params = list_remove_repeat(self.all_params)
def __init__(
self,
layer=None,
shape=None,
name='resize_bilinear_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
if shape == None:
shape = tf.shape(self.inputs)[1:3, ]
self.outputs = tf.image.resize_bilinear(self.inputs, shape)
print(" [TL] ResizeBilinearLayer %s: %s" %
(self.name, self.outputs.get_shape()))
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers = list_remove_repeat(self.all_layers)
self.all_params = list_remove_repeat(self.all_params)
def __init__(
self,
layer=None,
channel_shared=False,
a_init=tf.constant_initializer(value=0.0),
a_init_args={},
# restore = True,
name="prelu_layer"):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
print(" [TL] PReluLayer %s: channel_shared:%s" %
(self.name, channel_shared))
if channel_shared:
w_shape = (1, )
else:
w_shape = int(self.inputs.get_shape()[-1])
# with tf.name_scope(name) as scope:
with tf.variable_scope(name) as vs:
with tf.device('/cpu:0'):
alphas = tf.get_variable(name='alphas',
shape=w_shape,
initializer=a_init,
dtype=D_TYPE,
**a_init_args)
try: ## TF 1.0
self.outputs = tf.nn.relu(self.inputs) + tf.multiply(
alphas, (self.inputs - tf.abs(self.inputs))) * 0.5
except: ## TF 0.12
self.outputs = tf.nn.relu(self.inputs) + tf.mul(
alphas, (self.inputs - tf.abs(self.inputs))) * 0.5
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
self.all_params.extend([alphas])
def __init__(
self,
layer=None,
act=tf.identity,
shape=[5, 5, 1, 100],
strides=[1, 1, 1, 1],
padding='SAME',
W_init=tf.truncated_normal_initializer(stddev=0.02),
b_init=tf.constant_initializer(value=0.0),
W_init_args={},
b_init_args={},
use_cudnn_on_gpu=None,
data_format=None,
name='cnn_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
print(" [TL] Conv2dLayer %s: shape:%s strides:%s pad:%s act:%s" % (self.name, str(shape), str(strides), padding, act.__name__))
with tf.variable_scope(name) as vs:
with tf.device('/cpu:0'):
W = tf.get_variable(name='W_conv2d', shape=shape, initializer=W_init, dtype=D_TYPE, **W_init_args)
if b_init:
with tf.device('/cpu:0'):
b = tf.get_variable(name='b_conv2d', shape=(shape[-1]), initializer=b_init, dtype=D_TYPE, **b_init_args)
self.outputs = act(
tf.nn.conv2d(self.inputs, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) + b)
else:
self.outputs = act(tf.nn.conv2d(self.inputs, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format))
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
if b_init:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])
def SubVoxelConv(net, scale=2, only_z=False, n_out_channel=None, act=tf.identity, name='subpixel_conv3d'):
"""3-D version of tl.layers.SubpixelConv2d.
Inflate pixles in [channels] into [depth height width] to interpola the 3-d image
Parameters
net : [batch depth height width channels]
"""
scope_name = tf.get_variable_scope().name
if scope_name:
name = scope_name + '/' + name
def _PS(X, r, n_out_channel, z_only=False):
if n_out_channel >= 1:
batch_size, depth, height, width, channels = X.get_shape().as_list()
batch_size = tf.shape(X)[0]
Xs = tf.split(X, r, 4)
if z_only is False:
i = 0
for subx in Xs:
Xs[i] = _interpolateXY(subx, scale, nchannels_out=int(Xs[i].get_shape()[-1]) / (scale**2))
i += 1
Xr = tf.concat(Xs, 2) # concat at height dimension
if z_only is False:
X = tf.reshape(Xr, (batch_size, depth*scale, height*scale, width*scale, n_out_channel))
else:
X = tf.reshape(Xr, (batch_size, depth*scale, height, width, n_out_channel))
return X
inputs = net.outputs
down_factor = scale ** 3
if only_z:
down_factor = scale
if n_out_channel is None:
assert int(inputs.get_shape()[-1]) / down_factor % 1 == 0, "nchannels_in == ratio^3 * nchannels_out"
n_out_channel = int(int(inputs.get_shape()[-1]) / down_factor)
print(" SubvoxelConv %s : scale: %d n_out_channel: %s act: %s " % (name, scale, n_out_channel, act.__name__))
net_new = Layer(inputs, name=name)
with tf.variable_scope(name) as vs:
net_new.outputs = act(_PS(inputs, r=scale, n_out_channel=n_out_channel, z_only=only_z))
net_new.all_layers = list(net.all_layers)
net_new.all_params = list(net.all_params)
net_new.all_drop = dict(net.all_drop)
net_new.all_layers.extend( [net_new.outputs] )
return net_new
def __init__(
self,
inputs=None,
vocabulary_size=80000,
embedding_size=200,
num_shards=None,
E_init=tf.random_uniform_initializer(-0.1, 0.1),
E_init_args={},
name='embedding_layer',
):
Layer.__init__(self, name=name)
self.inputs = inputs
print(
" tensorlayer:Instantiate EmbeddingInputlayer %s: (%d, %d)" % (self.name, vocabulary_size, embedding_size))
with tf.variable_scope(name) as vs:
embeddings = sharded_variable("emb", [vocabulary_size, embedding_size], num_shards)
embed = tf.nn.embedding_lookup(embeddings, self.inputs)
self.outputs = embed
self.all_layers = [self.outputs]
self.all_params = [embeddings]
self.all_drop = {}
def __init__(
self,
layer=None,
decay=0.9,
epsilon=2e-5,
act=tf.identity,
is_train=False,
fix_gamma=True,
beta_init=tf.zeros_initializer,
gamma_init=tf.random_normal_initializer(
mean=1.0, stddev=0.002), # tf.ones_initializer,
# dtype = tf.float32,
trainable=None,
name='batchnorm_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
print(
" [TL] BatchNormLayer %s: decay:%f epsilon:%f act:%s is_train:%s"
% (self.name, decay, epsilon, act.__name__, is_train))
x_shape = self.inputs.get_shape()
params_shape = x_shape[-1:]
from tensorflow.python.training import moving_averages
from tensorflow.python.ops import control_flow_ops
with tf.variable_scope(name) as vs:
axis = list(range(len(x_shape) - 1))
## 1. beta, gamma
if tf.__version__ > '0.12.1' and beta_init == tf.zeros_initializer:
beta_init = beta_init()
beta = tf.get_variable('beta',
shape=params_shape,
initializer=beta_init,
dtype=tf.float32,
trainable=is_train) #, restore=restore)
gamma = tf.get_variable(
'gamma',
shape=params_shape,
initializer=gamma_init,
dtype=tf.float32,
trainable=fix_gamma,
) #restore=restore)
## 2.
if tf.__version__ > '0.12.1':
moving_mean_init = tf.zeros_initializer()
else:
moving_mean_init = tf.zeros_initializer
moving_mean = tf.get_variable(
'moving_mean',
params_shape,
initializer=moving_mean_init,
dtype=tf.float32,
trainable=False) # restore=restore)
moving_variance = tf.get_variable(
'moving_variance',
params_shape,
initializer=tf.constant_initializer(1.),
dtype=tf.float32,
trainable=False,
) # restore=restore)
## 3.
# These ops will only be preformed when training.
mean, variance = tf.nn.moments(self.inputs, axis)
try: # TF12
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay,
zero_debias=False) # if zero_debias=True, has bias
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay,
zero_debias=False) # if zero_debias=True, has bias
# print("TF12 moving")
except Exception as e: # TF11
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay)
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay)
# print("TF11 moving")
def mean_var_with_update():
with tf.control_dependencies(
[update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
if trainable:
mean, var = mean_var_with_update()
print(mean)
print(var)
self.outputs = act(
tf.nn.batch_normalization(self.inputs, mean, var, beta,
gamma, epsilon))
else:
self.outputs = act(
tf.nn.batch_normalization(self.inputs, moving_mean,
moving_variance, beta, gamma,
epsilon))
variables = [beta, gamma, moving_mean, moving_variance]
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
self.all_params.extend(variables)
def __init__(
self,
layer=None,
filters=2,
kernel_size=1,
strides=2,
act=tf.identity,
W_init=tf.contrib.layers.xavier_initializer_conv2d(),
b_init=None,
W_init_args={},
b_init_args={},
use_cudnn_on_gpu=True,
data_format='NCHW',
name='conv2d_fixed_padding',
):
if strides == 1:
padding = 'SAME'
else:
padding = 'VALID'
Layer.__init__(self, name=name)
self.inputs = layer.outputs
input_channel = self.inputs.get_shape()[1]
print(
" [TL] Resnet_Convolution %s: kernel_size:%s strides:%s pad:%s activation:%s"
%
(self.name, str(kernel_size), str(strides), padding, act.__name__))
with tf.variable_scope(name) as vs:
if strides > 1:
self.inputs = fixed_padding(self.inputs, kernel_size)
W = tf.get_variable(
name='W_conv2d',
shape=[kernel_size, kernel_size, input_channel, filters],
initializer=W_init,
**W_init_args)
# print(tf.shape(W))
if b_init:
b = tf.get_variable(name='b_conv2d',
shape=(filters),
initializer=b_init,
**b_init_args)
self.outputs = act(
tf.nn.conv2d(self.inputs,
W,
strides=[1, 1, strides, strides],
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format) + b)
else:
self.outputs = act(
tf.nn.conv2d(self.inputs,
W,
strides=[1, 1, strides, strides],
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format))
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
if b_init:
self.all_params.extend([W, b])
else:
self.all_params.extend([W])
def __init__(
self,
cnn_inputs,
cnn_size,
input_size,
layer=None,
n_hidden=100,
num_proj=None,
cell_init_args={},
initializer=tf.random_uniform_initializer(-0.1, 0.1),
n_steps=5,
initial_state=None,
return_last=False,
# is_reshape = True,
return_seq_2d=False,
name='lstm_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
cell_fn = CLSTMCell
print(" tensorlayer:Instantiate RNNLayer %s: n_hidden:%d, n_steps:%d, in_dim:%d %s, cell_fn:%s " % (
self.name, n_hidden,
n_steps, self.inputs.get_shape().ndims, self.inputs.get_shape(), cell_fn.__name__))
# You can get the dimension by .get_shape() or ._shape, and check the
# dimension by .with_rank() as follow.
# self.inputs.get_shape().with_rank(2)
# self.inputs.get_shape().with_rank(3)
# Input dimension should be rank 3 [batch_size, n_steps(max), n_features]
try:
self.inputs.get_shape().with_rank(3)
except:
raise Exception("RNN : Input dimension should be rank 3 : [batch_size, n_steps, n_features]")
# is_reshape : boolean (deprecate)
# Reshape the inputs to 3 dimension tensor.\n
# If input is［batch_size, n_steps, n_features], we do not need to reshape it.\n
# If input is [batch_size * n_steps, n_features], we need to reshape it.
# if is_reshape:
# self.inputs = tf.reshape(self.inputs, shape=[-1, n_steps, int(self.inputs._shape[-1])])
fixed_batch_size = self.inputs.get_shape().with_rank_at_least(1)[0]
if fixed_batch_size.value:
batch_size = fixed_batch_size.value
print(" RNN batch_size (concurrent processes): %d" % batch_size)
else:
from tensorflow.python.ops import array_ops
batch_size = array_ops.shape(self.inputs)[0]
print(" non specified batch_size, uses a tensor instead.")
self.batch_size = batch_size
# Simplified version of tensorflow.models.rnn.rnn.py's rnn().
# This builds an unrolled LSTM for tutorial purposes only.
# In general, use the rnn() or state_saving_rnn() from rnn.py.
#
# The alternative version of the code below is:
#
# from tensorflow.models.rnn import rnn
# inputs = [tf.squeeze(input_, [1])
# for input_ in tf.split(1, num_steps, inputs)]
# outputs, state = rnn.rnn(cell, inputs, initial_state=self._initial_state)
outputs = []
self.cell = cell = cell_fn(num_units=n_hidden, input_size=input_size, cnn_size=cnn_size, num_proj=num_proj,
**cell_init_args)
if initial_state is None:
self.initial_state = cell.zero_state(batch_size, dtype=tf.float32) # 1.2.3
state = self.initial_state
# with tf.variable_scope("model", reuse=None, initializer=initializer):
with tf.variable_scope(name, initializer=initializer) as vs:
for time_step in range(n_steps):
if time_step > 0: tf.get_variable_scope().reuse_variables()
(cell_output, state) = cell(self.inputs[:, time_step, :], cnn_inputs[:, time_step, :], state)
outputs.append(cell_output)
# Retrieve just the RNN variables.
# rnn_variables = [v for v in tf.all_variables() if v.name.startswith(vs.name)]
rnn_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=vs.name)
print(" n_params : %d" % (len(rnn_variables)))
output_size = num_proj if num_proj else n_hidden
if return_last:
# 2D Tensor [batch_size, n_hidden]
self.outputs = outputs[-1]
else:
if return_seq_2d:
# PTB tutorial: stack dense layer after that, or compute the cost from the output
# 2D Tensor [n_example, n_hidden]
self.outputs = tf.reshape(tf.concat(1, outputs), [-1, output_size])
else:
# <akara>: stack more RNN layer after that
# 3D Tensor [n_example/n_steps, n_steps, n_hidden]
self.outputs = tf.reshape(tf.concat(1, outputs), [-1, n_steps, output_size])
self.final_state = state
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
# print(type(self.outputs))
self.all_layers.extend([self.outputs])
self.all_params.extend(rnn_variables)
def __init__(
self,
layer=None,
theta_layer=None,
out_size=[40, 40],
name='sapatial_trans_2d_affine',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
self.theta_layer = theta_layer
print(
" [TL] SpatialTransformer2dAffineLayer %s: in_size:%s out_size:%s"
% (name, self.inputs.get_shape().as_list(), out_size))
with tf.variable_scope(name) as vs:
## 1. make the localisation network to [batch, 6] via Flatten and Dense.
# if self.theta_layer.outputs.get_shape().ndims > 2:
# self.theta_layer.outputs = flatten_reshape(self.theta_layer.outputs, 'flatten')
# ## 2. To initialize the network to the identity transform init.
# 2.1 W
#Converting from 3 parameter output from Localization Network to 6 parameter rotation matrix for STN
size = (int(self.theta_layer.outputs.get_shape()[0]), 6)
self.theta_lay = tf.get_variable(name='theta_layer',
initializer=tf.zeros(size))
self.theta_lay = tf.stack([
tf.cos(self.theta_layer.outputs[:, 0]),
-tf.sin(self.theta_layer.outputs[:, 0]),
self.theta_layer.outputs[:, 1],
tf.sin(self.theta_layer.outputs[:, 0]),
tf.cos(self.theta_layer.outputs[:, 0]),
self.theta_layer.outputs[:, 2]
],
axis=1)
shape = (int(self.theta_lay.get_shape()[-1]), 6)
W = tf.get_variable(name='W', initializer=tf.zeros(shape))
# 2.2 b
identity = tf.constant(
np.array([[1., 0, 0], [0, 1., 0]]).astype('float32').flatten())
b = tf.get_variable(name='b', initializer=identity)
# print(self.theta_layer.outputs)
# 2.3 transformation matrix
self.theta = tf.nn.tanh(tf.matmul(self.theta_lay, W) + b)
## 3. Spatial Transformer Sampling
# 3.1 transformation
self.outputs = transformer(self.inputs,
self.theta,
out_size=out_size)
# 3.2 automatically set batch_size and channels
# e.g. [?, 40, 40, ?] --> [64, 40, 40, 1] or [64, 20, 20, 4]/ Hao Dong
#
fixed_batch_size = self.inputs.get_shape().with_rank_at_least(1)[0]
if fixed_batch_size.value:
batch_size = fixed_batch_size.value
else:
from tensorflow.python.ops import array_ops
batch_size = array_ops.shape(self.inputs)[0]
size = self.inputs.get_shape().as_list()
n_channels = self.inputs.get_shape().as_list()[-1]
# print(self.outputs)
self.outputs = tf.reshape(
self.outputs,
shape=[batch_size, out_size[0], out_size[1], n_channels])
# print(self.outputs)
# exit()
## 4. Get all parameters
variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES,
scope=vs.name)
## fixed
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
## theta_layer
self.all_layers.extend(theta_layer.all_layers)
self.all_params.extend(theta_layer.all_params)
self.all_drop.update(theta_layer.all_drop)
## this layer
self.all_layers.extend([self.outputs])
self.all_params.extend(variables)
def __init__(
self,
layer=None,
embedding_name=None,
E_init=tf.random_uniform_initializer(-0.1, 0.1),
E_init_args={},
name='embedding_reverse_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
# Check if you define the embedding before
if embedding_name == None:
raise Exception(
"You should define the word embedding before. \nCheck if you did before using EmbeddingReverseLayer."
)
else:
have_found_embedding = False
for layer in layer.all_layers:
if embedding_name in layer.name:
have_found_embedding = True
if have_found_embedding == False:
raise Exception(
"You should assign the correct variable scope name of the previous embedding layer."
)
# Get origin embedded
with tf.variable_scope(embedding_name, reuse=True) as vs:
tl.layers.set_name_reuse(True)
origin_embeddings = tf.get_variable(name='embeddings',
**E_init_args)
vocabulary_size = int(origin_embeddings.get_shape()[0])
embedding_size = int(origin_embeddings.get_shape()[1])
print(" [TL] EmbeddingReverselayer %s: (%d, %d)" %
(self.name, vocabulary_size, embedding_size))
"""
The original size of inputs is [batch_size, num_steps, embedding_size]
The initial size of looked-up embedding is [vocabulary_size, embedding_size]
Expand as the equal size and calculate the L2 distance
Examples
--------
<before>
self.inputs = [None, 5, 128]
embed = [2000, 128]
<after>
embed - self.inputs
[None, 5, 2000, 128] - [None, 5, 1, 128]
"""
# Create range to lookup
with tf.variable_scope(name) as vs:
dummy_index = tf.range(0, limit=vocabulary_size)
embed = tf.nn.embedding_lookup(origin_embeddings, dummy_index)
embed_reshape = tf.reshape(embed, [-1])
embed_tile = tf.tile(
embed_reshape,
[tf.shape(self.inputs)[0] * tf.shape(self.inputs)[1]])
embed_tile_reshape = tf.reshape(embed_tile, [
tf.shape(self.inputs)[0],
int(self.inputs.get_shape()[1]), vocabulary_size,
embedding_size
])
input_embed = tf.expand_dims(self.inputs, axis=2)
"""
To generate the probability, the negative of L2 distance will be computed.
Next, the whole vector will shift over zero (add the minimun value).
At last, the softmax will process on the vector and give the probability of each words
"""
embed_diff = embed_tile_reshape - input_embed
embed_distance = tf.reduce_sum(tf.square(embed_diff), axis=-1)
embed_prob = tf.nn.softmax(-embed_distance +
tf.reduce_min(embed_distance),
dim=-1)
embed_index = tf.arg_max(embed_prob, -1)
self.outputs = embed_index
self.all_layers = [self.outputs]
self.all_params = [
dummy_index, embed, embed_distance, embed_prob, embed_index
]
self.all_drop = {}
def __init__(
self, # https://www.cnblogs.com/ranjiewen/articles/7748232.html 这个里面维度是(n,c,h,w)所以分通道求均值方差是axis=(0，2，3），而本项目是(n,h,w,c)，因此均值方差是axis=(0,1,2)
layer=None, # Last layer is: Conv2dLayer
decay=0.9, # 衰减系数。合适的衰减系数值接近1.0,特别是含多个9的值：0.999,0.99,0.9。如果训练集表现很好而验证/测试集表现得不好，选择小的系数（推荐使用0.9）
epsilon=2e-5, # 避免被零除
act=tf.identity, # 创建并返回一个和输入大小一样的tensor
is_train=False,
fix_gamma=True,
beta_init=tf.zeros_initializer, # bn层的参数 y = gamma * x' + beta
gamma_init=tf.random_normal_initializer(
mean=1.0,
stddev=0.002), # bn层的参数 y = gamma * x' + beta，tf.ones_initializer,
# dtype = tf.float32,
trainable=None, # ？和 is_train区别？
name='batchnorm_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs # Tensor("resnet_v1_50/conv1/Identity:0", shape=(?, 112, 112, 64), dtype=float32)
print(
" [TL] BatchNormLayer %s: decay:%f epsilon:%f act:%s is_train:%s"
% (self.name, decay, epsilon, act.__name__, is_train))
x_shape = self.inputs.get_shape(
) # cnn (?, 112, 112, 64) or # fc (?, 512) 全连接，一个batch会被扔到输出的512个里每个neural中去，也就是说一个batch会被扔512次，因此对每个neural都要加一组mean、var因此是(512,)个mean，(512,)个var
params_shape = x_shape[
-1:] # cnn (64, ) 均值应该有通道数那么多个，方差也应该有通道数那么多个，一个通道上求衣蛾均值，输入有c个通道就有c个均值c个方差, fc (512, )
from tensorflow.python.training import moving_averages
from tensorflow.python.ops import control_flow_ops
with tf.variable_scope(name) as vs: # 在命名空间'bn0'下，创建以下下变量
axis = list(
range(len(x_shape) - 1)
) # cnn：[0, 1, 2]，fc是[0] https://www.cnblogs.com/ranjiewen/articles/7748232.html 这个里面维度是(n,c,h,w)所以分通道求均值方差是axis=(0，2，3），而本项目是(n,h,w,c)，因此均值方差是axis=(0,1,2)
# fc axis = [0]
## 1. beta, gamma
if tf.__version__ > '0.12.1' and beta_init == tf.zeros_initializer:
beta_init = beta_init()
beta = tf.get_variable(
'beta',
shape=params_shape,
initializer=beta_init,
dtype=tf.float32,
trainable=is_train
) # , restore=restore) # <tf.Variable 'resnet_v1_50/bn0/beta:0' shape=(64,) dtype=float32_ref>
gamma = tf.get_variable(
'gamma',
shape=params_shape,
initializer=gamma_init,
dtype=tf.float32,
trainable=fix_gamma,
) # restore=restore) # <tf.Variable 'resnet_v1_50/bn0/gamma:0' shape=(64,) dtype=float32_ref>
## 2.
if tf.__version__ > '0.12.1':
moving_mean_init = tf.zeros_initializer()
else:
moving_mean_init = tf.zeros_initializer
moving_mean = tf.get_variable(
'moving_mean',
params_shape,
initializer=moving_mean_init,
dtype=tf.float32,
trainable=False
) # restore=restore) 如果有则取已有'moving_mean'，没有的话init新的，# <tf.Variable 'resnet_v1_50/bn0/moving_mean:0' shape=(64,) dtype=float32_ref>
moving_variance = tf.get_variable(
'moving_variance',
params_shape,
initializer=tf.constant_initializer(1.),
dtype=tf.float32,
trainable=False,
) # restore=restore) # <tf.Variable 'resnet_v1_50/bn0/moving_variance:0' shape=(64,) dtype=float32_ref>
## 3.
# These ops will only be preformed when training. # fc
mean, variance = tf.nn.moments(
self.inputs, axis
) # cnn (?,112,112,64), axis=[0,1,2] ; for i in 64 : mean(?,112,112), var(?,112,112)，out = (64:), (64:)， Tensor("resnet_v1_50/bn0/moments/Squeeze:0", shape=(64,), dtype=float32)，Tensor("resnet_v1_50/bn0/moments/Squeeze_1:0", shape=(64,), dtype=float32)
try: # TF12
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay,
zero_debias=False) # if zero_debias=True, has bias
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay, zero_debias=False
) # if zero_debias=True, has bias，如果想提高稳定性，zero_debias设为true
# print("TF12 moving")
except Exception as e: # TF11
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay)
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay)
# print("TF11 moving")
def mean_var_with_update():
with tf.control_dependencies(
[update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
if trainable:
mean, var = mean_var_with_update()
print('bn_mean:', mean)
print('bn_var:', var)
self.outputs = act(
tf.nn.batch_normalization(self.inputs, mean, var, beta,
gamma, epsilon))
else:
self.outputs = act(
tf.nn.batch_normalization(self.inputs, moving_mean,
moving_variance, beta, gamma,
epsilon)
) # Tensor("resnet_v1_50/bn0/Identity:0", shape=(?, 112, 112, 64), dtype=float32)
variables = [
beta, gamma, moving_mean, moving_variance
] # <class 'list'>: [<tf.Variable 'resnet_v1_50/bn0/beta:0' shape=(64,) dtype=float32_ref>, <tf.Variable 'resnet_v1_50/bn0/gamma:0' shape=(64,) dtype=float32_ref>, <tf.Variable 'resnet_v1_50/bn0/moving_mean:0' shape=(64,) dtype=float32_ref>, <tf.Variable 'resnet_v1_50/bn0/moving_variance:0' shape=(64,) dtype=float32_ref>]
self.all_layers = list(
layer.all_layers
) # <class 'list'>: [<tf.Tensor 'resnet_v1_50/conv1/Identity:0' shape=(?, 112, 112, 64) dtype=float32>]
self.all_params = list(
layer.all_params
) # <class 'list'>: [<tf.Variable 'resnet_v1_50/conv1/W_conv2d:0' shape=(3, 3, 3, 64) dtype=float32_ref>]
self.all_drop = dict(layer.all_drop)
self.all_layers.extend(
[self.outputs]
) # <class 'list'>: [<tf.Tensor 'resnet_v1_50/conv1/Identity:0' shape=(?, 112, 112, 64) dtype=float32>, <tf.Tensor 'resnet_v1_50/bn0/Identity:0' shape=(?, 112, 112, 64) dtype=float32>]
self.all_params.extend(variables)
def __init__(
self,
layer=None,
decay=0.9,
epsilon=2e-5,
act=tf.identity,
is_train=False,
fix_gamma=True,
beta_init=tf.zeros_initializer,
gamma_init=tf.random_normal_initializer(mean=1.0, stddev=0.002), # tf.ones_initializer,
# dtype = tf.float32,
trainable=None,
name='batchnorm_layer',
):
Layer.__init__(self, name=name)
self.inputs = layer.outputs
print(" [TL] BatchNormLayer %s: decay:%f epsilon:%f act:%s is_train:%s" % (self.name, decay, epsilon, act.__name__, is_train))
x_shape = self.inputs.get_shape()
params_shape = x_shape[-1:]
from tensorflow.python.training import moving_averages
from tensorflow.python.ops import control_flow_ops
with tf.variable_scope(name) as vs:
axis = list(range(len(x_shape) - 1))
## 1. beta, gamma
if tf.__version__ > '0.12.1' and beta_init == tf.zeros_initializer:
beta_init = beta_init()
with tf.device('/cpu:0'):
beta = tf.get_variable('beta', shape=params_shape, initializer=beta_init, dtype=tf.float32, trainable=is_train) #, restore=restore)
gamma = tf.get_variable(
'gamma',
shape=params_shape,
initializer=gamma_init,
dtype=tf.float32,
trainable=fix_gamma,
) #restore=restore)
## 2.
if tf.__version__ > '0.12.1':
moving_mean_init = tf.zeros_initializer()
else:
moving_mean_init = tf.zeros_initializer
with tf.device('/cpu:0'):
moving_mean = tf.get_variable('moving_mean', params_shape, initializer=moving_mean_init, dtype=tf.float32, trainable=False) # restore=restore)
moving_variance = tf.get_variable(
'moving_variance',
params_shape,
initializer=tf.constant_initializer(1.),
dtype=tf.float32,
trainable=False,
) # restore=restore)
## 3.
# These ops will only be preformed when training.
mean, variance = tf.nn.moments(self.inputs, axis)
try: # TF12
update_moving_mean = moving_averages.assign_moving_average(moving_mean, mean, decay, zero_debias=False) # if zero_debias=True, has bias
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay, zero_debias=False) # if zero_debias=True, has bias
# print("TF12 moving")
except Exception as e: # TF11
update_moving_mean = moving_averages.assign_moving_average(moving_mean, mean, decay)
update_moving_variance = moving_averages.assign_moving_average(moving_variance, variance, decay)
# print("TF11 moving")
def mean_var_with_update():
with tf.control_dependencies([update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
if trainable:
mean, var = mean_var_with_update()
print(mean)
print(var)
self.outputs = act(tf.nn.batch_normalization(self.inputs, mean, var, beta, gamma, epsilon))
else:
self.outputs = act(tf.nn.batch_normalization(self.inputs, moving_mean, moving_variance, beta, gamma, epsilon))
variables = [beta, gamma, moving_mean, moving_variance]
self.all_layers = list(layer.all_layers)
self.all_params = list(layer.all_params)
self.all_drop = dict(layer.all_drop)
self.all_layers.extend([self.outputs])
self.all_params.extend(variables)
