def vgg16(input, num_class):
network = tflearn.conv_2d(input,
KERNEL,
3,
activation='relu',
regularizer="L2",
weight_decay=0.0001)
network = tflearn.avg_pool_2d(network, 2)
network = tflearn.conv_2d(network,
KERNEL,
3,
activation='relu',
regularizer="L2",
weight_decay=0.0001)
network = tflearn.avg_pool_2d(network, 2)
network = tflearn.conv_2d(network,
KERNEL,
3,
activation='relu',
regularizer="L2",
weight_decay=0.0001)
network = tflearn.avg_pool_2d(network, 2)
network = tflearn.conv_2d(network,
KERNEL,
3,
activation='relu',
regularizer="L2",
weight_decay=0.0001)
network = tflearn.avg_pool_2d(network, 2)
x = tflearn.fully_connected(network,
num_class,
activation='sigmoid',
scope='fc8')
return x
def discriminator(x, reuse=False):
with tf.variable_scope('Discriminator', reuse=reuse):
x = tflearn.conv_2d(x, 64, 5, activation='tanh') # [n,28,28,64]
x = tflearn.avg_pool_2d(x, 2) # [n,14,14,64]
x = tflearn.conv_2d(x, 128, 5, activation='tanh') # [n,14,14,128]
x = tflearn.avg_pool_2d(x, 2) # [n,7,7,128]
x = tflearn.fully_connected(x, 1024, activation='tanh') # [n,1024]
x = tflearn.fully_connected(x, 2) # [n,2] 2类 0 False ;1 True
x = tf.nn.softmax(x)
return x
def discriminator(x, reuse=False):
with tf.variable_scope('Discriminator', reuse=reuse):
x = tflearn.conv_2d(x, 64, 5, activation='tanh')
x = tflearn.avg_pool_2d(x, 2)
x = tflearn.conv_2d(x, 128, 5, activation='tanh')
x = tflearn.avg_pool_2d(x, 2)
x = tflearn.fully_connected(x, 1024, activation='tanh')
x = tflearn.fully_connected(x, 2)
x = tf.nn.softmax(x)
return x
def residual_block(incoming,
nb_blocks,
out_channels,
downsample=False,
downsample_strides=2,
activation='relu',
batch_norm=True,
bias=True,
weights_init='variance_scaling',
bias_init='zeros',
regularizer='L2',
weight_decay=0.0001,
trainable=True,
restore=True,
reuse=False,
scope=None,
name="ResidualBlock"):
"""
Residual Block.
A residual block as described in MSRA's Deep Residual Network paper. Full pre-activation architecture is used here.
Input: 4-D Tensor [batch, height, width, in_channels].
Output: 4-D Tensor [batch, new height, new width, nb_filter].
Arguments: incoming: `Tensor`. Incoming 4-D Layer. nb_blocks: `int`. Number of layer blocks. out_channels: `int`. The number of convolutional filters of the convolution layers. downsample: `bool`. If True, apply downsampling using 'downsample_strides' for strides. downsample_strides: `int`. The strides to use when downsampling. activation: `str` (name) or `function` (returning a `Tensor`). Activation applied to this layer (see tflearn.activations). Default: 'linear'. batch_norm: `bool`. If True, apply batch normalization. bias: `bool`. If True, a bias is used. weights_init: `str` (name) or `Tensor`. Weights initialization. (see tflearn.initializations) Default: 'uniform_scaling'. bias_init: `str` (name) or `tf.Tensor`. Bias initialization. (see tflearn.initializations) Default: 'zeros'. regularizer: `str` (name) or `Tensor`. Add a regularizer to this layer weights (see tflearn.regularizers). Default: None. weight_decay: `float`. Regularizer decay parameter. Default: 0.001. trainable: `bool`. If True, weights will be trainable. restore: `bool`. If True, this layer weights will be restored when loading a model. reuse: `bool`. If True and 'scope' is provided, this layer variables will be reused (shared). scope: `str`. Define this layer scope (optional). A scope can be used to share variables between layers. Note that scope will override name. name: A name for this layer (optional). Default: 'ShallowBottleneck'.
References: - Deep Residual Learning for Image Recognition. Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. 2015. - Identity Mappings in Deep Residual Networks. Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. 2015.
Links: - [http://arxiv.org/pdf/1512.03385v1.pdf] (http://arxiv.org/pdf/1512.03385v1.pdf) - [Identity Mappings in Deep Residual Networks] (https://arxiv.org/pdf/1603.05027v2.pdf)
"""
resnet = incoming
in_channels = incoming.get_shape().as_list()[-1]
with tf.variable_op_scope([incoming], scope, name, reuse=reuse) as scope:
name = scope.name #TODO
for i in range(nb_blocks):
identity = resnet
if not downsample:
downsample_strides = 1
if batch_norm:
resnet = tflearn.batch_normalization(resnet)
resnet = tflearn.activation(resnet, activation)
resnet = conv_2d(resnet, out_channels, 3, downsample_strides,
'same', 'linear', bias, weights_init, bias_init,
regularizer, weight_decay, trainable, restore)
if batch_norm:
resnet = tflearn.batch_normalization(resnet)
resnet = tflearn.activation(resnet, activation)
resnet = conv_2d(resnet, out_channels, 3, 1, 'same', 'linear',
bias, weights_init, bias_init, regularizer,
weight_decay, trainable, restore)
# Downsampling
if downsample_strides > 1:
identity = tflearn.avg_pool_2d(identity, 1, downsample_strides)
# Projection to new dimension
if in_channels != out_channels:
ch = (out_channels - in_channels) // 2
identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch]])
in_channels = out_channels
resnet = resnet + identity
return resnet
def image_net(input_map, reuse=False):
with tf.variable_scope('image', reuse=False):
img_out = tfl.conv_2d(input_map, 8, [3, 3], activation='relu')
img_out = tfl.conv_2d(img_out, 8, [3, 3], activation='relu')
img_out = tfl.avg_pool_2d(img_out, [2, 2])
#img_out=tfl.conv_2d(img_out,8,[3,3], activation='relu')
#img_out=tfl.conv_2d(img_out,8,[3,3], activation='relu')
#img_out=tfl.avg_pool_2d(img_out,[2,2])
img_out = tfl.conv_2d(img_out, 8, [3, 3], activation='relu')
img_out = tfl.conv_2d(img_out, 8, [3, 3], activation='relu')
img_out = tfl.avg_pool_2d(img_out, [2, 2])
img_out = tf.reshape(
img_out,
[-1, np.prod(np.array(img_out.get_shape().as_list()[1:]))])
img_out = tfl.fully_connected(img_out, 32, 'tanh')
return img_out
def residual_block(incoming, nb_blocks, out_channels, downsample=False,
downsample_strides=2, activation='relu', batch_norm=True,
bias=True, weights_init='variance_scaling',
bias_init='zeros', regularizer='L2', weight_decay=0.0001,
trainable=True, restore=True, reuse=False, scope=None,
name="ResidualBlock"):
resnet = incoming
in_channels = incoming.get_shape().as_list()[-1]
# Variable Scope fix for older TF
with tf.variable_scope(scope, default_name=name, values=[incoming],
reuse=reuse) as scope:
name = scope.name #TODO
for i in range(nb_blocks):
identity = resnet
if not downsample:
downsample_strides = 1
if batch_norm:
#I think conv_2d should go before not after batch normalization and activation
resnet = tflearn.batch_normalization(resnet)
resnet = tflearn.activation(resnet, activation)
resnet = conv_2d(resnet, out_channels, 3,
downsample_strides, 'same', 'linear',
bias, weights_init, bias_init,
regularizer, weight_decay, trainable,
restore)
if batch_norm:
resnet = tflearn.batch_normalization(resnet)
resnet = tflearn.activation(resnet, activation)
resnet = conv_2d(resnet, out_channels, 3, 1, 'same',
'linear', bias, weights_init,
bias_init, regularizer, weight_decay,
trainable, restore)
# Downsampling
if downsample_strides > 1:
identity = tflearn.avg_pool_2d(identity, downsample_strides,
downsample_strides)
# Projection to new dimension
if in_channels != out_channels:
ch = (out_channels - in_channels)//2
identity = tf.pad(identity,
[[0, 0], [0, 0], [0, 0], [ch, ch]])
in_channels = out_channels
resnet = resnet + identity
return resnet
def discriminator(self, x, reuse=False):
with tf.variable_scope('Discriminator', reuse=reuse):
#self.noise_layer(x, 0.1)
x = tflearn.conv_2d(x, 64, 2, activation='relu')
x = tflearn.avg_pool_2d(x, 2)
# x = tflearn.batch_normalization(x)
x = tflearn.conv_2d(x, 128, 2, activation='relu')
x = tflearn.avg_pool_2d(x, 2)
# x = tflearn.batch_normalization(x)
x = tflearn.conv_2d(x, 64, 2, activation='relu')
x = tflearn.avg_pool_2d(x, 2)
# x = tflearn.batch_normalization(x)
x = tflearn.conv_2d(x, 128, 2, activation='relu')
x = tflearn.avg_pool_2d(x, 2)
# x = tflearn.batch_normalization(x)
x = tflearn.fully_connected(x, 1024, activation='relu')
x = tflearn.dropout(x, 0.8)
x = tflearn.fully_connected(x, 2, activation='softmax')
return x
def CNN_Core(self, x, reuse=False):
with tf.variable_scope('cnn_core', reuse=reuse):
network = tflearn.conv_2d(x,
KERNEL,
5,
activation='relu',
regularizer="L2",
weight_decay=0.0001)
network = tflearn.avg_pool_2d(network, 3)
network = tflearn.conv_2d(network,
KERNEL,
3,
activation='relu',
regularizer="L2",
weight_decay=0.0001)
network = tflearn.avg_pool_2d(network, 2)
network = tflearn.fully_connected(network,
DENSE_SIZE,
activation='relu')
split_flat = tflearn.flatten(network)
return split_flat
def create_model():
# Building Wide Residual Network
assert ((depth - 4) % 6 == 0)
n = (depth - 4) / 6
n_stages = [16, 16 * k, 32 * k, 64 * k]
net = tflearn.input_data(shape=[None, 32, 32, 3],
data_preprocessing=img_prep,
data_augmentation=img_aug)
conv1 = conv_2d(net,
n_stages[0],
3,
activation='linear',
bias=False,
regularizer='L2',
weight_decay=0.0001)
# Add wide residual blocks
block_fn = _wide_basic
conv2 = _layer(block_fn,
n_input_plane=n_stages[0],
n_output_plane=n_stages[1],
count=n,
stride=(1, 1))(conv1) # "Stage 1 (spatial size: 32x32)"
conv3 = _layer(block_fn,
n_input_plane=n_stages[1],
n_output_plane=n_stages[2],
count=n,
stride=(2, 2))(conv2) # "Stage 2 (spatial size: 16x16)"
conv4 = _layer(block_fn,
n_input_plane=n_stages[2],
n_output_plane=n_stages[3],
count=n,
stride=(2, 2))(conv3) # "Stage 3 (spatial size: 8x8)"
net = tflearn.batch_normalization(conv4)
net = tflearn.activation(net, 'twins_relu')
net = tflearn.avg_pool_2d(net, 8)
#net = tflearn.avg_pool_2d(net, kernel_size=8, strides=1, padding='same')
net = tflearn.fully_connected(net, 10, activation='softmax')
return net
def transition_layer(x, scope, reduction=0.5, keep_prob=1):
out_filters = int(int(x.get_shape()[-1]) * reduction)
with tf.name_scope(scope):
x = tflearn.batch_normalization(x, scope=scope + '_batch1')
x = tf.nn.relu(x)
x = tflearn.conv_2d(x,
nb_filter=out_filters,
filter_size=1,
strides=1,
padding='same',
activation='linear',
bias=False,
scope=scope + '_conv1',
regularizer='L2',
weight_decay=1e-4)
x = tflearn.dropout(x, keep_prob=keep_prob)
x = tflearn.avg_pool_2d(x, kernel_size=2, strides=2, padding='valid')
print(scope, x)
return x
def residual_bottleneck(incoming, nb_blocks, bottleneck_size, out_channels,
downsample=False, downsample_strides=2,
activation='relu', batch_norm=True, bias=True,
weights_init='variance_scaling', bias_init='zeros',
regularizer='L2', weight_decay=0.0001,
trainable=True, restore=True, name="ResidualBottleneck"):
""" Residual Bottleneck.
A residual bottleneck block as described in MSRA's Deep Residual Network
paper. Full pre-activation architecture is used here.
Input:
4-D Tensor [batch, height, width, in_channels].
Output:
4-D Tensor [batch, new height, new width, nb_filter].
Arguments:
incoming: `Tensor`. Incoming 4-D Layer.
nb_blocks: `int`. Number of layer blocks.
bottleneck_size: `int`. The number of convolutional filter of the
bottleneck convolutional layer.
out_channels: `int`. The number of convolutional filters of the
layers surrounding the bottleneck layer.
downsample: `bool`. If True, apply downsampling using
'downsample_strides' for strides.
downsample_strides: `int`. The strides to use when downsampling.
activation: `str` (name) or `function` (returning a `Tensor`).
Activation applied to this layer (see tflearn.activations).
Default: 'linear'.
batch_norm: `bool`. If True, apply batch normalization.
bias: `bool`. If True, a bias is used.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(see tflearn.initializations) Default: 'uniform_scaling'.
bias_init: `str` (name) or `tf.Tensor`. Bias initialization.
(see tflearn.initializations) Default: 'zeros'.
regularizer: `str` (name) or `Tensor`. Add a regularizer to this
layer weights (see tflearn.regularizers). Default: None.
weight_decay: `float`. Regularizer decay parameter. Default: 0.001.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model
name: A name for this layer (optional). Default: 'DeepBottleneck'.
References:
- Deep Residual Learning for Image Recognition. Kaiming He, Xiangyu
Zhang, Shaoqing Ren, Jian Sun. 2015.
- Identity Mappings in Deep Residual Networks. Kaiming He, Xiangyu
Zhang, Shaoqing Ren, Jian Sun. 2015.
Links:
- [http://arxiv.org/pdf/1512.03385v1.pdf]
(http://arxiv.org/pdf/1512.03385v1.pdf)
- [Identity Mappings in Deep Residual Networks]
(https://arxiv.org/pdf/1603.05027v2.pdf)
"""
resnet = incoming
in_channels = incoming.get_shape().as_list()[-1]
with tf.name_scope(name):
for i in range(nb_blocks):
identity = resnet
if not downsample:
downsample_strides = 1
if batch_norm:
resnet = tflearn.batch_normalization(resnet)
resnet = tflearn.activation(resnet, activation)
resnet = conv_2d(resnet, bottleneck_size, 1,
downsample_strides, 'valid',
'linear', bias, weights_init,
bias_init, regularizer, weight_decay,
trainable, restore)
if batch_norm:
resnet = tflearn.batch_normalization(resnet)
resnet = tflearn.activation(resnet, activation)
resnet = conv_2d(resnet, bottleneck_size, 3, 1, 'same',
'linear', bias, weights_init,
bias_init, regularizer, weight_decay,
trainable, restore)
resnet = conv_2d(resnet, out_channels, 1, 1, 'valid',
activation, bias, weights_init,
bias_init, regularizer, weight_decay,
trainable, restore)
# Downsampling
if downsample_strides > 1:
identity = tflearn.avg_pool_2d(identity, 1,
downsample_strides)
# Projection to new dimension
if in_channels != out_channels:
ch = (out_channels - in_channels)//2
identity = tf.pad(identity,
[[0, 0], [0, 0], [0, 0], [ch, ch]])
in_channels = out_channels
resnet = resnet + identity
resnet = tflearn.activation(resnet, activation)
return resnet
def wideresnet_block(incoming,
nb_blocks,
out_channels,
width,
downsample=False,
downsample_strides=2,
activation='relu',
batch_norm=True,
bias=True,
weights_init='variance_scaling',
bias_init='zeros',
regularizer='L2',
weight_decay=0.0001,
trainable=True,
restore=True,
reuse=False,
scope=None,
name="WideResNetBlock"):
out_channels = out_channels * width #layers are wider for a constant
widenet = incoming
in_channels = incoming.get_shape().as_list()[-1]
# Variable Scope fix for older TF
with tf.variable_scope(scope,
default_name=name,
values=[incoming],
reuse=reuse) as scope:
name = scope.name #TODO
for i in range(nb_blocks):
identity = widenet
if not downsample:
downsample_strides = 1
if batch_norm:
widenet = tflearn.batch_normalization(widenet)
widenet = tflearn.activation(widenet, activation)
widenet = conv_2d(widenet, out_channels, 3, downsample_strides,
'same', 'linear', bias, weights_init, bias_init,
regularizer, weight_decay, trainable, restore)
if batch_norm:
widenet = tflearn.batch_normalization(widenet)
widenet = tflearn.activation(widenet, activation)
widenet = tflearn.dropout(widenet,
0.7) #added dropout between layers
widenet = conv_2d(widenet, out_channels, 3, 1, 'same', 'linear',
bias, weights_init, bias_init, regularizer,
weight_decay, trainable, restore)
# Downsampling
if downsample_strides > 1:
identity = tflearn.avg_pool_2d(identity, downsample_strides,
downsample_strides)
# Projection to new dimension
if in_channels != out_channels:
ch = (out_channels - in_channels) // 2
identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch]])
in_channels = out_channels
widenet = widenet + identity
return widenet
#logits = tf.squeeze(conv3,[1,2])
#x = tf.nn.softmax(logits)
kernel = tf.get_variable(name='attention_conv3_weights',
initializer=tf.truncated_normal([1, 1, 512, 512],
dtype=tf.float32,
stddev=1e-1))
conv = tf.nn.conv2d(x, kernel, [1, 1, 1, 1], padding='SAME')
conv = tf.nn.relu(conv)
kernel2 = tf.get_variable(name='attention_conv4_weights',
initializer=tf.truncated_normal([1, 1, 512, 1],
dtype=tf.float32,
stddev=1e-1))
conv2 = tf.nn.conv2d(conv, kernel2, [1, 1, 1, 1], padding='SAME')
attention_score2 = tf.nn.relu(conv2)
x = x * attention_score2
c = tflearn.avg_pool_2d(x, 7)
c = tflearn.flatten(c)
c3 = tflearn.fully_connected(c, 32, activation='relu', scope='c3')
c4 = tflearn.fully_connected(c3, 512, activation='sigmoid', scope='c4')
c4 = tf.expand_dims(c4, 1)
c4 = tf.expand_dims(c4, 2)
x = x * c4
x = tflearn.fully_connected(x, 4096, activation='relu', scope='fc6')
x = tflearn.dropout(x, 0.5, name='dropout1')
fc7 = tflearn.fully_connected(x, 4096, activation='relu', scope='fc7')
import matplotlib.pyplot as plt
with tf.Session() as sess:
#sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
def res_block_aprelu(incoming, nb_blocks, out_channels, downsample=False,
downsample_strides=2, batch_norm=True,
bias=True, weights_init='variance_scaling',
bias_init='zeros', regularizer='L2', weight_decay=0.0001,
trainable=True, restore=True, reuse=False, scope=None,
name="ResidualBlock"):
resnet = incoming
in_channels = incoming.get_shape().as_list()[-1]
# Variable Scope fix for older TF
try:
vscope = tf.variable_scope(scope, default_name=name, values=[incoming],
reuse=reuse)
except Exception:
vscope = tf.variable_op_scope([incoming], scope, name, reuse=reuse)
with vscope as scope:
name = scope.name #TODO
for i in range(nb_blocks):
identity = resnet
if not downsample:
downsample_strides = 1
if batch_norm:
resnet = bn(resnet)
resnet = aprelu(resnet)
resnet = conv_2d(resnet, out_channels, 3,
downsample_strides, 'same', 'linear',
bias, weights_init, bias_init,
regularizer, weight_decay, trainable,
restore)
if batch_norm:
resnet = bn(resnet)
resnet = aprelu(resnet)
resnet = conv_2d(resnet, out_channels, 3, 1, 'same',
'linear', bias, weights_init,
bias_init, regularizer, weight_decay,
trainable, restore)
# Downsampling
if downsample_strides > 1:
identity = tflearn.avg_pool_2d(identity, 1, downsample_strides)
# Projection to new dimension
if in_channels != out_channels:
if (out_channels - in_channels) % 2 == 0:
ch = (out_channels - in_channels)//2
identity = tf.pad(identity,
[[0, 0], [0, 0], [0, 0], [ch, ch]])
else:
ch = (out_channels - in_channels)//2
identity = tf.pad(identity,
[[0, 0], [0, 0], [0, 0], [ch, ch+1]])
in_channels = out_channels
resnet = resnet + identity
return resnet
def network(self, input):
datadict = np.load('vgg16_weights.npz')
#print(datadict.keys())
#print(datadict['conv1_1_W'].shape)
conv1_1 = tf.constant(datadict['conv1_1_W'])
bias1_1 = tf.constant(datadict['conv1_1_b'])
conv1_2 = tf.constant(datadict['conv1_2_W'])
bias1_2 = tf.constant(datadict['conv1_1_b'])
conv2_1 = tf.constant(datadict['conv2_1_W'])
bias2_1 = tf.constant(datadict['conv2_1_b'])
conv2_2 = tf.constant(datadict['conv2_2_W'])
bias2_2 = tf.constant(datadict['conv2_2_b'])
conv3_1 = tf.constant(datadict['conv3_1_W'])
bias3_1 = tf.constant(datadict['conv3_1_b'])
conv3_2 = tf.constant(datadict['conv3_2_W'])
bias3_2 = tf.constant(datadict['conv3_2_b'])
conv3_3 = tf.constant(datadict['conv3_3_W'])
bias3_3 = tf.constant(datadict['conv3_3_b'])
conv4_1 = tf.constant(datadict['conv4_1_W'])
bias4_1 = tf.constant(datadict['conv4_1_b'])
conv4_2 = tf.constant(datadict['conv4_2_W'])
bias4_2 = tf.constant(datadict['conv4_2_b'])
conv4_3 = tf.constant(datadict['conv4_3_W'])
bias4_3 = tf.constant(datadict['conv4_3_b'])
conv5_1 = tf.constant(datadict['conv5_1_W'])
bias5_1 = tf.constant(datadict['conv5_1_b'])
conv5_2 = tf.constant(datadict['conv5_2_W'])
bias5_2 = tf.constant(datadict['conv5_2_b'])
conv5_3 = tf.constant(datadict['conv5_3_W'])
bias5_3 = tf.constant(datadict['conv5_3_b'])
fc6_w = tf.constant(datadict['fc6_W'])
fc6_b = tf.constant(datadict['fc6_b'])
fc7_w = tf.constant(datadict['fc7_W'])
fc7_b = tf.constant(datadict['fc7_b'])
x = tflearn.conv_2d(input,
64,
3,
activation='relu',
weights_init=conv1_1,
bias_init=bias1_1,
scope='conv1_1')
x = tflearn.conv_2d(x,
64,
3,
activation='relu',
weights_init=conv1_2,
bias_init=bias1_2,
scope='conv1_2')
x = tflearn.max_pool_2d(x, 2, strides=2)
'''kernel = tf.Variable(tf.truncated_normal([1,1,64,64],dtype=tf.float32,stddev=1e-1),name='attention_conv1_weights')
conv = tf.nn.conv2d(x,kernel,[1,1,1,1],padding='SAME')
conv = tf.nn.relu(conv)
kernel2 = tf.Variable(tf.truncated_normal([1,1,64,1],dtype=tf.float32,stddev=1e-1), name='attention_conv2_weights')
conv2 = tf.nn.conv2d(conv,kernel2,[1,1,1,1], padding='SAME')
attention_score = tf.nn.relu(conv2)
x = attention_score * x
c = tflearn.avg_pool_2d(x, 112)
c = tflearn.flatten(c)
c1 = tflearn.fully_connected(c, 4, activation='relu')
c2 = tflearn.fully_connected(c1, 64, activation='sigmoid')
c2 = tf.expand_dims(c2, 1)
c2 = tf.expand_dims(c2, 2)
x = x * c2'''
#attention_c = tf.get_variable('attention_c', shape=[64],initializer=tf.random_normal_initializer())
#x = x * attention_c
x = tflearn.conv_2d(x,
128,
3,
activation='relu',
weights_init=conv2_1,
bias_init=bias2_1,
scope='conv2_1')
x = tflearn.conv_2d(x,
128,
3,
activation='relu',
weights_init=conv2_2,
bias_init=bias2_2,
scope='conv2_2')
x = tflearn.max_pool_2d(x, 2, strides=2)
'''kernel = tf.get_variable(name='attention_conv3_weights',initializer=tf.truncated_normal([1,1,128,128],dtype=tf.float32,stddev=1e-1))
conv = tf.nn.conv2d(x, kernel,[1,1,1,1],padding='SAME')
conv = tf.nn.relu(conv)
kernel2 = tf.get_variable(name='attention_conv4_weights', initializer=tf.truncated_normal([1,1,128,1],dtype=tf.float32, stddev=1e-1))
conv2 = tf.nn.conv2d(conv,kernel2,[1,1,1,1], padding='SAME')
attention_score2= tf.nn.relu(conv2)
x = x * attention_score2
c = tflearn.avg_pool_2d(x,56)
c = tflearn.flatten(c)
c3 = tflearn.fully_connected(c,8,activation='relu',scope='c3')
c4 = tflearn.fully_connected(c3,128, activation='sigmoid',scope='c4')
c4 = tf.expand_dims(c4,1)
c4 = tf.expand_dims(c4, 2)
x = x * c4'''
x = tflearn.conv_2d(x,
256,
3,
activation='relu',
weights_init=conv3_1,
bias_init=bias3_1,
scope='conv3_1')
x = tflearn.conv_2d(x,
256,
3,
activation='relu',
weights_init=conv3_2,
bias_init=bias3_2,
scope='conv3_2')
x = tflearn.conv_2d(x,
256,
3,
activation='relu',
weights_init=conv3_3,
bias_init=bias3_3,
scope='conv3_3')
x = tflearn.max_pool_2d(x, 2, strides=2)
'''kernel = tf.get_variable(name='attention_conv3_weights',initializer=tf.truncated_normal([1,1,256,256],dtype=tf.float32,stddev=1e-1))
conv = tf.nn.conv2d(x, kernel,[1,1,1,1],padding='SAME')
conv = tf.nn.relu(conv)
kernel2 = tf.get_variable(name='attention_conv4_weights', initializer=tf.truncated_normal([1,1,256,1],dtype=tf.float32, stddev=1e-1))
conv2 = tf.nn.conv2d(conv,kernel2,[1,1,1,1], padding='SAME')
attention_score2= tf.nn.relu(conv2)
x = x * attention_score2
c = tflearn.avg_pool_2d(x,28)
c = tflearn.flatten(c)
c3 = tflearn.fully_connected(c,16,activation='relu',scope='c3')
c4 = tflearn.fully_connected(c3,256, activation='sigmoid',scope='c4')
c4 = tf.expand_dims(c4,1)
c4 = tf.expand_dims(c4, 2)
x = x * c4'''
x = tflearn.conv_2d(x,
512,
3,
activation='relu',
weights_init=conv4_1,
bias_init=bias4_1,
scope='conv4_1')
x = tflearn.conv_2d(x,
512,
3,
activation='relu',
weights_init=conv4_2,
bias_init=bias4_2,
scope='conv4_2')
x = tflearn.conv_2d(x,
512,
3,
activation='relu',
weights_init=conv4_3,
bias_init=bias4_3,
scope='conv4_3')
x = tflearn.max_pool_2d(x, 2, strides=2)
x = tflearn.conv_2d(x,
512,
3,
activation='relu',
weights_init=conv5_1,
bias_init=bias5_1,
scope='conv5_1')
x = tflearn.conv_2d(x,
512,
3,
activation='relu',
weights_init=conv5_2,
bias_init=bias5_2,
scope='conv5_2')
x = tflearn.conv_2d(x,
512,
3,
activation='relu',
weights_init=conv5_3,
bias_init=bias5_3,
scope='conv5_3')
x = tflearn.max_pool_2d(x, 2, strides=2)
'''kernel = tf.Variable(tf.truncated_normal([1,1,512,512],dtype=tf.float32,stddev=1e-1),name='attention_1_weights')
conv = tf.nn.conv2d(x, kernel,[1,1,1,1],padding='SAME')
f_conv1_attention = tf.nn.relu(conv)
kernel2 = tf.Variable(tf.truncated_normal([1,1,512,1],dtype=tf.float32,stddev=1e-1),name='attention_2_weights')
conv2 = tf.nn.conv2d(f_conv1_attention, kernel2,[1,1,1,1],padding='SAME')
attention_score = tf.nn.relu(conv2, name='score')
attention_feat = x * attention_score
#attention_feat = tf.expand_dims(tf.expand_dims(attention_feat, 1), 2)'''
#attention_c = tf.get_variable('attention_c_final', shape=[512],initializer=tf.random_normal_initializer())
#x = x * attention_c
#kernel3 = tf.Variable(tf.truncated_normal([1,1,512,num_class],dtype=tf.float32,
# stddev=1e-1),name='m2d_weights')
#conv3 = tf.nn.conv2d(x, kernel3,[1,1,1,1],padding='SAME')
#logits = tf.squeeze(conv3,[1,2])
#x = tf.nn.softmax(logits)
kernel = tf.get_variable(name='attention_conv3_weights',
initializer=tf.truncated_normal(
[1, 1, 512, 512],
dtype=tf.float32,
stddev=1e-1))
conv = tf.nn.conv2d(x, kernel, [1, 1, 1, 1], padding='SAME')
conv = tf.nn.relu(conv)
kernel2 = tf.get_variable(name='attention_conv4_weights',
initializer=tf.truncated_normal(
[1, 1, 512, 1],
dtype=tf.float32,
stddev=1e-1))
conv2 = tf.nn.conv2d(conv, kernel2, [1, 1, 1, 1], padding='SAME')
attention_score2 = tf.nn.relu(conv2)
x = x * attention_score2
c = tflearn.avg_pool_2d(x, 7)
c = tflearn.flatten(c)
c3 = tflearn.fully_connected(c, 32, activation='relu', scope='c3')
c4 = tflearn.fully_connected(c3, 512, activation='sigmoid', scope='c4')
c4 = tf.expand_dims(c4, 1)
c4 = tf.expand_dims(c4, 2)
x = x * c4
x = tflearn.fully_connected(x,
4096,
activation='relu',
weights_init=fc6_w,
bias_init=fc6_b,
scope='fc6')
x = tflearn.dropout(x, 0.5, name='dropout1')
x = tflearn.fully_connected(x,
4096,
activation='relu',
weights_init=fc7_w,
bias_init=fc7_b,
scope='fc7')
return x
def inception(input_layer, num_class):
conv1_7_7 = tflearn.conv_2d(input_layer,
64,
7,
strides=2,
activation='relu',
name='conv1_7_7_s2')
pool1_3_3 = tflearn.max_pool_2d(conv1_7_7, 3, strides=2)
pool1_3_3 = tflearn.local_response_normalization(pool1_3_3)
conv2_3_3_reduce = tflearn.conv_2d(pool1_3_3,
64,
1,
activation='relu',
name='conv2_3_3_reduce')
conv2_3_3 = tflearn.conv_2d(conv2_3_3_reduce,
192,
3,
activation='relu',
name='conv2_3_3')
conv2_3_3 = tflearn.local_response_normalization(conv2_3_3)
pool2_3_3 = tflearn.max_pool_2d(conv2_3_3,
kernel_size=3,
strides=2,
name='pool2_3_3_s2')
inception_3a_1_1 = tflearn.conv_2d(pool2_3_3,
64,
1,
activation='relu',
name='inception_3a_1_1')
inception_3a_3_3_reduce = tflearn.conv_2d(pool2_3_3,
96,
1,
activation='relu',
name='inception_3a_3_3_reduce')
inception_3a_3_3 = tflearn.conv_2d(inception_3a_3_3_reduce,
128,
filter_size=3,
activation='relu',
name='inception_3a_3_3')
inception_3a_5_5_reduce = tflearn.conv_2d(pool2_3_3,
16,
filter_size=1,
activation='relu',
name='inception_3a_5_5_reduce')
inception_3a_5_5 = tflearn.conv_2d(inception_3a_5_5_reduce,
32,
filter_size=5,
activation='relu',
name='inception_3a_5_5')
inception_3a_pool = tflearn.max_pool_2d(
pool2_3_3,
kernel_size=3,
strides=1,
)
inception_3a_pool_1_1 = tflearn.conv_2d(inception_3a_pool,
32,
filter_size=1,
activation='relu',
name='inception_3a_pool_1_1')
# merge the inception_3a__
inception_3a_output = tflearn.merge([
inception_3a_1_1, inception_3a_3_3, inception_3a_5_5,
inception_3a_pool_1_1
],
mode='concat',
axis=3)
inception_3b_1_1 = tflearn.conv_2d(inception_3a_output,
128,
filter_size=1,
activation='relu',
name='inception_3b_1_1')
inception_3b_3_3_reduce = tflearn.conv_2d(inception_3a_output,
128,
filter_size=1,
activation='relu',
name='inception_3b_3_3_reduce')
inception_3b_3_3 = tflearn.conv_2d(inception_3b_3_3_reduce,
192,
filter_size=3,
activation='relu',
name='inception_3b_3_3')
inception_3b_5_5_reduce = tflearn.conv_2d(inception_3a_output,
32,
filter_size=1,
activation='relu',
name='inception_3b_5_5_reduce')
inception_3b_5_5 = tflearn.conv_2d(inception_3b_5_5_reduce,
96,
filter_size=5,
name='inception_3b_5_5')
inception_3b_pool = tflearn.max_pool_2d(inception_3a_output,
kernel_size=3,
strides=1,
name='inception_3b_pool')
inception_3b_pool_1_1 = tflearn.conv_2d(inception_3b_pool,
64,
filter_size=1,
activation='relu',
name='inception_3b_pool_1_1')
# merge the inception_3b_*
inception_3b_output = tflearn.merge([
inception_3b_1_1, inception_3b_3_3, inception_3b_5_5,
inception_3b_pool_1_1
],
mode='concat',
axis=3,
name='inception_3b_output')
pool3_3_3 = tflearn.max_pool_2d(inception_3b_output,
kernel_size=3,
strides=2,
name='pool3_3_3')
inception_4a_1_1 = tflearn.conv_2d(pool3_3_3,
192,
filter_size=1,
activation='relu',
name='inception_4a_1_1')
inception_4a_3_3_reduce = tflearn.conv_2d(pool3_3_3,
96,
filter_size=1,
activation='relu',
name='inception_4a_3_3_reduce')
inception_4a_3_3 = tflearn.conv_2d(inception_4a_3_3_reduce,
208,
filter_size=3,
activation='relu',
name='inception_4a_3_3')
inception_4a_5_5_reduce = tflearn.conv_2d(pool3_3_3,
16,
filter_size=1,
activation='relu',
name='inception_4a_5_5_reduce')
inception_4a_5_5 = tflearn.conv_2d(inception_4a_5_5_reduce,
48,
filter_size=5,
activation='relu',
name='inception_4a_5_5')
inception_4a_pool = tflearn.max_pool_2d(pool3_3_3,
kernel_size=3,
strides=1,
name='inception_4a_pool')
inception_4a_pool_1_1 = tflearn.conv_2d(inception_4a_pool,
64,
filter_size=1,
activation='relu',
name='inception_4a_pool_1_1')
inception_4a_output = tflearn.merge([
inception_4a_1_1, inception_4a_3_3, inception_4a_5_5,
inception_4a_pool_1_1
],
mode='concat',
axis=3,
name='inception_4a_output')
inception_4b_1_1 = tflearn.conv_2d(inception_4a_output,
160,
filter_size=1,
activation='relu',
name='inception_4a_1_1')
inception_4b_3_3_reduce = tflearn.conv_2d(inception_4a_output,
112,
filter_size=1,
activation='relu',
name='inception_4b_3_3_reduce')
inception_4b_3_3 = tflearn.conv_2d(inception_4b_3_3_reduce,
224,
filter_size=3,
activation='relu',
name='inception_4b_3_3')
inception_4b_5_5_reduce = tflearn.conv_2d(inception_4a_output,
24,
filter_size=1,
activation='relu',
name='inception_4b_5_5_reduce')
inception_4b_5_5 = tflearn.conv_2d(inception_4b_5_5_reduce,
64,
filter_size=5,
activation='relu',
name='inception_4b_5_5')
inception_4b_pool = tflearn.max_pool_2d(inception_4a_output,
kernel_size=3,
strides=1,
name='inception_4b_pool')
inception_4b_pool_1_1 = tflearn.conv_2d(inception_4b_pool,
64,
filter_size=1,
activation='relu',
name='inception_4b_pool_1_1')
inception_4b_output = tflearn.merge([
inception_4b_1_1, inception_4b_3_3, inception_4b_5_5,
inception_4b_pool_1_1
],
mode='concat',
axis=3,
name='inception_4b_output')
inception_4c_1_1 = tflearn.conv_2d(inception_4b_output,
128,
filter_size=1,
activation='relu',
name='inception_4c_1_1')
inception_4c_3_3_reduce = tflearn.conv_2d(inception_4b_output,
128,
filter_size=1,
activation='relu',
name='inception_4c_3_3_reduce')
inception_4c_3_3 = tflearn.conv_2d(inception_4c_3_3_reduce,
256,
filter_size=3,
activation='relu',
name='inception_4c_3_3')
inception_4c_5_5_reduce = tflearn.conv_2d(inception_4b_output,
24,
filter_size=1,
activation='relu',
name='inception_4c_5_5_reduce')
inception_4c_5_5 = tflearn.conv_2d(inception_4c_5_5_reduce,
64,
filter_size=5,
activation='relu',
name='inception_4c_5_5')
inception_4c_pool = tflearn.max_pool_2d(inception_4b_output,
kernel_size=3,
strides=1)
inception_4c_pool_1_1 = tflearn.conv_2d(inception_4c_pool,
64,
filter_size=1,
activation='relu',
name='inception_4c_pool_1_1')
inception_4c_output = tflearn.merge([
inception_4c_1_1, inception_4c_3_3, inception_4c_5_5,
inception_4c_pool_1_1
],
mode='concat',
axis=3,
name='inception_4c_output')
inception_4d_1_1 = tflearn.conv_2d(inception_4c_output,
112,
filter_size=1,
activation='relu',
name='inception_4d_1_1')
inception_4d_3_3_reduce = tflearn.conv_2d(inception_4c_output,
144,
filter_size=1,
activation='relu',
name='inception_4d_3_3_reduce')
inception_4d_3_3 = tflearn.conv_2d(inception_4d_3_3_reduce,
288,
filter_size=3,
activation='relu',
name='inception_4d_3_3')
inception_4d_5_5_reduce = tflearn.conv_2d(inception_4c_output,
32,
filter_size=1,
activation='relu',
name='inception_4d_5_5_reduce')
inception_4d_5_5 = tflearn.conv_2d(inception_4d_5_5_reduce,
64,
filter_size=5,
activation='relu',
name='inception_4d_5_5')
inception_4d_pool = tflearn.max_pool_2d(inception_4c_output,
kernel_size=3,
strides=1,
name='inception_4d_pool')
inception_4d_pool_1_1 = tflearn.conv_2d(inception_4d_pool,
64,
filter_size=1,
activation='relu',
name='inception_4d_pool_1_1')
inception_4d_output = tflearn.merge([
inception_4d_1_1, inception_4d_3_3, inception_4d_5_5,
inception_4d_pool_1_1
],
mode='concat',
axis=3,
name='inception_4d_output')
inception_4e_1_1 = tflearn.conv_2d(inception_4d_output,
256,
filter_size=1,
activation='relu',
name='inception_4e_1_1')
inception_4e_3_3_reduce = tflearn.conv_2d(inception_4d_output,
160,
filter_size=1,
activation='relu',
name='inception_4e_3_3_reduce')
inception_4e_3_3 = tflearn.conv_2d(inception_4e_3_3_reduce,
320,
filter_size=3,
activation='relu',
name='inception_4e_3_3')
inception_4e_5_5_reduce = tflearn.conv_2d(inception_4d_output,
32,
filter_size=1,
activation='relu',
name='inception_4e_5_5_reduce')
inception_4e_5_5 = tflearn.conv_2d(inception_4e_5_5_reduce,
128,
filter_size=5,
activation='relu',
name='inception_4e_5_5')
inception_4e_pool = tflearn.max_pool_2d(inception_4d_output,
kernel_size=3,
strides=1,
name='inception_4e_pool')
inception_4e_pool_1_1 = tflearn.conv_2d(inception_4e_pool,
128,
filter_size=1,
activation='relu',
name='inception_4e_pool_1_1')
inception_4e_output = tflearn.merge([
inception_4e_1_1, inception_4e_3_3, inception_4e_5_5,
inception_4e_pool_1_1
],
axis=3,
mode='concat')
pool4_3_3 = tflearn.max_pool_2d(inception_4e_output,
kernel_size=3,
strides=2,
name='pool_3_3')
inception_5a_1_1 = tflearn.conv_2d(pool4_3_3,
256,
filter_size=1,
activation='relu',
name='inception_5a_1_1')
inception_5a_3_3_reduce = tflearn.conv_2d(pool4_3_3,
160,
filter_size=1,
activation='relu',
name='inception_5a_3_3_reduce')
inception_5a_3_3 = tflearn.conv_2d(inception_5a_3_3_reduce,
320,
filter_size=3,
activation='relu',
name='inception_5a_3_3')
inception_5a_5_5_reduce = tflearn.conv_2d(pool4_3_3,
32,
filter_size=1,
activation='relu',
name='inception_5a_5_5_reduce')
inception_5a_5_5 = tflearn.conv_2d(inception_5a_5_5_reduce,
128,
filter_size=5,
activation='relu',
name='inception_5a_5_5')
inception_5a_pool = tflearn.max_pool_2d(pool4_3_3,
kernel_size=3,
strides=1,
name='inception_5a_pool')
inception_5a_pool_1_1 = tflearn.conv_2d(inception_5a_pool,
128,
filter_size=1,
activation='relu',
name='inception_5a_pool_1_1')
inception_5a_output_ = tflearn.merge([
inception_5a_1_1, inception_5a_3_3, inception_5a_5_5,
inception_5a_pool_1_1
],
axis=3,
mode='concat')
inception_5a_output = tflearn.dropout(inception_5a_output_, 0.45)
inception_5b_1_1 = tflearn.conv_2d(inception_5a_output,
384,
filter_size=1,
activation='relu',
name='inception_5b_1_1')
inception_5b_3_3_reduce = tflearn.conv_2d(inception_5a_output,
192,
filter_size=1,
activation='relu',
name='inception_5b_3_3_reduce')
inception_5b_3_3 = tflearn.conv_2d(inception_5b_3_3_reduce,
384,
filter_size=3,
activation='relu',
name='inception_5b_3_3')
inception_5b_5_5_reduce = tflearn.conv_2d(inception_5a_output,
48,
filter_size=1,
activation='relu',
name='inception_5b_5_5_reduce')
inception_5b_5_5 = tflearn.conv_2d(inception_5b_5_5_reduce,
128,
filter_size=5,
activation='relu',
name='inception_5b_5_5')
inception_5b_pool = tflearn.max_pool_2d(inception_5a_output,
kernel_size=3,
strides=1,
name='inception_5b_pool')
inception_5b_pool_1_1 = tflearn.conv_2d(inception_5b_pool,
128,
filter_size=1,
activation='relu',
name='inception_5b_pool_1_1')
inception_5b_output = tflearn.merge([
inception_5b_1_1, inception_5b_3_3, inception_5b_5_5,
inception_5b_pool_1_1
],
axis=3,
mode='concat')
pool5_7_7 = tflearn.avg_pool_2d(inception_5b_output,
kernel_size=7,
strides=1)
pool5_7_7 = tflearn.dropout(pool5_7_7, 0.65)
loss = tflearn.fully_connected(pool5_7_7, num_class, activation='softmax')
return loss
def densenet_block(self,incoming, nb_layers, growth, bottleneck=True,
downsample=True, downsample_strides=2, activation='relu',
batch_norm=True, dropout=False, dropout_keep_prob=0.5,
weights_init='variance_scaling', regularizer='L2',
weight_decay=0.0001, bias=True, bias_init='zeros',
trainable=True, restore=True, reuse=False, scope=None,
name="DenseNetBlock"):
""" DenseNet Block.
A DenseNet block as described in DenseNet paper.
Input:
4-D Tensor [batch, height, width, in_channels].
Output:
4-D Tensor [batch, new height, new width, out_channels].
Arguments:
incoming: `Tensor`. Incoming 4-D Layer.
nb_blocks: `int`. Number of layer blocks.
growth: `int`. DenseNet 'growth': The number of convolutional
filters of each convolution.
bottleneck: `bool`. If True, add a 1x1 convolution before the 3x3
convolution to reduce the number of input features map.
downsample: `bool`. If True, apply downsampling using
'downsample_strides' for strides.
downsample_strides: `int`. The strides to use when downsampling.
activation: `str` (name) or `function` (returning a `Tensor`).
Activation applied to this layer (see tflearn.activations).
Default: 'linear'.
batch_norm: `bool`. If True, apply batch normalization.
dropout: `bool`. If True, apply dropout. Use 'dropout_keep_prob' to
specify the keep probability.
dropout_keep_prob: `float`. Keep probability parameter for dropout.
bias: `bool`. If True, a bias is used.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(see tflearn.initializations) Default: 'uniform_scaling'.
bias_init: `str` (name) or `tf.Tensor`. Bias initialization.
(see tflearn.initializations) Default: 'zeros'.
regularizer: `str` (name) or `Tensor`. Add a regularizer to this
layer weights (see tflearn.regularizers). Default: None.
weight_decay: `float`. Regularizer decay parameter. Default: 0.001.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
reuse: `bool`. If True and 'scope' is provided, this layer variables
will be reused (shared).
scope: `str`. Define this layer scope (optional). A scope can be
used to share variables between layers. Note that scope will
override name.
name: A name for this layer (optional). Default: 'ResNeXtBlock'.
References:
Densely Connected Convolutional Networks, G. Huang, Z. Liu,
K. Q. Weinberger, L. van der Maaten. 2016.
Links:
[https://arxiv.org/abs/1608.06993]
(https://arxiv.org/abs/1608.06993)
"""
densenet = incoming
with tf.variable_scope(scope, default_name=name, values=[incoming],
reuse=reuse) as scope:
for i in range(nb_layers):
# Identity
conn = densenet
# 1x1 Conv layer of the bottleneck block
if bottleneck:
if batch_norm:
densenet = tflearn.batch_normalization(densenet)
densenet = tflearn.activation(densenet, activation)
densenet = conv_2d(densenet, nb_filter=growth,
filter_size=1,
bias=bias,
weights_init=weights_init,
bias_init=bias_init,
regularizer=regularizer,
weight_decay=weight_decay,
trainable=trainable,
restore=restore)
# 3x3 Conv layer
if batch_norm:
densenet = tflearn.batch_normalization(densenet)
densenet = tflearn.activation(densenet, activation)
densenet = conv_2d(densenet, nb_filter=growth,
filter_size=3,
bias=bias,
weights_init=weights_init,
bias_init=bias_init,
regularizer=regularizer,
weight_decay=weight_decay,
trainable=trainable,
restore=restore)
# Connections
densenet = tf.concat([densenet, conn], 3)
# 1x1 Transition Conv
if batch_norm:
densenet = tflearn.batch_normalization(densenet)
densenet = tflearn.activation(densenet, activation)
densenet = conv_2d(densenet, nb_filter=growth,
filter_size=1,
bias=bias,
weights_init=weights_init,
bias_init=bias_init,
regularizer=regularizer,
weight_decay=weight_decay,
trainable=trainable,
restore=restore)
if dropout:
densenet = tflearn.dropout(densenet, keep_prob=dropout_keep_prob)
# Downsampling
if downsample:
densenet = tflearn.avg_pool_2d(densenet, kernel_size=2,
strides=downsample_strides)
return densenet
import tflearn
from tensorflow.examples.tutorials.mnist import input_data
net = tflearn.input_data([None, 28,28,1])
net = tflearn.conv_2d(net, nb_filter=6, filter_size=5,
strides=1, padding='SAME', activation='sigmoid')
net = tflearn.avg_pool_2d(net, kernel_size=2, strides=2, padding='SAME')
net = tflearn.conv_2d(net, nb_filter=16, filter_size=5,
strides=1, padding='VALID', activation='sigmoid')
net = tflearn.avg_pool_2d(net, kernel_size=2, strides=2, padding='SAME')
net = tflearn.fully_connected(net, n_units=120, activation='sigmoid')
net = tflearn.fully_connected(net, n_units=84, activation='sigmoid')
net = tflearn.fully_connected(net, n_units=10, activation='softmax')
net = tflearn.regression(net, learning_rate=0.001, optimizer='rmsprop', loss='categorical_crossentropy')
mnist = input_data.read_data_sets('mnist', one_hot=True)
x = mnist.train.images.reshape(-1, 28, 28, 1)
y = mnist.train.labels
model = tflearn.DNN(net)
model.fit(x, y, n_epoch=10, batch_size=128, shuffle=True, validation_set=0.2)
x = mnist.test.images.reshape(-1, 28, 28, 1)
y = mnist.test.labels
acc = model.evaluate(x, y)
def residual_shrinkage_block(incoming,
nb_blocks,
out_channels,
downsample=False,
downsample_strides=2,
activation='relu',
batch_norm=True,
bias=True,
weights_init='variance_scaling',
bias_init='zeros',
regularizer='L2',
weight_decay=0.0001,
trainable=True,
restore=True,
reuse=False,
scope=None,
name="ResidualBlock"):
# residual shrinkage blocks with channel-wise thresholds
residual = incoming
in_channels = incoming.get_shape().as_list()[-1]
# Variable Scope fix for older TF
try:
vscope = tf.variable_scope(scope,
default_name=name,
values=[incoming],
reuse=reuse)
except Exception:
vscope = tf.variable_op_scope([incoming], scope, name, reuse=reuse)
with vscope as scope:
name = scope.name #TODO
for i in range(nb_blocks):
identity = residual
if not downsample:
downsample_strides = 1
if batch_norm:
residual = tflearn.batch_normalization(residual)
residual = tflearn.activation(residual, activation)
residual = conv_2d(residual, out_channels, 3, downsample_strides,
'same', 'linear', bias, weights_init, bias_init,
regularizer, weight_decay, trainable, restore)
if batch_norm:
residual = tflearn.batch_normalization(residual)
residual = tflearn.activation(residual, activation)
residual = conv_2d(residual, out_channels, 3, 1, 'same', 'linear',
bias, weights_init, bias_init, regularizer,
weight_decay, trainable, restore)
# get thresholds and apply thresholding
abs_mean = tf.reduce_mean(tf.reduce_mean(tf.abs(residual),
axis=2,
keep_dims=True),
axis=1,
keep_dims=True)
scales = tflearn.fully_connected(abs_mean,
out_channels // 4,
activation='linear',
regularizer='L2',
weight_decay=0.0001,
weights_init='variance_scaling')
scales = tflearn.batch_normalization(scales)
scales = tflearn.activation(scales, 'relu')
scales = tflearn.fully_connected(scales,
out_channels,
activation='linear',
regularizer='L2',
weight_decay=0.0001,
weights_init='variance_scaling')
scales = tf.expand_dims(tf.expand_dims(scales, axis=1), axis=1)
thres = tf.multiply(abs_mean, tflearn.activations.sigmoid(scales))
# soft thresholding
residual = tf.multiply(tf.sign(residual),
tf.maximum(tf.abs(residual) - thres, 0))
# Downsampling
if downsample_strides > 1:
identity = tflearn.avg_pool_2d(identity, 1, downsample_strides)
# Projection to new dimension
if in_channels != out_channels:
if (out_channels - in_channels) % 2 == 0:
ch = (out_channels - in_channels) // 2
identity = tf.pad(identity,
[[0, 0], [0, 0], [0, 0], [ch, ch]])
else:
ch = (out_channels - in_channels) // 2
identity = tf.pad(identity,
[[0, 0], [0, 0], [0, 0], [ch, ch + 1]])
in_channels = out_channels
residual = residual + identity
return residual
addt = tf.add(resn_ud, coct, name='addt')
ups = tf.image.resize_nearest_neighbor(addt, (ih, iw), name='ups')
return ups
lr = 0.0001
tf.reset_default_graph()
net_input = input_data(shape=[None, parth, partw, n_channels],
name='net_input')
net_input_h = net_input.get_shape()[1]
net_input_w = net_input.get_shape()[2]
resn1 = resn_unit(net_input, 4, 64, name='resn1')
resn1_ds = avg_pool_2d(resn1, 2, name='resn1_ds')
resn1_ds_h = resn1_ds.get_shape()[1]
resn1_ds_w = resn1_ds.get_shape()[2]
resn2 = resn_unit(resn1_ds, 4, 128, name='resn2')
resn2_ds = avg_pool_2d(resn2, 2, name='resn2_ds')
resn2_ds_h = resn2_ds.get_shape()[1]
resn2_ds_w = resn2_ds.get_shape()[2]
resn3 = resn_unit(resn2_ds, 4, 256, name='resn3')
resn3_ds = avg_pool_2d(resn3, 2, name='resn3_ds')
resn_mid = resn_unit(resn3_ds, 4, 512, name='dens_mid')
resn_upadd1 = resn_ups_concat(resn_mid,
resn3_ds,
4,
256,
Y = du.to_categorical(Y, 10)
testY = du.to_categorical(testY, 10)
# Building Residual Network
net = tflearn.input_data(shape=[None, 32, 32, 3])
net = tflearn.conv_2d(net, 32, 3)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.shallow_residual_block(net, 4, 32, regularizer='L2')
net = tflearn.shallow_residual_block(net, 1, 32, downsample=True,
regularizer='L2')
net = tflearn.shallow_residual_block(net, 4, 64, regularizer='L2')
net = tflearn.shallow_residual_block(net, 1, 64, downsample=True,
regularizer='L2')
net = tflearn.shallow_residual_block(net, 5, 128, regularizer='L2')
net_shape = net.get_shape().as_list()
k_size = [1, net_shape[1], net_shape[2], 1]
net = tflearn.avg_pool_2d(net, k_size, strides=1)
# Regression
net = tflearn.fully_connected(net, 10, activation='softmax')
mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=16000, staircase=True)
net = tflearn.regression(net, optimizer=mom,
loss='categorical_crossentropy')
# Training
model = tflearn.DNN(net, checkpoint_path='model_resnet_cifar10',
max_checkpoints=10, tensorboard_verbose=0,
clip_gradients=1.0)
model.fit(X, Y, n_epoch=200, validation_set=(testX, testY),
show_metric=True, batch_size=128, shuffle=True,
run_id='resnet_cifar10')
X, mean = du.featurewise_zero_center(X)
testX = du.featurewise_zero_center(testX, mean)
# Building Residual Network
net = tflearn.input_data(shape=[None, 28, 28, 1])
net = tflearn.conv_2d(net, 64, 3, activation='relu', bias=False)
net = tflearn.batch_normalization(net)
# Residual blocks
net = tflearn.deep_residual_block(net, 3, 64)
net = tflearn.deep_residual_block(net, 1, 128, downsample=True)
net = tflearn.deep_residual_block(net, 3, 128)
net = tflearn.deep_residual_block(net, 1, 256, downsample=True)
net = tflearn.deep_residual_block(net, 3, 256)
net_shape = net.get_shape().as_list()
k_size = [1, net_shape[1], net_shape[2], 1]
net = tflearn.avg_pool_2d(net, k_size, padding='valid', strides=1)
# Regression
net = tflearn.fully_connected(net, 10, activation='softmax')
sgd = tflearn.SGD(learning_rate=0.1, lr_decay=0.96, decay_step=300)
net = tflearn.regression(net,
optimizer='sgd',
loss='categorical_crossentropy',
learning_rate=0.1)
# Training
model = tflearn.DNN(net,
checkpoint_path='model_resnet_mnist',
max_checkpoints=10,
tensorboard_verbose=0)
model.fit(X,
Y,
n_epoch=100,
def densenet_block(incoming,
nb_layers,
growth,
bottleneck=True,
downsample=True,
downsample_strides=2,
activation='relu',
batch_norm=True,
dropout=False,
dropout_keep_prob=0.5,
weights_init='variance_scaling',
regularizer='L2',
weight_decay=0.0001,
bias=True,
bias_init='zeros',
trainable=True,
restore=True,
reuse=False,
scope=None,
name="DenseNetBlock"):
densenet = incoming
with tf.variable_scope(scope,
default_name=name,
values=[incoming],
reuse=reuse) as scope:
for i in range(nb_layers):
# Identity
conn = densenet
# 1x1 Conv layer of the bottleneck block
if bottleneck:
if batch_norm:
densenet = tflearn.batch_normalization(densenet)
densenet = tflearn.activation(densenet, activation)
densenet = conv_2d(densenet,
nb_filter=growth,
filter_size=1,
bias=bias,
weights_init=weights_init,
bias_init=bias_init,
regularizer=regularizer,
weight_decay=weight_decay,
trainable=trainable,
restore=restore)
# 3x3 Conv layer
if batch_norm:
densenet = tflearn.batch_normalization(densenet)
densenet = tflearn.activation(densenet, activation)
densenet = conv_2d(densenet,
nb_filter=growth,
filter_size=3,
bias=bias,
weights_init=weights_init,
bias_init=bias_init,
regularizer=regularizer,
weight_decay=weight_decay,
trainable=trainable,
restore=restore)
# Connections
densenet = tf.concat([densenet, conn], 3)
# 1x1 Transition Conv
if batch_norm:
densenet = tflearn.batch_normalization(densenet)
densenet = tflearn.activation(densenet, activation)
densenet = conv_2d(densenet,
nb_filter=growth,
filter_size=1,
bias=bias,
weights_init=weights_init,
bias_init=bias_init,
regularizer=regularizer,
weight_decay=weight_decay,
trainable=trainable,
restore=restore)
if dropout:
densenet = tflearn.dropout(densenet, keep_prob=dropout_keep_prob)
# Downsampling
if downsample:
densenet = tflearn.avg_pool_2d(densenet,
kernel_size=2,
strides=downsample_strides)
return densenet
conv2 = tflearn.conv_2d(l1, 64, 1, activation='relu')
conv3 = tflearn.conv_2d(conv2, 192, 3, activation='relu')
l2 = tflearn.local_response_normalization(conv3)
m2 = tflearn.max_pool_2d(l2, kernel_size=3, strides=2)
i1 = inception(m2, 96, 16, 64, 128, 32, 32)
i2 = inception(i1, 128, 32, 128, 192, 96, 64)
m3 = tflearn.max_pool_2d(i2, kernel_size=3, strides=2)
i3 = inception(m3, 96, 16, 192, 208, 48, 64)
i4 = inception(i3, 112, 24, 160, 224, 64, 64)
i5 = inception(i4, 128, 24, 128, 256, 64, 64)
i6 = inception(i5, 144, 32, 112, 288, 64, 64)
i7 = inception(i6, 160, 32, 256, 320, 128, 128)
m4 = tflearn.max_pool_2d(i7, kernel_size=3, strides=2)
i8 = inception(m4, 160, 32, 256, 320, 128, 128)
i9 = inception(i8, 192, 48, 384, 384, 128, 128)
a1 = tflearn.avg_pool_2d(i9, kernel_size=7, strides=1)
d1 = tflearn.dropout(a1, 0.4)
f1 = tflearn.fully_connected(d1, 6, activation='softmax')
network = tflearn.regression(f1,
optimizer='momentum',
loss='categorical_crossentropy',
learning_rate=0.001)
# 학습
model = tflearn.DNN(network,
checkpoint_path='model_googlenet',
max_checkpoints=1,
tensorboard_verbose=3)
model.load("model_googlenet-24000")
3,
5,
activation='leaky_relu',
name="small_CONV")
network = tflearn.conv_2d(network,
3,
8,
activation='leaky_relu',
name="large_CONV")
network = tflearn.max_pool_2d(network, 2, name="max_POOL")
network = tflearn.conv_2d(network,
1,
5,
activation='leaky_relu',
name="final_CONV")
network = tflearn.avg_pool_2d(
network, 5, name="avg_POOL") # want avg here, to reduce outliers.
network = tflearn.reshape(network, (-1, 12, 28), name="final")
#no FULLY CONNECTED!
network = tflearn.regression(network,
optimizer='adam',
learning_rate=0.01,
loss='mean_square',
name='target')
model = tflearn.DNN(network,
tensorboard_verbose=3,
max_checkpoints=3,
tensorboard_dir="./logs")
print("Loading Model")
model.load("current.model")
for x in range(1000):
def resnext_block(incoming,
nb_blocks,
out_channels,
cardinality,
downsample=True,
regularizer='L2',
weight_decay=0.0001,
restore=True,
reuse=tf.AUTO_REUSE,
scope=None,
name="ResNeXtBlock"):
in_channels = incoming.get_shape().as_list()[-1]
for i in range(nb_blocks):
with tf.variable_scope((scope or "") + "_" + str(i),
default_name=name,
reuse=tf.AUTO_REUSE):
if i == 0 and downsample:
strides = 2
identity = tflearn.avg_pool_2d(incoming, 2, 2, padding='valid')
else:
strides = 1
identity = incoming
incoming = tflearn.conv_2d(incoming,
out_channels / 2,
1,
name='Conv2D_1',
bias=False,
regularizer='L2',
weight_decay=0.0001)
incoming = tflearn.batch_normalization(incoming, name='BN_1')
incoming = tf.nn.relu(incoming)
input_groups = tf.split(value=incoming,
num_or_size_splits=cardinality,
axis=3)
output_groups = [
split(inp, strides=strides, order=str(k))
for k, inp in enumerate(input_groups)
]
incoming = tf.concat(output_groups, axis=3)
incoming = tflearn.conv_2d(incoming,
out_channels,
1,
name='Conv2D_2',
bias=False,
regularizer='L2',
weight_decay=0.0001)
incoming = tflearn.batch_normalization(incoming, name='BN_2')
if in_channels != out_channels:
ch = (out_channels - in_channels) // 2
identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch]])
in_channels = out_channels
incoming = incoming + identity
incoming = tf.nn.relu(incoming)
return incoming
