def ModelVGGLike(X,is_training):
conv1 = ConvBNRelu(X, 3, 256, is_training)
conv1 = ConvBNRelu(conv1, 3, 256, is_training)
conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
conv2 = ConvBNRelu(conv1, 3, 128, is_training)
conv2 = ConvBNRelu(conv2, 3, 128, is_training)
conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
conv3 = ConvBNRelu(conv2, 3, 128, is_training)
conv3 = ConvBNRelu(conv3, 3, 128, is_training)
conv3 = tf.nn.max_pool(conv3, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
conv4 = ConvBNRelu(conv3, 3, 128, is_training)
conv4 = ConvBNRelu(conv4, 3, 128, is_training)
conv4 = tf.nn.max_pool(conv4, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
# print conv3
fc = FCRelu(conv4, 1024)
fc = tf.nn.dropout(fc, 0.5)
return linear(fc, classes)
def ModelVGGLike(X, is_training):
conv1 = ConvBNRelu(X, 3, 256, is_training)
conv1 = ConvBNRelu(conv1, 3, 256, is_training)
conv1 = tf.nn.max_pool(conv1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
conv2 = ConvBNRelu(conv1, 3, 128, is_training)
conv2 = ConvBNRelu(conv2, 3, 128, is_training)
conv2 = tf.nn.max_pool(conv2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
conv3 = ConvBNRelu(conv2, 3, 128, is_training)
conv3 = ConvBNRelu(conv3, 3, 128, is_training)
conv3 = tf.nn.max_pool(conv3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
conv4 = ConvBNRelu(conv3, 3, 128, is_training)
conv4 = ConvBNRelu(conv4, 3, 128, is_training)
conv4 = tf.nn.max_pool(conv4,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# print conv3
fc = FCRelu(conv4, 1024)
fc = tf.nn.dropout(fc, 0.5)
return linear(fc, classes)
def ModelSimple(X, is_training):
h_1 = lrelu(batch_norm(conv2d(X, 32, name='conv1'),
is_training, scope='bn1'), name='lrelu1')
h_2 = lrelu(batch_norm(conv2d(h_1, 64, name='conv2'),
is_training, scope='bn2'), name='lrelu2')
h_3 = lrelu(batch_norm(conv2d(h_2, 64, name='conv3'),
is_training, scope='bn3'), name='lrelu3')
h_3_flat = tf.reshape(h_3, [-1, 64 * 4 * 4])
return linear(h_3_flat, 10)
def ModelSimple(X, is_training):
h_1 = lrelu(batch_norm(conv2d(X, 32, name='conv1'),
is_training, scope='bn1'), name='lrelu1')
h_2 = lrelu(batch_norm(conv2d(h_1, 64, name='conv2'),
is_training, scope='bn2'), name='lrelu2')
h_3 = lrelu(batch_norm(conv2d(h_2, 64, name='conv3'),
is_training, scope='bn3'), name='lrelu3')
h_3_flat = tf.reshape(h_3, [-1, 64 * 4 * 4])
return linear(h_3_flat, 10)
def ModelVGGLikeSmall(X,is_training):
conv1 = ConvBNRelu(X, 3, 256, is_training)
conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
conv2 = ConvBNRelu(conv1, 3, 128, is_training)
conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
fc = FCRelu(conv2, 512)
fc = tf.nn.dropout(fc, 0.5)
return linear(fc, classes)
def ModelVGGLikeSmall(X,is_training):
conv1 = ConvBNRelu(X, 3, 32, is_training)
conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
conv2 = ConvBNRelu(conv1, 3, 32, is_training)
conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
fc = FCRelu(conv2, 256)
fc = tf.nn.dropout(fc, 0.5)
return linear(fc, classes)
def residual_network(x, n_outputs, activation=tf.nn.relu):
"""Builds a residual network.
Parameters
----------
x : Placeholder
Input to the network
n_outputs : TYPE
Number of outputs of final softmax
activation : Attribute, optional
Nonlinearity to apply after each convolution
Returns
-------
net : Tensor
Description
Raises
------
ValueError
If a 2D Tensor is input, the Tensor must be square or else
the network can't be converted to a 4D Tensor.
"""
# %%
LayerBlock = namedtuple("LayerBlock", ["num_layers", "num_filters", "bottleneck_size"])
blocks = [LayerBlock(3, 128, 32), LayerBlock(3, 256, 64), LayerBlock(3, 512, 128), LayerBlock(3, 1024, 256)]
# %%
input_shape = x.get_shape().as_list()
if len(input_shape) == 2:
ndim = int(sqrt(input_shape[1]))
if ndim * ndim != input_shape[1]:
raise ValueError("input_shape should be square")
x = tf.reshape(x, [-1, ndim, ndim, 1])
# %%
# First convolution expands to 64 channels and downsamples
net = conv2d(x, 64, k_h=7, k_w=7, batch_norm=True, name="conv1", activation=activation)
# %%
# Max pool and downsampling
net = tf.nn.max_pool(net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding="SAME")
# %%
# Setup first chain of resnets
net = conv2d(net, blocks[0].num_filters, k_h=1, k_w=1, stride_h=1, stride_w=1, padding="VALID", name="conv2")
# %%
# Loop through all res blocks
for block_i, block in enumerate(blocks):
for layer_i in range(block.num_layers):
name = "block_%d/layer_%d" % (block_i, layer_i)
conv = conv2d(
net,
block.num_filters,
k_h=1,
k_w=1,
padding="VALID",
stride_h=1,
stride_w=1,
activation=activation,
batch_norm=True,
name=name + "/conv_in",
)
conv = conv2d(
conv,
block.bottleneck_size,
k_h=3,
k_w=3,
padding="SAME",
stride_h=1,
stride_w=1,
activation=activation,
batch_norm=True,
name=name + "/conv_bottleneck",
)
conv = conv2d(
conv,
block.num_filters,
k_h=1,
k_w=1,
padding="VALID",
stride_h=1,
stride_w=1,
activation=activation,
batch_norm=True,
name=name + "/conv_out",
)
net = conv + net
try:
# upscale to the next block size
next_block = blocks[block_i + 1]
net = conv2d(
net,
next_block.num_filters,
k_h=1,
k_w=1,
padding="SAME",
stride_h=1,
stride_w=1,
bias=False,
name="block_%d/conv_upscale" % block_i,
)
except IndexError:
pass
# %%
net = tf.reshape(tf.reduce_mean(net, 3), [-1, net.get_shape().as_list()[1] * net.get_shape().as_list()[2]])
net = linear(net, n_outputs, activation=tf.nn.softmax)
# %%
return net
def residual_network(input_shape, n_outputs,
activation=tf.nn.relu, debug=False):
"""Builds a residual network.
Parameters
----------
input_shape : list
Input dimensions of tensor
n_outputs : TYPE
Number of outputs of final softmax
activation : Attribute, optional
Nonlinearity to apply after each convolution
Returns
-------
name : TYPE
Description
"""
# %%
LayerBlock = namedtuple(
'LayerBlock', ['num_layers', 'num_filters', 'bottleneck_size'])
blocks = [LayerBlock(3, 128, 32),
LayerBlock(3, 256, 64),
LayerBlock(3, 512, 128),
LayerBlock(3, 1024, 256)]
# %%
x = tf.placeholder(tf.float32, input_shape, 'x')
# %%
# First convolution expands to 64 channels and downsamples
net = conv2d(x, 64, k_h=7, k_w=7,
batch_norm=True, name='conv1',
activation=activation)
# %%
# Max pool and downsampling
net = tf.nn.max_pool(
net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# %%
# Setup first chain of resnets
net = conv2d(net, blocks[0].num_filters, k_h=1, k_w=1,
stride_h=1, stride_w=1, padding='VALID', name='conv2')
# %%
# Loop through all res blocks
for block_i, block in enumerate(blocks):
for layer_i in range(block.num_layers):
name = 'block_%d/layer_%d' % (block_i, layer_i)
conv = conv2d(net, block.num_filters, k_h=1, k_w=1,
padding='VALID', stride_h=1, stride_w=1,
activation=activation, batch_norm=True,
name=name + '/conv_in')
conv = conv2d(conv, block.bottleneck_size, k_h=3, k_w=3,
padding='SAME', stride_h=1, stride_w=1,
activation=activation, batch_norm=True,
name=name + '/conv_bottleneck')
conv = conv2d(conv, block.num_filters, k_h=1, k_w=1,
padding='VALID', stride_h=1, stride_w=1,
activation=activation, batch_norm=True,
name=name + '/conv_out')
net = conv + net
try:
# upscale to the next block size
next_block = blocks[block_i + 1]
net = conv2d(net, next_block.num_filters, k_h=1, k_w=1,
padding='SAME', stride_h=1, stride_w=1, bias=False,
name='block_%d/conv_upscale' % block_i)
except IndexError:
pass
# %%
net = tf.reshape(
tf.reduce_mean(net, 3),
[-1, net.get_shape().as_list()[1] * net.get_shape().as_list()[2]])
net = linear(net, n_outputs, activation=tf.nn.softmax)
# %%
return net
def residual_network(x, n_outputs, activation=tf.nn.relu):
"""Builds a residual network.
Parameters
----------
x : Placeholder
Input to the network
n_outputs : TYPE
Number of outputs of final softmax
activation : Attribute, optional
Nonlinearity to apply after each convolution
Returns
-------
net : Tensor
Description
Raises
------
ValueError
If a 2D Tensor is input, the Tensor must be square or else
the network can't be converted to a 4D Tensor.
"""
# %%
LayerBlock = namedtuple('LayerBlock',
['num_layers', 'num_filters', 'bottleneck_size'])
blocks = [
LayerBlock(3, 128, 32),
LayerBlock(3, 256, 64),
LayerBlock(3, 512, 128),
LayerBlock(3, 1024, 256)
]
# %%
input_shape = x.get_shape().as_list()
if len(input_shape) == 2:
ndim = int(sqrt(input_shape[1]))
if ndim * ndim != input_shape[1]:
raise ValueError('input_shape should be square')
x = tf.reshape(x, [-1, ndim, ndim, 1])
# %%
# First convolution expands to 64 channels and downsamples
net = conv2d(x,
64,
k_h=7,
k_w=7,
batch_norm=True,
name='conv1',
activation=activation)
# %%
# Max pool and downsampling
net = tf.nn.max_pool(net, [1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# %%
# Setup first chain of resnets
net = conv2d(net,
blocks[0].num_filters,
k_h=1,
k_w=1,
stride_h=1,
stride_w=1,
padding='VALID',
name='conv2')
# %%
# Loop through all res blocks
for block_i, block in enumerate(blocks):
for layer_i in range(block.num_layers):
name = 'block_%d/layer_%d' % (block_i, layer_i)
conv = conv2d(net,
block.num_filters,
k_h=1,
k_w=1,
padding='VALID',
stride_h=1,
stride_w=1,
activation=activation,
batch_norm=True,
name=name + '/conv_in')
conv = conv2d(conv,
block.bottleneck_size,
k_h=3,
k_w=3,
padding='SAME',
stride_h=1,
stride_w=1,
activation=activation,
batch_norm=True,
name=name + '/conv_bottleneck')
conv = conv2d(conv,
block.num_filters,
k_h=1,
k_w=1,
padding='VALID',
stride_h=1,
stride_w=1,
activation=activation,
batch_norm=True,
name=name + '/conv_out')
net = conv + net
try:
# upscale to the next block size
next_block = blocks[block_i + 1]
net = conv2d(net,
next_block.num_filters,
k_h=1,
k_w=1,
padding='SAME',
stride_h=1,
stride_w=1,
bias=False,
name='block_%d/conv_upscale' % block_i)
except IndexError:
pass
# %%
net = tf.reshape(
tf.reduce_mean(net, 3),
[-1, net.get_shape().as_list()[1] * net.get_shape().as_list()[2]])
net = linear(net, n_outputs, activation=tf.nn.softmax)
# %%
return net
# %% Setup input to the network and true output label. These are # simply placeholders which we'll fill in later. mnist = MNIST() x = tf.placeholder(tf.float32, [None, 784]) y = tf.placeholder(tf.float32, [None, 10]) x_tensor = tf.reshape(x, [-1, 28, 28, 1]) # %% Define the network: bn1 = batch_norm(-1, name='bn1') bn2 = batch_norm(-1, name='bn2') bn3 = batch_norm(-1, name='bn3') h_1 = lrelu(bn1(conv2d(x_tensor, 32, name='conv1')), name='lrelu1') h_2 = lrelu(bn2(conv2d(h_1, 64, name='conv2')), name='lrelu2') h_3 = lrelu(bn3(conv2d(h_2, 64, name='conv3')), name='lrelu3') h_3_flat = tf.reshape(h_3, [-1, 64 * 4 * 4]) h_4 = linear(h_3_flat, 10) y_pred = tf.nn.softmax(h_4) # %% Define loss/eval/training functions cross_entropy = -tf.reduce_sum(y * tf.log(y_pred)) train_step = tf.train.AdamOptimizer().minimize(cross_entropy) correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) # %% We now create a new session to actually perform the initialization the # variables: sess = tf.Session() sess.run(tf.initialize_all_variables()) # %% We'll train in minibatches and report accuracy: