def model(x, y, is_training):
# %% We'll convert our MNIST vector data to a 4-D tensor:
# N x W x H x C
x_tensor = tf.reshape(x, [-1, 28, 28, 1])
# %% We'll use a new method called batch normalization.
# This process attempts to "reduce internal covariate shift"
# which is a fancy way of saying that it will normalize updates for each
# batch using a smoothed version of the batch mean and variance
# The original paper proposes using this before any nonlinearities
h_1 = lrelu(batch_norm(conv2d(x_tensor, 32, name='conv1'),
is_training,
scope='bn1'),
name='lrelu1')
h_2 = lrelu(batch_norm(conv2d(h_1, 64, name='conv2'),
is_training,
scope='bn2'),
name='lrelu2')
h_3 = lrelu(batch_norm(conv2d(h_2, 64, name='conv3'),
is_training,
scope='bn3'),
name='lrelu3')
h_3_flat = tf.reshape(h_3, [-1, 64 * 4 * 4])
h_4 = linear(h_3_flat, 10)
y_pred = tf.nn.softmax(h_4)
# %% Define loss/eval/training functions
cross_entropy = -tf.reduce_sum(y * tf.log(y_pred))
train_step = tf.train.AdamOptimizer().minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
return [train_step, accuracy]
def residual_network(x, n_outputs, activation=tf.nn.relu):
LayerBlock = namedtuple('LayerBlock', ['num_repeats', 'num_filters', 'bottleneck_size'])
blocks = [
LayerBlock(3, 128, 32),
LayerBlock(3, 256, 64),
LayerBlock(3, 512, 128),
LayerBlock(3, 1024, 256)]
# 如果数据是二维的，则转一下，对标mnist
input_shape = x.get_shape().as_list()
if len(input_shape) == 2:
ndim = int(sqrt(input_shape[1]))
if ndim * ndim != input_shape[1]:
raise ValueError('input_shape should be square')
x = tf.reshape(x, [-1, ndim, ndim, 1])
net = conv2d(x, 64, k_h=7, k_w=7, name='conv1', activation=activation)
net = tf.nn.max_pool(net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
net = conv2d(net, blocks[0].num_filters, k_h=1, k_w=1,stride_h=1, stride_w=1, padding='VALID', name='conv2')
for block_i, block in enumerate(blocks):
for repeat_i in range(block.num_repeats):
name = 'block_%d/repeat_%d' % (block_i, repeat_i)
conv = conv2d(net, block.bottleneck_size, k_h=1, k_w=1, padding='VALID', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_in')
conv = conv2d(conv, block.bottleneck_size, k_h=3, k_w=3, padding='SAME', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_bottleneck')
conv = conv2d(conv, block.num_filters, k_h=1, k_w=1, padding='VALID', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_out')
net = conv + net
try:
next_block = blocks[block_i + 1]
net = conv2d(net, next_block.num_filters, k_h=1, k_w=1, padding='SAME', stride_h=1, stride_w=1, bias=False, name='block_%d/conv_upscale' % block_i)
except IndexError:
pass
net = tf.nn.avg_pool(net, ksize=[1, net.get_shape().as_list()[1], net.get_shape().as_list()[2], 1], strides=[1, 1, 1, 1], padding='VALID')
net = tf.reshape(net, [-1, net.get_shape().as_list()[1] * net.get_shape().as_list()[2] * net.get_shape().as_list()[3]])
net = linear(net, n_outputs, activation=tf.nn.softmax)
return net
def residual_network(x, n_outputs,
activation=tf.nn.relu):
"""Builds a residual network.
Parameters
----------
x : Placeholder
Input to the network
n_outputs : TYPE
Number of outputs of final softmax
activation : Attribute, optional
Nonlinearity to apply after each convolution
Returns
-------
net : Tensor
Description
Raises
------
ValueError
If a 2D Tensor is input, the Tensor must be square or else
the network can't be converted to a 4D Tensor.
"""
# %%
LayerBlock = namedtuple(
'LayerBlock', ['num_repeats', 'num_filters', 'bottleneck_size'])
blocks = [LayerBlock(3, 128, 32),
LayerBlock(3, 256, 64),
LayerBlock(3, 512, 128),
LayerBlock(3, 1024, 256)]
# %%
input_shape = x.get_shape().as_list()
if len(input_shape) == 2:
ndim = int(sqrt(input_shape[1]))
if ndim * ndim != input_shape[1]:
raise ValueError('input_shape should be square')
x = tf.reshape(x, [-1, ndim, ndim, 1])
# %%
# First convolution expands to 64 channels and downsamples
net = conv2d(x, 64, k_h=7, k_w=7,
name='conv1',
activation=activation)
# %%
# Max pool and downsampling
net = tf.nn.max_pool(
net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# %%
# Setup first chain of resnets
net = conv2d(net, blocks[0].num_filters, k_h=1, k_w=1,
stride_h=1, stride_w=1, padding='VALID', name='conv2')
# %%
# Loop through all res blocks
for block_i, block in enumerate(blocks):
for repeat_i in range(block.num_repeats):
name = 'block_%d/repeat_%d' % (block_i, repeat_i)
conv = conv2d(net, block.bottleneck_size, k_h=1, k_w=1,
padding='VALID', stride_h=1, stride_w=1,
activation=activation,
name=name + '/conv_in')
conv = conv2d(conv, block.bottleneck_size, k_h=3, k_w=3,
padding='SAME', stride_h=1, stride_w=1,
activation=activation,
name=name + '/conv_bottleneck')
conv = conv2d(conv, block.num_filters, k_h=1, k_w=1,
padding='VALID', stride_h=1, stride_w=1,
activation=activation,
name=name + '/conv_out')
net = conv + net
try:
# upscale to the next block size
next_block = blocks[block_i + 1]
net = conv2d(net, next_block.num_filters, k_h=1, k_w=1,
padding='SAME', stride_h=1, stride_w=1, bias=False,
name='block_%d/conv_upscale' % block_i)
except IndexError:
pass
# %%
net = tf.nn.avg_pool(net,
ksize=[1, net.get_shape().as_list()[1],
net.get_shape().as_list()[2], 1],
strides=[1, 1, 1, 1], padding='VALID')
net = tf.reshape(
net,
[-1, net.get_shape().as_list()[1] *
net.get_shape().as_list()[2] *
net.get_shape().as_list()[3]])
net = linear(net, n_outputs, activation=tf.nn.softmax)
# %%
return net
def residual_network(x, n_outputs,
activation=tf.nn.relu):
"""Builds a residual network.
Parameters
----------
x : Placeholder Input to the network
n_outputs : TYPE Number of outputs of final softmax
activation : Attribute, optional Nonlinearity to apply after each convolution
Returns
-------
net : Tensor Description
Raises
------
ValueError
If a 2D Tensor is input, the Tensor must be square or else
the network can't be converted to a 4D Tensor.
"""
#
LayerBlock = namedtuple(
'LayerBlock', ['num_repeats', 'num_filters', 'bottleneck_size'])
blocks = [LayerBlock(3, 128, 32),
LayerBlock(3, 256, 64),
LayerBlock(3, 512, 128),
LayerBlock(3, 1024, 256)]
#
input_shape = x.get_shape().as_list()
if len(input_shape) == 2:
ndim = int(sqrt(input_shape[1]))
if ndim * ndim != input_shape[1]:
raise ValueError('input_shape should be square')
x = tf.reshape(x, [-1, ndim, ndim, 1])
# First convolution expands to 64 channels and downsamples
net = conv2d(x, 64, k_h=7, k_w=7, name='conv1', activation=activation)
# Max pool and downsampling
net = tf.nn.max_pool(net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# Setup first chain of resnets
net = conv2d(net, blocks[0].num_filters, k_h=1, k_w=1, stride_h=1, stride_w=1, padding='VALID', name='conv2')
# Loop through all res blocks
for block_i, block in enumerate(blocks):
for repeat_i in range(block.num_repeats):
name = 'block_%d/repeat_%d' % (block_i, repeat_i)
conv = conv2d(net, block.bottleneck_size, k_h=1, k_w=1,
padding='VALID', stride_h=1, stride_w=1,
activation=activation,
name=name + '/conv_in')
conv = conv2d(conv, block.bottleneck_size, k_h=3, k_w=3,
padding='SAME', stride_h=1, stride_w=1,
activation=activation,
name=name + '/conv_bottleneck')
conv = conv2d(conv, block.num_filters, k_h=1, k_w=1,
padding='VALID', stride_h=1, stride_w=1,
activation=activation,
name=name + '/conv_out')
net = conv + net
try:
# upscale to the next block size
next_block = blocks[block_i + 1]
net = conv2d(net, next_block.num_filters, k_h=1, k_w=1,
padding='SAME', stride_h=1, stride_w=1, bias=False,
name='block_%d/conv_upscale' % block_i)
except IndexError:
pass
#
net = tf.nn.avg_pool(net,
ksize=[1, net.get_shape().as_list()[1],
net.get_shape().as_list()[2], 1],
strides=[1, 1, 1, 1], padding='VALID')
net = tf.reshape(
net,
[-1, net.get_shape().as_list()[1] *
net.get_shape().as_list()[2] *
net.get_shape().as_list()[3]])
net = linear(net, n_outputs, activation=tf.nn.softmax)
#
return net
'''
# The original paper proposes using this before any nonlinearities!!!!!!!!!!!!!!!
'''
# The original paper proposes using this before any nonlinearities!!!!!!!!!!!!!!!
h_1 = lrelu(batch_norm(conv2d(x_tensor, 32, name='conv1'),
is_training,
scope='bn1'),
name='lrelu1')
h_2 = lrelu(batch_norm(conv2d(h_1, 64, name='conv2'), is_training,
scope='bn2'),
name='lrelu2')
h_3 = lrelu(batch_norm(conv2d(h_2, 64, name='conv3'), is_training,
scope='bn3'),
name='lrelu3')
h_3_flat = tf.reshape(h_3, [-1, 64 * 4 * 4])
h_4 = linear(h_3_flat, 10)
y_pred = tf.nn.softmax(h_4)
# %% Define loss/eval/training functions
cross_entropy = -tf.reduce_sum(y * tf.log(y_pred))
train_step = tf.train.AdamOptimizer().minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
# %% We now create a new session to actually perform the initialization the
# variables:
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# %% We'll train in minibatches and report accuracy:
# N x W x H x C
x_tensor = tf.reshape(x, [-1, 28, 28, 1])
# %% We'll use a new method called batch normalization.
# This process attempts to "reduce internal covariate shift"
# which is a fancy way of saying that it will normalize updates for each
# batch using a smoothed version of the batch mean and variance
# The original paper proposes using this before any nonlinearities
h_1 = lrelu(batch_norm(conv2d(x_tensor, 32, name='conv1'),
is_training, scope='bn1'), name='lrelu1')
h_2 = lrelu(batch_norm(conv2d(h_1, 64, name='conv2'),
is_training, scope='bn2'), name='lrelu2')
h_3 = lrelu(batch_norm(conv2d(h_2, 64, name='conv3'),
is_training, scope='bn3'), name='lrelu3')
h_3_flat = tf.reshape(h_3, [-1, 64 * 4 * 4])
h_4 = linear(h_3_flat, 10)
y_pred = tf.nn.softmax(h_4)
# %% Define loss/eval/training functions
cross_entropy = -tf.reduce_sum(y * tf.log(y_pred))
train_step = tf.train.AdamOptimizer().minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
# %% We now create a new session to actually perform the initialization the
# variables:
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# %% We'll train in minibatches and report accuracy:
a2=tf.nn.conv2d(h1, K2, strides=[1,1,1,1], padding='VALID')
# 배치정규화
a2=tf.layers.batch_normalization(a2, training=True)
# 두 번째 합성곱층의 활성화함수 지정
a2=tf.nn.relu(a2)
# 두 번째 풀링층에 사용하는 풀링의 종류와 크기, 보폭 지정
h2=tf.nn.max_pool(a1,ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID')
#두 번째 풀링층의 출력을 1D로 변환
Flat=tf.reshape(h2,[-1,np.prod(h2.get_shape().as_list()[1:4])])
#완전 연결 신경망의 은닉층의 구조 지정
W1=tf.get_variable("W1",shape=[np.prod(h1.get_shape().as_list()[1:4]),50],initializer
=tf.contrib.layers.xavier_initializer())
b1=tf.Variable(tf.random_normal([50]))
L1=tf.matmul(Flat, W1)+b1
# 최종 출력을 위해 소프트맥스함수 지정
Y_pred =linear(L1, 10, activation=tf.nn.softmax)
cost=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=Y_pred,labels=Y))
optim=tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)
correct_predict=tf.equal(tf.argmax(Y_pred,1), tf.argmax(Y,1))
accuracy=tf.reduce_mean(tf.cast(correct_predict, tf.float32))
sess=tf.Session(); sess.run(tf.global_variables_initializer())
for epoch in range(epochs):
avg_cost=0
total_batch=int(mnist.train.num_examples/batch_size)
for i in range(total_batch):
batch_xs,batch_ys=mnist.train.next_batch(batch_size)
feed_dict={X:batch_xs, Y:batch_ys}
sess.run(optim, feed_dict=feed_dict)
ccost=sess.run(cost, feed_dict=feed_dict)
avg_cost+=ccost/total_batch
acc=sess.run(accuracy, feed_dict=feed_dict)
pass
# 평균 풀링을 이용하여 블록 구조의 최종 출력의 차원 변환
net = tf.nn.avg_pool(
net,
ksize=[1, net.get_shape().as_list()[1],
net.get_shape().as_list()[2], 1],
strides=[1, 1, 1, 1],
padding='VALID')
#ResNet 블록 구조의 최종 출력을 1D로 변환
Flat = tf.reshape(net, [
-1,
net.get_shape().as_list()[1] * net.get_shape().as_list()[2] *
net.get_shape().as_list()[3]
])
# 최종 출력을 위해 소프트맥스함수 지정
Y_pred = linear(Flat, 10, activation=tf.nn.softmax)
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=Y_pred, labels=Y))
optim = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
correct_predict = tf.equal(tf.argmax(Y_pred, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_predict, tf.float32))
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for epoch in range(epochs):
avg_cost = 0
total_batch = int(mnist.train.num_examples / batch_size)
for i in range(total_batch):
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
feed_dict = {X: batch_xs, Y: batch_ys}
sess.run(optim, feed_dict=feed_dict)
ccost = sess.run(cost, feed_dict=feed_dict)
def residual_network(x, n_outputs, activation=tf.nn.relu):
LayerBlock = namedtuple('LayerBlock',
['num_repeats', 'num_filters', 'bottleneck_size'])
blocks = [
LayerBlock(3, 128, 32),
LayerBlock(3, 256, 64),
LayerBlock(3, 512, 128),
LayerBlock(3, 1024, 256)
]
# 如果数据是二维的，则转一下，对标mnist
input_shape = x.get_shape().as_list()
if len(input_shape) == 2:
ndim = int(sqrt(input_shape[1]))
if ndim * ndim != input_shape[1]:
raise ValueError('input_shape should be square')
x = tf.reshape(x, [-1, ndim, ndim, 1])
net = conv2d(x, 64, k_h=7, k_w=7, name='conv1', activation=activation)
net = tf.nn.max_pool(net, [1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME')
net = conv2d(net,
blocks[0].num_filters,
k_h=1,
k_w=1,
stride_h=1,
stride_w=1,
padding='VALID',
name='conv2')
for block_i, block in enumerate(blocks):
for repeat_i in range(block.num_repeats):
name = 'block_%d/repeat_%d' % (block_i, repeat_i)
conv = conv2d(net,
block.bottleneck_size,
k_h=1,
k_w=1,
padding='VALID',
stride_h=1,
stride_w=1,
activation=activation,
name=name + '/conv_in')
conv = conv2d(conv,
block.bottleneck_size,
k_h=3,
k_w=3,
padding='SAME',
stride_h=1,
stride_w=1,
activation=activation,
name=name + '/conv_bottleneck')
conv = conv2d(conv,
block.num_filters,
k_h=1,
k_w=1,
padding='VALID',
stride_h=1,
stride_w=1,
activation=activation,
name=name + '/conv_out')
net = conv + net
try:
next_block = blocks[block_i + 1]
net = conv2d(net,
next_block.num_filters,
k_h=1,
k_w=1,
padding='SAME',
stride_h=1,
stride_w=1,
bias=False,
name='block_%d/conv_upscale' % block_i)
except IndexError:
pass
net = tf.nn.avg_pool(net,
ksize=[
1,
net.get_shape().as_list()[1],
net.get_shape().as_list()[2], 1
],
strides=[1, 1, 1, 1],
padding='VALID')
net = tf.reshape(net, [
-1,
net.get_shape().as_list()[1] * net.get_shape().as_list()[2] *
net.get_shape().as_list()[3]
])
net = linear(net, n_outputs, activation=tf.nn.softmax)
return net
phase_train=is_training,
scope='bn3'),
name='lrelu3')
#h_1 = lrelu((conv2d(x_tensor, 32, name='conv1', stride_h=1, k_h=1, k_w=3, pool_size=[1, 2], pool_stride=1)), name='lrelu1')
#h_2 = lrelu((conv2d(h_1, 64, name='conv2', stride_h=1, k_h=1, k_w=3, pool_size=[1, 2], pool_stride=1)), name='lrelu2')
#h_3 = lrelu((conv2d(h_2, 64, name='conv3', stride_h=1, k_h=1, k_w=3, pool_size=[1, 2], pool_stride=1)), name='lrelu3')
h_3_flat = tf.reshape(h_3, [-1, 64 * 4]) # FOR 26 dim
#h_3_flat = tf.reshape(h_3, [-1, 64 * 5]) # FOR CONVAE
h_4 = linear(h_3_flat,
2048,
scope='lin1',
activation=lambda x: lrelu(batch_norm_dense(
x, phase_train=is_training, scope='bn4'),
name='lrelu5'))
h4_dropout = tf.layers.dropout(inputs=h_4,
rate=0.5,
training=is_training,
name='dropout1')
h_5 = linear(h4_dropout,
2048,
scope='lin2',
activation=lambda x: lrelu(batch_norm_dense(
x, phase_train=is_training, scope='bn5'),
name='lrelu6'))
h5_dropout = tf.layers.dropout(inputs=h_5,
rate=0.5,
