def model(x, y, is_training): # %% We'll convert our MNIST vector data to a 4-D tensor: # N x W x H x C x_tensor = tf.reshape(x, [-1, 28, 28, 1]) # %% We'll use a new method called batch normalization. # This process attempts to "reduce internal covariate shift" # which is a fancy way of saying that it will normalize updates for each # batch using a smoothed version of the batch mean and variance # The original paper proposes using this before any nonlinearities h_1 = lrelu(batch_norm(conv2d(x_tensor, 32, name='conv1'), is_training, scope='bn1'), name='lrelu1') h_2 = lrelu(batch_norm(conv2d(h_1, 64, name='conv2'), is_training, scope='bn2'), name='lrelu2') h_3 = lrelu(batch_norm(conv2d(h_2, 64, name='conv3'), is_training, scope='bn3'), name='lrelu3') h_3_flat = tf.reshape(h_3, [-1, 64 * 4 * 4]) h_4 = linear(h_3_flat, 10) y_pred = tf.nn.softmax(h_4) # %% Define loss/eval/training functions cross_entropy = -tf.reduce_sum(y * tf.log(y_pred)) train_step = tf.train.AdamOptimizer().minimize(cross_entropy) correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) return [train_step, accuracy]
def residual_network(x, n_outputs, activation=tf.nn.relu): LayerBlock = namedtuple('LayerBlock', ['num_repeats', 'num_filters', 'bottleneck_size']) blocks = [ LayerBlock(3, 128, 32), LayerBlock(3, 256, 64), LayerBlock(3, 512, 128), LayerBlock(3, 1024, 256)] # 如果数据是二维的,则转一下,对标mnist input_shape = x.get_shape().as_list() if len(input_shape) == 2: ndim = int(sqrt(input_shape[1])) if ndim * ndim != input_shape[1]: raise ValueError('input_shape should be square') x = tf.reshape(x, [-1, ndim, ndim, 1]) net = conv2d(x, 64, k_h=7, k_w=7, name='conv1', activation=activation) net = tf.nn.max_pool(net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') net = conv2d(net, blocks[0].num_filters, k_h=1, k_w=1,stride_h=1, stride_w=1, padding='VALID', name='conv2') for block_i, block in enumerate(blocks): for repeat_i in range(block.num_repeats): name = 'block_%d/repeat_%d' % (block_i, repeat_i) conv = conv2d(net, block.bottleneck_size, k_h=1, k_w=1, padding='VALID', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_in') conv = conv2d(conv, block.bottleneck_size, k_h=3, k_w=3, padding='SAME', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_bottleneck') conv = conv2d(conv, block.num_filters, k_h=1, k_w=1, padding='VALID', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_out') net = conv + net try: next_block = blocks[block_i + 1] net = conv2d(net, next_block.num_filters, k_h=1, k_w=1, padding='SAME', stride_h=1, stride_w=1, bias=False, name='block_%d/conv_upscale' % block_i) except IndexError: pass net = tf.nn.avg_pool(net, ksize=[1, net.get_shape().as_list()[1], net.get_shape().as_list()[2], 1], strides=[1, 1, 1, 1], padding='VALID') net = tf.reshape(net, [-1, net.get_shape().as_list()[1] * net.get_shape().as_list()[2] * net.get_shape().as_list()[3]]) net = linear(net, n_outputs, activation=tf.nn.softmax) return net
def residual_network(x, n_outputs, activation=tf.nn.relu): """Builds a residual network. Parameters ---------- x : Placeholder Input to the network n_outputs : TYPE Number of outputs of final softmax activation : Attribute, optional Nonlinearity to apply after each convolution Returns ------- net : Tensor Description Raises ------ ValueError If a 2D Tensor is input, the Tensor must be square or else the network can't be converted to a 4D Tensor. """ # %% LayerBlock = namedtuple( 'LayerBlock', ['num_repeats', 'num_filters', 'bottleneck_size']) blocks = [LayerBlock(3, 128, 32), LayerBlock(3, 256, 64), LayerBlock(3, 512, 128), LayerBlock(3, 1024, 256)] # %% input_shape = x.get_shape().as_list() if len(input_shape) == 2: ndim = int(sqrt(input_shape[1])) if ndim * ndim != input_shape[1]: raise ValueError('input_shape should be square') x = tf.reshape(x, [-1, ndim, ndim, 1]) # %% # First convolution expands to 64 channels and downsamples net = conv2d(x, 64, k_h=7, k_w=7, name='conv1', activation=activation) # %% # Max pool and downsampling net = tf.nn.max_pool( net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') # %% # Setup first chain of resnets net = conv2d(net, blocks[0].num_filters, k_h=1, k_w=1, stride_h=1, stride_w=1, padding='VALID', name='conv2') # %% # Loop through all res blocks for block_i, block in enumerate(blocks): for repeat_i in range(block.num_repeats): name = 'block_%d/repeat_%d' % (block_i, repeat_i) conv = conv2d(net, block.bottleneck_size, k_h=1, k_w=1, padding='VALID', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_in') conv = conv2d(conv, block.bottleneck_size, k_h=3, k_w=3, padding='SAME', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_bottleneck') conv = conv2d(conv, block.num_filters, k_h=1, k_w=1, padding='VALID', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_out') net = conv + net try: # upscale to the next block size next_block = blocks[block_i + 1] net = conv2d(net, next_block.num_filters, k_h=1, k_w=1, padding='SAME', stride_h=1, stride_w=1, bias=False, name='block_%d/conv_upscale' % block_i) except IndexError: pass # %% net = tf.nn.avg_pool(net, ksize=[1, net.get_shape().as_list()[1], net.get_shape().as_list()[2], 1], strides=[1, 1, 1, 1], padding='VALID') net = tf.reshape( net, [-1, net.get_shape().as_list()[1] * net.get_shape().as_list()[2] * net.get_shape().as_list()[3]]) net = linear(net, n_outputs, activation=tf.nn.softmax) # %% return net
def residual_network(x, n_outputs, activation=tf.nn.relu): """Builds a residual network. Parameters ---------- x : Placeholder Input to the network n_outputs : TYPE Number of outputs of final softmax activation : Attribute, optional Nonlinearity to apply after each convolution Returns ------- net : Tensor Description Raises ------ ValueError If a 2D Tensor is input, the Tensor must be square or else the network can't be converted to a 4D Tensor. """ # LayerBlock = namedtuple( 'LayerBlock', ['num_repeats', 'num_filters', 'bottleneck_size']) blocks = [LayerBlock(3, 128, 32), LayerBlock(3, 256, 64), LayerBlock(3, 512, 128), LayerBlock(3, 1024, 256)] # input_shape = x.get_shape().as_list() if len(input_shape) == 2: ndim = int(sqrt(input_shape[1])) if ndim * ndim != input_shape[1]: raise ValueError('input_shape should be square') x = tf.reshape(x, [-1, ndim, ndim, 1]) # First convolution expands to 64 channels and downsamples net = conv2d(x, 64, k_h=7, k_w=7, name='conv1', activation=activation) # Max pool and downsampling net = tf.nn.max_pool(net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') # Setup first chain of resnets net = conv2d(net, blocks[0].num_filters, k_h=1, k_w=1, stride_h=1, stride_w=1, padding='VALID', name='conv2') # Loop through all res blocks for block_i, block in enumerate(blocks): for repeat_i in range(block.num_repeats): name = 'block_%d/repeat_%d' % (block_i, repeat_i) conv = conv2d(net, block.bottleneck_size, k_h=1, k_w=1, padding='VALID', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_in') conv = conv2d(conv, block.bottleneck_size, k_h=3, k_w=3, padding='SAME', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_bottleneck') conv = conv2d(conv, block.num_filters, k_h=1, k_w=1, padding='VALID', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_out') net = conv + net try: # upscale to the next block size next_block = blocks[block_i + 1] net = conv2d(net, next_block.num_filters, k_h=1, k_w=1, padding='SAME', stride_h=1, stride_w=1, bias=False, name='block_%d/conv_upscale' % block_i) except IndexError: pass # net = tf.nn.avg_pool(net, ksize=[1, net.get_shape().as_list()[1], net.get_shape().as_list()[2], 1], strides=[1, 1, 1, 1], padding='VALID') net = tf.reshape( net, [-1, net.get_shape().as_list()[1] * net.get_shape().as_list()[2] * net.get_shape().as_list()[3]]) net = linear(net, n_outputs, activation=tf.nn.softmax) # return net
''' # The original paper proposes using this before any nonlinearities!!!!!!!!!!!!!!! ''' # The original paper proposes using this before any nonlinearities!!!!!!!!!!!!!!! h_1 = lrelu(batch_norm(conv2d(x_tensor, 32, name='conv1'), is_training, scope='bn1'), name='lrelu1') h_2 = lrelu(batch_norm(conv2d(h_1, 64, name='conv2'), is_training, scope='bn2'), name='lrelu2') h_3 = lrelu(batch_norm(conv2d(h_2, 64, name='conv3'), is_training, scope='bn3'), name='lrelu3') h_3_flat = tf.reshape(h_3, [-1, 64 * 4 * 4]) h_4 = linear(h_3_flat, 10) y_pred = tf.nn.softmax(h_4) # %% Define loss/eval/training functions cross_entropy = -tf.reduce_sum(y * tf.log(y_pred)) train_step = tf.train.AdamOptimizer().minimize(cross_entropy) correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) # %% We now create a new session to actually perform the initialization the # variables: sess = tf.Session() sess.run(tf.global_variables_initializer()) # %% We'll train in minibatches and report accuracy:
# N x W x H x C x_tensor = tf.reshape(x, [-1, 28, 28, 1]) # %% We'll use a new method called batch normalization. # This process attempts to "reduce internal covariate shift" # which is a fancy way of saying that it will normalize updates for each # batch using a smoothed version of the batch mean and variance # The original paper proposes using this before any nonlinearities h_1 = lrelu(batch_norm(conv2d(x_tensor, 32, name='conv1'), is_training, scope='bn1'), name='lrelu1') h_2 = lrelu(batch_norm(conv2d(h_1, 64, name='conv2'), is_training, scope='bn2'), name='lrelu2') h_3 = lrelu(batch_norm(conv2d(h_2, 64, name='conv3'), is_training, scope='bn3'), name='lrelu3') h_3_flat = tf.reshape(h_3, [-1, 64 * 4 * 4]) h_4 = linear(h_3_flat, 10) y_pred = tf.nn.softmax(h_4) # %% Define loss/eval/training functions cross_entropy = -tf.reduce_sum(y * tf.log(y_pred)) train_step = tf.train.AdamOptimizer().minimize(cross_entropy) correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) # %% We now create a new session to actually perform the initialization the # variables: sess = tf.Session() sess.run(tf.global_variables_initializer()) # %% We'll train in minibatches and report accuracy:
a2=tf.nn.conv2d(h1, K2, strides=[1,1,1,1], padding='VALID') # 배치정규화 a2=tf.layers.batch_normalization(a2, training=True) # 두 번째 합성곱층의 활성화함수 지정 a2=tf.nn.relu(a2) # 두 번째 풀링층에 사용하는 풀링의 종류와 크기, 보폭 지정 h2=tf.nn.max_pool(a1,ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID') #두 번째 풀링층의 출력을 1D로 변환 Flat=tf.reshape(h2,[-1,np.prod(h2.get_shape().as_list()[1:4])]) #완전 연결 신경망의 은닉층의 구조 지정 W1=tf.get_variable("W1",shape=[np.prod(h1.get_shape().as_list()[1:4]),50],initializer =tf.contrib.layers.xavier_initializer()) b1=tf.Variable(tf.random_normal([50])) L1=tf.matmul(Flat, W1)+b1 # 최종 출력을 위해 소프트맥스함수 지정 Y_pred =linear(L1, 10, activation=tf.nn.softmax) cost=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=Y_pred,labels=Y)) optim=tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost) correct_predict=tf.equal(tf.argmax(Y_pred,1), tf.argmax(Y,1)) accuracy=tf.reduce_mean(tf.cast(correct_predict, tf.float32)) sess=tf.Session(); sess.run(tf.global_variables_initializer()) for epoch in range(epochs): avg_cost=0 total_batch=int(mnist.train.num_examples/batch_size) for i in range(total_batch): batch_xs,batch_ys=mnist.train.next_batch(batch_size) feed_dict={X:batch_xs, Y:batch_ys} sess.run(optim, feed_dict=feed_dict) ccost=sess.run(cost, feed_dict=feed_dict) avg_cost+=ccost/total_batch acc=sess.run(accuracy, feed_dict=feed_dict)
pass # 평균 풀링을 이용하여 블록 구조의 최종 출력의 차원 변환 net = tf.nn.avg_pool( net, ksize=[1, net.get_shape().as_list()[1], net.get_shape().as_list()[2], 1], strides=[1, 1, 1, 1], padding='VALID') #ResNet 블록 구조의 최종 출력을 1D로 변환 Flat = tf.reshape(net, [ -1, net.get_shape().as_list()[1] * net.get_shape().as_list()[2] * net.get_shape().as_list()[3] ]) # 최종 출력을 위해 소프트맥스함수 지정 Y_pred = linear(Flat, 10, activation=tf.nn.softmax) cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2(logits=Y_pred, labels=Y)) optim = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) correct_predict = tf.equal(tf.argmax(Y_pred, 1), tf.argmax(Y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_predict, tf.float32)) sess = tf.Session() sess.run(tf.global_variables_initializer()) for epoch in range(epochs): avg_cost = 0 total_batch = int(mnist.train.num_examples / batch_size) for i in range(total_batch): batch_xs, batch_ys = mnist.train.next_batch(batch_size) feed_dict = {X: batch_xs, Y: batch_ys} sess.run(optim, feed_dict=feed_dict) ccost = sess.run(cost, feed_dict=feed_dict)
def residual_network(x, n_outputs, activation=tf.nn.relu): LayerBlock = namedtuple('LayerBlock', ['num_repeats', 'num_filters', 'bottleneck_size']) blocks = [ LayerBlock(3, 128, 32), LayerBlock(3, 256, 64), LayerBlock(3, 512, 128), LayerBlock(3, 1024, 256) ] # 如果数据是二维的,则转一下,对标mnist input_shape = x.get_shape().as_list() if len(input_shape) == 2: ndim = int(sqrt(input_shape[1])) if ndim * ndim != input_shape[1]: raise ValueError('input_shape should be square') x = tf.reshape(x, [-1, ndim, ndim, 1]) net = conv2d(x, 64, k_h=7, k_w=7, name='conv1', activation=activation) net = tf.nn.max_pool(net, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME') net = conv2d(net, blocks[0].num_filters, k_h=1, k_w=1, stride_h=1, stride_w=1, padding='VALID', name='conv2') for block_i, block in enumerate(blocks): for repeat_i in range(block.num_repeats): name = 'block_%d/repeat_%d' % (block_i, repeat_i) conv = conv2d(net, block.bottleneck_size, k_h=1, k_w=1, padding='VALID', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_in') conv = conv2d(conv, block.bottleneck_size, k_h=3, k_w=3, padding='SAME', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_bottleneck') conv = conv2d(conv, block.num_filters, k_h=1, k_w=1, padding='VALID', stride_h=1, stride_w=1, activation=activation, name=name + '/conv_out') net = conv + net try: next_block = blocks[block_i + 1] net = conv2d(net, next_block.num_filters, k_h=1, k_w=1, padding='SAME', stride_h=1, stride_w=1, bias=False, name='block_%d/conv_upscale' % block_i) except IndexError: pass net = tf.nn.avg_pool(net, ksize=[ 1, net.get_shape().as_list()[1], net.get_shape().as_list()[2], 1 ], strides=[1, 1, 1, 1], padding='VALID') net = tf.reshape(net, [ -1, net.get_shape().as_list()[1] * net.get_shape().as_list()[2] * net.get_shape().as_list()[3] ]) net = linear(net, n_outputs, activation=tf.nn.softmax) return net
phase_train=is_training, scope='bn3'), name='lrelu3') #h_1 = lrelu((conv2d(x_tensor, 32, name='conv1', stride_h=1, k_h=1, k_w=3, pool_size=[1, 2], pool_stride=1)), name='lrelu1') #h_2 = lrelu((conv2d(h_1, 64, name='conv2', stride_h=1, k_h=1, k_w=3, pool_size=[1, 2], pool_stride=1)), name='lrelu2') #h_3 = lrelu((conv2d(h_2, 64, name='conv3', stride_h=1, k_h=1, k_w=3, pool_size=[1, 2], pool_stride=1)), name='lrelu3') h_3_flat = tf.reshape(h_3, [-1, 64 * 4]) # FOR 26 dim #h_3_flat = tf.reshape(h_3, [-1, 64 * 5]) # FOR CONVAE h_4 = linear(h_3_flat, 2048, scope='lin1', activation=lambda x: lrelu(batch_norm_dense( x, phase_train=is_training, scope='bn4'), name='lrelu5')) h4_dropout = tf.layers.dropout(inputs=h_4, rate=0.5, training=is_training, name='dropout1') h_5 = linear(h4_dropout, 2048, scope='lin2', activation=lambda x: lrelu(batch_norm_dense( x, phase_train=is_training, scope='bn5'), name='lrelu6')) h5_dropout = tf.layers.dropout(inputs=h_5, rate=0.5,