def vgg16(input, num_class): network = tflearn.conv_2d(input, KERNEL, 3, activation='relu', regularizer="L2", weight_decay=0.0001) network = tflearn.avg_pool_2d(network, 2) network = tflearn.conv_2d(network, KERNEL, 3, activation='relu', regularizer="L2", weight_decay=0.0001) network = tflearn.avg_pool_2d(network, 2) network = tflearn.conv_2d(network, KERNEL, 3, activation='relu', regularizer="L2", weight_decay=0.0001) network = tflearn.avg_pool_2d(network, 2) network = tflearn.conv_2d(network, KERNEL, 3, activation='relu', regularizer="L2", weight_decay=0.0001) network = tflearn.avg_pool_2d(network, 2) x = tflearn.fully_connected(network, num_class, activation='sigmoid', scope='fc8') return x
def discriminator(x, reuse=False): with tf.variable_scope('Discriminator', reuse=reuse): x = tflearn.conv_2d(x, 64, 5, activation='tanh') # [n,28,28,64] x = tflearn.avg_pool_2d(x, 2) # [n,14,14,64] x = tflearn.conv_2d(x, 128, 5, activation='tanh') # [n,14,14,128] x = tflearn.avg_pool_2d(x, 2) # [n,7,7,128] x = tflearn.fully_connected(x, 1024, activation='tanh') # [n,1024] x = tflearn.fully_connected(x, 2) # [n,2] 2类 0 False ;1 True x = tf.nn.softmax(x) return x
def discriminator(x, reuse=False): with tf.variable_scope('Discriminator', reuse=reuse): x = tflearn.conv_2d(x, 64, 5, activation='tanh') x = tflearn.avg_pool_2d(x, 2) x = tflearn.conv_2d(x, 128, 5, activation='tanh') x = tflearn.avg_pool_2d(x, 2) x = tflearn.fully_connected(x, 1024, activation='tanh') x = tflearn.fully_connected(x, 2) x = tf.nn.softmax(x) return x
def residual_block(incoming, nb_blocks, out_channels, downsample=False, downsample_strides=2, activation='relu', batch_norm=True, bias=True, weights_init='variance_scaling', bias_init='zeros', regularizer='L2', weight_decay=0.0001, trainable=True, restore=True, reuse=False, scope=None, name="ResidualBlock"): """ Residual Block. A residual block as described in MSRA's Deep Residual Network paper. Full pre-activation architecture is used here. Input: 4-D Tensor [batch, height, width, in_channels]. Output: 4-D Tensor [batch, new height, new width, nb_filter]. Arguments: incoming: `Tensor`. Incoming 4-D Layer. nb_blocks: `int`. Number of layer blocks. out_channels: `int`. The number of convolutional filters of the convolution layers. downsample: `bool`. If True, apply downsampling using 'downsample_strides' for strides. downsample_strides: `int`. The strides to use when downsampling. activation: `str` (name) or `function` (returning a `Tensor`). Activation applied to this layer (see tflearn.activations). Default: 'linear'. batch_norm: `bool`. If True, apply batch normalization. bias: `bool`. If True, a bias is used. weights_init: `str` (name) or `Tensor`. Weights initialization. (see tflearn.initializations) Default: 'uniform_scaling'. bias_init: `str` (name) or `tf.Tensor`. Bias initialization. (see tflearn.initializations) Default: 'zeros'. regularizer: `str` (name) or `Tensor`. Add a regularizer to this layer weights (see tflearn.regularizers). Default: None. weight_decay: `float`. Regularizer decay parameter. Default: 0.001. trainable: `bool`. If True, weights will be trainable. restore: `bool`. If True, this layer weights will be restored when loading a model. reuse: `bool`. If True and 'scope' is provided, this layer variables will be reused (shared). scope: `str`. Define this layer scope (optional). A scope can be used to share variables between layers. Note that scope will override name. name: A name for this layer (optional). Default: 'ShallowBottleneck'. References: - Deep Residual Learning for Image Recognition. Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. 2015. - Identity Mappings in Deep Residual Networks. Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. 2015. Links: - [http://arxiv.org/pdf/1512.03385v1.pdf] (http://arxiv.org/pdf/1512.03385v1.pdf) - [Identity Mappings in Deep Residual Networks] (https://arxiv.org/pdf/1603.05027v2.pdf) """ resnet = incoming in_channels = incoming.get_shape().as_list()[-1] with tf.variable_op_scope([incoming], scope, name, reuse=reuse) as scope: name = scope.name #TODO for i in range(nb_blocks): identity = resnet if not downsample: downsample_strides = 1 if batch_norm: resnet = tflearn.batch_normalization(resnet) resnet = tflearn.activation(resnet, activation) resnet = conv_2d(resnet, out_channels, 3, downsample_strides, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) if batch_norm: resnet = tflearn.batch_normalization(resnet) resnet = tflearn.activation(resnet, activation) resnet = conv_2d(resnet, out_channels, 3, 1, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) # Downsampling if downsample_strides > 1: identity = tflearn.avg_pool_2d(identity, 1, downsample_strides) # Projection to new dimension if in_channels != out_channels: ch = (out_channels - in_channels) // 2 identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch]]) in_channels = out_channels resnet = resnet + identity return resnet
def image_net(input_map, reuse=False): with tf.variable_scope('image', reuse=False): img_out = tfl.conv_2d(input_map, 8, [3, 3], activation='relu') img_out = tfl.conv_2d(img_out, 8, [3, 3], activation='relu') img_out = tfl.avg_pool_2d(img_out, [2, 2]) #img_out=tfl.conv_2d(img_out,8,[3,3], activation='relu') #img_out=tfl.conv_2d(img_out,8,[3,3], activation='relu') #img_out=tfl.avg_pool_2d(img_out,[2,2]) img_out = tfl.conv_2d(img_out, 8, [3, 3], activation='relu') img_out = tfl.conv_2d(img_out, 8, [3, 3], activation='relu') img_out = tfl.avg_pool_2d(img_out, [2, 2]) img_out = tf.reshape( img_out, [-1, np.prod(np.array(img_out.get_shape().as_list()[1:]))]) img_out = tfl.fully_connected(img_out, 32, 'tanh') return img_out
def residual_block(incoming, nb_blocks, out_channels, downsample=False, downsample_strides=2, activation='relu', batch_norm=True, bias=True, weights_init='variance_scaling', bias_init='zeros', regularizer='L2', weight_decay=0.0001, trainable=True, restore=True, reuse=False, scope=None, name="ResidualBlock"): resnet = incoming in_channels = incoming.get_shape().as_list()[-1] # Variable Scope fix for older TF with tf.variable_scope(scope, default_name=name, values=[incoming], reuse=reuse) as scope: name = scope.name #TODO for i in range(nb_blocks): identity = resnet if not downsample: downsample_strides = 1 if batch_norm: #I think conv_2d should go before not after batch normalization and activation resnet = tflearn.batch_normalization(resnet) resnet = tflearn.activation(resnet, activation) resnet = conv_2d(resnet, out_channels, 3, downsample_strides, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) if batch_norm: resnet = tflearn.batch_normalization(resnet) resnet = tflearn.activation(resnet, activation) resnet = conv_2d(resnet, out_channels, 3, 1, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) # Downsampling if downsample_strides > 1: identity = tflearn.avg_pool_2d(identity, downsample_strides, downsample_strides) # Projection to new dimension if in_channels != out_channels: ch = (out_channels - in_channels)//2 identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch]]) in_channels = out_channels resnet = resnet + identity return resnet
def discriminator(self, x, reuse=False): with tf.variable_scope('Discriminator', reuse=reuse): #self.noise_layer(x, 0.1) x = tflearn.conv_2d(x, 64, 2, activation='relu') x = tflearn.avg_pool_2d(x, 2) # x = tflearn.batch_normalization(x) x = tflearn.conv_2d(x, 128, 2, activation='relu') x = tflearn.avg_pool_2d(x, 2) # x = tflearn.batch_normalization(x) x = tflearn.conv_2d(x, 64, 2, activation='relu') x = tflearn.avg_pool_2d(x, 2) # x = tflearn.batch_normalization(x) x = tflearn.conv_2d(x, 128, 2, activation='relu') x = tflearn.avg_pool_2d(x, 2) # x = tflearn.batch_normalization(x) x = tflearn.fully_connected(x, 1024, activation='relu') x = tflearn.dropout(x, 0.8) x = tflearn.fully_connected(x, 2, activation='softmax') return x
def CNN_Core(self, x, reuse=False): with tf.variable_scope('cnn_core', reuse=reuse): network = tflearn.conv_2d(x, KERNEL, 5, activation='relu', regularizer="L2", weight_decay=0.0001) network = tflearn.avg_pool_2d(network, 3) network = tflearn.conv_2d(network, KERNEL, 3, activation='relu', regularizer="L2", weight_decay=0.0001) network = tflearn.avg_pool_2d(network, 2) network = tflearn.fully_connected(network, DENSE_SIZE, activation='relu') split_flat = tflearn.flatten(network) return split_flat
def create_model(): # Building Wide Residual Network assert ((depth - 4) % 6 == 0) n = (depth - 4) / 6 n_stages = [16, 16 * k, 32 * k, 64 * k] net = tflearn.input_data(shape=[None, 32, 32, 3], data_preprocessing=img_prep, data_augmentation=img_aug) conv1 = conv_2d(net, n_stages[0], 3, activation='linear', bias=False, regularizer='L2', weight_decay=0.0001) # Add wide residual blocks block_fn = _wide_basic conv2 = _layer(block_fn, n_input_plane=n_stages[0], n_output_plane=n_stages[1], count=n, stride=(1, 1))(conv1) # "Stage 1 (spatial size: 32x32)" conv3 = _layer(block_fn, n_input_plane=n_stages[1], n_output_plane=n_stages[2], count=n, stride=(2, 2))(conv2) # "Stage 2 (spatial size: 16x16)" conv4 = _layer(block_fn, n_input_plane=n_stages[2], n_output_plane=n_stages[3], count=n, stride=(2, 2))(conv3) # "Stage 3 (spatial size: 8x8)" net = tflearn.batch_normalization(conv4) net = tflearn.activation(net, 'twins_relu') net = tflearn.avg_pool_2d(net, 8) #net = tflearn.avg_pool_2d(net, kernel_size=8, strides=1, padding='same') net = tflearn.fully_connected(net, 10, activation='softmax') return net
def transition_layer(x, scope, reduction=0.5, keep_prob=1): out_filters = int(int(x.get_shape()[-1]) * reduction) with tf.name_scope(scope): x = tflearn.batch_normalization(x, scope=scope + '_batch1') x = tf.nn.relu(x) x = tflearn.conv_2d(x, nb_filter=out_filters, filter_size=1, strides=1, padding='same', activation='linear', bias=False, scope=scope + '_conv1', regularizer='L2', weight_decay=1e-4) x = tflearn.dropout(x, keep_prob=keep_prob) x = tflearn.avg_pool_2d(x, kernel_size=2, strides=2, padding='valid') print(scope, x) return x
def residual_bottleneck(incoming, nb_blocks, bottleneck_size, out_channels, downsample=False, downsample_strides=2, activation='relu', batch_norm=True, bias=True, weights_init='variance_scaling', bias_init='zeros', regularizer='L2', weight_decay=0.0001, trainable=True, restore=True, name="ResidualBottleneck"): """ Residual Bottleneck. A residual bottleneck block as described in MSRA's Deep Residual Network paper. Full pre-activation architecture is used here. Input: 4-D Tensor [batch, height, width, in_channels]. Output: 4-D Tensor [batch, new height, new width, nb_filter]. Arguments: incoming: `Tensor`. Incoming 4-D Layer. nb_blocks: `int`. Number of layer blocks. bottleneck_size: `int`. The number of convolutional filter of the bottleneck convolutional layer. out_channels: `int`. The number of convolutional filters of the layers surrounding the bottleneck layer. downsample: `bool`. If True, apply downsampling using 'downsample_strides' for strides. downsample_strides: `int`. The strides to use when downsampling. activation: `str` (name) or `function` (returning a `Tensor`). Activation applied to this layer (see tflearn.activations). Default: 'linear'. batch_norm: `bool`. If True, apply batch normalization. bias: `bool`. If True, a bias is used. weights_init: `str` (name) or `Tensor`. Weights initialization. (see tflearn.initializations) Default: 'uniform_scaling'. bias_init: `str` (name) or `tf.Tensor`. Bias initialization. (see tflearn.initializations) Default: 'zeros'. regularizer: `str` (name) or `Tensor`. Add a regularizer to this layer weights (see tflearn.regularizers). Default: None. weight_decay: `float`. Regularizer decay parameter. Default: 0.001. trainable: `bool`. If True, weights will be trainable. restore: `bool`. If True, this layer weights will be restored when loading a model name: A name for this layer (optional). Default: 'DeepBottleneck'. References: - Deep Residual Learning for Image Recognition. Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. 2015. - Identity Mappings in Deep Residual Networks. Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. 2015. Links: - [http://arxiv.org/pdf/1512.03385v1.pdf] (http://arxiv.org/pdf/1512.03385v1.pdf) - [Identity Mappings in Deep Residual Networks] (https://arxiv.org/pdf/1603.05027v2.pdf) """ resnet = incoming in_channels = incoming.get_shape().as_list()[-1] with tf.name_scope(name): for i in range(nb_blocks): identity = resnet if not downsample: downsample_strides = 1 if batch_norm: resnet = tflearn.batch_normalization(resnet) resnet = tflearn.activation(resnet, activation) resnet = conv_2d(resnet, bottleneck_size, 1, downsample_strides, 'valid', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) if batch_norm: resnet = tflearn.batch_normalization(resnet) resnet = tflearn.activation(resnet, activation) resnet = conv_2d(resnet, bottleneck_size, 3, 1, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) resnet = conv_2d(resnet, out_channels, 1, 1, 'valid', activation, bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) # Downsampling if downsample_strides > 1: identity = tflearn.avg_pool_2d(identity, 1, downsample_strides) # Projection to new dimension if in_channels != out_channels: ch = (out_channels - in_channels)//2 identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch]]) in_channels = out_channels resnet = resnet + identity resnet = tflearn.activation(resnet, activation) return resnet
def wideresnet_block(incoming, nb_blocks, out_channels, width, downsample=False, downsample_strides=2, activation='relu', batch_norm=True, bias=True, weights_init='variance_scaling', bias_init='zeros', regularizer='L2', weight_decay=0.0001, trainable=True, restore=True, reuse=False, scope=None, name="WideResNetBlock"): out_channels = out_channels * width #layers are wider for a constant widenet = incoming in_channels = incoming.get_shape().as_list()[-1] # Variable Scope fix for older TF with tf.variable_scope(scope, default_name=name, values=[incoming], reuse=reuse) as scope: name = scope.name #TODO for i in range(nb_blocks): identity = widenet if not downsample: downsample_strides = 1 if batch_norm: widenet = tflearn.batch_normalization(widenet) widenet = tflearn.activation(widenet, activation) widenet = conv_2d(widenet, out_channels, 3, downsample_strides, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) if batch_norm: widenet = tflearn.batch_normalization(widenet) widenet = tflearn.activation(widenet, activation) widenet = tflearn.dropout(widenet, 0.7) #added dropout between layers widenet = conv_2d(widenet, out_channels, 3, 1, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) # Downsampling if downsample_strides > 1: identity = tflearn.avg_pool_2d(identity, downsample_strides, downsample_strides) # Projection to new dimension if in_channels != out_channels: ch = (out_channels - in_channels) // 2 identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch]]) in_channels = out_channels widenet = widenet + identity return widenet
#logits = tf.squeeze(conv3,[1,2]) #x = tf.nn.softmax(logits) kernel = tf.get_variable(name='attention_conv3_weights', initializer=tf.truncated_normal([1, 1, 512, 512], dtype=tf.float32, stddev=1e-1)) conv = tf.nn.conv2d(x, kernel, [1, 1, 1, 1], padding='SAME') conv = tf.nn.relu(conv) kernel2 = tf.get_variable(name='attention_conv4_weights', initializer=tf.truncated_normal([1, 1, 512, 1], dtype=tf.float32, stddev=1e-1)) conv2 = tf.nn.conv2d(conv, kernel2, [1, 1, 1, 1], padding='SAME') attention_score2 = tf.nn.relu(conv2) x = x * attention_score2 c = tflearn.avg_pool_2d(x, 7) c = tflearn.flatten(c) c3 = tflearn.fully_connected(c, 32, activation='relu', scope='c3') c4 = tflearn.fully_connected(c3, 512, activation='sigmoid', scope='c4') c4 = tf.expand_dims(c4, 1) c4 = tf.expand_dims(c4, 2) x = x * c4 x = tflearn.fully_connected(x, 4096, activation='relu', scope='fc6') x = tflearn.dropout(x, 0.5, name='dropout1') fc7 = tflearn.fully_connected(x, 4096, activation='relu', scope='fc7') import matplotlib.pyplot as plt with tf.Session() as sess: #sess.run(tf.global_variables_initializer()) saver = tf.train.Saver()
def res_block_aprelu(incoming, nb_blocks, out_channels, downsample=False, downsample_strides=2, batch_norm=True, bias=True, weights_init='variance_scaling', bias_init='zeros', regularizer='L2', weight_decay=0.0001, trainable=True, restore=True, reuse=False, scope=None, name="ResidualBlock"): resnet = incoming in_channels = incoming.get_shape().as_list()[-1] # Variable Scope fix for older TF try: vscope = tf.variable_scope(scope, default_name=name, values=[incoming], reuse=reuse) except Exception: vscope = tf.variable_op_scope([incoming], scope, name, reuse=reuse) with vscope as scope: name = scope.name #TODO for i in range(nb_blocks): identity = resnet if not downsample: downsample_strides = 1 if batch_norm: resnet = bn(resnet) resnet = aprelu(resnet) resnet = conv_2d(resnet, out_channels, 3, downsample_strides, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) if batch_norm: resnet = bn(resnet) resnet = aprelu(resnet) resnet = conv_2d(resnet, out_channels, 3, 1, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) # Downsampling if downsample_strides > 1: identity = tflearn.avg_pool_2d(identity, 1, downsample_strides) # Projection to new dimension if in_channels != out_channels: if (out_channels - in_channels) % 2 == 0: ch = (out_channels - in_channels)//2 identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch]]) else: ch = (out_channels - in_channels)//2 identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch+1]]) in_channels = out_channels resnet = resnet + identity return resnet
def network(self, input): datadict = np.load('vgg16_weights.npz') #print(datadict.keys()) #print(datadict['conv1_1_W'].shape) conv1_1 = tf.constant(datadict['conv1_1_W']) bias1_1 = tf.constant(datadict['conv1_1_b']) conv1_2 = tf.constant(datadict['conv1_2_W']) bias1_2 = tf.constant(datadict['conv1_1_b']) conv2_1 = tf.constant(datadict['conv2_1_W']) bias2_1 = tf.constant(datadict['conv2_1_b']) conv2_2 = tf.constant(datadict['conv2_2_W']) bias2_2 = tf.constant(datadict['conv2_2_b']) conv3_1 = tf.constant(datadict['conv3_1_W']) bias3_1 = tf.constant(datadict['conv3_1_b']) conv3_2 = tf.constant(datadict['conv3_2_W']) bias3_2 = tf.constant(datadict['conv3_2_b']) conv3_3 = tf.constant(datadict['conv3_3_W']) bias3_3 = tf.constant(datadict['conv3_3_b']) conv4_1 = tf.constant(datadict['conv4_1_W']) bias4_1 = tf.constant(datadict['conv4_1_b']) conv4_2 = tf.constant(datadict['conv4_2_W']) bias4_2 = tf.constant(datadict['conv4_2_b']) conv4_3 = tf.constant(datadict['conv4_3_W']) bias4_3 = tf.constant(datadict['conv4_3_b']) conv5_1 = tf.constant(datadict['conv5_1_W']) bias5_1 = tf.constant(datadict['conv5_1_b']) conv5_2 = tf.constant(datadict['conv5_2_W']) bias5_2 = tf.constant(datadict['conv5_2_b']) conv5_3 = tf.constant(datadict['conv5_3_W']) bias5_3 = tf.constant(datadict['conv5_3_b']) fc6_w = tf.constant(datadict['fc6_W']) fc6_b = tf.constant(datadict['fc6_b']) fc7_w = tf.constant(datadict['fc7_W']) fc7_b = tf.constant(datadict['fc7_b']) x = tflearn.conv_2d(input, 64, 3, activation='relu', weights_init=conv1_1, bias_init=bias1_1, scope='conv1_1') x = tflearn.conv_2d(x, 64, 3, activation='relu', weights_init=conv1_2, bias_init=bias1_2, scope='conv1_2') x = tflearn.max_pool_2d(x, 2, strides=2) '''kernel = tf.Variable(tf.truncated_normal([1,1,64,64],dtype=tf.float32,stddev=1e-1),name='attention_conv1_weights') conv = tf.nn.conv2d(x,kernel,[1,1,1,1],padding='SAME') conv = tf.nn.relu(conv) kernel2 = tf.Variable(tf.truncated_normal([1,1,64,1],dtype=tf.float32,stddev=1e-1), name='attention_conv2_weights') conv2 = tf.nn.conv2d(conv,kernel2,[1,1,1,1], padding='SAME') attention_score = tf.nn.relu(conv2) x = attention_score * x c = tflearn.avg_pool_2d(x, 112) c = tflearn.flatten(c) c1 = tflearn.fully_connected(c, 4, activation='relu') c2 = tflearn.fully_connected(c1, 64, activation='sigmoid') c2 = tf.expand_dims(c2, 1) c2 = tf.expand_dims(c2, 2) x = x * c2''' #attention_c = tf.get_variable('attention_c', shape=[64],initializer=tf.random_normal_initializer()) #x = x * attention_c x = tflearn.conv_2d(x, 128, 3, activation='relu', weights_init=conv2_1, bias_init=bias2_1, scope='conv2_1') x = tflearn.conv_2d(x, 128, 3, activation='relu', weights_init=conv2_2, bias_init=bias2_2, scope='conv2_2') x = tflearn.max_pool_2d(x, 2, strides=2) '''kernel = tf.get_variable(name='attention_conv3_weights',initializer=tf.truncated_normal([1,1,128,128],dtype=tf.float32,stddev=1e-1)) conv = tf.nn.conv2d(x, kernel,[1,1,1,1],padding='SAME') conv = tf.nn.relu(conv) kernel2 = tf.get_variable(name='attention_conv4_weights', initializer=tf.truncated_normal([1,1,128,1],dtype=tf.float32, stddev=1e-1)) conv2 = tf.nn.conv2d(conv,kernel2,[1,1,1,1], padding='SAME') attention_score2= tf.nn.relu(conv2) x = x * attention_score2 c = tflearn.avg_pool_2d(x,56) c = tflearn.flatten(c) c3 = tflearn.fully_connected(c,8,activation='relu',scope='c3') c4 = tflearn.fully_connected(c3,128, activation='sigmoid',scope='c4') c4 = tf.expand_dims(c4,1) c4 = tf.expand_dims(c4, 2) x = x * c4''' x = tflearn.conv_2d(x, 256, 3, activation='relu', weights_init=conv3_1, bias_init=bias3_1, scope='conv3_1') x = tflearn.conv_2d(x, 256, 3, activation='relu', weights_init=conv3_2, bias_init=bias3_2, scope='conv3_2') x = tflearn.conv_2d(x, 256, 3, activation='relu', weights_init=conv3_3, bias_init=bias3_3, scope='conv3_3') x = tflearn.max_pool_2d(x, 2, strides=2) '''kernel = tf.get_variable(name='attention_conv3_weights',initializer=tf.truncated_normal([1,1,256,256],dtype=tf.float32,stddev=1e-1)) conv = tf.nn.conv2d(x, kernel,[1,1,1,1],padding='SAME') conv = tf.nn.relu(conv) kernel2 = tf.get_variable(name='attention_conv4_weights', initializer=tf.truncated_normal([1,1,256,1],dtype=tf.float32, stddev=1e-1)) conv2 = tf.nn.conv2d(conv,kernel2,[1,1,1,1], padding='SAME') attention_score2= tf.nn.relu(conv2) x = x * attention_score2 c = tflearn.avg_pool_2d(x,28) c = tflearn.flatten(c) c3 = tflearn.fully_connected(c,16,activation='relu',scope='c3') c4 = tflearn.fully_connected(c3,256, activation='sigmoid',scope='c4') c4 = tf.expand_dims(c4,1) c4 = tf.expand_dims(c4, 2) x = x * c4''' x = tflearn.conv_2d(x, 512, 3, activation='relu', weights_init=conv4_1, bias_init=bias4_1, scope='conv4_1') x = tflearn.conv_2d(x, 512, 3, activation='relu', weights_init=conv4_2, bias_init=bias4_2, scope='conv4_2') x = tflearn.conv_2d(x, 512, 3, activation='relu', weights_init=conv4_3, bias_init=bias4_3, scope='conv4_3') x = tflearn.max_pool_2d(x, 2, strides=2) x = tflearn.conv_2d(x, 512, 3, activation='relu', weights_init=conv5_1, bias_init=bias5_1, scope='conv5_1') x = tflearn.conv_2d(x, 512, 3, activation='relu', weights_init=conv5_2, bias_init=bias5_2, scope='conv5_2') x = tflearn.conv_2d(x, 512, 3, activation='relu', weights_init=conv5_3, bias_init=bias5_3, scope='conv5_3') x = tflearn.max_pool_2d(x, 2, strides=2) '''kernel = tf.Variable(tf.truncated_normal([1,1,512,512],dtype=tf.float32,stddev=1e-1),name='attention_1_weights') conv = tf.nn.conv2d(x, kernel,[1,1,1,1],padding='SAME') f_conv1_attention = tf.nn.relu(conv) kernel2 = tf.Variable(tf.truncated_normal([1,1,512,1],dtype=tf.float32,stddev=1e-1),name='attention_2_weights') conv2 = tf.nn.conv2d(f_conv1_attention, kernel2,[1,1,1,1],padding='SAME') attention_score = tf.nn.relu(conv2, name='score') attention_feat = x * attention_score #attention_feat = tf.expand_dims(tf.expand_dims(attention_feat, 1), 2)''' #attention_c = tf.get_variable('attention_c_final', shape=[512],initializer=tf.random_normal_initializer()) #x = x * attention_c #kernel3 = tf.Variable(tf.truncated_normal([1,1,512,num_class],dtype=tf.float32, # stddev=1e-1),name='m2d_weights') #conv3 = tf.nn.conv2d(x, kernel3,[1,1,1,1],padding='SAME') #logits = tf.squeeze(conv3,[1,2]) #x = tf.nn.softmax(logits) kernel = tf.get_variable(name='attention_conv3_weights', initializer=tf.truncated_normal( [1, 1, 512, 512], dtype=tf.float32, stddev=1e-1)) conv = tf.nn.conv2d(x, kernel, [1, 1, 1, 1], padding='SAME') conv = tf.nn.relu(conv) kernel2 = tf.get_variable(name='attention_conv4_weights', initializer=tf.truncated_normal( [1, 1, 512, 1], dtype=tf.float32, stddev=1e-1)) conv2 = tf.nn.conv2d(conv, kernel2, [1, 1, 1, 1], padding='SAME') attention_score2 = tf.nn.relu(conv2) x = x * attention_score2 c = tflearn.avg_pool_2d(x, 7) c = tflearn.flatten(c) c3 = tflearn.fully_connected(c, 32, activation='relu', scope='c3') c4 = tflearn.fully_connected(c3, 512, activation='sigmoid', scope='c4') c4 = tf.expand_dims(c4, 1) c4 = tf.expand_dims(c4, 2) x = x * c4 x = tflearn.fully_connected(x, 4096, activation='relu', weights_init=fc6_w, bias_init=fc6_b, scope='fc6') x = tflearn.dropout(x, 0.5, name='dropout1') x = tflearn.fully_connected(x, 4096, activation='relu', weights_init=fc7_w, bias_init=fc7_b, scope='fc7') return x
def inception(input_layer, num_class): conv1_7_7 = tflearn.conv_2d(input_layer, 64, 7, strides=2, activation='relu', name='conv1_7_7_s2') pool1_3_3 = tflearn.max_pool_2d(conv1_7_7, 3, strides=2) pool1_3_3 = tflearn.local_response_normalization(pool1_3_3) conv2_3_3_reduce = tflearn.conv_2d(pool1_3_3, 64, 1, activation='relu', name='conv2_3_3_reduce') conv2_3_3 = tflearn.conv_2d(conv2_3_3_reduce, 192, 3, activation='relu', name='conv2_3_3') conv2_3_3 = tflearn.local_response_normalization(conv2_3_3) pool2_3_3 = tflearn.max_pool_2d(conv2_3_3, kernel_size=3, strides=2, name='pool2_3_3_s2') inception_3a_1_1 = tflearn.conv_2d(pool2_3_3, 64, 1, activation='relu', name='inception_3a_1_1') inception_3a_3_3_reduce = tflearn.conv_2d(pool2_3_3, 96, 1, activation='relu', name='inception_3a_3_3_reduce') inception_3a_3_3 = tflearn.conv_2d(inception_3a_3_3_reduce, 128, filter_size=3, activation='relu', name='inception_3a_3_3') inception_3a_5_5_reduce = tflearn.conv_2d(pool2_3_3, 16, filter_size=1, activation='relu', name='inception_3a_5_5_reduce') inception_3a_5_5 = tflearn.conv_2d(inception_3a_5_5_reduce, 32, filter_size=5, activation='relu', name='inception_3a_5_5') inception_3a_pool = tflearn.max_pool_2d( pool2_3_3, kernel_size=3, strides=1, ) inception_3a_pool_1_1 = tflearn.conv_2d(inception_3a_pool, 32, filter_size=1, activation='relu', name='inception_3a_pool_1_1') # merge the inception_3a__ inception_3a_output = tflearn.merge([ inception_3a_1_1, inception_3a_3_3, inception_3a_5_5, inception_3a_pool_1_1 ], mode='concat', axis=3) inception_3b_1_1 = tflearn.conv_2d(inception_3a_output, 128, filter_size=1, activation='relu', name='inception_3b_1_1') inception_3b_3_3_reduce = tflearn.conv_2d(inception_3a_output, 128, filter_size=1, activation='relu', name='inception_3b_3_3_reduce') inception_3b_3_3 = tflearn.conv_2d(inception_3b_3_3_reduce, 192, filter_size=3, activation='relu', name='inception_3b_3_3') inception_3b_5_5_reduce = tflearn.conv_2d(inception_3a_output, 32, filter_size=1, activation='relu', name='inception_3b_5_5_reduce') inception_3b_5_5 = tflearn.conv_2d(inception_3b_5_5_reduce, 96, filter_size=5, name='inception_3b_5_5') inception_3b_pool = tflearn.max_pool_2d(inception_3a_output, kernel_size=3, strides=1, name='inception_3b_pool') inception_3b_pool_1_1 = tflearn.conv_2d(inception_3b_pool, 64, filter_size=1, activation='relu', name='inception_3b_pool_1_1') # merge the inception_3b_* inception_3b_output = tflearn.merge([ inception_3b_1_1, inception_3b_3_3, inception_3b_5_5, inception_3b_pool_1_1 ], mode='concat', axis=3, name='inception_3b_output') pool3_3_3 = tflearn.max_pool_2d(inception_3b_output, kernel_size=3, strides=2, name='pool3_3_3') inception_4a_1_1 = tflearn.conv_2d(pool3_3_3, 192, filter_size=1, activation='relu', name='inception_4a_1_1') inception_4a_3_3_reduce = tflearn.conv_2d(pool3_3_3, 96, filter_size=1, activation='relu', name='inception_4a_3_3_reduce') inception_4a_3_3 = tflearn.conv_2d(inception_4a_3_3_reduce, 208, filter_size=3, activation='relu', name='inception_4a_3_3') inception_4a_5_5_reduce = tflearn.conv_2d(pool3_3_3, 16, filter_size=1, activation='relu', name='inception_4a_5_5_reduce') inception_4a_5_5 = tflearn.conv_2d(inception_4a_5_5_reduce, 48, filter_size=5, activation='relu', name='inception_4a_5_5') inception_4a_pool = tflearn.max_pool_2d(pool3_3_3, kernel_size=3, strides=1, name='inception_4a_pool') inception_4a_pool_1_1 = tflearn.conv_2d(inception_4a_pool, 64, filter_size=1, activation='relu', name='inception_4a_pool_1_1') inception_4a_output = tflearn.merge([ inception_4a_1_1, inception_4a_3_3, inception_4a_5_5, inception_4a_pool_1_1 ], mode='concat', axis=3, name='inception_4a_output') inception_4b_1_1 = tflearn.conv_2d(inception_4a_output, 160, filter_size=1, activation='relu', name='inception_4a_1_1') inception_4b_3_3_reduce = tflearn.conv_2d(inception_4a_output, 112, filter_size=1, activation='relu', name='inception_4b_3_3_reduce') inception_4b_3_3 = tflearn.conv_2d(inception_4b_3_3_reduce, 224, filter_size=3, activation='relu', name='inception_4b_3_3') inception_4b_5_5_reduce = tflearn.conv_2d(inception_4a_output, 24, filter_size=1, activation='relu', name='inception_4b_5_5_reduce') inception_4b_5_5 = tflearn.conv_2d(inception_4b_5_5_reduce, 64, filter_size=5, activation='relu', name='inception_4b_5_5') inception_4b_pool = tflearn.max_pool_2d(inception_4a_output, kernel_size=3, strides=1, name='inception_4b_pool') inception_4b_pool_1_1 = tflearn.conv_2d(inception_4b_pool, 64, filter_size=1, activation='relu', name='inception_4b_pool_1_1') inception_4b_output = tflearn.merge([ inception_4b_1_1, inception_4b_3_3, inception_4b_5_5, inception_4b_pool_1_1 ], mode='concat', axis=3, name='inception_4b_output') inception_4c_1_1 = tflearn.conv_2d(inception_4b_output, 128, filter_size=1, activation='relu', name='inception_4c_1_1') inception_4c_3_3_reduce = tflearn.conv_2d(inception_4b_output, 128, filter_size=1, activation='relu', name='inception_4c_3_3_reduce') inception_4c_3_3 = tflearn.conv_2d(inception_4c_3_3_reduce, 256, filter_size=3, activation='relu', name='inception_4c_3_3') inception_4c_5_5_reduce = tflearn.conv_2d(inception_4b_output, 24, filter_size=1, activation='relu', name='inception_4c_5_5_reduce') inception_4c_5_5 = tflearn.conv_2d(inception_4c_5_5_reduce, 64, filter_size=5, activation='relu', name='inception_4c_5_5') inception_4c_pool = tflearn.max_pool_2d(inception_4b_output, kernel_size=3, strides=1) inception_4c_pool_1_1 = tflearn.conv_2d(inception_4c_pool, 64, filter_size=1, activation='relu', name='inception_4c_pool_1_1') inception_4c_output = tflearn.merge([ inception_4c_1_1, inception_4c_3_3, inception_4c_5_5, inception_4c_pool_1_1 ], mode='concat', axis=3, name='inception_4c_output') inception_4d_1_1 = tflearn.conv_2d(inception_4c_output, 112, filter_size=1, activation='relu', name='inception_4d_1_1') inception_4d_3_3_reduce = tflearn.conv_2d(inception_4c_output, 144, filter_size=1, activation='relu', name='inception_4d_3_3_reduce') inception_4d_3_3 = tflearn.conv_2d(inception_4d_3_3_reduce, 288, filter_size=3, activation='relu', name='inception_4d_3_3') inception_4d_5_5_reduce = tflearn.conv_2d(inception_4c_output, 32, filter_size=1, activation='relu', name='inception_4d_5_5_reduce') inception_4d_5_5 = tflearn.conv_2d(inception_4d_5_5_reduce, 64, filter_size=5, activation='relu', name='inception_4d_5_5') inception_4d_pool = tflearn.max_pool_2d(inception_4c_output, kernel_size=3, strides=1, name='inception_4d_pool') inception_4d_pool_1_1 = tflearn.conv_2d(inception_4d_pool, 64, filter_size=1, activation='relu', name='inception_4d_pool_1_1') inception_4d_output = tflearn.merge([ inception_4d_1_1, inception_4d_3_3, inception_4d_5_5, inception_4d_pool_1_1 ], mode='concat', axis=3, name='inception_4d_output') inception_4e_1_1 = tflearn.conv_2d(inception_4d_output, 256, filter_size=1, activation='relu', name='inception_4e_1_1') inception_4e_3_3_reduce = tflearn.conv_2d(inception_4d_output, 160, filter_size=1, activation='relu', name='inception_4e_3_3_reduce') inception_4e_3_3 = tflearn.conv_2d(inception_4e_3_3_reduce, 320, filter_size=3, activation='relu', name='inception_4e_3_3') inception_4e_5_5_reduce = tflearn.conv_2d(inception_4d_output, 32, filter_size=1, activation='relu', name='inception_4e_5_5_reduce') inception_4e_5_5 = tflearn.conv_2d(inception_4e_5_5_reduce, 128, filter_size=5, activation='relu', name='inception_4e_5_5') inception_4e_pool = tflearn.max_pool_2d(inception_4d_output, kernel_size=3, strides=1, name='inception_4e_pool') inception_4e_pool_1_1 = tflearn.conv_2d(inception_4e_pool, 128, filter_size=1, activation='relu', name='inception_4e_pool_1_1') inception_4e_output = tflearn.merge([ inception_4e_1_1, inception_4e_3_3, inception_4e_5_5, inception_4e_pool_1_1 ], axis=3, mode='concat') pool4_3_3 = tflearn.max_pool_2d(inception_4e_output, kernel_size=3, strides=2, name='pool_3_3') inception_5a_1_1 = tflearn.conv_2d(pool4_3_3, 256, filter_size=1, activation='relu', name='inception_5a_1_1') inception_5a_3_3_reduce = tflearn.conv_2d(pool4_3_3, 160, filter_size=1, activation='relu', name='inception_5a_3_3_reduce') inception_5a_3_3 = tflearn.conv_2d(inception_5a_3_3_reduce, 320, filter_size=3, activation='relu', name='inception_5a_3_3') inception_5a_5_5_reduce = tflearn.conv_2d(pool4_3_3, 32, filter_size=1, activation='relu', name='inception_5a_5_5_reduce') inception_5a_5_5 = tflearn.conv_2d(inception_5a_5_5_reduce, 128, filter_size=5, activation='relu', name='inception_5a_5_5') inception_5a_pool = tflearn.max_pool_2d(pool4_3_3, kernel_size=3, strides=1, name='inception_5a_pool') inception_5a_pool_1_1 = tflearn.conv_2d(inception_5a_pool, 128, filter_size=1, activation='relu', name='inception_5a_pool_1_1') inception_5a_output_ = tflearn.merge([ inception_5a_1_1, inception_5a_3_3, inception_5a_5_5, inception_5a_pool_1_1 ], axis=3, mode='concat') inception_5a_output = tflearn.dropout(inception_5a_output_, 0.45) inception_5b_1_1 = tflearn.conv_2d(inception_5a_output, 384, filter_size=1, activation='relu', name='inception_5b_1_1') inception_5b_3_3_reduce = tflearn.conv_2d(inception_5a_output, 192, filter_size=1, activation='relu', name='inception_5b_3_3_reduce') inception_5b_3_3 = tflearn.conv_2d(inception_5b_3_3_reduce, 384, filter_size=3, activation='relu', name='inception_5b_3_3') inception_5b_5_5_reduce = tflearn.conv_2d(inception_5a_output, 48, filter_size=1, activation='relu', name='inception_5b_5_5_reduce') inception_5b_5_5 = tflearn.conv_2d(inception_5b_5_5_reduce, 128, filter_size=5, activation='relu', name='inception_5b_5_5') inception_5b_pool = tflearn.max_pool_2d(inception_5a_output, kernel_size=3, strides=1, name='inception_5b_pool') inception_5b_pool_1_1 = tflearn.conv_2d(inception_5b_pool, 128, filter_size=1, activation='relu', name='inception_5b_pool_1_1') inception_5b_output = tflearn.merge([ inception_5b_1_1, inception_5b_3_3, inception_5b_5_5, inception_5b_pool_1_1 ], axis=3, mode='concat') pool5_7_7 = tflearn.avg_pool_2d(inception_5b_output, kernel_size=7, strides=1) pool5_7_7 = tflearn.dropout(pool5_7_7, 0.65) loss = tflearn.fully_connected(pool5_7_7, num_class, activation='softmax') return loss
def densenet_block(self,incoming, nb_layers, growth, bottleneck=True, downsample=True, downsample_strides=2, activation='relu', batch_norm=True, dropout=False, dropout_keep_prob=0.5, weights_init='variance_scaling', regularizer='L2', weight_decay=0.0001, bias=True, bias_init='zeros', trainable=True, restore=True, reuse=False, scope=None, name="DenseNetBlock"): """ DenseNet Block. A DenseNet block as described in DenseNet paper. Input: 4-D Tensor [batch, height, width, in_channels]. Output: 4-D Tensor [batch, new height, new width, out_channels]. Arguments: incoming: `Tensor`. Incoming 4-D Layer. nb_blocks: `int`. Number of layer blocks. growth: `int`. DenseNet 'growth': The number of convolutional filters of each convolution. bottleneck: `bool`. If True, add a 1x1 convolution before the 3x3 convolution to reduce the number of input features map. downsample: `bool`. If True, apply downsampling using 'downsample_strides' for strides. downsample_strides: `int`. The strides to use when downsampling. activation: `str` (name) or `function` (returning a `Tensor`). Activation applied to this layer (see tflearn.activations). Default: 'linear'. batch_norm: `bool`. If True, apply batch normalization. dropout: `bool`. If True, apply dropout. Use 'dropout_keep_prob' to specify the keep probability. dropout_keep_prob: `float`. Keep probability parameter for dropout. bias: `bool`. If True, a bias is used. weights_init: `str` (name) or `Tensor`. Weights initialization. (see tflearn.initializations) Default: 'uniform_scaling'. bias_init: `str` (name) or `tf.Tensor`. Bias initialization. (see tflearn.initializations) Default: 'zeros'. regularizer: `str` (name) or `Tensor`. Add a regularizer to this layer weights (see tflearn.regularizers). Default: None. weight_decay: `float`. Regularizer decay parameter. Default: 0.001. trainable: `bool`. If True, weights will be trainable. restore: `bool`. If True, this layer weights will be restored when loading a model. reuse: `bool`. If True and 'scope' is provided, this layer variables will be reused (shared). scope: `str`. Define this layer scope (optional). A scope can be used to share variables between layers. Note that scope will override name. name: A name for this layer (optional). Default: 'ResNeXtBlock'. References: Densely Connected Convolutional Networks, G. Huang, Z. Liu, K. Q. Weinberger, L. van der Maaten. 2016. Links: [https://arxiv.org/abs/1608.06993] (https://arxiv.org/abs/1608.06993) """ densenet = incoming with tf.variable_scope(scope, default_name=name, values=[incoming], reuse=reuse) as scope: for i in range(nb_layers): # Identity conn = densenet # 1x1 Conv layer of the bottleneck block if bottleneck: if batch_norm: densenet = tflearn.batch_normalization(densenet) densenet = tflearn.activation(densenet, activation) densenet = conv_2d(densenet, nb_filter=growth, filter_size=1, bias=bias, weights_init=weights_init, bias_init=bias_init, regularizer=regularizer, weight_decay=weight_decay, trainable=trainable, restore=restore) # 3x3 Conv layer if batch_norm: densenet = tflearn.batch_normalization(densenet) densenet = tflearn.activation(densenet, activation) densenet = conv_2d(densenet, nb_filter=growth, filter_size=3, bias=bias, weights_init=weights_init, bias_init=bias_init, regularizer=regularizer, weight_decay=weight_decay, trainable=trainable, restore=restore) # Connections densenet = tf.concat([densenet, conn], 3) # 1x1 Transition Conv if batch_norm: densenet = tflearn.batch_normalization(densenet) densenet = tflearn.activation(densenet, activation) densenet = conv_2d(densenet, nb_filter=growth, filter_size=1, bias=bias, weights_init=weights_init, bias_init=bias_init, regularizer=regularizer, weight_decay=weight_decay, trainable=trainable, restore=restore) if dropout: densenet = tflearn.dropout(densenet, keep_prob=dropout_keep_prob) # Downsampling if downsample: densenet = tflearn.avg_pool_2d(densenet, kernel_size=2, strides=downsample_strides) return densenet
import tflearn from tensorflow.examples.tutorials.mnist import input_data net = tflearn.input_data([None, 28,28,1]) net = tflearn.conv_2d(net, nb_filter=6, filter_size=5, strides=1, padding='SAME', activation='sigmoid') net = tflearn.avg_pool_2d(net, kernel_size=2, strides=2, padding='SAME') net = tflearn.conv_2d(net, nb_filter=16, filter_size=5, strides=1, padding='VALID', activation='sigmoid') net = tflearn.avg_pool_2d(net, kernel_size=2, strides=2, padding='SAME') net = tflearn.fully_connected(net, n_units=120, activation='sigmoid') net = tflearn.fully_connected(net, n_units=84, activation='sigmoid') net = tflearn.fully_connected(net, n_units=10, activation='softmax') net = tflearn.regression(net, learning_rate=0.001, optimizer='rmsprop', loss='categorical_crossentropy') mnist = input_data.read_data_sets('mnist', one_hot=True) x = mnist.train.images.reshape(-1, 28, 28, 1) y = mnist.train.labels model = tflearn.DNN(net) model.fit(x, y, n_epoch=10, batch_size=128, shuffle=True, validation_set=0.2) x = mnist.test.images.reshape(-1, 28, 28, 1) y = mnist.test.labels acc = model.evaluate(x, y)
def residual_shrinkage_block(incoming, nb_blocks, out_channels, downsample=False, downsample_strides=2, activation='relu', batch_norm=True, bias=True, weights_init='variance_scaling', bias_init='zeros', regularizer='L2', weight_decay=0.0001, trainable=True, restore=True, reuse=False, scope=None, name="ResidualBlock"): # residual shrinkage blocks with channel-wise thresholds residual = incoming in_channels = incoming.get_shape().as_list()[-1] # Variable Scope fix for older TF try: vscope = tf.variable_scope(scope, default_name=name, values=[incoming], reuse=reuse) except Exception: vscope = tf.variable_op_scope([incoming], scope, name, reuse=reuse) with vscope as scope: name = scope.name #TODO for i in range(nb_blocks): identity = residual if not downsample: downsample_strides = 1 if batch_norm: residual = tflearn.batch_normalization(residual) residual = tflearn.activation(residual, activation) residual = conv_2d(residual, out_channels, 3, downsample_strides, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) if batch_norm: residual = tflearn.batch_normalization(residual) residual = tflearn.activation(residual, activation) residual = conv_2d(residual, out_channels, 3, 1, 'same', 'linear', bias, weights_init, bias_init, regularizer, weight_decay, trainable, restore) # get thresholds and apply thresholding abs_mean = tf.reduce_mean(tf.reduce_mean(tf.abs(residual), axis=2, keep_dims=True), axis=1, keep_dims=True) scales = tflearn.fully_connected(abs_mean, out_channels // 4, activation='linear', regularizer='L2', weight_decay=0.0001, weights_init='variance_scaling') scales = tflearn.batch_normalization(scales) scales = tflearn.activation(scales, 'relu') scales = tflearn.fully_connected(scales, out_channels, activation='linear', regularizer='L2', weight_decay=0.0001, weights_init='variance_scaling') scales = tf.expand_dims(tf.expand_dims(scales, axis=1), axis=1) thres = tf.multiply(abs_mean, tflearn.activations.sigmoid(scales)) # soft thresholding residual = tf.multiply(tf.sign(residual), tf.maximum(tf.abs(residual) - thres, 0)) # Downsampling if downsample_strides > 1: identity = tflearn.avg_pool_2d(identity, 1, downsample_strides) # Projection to new dimension if in_channels != out_channels: if (out_channels - in_channels) % 2 == 0: ch = (out_channels - in_channels) // 2 identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch]]) else: ch = (out_channels - in_channels) // 2 identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch + 1]]) in_channels = out_channels residual = residual + identity return residual
addt = tf.add(resn_ud, coct, name='addt') ups = tf.image.resize_nearest_neighbor(addt, (ih, iw), name='ups') return ups lr = 0.0001 tf.reset_default_graph() net_input = input_data(shape=[None, parth, partw, n_channels], name='net_input') net_input_h = net_input.get_shape()[1] net_input_w = net_input.get_shape()[2] resn1 = resn_unit(net_input, 4, 64, name='resn1') resn1_ds = avg_pool_2d(resn1, 2, name='resn1_ds') resn1_ds_h = resn1_ds.get_shape()[1] resn1_ds_w = resn1_ds.get_shape()[2] resn2 = resn_unit(resn1_ds, 4, 128, name='resn2') resn2_ds = avg_pool_2d(resn2, 2, name='resn2_ds') resn2_ds_h = resn2_ds.get_shape()[1] resn2_ds_w = resn2_ds.get_shape()[2] resn3 = resn_unit(resn2_ds, 4, 256, name='resn3') resn3_ds = avg_pool_2d(resn3, 2, name='resn3_ds') resn_mid = resn_unit(resn3_ds, 4, 512, name='dens_mid') resn_upadd1 = resn_ups_concat(resn_mid, resn3_ds, 4, 256,
Y = du.to_categorical(Y, 10) testY = du.to_categorical(testY, 10) # Building Residual Network net = tflearn.input_data(shape=[None, 32, 32, 3]) net = tflearn.conv_2d(net, 32, 3) net = tflearn.batch_normalization(net) net = tflearn.activation(net, 'relu') net = tflearn.shallow_residual_block(net, 4, 32, regularizer='L2') net = tflearn.shallow_residual_block(net, 1, 32, downsample=True, regularizer='L2') net = tflearn.shallow_residual_block(net, 4, 64, regularizer='L2') net = tflearn.shallow_residual_block(net, 1, 64, downsample=True, regularizer='L2') net = tflearn.shallow_residual_block(net, 5, 128, regularizer='L2') net_shape = net.get_shape().as_list() k_size = [1, net_shape[1], net_shape[2], 1] net = tflearn.avg_pool_2d(net, k_size, strides=1) # Regression net = tflearn.fully_connected(net, 10, activation='softmax') mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=16000, staircase=True) net = tflearn.regression(net, optimizer=mom, loss='categorical_crossentropy') # Training model = tflearn.DNN(net, checkpoint_path='model_resnet_cifar10', max_checkpoints=10, tensorboard_verbose=0, clip_gradients=1.0) model.fit(X, Y, n_epoch=200, validation_set=(testX, testY), show_metric=True, batch_size=128, shuffle=True, run_id='resnet_cifar10')
X, mean = du.featurewise_zero_center(X) testX = du.featurewise_zero_center(testX, mean) # Building Residual Network net = tflearn.input_data(shape=[None, 28, 28, 1]) net = tflearn.conv_2d(net, 64, 3, activation='relu', bias=False) net = tflearn.batch_normalization(net) # Residual blocks net = tflearn.deep_residual_block(net, 3, 64) net = tflearn.deep_residual_block(net, 1, 128, downsample=True) net = tflearn.deep_residual_block(net, 3, 128) net = tflearn.deep_residual_block(net, 1, 256, downsample=True) net = tflearn.deep_residual_block(net, 3, 256) net_shape = net.get_shape().as_list() k_size = [1, net_shape[1], net_shape[2], 1] net = tflearn.avg_pool_2d(net, k_size, padding='valid', strides=1) # Regression net = tflearn.fully_connected(net, 10, activation='softmax') sgd = tflearn.SGD(learning_rate=0.1, lr_decay=0.96, decay_step=300) net = tflearn.regression(net, optimizer='sgd', loss='categorical_crossentropy', learning_rate=0.1) # Training model = tflearn.DNN(net, checkpoint_path='model_resnet_mnist', max_checkpoints=10, tensorboard_verbose=0) model.fit(X, Y, n_epoch=100,
def densenet_block(incoming, nb_layers, growth, bottleneck=True, downsample=True, downsample_strides=2, activation='relu', batch_norm=True, dropout=False, dropout_keep_prob=0.5, weights_init='variance_scaling', regularizer='L2', weight_decay=0.0001, bias=True, bias_init='zeros', trainable=True, restore=True, reuse=False, scope=None, name="DenseNetBlock"): densenet = incoming with tf.variable_scope(scope, default_name=name, values=[incoming], reuse=reuse) as scope: for i in range(nb_layers): # Identity conn = densenet # 1x1 Conv layer of the bottleneck block if bottleneck: if batch_norm: densenet = tflearn.batch_normalization(densenet) densenet = tflearn.activation(densenet, activation) densenet = conv_2d(densenet, nb_filter=growth, filter_size=1, bias=bias, weights_init=weights_init, bias_init=bias_init, regularizer=regularizer, weight_decay=weight_decay, trainable=trainable, restore=restore) # 3x3 Conv layer if batch_norm: densenet = tflearn.batch_normalization(densenet) densenet = tflearn.activation(densenet, activation) densenet = conv_2d(densenet, nb_filter=growth, filter_size=3, bias=bias, weights_init=weights_init, bias_init=bias_init, regularizer=regularizer, weight_decay=weight_decay, trainable=trainable, restore=restore) # Connections densenet = tf.concat([densenet, conn], 3) # 1x1 Transition Conv if batch_norm: densenet = tflearn.batch_normalization(densenet) densenet = tflearn.activation(densenet, activation) densenet = conv_2d(densenet, nb_filter=growth, filter_size=1, bias=bias, weights_init=weights_init, bias_init=bias_init, regularizer=regularizer, weight_decay=weight_decay, trainable=trainable, restore=restore) if dropout: densenet = tflearn.dropout(densenet, keep_prob=dropout_keep_prob) # Downsampling if downsample: densenet = tflearn.avg_pool_2d(densenet, kernel_size=2, strides=downsample_strides) return densenet
conv2 = tflearn.conv_2d(l1, 64, 1, activation='relu') conv3 = tflearn.conv_2d(conv2, 192, 3, activation='relu') l2 = tflearn.local_response_normalization(conv3) m2 = tflearn.max_pool_2d(l2, kernel_size=3, strides=2) i1 = inception(m2, 96, 16, 64, 128, 32, 32) i2 = inception(i1, 128, 32, 128, 192, 96, 64) m3 = tflearn.max_pool_2d(i2, kernel_size=3, strides=2) i3 = inception(m3, 96, 16, 192, 208, 48, 64) i4 = inception(i3, 112, 24, 160, 224, 64, 64) i5 = inception(i4, 128, 24, 128, 256, 64, 64) i6 = inception(i5, 144, 32, 112, 288, 64, 64) i7 = inception(i6, 160, 32, 256, 320, 128, 128) m4 = tflearn.max_pool_2d(i7, kernel_size=3, strides=2) i8 = inception(m4, 160, 32, 256, 320, 128, 128) i9 = inception(i8, 192, 48, 384, 384, 128, 128) a1 = tflearn.avg_pool_2d(i9, kernel_size=7, strides=1) d1 = tflearn.dropout(a1, 0.4) f1 = tflearn.fully_connected(d1, 6, activation='softmax') network = tflearn.regression(f1, optimizer='momentum', loss='categorical_crossentropy', learning_rate=0.001) # 학습 model = tflearn.DNN(network, checkpoint_path='model_googlenet', max_checkpoints=1, tensorboard_verbose=3) model.load("model_googlenet-24000")
3, 5, activation='leaky_relu', name="small_CONV") network = tflearn.conv_2d(network, 3, 8, activation='leaky_relu', name="large_CONV") network = tflearn.max_pool_2d(network, 2, name="max_POOL") network = tflearn.conv_2d(network, 1, 5, activation='leaky_relu', name="final_CONV") network = tflearn.avg_pool_2d( network, 5, name="avg_POOL") # want avg here, to reduce outliers. network = tflearn.reshape(network, (-1, 12, 28), name="final") #no FULLY CONNECTED! network = tflearn.regression(network, optimizer='adam', learning_rate=0.01, loss='mean_square', name='target') model = tflearn.DNN(network, tensorboard_verbose=3, max_checkpoints=3, tensorboard_dir="./logs") print("Loading Model") model.load("current.model") for x in range(1000):
def resnext_block(incoming, nb_blocks, out_channels, cardinality, downsample=True, regularizer='L2', weight_decay=0.0001, restore=True, reuse=tf.AUTO_REUSE, scope=None, name="ResNeXtBlock"): in_channels = incoming.get_shape().as_list()[-1] for i in range(nb_blocks): with tf.variable_scope((scope or "") + "_" + str(i), default_name=name, reuse=tf.AUTO_REUSE): if i == 0 and downsample: strides = 2 identity = tflearn.avg_pool_2d(incoming, 2, 2, padding='valid') else: strides = 1 identity = incoming incoming = tflearn.conv_2d(incoming, out_channels / 2, 1, name='Conv2D_1', bias=False, regularizer='L2', weight_decay=0.0001) incoming = tflearn.batch_normalization(incoming, name='BN_1') incoming = tf.nn.relu(incoming) input_groups = tf.split(value=incoming, num_or_size_splits=cardinality, axis=3) output_groups = [ split(inp, strides=strides, order=str(k)) for k, inp in enumerate(input_groups) ] incoming = tf.concat(output_groups, axis=3) incoming = tflearn.conv_2d(incoming, out_channels, 1, name='Conv2D_2', bias=False, regularizer='L2', weight_decay=0.0001) incoming = tflearn.batch_normalization(incoming, name='BN_2') if in_channels != out_channels: ch = (out_channels - in_channels) // 2 identity = tf.pad(identity, [[0, 0], [0, 0], [0, 0], [ch, ch]]) in_channels = out_channels incoming = incoming + identity incoming = tf.nn.relu(incoming) return incoming