def inference1(input_images): # with slim.arg_scope([slim.conv2d],padding='SAME',weights_regularizer=slim.l2_regularizer(0.001)): with slim.arg_scope([slim.conv2d], kernel_size=3, padding='SAME'): with slim.arg_scope([slim.max_pool2d], kernel_size=2): x = slim.conv2d(input_images, num_outputs=64, scope='conv1_1') x = slim.conv2d(x, num_outputs=64, scope='conv1_2') x = slim.max_pool2d(x, scope='pool1') x = slim.conv2d(x, num_outputs=64, scope='conv2_1') x = slim.conv2d(x, num_outputs=64, scope='conv2_2') x = slim.max_pool2d(x, scope='pool2') x = slim.conv2d(x, num_outputs=128, scope='conv3_1') x = slim.conv2d(x, num_outputs=128, scope='conv3_2') x = slim.max_pool2d(x, scope='pool3') x = slim.flatten(x, scope='flatten') x = slim.fully_connected(x, num_outputs=1024, activation_fn=None, scope='fc1') x = tflearn.prelu(x) feature = slim.fully_connected(x, num_outputs=2, activation_fn=None, scope='fc2') x = tflearn.prelu(feature) x = slim.dropout(x, scope='dropout2') x = slim.fully_connected(x, num_outputs=10, activation_fn=None, scope='fc3') return x, feature
def dnn(x): # Using TFLearn PReLU activations ops x = tflearn.prelu(tf.add(tf.matmul(x, W1), b1)) tflearn.summaries.monitor_activation(x) # Monitor activation x = tflearn.prelu(tf.add(tf.matmul(x, W2), b2)) tflearn.summaries.monitor_activation(x) # Monitor activation x = tf.nn.softmax(tf.add(tf.matmul(x, W3), b3)) return x
def dnn(x): # Using TFLearn PReLU activation ops x = tflearn.prelu(tf.add(tf.matmul(x, W1), b1)) tflearn.summaries.monitor_activation(x) # Monitor activation x = tflearn.prelu(tf.add(tf.matmul(x, W2), b2)) tflearn.summaries.monitor_activation(x) # Monitor activation x = tf.nn.softmax(tf.add(tf.matmul(x, W3), b3)) return x
def inference(input_images): with slim.arg_scope([slim.conv2d], kernel_size=3, padding='SAME'): with slim.arg_scope([slim.max_pool2d], kernel_size=2): x = slim.conv2d(input_images, num_outputs=32, scope='conv1_1') x = slim.conv2d(x, num_outputs=32, scope='conv1_2') x = slim.max_pool2d(x, scope='pool1') x = slim.conv2d(x, num_outputs=64, scope='conv2_1') x = slim.conv2d(x, num_outputs=64, scope='conv2_2') x = slim.max_pool2d(x, scope='pool2') x = slim.conv2d(x, num_outputs=128, scope='conv3_1') x = slim.conv2d(x, num_outputs=128, scope='conv3_2') x = slim.max_pool2d(x, scope='pool3') x = slim.flatten(x, scope='flatten') feature = slim.fully_connected(x, num_outputs=2, activation_fn=None, scope='fc1') x = tflearn.prelu(feature) x = slim.fully_connected(x, num_outputs=10, activation_fn=None, scope='fc2') return x, feature
def bn_prelu_conv(inputs, output_channels, kernel_size, stride, is_training, name, padding='same', use_bias=False): with tf.variable_scope(name_or_scope=name): '''device control?''' bn = tf.contrib.layers.batch_norm( inputs=inputs, # tensor, first dimension of batch_size decay=0.9, # recommend trying decay=0.9 scale= True, # If True, multiply by gamma. If False, gamma is not used epsilon= 1e-5, # Small float added to variance to avoid dividing by zero updates_collections=None, # tf.GraphKeys.UPDATE_OPS, # updates_collections: Collections to collect the update ops for computation. # The updates_ops need to be executed with the train_op. # If None, a control dependency would be added to make sure the updates are computed in place is_training=is_training, # In training mode it would accumulate the statistics of the moments into moving_mean # and moving_variance using an exponential moving average with the given decay. scope=name + '_batch_norm', # variable_scope ) prelu = tflearn.prelu(bn, name=name + '_prelu') conv = _conv3d(prelu, output_channels, kernel_size, stride, padding=padding, use_bias=use_bias, name=name + '_conv') return conv
def residual_unit(data, num_filter, stride, dim_match, name, **kwargs): use_se = kwargs.get('version_se', 1) bn1 = slim.batch_norm(data, scope=name + '_bn1') conv1 = slim.conv2d(bn1, num_filter, [3, 3], stride=(1, 1), scope=name + '_conv1') bn2 = slim.batch_norm(conv1, scope=name + '_bn2') act1 = tflearn.prelu(bn2) conv2 = conv2d_same(act1, num_filter, 3, stride=stride, scope=name + '_conv2') bn3 = slim.batch_norm(conv2, scope=name + '_bn3') if use_se: #se begin #body = tflearn.global_avg_pool (bn3,name=name+'_se_pool1') body = slim.avg_pool2d( bn3, kernel_size=[int(bn3.shape[1]), int(bn3.shape[2])]) body = slim.conv2d(body, num_filter // 16, [1, 1], stride=(1, 1), scope=name + "_se_conv1") body = tflearn.prelu(body) body = slim.conv2d(body, num_filter, [1, 1], stride=(1, 1), scope=name + "_se_conv2") body = tf.sigmoid(body) bn3 = tf.multiply(bn3, body) #se end if dim_match: shortcut = data else: conv1sc = slim.conv2d(data, num_filter, [1, 1], stride=stride, padding="VALID", scope=name + '_conv1sc') shortcut = slim.batch_norm(conv1sc, scope=name + '_sc') return bn3 + shortcut
def __init__(self): self.x_input = tf.placeholder(tf.float32, shape=[None, 784]) self.y_input = tf.placeholder(tf.int64, shape=[None]) self.x_image = tf.reshape(self.x_input, [-1, 28, 28, 1]) with slim.arg_scope([slim.conv2d], kernel_size=3, padding='SAME'): with slim.arg_scope([slim.max_pool2d], kernel_size=2): x = slim.conv2d(self.x_image, num_outputs=32, scope='conv1_1') x = slim.conv2d(x, num_outputs=32, scope='conv1_2') x = slim.max_pool2d(x, scope='pool1') x = slim.conv2d(x, num_outputs=64, scope='conv2_1') x = slim.conv2d(x, num_outputs=64, scope='conv2_2') x = slim.max_pool2d(x, scope='pool2') x = slim.conv2d(x, num_outputs=128, scope='conv3_1') x = slim.conv2d(x, num_outputs=128, scope='conv3_2') x = slim.max_pool2d(x, scope='pool3') x = slim.flatten(x, scope='flatten') x = slim.fully_connected(x, num_outputs=32, activation_fn=None, scope='fc1') self.feature = x = slim.fully_connected(x, num_outputs=2, activation_fn=None, scope='fc2') x = tflearn.prelu(x) # x = slim.fully_connected(x, num_outputs=10, activation_fn=None, scope='fc3') # output layer self.W_fc2 = self._weight_variable([2, 10]) self.b_fc2 = self._bias_variable([10]) self.pre_softmax = tf.matmul(self.feature, self.W_fc2) + self.b_fc2 y_xent = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=self.y_input, logits=self.pre_softmax) self.xent = tf.reduce_sum(y_xent) self.y_pred = tf.argmax(self.pre_softmax, 1) correct_prediction = tf.equal(self.y_pred, self.y_input) self.num_correct = tf.reduce_sum( tf.cast(correct_prediction, tf.int64)) self.accuracy = tf.reduce_mean( tf.cast(correct_prediction, tf.float32))
def slim_conv_net(input_images, NROFCLASSES): with slim.arg_scope([slim.conv2d], kernel_size=3, padding='SAME'): with slim.arg_scope([slim.max_pool2d], kernel_size=2): x = slim.conv2d( input_images, num_outputs=32, weights_initializer=initializers.xavier_initializer(), scope='conv1_1') x = slim.conv2d( x, num_outputs=32, weights_initializer=initializers.xavier_initializer(), biases_initializer=tf.zeros_initializer(), scope='conv1_2') x = slim.max_pool2d(x, scope='pool1') x = slim.conv2d( x, num_outputs=64, weights_initializer=initializers.xavier_initializer(), biases_initializer=tf.zeros_initializer(), scope='conv2_1') x = slim.conv2d( x, num_outputs=64, weights_initializer=initializers.xavier_initializer(), biases_initializer=tf.zeros_initializer(), scope='conv2_2') x = slim.max_pool2d(x, scope='pool2') x = slim.conv2d( x, num_outputs=128, weights_initializer=initializers.xavier_initializer(), biases_initializer=tf.zeros_initializer(), scope='conv3_1') x = slim.conv2d( x, num_outputs=128, weights_initializer=initializers.xavier_initializer(), biases_initializer=tf.zeros_initializer(), scope='conv3_2') x = slim.max_pool2d(x, scope='pool3') x = slim.flatten(x, scope='flatten') feature = slim.fully_connected(x, num_outputs=128, activation_fn=None, scope='fc1') x = tflearn.prelu(feature) x = slim.fully_connected(x, num_outputs=NROFCLASSES, activation_fn=None, scope='fc2') return x
def resnet(images, units, is_training, num_stages, filter_list, bottleneck_layer_size, **kwargs): with slim.arg_scope( [slim.batch_norm], updates_collections=None, variables_collections=[tf.GraphKeys.TRAINABLE_VARIABLES], is_training=is_training, center=True, scale=True, epsilon=2e-5, decay=0.9): body = images body = slim.conv2d(body, filter_list[0], [3, 3], stride=(1, 1), padding="SAME", scope="conv0") body = slim.batch_norm(body, scope='bn0') body = tflearn.prelu(body) for i in range(num_stages): body = residual_unit(body, filter_list[i + 1], (2, 2), False, name='stage%d_unit%d' % (i + 1, 1), **kwargs) for j in range(units[i] - 1): body = residual_unit(body, filter_list[i + 1], (1, 1), True, name='stage%d_unit%d' % (i + 1, j + 2), **kwargs) body = slim.batch_norm(body, scope='bn1') print(body) body = slim.dropout(body, 0.6, is_training=is_training, scope='Dropout') #body=slim.flatten(body) #body = slim.fully_connected(body, 512, activation_fn=None,scope='pre_fc1') net = slim.conv2d(body, bottleneck_layer_size, 7, activation_fn=None, padding="VALID", stride=1, scope='fc1') net = tf.squeeze(net, [1, 2], name='Bottleneck') body = slim.batch_norm(body, scope='fcend') body = tf.identity(body, name="output") return body
def activate(x, acti_mode, scope=None): if acti_mode==0: return x elif acti_mode==1: return tf.nn.relu(x) elif acti_mode==2: return tf.nn.sigmoid(x) elif acti_mode==3: return (tf.nn.tanh(x) + x) / 2 elif acti_mode==4: return (tf.nn.sigmoid(x) + x) / 2 elif acti_mode==5: return tf.nn.leaky_relu(x) elif acti_mode==6: return tflearn.prelu(x)
def bn_prelu_deconv(inputs, output_channels, is_training, name): with tf.variable_scope(name): '''device control?''' bn = tf.contrib.layers.batch_norm(inputs=inputs, decay=0.9, scale=True, epsilon=1e-5, updates_collections=None, is_training=is_training, scope=name + '_batch_norm') prelu = tflearn.prelu(bn, name=name + '_prelu') deconv = deconv3d(inputs=prelu, output_channels=output_channels, name=name + '_deconv') return deconv
def inference(input_images, num_class=200, reuse=False): with slim.arg_scope([slim.conv2d], kernel_size=3, padding='SAME'): with slim.arg_scope([slim.max_pool2d], kernel_size=2): x = slim.conv2d(input_images, num_outputs=64, scope='conv1_1') x = slim.conv2d(x, num_outputs=64, scope='conv1_2') x = slim.max_pool2d(x, scope='pool1') x = slim.conv2d(x, num_outputs=128, scope='conv2_1') x = slim.conv2d(x, num_outputs=128, scope='conv2_2') x = slim.max_pool2d(x, scope='pool2') x = slim.conv2d(x, num_outputs=256, scope='conv3_1') x = slim.conv2d(x, num_outputs=256, scope='conv3_2') x = slim.max_pool2d(x, scope='pool3') x = slim.conv2d(x, num_outputs=512, scope='conv4_1') x = slim.conv2d(x, num_outputs=512, scope='conv4_2') x = slim.max_pool2d(x, scope='pool3') x = slim.flatten(x, scope='flatten') feature3 = x = slim.fully_connected(x, num_outputs=512, activation_fn=None, scope='fc0') feature2 = x = slim.fully_connected(x, num_outputs=32, activation_fn=None, scope='fc1') feature1 = x = slim.fully_connected(x, num_outputs=2, activation_fn=None, scope='fc2') x = tflearn.prelu(feature3) x = slim.fully_connected(x, num_outputs=num_class, activation_fn=None, scope='fc3') feature_list = [feature1, feature2] return x, feature_list
def inference(input_images): x = tf.layers.conv2d(inputs=input_images, filters=32, kernel_size=3, padding='SAME', name='conv1_1') x = tf.layers.conv2d(inputs=x, filters=32, kernel_size=3, padding='SAME', name='conv1_2') x = tf.layers.max_pooling2d(inputs=x, pool_size=2, strides=2, name='pool1') x = tf.layers.conv2d(inputs=x, filters=64, kernel_size=3, padding='SAME', name='conv2_1') x = tf.layers.conv2d(inputs=x, filters=64, kernel_size=3, padding='SAME', name='conv2_2') x = tf.layers.max_pooling2d(inputs=x, pool_size=2, strides=2, name='pool2') x = tf.layers.conv2d(inputs=x, filters=128, kernel_size=3, padding='SAME', name='conv3_1') x = tf.layers.conv2d(inputs=x, filters=128, kernel_size=3, padding='SAME', name='conv3_2') x = tf.layers.max_pooling2d(inputs=x, pool_size=2, strides=2, name='pool3') x = tf.layers.flatten(inputs=x, name='flatten') feature = tf.layers.dense(inputs=x, units=2, name='fc1') x = tflearn.prelu(feature) x = tf.layers.dense(inputs=x, units=NUM_CLASSES, name='fc2') return x, feature
def _active_layer(_input, name, is_training): bn = tf.contrib.layers.batch_norm( inputs=_input, # tensor, first dimension of batch_size decay=0.9, # recommend trying decay=0.9 scale=True, # If True, multiply by gamma. If False, gamma is not used epsilon=1e-5, # Small float added to variance to avoid dividing by zero updates_collections=None, # tf.GraphKeys.UPDATE_OPS, # updates_collections: Collections to collect the update ops for computation. # The updates_ops need to be executed with the train_op. # If None, a control dependency would be added to make sure the updates are computed in place is_training=is_training, # In training mode it would accumulate the statistics of the moments into moving_mean # and moving_variance using an exponential moving average with the given decay. scope=name + '_batch_norm', # variable_scope ) prelu = tflearn.prelu(bn, name=name + '_prelu') return prelu
def __init__(self): self.x_input = tf.placeholder(tf.float32, shape=[None, 784]) self.y_input = tf.placeholder(tf.int64, shape=[None]) self.x_image = tf.reshape(self.x_input, [-1, 28, 28, 1]) with slim.arg_scope([slim.conv2d], kernel_size=3, padding='SAME'): with slim.arg_scope([slim.max_pool2d], kernel_size=2): x = slim.conv2d(self.x_image, num_outputs=32, scope='conv1_1') x = slim.conv2d(x, num_outputs=32, scope='conv1_2') x = slim.max_pool2d(x, scope='pool1') x = slim.conv2d(x, num_outputs=64, scope='conv2_1') x = slim.conv2d(x, num_outputs=64, scope='conv2_2') x = slim.max_pool2d(x, scope='pool2') x = slim.conv2d(x, num_outputs=128, scope='conv3_1') x = slim.conv2d(x, num_outputs=128, scope='conv3_2') x = slim.max_pool2d(x, scope='pool3') x = slim.flatten(x, scope='flatten') x = slim.fully_connected(x, num_outputs=32, activation_fn=None, scope='fc1') x = slim.fully_connected(x, num_outputs=2, activation_fn=None, scope='fc2') self.feature = x = tflearn.prelu(x) self.xent, logits, tmp = cos_loss(x, self.y_input, 10, alpha=0.25) self.y_pred = tf.arg_max( tf.matmul(tmp['x_feat_norm'], tmp['w_feat_norm']), 1) self.accuracy = tf.reduce_mean( tf.cast(tf.equal(self.y_pred, self.y_input), tf.float32))
def inference(input_images, num_class=10, reuse=False): with slim.arg_scope([slim.conv2d], kernel_size=3, padding='SAME'): with slim.arg_scope([slim.max_pool2d], kernel_size=2): x = slim.conv2d(input_images, num_outputs=32, scope='conv1_1') x = slim.conv2d(x, num_outputs=32, scope='conv1_2') x = slim.max_pool2d(x, scope='pool1') x = slim.conv2d(x, num_outputs=64, scope='conv2_1') x = slim.conv2d(x, num_outputs=64, scope='conv2_2') h_conv1 = x = slim.max_pool2d(x, scope='pool2') x = slim.conv2d(x, num_outputs=128, scope='conv3_1') x = slim.conv2d(x, num_outputs=128, scope='conv3_2') h_conv2 = x = slim.max_pool2d(x, scope='pool3') x = slim.flatten(x, scope='flatten') h_fc1 = x = slim.fully_connected(x, num_outputs=32, activation_fn=None, scope='fc1') feature = x = slim.fully_connected(x, num_outputs=32, activation_fn=None, scope='fc2') x = tflearn.prelu(x) x = slim.fully_connected(x, num_outputs=num_class, activation_fn=None, scope='fc3') return x, [h_conv1, h_conv2, h_fc1, feature]
def get_fc1(last_conv, bottleneck_layer_size, fc_type, **kwargs): bn_mom = 0.9 body = last_conv is_training = kwargs.get('phase_train', True) with slim.arg_scope( [slim.batch_norm], updates_collections=None, variables_collections=[tf.GraphKeys.TRAINABLE_VARIABLES], is_training=is_training): if fc_type == 'E': body = slim.batch_norm(body, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope='bn1') body = slim.dropout(body, keep_prob=0.4, is_training=is_training) fc1 = slim.fully_connected(body, bottleneck_layer_size, scope='pre_fc1') fc1 = slim.batch_norm(fc1, center=False, scale=False, epsilon=2e-5, decay=bn_mom, scope='fc1') elif fc_type == 'F': body = slim.batch_norm(body, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope='bn1') body = slim.dropout(body, keep_prob=0.4, is_training=is_training) fc1 = slim.fully_connected(body, bottleneck_layer_size, scope='fc1') elif fc_type == 'G': body = slim.batch_norm(body, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope='bn1') fc1 = slim.fully_connected(body, bottleneck_layer_size, scope='fc1') elif fc_type == 'H': fc1 = slim.fully_connected(body, bottleneck_layer_size, scope='fc1') elif fc_type == 'I': body = slim.batch_norm(body, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope='bn1') fc1 = slim.fully_connected(body, bottleneck_layer_size, scope='pre_fc1') fc1 = slim.batch_norm(fc1, center=False, scale=False, epsilon=2e-5, decay=bn_mom, scope='fc1') elif fc_type == 'J': fc1 = mslim.fully_connected(body, bottleneck_layer_size, scope='pre_fc1') fc1 = slim.batch_norm(fc1, center=False, scale=False, epsilon=2e-5, decay=bn_mom, scope='fc1') else: bn1 = slim.batch_norm(body, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope='bn1') relu1 = tflearn.prelu(bn1) # Although kernel is not used here when global_pool=True, we should put one #pool1 = tflearn.global_avg_pool (relu1, name='pool1') pool1 = slim.avg_pool2d( relu1, kernel_size=[int(relu1.shape[1]), int(relu1.shape[2])]) flat = slim.flatten(pool1) if len(fc_type) > 1: if fc_type[1] == 'X': print('dropout mode') flat = slim.dropout(flat, keep_prob=0.2, is_training=is_training) fc_type = fc_type[0] if fc_type == 'A': fc1 = flat else: #B-D #B fc1 = slim.fully_connected(flat, num_classes, name='pre_fc1') if fc_type == 'C': fc1 = slim.batch_norm(fc1, center=False, scale=False, epsilon=2e-5, decay=bn_mom, scope='fc1') elif fc_type == 'D': fc1 = slim.batch_norm(fc1, center=False, scale=False, epsilon=2e-5, decay=bn_mom, scope='fc1') fc1 = tflearn.prelu(fc1) return fc1
def model_graph(pssm, encoding, domain, keep_prob, y_): with tf.name_scope('pretrained_cnn'): with tf.device('/gpu:0'): with tf.name_scope('convnet_for_pssm_feature'): #Reshape the input into 4D Tensor with tf.name_scope('pssm_feature_reshape'): x_pssm=tf.reshape(pssm,[-1,MAX_LENGTH,TYPE_OF_AA,1]) #ADD FIRST CONVELUTIONAL LAYER# #add the variables with tf.name_scope('pssm_conv_layer_1'): w_conv1_pssm=weight_variable([40,4,1,32]) b_conv1_pssm=bias_variable([32]) #add the first conv layer and pooling layer h_conv1_pssm=tf.nn.relu(conv2d(x_pssm,w_conv1_pssm)+b_conv1_pssm) with tf.name_scope('pssm_pool_layer_1'): h_pool1_pssm=max_pool2d(h_conv1_pssm,5,2) h_pool1_pssm = tflearn.batch_normalization(h_pool1_pssm) #ADD THE SECOND CONVERLUTIONAL LAYER# with tf.name_scope('pssm_conv_layer_2'): w_conv2_pssm=weight_variable([30,4,32,64]) b_conv2_pssm=bias_variable([64]) h_conv2_pssm=tf.nn.relu(conv2d(h_pool1_pssm,w_conv2_pssm)+b_conv2_pssm) h_conv2_pssm = tflearn.batch_normalization(h_conv2_pssm) #ADD THE THIRD CONVER LAYER AND THE SECOND POOLING LAYER with tf.name_scope('pssm_conv_layer_3'): w_conv3_pssm=weight_variable([30,4,64,128]) b_conv3_pssm=bias_variable([128]) h_conv3_pssm=tf.nn.relu(conv2d(h_conv2_pssm,w_conv3_pssm)+b_conv3_pssm) with tf.name_scope('pssm_pool_layer_2'): h_pool2_pssm=max_pool2d(h_conv3_pssm,5,2) h_pool2_pssm = tflearn.batch_normalization(h_pool2_pssm) #ADD THE FORTH CONVER LAYER with tf.name_scope('pssm_conv_layer_4'): w_conv4_pssm=weight_variable([20,3,128,256]) b_conv4_pssm=bias_variable([256]) h_conv4_pssm=tf.nn.relu(conv2d(h_pool2_pssm,w_conv4_pssm)+b_conv4_pssm) h_conv4_pssm = tflearn.batch_normalization(h_conv4_pssm) #ADD THE FIFTH CONVER LAYER AND THIRD POOLING LAYER with tf.name_scope('pssm_conv_layer_5'): w_conv5_pssm=weight_variable([20,3,256,256]) b_conv5_pssm=bias_variable([256]) h_conv5_pssm=tf.nn.relu(conv2d(h_conv4_pssm,w_conv5_pssm)+b_conv5_pssm) with tf.name_scope('pssm_pool_layer_3'): h_pool3_pssm=max_pool2d(h_conv5_pssm,4,1) h_pool3_pssm = tflearn.batch_normalization(h_pool3_pssm) #ADD THE SIXTH CONVER LAYER AND FORTH POOLING LAYER with tf.name_scope('pssm_conv_layer_6'): w_conv6_pssm=weight_variable([20,3,256,256]) b_conv6_pssm=bias_variable([256]) h_conv6_pssm=tf.nn.relu(conv2d(h_pool3_pssm,w_conv6_pssm)+b_conv6_pssm) with tf.name_scope('pssm_pool_layer_4'): h_pool4_pssm=max_pool2d(h_conv6_pssm,2,1) h_pool4_pssm = tflearn.batch_normalization(h_pool4_pssm) with tf.device('/gpu:0'): with tf.name_scope('convnet_for_encoding_feature'): #Reshape the input into 4D Tensor with tf.name_scope('encoding_feature_reshape'): x_encoding=tf.reshape(encoding,[-1,1,MAX_LENGTH,1]) #ADD FIRST CONVELUTIONAL LAYER# with tf.name_scope('encoding_conv_layer_1'): #add the variables w_conv1_encoding=weight_variable([1,40,1,32]) b_conv1_encoding=bias_variable([32]) #add the first conv layer and pooling layer h_conv1_encoding=tf.nn.relu(conv2d(x_encoding,w_conv1_encoding)+b_conv1_encoding) with tf.name_scope('encoding_pool_layer_1'): h_pool1_encoding=max_pool2d(h_conv1_encoding,1,5) h_pool1_encoding = tflearn.batch_normalization(h_pool1_encoding) #ADD THE SECOND CONVERLUTIONAL LAYER# with tf.name_scope('encoding_conv_layer_2'): w_conv2_encoding=weight_variable([1,30,32,64]) b_conv2_encoding=bias_variable([64]) h_conv2_encoding=tf.nn.relu(conv2d(h_pool1_encoding,w_conv2_encoding)+b_conv2_encoding) h_conv2_encoding = tflearn.batch_normalization(h_conv2_encoding) #ADD THE THIRD CONVER LAYER AND THE SECOND POOLING LAYER with tf.name_scope('encoding_conv_layer_3'): w_conv3_encoding=weight_variable([1,30,64,128]) b_conv3_encoding=bias_variable([128]) h_conv3_encoding=tf.nn.relu(conv2d(h_conv2_encoding,w_conv3_encoding)+b_conv3_encoding) with tf.name_scope('encoding_pool_layer_2'): h_pool2_encoding=max_pool2d(h_conv3_encoding,1,5) h_pool2_encoding = tflearn.batch_normalization(h_pool2_encoding) #ADD THE FORTH CONVER LAYER with tf.name_scope('encoding_conv_layer_4'): w_conv4_encoding=weight_variable([1,20,128,256]) b_conv4_encoding=bias_variable([256]) h_conv4_encoding=tf.nn.relu(conv2d(h_pool2_encoding,w_conv4_encoding)+b_conv4_encoding) h_conv4_encoding = tflearn.batch_normalization(h_conv4_encoding) #ADD THE FIFTH CONVER LAYER AND THIRD POOLING LAYER with tf.name_scope('encoding_conv_layer_5'): w_conv5_encoding=weight_variable([1,20,256,256]) b_conv5_encoding=bias_variable([256]) h_conv5_encoding=tf.nn.relu(conv2d(h_conv4_encoding,w_conv5_encoding)+b_conv5_encoding) with tf.name_scope('encoding_pool_layer_3'): h_pool3_encoding=max_pool2d(h_conv5_encoding,1,4) h_pool3_encoding = tflearn.batch_normalization(h_pool3_encoding) #ADD THE SIXTH CONVER LAYER AND FORTH POOLING LAYER with tf.name_scope('encoding_conv_layer_6'): w_conv6_encoding=weight_variable([1,20,256,256]) b_conv6_encoding=bias_variable([256]) h_conv6_encoding=tf.nn.relu(conv2d(h_pool3_encoding,w_conv6_encoding)+b_conv6_encoding) with tf.name_scope('encoding_pool_layer_4'): h_pool4_encoding=max_pool2d(h_conv6_encoding,1,2) h_pool4_encoding = tflearn.batch_normalization(h_pool4_encoding) #consturct dimensionality redution for functional domain encoding with tf.name_scope('fine_tune_layers'): with tf.name_scope('functional_domain_layers'): with tf.name_scope('functional_domain_fc_1'): w_dr1_domain=weight_variable([DOMAIN,4096]) b_dr1_domain=bias_variable([4096]) h_dr1_domain=tflearn.prelu(tf.matmul(domain,w_dr1_domain)+b_dr1_domain) h_dr1_domain=tflearn.batch_normalization(h_dr1_domain) with tf.name_scope('functional_domain_fc_2'): w_dr2_domain=weight_variable([4096,1024]) b_dr2_domain=bias_variable([1024]) h_dr2_domain=tflearn.prelu(tf.matmul(h_dr1_domain,w_dr2_domain)+b_dr2_domain) h_dr2_domain=tflearn.batch_normalization(h_dr2_domain) #ADD THE DENSELY CONNECTED LAYER# with tf.name_scope('densely_connected_layers'): with tf.name_scope('fc_1'): b_fc1=bias_variable([1024]) w_fc1_pssm=weight_variable([25*5*256,1024]) h_pool4_pssm_flat=tf.reshape(h_pool4_pssm,[-1,25*5*256]) w_fc1_encoding=weight_variable([1*25*256,1024]) h_pool4_encoding_flat=tf.reshape(h_pool4_encoding,[-1,1*25*256]) #incoporate functional domain encoding information w_fc1_domain=weight_variable([1024,1024]) h_fc1=tf.nn.relu(tf.matmul(h_pool4_pssm_flat,w_fc1_pssm)+tf.matmul(h_pool4_encoding_flat, w_fc1_encoding)+tf.matmul(h_dr2_domain,w_fc1_domain)+b_fc1) h_fc1=tflearn.batch_normalization(h_fc1) if DROPOUT==True: h_fc1=tf.nn.dropout(h_fc1,keep_prob) with tf.name_scope('fc_3'): #Add the third densely connected layer w_fc3=weight_variable([1024,1024]) b_fc3=bias_variable([1024]) h_fc3=tflearn.prelu(tf.matmul(h_fc1,w_fc3)+b_fc3) h_fc3=tflearn.batch_normalization(h_fc3) if DROPOUT==True: h_fc3=tf.nn.dropout(h_fc3,keep_prob) #ADD SOFTMAX LAYER with tf.name_scope('softmax_layer'): w_fc4=weight_variable([1024,y_.get_shape().as_list()[1]]) b_fc4=bias_variable([y_.get_shape().as_list()[1]]) y_conv_logit=tf.matmul(h_fc3,w_fc4)+b_fc4 y_conv=tf.nn.softmax(y_conv_logit) return y_conv_logit, y_conv, h_fc3
def inference(incoming): with tf.variable_scope('conv1_1', reuse=tf.AUTO_REUSE): # 28*28*3->28*28*32 net = tflearn.conv_2d(incoming, nb_filter=32, filter_size=3, bias=False, regularizer='L2', weight_decay=0.0001) net = tflearn.batch_normalization(net) net = tf.nn.relu(net) with tf.variable_scope('conv1_2', reuse=tf.AUTO_REUSE): # 28*28*32->28*28*32 net = tflearn.conv_2d(net, nb_filter=32, filter_size=3, bias=False, regularizer='L2', weight_decay=0.0001) net = tflearn.batch_normalization(net) net = tf.nn.relu(net) print(net) net = tflearn.max_pool_2d(net, kernel_size=2, strides=2, padding='valid', name='pool1') print(net) with tf.variable_scope('conv2_1', reuse=tf.AUTO_REUSE): # 14*14*32->14*14*64 net = tflearn.conv_2d(net, nb_filter=64, filter_size=3, bias=False, regularizer='L2', weight_decay=0.0001) net = tflearn.batch_normalization(net) net = tf.nn.relu(net) with tf.variable_scope('conv2_2', reuse=tf.AUTO_REUSE): # 14*14*64->14*14*64 net = tflearn.conv_2d(net, nb_filter=64, filter_size=3, bias=False, regularizer='L2', weight_decay=0.0001) net = tflearn.batch_normalization(net) net = tf.nn.relu(net) net = tflearn.max_pool_2d(net, kernel_size=2, strides=2, padding='valid', name='pool2') print(net) with tf.variable_scope('conv3_1', reuse=tf.AUTO_REUSE): # 7*7*64->7*7*128 net = tflearn.conv_2d(net, nb_filter=128, filter_size=3, bias=False, regularizer='L2', weight_decay=0.0001) net = tflearn.batch_normalization(net) net = tf.nn.relu(net) with tf.variable_scope('conv3_2', reuse=tf.AUTO_REUSE): # 7*7*128->7*7*128 net = tflearn.conv_2d(net, nb_filter=128, filter_size=3, bias=False, regularizer='L2', weight_decay=0.0001) net = tflearn.batch_normalization(net) net = tf.nn.relu(net) net = tflearn.global_avg_pool(net) feature = tflearn.fully_connected(net, n_units=2, regularizer='L2', weight_decay=0.0001) print(net) net = tflearn.prelu(feature) net = tflearn.fully_connected(net, n_units=10, regularizer='L2', weight_decay=0.0001) return net, feature
def PRelu(self, x): return tflearn.prelu(x, name='prelu')
def prelu(x): pre = tflearn.prelu(x) #alpha = tf.get_variable('alpha', shape=x.get_shape()[-1], dtype=x.dtype, initializer=tf.constant_initializer(0.1)) #return tf.maximum(0.0, x) + alpha * tf.minimum(0.0, x) return pre
def network2(frame1, frame2, frame3, reuse=False, scope='netflow'): # design for QP22,27,32 with tf.variable_scope(scope, reuse=reuse): with slim.arg_scope( [slim.conv2d], activation_fn=tflearn.activations.prelu, weights_initializer=tf.contrib.layers.xavier_initializer( uniform=True), biases_initializer=tf.constant_initializer(0.0)): # Multi-scale c3_1 = slim.conv2d(frame1, 32, [3, 3], scope='conv3_1') c5_1 = slim.conv2d(frame1, 32, [5, 5], scope='conv5_1') c7_1 = slim.conv2d(frame1, 32, [7, 7], scope='conv7_1') cc_1 = tf.concat([c3_1, c5_1, c7_1], 3, name='concat_1') c3_2 = slim.conv2d(frame2, 32, [3, 3], scope='conv3_2') c5_2 = slim.conv2d(frame2, 32, [5, 5], scope='conv5_2') c7_2 = slim.conv2d(frame2, 32, [7, 7], scope='conv7_2') cc_2 = tf.concat([c3_2, c5_2, c7_2], 3, name='concat_2') c3_3 = slim.conv2d(frame3, 32, [3, 3], scope='conv3_3') c5_3 = slim.conv2d(frame3, 32, [5, 5], scope='conv5_3') c7_3 = slim.conv2d(frame3, 32, [7, 7], scope='conv7_3') cc_3 = tf.concat([c3_3, c5_3, c7_3], 3, name='concat_3') # Merge c_concat = tf.concat([cc_1, cc_2, cc_3], 3, name='c_concat') # General CNN ##### c1_w = tf.get_variable("c1_w", shape=[3, 3, 32*3*3, 32],initializer=tf.contrib.layers.xavier_initializer(uniform=True)) # cc1 = slim.conv2d(c_concat, 32, [3, 3], scope='cconv1') # cconv1_mask = tf.Variable(np.ones([128,64,64,32]),trainable=False,name='cconv1_mask',dtype=tf.float32) #参数 # cc2 = slim.conv2d(entry_stop_gradients(cc1,cconv1_mask), 32, [3, 3], scope='cconv2') with tf.variable_scope("cconv1"): cconv1_mask = tf.get_variable('cconv1_mask', [3, 3, 288, 32], trainable=False) cc1_weights = tf.get_variable("weights", [3, 3, 288, 32]) cc1_bias = tf.get_variable("biases", [32]) cc1_conv = tf.nn.conv2d(c_concat, entry_stop_gradients( cc1_weights, cconv1_mask), strides=[1, 1, 1, 1], padding='SAME') + cc1_bias cc1_h_conv = tflearn.prelu(cc1_conv) # 参数 cc2 = slim.conv2d(cc1_h_conv, 32, [3, 3], scope='cconv2') ### cc3 = slim.conv2d(cc2, 32, [3, 3], scope='cconv3') cc4 = slim.conv2d(cc3, 32, [3, 3], scope='cconv4') cc5 = slim.conv2d(cc4, 32, [3, 3], scope='cconv5') cc6 = slim.conv2d(cc5, 32, [3, 3], scope='cconv6') cc7 = slim.conv2d(cc6, 32, [3, 3], scope='cconv7') cc8 = slim.conv2d(cc7, 16, [3, 3], scope='cconv8') cout = slim.conv2d(cc8, 1, [3, 3], activation_fn=None, scope='cout') # 1 channel output output = tf.add(cout, frame2) # ResNet return output
def residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): use_se = kwargs.get('version_se', 1) bn_mom = kwargs.get('bn_mom', 0.9) is_training = kwargs.get('phase_train', True) with slim.arg_scope( [slim.batch_norm], updates_collections=None, variables_collections=[tf.GraphKeys.TRAINABLE_VARIABLES], is_training=is_training): #print('in unit2') if bottle_neck: # the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper bn1 = slim.batch_norm(data, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn1') act1 = tflearn.prelu(bn1) conv1 = slim.conv2d(act1, int(num_filter * 0.25), [1, 1], stride=(1, 1), biases_initializer=None, scope=name + '_conv1') bn2 = slim.batch_norm(conv1, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn2') act2 = tflearn.prelu(bn2) conv2 = slim.conv2d(act2, int(num_filter * 0.25), [3, 3], stride=stride, biases_initializer=None, scope=name + '_conv2') bn3 = slim.batch_norm(conv2, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn3') act3 = tflearn.prelu(bn3) conv3 = slim.conv2d(act3, num_filter, [1, 1], stride=(1, 1), biases_initializer=None, scope=name + '_conv3') if use_se: #se begin #body = tflearn.global_avg_pool (conv3, name=name+'_se_pool1') body = slim.avg_pool2d( conv3, kernel_size=[int(conv3.shape[1]), int(conv3.shape[2])]) body = slim.conv2d(body, num_filter // 16, [1, 1], stride=(1, 1), scope=name + "_se_conv1") body = tflearn.prelu(body) body = slim.conv2d(body, num_filter, [1, 1], stride=(1, 1), scope=name + "_se_conv2") body = tf.sigmoid(body) conv3 = tf.multiply(conv3, body) if dim_match: shortcut = data else: shortcut = slim.conv2d(act1, num_filter, [1, 1], stride=stride, biases_initializer=None, scope=name + '_sc') return conv3 + shortcut else: bn1 = slim.batch_norm(data, center=True, scale=True, decay=bn_mom, epsilon=2e-5, scope=name + '_bn1') act1 = tflearn.prelu(bn1) conv1 = slim.conv2d(act1, num_filter, [3, 3], stride=stride, biases_initializer=None, scope=name + '_conv1') bn2 = slim.batch_norm(conv1, center=True, scale=True, decay=bn_mom, epsilon=2e-5, scope=name + '_bn2') act2 = tflearn.prelu(bn2) conv2 = slim.conv2d(act2, num_filter, [3, 3], stride=1, biases_initializer=None, scope=name + '_conv2') if use_se: #se begin #body = tflearn.global_avg_pool (conv2, name=name+'_se_pool1') body = slim.avg_pool2d( conv2, kernel_size=[int(conv2.shape[1]), int(conv2.shape[2])]) body = slim.conv2d(body, num_filter // 16, [1, 1], stride=(1, 1), name=name + "_se_conv1") body = tflearn.prelu(body) body = slim.conv2d(body, num_filter, [1, 1], stride=(1, 1), scope=name + "_se_conv2") body = slim.sigmoid(body) conv2 = tf.multiply(conv2, body) if dim_match: shortcut = data else: shortcut = slim.conv2d(act1, num_filter, [1, 1], stride=stride, biases_initializer=None, scope=name + '_sc') output = tf.concat([conv2, shortcut], 3) return output
def resnet(images, units, num_stages, filter_list, bottle_neck, bottleneck_layer_size, **kwargs): bn_mom = kwargs.get('bn_mom', 0.9) version_se = kwargs.get('version_se', 1) version_input = kwargs.get('version_input', 1) assert version_input >= 0 version_output = kwargs.get('version_output', 'E') fc_type = version_output version_unit = kwargs.get('version_unit', 3) print(version_se, version_input, version_output, version_unit) num_unit = len(units) is_training = kwargs.get("is_training", True) assert (num_unit == num_stages) with slim.arg_scope( [slim.batch_norm], updates_collections=None, variables_collections=[tf.GraphKeys.TRAINABLE_VARIABLES]): if version_input == 0: body = slim.conv2d(images, filter_list[0], [7, 7], stride=(2, 2), biases_initializer=None, scope="conv0") body = slim.batch_norm(body, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope='bn0') body = tflearn.prelu(body) body = slim.max_pool2d(body, kernel_size=[3, 3], stride=(2, 2), padding="SAME") else: body = images body = slim.conv2d(body, filter_list[0], [3, 3], stride=(1, 1), biases_initializer=None, scope="conv0") body = slim.batch_norm(body, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope='bn0') body = tflearn.prelu(body) for i in range(num_stages): if version_input == 0: body = residual_unit(body, filter_list[i + 1], (1 if i == 0 else 2, 1 if i == 0 else 2), False, name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) else: body = residual_unit(body, filter_list[i + 1], (2, 2), False, name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) for j in range(units[i] - 1): body = residual_unit(body, filter_list[i + 1], (1, 1), True, name='stage%d_unit%d' % (i + 1, j + 2), bottle_neck=bottle_neck, **kwargs) fc1 = get_fc1(body, bottleneck_layer_size, fc_type) return fc1
def residual_unit_v4(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): use_se = kwargs.get('version_se', 1) bn_mom = kwargs.get('bn_mom', 0.9) is_training = kwargs.get('phase_train', True) with slim.arg_scope( [slim.batch_norm], updates_collections=None, variables_collections=[tf.GraphKeys.TRAINABLE_VARIABLES], is_training=is_training): #print('in unit3') if bottle_neck: bn1 = slim.batch_norm(data, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn1') conv1 = slim.conv2d(bn1, int(num_filter * 0.25), [1, 1], stride=(1, 1), biases_initializer=None, scope=name + '_conv1') bn2 = slim.batch_norm(conv1, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn2') act1 = tflearn.prelu(bn2) conv2 = slim.conv2d(act1, int(num_filter * 0.25), [3, 3], stride=(1, 1), biases_initializer=None, scope=name + '_conv2') bn3 = slim.batch_norm(conv2, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn3') act2 = tflearn.prelu(bn3) conv3 = slim.conv2d(act2, num_filter, [1, 1], stride=stride, biases_initializer=None, scope=name + '_conv3') bn4 = slim.batch_norm(conv3, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn4') if use_se: #se begin #body = tflearn.global_avg_pool (bn4, name=name+'_se_pool1') body = slim.avg_pool2d( bn4, kernel_size=[int(bn4.shape[1]), int(bn4.shape[2])]) body = slim.conv2d(body, num_filter // 16, [1, 1], stride=(1, 1), scope=name + "_se_conv1") body = tflearn.prelu(body) body = slim.conv2d(body, num_filter, [1, 1], stride=(1, 1), scope=name + "_se_conv2") body = tf.sigmoid(body) bn4 = tf.multiply(bn4, body) #se end if dim_match: shortcut = data return bn4 + shortcut else: return bn4 else: bn1 = slim.batch_norm(data, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn1') conv1 = slim.conv2d(bn1, num_filter, [3, 3], stride=(1, 1), biases_initializer=None, scope=name + '_conv1') bn2 = slim.batch_norm(conv1, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn2') act1 = tflearn.prelu(bn2) conv2 = slim.conv2d(act1, num_filter, [3, 3], stride=stride, biases_initializer=None, scope=name + '_conv2') bn3 = slim.batch_norm(conv2, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn3') if use_se: #se begin #body = tflearn.global_avg_pool (bn3,name=name+'_se_pool1') body = slim.avg_pool2d( bn3, kernel_size=[int(bn3.shape[1]), int(bn3.shape[2])]) body = slim.conv2d(body, num_filter // 16, [1, 1], stride=(1, 1), scoep=name + "_se_conv1") body = tflearn.prelu(body) body = slim.conv2d(body, num_filter, [1, 1], stride=(1, 1), scope=name + "_se_conv2") body = tf.sigmoid(body) bn3 = tf.multiply(bn3, body) #se end if dim_match: shortcut = data return bn3 + shortcut else: return bn3
def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): print(data.shape) use_se = kwargs.get('version_se', 1) bn_mom = kwargs.get('bn_mom', 0.9) is_training = kwargs.get('phase_train', True) with slim.arg_scope( [slim.batch_norm], updates_collections=None, variables_collections=[tf.GraphKeys.TRAINABLE_VARIABLES], is_training=is_training): num_group = 32 #print('in unit3') bn1 = slim.batch_norm(data, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn1') #conv1 = slim.conv2d(bn1, int(num_filter*0.5), [1,1],padding="VALID",biases_initializer=None,scope=name + '_conv1') conv1 = tflearn.layers.conv.grouped_conv_2d(bn1, num_group, int(num_filter * 0.5), kernel=(3, 3), stride=(1, 1), biases_initializer=None, scope=name + '_conv2') bn2 = slim.batch_norm(conv1, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn2') act1 = tflearn.prelu(bn2) conv2 = slim.conv2d(act1, int(num_filter * 0.5), [3, 3], biases_initializer=None, scope=name + '_conv2') #conv2 = tfleaen.grouped_conv_2d(act1, num_group, int(num_filter*0.5), kernel=(3,3), stride=(1,1),biases_initializer=None,scope=name + '_conv2') bn3 = slim.batch_norm(conv2, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn3') act2 = tflearn.prelu(bn3) conv3 = slim.conv2d(act2, num_filter, [1, 1], stride=stride, padding="VALID", scope=name + '_conv3') bn4 = slim.batch_norm(conv3, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_bn4') print(bn4.shape) if use_se: #se begin #2 choice body = tflearn.global_avg_pool (bn4, name=name+'_se_pool1') body = slim.avg_pool2d( bn4, kernel_size=[int(bn4.shape[1]), int(bn4.shape[2])]) print("-->", body.shape) body = slim.conv2d(body, num_filter // 16, [1, 1], scope=name + "_se_conv1") body = tflearn.prelu(body) body = slim.conv2d(body, num_filter, [1, 1], stride=(1, 1), scope=name + "_se_conv2") body = tf.sigmoid(body) bn4 = tf.multiply(bn4, body) #se end if dim_match: shortcut = data else: conv1sc = slim.conv2d(data, num_filter, [1, 1], stride=stride, biases_initializer=None, scope=name + '_conv1sc') shortcut = slim.batch_norm(conv1sc, center=True, scale=True, epsilon=2e-5, decay=bn_mom, scope=name + '_sc') print(bn4.shape) print(shortcut.shape) return bn4 + shortcut