def residual_block(conf, X, filters, block_num, dropout, scope_name): f0 = X.get_shape().as_list()[-1] f1, f2 = filters X_shortcut = X with tf.variable_scope(scope_name): X = ops.conv_layer(conf, X, k_shape=[3, 3, f0, f1], stride=1, padding='SAME', w_init='tn', scope_name='conv_1', add_smry=False) X = ops.batch_norm(conf, X, scope_name='bn_1') X = ops.activation(X, "relu") logging.info('%s : conv_1 shape: %s', str(scope_name), str(X.shape)) if dropout is not None: logging.info('%s : dropout = %s shape: %s', str(scope_name), str(dropout), str(X.shape)) X = tf.nn.dropout(X, dropout) X = ops.conv_layer(conf, X, k_shape=[3, 3, f1, f2], stride=1, padding='SAME', w_init='tn', scope_name='conv_2', add_smry=False) X = ops.batch_norm(conf, X, scope_name='bn_2') logging.info('%s : conv_2 shape: %s', str(scope_name), str(X.shape)) # Add skip connection X = X + X_shortcut X = ops.activation(X, 'relu') logging.info('%s : Skip add shape: %s', str(scope_name), str(X.shape)) return X
def resnet(conf, img_shape, device_type, use_dropout): inpX = tf.placeholder(dtype=tf.float32, shape=[None, img_shape[0], img_shape[1], img_shape[2]], name='X') inpY = tf.placeholder(dtype=tf.float32, shape=[None, conf['myNet']['num_labels']], name='Y') with tf.device(device_type): X_embeddings = embeddings(conf, inpX, use_dropout) X_embeddings = ops.activation(X_embeddings, 'relu', scope_name='relu_fc') logging.info('X - FC Layer (RELU): %s', str(X_embeddings.get_shape().as_list())) # SOFTMAX Layer X_logits = ops.fc_layers(conf, X_embeddings, [512, 2], w_init='tn', scope_name='fc_layer2', add_smry=False) logging.info('LOGITS - Softmax Layer: %s', str(X_logits.get_shape().as_list())) Y_probs = tf.nn.softmax(X_logits) logging.info('Softmax Y-Prob shape: shape %s', str(Y_probs.shape)) loss = ops.get_loss(y_true=inpY, y_logits=X_logits, which_loss='sigmoid_cross_entropy', lamda=None) optimizer, l_rate = ops.optimize(conf, loss=loss, learning_rate_decay=True, add_smry=False) acc = ops.accuracy(labels=inpY, logits=X_logits, type='training', add_smry=False) return dict(inpX=inpX, inpY=inpY, outProbs=Y_probs, accuracy=acc, loss=loss, optimizer=optimizer, l_rate=l_rate)
def conv_1(conf, X, filters, scope_name): f = filters with tf.variable_scope(scope_name): X = ops.conv_layer(conf, X, k_shape=[7, 7, 3, f], stride=2, padding='SAME', w_init='tn', scope_name='conv_1', add_smry=False) X = ops.batch_norm(conf, X, scope_name='bn_1') X = ops.activation(X, "relu") X = tf.layers.max_pooling2d(X, pool_size=3, padding='SAME', strides=2) logging.info('%s : conv_1 shape: %s', str(scope_name), str(X.shape)) return X
def residual_block_first(conf, X, filters, block_num, dropout, scope_name): ''' Why need this? Normally we have skip connections between 2 layers in one residual block. When going from 1 residual block to another we decrease in the image size, In-order to maintain skip connection between the layers, we need to have the same dimension for input and output. ''' f0 = X.get_shape().as_list()[-1] f1, f2 = filters with tf.variable_scope(scope_name): # We perform a 1x1 conv increasing the num_out channels to equal the number of dimensions. We also perform # down sampling of convolutional layer by using a stride of 2 X_shortcut = ops.conv_layer(conf, X, [1, 1, f0, f1], stride=2, padding='SAME', w_init='tn', scope_name='X_Shortcut', add_smry=False) logging.info('%s : conv_shortcut shape: %s', str(scope_name), str(X_shortcut.shape)) X = ops.conv_layer(conf, X, [3, 3, f0, f1], stride=2, padding='SAME', w_init='tn', scope_name='conv_1', add_smry=False) X = ops.batch_norm(conf, X, scope_name='bn_1') X = ops.activation(X, 'relu', scope_name='relu_1') logging.info('%s : conv_1 shape: %s', str(scope_name), str(X.shape)) #if dropout is not None: # logging.info('%s : dropout = %s shape: %s', str(scope_name), str(dropout), str(X.shape)) # X = tf.nn.dropout(X, dropout) X = ops.conv_layer(conf, X, [3, 3, f1, f2], stride=1, padding='SAME', w_init='tn', scope_name='conv_2', add_smry=False) X = ops.batch_norm(conf, X, scope_name='bn_2') logging.info('%s : conv_2 shape: %s', str(scope_name), str(X.shape)) X = X + X_shortcut X = ops.activation(X, 'relu', scope_name='relu_2') logging.info('%s : Skip add shape: %s', str(scope_name), str(X.shape)) return X
def mixture_of_experts(conf, img_shape, device_type, use_dropout): inpX1 = tf.placeholder(dtype=tf.float32, shape=[None, img_shape[0], img_shape[1], img_shape[2]], name='expert1') inpX2 = tf.placeholder(dtype=tf.float32, shape=[None, img_shape[0], img_shape[1], img_shape[2]], name='expert2') inpY = tf.placeholder(dtype=tf.float32, shape=[None, conf['myNet']['num_labels']], name='Y') with tf.device(device_type): logging.info('Expert 1: Creating Computation graph for Expert 1 ............... ') with tf.variable_scope('Expert1'): embeddings_m1 = embeddings(inpX1, use_dropout) logging.info('Expert 2: Creating Computation graph for Expert 2 ............... ') with tf.variable_scope('Expert2'): embeddings_m2 = embeddings(inpX2, use_dropout) expert_embeddings = tf.concat(values=[embeddings_m1, embeddings_m2], axis=-1) expert_embeddings = ops.activation(expert_embeddings, type='sigmoid', scope_name='sigmoid') logging.info('EMBEDDINGS: Stacked (sigmoid Gate) %s', str(expert_embeddings.get_shape().as_list())) # SOFTMAX Layer X_logits = ops.fc_layers(conf, expert_embeddings, [1024, 2], w_init='tn', scope_name='softmax', add_smry=False) logging.info('LOGITS - Softmax Layer: %s', str(X_logits.get_shape().as_list())) Y_probs = tf.nn.softmax(X_logits) logging.info('Softmax Y-Prob shape: shape %s', str(Y_probs.shape)) loss = ops.get_loss(y_true=inpY, y_logits=X_logits, which_loss='sigmoid_cross_entropy', lamda=None) optimizer, l_rate = ops.optimize(loss=loss, learning_rate_decay=True, add_smry=False) acc = ops.accuracy(labels=inpY, logits=X_logits, type='training', add_smry=False) return dict(inpX1=inpX1, inpX2=inpX2, inpY=inpY, outProbs=Y_probs, accuracy=acc, loss=loss, optimizer=optimizer, l_rate=l_rate)
def encoder(X, encoding_filters, latent_dim): logging.info('INPUT shape: %s', str(X.shape)) # Downsampling 1 X = ops.conv_layer( X, k_shape=[3, 3, encoding_filters[0], encoding_filters[1]], stride=1, padding='SAME', w_init='gu', w_decay=None, scope_name='conv_1', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_1') logging.info('ENCODER: conv_1 shape: %s' % (str(X.shape))) X = ops.activation(X, 'relu', 'relu_1') # print(X.shape) X = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool_1') logging.info('ENCODER : pool_1 shape: %s' % (str(X.shape))) # print(X.shape) # Downsampling 2 X = ops.conv_layer( X, k_shape=[3, 3, encoding_filters[1], encoding_filters[2]], stride=1, padding='SAME', w_init='gu', w_decay=None, scope_name='conv_2', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_2') logging.info('ENCODER : conv_2 shape: %s' % (str(X.shape))) X = ops.activation(X, 'relu', 'relu_2') # print(X.shape) X = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool_2') logging.info('ENCODER : pool_2 shape: %s' % (str(X.shape))) # print(X.shape) # Downsampling 3 X = ops.conv_layer( X, k_shape=[3, 3, encoding_filters[2], encoding_filters[3]], stride=1, padding='SAME', w_init='gu', w_decay=None, scope_name='conv_3', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_3') logging.info('ENCODER : conv_3 shape: %s' % (str(X.shape))) X = ops.activation(X, 'relu', 'relu_3') # print(X.shape) X = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool_3') logging.info('ENCODER : pool_3 shape: %s' % (str(X.shape))) # print(X.shape) # Downsampling 4 X = ops.conv_layer( X, k_shape=[3, 3, encoding_filters[3], encoding_filters[4]], stride=1, padding='SAME', w_init='gu', w_decay=None, scope_name='conv_4', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_4') logging.info('ENCODER : conv_4 shape: %s' % (str(X.shape))) X = ops.activation(X, 'relu', 'relu_4') # print(X.shape) X = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool_4') logging.info('ENCODER : pool_4 shape: %s' % (str(X.shape))) # Downloading 5 X = ops.conv_layer( X, k_shape=[3, 3, encoding_filters[4], encoding_filters[5]], stride=1, padding='SAME', w_init='gu', w_decay=None, scope_name='conv_5', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_5') logging.info('ENCODER : conv_5 shape: %s' % (str(X.shape))) X = ops.activation(X, 'relu', 'relu_5') # print(X.shape) X = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool_5') logging.info('ENCODER : pool_5 shape: %s' % (str(X.shape))) # Downloading 5 X = ops.conv_layer( X, k_shape=[3, 3, encoding_filters[5], encoding_filters[6]], stride=1, padding='SAME', w_init='gu', w_decay=None, scope_name='conv_6', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_6') logging.info('ENCODER : conv_6 shape: %s' % (str(X.shape))) X = ops.activation(X, 'relu', 'relu_5') # print(X.shape) X = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool_6') logging.info('ENCODER : pool_6 shape: %s' % (str(X.shape))) X_flat = tf.layers.flatten(X, 'X_flattened') X = ops.fc_layers(X_flat, k_shape=[X_flat.get_shape().as_list()[-1], latent_dim], w_init='gu', scope_name='fc_layer', add_smry=False) X = ops.activation(X, 'relu', 'relu_5') logging.info('ENCODER : Flattened shape: %s' % (str(X.shape))) # # # Downloading 2: Downsample to 2 features, to plot data points # X = ops.conv_layer(X, k_shape=[3, 3, encoding_filters[6], encoding_filters[7]], stride=1, padding='SAME', # w_init='gu', w_decay=None, scope_name='conv_6', add_smry=False) # logging.info('%s : conv_7 shape: %s', 'ENCODER: ', str(X.shape)) # X = ops.activation(X, 'relu', 'relu_7') # # print(X.shape) # X = tf.nn.max_pool(X, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool_7') # logging.info('%s : pool_7 shape: %s', 'ENCODER: ', str(X.shape)) # return X
def decoder(X, decoding_filters, latent_dim): flattened_shape = [2, 2, 64] X = ops.fc_layers(X, k_shape=[ latent_dim, flattened_shape[0] * flattened_shape[1] * flattened_shape[2] ], w_init='gu', scope_name='dec_fc_layer', add_smry=False) X = ops.activation(X, 'relu', 'relu_5') print(X.shape) X = tf.reshape(X, [ tf.shape(X)[0], flattened_shape[0], flattened_shape[1], flattened_shape[2] ]) print(X.shape) # Upsample 1 # X = ops.conv_layer(X, k_shape=[3,3,enc_fn,decoding_filters[0]], stride=1, padding='SAME', w_init='gu', # w_decay=None, scope_name='conv_4', add_smry=False) print(X.get_shape().as_list()[-1]) X = ops.conv2D_transposed_strided(X, k_shape=[3, 3, 64, decoding_filters[0]], stride=2, padding='SAME', w_init='gu', out_shape=None, scope_name='dconv_1', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_7') logging.info('%s : dconv_1 shape: %s', 'DECODER: ', str(X.shape)) X = ops.activation(X, 'relu', 'relu_6') print(X.shape) # Upsample 2 # X = ops.conv_layer(X, k_shape=[3, 3, decoding_filters[0], decoding_filters[1]], stride=1, padding='SAME', # w_init='gu', w_decay=None, scope_name='conv_5', add_smry=False) # print(X.shape) X = ops.conv2D_transposed_strided( X, k_shape=[3, 3, decoding_filters[0], decoding_filters[1]], stride=2, padding='SAME', w_init='gu', out_shape=None, scope_name='dconv_2', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_8') logging.info('%s : dconv_2 shape: %s', 'DECODER: ', str(X.shape)) X = ops.activation(X, 'relu', 'relu_7') # print(X.shape) # Upsampling 3 # X = ops.conv_layer(X, k_shape=[3, 3, decoding_filters[1], decoding_filters[2]], stride=1, padding='SAME', # w_init='gu', w_decay=None, scope_name='conv_6', add_smry=False) # print(X.shape) X = ops.conv2D_transposed_strided( X, k_shape=[3, 3, decoding_filters[1], decoding_filters[2]], stride=2, padding='SAME', w_init='gu', out_shape=None, scope_name='dconv_3', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_9') logging.info('%s : dconv_3 shape: %s', 'DECODER: ', str(X.shape)) X = ops.activation(X, 'relu', 'relu_8') # print(X.shape) # Upsampling 4 X = ops.conv2D_transposed_strided( X, k_shape=[3, 3, decoding_filters[2], decoding_filters[3]], stride=2, padding='SAME', w_init='gu', out_shape=None, scope_name='dconv_4', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_10') logging.info('%s : dconv_4 shape: %s', 'DECODER: ', str(X.shape)) X = ops.activation(X, 'relu', 'relu_9') # Upsampling 5 X = ops.conv2D_transposed_strided( X, k_shape=[3, 3, decoding_filters[3], decoding_filters[4]], stride=2, padding='SAME', w_init='gu', out_shape=None, scope_name='dconv_5', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_11') logging.info('%s : dconv_5 shape: %s', 'DECODER: ', str(X.shape)) X = ops.activation(X, 'relu', 'relu_10') # Upsampling 6 X = ops.conv2D_transposed_strided( X, k_shape=[3, 3, decoding_filters[4], decoding_filters[5]], stride=2, padding='SAME', w_init='gu', out_shape=None, scope_name='dconv_6', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_12') logging.info('%s : dconv_6 shape: %s', 'DECODER: ', str(X.shape)) X = ops.activation(X, 'relu', 'relu_11') # Decoding X = ops.conv_layer( X, k_shape=[3, 3, decoding_filters[5], decoding_filters[6]], stride=1, padding='SAME', w_init='gu', w_decay=None, scope_name='conv_7', add_smry=False) # X = ops.batch_norm(X, axis=[0, 1, 2], scope_name='bn_13') logging.info('%s : conv_7 shape: %s', 'DECODER: ', str(X.shape)) sigmoid_logits = ops.activation(X, 'sigmoid', 'sigmoid_1') # print(X.shape) return X, sigmoid_logits