def conv_layer(x, k=3, n=8, stride=1, bias=0.1, winit=he_weights_initializer(), pad='SAME', name="conv_layer"): num_filters_in = int(x.get_shape()[-1]) # The number of chanels coming in with tf.name_scope(name) as scope: weights = tf.Variable(winit([k, k, num_filters_in, n]), name="weights", dtype=tf.float32) if bias is not None: conv = tf.nn.conv2d(x, weights, strides=[1, stride, stride, 1], padding=pad, name="conv") bias = tf.Variable(tf.constant(bias, shape=[n]), name="bias") preactivation = tf.add(conv, bias, name=scope) else: preactivation = tf.nn.conv2d(x, weights, strides=[1, stride, stride, 1], padding=pad, name=scope) return preactivation
def multi_digit_loss(logits_list, Y, max_digits=5, name="loss"): """ Calculates the loss for the multi-digit recognition task, given a list of the logits for each digit, and the correct labels. Args: logits: (list of tensors) list of the logits from each of the branches Y: (tensor) correct labels, shaped as [n_batch, max_digits] name: (str) Name for the scope of this loss. Returns: (tensor) the loss """ with tf.name_scope(name) as scope: # LOSSES FOR EACH DIGIT BRANCH losses = [None] * (max_digits) losses[0] = tf.nn.sparse_softmax_cross_entropy_with_logits( logits_list[0], Y[:, 0]) losses[1] = tf.nn.sparse_softmax_cross_entropy_with_logits( logits_list[1], Y[:, 1]) losses[2] = tf.nn.sparse_softmax_cross_entropy_with_logits( logits_list[2], Y[:, 2]) losses[3] = tf.nn.sparse_softmax_cross_entropy_with_logits( logits_list[3], Y[:, 3]) losses[4] = tf.nn.sparse_softmax_cross_entropy_with_logits( logits_list[4], Y[:, 4]) # AVERAGE LOSS loss = sum(losses) / float(max_digits) loss = tf.reduce_mean(loss, name=scope) return loss
def fc_layer(x, n=32, bias=0.1, winit=he_weights_initializer(), dtype=tf.float32, name="fc_layer"): """Fully Connected Layer Args: x: The tensor from the previous layer. n: The number of nodes for this layer (width). bias: (float or None)(default=0.1) Initial value for bias units. If None, then no Biases will be used. winit: weights initializer name: Name for this operation """ in_nodes = int(x.get_shape()[-1]) # The number of nodes from input x with tf.name_scope(name) as scope: weights = tf.Variable(winit([in_nodes, n]), name="weights", dtype=dtype) if bias is not None: bias = tf.Variable(tf.constant(bias, shape=[n], dtype=dtype), name="bias") preactivation = tf.add(tf.matmul(x, weights), bias, name=scope) else: preactivation = tf.matmul(x, weights, name=scope) return preactivation
def fc_battery(x, global_step, n=1024, bias=0.1, is_training=False, dropout=0.0, winit=None, verbose=False, name="FC"): # BATCH NORM SETTINGS bn_offset = 0.0 bn_scale = 1.0 # DROPOUT dropout = tf.cond(is_training, lambda: tf.constant(dropout), lambda: tf.constant(0.0)) # DEFAULT WEIGHTS INITIALIZATION if winit is None: winit = he_weights_initializer() # identity_weights_initializer() # FC STACK with tf.name_scope(name) as scope: x = fc_layer(x, n=n, bias=bias, winit=winit, name="FC") x = batchnormC(x, is_training=is_training, iteration=global_step, conv=False, offset=bn_offset, scale=bn_scale) x = tf.nn.dropout(x, keep_prob=1 - dropout) x = leaky_relu(x, rate=0.01, name="relu") print_tensor_shape(x, name=scope, verbose=verbose) return x
def conv_battery(x, global_step, convk=3, n=32, mpk=2, mpstride=1, dropout=0.0, is_training=False, name="C", verbose=False): # BATCH NORM SETTINGS bn_offset = 0.0 bn_scale = 1.0 # DROPOUT conv_dropout = tf.cond(is_training, lambda: tf.constant(dropout), lambda: tf.constant(0.0)) # CONV STACK with tf.name_scope(name) as scope: x = conv_layer(x, k=convk, n=n, bias=None, stride=1, name="conv") x = batchnormC(x, is_training=is_training, iteration=global_step, conv=True, offset=bn_offset, scale=bn_scale) x = tf.nn.dropout(x, keep_prob=1 - conv_dropout) x = max_pool_layer(x, k=mpk, stride=mpstride, name="maxpool") x = leaky_relu(x, name="relu") print_tensor_shape(x, name=scope, verbose=verbose) return x
def flatten(x, name="flatten"): """Given a tensor whose first dimension is the batch_size, and all other dimensions are elements of data, then it flattens the data so you get a [batch_size, num_elements] sized tensor""" with tf.name_scope(name) as scope: num_elements = np.product(x.get_shape().as_list()[1:]) x = tf.reshape(x, [-1, num_elements], name=scope) return x
def leaky_relu(x, rate=0.01, name="leaky_relu"): """Leaky Rectified Linear Activation Unit Args: x: preactivation tensor rate: Amount of leakiness name: name for this op """ with tf.name_scope(name) as scope: leak_rate = tf.multiply(x, rate, name="leak_rate") activation = tf.maximum(x, leak_rate, name=scope) # activation_summary(activation) # tf.histogram_summary(scope + '/activation', activation) return activation