def fully_connected(x, n_output, is_training, reuse, activation=None, batch_norm=None, batch_norm_args=None, w_init=initz.he_normal(), use_bias=True, b_init=0.0, w_regularizer=tf.nn.l2_loss, outputs_collections=None, trainable=True, name='fc'): input_shape = helper.get_input_shape(x) assert len(input_shape) > 1, "Input Tensor shape must be > 1-D" if len(x.get_shape()) != 2: x = _flatten(x) n_input = helper.get_input_shape(x)[1] with tf.variable_scope(name, reuse=reuse) as curr_scope: shape = [n_input, n_output] if hasattr(w_init, '__call__') else None W = tf.get_variable(name='weights', shape=shape, initializer=w_init, regularizer=w_regularizer, trainable=trainable) output = tf.matmul(x, W) if use_bias: b = tf.get_variable(name='biases', shape=[n_output], initializer=tf.constant_initializer(b_init), trainable=trainable) output = tf.nn.bias_add(value=output, bias=b) if batch_norm is not None and batch_norm is not False: if isinstance(batch_norm, bool): batch_norm = batch_norm_tf batch_norm_args = batch_norm_args or {} output = batch_norm(output, is_training=is_training, reuse=reuse, trainable=trainable, **batch_norm_args) if activation: output = activation(output, is_training=is_training, reuse=reuse, trainable=trainable) return _collect_named_outputs(outputs_collections, curr_scope.original_name_scope, name, output)
def global_avg_pool(x, outputs_collections=None, name="global_avg_pool", **unused): _check_unused(unused, name) input_shape = helper.get_input_shape(x) assert len(input_shape) == 4, "Input Tensor shape must be 4-D" with tf.name_scope(name) as curr_scope: output = tf.reduce_mean(x, [1, 2]) return _collect_named_outputs(outputs_collections, curr_scope, name, output)
def _setup_summaries(self): def image_batch_like(shape): return len(shape) == 4 and shape[3] in {1, 3, 4} with tf.name_scope('summaries'): self.epoch_loss = tf.placeholder(tf.float32, shape=[], name="epoch_loss") # Training summaries tf.summary.scalar('learning rate', self.learning_rate, collections=[TRAINING_EPOCH_SUMMARIES]) tf.summary.scalar('training (cross entropy) loss', self.epoch_loss, collections=[TRAINING_EPOCH_SUMMARIES]) if image_batch_like(util.get_input_shape(self.inputs)): tf.summary.image('input', self.inputs, 10, collections=[TRAINING_BATCH_SUMMARIES]) for key, val in self.training_end_points.iteritems(): variable_summaries(val, key, collections=[TRAINING_BATCH_SUMMARIES]) # for var in tf.trainable_variables(): # variable_summaries(var, var.op.name, collections=[TRAINING_BATCH_SUMMARIES]) for grad, var in self.grads_and_vars: variable_summaries(var, var.op.name, collections=[TRAINING_BATCH_SUMMARIES]) variable_summaries(grad, var.op.name + '/grad', collections=[TRAINING_BATCH_SUMMARIES]) # Validation summaries for key, val in self.validation_end_points.iteritems(): variable_summaries(val, key, collections=[VALIDATION_BATCH_SUMMARIES]) tf.summary.scalar('validation loss', self.epoch_loss, collections=[VALIDATION_EPOCH_SUMMARIES]) self.validation_metric_placeholders = [] for metric_name, _ in self.validation_metrics_def: validation_metric = tf.placeholder(tf.float32, shape=[], name=metric_name.replace(' ', '_')) self.validation_metric_placeholders.append(validation_metric) tf.summary.scalar(metric_name, validation_metric, collections=[VALIDATION_EPOCH_SUMMARIES]) self.validation_metric_placeholders = tuple(self.validation_metric_placeholders)
def _flatten(x, name='flatten'): input_shape = helper.get_input_shape(x) assert len(input_shape) > 1, "Input Tensor shape must be > 1-D" with tf.name_scope(name): dims = int(np.prod(input_shape[1:])) flattened = tf.reshape(x, [-1, dims]) return flattened
def feature_max_pool_1d(x, stride=2, outputs_collections=None, name='feature_max_pool', **unused): _check_unused(unused, name) input_shape = helper.get_input_shape(x) assert len(input_shape) == 2, "Input Tensor shape must be 2-D" with tf.name_scope(name) as curr_scope: x = tf.reshape(x, (-1, input_shape[1] // stride, stride)) output = tf.reduce_max( input_tensor=x, reduction_indices=[2], ) return _collect_named_outputs(outputs_collections, curr_scope, name, output)
def avg_pool_2d(x, filter_size=(3, 3), stride=(2, 2), padding='SAME', outputs_collections=None, name='avg_pool', **unused): _check_unused(unused, name) input_shape = helper.get_input_shape(x) assert len(input_shape) == 4, "Input Tensor shape must be 4-D" with tf.name_scope(name) as curr_scope: output = tf.nn.avg_pool( value=x, ksize=[1, filter_size[0], filter_size[1], 1], strides=[1, stride[0], stride[1], 1], padding=padding, ) return _collect_named_outputs(outputs_collections, curr_scope, name, output)
def _flatten(x): input_shape = helper.get_input_shape(x) assert len(input_shape) > 1, "Input Tensor shape must be > 1-D" dims = int(np.prod(input_shape[1:])) flattened = tf.reshape(x, [-1, dims]) return flattened
def batch_norm_lasagne(x, is_training, reuse, decay=0.9, epsilon=1e-4, updates_collections=tf.GraphKeys.UPDATE_OPS, outputs_collections=None, trainable=True, name='bn'): with tf.variable_scope(name, reuse=reuse) as curr_scope: beta = tf.get_variable(name='beta', initializer=tf.constant( 0.0, shape=[x.get_shape()[-1]]), trainable=trainable) gamma = tf.get_variable(name='gamma', initializer=tf.constant( 1.0, shape=[x.get_shape()[-1]]), trainable=trainable) moving_mean = tf.get_variable(name='moving_mean', shape=[x.get_shape()[-1]], initializer=tf.zeros_initializer, trainable=False) moving_inv_std = tf.get_variable(name='moving_inv_std', shape=[x.get_shape()[-1]], initializer=tf.ones_initializer(), trainable=False) input_shape = helper.get_input_shape(x) moments_axes = list(range(len(input_shape) - 1)) def mean_inv_std_with_update(): mean, variance = tf.nn.moments(x, moments_axes, shift=moving_mean, name='bn-moments') inv_std = math_ops.rsqrt(variance + epsilon) update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean, decay, zero_debias=False) update_moving_inv_std = moving_averages.assign_moving_average( moving_inv_std, inv_std, decay, zero_debias=False) with tf.control_dependencies( [update_moving_mean, update_moving_inv_std]): m, v = tf.identity(mean), tf.identity(inv_std) return m, v def mean_inv_std_with_pending_update(): mean, variance = tf.nn.moments(x, moments_axes, shift=moving_mean, name='bn-moments') inv_std = math_ops.rsqrt(variance + epsilon) update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean, decay, zero_debias=False) update_moving_inv_std = moving_averages.assign_moving_average( moving_inv_std, inv_std, decay, zero_debias=False) tf.add_to_collection(updates_collections, update_moving_mean) tf.add_to_collection(updates_collections, update_moving_inv_std) return mean, inv_std mean_inv_std_with_relevant_update = \ mean_inv_std_with_pending_update if updates_collections is not None else mean_inv_std_with_update (mean, inv_std) = mean_inv_std_with_relevant_update() if is_training else ( moving_mean, moving_inv_std) def _batch_normalization(x, mean, inv, offset, scale): with tf.name_scope(name, "batchnorm", [x, mean, inv, scale, offset]): if scale is not None: inv *= scale return x * inv + (offset - mean * inv if offset is not None else -mean * inv) output = _batch_normalization(x, mean, inv_std, beta, gamma) return _collect_named_outputs(outputs_collections, curr_scope.original_name_scope, output)
def conv2d(x, n_output_channels, is_training, reuse, filter_size=(3, 3), stride=(1, 1), dilation_rate=1, padding='SAME', activation=None, batch_norm=None, batch_norm_args=None, w_init=initz.he_normal(), use_bias=True, untie_biases=False, b_init=0.0, w_regularizer=tf.nn.l2_loss, outputs_collections=None, trainable=True, name='conv2d'): input_shape = helper.get_input_shape(x) assert len(input_shape) == 4, "Input Tensor shape must be 4-D" with tf.variable_scope(name, reuse=reuse) as curr_scope: shape = [ filter_size[0], filter_size[1], x.get_shape()[-1], n_output_channels ] if hasattr(w_init, '__call__') else None W = tf.get_variable(name='weights', shape=shape, initializer=w_init, regularizer=w_regularizer, trainable=trainable) if dilation_rate == 1: output = tf.nn.conv2d(input=x, filter=W, strides=[1, stride[0], stride[1], 1], padding=padding) else: if len([s for s in stride if s > 1]) > 0: raise ValueError( "Stride (%s) cannot be more than 1 if rate (%d) is not 1" % (stride, dilation_rate)) output = tf.nn.atrous_conv2d(value=x, filters=W, rate=dilation_rate, padding=padding) if use_bias: if untie_biases: b = tf.get_variable( name='biases', shape=output.get_shape()[1:], initializer=tf.constant_initializer(b_init), trainable=trainable) output = tf.add(output, b) else: b = tf.get_variable( name='biases', shape=[n_output_channels], initializer=tf.constant_initializer(b_init), trainable=trainable) output = tf.nn.bias_add(value=output, bias=b) if batch_norm is not None: if isinstance(batch_norm, bool): batch_norm = batch_norm_tf batch_norm_args = batch_norm_args or {} output = batch_norm(output, is_training=is_training, reuse=reuse, trainable=trainable, **batch_norm_args) if activation: output = activation(output, is_training=is_training, reuse=reuse, trainable=trainable) return _collect_named_outputs(outputs_collections, curr_scope.original_name_scope, output)