def bottleneck_hole(inputs, depth, depth_bottleneck, stride, rate=2, outputs_collections=None, scope=None): with variable_scope.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') else: shortcut = layers.conv2d( inputs, depth, [1, 1], stride=stride, activation_fn=None, scope='shortcut') residual = layers.conv2d( inputs, depth_bottleneck, [1, 1], stride=1, scope='conv1') residual = layers_lib.conv2d(residual, depth_bottleneck, [3, 3], stride=1, rate=rate, padding='SAME', scope='conv2') residual = layers.conv2d( residual, depth, [1, 1], stride=1, activation_fn=None, scope='conv3') output = nn_ops.relu(shortcut + residual) return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, scope=None): with tf.variable_scope(scope, 'bottleneck_v1') as sc: depth_in = utils.last_dimension(inputs.outputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = tl.layers.Conv2d(inputs, depth, filter_size=(1, 1), strides=(stride, stride), act=None, b_init=None, name='shortcut_conv') shortcut = tl.layers.BatchNormLayer(shortcut, act=tf.identity, is_train=True, name='shortcut_bn/BatchNorm') # bottleneck layer 1 residual = tl.layers.Conv2d(inputs, depth_bottleneck, filter_size=(1, 1), strides=(1, 1), act=None, b_init=None, name='conv1') residual = tl.layers.BatchNormLayer(residual, act=tf.nn.relu, is_train=True, name='conv1_bn/BatchNorm') # bottleneck layer 2 residual = conv2d_same(residual, depth_bottleneck, kernel_size=3, strides= stride, rate=rate, scope='conv2') # bottleneck layer 3 residual = tl.layers.Conv2d(residual, depth, filter_size=(1, 1), strides=(1, 1), act=None, b_init=None, name='conv3') residual = tl.layers.BatchNormLayer(residual, act=tf.identity, is_train=True, name='conv3_bn/BatchNorm') output = ElementwiseLayer(layer=[shortcut, residual], combine_fn=tf.add, name='combine_layer', act=tf.nn.relu) return output
def inference (images, train=True, resnet_stride=8): with slim.arg_scope(resnet_v1.resnet_arg_scope(train)): net, end_points = resnet_v1_slim(images, num_classes = None, global_pool = False, output_stride = resnet_stride) # replace resnet_v1_slim above with resnet_v1.resnet_v1_50/101/... # to use standard architectures. # num_classes: Number of predicted classes for classification tasks. If None # we return the features before the logit layer. # global_pool: If True, we perform global average pooling before computing the # logits. Set to True for image classification, False for dense prediction. # output_stride: If None, then the output will be computed at the nominal # network stride. If output_stride is not None, it specifies the requested # ratio of input to output spatial resolution. resnet_depth = utils.last_dimension(net.get_shape(), min_rank=4) shape = tf.unpack(tf.shape(images)) print(shape.__class__) shape.pop() shape.append(tf.constant(FLAGS.out_channels, dtype=tf.int32)) print(len(shape)) filters = tf.Variable( tf.truncated_normal( [resnet_stride*2+1, resnet_stride*2+1, FLAGS.out_channels, resnet_depth], dtype=tf.float32, stddev=0.01), name='filters') logits = tf.nn.conv2d_transpose(net, filters, tf.pack(shape), [1,resnet_stride,resnet_stride,1], padding='SAME', name='upscale') return logits
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, outputs_collections=None, scope=None): """Bottleneck residual unit variant with BN before convolutions. This is the full preactivation residual unit variant proposed in [2]. See Fig. 1(b) of [2] for its definition. Note that we use here the bottleneck variant which has an extra bottleneck layer. When putting together two consecutive ResNet blocks that use this unit, one should use stride = 2 in the last unit of the first block. Args: inputs: A tensor of size [batch, height, width, channels]. depth: The depth of the ResNet unit output. depth_bottleneck: The depth of the bottleneck layers. stride: The ResNet unit's stride. Determines the amount of downsampling of the units output compared to its input. rate: An integer, rate for atrous convolution. outputs_collections: Collection to add the ResNet unit output. scope: Optional variable_scope. Returns: The ResNet unit's output. """ with variable_scope.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) preact = layers.batch_norm( inputs, activation_fn=nn_ops.relu, scope='preact') if depth == depth_in: shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') else: shortcut = layers_lib.conv2d( preact, depth, [1, 1], stride=stride, normalizer_fn=None, activation_fn=None, scope='shortcut') residual = layers_lib.conv2d( preact, depth_bottleneck, [1, 1], stride=1, scope='conv1') residual = resnet_utils.conv2d_same( residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2') residual = layers_lib.conv2d( residual, depth, [1, 1], stride=1, normalizer_fn=None, activation_fn=None, scope='conv3') output = shortcut + residual return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, outputs_collections=None, scope=None): """Bottleneck residual unit variant with BN before convolutions. This is the full preactivation residual unit variant proposed in [2]. See Fig. 1(b) of [2] for its definition. Note that we use here the bottleneck variant which has an extra bottleneck layer. When putting together two consecutive ResNet blocks that use this unit, one should use stride = 2 in the last unit of the first block. Args: inputs: A tensor of size [batch, height, width, channels]. depth: The depth of the ResNet unit output. depth_bottleneck: The depth of the bottleneck layers. stride: The ResNet unit's stride. Determines the amount of downsampling of the units output compared to its input. rate: An integer, rate for atrous convolution. outputs_collections: Collection to add the ResNet unit output. scope: Optional variable_scope. Returns: The ResNet unit's output. """ with variable_scope.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) preact = layers.batch_norm( inputs, activation_fn=nn_ops.relu, scope='preact') if depth == depth_in: shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') else: shortcut = layers_lib.conv2d( preact, depth, [1, 1], stride=stride, normalizer_fn=None, activation_fn=None, scope='shortcut') residual = layers_lib.conv2d( preact, depth_bottleneck, [1, 1], stride=1, scope='conv1') residual = resnet_utils.conv2d_same( residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2') residual = layers_lib.conv2d( residual, depth, [1, 1], stride=1, normalizer_fn=None, activation_fn=None, scope='conv3') output = shortcut + residual return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, outputs_collections=None, scope=None): """Bottleneck residual unit variant with BN after convolutions. This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for its definition. Note that we use here the bottleneck variant which has an extra bottleneck layer. When putting together two consecutive ResNet blocks that use this unit, one should use stride = 2 in the last unit of the first block. Args: inputs: A tensor of size [batch, height, width, channels]. depth: The depth of the ResNet unit output. depth_bottleneck: The depth of the bottleneck layers. stride: The ResNet unit's stride. Determines the amount of downsampling of the units output compared to its input. rate: An integer, rate for atrous convolution. outputs_collections: Collection to add the ResNet unit output. scope: Optional variable_scope. use_bounded_activations: Whether or not to use bounded activations. Bounded activations better lend themselves to quantized inference. Returns: The ResNet unit's output. """ with variable_scope.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') else: shortcut = layers.conv2d( inputs, depth, [1, 1], stride=stride, activation_fn=None, scope='shortcut') residual = layers.conv2d(inputs, depth_bottleneck, [1, 1], stride=1, scope='conv1') # I don't know use activation_fn? #residual = resnet_utils.conv2d_same(residual, depth_bottleneck, 3, stride, # rate=rate, scope='conv2') if stride == 1: residual = layers.separable_conv2d(residual, depth_bottleneck, [3,3], depth_multiplier=1,stride=1,scope='dws_conv2') else: residual = layers.separable_conv2d(residual, depth_bottleneck, [3,3], depth_multiplier=1,stride=stride,scope='dws_conv2') residual = layers.conv2d(residual, depth, [1, 1], stride=1, activation_fn=None, scope='conv3') output = nn_ops.relu(shortcut + residual) return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bottleneck(self, inputs, depth, depth_bottleneck, stride, rate=1, outputs_collections=None, scope=None): """Bottleneck residual unit variant with BN after convolutions. This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for its definition. Note that we use here the bottleneck variant which has an extra bottleneck layer. When putting together two consecutive ResNet blocks that use this unit, one should use stride = 2 in the last unit of the first block. Args: inputs: A tensor of size [batch, height, width, channels]. depth: The depth of the ResNet unit output. depth_bottleneck: The depth of the bottleneck layers. stride: The ResNet unit's stride. Determines the amount of downsampling of the units output compared to its input. rate: An integer, rate for atrous convolution. outputs_collections: Collection to add the ResNet unit output. scope: Optional variable_scope. Returns: The ResNet unit's output. """ with variable_scope.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') else: shortcut = layers.conv2d( inputs, depth, [1, 1], stride=stride, activation_fn=None, scope='shortcut') residual = layers.conv2d( inputs, depth_bottleneck, [1, 1], stride=1, scope='conv1') layer_name = LayerName(sc.name + '/conv2', 'net_layer') # name = sc.name.replace(self._resnet_scope,'') + '/conv2' if layer_name in self._comp_weights_dict.keys(): residual = self.separate_conv_layer(residual, depth_bottleneck, 3, stride, rate=rate, layer_name='conv2', full_layer_name=layer_name) else: residual = resnet_utils.conv2d_same(residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2') residual = layers.conv2d( residual, depth, [1, 1], stride=1, activation_fn=None, scope='conv3') output = nn_ops.relu(shortcut + residual) return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bottleneck_IR(inputs, depth, depth_bottleneck, stride, rate=1, w_init=None, scope=None, trainable=None): with tf.variable_scope(scope, 'bottleneck_v1') as sc: depth_in = utils.last_dimension(inputs.outputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = tl.layers.Conv2d(inputs, depth, filter_size=(1, 1), strides=(stride, stride), act=None, W_init=w_init, b_init=None, name='shortcut_conv', use_cudnn_on_gpu=True) shortcut = GroupNormLayer(layer=shortcut, act=tf.identity, name='shortcut_bn/BatchNorm') # bottleneck layer 1 residual = GroupNormLayer(layer=inputs, act=tf.identity, name='conv1_bn1') residual = tl.layers.Conv2d(residual, depth_bottleneck, filter_size=(3, 3), strides=(1, 1), act=None, b_init=None, W_init=w_init, name='conv1', use_cudnn_on_gpu=True) residual = GroupNormLayer(layer=residual, act=tf.identity, name='conv1_bn2') # bottleneck prelu residual = tl.layers.PReluLayer(residual) # bottleneck layer 2 residual = conv2d_same(residual, depth, kernel_size=3, strides=stride, rate=rate, w_init=w_init, scope='conv2', trainable=trainable) output = ElementwiseLayer(layer=[shortcut, residual], combine_fn=tf.add, name='combine_layer', act=None) return output
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, scope=None): with tf.variable_scope(scope, 'bottleneck_v1') as sc: depth_in = utils.last_dimension(inputs.outputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = tl.layers.Conv2d(inputs, depth, filter_size=(1, 1), strides=(stride, stride), act=None, b_init=None, name='shortcut_conv') shortcut = tl.layers.BatchNormLayer(shortcut, act=tf.identity, is_train=True, name='shortcut_bn/BatchNorm') # bottleneck layer 1 residual = tl.layers.Conv2d(inputs, depth_bottleneck, filter_size=(1, 1), strides=(1, 1), act=None, b_init=None, name='conv1') residual = tl.layers.BatchNormLayer(residual, act=tf.nn.relu, is_train=True, name='conv1_bn/BatchNorm') # bottleneck layer 2 residual = conv2d_same(residual, depth_bottleneck, kernel_size=3, strides=stride, rate=rate, scope='conv2') # bottleneck layer 3 residual = tl.layers.Conv2d(residual, depth, filter_size=(1, 1), strides=(1, 1), act=None, b_init=None, name='conv3') residual = tl.layers.BatchNormLayer(residual, act=tf.identity, is_train=True, name='conv3_bn/BatchNorm') output = ElementwiseLayer(layer=[shortcut, residual], combine_fn=tf.add, name='combine_layer', act=tf.nn.relu) return output
def bottleneck(inputs, depth, depth_bottleneck, stride, outputs_collections=None, scope=None): """ Args: inputs: A tensor of size [batch, height, width, channels]. depth、depth_bottleneck:、stride三个参数是前面blocks类中的args rate: An integer, rate for atrous convolution. outputs_collections: 是收集end_points的collection """ with tf.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) # 最后一个维度,即输出通道数 preact = slim.batch_norm(inputs, activation_fn=tf.nn.relu, scope='preact') if depth == depth_in: # 如果残差单元的输入通道数和输出通道数一致,那么按步长对inputs进行降采样 shortcut = subsample(inputs, stride, 'shortcut') else: # 如果不一样就按步长和1*1的卷积改变其通道数,使得输入、输出通道数一致 shortcut = slim.conv2d(preact, depth, [1, 1], stride=stride, normalizer_fn=None, activation_fn=None, scope='shortcut') # 先是一个1*1尺寸,步长1,输出通道数为depth_bottleneck的卷积 residual = slim.conv2d(preact, depth_bottleneck, [1, 1], stride=1, scope='conv1') # 然后是3*3尺寸,步长为stride,输出通道数为depth_bottleneck的卷积 residual = conv2d_same(residual, depth_bottleneck, 3, stride, scope='conv2') # 最后是1*1卷积,步长1,输出通道数depth的卷积,得到最终的residual。最后一层没有正则项也没有激活函数 residual = slim.conv2d(residual, depth, [1, 1], stride=1, normalizer_fn=None, activation_fn=None, scope='conv3') # 将降采样的结果和residual相加 output = shortcut + residual return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bias_add(inputs, activation_fn=None, initializer=init_ops.zeros_initializer, regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, scope=None): """Adds a bias to the inputs. Can be used as a normalizer function for conv2d and fully_connected. Args: inputs: a tensor of with at least rank 2 and value for the last dimension, e.g. `[batch_size, depth]`, `[None, None, None, depth]`. activation_fn: Optional activation function. initializer: An initializer for the bias, defaults to 0. regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. reuse: whether or not the layer and its variables should be reused. To be able to reuse the layer scope must be given. variables_collections: optional collections for the variables. outputs_collections: collections to add the outputs. trainable: If `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). scope: Optional scope for variable_op_scope. Returns: a tensor representing the result of adding biases to the inputs. """ with variable_scope.variable_op_scope([inputs], scope, 'BiasAdd', reuse=reuse) as sc: inputs = ops.convert_to_tensor(inputs) dtype = inputs.dtype.base_dtype num_features = utils.last_dimension(inputs.get_shape(), min_rank=2) biases_collections = utils.get_variable_collections( variables_collections, 'biases') biases = variables.model_variable('biases', shape=[ num_features, ], dtype=dtype, initializer=initializer, regularizer=regularizer, collections=biases_collections, trainable=trainable) outputs = nn.bias_add(inputs, biases) if activation_fn: outputs = activation_fn(outputs) return utils.collect_named_outputs(outputs_collections, sc.name, outputs)
def mlp_nobias(x, hidden_size, output_size, activation=tf.nn.relu, scope=None): scope = 'mlp_nobias' if scope is None else scope with tf.variable_scope(scope): input_dim = utils.last_dimension(x.get_shape(), min_rank=2) if isinstance(x, tf.Tensor): w_h = melt.get_weights('w_h', [input_dim, hidden_size]) else: with tf.device('/cpu:0'): w_h = melt.get_weights('w_h', [input_dim, hidden_size]) w_o = melt.get_weights('w_o', [hidden_size, output_size]) return melt.mlp_forward_nobias(x, w_h, w_o, activation)
def blur_pool(x, filt_size=7, stride=2, pad_off=0): channels = utils.last_dimension(x.get_shape(), min_rank=4) if (filt_size == 1): a = np.array([ 1., ]) elif (filt_size == 2): a = np.array([1., 1.]) elif (filt_size == 3): a = np.array([1., 2., 1.]) elif (filt_size == 4): a = np.array([1., 3., 3., 1.]) elif (filt_size == 5): a = np.array([1., 4., 6., 4., 1.]) elif (filt_size == 6): a = np.array([1., 5., 10., 10., 5., 1.]) elif (filt_size == 7): a = np.array([1., 6., 15., 20., 15., 6., 1.]) kernel_1 = a[:, None] * a[None, :] kernel_1 = np.expand_dims(kernel_1 / np.sum(np.sum(kernel_1)), -1) kernel_1 = np.expand_dims(kernel_1.repeat(axis=-1, repeats=channels), axis=-1).repeat(axis=-1, repeats=1).astype(np.float32) kernel = tf.Variable(kernel_1, trainable=False) pad_sizes = [ int(1. * (filt_size - 1) / 2), int(np.ceil(1. * (filt_size - 1) / 2)), int(1. * (filt_size - 1) / 2), int(np.ceil(1. * (filt_size - 1) / 2)) ] pad_sizes = [pad_size + pad_off for pad_size in pad_sizes] l1 = list(pad_sizes[:2]) l2 = list(pad_sizes[2:]) if filt_size == 1: if pad_off == 0: return x[:, :, ::stride, ::stride] else: return tf.pad(x, [[0, 0], l1, l2, [0, 0]], 'REFLECT')[:, :, ::stride, ::stride] else: x = tf.pad(x, [[0, 0], l1, l2, [0, 0]], 'REFLECT') x = tf.nn.depthwise_conv2d(x, kernel, strides=[1, stride, stride, 1], padding='VALID') return x
def resid_unit(inputs, depth, depth_bottleneck, stride, rate=1, outputs_collections=None, scope=None): """Residual unit with BN after convolutions. This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for its definition. When putting together two consecutive ResNet blocks that use this unit, one should use stride = 2 in the last unit of the first block. Args: inputs: A tensor of size [batch, height, width, channels]. depth: The depth of the ResNet unit output. depth_bottleneck: The depth of the bottleneck layers. stride: The ResNet unit's stride. Determines the amount of downsampling of the units output compared to its input. rate: An integer, rate for atrous convolution. outputs_collections: Collection to add the ResNet unit output. scope: Optional variable_scope. Returns: The ResNet unit's output. """ with variable_scope.variable_scope(scope, 'resid_v1', [inputs]) as sc: # print (inputs.shape) depth_in = utils.last_dimension(inputs.get_shape(), min_rank=5) if depth == depth_in: shortcut = resnet_3d_utils.subsample(inputs, stride, 'shortcut') else: shortcut = layers.conv3d(inputs, depth, [1, 1, 1], stride=stride, activation_fn=None, scope='shortcut') residual = resnet_3d_utils.conv3d_same(inputs, depth_bottleneck, 3, stride=1, scope='conv1') residual = layers.conv3d(residual, depth_bottleneck, 3, stride, scope='conv2') output = nn_ops.relu(shortcut + residual) return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, scope=None, **common_args): conv_args_common = make_args(use_bias=False, batch_norm=batch_norm_tf, batch_norm_args=batch_norm_params, **common_args) conv_args_relu = make_args(activation=relu, **conv_args_common) conv_args_none = make_args(activation=None, **conv_args_common) with tf.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut', **common_args) else: shortcut = conv2d(inputs, depth, filter_size=(1, 1), stride=(stride, stride), name='shortcut', **conv_args_none) residual = conv2d(inputs, depth_bottleneck, filter_size=(1, 1), stride=(1, 1), name='conv1', **conv_args_relu) residual = conv2d_same(residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2', **conv_args_relu) residual = conv2d(residual, depth, filter_size=(1, 1), stride=(1, 1), name='conv3', **conv_args_none) # not in endpoints. does that matter? output = tf.nn.relu(shortcut + residual) return collect_named_outputs(common_args['outputs_collections'], sc.name, output)
def bottleneck2(inputs, depth, depth_bottleneck, stride, outputs_collections=None, scope=None): with tf.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: # preact = slim.batch_norm(inputs, activation_fn=tf.nn.relu, scope='preact') preact = inputs #tf.nn.relu(inputs) depth_in = utils.last_dimension(inputs.get_shape(), min_rank=3) if depth_in == depth: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = slim.convolution(inputs, depth, 1, stride=stride, padding='SAME', normalizer_fn=None, activation_fn=None, scope='shortcut') # residual = slim.dropout(preact, keep_prob=0.8, scope='dropout1') residual = conv2d_same(preact, depth_bottleneck, 16, stride, scope='conv1') # residual = slim.dropout(residual, keep_prob=0.8, scope='dropout2') residual = slim.convolution(residual, depth, 16, stride=1, padding='SAME', scope='conv2') # residual = slim.convolution(residual, depth, 16, # stride=1, # padding='SAME', # normalizer_fn=None, # activation_fn=None, # scope='conv2' # ) output = shortcut + residual return utils.collect_named_outputs(outputs_collections, sc.name, output)
def _bottleneck(inputs, filters, kernel, t, s, is_training, r=False): num_filters_in = utils.last_dimension(inputs.get_shape(), min_rank=4) # "Expension layer" + BN + activation x = _conv_block( inputs=inputs, filters=num_filters_in * t, kernel=(1, 1), strides=(1, 1), is_training=is_training) # Depthwise convolution + BN + activation x = tf.contrib.layers.separable_conv2d( inputs=x, num_outputs=None, kernel_size=kernel, depth_multiplier=1, stride=(s, s), padding='SAME', activation_fn=None, # tf.nn.relu6, weights_initializer=tf.contrib.layers.xavier_initializer(seed=481), normalizer_fn=None) x = tf.layers.batch_normalization( inputs=x, training=is_training) x = tf.nn.relu6(features=x) # x = tf.nn.leaky_relu(features=x) # RAN - trial # "Projection" layer + BN x = tf.layers.conv2d( inputs=x, filters=filters, kernel_size=(1, 1), strides=(1, 1), padding='same', kernel_initializer=tf.contrib.layers.xavier_initializer(seed=481)) x = tf.layers.batch_normalization( inputs=x, training=is_training) if r: x = tf.add(x, inputs) return x
def bias_add(inputs, activation_fn=None, initializer=init_ops.zeros_initializer, regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, scope=None): """Adds a bias to the inputs. Can be used as a normalizer function for conv2d and fully_connected. Args: inputs: a tensor of with at least rank 2 and value for the last dimension, e.g. `[batch_size, depth]`, `[None, None, None, depth]`. activation_fn: Optional activation function. initializer: An initializer for the bias, defaults to 0. regularizer: A regularizer like the result of `l1_regularizer` or `l2_regularizer`. reuse: whether or not the layer and its variables should be reused. To be able to reuse the layer scope must be given. variables_collections: optional collections for the variables. outputs_collections: collections to add the outputs. trainable: If `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). scope: Optional scope for variable_op_scope. Returns: a tensor representing the result of adding biases to the inputs. """ with variable_scope.variable_op_scope([inputs], scope, 'BiasAdd', reuse=reuse) as sc: inputs = ops.convert_to_tensor(inputs) dtype = inputs.dtype.base_dtype num_features = utils.last_dimension(inputs.get_shape(), min_rank=2) biases_collections = utils.get_variable_collections(variables_collections, 'biases') biases = variables.model_variable('biases', shape=[num_features,], dtype=dtype, initializer=initializer, regularizer=regularizer, collections=biases_collections, trainable=trainable) outputs = nn.bias_add(inputs, biases) if activation_fn: outputs = activation_fn(outputs) return utils.collect_named_outputs(outputs_collections, sc.name, outputs)
def linear(name_scope, inputs, nb_output_channels): with tf.variable_scope(name_scope): dtype = inputs.dtype.base_dtype nb_input_channels = utils.last_dimension(inputs.get_shape(), min_rank=2) weights_shape = [nb_input_channels, nb_output_channels] weights = tf.get_variable( 'weights', weights_shape, initializer=initializers.xavier_initializer(), dtype=dtype) bias = tf.get_variable('bias', [nb_output_channels], initializer=init_ops.zeros_initializer, dtype=dtype) output = tf.nn.bias_add(tf.matmul(inputs, weights), bias) return output
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, outputs_collections=None, scope=None): """Bottleneck residual unit variant with BN before convolutions.""" with variable_scope.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=3) preact = layers.batch_norm(inputs, activation_fn=nn_ops.relu, scope='preact') if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = layers.convolution(preact, depth, 1, stride=stride, normalizer_fn=None, activation_fn=None, scope='shortcut') residual = layers.convolution(preact, depth_bottleneck, 1, stride=1, scope='conv1') residual = conv1d_same(residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2') residual = layers.convolution(residual, depth, 1, stride=1, normalizer_fn=None, activation_fn=None, scope='conv3') output = shortcut + residual return utils.collect_named_outputs(outputs_collections, sc.name, output)
def fully_connected(x, output_size, activation=tf.nn.relu, scope=None): #@TODO -1 or last dim ? NotImplementedError("Negative indices are currently unsupported") #input_dim = tf.shape(x)[-1] #@TODO how is slim.fully_connected get inputdim and use.. #below will now work int() argument must be a string or a number, not 'Tensor' [input_dim, output_size]) #input_dim = tf.shape(x)[1] #check contrib\layers\python\layers\layers.py scope = 'fc' if scope is None else scope with tf.variable_scope(scope): input_dim = utils.last_dimension(x.get_shape(), min_rank=2) if isinstance(x, tf.Tensor): w_h = melt.get_weights('w_h', [input_dim, output_size]) else: with tf.device('/cpu:0'): w_h = melt.get_weights('w_h', [input_dim, output_size]) b_h = melt.get_bias('b_h', [output_size]) return activation(melt.matmul(x, w_h) + b_h)
def preact_conv2d(inputs, num_outputs, kernel_size, stride=1, padding='SAME', activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, scope=None): """Adds a 2D convolution preceded by batch normalization and activation. """ with variable_scope.variable_scope(scope, 'Conv', values=[inputs], reuse=reuse) as sc: inputs = ops.convert_to_tensor(inputs) dtype = inputs.dtype.base_dtype if normalizer_fn: normalizer_params = normalizer_params or {} inputs = normalizer_fn(inputs, activation_fn=activation_fn, **normalizer_params) kernel_h, kernel_w = utils.two_element_tuple(kernel_size) stride_h, stride_w = utils.two_element_tuple(stride) num_filters_in = utils.last_dimension(inputs.get_shape(), min_rank=4) weights_shape = [kernel_h, kernel_w, num_filters_in, num_outputs] weights_collections = utils.get_variable_collections( variables_collections, 'weights') weights = variables.model_variable('weights', shape=weights_shape, dtype=dtype, initializer=weights_initializer, regularizer=weights_regularizer, collections=weights_collections, trainable=trainable) outputs = nn.conv2d(inputs, weights, [1, stride_h, stride_w, 1], padding=padding) return utils.collect_named_outputs(outputs_collections, sc.name, outputs)
def bottleneck( inputs, depth, expansion, stride, outputs_collections=None, scope=None): """ :param inputs: :param depth: :param expansion: :param stride: :param outputs_collections: :param scope: :return output: """ with tf.variable_scope(scope, 'bottleneck', [inputs]) as sc: residual = inputs depth_in = utils.last_dimension( inputs.get_shape(), min_rank=4) output = slim.conv2d( inputs, expansion*depth_in, 1, 1, scope='conv1x1_1') output = slim.batch_norm( output, scope='conv1x1_1_bn') output = tf.nn.relu6(output) output = slim.separable_conv2d( output, depth, 3, 1, stride, scope='separable_conv3x3') output = slim.batch_norm( output, scope='separable_conv3x3_bn') output = tf.nn.relu6(output) output = slim.conv2d( output, depth, 1, 1, scope='conv1x1_2') output = slim.batch_norm(output, scope='conv1x1_2_bn') if stride == 1: output += residual return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, outputs_collections=None, scope=None): with variable_scope.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) preact = layers.batch_norm( inputs, activation_fn=nn_ops.relu, scope='preact') if depth == depth_in: shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') else: shortcut = layers_lib.conv2d( preact, depth, [1, 1], stride=stride, normalizer_fn=None, activation_fn=None, scope='shortcut') residual = preact residual = tf.layers.batch_normalization(residual) residual = tf.nn.relu(residual) residual = layers_lib.conv2d( residual, depth_bottleneck, [1, 1], stride=1, scope='conv1') residual = tf.layers.batch_normalization(residual) residual = tf.nn.relu(residual) residual = resnet_utils.conv2d_same( residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2') residual = tf.layers.batch_normalization(residual) residual = tf.nn.relu(residual) residual = layers_lib.conv2d( residual, depth, [1, 1], stride=1, normalizer_fn=None, activation_fn=None, scope='conv3') output = shortcut + residual return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bottleneck(inputs, depth, depth_bottleneck, stride, outputs_collections=None, scope=None): with tf.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: # 获取输入的最后一个维度,输出通道数 depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) # 对输入进行batch_borm,接着用relu进行预激活 preact = slim.batch_norm(inputs, activation_fn=tf.nn.relu, scope='preact') if depth == depth_in: # 如果残差单元输入通道和输出通道数一样,就对inputs进行降采样 shortcut = subsample(inputs, stride, 'shortcut') else: # 如果残差单元输入通道与输出通道数不一样,就使用stride步长的1*1卷积改变其通道数,是的输入通道和输出通道数一样 shortcut = slim.conv2d(preact, depth, [1, 1], stride=stride, normalizer_fn=None, activation_fn=None, scope='shortcut') # 定义残差 # 第一步:1*1,stride=1,输出通道数为depth_bottleneck的卷积 # 第二步:3*3,stride=stride,输出通道数为depth_bottleneck的卷积 # 第三步:1*1,stride=1,输出通道数为depth的卷积 residual = slim.conv2d(preact, depth_bottleneck, [1, 1], stride=1, scope='conv1') residual = slim.conv2d(residual, depth_bottleneck, [3, 3], stride=stride, scope='conv2') residual = slim.conv2d(residual, depth, [1, 1], stride=1, scope='conv3') output = shortcut + residual # 将结果添加到outputs_collections return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bottleneck_IR(inputs, depth, depth_bottleneck, stride, rate=1, w_init=None, scope=None, trainable=None): with tf.variable_scope(scope, 'bottleneck_v1') as sc: depth_in = utils.last_dimension(inputs.outputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = tl.layers.Conv2d(inputs, depth, filter_size=(1, 1), strides=(stride, stride), act=None, W_init=w_init, b_init=None, name='shortcut_conv', use_cudnn_on_gpu=True) shortcut.outputs = tf.layers.batch_normalization(inputs=shortcut.outputs, momentum=0.9, training=trainable, renorm=True, renorm_clipping={'rmax': 3, 'rmin': 0.3333, 'dmax': 5}, renorm_momentum=0.9, name='shortcut_bn/BatchNorm') # bottleneck layer 1 inputs.outputs = tf.layers.batch_normalization(inputs=inputs.outputs, momentum=0.9, training=trainable, renorm=True, renorm_clipping={'rmax': 3, 'rmin': 0.3333, 'dmax': 5}, renorm_momentum=0.9, name='conv1_bn1') residual = tl.layers.Conv2d(inputs, depth_bottleneck, filter_size=(3, 3), strides=(1, 1), act=None, b_init=None, W_init=w_init, name='conv1', use_cudnn_on_gpu=True) residual.outputs = tf.layers.batch_normalization(inputs=residual.outputs, momentum=0.9, training=trainable, renorm=True, renorm_clipping={'rmax': 3, 'rmin': 0.3333, 'dmax': 5}, renorm_momentum=0.9, name='conv1_bn2') # bottleneck prelu residual = tl.layers.PReluLayer(residual) # bottleneck layer 2 residual = conv2d_same(residual, depth, kernel_size=3, strides=stride, rate=rate, w_init=w_init, scope='conv2', trainable=trainable) output = ElementwiseLayer(layer=[shortcut, residual], combine_fn=tf.add, name='combine_layer', act=None) return output
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1): with tf.variable_scope('bottleneck_v2'): depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) preact = layers.batch_norm(inputs, activation_fn=nn_ops.relu, scope='preact') if depth == depth_in: shortcut = max_pool(inputs=inputs, kernel_size=1, stride=stride, scope='shortcut') else: with arg_scope([layers_lib.conv2d], normalizer_fn=None, activation_fn=None): shortcut = conv2d(preact, depth, 1, stride=stride, scope='shortcut') residual = layers_lib.conv2d(preact, depth_bottleneck, 1, stride=1, scope='conv1') residual = conv2d(residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2') residual = layers_lib.conv2d(residual, depth, 1, stride=1, normalizer_fn=None, activation_fn=None, scope='conv3') output = shortcut + residual return output
def bottleneck(inputs, depth, depth_bottleneck, stride, outputs_collections=None, scope=None): with tf.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: # resnet_v2 depin_in = utils.last_dimension(inputs.get_shape(), min_rank=4) preact = slim.batch_norm(inputs, activation_fn=tf.nn.relu, scope='preact') if depth == depin_in: # depth是第三层的输出通道数 shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = slim.conv2d(preact, depth, [1, 1], stride=stride, normalizer_fn=None, activation_fn=None, scope='shortcut') residual = slim.conv2d(preact, depth_bottleneck, [1, 1], stride=1, scope='conv1') residual = conv2d_same(residual, depth_bottleneck, 3, stride, scope='conv2') residual = slim.conv2d(residual, depth, [1, 1], stride=1, normalizer_fn=None, activation_fn=None, scope='conv3') output = residual + shortcut return utils.collect_named_outputs(outputs_collections, sc.name, output)
def resnet_reduction(input, kernel_size, is_training, scope=None): with variable_scope.variable_scope(scope, 'resnet_reduction', [input]) as sc: concat_dim = len(input.get_shape().as_list()) - 1 last_dim = utils.last_dimension(input.get_shape()) result_pool = tf.nn.max_pool3d( input=input, ksize=[1, kernel_size, kernel_size, kernel_size, 1], strides=[1, 2, 2, 2, 1], padding='SAME', name=scope) result_resnet = resnet_conv(input=input, kernel_size=kernel_size, stride=2, num_outputs=last_dim, is_training=is_training) return tf.concat(concat_dim, [result_pool, result_resnet])
def bottleneck_SE(inputs, depth, depth_bottleneck, stride, rate=1, scope=None): with tf.variable_scope(scope, 'bottleneck_v1') as sc: depth_in = utils.last_dimension(inputs.outputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = tl.layers.Conv2d(inputs, depth, filter_size=(1, 1), strides=(stride, stride), act=None, b_init=None, name='shortcut_conv') shortcut = tl.layers.BatchNormLayer(shortcut, act=tf.identity, is_train=True, name='shortcut_bn/BatchNorm') # bottleneck layer 1 residual = tl.layers.Conv2d(inputs, depth_bottleneck, filter_size=(1, 1), strides=(1, 1), act=None, b_init=None, name='conv1') residual = tl.layers.BatchNormLayer(residual, act=tf.nn.relu, is_train=True, name='conv1_bn/BatchNorm') # bottleneck layer 2 residual = conv2d_same(residual, depth_bottleneck, kernel_size=3, strides= stride, rate=rate, scope='conv2') # bottleneck layer 3 residual = tl.layers.Conv2d(residual, depth, filter_size=(1, 1), strides=(1, 1), act=None, b_init=None, name='conv3') residual = tl.layers.BatchNormLayer(residual, act=tf.identity, is_train=True, name='conv3_bn/BatchNorm') # squeeze squeeze = tl.layers.InputLayer(tf.reduce_mean(residual.outputs, axis=[1, 2]), name='squeeze_layer') # excitation excitation1 = tl.layers.DenseLayer(squeeze, n_units=int(depth/16.0), act=tf.nn.relu, name='excitation_1') excitation2 = tl.layers.DenseLayer(excitation1, n_units=depth, act=tf.nn.sigmoid, name='excitation_2') # scale scale = tl.layers.ReshapeLayer(excitation2, shape=[tf.shape(excitation2.outputs)[0], 1, 1, depth], name='excitation_reshape') residual_se = ElementwiseLayer(layer=[residual, scale], combine_fn=tf.multiply, name='scale_layer', act=None) output = ElementwiseLayer(layer=[shortcut, residual_se], combine_fn=tf.add, name='combine_layer', act=tf.nn.relu) return output
def bottleneck_IR_SE(inputs, depth, depth_bottleneck, stride, rate=1, w_init=None, scope=None, trainable=None): with tf.variable_scope(scope, 'bottleneck_v1') as sc: depth_in = utils.last_dimension(inputs.outputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = tl.layers.Conv2d(inputs, depth, filter_size=(1, 1), strides=(stride, stride), act=None, W_init=w_init, b_init=None, name='shortcut_conv', use_cudnn_on_gpu=True) shortcut = BatchNormLayer(shortcut, act=tf.identity, is_train=True, trainable=trainable, name='shortcut_bn/BatchNorm') # bottleneck layer 1 residual = BatchNormLayer(inputs, act=tf.identity, is_train=True, trainable=trainable, name='conv1_bn1') residual = tl.layers.Conv2d(residual, depth_bottleneck, filter_size=(3, 3), strides=(1, 1), act=None, b_init=None, W_init=w_init, name='conv1', use_cudnn_on_gpu=True) residual = BatchNormLayer(residual, act=tf.identity, is_train=True, trainable=trainable, name='conv1_bn2') # bottleneck prelu residual = tl.layers.PReluLayer(residual) # bottleneck layer 2 residual = conv2d_same(residual, depth, kernel_size=3, strides=stride, rate=rate, w_init=w_init, scope='conv2', trainable=trainable) # squeeze squeeze = tl.layers.InputLayer(tf.reduce_mean(residual.outputs, axis=[1, 2]), name='squeeze_layer') # excitation excitation1 = tl.layers.DenseLayer(squeeze, n_units=int(depth/16.0), act=tf.nn.relu, W_init=w_init, name='excitation_1') # excitation1 = tl.layers.PReluLayer(excitation1, name='excitation_prelu') excitation2 = tl.layers.DenseLayer(excitation1, n_units=depth, act=tf.nn.sigmoid, W_init=w_init, name='excitation_2') # scale scale = tl.layers.ReshapeLayer(excitation2, shape=[tf.shape(excitation2.outputs)[0], 1, 1, depth], name='excitation_reshape') residual_se = ElementwiseLayer(layer=[residual, scale], combine_fn=tf.multiply, name='scale_layer', act=None) output = ElementwiseLayer(layer=[shortcut, residual_se], combine_fn=tf.add, name='combine_layer', act=tf.nn.relu) return output
def bottleneck(self, inputs, depth, depth_bottleneck, stride, rate=1, outputs_collections=None, scope=None): with tf.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) # 如果不能在深度上变化,就在宽度上变化 if depth == depth_in: shortcut = self.subsample(inputs, stride, 'shortcut') else: shortcut = slim.conv2d(inputs, depth, [1, 1], stride=stride, activation_fn=None, scope='shortcut') # 3x128x128x256 residual = slim.conv2d(inputs, depth_bottleneck, [1, 1], stride=1, scope='conv1') # 3x128x128x64 residual = self.conv2d_same(residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2') # 3x128x128x64 residual = slim.conv2d(residual, depth, [1, 1], activation_fn=None, stride=1, scope='conv3') # 3x128x128x256 output = tf.nn.relu(shortcut + residual) return utils.collect_named_outputs(outputs_collections, sc.original_name_scope, output)
def bottleneck_hole(inputs, depth, depth_bottleneck, stride, rate=2, outputs_collections=None, scope=None): with variable_scope.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = resnet_utils.subsample(inputs, stride, 'shortcut') else: shortcut = layers.conv2d(inputs, depth, [1, 1], stride=stride, activation_fn=None, scope='shortcut') residual = layers.conv2d(inputs, depth_bottleneck, [1, 1], stride=1, scope='conv1') residual = layers_lib.conv2d(residual, depth_bottleneck, [3, 3], stride=1, rate=rate, padding='SAME', scope='conv2') residual = layers.conv2d(residual, depth, [1, 1], stride=1, activation_fn=None, scope='conv3') output = nn_ops.relu(shortcut + residual) return utils.collect_named_outputs(outputs_collections, sc.name, output)
def bottleneck_IR(inputs, depth, depth_bottleneck, stride, rate=1, w_init=None, scope=None, trainable=None): with tf.variable_scope(scope, 'bottleneck_v1') as sc: depth_in = utils.last_dimension(inputs.outputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = Conv2d(inputs, depth, filter_size=(1, 1), strides=(stride, stride), act=None, W_init=w_init, b_init=None, name='shortcut_conv', use_cudnn_on_gpu=True) shortcut = BatchNormLayer(shortcut, act=tf.identity, is_train=True, trainable=trainable, name='shortcut_bn/BatchNorm') # bottleneck layer 1 residual = BatchNormLayer(inputs, act=tf.identity, is_train=True, trainable=trainable, name='conv1_bn1') residual = Conv2d(residual, depth_bottleneck, filter_size=(3, 3), strides=(1, 1), act=None, b_init=None, W_init=w_init, name='conv1', use_cudnn_on_gpu=True) residual = BatchNormLayer(residual, act=tf.identity, is_train=True, trainable=trainable, name='conv1_bn2') # bottleneck prelu residual = PReluLayer(residual) # bottleneck layer 2 residual = conv2d_same(residual, depth, kernel_size=3, strides=stride, rate=rate, w_init=w_init, scope='conv2', trainable=trainable) output = ElementwiseLayer(layer=[shortcut, residual], combine_fn=tf.add, name='combine_layer', act=None) return output
def inference(images, train=True, resnet_stride=8): with slim.arg_scope(resnet_v1.resnet_arg_scope(train)): net, end_points = resnet_v1_slim(images, num_classes=None, global_pool=False, output_stride=resnet_stride) # replace resnet_v1_slim above with resnet_v1.resnet_v1_50/101/... # to use standard architectures. # num_classes: Number of predicted classes for classification tasks. If None # we return the features before the logit layer. # global_pool: If True, we perform global average pooling before computing the # logits. Set to True for image classification, False for dense prediction. # output_stride: If None, then the output will be computed at the nominal # network stride. If output_stride is not None, it specifies the requested # ratio of input to output spatial resolution. resnet_depth = utils.last_dimension(net.get_shape(), min_rank=4) shape = tf.unpack(tf.shape(images)) print(shape.__class__) shape.pop() shape.append(tf.constant(FLAGS.out_channels, dtype=tf.int32)) print(len(shape)) filters = tf.Variable(tf.truncated_normal([ resnet_stride * 2 + 1, resnet_stride * 2 + 1, FLAGS.out_channels, resnet_depth ], dtype=tf.float32, stddev=0.01), name='filters') logits = tf.nn.conv2d_transpose(net, filters, tf.pack(shape), [1, resnet_stride, resnet_stride, 1], padding='SAME', name='upscale') return logits
def Squeeze_excitation_layer2(inputs, ratio, outputs_collections=None, scope=None): channal = utils.last_dimension(inputs.get_shape(), min_rank=4) num_outputs = channal // ratio with tf.name_scope(scope, 'SE_layer2', [inputs]) as sc: squeeze = slim.conv2d(inputs, num_outputs, [1, 1], stride=1, activation_fn=tf.nn.relu, normalizer_fn=None, scope='squeeze') excitation = slim.conv2d(squeeze, channal, [1, 1], stride=1, activation_fn=tf.nn.sigmoid, normalizer_fn=None, scope='excitation') scale = inputs * excitation return utils.collect_named_outputs(outputs_collections, sc, scale)
def bottleneck_Xt(inputs, depth, stride, cardinality, cardinality_dim, rate=1, scope=None): with tf.variable_scope(scope, 'bottleneck_v1') as sc: depth_in = utils.last_dimension(inputs.outputs.get_shape(), min_rank=4) if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = tl.layers.Conv2d(inputs, depth, filter_size=(1, 1), strides=(stride, stride), act=None, b_init=None, name='shortcut_conv') shortcut = tl.layers.BatchNormLayer(shortcut, act=tf.identity, is_train=True, name='shortcut_bn/BatchNorm') cardinality_layers = [] for i in range(cardinality): # bottleneck layer 1 residual = tl.layers.Conv2d(inputs, cardinality_dim, filter_size=(1, 1), strides=(1, 1), act=None, b_init=None, name='cardinality_%d/conv1' % i) residual = tl.layers.BatchNormLayer(residual, act=tf.nn.relu, is_train=True, name='cardinality_%d/conv1_bn/BatchNorm' % i) # bottleneck layer 2 residual = conv2d_same(residual, cardinality_dim, kernel_size=3, strides= stride, rate=rate, scope='cardinality_%d/conv2' % i) # bottleneck layer 3 residual = tl.layers.Conv2d(residual, depth, filter_size=(1, 1), strides=(1, 1), act=None, b_init=None, name='cardinality_%d/conv3' % i) residual = tl.layers.BatchNormLayer(residual, act=tf.identity, is_train=True, name='cardinality_%d/conv3_bn/BatchNorm' % i) cardinality_layers.append(residual) residual_total = ElementwiseLayer(layer=cardinality_layers, combine_fn=tf.add, name='cardinality_cmobine', act=None) with tf.control_dependencies([residual_total.outputs]): output = ElementwiseLayer(layer=[shortcut, residual_total], combine_fn=tf.add, name='combine_layer', act=tf.nn.relu) return output
def fully_connected(inputs, num_outputs, activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, weights_normalizer_fn=None, weights_normalizer_params=None, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, biases_initializer=init_ops.zeros_initializer(), biases_regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, scope=None): # Be copied and modified from tensorflow-0.12.0.contrib.layer.fully_connected, # add weights_nomalizer_* options. """Adds a fully connected layer. `fully_connected` creates a variable called `weights`, representing a fully connected weight matrix, which is multiplied by the `inputs` to produce a `Tensor` of hidden units. If a `normalizer_fn` is provided (such as `batch_norm`), it is then applied. Otherwise, if `normalizer_fn` is None and a `biases_initializer` is provided then a `biases` variable would be created and added the hidden units. Finally, if `activation_fn` is not `None`, it is applied to the hidden units as well. Note: that if `inputs` have a rank greater than 2, then `inputs` is flattened prior to the initial matrix multiply by `weights`. Args: inputs: A tensor of with at least rank 2 and value for the last dimension, i.e. `[batch_size, depth]`, `[None, None, None, channels]`. num_outputs: Integer or long, the number of output units in the layer. activation_fn: activation function, set to None to skip it and maintain a linear activation. normalizer_fn: normalization function to use instead of `biases`. If `normalizer_fn` is provided then `biases_initializer` and `biases_regularizer` are ignored and `biases` are not created nor added. default set to None for no normalizer function normalizer_params: normalization function parameters. weights_normalizer_fn: weights normalization function. weights_normalizer_params: weights normalization function parameters. weights_initializer: An initializer for the weights. weights_regularizer: Optional regularizer for the weights. biases_initializer: An initializer for the biases. If None skip biases. biases_regularizer: Optional regularizer for the biases. reuse: whether or not the layer and its variables should be reused. To be able to reuse the layer scope must be given. variables_collections: Optional list of collections for all the variables or a dictionary containing a different list of collections per variable. outputs_collections: collection to add the outputs. trainable: If `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). scope: Optional scope for variable_scope. Returns: the tensor variable representing the result of the series of operations. Raises: ValueError: if x has rank less than 2 or if its last dimension is not set. """ if not (isinstance(num_outputs, six.integer_types)): raise ValueError('num_outputs should be int or long, got %s.', num_outputs) with variable_scope.variable_scope(scope, 'fully_connected', [inputs], reuse=reuse) as sc: inputs = ops.convert_to_tensor(inputs) dtype = inputs.dtype.base_dtype inputs_shape = inputs.get_shape() num_input_units = utils.last_dimension(inputs_shape, min_rank=2) static_shape = inputs_shape.as_list() static_shape[-1] = num_outputs out_shape = array_ops.unpack(array_ops.shape(inputs), len(static_shape)) out_shape[-1] = num_outputs weights_shape = [num_input_units, num_outputs] weights_collections = utils.get_variable_collections( variables_collections, 'weights') weights = variables.model_variable('weights', shape=weights_shape, dtype=dtype, initializer=weights_initializer, regularizer=weights_regularizer, collections=weights_collections, trainable=trainable) if weights_normalizer_fn is not None: weights_normalizer_params = weights_normalizer_params or {} weights = weights_normalizer_fn(weights, **weights_normalizer_params) if len(static_shape) > 2: # Reshape inputs inputs = array_ops.reshape(inputs, [-1, num_input_units]) outputs = standard_ops.matmul(inputs, weights) if normalizer_fn is not None: normalizer_params = normalizer_params or {} outputs = normalizer_fn(outputs, **normalizer_params) else: if biases_initializer is not None: biases_collections = utils.get_variable_collections( variables_collections, 'biases') biases = variables.model_variable( 'biases', shape=[ num_outputs, ], dtype=dtype, initializer=biases_initializer, regularizer=biases_regularizer, collections=biases_collections, trainable=trainable) outputs = nn.bias_add(outputs, biases) if activation_fn is not None: outputs = activation_fn(outputs) if len(static_shape) > 2: # Reshape back outputs outputs = array_ops.reshape(outputs, array_ops.pack(out_shape)) outputs.set_shape(static_shape) return utils.collect_named_outputs(outputs_collections, sc.original_name_scope, outputs)
def fully_connected(inputs, num_outputs, activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, biases_initializer=init_ops.zeros_initializer, biases_regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, scope=None): """Adds a fully connected layer. `fully_connected` creates a variable called `weights`, representing a fully connected weight matrix, which is multiplied by the `inputs` to produce a `Tensor` of hidden units. If a `normalizer_fn` is provided (such as `batch_norm`), it is then applied. Otherwise, if `normalizer_fn` is None and a `biases_initializer` is provided then a `biases` variable would be created and added the hidden units. Finally, if `activation_fn` is not `None`, it is applied to the hidden units as well. Note: that if `inputs` have a rank greater than 2, then `inputs` is flattened prior to the initial matrix multiply by `weights`. Args: inputs: A tensor of with at least rank 2 and value for the last dimension, i.e. `[batch_size, depth]`, `[None, None, None, channels]`. num_outputs: Integer, the number of output units in the layer. activation_fn: activation function. normalizer_fn: normalization function to use instead of `biases`. If `normalize_fn` is provided then `biases_initializer` and `biases_regularizer` are ignored and `biases` are not created nor added. normalizer_params: normalization function parameters. weights_initializer: An initializer for the weights. weights_regularizer: Optional regularizer for the weights. biases_initializer: An initializer for the biases. If None skip biases. biases_regularizer: Optional regularizer for the biases. reuse: whether or not the layer and its variables should be reused. To be able to reuse the layer scope must be given. variables_collections: Optional list of collections for all the variables or a dictionary containing a different list of collections per variable. outputs_collections: collection to add the outputs. trainable: If `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). scope: Optional scope for variable_op_scope. Returns: the tensor variable representing the result of the series of operations. Raises: ValueError: if x has rank less than 2 or if its last dimension is not set. """ if not isinstance(num_outputs, int): raise ValueError('num_outputs should be integer, got %s.', num_outputs) with variable_scope.variable_op_scope([inputs], scope, 'fully_connected', reuse=reuse) as sc: dtype = inputs.dtype.base_dtype num_input_units = utils.last_dimension(inputs.get_shape(), min_rank=2) static_shape = inputs.get_shape().as_list() static_shape[-1] = num_outputs out_shape = array_ops.unpack(array_ops.shape(inputs)) out_shape[-1] = num_outputs weights_shape = [num_input_units, num_outputs] weights_collections = utils.get_variable_collections( variables_collections, 'weights') weights = variables.model_variable('weights', shape=weights_shape, dtype=dtype, initializer=weights_initializer, regularizer=weights_regularizer, collections=weights_collections, trainable=trainable) if len(static_shape) > 2: # Reshape inputs inputs = array_ops.reshape(inputs, [-1, num_input_units]) outputs = standard_ops.matmul(inputs, weights) if normalizer_fn: normalizer_params = normalizer_params or {} outputs = normalizer_fn(outputs, **normalizer_params) else: if biases_initializer is not None: biases_collections = utils.get_variable_collections( variables_collections, 'biases') biases = variables.model_variable('biases', shape=[num_outputs,], dtype=dtype, initializer=biases_initializer, regularizer=biases_regularizer, collections=biases_collections, trainable=trainable) outputs = nn.bias_add(outputs, biases) if len(static_shape) > 2: # Reshape back outputs outputs = array_ops.reshape(outputs, array_ops.pack(out_shape)) outputs.set_shape(static_shape) if activation_fn: outputs = activation_fn(outputs) return utils.collect_named_outputs(outputs_collections, sc.name, outputs)
def convolution2d(inputs, num_outputs, kernel_size, stride=1, padding='SAME', activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, biases_initializer=init_ops.zeros_initializer, biases_regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, scope=None): """Adds a 2D convolution followed by an optional batch_norm layer. `convolution2d` creates a variable called `weights`, representing the convolutional kernel, that is convolved with the `inputs` to produce a `Tensor` of activations. If a `normalizer_fn` is provided (such as `batch_norm`), it is then applied. Otherwise, if `normalizer_fn` is None and a `biases_initializer` is provided then a `biases` variable would be created and added the activations. Finally, if `activation_fn` is not `None`, it is applied to the activations as well. Args: inputs: a 4-D tensor `[batch_size, height, width, channels]`. num_outputs: integer, the number of output filters. kernel_size: a list of length 2 `[kernel_height, kernel_width]` of of the filters. Can be an int if both values are the same. stride: a list of length 2 `[stride_height, stride_width]`. Can be an int if both strides are the same. Note that presently both strides must have the same value. padding: one of `VALID` or `SAME`. activation_fn: activation function. normalizer_fn: normalization function to use instead of `biases`. If `normalize_fn` is provided then `biases_initializer` and `biases_regularizer` are ignored and `biases` are not created nor added. normalizer_params: normalization function parameters. weights_initializer: An initializer for the weights. weights_regularizer: Optional regularizer for the weights. biases_initializer: An initializer for the biases. If None skip biases. biases_regularizer: Optional regularizer for the biases. reuse: whether or not the layer and its variables should be reused. To be able to reuse the layer scope must be given. variables_collections: optional list of collections for all the variables or a dictionay containing a different list of collection per variable. outputs_collections: collection to add the outputs. trainable: If `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). scope: Optional scope for `variable_op_scope`. Returns: a tensor representing the output of the operation. """ with variable_scope.variable_op_scope([inputs], scope, 'Conv', reuse=reuse) as sc: dtype = inputs.dtype.base_dtype kernel_h, kernel_w = utils.two_element_tuple(kernel_size) stride_h, stride_w = utils.two_element_tuple(stride) num_filters_in = utils.last_dimension(inputs.get_shape(), min_rank=4) weights_shape = [kernel_h, kernel_w, num_filters_in, num_outputs] weights_collections = utils.get_variable_collections( variables_collections, 'weights') weights = variables.model_variable('weights', shape=weights_shape, dtype=dtype, initializer=weights_initializer, regularizer=weights_regularizer, collections=weights_collections, trainable=trainable) outputs = nn.conv2d(inputs, weights, [1, stride_h, stride_w, 1], padding=padding) if normalizer_fn: normalizer_params = normalizer_params or {} outputs = normalizer_fn(outputs, **normalizer_params) else: if biases_initializer is not None: biases_collections = utils.get_variable_collections( variables_collections, 'biases') biases = variables.model_variable('biases', shape=[num_outputs,], dtype=dtype, initializer=biases_initializer, regularizer=biases_regularizer, collections=biases_collections, trainable=trainable) outputs = nn.bias_add(outputs, biases) if activation_fn: outputs = activation_fn(outputs) return utils.collect_named_outputs(outputs_collections, sc.name, outputs)
def residual_unit(self, net, group_i, unit_i): """pre-activation Residual Units from https://arxiv.org/abs/1603.05027.""" name = 'group_%d/unit_%d' % (group_i, unit_i) group = self.groups[group_i] if group.is_downsample and unit_i == 0: stride1 = 2 else: stride1 = 1 def conv_pre(name, net, num_ker, kernel_size, stride, conv_i): """ 1D pre-activation convolution. args: num_ker (int): number of kernels (out_channels). ker_size (int): size of 1D kernel. stride (int) """ with tf.variable_scope(name): if not (self.special_first and group_i == unit_i == conv_i == 0): net = self.BN_ReLU(net) # 1D-convolution net = self.conv1d(net, num_ker, kernel_size, stride) return net def conv_post(name, net, num_ker, kernel_size, stride, conv_i): """ 1D post-activation convolution. args: num_ker (int): number of kernels (out_channels). ker_size (int): size of 1D kernel. stride (int) """ with tf.variable_scope(name): # 1D-convolution net = self.conv1d(net, num_ker, kernel_size, stride) net = self.BN_ReLU(net) return net ### residual function net_residual = net if self.unit_type == 0 and not self.special_first: unit_conv = conv_post elif self.unit_type == 1: unit_conv = conv_pre else: raise ValueError("wrong residual unit type:{}".format( self.unit_type)) if self.residual_type == 0: # 1x1 convolution responsible for reducing dimension net_residual = unit_conv(name + '/conv_reduce', net_residual, group.reduced_ker, self.bott_size13, stride1, 0) # 3x1 convolution bottleneck net_residual = unit_conv(name + '/conv_bottleneck', net_residual, group.reduced_ker, self.bott_size, 1, 1) # 1x1 convolution responsible for restoring dimension net_residual = unit_conv(name + '/conv_restore', net_residual, group.num_ker, self.bott_size13, 1, 2) elif self.residual_type == 1: net_residual = unit_conv(name + '/conv_one', net_residual, group.num_ker, self.bott_size, stride1, 0) # if self.if_drop and group_i == 2: if self.if_drop and unit_i == 0: with tf.name_scope("dropout"): net_residual = tf.nn.dropout(net_residual, self.dropout_keep_prob) net_residual = unit_conv(name + '/conv_two', net_residual, group.num_ker, self.bott_size, 1, 1) else: raise ValueError("residual_type error") ### shortcut connection num_ker_in = utils.last_dimension(net.get_shape(), min_rank=4) if self.shortcut == 0 and unit_i == 0: # average pooling for data downsampling if group.is_downsample: net = tf.nn.avg_pool(net, ksize=[1, 2, 1, 1], strides=[1, 2, 1, 1], padding='SAME') # zero-padding for increasing kernel numbers if group.num_ker / num_ker_in == 2: net = tf.pad(net, [[0, 0], [0, 0], [0, 0], [int(num_ker_in / 2), int(num_ker_in / 2)]]) elif group.num_ker != num_ker_in: raise ValueError("illigal kernel numbers at group {} unit {}" .format(group_i, unit_i)) elif self.shortcut == 1 and unit_i == 0 or self.shortcut == 2: with tf.variable_scope(name+'_sc'): # projection net = self.BN_ReLU(net) net = self.conv1d(net, group.num_ker, 1, stride1) ### element-wise addition net = net + net_residual return net
def conv2d_leaders(inputs, num_outputs, kernel_size, rates=[1], stride=1, padding='SAME', activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, biases_initializer=init_ops.zeros_initializer, biases_regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, scope=None,): """Adds a 2D convolution followed by an optional batch_norm layer. `convolution2d` creates a variable called `weights`, representing the convolutional kernel, that is convolved with the `inputs` to produce a `Tensor` of activations. If a `normalizer_fn` is provided (such as `batch_norm`), it is then applied. Otherwise, if `normalizer_fn` is None and a `biases_initializer` is provided then a `biases` variable would be created and added the activations. Finally, if `activation_fn` is not `None`, it is applied to the activations as well. Performs a'trous convolution with input stride equal to rate if rate is greater than one. Args: inputs: a 4-D tensor `[batch_size, height, width, channels]`. num_outputs: integer, the number of output filters. kernel_size: a list of length 2 `[kernel_height, kernel_width]` of of the filters. Can be an int if both values are the same. stride: a list of length 2 `[stride_height, stride_width]`. Can be an int if both strides are the same. Note that presently both strides must have the same value. padding: one of `VALID` or `SAME`. rate: integer. If less than or equal to 1, a standard convolution is used. If greater than 1, than the a'trous convolution is applied and `stride` must be set to 1. activation_fn: activation function. normalizer_fn: normalization function to use instead of `biases`. If `normalize_fn` is provided then `biases_initializer` and `biases_regularizer` are ignored and `biases` are not created nor added. normalizer_params: normalization function parameters. weights_initializer: An initializer for the weights. weights_regularizer: Optional regularizer for the weights. biases_initializer: An initializer for the biases. If None skip biases. biases_regularizer: Optional regularizer for the biases. reuse: whether or not the layer and its variables should be reused. To be able to reuse the layer scope must be given. variables_collections: optional list of collections for all the variables or a dictionay containing a different list of collection per variable. outputs_collections: collection to add the outputs. trainable: If `True` also add variables to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable). scope: Optional scope for `variable_op_scope`. Returns: a tensor representing the output of the operation. Raises: ValueError: if both 'rate' and `stride` are larger than one. """ with variable_scope.variable_scope(scope, 'Conv', [inputs], reuse=reuse) as sc: inputs = ops.convert_to_tensor(inputs) dtype = inputs.dtype.base_dtype # inshape = tf.shape(inputs) # Leading kernel size. kernel_h, kernel_w = utils.two_element_tuple(kernel_size) stride_h, stride_w = utils.two_element_tuple(stride) num_filters_in = utils.last_dimension(inputs.get_shape(), min_rank=4) # Weights variable. weights_shape = [kernel_h, kernel_w, num_filters_in, num_outputs] weights_collections = utils.get_variable_collections( variables_collections, 'weights') weights = variables.model_variable('weights', shape=weights_shape, dtype=dtype, initializer=weights_initializer, regularizer=weights_regularizer, collections=weights_collections, trainable=trainable) # # Bias variable. # biases = None # if biases_initializer is not None: # biases_collections = utils.get_variable_collections( # variables_collections, 'biases') # biases = variables.model_variable('biases', # shape=[num_outputs, ], # dtype=dtype, # initializer=biases_initializer, # regularizer=biases_regularizer, # collections=biases_collections, # trainable=trainable) # Convolution at different scales. outputs_pool = [] for rate in rates: if rate > 1: conv = nn.atrous_conv2d(inputs, weights, rate, padding='SAME') else: conv = nn.conv2d(inputs, weights, [1, 1, 1, 1], padding='SAME') outputs_pool.append(conv) # 'Pooling' at different scales. A bit hacky. Use of concat + max_pool? outputs = None outputs_pool.reverse() for node in outputs_pool: if outputs is None: outputs = node else: outputs = tf.maximum(outputs, node) # # Add bias? # if biases is not None: # outputs = tf.nn.bias_add(outputs, biases) # Fix padding and stride. A bit hacky too and not so efficient! if padding == 'VALID' or stride > 1: padfilter = np.zeros(shape=(kernel_h, kernel_w, num_filters_in, 1), dtype=dtype) x = (kernel_h - 1) / 2 y = (kernel_w - 1) / 2 padfilter[x, y, :, 0] = 1. outputs = tf.nn.depthwise_conv2d(outputs, padfilter, [1, stride_h, stride_w, 1], padding=padding) # Batch norm / bias and activation... if normalizer_fn is not None: normalizer_params = normalizer_params or {} outputs = normalizer_fn(outputs, **normalizer_params) else: if biases_initializer is not None: biases_collections = utils.get_variable_collections( variables_collections, 'biases') biases = variables.model_variable('biases', shape=[num_outputs, ], dtype=dtype, initializer=biases_initializer, regularizer=biases_regularizer, collections=biases_collections, trainable=trainable) outputs = nn.bias_add(outputs, biases) if activation_fn is not None: outputs = activation_fn(outputs) return utils.collect_named_outputs(outputs_collections, sc.name, outputs)
def depthwise_convolution2d( inputs, kernel_size, depth_multiplier=1, stride=1, padding='SAME', rate=1, activation_fn=nn.relu, normalizer_fn=None, normalizer_params=None, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, biases_initializer=init_ops.zeros_initializer(), biases_regularizer=None, reuse=None, variables_collections=None, outputs_collections=None, trainable=True, data_format='NHWC', scope=None): """Adds a depthwise 2D convolution with optional batch_norm layer. This op performs a depthwise convolution that acts separately on channels, creating a variable called `depthwise_weights`. Then, if `normalizer_fn` is None, it adds bias to the result, creating a variable called 'biases', otherwise, the `normalizer_fn` is applied. It finally applies an activation function to produce the end result. Args: inputs: A tensor of size [batch_size, height, width, channels]. num_outputs: The number of pointwise convolution output filters. If is None, then we skip the pointwise convolution stage. kernel_size: A list of length 2: [kernel_height, kernel_width] of of the filters. Can be an int if both values are the same. depth_multiplier: The number of depthwise convolution output channels for each input channel. The total number of depthwise convolution output channels will be equal to `num_filters_in * depth_multiplier`. stride: A list of length 2: [stride_height, stride_width], specifying the depthwise convolution stride. Can be an int if both strides are the same. padding: One of 'VALID' or 'SAME'. rate: A list of length 2: [rate_height, rate_width], specifying the dilation rates for atrous convolution. Can be an int if both rates are the same. If any value is larger than one, then both stride values need to be one. activation_fn: Activation function. The default value is a ReLU function. Explicitly set it to None to skip it and maintain a linear activation. normalizer_fn: Normalization function to use instead of `biases`. If `normalizer_fn` is provided then `biases_initializer` and `biases_regularizer` are ignored and `biases` are not created nor added. default set to None for no normalizer function normalizer_params: Normalization function parameters. weights_initializer: An initializer for the weights. weights_regularizer: Optional regularizer for the weights. biases_initializer: An initializer for the biases. If None skip biases. biases_regularizer: Optional regularizer for the biases. reuse: Whether or not the layer and its variables should be reused. To be able to reuse the layer scope must be given. variables_collections: Optional list of collections for all the variables or a dictionary containing a different list of collection per variable. outputs_collections: Collection to add the outputs. trainable: Whether or not the variables should be trainable or not. scope: Optional scope for variable_scope. Returns: A `Tensor` representing the output of the operation. """ with variable_scope.variable_scope(scope, 'DepthwiseConv2d', [inputs], reuse=reuse) as sc: inputs = ops.convert_to_tensor(inputs) # Actually apply depthwise conv instead of separable conv. dtype = inputs.dtype.base_dtype kernel_h, kernel_w = utils.two_element_tuple(kernel_size) stride_h, stride_w = utils.two_element_tuple(stride) if data_format == 'NHWC': num_filters_in = utils.last_dimension(inputs.get_shape(), min_rank=4) strides = [1, stride_h, stride_w, 1] else: num_filters_in = inputs.get_shape().as_list()[1] strides = [1, 1, stride_h, stride_w] weights_collections = utils.get_variable_collections( variables_collections, 'weights') # Depthwise weights variable. depthwise_shape = [kernel_h, kernel_w, num_filters_in, depth_multiplier] depthwise_weights = variables.model_variable( 'depthwise_weights', shape=depthwise_shape, dtype=dtype, initializer=weights_initializer, regularizer=weights_regularizer, trainable=trainable, collections=weights_collections) outputs = nn.depthwise_conv2d(inputs, depthwise_weights, strides, padding, rate=utils.two_element_tuple(rate), data_format=data_format) num_outputs = depth_multiplier * num_filters_in if normalizer_fn is not None: normalizer_params = normalizer_params or {} outputs = normalizer_fn(outputs, **normalizer_params) else: if biases_initializer is not None: biases_collections = utils.get_variable_collections( variables_collections, 'biases') biases = variables.model_variable('biases', shape=[num_outputs,], dtype=dtype, initializer=biases_initializer, regularizer=biases_regularizer, trainable=trainable, collections=biases_collections) outputs = nn.bias_add(outputs, biases, data_format=data_format) if activation_fn is not None: outputs = activation_fn(outputs) return utils.collect_named_outputs(outputs_collections, sc.original_name_scope, outputs)