def dense( inputs, units, use_bias=True, trainable=True, kernel_initializer=tf.variance_scaling_initializer(), bias_initializer=tf.zeros_initializer() ): net = tf.layers.dense( inputs, units=units, activation=None, use_bias=use_bias, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, trainable=trainable ) _log_hparams( classname='Dense', layername=net.name, units=units, use_bias=use_bias, trainable=trainable, out_shape=str(net.get_shape()), out_dtype=net.dtype ) return net
def prelu(inputs, channel_shared=False, trainable=True, name='prelu'): def parametric_relu(_x): if channel_shared: w_shape = (1, ) else: w_shape = int(_x.get_shape()[-1]) alphas = tf.get_variable('alpha', w_shape, trainable=trainable, initializer=tf.initializers.truncated_normal( mean=-1.0, stddev=0.2)) alphas = tf.nn.sigmoid(alphas, name="constraining_alpha_var_in_0_1") return tf.maximum(_x, _x * alphas) with tf.variable_scope(name): net = parametric_relu(inputs) _log_hparams(classname='PReLU', layername=net.name, channel_shared=channel_shared, trainable=trainable, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def sigmoid(x, name='sigmoid'): net = tf.math.sigmoid(x, name=name) _log_hparams(classname='Sigmoid', layername=net.name, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def tanh(inputs, name='tanh'): net = tf.math.tanh(inputs, name=name) _log_hparams(classname='TanH', layername=net.name, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def elu(features, name='elu'): net = tf.nn.elu(features, name=name) _log_hparams(classname='ELU', layername=net.name, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def flatten(inputs, name='flatten'): net = tf.layers.flatten(inputs, name=name) _log_hparams(classname='Flatten', layername=net.name, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def relu6(inputs, name='relu6'): net = tf.nn.relu6(inputs, name=name) _log_hparams(classname='ReLU6', layername=net.name, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def squeeze(tensor, axis, name='squeeze'): net = tf.squeeze(tensor, axis=axis, name=name) _log_hparams(classname='Squeeze', layername=net.name, axis=axis, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def reshape(tensor, shape, name='reshape'): net = tf.reshape(tensor, shape=shape, name=name) _log_hparams(classname='Reshape', layername=net.name, shape=shape, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def concat(values, axis, name='concat'): net = tf.concat(values=values, axis=axis, name=name) _log_hparams(classname='Concat', layername=net.name, axis=axis, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def leaky_relu(features, alpha=0.2, name='leaky_relu'): net = tf.nn.leaky_relu(features, alpha=alpha, name=name) _log_hparams(classname='LeakyReLU', layername=net.name, alpha=alpha, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def crelu(features, name='crelu', axis=-1): net = tf.nn.crelu(features, name=name, axis=axis) _log_hparams(classname='CReLU', layername=net.name, axis=axis, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def conv2d(inputs, n_channels=8, kernel_size=(3, 3), strides=(1, 1), padding='VALID', data_format='NHWC', dilation_rate=(1, 1), use_bias=True, kernel_initializer=tf.variance_scaling_initializer(), bias_initializer=tf.zeros_initializer(), trainable=True): if data_format not in ['NHWC', 'NCHW']: raise ValueError( "Unknown data format: `%s` (accepted: ['NHWC', 'NCHW'])" % data_format) if padding.upper() not in ['SAME', 'VALID']: raise ValueError( "Unknown padding: `%s` (accepted: ['SAME', 'VALID'])" % padding.upper()) net = tf.layers.conv2d(inputs, filters=n_channels, kernel_size=kernel_size, strides=strides, padding=padding, dilation_rate=dilation_rate, data_format='channels_last' if data_format == 'NHWC' else 'channels_first', use_bias=use_bias, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, trainable=trainable, activation=None) _log_hparams(classname='Conv2D', layername=net.name, n_channels=n_channels, kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, dilation_rate=dilation_rate, use_bias=use_bias, trainable=trainable, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def softmax(inputs, axis=None, name="softmax"): net = tf.nn.softmax( inputs, axis=axis, name=name, ) _log_hparams(classname='Softmax', layername=net.name, axis=axis, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def max_pooling2d(inputs, pool_size=(2, 2), strides=None, padding='valid', data_format=None, name="max_pooling2d"): if data_format not in ['NHWC', 'NCHW']: raise ValueError( "Unknown data format: `%s` (accepted: ['NHWC', 'NCHW'])" % data_format) if padding.lower() not in ['same', 'valid']: raise ValueError( "Unknown padding: `%s` (accepted: ['same', 'valid'])" % padding) ''' net = tf.keras.layers.MaxPool2D( pool_size=pool_size, strides=strides, padding=padding, data_format='channels_first' if data_format == 'NCHW' else 'channels_last', name=name, )(inputs) ''' net = tf.layers.max_pooling2d(inputs, pool_size=pool_size, strides=strides, padding=padding, data_format='channels_first' if data_format == 'NCHW' else 'channels_last', name=name) _log_hparams(classname='MaxPooling2D', layername=net.name, pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def pad(inputs, paddings, mode='CONSTANT', name='padding', constant_values=0): if mode.upper() not in ['CONSTANT', 'REFLECT', 'SYMMETRIC']: raise ValueError( "Unknown padding mode: `%s` (accepted: ['CONSTANT', 'REFLECT', 'SYMMETRIC'])" % mode) net = tf.pad(inputs, paddings=paddings, mode=mode, name=name, constant_values=constant_values) _log_hparams(classname='Padding', layername=net.name, paddings=paddings, mode=mode, constant_values=constant_values, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def reduce_mean(inputs, keepdims=None, data_format='channels_last', name='spatial_mean'): if data_format not in ['NHWC', 'NCHW']: raise ValueError( "Unknown data format: `%s` (accepted: ['NHWC', 'NCHW'])" % data_format) axes = [1, 2] if data_format == 'NHWC' else [2, 3] net = tf.math.reduce_mean(inputs, axis=axes, keepdims=keepdims, name=name) _log_hparams(classname='ReduceMean', layername=net.name, axis=axes, keepdims=keepdims, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def dropout(inputs, rate=0.5, noise_shape=None, seed=None, training=False, name=None): layer = tf.keras.layers.Dropout(rate, noise_shape=noise_shape, seed=seed, name=name) net = layer.apply(inputs, training=training) _log_hparams(classname='Dropout', layername=net.name, noise_shape=noise_shape, training=training, seed=seed, out_shape=str(net.get_shape()), out_dtype=net.dtype) return net
def deconv2d(inputs, n_channels=8, kernel_size=(3, 3), padding='VALID', data_format='NHWC', use_bias=True, kernel_initializer=tf.variance_scaling_initializer(), bias_initializer=tf.zeros_initializer(), trainable=True, use_upscale_conv=True): padding = padding.upper() # Enforce capital letters for the padding mode if data_format not in ['NHWC', 'NCHW']: raise ValueError( "Unknown data format: `%s` (accepted: ['NHWC', 'NCHW'])" % data_format) if padding not in ['SAME', 'VALID']: raise ValueError( "Unknown padding: `%s` (accepted: ['SAME', 'VALID'])" % padding) with tf.variable_scope("deconv2d"): if use_upscale_conv: layer = layers.upscale_2d( inputs, size=(2, 2), method=tf.image.ResizeMethod. NEAREST_NEIGHBOR, # [BILINEAR, NEAREST_NEIGHBOR, BICUBIC, AREA] align_corners=True, is_scale=True, data_format=data_format) layer = layers.conv2d(layer, n_channels=n_channels, kernel_size=kernel_size, strides=(1, 1), padding=padding, data_format=data_format, use_bias=use_bias, trainable=trainable, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer) else: input_shape = inputs.get_shape() layer = tf.layers.conv2d_transpose( inputs=inputs, filters=n_channels, kernel_size=kernel_size, strides=(2, 2), padding=padding, data_format='channels_first' if data_format == "NCHW" else "channels_last", use_bias=use_bias, trainable=trainable, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer) _log_hparams(classname='Conv2DTranspose', layername=layer.name, n_channels=n_channels, kernel_size=kernel_size, strides=(2, 2), padding=padding, data_format=data_format, use_bias=use_bias, trainable=trainable, input_shape=str(input_shape), out_shape=str(layer.get_shape()), out_dtype=layer.dtype) return layer
def upscale_2d(inputs, size, is_scale=True, method=0, align_corners=True, data_format='NHWC', name='upsample2d_layer'): if not isinstance(size, (list, tuple)) and len(size) == 2: raise AssertionError() if data_format not in ['NHWC', 'NCHW']: raise ValueError( "Unknown data format received: `%s` (allowed: `NHWC`, `NCHW`)" % data_format) input_shape = inputs.get_shape() if len(inputs.get_shape()) == 3: if is_scale: size_h = size[0] * int(inputs.get_shape()[0]) size_w = size[1] * int(inputs.get_shape()[1]) _size = [size_h, size_w] else: _size = size elif len(inputs.get_shape()) == 4: if data_format == 'NCHW': inputs = tf.transpose(inputs, [0, 2, 3, 1]) # NCHW => NHWC if is_scale: size_h = size[0] * int(inputs.get_shape()[1]) size_w = size[1] * int(inputs.get_shape()[2]) _size = [size_h, size_w] else: _size = size else: raise Exception("Do not support shape %s" % str(inputs.get_shape())) with tf.variable_scope(name): net = tf.image.resize_images(inputs, size=_size, method=method, align_corners=align_corners) if data_format == 'NCHW' and len(inputs.get_shape()) == 4: net = tf.transpose(net, [0, 3, 1, 2]) # NHWC => NCHW _log_hparams(classname='Upscale2D', layername=net.name, size=size, is_scale=is_scale, method=method, align_corners=align_corners, data_format=data_format, input_shape=str(input_shape), out_shape=str(net.get_shape()), out_dtype=net.dtype) return net