def __call__(self, kwargs=None):
input_tensor = ModelAssign(kwargs, 'input_tensor', None)
self.input_shape = input_tensor.get_shape()[1:]
self.skip_from_names = ModelAssign(kwargs, 'skip_from_names', None)
skip_from = ModelAssign(kwargs, 'skip_from', None)
self.is_training = ModelAssign(kwargs, 'is_training')
initializer = ModelAssign(kwargs, 'initializer',
G.BACKEND_DEFAULT_CONV_INITIALIZER)
regularizer = ModelAssign(kwargs, 'regularizer',
G.BACKEND_DEFAULT_REGULARIZER)
regularizer_strength = ModelAssign(kwargs, 'regularizer_strength',
1e-4)
# infer filters during compiling
self.filters = int(
input_tensor.get_shape()[-1]) * self.depthwise_multiplier
with tf.variable_scope(self.name) as scope:
self.output_tensor = util.depthwise_conv2d(
inputs=input_tensor,
depth_multiplier=self.depthwise_multiplier,
kernel_size=self.kernel_size,
strides=self.strides,
padding=self.padding,
batchnorm=self.batchnorm,
activation=self.activation,
initializer=initializer(),
regularizer=regularizer(regularizer_strength),
use_bias=self.use_bias,
is_training=self.is_training,
mode=G.EXEC_CONV_MODE)
self.output_shape = self.output_tensor.get_shape()[1:]
# Normal
self.num_trainable_parameters += (self.kernel_size *
self.kernel_size) * self.filters
if self.use_bias:
self.num_trainable_parameters += self.filters
self.MACs += self.kernel_size * self.kernel_size * self.filters * int(
self.output_shape[0]) * int(self.output_shape[1])
if self.batchnorm and not G.DISABLE_BATCHNORM_COUNT:
self.num_trainable_parameters += 2 * self.filters
self.num_non_trainable_parameters += 2 * self.filters
self.MACs += int(self.output_shape[0]) * int(
self.output_shape[1]) * int(self.output_shape[2]) * 2
self.mem_cost = self.num_non_trainable_parameters + self.num_trainable_parameters
self.peak_activation_mem += int(self.input_shape[0]) * int(
self.input_shape[1]) * int(self.input_shape[2])
self.peak_activation_mem += int(self.output_shape[0]) * int(
self.output_shape[1]) * int(self.output_shape[2])
def __call__(self, kwargs=None):
input_tensor = ModelAssign(kwargs, 'input_tensor', None)
self.input_shape = input_tensor.get_shape()[1:]
with tf.variable_scope(self.name, tf.AUTO_REUSE) as scope:
output = util.flatten(input_tensor)
self.output_shape = output.get_shape()[1:]
self.output_tensor = output
def __call__(self, kwargs=None):
skip_from = ModelAssign(kwargs, 'skip_from', None)
self.skip_from_names = ModelAssign(kwargs, 'skip_from_names', None)
input_tensor = ModelAssign(kwargs, 'input_tensor', None)
self.input_shape = input_tensor.get_shape()[1:]
self.is_training = ModelAssign(kwargs, 'is_training', None)
print(self.is_training)
in_dim = int(input_tensor.get_shape()[-1])
initializer = ModelAssign(kwargs, 'initializer',
G.BACKEND_DEFAULT_FC_INITIALIZER)
regularizer = ModelAssign(kwargs, 'regularizer',
G.BACKEND_DEFAULT_REGULARIZER)
regularizer_strength = ModelAssign(kwargs, 'regularizer_strength',
1e-4)
print("Intializing Dense Layer with L2-reg=%f" % regularizer_strength)
with tf.variable_scope(self.name) as scope:
output = util.dense(input_tensor,
units=self.units,
activation=self.activation,
batchnorm=self.batchnorm,
initializer=initializer(),
regularizer=regularizer(regularizer_strength),
is_training=self.is_training,
trainable=self.trainable,
use_bias=self.use_bias)
self.num_trainable_parameters += (in_dim + 1) * self.units
self.MACs += int(in_dim) * self.units
if self.batchnorm:
self.num_non_trainable_parameters += 2 * self.units
self.num_trainable_parameters += 2 * self.units
self.MACs += int(self.units) * 2
if self.dropout > 1e-12:
self.dropout_tensor = tf.placeholder(dtype=tf.float32,
shape=())
output = tf.nn.dropout(output,
keep_prob=1 - self.dropout_tensor,
name="dropout")
self.output_tensor = output
self.output_shape = self.output_tensor.get_shape()[1:]
self.mem_cost = self.num_non_trainable_parameters + self.num_trainable_parameters
self.peak_activation_mem += int(self.input_shape[0])
self.peak_activation_mem += int(self.output_shape[0])
if self.dropout > 1e-12:
return {'dropout': self.dropout_tensor}
def __call__(self, kwargs=None):
input_tensor = ModelAssign(kwargs, 'input_tensor', None)
self.input_shape = input_tensor.get_shape()[1:]
self.is_training = ModelAssign(kwargs, 'is_training')
with tf.variable_scope(self.name, tf.AUTO_REUSE) as scope:
output = batch_normalization(input_tensor,
is_training=self.is_training,
activation='linear',
trainable=self.trainable)
self.output_shape = output.get_shape()[1:]
self.output_tensor = output
self.MACs = int(self.output_shape[0]) * int(
self.output_shape[1]) * int(self.output_shape[2]) * 2
def __call__(self, kwargs=None):
"""
Call the attention module and returns an output tensor
:param kwargs: configurations
:return: an output tensor after feeding the input into attention model
"""
input_tensor = ModelAssign(kwargs, 'input_tensor', None)
self.input_shape = input_tensor.get_shape()[1:]
if self.method == 'softmax':
with tf.variable_scope(self.name, tf.AUTO_REUSE):
self.outputs = util.softmax_self_attention(inputs=input_tensor)
else:
raise NotImplementedError
pass
def __call__(self, kwargs=None):
input_tensor = ModelAssign(kwargs, 'input_tensor', None)
self.input_shape = input_tensor.get_shape()[1:]
self.dropout_tensor = tf.placeholder(dtype=tf.float32)
if self.dropout > 1e-12:
output = tf.nn.dropout(input_tensor,
keep_prob=1 - self.dropout_tensor,
name="dropout")
else:
output = input_tensor
self.output_shape = output.get_shape()[1:]
self.output_tensor = output
if self.dropout > 1e-12:
return {'dropout': self.dropout_tensor}
class SoftmaxLoss(Layer):
def __init__(self, kwargs):
Layer.__init__(self, kwargs)
self.input_shape = None
self.name = ModelAssign(kwargs, 'name', None)
self.label_smoothing = ModelAssign(kwargs, 'label_smoothing', 0.0)
self.output_tensor = None
self.num_trainable_parameters = 0
self.num_non_trainable_parameters = 0
self.shared_trainable_parameters = 0
self.MACs = 0
def __call__(self, kwargs=None, mode="training"):
self.label = ModelAssign(kwargs, 'label', None)
self.prediction = ModelAssign(kwargs, 'input', None)
self.input_shape = self.label.get_shape()[1:]
self.output_tensor = tf.losses.softmax_cross_entropy(
self.label, self.prediction, label_smoothing=self.label_smoothing)
def summary(self):
format_str = '|SoftmaxLoss(%s)' % self.name + ' ' * (17 -
len(self.name))
conv_str = "%s" % (self.input_shape)
space = " " * (36 - len(conv_str))
format_str += "|" + conv_str + space
ts = '%s' % 0
tstr = '| ' + ts + ' ' * (22 - len(ts))
format_str += tstr
ts = '%s' % 0
tstr = '| ' + ts + ' ' * (26 - len(ts))
format_str += tstr
ts = '%s' % 0
tstr = '| ' + ts + ' ' * (15 - len(ts))
format_str += tstr
ts = '%s' % None
tstr = '| ' + ts + ' ' * (14 - len(ts)) + '|'
format_str += tstr
print(format_str)
def __call__(self, kwargs=None):
input_tensor = ModelAssign(kwargs, 'input_tensor', None)
self.input_shape = input_tensor.get_shape()[1:]
self.skip_from_names = ModelAssign(kwargs, 'skip_from_names', None)
input_filters = int(self.input_shape[-1])
skip_from = ModelAssign(kwargs, 'skip_from', None)
initializer = ModelAssign(kwargs, 'initializer',
G.BACKEND_DEFAULT_CONV_INITIALIZER)
regularizer = ModelAssign(kwargs, 'regularizer',
G.BACKEND_DEFAULT_REGULARIZER)
self.is_training = ModelAssign(kwargs, 'is_training')
regularizer_strength = ModelAssign(kwargs, 'regularizer_strength',
1e-4)
# infer filters during compiling
if self.filters == 1:
self.filters = int(input_tensor.get_shape()[-1])
print("Intializing Conv Layer with L2-reg=%f" % regularizer_strength)
with tf.variable_scope(self.name) as scope:
if not self.hyper:
output = util.convolution2d(
inputs=input_tensor,
kernel_size=self.kernel_size,
filters=self.filters,
strides=self.strides,
padding=self.padding,
batchnorm=self.batchnorm,
activation=self.activation,
initializer=initializer(),
regularizer=regularizer(regularizer_strength),
use_bias=self.use_bias,
is_training=self.is_training,
mode=G.EXEC_CONV_MODE)
self.output_shape = output.get_shape()[1:]
# Normal
self.num_trainable_parameters += (self.kernel_size *
self.kernel_size *
input_filters) * self.filters
if self.use_bias:
self.num_trainable_parameters += self.filters
# FLOPS-MAC
self.MACs += int(self.input_shape[0]) * int(self.input_shape[1]) * \
self.kernel_size * self.kernel_size * input_filters * self.filters / self.strides / self.strides
if self.batchnorm and not G.DISABLE_BATCHNORM_COUNT:
self.num_trainable_parameters += 2 * self.filters
self.num_non_trainable_parameters += 2 * self.filters
self.MACs += int(self.output_shape[0]) * int(
self.output_shape[1]) * int(self.output_shape[2]) * 2
else:
raise NotImplementedError
self.mem_cost = self.num_non_trainable_parameters + self.num_trainable_parameters
self.peak_activation_mem += int(self.input_shape[0]) * int(
self.input_shape[1]) * int(self.input_shape[2])
self.peak_activation_mem += int(self.output_shape[0]) * int(
self.output_shape[1]) * int(self.output_shape[2])
if self.dropout > 1e-12:
self.dropout_tensor = tf.placeholder(dtype=tf.float32)
output = tf.nn.dropout(output,
keep_prob=1 - self.dropout_tensor,
name="dropout")
self.output_tensor = output
ret_dict = {}
if self.hyper:
ret_dict.update({
'layerinfo': {
self.layer_info: [
self.kernel_size, self.kernel_size, input_filters,
self.filters
]
}
})
if self.dropout > 1e-12:
ret_dict.update({'dropout': self.dropout_tensor})
return ret_dict
def __call__(self, kwargs=None):
input_tensor = ModelAssign(kwargs, 'input_tensor', None)
self.input_shape = input_tensor.get_shape()[1:]
output = input_tensor
self.output_shape = output.get_shape()[1:]
self.output_tensor = output