def __call__(self, inputs, *args, **kwargs):
with vs.variable_scope(self._scope, reuse=self.built
or self._reuse) as scope:
with ops.name_scope(scope.original_name_scope):
if not self.built:
input_shapes = [
x.get_shape() for x in nest.flatten(inputs)
]
if len(input_shapes) == 1:
self.build(input_shapes[0])
else:
self.build(input_shapes)
outputs = self.call(inputs, *args, **kwargs)
# # Apply activity regularization.
# # Note that it should be applied every time the layer creates a new
# # output, since it is output-specific.
# if hasattr(self, 'activity_regularizer') and self.activity_regularizer:
# output_list = _to_list(outputs)
# for output in output_list:
# with ops.name_scope('ActivityRegularizer'):
# activity_regularization = self.activity_regularizer(output)
# self.add_loss(activity_regularization)
# _add_elements_to_collection(
# activity_regularization, ops.GraphKeys.REGULARIZATION_LOSSES)
# Update global default collections.
_add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS)
self.built = True
return outputs
def add_variable(self,
name,
shape,
dtype=None,
trainable=True,
initializer=None,
regularizer=None):
if dtype is None:
dtype = self.dtype
existing_variables = set(tf_variables.global_variables())
with vs.variable_scope(self._scope, reuse=self.built
or self._reuse) as scope:
with ops.name_scope(scope.original_name_scope):
full_name = get_tensor_scope() + name
variable = vs.get_variable(name,
shape=shape,
initializer=initializer,
dtype=dtypes.as_dtype(dtype),
trainable=trainable
and self.trainable)
if variable in existing_variables:
# Work only if the layer is built
return variable
if regularizer:
raise NotImplementedError()
if trainable:
self._trainable_weights.append(variable)
else:
self._non_trainable_weights.append(variable)
return variable
def batch_norm(inputs,
decay=0.999,
center=True,
scale=False,
epsilon=0.001,
activation_fn=None,
param_initializers=None,
param_regularizers=None,
updates_collections=ops.GraphKeys.UPDATE_OPS,
is_training=True,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
batch_weights=None,
fused=False,
data_format=DATA_FORMAT_NHWC,
zero_debias_moving_mean=False,
scope=None,
renorm=False,
renorm_clipping=None,
renorm_decay=0.99):
scope = _default_scope(scope, 'BATCH_NORM', 'BatchNorm')
if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC):
raise ValueError('data_format has to be either NCHW or NHWC.')
axis = 1 if data_format == DATA_FORMAT_NCHW else -1
with vs.variable_scope(scope, reuse=reuse) as sc:
if not param_initializers:
param_initializers = {}
beta_initializer = param_initializers.get('beta', init_ops.zeros_initializer())
gamma_initializer = param_initializers.get('gamma', init_ops.ones_initializer())
moving_mean_initializer = param_initializers.get('moving_mean', init_ops.zeros_initializer())
moving_variance_initializer = param_initializers.get('moving_variance', init_ops.ones_initializer())
if not param_regularizers:
param_regularizers = {}
beta_regularizer = param_regularizers.get('beta')
gamma_regularizer = param_regularizers.get('gamma')
return layers.batch_norm(
inputs=inputs,
axis=axis,
momentum=decay,
epsilon=epsilon,
center=center,
scale=scale,
beta_initializer=beta_initializer,
gamma_initializer=gamma_initializer,
moving_mean_initializer=moving_mean_initializer,
moving_variance_initializer=moving_variance_initializer,
beta_regularizer=beta_regularizer,
gamma_regularizer=gamma_regularizer,
trainable=trainable,
renorm=renorm,
renorm_clipping=renorm_clipping,
renorm_momentum=renorm_decay,
fused=fused,
training=is_training)
def fully_connected(inputs,
num_outputs,
activation_fn=nn.relu,
normalizer_fn=None,
normalizer_params=None,
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
biases_initializer=init_ops.zeros_initializer(),
biases_regularizer=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
scope=None):
scope = _default_scope(scope, 'FULLY_CONNECTED', 'fully_connected')
with vs.variable_scope(scope, reuse=reuse) as sc:
return layers.dense(
inputs=inputs,
units=num_outputs,
activation=activation_fn,
use_bias=True if biases_initializer is not None else False,
kernel_initializer=weights_initializer,
bias_initializer=biases_initializer,
bias_regularizer=biases_regularizer,
activity_regularizer=None,
trainable=trainable,
reuse=reuse)
def fully_connected(inputs,
num_outputs,
activation_fn=nn.relu,
normalizer_fn=None,
normalizer_params=None,
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
biases_initializer=init_ops.zeros_initializer(),
biases_regularizer=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
scope=None):
scope = _default_scope(scope, 'FULLY_CONNECTED', 'fully_connected')
with vs.variable_scope(scope, reuse=reuse) as sc:
return layers.dense(
inputs=inputs,
units=num_outputs,
activation=activation_fn,
use_bias=True if biases_initializer is not None else False,
kernel_initializer=weights_initializer,
bias_initializer=biases_initializer,
bias_regularizer=biases_regularizer,
activity_regularizer=None,
trainable=trainable,
reuse=reuse)
def add_variable(self, name, shape, dtype=None, trainable=True,
initializer=None, regularizer=None):
if dtype is None:
dtype = self.dtype
existing_variables = set(tf_variables.global_variables())
with vs.variable_scope(self._scope,
reuse=self.built or self._reuse) as scope:
with ops.name_scope(scope.original_name_scope):
full_name = get_tensor_scope() + name
variable = vs.get_variable(name,
shape=shape,
initializer=initializer,
dtype=dtypes.as_dtype(dtype),
trainable=trainable and self.trainable)
if variable in existing_variables:
# Work only if the layer is built
return variable
if regularizer:
raise NotImplementedError()
if trainable:
self._trainable_weights.append(variable)
else:
self._non_trainable_weights.append(variable)
return variable
def __call__(self, inputs, *args, **kwargs):
with vs.variable_scope(self._scope,
reuse=self.built or self._reuse) as scope:
with ops.name_scope(scope.original_name_scope):
if not self.built:
input_shapes = [x.get_shape()
for x in nest.flatten(inputs)]
if len(input_shapes) == 1:
self.build(input_shapes[0])
else:
self.build(input_shapes)
outputs = self.call(inputs, *args, **kwargs)
# # Apply activity regularization.
# # Note that it should be applied every time the layer creates a new
# # output, since it is output-specific.
# if hasattr(self, 'activity_regularizer') and self.activity_regularizer:
# output_list = _to_list(outputs)
# for output in output_list:
# with ops.name_scope('ActivityRegularizer'):
# activity_regularization = self.activity_regularizer(output)
# self.add_loss(activity_regularization)
# _add_elements_to_collection(
# activity_regularization, ops.GraphKeys.REGULARIZATION_LOSSES)
# Update global default collections.
_add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS)
self.built = True
return outputs
def batch_norm(inputs,
decay=0.999,
center=True,
scale=False,
epsilon=0.001,
activation_fn=None,
param_initializers=None,
param_regularizers=None,
updates_collections=ops.GraphKeys.UPDATE_OPS,
is_training=True,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
batch_weights=None,
fused=False,
data_format=DATA_FORMAT_NHWC,
zero_debias_moving_mean=False,
scope=None,
renorm=False,
renorm_clipping=None,
renorm_decay=0.99):
scope = _default_scope(scope, 'BATCH_NORM', 'BatchNorm')
if data_format not in (DATA_FORMAT_NCHW, DATA_FORMAT_NHWC):
raise ValueError('data_format has to be either NCHW or NHWC.')
axis = 1 if data_format == DATA_FORMAT_NCHW else -1
with vs.variable_scope(scope, reuse=reuse) as sc:
if not param_initializers:
param_initializers = {}
beta_initializer = param_initializers.get('beta', init_ops.zeros_initializer())
gamma_initializer = param_initializers.get('gamma', init_ops.ones_initializer())
moving_mean_initializer = param_initializers.get('moving_mean', init_ops.zeros_initializer())
moving_variance_initializer = param_initializers.get('moving_variance', init_ops.ones_initializer())
if not param_regularizers:
param_regularizers = {}
beta_regularizer = param_regularizers.get('beta')
gamma_regularizer = param_regularizers.get('gamma')
return layers.batch_norm(
inputs=inputs,
axis=axis,
momentum=decay,
epsilon=epsilon,
center=center,
scale=scale,
beta_initializer=beta_initializer,
gamma_initializer=gamma_initializer,
moving_mean_initializer=moving_mean_initializer,
moving_variance_initializer=moving_variance_initializer,
beta_regularizer=beta_regularizer,
gamma_regularizer=gamma_regularizer,
trainable=trainable,
renorm=renorm,
renorm_clipping=renorm_clipping,
renorm_momentum=renorm_decay,
fused=fused,
training=is_training)
def __call__(self, inputs, *args, **kwargs):
with vs.variable_scope(self.name, reuse=self.built
or self._reuse) as scope:
if not self.built:
input_shapes = [x.get_shape() for x in nest.flatten(inputs)]
if len(input_shapes) == 1: self.build(input_shapes[0])
else: self.build(input_shapes)
outputs = self.call(inputs, *args, **kwargs)
# Update global default collections.
_add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS)
return outputs
def convolution(
inputs,
num_outputs,
kernel_size,
stride=1,
padding='SAME',
data_format=None,
rate=1,
activation_fn=nn.relu,
normalizer_fn=None,
normalizer_params=None,
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
biases_initializer=init_ops.zeros_initializer(),
biases_regularizer=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
scope=None,
):
scope = _default_scope(scope, 'Conv')
if data_format not in [None, 'NHWC', 'NCHW']:
raise ValueError('Invalid data_format: %r' % (data_format,))
data_format = 'channels_first' if data_format == 'NCHW' else 'channels_last'
input_rank = inputs.get_shape().ndims
with vs.variable_scope(scope, reuse=reuse) as sc:
if input_rank == 4:
return layers.conv2d(
inputs=inputs,
filters=num_outputs,
kernel_size=kernel_size,
strides=stride,
padding=padding,
data_format=data_format,
dilation_rate=rate,
activation=activation_fn,
use_bias=True if biases_initializer is not None else False,
kernel_initializer=weights_initializer,
bias_initializer=biases_initializer,
bias_regularizer=biases_regularizer,
activity_regularizer=None,
trainable=trainable,
reuse=reuse,
)
def convolution(inputs,
num_outputs,
kernel_size,
stride=1,
padding='SAME',
data_format=None,
rate=1,
activation_fn=nn.relu,
normalizer_fn=None,
normalizer_params=None,
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
biases_initializer=init_ops.zeros_initializer(),
biases_regularizer=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
scope=None):
scope = _default_scope(scope, 'CONVOLUTION', 'Conv')
if data_format not in [None, 'NHWC', 'NCHW']:
raise ValueError('Invalid data_format: %r' % (data_format,))
data_format = 'channels_first' if data_format == 'NCHW' else 'channels_last'
input_rank = inputs.get_shape().ndims
with vs.variable_scope(scope, reuse=reuse) as sc:
if input_rank == 4:
return layers.conv2d(
inputs=inputs,
filters=num_outputs,
kernel_size=kernel_size,
strides=stride,
padding=padding,
data_format=data_format,
dilation_rate=rate,
activation=activation_fn,
use_bias=True if biases_initializer is not None else False,
kernel_initializer=weights_initializer,
bias_initializer=biases_initializer,
bias_regularizer=biases_regularizer,
activity_regularizer=None,
trainable=trainable,
reuse=reuse)