def call(self,inputs):
inputs_reshaped = tf.reshape(
inputs,
(-1,
inputs.shape[1], inputs.shape[2],
self.G_in.size * self.ch_in))
filterbank_reshaped = tf.reshape(
self.filterbank,
(self.kw,
self.kh,
self.G_in.size * self.ch_in,
self.G_out.size * self.ch_out))
outputs = nn_ops.Convolution(
inputs_reshaped.shape,
filter_shape=filterbank_reshaped.shape,
padding='VALID')(inputs_reshaped, filterbank_reshaped)
outputs = tf.reshape(outputs,
(-1,
outputs.shape[1], outputs.shape[2],
self.G_out.size,
self.ch_out))
# outputs = nn.bias_add(outputs, self.bias, data_format='NHWC')
return self.activation(outputs)
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
input_channel = self._get_input_channel(input_shape)
kernel_shape = self.kernel_size + (input_channel, self.filters)
self.kernel = self.add_weight(name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
try:
# Disable variable partitioning when creating the variable
if hasattr(self, '_scope') and self._scope:
partitioner = self._scope.partitioner
self._scope.set_partitioner(None)
else:
partitioner = None
self.u = self.add_weight(
name='sn_u',
shape=(1, tf.reduce_prod(kernel_shape[:-1])),
dtype=self.dtype,
initializer=tf.keras.initializers.ones,
synchronization=tf.VariableSynchronization.ON_READ,
trainable=False,
aggregation=tf.VariableAggregation.MEAN)
finally:
if partitioner:
self._scope.set_partitioner(partitioner)
if self.use_bias:
self.bias = self.add_weight(name='bias',
shape=(self.filters, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
channel_axis = self._get_channel_axis()
self.input_spec = InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_channel})
self._build_conv_op_input_shape = input_shape
self._build_input_channel = input_channel
self._padding_op = self._get_padding_op()
self._conv_op_data_format = conv_utils.convert_data_format(
self.data_format, self.rank + 2)
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format)
self.built = True
def call(self, inputs):
if self.group > 1:
if self._recreate_conv_op(inputs):
input_shape = inputs.get_shape()
if self.data_format == "channels_last":
input_shapes = [
tensor_shape.TensorShape([
input_shape[0], input_shape[1], input_shape[2],
input_shape[3] // self.group
])
] * self.group
elif self.data_format == "channels_first":
input_shapes = [
tensor_shape.TensorShape([
input_shape[0] // self.group, input_shape[1],
input_shape[2], input_shape[3]
])
] * self.group
else:
raise ValueError('Invalid data_format:', self.data_format)
self._convolution_ops = [
nn_ops.Convolution(input_shapes[i],
filter_shape=self.kernels[i].shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format)
for i in range(self.group)
]
kernels = tf.split(self.kernel, self.group, self._channel_axis)
inputs = tf.split(inputs, self.group, self._channel_axis)
outputs = [
self._convolution_ops[i](inputs[i], kernels[i])
for i in range(self.group)
]
outputs = tf.concat(outputs, self._channel_axis)
if self.use_bias:
if self.data_format == 'channels_first':
if self.rank == 1:
# nn.bias_add does not accept a 1D input tensor.
bias = array_ops.reshape(self.bias,
(1, self.filters, 1))
outputs += bias
else:
outputs = nn.bias_add(outputs,
self.bias,
data_format='NCHW')
else:
outputs = nn.bias_add(outputs,
self.bias,
data_format='NHWC')
if self.activation is not None:
return self.activation(outputs)
return outputs
else:
return super(GroupConv2D, self).call(inputs)
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
input_channel = self._get_input_channel(input_shape)
kernel_shape = self.kernel_size + (input_channel, self.filters)
self.kernel = self.add_weight(name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
channel_axis = self._get_channel_axis()
self.input_spec = InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_channel})
self._build_conv_op_input_shape = input_shape
self._build_input_channel = input_channel
self._padding_op = self._get_padding_op()
self._conv_op_data_format = conv_utils.convert_data_format(
self.data_format, self.rank + 2)
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format)
self.mask = self.create_mask(self.mask_type, self.color_conditioning)
self.built = True
def build(self, input_shape):
if len(input_shape) > self.rank + 2:
raise ValueError("Input to {} should has {:d} rank,"
"but received input shape {}".format(
self.name, self.rank, str(input_shape)))
channel_axis = -1 if self.data_format[-1] == 'C' else 1
in_channels = input_shape[channel_axis]
if not in_channels:
raise ValueError("Input channel must be defined legally,"
"but received {}".format(str(in_channels)))
kernel_shape = self.kernel_size + (in_channels, self.out_channels)
self.kernel = self.add_weight(shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
name='kernel')
if self.use_bias:
self.bias = self.add_weight(shape=(self.out_channels, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
name='bias')
else:
self.bias = None
self._convolution_op = nn_ops.Convolution(
input_shape=tensor_shape.TensorShape(input_shape),
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding,
data_format=self.data_format)
def call(self, inputs):
k = self.kernel * self.mask
# Check if the input_shape in call() is different from that in build().
# If they are different, recreate the _convolution_op to avoid the stateful
# behavior.
call_input_shape = inputs.get_shape()
recreate_conv_op = (call_input_shape[1:] != self._build_conv_op_input_shape[1:])
if recreate_conv_op:
self._convolution_op = nn_ops.Convolution( call_input_shape, filter_shape=self.kernel.shape, dilation_rate=self.dilation_rate, strides=self.strides, padding=self._padding_op, data_format=self._conv_op_data_format)
# Apply causal padding to inputs for Conv1D.
if self.padding == 'causal' and self.__class__.__name__ == 'Conv1D':
inputs = array_ops.pad(inputs, self._compute_causal_padding())
outputs = self._convolution_op(inputs, k)
if self.use_bias:
if self.data_format == 'channels_first':
if self.rank == 1:
# nn.bias_add does not accept a 1D input tensor.
bias = array_ops.reshape(self.bias, (1, self.filters, 1))
outputs += bias
else:
outputs = nn.bias_add(outputs, self.bias, data_format='NCHW')
else:
outputs = nn.bias_add(outputs, self.bias, data_format='NHWC')
if self.activation is not None:
return self.activation(outputs)
return outputs
def build_dilated_conv(self, input_shape, input_dim):
"""
Define trainable kernel and bias variables for dilated conv layers
Prepare conv operations
"""
kernel_shape = self.kernel_size + (input_dim, self.filters)
self.kernel = self.add_weight(name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
if self.use_bias:
self.bias = self.add_weight(name='bias',
shape=(self.filters, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
for dilation in self.dilations:
convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.get_shape(),
dilation_rate=dilation,
strides=self.strides,
padding="SAME",
data_format=conv_utils.convert_data_format(
self.data_format, self.rank + 2))
self.conv_ops.append(convolution_op)
def call(self, inputs):
if self._recreate_conv_op(inputs):
self._convolution_op = nn_ops.Convolution(
inputs.get_shape(),
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format)
self._build_conv_op_input_shape = inputs.get_shape()
# Apply causal padding to inputs for Conv1D.
if self.padding == 'causal' and self.__class__.__name__ == 'Conv1D':
inputs = array_ops.pad(inputs, self._compute_causal_padding())
outputs = self._convolution_op(inputs, self.kernel)
if self.use_bias:
if self.data_format == 'channels_first':
if self.rank == 1:
# nn.bias_add does not accept a 1D input tensor.
bias = array_ops.reshape(self.bias, (1, self.filters, 1))
outputs += bias
else:
outputs = nn.bias_add(outputs,
self.bias,
data_format='NCHW')
else:
outputs = nn.bias_add(outputs, self.bias, data_format='NHWC')
if self.activation is not None:
return self.activation(outputs)
return outputs
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape.dims[channel_axis].value is None:
raise ValueError(
'The channel dimension of the inputs should be defined. Found `None`.'
)
input_dim = int(input_shape[channel_axis])
wshape = [
self.kernel_size[0] * self.kernel_size[1] - 1, input_dim,
self.filters
]
W = self.add_weight(name='kernel',
shape=wshape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
self.kernel = array_ops.concat(
(W[:wshape[0] // 2], array_ops.zeros(
(1, wshape[1], wshape[2])), W[wshape[0] // 2:]),
axis=0)
self.kernel = array_ops.reshape(self.kernel, [
self.kernel_size[0], self.kernel_size[1], input_dim, self.filters
])
if self.use_bias:
self.bias = self.add_weight(name='bias',
shape=(self.filters, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
self.input_spec = InputSpec(ndim=4, axes={channel_axis: input_dim})
if self.padding == 'causal':
op_padding = 'valid'
else:
op_padding = self.padding
if not isinstance(op_padding, (list, tuple)):
op_padding = op_padding.upper()
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=op_padding,
data_format=conv_utils.convert_data_format(self.data_format, 4))
self.built = True
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape.dims[channel_axis].value is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
input_dim = int(input_shape[channel_axis])
kernel_shape = self.kernel_size + (input_dim, self.filters)
self.wn_g = self.add_weight(
name='wn_g',
shape=(self.filters, ),
initializer=tf.keras.initializers.RandomUniform(1, 1),
trainable=True,
dtype=self.dtype)
self.kernel = self.add_weight(name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
if self.use_bias:
self.bias = self.add_weight(name='bias',
shape=(self.filters, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
self.input_spec = InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_dim})
if self.padding == 'causal':
op_padding = 'valid'
else:
op_padding = self.padding
if not isinstance(op_padding, (list, tuple)):
op_padding = op_padding.upper()
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=op_padding,
data_format=conv_utils.convert_data_format(self.data_format,
self.rank + 2))
self.built = True
def call(self, inputs, training=None):
# Check if the input_shape in call() is different from that in build().
# If they are different, recreate the _convolution_op to avoid the stateful
# behavior.
if training is None:
training = K.learning_phase()
call_input_shape = inputs.get_shape()
recreate_conv_op = (call_input_shape[1:] !=
self._build_conv_op_input_shape[1:])
if recreate_conv_op:
self._convolution_op = nn_ops.Convolution(
call_input_shape,
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format)
# Apply causal padding to inputs for Conv1D.
if self.padding == 'causal' and self.__class__.__name__ == 'Conv1D':
inputs = array_ops.pad(inputs, self._compute_causal_padding())
# Update SpectralNormalization variable
u, v, w = self.calc_u(self.kernel)
def u_update():
# TODO u_update maybe need `training control`
return tf_utils.smart_cond(
training, lambda: self._assign_new_value(self.u, u),
lambda: array_ops.identity(u))
# NOTE add update must in call function scope
self.add_update(u_update)
sigma = self.calc_sigma(u, v, w)
new_kernel = tf_utils.smart_cond(training, lambda: self.kernel / sigma,
lambda: self.kernel)
outputs = self._convolution_op(inputs, new_kernel)
if self.use_bias:
if self.data_format == 'channels_first':
if self.rank == 1:
# nn.bias_add does not accept a 1D input tensor.
bias = array_ops.reshape(self.bias, (1, self.filters, 1))
outputs += bias
else:
outputs = nn.bias_add(outputs,
self.bias,
data_format='NCHW')
else:
outputs = nn.bias_add(outputs, self.bias, data_format='NHWC')
if self.activation is not None:
return self.activation(outputs)
return outputs
def _testConvReparameterization(self, layer_class):
batch_size, depth, height, width, channels, filters = 2, 4, 4, 4, 3, 5
with self.test_session() as sess:
(kernel_posterior, kernel_prior, kernel_divergence,
bias_posterior, bias_prior, bias_divergence, layer, inputs,
outputs, kl_penalty, kernel_shape) = self._testConvSetUp(
layer_class, batch_size,
depth=depth, height=height, width=width, channels=channels,
filters=filters)
convolution_op = nn_ops.Convolution(
tf.TensorShape(inputs.shape),
filter_shape=tf.TensorShape(kernel_shape),
padding='SAME')
expected_outputs = convolution_op(inputs, kernel_posterior.result_sample)
expected_outputs = tf.nn.bias_add(expected_outputs,
bias_posterior.result_sample,
data_format='NHWC')
[
expected_outputs_, actual_outputs_,
expected_kernel_, actual_kernel_,
expected_kernel_divergence_, actual_kernel_divergence_,
expected_bias_, actual_bias_,
expected_bias_divergence_, actual_bias_divergence_,
] = sess.run([
expected_outputs, outputs,
kernel_posterior.result_sample, layer.kernel_posterior_tensor,
kernel_divergence.result, kl_penalty[0],
bias_posterior.result_sample, layer.bias_posterior_tensor,
bias_divergence.result, kl_penalty[1],
])
self.assertAllClose(
expected_kernel_, actual_kernel_,
rtol=1e-6, atol=0.)
self.assertAllClose(
expected_bias_, actual_bias_,
rtol=1e-6, atol=0.)
self.assertAllClose(
expected_outputs_, actual_outputs_,
rtol=1e-6, atol=0.)
self.assertAllClose(
expected_kernel_divergence_, actual_kernel_divergence_,
rtol=1e-6, atol=0.)
self.assertAllClose(
expected_bias_divergence_, actual_bias_divergence_,
rtol=1e-6, atol=0.)
self.assertAllEqual(
[[kernel_posterior, kernel_prior, kernel_posterior.result_sample]],
kernel_divergence.args)
self.assertAllEqual(
[[bias_posterior, bias_prior, bias_posterior.result_sample]],
bias_divergence.args)
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape[channel_axis].value is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
input_dim = input_shape[channel_axis].value
kernel_shape = self.kernel_size + (input_dim, self.filters)
self.kernel = self.add_variable(name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
if self.weight_norm:
self.V = self.add_variable(
name='V_weight_norm',
shape=kernel_shape,
dtype=tf.float32,
initializer=tf.random_normal_initializer(0, 0.05),
trainable=True)
self.g = self.add_variable(name='g_weight_norm',
shape=(self.filters, ),
initializer=init_ops.ones_initializer(),
dtype=self.dtype,
trainable=True)
if self.mean_only_batch_norm:
self.batch_norm_running_average = []
if self.use_bias:
self.bias = self.add_variable(name='bias',
shape=(self.filters, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
self.input_spec = base.InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_dim})
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.get_shape(),
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=utils.convert_data_format(self.data_format,
self.rank + 2))
self.built = True
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape[channel_axis].value is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
input_dim = input_shape[channel_axis].value
# kernel_shape=(self.kernel_size, input_dim, self.filters)
kernel_shape = self.kernel_size + (input_dim, self.filters)
kernel = self.add_variable(name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
# weight normalization
if self.weight_norm:
g = self.add_variable(name='wn/g',
shape=(self.filters, ),
initializer=init_ops.ones_initializer(),
dtype=kernel.dtype,
trainable=True)
self.kernel = tf.reshape(
g, [1, 1, self.filters]) * nn_impl.l2_normalize(
kernel, [0, 1])
else:
self.kernel = kernel
if self.use_bias:
self.bias = self.add_variable(name='bias',
shape=(self.filters, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
self.input_spec = base.InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_dim})
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.get_shape(),
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=utils.convert_data_format(self.data_format,
self.rank + 2))
self.built = True
def build(
self,
input_shape,
):
input_shape = tf.TensorShape(input_shape)
input_channel = self._get_input_channel(input_shape)
kernel_shape = self.kernel_size + (input_channel, self.filters)
self.kernel = self.add_weight(
name="kernel",
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype,
)
# Add mask to remove weights on half of the kernel to the left
# (only keep future
# context)
left_kernel_dims = (self.future_context, input_channel, self.filters)
left_kernel = tf.fill(dims=left_kernel_dims, value=0)
right_kernel_dims = (self.future_context + 1, input_channel, self.filters)
right_kernel = tf.fill(dims=right_kernel_dims, value=1)
mask_kernel = tf.cast(tf.concat([left_kernel, right_kernel], axis=0), dtype=self.dtype)
self.kernel = tf.multiply(self.kernel, mask_kernel)
if self.use_bias:
self.bias = self.add_weight(
name="bias",
shape=(self.filters,),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype,
)
else:
self.bias = None
channel_axis = self._get_channel_axis()
self.input_spec = tf.keras.layers.InputSpec(ndim=self.rank + 2, axes={channel_axis: input_channel})
self.make_conv_op_input_shape = input_shape
self.make_input_channel = input_channel
self._padding_op = self._get_padding_op()
self._conv_op_data_format = conv_utils.convert_data_format(self.data_format, self.rank + 2)
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format,
)
self.built = True
def build(self, input_shape):
input_shape = tf.TensorShape(input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape[channel_axis].value is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
input_dim = input_shape[channel_axis].value
kernel_shape = self.kernel_size + (input_dim, self.filters)
self.kernel_mu = self.add_variable(
'posterior_kernel_mu',
shape=kernel_shape,
initializer=self.kernel_mu_initializer,
trainable=True,
dtype=self.dtype)
self.kernel_rho = self.add_variable(
'posterior_kernel_rho',
shape=kernel_shape,
initializer=self.kernel_rho_initializer,
trainable=True,
dtype=self.dtype)
if self.use_bias:
self.bias_mu = self.add_variable(
'posterior_bias_mu',
shape=[
self.filters,
],
initializer=self.bias_mu_initializer,
dtype=self.dtype,
trainable=True)
self.bias_rho = self.add_variable(
'posterior_bias_rho',
shape=[
self.filters,
],
initializer=self.bias_rho_initializer)
else:
self.bias_mu = None
self.bias_rho = None
self.input_spec = base.InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_dim})
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel_mu.get_shape(),
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=utils.convert_data_format(self.data_format,
self.rank + 2))
self.built = True
def build(self, input_shape):
input_shape = tf.TensorShape(input_shape)
if self.data_format == "channels_first":
channel_axis = 1
else:
channel_axis = -1
input_dim = tf.compat.dimension_value(input_shape[channel_axis])
if input_dim is None:
raise ValueError("The channel dimension of inputs Found `None`.")
kernel_shape = self.kernel_size + (input_dim, self.filters)
# If self.dtype is None, build weights using the default dtype.
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
# Must have a posterior kernel.
self.kernel_posterior = self.kernel_posterior_fn(
dtype, kernel_shape, "kernel_posterior", self.trainable, self.add_variable
)
if self.kernel_prior_fn is None:
self.kernel_prior = None
else:
self.kernel_prior = self.kernel_prior_fn(
dtype, kernel_shape, "kernel_prior", self.trainable, self.add_variable
)
if self.bias_posterior_fn is None:
self.bias_posterior = None
else:
self.bias_posterior = self.bias_posterior_fn(
dtype,
(self.filters,),
"bias_posterior",
self.trainable,
self.add_variable,
)
if self.bias_prior_fn is None:
self.bias_prior = None
else:
self.bias_prior = self.bias_prior_fn(
dtype, (self.filters,), "bias_prior", self.trainable, self.add_variable
)
self.input_spec = tf.keras.layers.InputSpec(
ndim=self.rank + 2, axes={channel_axis: input_dim}
)
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=tf.TensorShape(kernel_shape),
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=tf_layers_util.convert_data_format(
self.data_format, self.rank + 2
),
)
self.built = True
def build(self, input_shape):
stack_size, input_dim = input_shape[-2:]
if stack_size is None or input_dim is None:
raise ValueError(
"The two last dimensions of the inputs should be defined. Found `None`."
)
kernel_shape = (stack_size, *self.kernel_size, stack_size, input_dim,
self.filters)
self.kernel = self.add_weight(name="kernel",
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
if self.use_bias:
bias_shape = (stack_size, self.filters)
self.bias = self.add_weight(name="bias",
shape=bias_shape,
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
kernel_mask = get_stack_kernel_mask(self.kernel_size, stack_size,
self.mask_center)
self.kernel_mask = tf.constant(kernel_mask,
dtype=tf.float32,
name="kernel_mask")
self.input_spec = InputSpec(ndim=self.rank + 2,
axes={
-1: input_dim,
-2: stack_size
})
strides = conv_utils.normalize_tuple(1, self.rank, "strides")
dilation_rate = conv_utils.normalize_tuple(1, self.rank,
"dilation_rate")
self._convolution_op = nn_ops.Convolution(
input_shape=input_shape,
filter_shape=tf.TensorShape(kernel_shape[1:]),
dilation_rate=dilation_rate,
strides=strides,
padding=get_stack_padding(self.kernel_size),
data_format=conv_utils.convert_data_format("channels_last",
self.rank + 2))
def testConvolutionClass3DExpandedBatch(self):
tensor_in_sizes_batch = [10, 2, 3, 1, 3]
tensor_in_sizes_expanded_batch = [2, 5, 2, 3, 1, 3]
filter_in_sizes = [1, 1, 1, 3, 3]
filter_in = self._CreateNumpyTensor(filter_in_sizes)
x1 = self._CreateNumpyTensor(tensor_in_sizes_batch)
x2 = x1.reshape(tensor_in_sizes_expanded_batch)
convolver1 = nn_ops.Convolution(input_shape=x1.shape,
filter_shape=filter_in.shape,
strides=[1, 1, 1],
padding="VALID")
self.assertEqual(convolver1.num_batch_dims, 1)
convolver2 = nn_ops.Convolution(input_shape=x2.shape,
filter_shape=filter_in.shape,
strides=[1, 1, 1],
padding="VALID")
self.assertEqual(convolver2.num_batch_dims, 2)
conv1 = convolver1(x1, filter_in)
conv2 = convolver2(x2, filter_in)
self.assertEqual(conv1.shape, tensor_in_sizes_batch)
self.assertEqual(conv2.shape, tensor_in_sizes_expanded_batch)
self.assertAllEqual(conv1, self.evaluate(conv2).reshape(conv1.shape))
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape[channel_axis].value is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
input_dim = int(input_shape[channel_axis])
kernel_shape = self.kernel_size + (input_dim, self.filters)
self.kernel = self.add_weight(name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
self.loc = self.add_variable(
'loc',
shape=kernel_shape,
initializer=tf.keras.initializers.RandomNormal(
mean=self.loc_mean, stddev=self.loc_stddev, seed=None),
regularizer=None,
constraint=None,
dtype=self.dtype,
trainable=True)
self.loc2 = self.loc.numpy()
if self.use_bias:
self.bias = self.add_weight(name='bias',
shape=(self.filters, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
self.input_spec = InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_dim})
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.get_shape(),
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=conv_utils.convert_data_format(self.data_format,
self.rank + 2))
self.built = True
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
# pylint: disable=no-member
if input_shape[channel_axis].value is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
# pylint: disable=no-member
input_dim = input_shape[channel_axis].value
kernel_shape = self.kernel_size + (input_dim, self.filters)
# The variables defined below are specific to the weight normed conv class
self.kernel_v = self.add_variable(name='kernel_v',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
self.kernel_g = self.add_variable(name='kernel_g',
shape=[],
trainable=True,
dtype=self.dtype)
self.kernel = self.kernel_g * tf.nn.l2_normalize(self.kernel_v)
if self.use_bias:
self.bias = self.add_variable(name='bias',
shape=(self.filters, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
self.input_spec = base.InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_dim})
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.get_shape(),
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=utils.convert_data_format(self.data_format,
self.rank + 2))
self.built = True
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
input_channel = self._get_input_channel(input_shape)
kernel_shape = self.kernel_size + (input_channel, self.filters)
self.kernel_weights = self.add_weight(
name='kernel_weights',
shape=kernel_shape,
initializer=tf.random_normal_initializer(mean=0.0, stddev=0.05),
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
self.kernel_log_scale = self.add_weight(name='kernel_log_scale',
shape=(1, 1, 1, self.filters),
initializer=tf.constant_initializer(value=0.),
regularizer=None,
constraint=None,
trainable=True,
dtype=self.dtype)
if self.use_bias:
self.bias = self.add_weight(
name='bias',
shape=(self.filters,),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
channel_axis = self._get_channel_axis()
self.input_spec = InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_channel})
self._build_conv_op_input_shape = input_shape
self._build_input_channel = input_channel
self._padding_op = self._get_padding_op()
self._conv_op_data_format = conv_utils.convert_data_format(
self.data_format, self.rank + 2)
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format)
self.built = True
def build(self, input_shape):
input_shape = tf.TensorShape(input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape[channel_axis].value is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
input_dim = input_shape[channel_axis].value
kernel_shape = self.kernel_size + (input_dim, self.filters)
self.kernel = self.add_variable(name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
if self.spectral_normalization:
# TODO pass update_collection?
vs = tf.get_variable_scope()
upd_coll = None if not vs.reuse else NO_OPS
# print("update collection = ", upd_coll)
self.kernel = spectral_normed_weight(self.kernel,
update_collection=upd_coll)
if self.use_bias:
self.bias = self.add_variable(name='bias',
shape=(self.filters, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
self.input_spec = tf.layers.InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_dim})
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.get_shape(),
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=convert_data_format(self.data_format, self.rank + 2))
self.built = True
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
input_channel = self._get_input_channel(input_shape)
kernel_shape = self.kernel_size + (input_channel, self.filters)
self.kernelA = self.add_weight(name='kernelA',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint,
trainable=True,
dtype=self.dtype)
self.kernelB = K.constant(self.kernelB_init_weight)
self.kernel = K.transpose(
K.dot(K.transpose(self.kernelA), self.kernelB))
if self.use_bias:
self.bias = self.add_weight(name='bias',
shape=(self.filters, ),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint,
trainable=True,
dtype=self.dtype)
else:
self.bias = None
channel_axis = self._get_channel_axis()
self.input_spec = InputSpec(ndim=self.rank + 2,
axes={channel_axis: input_channel})
self._build_conv_op_input_shape = input_shape
self._build_input_channel = input_channel
self._padding_op = self._get_padding_op()
self._conv_op_data_format = conv_utils.convert_data_format(
self.data_format, self.rank + 2)
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format)
self.built = True
def call(self, inputs):
aw = self.aw if tf.keras.backend.learning_phase() else self.l1_pruning(
self.aw, self.lambda_l1)
mask = self.mask if tf.keras.backend.learning_phase(
) else self.l1_pruning(self.mask, self.lambda_mask)
atten = self.atten
aw_kbs = self.aw_kb
############################### Decomposed Kernel #################################
self.my_theta = self.sw * mask + aw + tf.keras.backend.sum(
aw_kbs * atten, axis=-1)
###################################################################################
# if self._recreate_conv_op(inputs):
self._convolution_op = nn_ops.Convolution(
inputs.get_shape(),
filter_shape=self.my_theta.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format)
# Apply causal padding to inputs for Conv1D.
if self.padding == 'causal' and self.__class__.__name__ == 'Conv1D':
inputs = array_ops.pad(inputs, self._compute_causal_padding())
outputs = self._convolution_op(inputs, self.my_theta)
if self.use_bias:
if self.data_format == 'channels_first':
if self.rank == 1:
# nn.bias_add does not accept a 1D input tensor.
bias = array_ops.reshape(self.bias, (1, self.filters, 1))
outputs += bias
else:
outputs = nn.bias_add(outputs,
self.bias,
data_format='NCHW')
else:
outputs = nn.bias_add(outputs, self.bias, data_format='NHWC')
if self.activation is not None:
return self.activation(outputs)
return outputs
def build(self, input_shape):
input_shape = tensor_shape.TensorShape(input_shape)
dau_params_shape = self.get_dau_variable_shape(input_shape)
if self.dau_weights is None:
self.dau_weights = self.add_dau_weights_var(input_shape)
elif np.any(self.dau_weights.shape != dau_params_shape):
raise ValueError('Shape mismatch for variable `dau_weights`')
if self.dau_mu1 is None:
self.dau_mu1 = self.add_dau_mu1_var(input_shape)
elif np.any(self.dau_mu1.shape != dau_params_shape):
raise ValueError('Shape mismatch for variable `dau_mu1`')
if self.dau_mu2 is None:
self.dau_mu2 = self.add_dau_mu2_var(input_shape)
elif np.any(self.dau_mu2.shape != dau_params_shape):
raise ValueError('Shape mismatch for variable `dau_mu2`')
if self.dau_sigma is None:
self.dau_sigma = self.add_dau_sigma_var(input_shape, trainable=self.dau_sigma_trainable)
elif np.any(self.dau_sigma.shape != dau_params_shape):
raise ValueError('Shape mismatch for variable `dau_sigma`')
if self.use_bias:
self.bias = self.add_bias_var()
else:
self.bias = None
input_channel_axis = self._get_input_channel_axis()
num_input_channels = self._get_input_channels(input_shape)
self.input_spec = base.InputSpec(ndim=self.rank + 2,
axes={input_channel_axis: num_input_channels})
kernel_shape = tf.TensorShape((self.max_kernel_size, self.max_kernel_size, num_input_channels, self.filters))
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=kernel_shape,
dilation_rate=(1,1),
strides=(self.strides,self.strides),
padding="SAME",
data_format=utils.convert_data_format(self.data_format,
self.rank + 2))
self.built = True
def call(self, double_input_matrix, training=False):
[target_inputs, inputs] = double_input_matrix
if self._recreate_conv_op(inputs):
self._convolution_op = nn_ops.Convolution(
inputs.get_shape(),
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format)
# Apply causal padding to inputs for Conv1D.
if self.padding == 'causal' and self.__class__.__name__ == 'Conv1D':
inputs = array_ops.pad(inputs, self._compute_causal_padding())
b, W = self.calculate_internal_weight_matrix(target_inputs, training)
outputs = self.propagate_layer(inputs, W, b)
target_outputs = self.propagate_layer(target_inputs, W, b)
return [target_outputs, outputs]
def build(self, input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
if not (dtype.is_floating or dtype.is_complex):
raise TypeError(
'Unable to build `Dense` layer with non-floating point dtype {:s}'
.format(dtype))
input_shape = tf.TensorShape(input_shape)
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
input_dim = tf.compat.dimension_value(input_shape[channel_axis])
if input_dim is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
self.kernel_shape = self.kernel_size + (input_dim, self.filters)
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
self.input_spec = tf.keras.layers.InputSpec(
ndim=self.rank + 2, axes={channel_axis: input_dim})
self._convolution_op = nn_ops.Convolution(
input_shape,
filter_shape=tf.TensorShape(self.kernel_shape),
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=conv_utils.convert_data_format(self.data_format,
self.rank + 2))
self.num_weights = np.prod(self.kernel_shape)
self._posterior = self._make_posterior_fn(
self.num_weights, self.filters if self.use_bias else 0, dtype)
self._prior = self._make_prior_fn(self.num_weights,
self.filters if self.use_bias else 0,
dtype)
self.built = True
def build(self, input_shape):
super().build(input_shape) # super of host class
if self.mask_threshold:
# just for given threshold
size = self._get_window_size()
self.mask_kernel = tf.ones((size, 1, 1), dtype=self.dtype)
if self.padding == 'causal':
op_padding = 'valid'
else:
op_padding = self.padding
mask_shape = (*input_shape.as_list()[:-1], 1)
self._mask_op = nn_ops.Convolution(
tf.TensorShape(mask_shape),
filter_shape=self.mask_kernel.get_shape(),
strides=self.strides,
padding=op_padding.upper(),
data_format=conv_utils.convert_data_format(
self.data_format, 3),
)
def call(self, inputs):
print(self.data_format)
#inputs = backend.permute_dimensions(inputs, (0, 2, 3, 1))
print(inputs.get_shape())
print(self.kernel.shape)
print(self.dilation_rate)
print(self.strides)
print(self._padding_op)
print(self._conv_op_data_format)
if self._recreate_conv_op(inputs):
print("Check point 0---------------------------")
self._convolution_op = nn_ops.Convolution(
inputs.get_shape(),
filter_shape=self.kernel.shape,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self._padding_op,
data_format=self._conv_op_data_format)
print("Check point 1---------------------------")
# Apply causal padding to inputs for Conv1D.
if self.padding == 'causal' and self.__class__.__name__ == 'Conv1D':
inputs = array_ops.pad(inputs, self._compute_causal_padding())
print("Check point 2---------------------------")
outputs = self._convolution_op(inputs, self.kernel)
print("Check point 3---------------------------")
if self.use_bias:
if self.data_format == 'channels_first':
if self.rank == 1:
# nn.bias_add does not accept a 1D input tensor.
bias = array_ops.reshape(self.bias, (1, self.filters, 1))
outputs += bias
else:
outputs = nn.bias_add(outputs,
self.bias,
data_format='NCHW')
else:
outputs = nn.bias_add(outputs, self.bias, data_format='NHWC')
print("Check point 4---------------------------")
if self.activation is not None:
return self.activation(outputs)
return outputs
