def _scharr_edges(cls, image, magnitude):
""" Returns a tensor holding modified Scharr edge maps.
Parameters
----------
image: tensor
Image tensor with shape [batch_size, h, w, d] and type float32. The image(s) must be
2x2 or larger.
magnitude: bool
Boolean to determine if the edge magnitude or edge direction is returned
Returns
-------
tensor
Tensor holding edge maps for each channel. Returns a tensor with shape `[batch_size, h,
w, d, 2]` where the last two dimensions hold `[[dy[0], dx[0]], [dy[1], dx[1]], ...,
[dy[d-1], dx[d-1]]]` calculated using the Scharr filter.
"""
# Define vertical and horizontal Scharr filters.
# TODO PlaidML: AttributeError: 'Value' object has no attribute 'get_shape'
static_image_shape = image.get_shape()
image_shape = K.shape(image)
# 5x5 modified Scharr kernel ( reshape to (5,5,1,2) )
matrix = np.array([[[[0.00070, 0.00070]], [[0.00520, 0.00370]],
[[0.03700, 0.00000]], [[0.00520, -0.0037]],
[[0.00070, -0.0007]]],
[[[0.00370, 0.00520]], [[0.11870, 0.11870]],
[[0.25890, 0.00000]], [[0.11870, -0.1187]],
[[0.00370, -0.0052]]],
[[[0.00000, 0.03700]], [[0.00000, 0.25890]],
[[0.00000, 0.00000]], [[0.00000, -0.2589]],
[[0.00000, -0.0370]]],
[[[-0.0037, 0.00520]], [[-0.1187, 0.11870]],
[[-0.2589, 0.00000]], [[-0.1187, -0.1187]],
[[-0.0037, -0.0052]]],
[[[-0.0007, 0.00070]], [[-0.0052, 0.00370]],
[[-0.0370, 0.00000]], [[-0.0052, -0.0037]],
[[-0.0007, -0.0007]]]])
num_kernels = [2]
kernels = K.constant(matrix, dtype='float32')
kernels = K.tile(kernels, [1, 1, image_shape[-1], 1])
# Use depth-wise convolution to calculate edge maps per channel.
# Output tensor has shape [batch_size, h, w, d * num_kernels].
pad_sizes = [[0, 0], [2, 2], [2, 2], [0, 0]]
padded = pad(image, pad_sizes, mode='REFLECT')
output = K.depthwise_conv2d(padded, kernels)
if not magnitude: # direction of edges
# Reshape to [batch_size, h, w, d, num_kernels].
shape = K.concatenate([image_shape, num_kernels], axis=0)
output = K.reshape(output, shape=shape)
output.set_shape(static_image_shape.concatenate(num_kernels))
output = tf.atan(
K.squeeze(output[:, :, :, :, 0] / output[:, :, :, :, 1],
axis=None))
# magnitude of edges -- unified x & y edges don't work well with Neural Networks
return output
def call(self, var_x, mask=None): # pylint:disable=unused-argument,arguments-differ
"""This is where the layer's logic lives.
Parameters
----------
inputs: tensor
Input tensor, or list/tuple of input tensors
kwargs: dict
Additional keyword arguments
Returns
-------
tensor
A tensor or list/tuple of tensors
"""
input_shape = self.input_spec[0].shape
in_width, in_height = input_shape[2], input_shape[1]
kernel_width, kernel_height = self.kernel_size, self.kernel_size
if (in_height % self.stride) == 0:
padding_height = max(kernel_height - self.stride, 0)
else:
padding_height = max(kernel_height - (in_height % self.stride), 0)
if (in_width % self.stride) == 0:
padding_width = max(kernel_width - self.stride, 0)
else:
padding_width = max(kernel_width - (in_width % self.stride), 0)
padding_top = padding_height // 2
padding_bot = padding_height - padding_top
padding_left = padding_width // 2
padding_right = padding_width - padding_left
return pad(var_x, [[0, 0], [padding_top, padding_bot],
[padding_left, padding_right], [0, 0]], 'REFLECT')
def _shrink_images(
cls, images: List[plaidml.tile.Value]) -> List[plaidml.tile.Value]:
""" Reduce the dimensional space of a batch of images in half. If the images are an odd
number of pixels then pad them to an even dimension prior to shrinking
All incoming images are assumed square.
Parameters
----------
images: list
The y_true, y_pred batch of images to be shrunk
Returns
-------
list
The y_true, y_pred batch shrunk by half
"""
if any(x % 2 != 0 for x in K.int_shape(images[1])[1:2]):
images = [
pad(img, [[0, 0], [0, 1], [0, 1], [0, 0]], mode="REFLECT")
for img in images
]
images = [
K.pool2d(img, (2, 2),
strides=(2, 2),
padding="valid",
pool_mode="avg") for img in images
]
return images
def _conv_gaussian(self, inputs: plaidml.tile.Value) -> plaidml.tile.Value:
""" Perform Gaussian convolution on a batch of images.
Parameters
----------
inputs: :class:`plaidml.tile.Value`
The input batch of images to perform Gaussian convolution on.
Returns
-------
:class:`plaidml.tile.Value`
The convolved images
"""
channels = self._shape[-1]
gauss = K.tile(self._gaussian_kernel, (1, 1, 1, channels))
# PlaidML doesn't implement replication padding like pytorch. This is an inefficient way to
# implement it for a square guassian kernel
size = K.int_shape(self._gaussian_kernel)[1] // 2
padded_inputs = inputs
for _ in range(size):
padded_inputs = pad(
padded_inputs, # noqa,pylint:disable=no-value-for-parameter,unexpected-keyword-arg
([0, 0], [1, 1], [1, 1], [0, 0]),
mode="REFLECT")
retval = K.conv2d(padded_inputs,
gauss,
strides=(1, 1),
padding="valid")
return retval
def _scharr_edges(
cls, image: plaidml.tile.Value, magnitude: bool,
image_shape: Tuple[None, int, int, int]) -> plaidml.tile.Value:
""" Returns a tensor holding modified Scharr edge maps.
Parameters
----------
image: :class:`plaidml.tile.Value`
Image tensor with shape [batch_size, h, w, d] and type float32. The image(s) must be
2x2 or larger.
magnitude: bool
Boolean to determine if the edge magnitude or edge direction is returned
image_shape: tuple
The shape of the incoming image
Returns
-------
:class:`plaidml.tile.Value`
Tensor holding edge maps for each channel. Returns a tensor with shape `[batch_size, h,
w, d, 2]` where the last two dimensions hold `[[dy[0], dx[0]], [dy[1], dx[1]], ...,
[dy[d-1], dx[d-1]]]` calculated using the Scharr filter.
"""
# Define vertical and horizontal Scharr filters.
# 5x5 modified Scharr kernel ( reshape to (5,5,1,2) )
matrix = np.array([[[[0.00070, 0.00070]], [[0.00520, 0.00370]],
[[0.03700, 0.00000]], [[0.00520, -0.0037]],
[[0.00070, -0.0007]]],
[[[0.00370, 0.00520]], [[0.11870, 0.11870]],
[[0.25890, 0.00000]], [[0.11870, -0.1187]],
[[0.00370, -0.0052]]],
[[[0.00000, 0.03700]], [[0.00000, 0.25890]],
[[0.00000, 0.00000]], [[0.00000, -0.2589]],
[[0.00000, -0.0370]]],
[[[-0.0037, 0.00520]], [[-0.1187, 0.11870]],
[[-0.2589, 0.00000]], [[-0.1187, -0.1187]],
[[-0.0037, -0.0052]]],
[[[-0.0007, 0.00070]], [[-0.0052, 0.00370]],
[[-0.0370, 0.00000]], [[-0.0052, -0.0037]],
[[-0.0007, -0.0007]]]])
# num_kernels = [2]
kernels = K.constant(matrix, dtype='float32')
kernels = K.tile(kernels, [1, 1, image_shape[-1], 1])
# Use depth-wise convolution to calculate edge maps per channel.
# Output tensor has shape [batch_size, h, w, d * num_kernels].
pad_sizes = [[0, 0], [2, 2], [2, 2], [0, 0]]
padded = pad(image, pad_sizes, mode='REFLECT')
output = K.depthwise_conv2d(padded, kernels)
# TODO magnitude not implemented for plaidml
if not magnitude: # direction of edges
raise FaceswapError(
"Magnitude for GMSD Loss is not implemented in PlaidML")
# # Reshape to [batch_size, h, w, d, num_kernels].
# shape = K.concatenate([image_shape, num_kernels], axis=0)
# output = K.reshape(output, shape=shape)
# output.set_shape(static_image_shape.concatenate(num_kernels))
# output = tf.atan(K.squeeze(output[:, :, :, :, 0] / output[:, :, :, :, 1], axis=None))
# magnitude of edges -- unified x & y edges don't work well with Neural Networks
return output
def call(self, x, mask=None):
input_shape = self.input_spec[0].shape
in_width, in_height = input_shape[2], input_shape[1]
kernel_width, kernel_height = self.kernel_size, self.kernel_size
if (in_height % self.stride) == 0:
padding_height = max(kernel_height - self.stride, 0)
else:
padding_height = max(kernel_height - (in_height % self.stride), 0)
if (in_width % self.stride) == 0:
padding_width = max(kernel_width - self.stride, 0)
else:
padding_width = max(kernel_width - (in_width % self.stride), 0)
padding_top = padding_height // 2
padding_bot = padding_height - padding_top
padding_left = padding_width // 2
padding_right = padding_width - padding_left
return pad(x, [[0, 0], [padding_top, padding_bot],
[padding_left, padding_right], [0, 0]], 'REFLECT')
def call(self, x, mask=None):
input_shape = self.input_spec[0].shape
in_width, in_height = input_shape[2], input_shape[1]
kernel_width, kernel_height = self.kernel_size, self.kernel_size
if (in_height % self.stride == 0):
padding_height = max(kernel_height - self.stride, 0)
else:
padding_height = max(kernel_height - (in_height % self.stride), 0)
if (in_width % self.stride == 0):
padding_width = max(kernel_width - self.stride, 0)
else:
padding_width = max(kernel_width- (in_width % self.stride), 0)
padding_top = padding_height // 2
padding_bot = padding_height - padding_top
padding_left = padding_width // 2
padding_right = padding_width - padding_left
return pad(x, [[0,0],
[padding_top, padding_bot],
[padding_left, padding_right],
[0,0] ],
'REFLECT')
