def _scharr_edges(cls, image, magnitude):
""" Returns a tensor holding modified Scharr edge maps.
Parameters
----------
image: tensor
Image tensor with shape [batch_size, h, w, d] and type float32. The image(s) must be
2x2 or larger.
magnitude: bool
Boolean to determine if the edge magnitude or edge direction is returned
Returns
-------
tensor
Tensor holding edge maps for each channel. Returns a tensor with shape `[batch_size, h,
w, d, 2]` where the last two dimensions hold `[[dy[0], dx[0]], [dy[1], dx[1]], ...,
[dy[d-1], dx[d-1]]]` calculated using the Scharr filter.
"""
# Define vertical and horizontal Scharr filters.
static_image_shape = image.get_shape()
image_shape = K.shape(image)
# 5x5 modified Scharr kernel ( reshape to (5,5,1,2) )
matrix = np.array([[[[0.00070, 0.00070]], [[0.00520, 0.00370]],
[[0.03700, 0.00000]], [[0.00520, -0.0037]],
[[0.00070, -0.0007]]],
[[[0.00370, 0.00520]], [[0.11870, 0.11870]],
[[0.25890, 0.00000]], [[0.11870, -0.1187]],
[[0.00370, -0.0052]]],
[[[0.00000, 0.03700]], [[0.00000, 0.25890]],
[[0.00000, 0.00000]], [[0.00000, -0.2589]],
[[0.00000, -0.0370]]],
[[[-0.0037, 0.00520]], [[-0.1187, 0.11870]],
[[-0.2589, 0.00000]], [[-0.1187, -0.1187]],
[[-0.0037, -0.0052]]],
[[[-0.0007, 0.00070]], [[-0.0052, 0.00370]],
[[-0.0370, 0.00000]], [[-0.0052, -0.0037]],
[[-0.0007, -0.0007]]]])
num_kernels = [2]
kernels = K.constant(matrix, dtype='float32')
kernels = K.tile(kernels, [1, 1, image_shape[-1], 1])
# Use depth-wise convolution to calculate edge maps per channel.
# Output tensor has shape [batch_size, h, w, d * num_kernels].
pad_sizes = [[0, 0], [2, 2], [2, 2], [0, 0]]
padded = tf.pad(image, pad_sizes, mode='REFLECT')
output = K.depthwise_conv2d(padded, kernels)
if not magnitude: # direction of edges
# Reshape to [batch_size, h, w, d, num_kernels].
shape = K.concatenate([image_shape, num_kernels], axis=0)
output = K.reshape(output, shape=shape)
output.set_shape(static_image_shape.concatenate(num_kernels))
output = tf.atan(
K.squeeze(output[:, :, :, :, 0] / output[:, :, :, :, 1],
axis=None))
# magnitude of edges -- unified x & y edges don't work well with Neural Networks
return output
def custom_loss(self, y_true, y_pred):
r1p = y_pred[0]
l1p = y_pred[1]
r2p = y_pred[2]
l2p = y_pred[3]
img1 = self.model.get_layer('input_1').output
img2 = self.model.get_layer('input_2').output
r1 = self.model.get_layer('reflectance_1').output
l1 = self.model.get_layer('illumination_1').output
r2 = self.model.get_layer('reflectance_2').output
l2 = self.model.get_layer('illumination_2').output
img1_pp = keras.layers.Multiply()([r1p, l2p])
img2_pp = keras.layers.Multiply()([r2p, l1p])
# reconstruction loss
reconstruction_loss = K.mean(K.square(img1 - img1_pp)) + K.mean(
K.square(img2 - img2_pp))
# consistency loss
consistency_loss = K.mean(K.square(l1 - l2p)) + K.mean(K.square(l2 - l1p)) + \
K.mean(K.square(r1 - r1p)) + K.mean(K.square(r2 - r2p))
# entropy - convert to greyscale, then find bins?
entropy_loss = self.getEntropy(r1) + self.getEntropy(r2)
# sobel filter
l1_sobelFilter = self.getSobelFilter(l1)
l2_sobelFilter = self.getSobelFilter(l2)
l1f = K.mean(K.depthwise_conv2d(l1, l1_sobelFilter))
l2f = K.mean(K.depthwise_conv2d(l2, l2_sobelFilter))
smoothness_loss = l1f + l2f
# weights = K.constant([0.7, 0.1, 0.1, 0.1], dtype=tf.float32)
# losses = K.variable([reconstruction_loss, consistency_loss, entropy_loss, smoothness_loss], dtype=tf.float32)
# total_loss = tf.tensordot(weights, losses, axes=1)
total_loss = 0.7 * reconstruction_loss + 0.1 * consistency_loss + 0.1 * entropy_loss + 0.1 * smoothness_loss
return total_loss
def call(self, x):
x = tf.nn.pool(x, (self.pool_size,), strides=(1,), padding='SAME', pooling_type='AVG',
data_format='NWC')
x = K.expand_dims(x, axis=-2)
x = K.depthwise_conv2d(x, self.blur_kernel, padding='same', strides=(self.pool_size, self.pool_size))
x = K.squeeze(x, axis=-2)
return x
def call(self, x, **kwargs):
if self.f.shape == (1, 1) and self.f[0, 0] == 1:
return x
else:
x = K.depthwise_conv2d(x,
self.filters,
strides=(self.stride, self.stride),
padding='same')
return x
def call(self, x):
k = self.a
k = k[:, None] * k[None, :]
k = k / K.sum(k)
k = K.tile(k[:, :, None, None], (1, 1, K.int_shape(x)[-1], 1))
k = K.constant(k, dtype=K.floatx())
x = K.spatial_2d_padding(x, padding=self.padding)
x = K.depthwise_conv2d(x, k, strides=self.strides, padding='valid')
return x
def call(self, x):
x = K.expand_dims(x, axis=-2)
x = K.depthwise_conv2d(x,
self.blur_kernel,
padding='same',
strides=(self.pool_size, self.pool_size))
x = K.squeeze(x, axis=-2)
return x
def call(self, x):
# x = tf.nn.pool(x, (self.pool_size, self.pool_size),
# strides=(1, 1), padding='SAME', pooling_type='MAX', data_format='NHWC')
x = K.depthwise_conv2d(x,
self.blur_kernel,
padding='same',
strides=self.pool_strides)
return x
def call(self, inputs, training=None):
outputs = K.depthwise_conv2d(inputs,
self.depthwise_kernel,
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate,
data_format=self.data_format)
if self.bias:
outputs = K.bias_add(outputs,
self.bias,
data_format=self.data_format)
if self.activation is not None:
return self.activation(outputs)
return outputs
def call(self, inp):
if self.stride == 1: # Even if the filter is >1, do not blur if there is no downsampling
return inp
if (self.filt_size == 1):
if (self.pad_off == 0):
x = inp
else:
x = self.pad(inp)
if K.image_data_format() == 'channels_first':
return x[:, :, ::self.stride, ::self.stride]
else:
return x[:, ::self.stride, ::self.stride, :]
else:
return K.depthwise_conv2d(self.pad(inp),
self.filt,
strides=(self.stride, self.stride))
def call(self, inp):
if self.stride==1: # Even if the filter is >1, do not blur if there is no downsampling
return inp
if(self.filt_size==1 or self.filt_size==0):
if(self.pad_off==0):
x = inp
else:
x = self.pad(inp)
if K.image_data_format()=='channels_first':
return x[:,:,::self.stride,::self.stride]
else:
return x[:,::self.stride,::self.stride,:]
else:
#print ("shape after pad: %s" % str(self.pad.compute_output_shape(inp.shape)))
return K.depthwise_conv2d(self.pad(inp), self.filt,
strides=(self.stride,self.stride))
def call(self, x, *kwargs):
kernel = K.tile(self.W, [1, 1, self.group_size * self.in_channels, 1])
print(x)
print(kernel.shape, K.shape(kernel))
act = K.depthwise_conv2d(x,
depthwise_kernel=kernel,
padding=self.padding,
data_format=self.data_format)
#act = K.conv2d(x,kernel,self.strides,padding=self.padding,data_format=self.data_format,dilation_rate=self.dilation_rate)
if self.use_bias:
bias_replicated = tf.tile(self.b, [1, self.group_size])
bias_reshaped = tf.reshape(bias_replicated,
[1, 1, 1, self.filters])
#act_reshaped = tf.reshape()
#act = tf.nn.bias_add(act,bias_replicated)
act = act + bias_reshaped
#act = K.tile(act,[1,1,1,self.group_size])
return act
def call(self, inputs, training=None):
outputs = K.depthwise_conv2d(
inputs,
self.depthwise_kernel,
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate,
data_format=self.data_format)
if self.bias:
outputs = K.bias_add(
outputs,
self.bias,
data_format=self.data_format)
if self.activation is not None:
return self.activation(outputs)
return outputs
def _depthwise_conv2d(cls, image: plaidml.tile.Value,
kernel: plaidml.tile.Value) -> plaidml.tile.Value:
""" Perform a standardized depthwise convolution.
Parameters
----------
image: :class:`plaidml.tile.Value`
Batch of images, channels last, to perform depthwise convolution
kernel: :class:`plaidml.tile.Value`
convolution kernel
Returns
-------
:class:`plaidml.tile.Value`
The output from the convolution
"""
return K.depthwise_conv2d(image,
kernel,
strides=(1, 1),
padding="valid")
def call(self, inputs, early_return=None):
if early_return is not None:
assert early_return in ['prod', 'bias']
if self.slalom_privacy:
inputs = K.cast(inputs, tf.float64)
outputs = K.depthwise_conv2d(inputs,
self.kernel_q,
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate,
data_format=self.data_format)
if self.log:
outputs = tf.Print(outputs, [
log2(tf.reduce_max(tf.abs(outputs))),
tf.reduce_mean(tf.abs(outputs)),
tf.greater_equal(log2(tf.reduce_max(tf.abs(outputs))),
np.log2(MID))
],
message="DepthConv log: ")
if early_return == 'prod':
# X .* W
return tf.cast(remainder(outputs, P), tf.float32)
if self.use_bias:
outputs = K.bias_add(outputs,
self.bias_q,
data_format=self.data_format)
if self.slalom_privacy:
outputs = tf.cast(remainder(outputs, P), tf.float32)
if early_return == 'bias':
# X .* W + b
return outputs
return outputs
def call(self,x):
depthwise_outputs=K.depthwise_conv2d(x,self.depthwise_kernel,
strides=self.strides,
padding=self.padding,)
if self.fist_layer:
x1=depthwise_outputs
else:
num=int(self.alph*K.int_shape(depthwise_outputs)[-1])
step=int(1/self.alph)
T=[]
for i in range(0,num,2):
a1=depthwise_outputs[:,:,:,(i*step):(i+1)*step]
a2=depthwise_outputs[:,:,:,((i+1)*step):(i+2)*step]
T.append(K.sum(a1,axis=-1,keepdims=True))
T.append(K.sum(a2,axis=-1,keepdims=True))
x1=K.concatenate(T)
x1=K.relu(x1)
outputs=K.conv2d(x1,self.kernel)
return outputs
def call(self, inputs):
c = inputs.get_shape().as_list()[-1]
kernel = np.ones((5, 5)) / 25.0
kernel = K.constant(kernel)
kernel = K.expand_dims(kernel, -1)
kernel = K.expand_dims(kernel, -1)
kernel = K.tile(kernel, (1, 1, c, 1))
outs = K.depthwise_conv2d(inputs,
kernel,
strides=(1, 1),
padding='same')
outs = inputs - outs
self.outs_size = outs.get_shape().as_list()
return outs
def call(self, inputs, training=None):
self.depthwise_kernel = self.tf_gaussian_filter(
kernel_shape=self.kernel_size,
input_dim=self.input_dim,
sigma=self.sigma_kernel,
depth_multiplier=self.depth_multiplier)
# print(self.depthwise_kernel)
outputs = K.depthwise_conv2d(inputs,
self.depthwise_kernel,
strides=self.strides,
padding=self.padding,
dilation_rate=self.dilation_rate,
data_format=self.data_format)
if self.use_bias:
outputs = K.bias_add(outputs,
self.bias,
data_format=self.data_format)
if self.activation is not None:
return self.activation(outputs)
return outputs
def __call__(self, x):
filt = self.expandedSobel(x)
sobel = K.depthwise_conv2d(x, filt)
return sobel
def sobelNorm(y):
filt = expandedSobel(y)
sobel = K.depthwise_conv2d(y, filt, padding='same')
return K.mean(K.square(sobel))
def call(self, x):
x = K.depthwise_conv2d(x, self.blur_kernel, padding='same', strides=(self.pool_size, self.pool_size))
return x
