def cross_input_both(X):
tensor_left = X[0]
tensor_right = X[1]
x_length = K.int_shape(tensor_left)[1]
y_length = K.int_shape(tensor_left)[2]
cross_y_left = []
cross_x_left = []
cross_y_right = []
cross_x_right = []
scalar = K.ones([5,5])
tensor_left_padding = K.spatial_2d_padding(tensor_left,padding=(2,2))
tensor_right_padding = K.spatial_2d_padding(tensor_right,padding=(2,2))
for i_x in range(2, x_length + 2):
for i_y in range(2, y_length + 2):
cross_y_left.append(tensor_left_padding[:,i_x-2:i_x+3,i_y-2:i_y+3,:]
- concat_iterat(tensor_right_padding[:,i_x,i_y,:]))
cross_y_right.append(tensor_right_padding[:,i_x-2:i_x+3,i_y-2:i_y+3,:]
- concat_iterat(tensor_left_padding[:,i_x,i_y,:]))
cross_x_left.append(K.concatenate(cross_y_left, axis=2))
cross_x_right.append(K.concatenate(cross_y_right, axis=2))
cross_y_left = []
cross_y_right = []
cross_out_left = K.concatenate(cross_x_left,axis=1)
cross_out_right = K.concatenate(cross_x_right,axis=1)
cross_out = K.concatenate([cross_out_left, cross_out_right], axis=3)
return K.abs(cross_out)
def call(self, inputs, mask=None):
"""Define PConv2D layer's logic.
Args:
inputs: list=[K.tensor, K.tensor]
Returns: list=[K.tensor, K.tensor]
"""
# Both sound field and mask must be supplied
if type(inputs) is not list or len(inputs) != 2:
raise Exception('PartialConvolution2D must be called on a list of two tensors [sf, mask]. Instead got: ' + str(inputs))
# Padding done explicitly so that padding becomes part of the masked partial convolution
sfs = K.spatial_2d_padding(inputs[0], self.pconv_padding, self.data_format)
masks = K.spatial_2d_padding(inputs[1], self.pconv_padding, self.data_format)
# Apply convolutions to mask
mask_output = K.conv2d(
masks, self.kernel_mask,
strides=self.strides,
padding='valid',
data_format=self.data_format,
dilation_rate=self.dilation_rate
)
# Apply convolutions to sound field
sf_output = K.conv2d(
(sfs*masks), self.kernel,
strides=self.strides,
padding='valid',
data_format=self.data_format,
dilation_rate=self.dilation_rate
)
# Calculate the mask ratio on each psoition in the output mask
mask_ratio = self.window_size / (mask_output + 1e-8)
# Clip output to be between 0 and 1
mask_output = K.clip(mask_output, 0, 1)
# Remove ratio values where there are holes
mask_ratio = mask_ratio * mask_output
# Normalize sound field output
sf_output = sf_output * mask_ratio
# Apply bias only to the sound field (if chosen to do so)
if self.use_bias:
sf_output = K.bias_add(
sf_output,
self.bias,
data_format=self.data_format)
# Apply activations on the sound field
if self.activation is not None:
sf_output = self.activation(sf_output)
return [sf_output, mask_output]
def f(y_true, y_pred):
padding_pattern = ((kernal[0]//2, kernal[0]//2), (kernal[1]//2, kernal[1]//2))
y_true = K.spatial_2d_padding(y_true, padding_pattern)
y_pred = K.spatial_2d_padding(y_pred, padding_pattern)
kernal_x = kernal[0]
kernal_y = kernal[1]
stride_x, stride_y = strides
x = 0
for idx_x in range(0, height, stride_y):
print(idx_x)
y = 0
for idx_y in range(0, width, stride_x):
if K.image_data_format() == 'channels_first':
tmp=1-SSMI(y_true[:, :, idx_x:idx_x+kernal_x, idx_y:idx_y+kernal_y], y_pred[:, :, idx_x:idx_x+kernal_x, idx_y:idx_y+kernal_y])
else:
tmp=1-SSMI(y_true[:, idx_x:idx_x+kernal_x, idx_y:idx_y+kernal_y, :], y_pred[:, idx_x:idx_x+kernal_x, idx_y:idx_y+kernal_y, :])
try:
loss_ssmi=K.concatenate([loss_ssmi, K.expand_dims(tmp,0)],0)
except Exception as e:
loss_ssmi= K.expand_dims(tmp,0)
y += 1
x += 1
loss_ssmi=K.transpose(loss_ssmi)
true_loss_ssmi = loss_ssmi[:K.shape(y_true)[0]]
return true_loss_ssmi
def call(self, X, mask=None):
if self.data_format == 'channels_last':
b, r, c, ch = K.int_shape(X)
else:
b, ch, r, c = K.int_shape(X)
half = self.n // 2
square = K.square(X)
if self.data_format != 'channels_last':
extra_channels = K.spatial_2d_padding(square,
padding=((half, half), (0,
0)),
data_format='channels_last')
else:
extra_channels = K.spatial_2d_padding(
K.permute_dimensions(square, (0, 3, 1, 2)),
padding=((half, half), (0, 0)),
data_format='channels_last')
scale = self.k
for i in range(self.n):
scale += (self.alpha / self.n) * extra_channels[:,
i:(i + ch), :, :]
scale = scale**self.beta
if self.data_format == 'channels_last':
scale = K.permute_dimensions(scale, (0, 2, 3, 1))
return X / scale
def call(self, inputs, mask=None):
'''
We will be using the Keras conv2d method, and essentially we have
to do here is multiply the mask with the input X, before we apply the
convolutions. For the mask itself, we apply convolutions with all weights
set to 1.
Subsequently, we clip mask values to between 0 and 1
'''
# Both image and mask must be supplied
if type(inputs) is not list or len(inputs) != 2:
raise Exception(
'PartialConvolution2D must be called on a list of two tensors [img, mask]. Instead got: '
+ str(inputs))
# Padding done explicitly so that padding becomes part of the masked partial convolution
images = K.spatial_2d_padding(inputs[0], self.pconv_padding,
self.data_format)
masks = K.spatial_2d_padding(inputs[1], self.pconv_padding,
self.data_format)
# Apply convolutions to mask
mask_output = K.conv2d(masks,
self.kernel_mask,
strides=self.strides,
padding='valid',
data_format=self.data_format,
dilation_rate=self.dilation_rate)
# Apply convolutions to image
img_output = K.conv2d((images * masks),
self.kernel,
strides=self.strides,
padding='valid',
data_format=self.data_format,
dilation_rate=self.dilation_rate)
# Calculate the mask ratio on each pixel in the output mask
mask_ratio = self.window_size / (mask_output + 1e-8)
# Clip output to be between 0 and 1
mask_output = K.clip(mask_output, 0, 1)
# Remove ratio values where there are holes
mask_ratio = mask_ratio * mask_output
# Normalize iamge output
img_output = img_output * mask_ratio
# Apply bias only to the image (if chosen to do so)
if self.use_bias:
img_output = K.bias_add(img_output,
self.bias,
data_format=self.data_format)
# Apply activations on the image
if self.activation is not None:
img_output = self.activation(img_output)
return [img_output, mask_output]
def call(self, x, mask=None):
input_1, input_2 = x
input_shape = input_1._keras_shape
assert input_shape == input_2._keras_shape
self.H = input_shape[1]
self.W = input_shape[2]
self.C = input_shape[3]
padding1 = K.spatial_2d_padding(input_1,
padding=((self.k, self.k + 1),
(self.k, self.k + 1)),
data_format='channels_last')
padding2 = K.spatial_2d_padding(
input_2,
padding=(((self.D_stride - 1) / 2, (self.D_stride - 1) / 2 + 1),
((self.D_stride - 1) / 2, (self.D_stride - 1) / 2 + 1)),
data_format='channels_last')
# padding1&2: [nS, w, h, c]
out = tf.scan(self.single_sample_corr,
elems=[padding2, padding1],
initializer=(K.zeros(
(self.H / self.s1, self.W / self.s1, self.D**2))))
return out
def call(self, inputs):
''' This portion includes the actual implementation :
the image is elementally multiplied to the masks
which is fed to the network and later the masks are
updated on the basis of the output generated'''
assert isinstance(inputs, list)
images = inputs[0]
masks = inputs[1]
#padding layer
images = K.spatial_2d_padding(inputs[0], self.pconv_padding,
self.data_format)
masks = K.spatial_2d_padding(inputs[1], self.pconv_padding,
self.data_format)
X = images * masks
out = K.conv2d(X,
self.kernel,
strides=self.strides,
padding='valid',
data_format=self.data_format)
#################################################
def mask_update(masks):
''' mask update function that updates the mask
after a succesful Partial conv layer
'''
out = K.conv2d(masks,
self.kernel_mask,
strides=self.strides,
padding='valid',
data_format=self.data_format)
out = K.clip(out, 0, 1)
eps = 1e-6
mask_ratio = self.window_size / (out + eps)
mask_ratio *= out
return out, mask_ratio
#################################################
updated_mask, mask_ratio = mask_update(masks)
out = out * mask_ratio
if self.use_bias:
out = K.bias_add(out, self.bias, data_format=self.data_format)
if self.activation:
out = self.activation(out)
return [out, updated_mask]
def call(self, inputs, **kwargs):
"""
:param inputs:
:param kwargs:
:return:
"""
_, ch, r, c = K.int_shape(inputs)
# print("Call Fcn: Channel First Input shape ", K.int_shape(inputs))
# 1. Inputs Formatting
# --------------------
# Pad the rows and columns to allow full matrix multiplication
# Note that this function is aware of which dimension the columns and rows are
padded_inputs = K.spatial_2d_padding(inputs,
((self.n / 2, self.n / 2),
(self.n / 2, self.n / 2)))
# print("Call Fcn: padded_inputs shape ", K.int_shape(padded_inputs))
# Channel first, batch second. This is done to take the unknown batch size into the matrix
# multiply where it can be handled more easily
inputs_chan_first = K.permute_dimensions(padded_inputs, [1, 0, 2, 3])
# print("Call Fcn: inputs_chan_first shape: ", inputs_chan_first.shape)
# 2. Kernel Formatting
# --------------------
# Flatten rows and columns into a single dimension
k_ch, k_r, k_c = K.int_shape(self.kernel)
apply_kernel = K.reshape(self.kernel, (k_ch, k_r * k_c, 1))
# print("Call Fcn: kernel for matrix multiply: ", apply_kernel.shape)
# 3. Get outputs at each spatial location
# ---------------------------------------
xs = []
for i in range(r):
for j in range(c):
input_slice = inputs_chan_first[:, :, i:i + self.n,
j:j + self.n]
input_slice_apply = K.reshape(input_slice, (ch, -1, self.n**2))
output_slice = K.batch_dot(input_slice_apply, apply_kernel)
# Reshape the output slice to put batch first
output_slice = K.permute_dimensions(output_slice, [1, 0, 2])
xs.append(output_slice)
# print("Call Fcn: len of xs", len(xs))
# print("Call Fcn: shape of each element of xs", xs[0].shape)
# Reshape the output to correct format
outputs = K.concatenate(xs, axis=2)
outputs = K.reshape(outputs,
(-1, ch, r, c)) # Break into row and column
# 4. Add the lateral and the feed-forward activations
# ------------------------------------------------------
outputs = outputs * inputs + self.bias
outputs = self.activation(outputs)
return outputs + inputs
def call(self, inputs):
"""Performs the actual padding.
Parameters
----------
inputs : Tensor, rank 4
4D tensor with shape:
- If `data_format` is `"channels_last"`:
`(batch, rows, cols, channels)`
- If `data_format` is `"channels_first"`:
`(batch, channels, rows, cols)`
Returns
-------
outputs : Tensor, rank 4
4D tensor with shape:
- If `data_format` is `"channels_last"`:
`(batch, padded_rows, padded_cols, channels)`
- If `data_format` is `"channels_first"`:
`(batch, channels, padded_rows, padded_cols)`
"""
outputs = K.spatial_2d_padding(inputs,
padding=self.padding,
data_format=self.data_format)
p00, p01 = self.padding[0][0], self.padding[0][1]
p10, p11 = self.padding[1][0], self.padding[1][1]
if self.data_format == "channels_last":
row0 = K.concatenate([inputs[:, p00:0:-1, p10:0:-1, :],
inputs[:, p00:0:-1, :, :],
inputs[:, p00:0:-1, -2:-2-p11:-1, :]],
axis=2)
row1 = K.concatenate([inputs[:, :, p10:0:-1, :],
inputs,
inputs[:, :, -2:-2-p11:-1, :]],
axis=2)
row2 = K.concatenate([inputs[:, -2:-2-p01:-1, p10:0:-1, :],
inputs[:, -2:-2-p01:-1, :, :],
inputs[:, -2:-2-p01:-1, -2:-2-p11:-1, :]],
axis=2)
outputs = K.concatenate([row0, row1, row2], axis=1)
else: # self.data_format == "channels_first"
row0 = K.concatenate([inputs[:, :, p00:0:-1, p10:0:-1],
inputs[:, :, p00:0:-1, :],
inputs[:, :, p00:0:-1, -2:-2-p11:-1]],
axis=3)
row1 = K.concatenate([inputs[:, :, :, p10:0:-1],
inputs,
inputs[:, :, :, -2:-2-p11:-1]],
axis=3)
row2 = K.concatenate([inputs[:, :, -2:-2-p01:-1, p10:0:-1],
inputs[:, :, -2:-2-p01:-1, :],
inputs[:, :, -2:-2-p01:-1, -2:-2-p11:-1]],
axis=3)
outputs = K.concatenate([row0, row1, row2], axis=2)
return outputs
def __call__(self, inputs):
if self.mask:
masked = layers.Lambda(capsule.mask)(inputs)
masked = layers.Flatten()(masked)
elif K.ndim(inputs) > 2:
masked = layers.Flatten()(inputs)
else:
masked = inputs
if self.conv:
fc = layers.Dense(256, activation='relu')(masked)
reshape = layers.Reshape((8, 8, 4))(fc)
up1 = layers.Conv2DTranspose(256,
9,
strides=2,
activation='relu',
padding='valid')(reshape)
pad = layers.Lambda(
lambda x: K.spatial_2d_padding(x, ((0, 1), (0, 1))))(up1)
up2 = layers.Conv2DTranspose(256,
9,
activation='relu',
padding='valid')(pad)
outputs = layers.Conv2D(self.image_shape[-1],
1,
activation='sigmoid',
name='reconstruction')(up2)
else:
up1 = layers.Dense(512, activation='relu')(masked)
up2 = layers.Dense(1024, activation='relu')(up1)
outputs = layers.Dense(np.prod(self.image_shape),
activation='sigmoid')(up2)
outputs = layers.Reshape(self.image_shape,
name='reconstruction')(outputs)
return outputs
def blur_pool_func(x, filt_size=3, stride=2):
stride = (stride, stride)
padding = ((int(1. * (filt_size - 1) / 2),
int(np.ceil(1. * (filt_size - 1) / 2))),
(int(1. * (filt_size - 1) / 2),
int(np.ceil(1. * (filt_size - 1) / 2))))
if (filt_size == 1):
k = np.array([
1.,
])
elif (filt_size == 2):
k = np.array([1., 1.])
elif (filt_size == 3):
k = np.array([1., 2., 1.])
elif (filt_size == 4):
k = np.array([1., 3., 3., 1.])
elif (filt_size == 5):
k = np.array([1., 4., 6., 4., 1.])
elif (filt_size == 6):
k = np.array([1., 5., 10., 10., 5., 1.])
elif (filt_size == 7):
k = np.array([1., 6., 15., 20., 15., 6., 1.])
#k = a
k = k[:, None] * k[None, :]
k = k / np.sum(k)
k = np.tile(k[:, :, None, None], (1, 1, K.int_shape(x)[-1], 1))
k = K.constant(k, dtype=K.floatx())
x = K.spatial_2d_padding(x, padding=padding)
x = K.depthwise_conv2d(x, k, strides=stride, padding='valid')
return x
def get_keras_pad(cls, x, pads, dim, data_format=None):
"""
implement pad in conv or pool operator
:param x: input tensor
:param pads: pads attribute in conv or pool operator
:param dim: the pad dim
:param data_format: data format of x
:return: the result tensor of input tensor implementing padding operation
"""
if sum(pads) == 0:
return x
if len(pads) == dim * 2:
pads = list(
np.transpose(
np.array(pads).reshape([2, dim]).astype(np.int32)))
pads = tuple(tuple(i) for i in pads)
elif len(pads) == dim:
pads = tuple((i, i) for i in pads)
if dim == 1:
return Lambda(lambda _x: K.temporal_padding(_x, pads))(x)
elif dim == 2:
return Lambda(
lambda _x: K.spatial_2d_padding(_x, pads, data_format))(x)
elif dim == 3:
return Lambda(
lambda _x: K.spatial_3d_padding(_x, pads, data_format))(x)
else:
raise NotImplementedError(
"padding with dim {} is not implemented.".format(dim))
def _deconv(self, X, lname, d_switch, feat_map=None):
o_width, o_height = self[lname].output_shape[-2:]
# Get filter size
f_width = self[lname].W_shape[2]
f_height = self[lname].W_shape[3]
# Keras 2.0.
# f_width = self[lname].kernel_size[0]
# f_height = self[lname].kernel_size[1]
# Compute padding needed
i_width, i_height = X.shape[-2:]
pad_width = (o_width - i_width + f_width - 1) / 2
pad_height = (o_height - i_height + f_height - 1) / 2
pad_width = int(pad_width)
pad_height = int(pad_height)
assert isinstance(pad_width, int), "Pad width size issue at layer %s" % lname
assert isinstance(pad_height, int), "Pad height size issue at layer %s" % lname
# Set to zero based on switch values
X[d_switch[lname]] = 0
# Get activation function
activation = self[lname].activation
X = activation(X)
if feat_map is not None:
logger.debug("Setting other feat map to zero")
for i in range(X.shape[1]):
if i != feat_map:
X[:, i, :, :] = 0
logger.debug("Setting non max activations to zero")
for i in range(X.shape[0]):
iw, ih = np.unravel_index(X[i, feat_map, :, :].argmax(), X[i, feat_map, :, :].shape)
m = np.max(X[i, feat_map, :, :])
X[i, feat_map, :, :] = 0
X[i, feat_map, iw, ih] = m
# Get filters. No bias for now
W = self[lname].W
# W = self[lname].kernel # keras 2.0
# Transpose filter
W = W.transpose([1, 0, 2, 3])
W = W[:, :, ::-1, ::-1]
# CUDNN for conv2d ?
conv_out = K.T.nnet.conv2d(input=self.x, filters=W, border_mode='valid')
# Add padding to get correct size
pad = K.function([self.x], K.spatial_2d_padding(self.x, padding=(pad_width, pad_height), dim_ordering="th"))
# Keras 2.0 but not sure
# pad = K.function([self.x], K.spatial_2d_padding(
# self.x, padding=((pad_width, 0), (0, pad_height)), data_format="channels_first"))
X_pad = pad([X])
# Get Deconv output
deconv_func = K.function([self.x], conv_out)
X_deconv = deconv_func([X_pad])
assert X_deconv.shape[-2:] == (o_width, o_height), "Deconv output at %s has wrong size" % lname
return X_deconv
def _initialize_ii_buffer(self, x):
x_pad = K.spatial_2d_padding(
x, ((self.max_wh // 2 + 1, self.max_wh // 2 + 1),
(self.max_ww // 2 + 1, self.max_ww // 2 + 1)))
ii_x = K.cumsum(x_pad, axis=1)
ii_x2 = K.cumsum(ii_x, axis=2)
return ii_x2
def local_response_normalization(x, alpha=1e-4, k=2, beta=0.75, n=5):
"""
:param x:
:param alpha:
:param k:
:param beta:
:param n:
:return:
"""
if K.image_data_format() == 'channel_first':
b, ch, r, c = K.int_shape(x)
else:
b, r, c, ch = K.int_shape(x)
x = K.permute_dimensions(x, (0, 3, 1, 2))
square = K.square(x)
half = n // 2
extra_channels = K.spatial_2d_padding(square, ((half, half), (0, 0)))
scale = k
for i in range(n):
scale += alpha * extra_channels[:, i:i + ch, :, :]
scale**beta
output = x / scale
if K.image_data_format() == 'channels_last':
output = K.permute_dimensions(output, (0, 2, 3, 1))
return output
def inst_weight(output_y, output_x, output_dr, output_dl, config=None):
dy = output_y[:,2:,2:]-output_y[:, :-2,2:] + \
2*(output_y[:,2:,1:-1]- output_y[:,:-2,1:-1]) + \
output_y[:,2:,:-2]-output_y[:,:-2,:-2]
dx = output_x[:,2:,2:]- output_x[:,2:,:-2] + \
2*( output_x[:,1:-1,2:]- output_x[:,1:-1,:-2]) +\
output_x[:,:-2,2:]- output_x[:,:-2,:-2]
ddr= (output_dr[:,2:,2:]-output_dr[:,:-2,:-2] +\
output_dr[:,1:-1,2:]-output_dr[:,:-2,1:-1]+\
output_dr[:,2:,1:-1]-output_dr[:,1:-1,:-2])*K.constant(2)
ddl= (output_dl[:,2:,:-2]-output_dl[:,:-2,2:] +\
output_dl[:,2:,1:-1]-output_dl[:,1:-1,2:]+\
output_dl[:,1:-1,:-2]-output_dl[:,:-2,1:-1])*K.constant(2)
dpred = K.concatenate([dy,dx,ddr,ddl],axis=-1)
dpred = K.spatial_2d_padding(dpred)
weight_fg = K.cast(K.all(dpred>K.constant(config.GRADIENT_THRES), axis=3,
keepdims=True), K.floatx())
weight = K.clip(K.sqrt(weight_fg*K.prod(dpred, axis=3, keepdims=True)),
config.WEIGHT_AREA/config.CLIP_AREA_HIGH,
config.WEIGHT_AREA/config.CLIP_AREA_LOW)
weight +=(1-weight_fg)*config.WEIGHT_AREA/config.BG_AREA
weight = K.conv2d(weight, K.constant(config.GAUSSIAN_KERNEL),
padding='same')
return K.stop_gradient(weight)
def f(X):
b, ch, r, c = X.shape
half = n // 2
square = K.square(X)
# extra_channels = K.spatial_2d_padding(K.permute_dimensions(square, (0,2,3,1))
# , (0,half))
#extra_channels = K.permute_dimensions(extra_channels, (0,3,1,2))
extra_channels = K.spatial_2d_padding(
K.permute_dimensions(square, (0, 2, 3, 1)), ((0, 0), (half, half)))
extra_channels = K.spatial_2d_padding(
K.permute_dimensions(square, (0, 2, 3, 1)), ((0, 0), (0, n)))
scale = k
for i in range(n):
scale += alpha * extra_channels[:, i:i + ch, :, :]
scale = scale**beta
return X / scale
def cross_input_asym(X):
tensor_left = X[0]
tensor_right = X[1]
x_length = K.int_shape(tensor_left)[1]
y_length = K.int_shape(tensor_left)[2]
cross_y = []
cross_x = []
tensor_left_padding = K.spatial_2d_padding(tensor_left,padding=(2,2))
tensor_right_padding = K.spatial_2d_padding(tensor_right,padding=(2,2))
for i_x in range(2, x_length + 2):
for i_y in range(2, y_length + 2):
cross_y.append(tensor_left_padding[:,i_x-2:i_x+3,i_y-2:i_y+3,:]
- concat_iterat(tensor_right_padding[:,i_x,i_y,:]))
cross_x.append(K.concatenate(cross_y,axis=2))
cross_y = []
cross_out = K.concatenate(cross_x,axis=1)
return K.abs(cross_out)
def call(self, x):
k = self.a
k = k[:, None] * k[None, :]
k = k / K.sum(k)
k = K.tile(k[:, :, None, None], (1, 1, K.int_shape(x)[-1], 1))
k = K.constant(k, dtype=K.floatx())
x = K.spatial_2d_padding(x, padding=self.padding)
x = K.depthwise_conv2d(x, k, strides=self.strides, padding='valid')
return x
def _interpolate(im, x, y):
_edge_size = 0
if _wrap_mode == 'border':
_edge_size = 1
im = K.spatial_2d_padding(im, padding=((1, 1), (1, 1)))
x = x + _edge_size
y = y + _edge_size
elif _wrap_mode == 'edge':
_edge_size = 0
else:
return None
x = K.clip(x, 0.0, K.eval(_width_f) - 1 + 2 * _edge_size)
x0_f = K.round(x)
y0_f = K.round(y)
x1_f = x0_f + 1
x0 = K.cast(x0_f, 'int32')
y0 = K.cast(y0_f, 'int32')
x1 = K.cast(K.minimum(x1_f,
K.eval(_width_f) - 1 + 2 * _edge_size), 'int32')
dim2 = (_width + 2 * _edge_size)
dim1 = (_width + 2 * _edge_size) * (_height + 2 * _edge_size)
base = _repeat(K.arange(_num_batch) * dim1, _height * _width)
base_y0 = base + y0 * dim2
idx_l = base_y0 + x0
idx_r = base_y0 + x1
im_flat = K.reshape(im, K.stack([-1, _num_channels]))
pix_l = K.gather(im_flat, idx_l)
pix_r = K.gather(im_flat, idx_r)
weight_l = K.expand_dims(x1_f - x, 1)
weight_r = K.expand_dims(x - x0_f, 1)
return weight_l * pix_l + weight_r * pix_r
def f(X):
b, ch, r, c = X.shape.as_list()
half = n // 2
# Add some zero channels
extra_channels = K.spatial_2d_padding(K.square(X), ((half, half), (0, 0)), data_format='channels_last')
scale = k
for i in range(n):
scale += alpha * extra_channels[:, i:i + ch, :, :]
scale = scale ** beta
return X / scale
def f(X):
b, r, c, ch = X.shape
half = n // 2
square = K.square(X)
extra_channels = K.spatial_2d_padding(square, ((0, 0), (half, half)), data_format='channels_first')
scale = k
for i in range(n):
scale += alpha * extra_channels[:, :, :, i:i + int(ch)]
scale = scale ** beta
return X / scale
def _concat_features(lf, states):
b, f, h, w = lf.get_shape().as_list()
rf = states[0]
rfs = rf[:, :, :, :-1]
disp_rfs = K.spatial_2d_padding(rfs,
padding=((0, 0), (1, 0)),
data_format='channels_first')
concat = K.concatenate([lf, rf], axis=2)
output = K.reshape(concat, (-1, 2 * f, h, w))
return output, [disp_rfs]
def call(self, inputs):
"""
:param inputs:
:return:
"""
_, r, c, ch = K.int_shape(inputs)
# print("Call Fcn: Input shape ", r, c, ch)
# 1. Inputs Formatting
# Channel First, batch second. This is done to take the unknown batch size into the matrix multiply
# where it can be handled more easily
padded_inputs = K.spatial_2d_padding(inputs,
((self.n / 2, self.n / 2),
(self.n / 2, self.n / 2)))
inputs_chan_first = K.permute_dimensions(padded_inputs, [3, 0, 1, 2])
# print("Call Fcn: inputs_chan_first shape: ", inputs_chan_first.shape)
# 2. Kernel
kernel_chan_first = K.permute_dimensions(self.kernel, (2, 0, 1))
# print("Call Fcn: kernel_chan_first shape", kernel_chan_first.shape)
k_ch, k_r, k_c = K.int_shape(kernel_chan_first)
apply_kernel = K.reshape(kernel_chan_first, (k_ch, k_r * k_c, 1))
# print("Call Fcn: kernel for matrix multiply: ", apply_kernel.shape)
# 3. Get outputs at each spatial location
xs = []
for i in range(r):
for j in range(c):
input_slice = inputs_chan_first[:, :, i:i + self.n,
j:j + self.n]
input_slice_apply = K.reshape(input_slice, (ch, -1, self.n**2))
output_slice = K.batch_dot(input_slice_apply, apply_kernel)
# Reshape the output slice to put batch first
output_slice = K.permute_dimensions(output_slice, [1, 0, 2])
xs.append(output_slice)
# print("Call Fcn: len of xs", len(xs))
# print("Call Fcn: shape of each element of xs", xs[0].shape)
# 4. Reshape the output to correct format
outputs = K.concatenate(xs, axis=2)
outputs = K.reshape(outputs,
(-1, ch, r, c)) # Break into row and column
outputs = K.permute_dimensions(outputs,
[0, 2, 3, 1]) # Back to batch first
# print("Call Fcn: shape of output", outputs.shape)
# 5. Add the lateral and the feed-forward activations
outputs += inputs
return outputs
def f(X):
b, r, c, ch = X.shape
half = n // 2
square = keras.backend.square(X)
extra_channels = K.spatial_2d_padding(K.permute_dimensions(square, (0, 2, 3, 1)), ((0, 0), (half, half)))
extra_channels = K.permute_dimensions(extra_channels, (0, 3, 1, 2))
scale = k
for i in range(n):
scale += alpha * extra_channels[:, :, :, i:i + int(ch)]
scale = scale ** beta
return X / scale
def cut_and_paste(x):
background = x[0]
crop = x[1]
mask = x[2]
mask_to_paste = mask * crop
mask_to_paste = K.spatial_2d_padding(mask_to_paste,
padding=((PADDING*2, 0), (PADDING, PADDING)),
data_format='channels_last')
prep_mask = K.spatial_2d_padding(mask,
padding=((PADDING*2, 0), (PADDING, PADDING)),
data_format='channels_last')
inverted_mask = 1 - prep_mask
cp_img = (background * inverted_mask) + mask_to_paste
return cp_img
def f(X):
b, ch, r, c = X.shape # batch, channel, row, column
half = n // 2
square = K.square(X)
extra_channels = K.spatial_2d_padding(K.permute_dimensions(square, (0, 2, 3, 1)), (0, half))
extra_channels = K.permute_dimensions(extra_channels, (0, 3, 1, 2))
scale = k
for i in range(n):
scale += alpha * extra_channels[:, i:i + ch, :, :]
scale = scale ** beta
return X / scale
def CVLayer(inputs, D):
lfeatures = inputs[0]
rfeatures = inputs[1]
output = []
for disparity in np.arange(D):
rfs = rfeatures[:, :, :, disparity:]
disp_rfs = K.spatial_2d_padding(rfs,
padding=((0, 0), (0, disparity)),
data_format='channels_first')
output.append(K.concatenate([lfeatures, disp_rfs], axis=1))
return K.stack(output, axis=2)
def call(self, x):
if self.binarize_input_ == True:
x = Binarization(x)
self.binary_kernel = Binarization(K.clip(self.float_kernel, -1.0, 1.0))
x = K.spatial_2d_padding(x,
padding=((self.pad_, self.pad_), (self.pad_,
self.pad_)))
return K.conv2d(x,
self.binary_kernel,
strides=(self.stride_, self.stride_),
padding='valid',
data_format='channels_last')
def _getCostVolume_(inputs, max_d):
left_tensor, right_tensor = inputs
shape = K.shape(right_tensor) #(batch, height, width, channel)
right_tensor = K.spatial_2d_padding(right_tensor, padding=((0, 0), (max_d, 0)))
disparity_costs = []
for d in reversed(range(max_d)):
left_tensor_slice = left_tensor
right_tensor_slice = tf.slice(right_tensor, begin=[0, 0, d, 0], size=[-1, -1, shape[2], -1])
cost = K.concatenate([left_tensor_slice, right_tensor_slice], axis=3)
disparity_costs.append(cost)
cost_volume = K.stack(disparity_costs, axis=1) #(batch, D, height, width, 2*channel)
return cost_volume
def f(X):
if K.image_dim_ordering()=='tf':
b, r, c, ch = X.get_shape()
else:
b, ch, r, c = X.shape
half = n // 2
square = K.square(X)
scale = k
if K.image_dim_ordering() == 'th':
extra_channels = K.spatial_2d_padding(K.permute_dimensions(square, (0, 2, 3, 1)), (0, half))
extra_channels = K.permute_dimensions(extra_channels, (0, 3, 1, 2))
for i in range(n):
scale += alpha * extra_channels[:, i:i+ch, :, :]
if K.image_dim_ordering() == 'tf':
extra_channels = K.spatial_2d_padding(K.permute_dimensions(square, (0, 3, 1, 2)), (half, 0))
extra_channels = K.permute_dimensions(extra_channels, (0, 2, 3, 1))
for i in range(n):
scale += alpha * extra_channels[:, :, :, i:i+int(ch)]
scale = scale ** beta
return X / scale
def _deconv(self, X, lname, d_switch, feat_map=None):
o_width, o_height = self[lname].output_shape[-2:]
# Get filter size
f_width = self[lname].W_shape[2]
f_height = self[lname].W_shape[3]
# Compute padding needed
i_width, i_height = X.shape[-2:]
pad_width = (o_width - i_width + f_width - 1) / 2
pad_height = (o_height - i_height + f_height - 1) / 2
assert isinstance(
pad_width, int), "Pad width size issue at layer %s" % lname
assert isinstance(
pad_height, int), "Pad height size issue at layer %s" % lname
# Set to zero based on switch values
X[d_switch[lname]] = 0
# Get activation function
activation = self[lname].activation
X = activation(X)
if feat_map is not None:
print "Setting other feat map to zero"
for i in range(X.shape[1]):
if i != feat_map:
X[:, i, :, :] = 0
print "Setting non max activations to zero"
for i in range(X.shape[0]):
iw, ih = np.unravel_index(
X[i, feat_map, :, :].argmax(), X[i, feat_map, :, :].shape)
m = np.max(X[i, feat_map, :, :])
X[i, feat_map, :, :] = 0
X[i, feat_map, iw, ih] = m
# Get filters. No bias for now
W = self[lname].W
# Transpose filter
W = W.transpose([1, 0, 2, 3])
W = W[:, :, ::-1, ::-1]
# CUDNN for conv2d ?
conv_out = K.T.nnet.conv2d(
input=self.x, filters=W, border_mode='valid')
# Add padding to get correct size
pad = K.function([self.x], K.spatial_2d_padding(
self.x, padding=(pad_width, pad_height), dim_ordering="th"))
X_pad = pad([X])
# Get Deconv output
deconv_func = K.function([self.x], conv_out)
X_deconv = deconv_func([X_pad])
assert X_deconv.shape[-2:] == (o_width, o_height),\
"Deconv output at %s has wrong size" % lname
return X_deconv
def _deconv(self, X, lname, conv_input_shape = None, W = None):
if conv_input_shape is None:
o_width, o_height = self[lname].output_shape[-2:]
else:
o_width, o_height = self.input_shape_withoutpadding[lname][-2:]
if W is None:
# Get filters. No bias for now
W = self[lname].W
# Get filter size
if conv_input_shape is None:
f_width = self[lname].W_shape[2]
f_height = self[lname].W_shape[3]
else:
f_width = f_height = conv_input_shape[2]
# Compute padding needed
i_width, i_height = X.shape[-2:]
pad_width = (o_width - i_width + f_width - 1) / 2
pad_height = (o_height - i_height + f_height - 1) / 2
assert isinstance(pad_width, int), "Pad width size issue at layer %s" % lname
assert isinstance(pad_height, int), "Pad height size issue at layer %s" % lname
if self.lnames.index(lname) != len(self.lnames)-1:
# Get activation function, do not apply to last layer (softmax)
activation = self[lname].activation
X = activation(X)
# Transpose filter
W = W.transpose([1, 0, 2, 3])
W = W[:,:, ::-1, ::-1]
# CUDNN for conv2d ?
conv_out = K.T.nnet.conv2d(input=self.x, filters=W, border_mode='valid')
# Add padding to get correct size
pad = K.function([self.x,K.learning_phase()], K.spatial_2d_padding(
self.x, padding=(pad_width, pad_height), dim_ordering="th"))
X_pad = pad([X,1])
# Get Deconv output
deconv_func = K.function([self.x,K.learning_phase()], conv_out)
X_deconv = deconv_func([X_pad,1])
assert X_deconv.shape[-2:] == (o_width, o_height),\
"Deconv output at %s has wrong size" % lname
return X_deconv
