def call(self, x, mask=None):
x = K.permute_dimensions(x, (0, 2, 1))
x = K.reshape(x, (-1, self.input_length))
x = K.expand_dims(x, 1)
x = K.expand_dims(x, -1)
if self.real_filts is not None:
conv_out_r = K.conv2d(x, self.W_r, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering='th')
else:
conv_out_r = x
if self.complex_filts is not None:
conv_out_c1 = K.conv2d(x, self.W_c1, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering='th')
conv_out_c2 = K.conv2d(x, self.W_c2, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering='th')
conv_out_c = K.sqrt(K.square(conv_out_c1) + K.square(conv_out_c2) + K.epsilon())
output = K.concatenate((conv_out_r, conv_out_c), axis=1)
else:
output = conv_out_r
output_shape = self.get_output_shape_for((None, self.input_length, self.input_dim))
output = K.squeeze(output, 3) # remove the dummy 3rd dimension
output = K.permute_dimensions(output, (2, 1, 0))
output = K.reshape(output, (-1, output_shape[1], output.shape[1]*output.shape[2]))
return output
def eigen_loss(y_true, y_pred):
y_true = tf.Print(y_true, [y_true], message='y_true', summarize=30)
y_pred = tf.Print(y_pred, [y_pred], message='y_pred', summarize=30)
y_true_clipped = K.clip(y_true, K.epsilon(), None)
y_pred_clipped = K.clip(y_pred, K.epsilon(), None)
first_log = K.log(y_pred_clipped + 1.)
second_log = K.log(y_true_clipped + 1.)
w_x = K.variable(np.array([[-1., 0., 1.],
[-1., 0., 1.],
[-1., 0., 1.]]).reshape(3, 3, 1, 1))
grad_x_pred = K.conv2d(first_log, w_x, padding='same')
grad_x_true = K.conv2d(second_log, w_x, padding='same')
w_y = K.variable(np.array([[-1., -1., -1.],
[0., 0., 0.],
[1., 1., 1.]]).reshape(3, 3, 1, 1))
grad_y_pred = K.conv2d(first_log, w_y, padding='same')
grad_y_true = K.conv2d(second_log, w_y, padding='same')
diff_x = grad_x_pred - grad_x_true
diff_y = grad_y_pred - grad_y_true
log_term = K.mean(K.square((first_log - second_log)), axis=-1)
sc_inv_term = K.square(K.mean((first_log - second_log),axis=-1))
grad_loss = K.mean(K.square(diff_x) + K.square(diff_y), axis=-1)
return log_term - (0.5 * sc_inv_term) + grad_loss
def get_output(self, train=False):
print "Input Shape", self.input_shape
print "ConvolutionDNASequenceBinding", self.output_shape
X = self.get_input(train)
if self.use_three_base_encoding:
X_fwd = X[:,1:,:,:]
X_rc = X[:,:3,:,:]
else:
X_fwd = X
X_rc = X
print self.W
print self.b
if self.W[1] is not None:
W = self.W[0][self.W[1],:,:,:]
else:
W = self.W[0]
if self.b[1] is not None:
b = self.b[0][self.b[1]]
else:
b = self.b[0]
fwd_rv = K.conv2d(X_fwd, W, border_mode='valid') \
+ K.reshape(b, (1, self.nb_motifs, 1, 1))
rc_rv = K.conv2d(X_rc, W[:,::-1,:,::-1], border_mode='valid') \
+ K.reshape(b, (1, self.nb_motifs, 1, 1))
rv = K.concatenate((fwd_rv, rc_rv), axis=2)
#return rv.dimshuffle((0,3,2,1))
return rv # K.permute_dimensions(rv, (0,3,2,1))
def get_output(self, train=False):
print "LogNormalizedOccupancy", self.output_shape
X = self.get_input(train)
# calculate the log occupancies
log_occs = theano_calc_log_occs(-X, self.chem_affinity)
# reshape the output so that the forward and reverse complement
# occupancies are viewed as different tracks
log_occs = K.reshape(log_occs, (X.shape[0], 1, 2*X.shape[1], X.shape[3]))
if self.steric_hindrance_win_len == 0:
log_norm_factor = 0
else:
# correct occupancies for overlapping binding sites
occs = K.exp(log_occs)
kernel = K.ones((1, 1, 1, 2*self.steric_hindrance_win_len-1), dtype='float32')
win_occ_sum = K.conv2d(occs, kernel, border_mode='same').sum(axis=2, keepdims=True)
win_prb_all_unbnd = TT.exp(
K.conv2d(K.log(1-occs), kernel, border_mode='same')).sum(axis=2, keepdims=True)
log_norm_factor = TT.log(win_occ_sum + win_prb_all_unbnd)
#start = max(0, self.steric_hindrance_win_len-1)
#stop = min(self.output_shape[3],
# self.output_shape[3]-(self.steric_hindrance_win_len-1))
#rv = log_occs[:,:,:,start:stop] - log_norm_factor
rv = (log_occs - log_norm_factor)
return K.reshape(
rv,
(X.shape[0], 2*X.shape[1], 1, X.shape[3])
)
def get_output(self, train=False):
X = self.get_input(train)
fwd_rv = K.conv2d(X[:,:,:,:6],
self.W,
border_mode='valid') \
+ K.reshape(self.b, (1, self.nb_motifs, 1, 1))
rc_rv = K.conv2d(X[:,:,:,-6:][:,::-1,::-1],
self.W,
border_mode='valid') \
+ K.reshape(self.b, (1, self.nb_motifs, 1, 1))
return K.concatenate((fwd_rv, rc_rv), axis=3)
def normals_metric(y_true, y_pred):
y_true = K.variable(y_true)
y_pred = K.variable(y_pred)
y_true = K.expand_dims(y_true,0)
filter_y = K.variable(np.array([[ 0., -0.5 , 0.],
[0., 0., 0.],
[0., 0.5, 0.]]).reshape(3, 3, 1, 1))
filter_x = K.variable(np.array([ [0, 0., 0.],
[0.5, 0., -0.5],
[0., 0., 0.]]).reshape(3, 3, 1, 1))
dzdx = K.conv2d(K.exp(y_true), filter_x, padding='same')
dzdy = K.conv2d(K.exp(y_true), filter_y, padding='same')
dzdx_ = dzdx * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdx))
dzdy_ = dzdy * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdy))
mag_norm = K.pow(dzdx,2) + K.pow(dzdy,2) + 1.0#K.constant(1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(1.0, shape=K.int_shape(dzdx))
mag_norm = K.sqrt(mag_norm)
N3 = 1.0 / mag_norm #K.constant(1.0, shape=K.int_shape(dzdx)) / mag_norm
N1 = dzdx_ / mag_norm
N2 = dzdy_ / mag_norm
normals = K.concatenate(tensors=[N1,N2,N3],axis=-1)
dzdx_pred = K.conv2d(K.exp(y_pred), filter_x, padding='same')
dzdy_pred = K.conv2d(K.exp(y_pred), filter_y, padding='same')
mag_norm_pred = K.pow(dzdx_pred,2) + K.pow(dzdy_pred,2) + 1.0
mag_norm_pred = K.sqrt(mag_norm_pred)
grad_x = K.concatenate(tensors=[1.0/ mag_norm_pred,
0.0/ mag_norm_pred, dzdx_pred/ mag_norm_pred],axis=-1)
grad_y = K.concatenate(tensors=[0.0/ mag_norm_pred,
1.0/ mag_norm_pred, dzdy_pred/ mag_norm_pred],axis=-1)
dot_term_x = K.mean(K.sum(normals[0,:,:,:] * grad_x[0,:,:,:], axis=-1, keepdims=True), axis=-1)
dot_term_y = K.mean(K.sum(normals[0,:,:,:] * grad_y[0,:,:,:], axis=-1, keepdims=True), axis=-1)
dot_term_x = K.abs(dot_term_x)
dot_term_y = K.abs(dot_term_y)
return K.eval(K.mean(dot_term_x)),K.eval(K.mean(dot_term_y))
def step(self, x, states):
h_tm1 = states[0]
c_tm1 = states[1]
x_i = K.conv2d(x, self.W_i, border_mode="same")
x_f = K.conv2d(x, self.W_f, border_mode="same")
x_c = K.conv2d(x, self.W_c, border_mode="same")
x_o = K.conv2d(x, self.W_o, border_mode="same")
h_i = K.conv2d(h_tm1, self.U_i, border_mode="same")
h_f = K.conv2d(h_tm1, self.U_f, border_mode="same")
h_c = K.conv2d(h_tm1, self.U_c, border_mode="same")
h_o = K.conv2d(h_tm1, self.U_o, border_mode="same")
c_i = self.C_i * c_tm1
c_f = self.C_f * c_tm1
c_o = self.C_o * c_tm1
b_i = K.reshape(self.b_i, (1, -1, 1, 1))
b_f = K.reshape(self.b_f, (1, -1, 1, 1))
b_c = K.reshape(self.b_c, (1, -1, 1, 1))
b_o = K.reshape(self.b_o, (1, -1, 1, 1))
i = self.inner_activation(x_i + h_i + c_i + b_i)
f = self.inner_activation(x_f + h_f + c_f + b_f)
c = f * c_tm1 + i * self.activation(x_c + h_c + b_c)
o = self.inner_activation(x_o + h_o + c_o + b_o)
h = o * self.activation(c)
return h, [h, c]
def call(self, inputs):
if self.rank == 1:
outputs = K.conv1d(
inputs,
self.kernel,
strides=self.strides[0],
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate[0])
if self.rank == 2:
outputs = K.conv2d(
inputs,
self.kernel,
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
if self.rank == 3:
outputs = K.conv3d(
inputs,
self.kernel,
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
if self.use_bias:
outputs = K.bias_add(
outputs,
self.bias,
data_format=self.data_format)
if self.activation is not None:
return self.activation(outputs)
return outputs
def deconv2d_tied(x, kernel, strides=(1, 1), border_mode='valid', dim_ordering='th',
image_shape=None, filter_shape=None):
'''
Run on cuDNN if available.
border_mode: string, "same" or "valid".
'''
if dim_ordering not in {'th', 'tf'}:
raise Exception('Unknown dim_ordering ' + str(dim_ordering))
###
if dim_ordering == 'default':
dim_ordering = image_dim_ordering()
if dim_ordering not in {'th', 'tf'}:
raise ValueError('Unknown dim_ordering ', dim_ordering)
x = K._preprocess_conv2d_input(x, dim_ordering)
kernel = K._preprocess_conv2d_kernel(kernel, dim_ordering)
th_border_mode = K._preprocess_border_mode(border_mode)
np_kernel = kernel.eval()
image_shape = K._preprocess_conv2d_image_shape(dim_ordering, image_shape)
filter_shape = K._preprocess_conv2d_filter_shape(dim_ordering, filter_shape)
conv_out = K.conv2d(x, kernel,
border_mode=th_border_mode,
subsample=strides,
filter_flip=False, # <<<<< IMPORTANT 111, dont flip kern
input_shape=image_shape,
filter_shape=filter_shape)
conv_out = K._postprocess_conv2d_output(conv_out, x, border_mode, np_kernel,
strides, dim_ordering)
return conv_out
def call(self, x, mask=None):
x = K.permute_dimensions(x, (0, 2, 1))
x = K.expand_dims(x, -1)
output = K.square(K.permute_dimensions(K.squeeze(K.conv2d(x, self.kernel), -1), (0, 2, 1)))
return output
def call(self, x, mask=None):
conv_out = K.conv2d(x, self.W, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering=self.dim_ordering,
filter_shape=self.W_shape)
output = self.activation(conv_out)
return output
def call(self, inputs):
filter_in_group = self.filters / self.num_group
if self.data_format == 'channels_first':
channel_axis = 1
input_in_group = self.channel_num / self.num_group
outputs_list = []
for i in range(self.num_group):
outputs = K.conv2d(
inputs[:,i*input_in_group:(i+1)*input_in_group,:,:],
self.kernel[:, :, :, i*filter_in_group:(i+1)*filter_in_group],
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
if self.use_bias:
outputs = K.bias_add(
outputs,
self.bias[i*filter_in_group:(i+1)*filter_in_group],
data_format=self.data_format)
outputs_list.append(outputs)
elif self.data_format == 'channels_last':
outputs_list = []
channel_axis = -1
input_in_group = self.channel_num / self.num_group
for i in range(self.num_group):
outputs = K.conv2d(
inputs[:, :, :, i*input_in_group:(i+1)*input_in_group],
self.kernel[:, :, :, i*filter_in_group:(i+1)*filter_in_group],
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
if self.use_bias:
outputs = K.bias_add(
outputs,
self.bias[i*filter_in_group:(i+1)*filter_in_group],
data_format=self.data_format)
outputs_list.append(outputs)
outputs = concatenate(outputs_list, axis=channel_axis)
return outputs
def gen_conv(x_slice):
x_slice = K.expand_dims(x_slice,
1) # shape (num_batches, 1, input_length)
x_slice = K.expand_dims(
x_slice, 2) # shape (num_batches, 1, 1, input_length)
return K.conv2d(x_slice,
wav_kernels,
strides=self.subsample,
padding=self.padding,
data_format='channels_first')
def convolve_tensor(x, kernel_tensor=None):
'''
conv2d
input tensor: [batch, in_height, in_width, in_channels]
kernel tensor: [filter_height, filter_width, in_channels, out_channels]
'''
# x = tf.image.rgb_to_grayscale(x)
print(x.shape)
print(kernel_tensor.shape)
return K.conv2d(x, kernel_tensor, padding='same')
def _loss(y_true, y_pred):
kh, kv = get_sobel_kernel(size) # (5,5)
kh, kv = np.expand_dims(kh, -1), np.expand_dims(kv, -1)
kernel = np.concatenate([kh, kv],
axis=-1)[...,
None] # (H, W, in_depth, out_depth)
kernel = K.constant(kernel, dtype=tf.float32)
gt_mask = K.cast(K.greater_equal(y_true[..., 0], K.epsilon()),
tf.float32)
M = K.sum(gt_mask)
# approximate to h/v grad and sum together
gt_grad = K.conv2d(y_true, kernel, strides=(1, 1),
padding='same') # (N,H,W,1)
pr_grad = K.conv2d(y_pred, kernel, strides=(1, 1), padding='same')
gmse_loss = K.square(gt_grad - pr_grad)[..., 0] * gt_mask
return K.sum(gmse_loss) / M
def input_conv(self, x, w, b=None, padding='valid'):
conv_out = K.conv2d(x,
w,
strides=self.strides,
padding=padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
if b is not None:
conv_out = K.bias_add(conv_out, b, data_format=self.data_format)
return conv_out
def __call__(self, image: plaidml.tile.Value,
feature_type: str) -> plaidml.tile.Value:
""" Run the feature detection
Parameters
----------
image: Tensor
Batch of images in YCxCz color space with normalized Y values
feature_type: str
Type of features to detect (`"edge"` or `"point"`)
Returns
-------
Tensor
Detected features in the 0-1 range
"""
feature_type = feature_type.lower()
if feature_type == 'edge':
grad_x = np.multiply(-self._grid[0], self._gradient)
else:
grad_x = np.multiply(self._grid[0]**2 / (self._std**2) - 1,
self._gradient)
negative_weights_sum = -np.sum(grad_x[grad_x < 0])
positive_weights_sum = np.sum(grad_x[grad_x > 0])
grad_x = K.constant(grad_x)
grad_x = K.switch(grad_x < 0, grad_x / negative_weights_sum,
grad_x / positive_weights_sum)
kernel = K.expand_dims(K.expand_dims(grad_x, axis=-1), axis=-1)
features_x = K.conv2d(replicate_pad(image, self._radius),
kernel,
strides=(1, 1),
padding="valid")
kernel = K.permute_dimensions(kernel, (1, 0, 2, 3))
features_y = K.conv2d(replicate_pad(image, self._radius),
kernel,
strides=(1, 1),
padding="valid")
features = K.concatenate([features_x, features_y], axis=-1)
return features
def get_output(self, train=False):
print "ConvolutionBindingSubDomains", self.output_shape
X = self.get_input(train)
if self.W[1] is not None:
W = self.W[0][self.W[1], :, :, :]
else:
W = self.W[0]
if self.b[1] is not None:
b = self.b[0][self.b[1]]
else:
b = self.b[0]
fwd_rv = K.conv2d(X[:,:,0:1,:], W, border_mode='valid') \
+ K.reshape(b, (1, self.nb_domains, 1, 1))
# # [:,:,::-1,::-1]
rc_rv = K.conv2d(X[:,:,1:2,:], W[:,:,:,::-1], border_mode='valid') \
+ K.reshape(b, (1, self.nb_domains, 1, 1))
rv = K.concatenate((fwd_rv, rc_rv), axis=2)
#return rv.dimshuffle((0,3,2,1))
return rv #K.permute_dimensions(rv, (0,3,2,1))
def call(self, inputs):
multihead_attended = inputs
o = K.conv2d(multihead_attended,
kernel=self.kernel_o,
strides=(1, 1),
padding='same')
o = K.bias_add(o, self.bias_o)
o = kl.Activation(activation)(o)
self.o_sh = tuple(o.shape.as_list())
return o
def get_grad_tensor(img_tensor, apply_gauss=True):
grad_x = K.conv2d(img_tensor, SOBEL_X, padding='same')
grad_y = K.conv2d(img_tensor, SOBEL_Y, padding='same')
grad_tensor = K.sqrt(grad_x * grad_y + grad_y * grad_y)
grad_tensor = K.greater(grad_tensor, 100.0 * K.epsilon())
grad_tensor = K.cast(grad_tensor, K.floatx())
grad_tensor = K.clip(grad_tensor, K.epsilon(), 1.0)
grad_map = K.sum(grad_tensor, axis=CHANNEL_AXIS, keepdims=True)
grad_map = [grad_map, grad_map]
grad_tensor = K.concatenate(grad_map, axis=CHANNEL_AXIS)
del grad_map
grad_tensor = K.concatenate([grad_tensor, grad_tensor], axis=CHANNEL_AXIS)
grad_tensor = K.greater(grad_tensor, 100.0 * K.epsilon())
grad_tensor = K.cast(grad_tensor, K.floatx())
print("K.floatx", K.floatx())
if apply_gauss:
grad_tensor = K.conv2d(grad_tensor, GAUSS_KERNEL, padding='same')
return grad_tensor
def call(self, inputs):
outputs = K.conv2d(inputs,
self.kernel,
strides=self.strides,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
outputs = K.bias_add(outputs, self.bias, data_format=self.data_format)
return self.activation(outputs)
def call(self, x, mask=None):
b, xb = 0., 0.
if self.data_format == 'channels_first':
kernel_sum_axes = [1, 2, 3]
if self.use_bias:
b = K.reshape(self.b, (self.filters, 1, 1, 1))
xb = 1.
elif self.data_format == 'channels_last':
kernel_sum_axes = [0, 1, 2]
if self.use_bias:
b = K.reshape(self.b, (1, 1, 1, self.filters))
xb = 1.
Wnorm = K.sqrt(K.sum(K.square(self.W), axis=kernel_sum_axes, keepdims=True) + K.square(b) + K.epsilon())
xnorm = K.sqrt(K.conv2d(K.square(x), self.kernel_norm, strides=self.strides,
padding=self.padding,
data_format=self.data_format,
filter_shape=self.kernel_norm_shape) + xb + K.epsilon())
W = self.W / Wnorm
output = K.conv2d(x, W, strides=self.strides,
padding=self.padding,
data_format=self.data_format,
filter_shape=self.kernel_shape)
if K.backend() == 'theano':
xnorm = K.pattern_broadcast(xnorm, [False, True, False, False])
output /= xnorm
if self.use_bias:
b /= Wnorm
if self.data_format == 'channels_first':
b = K.reshape(b, (1, self.filters, 1, 1))
elif self.data_format == 'channels_last':
b = K.reshape(b, (1, 1, 1, self.filters))
else:
raise ValueError('Invalid data_format:', self.data_format)
b /= xnorm
output += b
output = self.activation(output)
return output
def _computeSoftArgMin_(cv, d):
softmax = tf.nn.softmax(cv, dim=1)
#softmax = K.permute_dimensions(softmax, (0,2,3,1))
disp_map = K.reshape(K.arange(0, d, dtype='float32'), (1, 1, d, 1))
output = K.conv2d(softmax,
disp_map,
strides=(1, 1),
data_format='channels_first',
padding='valid')
return K.squeeze(output, axis=1)
def get_output(self, train=False):
print "ConvolutionBindingSubDomains", self.output_shape
X = self.get_input(train)
if self.W[1] is not None:
W = self.W[0][self.W[1],:,:,:]
else:
W = self.W[0]
if self.b[1] is not None:
b = self.b[0][self.b[1]]
else:
b = self.b[0]
fwd_rv = K.conv2d(X[:,:,0:1,:], W, border_mode='valid') \
+ K.reshape(b, (1, self.nb_domains, 1, 1))
# # [:,:,::-1,::-1]
rc_rv = K.conv2d(X[:,:,1:2,:], W[:,:,:,::-1], border_mode='valid') \
+ K.reshape(b, (1, self.nb_domains, 1, 1))
rv = K.concatenate((fwd_rv, rc_rv), axis=2)
#return rv.dimshuffle((0,3,2,1))
return rv #K.permute_dimensions(rv, (0,3,2,1))
def call(self, inputs):
assert isinstance(inputs, list) and len(inputs) == 2
#img, mask = inputs
# Masked convolution:
img_output = K.conv2d(inputs[0] * inputs[1],
self.kernel,
strides=self.strides,
padding=self.padding,
data_format=self.data_format)
# Image scaling:
sum_m = K.conv2d(inputs[1],
self.kernel_mask,
strides=self.strides,
padding=self.padding,
data_format=self.data_format)
# Note, sum_1 does not need to be created via conv2d (as sum_m), it can be generated straight away:
sum_1i = self.kernel_size[0] * self.kernel_size[1] * self.input_dim
sum_1 = sum_1i * K.ones(K.shape(sum_m))
# Prevent division by zero:
sum_m_clip = K.clip(sum_m, 1., None)
# Scale the image:
img_output = img_output * (sum_1 / sum_m_clip)
# Apply bias only to the image (if chosen to do so):
if self.use_bias:
img_output = K.bias_add(img_output,
self.bias,
data_format=self.data_format)
# Apply activation if needed. Note, in the paper, activation is applied after BatchNormalization.
if self.activation is not None:
img_output = self.activation(img_output)
if self.last_layer:
return img_output
# Update the mask:
mask_output = K.clip(sum_m, 0., 1.)
return [img_output, mask_output]
def func(input_):
inputs = [
input_[:, :, :, i:i + 1] for i in range(K.int_shape(input_)[-1])
]
outputs = [
K.conv2d(inp,
K.constant(gauss_kernel),
strides=(1, 1),
padding="same") for inp in inputs
]
return K.concatenate(outputs, axis=-1)
def call(self, inputs): bs = K.shape(inputs)[0:1] shape1 = K.variable(np.array([self.n, self.filters]), dtype='int32') shape2 = K.variable(np.array([self.n, self.channels]), dtype='int32') shape1 = K.concatenate([bs, shape1]) shape2 = K.concatenate([bs, shape2]) f = K.conv2d(inputs, kernel = self.Wf, data_format = data_format) g = K.conv2d(inputs, kernel = self.Wg, data_format = data_format) h = K.conv2d(inputs, kernel = self.Wh, data_format = data_format) ff = K.reshape(f, shape1) gf = K.reshape(g, shape1) hf = K.reshape(h,shape2)#bs × n x c s = K.batch_dot(ff, gf, axes=(2,2))#bs x n x n beta = K.softmax(s) o = K.batch_dot(beta, hf, axes=(2,1))#bs x n x c o = K.reshape(o, K.shape(inputs)) o = K.conv2d(o, kernel = self.Wv, data_format = data_format) y = self.gamma * o + inputs return y
def loss_tv(self, mask, y_comp):
kernel = K.ones(shape=(3, 3, mask.shape[3], mask.shape[3]))
dilated_mask = K.conv2d(1-mask, kernel, data_format='channels_last', padding='same')
dilated_mask = K.cast(K.greater(dilated_mask, 0), 'float32')
P = dilated_mask * y_comp
a = self.l1(P[:,1:,:,:], P[:,:-1,:,:])
b = self.l1(P[:,:,1:,:], P[:,:,:-1,:])
return a+b
def call(self, inputs): # pylint: disable=arguments-differ
pin = K.permute_dimensions(inputs, (0, 1, 3, 2))
avg_conv = K.conv2d(pin,
self.kernel,
strides=(1, 1),
padding="valid",
data_format='channels_last',
dilation_rate=(1, 1))
output = K.permute_dimensions(avg_conv, (0, 1, 3, 2))
return output
def local_mean_subtraction(self, X, kernel_size=5):
filter_shape = (1, 1, kernel_size, kernel_size)
filters = self.mean_filter(kernel_size).reshape(filter_shape)
filters = K.variable(filters)
mean = K.conv2d(X,
filters,
filter_shape=filter_shape,
border_mode='same')
return X - mean
def call(self, inputs, mask=None):
#print(inputs * inputs)
one_kernel = K.ones((self.kernelSize[0], self.kernelSize[0],
self.nInputPlane, self.nOutputPlane))
inputs_norm = K.conv2d(inputs * inputs,
one_kernel,
strides=self.strides,
padding=self.padding)
inputs_norm = K.sqrt(inputs_norm)
#print(inputs_norm)
conv = K.conv2d(inputs,
self.kernelWeights,
strides=self.strides,
padding=self.padding)
#print("+++", conv / ( inputs_norm * K.sqrt(K.sum(self.kernelWeights*self.kernelWeights))))
#print(K.sqrt(K.sum(self.kernelWeights*self.kernelWeights)))
g = conv / (inputs_norm *
K.sqrt(K.sum(self.kernelWeights * self.kernelWeights)))
h = self.alpha * inputs_norm
return h * g
def testDilatedConv2D(self):
I = K.variable(m(2, 6, 10, 3))
kernel = K.variable(m(3, 2, 3, 2))
code = """function (I[N, Lx, Ly, CI], K[LKx, LKy, CI, CO]) -> (O) {
O[n, x, y, co: N, Lx - 2 * (LKx - 1), Ly - 3 * (LKy - 1), CO] =
+(I[n, x + 2 * kx, y + 3 * ky, ci] * K[kx, ky, ci, co]);
}"""
op = K._Op('cumulative_sum', I.dtype, (2, 2, 7, 2), code,
OrderedDict([('I', I), ('K', kernel)]), ['O'])
reference = K.conv2d(I, kernel, padding='valid', dilation_rate=(2, 3))
npt.assert_allclose(op.eval(), reference.eval())
def call(self, inputs):
if self.rank == 1:
expert_outputs = K.conv1d(inputs,
self.expert_kernel,
strides=self.strides[0],
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate[0])
if self.rank == 2:
expert_outputs = K.conv2d(inputs,
self.expert_kernel,
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
if self.rank == 3:
expert_outputs = K.conv3d(inputs,
self.expert_kernel,
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
expert_outputs = K.reshape(expert_outputs,
(-1, ) + self.o_shape[1:-1] +
(self.n_filters, self.n_experts_per_filter))
if self.use_expert_bias:
expert_outputs = K.bias_add(expert_outputs,
self.expert_bias,
data_format=self.data_format)
if self.expert_activation is not None:
expert_outputs = self.expert_activation(expert_outputs)
gating_outputs = tf.tensordot(
inputs, self.gating_kernel,
axes=self.rank + 1) # samples x n_filters x n_experts_per_filter
if self.use_gating_bias:
gating_outputs = K.bias_add(gating_outputs,
self.gating_bias,
data_format=self.data_format)
if self.gating_activation is not None:
gating_outputs = self.gating_activation(gating_outputs)
gating_outputs = K.reshape(gating_outputs,
self.new_gating_outputs_shape)
outputs = K.sum(expert_outputs * gating_outputs,
axis=-1,
keepdims=False)
return outputs
def call(self, inputs):
output = K.conv2d(inputs,
self.kernel * self.scale,
padding=self.padding)
if self.use_bias:
output = K.bias_add(output, self.bias)
if self.activation is not tf.keras.activations.linear:
output = self.activation(output)
elif self.lrelu:
output = LeakyReLU(alpha=0.2)(output)
return output
def get_initial_states(self, x):
initial_state = K.sum(x, axis=1)
print('what is this: ',
(self.nb_filters_out, self.nb_filters_in, 1, 1))
initial_state = K.conv2d(initial_state,
K.zeros((self.nb_filters_out,
self.nb_filters_in, 1, 1)),
padding='same')
initial_states = [initial_state for _ in range(len(self.states))]
return initial_states
def call(self, inputs):
shape = K.shape(inputs)
if self.keep_dims:
y = K.conv2d(inputs, self.dct_kernel)
else:
y = K.conv2d_transpose(inputs,
self.dct_kernel,
output_shape=(shape[0], shape[1] + 7,
shape[2] + 7, 1))
y = y * self.scale_kernel
return y
def call(self, inputs, **kwargs):
tower1 = K.conv2d(x=inputs,
kernel=self.K1,
strides=(1, 1),
padding='same',
dilation_rate=self.dilation_rate
)
tower1 = K.conv2d(x=tower1,
kernel=self.K2,
strides=(1, 1),
padding='same',
dilation_rate=self.dilation_rate
)
tower2 = K.conv2d(x=inputs,
kernel=self.K3,
strides=(1, 1),
padding='same',
dilation_rate=self.dilation_rate
)
tower3 = MaxPool2D(
pool_size=(2*self.dilation_rate[0],2*self.dilation_rate[1]),
padding='same',
strides=(1,1),
)(inputs)
tower3 = K.conv2d(x=tower3,
kernel=self.K4,
strides=(1,1),
padding='same',
dilation_rate=(1, 1)
)
output = K.concatenate((tower1,tower2,tower3))
output = K.conv2d(x=output,
kernel=self.K5,
strides=(1,1),
padding='same',
dilation_rate=(1,1)
)
if self.use_softmax:
return K.softmax(output)
else:
return K.relu(output)
def call(self, inputs, **kwargs):
def sqrt(x):
return K.sqrt(K.maximum(x, K.epsilon()))
# Both components and input given
if isinstance(inputs, list) and len(inputs) > 1:
signals, kernel = inputs
else:
signals = inputs
kernel = self.components.astype(K.floatx())
# move component_number to channel dimension
kernel = K.permute_dimensions(kernel, (1, 2, 3, 0))
# normalize kernel
normed_kernel = kernel / sqrt(
K.sum(K.square(kernel), (0, 1, 2), keepdims=True))
# get norm of signals
signals_norm = sqrt(
K.conv2d(K.square(signals),
np.ones(K.int_shape(kernel)[:3] + (1, ),
dtype=K.floatx()),
strides=self.strides,
padding=self.padding,
data_format='channels_last',
dilation_rate=self.dilation_rate))
diss = K.conv2d(signals,
normed_kernel,
strides=self.strides,
padding=self.padding,
data_format='channels_last',
dilation_rate=self.dilation_rate) / signals_norm
if self.n_replicas != 1:
shape = K.int_shape(diss)
diss = K.reshape(diss,
(-1, shape[1], shape[2],
shape[3] // self.n_replicas, self.n_replicas))
return self.activation(diss)
def call(self, inputs):
output = []
for eachlen in range(self.seq_len):
tmp = K.bias_add(
K.conv2d(inputs[:, eachlen, :, :, :],
self.kernel[eachlen],
strides=self.strides,
padding=self.padding), self.bias[eachlen])
if self.activation is not None:
output += [self.activation(tmp)]
output = tf.stack(output, axis=1)
return output
def call(self, x):
if self.binarize_input_ == True:
x = Binarization(x)
self.binary_kernel = Binarization(K.clip(self.float_kernel, -1.0, 1.0))
x = K.spatial_2d_padding(x,
padding=((self.pad_, self.pad_), (self.pad_,
self.pad_)))
return K.conv2d(x,
self.binary_kernel,
strides=(self.stride_, self.stride_),
padding='valid',
data_format='channels_last')
def ori_loss(y_true, y_pred, lamb=1.):
# clip
y_pred = K.tf.clip_by_value(y_pred, K.epsilon(), 1 - K.epsilon())
# get ROI
label_seg = K.sum(y_true, axis=-1, keepdims=True)
label_seg = K.tf.cast(K.tf.greater(label_seg, 0), K.tf.float32)
# weighted cross entropy loss
lamb_pos, lamb_neg = 1., 1.
logloss = lamb_pos*y_true*K.log(y_pred)+lamb_neg*(1-y_true)*K.log(1-y_pred)
logloss = logloss*label_seg # apply ROI
logloss = -K.sum(logloss) / (K.sum(label_seg) + K.epsilon())
# coherence loss, nearby ori should be as near as possible
mean_kernal = np.reshape(np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]], dtype=np.float32)/8, [3, 3, 1, 1])
sin2angle_ori, cos2angle_ori, modulus_ori = ori2angle(y_pred)
sin2angle = K.conv2d(sin2angle_ori, mean_kernal, padding='same')
cos2angle = K.conv2d(cos2angle_ori, mean_kernal, padding='same')
modulus = K.conv2d(modulus_ori, mean_kernal, padding='same')
coherence = K.sqrt(K.square(sin2angle) + K.square(cos2angle)) / (modulus + K.epsilon())
coherenceloss = K.sum(label_seg) / (K.sum(coherence*label_seg) + K.epsilon()) - 1
loss = logloss + lamb*coherenceloss
return loss
def call(self, x):
f = K.conv2d(x,
kernel=self.kernel_f,
strides=(1, 1), padding='same') # [bs, h, w, c']
g = K.conv2d(x,
kernel=self.kernel_g,
strides=(1, 1), padding='same') # [bs, h, w, c']
h = K.conv2d(x,
kernel=self.kernel_h,
strides=(1, 1), padding='same') # [bs, h, w, c]
s = K.batch_dot(_hw_flatten(g), K.permute_dimensions(_hw_flatten(f), (0, 2, 1))) # # [bs, N, N]
beta = K.softmax(s, axis=-1) # Attention map
o = K.batch_dot(beta, _hw_flatten(h)) # [bs, N, C]
o = K.reshape(o, shape=K.shape(x)) # [bs, h, w, C]
x = self.gamma * o + x
return x
def call(self, inputs):
print(inputs.shape)
filters = self.U()
print(filters.shape)
#filters /= T.sum(filters, axis=(2, 3)).dimshuffle(0, 1, 'x', 'x')
# channel_first means tensofrlow
conved = K.conv2d(inputs,
filters,
padding=self.padding,
data_format=self.data_format)
return conved
def find_patch_matches(a, a_norm, b):
'''For each patch in A, find the best matching patch in B'''
convs = None
if K.backend() == 'theano':
# HACK: This was not being performed on the GPU for some reason.
from theano.sandbox.cuda import dnn
if dnn.dnn_available():
convs = dnn.dnn_conv(
img=a, kerns=b[:, :, ::-1, ::-1], border_mode='valid')
if convs is None:
convs = K.conv2d(a, b[:, :, ::-1, ::-1], border_mode='valid')
argmax = K.argmax(convs / a_norm, axis=1)
return argmax
def call(self, x, mask=None):
x = K.permute_dimensions(x, (0, 2, 1))
x = K.expand_dims(x, -1)
conv_out = K.permute_dimensions(K.squeeze(K.conv2d(x, self.kernel), -1), (0, 2, 1))
conv_out_s = conv_out[:,:,:self.nb_simple]
conv_out_c = K.square(conv_out[:,:,self.nb_simple:])
output = K.concatenate((conv_out_s, conv_out_c), axis=-1)
return output
def get_output(self, train=False):
X = train
X = K.expand_dims(X, -1) # add a dimension of the right
X = K.permute_dimensions(X, (0, 2, 1, 3))
conv_out = K.conv2d(X, self.W, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering='th')
output = conv_out + K.reshape(self.b, (1, self.nb_filter, 1, 1))
output = self.activation(output)
output = K.squeeze(output, 3) # remove the dummy 3rd dimension
output = K.permute_dimensions(output, (0, 2, 1))
return output
def get_output(self, train=False):
X = self.get_input(train)
conv_out = K.conv2d(X, self.W1, strides=(1, 1),
border_mode='same',
dim_ordering=self.dim_ordering)
if self.dim_ordering == 'tf':
channels = [conv_out[:, :, :, i:i+1] for i in range(self.nb_filter)]
elif self.dim_ordering == 'th':
channels = [conv_out[:, i:i+1, :, :] for i in range(self.nb_filter)]
conv_channels = [K.conv2d(channels[i], self.conv_Ws[i],
strides=self.subsample, border_mode=self.border_mode,
dim_ordering=self.dim_ordering) for i in range(self.nb_filter)]
if self.dim_ordering == 'tf':
conv_out = K.concatenate(conv_channels, axis=3)
if self.dim_ordering == 'th':
conv_out = K.concatenate(conv_channels, axis=1)
if self.dim_ordering == 'th':
output = conv_out + K.reshape(self.b, (1, self.nb_filter, 1, 1))
elif self.dim_ordering == 'tf':
output = conv_out + K.reshape(self.b, (1, 1, 1, self.nb_filter))
output = self.activation(output)
return output
def call(self, x, mask=None):
output = K.conv2d(x, self.W, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering=self.dim_ordering,
filter_shape=self.W_shape)
if self.bias:
if self.dim_ordering == 'th':
output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
elif self.dim_ordering == 'tf':
output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
output = K.square(self.activation(output))
return output
def get_output(self, train=False):
print "ConvolutionCoOccupancy", self.output_shape, self.input_shape
X = self.get_input(train)
if self.W[1] is not None:
W = self.W[0][self.W[1],:,:,:]
else:
W = self.W[0]
if self.b[1] is not None:
b = self.b[0][self.b[1]]
else:
b = self.b[0]
rv = K.conv2d(X[:,:,:,:], W, border_mode='valid') \
+ K.reshape(b, (1, self.nb_domains, 1, 1))
#return rv.dimshuffle((0,3,2,1))
return rv #K.permute_dimensions(rv, (0,3,2,1))
def get_output(self, train=False):
X = self.get_input(train)
conv_out = K.conv2d(X, self.kernel, strides=self.strides,
border_mode='same',
dim_ordering=self.dim_ordering,
image_shape=self.input_shape,
filter_shape=self.kernel_shape)
if self.dim_ordering == 'th':
output = conv_out + K.reshape(self.biases, (1, self.nb_filter, 1, 1))
elif self.dim_ordering == 'tf':
output = conv_out + K.reshape(self.biases, (1, 1, 1, self.nb_filter))
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
output = K.relu(output)
return output
def call(self, x, mask=None):
input_shape = x._keras_shape
if self.convolution_type=='1D':
output = []
for i in range(input_shape[-1]):
x_col = x[:,:,:,i].dimshuffle(0,1,2,'x')
w_col = self.W[:,:,:,i].dimshuffle(0,1,2,'x')
filter_shape=list(self.W_shape[:-1])+[1]
col_output = K.conv2d(x_col, w_col, strides=self.subsample,
border_mode=self.border_mode,
dim_ordering=self.dim_ordering,
filter_shape=tuple(filter_shape))
if self.bias:
if self.dim_ordering == 'th':
col_output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
elif self.dim_ordering == 'tf':
col_output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
col_output = self.activation(col_output)
output.append(col_output)
y = K.concatenate(output,axis=-1)
return y
if self.convolution_type=='bow':
# bow-conv
start = range(0, self.bow_output_length )
y = []
for s in start:
y.append(K.sum(x[:, :, s:s + self.bow_size, :], axis=2))
y = K.concatenate(y, axis=2)
x = K.reshape(y, (-1, input_shape[1], self.bow_output_length, input_shape[3]))
x = super(Convolution2DWrapper, self).call(x, mask)
if self.convolution_type=='bow':
x = T.transpose(x,[0,3,2,1])
return x
def build(self):
if self.dim_ordering == 'th':
stack_size = self.input_shape[1]
input_width = self.input_shape[2]
input_height = self.input_shape[3]
self.input_space_dim = stack_size * self.nb_row * self.nb_col
self.kernel_shape = (self.nb_filter, stack_size, self.nb_row, self.nb_col)
self.identity_kernel_shape = (self.input_space_dim, stack_size, 1, 1)
elif self.dim_ordering == 'tf':
stack_size = self.input_shape[3]
input_width = self.input_shape[1]
input_height = self.input_shape[2]
self.input_space_dim = stack_size * self.nb_row * self.nb_col
self.kernel_shape = (self.nb_row, self.nb_col, stack_size, self.nb_filter)
self.identity_kernel_shape = (1, 1, stack_size, self.input_space_dim)
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
self.kernel = self.init(self.kernel_shape)
self.biases = K.zeros((self.nb_filter,))
self.trainable_weights = [self.kernel, self.biases]
self.feature_sums = K.zeros((self.input_space_dim,))
self.feature_covariance_sums = K.zeros((self.input_space_dim, self.input_space_dim))
self.non_trainable_weights = [self.feature_sums, self.feature_covariance_sums]
np_identity_kernel = np.ones(self.input_space_dim * stack_size).reshape(self.identity_kernel_shape)
self.identity_kernel = K.variable(np_identity_kernel)
x = self.get_input(train=False)
x_flat = K.conv2d(x, self.identity_kernel,
border_mode='same',
dim_ordering=self.dim_ordering,
image_shape=self.input_shape,
filter_shape=self.identity_kernel_shape)
batch_size = self.input_shape[0]
x_flat = K.reshape(x_flat, (batch_size * input_width * input_height, self.input_space_dim))
feature_sums_update = self.momentum * self.feature_sums + (1 - self.momentum) * K.sum(x_flat, axis=0)
feature_covariance_sums_update = self.momentum * self.feature_covariance_sums + (1 - self.momentum) * K.dot(K.transpose(x_flat), x_flat)
self.updates = [(self.feature_sums, feature_sums_update),
(self.feature_covariance_sums, feature_covariance_sums_update)]
self.regularizers = [self.pmi_regularizer]
def conv_step(self, x, W, b=None, border_mode="valid"):
input_shape = self.input_spec[0].shape
conv_out = K.conv2d(x, W, strides=self.subsample,
border_mode=border_mode,
dim_ordering=self.dim_ordering,
image_shape=(input_shape[0],
input_shape[2],
input_shape[3],
input_shape[4]),
filter_shape=self.W_shape)
if b:
if self.dim_ordering == 'th':
conv_out = conv_out + K.reshape(b, (1, self.nb_filter, 1, 1))
elif self.dim_ordering == 'tf':
conv_out = conv_out + K.reshape(b, (1, 1, 1, self.nb_filter))
else:
raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
return conv_out
def conv_step_hidden(self, x, W, border_mode="valid"):
# This new function was defined because the
# image shape must be hardcoded
input_shape = self.input_spec[0].shape
output_shape = self.get_output_shape_for(input_shape)
if self.return_sequences:
out_row, out_col, out_filter = output_shape[2:]
else:
out_row, out_col, out_filter = output_shape[1:]
conv_out = K.conv2d(x, W, strides=(1, 1),
border_mode=border_mode,
dim_ordering=self.dim_ordering,
image_shape=(input_shape[0],
out_row, out_col,
out_filter),
filter_shape=self.W_shape1)
return conv_out
def call(self, inputs):
assert self.rank == 2, 'only conv2d supported for now...'
if self.rank == 2:
outputs = K.conv2d(
inputs,
self.kernel,
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
if self.use_bias:
outputs = K.bias_add(
outputs,
self.bias,
data_format=self.data_format)
#if self.activation is not None:
# assert False,'activation functions not supported'
# return self.activation(outputs)
return outputs
def make_soft(y_true, fragment_length, nb_output_bins, train_with_soft_target_stdev, with_prints=False):
receptive_field, _ = compute_receptive_field()
n_outputs = fragment_length - receptive_field + 1
# Make a gaussian kernel.
kernel_v = scipy.signal.gaussian(9, std=train_with_soft_target_stdev)
print(kernel_v)
kernel_v = np.reshape(kernel_v, [1, 1, -1, 1])
kernel = K.variable(kernel_v)
if with_prints:
y_true = print_t(y_true, 'y_true initial')
# y_true: [batch, timesteps, input_dim]
y_true = K.reshape(y_true, (-1, 1, nb_output_bins, 1)) # Same filter for all output; combine with batch.
# y_true: [batch*timesteps, n_channels=1, input_dim, dummy]
y_true = K.conv2d(y_true, kernel, padding='same')
y_true = K.reshape(y_true, (-1, n_outputs, nb_output_bins)) # Same filter for all output; combine with batch.
# y_true: [batch, timesteps, input_dim]
y_true /= K.sum(y_true, axis=-1, keepdims=True)
if with_prints:
y_true = print_t(y_true, 'y_true after')
return y_true
def offset_conv2d_eval(depth, padding, x):
"""Perform a conv2d on x with a given padding"""
kernel = K.variable(value=np.array([[[[1]] + [[0]] * (depth - 1)]]),
dtype='float32')
return K.conv2d(x, kernel, strides=(3, 3), padding=padding)
def deconv2d_fast(x, kernel, strides=(1, 1), border_mode='valid', dim_ordering='th',
image_shape=None, filter_shape=None):
'''
Run on cuDNN if available.
border_mode: string, "same" or "valid".
'''
if dim_ordering not in {'th', 'tf'}:
raise Exception('Unknown dim_ordering ' + str(dim_ordering))
if dim_ordering == 'tf':
# TF uses the last dimension as channel dimension,
# instead of the 2nd one.
# TH input shape: (samples, input_depth, rows, cols)
# TF input shape: (samples, rows, cols, input_depth)
# TH kernel shape: (depth, input_depth, rows, cols)
# TF kernel shape: (rows, cols, input_depth, depth)
x = x.dimshuffle((0, 3, 1, 2))
kernel = kernel.dimshuffle((3, 2, 0, 1))
if image_shape:
image_shape = (image_shape[0], image_shape[3],
image_shape[1], image_shape[2])
if filter_shape:
filter_shape = (filter_shape[3], filter_shape[2],
filter_shape[0], filter_shape[1])
if _on_gpu() and dnn.dnn_available():
if border_mode == 'same':
assert (strides == (1, 1))
conv_out = dnn.dnn_conv(img=x,
kerns=kernel,
border_mode='full')
shift_x = (kernel.shape[2] - 1) // 2
shift_y = (kernel.shape[3] - 1) // 2
conv_out = conv_out[:, :,
shift_x:x.shape[2] + shift_x,
shift_y:x.shape[3] + shift_y]
else:
conv_out = dnn.dnn_conv(img=x,
conv_mode='cross',
kerns=kernel,
border_mode=border_mode,
subsample=strides)
else:
if border_mode == 'same':
th_border_mode = 'full'
assert (strides == (1, 1))
elif border_mode == 'valid':
th_border_mode = 'valid'
else:
raise Exception('Border mode not supported: ' + str(border_mode))
conv_out = K.conv2d(x, kernel,
border_mode=th_border_mode,
subsample=strides,
filter_flip=False, # <<<<< IMPORTANT 111, dont flip kern
input_shape=image_shape,
filter_shape=filter_shape)
if border_mode == 'same':
shift_x = (kernel.shape[2] - 1) // 2
shift_y = (kernel.shape[3] - 1) // 2
conv_out = conv_out[:, :,
shift_x:x.shape[2] + shift_x,
shift_y:x.shape[3] + shift_y]
if dim_ordering == 'tf':
conv_out = conv_out.dimshuffle((0, 2, 3, 1))
return conv_out
def find_patch_matches(a, a_norm, b):
'''For each patch in A, find the best matching patch in B'''
# we want cross-correlation here so flip the kernels
convs = K.conv2d(a, b[:, :, ::-1, ::-1], border_mode='valid')
argmax = K.argmax(convs / a_norm, axis=1)
return argmax
def log_normals_loss(y_true, y_pred):
y_true = tf.Print(y_true, [y_true], message='y_true', summarize=30)
y_pred = tf.Print(y_pred, [y_pred], message='y_pred', summarize=30)
#compute normals with convolution approach
# (http://answers.opencv.org/question/82453/calculate-surface-normals-from-depth-image-using-neighboring-pixels-cross-product/)
config, unparsed = get_config()
batch_size = config.batch_size
#y_true_clipped = K.clip(y_true, K.epsilon(), None)#40.0)
#y_pred_clipped = K.clip(y_pred, K.epsilon(), None)
y_true_clipped = y_true
y_pred_clipped = y_pred
filter_y = K.variable(np.array([[ 0., -0.5 , 0.],
[0., 0., 0.],
[0., 0.5, 0.]]).reshape(3, 3, 1, 1))
filter_x = K.variable(np.array([ [0, 0., 0.],
[0.5, 0., -0.5],
[0., 0., 0.]]).reshape(3, 3, 1, 1))
w_x = K.variable(np.array([[-1., 0., 1.],
[-1., 0., 1.],
[-1., 0., 1.]]).reshape(3, 3, 1, 1))
w_y = K.variable(np.array([[-1., -1., -1.],
[0., 0., 0.],
[1., 1., 1.]]).reshape(3, 3, 1, 1))
#dzdx = K.conv2d(K.exp(y_true_clipped), w_x, padding='same')
#dzdy = K.conv2d(K.exp(y_true_clipped), w_y, padding='same')
dzdx = K.conv2d(y_true_clipped, w_x, padding='same')
dzdy = K.conv2d(y_true_clipped, w_y, padding='same')
dzdx_ = dzdx * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdx))
dzdy_ = dzdy * -1.0#K.constant(-1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(-1.0, shape=K.int_shape(dzdy))
mag_norm = K.pow(dzdx,2) + K.pow(dzdy,2) + 1.0#K.constant(1.0, shape=[batch_size,K.int_shape(y_pred)[1],K.int_shape(y_pred)[2],K.int_shape(y_pred)[3]]) #K.constant(1.0, shape=K.int_shape(dzdx))
mag_norm = K.sqrt(mag_norm)
N3 = 1.0 / mag_norm #K.constant(1.0, shape=K.int_shape(dzdx)) / mag_norm
N1 = dzdx_ / mag_norm
N2 = dzdy_ / mag_norm
normals = K.concatenate(tensors=[N1,N2,N3],axis=-1)
#dzdx_pred = K.conv2d(K.exp(y_pred_clipped), w_x, padding='same')
#dzdy_pred = K.conv2d(K.exp(y_pred_clipped), w_y, padding='same')
dzdx_pred = K.conv2d(y_pred_clipped, w_x, padding='same')
dzdy_pred = K.conv2d(y_pred_clipped, w_y, padding='same')
mag_norm_pred_x = K.pow(dzdx_pred,2) + 1.0
mag_norm_pred_x = K.sqrt(mag_norm_pred_x)
mag_norm_pred_y = K.pow(dzdy_pred, 2) + 1.0
mag_norm_pred_y = K.sqrt(mag_norm_pred_y)
grad_x = K.concatenate(tensors=[K.constant(1.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_x,
K.constant(0.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_x, dzdx_pred/ mag_norm_pred_x],axis=-1)
grad_y = K.concatenate(tensors=[K.constant(0.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_y,
K.constant(1.0, shape=[batch_size, K.int_shape(y_pred)[1], K.int_shape(y_pred)[2], K.int_shape(y_pred)[3]])/ mag_norm_pred_y, dzdy_pred/ mag_norm_pred_y],axis=-1)
first_log = K.log(y_pred_clipped + 1.)
second_log = K.log(y_true_clipped + 1.)
log_term = K.sqrt(K.mean(K.square(first_log - second_log), axis=-1) + 0.00001)
dot_term_x = K.sum(normals[:,:,:,:] * grad_x[:,:,:,:], axis=-1, keepdims=True)
dot_term_y = K.sum(normals[:,:,:,:] * grad_y[:,:,:,:], axis=-1, keepdims=True)
#dot_term_x = K.mean(K.sum(normals[:, :, :, :] * grad_x[:, :, :, :], axis=-1, keepdims=True), axis=-1)
#dot_term_y = K.mean(K.sum(normals[:, :, :, :] * grad_y[:, :, :, :], axis=-1, keepdims=True), axis=-1)
dot_term_x = tf.Print(dot_term_x, [dot_term_x], message='dot_term_x', summarize=30)
dot_term_y = tf.Print(dot_term_y, [dot_term_y], message='dot_term_y', summarize=30)
#commentare per vecchia versione
sc_inv_term = K.square(K.mean((first_log - second_log), axis=-1))
norm_term = K.mean(K.square(dot_term_x), axis=-1) + K.mean(K.square(dot_term_y), axis=-1)
diff_x = dzdx_pred - dzdx
diff_y = dzdy_pred - dzdy
grad_loss = K.mean(K.square(diff_x) + K.square(diff_y), axis=-1)
loss = log_term - (0.5 * sc_inv_term) + norm_term #+ grad_loss
#loss = log_term + K.square(dot_term_x) + K.square(dot_term_y)
return loss