def loss1(y_true, y_pred):
#first term
#only consider pixels that are annotated
y_true = K.clip(y_true, K.epsilon(), 1 - K.epsilon())
y_pred = K.clip(y_pred, K.epsilon(), 1 - K.epsilon())
#get point and boundary annotation separately
y_true_pos = y_true[:, :, :, 0]
y_true_neg = y_true[:, :, :, 1]
#flatten, (batch, pixels)
y_true_pos_f = K.batch_flatten(y_true_pos)
y_true_neg_f = K.batch_flatten(y_true_neg)
y_pred_f = K.batch_flatten(y_pred)
#only consider annotated pixels
#(batch, )
y_true_pos_count = K.sum(y_true_pos_f, axis=-1)
y_true_neg_count = K.sum(y_true_neg_f, axis=-1)
#cross_entropy of each image
#(batch, )
cross_entropy_pos = K.sum(-y_true_pos_f * K.log(y_pred_f), axis=-1)
cross_entropy_neg = K.sum(-y_true_neg_f * K.log(1 - y_pred_f), axis=-1)
#loss_pos = K.mean(cross_entropy_pos / y_true_pos_count)
#loss_neg = K.mean(cross_entropy_neg / y_true_neg_count)
loss_pos = cross_entropy_pos / y_true_pos_count
loss_neg = cross_entropy_neg / y_true_neg_count
return 1.0 * loss_pos + 0.1 * loss_neg
def dice_coef(y_true, y_pred):
y_true_f = K.batch_flatten(y_true)
y_pred_f = K.batch_flatten(y_pred)
intersection = 2 * K.sum(y_true_f * y_pred_f, axis=1, keepdims=True) + 1
union = K.sum(y_true_f, axis=1, keepdims=True) + K.sum(
y_pred_f, axis=1, keepdims=True) + 1
return K.mean(intersection / union)
def get_flux_loss(m, state, state_pred):
'''
@params: state, state_pred shape (batch_size, 60, 60, 2)
p, p_pred shape (batch_size, 60, 60, 1)
m shape (batch_size, 60, 60, 1)
@return: loss_flux: scalar
Only consider discrepancies in total flux, not in phases (saturation not used)
'''
perm = K.exp(m)
p = K.expand_dims(state[:, :, :, 1], -1)
p_pred = K.expand_dims(state_pred[:, :, :, 1], -1)
tran_x = 1. / perm[:, 1:, ...] + 1. / perm[:, :-1, ...]
tran_y = 1. / perm[:, :, 1:, ...] + 1. / perm[:, :, :-1, ...]
flux_x = (p[:, 1:, ...] - p[:, :-1, ...]) / tran_x
flux_y = (p[:, :, 1:, :] - p[:, :, :-1, :]) / tran_y
flux_x_pred = (p_pred[:, 1:, ...] - p_pred[:, :-1, ...]) / tran_x
flux_y_pred = (p_pred[:, :, 1:, :] - p_pred[:, :, :-1, :]) / tran_y
loss_x = K.sum(
K.abs(K.batch_flatten(flux_x) - K.batch_flatten(flux_x_pred)), axis=-1)
loss_y = K.sum(
K.abs(K.batch_flatten(flux_y) - K.batch_flatten(flux_y_pred)), axis=-1)
loss_flux = K.mean(loss_x + loss_y)
return loss_flux
def gram_matrix(x):
assert K.ndim(x) == 3
if K.image_data_format() == 'channels_first':
features = K.batch_flatten(x)
else:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return gram
def closs(y_true,y_pred):
mask = K.batch_flatten(K.sign(y_true[...,4]))
N = K.sum(mask)
heatmaps = K.batch_flatten(y_true[...,0])
cls_pred = K.batch_flatten(y_pred[...,1])
cls_pred = K.clip(cls_pred,1e-7,1-1e-7)
cls_true = K.batch_flatten(y_true[...,1])
cls_loss = K.sum(focal_loss(gamma1,gamma2,cls_true,cls_pred,heatmaps))/N
return cls_loss
def get_reconstruction_loss(x, t_decoded):
'''
Reconstruction loss for the plain VAE
'''
v = 0.1
# return K.mean(K.sum((K.batch_flatten(x) - K.batch_flatten(t_decoded)) ** 2 / (2*v) + 0.5*K.log(2*np.pi*v), axis=-1))
return K.mean(
K.sum((K.batch_flatten(x) - K.batch_flatten(t_decoded))**2 / (2 * v),
axis=-1))
def sloss(y_true,y_pred):
mask = K.batch_flatten(K.sign(y_true[...,4]))
N = K.sum(mask)
sizey_true = K.batch_flatten(y_true[...,4])
sizey_pred = K.batch_flatten(y_pred[...,4])
sizex_true = K.batch_flatten(y_true[...,5])
sizex_pred = K.batch_flatten(y_pred[...,5])
size_loss = K.sum(K.abs(sizex_pred*mask-sizex_true)+K.abs(sizey_pred*mask-sizey_true))/N
return size_loss
def __call__(self, y_true, y_pred):
bb = self.beta * self.beta
y_true_f = K.batch_flatten(y_true)
y_pred_f = K.batch_flatten(y_pred)
intersection = K.sum(y_true_f * y_pred_f, axis=-1)
weighted_union = bb * K.sum(y_true_f, axis=-1) + \
K.sum(y_pred_f, axis=-1)
score = -((1 + bb) * intersection + self.smooth) / \
(weighted_union + self.smooth)
return score
def off_loss(y_true,y_pred):
mask = K.batch_flatten(K.sign(y_true[...,4]))
N = K.sum(mask)
offy_pred = K.batch_flatten(y_pred[...,2])
offx_pred = K.batch_flatten(y_pred[...,3])
offy_true = K.batch_flatten(y_true[...,2])
offx_true = K.batch_flatten(y_true[...,3])
offloss = K.abs(offx_pred*mask-offx_true) + K.abs(offy_pred*mask-offy_true)
offloss = K.sum(offloss)/N
return offloss
def dice_coef_multiclass_2D(y_true, y_pred):
"""A keras implementation of the multiclass Dice coefficient
Adds up Dice coefficients for each non-background class individually. Note there is a small value added to the
denominator to avoid division by zero, so this value should not be reported as the true Dice coefficient
(the difference will be negligible for large arrays).
Parameters:
-----------
y_true : keras layer
The true classes
y_pred : keras layer
The keras layer that computes the classification softmax values
Returns:
--------
keras layer
Multiclass Dice coefficient output calculated across every pixel in the batch
"""
if K.image_data_format() == "channels_first":
b_ax, h_ax, w_ax, c_ax = 0, 2, 3, 1
elif K.image_data_format() == "channels_last":
b_ax, h_ax, w_ax, c_ax = 0, 1, 2, 3
# Flatten predictions, preserving the class dimension
y_pred_f = K.batch_flatten(
K.permute_dimensions(y_pred, (c_ax, b_ax, h_ax, w_ax)))
# Number of output classes
num_classes = y_pred.shape[c_ax]
# Create a one hot coded array of the same shape for the ground truth
y_true = K.one_hot(K.cast(K.squeeze(y_true, axis=c_ax), dtype='uint8'),
num_classes)
true_one_hot = K.permute_dimensions(y_true, (3, 0, 1, 2))
true_one_hot = K.batch_flatten(true_one_hot)
# Find dice coeffcient for each class individually
# Ignore class 0, assumed to be background
class_losses = []
for c in range(1, num_classes):
this_class_intersection = K.sum(true_one_hot[c, :] * y_pred_f[c, :])
this_class_loss = (2. * this_class_intersection + smooth) /\
(K.sum(true_one_hot[c, :]) + K.sum(y_pred_f[c, :]) + smooth)
class_losses.append(this_class_loss)
# Total loss is sum of class losses
total_loss = class_losses[0]
for cl in class_losses[1:]:
total_loss += cl
return total_loss
def style_loss(style_layer, generated_style_layer):
img_channels, img_size = CHANNELS, IMG_HEIGHT * IMG_WIDTH
style_features = K.batch_flatten(K.permute_dimensions(style_layer, (2, 0, 1)))
generated_features = K.batch_flatten(K.permute_dimensions(generated_style_layer, (2, 0, 1)))
style_gram_matrix = gram_matrix(style_features)
generated_gram_matrix = gram_matrix(generated_features)
return K.sum(K.square(style_gram_matrix - generated_gram_matrix)) / (4.0 * (img_channels ** 2) * (img_size ** 2))
def loss(y_true, y_pred):
"""Hellinger loss or Helliger distance, is distance between two discrete distributions.
Hellinger distance:
H(p,q)= 1/sqrt(2) * sqrt(Sum((sqrt(p_i) - sqrt(q_i))**2))
"""
# Reshape tensors (batch_flatten)
y_true_ = K.batch_flatten(y_true)
y_pred_ = K.batch_flatten(y_pred)
# Hellinger distance
z = K.sqrt(y_true_) - K.sqrt(y_pred_)
return sqrt_two * K.sqrt(K.batch_dot(z, z))
def dice_coeff(y_true, y_pred, smooth=1, reduction="mean"):
"""
DSC = 2TP / (2TP + FN + FP)
"""
y_true_f = K.abs(K.batch_flatten(y_true))
y_pred_f = K.abs(K.batch_flatten(y_pred))
intersection = K.sum(y_true_f * y_pred_f, axis=[1])
union = K.sum(y_true_f, axis=[1]) + K.sum(y_pred_f, axis=[1])
dice = (2. * intersection + smooth)/(union + smooth)
return apply_reduction(dice, reduction=reduction)
def jaccard_index(y_true, y_pred, smooth=1, reduction="mean"):
"""
JI = TP / (TP + FP + FN)
"""
y_true_f = K.abs(K.batch_flatten(y_true))
y_pred_f = K.abs(K.batch_flatten(y_pred))
intersection = K.sum(y_true_f * y_pred_f, axis=[1])
union = K.sum(y_true_f, axis=[1]) + \
K.sum(y_pred_f, axis=[1]) - intersection
iou = (intersection + smooth) / (union + smooth)
return apply_reduction(iou, reduction=reduction)
def class_tversky(y_true, y_pred):
smooth = 1
y_true = K.permute_dimensions(y_true, (3, 1, 2, 0))
y_pred = K.permute_dimensions(y_pred, (3, 1, 2, 0))
y_true_pos = K.batch_flatten(y_true)
y_pred_pos = K.batch_flatten(y_pred)
true_pos = K.sum(y_true_pos * y_pred_pos, 1)
false_neg = K.sum(y_true_pos * (1 - y_pred_pos), 1)
false_pos = K.sum((1 - y_true_pos) * y_pred_pos, 1)
alpha = 0.7
return (true_pos + smooth) / (true_pos + alpha * false_neg +
(1 - alpha) * false_pos + smooth)
def gram_matrix(x):
assert K.ndim(x) == 3
if K.image_dim_ordering() == 'th':
features = K.batch_flatten(x)
else:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
shape = K.shape(x)
C, W, H = (shape[0], shape[1], shape[2])
cf = K.reshape(features, (C, -1))
gram = K.dot(cf, K.transpose(cf)) / K.cast(C * W * H, dtype='float32')
return gram
def call(self, x):
x_orig = x
# x reshape
this_bs_int = K.shape(x)[0]
this_bs = tf.cast(this_bs_int, 'float32') # this batch size
prev_count = self.count
x = K.batch_flatten(x) # B x N
# update mean
new_mean, new_count = _mean_update(self.mean, self.count, x, self.cap)
# new C update. Should be B x N x N
x = K.expand_dims(x, -1)
C_delta = K.batch_dot(x, K.permute_dimensions(x, [0, 2, 1]))
# update cov
prev_cap = K.minimum(prev_count, self.cap)
C = self.cov * (prev_cap - 1) + K.sum(C_delta, 0)
new_cov = C / (prev_cap + this_bs - 1)
# updates
updates = [(self.count, new_count), (self.mean, new_mean), (self.cov, new_cov)]
self.add_update(updates, x_orig)
# prep for broadcasting :(
p = tf.concat((K.reshape(this_bs_int, (1,)), K.shape(self.cov)), 0)
z = K.ones(p)
return K.minimum(1., new_count/self.cap) * (z * K.expand_dims(new_cov, 0))
def step(self, x, states):
# x.shape=(1, 512, 30, 40)
# states : lista de tensores shape=(1, 512, 30, 40)
h_tm1 = states[0]
c_tm1 = states[1]
#print("Checkpoint 1--------------")
e = self.V_a(K.tanh(self.W_a(h_tm1) + self.U_a(x))) #e.shape (1, 1, 30, 40)
#print("Checkpoint 2--------------")
a = K.reshape(K.softmax(K.batch_flatten(e)), (x.shape[0], 1, x.shape[2], x.shape[3])) #Nueva version a.shape (1, 1, 30, 40)
#a = K.reshape(K.softmax(K.batch_flatten(e)), (x_shape[0], 1, x_shape[2], x_shape[3]))
#print("Checkpoint 3--------------")
x_tilde = x * K.repeat_elements(a, x.shape[1], 1) #Nueva version x_tilde.shape=(1, 512, 30, 40)
#x_tilde = x * K.repeat_elements(a, x_shape[1], 1)
#print("Checkpoint 4--------------")
x_i = self.W_i(x_tilde)
x_f = self.W_f(x_tilde)
x_c = self.W_c(x_tilde)
x_o = self.W_o(x_tilde)
i = self.inner_activation(x_i + self.U_i(h_tm1))
f = self.inner_activation(x_f + self.U_f(h_tm1))
c = f * c_tm1 + i * self.activation(x_c + self.U_c(h_tm1))
o = self.inner_activation(x_o + self.U_o(h_tm1))
h = o * self.activation(c)
#print("Dime que llegaste/////////////////////")
return h, [h, c]
def cross_correlation(x, y):
#how should the normalization be done??
x = K.l2_normalize(x, axis=1)
y = K.l2_normalize(y, axis=1)
x = K.batch_flatten(x)
y = K.batch_flatten(y)
#x = tf.reshape(x, (x.shape[0]*x.shape[1], x.shape[2], x.shape[3]))
#y = tf.reshape(y, (y.shape[0]*y.shape[1], y.shape[2], y.shape[3]))
a = K.batch_dot(x, y, axes=1)
b = K.batch_dot(x, x, axes=1)
c = K.batch_dot(y, y, axes=1)
return 1 - (a / (K.sqrt(b) * K.sqrt(c)))
def call(self, x):
# soft-assignment.
s = K.conv2d(x, self.kernel, padding='same') + self.bias
print('s.shape=', s.shape)
a = K.softmax(s)
self.amap = K.argmax(a, -1)
# print 'amap.shape', self.amap.shape
# Dims used hereafter: batch, H, W, desc_coeff, cluster
a = K.expand_dims(a, -2)
# print 'a.shape=',a.shape
# Core
v = K.expand_dims(x, -1) + self.C
# print 'v.shape', v.shape
v = a * v
# print 'v.shape', v.shape
v = K.sum(v, axis=[1, 2])
# print 'v.shape', v.shape
v = K.permute_dimensions(v, pattern=[0, 2, 1])
# print 'v.shape', v.shape
#v.shape = None x K x D
# Normalize v (Intra Normalization)
v = K.l2_normalize(v, axis=-1)
v = K.batch_flatten(v)
v = K.l2_normalize(v, axis=-1)
# return [v, self.amap]
return v
def nss(y_true, y_pred):
max_y_pred = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(K.max(K.max(y_pred, axis=2), axis=2)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
y_pred /= max_y_pred
y_pred_flatten = K.batch_flatten(y_pred)
y_mean = K.mean(y_pred_flatten, axis=-1)
y_mean = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(K.expand_dims(y_mean)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
y_std = K.std(y_pred_flatten, axis=-1)
y_std = K.repeat_elements(K.expand_dims(
K.repeat_elements(K.expand_dims(K.expand_dims(y_std)),
shape_r_out,
axis=-1)),
shape_c_out,
axis=-1)
y_pred = (y_pred - y_mean) / (y_std + K.epsilon())
return -(K.sum(K.sum(y_true * y_pred, axis=2), axis=2) /
K.sum(K.sum(y_true, axis=2), axis=2))
def gramMatrix(x):
if K.image_data_format() == "channels_first":
features = K.flatten(x)
else:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return gram
def _label_to_one_hot(tens, nb_labels):
"""
Transform a label nD Tensor to a one-hot 3D Tensor. The input tensor is first
batch-flattened, and then each batch and each voxel gets a one-hot representation
"""
y = K.batch_flatten(tens)
return K.one_hot(y, nb_labels)
def gram_matrix(x, norm_by_channels=False):
'''
Returns the Gram matrix of the tensor x.
'''
if K.ndim(x) == 3:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
shape = K.shape(x)
C, H, W = shape[0], shape[1], shape[2]
gram = K.dot(features, K.transpose(features))
elif K.ndim(x) == 4:
# Swap from (H, W, C) to (B, C, H, W)
x = K.permute_dimensions(x, (0, 3, 1, 2))
shape = K.shape(x)
B, C, H, W = shape[0], shape[1], shape[2], shape[3]
# Reshape as a batch of 2D matrices with vectorized channels
features = K.reshape(x, K.stack([B, C, H * W]))
# This is a batch of Gram matrices (B, C, C).
gram = K.batch_dot(features, features, axes=2)
else:
raise ValueError(
'The input tensor should be either a 3d (H, W, C) or 4d (B, H, W, C) tensor.'
)
# Normalize the Gram matrix
if norm_by_channels:
denominator = C * H * W # Normalization from Johnson
else:
denominator = H * W # Normalization from Google
gram = gram / K.cast(denominator, x.dtype)
return gram
def gram_matrix(img):
# input is (H, W, C) (C = # feature maps)
# convert to (C, H*W)
X = K.batch_flatten(K.permute_dimensions(img, (2, 0, 1)))
# gram = XX^T / N
G = K.dot(X, K.transpose(X) / img.get_shape().num_elements())
return G
def correlation_decoder_loss(x, y):
x = K.l2_normalize(x, axis=1)
y = K.l2_normalize(y, axis=1)
x = K.batch_flatten(x)
y = K.batch_flatten(y)
#x = tf.reshape(x, (tf.shape(x)[0]*tf.shape(x)[1], tf.shape(x)[2], tf.shape(x)[3]))
#y = tf.reshape(y, (tf.shape(y)[0]*tf.shape(y)[1], tf.shape(y)[2], tf.shape(y)[3]))
x = K.cast(x, 'float32')
a = K.batch_dot(x, y, axes=1)
b = K.batch_dot(x, x, axes=1)
c = K.batch_dot(y, y, axes=1)
return 1 - (a / (K.sqrt(b) * K.sqrt(c)))
def call(self, inputs, **kwargs):
if type(inputs) is list:
inputs, mask = inputs
else:
x = K.sqrt(K.sum(K.square(inputs), -1))
mask = K.one_hot(indices=K.argmax(x, 1), num_classes=x.get_shape().as_list()[1])
masked = K.batch_flatten(inputs * K.expand_dims(mask, -1))
return masked
def last_dim_flatten(input_tensor):
'''
Takes a tensor and returns a matrix while preserving only the last dimension from the input.
'''
input_ndim = K.ndim(input_tensor)
shuffle_pattern = (input_ndim - 1, ) + tuple(range(input_ndim - 1))
dim_shuffled_input = K.permute_dimensions(input_tensor, shuffle_pattern)
return K.transpose(K.batch_flatten(dim_shuffled_input))
def call(self, args):
if not isinstance(args, (list, tuple)):
args = [args]
self.cargs = len(args)
# flatten
if len(args) == 2: # input y, m
# get inputs
y, y_mask = args
a_fact = int(y.get_shape().as_list()[-1] / y_mask.get_shape().as_list()[-1])
y_mask = K.repeat_elements(y_mask, a_fact, -1)
y_flat = K.batch_flatten(y) # N x D
y_mask_flat = K.batch_flatten(y_mask) # N x D
# prepare switching matrix
W = self.W # d x D
w_tmp = K.expand_dims(W, 0) # 1 x d x D
Wo = K.permute_dimensions(w_tmp, [0, 2, 1]) * K.expand_dims(y_mask_flat, -1) # N x D x d
WoT = K.permute_dimensions(Wo, [0, 2, 1]) # N x d x D
WotWo_inv = tf.matrix_inverse(K.batch_dot(WoT, Wo)) # N x d x d
pre = K.batch_dot(WotWo_inv, WoT) # N x d x D
res = K.batch_dot(pre, y_flat) # N x d
if self.use_bias:
res += K.expand_dims(self.bias, 0)
else:
x_data = args[0]
shape = K.shape(x_data)
x_data = K.batch_flatten(x_data) # N x d
if self.use_bias:
x_data -= self.bias
res = K.dot(x_data, self.W)
# reshape
# Here you can mix integers and symbolic elements of `shape`
pool_shape = tf.stack([shape[0], *self.orig_input_shape])
res = K.reshape(res, pool_shape)
return res
def gram_matrix(img):
"""
A imagem de input tem o formato (height, width, c) onde c = feature maps.
Primeiro precisamos converter o formato para (c, height * width) = X
depois calculamos gram matrix = XX ^ T = G.
"""
X = K.batch_flatten(K.permute_dimensions(img, (2, 0, 1)))
G = K.dot(X, K.transpose(X)) / img.get_shape().num_elements()
return G
