def loss_DSSIM_theano(y_true, y_pred):
# expected net output is of shape [batch_size, row, col, image_channels]
# e.g. [10, 480, 640, 3] for a batch of 10 640x480 RGB images
# We need to shuffle this to [Batch_size, image_channels, row, col]
y_true = y_true.dimshuffle([0, 3, 1, 2])
y_pred = y_pred.dimshuffle([0, 3, 1, 2])
# There are additional parameters for this function
# Note: some of the 'modes' for edge behavior do not yet have a gradient definition in the Theano tree
# and cannot be used for learning
patches_true = T.nnet.neighbours.images2neibs(y_true, [4, 4])
patches_pred = T.nnet.neighbours.images2neibs(y_pred, [4, 4])
u_true = K.mean(patches_true, axis=-1)
u_pred = K.mean(patches_pred, axis=-1)
var_true = K.var(patches_true, axis=-1)
var_pred = K.var(patches_pred, axis=-1)
std_true = K.sqrt(var_true)
std_pred = K.sqrt(var_pred)
c1 = 0.01 ** 2
c2 = 0.03 ** 2
ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
denom = (u_true ** 2 + u_pred ** 2 + c1) * (var_pred + var_true + c2)
ssim /= K.clip(denom, K.epsilon(), np.inf)
#ssim = tf.select(tf.is_nan(ssim), K.zeros_like(ssim), ssim)
return K.mean((1.0 - ssim) / 2.0)
def keras_SSIM_cs(y_true, y_pred):
axis=None
gaussian = make_kernel(1.5)
x = tf.nn.conv2d(y_true, gaussian, strides=[1, 1, 1, 1], padding='SAME')
y = tf.nn.conv2d(y_pred, gaussian, strides=[1, 1, 1, 1], padding='SAME')
u_x=K.mean(x, axis=axis)
u_y=K.mean(y, axis=axis)
var_x=K.var(x, axis=axis)
var_y=K.var(y, axis=axis)
cov_xy = K.mean(x*y, axis=axis) - u_x*u_y
#cov_xy=cov_keras(x, y, axis)
K1=0.01
K2=0.03
L=1 # depth of image (255 in case the image has a differnt scale)
C1=(K1*L)**2
C2=(K2*L)**2
C3=C2/2
l = ((2*u_x*u_y)+C1) / (K.pow(u_x,2) + K.pow(u_x,2) + C1)
c = ((2*K.sqrt(var_x)*K.sqrt(var_y))+C2) / (var_x + var_y + C2)
s = (cov_xy+C3) / (K.sqrt(var_x)*K.sqrt(var_y) + C3)
return [c,s,l]
def DSSIM(y_true, y_pred):
# y_true = inputs[0]
# y_pred = inputs[1]
# print(y_true.shape)
patches_true = tf.extract_image_patches(y_true, [1, 5, 5, 1], [1, 2, 2, 1], [1, 1, 1, 1], "SAME")
patches_pred = tf.extract_image_patches(y_pred, [1, 5, 5, 1], [1, 2, 2, 1], [1, 1, 1, 1], "SAME")
eps = 1e-9
u_true = K.mean(patches_true, axis=3)
u_pred = K.mean(patches_pred, axis=3)
var_true = K.var(patches_true, axis=3)
var_pred = K.var(patches_pred, axis=3)
covar_true_pred = K.mean(patches_true * patches_pred, axis=3) - u_true * u_pred
std_true = K.sqrt(var_true+eps)
std_pred = K.sqrt(var_pred+eps)
c1 = 0.01 ** 2
c2 = 0.03 ** 2
ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
denom = (K.sqrt(u_true+eps) + K.sqrt(u_pred+eps) + c1) * (var_pred + var_true + c2)
ssim /= denom
ssim = tf.where(tf.is_nan(ssim), K.zeros_like(ssim), ssim)
return K.mean(((1.0 - ssim) / 2))
def loss_DSSIS_tf11(self, y_true, y_pred):
"""
DSSIM loss function to get the structural dissimilarity between y_true and y_pred
:param y_true: groundtruth
:param y_pred: output from the model
:return: The loss value
:note Need tf0.11rc to work
"""
y_true = tf.reshape(y_true, [self.batch_size] + get_shape(y_pred)[1:])
y_pred = tf.reshape(y_pred, [self.batch_size] + get_shape(y_pred)[1:])
y_true_tf = tf.transpose(y_true, [0, 2, 3, 1])
y_pred_tf = tf.transpose(y_pred, [0, 2, 3, 1])
patches_true = tf.extract_image_patches(y_true_tf, [1, 5, 5, 1],
[1, 2, 2, 1], [1, 1, 1, 1],
"SAME")
patches_pred = tf.extract_image_patches(y_pred_tf, [1, 5, 5, 1],
[1, 2, 2, 1], [1, 1, 1, 1],
"SAME")
u_true = K.mean(patches_true, axis=3)
u_pred = K.mean(patches_pred, axis=3)
var_true = K.var(patches_true, axis=3)
var_pred = K.var(patches_pred, axis=3)
std_true = K.sqrt(var_true)
std_pred = K.sqrt(var_pred)
c1 = 0.01**2
c2 = 0.03**2
ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
denom = (u_true**2 + u_pred**2 + c1) * (var_pred + var_true + c2)
ssim /= denom
ssim = tf.select(tf.is_nan(ssim), K.zeros_like(ssim), ssim)
norma = K.mean(K.abs(y_true - y_pred))
norma = tf.select(tf.is_nan(norma), K.ones_like(norma), norma)
return K.mean(((1.0 - ssim) / 2)) + (norma / 2)
def my_loss(x, z): #batch, m = z.get_shape() #batch = K.cast(batch, x.dtype) #div_N = Lambda(lambda v: v / batch) z = z - K.mean(z, axis = 0, keepdims = True) x = x - K.mean(x, axis = 0, keepdims = True) x_2 = K.var(x, axis = 0, keepdims = True) #1xi z_2 = K.var(z, axis = 0, keepdims = True) #1xj #zj_xi = div_N(K.dot(K.transpose(z), x)) zj_xi = tf.divide(K.dot(K.transpose(z), x), batch) z2x2 = K.dot(K.transpose(z_2),x_2) R = 1./(1+K.sum(tf.divide(zj_xi**2, z2x2- zj_xi**2+EPS), axis = 0, keepdims = True)) #1 x i R = tf.divide(R, x_2 + EPS) x_decode = tf.multiply(R, K.transpose(K.dot(K.transpose(tf.divide(zj_xi, (z2x2 - zj_xi**2+EPS))), K.transpose(z)))) dist = tf.contrib.distributions.Normal(0.0, 1.0) if z_log_noise is None: # for each yj, calculate prod (p(y_j) ) under gaussian prod_zj = K.sum(K.log(dist.prob(tf.divide(z, K.sqrt(z_2)))), axis = -1, keepdims = True) # batch x 1 # why log? else: prod_zj = 0 # z_log_noise 1 x j # each real z, # 1/n sum_y p_CI(xi|y) log p_CI(y) / prod(y_j) ci_wms = K.mean(tf.multiply(x_decode, K.log(K.dot(K.exp(prod_zj), R)+EPS)), axis = 0) #)), axis = 0) # 1 x i (mean over batch) #ci_wms = K.mean(tf.multiply(x_decode, K.log(1./R)), axis = 0) #)), axis = 0) # 1 x i (mean over batch) #return K.sum(tf.divide(zj_xi**2, z2x2-zj_xi**2+EPS)) return K.sum(ci_wms) #we want min negative wms == minimize TC == minimize redundancy
def gaussian_cond_var(x_, z, invert_sigmoid = False, subtract_log_det = False, binary_z = False, return_all = False): batch_size = K.cast(K.shape(x_)[0], x_.dtype) div_N = Lambda(lambda v: v / batch_size) size = K.cast(K.shape(x_)[1], dtype='int32') if invert_sigmoid: #log_det_jac = -K.mean(K.log(K.clip(tf.multiply(x_, 1-x_), EPS, 1-EPS))) #log_det_jac = -K.mean(K.log(K.clip(tf.multiply(x, 1-x), EPS, 1-EPS))) x = K.clip(x_, EPS, 1-EPS) x = K.log(x) - K.log(1-x) if binary_z: z = K.clip(z, EPS, 1-EPS) z = K.log(z) - K.log(1-z) # z is likely continuous, but invert sigmoid if binary else: x = x_ log_det_jac = 0 mi = K.expand_dims(K.mean(x, axis=0), 0) # mean of x_i mj = K.expand_dims(K.mean(z, axis=0), 1) # mean of z_j vj = K.expand_dims(K.var(z, axis=0), 1) # sigma_j^2 vi = K.expand_dims(K.var(x, axis=0), 0) # sigma_i^2 V = div_N(K.dot(K.transpose(z-K.transpose(mj)), x- mi)) #jxi cond_var = vi - tf.divide(V**2, vj) if return_all: return cond_var, mi, mj, vi, vj, V else: return cond_var
def loss_DSSIM_theano(y_true, y_pred):
# There are additional parameters for this function
# Note: some of the 'modes' for edge behavior do not yet have a gradient definition in the Theano tree
# and cannot be used for learning
y_true = y_true.dimshuffle([0, 3, 1, 2])
y_pred = y_pred.dimshuffle([0, 3, 1, 2])
patches_true = images2neibs(y_true, [4, 4])
patches_pred = images2neibs(y_pred, [4, 4])
u_true = K.mean(patches_true, axis=-1)
u_pred = K.mean(patches_pred, axis=-1)
var_true = K.var(patches_true, axis=-1)
var_pred = K.var(patches_pred, axis=-1)
std_true = K.sqrt(var_true + K.epsilon())
std_pred = K.sqrt(var_pred + K.epsilon())
c1 = 0.01**2
c2 = 0.03**2
ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
denom = (u_true**2 + u_pred**2 + c1) * (var_pred + var_true + c2)
ssim /= denom # no need for clipping, c1 and c2 make the denom non-zero
return (alpha * K.mean((1.0 - ssim) / 2.0) +
beta * K.mean(K.square(y_pred - y_true), axis=-1))
def loss_G(self,y_true, y_pred):
# L1 Error, I am not using MSE
L1_distance = K.mean(K.abs(y_true-y_pred))
# SSIM error, or D-SSIM = (1-SSIM)
# x = K.mean(y_pred,axis=0)
# y = K.mean(y_true,axis=0)
x = y_true
y = y_pred
ave_x = K.mean(x)
ave_y = K.mean(y)
var_x = K.var(x)
var_y = K.var(y)
covariance = K.mean(x*y) - ave_x*ave_y
c1 = 0.01**2
c2 = 0.03**2
ssim = (2*ave_y*ave_x+c1)*(2*covariance+c2)
ssim = ssim/((K.pow(ave_x,2)+K.pow(ave_y,2)+c1) * (var_x+var_y+c2))
dssim = 1 - ssim
#Possibly add something here as well to penalize noise
# SNR
snr = K.sigmoid(K.var(y_pred-y_true)/K.var(y_true))
return self.l1_loss* L1_distance + self.dssim_loss*dssim
def loss_DSSIS_tf11(self, y_true, y_pred):
"""Need tf0.11rc to work"""
y_true = tf.reshape(y_true,
[self.batch_size] + get_shape(y_pred)[1:])
y_pred = tf.reshape(y_pred,
[self.batch_size] + get_shape(y_pred)[1:])
y_true = tf.transpose(y_true, [0, 2, 3, 1])
y_pred = tf.transpose(y_pred, [0, 2, 3, 1])
patches_true = tf.extract_image_patches(y_true, [1, 5, 5, 1],
[1, 2, 2, 1], [1, 1, 1, 1],
"SAME")
patches_pred = tf.extract_image_patches(y_pred, [1, 5, 5, 1],
[1, 2, 2, 1], [1, 1, 1, 1],
"SAME")
u_true = K.mean(patches_true, axis=3)
u_pred = K.mean(patches_pred, axis=3)
var_true = K.var(patches_true, axis=3)
var_pred = K.var(patches_pred, axis=3)
std_true = K.sqrt(var_true)
std_pred = K.sqrt(var_pred)
c1 = 0.01**2
c2 = 0.03**2
ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
denom = (u_true**2 + u_pred**2 + c1) * (var_pred + var_true + c2)
ssim /= denom
ssim = tf.select(tf.is_nan(ssim), K.zeros_like(ssim), ssim)
return K.mean(((1.0 - ssim) / 2))
def ssim(y_true, y_pred):
# source: https://gist.github.com/Dref360/a48feaecfdb9e0609c6a02590fd1f91b
y_true = tf.expand_dims(y_true, -1)
y_pred = tf.expand_dims(y_pred, -1)
y_true = tf.transpose(y_true, [0, 2, 3, 1])
y_pred = tf.transpose(y_pred, [0, 2, 3, 1])
patches_true = tf.extract_image_patches(y_true, [1, 5, 5, 1], [1, 2, 2, 1], [1, 1, 1, 1], "SAME")
patches_pred = tf.extract_image_patches(y_pred, [1, 5, 5, 1], [1, 2, 2, 1], [1, 1, 1, 1], "SAME")
u_true = K.mean(patches_true, axis=3)
u_pred = K.mean(patches_pred, axis=3)
var_true = K.var(patches_true, axis=3)
var_pred = K.var(patches_pred, axis=3)
std_true = K.sqrt(var_true)
std_pred = K.sqrt(var_pred)
c1 = 0.01 ** 2
c2 = 0.03 ** 2
ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
denom = (u_true ** 2 + u_pred ** 2 + c1) * (var_pred + var_true + c2)
ssim /= denom
ssim = tf.where(tf.is_nan(ssim), K.zeros_like(ssim), ssim)
return ssim
# ----------------------------------------------------------------------------------------------------------------------
def DSSIM_base(y_true, y_pred, kernel_size):
kernel = [kernel_size, kernel_size]
y_true = K.reshape(y_true, [-1] + list(K.int_shape(y_pred)[1:]))
y_pred = K.reshape(y_pred, [-1] + list(K.int_shape(y_pred)[1:]))
patches_pred = KC.extract_image_patches(y_pred, kernel, kernel, 'valid',
K.image_data_format())
patches_true = KC.extract_image_patches(y_true, kernel, kernel, 'valid',
K.image_data_format())
# Reshape to get the var in the cells
bs, w, h, c1, c2, c3 = K.int_shape(patches_pred)
patches_pred = K.reshape(patches_pred, [-1, w, h, c1 * c2 * c3])
patches_true = K.reshape(patches_true, [-1, w, h, c1 * c2 * c3])
# Get mean
u_true = K.mean(patches_true, axis=-1)
u_pred = K.mean(patches_pred, axis=-1)
# Get variance
var_true = K.var(patches_true, axis=-1)
var_pred = K.var(patches_pred, axis=-1)
# Get std dev
covar_true_pred = K.mean(patches_true * patches_pred,
axis=-1) - u_true * u_pred
ssim = (2 * u_true * u_pred + 0.01**2) * (2 * covar_true_pred + 0.03**2)
denom = ((K.square(u_true) + K.square(u_pred) + 0.01**2) *
(var_pred + var_true + 0.03**2))
ssim /= denom # no need for clipping, c1 and c2 make the denom non-zero
print('----------------------', K.mean((1.0 - ssim) / 2.0))
return K.mean((1.0 - ssim) / 2.0)
def SSIM_cs(y_true, y_pred):
patches_true = tf.extract_image_patches(y_true,
ksizes=[1, 8, 8, 1],
strides=[1, 8, 8, 1],
rates=[1, 1, 1, 1],
padding="VALID")
patches_pred = tf.extract_image_patches(y_pred,
ksizes=[1, 8, 8, 1],
strides=[1, 8, 8, 1],
rates=[1, 1, 1, 1],
padding="VALID")
var_true = K.var(patches_true, axis=(1, 2))
var_pred = K.var(patches_pred, axis=(1, 2))
mean_true = K.mean(patches_true, axis=(1, 2))
mean_pred = K.mean(patches_pred, axis=(1, 2))
std_true = K.sqrt(var_true)
std_pred = K.sqrt(var_pred)
covar_true_pred = K.mean(patches_true * patches_pred,
axis=(1, 2)) - mean_true * mean_pred
c1 = (0.01 * 11.09)**2
c2 = (0.03 * 11.09)**2
#contrast = (2 * std_pred * std_true + c2)/(var_pred+var_true+c2);
#lumi = (2 * mean_pred * mean_true + c1)/(mean_pred**2+mean_true**2+c1);
#struct = (covar_true_pred+c2/2)/(std_true*std_pred+c2/2);
ssim = (2 * mean_true * mean_pred + c1) * (2 * covar_true_pred + c2)
denom = (mean_pred**2 + mean_true**2 + c1) * (var_pred + var_true + c2)
ssim /= denom
#ssim = contrast*struct;
ssim = tf.where(tf.is_nan(ssim), K.zeros_like(ssim), ssim)
return K.mean(ssim)
def __call__(self, y_true, y_pred):
# There are additional parameters for this function
# Note: some of the 'modes' for edge behavior do not yet have a gradient definition in the Theano tree
# and cannot be used for learning
kernel = [self.kernel_size, self.kernel_size]
y_true = KC.reshape(y_true, [-1] + list(self.__int_shape(y_pred)[1:]))
y_pred = KC.reshape(y_pred, [-1] + list(self.__int_shape(y_pred)[1:]))
patches_pred = KC.extract_image_patches(y_pred, kernel, kernel,
'valid', self.dim_ordering)
patches_true = KC.extract_image_patches(y_true, kernel, kernel,
'valid', self.dim_ordering)
# Reshape to get the var in the cells
bs, w, h, c1, c2, c3 = self.__int_shape(patches_pred)
patches_pred = KC.reshape(patches_pred, [-1, w, h, c1 * c2 * c3])
patches_true = KC.reshape(patches_true, [-1, w, h, c1 * c2 * c3])
# Get mean
u_true = KC.mean(patches_true, axis=-1)
u_pred = KC.mean(patches_pred, axis=-1)
# Get variance
var_true = K.var(patches_true, axis=-1)
var_pred = K.var(patches_pred, axis=-1)
# Get std dev
covar_true_pred = K.mean(patches_true * patches_pred,
axis=-1) - u_true * u_pred
ssim = (2 * u_true * u_pred + self.c1) * (2 * covar_true_pred +
self.c2)
denom = (K.square(u_true) + K.square(u_pred) +
self.c1) * (var_pred + var_true + self.c2)
ssim /= denom # no need for clipping, c1 and c2 make the denom non-zero
return K.mean(ssim)
def CCC(actual, predicted):
pred_mean = K.mean(predicted, axis=0)
ref_mean = K.mean(actual, axis=0)
pred_var = K.var(predicted, axis=0)
ref_var = K.var(actual, axis=0)
covariance = K.mean((predicted - pred_mean) * (actual - ref_mean), axis=0)
CCC = (2 * covariance) / (pred_var + ref_var + K.pow((pred_mean - ref_mean), 2))
return K.sum(CCC) / 2
def inverse_variance_combination_content_loss(base, combination):
num = K.sum(combination / K.var(combination))
den = K.sum(1 / K.var(combination))
inverse_variance = num / den
loss = K.sqrt(inverse_variance * K.sum(combination - base))
return loss
def ccc_err_tensor(y_true, y_pred):
""" Symbolic CCC error for two 1D time series. Can be used as objective.
"""
cov_xy = K.mean(y_pred * y_true) - (K.mean(y_pred) * K.mean(y_true))
mean_x = K.mean(y_pred)
mean_y = K.mean(y_true)
var_x = K.var(y_pred)
var_y = K.var(y_true)
return 1 - (2 * cov_xy / (var_x + var_y + K.square(mean_x - mean_y)))
def call(self, inputs, mask=None):
input_shape = K.int_shape(inputs)
if len(input_shape) != 4 and len(input_shape) != 2:
raise ValueError('Inputs should have rank ' +
str(4) + " or " + str(2) +
'; Received input shape:', str(input_shape))
if len(input_shape) == 4:
if self.data_format == 'channels_last':
batch_size, h, w, c = input_shape
if batch_size is None:
batch_size = -1
if c < self.group:
raise ValueError('Input channels should be larger than group size' +
'; Received input channels: ' + str(c) +
'; Group size: ' + str(self.group)
)
x = K.reshape(inputs, (batch_size, h, w, self.group, c // self.group))
mean = K.mean(x, axis=[1, 2, 4], keepdims=True)
std = K.sqrt(K.var(x, axis=[1, 2, 4], keepdims=True) + self.epsilon)
x = (x - mean) / std
x = K.reshape(x, (batch_size, h, w, c))
return self.gamma * x + self.beta
elif self.data_format == 'channels_first':
batch_size, c, h, w = input_shape
if batch_size is None:
batch_size = -1
if c < self.group:
raise ValueError('Input channels should be larger than group size' +
'; Received input channels: ' + str(c) +
'; Group size: ' + str(self.group)
)
x = K.reshape(inputs, (batch_size, self.group, c // self.group, h, w))
mean = K.mean(x, axis=[2, 3, 4], keepdims=True)
std = K.sqrt(K.var(x, axis=[2, 3, 4], keepdims=True) + self.epsilon)
x = (x - mean) / std
x = K.reshape(x, (batch_size, c, h, w))
return self.gamma * x + self.beta
elif len(input_shape) == 2:
reduction_axes = list(range(0, len(input_shape)))
del reduction_axes[0]
batch_size, _ = input_shape
if batch_size is None:
batch_size = -1
mean = K.mean(inputs, keepdims=True)
std = K.sqrt(K.var(inputs, keepdims=True) + self.epsilon)
x = (inputs - mean) / std
return self.gamma * x + self.beta
def pearson_error(y_true, y_pred):
true_mean = K.mean(y_true)
true_variance = K.var(y_true)
pred_mean = K.mean(y_pred)
pred_variance = K.var(y_pred)
x = y_true - true_mean
y = y_pred - pred_mean
rho = K.sum(x * y) / K.sqrt(K.sum(x**2) * K.sum(y**2))
return 1 - rho
def gaussian_mi(x, z): # returns j i matrix of I(X_i:Y_j) batch_size = K.cast(K.shape(x)[0], x.dtype) # This is a node tensor, so we can't treat as integer div_N = Lambda(lambda v: v / batch_size) mi = K.expand_dims(K.mean(x, axis=0), 0) # mean of x_i mj = K.expand_dims(K.mean(z, axis=0), 1) # mean of z_j vj = K.expand_dims(K.var(z, axis=0) + EPS, 1) # sigma_j^2 vi = K.expand_dims(K.var(x, axis=0) + EPS, 0) # sigma_i^2 V = div_N(K.dot(K.transpose(z-K.transpose(mj)), x- mi)) rho = V / K.sqrt(vi*vj) return -.5*K.log(1-rho**2) #j i
def SSIM(y_true, y_pred):
u_true = k.mean(y_true)
u_pred = k.mean(y_pred)
var_true = k.var(y_true)
var_pred = k.var(y_pred)
std_true = k.sqrt(var_true)
std_pred = k.sqrt(var_pred)
c1 = k.square(0.01 * 7)
c2 = k.square(0.03 * 7)
ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
denom = (u_true**2 + u_pred**2 + c1) * (var_pred + var_true + c2)
return ssim / denom
def layer_style_loss(self, content, style):
content_mean = K.mean(content, axis=[1, 2])
content_var = K.sqrt(K.var(content, axis=[1, 2]) + 1e-03)
style_mean = K.mean(style, axis=[1, 2])
style_std = K.sqrt(K.var(style, axis=[1, 2]) + 1e-03)
m_loss = LossFunction.sse(content_mean, style_mean) / self.batch_size
s_loss = LossFunction.sse(content_var, style_std) / self.batch_size
return m_loss + s_loss
def call(self, inputs):
shape = inputs.shape
if K.image_data_format() == 'channels_first':
inputs = K.reshape(inputs,(-1,int(shape[1]),int(shape[2])*int(shape[3])))
m = K.mean(inputs, axis=-1, keepdims=False)
v = K.sqrt(K.update_add(K.var(inputs, axis=-1, keepdims=False),1.0e-5))
return K.concatenate([m,v],axis = -1)
else:
inputs = (K.permute_dimensions(inputs, (0, 3, 1, 2)))
inputs = K.reshape(inputs, (-1, int(shape[3]), int(shape[1]) * int(shape[2])))
m = K.mean(inputs, axis=-1, keepdims=False)
v = K.sqrt(K.var(inputs, axis=-1, keepdims=False)+K.constant(1.0e-5, dtype=inputs.dtype.base_dtype))
return K.concatenate([m,v],axis = -1)
def loss_DSSIS(self, y_true, y_pred):
u_true = K.mean(y_true)
u_pred = K.mean(y_pred)
var_true = K.var(y_true)
var_pred = K.var(y_pred)
std_true = K.std(y_true)
std_pred = K.std(y_pred)
c1 = 0.01**2
c2 = 0.03**2
ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
ssim /= (u_true**2 + u_pred**2 + c1) * (var_pred + var_true + c2)
return ((1.0 - ssim) / 2 +
K.binary_crossentropy(y_pred, y_true, True)) / 2.0
def __call__(self, y_true, y_pred):
""" Call the DSSIM Loss Function.
Parameters
----------
y_true: tensor or variable
The ground truth value
y_pred: tensor or variable
The predicted value
Returns
-------
tensor
The DSSIM Loss value
Notes
-----
There are additional parameters for this function. some of the 'modes' for edge behavior
do not yet have a gradient definition in the Theano tree and cannot be used for learning
"""
kernel = [self.kernel_size, self.kernel_size]
y_true = K.reshape(y_true, [-1] + list(self._int_shape(y_pred)[1:]))
y_pred = K.reshape(y_pred, [-1] + list(self._int_shape(y_pred)[1:]))
patches_pred = self.extract_image_patches(y_pred,
kernel,
kernel,
'valid',
self.dim_ordering)
patches_true = self.extract_image_patches(y_true,
kernel,
kernel,
'valid',
self.dim_ordering)
# Get mean
u_true = K.mean(patches_true, axis=-1)
u_pred = K.mean(patches_pred, axis=-1)
# Get variance
var_true = K.var(patches_true, axis=-1)
var_pred = K.var(patches_pred, axis=-1)
# Get standard deviation
covar_true_pred = K.mean(
patches_true * patches_pred, axis=-1) - u_true * u_pred
ssim = (2 * u_true * u_pred + self.c_1) * (
2 * covar_true_pred + self.c_2)
denom = (K.square(u_true) + K.square(u_pred) + self.c_1) * (
var_pred + var_true + self.c_2)
ssim /= denom # no need for clipping, c_1 + c_2 make the denorm non-zero
return (1.0 - ssim) / 2.0
def loss_MS_SSIM(y_true, y_pred):
# expected net output is of shape [batch_size, row, col, image_channels]
# We need to shuffle this to [Batch_size, image_channels, row, col]
y_true = y_true.dimshuffle([0, 3, 1, 2])
y_pred = y_pred.dimshuffle([0, 3, 1, 2])
c1 = 0.01**2
c2 = 0.03**2
ssim = 1.0
alpha = 1.0
beta = 1.0
gamma = 1.0
for i in range(0, 3):
patches_true = K.T.nnet.neighbours.images2neibs(
y_true, [10, 10], [5, 5])
patches_pred = K.T.nnet.neighbours.images2neibs(
y_pred, [10, 10], [5, 5])
mx = K.mean(patches_true, axis=-1)
my = K.mean(patches_pred, axis=-1)
varx = K.var(patches_true, axis=-1)
vary = K.var(patches_pred, axis=-1)
covxy = K.mean(patches_true * patches_pred, axis=-1) - mx * my
if i == 0:
ssimLn = (2 * mx * my + c1)
ssimLd = (mx**2 + my**2 + c1)
ssimLn /= K.clip(ssimLd, K.epsilon(), np.inf)
ssim = K.mean(ssimLn**(alpha * 3))
ssimCn = (2 * K.sqrt(varx * vary + K.epsilon()) + c2)
ssimCd = (varx + vary + c2)
ssimCn /= K.clip(ssimCd, K.epsilon(), np.inf)
ssimSn = (covxy + c2 / 2)
ssimSd = (K.sqrt(varx * vary + K.epsilon()) + c2 / 2)
ssimSn /= K.clip(ssimSd, K.epsilon(), np.inf)
ssim *= K.mean((ssimCn**beta) * (ssimSn**gamma))
y_true = K.pool2d(y_true, (2, 2), (2, 2),
data_format='channels_first',
pool_mode='avg')
y_pred = K.pool2d(y_pred, (2, 2), (2, 2),
data_format='channels_first',
pool_mode='avg')
return (1.0 - ssim)
def __call__(self, y_true, y_pred):
# There are additional parameters for this function
# Note: some of the 'modes' for edge behavior do not yet have a gradient definition in the Theano tree
# and cannot be used for learning
#print(y_true)
kernel = [1, self.kernel_size, self.kernel_size, 1]
y_true = K.reshape(y_true, [-1] + list(self.__int_shape(y_pred)[1:]))
y_pred = K.reshape(y_pred, [-1] + list(self.__int_shape(y_pred)[1:]))
#print(y_true)
patches_pred = tf.extract_image_patches(
images=y_pred,
ksizes=kernel,
strides=kernel,
rates=[1, 1, 1, 1],
padding='SAME')
patches_true = tf.extract_image_patches(
images=y_true,
ksizes=kernel,
strides=kernel,
rates=[1, 1, 1, 1],
padding='SAME')
#print(patches_true)
# Get mean
u_true = K.mean(patches_true, axis=-1)
u_pred = K.mean(patches_pred, axis=-1)
# Get variance
var_true = K.var(patches_true, axis=-1)
var_pred = K.var(patches_pred, axis=-1)
# Get std dev
covar_true_pred = K.mean(
patches_true * patches_pred, axis=-1) - u_true * u_pred
#print(var_true)
ssim = (2 * u_true * u_pred + self.c1) * \
(2 * covar_true_pred + self.c2)
denom = (K.square(u_true) + K.square(u_pred) + self.c1) * \
(var_pred + var_true + self.c2)
ssim /= denom # no need for clipping, c1 and c2 make the denom non-zero
ssim = tf.clip_by_value(ssim, -1.0, 1.0)
#print(ssim)
#print(ssim.eval(session=tf.Session()))
return K.mean((1.0 - ssim) / 2.0)
def SSIM_Loss(y_true, y_pred):
# assert y_true.shape == y_pred.shape, 'Cannot compute PNSR if two input shapes are not same: %s and %s' % (str(
# y_true.shape), str(y_pred.shape))
u_true = K.mean(y_true)
u_pred = K.mean(y_pred)
var_true = K.var(y_true)
var_pred = K.var(y_pred)
std_true = K.sqrt(var_true)
std_pred = K.sqrt(var_pred)
c1 = K.square(0.01 * 7)
c2 = K.square(0.03 * 7)
ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
denom = (u_true**2 + u_pred**2 + c1) * (var_pred + var_true + c2)
return ssim / denom
def tlayer(self, O, T, W, P, K, D, b):
if self.normalize:
O = BE.batch_normalization(x=O,
mean=BE.mean(O),
var=BE.var(O),
gamma=1.,
beta=0.,
epsilon=0.0001)
P = BE.batch_normalization(x=P,
mean=BE.mean(P),
var=BE.var(P),
gamma=1.,
beta=0.,
epsilon=0.0001)
T_ = BE.reshape(T, [D, D * K])
OT = BE.dot(O, T_)
OT = BE.reshape(OT, [-1, D, K])
P_ = BE.reshape(P, [-1, D, 1])
OTP = BE.batch_dot(OT, P_, axes=(1, 1))
OP = BE.concatenate([O, P], axis=1)
W_ = BE.transpose(W)
WOP = BE.dot(OP, W_)
WOP = BE.reshape(WOP, [-1, K, 1])
b_ = BE.reshape(b, [K, 1])
S = merge([OTP, WOP, b_], mode='sum')
S_ = BE.reshape(S, [-1, K])
R = BE.tanh(S_)
# print('O shape: ', BE.int_shape(O))
# print('T_ shape: ', BE.int_shape(T_))
# print('OT shape:', BE.int_shape(OT))
# print('P shape: ', BE.int_shape(P))
# print('P_ shape: ', BE.int_shape(P_))
# print('OTP shape:', BE.int_shape(OTP))
# print('OP shape: ', BE.int_shape(OP))
# print('WOP shape: ', BE.int_shape(WOP))
# print('WOP reshape: ', BE.int_shape(WOP))
# print('b_ shape: ', BE.int_shape(b_))
# print('S shape: ', BE.int_shape(S))
# print('S_ shape: ', BE.int_shape(S_))
return R
def SSIM_Loss(self, y_true, y_pred):
y_pred_hsv = rgb_to_hsv(y_pred)
# mae_loss
mae = K.mean(K.abs(y_pred - y_true), axis=-1)
# tv_loss
shape = tf.shape(y_pred)
height, width = shape[1], shape[2]
y = tf.slice(y_pred, [0, 0, 0, 0],
tf.stack([-1, height - 1, -1, -1])) - tf.slice(
y_pred, [0, 1, 0, 0], [-1, -1, -1, -1])
x = tf.slice(y_pred, [0, 0, 0, 0],
tf.stack([-1, -1, width - 1, -1])) - tf.slice(
y_pred, [0, 0, 1, 0], [-1, -1, -1, -1])
tv_loss = tf.nn.l2_loss(x) / tf.to_float(
tf.size(x)) + tf.nn.l2_loss(y) / tf.to_float(tf.size(y))
# ssim_loss
c1 = 0.01**2
c2 = 0.03**2
y_true = tf.transpose(y_true, [0, 2, 3, 1])
y_pred = tf.transpose(y_pred, [0, 2, 3, 1])
patches_true = tf.extract_image_patches(y_true, [1, 8, 8, 1],
[1, 2, 2, 1], [1, 1, 1, 1],
"SAME")
patches_pred = tf.extract_image_patches(y_pred, [1, 8, 8, 1],
[1, 2, 2, 1], [1, 1, 1, 1],
"SAME")
# Get mean
u_true = K.mean(patches_true, axis=-1)
u_pred = K.mean(patches_pred, axis=-1)
# Get variance
var_true = K.var(patches_true, axis=-1)
var_pred = K.var(patches_pred, axis=-1)
# Get std dev
std_true = K.sqrt(var_true)
std_pred = K.sqrt(var_pred)
covar_true_pred = std_pred * std_true
ssim = (2 * u_true * u_pred + c1) * (2 * covar_true_pred + c2)
denom = (K.square(u_true) + K.square(u_pred) + c1) * (var_pred +
var_true + c2)
ssim /= denom
size = tf.size(y_pred_hsv)
light_loss = tf.nn.l2_loss(y_pred_hsv[:, :, :, 2]) / tf.to_float(size)
total_loss = -0.07 * light_loss + 1.0 * mae - 0.0005 * tv_loss
return total_loss
def ssim(y_true, y_pred):
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
u_true = K.mean(y_true_f)
u_pred = K.mean(y_pred_f)
var_true = K.var(y_true_f)
var_pred = K.var(y_pred_f)
std_true = K.sqrt(var_true)
std_pred = K.sqrt(var_pred)
c1 = 0.01
c2 = 0.03
return ((2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)) / ((u_true ** 2 + u_pred ** 2 + c1) * (var_pred + var_true + c2))
def loss_DSSIM_theano(y_true, y_pred):
patches_true = T.nnet.neighbours.images2neibs(y_true, [4, 4])
patches_pred = T.nnet.neighbours.images2neibs(y_pred, [4, 4])
u_true = K.mean(patches_true, axis=-1)
u_pred = K.mean(patches_pred, axis=-1)
var_true = K.var(patches_true, axis=-1)
var_pred = K.var(patches_pred, axis=-1)
eps = 1e-9
std_true = K.sqrt(var_true + eps)
std_pred = K.sqrt(var_pred + eps)
c1 = 0.01 ** 2
c2 = 0.03 ** 2
ssim = (2 * u_true * u_pred + c1) * (2 * std_pred * std_true + c2)
denom = (u_true ** 2 + u_pred ** 2 + c1) * (var_pred + var_true + c2)
ssim /= denom # no need for clipping, c1 and c2 make the denom non-zero
return K.mean((1.0 - ssim) / 2.0)
def __call__(self, y_true, y_pred):
# There are additional parameters for this function
# Note: some of the 'modes' for edge behavior do not yet have a
# gradient definition in the Theano tree and cannot be used for
# learning
kernel = [self.kernel_size, self.kernel_size]
y_true = K.reshape(y_true, [-1] + list(self.__int_shape(y_pred)[1:]))
y_pred = K.reshape(y_pred, [-1] + list(self.__int_shape(y_pred)[1:]))
patches_pred = self.extract_image_patches(y_pred,
kernel,
kernel,
'valid',
self.dim_ordering)
patches_true = self.extract_image_patches(y_true,
kernel,
kernel,
'valid',
self.dim_ordering)
# Get mean
u_true = K.mean(patches_true, axis=-1)
u_pred = K.mean(patches_pred, axis=-1)
# Get variance
var_true = K.var(patches_true, axis=-1)
var_pred = K.var(patches_pred, axis=-1)
# Get std dev
covar_true_pred = K.mean(
patches_true * patches_pred, axis=-1) - u_true * u_pred
ssim = (2 * u_true * u_pred + self.c_1) * (
2 * covar_true_pred + self.c_2)
denom = (K.square(u_true) + K.square(u_pred) + self.c_1) * (
var_pred + var_true + self.c_2)
ssim /= denom # no need for clipping, c_1 + c_2 make the denom non-zero
return K.mean((1.0 - ssim) / 2.0)
def norm(self, xs, norm_id): mu = K.mean(xs, axis=-1, keepdims=True) sigma = K.sqrt(K.var(xs, axis=-1, keepdims=True) + 1e-3) xs = self.gs[norm_id] * (xs - mu) / (sigma + 1e-3) + self.bs[norm_id] return xs
def _get_mean_and_std(self, x):
reduction_axes = list(range(K.ndim(x)))
del reduction_axes[self.axis]
mean = K.mean(x, axis=reduction_axes)
std = K.sqrt(K.var(x, axis=reduction_axes) + K.epsilon())
return mean, std
def main(self, name, opts):
logging.basicConfig(filename=opts.log_file,
format='%(levelname)s (%(asctime)s): %(message)s')
log = logging.getLogger(name)
if opts.verbose:
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.INFO)
log.debug(opts)
if opts.seed is not None:
np.random.seed(opts.seed)
if not opts.model_files:
raise ValueError('No model files provided!')
log.info('Loading model ...')
K.set_learning_phase(0)
model = mod.load_model(opts.model_files)
# Get DNA layer.
dna_layer = None
for layer in model.layers:
if layer.name == 'dna':
dna_layer = layer
break
if not dna_layer:
raise ValueError('The provided model is not a DNA model!')
# Create output vector.
outputs = []
for output in model.outputs:
outputs.append(K.reshape(output, (-1, 1)))
outputs = K.concatenate(outputs, axis=1)
# Compute gradient of outputs wrt. DNA layer.
grads = []
for name in opts.targets:
if name == 'mean':
target = K.mean(outputs, axis=1)
elif name == 'var':
target = K.var(outputs, axis=1)
else:
raise ValueError('Invalid effect size "%s"!' % name)
grad = K.gradients(target, dna_layer.output)
grads.extend(grad)
grad_fun = K.function(model.inputs, grads)
log.info('Reading data ...')
nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample)
replicate_names = dat.get_replicate_names(
opts.data_files[0],
regex=opts.replicate_names,
nb_key=opts.nb_replicate)
data_reader = mod.data_reader_from_model(
model, outputs=False, replicate_names=replicate_names)
data_reader = data_reader(opts.data_files,
nb_sample=nb_sample,
batch_size=opts.batch_size,
loop=False,
shuffle=False)
meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'],
nb_sample=nb_sample,
batch_size=opts.batch_size,
loop=False,
shuffle=False)
out_file = h5.File(opts.out_file, 'w')
out_group = out_file
def h5_dump(path, data, idx, dtype=None, compression='gzip'):
if path not in out_group:
if dtype is None:
dtype = data.dtype
out_group.create_dataset(
name=path,
shape=[nb_sample] + list(data.shape[1:]),
dtype=dtype,
compression=compression
)
out_group[path][idx:idx+len(data)] = data
log.info('Computing effects ...')
progbar = ProgressBar(nb_sample, log.info)
idx = 0
for inputs in data_reader:
if isinstance(inputs, dict):
inputs = list(inputs.values())
batch_size = len(inputs[0])
progbar.update(batch_size)
# Compute gradients.
grads = grad_fun(inputs)
# Slice window at center.
if opts.dna_wlen:
for i, grad in enumerate(grads):
delta = opts.dna_wlen // 2
ctr = grad.shape[1] // 2
grads[i] = grad[:, (ctr-delta):(ctr+delta+1)]
# Aggregate effects in window
if opts.agg_effects:
for i, grad in enumerate(grads):
if opts.agg_effects == 'mean':
grad = grad.mean(axis=1)
elif opts.agg_effects == 'wmean':
weights = linear_weights(grad.shape[1])
grad = np.average(grad, axis=1, weights=weights)
elif opts.agg_effects == 'max':
grad = grad.max(axis=1)
else:
tmp = 'Invalid function "%s"!' % (opts.agg_effects)
raise ValueError(tmp)
grads[i] = grad
# Write computed effects
for name, grad in zip(opts.targets, grads):
h5_dump(name, grad, idx)
# Store inputs
if opts.store_inputs:
for name, value in zip(model.input_names, inputs):
h5_dump(name, value, idx)
# Store positions
for name, value in next(meta_reader).items():
h5_dump(name, value, idx)
idx += batch_size
progbar.close()
out_file.close()
log.info('Done!')
return 0
def call(self, inputs, mask=None):
input_shape = K.int_shape(inputs)
if len(input_shape) != 4 and len(input_shape) != 2:
raise ValueError('Inputs should have rank ' +
str(4) + " or " + str(2) +
'; Received input shape:', str(input_shape))
if len(input_shape) == 4:
if self.data_format == 'channels_last':
batch_size, height, width, channels = input_shape
if batch_size is None:
batch_size = -1
if channels < self.group:
raise ValueError('Input channels should be larger than group size' +
'; Received input channels: ' + str(channels) +
'; Group size: ' + str(self.group))
var_x = K.reshape(inputs, (batch_size,
height,
width,
self.group,
channels // self.group))
mean = K.mean(var_x, axis=[1, 2, 4], keepdims=True)
std = K.sqrt(K.var(var_x, axis=[1, 2, 4], keepdims=True) + self.epsilon)
var_x = (var_x - mean) / std
var_x = K.reshape(var_x, (batch_size, height, width, channels))
retval = self.gamma * var_x + self.beta
elif self.data_format == 'channels_first':
batch_size, channels, height, width = input_shape
if batch_size is None:
batch_size = -1
if channels < self.group:
raise ValueError('Input channels should be larger than group size' +
'; Received input channels: ' + str(channels) +
'; Group size: ' + str(self.group))
var_x = K.reshape(inputs, (batch_size,
self.group,
channels // self.group,
height,
width))
mean = K.mean(var_x, axis=[2, 3, 4], keepdims=True)
std = K.sqrt(K.var(var_x, axis=[2, 3, 4], keepdims=True) + self.epsilon)
var_x = (var_x - mean) / std
var_x = K.reshape(var_x, (batch_size, channels, height, width))
retval = self.gamma * var_x + self.beta
elif len(input_shape) == 2:
reduction_axes = list(range(0, len(input_shape)))
del reduction_axes[0]
batch_size, _ = input_shape
if batch_size is None:
batch_size = -1
mean = K.mean(inputs, keepdims=True)
std = K.sqrt(K.var(inputs, keepdims=True) + self.epsilon)
var_x = (inputs - mean) / std
retval = self.gamma * var_x + self.beta
return retval
