def model_DIBCO(inp_shape=(256, 256, 3)):
I = Input(inp_shape)
#I=Input(shape=(None,None,1))
z1 = Dilation2D(2, (8, 8), padding="same", strides=(1, 1))(I)
z2 = Erosion2D(2, (8, 8), padding="same", strides=(1, 1))(I)
z3 = Concatenate()([z1, z2])
z3 = Conv2D(8, (1, 1), padding='same')(z3)
for j in range(1):
z1 = Dilation2D(2, (8, 8), padding="same", strides=(1, 1))(z3)
z2 = Erosion2D(2, (8, 8), padding="same", strides=(1, 1))(z3)
z3 = Concatenate()([z1, z2])
z3 = Conv2D(8, (1, 1), padding='same')(z3)
z1 = Dilation2D(2, (8, 8), padding="same", strides=(1, 1))(z3)
z2 = Erosion2D(2, (8, 8), padding="same", strides=(1, 1))(z3)
z3 = Concatenate()([z1, z2])
z3 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(z3)
model = Model(inputs=[I], outputs=[z3])
model.compile(loss=DSSIMObjective(kernel_size=100),
optimizer="adam",
metrics=['mse', DSSIMObjective(kernel_size=100)])
return model
def test_DSSIM_channels_first():
prev_data = K.image_data_format()
K.set_image_data_format('channels_first')
for input_dim, kernel_size in zip([32, 33], [2, 3]):
input_shape = [3, input_dim, input_dim]
X = np.random.random_sample(4 * input_dim * input_dim * 3)
X = X.reshape([4] + input_shape)
y = np.random.random_sample(4 * input_dim * input_dim * 3)
y = y.reshape([4] + input_shape)
model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same', input_shape=input_shape,
activation='relu'))
model.add(Conv2D(3, (3, 3), padding='same', input_shape=input_shape,
activation='relu'))
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
model.compile(loss=DSSIMObjective(kernel_size=kernel_size), metrics=['mse'],
optimizer=adam)
model.fit(X, y, batch_size=2, epochs=1, shuffle='batch')
# Test same
x1 = K.constant(X, 'float32')
x2 = K.constant(X, 'float32')
dssim = DSSIMObjective(kernel_size=kernel_size)
assert_allclose(0.0, K.eval(dssim(x1, x2)), atol=1e-4)
# Test opposite
x1 = K.zeros([4] + input_shape)
x2 = K.ones([4] + input_shape)
dssim = DSSIMObjective(kernel_size=kernel_size)
assert_allclose(0.5, K.eval(dssim(x1, x2)), atol=1e-4)
K.set_image_data_format(prev_data)
def compile_model(model, lambda1=0.05):
I1 = model.inputs[0]
I2 = model.inputs[1]
o1 = model.outputs[0]
# this is to calculate the inverse_warp
o2 = image_warp(-o1, o1)
I2_rec = image_warp(I1, o1)
I1_rec = image_warp(I2, o2)
ux, uy = grad_xy(o1[:, :, :, :1])
vx, vy = grad_xy(o1[:, :, :, 1:2])
sm_loss = lambda1 * (K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)))
re_loss1 = DSSIMObjective(kernel_size=50)(I2, I2_rec)
re_loss2 = DSSIMObjective(kernel_size=50)(I1, I1_rec)
total_loss = lambda1 * sm_loss + re_loss1 + re_loss2
model = Model(inputs=[I1, I2], outputs=[o1])
model.add_loss(total_loss)
model.compile(optimizer=keras.optimizers.Adadelta(
lr=1.0, rho=0.95, epsilon=None, decay=0.0))
return model
def compile_model(model,lambda_smoothness = 0, lambda_flow=0.0001, lambda_mse=0, occ_punishment = 0):
i1=model.inputs[0]
i2=model.inputs[1]
o1=model.outputs[0]
o2 = image_warp(-o1,o1)
oxf, oxb = mask(i1,i2,o1,o2)
mask_f = oxf[:,:,:,0]
mask_b = oxb[:,:,:,1]
err_f, err_b = photometric_error(i1,i2,o1,o2)
flow_f, flow_b = flow_error(o1,o2)
###--------Occlusion_aware_mse_rec_image-------------------------------------
occ_loss1 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_f), mask_f)))#/(436*1024)
occ_loss2 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_b), mask_b)))#/(436*1024)
occ_loss = (occ_loss1 + occ_loss2)*lambda_mse
###--------Occlusion_aware_mse_flow------------------------------------
flow_loss1 = tf.reduce_sum(tf.boolean_mask(charbonnier(flow_f), mask_f))
flow_loss2 = tf.reduce_sum(tf.boolean_mask(charbonnier(flow_b), mask_f))
flow_loss = (flow_loss1 + flow_loss2)*lambda_flow
###--------Punishment_for_occlusion-----------------------------------------
occ_punish1 = tf.multiply(tf.reduce_sum(tf.cast(mask_f, tf.float32)),occ_punishment)
occ_punish2 = tf.multiply(tf.reduce_sum(tf.cast(mask_b, tf.float32)),occ_punishment)
occ_punish = occ_punish1 + occ_punish2
###--------Gradient_smoothness--------------------------------------------
ux,uy=grad_xy(o1[:,:,:,:1])
vx,vy=grad_xy(o1[:,:,:,1:2])
sm_loss_o1 = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy))
ux,uy=grad_xy(o2[:,:,:,:1])
vx,vy=grad_xy(o2[:,:,:,1:2])
sm_loss_o2 = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy))
sm_loss = (sm_loss_o1 + sm_loss_o2)*lambda_smoothness
### Reconstruction_loss_ssim_(occlusion_not considered)
i2_rec=image_warp(i1,o1)
i1_rec=image_warp(i2,o2)
re_loss1=DSSIMObjective(kernel_size=50)(i2,i2_rec)
re_loss2=DSSIMObjective(kernel_size=50)(i1,i1_rec)
re_loss_ssim = re_loss1 + re_loss2
total_loss = sm_loss + occ_loss + occ_punish + re_loss_ssim + flow_loss
model = Model(inputs=[i1,i2], outputs=[o1])
model.add_loss(total_loss)
model.compile(optimizer=keras.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0))
return model
def compile_model(model, lambda1=0.005):
s1 = tf.get_variable("sig1",
trainable=True,
initializer=tf.constant([0.3]))
s2 = tf.get_variable("sig2",
trainable=True,
initializer=tf.constant([0.7]))
s1_2 = s1 * s1
s2_2 = s2 * s2
I1 = model.inputs[0]
I2 = model.inputs[1]
o1 = model.outputs[0]
I2_rec = image_warp(I1, o1)
I1_rec = image_warp(I2, -o1)
ux, uy = grad_xy(o1[:, :, :, :1])
vx, vy = grad_xy(o1[:, :, :, 1:2])
sm_loss = (K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)))
# re_loss_mse = K.mean(K.square(I2 - input1_rec))
re_loss1 = DSSIMObjective(kernel_size=50)(I2, I2_rec)
re_loss2 = DSSIMObjective(kernel_size=50)(I1, I1_rec)
###############################################
# input1_rec=image_warp(model.inputs[0],model.outputs[0],num_batch=b_size)
# input0_rec=image_warp(model.inputs[1],-model.outputs[0],num_batch=b_size)
# ux,uy=grad_xy(model.outputs[0][:,:,:,:1])
# vx,vy=grad_xy(model.outputs[0][:,:,:,1:2])
# sm_loss=lambda1*(K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy)))
# re_loss=DSSIMObjective(kernel_size=50)(model.inputs[1],input1_rec)
################################################
# loss_mse = K.mean(K.square(model.outputs[0] - Y))
re_loss = re_loss1 + re_loss2
total_loss = (1 / s1_2) * re_loss + (1 / s2_2) * sm_loss + K.log(
s1_2) + K.log(s2_2)
model = Model(inputs=[I1, I2], outputs=[o1])
model.add_loss(total_loss)
model.compile(loss="mse",
optimizer=keras.optimizers.Adadelta(lr=1.0,
rho=0.95,
epsilon=None,
decay=0.0))
#model.compile(optimizer='rmsprop')
return model
def total_loss(y_true, y_pred):
s1 = tf.get_variable("sig1", shape=(1, ), trainable=True)
s2 = tf.get_variable("sig2", shape=(1, ), trainable=True)
loss = (1 / (s1 * s1)) * DSSIMObjective(kernel_size=100)(
y_true, y_pred) + (1 / (s2 * s2)) * (PSNRLoss(y_true, y_pred)) + K.log(
s1 * s1) + K.log(s2 * s2)
return loss
def total_loss1(y_true,y_pred):
s1 = tf.get_variable("sig1", shape=(1,), trainable=True)
s2 = tf.get_variable("sig2", shape=(1,), trainable=True)
#loss=(1/(s1*s1+K.epsilon()))*DSSIMObjective(kernel_size=100)(y_true,y_pred)+ (1/(s2*s2+K.epsilon()))*K.mean(K.abs(y_true-y_pred))+2*K.log(s1)+2*K.log(s2)
loss=(1/(s1*s1))*DSSIMObjective(kernel_size=100)(y_true,y_pred)+ (1/(s2*s2))*K.mean(K.abs(y_true-y_pred))+K.log(s1*s1)+K.log(s2*s2)
return(loss)
def get_dssim_l1_loss(alpha=0.84, kernel_size=3, max_value=1.0):
from keras_contrib.losses import DSSIMObjective
from keras.losses import mean_absolute_error
dssim = DSSIMObjective(kernel_size=kernel_size, max_value=max_value)
def DSSIM_L1(y_true, y_pred):
return alpha*dssim(y_true, y_pred) + (1.0-alpha)*mean_absolute_error(y_true, y_pred)
return DSSIM_L1
def loss_new(y_true, y_pred):
#loss=DSSIMObjective(kernel_size=20)(y_true,y_pred)+ K.mean(K.abs(y_true-y_pred))
loss_ssim = DSSIMObjective(kernel_size=100)(
y_true, y_pred) #+ K.mean(K.abs(y_true-y_pred))
loss_psnr = PSNRLoss(y_true, y_pred)
loss = loss_ssim + 0.006 * loss_psnr
return loss
def DSSIM_RGB(y_true, y_pred):
#loss1=DSSIMObjective(kernel_size=23)(y_true[:,:,:,:1],y_pred[:,:,:,:1])
#loss2=DSSIMObjective(kernel_size=23)(y_true[:,:,:,1:2],y_pred[:,:,:,1:2])
#loss3=DSSIMObjective(kernel_size=23)(y_true[:,:,:,2:3],y_pred[:,:,:,2:3])
#loss=K.mean(loss1+loss2+loss3)
#loss=(loss1+loss2+loss3)/3.0
loss = DSSIMObjective(kernel_size=23)(y_true[:, :, :, :3],
y_pred[:, :, :, :3])
return loss
def fcnn_loss(input_img, output, beta = 1):
# Compute error in reconstruction
reconstruction_loss = mae(input_img, output)
# compute the structural similarity index
dssim = DSSIMObjective()
structural_loss = dssim(input_img, output)
total_loss = reconstruction_loss + (beta * structural_loss)
return total_loss
def mse_ssim_loss():
dssim = DSSIMObjective()
mse = losses.mean_squared_error
# Create a loss function that adds the MSE loss to SSIM loss
def loss(y_true, y_pred):
#return K.mean(mse(y_true, y_pred), dssim(y_true,y_pred), axis=-1)
return mse(y_true, y_pred) + dssim(y_true, y_pred)
# Return a function
return loss
def compile_model(model, lambda1=0.05):
s1 = tf.get_variable("sig1",
trainable=True,
initializer=tf.constant([0.3]))
s2 = tf.get_variable("sig2",
trainable=True,
initializer=tf.constant([0.7]))
s1_2 = s1 * s1
s2_2 = s1 * s1
I1 = model.inputs[0]
I2 = model.inputs[1]
o1 = model.outputs[0]
# this is to calculate the inverse_warp
o2 = image_warp(-o1, o1)
I2_rec = image_warp(I1, o1)
I1_rec = image_warp(I2, o2)
ux, uy = grad_xy(o1[:, :, :, :1])
vx, vy = grad_xy(o1[:, :, :, 1:2])
sm_loss = (K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)))
# re_loss_mse = K.mean(K.square(I2 - input1_rec))
re_loss1 = DSSIMObjective(kernel_size=50)(I2, I2_rec)
re_loss2 = DSSIMObjective(kernel_size=50)(I1, I1_rec)
re_loss = re_loss1 + re_loss2
total_loss = (1 / s1_2) * re_loss + (1 / s2_2) * sm_loss + K.log(
s1_2) + K.log(s2_2)
model = Model(inputs=[I1, I2], outputs=[o1])
model.add_loss(total_loss)
model.compile(optimizer=keras.optimizers.Adadelta(
lr=1.0, rho=0.95, epsilon=None, decay=0.0))
return model
def compile_model_new(model, b_size, lambda1=0.002):
"""
session=tf.Session()
session.run(tf.global_variables_initializer())
var = [v for v in tf.trainable_variables() if v.name == "sig1:0"][0]
session.run(var)
"""
#s1 = tf.get_variable("sig1", shape=(1,), trainable=True,initializer=tf.constant([0.3]))
#s2 = tf.get_variable("sig2", shape=(1,), trainable=True,initializer=tf.constant([0.7]))
#s1 = tf.get_variable("sig1", trainable=True,initializer=tf.constant([0.3]))
#s2 = tf.get_variable("sig2", trainable=True,initializer=tf.constant([0.7]))
#s1_2=s1*s1
#s2_2=s1*s1
input1_rec = image_warp(model.inputs[0],
model.outputs[0],
num_batch=b_size)
input0_rec = image_warp(model.inputs[1],
-model.outputs[0],
num_batch=b_size)
ux, uy = grad_xy(model.outputs[0][:, :, :, :1])
vx, vy = grad_xy(model.outputs[0][:, :, :, 1:2])
sm_loss = lambda1 * (K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)))
re_loss = DSSIMObjective(kernel_size=50)(model.inputs[1], input1_rec)
loss_mse = K.mean(K.square(model.outputs[0] - model.inputs[1]))
#total_loss=(1/s1_2)*re_loss+(1/s2_2)*sm_loss+K.log(s1_2)+K.log(s2_2)
total_loss = lambda1 * sm_loss + re_loss + loss_mse
model.add_loss(total_loss)
model.compile(optimizer='rmsprop')
return model
def hourglass_net():
def conv(x,
filters,
kernel_size=1,
strides=(1, 1),
padding='same',
name="conv"):
x = Conv2D(filters,
kernel_size,
strides=strides,
padding=padding,
use_bias=False)(x)
return x
def max_pool2d(x, pool_size=(2, 2), strides=(2, 2), padding="valid"):
x = MaxPooling2D(pool_size=pool_size, strides=strides,
padding=padding)(x)
return x
def conv_bn(x,
filters,
kernel_size=1,
strides=(1, 1),
padding='same',
name="conv_bn"):
x = conv(x, filters, kernel_size, strides, padding, name)
x = BatchNormalization(axis=-1, scale=False)(x)
return x
def conv_bn_relu(x,
filters,
kernel_size=1,
strides=1,
padding="same",
name="conv_bn_rel"):
x = conv(x, filters, kernel_size, strides, padding, name)
x = BatchNormalization(axis=-1, scale=False)(x)
x = Activation('relu')(x)
return x
def conv_block(x, numOut, name="conv_block"):
x = BatchNormalization(axis=-1, scale=False)(x)
x = Activation('relu')(x)
x = conv(x, int(numOut))
return x
def skip_layer(x, numOut, name='skip_layer'):
if x.shape[3] == numOut: # check if right
return x
x = conv(x, numOut)
return x
def residual_block(x, numOut, name="residual_block"):
convb = conv_block(x, numOut)
skip_l = skip_layer(x, numOut)
x = Add()([convb, skip_l])
x = Activation('relu')(x)
return x
def up_block(x, numOut, kernel_size=(3, 3), strides=(1, 1)):
x = Conv2DTranspose(filters=filters,
kernel_size=kernel_size,
padding='same',
strides=strides,
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
return x
def hourglass(x, numOut, name='hourglass'):
# upper branch
up_1 = residual_block(x, numOut, name="up_1")
# lower branch
low_ = max_pool2d(x)
low_1 = residual_block(low_, numOut, name="low_1")
up_2 = residual_block(low_1, numOut, name="up_2")
low_2 = max_pool2d(low_1)
low_2 = residual_block(low_2, numOut, name="low_2")
up_3 = residual_block(low_2, numOut, name="up_3")
low_3 = max_pool2d(low_2)
low_3 = residual_block(low_3, numOut, name="low_3_1")
low_3 = residual_block(low_3, numOut, name="low_3_2")
low_3 = residual_block(low_3, numOut, name="low_3_3")
#low_up_3 = UpSampling2D((2,2))(low_3)
low_up_3 = up_block(low_3, numOut, strides=(2, 2))
low_up_3 = Add()([low_up_3, up_3])
low_up_2 = residual_block(low_up_3, numOut, name="low_up_2")
#low_up_2 = UpSampling2D((2,2))(low_up_2)
low_up_2 = up_block(low_up_2, numOut, strides=(2, 2))
low_up_1 = Add()([low_up_2, up_2])
low_up_1 = residual_block(low_up_1, numOut, name="low_up_1")
#low_up_1 = UpSampling2D((2,2))(low_up_1)
low_up_1 = up_block(low_up_1, numOut, strides=(2, 2))
x = Add()([low_up_1, up_1])
x = Activation('relu')(x)
x = Dropout(0.2)(x)
return x
def up_block(input, filters=64, kernel_size=(3, 3), strides=(1, 1)):
x = Conv2DTranspose(filters=filters,
kernel_size=kernel_size,
padding='same',
strides=strides,
kernel_initializer='he_normal')(input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
return x
inputs = Input((image_rows_low, image_cols, channel_num))
filters = 128
# upscaling
x0 = inputs
for _ in range(int(np.log(upscaling_factor) / np.log(2))):
x0 = up_block(x0, filters, strides=(2, 1))
hg1 = hourglass(x0, filters, name='hourglass_1')
hg2 = hourglass(hg1, filters, name='hourglass_2')
hg3 = hourglass(hg2, filters, name='hourglass_3')
# last_put = conv_bn_relu(hg3, 1, kernel_size = 1, strides = 1, padding="same", name="conv_bn_rel")
last_put = Conv2D(1, (1, 1), activation='relu')(hg3)
def PSNR(y_true, y_pred):
max_pixel = 1.0
return -(10.0 * K.log(
(max_pixel**2) / (K.mean(K.square(y_pred - y_true))))) / 2.303
from keras_contrib.losses import DSSIMObjective
loss_func = DSSIMObjective()
model = Model(inputs=inputs, outputs=last_put)
model.compile(optimizer=Adam(lr=0.0001, decay=0.00001),
loss=loss_func,
metrics=['accuracy', 'mse', 'mae', PSNR, loss_func])
model.summary()
print("Input shape: ", inputs.shape)
print("Output length: ", last_put.shape)
return model
train_generator = aae.generate_keras_input('train')#val')
mse = []
ssim = []
for i in range(973):
imgs_true, under, _ = next(train_generator)
z = encoder.predict(under)
imgs_rec = generator.predict(z)
mse.append(np.mean((imgs_true - imgs_rec)**2))
imgs_true = K.constant(imgs_true)
imgs_rec = K.constant(imgs_rec)
ssim.append(K.eval(DSSIMObjective()(imgs_true, imgs_rec)))
print(np.mean(np.asarray(mse)))
print(np.mean(np.asarray(ssim)))
val_generator = aae.generate_keras_input('val')#val')
mse = []
ssim = []
for i in range(100):
imgs_true, under, _ = next(val_generator)
z = encoder.predict(under)
imgs_rec = generator.predict(z)
mse.append(np.mean((imgs_true - imgs_rec)**2))
file_in, file_out = get_in_out_file()
X, Y_gt = read_files(file_in, file_out)
model_path1 = path1_old()
model_path2 = path2_old()
model_path12_new = model_new()
model_cnn = create_CNN_model()
model_list = [model_path1, model_path2, model_path12_new, model_cnn]
model_list = load_weights(model_list)
#our_loss=DSSIMObjective(kernel_size=23)
#our_loss=loss_all
num_epochs = 1
for i in range(len(model_list)):
model_list[i].compile(loss=DSSIMObjective(kernel_size=23),
optimizer="RMSprop",
metrics=['mse',
DSSIMObjective(kernel_size=23)])
model_list[i].fit(X, Y_gt, epochs=num_epochs, batch_size=4)
#save_models(model_list)
"""
#### Test ##################
datagen=gen_data(batch_size=4)
t1,t2=datagen.next()
t2_out=model.predict(t1)
"""
import matplotlib.pyplot as plt
from tensorflow import keras
from keras_contrib.losses import DSSIMObjective
from keras.losses import mean_squared_error
smiloss = DSSIMObjective()
def rmse_loss(y_true,y_pred):
return smiloss(y_true,y_pred) + K.sqrt(K.mean(K.square(y_pred - y_true)))
def rmse_smi_loss(y_true,y_pred):
return smiloss(y_true,y_pred) + K.sqrt(K.mean(K.square(y_pred - y_true)))
Input = keras.layers.Input
Dense = keras.layers.Dense
Conv2D = keras.layers.Conv2D
MaxPooling2D = keras.layers.MaxPooling2D
UpSampling2D = keras.layers.UpSampling2D
Model = keras.models.Model
K = keras.backend
input_img = Input(shape=(256, 256, 1)) # adapt this if using `channels_first` image data format
x = Conv2D(128, (3, 3), activation='relu', padding='same')(input_img)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(64, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(16, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
x = upscale(64)(x)
x = res_block(x, 64)
x = upscale(32)(x)
#x = res_block(x, 32)
#x = upscale(16)(x)
x = Conv2D(3, kernel_size=5, padding='same', activation='sigmoid')(x)
m1 = Input(shape=(128, 128, 1))
sumModel = Model([input_warped, input_examples, m1], [x])
print(sumModel.summary())
try:
sumModel.load_weights('weights.dat')
except:
print('Weights not found')
DSSIM = DSSIMObjective()
sumModel.compile(optimizer=optimizer, loss=['mae'])
from keras_contrib.losses import DSSIMObjective
for n in range(900000):
imaegsAGen = get_training_data(images_A, landmarks_A, batch_size)
imaegsBGen = get_training_data(images_B, landmarks_B, batch_size)
imaegsCGen = get_training_data(images_C, landmarks_C, batch_size)
imaegsDGen = get_training_data(images_D, landmarks_D, batch_size)
xa, xae, ya, ma = next(imaegsAGen)
xb, xbe, yb, mb = next(imaegsBGen)
xc, xce, yc, mc = next(imaegsCGen)
xd, xde, yd, md = next(imaegsDGen)
def reconstruction(img, reconstructed_img):
#return huber_loss(img, reconstructed_img)
return DSSIMObjective()(img, reconstructed_img)
import keras.models
import keras.losses
import scipy.misc
import numpy as np
import skimage.measure
import os
from keras_contrib.losses import DSSIMObjective
keras.losses.dssim = DSSIMObjective()
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import cmocean
import cmocean.cm
# from keras.utils.generic_utils import get_custom_objects
#
# loss = DSSIMObjective()
# get_custom_objects().update({"dssim": loss})
from src.data.loader import DataLoader
from src.processing.folders import Folders
from PIL import Image
from src.visualization.ssim_plotter import SSIMPlotter
from keras_contrib.losses import DSSIMObjective
from keras import backend as K
import numpy as np
from keras_contrib.losses import DSSIMObjective
from keras import backend as K
#Shape should be (batch,x,y,channels)
imga = np.random.normal(size=(1,256,256,3))
imgb = np.random.normal(size=(1,256,256,3))
loss_func = DSSIMObjective()
resulting_loss1 = K.eval(loss_func(K.variable(imga),K.variable(imgb)))
resulting_loss2 = K.eval(loss_func(K.variable(imga),K.variable(imga)))
print ("Loss for different images: %.2f" % resulting_loss1)
print ("Loss for same image: %.2f" % resulting_loss2)
import matplotlib.image as mpimg
from skimage.transform import resize
from keras.models import Model, load_model
from keras.layers import Input, Dense
import math
from keras.layers import Conv2D, Conv2DTranspose, Dense, Dropout, Flatten, Input, MaxPooling2D, UpSampling2D
from keras.applications.vgg16 import VGG16
from keras_contrib.losses import DSSIMObjective
import glob
import time
import sys
#--------------------------------------main code-----------------------------------------------------------------------
model = load_model(
'model.h5',
custom_objects={'DSSIMObjective': DSSIMObjective(kernel_size=23)})
model.load_weights('model_weights.h5')
def patchextractor(img, patchsize, stride):
patch = []
(a, b) = img.shape
xlim = int(
np.ceil(float(max(a, patchsize[0]) - patchsize[0] + stride) /
stride)) * stride
ylim = int(
np.ceil(float(max(b, patchsize[1]) - patchsize[1] + stride) /
stride)) * stride
# print(xlim,ylim)
for i in range(0, xlim, stride):
] = load_weights(model_list)
model_list = [
model_path1,
model_path2,
model_path12,
model_cnn,
model_path12_new,
model_morph_type2,
]
# our_loss=DSSIMObjective(kernel_size=23)
our_loss = loss_all
num_epochs = 20
for i in range(len(model_list)):
datagen = gen_data(batch_size=4)
model_list[i].compile(
loss=our_loss,
optimizer="RMSprop",
metrics=["mse", DSSIMObjective(kernel_size=23)],
)
model_list[i].fit_generator(datagen, epochs=num_epochs, steps_per_epoch=20)
save_models(model_list)
"""
#### Test ##################
datagen=gen_data(batch_size=4)
t1,t2=datagen.next()
t2_out=model.predict(t1)
"""
def SSIM(y_true, y_pred):
# DSSIM = (1 - SSIM) / 2 => SSIM = 1 - 2 * DSSIM
dssim = DSSIMObjective()
return 1 - 2 * dssim(y_true, y_pred)
def loss_new(y_true, y_pred):
loss = DSSIMObjective(kernel_size=100)(y_true, y_pred) + K.mean(
K.abs(y_true - y_pred))
return loss
def LSTM():
def conv_block(input, filters=64, kernel_size=(3, 3)):
x = Conv2D(filters=filters,
kernel_size=kernel_size,
padding='same',
kernel_initializer=kernel_init)(input)
x = BatchNormalization()(x)
x = Activation(act_func)(x)
x = Conv2D(filters=filters,
kernel_size=kernel_size,
padding='same',
kernel_initializer=kernel_init)(x)
x = BatchNormalization()(x)
x = Activation(act_func)(x)
return x
def up_block(input, filters=64, kernel_size=(3, 3), strides=(1, 1)):
x = Conv2DTranspose(filters=filters,
kernel_size=kernel_size,
padding='same',
strides=strides,
kernel_initializer=kernel_init)(input)
x = BatchNormalization()(x)
x = Activation(act_func)(x)
return x
filters = 64
dropout_rate = 0.25
act_func = 'relu'
kernel_init = 'he_normal'
inputs = Input((image_rows_low, image_cols, channel_num))
# upscailing
x0 = inputs
for _ in range(int(np.log(upscaling_factor) / np.log(2))):
x0 = up_block(x0, filters, strides=(2, 1))
x1 = conv_block(x0, filters)
x2 = AveragePooling2D((2, 2))(x1)
x2 = Dropout(dropout_rate)(x2, training=True)
x2 = conv_block(x2, filters * 2)
x3 = AveragePooling2D((2, 2))(x2)
x3 = Dropout(dropout_rate)(x3, training=True)
x3 = conv_block(x3, filters * 4)
x4 = AveragePooling2D((2, 2))(x3)
x4 = Dropout(dropout_rate)(x4, training=True)
x4 = conv_block(x4, filters * 8)
y4 = AveragePooling2D((2, 2))(x4)
y4 = Dropout(dropout_rate)(y4, training=True)
y4 = conv_block(y4, filters * 16)
y4 = Dropout(dropout_rate)(y4, training=True)
y4 = up_block(y4, filters * 8, strides=(2, 2))
y3 = concatenate([x4, y4], axis=3)
y3 = conv_block(y3, filters * 8)
y3 = Dropout(dropout_rate)(y3, training=True)
y3 = up_block(y3, filters * 4, strides=(2, 2))
y2 = concatenate([x3, y3], axis=3)
y2 = conv_block(y2, filters * 4)
y2 = Dropout(dropout_rate)(y2, training=True)
y2 = up_block(y2, filters * 2, strides=(2, 2))
y1 = concatenate([x2, y2], axis=3)
y1 = conv_block(y1, filters * 2)
y1 = Dropout(dropout_rate)(y1, training=True)
y1 = up_block(y1, filters, strides=(2, 2))
y0 = concatenate([x1, y1], axis=3)
y0 = conv_block(y0, filters)
outputs = Conv2D(1, (1, 1), activation=act_func)(y0)
def PSNR(y_true, y_pred):
max_pixel = 1.0
return (10.0 * K.log(
(max_pixel**2) / (K.mean(K.square(y_pred - y_true))))) / 2.303
from keras_contrib.losses import DSSIMObjective
loss_func = DSSIMObjective()
model = Model(inputs=inputs, outputs=outputs)
# model.compile(optimizer=Adam(lr=0.0001, decay=0.00001),loss=loss_func ,metrics =['accuracy', 'mse' , 'mae', PSNR, loss_func ] )
model.compile(optimizer=Adam(lr=0.0001, decay=0.00001),
loss=loss_func,
metrics=['accuracy', 'mse', 'mae', PSNR, loss_func])
model.summary()
return model
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '4, 5'
import math
import numpy as np
from keras_contrib.losses import DSSIMObjective
import tensorflow as tf
from keras import backend as K
import cv2
'''
- contain code for reconstruct, blockify, finding nearest codevector
'''
ssim = lambda im1, im2: 1 - K.get_value(DSSIMObjective(kernel_size=3).__call__(tf.cast(np.array(im1), tf.float32), tf.cast(np.array(im2), tf.float32)))
def reconstruct_image(blocks, image_size):
image = np.zeros(image_size)
avg = np.zeros(image_size)
bh = blocks.shape[1]
bw = blocks.shape[2]
bd = blocks.shape[3]
for i in range(blocks.shape[0]):
fitH = math.ceil(image_size[0]/bh)
overH = image_size[0]%bh
fitW = math.ceil(image_size[1]/bw)
overW = image_size[1]%bw
h0 = image_size[0]-bh if bh*(i//fitW)+bh>image_size[0] else bh*(i//fitW)
h1 = h0+bh
w0 = image_size[1]-bw if bw*(i%fitW)+bw>image_size[1] else bw*(i%fitW)
w1 = w0+bw
# x= MaxPooling2D(pool_size=(2, 2), strides=(2,2),padding='valid')(x) x=Conv2D(256,(2,2),name='conv4',activation=actFunc, padding='valid',strides=(2,2))(x)#8 # x= MaxPooling2D(pool_size=(2, 2), strides=(2sigmoid,2),padding='valid')(x) #x=Conv2D(256,(5,5),name='conv5',activation=actFunc, padding='valid',strides=(2,2))(x)#1 # x= MaxPooling2D(pool_size=(2, 2), strides=(2,2), padding='valid')(x) y4=x x= Conv2DTranspose(128,(4,4),activation=actFunc, padding='valid',strides=(2,2))(x) # x=Conv2D(256,(3,3),activation='tanh', padding='valid')(x) x= Conv2DTranspose(64,(2,2),activation=actFunc, padding='valid',strides=(2,2))(x) # x=Conv2D(128,(3,3),activation='tanh', padding='valid')(x) x= Conv2DTranspose(64,(2,2),activation=actFunc, padding='valid',strides=(2,2))(x) # x=Conv2D(64,(3,3),activation='tanh', paddactFuncg='valid')(x) x= Conv2DTranspose(16,(1,1),activation=actFunc, padding='same',strides=(2,2))(x) x= Conv2DTranspose(8,(2,2),activation=actFunc, padding='same',strides=(1,1))(x) # x=Conv2D(64,(3,3),activation='tanh', paddactFuncg='valid')(x) x= Conv2DTranspose(1,(1,1),activation="sigmoid", padding='same',strides=(1,1))(x) # x=Conv2D(1,(3,3),activation='sigmoid', padding='valid')(x) #x= Conv2DTranspose(1,(2,2),activation='sigmoid', padding='valid',strides=(1,1))(x) model = Model(inputs = [I], outputs= [x]) model.summary() return model model=build_model() model.compile(optimizer = 'rmsprop', loss=DSSIMObjective(kernel_size=23)) history4= model.fit(X,Y,batch_size=5,epochs=5)
def TCN_net():
def conv_block(input, filters=32, kernel_size=(3, 3)):
x = Conv2D(filters=filters,
kernel_size=kernel_size,
padding='same',
kernel_initializer='he_normal')(input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters=filters,
kernel_size=kernel_size,
padding='same',
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
return x
def BandA(input):
x = BatchNormalization()(input)
x = Activation('relu')(x)
return x
inputs = Input((seq_length, image_rows_low, image_cols, channel_num))
# upscailing
x0 = inputs
# split in seq_length
split_x = Lambda(tf.split,
arguments={
'axis': 1,
'num_or_size_splits': seq_length
})(inputs)
# shared parameteres layer
up0 = Conv2DTranspose(filters=32,
kernel_size=(3, 3),
padding='same',
strides=(2, 1),
kernel_initializer='he_normal')
up1 = Conv2DTranspose(filters=32,
kernel_size=(3, 3),
padding='same',
strides=(2, 1),
kernel_initializer='he_normal')
output1 = [] # 0,1,2,3,4,5,6...15
for i in range(seq_length):
slice = split_x[i]
slice = Lambda(tf.squeeze, arguments={
'axis': 1,
})(slice)
slice = up0(slice)
slice = BatchNormalization()(slice)
slice = Activation('relu')(slice)
slice = up1(slice)
slice = BatchNormalization()(slice)
slice = Activation('relu')(slice)
output1.append(slice)
output2 = [] # 0,1,2,3,4,5,6,7
conv1 = Conv2D(filters=64,
kernel_size=(3, 3),
kernel_regularizer=regularizers.l2(0.01),
dilation_rate=(2, 2),
padding='same',
kernel_initializer='he_normal')
conv2 = AtrousConvolution2D(filters=64,
kernel_size=(3, 3),
kernel_regularizer=regularizers.l2(0.01),
dilation_rate=(2, 2),
strides=(2, 2),
padding='same',
kernel_initializer='he_normal')
for i in range(int(seq_length / 2)):
slice = concatenate([output1[2 * i], output1[2 * i + 1]], axis=3)
slice = conv1(slice)
slice = BatchNormalization()(slice)
slice = Activation('relu')(slice)
slice = conv2(slice)
slice = BatchNormalization()(slice)
slice = Activation('relu')(slice)
output2.append(slice)
output3 = [] # 0,1,2,3
conv3 = Conv2D(filters=128,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
dilation_rate=(2, 2),
padding='same',
kernel_initializer='he_normal')
conv4 = AtrousConvolution2D(filters=128,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
dilation_rate=(2, 2),
strides=(2, 2),
padding='same',
kernel_initializer='he_normal')
for i in range(int(seq_length / 4)):
slice = concatenate([output2[i * 2], output2[2 * i + 1]], axis=3)
slice = conv3(slice)
slice = BatchNormalization()(slice)
slice = Activation('relu')(slice)
slice = conv4(slice)
slice = BatchNormalization()(slice)
slice = Activation('relu')(slice)
output3.append(slice)
output4 = [] # 0,1
conv5 = Conv2D(filters=256,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
dilation_rate=(2, 2),
padding='same',
kernel_initializer='he_normal')
conv6 = AtrousConvolution2D(filters=256,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
dilation_rate=(2, 2),
strides=(2, 2),
padding='same',
kernel_initializer='he_normal')
for i in range(int(seq_length / 8)):
slice = concatenate([output3[i * 2], output3[2 * i + 1]], axis=3)
slice = conv5(slice)
slice = BatchNormalization()(slice)
slice = Activation('relu')(slice)
slice = conv6(slice)
slice = BatchNormalization()(slice)
slice = Activation('relu')(slice)
output4.append(slice)
output5 = [] # 0
conv7 = Conv2D(filters=512,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
dilation_rate=(2, 2),
padding='same',
kernel_initializer='he_normal')
conv8 = AtrousConvolution2D(filters=512,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
dilation_rate=(2, 2),
strides=(2, 2),
padding='same',
kernel_initializer='he_normal')
for i in range(int(seq_length / 16)):
slice = concatenate([output4[i * 2], output4[2 * i + 1]], axis=3)
slice = conv7(slice)
slice = BatchNormalization()(slice)
slice = Activation('relu')(slice)
slice = conv8(slice)
slice = BatchNormalization()(slice)
slice = Activation('relu')(slice)
output5.append(slice)
highest_map = output5[0]
highest_map = Conv2D(filters=1024,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal')(highest_map)
highest_map = BatchNormalization()(highest_map)
highest_map = Activation('relu')(highest_map)
highest_map = Conv2D(filters=1024,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal')(highest_map)
highest_map = BatchNormalization()(highest_map)
highest_map = Activation('relu')(highest_map)
up1_map = Conv2DTranspose(filters=512,
kernel_size=(3, 3),
padding='same',
strides=(2, 2),
kernel_initializer='he_normal')(highest_map)
up1_map = BandA(up1_map)
up1_concat = concatenate([up1_map, output4[1]], axis=3)
up1_concat = Conv2D(filters=512,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal')(up1_concat)
up1_concat = BandA(up1_concat)
up1_concat = Conv2D(filters=512,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal')(up1_concat)
up1_concat = BandA(up1_concat)
up2_map = Conv2DTranspose(filters=256,
kernel_size=(3, 3),
padding='same',
strides=(2, 2),
kernel_initializer='he_normal')(up1_concat)
up2_map = BandA(up2_map)
up2_concat = concatenate([up2_map, output3[3]], axis=3)
up2_concat = Conv2D(filters=256,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal')(up2_concat)
up2_concat = BandA(up2_concat)
up2_concat = Conv2D(filters=256,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal')(up2_concat)
up2_concat = BandA(up2_concat)
up3_map = Conv2DTranspose(filters=128,
kernel_size=(3, 3),
padding='same',
strides=(2, 2),
kernel_initializer='he_normal')(up2_concat)
up3_map = BandA(up3_map)
up3_concat = concatenate([up3_map, output2[7]], axis=3)
up3_concat = Conv2D(filters=128,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal')(up3_concat)
up3_concat = BandA(up3_concat)
up3_concat = Conv2D(filters=128,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal')(up3_concat)
up3_concat = BandA(up3_concat)
up4_map = Conv2DTranspose(filters=64,
kernel_size=(3, 3),
padding='same',
strides=(2, 2),
kernel_initializer='he_normal')(up3_concat)
up4_map = BandA(up4_map)
up4_concat = concatenate([up4_map, output1[15]], axis=3)
up4_concat = Conv2D(filters=64,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal')(up4_concat)
up4_concat = BandA(up4_concat)
up4_concat = Conv2D(filters=64,
kernel_regularizer=regularizers.l2(0.01),
kernel_size=(3, 3),
padding='same',
kernel_initializer='he_normal')(up4_concat)
up4_concat = BandA(up4_concat)
lowest_map = Conv2DTranspose(filters=64,
kernel_size=(3, 3),
padding='same',
strides=(2, 2),
kernel_initializer='he_normal')(up3_concat)
print(K.int_shape(lowest_map))
gen_image = Conv2D(1, (1, 1), activation='relu')(lowest_map)
print(K.int_shape(gen_image))
outputs = gen_image
def PSNR(y_true, y_pred):
max_pixel = 1.0
return -(10.0 * K.log(
(max_pixel**2) / (K.mean(K.square(y_pred - y_true))))) / 2.303
from keras_contrib.losses import DSSIMObjective
ssim_loss = DSSIMObjective()
model = Model(inputs=inputs, outputs=outputs)
model.compile(optimizer=Adam(lr=0.0001, decay=0.00001),
loss='mse',
metrics=['accuracy', 'mse', 'mae', PSNR, ssim_loss])
# model.compile(optimizer=Adam(lr=0.0001, decay=0.00001),loss=PSNR ,metrics =['accuracy', 'mse' , 'mae', PSNR, ssim_loss ] )
model.summary()
return model
