def compile_model(model, lambda1=0.05):
I1 = model.inputs[0]
I2 = model.inputs[1]
o1 = model.outputs[0]
# this is to calculate the inverse_warp
o2 = image_warp(-o1, o1)
I2_rec = image_warp(I1, o1)
I1_rec = image_warp(I2, o2)
ux, uy = grad_xy(o1[:, :, :, :1])
vx, vy = grad_xy(o1[:, :, :, 1:2])
sm_loss = lambda1 * (K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)))
re_loss1 = DSSIMObjective(kernel_size=50)(I2, I2_rec)
re_loss2 = DSSIMObjective(kernel_size=50)(I1, I1_rec)
total_loss = lambda1 * sm_loss + re_loss1 + re_loss2
model = Model(inputs=[I1, I2], outputs=[o1])
model.add_loss(total_loss)
model.compile(optimizer=keras.optimizers.Adadelta(
lr=1.0, rho=0.95, epsilon=None, decay=0.0))
return model
def compile_model(model, lambda1=0.005):
I1 = model.inputs[0]
I2 = model.inputs[1]
o1 = model.outputs[0]
# this is to calculate the inverse_warp
o2 = image_warp(-o1, o1)
I2_rec = image_warp(I1, o1)
I1_rec = image_warp(I2, o2)
ux, uy = grad_xy(o1[:, :, :, :1])
vx, vy = grad_xy(o1[:, :, :, 1:2])
sm_loss = lambda1 * (K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)))
re_loss_mse = K.mean(K.square(I2 - I1_rec))
# loss_mse = K.mean(K.square(model.outputs[0] - Y))
total_loss = lambda1 * sm_loss + re_loss_mse # + loss_mse
model.add_loss(total_loss)
model = Model(inputs=[I1, I2], outputs=[o1])
model.compile(loss="mse", optimizer='Adam')
return model
def compile_model(model,lambda_smoothness = 0, lambda_flow=0.0001, lambda_mse=0, occ_punishment = 0):
i1=model.inputs[0]
i2=model.inputs[1]
o1=model.outputs[0]
o2 = image_warp(-o1,o1)
oxf, oxb = mask(i1,i2,o1,o2)
mask_f = oxf[:,:,:,0]
mask_b = oxb[:,:,:,1]
err_f, err_b = photometric_error(i1,i2,o1,o2)
flow_f, flow_b = flow_error(o1,o2)
###--------Occlusion_aware_mse_rec_image-------------------------------------
occ_loss1 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_f), mask_f)))#/(436*1024)
occ_loss2 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_b), mask_b)))#/(436*1024)
occ_loss = (occ_loss1 + occ_loss2)*lambda_mse
###--------Occlusion_aware_mse_flow------------------------------------
flow_loss1 = tf.reduce_sum(tf.boolean_mask(charbonnier(flow_f), mask_f))
flow_loss2 = tf.reduce_sum(tf.boolean_mask(charbonnier(flow_b), mask_f))
flow_loss = (flow_loss1 + flow_loss2)*lambda_flow
###--------Punishment_for_occlusion-----------------------------------------
occ_punish1 = tf.multiply(tf.reduce_sum(tf.cast(mask_f, tf.float32)),occ_punishment)
occ_punish2 = tf.multiply(tf.reduce_sum(tf.cast(mask_b, tf.float32)),occ_punishment)
occ_punish = occ_punish1 + occ_punish2
###--------Gradient_smoothness--------------------------------------------
ux,uy=grad_xy(o1[:,:,:,:1])
vx,vy=grad_xy(o1[:,:,:,1:2])
sm_loss_o1 = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy))
ux,uy=grad_xy(o2[:,:,:,:1])
vx,vy=grad_xy(o2[:,:,:,1:2])
sm_loss_o2 = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy))
sm_loss = (sm_loss_o1 + sm_loss_o2)*lambda_smoothness
### Reconstruction_loss_ssim_(occlusion_not considered)
i2_rec=image_warp(i1,o1)
i1_rec=image_warp(i2,o2)
re_loss1=DSSIMObjective(kernel_size=50)(i2,i2_rec)
re_loss2=DSSIMObjective(kernel_size=50)(i1,i1_rec)
re_loss_ssim = re_loss1 + re_loss2
total_loss = sm_loss + occ_loss + occ_punish + re_loss_ssim + flow_loss
model = Model(inputs=[i1,i2], outputs=[o1])
model.add_loss(total_loss)
model.compile(optimizer=keras.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0))
return model
def compile_model(model, lambda1=0.005):
s1 = tf.get_variable("sig1",
trainable=True,
initializer=tf.constant([0.3]))
s2 = tf.get_variable("sig2",
trainable=True,
initializer=tf.constant([0.7]))
s1_2 = s1 * s1
s2_2 = s2 * s2
I1 = model.inputs[0]
I2 = model.inputs[1]
o1 = model.outputs[0]
I2_rec = image_warp(I1, o1)
I1_rec = image_warp(I2, -o1)
ux, uy = grad_xy(o1[:, :, :, :1])
vx, vy = grad_xy(o1[:, :, :, 1:2])
sm_loss = (K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)))
# re_loss_mse = K.mean(K.square(I2 - input1_rec))
re_loss1 = DSSIMObjective(kernel_size=50)(I2, I2_rec)
re_loss2 = DSSIMObjective(kernel_size=50)(I1, I1_rec)
###############################################
# input1_rec=image_warp(model.inputs[0],model.outputs[0],num_batch=b_size)
# input0_rec=image_warp(model.inputs[1],-model.outputs[0],num_batch=b_size)
# ux,uy=grad_xy(model.outputs[0][:,:,:,:1])
# vx,vy=grad_xy(model.outputs[0][:,:,:,1:2])
# sm_loss=lambda1*(K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy)))
# re_loss=DSSIMObjective(kernel_size=50)(model.inputs[1],input1_rec)
################################################
# loss_mse = K.mean(K.square(model.outputs[0] - Y))
re_loss = re_loss1 + re_loss2
total_loss = (1 / s1_2) * re_loss + (1 / s2_2) * sm_loss + K.log(
s1_2) + K.log(s2_2)
model = Model(inputs=[I1, I2], outputs=[o1])
model.add_loss(total_loss)
model.compile(loss="mse",
optimizer=keras.optimizers.Adadelta(lr=1.0,
rho=0.95,
epsilon=None,
decay=0.0))
#model.compile(optimizer='rmsprop')
return model
def mask(i1,i2,o1,o2, alpha1=0.01, alpha2=0.5): left_f = tf.square( o1 + image_warp(o2,o2)) right_f = alpha1*( tf.square(o1) + tf.square(image_warp(o2,o2))) + alpha2 oxf = tf.less(right_f, left_f) left_b = tf.square( o2 + image_warp(o1,o1) ) right_b = alpha1*( tf.square(o2) + tf.square(image_warp(o1,o1))) + alpha2 oxb = tf.less(right_b, right_b) return oxf, oxb
def train(train_loader, model, optimizer):
# switch to train mode
train_loader = torch.tensor(train_loader).type(torch.cuda.FloatTensor)
## train_loader.size = Bx3x(C==3)xHxW
model.train()
out_flow12 = model(
train_loader[:, 0:2]) ## list of 5 optical flows not of image size
out_flow23 = model(
train_loader[:, 1:3]) ## list of 5 optical flows not of image size
h, w = train_loader.size()[-2:] ## image size
out_flow12 = [F.interpolate(oflow, (h, w))
for oflow in out_flow12] ## upsampling flow to image size
out_flow23 = [F.interpolate(oflow, (h, w))
for oflow in out_flow23] ## upsampling flow to image size
out_image = []
im1 = train_loader[:, 0] ## the 1st image of the triplets in the batch
im2 = train_loader[:, 1] ## the 2nd image of the triplets in the batch
im3 = train_loader[:, 2] ## the 3rd image of the triplets in the batch
weights = [0.32, 0.08, 0.02, 0.01, 0.005]
loss = 0
combined_loss = []
for i in range(0, len(out_flow12)):
oflow12 = out_flow12[i]
oflow23 = out_flow23[i]
warped_img12 = image_warp(im1, oflow12)
warped_img23 = image_warp(im2, oflow23)
temp_loss = flow_loss(train_loader, warped_img12, warped_img23,
oflow12, oflow23, weights[i])
loss += temp_loss
combined_loss.append(temp_loss / weights[i])
# print("combined_loss",combined_loss)
# total_loss = torch.sum(torch.tensor(combined_loss).type(torch.cuda.FloatTensor))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# print(loss.grad)
# print(loss)
return loss
def flow_loss(im1, im2, flow_fw):
"""Calculate the loss from the logits and the labels.
Args:
img1: tensor, float - the first frame.
img2: tensor, float - the second frame.
optical: tensor, float - the optical estimation from flownet.
Returns:
loss: Loss tensor of type float.
"""
with tf.name_scope('optical_flow_loss'):
losses = {}
im2_warped = image_warp(im2, flow_fw)
#im1_warped = image_warp(im1, flow_bw)
im_diff_fw = im1 - im2_warped
#im_diff_bw = im2 - im1_warped
mask_fw = tf.ones_like(im_diff_fw)
losses['photo'] = (photometric_loss(im_diff_fw, mask_fw))
losses['grad'] = (gradient_loss(im1, im2_warped, mask_fw))
losses['smooth_1st'] = (smoothness_loss(flow_fw))
return losses
def return_func():
I = Input(shape=(436, 1024, 3))
F = Input(shape=(436, 1024, 2))
F_new = image_warp(-F, F)
# I2=image_warp(I,F)
# f=K.function(inputs=[I,F],outputs=[I2])
return f
def evaluate(train_loader, model):
# switch to train mode
train_loader = torch.tensor(train_loader).type(torch.cuda.FloatTensor)
## train_loader.size = Bx3x(C==3)xHxW
model.eval()
# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
# epoch = checkpoint['epoch']
# loss = checkpoint['loss']
out_flow12 = model(train_loader[:,0:2]) ## list of 5 optical flows not of image size
out_flow23 = model(train_loader[:,1:3]) ## list of 5 optical flows not of image size
# print("out_flow12.shape",out_flow12.size())
h, w = train_loader.size()[-2:] ## image size
print("h,w",h,w)
out_flow12 = F.interpolate(out_flow12, (h,w)) ## upsampling flow to image size
out_flow23 = F.interpolate(out_flow23, (h,w)) ## upsampling flow to image size
out_image = []
im1 = train_loader[:,0] ## the 1st image of the triplets in the batch
im2 = train_loader[:,1] ## the 2nd image of the triplets in the batch
im3 = train_loader[:,2] ## the 3rd image of the triplets in the batch
# weights = [0.32, 0.08, 0.02, 0.01, 0.005]
loss = 0
combined_loss = []
warped_img12 = image_warp(im1,out_flow12)
warped_img23 = image_warp(im2,out_flow23)
criterion = torch.nn.MSELoss()
loss12 = criterion(im2,warped_img12)
loss23 = criterion(im3,warped_img23)
return loss12, loss23, warped_img12, warped_img23, out_flow12, out_flow23
def recons_img(input_image, F):
inp1 = K.placeholder(shape=input_image.shape)
F_in = K.placeholder(shape=F.shape)
input1_rec = image_warp(inp1, F, num_batch=F.shape[0])
f1 = K.function(inputs=[inp1, F_in], outputs=[input1_rec])
out = f1([input_image, F])
return out
def compile_model(model, lambda1=0.05):
s1 = tf.get_variable("sig1",
trainable=True,
initializer=tf.constant([0.3]))
s2 = tf.get_variable("sig2",
trainable=True,
initializer=tf.constant([0.7]))
s1_2 = s1 * s1
s2_2 = s1 * s1
I1 = model.inputs[0]
I2 = model.inputs[1]
o1 = model.outputs[0]
# this is to calculate the inverse_warp
o2 = image_warp(-o1, o1)
I2_rec = image_warp(I1, o1)
I1_rec = image_warp(I2, o2)
ux, uy = grad_xy(o1[:, :, :, :1])
vx, vy = grad_xy(o1[:, :, :, 1:2])
sm_loss = (K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)))
# re_loss_mse = K.mean(K.square(I2 - input1_rec))
re_loss1 = DSSIMObjective(kernel_size=50)(I2, I2_rec)
re_loss2 = DSSIMObjective(kernel_size=50)(I1, I1_rec)
re_loss = re_loss1 + re_loss2
total_loss = (1 / s1_2) * re_loss + (1 / s2_2) * sm_loss + K.log(
s1_2) + K.log(s2_2)
model = Model(inputs=[I1, I2], outputs=[o1])
model.add_loss(total_loss)
model.compile(optimizer=keras.optimizers.Adadelta(
lr=1.0, rho=0.95, epsilon=None, decay=0.0))
return model
def compile_model(model1,
lambda_smoothness=0.01,
lambda_ssim=5,
lambda_mse=0.0002,
lambda_flow=0.0001,
occ_punishment=0.0002):
i1 = model1.inputs[0]
i2 = model1.inputs[1]
o1 = model1.outputs[0]
o2 = model1.outputs[1]
###--------Flow_continuity------------------------------------------------
double_recon1 = image_warp((image_warp(i1, o2)), o1)
double_recon2 = image_warp((image_warp(i2, o1)), o2)
mse_image_recon = K.mean(K.square(i1 - double_recon1)) + K.mean(
K.square(i2 - double_recon2))
###--------Smoothness--------------------------------------------
g1 = tf.image.rgb_to_grayscale(i1[:, :, :, :])
i1x, i1y = grad_xy(g1[:, :, :, :1])
fm = flow_mag_tensor(o1)
fmx, fmy = grad_xy(fm[:, :, :])
# ux,uy=grad_xy(o1[:,:,:,:1])
# vx,vy=grad_xy(o1[:,:,:,1:2])
# body_sm_loss = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy))
edge_sm_new = K.abs(fmx) * (1 - K.exp(-(K.abs(i1y)))) + K.abs(fmy) * (
1 - K.exp(-(K.abs(i1x))))
# edge_sm_old = K.abs(fmx)*K.exp(-(K.abs(i1x))) + K.abs(fmy)*K.exp(-(K.abs(i1y)))
# sm_loss = body_sm_loss*(0.01) + edge_sm_old*(0.01)
###--------Reconstruction_loss_ssim_(occlusion_not considered)
i1_rec = image_warp(i2, o1)
i2_rec = image_warp(i1, o2)
###-----contrast_and_luminous-----------------------------------
# CL_loss = DSSIM_updated(i2,i2_rec) + DSSIM_updated(i1,i1_rec)
###-------------------------------------------------------------
#gradient loss
mse = K.mean(K.square(i2 - i2_rec)) + K.mean(K.square(i1 - i1_rec))
i1_ux, i1_uy = grad_xy(i1)
i2_ux, i2_uy = grad_xy(i2)
i1_rec_ux, i1_rec_uy = grad_xy(i1_rec)
i2_rec_ux, i2_rec_uy = grad_xy(i2_rec)
mse_grad = K.mean(K.square(i2_ux - i2_rec_ux)) + K.mean(
K.square(i2_uy - i2_rec_uy)) + K.mean(
K.square(i1_ux - i1_rec_ux)) + K.mean(K.square(i1_uy - i1_rec_uy))
grad_loss = mse + mse_grad
total_loss = edge_sm_new * (0.1) + mse_image_recon + grad_loss #+ CL_loss
model1.add_loss(total_loss)
model1.compile(optimizer="adadelta")
return model1
def compile_model_new(model, b_size, lambda1=0.002):
"""
session=tf.Session()
session.run(tf.global_variables_initializer())
var = [v for v in tf.trainable_variables() if v.name == "sig1:0"][0]
session.run(var)
"""
#s1 = tf.get_variable("sig1", shape=(1,), trainable=True,initializer=tf.constant([0.3]))
#s2 = tf.get_variable("sig2", shape=(1,), trainable=True,initializer=tf.constant([0.7]))
#s1 = tf.get_variable("sig1", trainable=True,initializer=tf.constant([0.3]))
#s2 = tf.get_variable("sig2", trainable=True,initializer=tf.constant([0.7]))
#s1_2=s1*s1
#s2_2=s1*s1
input1_rec = image_warp(model.inputs[0],
model.outputs[0],
num_batch=b_size)
input0_rec = image_warp(model.inputs[1],
-model.outputs[0],
num_batch=b_size)
ux, uy = grad_xy(model.outputs[0][:, :, :, :1])
vx, vy = grad_xy(model.outputs[0][:, :, :, 1:2])
sm_loss = lambda1 * (K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)))
re_loss = DSSIMObjective(kernel_size=50)(model.inputs[1], input1_rec)
loss_mse = K.mean(K.square(model.outputs[0] - model.inputs[1]))
#total_loss=(1/s1_2)*re_loss+(1/s2_2)*sm_loss+K.log(s1_2)+K.log(s2_2)
total_loss = lambda1 * sm_loss + re_loss + loss_mse
model.add_loss(total_loss)
model.compile(optimizer='rmsprop')
return model
def model():
# x = tf.placeholder(tf.float32, shape=(4,436,1024,6), name='x')
x1 = tf.placeholder(tf.float32, shape=(4,436,1024,3), name='x1')
x2 = tf.placeholder(tf.float32, shape=(4,436,1024,3), name='x2')
W1 = tf.get_variable("W1", [5, 5, 6, 12], initializer = tf.contrib.layers.xavier_initializer())
W2 = tf.get_variable("W2", [5, 5, 12, 24], initializer = tf.contrib.layers.xavier_initializer())
W3 = tf.get_variable("W3", [5, 5, 24, 12], initializer = tf.contrib.layers.xavier_initializer())
W3u = tf.get_variable("W3u", [5, 5, 12, 12], initializer = tf.contrib.layers.xavier_initializer())
W4 = tf.get_variable("W4", [5, 5, 12, 6], initializer = tf.contrib.layers.xavier_initializer())
W4u = tf.get_variable("W4u", [5, 5, 6, 6], initializer = tf.contrib.layers.xavier_initializer())
W5 = tf.get_variable("W5", [5, 5, 6, 2], initializer = tf.contrib.layers.xavier_initializer())
# W1 = parameters['W1']
# W2 = parameters['W2']
# W3 = parameters['W3']
# W3u = parameters['W3u']
# W4 = parameters['W4']
# W4u = parameters['W4u']
# W5 = parameters['W5']
x = tf.concat([x1,x2],3)
Z1 = tf.nn.conv2d(x, W1, strides=[1,1,1,1], padding="SAME")
A1 = tf.nn.relu(Z1)
P1 = tf.nn.max_pool(A1, ksize=[1,2,2,1], strides=[1,2,2,1], padding="VALID")
Z2 = tf.nn.conv2d(P1, W2, strides=[1,1,1,1], padding="SAME")
A2 = tf.nn.relu(Z2)
P2 = tf.nn.max_pool(A2, ksize=[1,2,2,1], strides=[1,2,2,1], padding="VALID")
Z3 = tf.nn.conv2d(P2,W3, strides=[1,1,1,1], padding="SAME")
A3 = tf.nn.relu(Z3)
P3 = tf.nn.conv2d_transpose(A3,W3u, output_shape=[4,218,512,12],strides=[1,2,2,1])#, padding="SAME")
Z4 = tf.nn.conv2d(P3,W4, strides=[1,1,1,1], padding="SAME")
A4 = tf.nn.relu(Z4)
P4 = tf.nn.conv2d_transpose(A4,W4u, output_shape=[4,436,1024,6],strides=[1,2,2,1])#, padding="SAME")
Z5 = tf.nn.conv2d(P4,W5, strides=[1,1,1,1], padding="SAME")
F = tf.nn.relu(Z5)
with tf.variable_scope('image_warp') as scope:
# x1 = x[:,:,:,:3]
# x2 = x[:,:,:,3:]
warped = image_warp(x1,F)
loss = tf.reduce_mean(tf.square(warped - x2))
return loss, x1, x2, F, warped
def c_grad_rec(y_true,y_pred,model):
lambda1 = 0.5
input1_rec=image_warp(model.inputs[0],model.outputs[0])
# input0_rec=image_warp(model.inputs[1],-model.outputs[0],num_batch = 2)
ux,uy=grad_xy(model.outputs[0][:,:,:,:1])
vx,vy=grad_xy(model.outputs[0][:,:,:,1:2])
sm_loss=lambda1*(K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy)))
re_loss_mse = K.mean(K.square(model.inputs[1] - input1_rec))
total_loss = lambda1*sm_loss+re_loss_mse
return total_loss
def occLoss(i1,i2,o1, occ_punishment =0.1): i1 = tf.convert_to_tensor(i1) i2 = tf.convert_to_tensor(i2) o1 = tf.convert_to_tensor(o1) o2 = image_warp(-o1,o1) oxf, oxb = mask(i1,i2,o1,o2) mask_f = oxf[:,:,:,0] mask_b = oxb[:,:,:,1] err_f, err_b = photometric_error(i1,i2,o1,o2) occ_loss1 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_f), mask_f)))/(436*1024) occ_loss2 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_b), mask_b)))/(436*1024) occ_punish1 = tf.multiply(tf.reduce_sum(tf.cast(mask_f, tf.float32)),occ_punishment) occ_punish2 = tf.multiply(tf.reduce_sum(tf.cast(mask_b, tf.float32)),occ_punishment) occ_loss = occ_loss1 + occ_loss2 + occ_punish1 + occ_punish2 return occ_loss
def photometric_error(i1,i2,o1,o2): err_f = tf.reduce_sum(tf.subtract(i1, image_warp(i2,o2)),-1) err_b = tf.reduce_sum(tf.subtract(i2, image_warp(i1,o1)),-1) return err_f, err_b
# f=K.function(inputs=[I,F],outputs=[I2])
return f
###############-------------------------------------------------
imgen = ImageSequence_fixed()
[X1, X2], Y = imgen.__getitem__()
f = return_func()
X1_out = f([X1, Y])
plt.imshow(X1[0])
im_rec = image_warp(X1[0], Y[0])
# plt.figure("rec")
# plt.imshow(im_rec)
# def read_image(file_path):
# Img=misc.imread(file_path)
# Img=resize(Img,(436,1024))
# return Img
# I1=read_image("../data/frame_0001.png")
# I2=read_image("../data/frame_0001.png")
# x_batch = []
# x_batch.append([I1,I2])
# x_batch = np.array(x_batch, np.float32)
def compile_model(model1,
lambda_smoothness=0.01,
lambda_ssim=5,
lambda_mse=0.0002,
lambda_flow=0.0001,
occ_punishment=0.0002):
i1 = model1.inputs[0]
i2 = model1.inputs[1]
o1 = model1.outputs[0]
o2 = model1.outputs[1]
#------flow consistiancy loss------
# grid=return_grid(batch_size=2)
# grid_recon1 = image_warp((image_warp(grid,o2)),o1)
# grid_recon2 = image_warp((image_warp(grid,o1)),o2)
#
# grid_ux,grid_uy = grad_xy(grid)
# grid_rec_ux,grid_rec_uy = grad_xy(grid_recon1)
# grid_rec_ux,grid_rec_uy = grad_xy(grid_recon2)
# mse_grid_grad = K.mean(K.square(grid_ux-grid_rec_ux)) + K.mean(K.square(grid_ux-grid_rec_ux)) + K.mean(K.square(grid_uy-grid_rec_uy)) + K.mean(K.square(grid_uy-grid_rec_uy))
# mse_grid = K.mean(K.square(grid-grid_recon1)) + K.mean(K.square(grid-grid_recon2))
###--------Flow_smoothness------------------------------------------------
double_recon1 = image_warp((image_warp(i1, o2)), o1)
double_recon2 = image_warp((image_warp(i2, o1)), o2)
mse_image_recon = K.mean(K.square(i1 - double_recon1)) + K.mean(
K.square(i2 - double_recon2))
###--------Gradient_smoothness--------------------------------------------
ux, uy = grad_xy(o1[:, :, :, :1])
vx, vy = grad_xy(o1[:, :, :, 1:2])
#-----------------------------------
# sm_loss_o1 = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy))
# ux,uy=grad_xy(o2[:,:,:,:1])
# vx,vy=grad_xy(o2[:,:,:,1:2])
# sm_loss_o2 = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy))
# sm_loss = (sm_loss_o1 + sm_loss_o2)
###--------Spatial_smoothness---------------------------------------------
g1 = tf.image.rgb_to_grayscale(i1[:, :, :, :])
i1x, i1y = grad_xy(g1[:, :, :, :1])
# edge_sm = K.abs(ux)*K.exp(-(K.abs(i1x))) + K.abs(vy)*K.exp(-(K.abs(i1y)))
edge_sm = K.abs(ux) * (1 - K.exp(-(K.abs(i1y)))) + K.abs(vy) * (
1 - K.exp(-(K.abs(i1x))))
###--------Reconstruction_loss_ssim_(occlusion_not considered)
i1_rec = image_warp(i2, o1)
i2_rec = image_warp(i1, o2)
# """
#SSIM LOSS
# re_loss1=DSSIMObjective(kernel_size=50)(i2,i2_rec)
# re_loss2=DSSIMObjective(kernel_size=50)(i1,i1_rec)
# re_loss_ssim = (re_loss1 + re_loss2)
# """
#gradient loss
mse = K.mean(K.square(i2 - i2_rec)) + K.mean(K.square(i1 - i1_rec))
i1_ux, i1_uy = grad_xy(i1)
i2_ux, i2_uy = grad_xy(i2)
i1_rec_ux, i1_rec_uy = grad_xy(i1_rec)
i2_rec_ux, i2_rec_uy = grad_xy(i2_rec)
mse_grad = K.mean(K.square(i2_ux - i2_rec_ux)) + K.mean(
K.square(i2_uy - i2_rec_uy)) + K.mean(
K.square(i1_ux - i1_rec_ux)) + K.mean(K.square(i1_uy - i1_rec_uy))
grad_loss = mse + mse_grad + mse_image_recon
#total_loss = sm_loss + occ_loss + occ_punish + re_loss_ssim + flow_loss
# total_loss = grad_loss+0.001*sm_loss + 0.1*mse_grid
# total_loss = grad_loss #+ edge_sm*(0.1) + re_loss_ssim
# total_loss = grad_loss + edge_sm*(0.1) + mse_image_recon
total_loss = grad_loss + mse_image_recon + edge_sm * (0.1)
#### model = Model(inputs=[i1,i2], outputs=[o1])
model1.add_loss(total_loss)
model1.compile(optimizer="adadelta")
return model1
def flow_error(o1,o2): ff = o1 + image_warp(o2,o2) fb = o2 + image_warp(o1,o1) return ff, fb
def compile_model(model1,
lambda_smoothness=0.01,
lambda_ssim=5,
lambda_mse=0.0002,
lambda_flow=0.0001,
occ_punishment=0.0002):
i1 = model1.inputs[0]
i2 = model1.inputs[1]
o1 = model1.outputs[0]
o2 = model1.outputs[1]
oxf, oxb = mask(i1, i2, o1, o2)
mask_f = oxf[:, :, :, 0]
mask_b = oxb[:, :, :, 1]
err_f, err_b = photometric_error(i1, i2, o1, o2)
flow_f, flow_b = flow_error(o1, o2)
###--------Occlusion_aware_mse_rec_image-------------------------------------
occ_loss1 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_f),
mask_f))) # /(436*1024)
occ_loss2 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_b),
mask_b))) # /(436*1024)
occ_loss = (occ_loss1 + occ_loss2) * lambda_mse
###--------Occlusion_aware_mse_flow------------------------------------
flow_loss1 = tf.reduce_sum(tf.boolean_mask(charbonnier(flow_f), mask_f))
flow_loss2 = tf.reduce_sum(tf.boolean_mask(charbonnier(flow_b), mask_f))
flow_loss = (flow_loss1 + flow_loss2) * lambda_flow
###--------Punishment_for_occlusion-----------------------------------------
occ_punish1 = tf.multiply(tf.reduce_sum(tf.cast(mask_f, tf.float32)),
occ_punishment)
occ_punish2 = tf.multiply(tf.reduce_sum(tf.cast(mask_b, tf.float32)),
occ_punishment)
occ_punish = (occ_punish1 + occ_punish2)
###--------Gradient_smoothness--------------------------------------------
ux, uy = grad_xy(o1[:, :, :, :1])
vx, vy = grad_xy(o1[:, :, :, 1:2])
sm_loss_o1 = K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy))
ux, uy = grad_xy(o2[:, :, :, :1])
vx, vy = grad_xy(o2[:, :, :, 1:2])
sm_loss_o2 = K.mean(
K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy))
sm_loss = (sm_loss_o1 + sm_loss_o2) * lambda_smoothness
### Reconstruction_loss_ssim_(occlusion_not considered)
i2_rec = image_warp(i1, o1)
i1_rec = image_warp(i2, o2)
re_loss1 = DSSIMObjective(kernel_size=50)(i2, i2_rec)
re_loss2 = DSSIMObjective(kernel_size=50)(i1, i1_rec)
re_loss_ssim = (re_loss1 + re_loss2) * lambda_ssim
# s1 = tf.get_variable("sig1", trainable=True,initializer=tf.constant([1]))
# s2 = tf.get_variable("sig2", trainable=True,initializer=tf.constant([1]))
# s3 = tf.get_variable("sig3", trainable=True,initializer=tf.constant([1]))
# s4 = tf.get_variable("sig4", trainable=True,initializer=tf.constant([1]))
# s1_2=s1*s1
# s2_2=s2*s2
# s3_2=s3*s3
# s4_2=s4*s4
# loss=(1/s1_2)*re_loss_ssim +(1/s2_2)*sm_loss +(1/s3_2)*(occ_loss+occ_punishment) +(1/s4_2)*flow_loss
# sigma_punishment = K.log(s1_2) + K.log(s2_2) + K.log(s3_2) + K.log(s4_2)
# total_loss = loss + sigma_punishment
total_loss = sm_loss + occ_loss + occ_punish + re_loss_ssim + flow_loss
#### model = Model(inputs=[i1,i2], outputs=[o1])
model1.add_loss(total_loss)
model1.compile(optimizer=keras.optimizers.Adadelta(
lr=1.0, rho=0.95, epsilon=None, decay=0.0))
return model1