def compile_model(model, lambda1=0.05): I1 = model.inputs[0] I2 = model.inputs[1] o1 = model.outputs[0] # this is to calculate the inverse_warp o2 = image_warp(-o1, o1) I2_rec = image_warp(I1, o1) I1_rec = image_warp(I2, o2) ux, uy = grad_xy(o1[:, :, :, :1]) vx, vy = grad_xy(o1[:, :, :, 1:2]) sm_loss = lambda1 * (K.mean( K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy))) re_loss1 = DSSIMObjective(kernel_size=50)(I2, I2_rec) re_loss2 = DSSIMObjective(kernel_size=50)(I1, I1_rec) total_loss = lambda1 * sm_loss + re_loss1 + re_loss2 model = Model(inputs=[I1, I2], outputs=[o1]) model.add_loss(total_loss) model.compile(optimizer=keras.optimizers.Adadelta( lr=1.0, rho=0.95, epsilon=None, decay=0.0)) return model
def compile_model(model, lambda1=0.005): I1 = model.inputs[0] I2 = model.inputs[1] o1 = model.outputs[0] # this is to calculate the inverse_warp o2 = image_warp(-o1, o1) I2_rec = image_warp(I1, o1) I1_rec = image_warp(I2, o2) ux, uy = grad_xy(o1[:, :, :, :1]) vx, vy = grad_xy(o1[:, :, :, 1:2]) sm_loss = lambda1 * (K.mean( K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy))) re_loss_mse = K.mean(K.square(I2 - I1_rec)) # loss_mse = K.mean(K.square(model.outputs[0] - Y)) total_loss = lambda1 * sm_loss + re_loss_mse # + loss_mse model.add_loss(total_loss) model = Model(inputs=[I1, I2], outputs=[o1]) model.compile(loss="mse", optimizer='Adam') return model
def compile_model(model,lambda_smoothness = 0, lambda_flow=0.0001, lambda_mse=0, occ_punishment = 0): i1=model.inputs[0] i2=model.inputs[1] o1=model.outputs[0] o2 = image_warp(-o1,o1) oxf, oxb = mask(i1,i2,o1,o2) mask_f = oxf[:,:,:,0] mask_b = oxb[:,:,:,1] err_f, err_b = photometric_error(i1,i2,o1,o2) flow_f, flow_b = flow_error(o1,o2) ###--------Occlusion_aware_mse_rec_image------------------------------------- occ_loss1 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_f), mask_f)))#/(436*1024) occ_loss2 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_b), mask_b)))#/(436*1024) occ_loss = (occ_loss1 + occ_loss2)*lambda_mse ###--------Occlusion_aware_mse_flow------------------------------------ flow_loss1 = tf.reduce_sum(tf.boolean_mask(charbonnier(flow_f), mask_f)) flow_loss2 = tf.reduce_sum(tf.boolean_mask(charbonnier(flow_b), mask_f)) flow_loss = (flow_loss1 + flow_loss2)*lambda_flow ###--------Punishment_for_occlusion----------------------------------------- occ_punish1 = tf.multiply(tf.reduce_sum(tf.cast(mask_f, tf.float32)),occ_punishment) occ_punish2 = tf.multiply(tf.reduce_sum(tf.cast(mask_b, tf.float32)),occ_punishment) occ_punish = occ_punish1 + occ_punish2 ###--------Gradient_smoothness-------------------------------------------- ux,uy=grad_xy(o1[:,:,:,:1]) vx,vy=grad_xy(o1[:,:,:,1:2]) sm_loss_o1 = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy)) ux,uy=grad_xy(o2[:,:,:,:1]) vx,vy=grad_xy(o2[:,:,:,1:2]) sm_loss_o2 = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy)) sm_loss = (sm_loss_o1 + sm_loss_o2)*lambda_smoothness ### Reconstruction_loss_ssim_(occlusion_not considered) i2_rec=image_warp(i1,o1) i1_rec=image_warp(i2,o2) re_loss1=DSSIMObjective(kernel_size=50)(i2,i2_rec) re_loss2=DSSIMObjective(kernel_size=50)(i1,i1_rec) re_loss_ssim = re_loss1 + re_loss2 total_loss = sm_loss + occ_loss + occ_punish + re_loss_ssim + flow_loss model = Model(inputs=[i1,i2], outputs=[o1]) model.add_loss(total_loss) model.compile(optimizer=keras.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)) return model
def compile_model(model, lambda1=0.005): s1 = tf.get_variable("sig1", trainable=True, initializer=tf.constant([0.3])) s2 = tf.get_variable("sig2", trainable=True, initializer=tf.constant([0.7])) s1_2 = s1 * s1 s2_2 = s2 * s2 I1 = model.inputs[0] I2 = model.inputs[1] o1 = model.outputs[0] I2_rec = image_warp(I1, o1) I1_rec = image_warp(I2, -o1) ux, uy = grad_xy(o1[:, :, :, :1]) vx, vy = grad_xy(o1[:, :, :, 1:2]) sm_loss = (K.mean( K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy))) # re_loss_mse = K.mean(K.square(I2 - input1_rec)) re_loss1 = DSSIMObjective(kernel_size=50)(I2, I2_rec) re_loss2 = DSSIMObjective(kernel_size=50)(I1, I1_rec) ############################################### # input1_rec=image_warp(model.inputs[0],model.outputs[0],num_batch=b_size) # input0_rec=image_warp(model.inputs[1],-model.outputs[0],num_batch=b_size) # ux,uy=grad_xy(model.outputs[0][:,:,:,:1]) # vx,vy=grad_xy(model.outputs[0][:,:,:,1:2]) # sm_loss=lambda1*(K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy))) # re_loss=DSSIMObjective(kernel_size=50)(model.inputs[1],input1_rec) ################################################ # loss_mse = K.mean(K.square(model.outputs[0] - Y)) re_loss = re_loss1 + re_loss2 total_loss = (1 / s1_2) * re_loss + (1 / s2_2) * sm_loss + K.log( s1_2) + K.log(s2_2) model = Model(inputs=[I1, I2], outputs=[o1]) model.add_loss(total_loss) model.compile(loss="mse", optimizer=keras.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)) #model.compile(optimizer='rmsprop') return model
def mask(i1,i2,o1,o2, alpha1=0.01, alpha2=0.5): left_f = tf.square( o1 + image_warp(o2,o2)) right_f = alpha1*( tf.square(o1) + tf.square(image_warp(o2,o2))) + alpha2 oxf = tf.less(right_f, left_f) left_b = tf.square( o2 + image_warp(o1,o1) ) right_b = alpha1*( tf.square(o2) + tf.square(image_warp(o1,o1))) + alpha2 oxb = tf.less(right_b, right_b) return oxf, oxb
def train(train_loader, model, optimizer): # switch to train mode train_loader = torch.tensor(train_loader).type(torch.cuda.FloatTensor) ## train_loader.size = Bx3x(C==3)xHxW model.train() out_flow12 = model( train_loader[:, 0:2]) ## list of 5 optical flows not of image size out_flow23 = model( train_loader[:, 1:3]) ## list of 5 optical flows not of image size h, w = train_loader.size()[-2:] ## image size out_flow12 = [F.interpolate(oflow, (h, w)) for oflow in out_flow12] ## upsampling flow to image size out_flow23 = [F.interpolate(oflow, (h, w)) for oflow in out_flow23] ## upsampling flow to image size out_image = [] im1 = train_loader[:, 0] ## the 1st image of the triplets in the batch im2 = train_loader[:, 1] ## the 2nd image of the triplets in the batch im3 = train_loader[:, 2] ## the 3rd image of the triplets in the batch weights = [0.32, 0.08, 0.02, 0.01, 0.005] loss = 0 combined_loss = [] for i in range(0, len(out_flow12)): oflow12 = out_flow12[i] oflow23 = out_flow23[i] warped_img12 = image_warp(im1, oflow12) warped_img23 = image_warp(im2, oflow23) temp_loss = flow_loss(train_loader, warped_img12, warped_img23, oflow12, oflow23, weights[i]) loss += temp_loss combined_loss.append(temp_loss / weights[i]) # print("combined_loss",combined_loss) # total_loss = torch.sum(torch.tensor(combined_loss).type(torch.cuda.FloatTensor)) optimizer.zero_grad() loss.backward() optimizer.step() # print(loss.grad) # print(loss) return loss
def flow_loss(im1, im2, flow_fw): """Calculate the loss from the logits and the labels. Args: img1: tensor, float - the first frame. img2: tensor, float - the second frame. optical: tensor, float - the optical estimation from flownet. Returns: loss: Loss tensor of type float. """ with tf.name_scope('optical_flow_loss'): losses = {} im2_warped = image_warp(im2, flow_fw) #im1_warped = image_warp(im1, flow_bw) im_diff_fw = im1 - im2_warped #im_diff_bw = im2 - im1_warped mask_fw = tf.ones_like(im_diff_fw) losses['photo'] = (photometric_loss(im_diff_fw, mask_fw)) losses['grad'] = (gradient_loss(im1, im2_warped, mask_fw)) losses['smooth_1st'] = (smoothness_loss(flow_fw)) return losses
def return_func(): I = Input(shape=(436, 1024, 3)) F = Input(shape=(436, 1024, 2)) F_new = image_warp(-F, F) # I2=image_warp(I,F) # f=K.function(inputs=[I,F],outputs=[I2]) return f
def evaluate(train_loader, model): # switch to train mode train_loader = torch.tensor(train_loader).type(torch.cuda.FloatTensor) ## train_loader.size = Bx3x(C==3)xHxW model.eval() # optimizer.load_state_dict(checkpoint['optimizer_state_dict']) # epoch = checkpoint['epoch'] # loss = checkpoint['loss'] out_flow12 = model(train_loader[:,0:2]) ## list of 5 optical flows not of image size out_flow23 = model(train_loader[:,1:3]) ## list of 5 optical flows not of image size # print("out_flow12.shape",out_flow12.size()) h, w = train_loader.size()[-2:] ## image size print("h,w",h,w) out_flow12 = F.interpolate(out_flow12, (h,w)) ## upsampling flow to image size out_flow23 = F.interpolate(out_flow23, (h,w)) ## upsampling flow to image size out_image = [] im1 = train_loader[:,0] ## the 1st image of the triplets in the batch im2 = train_loader[:,1] ## the 2nd image of the triplets in the batch im3 = train_loader[:,2] ## the 3rd image of the triplets in the batch # weights = [0.32, 0.08, 0.02, 0.01, 0.005] loss = 0 combined_loss = [] warped_img12 = image_warp(im1,out_flow12) warped_img23 = image_warp(im2,out_flow23) criterion = torch.nn.MSELoss() loss12 = criterion(im2,warped_img12) loss23 = criterion(im3,warped_img23) return loss12, loss23, warped_img12, warped_img23, out_flow12, out_flow23
def recons_img(input_image, F): inp1 = K.placeholder(shape=input_image.shape) F_in = K.placeholder(shape=F.shape) input1_rec = image_warp(inp1, F, num_batch=F.shape[0]) f1 = K.function(inputs=[inp1, F_in], outputs=[input1_rec]) out = f1([input_image, F]) return out
def compile_model(model, lambda1=0.05): s1 = tf.get_variable("sig1", trainable=True, initializer=tf.constant([0.3])) s2 = tf.get_variable("sig2", trainable=True, initializer=tf.constant([0.7])) s1_2 = s1 * s1 s2_2 = s1 * s1 I1 = model.inputs[0] I2 = model.inputs[1] o1 = model.outputs[0] # this is to calculate the inverse_warp o2 = image_warp(-o1, o1) I2_rec = image_warp(I1, o1) I1_rec = image_warp(I2, o2) ux, uy = grad_xy(o1[:, :, :, :1]) vx, vy = grad_xy(o1[:, :, :, 1:2]) sm_loss = (K.mean( K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy))) # re_loss_mse = K.mean(K.square(I2 - input1_rec)) re_loss1 = DSSIMObjective(kernel_size=50)(I2, I2_rec) re_loss2 = DSSIMObjective(kernel_size=50)(I1, I1_rec) re_loss = re_loss1 + re_loss2 total_loss = (1 / s1_2) * re_loss + (1 / s2_2) * sm_loss + K.log( s1_2) + K.log(s2_2) model = Model(inputs=[I1, I2], outputs=[o1]) model.add_loss(total_loss) model.compile(optimizer=keras.optimizers.Adadelta( lr=1.0, rho=0.95, epsilon=None, decay=0.0)) return model
def compile_model(model1, lambda_smoothness=0.01, lambda_ssim=5, lambda_mse=0.0002, lambda_flow=0.0001, occ_punishment=0.0002): i1 = model1.inputs[0] i2 = model1.inputs[1] o1 = model1.outputs[0] o2 = model1.outputs[1] ###--------Flow_continuity------------------------------------------------ double_recon1 = image_warp((image_warp(i1, o2)), o1) double_recon2 = image_warp((image_warp(i2, o1)), o2) mse_image_recon = K.mean(K.square(i1 - double_recon1)) + K.mean( K.square(i2 - double_recon2)) ###--------Smoothness-------------------------------------------- g1 = tf.image.rgb_to_grayscale(i1[:, :, :, :]) i1x, i1y = grad_xy(g1[:, :, :, :1]) fm = flow_mag_tensor(o1) fmx, fmy = grad_xy(fm[:, :, :]) # ux,uy=grad_xy(o1[:,:,:,:1]) # vx,vy=grad_xy(o1[:,:,:,1:2]) # body_sm_loss = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy)) edge_sm_new = K.abs(fmx) * (1 - K.exp(-(K.abs(i1y)))) + K.abs(fmy) * ( 1 - K.exp(-(K.abs(i1x)))) # edge_sm_old = K.abs(fmx)*K.exp(-(K.abs(i1x))) + K.abs(fmy)*K.exp(-(K.abs(i1y))) # sm_loss = body_sm_loss*(0.01) + edge_sm_old*(0.01) ###--------Reconstruction_loss_ssim_(occlusion_not considered) i1_rec = image_warp(i2, o1) i2_rec = image_warp(i1, o2) ###-----contrast_and_luminous----------------------------------- # CL_loss = DSSIM_updated(i2,i2_rec) + DSSIM_updated(i1,i1_rec) ###------------------------------------------------------------- #gradient loss mse = K.mean(K.square(i2 - i2_rec)) + K.mean(K.square(i1 - i1_rec)) i1_ux, i1_uy = grad_xy(i1) i2_ux, i2_uy = grad_xy(i2) i1_rec_ux, i1_rec_uy = grad_xy(i1_rec) i2_rec_ux, i2_rec_uy = grad_xy(i2_rec) mse_grad = K.mean(K.square(i2_ux - i2_rec_ux)) + K.mean( K.square(i2_uy - i2_rec_uy)) + K.mean( K.square(i1_ux - i1_rec_ux)) + K.mean(K.square(i1_uy - i1_rec_uy)) grad_loss = mse + mse_grad total_loss = edge_sm_new * (0.1) + mse_image_recon + grad_loss #+ CL_loss model1.add_loss(total_loss) model1.compile(optimizer="adadelta") return model1
def compile_model_new(model, b_size, lambda1=0.002): """ session=tf.Session() session.run(tf.global_variables_initializer()) var = [v for v in tf.trainable_variables() if v.name == "sig1:0"][0] session.run(var) """ #s1 = tf.get_variable("sig1", shape=(1,), trainable=True,initializer=tf.constant([0.3])) #s2 = tf.get_variable("sig2", shape=(1,), trainable=True,initializer=tf.constant([0.7])) #s1 = tf.get_variable("sig1", trainable=True,initializer=tf.constant([0.3])) #s2 = tf.get_variable("sig2", trainable=True,initializer=tf.constant([0.7])) #s1_2=s1*s1 #s2_2=s1*s1 input1_rec = image_warp(model.inputs[0], model.outputs[0], num_batch=b_size) input0_rec = image_warp(model.inputs[1], -model.outputs[0], num_batch=b_size) ux, uy = grad_xy(model.outputs[0][:, :, :, :1]) vx, vy = grad_xy(model.outputs[0][:, :, :, 1:2]) sm_loss = lambda1 * (K.mean( K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy))) re_loss = DSSIMObjective(kernel_size=50)(model.inputs[1], input1_rec) loss_mse = K.mean(K.square(model.outputs[0] - model.inputs[1])) #total_loss=(1/s1_2)*re_loss+(1/s2_2)*sm_loss+K.log(s1_2)+K.log(s2_2) total_loss = lambda1 * sm_loss + re_loss + loss_mse model.add_loss(total_loss) model.compile(optimizer='rmsprop') return model
def model(): # x = tf.placeholder(tf.float32, shape=(4,436,1024,6), name='x') x1 = tf.placeholder(tf.float32, shape=(4,436,1024,3), name='x1') x2 = tf.placeholder(tf.float32, shape=(4,436,1024,3), name='x2') W1 = tf.get_variable("W1", [5, 5, 6, 12], initializer = tf.contrib.layers.xavier_initializer()) W2 = tf.get_variable("W2", [5, 5, 12, 24], initializer = tf.contrib.layers.xavier_initializer()) W3 = tf.get_variable("W3", [5, 5, 24, 12], initializer = tf.contrib.layers.xavier_initializer()) W3u = tf.get_variable("W3u", [5, 5, 12, 12], initializer = tf.contrib.layers.xavier_initializer()) W4 = tf.get_variable("W4", [5, 5, 12, 6], initializer = tf.contrib.layers.xavier_initializer()) W4u = tf.get_variable("W4u", [5, 5, 6, 6], initializer = tf.contrib.layers.xavier_initializer()) W5 = tf.get_variable("W5", [5, 5, 6, 2], initializer = tf.contrib.layers.xavier_initializer()) # W1 = parameters['W1'] # W2 = parameters['W2'] # W3 = parameters['W3'] # W3u = parameters['W3u'] # W4 = parameters['W4'] # W4u = parameters['W4u'] # W5 = parameters['W5'] x = tf.concat([x1,x2],3) Z1 = tf.nn.conv2d(x, W1, strides=[1,1,1,1], padding="SAME") A1 = tf.nn.relu(Z1) P1 = tf.nn.max_pool(A1, ksize=[1,2,2,1], strides=[1,2,2,1], padding="VALID") Z2 = tf.nn.conv2d(P1, W2, strides=[1,1,1,1], padding="SAME") A2 = tf.nn.relu(Z2) P2 = tf.nn.max_pool(A2, ksize=[1,2,2,1], strides=[1,2,2,1], padding="VALID") Z3 = tf.nn.conv2d(P2,W3, strides=[1,1,1,1], padding="SAME") A3 = tf.nn.relu(Z3) P3 = tf.nn.conv2d_transpose(A3,W3u, output_shape=[4,218,512,12],strides=[1,2,2,1])#, padding="SAME") Z4 = tf.nn.conv2d(P3,W4, strides=[1,1,1,1], padding="SAME") A4 = tf.nn.relu(Z4) P4 = tf.nn.conv2d_transpose(A4,W4u, output_shape=[4,436,1024,6],strides=[1,2,2,1])#, padding="SAME") Z5 = tf.nn.conv2d(P4,W5, strides=[1,1,1,1], padding="SAME") F = tf.nn.relu(Z5) with tf.variable_scope('image_warp') as scope: # x1 = x[:,:,:,:3] # x2 = x[:,:,:,3:] warped = image_warp(x1,F) loss = tf.reduce_mean(tf.square(warped - x2)) return loss, x1, x2, F, warped
def c_grad_rec(y_true,y_pred,model): lambda1 = 0.5 input1_rec=image_warp(model.inputs[0],model.outputs[0]) # input0_rec=image_warp(model.inputs[1],-model.outputs[0],num_batch = 2) ux,uy=grad_xy(model.outputs[0][:,:,:,:1]) vx,vy=grad_xy(model.outputs[0][:,:,:,1:2]) sm_loss=lambda1*(K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy))) re_loss_mse = K.mean(K.square(model.inputs[1] - input1_rec)) total_loss = lambda1*sm_loss+re_loss_mse return total_loss
def occLoss(i1,i2,o1, occ_punishment =0.1): i1 = tf.convert_to_tensor(i1) i2 = tf.convert_to_tensor(i2) o1 = tf.convert_to_tensor(o1) o2 = image_warp(-o1,o1) oxf, oxb = mask(i1,i2,o1,o2) mask_f = oxf[:,:,:,0] mask_b = oxb[:,:,:,1] err_f, err_b = photometric_error(i1,i2,o1,o2) occ_loss1 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_f), mask_f)))/(436*1024) occ_loss2 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_b), mask_b)))/(436*1024) occ_punish1 = tf.multiply(tf.reduce_sum(tf.cast(mask_f, tf.float32)),occ_punishment) occ_punish2 = tf.multiply(tf.reduce_sum(tf.cast(mask_b, tf.float32)),occ_punishment) occ_loss = occ_loss1 + occ_loss2 + occ_punish1 + occ_punish2 return occ_loss
def photometric_error(i1,i2,o1,o2): err_f = tf.reduce_sum(tf.subtract(i1, image_warp(i2,o2)),-1) err_b = tf.reduce_sum(tf.subtract(i2, image_warp(i1,o1)),-1) return err_f, err_b
# f=K.function(inputs=[I,F],outputs=[I2]) return f ###############------------------------------------------------- imgen = ImageSequence_fixed() [X1, X2], Y = imgen.__getitem__() f = return_func() X1_out = f([X1, Y]) plt.imshow(X1[0]) im_rec = image_warp(X1[0], Y[0]) # plt.figure("rec") # plt.imshow(im_rec) # def read_image(file_path): # Img=misc.imread(file_path) # Img=resize(Img,(436,1024)) # return Img # I1=read_image("../data/frame_0001.png") # I2=read_image("../data/frame_0001.png") # x_batch = [] # x_batch.append([I1,I2]) # x_batch = np.array(x_batch, np.float32)
def compile_model(model1, lambda_smoothness=0.01, lambda_ssim=5, lambda_mse=0.0002, lambda_flow=0.0001, occ_punishment=0.0002): i1 = model1.inputs[0] i2 = model1.inputs[1] o1 = model1.outputs[0] o2 = model1.outputs[1] #------flow consistiancy loss------ # grid=return_grid(batch_size=2) # grid_recon1 = image_warp((image_warp(grid,o2)),o1) # grid_recon2 = image_warp((image_warp(grid,o1)),o2) # # grid_ux,grid_uy = grad_xy(grid) # grid_rec_ux,grid_rec_uy = grad_xy(grid_recon1) # grid_rec_ux,grid_rec_uy = grad_xy(grid_recon2) # mse_grid_grad = K.mean(K.square(grid_ux-grid_rec_ux)) + K.mean(K.square(grid_ux-grid_rec_ux)) + K.mean(K.square(grid_uy-grid_rec_uy)) + K.mean(K.square(grid_uy-grid_rec_uy)) # mse_grid = K.mean(K.square(grid-grid_recon1)) + K.mean(K.square(grid-grid_recon2)) ###--------Flow_smoothness------------------------------------------------ double_recon1 = image_warp((image_warp(i1, o2)), o1) double_recon2 = image_warp((image_warp(i2, o1)), o2) mse_image_recon = K.mean(K.square(i1 - double_recon1)) + K.mean( K.square(i2 - double_recon2)) ###--------Gradient_smoothness-------------------------------------------- ux, uy = grad_xy(o1[:, :, :, :1]) vx, vy = grad_xy(o1[:, :, :, 1:2]) #----------------------------------- # sm_loss_o1 = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy)) # ux,uy=grad_xy(o2[:,:,:,:1]) # vx,vy=grad_xy(o2[:,:,:,1:2]) # sm_loss_o2 = K.mean(K.abs(ux*ux)+ K.abs(uy*uy)+ K.abs(vx*vx)+ K.abs(vy*vy)) # sm_loss = (sm_loss_o1 + sm_loss_o2) ###--------Spatial_smoothness--------------------------------------------- g1 = tf.image.rgb_to_grayscale(i1[:, :, :, :]) i1x, i1y = grad_xy(g1[:, :, :, :1]) # edge_sm = K.abs(ux)*K.exp(-(K.abs(i1x))) + K.abs(vy)*K.exp(-(K.abs(i1y))) edge_sm = K.abs(ux) * (1 - K.exp(-(K.abs(i1y)))) + K.abs(vy) * ( 1 - K.exp(-(K.abs(i1x)))) ###--------Reconstruction_loss_ssim_(occlusion_not considered) i1_rec = image_warp(i2, o1) i2_rec = image_warp(i1, o2) # """ #SSIM LOSS # re_loss1=DSSIMObjective(kernel_size=50)(i2,i2_rec) # re_loss2=DSSIMObjective(kernel_size=50)(i1,i1_rec) # re_loss_ssim = (re_loss1 + re_loss2) # """ #gradient loss mse = K.mean(K.square(i2 - i2_rec)) + K.mean(K.square(i1 - i1_rec)) i1_ux, i1_uy = grad_xy(i1) i2_ux, i2_uy = grad_xy(i2) i1_rec_ux, i1_rec_uy = grad_xy(i1_rec) i2_rec_ux, i2_rec_uy = grad_xy(i2_rec) mse_grad = K.mean(K.square(i2_ux - i2_rec_ux)) + K.mean( K.square(i2_uy - i2_rec_uy)) + K.mean( K.square(i1_ux - i1_rec_ux)) + K.mean(K.square(i1_uy - i1_rec_uy)) grad_loss = mse + mse_grad + mse_image_recon #total_loss = sm_loss + occ_loss + occ_punish + re_loss_ssim + flow_loss # total_loss = grad_loss+0.001*sm_loss + 0.1*mse_grid # total_loss = grad_loss #+ edge_sm*(0.1) + re_loss_ssim # total_loss = grad_loss + edge_sm*(0.1) + mse_image_recon total_loss = grad_loss + mse_image_recon + edge_sm * (0.1) #### model = Model(inputs=[i1,i2], outputs=[o1]) model1.add_loss(total_loss) model1.compile(optimizer="adadelta") return model1
def flow_error(o1,o2): ff = o1 + image_warp(o2,o2) fb = o2 + image_warp(o1,o1) return ff, fb
def compile_model(model1, lambda_smoothness=0.01, lambda_ssim=5, lambda_mse=0.0002, lambda_flow=0.0001, occ_punishment=0.0002): i1 = model1.inputs[0] i2 = model1.inputs[1] o1 = model1.outputs[0] o2 = model1.outputs[1] oxf, oxb = mask(i1, i2, o1, o2) mask_f = oxf[:, :, :, 0] mask_b = oxb[:, :, :, 1] err_f, err_b = photometric_error(i1, i2, o1, o2) flow_f, flow_b = flow_error(o1, o2) ###--------Occlusion_aware_mse_rec_image------------------------------------- occ_loss1 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_f), mask_f))) # /(436*1024) occ_loss2 = (tf.reduce_sum(tf.boolean_mask(charbonnier(err_b), mask_b))) # /(436*1024) occ_loss = (occ_loss1 + occ_loss2) * lambda_mse ###--------Occlusion_aware_mse_flow------------------------------------ flow_loss1 = tf.reduce_sum(tf.boolean_mask(charbonnier(flow_f), mask_f)) flow_loss2 = tf.reduce_sum(tf.boolean_mask(charbonnier(flow_b), mask_f)) flow_loss = (flow_loss1 + flow_loss2) * lambda_flow ###--------Punishment_for_occlusion----------------------------------------- occ_punish1 = tf.multiply(tf.reduce_sum(tf.cast(mask_f, tf.float32)), occ_punishment) occ_punish2 = tf.multiply(tf.reduce_sum(tf.cast(mask_b, tf.float32)), occ_punishment) occ_punish = (occ_punish1 + occ_punish2) ###--------Gradient_smoothness-------------------------------------------- ux, uy = grad_xy(o1[:, :, :, :1]) vx, vy = grad_xy(o1[:, :, :, 1:2]) sm_loss_o1 = K.mean( K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)) ux, uy = grad_xy(o2[:, :, :, :1]) vx, vy = grad_xy(o2[:, :, :, 1:2]) sm_loss_o2 = K.mean( K.abs(ux * ux) + K.abs(uy * uy) + K.abs(vx * vx) + K.abs(vy * vy)) sm_loss = (sm_loss_o1 + sm_loss_o2) * lambda_smoothness ### Reconstruction_loss_ssim_(occlusion_not considered) i2_rec = image_warp(i1, o1) i1_rec = image_warp(i2, o2) re_loss1 = DSSIMObjective(kernel_size=50)(i2, i2_rec) re_loss2 = DSSIMObjective(kernel_size=50)(i1, i1_rec) re_loss_ssim = (re_loss1 + re_loss2) * lambda_ssim # s1 = tf.get_variable("sig1", trainable=True,initializer=tf.constant([1])) # s2 = tf.get_variable("sig2", trainable=True,initializer=tf.constant([1])) # s3 = tf.get_variable("sig3", trainable=True,initializer=tf.constant([1])) # s4 = tf.get_variable("sig4", trainable=True,initializer=tf.constant([1])) # s1_2=s1*s1 # s2_2=s2*s2 # s3_2=s3*s3 # s4_2=s4*s4 # loss=(1/s1_2)*re_loss_ssim +(1/s2_2)*sm_loss +(1/s3_2)*(occ_loss+occ_punishment) +(1/s4_2)*flow_loss # sigma_punishment = K.log(s1_2) + K.log(s2_2) + K.log(s3_2) + K.log(s4_2) # total_loss = loss + sigma_punishment total_loss = sm_loss + occ_loss + occ_punish + re_loss_ssim + flow_loss #### model = Model(inputs=[i1,i2], outputs=[o1]) model1.add_loss(total_loss) model1.compile(optimizer=keras.optimizers.Adadelta( lr=1.0, rho=0.95, epsilon=None, decay=0.0)) return model1