def forward(self, in0, in1): in0_sc = (in0 - self.shift.expand_as(in0)) / self.scale.expand_as(in0) in1_sc = (in1 - self.shift.expand_as(in0)) / self.scale.expand_as(in0) if (self.version == '0.0'): # v0.0 - original release had a bug, where input was not scaled in0_input = in0 in1_input = in1 else: # v0.1 in0_input = in0_sc in1_input = in1_sc if (self.pnet_tune): outs0 = self.net.forward(in0_input) outs1 = self.net.forward(in1_input) else: outs0 = self.net[0].forward(in0_input) outs1 = self.net[0].forward(in1_input) feats0 = {} feats1 = {} diffs = [0] * len(outs0) for (kk, out0) in enumerate(outs0): feats0[kk] = util.normalize_tensor(outs0[kk]) feats1[kk] = util.normalize_tensor(outs1[kk]) diffs[kk] = (feats0[kk] - feats1[kk])**2 if self.spatial: lin_models = [ self.lin0, self.lin1, self.lin2, self.lin3, self.lin4 ] if (self.pnet_type == 'squeeze'): lin_models.extend([self.lin5, self.lin6]) res = [lin_models[kk].model(diffs[kk]) for kk in range(len(diffs))] return res val = torch.mean(torch.mean(self.lin0.model(diffs[0]), dim=3), dim=2) val = val + torch.mean(torch.mean(self.lin1.model(diffs[1]), dim=3), dim=2) val = val + torch.mean(torch.mean(self.lin2.model(diffs[2]), dim=3), dim=2) val = val + torch.mean(torch.mean(self.lin3.model(diffs[3]), dim=3), dim=2) val = val + torch.mean(torch.mean(self.lin4.model(diffs[4]), dim=3), dim=2) if (self.pnet_type == 'squeeze'): val = val + torch.mean( torch.mean(self.lin5.model(diffs[5]), dim=3), dim=2) val = val + torch.mean( torch.mean(self.lin6.model(diffs[6]), dim=3), dim=2) val = val.view(val.size()[0], val.size()[1], 1, 1) return val
def distance(self, in1_tensor, in2_tensor): # The input must be 64x64 scale # Both Input1 and Input2 are input images with shape [N, H, W, C] ndarray # Batch Normalization input1_bn = (in1_tensor - tf.contrib.framework.broadcast_to( self.alexnetinnet.shift, shape=tf.shape(in1_tensor))) / tf.contrib.framework.broadcast_to( self.alexnetinnet.scale, shape=tf.shape(in1_tensor)) input2_bn = (in2_tensor - tf.contrib.framework.broadcast_to( self.alexnetinnet.shift, shape=tf.shape(in2_tensor))) / tf.contrib.framework.broadcast_to( self.alexnetinnet.scale, shape=tf.shape(in2_tensor)) # Resize the input to 68x68 to ensure the output of the first layer is the same to that in torch # Step 2. Manully padding the input to Nx68x68x3 # [N, H, W, C] input1_value_68 = tf.pad(input1_bn, paddings=tf.constant([[0, 0], [2, 2], [2, 2], [0, 0]]), mode='CONSTANT') input2_value_68 = tf.pad(input2_bn, paddings=tf.constant([[0, 0], [2, 2], [2, 2], [0, 0]]), mode='CONSTANT') relu_values1 = self.alexnet.forward(input1_value_68) relu_values2 = self.alexnet.forward(input2_value_68) # # Assert each output of the layers in alext has the correct shape. # assert relu_values1[0].shape[1:] == (15, 15, 64) # assert relu_values1[1].shape[1:] == (7, 7, 192) # assert relu_values1[2].shape[1:] == (3, 3, 384) # assert relu_values1[3].shape[1:] == (3, 3, 256) # assert relu_values1[4].shape[1:] == (3, 3, 256) diffs = [] for out1, out2 in zip(relu_values1, relu_values2): # Step 3. Normalize out1_norm = util.normalize_tensor(out1) out2_norm = util.normalize_tensor(out2) diffs.append((out1_norm - out2_norm)**2) # The difference is the same as that in pytorch convs_value = self.alexnetinnet.forward(diffs) result = tf.reduce_mean(tf.reduce_mean(convs_value[0], axis=2), axis=1) for conv in convs_value[1:]: result += tf.reduce_mean(tf.reduce_mean(conv, axis=2), axis=1) #result = tf.reshape(result, shape=(result.shape[0], 1, 1, result.shape[1])) result = tf.squeeze(result) return result
def forward(self, in0, in1, retPerLayer=False): # v0.0 - original release had a bug, where input was not scaled in0_input, in1_input = ( self.scaling_layer(in0), self.scaling_layer(in1)) if self.version == '0.1' else (in0, in1) outs0, outs1 = self.net.forward(in0_input), self.net.forward(in1_input) feats0, feats1, diffs = {}, {}, {} for kk in range(self.L): feats0[kk], feats1[kk] = util.normalize_tensor( outs0[kk]), util.normalize_tensor(outs1[kk]) diffs[kk] = (feats0[kk] - feats1[kk])**2 if (self.lpips): if (self.spatial): res = [ upsample(self.lins[kk].model(diffs[kk]), out_H=in0.shape[2]) for kk in range(self.L) ] else: res = [ spatial_average(self.lins[kk].model(diffs[kk]), keepdim=True) for kk in range(self.L) ] else: if (self.spatial): res = [ upsample(diffs[kk].sum(dim=1, keepdim=True), out_H=in0.shape[2]) for kk in range(self.L) ] else: res = [ spatial_average(diffs[kk].sum(dim=1, keepdim=True), keepdim=True) for kk in range(self.L) ] val = res[0] for l in range(1, self.L): val += res[l] if (retPerLayer): return (val, res) else: return val
def forward(self, in0, in1): in0_sc = (in0 - self.shift.expand_as(in0))/self.scale.expand_as(in0) in1_sc = (in1 - self.shift.expand_as(in0))/self.scale.expand_as(in0) if(self.pnet_tune): outs0 = self.net.forward(in0) outs1 = self.net.forward(in1) else: outs0 = self.net[0].forward(in0) outs1 = self.net[0].forward(in1) feats0 = {} feats1 = {} diffs = [0]*len(outs0) for (kk,out0) in enumerate(outs0): feats0[kk] = util.normalize_tensor(outs0[kk]) feats1[kk] = util.normalize_tensor(outs1[kk]) diffs[kk] = (feats0[kk]-feats1[kk])**2 if self.spatial: lin_models = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4] if(self.pnet_type=='squeeze'): lin_models.extend([self.lin5, self.lin6]) res = [lin_models[kk].model(diffs[kk]) for kk in range(len(diffs))] return res val = torch.mean(torch.mean(self.lin0.model(diffs[0]),dim=3),dim=2) val = val + torch.mean(torch.mean(self.lin1.model(diffs[1]),dim=3),dim=2) val = val + torch.mean(torch.mean(self.lin2.model(diffs[2]),dim=3),dim=2) val = val + torch.mean(torch.mean(self.lin3.model(diffs[3]),dim=3),dim=2) val = val + torch.mean(torch.mean(self.lin4.model(diffs[4]),dim=3),dim=2) if(self.pnet_type=='squeeze'): val = val + torch.mean(torch.mean(self.lin5.model(diffs[5]),dim=3),dim=2) val = val + torch.mean(torch.mean(self.lin6.model(diffs[6]),dim=3),dim=2) val = val.view(val.size()[0],val.size()[1],1,1) return val
def forward(self, in0, in1): in0_sc = (in0 - self.shift.expand_as(in0)) / self.scale.expand_as(in0) in1_sc = (in1 - self.shift.expand_as(in0)) / self.scale.expand_as(in0) if (self.pnet_tune): outs0 = self.net.forward(in0) outs1 = self.net.forward(in1) else: outs0 = self.net[0].forward(in0) outs1 = self.net[0].forward(in1) feats0 = {} feats1 = {} diffs = {} for (kk, out0) in enumerate(outs0): feats0[kk] = util.normalize_tensor(outs0[kk]) feats1[kk] = util.normalize_tensor(outs1[kk]) diffs[kk] = (feats0[kk] - feats1[kk])**2 val = torch.mean(torch.mean(self.lin0.model(diffs[0]), dim=3), dim=2) val = val + \ torch.mean(torch.mean(self.lin1.model(diffs[1]), dim=3), dim=2) val = val + \ torch.mean(torch.mean(self.lin2.model(diffs[2]), dim=3), dim=2) val = val + \ torch.mean(torch.mean(self.lin3.model(diffs[3]), dim=3), dim=2) val = val + \ torch.mean(torch.mean(self.lin4.model(diffs[4]), dim=3), dim=2) if (self.pnet_type == 'squeeze'): val = val + \ torch.mean(torch.mean(self.lin5.model(diffs[5]), dim=3), dim=2) val = val + \ torch.mean(torch.mean(self.lin6.model(diffs[6]), dim=3), dim=2) val = val.view(val.size()[0], val.size()[1], 1, 1) return val
def forward(self, in0, in1, retPerLayer=False): in0_sc = (in0 - self.shift.expand_as(in0)) / self.scale.expand_as(in0) in1_sc = (in1 - self.shift.expand_as(in0)) / self.scale.expand_as(in0) outs0 = self.net.forward(in0_sc) outs1 = self.net.forward(in1_sc) if (retPerLayer): all_scores = [] for (kk, out0) in enumerate(outs0): # # cur_score = (1.-util.cos_sim(outs0[kk],outs1[kk])) outs0_norm = util.normalize_tensor(outs0[kk]) outs1_norm = util.normalize_tensor(outs1[kk]) # outs0_norm = outs0[kk] # outs1_norm = outs1[kk] # outs0_mean = torch.mean(outs0_norm,dim=(2,3)) # outs1_mean = torch.mean(outs1_norm,dim=(2,3)) # mean_diff = torch.mean(torch.abs(torch.add(outs0_mean, -1, outs1_mean)),1) # cur_score = mean_diff # Fourier1 = torch.rfft(torch.reshape(outs0_norm,(outs0_norm.size()[3],outs0_norm.size()[2],outs0_norm.size()[1],outs0_norm.size()[0])),2,True,False) # Fourier2 = torch.rfft(torch.reshape(outs1_norm,(outs0_norm.size()[3], outs0_norm.size()[2], outs0_norm.size()[1], outs0_norm.size()[0])), 2, True, False) Fourier1 = torch.div(torch.rfft(outs0_norm, 2, False, False), (outs0_norm.size()[3]) * (outs0_norm.size()[2])) Fourier2 = torch.div(torch.rfft(outs1_norm, 2, False, False), (outs0_norm.size()[3]) * (outs0_norm.size()[2])) Fourier1 = Fourier1**2 Fourier2 = Fourier2**2 Fourier1 = torch.sqrt( torch.add(Fourier1[:, :, :, :, 0], 1, Fourier1[:, :, :, :, 1])) # Fourier1[(Fourier1==0).nonzero()] = 1 # Fourier1 = torch.add(Fourier1,1e-4) # Fourier1 = torch.log(Fourier1) Fourier2 = torch.sqrt( torch.add(Fourier2[:, :, :, :, 0], 1, Fourier2[:, :, :, :, 1])) # Fourier2[(Fourier2 == 0).nonzero()] = 1 # Fourier2 = torch.add(Fourier2, 1e-4) # Fourier2 = torch.log(Fourier2) factor = torch.zeros(outs0_norm.size()[2], outs0_norm.size()[3]) for i in range(outs0_norm.size()[2]): for j in range(outs0_norm.size()[3]): factor[i, j] = (i + j) factor = torch.exp(-1. * factor) factor = factor.expand_as(Fourier1).cuda() diff = torch.mul(torch.abs(torch.add(Fourier1, -1, Fourier2)), factor) # sum = torch.add(Fourier1,1,Fourier2) # norm_diff = torch.div(diff,sum+1e-8) mean_diff = torch.mean(torch.sum(diff, (2, 3)), 1) cur_score = mean_diff # cur_score = torch.mean(torch.abs(torch.add(outs0_mean, -1, outs1_mean)),1) # idx = list(range(1, outs0[kk].size()[2])) + [0] # tv_rows0 = torch.mean(torch.abs(torch.add(outs0_norm, -1, outs0_norm[:, :, idx, :])), (2, 3)) # tv_rows1 = torch.mean(torch.abs(torch.add(outs1_norm, -1, outs1_norm[:, :, idx, :])), (2, 3)) # tv_cols0 = torch.mean(torch.abs(torch.add(outs0_norm, -1, outs0_norm[:, :, :, idx])), (2, 3)) # tv_cols1 = torch.mean(torch.abs(torch.add(outs1_norm, -1, outs1_norm[:, :, :, idx])), (2, 3)) # total_tv0 = torch.add(tv_cols0, 1, tv_rows0) # total_tv1 = torch.add(tv_cols1, 1, tv_rows1) # tv_diff = 0.5*torch.mean(torch.abs(torch.add(total_tv0, -1, total_tv1)),1) # idx_up = list(range(1, outs0_norm.size()[2])) + [0] # idx_down = [outs0_norm.size()[2] - 1] + list(range(0, outs0_norm.size()[2] - 1)) # outs0_shift_up = outs0_norm[:, :, idx_up, :] # outs0_shift_down = outs0_norm[:, :, idx_down, :] # outs0_shift_right = outs0_norm[:, :, :, idx_up] # outs0_shift_left = outs0_norm[:, :, :, idx_down] # outs1_shift_up = outs1_norm[:, :, idx_up, :] # outs1_shift_down = outs1_norm[:, :, idx_down, :] # outs1_shift_right = outs1_norm[:, :, :, idx_up] # outs1_shift_left = outs1_norm[:, :, :, idx_down] # laplacian0 = torch.add(torch.add(torch.add(torch.add(outs0_shift_up, 1, outs0_shift_down), 1, outs0_shift_left), 1,outs0_shift_right), -4, outs0_norm) # laplacian1 = torch.add(torch.add(torch.add(torch.add(outs1_shift_up, 1, outs1_shift_down), 1, outs1_shift_left), 1,outs1_shift_right), -4, outs1_norm) # tot_lap0 = torch.mean(torch.abs(laplacian0), (2, 3)) # tot_lap1 = torch.mean(torch.abs(laplacian1), (2, 3)) if (kk == 0): val = 1. * cur_score else: val = val + cur_score if (retPerLayer): all_scores += [cur_score] if (retPerLayer): return (val, all_scores) else: return val
def forward(self, in0, in1): in0_sc = (in0 - self.shift.expand_as(in0)) / self.scale.expand_as(in0) in1_sc = (in1 - self.shift.expand_as(in0)) / self.scale.expand_as(in0) if (self.version == '0.0'): # v0.0 - original release had a bug, where input was not scaled in0_input = in0 in1_input = in1 else: # v0.1 in0_input = in0_sc in1_input = in1_sc if (self.pnet_tune): outs0 = self.net.forward(in0_input) outs1 = self.net.forward(in1_input) else: outs0 = self.net[0].forward(in0_input) outs1 = self.net[0].forward(in1_input) feats0 = {} feats1 = {} diffs = [0] * len(outs0) # outs0_mean = torch.mean(util.normalize_tensor(outs0[kk]), dim=(2, 3)) # outs1_mean = torch.mean(util.normalize_tensor(outs1[kk]), dim=(2, 3)) # cur_score = torch.mean(torch.abs(torch.add(outs0_mean, -1, outs1_mean)), 1) for (kk, out0) in enumerate(outs0): # feats0[kk] = util.normalize_tensor(outs0[kk]) # feats1[kk] = util.normalize_tensor(outs1[kk]) # diffs[kk] = (feats0[kk]-feats1[kk])**2 # outs0_norm = util.normalize_tensor(outs0[kk]) # outs1_norm = util.normalize_tensor(outs1[kk]) # idx = list(range(1, outs0[kk].size()[2])) + [0] # tv_rows0 = torch.sum(torch.abs(torch.add(outs0_norm, -1, outs0_norm[:, :, idx, :])), (2, 3), keepdim=True) # tv_rows1 = torch.sum(torch.abs(torch.add(outs1_norm, -1, outs1_norm[:, :, idx, :])), (2, 3), keepdim=True) # tv_cols0 = torch.sum(torch.abs(torch.add(outs0_norm, -1, outs0_norm[:, :, :, idx])), (2, 3), keepdim=True) # tv_cols1 = torch.sum(torch.abs(torch.add(outs1_norm, -1, outs1_norm[:, :, :, idx])), (2, 3), keepdim=True) # total_tv0 = torch.add(tv_cols0, 1, tv_rows0) # total_tv1 = torch.add(tv_cols1, 1, tv_rows1) outs0_mean = torch.mean(util.normalize_tensor(outs0[kk]), dim=(2, 3), keepdim=True) outs1_mean = torch.mean(util.normalize_tensor(outs1[kk]), dim=(2, 3), keepdim=True) diffs[kk] = torch.abs(torch.add(outs0_mean, -1, outs1_mean)) if self.spatial: lin_models = [ self.lin0, self.lin1, self.lin2, self.lin3, self.lin4 ] if (self.pnet_type == 'squeeze'): lin_models.extend([self.lin5, self.lin6]) res = [lin_models[kk].model(diffs[kk]) for kk in range(len(diffs))] return res # val = torch.mean(torch.mean(self.lin0.model(diffs[0]),dim=3),dim=2) # val = val + torch.mean(torch.mean(self.lin1.model(diffs[1]),dim=3),dim=2) # val = val + torch.mean(torch.mean(self.lin2.model(diffs[2]),dim=3),dim=2) # val = val + torch.mean(torch.mean(self.lin3.model(diffs[3]),dim=3),dim=2) # val = val + torch.mean(torch.mean(self.lin4.model(diffs[4]),dim=3),dim=2) # if(self.pnet_type=='squeeze'): # val = val + torch.mean(torch.mean(self.lin5.model(diffs[5]),dim=3),dim=2) # val = val + torch.mean(torch.mean(self.lin6.model(diffs[6]),dim=3),dim=2) val = self.lin0.model(diffs[0]) val = val + self.lin1.model(diffs[1]) val = val + self.lin2.model(diffs[2]) val = val + self.lin3.model(diffs[3]) val = val + self.lin4.model(diffs[4]) val = val.view(val.size()[0], val.size()[1], 1, 1) return val