def forward(self, in0, in1):
in0_sc = (in0 - self.shift.expand_as(in0)) / self.scale.expand_as(in0)
in1_sc = (in1 - self.shift.expand_as(in0)) / self.scale.expand_as(in0)
if (self.version == '0.0'):
# v0.0 - original release had a bug, where input was not scaled
in0_input = in0
in1_input = in1
else:
# v0.1
in0_input = in0_sc
in1_input = in1_sc
if (self.pnet_tune):
outs0 = self.net.forward(in0_input)
outs1 = self.net.forward(in1_input)
else:
outs0 = self.net[0].forward(in0_input)
outs1 = self.net[0].forward(in1_input)
feats0 = {}
feats1 = {}
diffs = [0] * len(outs0)
for (kk, out0) in enumerate(outs0):
feats0[kk] = util.normalize_tensor(outs0[kk])
feats1[kk] = util.normalize_tensor(outs1[kk])
diffs[kk] = (feats0[kk] - feats1[kk])**2
if self.spatial:
lin_models = [
self.lin0, self.lin1, self.lin2, self.lin3, self.lin4
]
if (self.pnet_type == 'squeeze'):
lin_models.extend([self.lin5, self.lin6])
res = [lin_models[kk].model(diffs[kk]) for kk in range(len(diffs))]
return res
val = torch.mean(torch.mean(self.lin0.model(diffs[0]), dim=3), dim=2)
val = val + torch.mean(torch.mean(self.lin1.model(diffs[1]), dim=3),
dim=2)
val = val + torch.mean(torch.mean(self.lin2.model(diffs[2]), dim=3),
dim=2)
val = val + torch.mean(torch.mean(self.lin3.model(diffs[3]), dim=3),
dim=2)
val = val + torch.mean(torch.mean(self.lin4.model(diffs[4]), dim=3),
dim=2)
if (self.pnet_type == 'squeeze'):
val = val + torch.mean(
torch.mean(self.lin5.model(diffs[5]), dim=3), dim=2)
val = val + torch.mean(
torch.mean(self.lin6.model(diffs[6]), dim=3), dim=2)
val = val.view(val.size()[0], val.size()[1], 1, 1)
return val
def distance(self, in1_tensor, in2_tensor):
# The input must be 64x64 scale
# Both Input1 and Input2 are input images with shape [N, H, W, C] ndarray
# Batch Normalization
input1_bn = (in1_tensor - tf.contrib.framework.broadcast_to(
self.alexnetinnet.shift,
shape=tf.shape(in1_tensor))) / tf.contrib.framework.broadcast_to(
self.alexnetinnet.scale, shape=tf.shape(in1_tensor))
input2_bn = (in2_tensor - tf.contrib.framework.broadcast_to(
self.alexnetinnet.shift,
shape=tf.shape(in2_tensor))) / tf.contrib.framework.broadcast_to(
self.alexnetinnet.scale, shape=tf.shape(in2_tensor))
# Resize the input to 68x68 to ensure the output of the first layer is the same to that in torch
# Step 2. Manully padding the input to Nx68x68x3
# [N, H, W, C]
input1_value_68 = tf.pad(input1_bn,
paddings=tf.constant([[0, 0], [2, 2], [2, 2],
[0, 0]]),
mode='CONSTANT')
input2_value_68 = tf.pad(input2_bn,
paddings=tf.constant([[0, 0], [2, 2], [2, 2],
[0, 0]]),
mode='CONSTANT')
relu_values1 = self.alexnet.forward(input1_value_68)
relu_values2 = self.alexnet.forward(input2_value_68)
# # Assert each output of the layers in alext has the correct shape.
# assert relu_values1[0].shape[1:] == (15, 15, 64)
# assert relu_values1[1].shape[1:] == (7, 7, 192)
# assert relu_values1[2].shape[1:] == (3, 3, 384)
# assert relu_values1[3].shape[1:] == (3, 3, 256)
# assert relu_values1[4].shape[1:] == (3, 3, 256)
diffs = []
for out1, out2 in zip(relu_values1, relu_values2):
# Step 3. Normalize
out1_norm = util.normalize_tensor(out1)
out2_norm = util.normalize_tensor(out2)
diffs.append((out1_norm - out2_norm)**2)
# The difference is the same as that in pytorch
convs_value = self.alexnetinnet.forward(diffs)
result = tf.reduce_mean(tf.reduce_mean(convs_value[0], axis=2), axis=1)
for conv in convs_value[1:]:
result += tf.reduce_mean(tf.reduce_mean(conv, axis=2), axis=1)
#result = tf.reshape(result, shape=(result.shape[0], 1, 1, result.shape[1]))
result = tf.squeeze(result)
return result
def forward(self, in0, in1, retPerLayer=False):
# v0.0 - original release had a bug, where input was not scaled
in0_input, in1_input = (
self.scaling_layer(in0),
self.scaling_layer(in1)) if self.version == '0.1' else (in0, in1)
outs0, outs1 = self.net.forward(in0_input), self.net.forward(in1_input)
feats0, feats1, diffs = {}, {}, {}
for kk in range(self.L):
feats0[kk], feats1[kk] = util.normalize_tensor(
outs0[kk]), util.normalize_tensor(outs1[kk])
diffs[kk] = (feats0[kk] - feats1[kk])**2
if (self.lpips):
if (self.spatial):
res = [
upsample(self.lins[kk].model(diffs[kk]),
out_H=in0.shape[2]) for kk in range(self.L)
]
else:
res = [
spatial_average(self.lins[kk].model(diffs[kk]),
keepdim=True) for kk in range(self.L)
]
else:
if (self.spatial):
res = [
upsample(diffs[kk].sum(dim=1, keepdim=True),
out_H=in0.shape[2]) for kk in range(self.L)
]
else:
res = [
spatial_average(diffs[kk].sum(dim=1, keepdim=True),
keepdim=True) for kk in range(self.L)
]
val = res[0]
for l in range(1, self.L):
val += res[l]
if (retPerLayer):
return (val, res)
else:
return val
def forward(self, in0, in1):
in0_sc = (in0 - self.shift.expand_as(in0))/self.scale.expand_as(in0)
in1_sc = (in1 - self.shift.expand_as(in0))/self.scale.expand_as(in0)
if(self.pnet_tune):
outs0 = self.net.forward(in0)
outs1 = self.net.forward(in1)
else:
outs0 = self.net[0].forward(in0)
outs1 = self.net[0].forward(in1)
feats0 = {}
feats1 = {}
diffs = [0]*len(outs0)
for (kk,out0) in enumerate(outs0):
feats0[kk] = util.normalize_tensor(outs0[kk])
feats1[kk] = util.normalize_tensor(outs1[kk])
diffs[kk] = (feats0[kk]-feats1[kk])**2
if self.spatial:
lin_models = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4]
if(self.pnet_type=='squeeze'):
lin_models.extend([self.lin5, self.lin6])
res = [lin_models[kk].model(diffs[kk]) for kk in range(len(diffs))]
return res
val = torch.mean(torch.mean(self.lin0.model(diffs[0]),dim=3),dim=2)
val = val + torch.mean(torch.mean(self.lin1.model(diffs[1]),dim=3),dim=2)
val = val + torch.mean(torch.mean(self.lin2.model(diffs[2]),dim=3),dim=2)
val = val + torch.mean(torch.mean(self.lin3.model(diffs[3]),dim=3),dim=2)
val = val + torch.mean(torch.mean(self.lin4.model(diffs[4]),dim=3),dim=2)
if(self.pnet_type=='squeeze'):
val = val + torch.mean(torch.mean(self.lin5.model(diffs[5]),dim=3),dim=2)
val = val + torch.mean(torch.mean(self.lin6.model(diffs[6]),dim=3),dim=2)
val = val.view(val.size()[0],val.size()[1],1,1)
return val
def forward(self, in0, in1):
in0_sc = (in0 - self.shift.expand_as(in0)) / self.scale.expand_as(in0)
in1_sc = (in1 - self.shift.expand_as(in0)) / self.scale.expand_as(in0)
if (self.pnet_tune):
outs0 = self.net.forward(in0)
outs1 = self.net.forward(in1)
else:
outs0 = self.net[0].forward(in0)
outs1 = self.net[0].forward(in1)
feats0 = {}
feats1 = {}
diffs = {}
for (kk, out0) in enumerate(outs0):
feats0[kk] = util.normalize_tensor(outs0[kk])
feats1[kk] = util.normalize_tensor(outs1[kk])
diffs[kk] = (feats0[kk] - feats1[kk])**2
val = torch.mean(torch.mean(self.lin0.model(diffs[0]), dim=3), dim=2)
val = val + \
torch.mean(torch.mean(self.lin1.model(diffs[1]), dim=3), dim=2)
val = val + \
torch.mean(torch.mean(self.lin2.model(diffs[2]), dim=3), dim=2)
val = val + \
torch.mean(torch.mean(self.lin3.model(diffs[3]), dim=3), dim=2)
val = val + \
torch.mean(torch.mean(self.lin4.model(diffs[4]), dim=3), dim=2)
if (self.pnet_type == 'squeeze'):
val = val + \
torch.mean(torch.mean(self.lin5.model(diffs[5]), dim=3), dim=2)
val = val + \
torch.mean(torch.mean(self.lin6.model(diffs[6]), dim=3), dim=2)
val = val.view(val.size()[0], val.size()[1], 1, 1)
return val
def forward(self, in0, in1, retPerLayer=False):
in0_sc = (in0 - self.shift.expand_as(in0)) / self.scale.expand_as(in0)
in1_sc = (in1 - self.shift.expand_as(in0)) / self.scale.expand_as(in0)
outs0 = self.net.forward(in0_sc)
outs1 = self.net.forward(in1_sc)
if (retPerLayer):
all_scores = []
for (kk, out0) in enumerate(outs0):
# # cur_score = (1.-util.cos_sim(outs0[kk],outs1[kk]))
outs0_norm = util.normalize_tensor(outs0[kk])
outs1_norm = util.normalize_tensor(outs1[kk])
# outs0_norm = outs0[kk]
# outs1_norm = outs1[kk]
# outs0_mean = torch.mean(outs0_norm,dim=(2,3))
# outs1_mean = torch.mean(outs1_norm,dim=(2,3))
# mean_diff = torch.mean(torch.abs(torch.add(outs0_mean, -1, outs1_mean)),1)
# cur_score = mean_diff
# Fourier1 = torch.rfft(torch.reshape(outs0_norm,(outs0_norm.size()[3],outs0_norm.size()[2],outs0_norm.size()[1],outs0_norm.size()[0])),2,True,False)
# Fourier2 = torch.rfft(torch.reshape(outs1_norm,(outs0_norm.size()[3], outs0_norm.size()[2], outs0_norm.size()[1], outs0_norm.size()[0])), 2, True, False)
Fourier1 = torch.div(torch.rfft(outs0_norm, 2, False,
False), (outs0_norm.size()[3]) *
(outs0_norm.size()[2]))
Fourier2 = torch.div(torch.rfft(outs1_norm, 2, False,
False), (outs0_norm.size()[3]) *
(outs0_norm.size()[2]))
Fourier1 = Fourier1**2
Fourier2 = Fourier2**2
Fourier1 = torch.sqrt(
torch.add(Fourier1[:, :, :, :, 0], 1, Fourier1[:, :, :, :, 1]))
# Fourier1[(Fourier1==0).nonzero()] = 1
# Fourier1 = torch.add(Fourier1,1e-4)
# Fourier1 = torch.log(Fourier1)
Fourier2 = torch.sqrt(
torch.add(Fourier2[:, :, :, :, 0], 1, Fourier2[:, :, :, :, 1]))
# Fourier2[(Fourier2 == 0).nonzero()] = 1
# Fourier2 = torch.add(Fourier2, 1e-4)
# Fourier2 = torch.log(Fourier2)
factor = torch.zeros(outs0_norm.size()[2], outs0_norm.size()[3])
for i in range(outs0_norm.size()[2]):
for j in range(outs0_norm.size()[3]):
factor[i, j] = (i + j)
factor = torch.exp(-1. * factor)
factor = factor.expand_as(Fourier1).cuda()
diff = torch.mul(torch.abs(torch.add(Fourier1, -1, Fourier2)),
factor)
# sum = torch.add(Fourier1,1,Fourier2)
# norm_diff = torch.div(diff,sum+1e-8)
mean_diff = torch.mean(torch.sum(diff, (2, 3)), 1)
cur_score = mean_diff
# cur_score = torch.mean(torch.abs(torch.add(outs0_mean, -1, outs1_mean)),1)
# idx = list(range(1, outs0[kk].size()[2])) + [0]
# tv_rows0 = torch.mean(torch.abs(torch.add(outs0_norm, -1, outs0_norm[:, :, idx, :])), (2, 3))
# tv_rows1 = torch.mean(torch.abs(torch.add(outs1_norm, -1, outs1_norm[:, :, idx, :])), (2, 3))
# tv_cols0 = torch.mean(torch.abs(torch.add(outs0_norm, -1, outs0_norm[:, :, :, idx])), (2, 3))
# tv_cols1 = torch.mean(torch.abs(torch.add(outs1_norm, -1, outs1_norm[:, :, :, idx])), (2, 3))
# total_tv0 = torch.add(tv_cols0, 1, tv_rows0)
# total_tv1 = torch.add(tv_cols1, 1, tv_rows1)
# tv_diff = 0.5*torch.mean(torch.abs(torch.add(total_tv0, -1, total_tv1)),1)
# idx_up = list(range(1, outs0_norm.size()[2])) + [0]
# idx_down = [outs0_norm.size()[2] - 1] + list(range(0, outs0_norm.size()[2] - 1))
# outs0_shift_up = outs0_norm[:, :, idx_up, :]
# outs0_shift_down = outs0_norm[:, :, idx_down, :]
# outs0_shift_right = outs0_norm[:, :, :, idx_up]
# outs0_shift_left = outs0_norm[:, :, :, idx_down]
# outs1_shift_up = outs1_norm[:, :, idx_up, :]
# outs1_shift_down = outs1_norm[:, :, idx_down, :]
# outs1_shift_right = outs1_norm[:, :, :, idx_up]
# outs1_shift_left = outs1_norm[:, :, :, idx_down]
# laplacian0 = torch.add(torch.add(torch.add(torch.add(outs0_shift_up, 1, outs0_shift_down), 1, outs0_shift_left), 1,outs0_shift_right), -4, outs0_norm)
# laplacian1 = torch.add(torch.add(torch.add(torch.add(outs1_shift_up, 1, outs1_shift_down), 1, outs1_shift_left), 1,outs1_shift_right), -4, outs1_norm)
# tot_lap0 = torch.mean(torch.abs(laplacian0), (2, 3))
# tot_lap1 = torch.mean(torch.abs(laplacian1), (2, 3))
if (kk == 0):
val = 1. * cur_score
else:
val = val + cur_score
if (retPerLayer):
all_scores += [cur_score]
if (retPerLayer):
return (val, all_scores)
else:
return val
def forward(self, in0, in1):
in0_sc = (in0 - self.shift.expand_as(in0)) / self.scale.expand_as(in0)
in1_sc = (in1 - self.shift.expand_as(in0)) / self.scale.expand_as(in0)
if (self.version == '0.0'):
# v0.0 - original release had a bug, where input was not scaled
in0_input = in0
in1_input = in1
else:
# v0.1
in0_input = in0_sc
in1_input = in1_sc
if (self.pnet_tune):
outs0 = self.net.forward(in0_input)
outs1 = self.net.forward(in1_input)
else:
outs0 = self.net[0].forward(in0_input)
outs1 = self.net[0].forward(in1_input)
feats0 = {}
feats1 = {}
diffs = [0] * len(outs0)
# outs0_mean = torch.mean(util.normalize_tensor(outs0[kk]), dim=(2, 3))
# outs1_mean = torch.mean(util.normalize_tensor(outs1[kk]), dim=(2, 3))
# cur_score = torch.mean(torch.abs(torch.add(outs0_mean, -1, outs1_mean)), 1)
for (kk, out0) in enumerate(outs0):
# feats0[kk] = util.normalize_tensor(outs0[kk])
# feats1[kk] = util.normalize_tensor(outs1[kk])
# diffs[kk] = (feats0[kk]-feats1[kk])**2
# outs0_norm = util.normalize_tensor(outs0[kk])
# outs1_norm = util.normalize_tensor(outs1[kk])
# idx = list(range(1, outs0[kk].size()[2])) + [0]
# tv_rows0 = torch.sum(torch.abs(torch.add(outs0_norm, -1, outs0_norm[:, :, idx, :])), (2, 3), keepdim=True)
# tv_rows1 = torch.sum(torch.abs(torch.add(outs1_norm, -1, outs1_norm[:, :, idx, :])), (2, 3), keepdim=True)
# tv_cols0 = torch.sum(torch.abs(torch.add(outs0_norm, -1, outs0_norm[:, :, :, idx])), (2, 3), keepdim=True)
# tv_cols1 = torch.sum(torch.abs(torch.add(outs1_norm, -1, outs1_norm[:, :, :, idx])), (2, 3), keepdim=True)
# total_tv0 = torch.add(tv_cols0, 1, tv_rows0)
# total_tv1 = torch.add(tv_cols1, 1, tv_rows1)
outs0_mean = torch.mean(util.normalize_tensor(outs0[kk]),
dim=(2, 3),
keepdim=True)
outs1_mean = torch.mean(util.normalize_tensor(outs1[kk]),
dim=(2, 3),
keepdim=True)
diffs[kk] = torch.abs(torch.add(outs0_mean, -1, outs1_mean))
if self.spatial:
lin_models = [
self.lin0, self.lin1, self.lin2, self.lin3, self.lin4
]
if (self.pnet_type == 'squeeze'):
lin_models.extend([self.lin5, self.lin6])
res = [lin_models[kk].model(diffs[kk]) for kk in range(len(diffs))]
return res
# val = torch.mean(torch.mean(self.lin0.model(diffs[0]),dim=3),dim=2)
# val = val + torch.mean(torch.mean(self.lin1.model(diffs[1]),dim=3),dim=2)
# val = val + torch.mean(torch.mean(self.lin2.model(diffs[2]),dim=3),dim=2)
# val = val + torch.mean(torch.mean(self.lin3.model(diffs[3]),dim=3),dim=2)
# val = val + torch.mean(torch.mean(self.lin4.model(diffs[4]),dim=3),dim=2)
# if(self.pnet_type=='squeeze'):
# val = val + torch.mean(torch.mean(self.lin5.model(diffs[5]),dim=3),dim=2)
# val = val + torch.mean(torch.mean(self.lin6.model(diffs[6]),dim=3),dim=2)
val = self.lin0.model(diffs[0])
val = val + self.lin1.model(diffs[1])
val = val + self.lin2.model(diffs[2])
val = val + self.lin3.model(diffs[3])
val = val + self.lin4.model(diffs[4])
val = val.view(val.size()[0], val.size()[1], 1, 1)
return val
