def pdist(self, fX):
"""Compute pdist à-la scipy.spatial.distance.pdist
Parameters
----------
fX : (n, d) torch.Tensor
Embeddings.
Returns
-------
distances : (n * (n-1) / 2,) torch.Tensor
Condensed pairwise distance matrix
"""
n_sequences, _ = fX.size()
distances = []
for i in range(n_sequences - 1):
if self.metric in ('cosine', 'angular'):
d = 1. - F.cosine_similarity(
fX[i, :].expand(n_sequences - 1 - i, -1),
fX[i+1:, :], dim=1, eps=1e-8)
if self.metric == 'angular':
d = torch.acos(torch.clamp(1. - d, -1 + 1e-6, 1 - 1e-6))
elif self.metric == 'euclidean':
d = F.pairwise_distance(
fX[i, :].expand(n_sequences - 1 - i, -1),
fX[i+1:, :], p=2, eps=1e-06).view(-1)
distances.append(d)
return torch.cat(distances)
def forward(self, input1):
self.batchgrid3d = torch.zeros(torch.Size([input1.size(0)]) + self.grid3d.size())
for i in range(input1.size(0)):
self.batchgrid3d[i] = self.grid3d
self.batchgrid3d = Variable(self.batchgrid3d)
#print(self.batchgrid3d)
x = torch.sum(torch.mul(self.batchgrid3d, input1[:,:,:,0:4]), 3)
y = torch.sum(torch.mul(self.batchgrid3d, input1[:,:,:,4:8]), 3)
z = torch.sum(torch.mul(self.batchgrid3d, input1[:,:,:,8:]), 3)
#print(x)
r = torch.sqrt(x**2 + y**2 + z**2) + 1e-5
#print(r)
theta = torch.acos(z/r)/(np.pi/2) - 1
#phi = torch.atan(y/x)
phi = torch.atan(y/(x + 1e-5)) + np.pi * x.lt(0).type(torch.FloatTensor) * (y.ge(0).type(torch.FloatTensor) - y.lt(0).type(torch.FloatTensor))
phi = phi/np.pi
output = torch.cat([theta,phi], 3)
return output
def forward(self, depth, trans0, trans1, rotate):
self.batchgrid3d = torch.zeros(torch.Size([depth.size(0)]) + self.grid3d.size())
for i in range(depth.size(0)):
self.batchgrid3d[i] = self.grid3d
self.batchgrid3d = Variable(self.batchgrid3d)
self.batchgrid = torch.zeros(torch.Size([depth.size(0)]) + self.grid.size())
for i in range(depth.size(0)):
self.batchgrid[i] = self.grid
self.batchgrid = Variable(self.batchgrid)
if depth.is_cuda:
self.batchgrid = self.batchgrid.cuda()
self.batchgrid3d = self.batchgrid3d.cuda()
x_ = self.batchgrid3d[:,:,:,0:1] * depth + trans0.view(-1,1,1,1).repeat(1, self.height, self.width, 1)
y_ = self.batchgrid3d[:,:,:,1:2] * depth + trans1.view(-1,1,1,1).repeat(1, self.height, self.width, 1)
z = self.batchgrid3d[:,:,:,2:3] * depth
#print(x.size(), y.size(), z.size())
rotate_z = rotate.view(-1,1,1,1).repeat(1,self.height, self.width,1) * np.pi
x = x_ * torch.cos(rotate_z) - y_ * torch.sin(rotate_z)
y = x_ * torch.sin(rotate_z) + y_ * torch.cos(rotate_z)
r = torch.sqrt(x**2 + y**2 + z**2) + 1e-5
#print(r)
theta = torch.acos(z/r)/(np.pi/2) - 1
#phi = torch.atan(y/x)
if depth.is_cuda:
phi = torch.atan(y/(x + 1e-5)) + np.pi * x.lt(0).type(torch.cuda.FloatTensor) * (y.ge(0).type(torch.cuda.FloatTensor) - y.lt(0).type(torch.cuda.FloatTensor))
else:
phi = torch.atan(y/(x + 1e-5)) + np.pi * x.lt(0).type(torch.FloatTensor) * (y.ge(0).type(torch.FloatTensor) - y.lt(0).type(torch.FloatTensor))
phi = phi/np.pi
output = torch.cat([theta,phi], 3)
return output
def zero_mean_covariance(covariance, stability=0.0):
'''Output covariance of ReLU for zero-mean Gaussian input.
f(x) = max(x, 0).
Args:
covariance: Input covariance matrix (Size, Size).
stability: For accurate results this should be zero
if used in training, use a value like 1e-4 for stability.
Returns:
Output covariance of ReLU for zero-mean Gaussian input (Size, Size).
'''
S = outer(torch.sqrt(torch.diagonal(covariance, 0, -2, -1)))
V = (covariance / S).clamp_(stability - 1.0, 1.0 - stability)
Q = torch.acos(-V) * V + torch.sqrt(1.0 - (V**2.0)) - 1.0
cov = S * Q * (1.0 / (2.0 * math.pi))
# handle degenerate case when we have zero variance
cov[cov != cov] = 0 # replace nans with zeros
return cov
def forward(self, depth, trans0, trans1, rotate):
self.batchgrid3d = torch.zeros(torch.Size([depth.size(0)]) + self.grid3d.size())
for i in range(depth.size(0)):
self.batchgrid3d[i] = self.grid3d
self.batchgrid3d = Variable(self.batchgrid3d)
self.batchgrid = torch.zeros(torch.Size([depth.size(0)]) + self.grid.size())
for i in range(depth.size(0)):
self.batchgrid[i] = self.grid
self.batchgrid = Variable(self.batchgrid)
x = self.batchgrid3d[:,:,:,0:1] * depth + trans0.view(-1,1,1,1).repeat(1, self.height, self.width, 1)
y = self.batchgrid3d[:,:,:,1:2] * depth + trans1.view(-1,1,1,1).repeat(1, self.height, self.width, 1)
z = self.batchgrid3d[:,:,:,2:3] * depth
#print(x.size(), y.size(), z.size())
r = torch.sqrt(x**2 + y**2 + z**2) + 1e-5
#print(r)
theta = torch.acos(z/r)/(np.pi/2) - 1
#phi = torch.atan(y/x)
phi = torch.atan(y/(x + 1e-5)) + np.pi * x.lt(0).type(torch.FloatTensor) * (y.ge(0).type(torch.FloatTensor) - y.lt(0).type(torch.FloatTensor))
phi = phi/np.pi
#print(theta.size(), phi.size())
input_u = rotate.view(-1,1,1,1).repeat(1,self.height, self.width,1)
output = torch.cat([theta,phi], 3)
#print(output.size())
output1 = torch.atan(torch.tan(np.pi/2.0*(output[:,:,:,1:2] + self.batchgrid[:,:,:,2:] * input_u[:,:,:,:]))) /(np.pi/2)
output2 = torch.cat([output[:,:,:,0:1], output1], 3)
return output2
def forward(self, input1, input2):
self.batchgrid3d = torch.zeros(torch.Size([input1.size(0)]) + self.grid3d.size())
for i in range(input1.size(0)):
self.batchgrid3d[i] = self.grid3d
self.batchgrid3d = Variable(self.batchgrid3d)
self.batchgrid = torch.zeros(torch.Size([input1.size(0)]) + self.grid.size())
for i in range(input1.size(0)):
self.batchgrid[i] = self.grid
self.batchgrid = Variable(self.batchgrid)
#print(self.batchgrid3d)
x = torch.sum(torch.mul(self.batchgrid3d, input1[:,:,:,0:4]), 3)
y = torch.sum(torch.mul(self.batchgrid3d, input1[:,:,:,4:8]), 3)
z = torch.sum(torch.mul(self.batchgrid3d, input1[:,:,:,8:]), 3)
#print(x)
r = torch.sqrt(x**2 + y**2 + z**2) + 1e-5
#print(r)
theta = torch.acos(z/r)/(np.pi/2) - 1
#phi = torch.atan(y/x)
phi = torch.atan(y/(x + 1e-5)) + np.pi * x.lt(0).type(torch.FloatTensor) * (y.ge(0).type(torch.FloatTensor) - y.lt(0).type(torch.FloatTensor))
phi = phi/np.pi
input_u = input2.view(-1,1,1,1).repeat(1,self.height, self.width,1)
output = torch.cat([theta,phi], 3)
output1 = torch.atan(torch.tan(np.pi/2.0*(output[:,:,:,1:2] + self.batchgrid[:,:,:,2:] * input_u[:,:,:,:]))) /(np.pi/2)
output2 = torch.cat([output[:,:,:,0:1], output1], 3)
return output2
def get_elliptical_mask(self, feat, img):
"""Gets the elliptical scaling mask according to the equation of a
rotated ellipse
:param feat: features from the backbone
:param img: image
:returns: elliptical adjustment maps for each channel
:rtype: Tensor
"""
# The two eps parameters are used to avoid numerical issues in the learning
eps2 = 1e-7
eps1 = 1e-10
# max_scale is the maximum an ellipse can scale the image R,G,B values by
max_scale = 2
min_scale = 0
feat_elliptical = torch.cat((feat, img), 1)
feat_elliptical = self.upsample(feat_elliptical)
# The following layers calculate the parameters of the ellipses that we use for image enhancement
x = self.elliptical_layer1(feat_elliptical)
x = self.elliptical_layer2(x)
x = self.elliptical_layer3(x)
x = self.elliptical_layer4(x)
x = self.elliptical_layer5(x)
x = self.elliptical_layer6(x)
x = self.elliptical_layer7(x)
x = self.elliptical_layer8(x)
x = x.view(x.size()[0], -1)
x = self.dropout(x)
G = self.fc_elliptical(x)
# The next code implements a rotated ellipse according to:
# https://math.stackexchange.com/questions/426150/what-is-the-general-equation-of-the-ellipse-that-is-not-in-the-origin-and-rotate
# Normalised coordinates for x and y-axes, we instantiate the ellipses in these coordinates
x_axis = Variable(
torch.arange(img.shape[2]).view(-1, 1).repeat(
1, img.shape[3]).cuda()) / img.shape[2]
y_axis = Variable(
torch.arange(img.shape[3]).repeat(img.shape[2],
1).cuda()) / img.shape[3]
# x coordinate - h position
right_x = (img.shape[2] - 1) / img.shape[2]
left_x = 0
# Centre of ellipse, x-coordinate
x_coord1 = self.tanh01(G[0, 0]) + eps1
x_coord2 = self.tanh01(G[0, 1]) + eps1
x_coord3 = self.tanh01(G[0, 2]) + eps1
# y coordinate - k coordinate
right_y = (img.shape[3] - 1) // img.shape[3]
left_y = 0
# Centre of ellipse, y-coordinate
y_coord1 = self.tanh01(G[0, 3]) + eps1
y_coord2 = self.tanh01(G[0, 4]) + eps1
y_coord3 = self.tanh01(G[0, 5]) + eps1
# a value of ellipse
a1 = self.tanh01(G[0, 6]) + eps1
a2 = self.tanh01(G[0, 7]) + eps1
a3 = self.tanh01(G[0, 8]) + eps1
# b value
b1 = self.tanh01(G[0, 9]) + eps1
b2 = self.tanh01(G[0, 10]) + eps1
b3 = self.tanh01(G[0, 11]) + eps1
# A value is angle to the x-axis
A1 = self.tanh01(G[0, 12]) * math.pi + eps1
A2 = self.tanh01(G[0, 13]) * math.pi + eps1
A3 = self.tanh01(G[0, 14]) * math.pi + eps1
'''
The following are the scale factors for each of the 9 ellipses
'''
scale1 = self.tanh01(G[0, 15]) * max_scale + eps1
scale2 = self.tanh01(G[0, 16]) * max_scale + eps1
scale3 = self.tanh01(G[0, 17]) * max_scale + eps1
scale4 = self.tanh01(G[0, 18]) * max_scale + eps1
scale5 = self.tanh01(G[0, 19]) * max_scale + eps1
scale6 = self.tanh01(G[0, 20]) * max_scale + eps1
scale7 = self.tanh01(G[0, 21]) * max_scale + eps1
scale8 = self.tanh01(G[0, 22]) * max_scale + eps1
scale9 = self.tanh01(G[0, 23]) * max_scale + eps1
############ Angle of orientation of the ellipses with respect to the y semi-axis
angle_1 = torch.acos(
torch.clamp((y_axis - y_coord1) /
(torch.sqrt((x_axis - x_coord1)**2 +
(y_axis - y_coord1)**2 + eps1)), -1 + eps2,
1 - eps2)) - A1
angle_2 = torch.acos(
torch.clamp((y_axis - y_coord2) /
(torch.sqrt((x_axis - x_coord2)**2 +
(y_axis - y_coord2)**2 + eps1)), -1 + eps2,
1 - eps2)) - A2
angle_3 = torch.acos(
torch.clamp((y_axis - y_coord3) /
(torch.sqrt((x_axis - x_coord3)**2 +
(y_axis - y_coord3)**2 + eps1)), -1 + eps2,
1 - eps2)) - A3
############ Radius of the ellipses
# https://math.stackexchange.com/questions/432902/how-to-get-the-radius-of-an-ellipse-at-a-specific-angle-by-knowing-its-semi-majo
radius_1 = (a1 * b1) / torch.sqrt((a1**2) * (torch.sin(angle_1)**2) +
(b1**2) *
(torch.cos(angle_1)**2) + eps1)
radius_2 = (a2 * b2) / torch.sqrt((a2**2) * (torch.sin(angle_2)**2) +
(b2**2) *
(torch.cos(angle_2)**2) + eps1)
radius_3 = (a3 * b3) / torch.sqrt((a3**2) * (torch.sin(angle_3)**2) +
(b3**2) *
(torch.cos(angle_3)**2) + eps1)
############ Scaling factors for the R,G,B channels, here we learn three ellipses
mask_scale1 = self.get_mask(x_axis,
y_axis,
shift_x=x_coord1,
shift_y=y_coord1,
semi_axis_x=a1,
semi_axis_y=b1,
alpha=angle_1,
scale_factor=scale1,
radius=radius_1)
mask_scale2 = self.get_mask(x_axis,
y_axis,
shift_x=x_coord1,
shift_y=y_coord1,
semi_axis_x=a1,
semi_axis_y=b1,
alpha=angle_1,
scale_factor=scale2,
radius=radius_1)
mask_scale3 = self.get_mask(x_axis,
y_axis,
shift_x=x_coord1,
shift_y=y_coord1,
semi_axis_x=a1,
semi_axis_y=b1,
alpha=angle_1,
scale_factor=scale3,
radius=radius_1)
mask_scale_1 = torch.cat((mask_scale1, mask_scale2, mask_scale3),
dim=0)
mask_scale_1_rad = torch.clamp(mask_scale_1.unsqueeze(0), 0, max_scale)
############ Scaling factors for the R,G,B channels, here we learn three ellipses
mask_scale4 = self.get_mask(x_axis,
y_axis,
shift_x=x_coord2,
shift_y=y_coord2,
semi_axis_x=a2,
semi_axis_y=b2,
alpha=angle_2,
scale_factor=scale4,
radius=radius_2)
mask_scale5 = self.get_mask(x_axis,
y_axis,
shift_x=x_coord2,
shift_y=y_coord2,
semi_axis_x=a2,
semi_axis_y=b2,
alpha=angle_2,
scale_factor=scale5,
radius=radius_2)
mask_scale6 = self.get_mask(x_axis,
y_axis,
shift_x=x_coord2,
shift_y=y_coord2,
semi_axis_x=a2,
semi_axis_y=b3,
alpha=angle_2,
scale_factor=scale6,
radius=radius_2)
mask_scale_4 = torch.cat((mask_scale4, mask_scale5, mask_scale6),
dim=0)
mask_scale_4_rad = torch.clamp(mask_scale_4.unsqueeze(0), 0, max_scale)
############ Scaling factors for the R,G,B channels, here we learn three ellipses
mask_scale7 = self.get_mask(x_axis,
y_axis,
shift_x=x_coord3,
shift_y=y_coord3,
semi_axis_x=a3,
semi_axis_y=b3,
alpha=angle_3,
scale_factor=scale7,
radius=radius_3)
mask_scale8 = self.get_mask(x_axis,
y_axis,
shift_x=x_coord3,
shift_y=y_coord3,
semi_axis_x=a3,
semi_axis_y=b3,
alpha=angle_3,
scale_factor=scale8,
radius=radius_3)
mask_scale9 = self.get_mask(x_axis,
y_axis,
shift_x=x_coord3,
shift_y=y_coord3,
semi_axis_x=a3,
semi_axis_y=b3,
alpha=angle_3,
scale_factor=scale9,
radius=radius_3)
mask_scale_7 = torch.cat((mask_scale7, mask_scale8, mask_scale9),
dim=0)
mask_scale_7_rad = torch.clamp(mask_scale_7.unsqueeze(0), 0, max_scale)
############ Mix the ellipses together by multiplication
mask_scale_elliptical = torch.clamp(
mask_scale_1_rad * mask_scale_4_rad * mask_scale_7_rad, 0,
max_scale)
return mask_scale_elliptical
def forward(self, depth, trans0, trans1, rotate):
self.batchgrid3d = torch.zeros(
torch.Size([depth.size(0)]) + self.grid3d.size())
for i in range(depth.size(0)):
self.batchgrid3d[i] = self.grid3d
self.batchgrid3d = Variable(self.batchgrid3d)
self.batchgrid = torch.zeros(
torch.Size([depth.size(0)]) + self.grid.size())
for i in range(depth.size(0)):
self.batchgrid[i] = self.grid
self.batchgrid = Variable(self.batchgrid)
if depth.is_cuda:
self.batchgrid = self.batchgrid.cuda()
self.batchgrid3d = self.batchgrid3d.cuda()
x_ = self.batchgrid3d[:, :, :, 0:1] * depth + trans0.view(-1, 1, 1,
1).repeat(1,
self.height,
self.width,
1)
y_ = self.batchgrid3d[:, :, :, 1:2] * depth + trans1.view(-1, 1, 1,
1).repeat(1,
self.height,
self.width,
1)
z = self.batchgrid3d[:, :, :, 2:3] * depth
# print(x.size(), y.size(), z.size())
rotate_z = rotate.view(-1, 1, 1, 1).repeat(1, self.height, self.width,
1) * np.pi
x = x_ * torch.cos(rotate_z) - y_ * torch.sin(rotate_z)
y = x_ * torch.sin(rotate_z) + y_ * torch.cos(rotate_z)
r = torch.sqrt(x ** 2 + y ** 2 + z ** 2) + 1e-5
# print(r)
theta = torch.acos(z / r) / (np.pi / 2) - 1
# phi = torch.atan(y/x)
if depth.is_cuda:
phi = torch.atan(y / (x + 1e-5)) + np.pi * x.lt(0).type(
torch.cuda.FloatTensor) * (
y.ge(0).type(torch.cuda.FloatTensor) - y.lt(0).type(
torch.cuda.FloatTensor))
else:
phi = torch.atan(y / (x + 1e-5)) + np.pi * x.lt(0).type(
torch.FloatTensor) * (
y.ge(0).type(torch.FloatTensor) - y.lt(0).type(
torch.FloatTensor))
phi = phi / np.pi
output = torch.cat([theta, phi], 3)
return output
def train(pklfile, trainedh5):
# input dataset from h5, then divide it into train dataset and test dataset(10:1)
x = torch.unsqueeze(torch.linspace(-1, 1, 10000),
dim=1).double() # x data (tensor), shape=(100, 1)
y = torch.acos(x * 0.02) + x.pow(5) - x.pow(4) * 2 - x.pow(3) * 2 + x.pow(
2) + 0.002 * torch.rand(
x.size()).double() # noisy y data (tensor), shape=(100, 1)
x_1 = torch.unsqueeze(torch.linspace(-1, 1, 300),
dim=1).double() # x data (tensor), shape=(100, 1)
y_1 = torch.acos(x_1 * 0.02) + x_1.pow(
5) - x_1.pow(4) * 2 - x_1.pow(3) * 2 + x_1.pow(2) + 0.002 * torch.rand(
x_1.size()).double() # noisy y data (tensor), shape=(100, 1)
# y = x.pow(3) # noisy y data (tensor), shape=(100, 1)
torch_dataset = Data.TensorDataset(x, y)
torch_dataset_1 = Data.TensorDataset(x_1, y_1)
concat_dataset = Data.ConcatDataset((torch_dataset, torch_dataset_1))
print(concat_dataset.datasets)
loader = Data.DataLoader(
dataset=concat_dataset,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=2,
)
x_v = torch.unsqueeze(torch.linspace(
-1, 1, 100), dim=1).double() # x test data (tensor), shape=(100, 1)
y_v = torch.acos(x_v * 0.02) + x_v.pow(
5) - x_v.pow(4) * 2 - x_v.pow(3) * 2 + x_v.pow(2) + 0.002 * torch.rand(
x_v.size()).double() # noisy y test data (tensor), shape=(100, 1)
y_cacu = torch.acos(
x_v * 0.02) + x_v.pow(5) - x_v.pow(4) * 2 - x_v.pow(3) * 2 + x_v.pow(
2) # noisy y test data (tensor), shape=(100, 1)
# y_v = x_v.pow(3) # noisy y test data (tensor), shape=(100, 1)
net = Net(n_feature=1, n_output=1)
net = net.double()
print(net)
logdir = './NN_logs_' + h5key
if os.path.isdir(logdir):
shutil.rmtree(logdir)
logger = Logger(logdir)
res = net(x_v)
writer = SummaryWriter(logdir)
writer.add_graph(net, res)
writer.close()
# optimizer = torch.optim.SGD(net.parameters(), lr=LR, weight_decay=0.1)
# optimizer = torch.optim.SGD(net.parameters(), lr=LR)
# optimizer = torch.optim.Adam(net.parameters(), lr=LR, weight_decay=0.0001)
optimizer = torch.optim.Adam(net.parameters(), lr=LR)
# optimizer = torch.optim.Adagrad(net.parameters(), lr=LR, lr_decay=0.001)
loss_func = nn.MSELoss()
plt.ion()
plt.figure(figsize=(13, 3.5))
loss_list = []
loss_list_test = []
#par_np = net.parameters()
Step = 0
lri = LR
for epoch in range(EPOCH):
print('Epoch: ', epoch)
for step, (b_x, b_y) in enumerate(loader):
print('Step: ', step)
prediction = net(b_x)
loss = loss_func(prediction, b_y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
Step += 1
if (Step + 1) % 5 == 0:
lri = lri / (1 + 0.001)
print("lri: ", lri)
for param_group in optimizer.param_groups:
param_group['lr'] = lri
pre_y_v = net(x_v)
loss_test = loss_func(pre_y_v, y_v)
loss_best = loss_func(y_cacu, y_v)
loss_list.append(loss.data[0])
loss_list_test.append(loss_test.data[0])
plt.subplot(131)
plt.cla()
plt.scatter(b_x.data.numpy(), b_y.data.numpy())
plt.scatter(b_x.data.numpy(),
prediction.data.numpy(),
s=2,
color='red')
plt.text(-0.3,
0,
'Loss=%.6f' % loss.data.numpy(),
fontdict={
'size': 10,
'color': 'red'
})
plt.subplot(132)
plt.cla()
plt.scatter(x_v.data.numpy(), y_v.data.numpy())
plt.plot(x_v.data.numpy(), pre_y_v.data.numpy(), 'r-', lw=2)
plt.text(-0.3,
0,
'Loss =%.6f' % loss_test.data.numpy(),
fontdict={
'size': 10,
'color': 'red'
})
plt.text(-0.3,
0.2,
'Loss_b=%.6f' % loss_best.data.numpy(),
fontdict={
'size': 10,
'color': 'red'
})
plt.subplot(133)
plt.cla()
plt.plot(loss_list, 'b-', lw=1, label='train')
plt.plot(loss_list_test, 'r-', lw=1, label='test')
plt.legend(loc='best')
plt.pause(0.1)
# ================================================================== #
# Tensorboard Logging #
# ================================================================== #
# 1. Log scalar values (scalar summary)
info = {'loss': loss.item()}
for tag, value in info.items():
print(tag, value)
logger.scalar_summary(tag, value, Step + 1)
# 2. Log values and gradients of the parameters (histogram summary)
for tag, value in net.named_parameters():
tag = tag.replace('.', '/')
logger.histo_summary(tag,
value.data.cpu().numpy(), Step + 1)
logger.histo_summary(tag + '/grad',
value.grad.data.cpu().numpy(),
Step + 1)
# 3. Log training images (image summary)
info = {'images': x.view(-1, 5, 5)[:10].cpu().numpy()}
for tag, images in info.items():
logger.image_summary(tag, images, Step + 1)
plt.ioff()
plt.show()
def _get_auxiliary_parameter(self, p, rot_mat, gc):
p0_2 = torch.tensor([0., 0., self.d_bs]).double()
p6_7 = torch.tensor([0., 0., self.d_wf]).double()
R0_7 = rot_mat
self.gc = np.array([gc[0], gc[0], gc[0], gc[1], gc[2], gc[2], gc[2]])
# get q4 and p2_6 Shoulder-Elbow-Wrist (SEW) Plane
p2_6 = p - p0_2 - rot_mat @ p6_7
cosq4_v = (torch.norm(p2_6, dim=0)**2 - self.d_se**2 -
self.d_ew**2) / (2 * self.d_se * self.d_ew)
if abs(cosq4_v) > 1 + 1e-9:
self.q4_v = None
self.a = torch.zeros((7, 2))
self.b = torch.zeros((7, 2))
self.c = torch.zeros((7, 2))
return False
else:
cosq4_v = torch.clamp(cosq4_v, -1., 1.)
self.q4_v = gc[1] * torch.acos(cosq4_v)
p2_6_norm = torch.norm(p2_6)
q1_v = torch.atan2(p2_6[1], p2_6[0])
phi = torch.acos((self.d_se**2 + p2_6_norm**2 - self.d_ew**2) /
(2 * self.d_se * p2_6_norm))
q2_v = torch.atan2(torch.sqrt(p2_6[0]**2 + p2_6[1]**2),
p2_6[2]) + gc[1] * phi
q3_v = torch.tensor(0.0)
T_0_3_v = self._transform_i(0, q1_v) @ self._transform_i(
1, q2_v) @ self._transform_i(2, q3_v)
R_0_3_v = T_0_3_v[:3, :3]
# cross product matrixq_true
p2_6_hat = p2_6 / p2_6_norm
cpm = torch.tensor([[0., -p2_6_hat[2], p2_6_hat[1]],
[p2_6_hat[2], 0., -p2_6_hat[0]],
[-p2_6_hat[1], p2_6_hat[0], 0.]])
A_s = cpm @ R_0_3_v
B_s = -(cpm @ cpm) @ R_0_3_v
C_s = torch.ger(p2_6_hat, p2_6_hat) @ R_0_3_v
T_3_4 = self._transform_i(3, self.q4_v)
R_3_4 = T_3_4[:3, :3]
A_w = R_3_4.T @ A_s.T @ R0_7
B_w = R_3_4.T @ B_s.T @ R0_7
C_w = R_3_4.T @ C_s.T @ R0_7
# x[3, :]=0 For joint 4 is zero
# x[1 5, 1]=0 a_d, b_d, c_d =0
self.a = torch.zeros((7, 2))
self.b = torch.zeros((7, 2))
self.c = torch.zeros((7, 2))
self.a[0, 0] = A_s[1, 1]
self.b[0, 0] = B_s[1, 1]
self.c[0, 0] = C_s[1, 1]
self.a[0, 1] = A_s[0, 1]
self.b[0, 1] = B_s[0, 1]
self.c[0, 1] = C_s[0, 1]
self.a[1, 0] = A_s[2, 1]
self.b[1, 0] = B_s[2, 1]
self.c[1, 0] = C_s[2, 1]
self.a[2, 0] = -A_s[2, 2]
self.b[2, 0] = -B_s[2, 2]
self.c[2, 0] = -C_s[2, 2]
self.a[2, 1] = -A_s[2, 0]
self.b[2, 1] = -B_s[2, 0]
self.c[2, 1] = -C_s[2, 0]
self.a[4, 0] = A_w[1, 2]
self.a[4, 1] = A_w[0, 2]
self.b[4, 0] = B_w[1, 2]
self.b[4, 1] = B_w[0, 2]
self.c[4, 0] = C_w[1, 2]
self.c[4, 1] = C_w[0, 2]
self.a[5, 0] = A_w[2, 2]
self.b[5, 0] = B_w[2, 2]
self.c[5, 0] = C_w[2, 2]
self.a[6, 0] = A_w[2, 1]
self.a[6, 1] = -A_w[2, 0]
self.b[6, 0] = B_w[2, 1]
self.b[6, 1] = -B_w[2, 0]
self.c[6, 0] = C_w[2, 1]
self.c[6, 1] = -C_w[2, 0]
return True
def compute_loss(pred, target, outputs, output_pred_ind, output_target_ind, output_loss_weight, normal_loss_type, arch,
patch_rot=None, phase='train',
use_consistency=False, point_weights=None, neighbor_normals=None, opt=None, trans=None, trans2=None):
loss = torch.zeros(1, device=pred.device, dtype=pred.dtype)
n_loss = torch.zeros(1, device=pred.device, dtype=pred.dtype)
consistency_loss = torch.zeros(1, device=pred.device, dtype=pred.dtype)
# # generate a inv normal distribution for weight kl div
# add pointnet transformation regularization
regularizer_trans = 0
if trans is not None:
regularizer_trans += 0.1 * torch.nn.MSELoss()(trans * trans.permute(0, 2, 1),
torch.eye(3, device=trans.device).unsqueeze(0).repeat(
trans.size(0), 1, 1))
if trans2 is not None:
regularizer_trans += 0.01 * torch.nn.MSELoss()(trans2 * trans2.permute(0, 2, 1),
torch.eye(64, device=trans.device).unsqueeze(0).repeat(
trans.size(0), 1, 1))
for oi, o in enumerate(outputs):
if o == 'unoriented_normals' or o == 'oriented_normals':
o_pred = pred[:, output_pred_ind[oi]:output_pred_ind[oi]+3]
o_target = target[output_target_ind[oi]]
if patch_rot is not None:
# transform predictions with inverse transform
# since we know the transform to be a rotation (QSTN), the transpose is the inverse
o_pred = torch.bmm(o_pred.unsqueeze(1), patch_rot.transpose(2, 1)).squeeze(1)
if o == 'unoriented_normals':
if normal_loss_type == 'ms_euclidean':
normal_loss = torch.min((o_pred-o_target).pow(2).sum(1), (o_pred+o_target).pow(2).sum(1)).mean() * output_loss_weight[oi]
elif normal_loss_type == 'ms_oneminuscos':
cos_ang = normal_estimation_utils.cos_angle(o_pred, o_target)
normal_loss = (1-torch.abs(cos_ang)).pow(2).mean() * output_loss_weight[oi]
elif normal_loss_type == 'sin':
normal_loss = 0.5 * torch.norm(torch.cross(o_pred, o_target, dim=-1), p=2, dim=1).mean()
else:
raise ValueError('Unsupported loss type: %s' % (normal_loss_type))
loss = normal_loss
# get angle value at test time (not in training to save runtime)
if phase == 'test':
if not normal_loss_type == 'ms_oneminuscos':
cos_ang = torch.abs(normal_estimation_utils.cos_angle(o_pred, o_target))
cos_ang[cos_ang>1] = 1
angle = torch.acos(cos_ang)
err_angle = torch.mean(angle)
else:
err_angle = None
else:
raise ValueError('Unsupported output type: %s' % (o))
elif o == 'max_curvature' or o == 'min_curvature':
o_pred = pred[:, output_pred_ind[oi]:output_pred_ind[oi]+1]
o_target = target[output_target_ind[oi]]
# Rectified mse loss: mean square of (pred - gt) / max(1, |gt|)
normalized_diff = (o_pred - o_target) / torch.clamp(torch.abs(o_target), min=1)
loss += normalized_diff.pow(2).mean() * output_loss_weight[oi]
elif o == 'neighbor_normals':
if use_consistency:
o_pred = neighbor_normals
o_target = target[output_target_ind[oi]]
batch_size, n_points, _ = neighbor_normals.shape
if patch_rot is not None:
# transform predictions with inverse transform
o_pred = torch.bmm(o_pred.view(-1, 1, 3),
patch_rot.transpose(2, 1).repeat(1, n_points, 1, 1).view(-1, 3, 3)).view(batch_size, n_points, 3)
# if opt.jet_order < 2: # when the jet has order higher than 2 the normal vector orientation matters.
if normal_loss_type == 'ms_euclidean':
consistency_loss = torch.mean(point_weights * torch.min((o_pred - o_target).pow(2).sum(2),
(o_pred + o_target).pow(2).sum(2)) )
elif normal_loss_type == 'ms_oneminuscos':
cos_ang = normal_estimation_utils.cos_angle(o_pred.view(-1, 3),
o_target.view(-1, 3)).view(batch_size, n_points)
consistency_loss = torch.mean(point_weights * (1 - torch.abs(cos_ang)).pow(2))
elif normal_loss_type == 'sin':
consistency_loss = 0.25 * torch.mean(point_weights *
torch.norm(torch.cross(o_pred.view(-1, 3),
o_target.view(-1, 3), dim=-1).view(batch_size, -1, 3), p=2, dim=2))
else:
raise ValueError('Unsupported loss type: %s' % (normal_loss_type))
if opt.con_reg == 'mean':
regularizer = - 0.01 * torch.mean(point_weights)
elif opt.con_reg == "log":
regularizer = - 0.05 * torch.mean(point_weights.log())
elif opt.con_reg == 'norm':
regularizer = torch.mean((1/n_points)*torch.norm(point_weights-1, dim=1))
else:
raise ValueError("Unkonwn consistency regularizer")
regularizer = regularizer_trans + regularizer
consistency_loss = consistency_loss + regularizer
loss = consistency_loss + normal_loss
else:
raise ValueError('Unsupported output type: %s' % (o))
return loss, n_loss, err_angle, consistency_loss, normal_loss
def test_acos(x, y):
c = torch.acos(torch.add(x, y))
return c
def compute_angle_error(preds, labels):
pred_x, pred_y, pred_z = convert_to_unit_vector(preds)
label_x, label_y, label_z = convert_to_unit_vector(labels)
angles = pred_x * label_x + pred_y * label_y + pred_z * label_z
return torch.acos(angles) * 180 / np.pi
def tensorAngularAllDiffs(label_qs, verts):
return 2 * torch.acos(
torch.abs(
torch.transpose(torch.mm(verts, torch.transpose(label_qs, 0, 1)),
0, 1)).clamp(max=1 - eps))
def tensorSignedAngularDiff(q1, q2):
return 2 * torch.acos(torch.mm(q1, q2.t()).clamp(min=eps - 1, max=1 - eps))
def forward(self, inputs,targets=None,batch=None,total_batch=None,):
cosine = inputs[0]
soft_cosine = inputs[1]
# --------------------------- convert label to one-hot ---------------------------
one_hot = torch.zeros_like(cosine).detach()
one_hot.scatter_(1, targets.view(-1, 1).long(), 1)
Tempture = 0.1
Theta_soft_cosine = torch.acos(soft_cosine).detach()
postive_Theta_soft_cosine = (torch.clamp(math.pi/2 - Theta_soft_cosine,min=0)*Tempture).detach()
# Theta_soft_cosine = (Theta_soft_cosine*Tempture).detach()
Theta_cosine = torch.acos(cosine)
Theta_cosine = one_hot * (Theta_cosine ) + \
(1.0 - one_hot) * (Theta_cosine+postive_Theta_soft_cosine)
cosine = torch.cos(Theta_cosine)
# cosine = (one_hot * cosine) + (
# (1.0 - one_hot) * phi_cosine)
###=========== warmup =========####
if batch < self.warm_batch:
return cosine * self.s
####======= add Margin ========####
phi_sine = torch.sqrt(1.0 - torch.pow(cosine, 2))
phi = cosine * self.cos_m - phi_sine * self.sin_m
if self.easy_margin:
phi = torch.where(cosine > 0, phi, cosine)
else:
# phi = torch.where(cosine > self.th, phi, cosine - self.mm)
with torch.no_grad():
soft_cosine_norm = (soft_cosine-1)*0.5
th = self.th + soft_cosine_norm
mm = soft_cosine_norm + self.mm
phi = torch.where(cosine > th, phi, cosine - mm )
if self.adacos and batch != 0:
theta = torch.acos(torch.clamp(cosine, -1.0 + 1e-7, 1.0 - 1e-7))
with torch.no_grad():
# B_avg = torch.where(one_hot < 1, torch.exp(self.s * cosine), torch.zeros_like(cosine))
B_avg_ = cosine[one_hot != 1]
B_avg = torch.sum(torch.exp(self.s * B_avg_)) / input.size(0)
theta_med = torch.median(theta[one_hot == 1])
theta_sum = torch.sum(theta[one_hot != 1])
self.s = torch.log(B_avg) / torch.cos(torch.min(math.pi / 4 * torch.ones_like(theta_med), theta_med))
if self.s > 32:
self.s = 32
# s = self.s * self.decay_rate ** (batch / self.decay_steps)
print("=" * 60)
print("s={} theta_med={} theta_sum={} B_avg={}".format(self.s, theta_med, theta_sum, B_avg))
print("=" * 60)
# -------------torch.where(out_i = {x_i if condition_i else y_i) -------------
output = (one_hot * phi) + (
(1.0 - one_hot) * cosine)
output *= self.s
return output
def get_elliptical_mask(self, feat, img):
"""Gets the elliptical scaling mask according to the equation of a
rotated ellipse
:param feat: features from the backbone
:param img: image
:returns: elliptical adjustment maps for each channel
:rtype: Tensor
"""
eps = 1e-10
max_scale = 2
min_scale = 0
feat_elliptical = torch.cat((feat, img), 1)
feat_elliptical = self.upsample(feat_elliptical)
x = self.elliptical_layer1(feat_elliptical)
x = self.elliptical_layer2(x)
x = self.elliptical_layer3(x)
x = self.elliptical_layer4(x)
x = self.elliptical_layer5(x)
x = self.elliptical_layer6(x)
x = self.elliptical_layer7(x)
x = self.elliptical_layer8(x)
x = x.view(x.size()[0], -1)
x = self.dropout(x)
G = self.fc_elliptical(x)
x_axis = Variable(
torch.arange(img.shape[2]).view(-1, 1).repeat(
1, img.shape[3]).cuda()) / img.shape[2]
y_axis = Variable(
torch.arange(img.shape[3]).repeat(img.shape[2],
1).cuda()) / img.shape[3]
# x coordinate - h position
right_x = (img.shape[2] - 1) / img.shape[2]
left_x = 0
G[0, 0] = self.tanh01(G[0, 0]) + eps
G[0, 8] = self.tanh01(G[0, 8]) + eps
G[0, 16] = self.tanh01(G[0, 16]) + eps
# y coordinate - k coordinate
right_y = (img.shape[3] - 1) // img.shape[3]
left_y = 0
G[0, 1] = self.tanh01(G[0, 1]) + eps
G[0, 9] = self.tanh01(G[0, 9]) + eps
G[0, 17] = self.tanh01(G[0, 17]) + eps
# a value
G[0, 2] = self.tanh01(G[0, 2]) + eps
G[0, 10] = self.tanh01(G[0, 10]) + eps
G[0, 18] = self.tanh01(G[0, 18]) + eps
# b value
G[0, 3] = self.tanh01(G[0, 3]) + eps # * (right_x - left_x) + left_x
G[0, 11] = self.tanh01(G[0, 11]) + eps
G[0, 19] = self.tanh01(G[0, 19]) + eps
# A value
G[0, 4] = self.tanh01(G[0, 4]) * math.pi + eps
G[0, 12] = self.tanh01(G[0, 12]) * math.pi + eps
G[0, 20] = self.tanh01(G[0, 20]) * math.pi + eps
'''
The following are the scale factors for each ellipse
'''
G[0, 5] = self.tanh01(G[0, 5]) * max_scale + eps
G[0, 6] = self.tanh01(G[0, 6]) * max_scale + eps
G[0, 7] = self.tanh01(G[0, 7]) * max_scale + eps
G[0, 13] = self.tanh01(G[0, 13]) * max_scale + eps
G[0, 14] = self.tanh01(G[0, 14]) * max_scale + eps
G[0, 15] = self.tanh01(G[0, 15]) * max_scale + eps
G[0, 21] = self.tanh01(G[0, 21]) * max_scale + eps
G[0, 22] = self.tanh01(G[0, 22]) * max_scale + eps
G[0, 23] = self.tanh01(G[0, 23]) * max_scale + eps
angle_1 = torch.acos(
torch.clamp((y_axis - G[0, 1]) /
(torch.sqrt((x_axis - G[0, 0])**2 +
(y_axis - G[0, 1])**2 + eps) + eps),
-1 + 1e-7, 1 - 1e-7)) - G[0, 4]
angle_2 = torch.acos(
torch.clamp((y_axis - G[0, 9]) /
(torch.sqrt((x_axis - G[0, 8])**2 +
(y_axis - G[0, 9])**2 + eps) + eps),
-1 + 1e-7, 1 - 1e-7)) - G[0, 12]
angle_3 = torch.acos(
torch.clamp((y_axis - G[0, 17]) /
(torch.sqrt((x_axis - G[0, 16])**2 +
(y_axis - G[0, 17])**2 + eps) + eps),
-1 + 1e-7, 1 - 1e-7)) - G[0, 20]
radius_1 = ((G[0, 2] * G[0, 3]) /
(torch.sqrt((G[0, 2]**2) * (torch.sin(angle_1)**2) +
(G[0, 3]**2) *
(torch.cos(angle_1)**2) + eps) + eps)) + eps
radius_2 = ((G[0, 10] * G[0, 11]) /
(torch.sqrt((G[0, 10]**2) * (torch.sin(angle_2)**2) +
(G[0, 11]**2) *
(torch.cos(angle_2)**2) + eps) + eps)) + eps
radius_3 = ((G[0, 18] * G[0, 19]) /
(torch.sqrt((G[0, 18]**2) * (torch.sin(angle_3)**2) +
(G[0, 19]**2) *
(torch.cos(angle_3)**2) + eps) + eps)) + eps
mask_scale1 = self.get_mask(x_axis,
y_axis,
shift_x=G[0, 0],
shift_y=G[0, 1],
semi_axis_x=G[0, 2],
semi_axis_y=G[0, 3],
alpha=G[0, 4],
scale_factor=G[0, 5],
radius=radius_1)
mask_scale2 = self.get_mask(x_axis,
y_axis,
shift_x=G[0, 0],
shift_y=G[0, 1],
semi_axis_x=G[0, 2],
semi_axis_y=G[0, 3],
alpha=G[0, 4],
scale_factor=G[0, 6],
radius=radius_1)
mask_scale3 = self.get_mask(x_axis,
y_axis,
shift_x=G[0, 0],
shift_y=G[0, 1],
semi_axis_x=G[0, 2],
semi_axis_y=G[0, 3],
alpha=G[0, 4],
scale_factor=G[0, 7],
radius=radius_1)
mask_scale_1 = torch.cat((mask_scale1, mask_scale2, mask_scale3),
dim=0)
mask_scale_1_rad = torch.clamp(mask_scale_1.unsqueeze(0), 0, max_scale)
############
mask_scale4 = self.get_mask(x_axis,
y_axis,
shift_x=G[0, 8],
shift_y=G[0, 9],
semi_axis_x=G[0, 10],
semi_axis_y=G[0, 11],
alpha=G[0, 12],
scale_factor=G[0, 13],
radius=radius_2)
mask_scale5 = self.get_mask(x_axis,
y_axis,
shift_x=G[0, 8],
shift_y=G[0, 9],
semi_axis_x=G[0, 10],
semi_axis_y=G[0, 11],
alpha=G[0, 12],
scale_factor=G[0, 14],
radius=radius_2)
mask_scale6 = self.get_mask(x_axis,
y_axis,
shift_x=G[0, 8],
shift_y=G[0, 9],
semi_axis_x=G[0, 10],
semi_axis_y=G[0, 11],
alpha=G[0, 12],
scale_factor=G[0, 15],
radius=radius_2)
mask_scale_4 = torch.cat((mask_scale4, mask_scale5, mask_scale6),
dim=0)
mask_scale_4_rad = torch.clamp(mask_scale_4.unsqueeze(0), 0, max_scale)
############
mask_scale7 = self.get_mask(x_axis,
y_axis,
shift_x=G[0, 16],
shift_y=G[0, 17],
semi_axis_x=G[0, 18],
semi_axis_y=G[0, 19],
alpha=G[0, 20],
scale_factor=G[0, 21],
radius=radius_3)
mask_scale8 = self.get_mask(x_axis,
y_axis,
shift_x=G[0, 16],
shift_y=G[0, 17],
semi_axis_x=G[0, 18],
semi_axis_y=G[0, 19],
alpha=G[0, 20],
scale_factor=G[0, 22],
radius=radius_3)
mask_scale9 = self.get_mask(x_axis,
y_axis,
shift_x=G[0, 16],
shift_y=G[0, 17],
semi_axis_x=G[0, 18],
semi_axis_y=G[0, 19],
alpha=G[0, 20],
scale_factor=G[0, 23],
radius=radius_3)
mask_scale_7 = torch.cat((mask_scale7, mask_scale8, mask_scale9),
dim=0)
mask_scale_7_rad = torch.clamp(mask_scale_7.unsqueeze(0), 0, max_scale)
mask_scale_elliptical = torch.clamp(
mask_scale_1_rad * mask_scale_4_rad * mask_scale_7_rad, 0,
max_scale)
return mask_scale_elliptical
def get_eig_adjacency(adj,
eig_idx,
normalization='none',
add_diag=True,
absolute_adj=False,
normalize_L=False,
eig_acos=True):
r"""
Computes an adjacency matrix from the gradient of the eigenvectors of the graph Laplacian,
with the graph being described by the adjacency matrix ``adj``.
The gradient is a function on the edges, computed from these node features.
For 2 nodes ``n_i, n_j`` with node features ``f_i, f_j`` and an edge weight of ``w_ij``,
the gradient on edge ``e_ij`` is given by ``e_ij = w_ij * (f_j - f_i)``.
If ``X`` is a function of the nodes, then ``matmul(grad_adj, X)`` behaves as the derivative of
``X`` in the direction of the gradient of ``features``.
Parameters
--------------
adj: tensor(..., N, N)
Batches of adjacency matrices of graphs. It is used to compute the Laplacian matrix,
and the eigenvectors of the Laplacian.
eig_idx: int, iterator(int)
Indexes of the eigenvectors to compute, sorted by smallest eigenvalues.
The current function is efficient for low indexes.
e.g. ``eig_idx=[1, 2, 4]`` will compute the eigenvectors ``[0, 1, 2, 3, 4]``, but will
then ignore the eigenvectors 0 and 3.
If ``0`` is in the ``eig_idx``, then the normalized adjacency matrix is returned for
that specific index, no matter the normalization chosen.
normalization: str, Optional
Normalization strategy for the ``grad_adj`` matrix. It does not apply to eig_idx==0.
- 'none': No normalization is applied
- 'row-abs': Normalize such that the absolute sum of each row of ``grad_adj`` is 1.
All values below the global variable ``EPS=1e-5`` will be set to 0.
- 'in-out-field': Normalize such that the the norm of the input and output field
are 1. The input field is the negative values in the ``grad_adj``, while the
output field is the positive values. The norm is computed as the root of the squared sum.
All values below the global variable ``EPS=1e-5`` will be set to 0.
For example, if there are no positive values above EPS for a specific node, then
the output field of that specific node will be set to 0.
(Default='none')
add_diag: bool, Optional
Whether to add a diagonal element such as each row of ``grad_adj`` sums to 0.
The diagonal is added after the normalization. It does not apply to eig_idx==0.
- If True, ``matmul(grad_adj, X)`` will behave like a forward/backward derivative
at the local minima/maxima from the features function.
- If False, ``matmul(grad_adj, X)`` will behave similarily to a center derivative
with a zero-padding added before the local minima/maxima
(Default=True)
absolute_adj: bool, Optional
Return the absolute value of ``grad_adj``.
``matmul(grad_adj, X)`` becomes a directional smoothing instead of a directional
derivative.
(Default=False)
normalize_L: bool, Optional
Whether to normalize the Laplacian matrix
If `False`, then `L = D - A`
If `True`, then `L = D^-1 (D - A)`
(Default=False)
eig_acos: bool, Optional
Whether to compute the arcosine of the eigenvectors instead of the eigenvector.
This will sort-of `linearize` the eigenvector. If ``normalize=='in-out-field'``,
this parameter shouldn't change anything, appart from numerical error. For other
normalization, it will have an important impact, especially near the borders
of the graph.
Returns
-------------
eig_adj: dict(tensor(..., N, N))
Dictionary of Batches of adjacency matrices representing the gradient of
the eigenvectors on the graph. Each key corresponds to a specific ``eig_idx``,
and the value corresponds to the associated batch of eigen-adjacencies.
"""
# Convert the eig_idx into a list
try:
eig_idx = list(eig_idx)
except:
eig_idx = [eig_idx]
# Generate an adjacency matrix for each eigenvectors from `eig_idx`
eigvec = get_k_lowest_eig(adj, max(eig_idx) + 1, normalize_L=normalize_L)
eig_adj = {}
for ii in eig_idx:
if ii != 0:
this_eigvec = eigvec[..., ii]
if eig_acos:
this_eigvec = torch.acos(this_eigvec /
torch.max(torch.abs(this_eigvec)))
w_adj = get_adjacency_from_gradient_of_features(
adj,
features=this_eigvec,
normalization=normalization,
add_diag=add_diag,
absolute_adj=absolute_adj)
eig_adj[ii] = w_adj
else:
eig_adj[ii] = adj / (torch.sum(adj.abs(), dim=-1, keepdims=True) +
EPS)
return eig_adj
def acos(*, input):
return torch.acos(**locals())
def nn_batch_angular_distance(a, b):
sim = F.cosine_similarity(a, b, dim=-1, eps=1e-6)
sim = F.hardtanh(sim, 1e-6, 1.0 - 1e-6)
return torch.mean(torch.acos(sim) * (180 / np.pi), dim=1)
def so3_log(R):
#input: R 64x3x3
#output: log(R) 64x3
batch_size = R.size(0)
# The rotation axis (not unit-length) is given by
axes = R.new(batch_size, 3).zero_()
axes[:,0] = R[:, 2, 1] - R[:, 1, 2]
axes[:,1] = R[:, 0, 2] - R[:, 2, 0]
axes[:,2] = R[:, 1, 0] - R[:, 0, 1]
# The sine of the rotation angle is half the norm of the axis
# This does not work well??
#sin_angles = 0.5 * vec_norms(axes)
#angles = torch.atan2(sin_angles, cos_angles)
# The cosine of the rotation angle is related to the trace of C
#NOTE: clamp ensures that we don't get any nan's due to out of range numerical errors
angles = torch.acos((0.5 * batch_trace(R) - 0.5).clamp(-1+EPS,1-EPS))
sin_angles = torch.sin(angles)
# If angle is close to zero, use first-order Taylor expansion
small_angles_mask = angles.lt(EPS).view(-1)
small_angles_num = small_angles_mask.sum()
#This tensor is used to extract the 3x3 R's that correspond to small angles
small_angles_indices = small_angles_mask.nonzero().squeeze()
#print('small angles: {}/{}.'.format(small_angles_num, batch_size))
if small_angles_num == 0:
#Regular log
ax_sin = axes / sin_angles.expand_as(axes)
logs = 0.5 * angles.expand_as(ax_sin) * ax_sin
elif small_angles_num == batch_size:
#Small angle Log
I = R.new(3, 3).zero_()
I[0,0] = I[1,1] = I[2,2] = 1.0
I = I.expand(batch_size, 3,3) #I is now batch_sizex3x3
logs = so3_vee(R - I)
else:
#Some combination of both
I = R.new(3, 3).zero_()
I[0,0] = I[1,1] = I[2,2] = 1.0
I = I.expand(small_angles_num, 3,3) #I is now small_angles_numx3x3
ax_sin = (axes / sin_angles.expand_as(axes))
logs = 0.5 * angles.expand_as(ax_sin) * ax_sin
small_logs = so3_vee(R[small_angles_indices] - I)
logs[small_angles_indices] = small_logs
return logs
def tensorAngularDiff(q1, q2):
return 2 * torch.acos(torch.abs((q1 * q2).sum(1)).clamp(max=1 - eps))
def main():
class_id = 0
class_file = open('datasets/ycb/dataset_config/classes.txt')
cld = {}
while 1:
class_input = class_file.readline()
if not class_input:
break
input_file = open('{0}/models/{1}/points.xyz'.format(
opt.dataset_root, class_input[:-1]))
cld[class_id] = []
while 1:
input_line = input_file.readline()
if not input_line:
break
input_line = input_line[:-1].split(' ')
cld[class_id].append([
float(input_line[0]),
float(input_line[1]),
float(input_line[2])
])
cld[class_id] = np.array(cld[class_id])
input_file.close()
class_id += 1
opt.manualSeed = random.randint(1, 10000)
random.seed(opt.manualSeed)
torch.manual_seed(opt.manualSeed)
symmetry_obj_idx = [12, 15, 18, 19, 20]
if opt.dataset == 'ycb':
opt.num_objects = 21 # number of object classes in the dataset
opt.num_points = 1000 # number of points on the input pointcloud
opt.outf = 'trained_models/ycb/' + opt.output_dir # folder to save trained models
opt.test_output = 'experiments/output/ycb/' + opt.output_dir
if not os.path.exists(opt.test_output):
os.makedirs(opt.test_output, exist_ok=True)
opt.repeat_epoch = 1 # number of repeat times for one epoch training
elif opt.dataset == 'linemod':
opt.num_objects = 13
opt.num_points = 500
opt.outf = 'trained_models/linemod'
opt.log_dir = 'experiments/logs/linemod'
opt.repeat_epoch = 20
else:
print('Unknown dataset')
return
estimator = PoseNet(num_points=opt.num_points,
num_obj=opt.num_objects,
object_max=opt.object_max)
estimator.cuda()
if opt.resume_posenet != '':
estimator.load_state_dict(
torch.load('{0}/{1}'.format(opt.outf, opt.resume_posenet)))
opt.refine_start = False
opt.decay_start = False
dataset = PoseDataset_ycb('train', opt.num_points, False, opt.dataset_root,
opt.noise_trans, opt.seg_type, True)
test_dataset = PoseDataset_ycb('test', opt.num_points, False,
opt.dataset_root, 0.0, opt.seg_type, True)
testdataloader = torch.utils.data.DataLoader(test_dataset,
shuffle=False,
num_workers=opt.workers)
opt.sym_list = dataset.get_sym_list()
opt.num_points_mesh = dataset.get_num_points_mesh()
print(
'>>>>>>>>----------Dataset loaded!---------<<<<<<<<\nlength of the training set: {0}\nlength of the testing set: {1}\nnumber of sample points on mesh: {2}\nsymmetry object list: {3}'
.format(len(dataset), len(test_dataset), opt.num_points_mesh,
opt.sym_list))
criterion = Loss(opt.num_points_mesh, opt.sym_list)
logger = setup_logger(
'final_eval_tf_with_seg_square',
os.path.join(opt.test_output, 'final_eval_tf_with_seg_square.txt'))
object_max = opt.object_max
total_test_dis = {key: [] for key in range(0, object_max)}
total_test_count = {key: [] for key in range(0, object_max)}
dir_test_dis = {key: [] for key in range(0, object_max)}
dir_test_count = {key: [] for key in range(0, object_max)}
# for add
total_unseen_objects = {key: [] for key in range(0, object_max)}
total_object_without_pose = {key: [] for key in range(0, object_max)}
dir_add_count = {key: [] for key in range(0, object_max)}
dir_add_count_unseen = {key: [] for key in range(0, object_max)}
dir_add_02_count_unseen = {key: [] for key in range(0, object_max)}
dir_add_pure_count = {key: [] for key in range(0, object_max)}
dir_add_s_count = {key: [] for key in range(0, object_max)}
dir_add_02_count = {key: [] for key in range(0, object_max)}
dir_add_pure_02_count = {key: [] for key in range(0, object_max)}
dir_add_s_02_count = {key: [] for key in range(0, object_max)}
total_add_count = {key: [] for key in range(0, object_max)}
total_add_count_unseen = {key: [] for key in range(0, object_max)}
total_add_02_count_unseen = {key: [] for key in range(0, object_max)}
total_add_pure_count = {key: [] for key in range(0, object_max)}
total_add_s_count = {key: [] for key in range(0, object_max)}
total_add_02_count = {key: [] for key in range(0, object_max)}
total_add_pure_02_count = {key: [] for key in range(0, object_max)}
total_add_s_02_count = {key: [] for key in range(0, object_max)}
dir_dbd_count = {key: [] for key in range(0, object_max)}
dir_drr_count = {key: [] for key in range(0, object_max)}
dir_ada_count = {key: [] for key in range(0, object_max)}
dir_distance_1_count = {key: [] for key in range(0, object_max)}
total_dbd_count = {key: [] for key in range(0, object_max)}
total_drr_count = {key: [] for key in range(0, object_max)}
total_ada_count = {key: [] for key in range(0, object_max)}
total_distance_1_count = {key: [] for key in range(0, object_max)}
last_dis = {key: [] for key in range(0, object_max)}
for i in range(object_max):
total_unseen_objects[i] = 0
total_object_without_pose[i] = 0
total_test_dis[i] = 0.
total_test_count[i] = 0
dir_test_dis[i] = 0.
dir_test_count[i] = 0
# for add
dir_add_count[i] = 0
dir_add_count_unseen[i] = 0
dir_add_02_count_unseen[i] = 0
dir_add_pure_count[i] = 0
dir_add_s_count[i] = 0
dir_add_02_count[i] = 0
total_add_count[i] = 0
total_add_count_unseen[i] = 0
total_add_02_count_unseen[i] = 0
total_add_pure_count[i] = 0
total_add_s_count[i] = 0
total_add_02_count[i] = 0
dir_add_pure_02_count[i] = 0
dir_add_s_02_count[i] = 0
total_add_pure_02_count[i] = 0
total_add_s_02_count[i] = 0
# for stable
dir_dbd_count[i] = 0.
dir_drr_count[i] = 0
dir_ada_count[i] = 0.
dir_distance_1_count[i] = 0.
total_dbd_count[i] = 0.
total_drr_count[i] = 0
total_ada_count[i] = 0.
total_distance_1_count[i] = 0.
last_dis[i] = None
st_time = time.time()
isFirstInitLastDatafolder = True
estimator.eval()
with torch.no_grad():
for j, data in enumerate(testdataloader, 0):
if opt.dataset == 'ycb':
list_points, list_choose, list_img, list_target, list_model_points, list_idx, list_filename, \
list_full_img, list_focal_length, list_principal_point, list_motion = data
output_image = Image.open('{0}/{1}-color-masked-square.png'.format(
opt.dataset_root, list_filename[0][0]))
OUTPUT_IMAGE_PATH = '{0}/{1}-color-seg-square-output-tf.png'.format(
opt.dataset_root, list_filename[0][0])
for list_index in range(len(list_points)):
points, choose, img, target, model_points, idx, filename, full_img, focal_length, principal_point, motion \
= list_points[list_index], list_choose[list_index], list_img[list_index], \
list_target[list_index], list_model_points[list_index], list_idx[list_index], \
list_filename[list_index], list_full_img[list_index], list_focal_length[list_index], \
list_principal_point[list_index], list_motion[list_index]
# Temporal Clean when Changing datafolder
datafolder = filename[0].split('/')[1]
filehead = filename[0].split('/')[2]
if isFirstInitLastDatafolder:
lastdatafolder = datafolder
isFirstInitLastDatafolder = False
if datafolder != lastdatafolder:
logger.info('changing folder from {0} to {1}'.format(
lastdatafolder, datafolder))
estimator.temporalClear(opt.object_max)
# handle dir output
for i in range(0, object_max):
if dir_test_count[i] != 0:
logger.info(
'Dir {0} Object {1} dis:{2} with {3} samples'.
format(lastdatafolder, i,
dir_test_dis[i] / dir_test_count[i],
dir_test_count[i]))
if dir_add_count[i] != 0:
logger.info(
'Dir {0} Object {1} add:{2} with 0.02: {3}'
.format(
lastdatafolder, i,
dir_add_count[i] / dir_test_count[i],
dir_add_02_count[i] /
dir_add_count[i]))
else:
logger.info(
'Dir {0} Object {1} add:{2} with 0.02: {3}'
.format(
lastdatafolder, i,
dir_add_count[i] / dir_test_count[i],
0))
if dir_add_pure_count[i] != -0:
logger.info(
'Dir {0} Object {1} add_pure:{2} with 0.02: {3}'
.format(
lastdatafolder, i,
dir_add_pure_count[i] /
dir_test_count[i],
dir_add_pure_02_count[i] /
dir_add_pure_count[i]))
else:
logger.info(
'Dir {0} Object {1} add_pure:{2} with 0.02: {3}'
.format(
lastdatafolder, i,
dir_add_pure_count[i] /
dir_test_count[i], 0))
if dir_add_s_count[i] != 0:
logger.info(
'Dir {0} Object {1} add_s:{2} with 0.02: {3}'
.format(
lastdatafolder, i,
dir_add_s_count[i] / dir_test_count[i],
dir_add_s_02_count[i] /
dir_add_s_count[i]))
else:
logger.info(
'Dir {0} Object {1} add_s:{2} with 0.02: {3}'
.format(
lastdatafolder, i,
dir_add_s_count[i] / dir_test_count[i],
0))
logger.info('Dir {0} Object {1} dbd:{2}'.format(
lastdatafolder, i,
dir_dbd_count[i] / dir_test_count[i]))
logger.info('Dir {0} Object {1} drr:{2}'.format(
lastdatafolder, i,
dir_drr_count[i] / dir_test_count[i]))
logger.info('Dir {0} Object {1} ada:{2}'.format(
lastdatafolder, i,
dir_ada_count[i] / dir_test_count[i]))
logger.info(
'Dir {0} Object {1} distance_1:{2}'.format(
lastdatafolder, i,
dir_distance_1_count[i] /
dir_test_count[i]))
dir_dbd = 0.
dir_drr = 0.
dir_ada = 0.
dir_distance_1 = 0.
dir_dis = 0.
dir_add = 0
dir_add_s = 0
dir_add_pure = 0
dir_add_02 = 0
dir_add_s_02 = 0
dir_add_pure_02 = 0
dir_count = 0
for i in range(object_max):
if total_test_count[i] != 0:
dir_count += dir_test_count[i]
dir_dis += dir_test_dis[i]
dir_add += dir_add_count[i]
dir_add_pure += dir_add_pure_count[i]
dir_add_s += dir_add_s_count[i]
dir_add_02 += dir_add_02_count[i]
dir_add_pure_02 += dir_add_pure_02_count[i]
dir_add_s_02 += dir_add_s_02_count[i]
dir_dbd += dir_dbd_count[i]
dir_drr += dir_drr_count[i]
dir_ada += dir_ada_count[i]
dir_distance_1 += dir_distance_1_count[i]
dir_test_dis[i] = 0
dir_test_count[i] = 0
dir_add_count[i] = 0
dir_add_pure_count[i] = 0
dir_add_s_count[i] = 0
dir_add_02_count[i] = 0
dir_add_pure_02_count[i] = 0
dir_add_s_02_count[i] = 0
dir_dbd_count[i] = 0
dir_drr_count[i] = 0
dir_ada_count[i] = 0
dir_distance_1_count[i] = 0
last_dis[i] = None
logger.info(
'Dir {0} \'s total dis:{1} with {2} samples'.format(
lastdatafolder, dir_dis / dir_count, dir_count))
logger.info(
'Dir {0} \'s total add:{1} with 0.02: {2}'.format(
lastdatafolder, dir_add / dir_count,
dir_add_02 / dir_add))
logger.info(
'Dir {0} \'s total add_s:{1} with 0.02: {2}'.format(
lastdatafolder, dir_add_s / dir_count,
dir_add_s_02 / dir_add_s))
logger.info(
'Dir {0} \'s total add_pure:{1} with 0.02: {2}'.format(
lastdatafolder, dir_add_pure / dir_count,
dir_add_pure_02 / dir_add_pure))
logger.info('Dir {0} \'s total dbd:{1}'.format(
lastdatafolder, dir_dbd / dir_count))
logger.info('Dir {0} \'s total drr:{1}'.format(
lastdatafolder, dir_drr / dir_count))
logger.info('Dir {0} \'s total ada:{1}'.format(
lastdatafolder, dir_ada / dir_count))
logger.info('Dir {0} \'s total distance_1:{1}'.format(
lastdatafolder, dir_distance_1 / dir_count))
# end of handle dir output
lastdatafolder = datafolder
points, choose, img, target, model_points, idx = points.cuda(), \
choose.cuda(), \
img.cuda(), \
target.cuda(), \
model_points.cuda(), \
idx.cuda()
cloud_path = "experiments/clouds/ycb/{0}/{1}/{2}/{3}_{4}".format(
opt.output_dir, 1, datafolder, filehead,
int(idx)) # folder to save logs
pred_r, pred_t, pred_c, x_return = estimator(
img, points, choose, idx, focal_length, principal_point,
motion, cloud_path)
# count for unseen object
if pred_r is None:
last_dis[int(idx)] = None
total_unseen_objects[int(idx)] += 1
total_object_without_pose[int(idx)] += 1
continue
pred_r_ori = copy.deepcopy(pred_r)
pred_t_ori = copy.deepcopy(pred_t)
pred_c_ori = copy.deepcopy(pred_c)
x_return_ori = copy.deepcopy(x_return)
gt_r, gt_t = get_target(opt.dataset_root, filename, idx)
if gt_r is None: print('gtr is None')
is_sym = int(idx) in symmetry_obj_idx
dis, dis_vector, pred_cloud = calDistance(
pred_r_ori, pred_t_ori, pred_c_ori, x_return_ori, gt_r,
gt_t, cld[int(idx)], is_sym)
dis_s, dis_vector_s, _ = calDistance(pred_r_ori, pred_t_ori,
pred_c_ori, x_return_ori,
gt_r, gt_t, cld[int(idx)],
True)
dis_pure, dis_vector_pure, _ = calDistance(
pred_r_ori, pred_t_ori, pred_c_ori, x_return_ori, gt_r,
gt_t, cld[int(idx)], False)
if last_dis[int(idx)] is not None:
dir_dbd_count[int(idx)] += torch.norm(dis_vector -
last_dis[int(idx)])
total_dbd_count[int(idx)] += torch.norm(dis_vector -
last_dis[int(idx)])
dir_distance_1_count[int(idx)] += torch.norm(
(dis_vector / torch.norm(dis_vector)) -
(last_dis[int(idx)] / torch.norm(last_dis[int(idx)])))
total_distance_1_count[int(idx)] += torch.norm(
(dis_vector / torch.norm(dis_vector)) -
(last_dis[int(idx)] / torch.norm(last_dis[int(idx)])))
if torch.dot(last_dis[int(idx)], dis_vector) < 0:
dir_drr_count[int(idx)] += 1
total_drr_count[int(idx)] += 1
dir_ada_count[int(idx)] += torch.acos(
(torch.dot(last_dis[int(idx)], dis_vector)) /
(torch.norm(last_dis[int(idx)]) *
torch.norm(dis_vector)))
total_ada_count[int(idx)] += torch.acos(
(torch.dot(last_dis[int(idx)], dis_vector)) /
(torch.norm(last_dis[int(idx)]) *
torch.norm(dis_vector)))
last_dis[int(idx)] = dis_vector
# calc adds
if img.shape[1] != 0:
dir_test_dis[int(idx)] += dis.item()
total_test_dis[int(idx)] += dis.item()
dir_test_count[int(idx)] += 1
total_test_count[int(idx)] += 1
if dis < 0.1:
dir_add_count[int(idx)] += 1
total_add_count[int(idx)] += 1
if dis < 0.02:
dir_add_02_count[int(idx)] += 1
total_add_02_count[int(idx)] += 1
if dis_s < 0.1:
dir_add_s_count[int(idx)] += 1
total_add_s_count[int(idx)] += 1
if dis_s < 0.02:
dir_add_s_02_count[int(idx)] += 1
total_add_s_02_count[int(idx)] += 1
if dis_pure < 0.1:
dir_add_pure_count[int(idx)] += 1
total_add_pure_count[int(idx)] += 1
if dis_pure < 0.02:
dir_add_pure_02_count[int(idx)] += 1
total_add_pure_02_count[int(idx)] += 1
else:
last_dis[int(idx)] = None
if dis < 0.1:
dir_add_count_unseen[int(idx)] += 1
total_add_count_unseen[int(idx)] += 1
total_unseen_objects[int(idx)] += 1
if dis < 0.02:
dir_add_02_count_unseen[int(idx)] += 1
total_add_02_count_unseen[int(idx)] += 1
total_unseen_objects[int(idx)] += 1
output_image = output_transformed_image(
OUTPUT_IMAGE_PATH, output_image, pred_cloud, focal_length,
principal_point, int(idx))
logger.info('Test time {0} Test Frame {1} {2} dis:{3}'.format(
time.strftime("%Hh %Mm %Ss",
time.gmtime(time.time() - st_time)),
filename, idx.item(), dis))
output_image.save(OUTPUT_IMAGE_PATH)
# handle dir output
for i in range(0, object_max):
if dir_test_count[i] != 0:
logger.info(
'Dir {0} Object {1} dis:{2} with {3} samples'.format(
lastdatafolder, i, dir_test_dis[i] / dir_test_count[i],
dir_test_count[i]))
if dir_add_count[i] != 0:
logger.info(
'Dir {0} Object {1} add:{2} with 0.02: {3}'.format(
lastdatafolder, i,
dir_add_count[i] / dir_test_count[i],
dir_add_02_count[i] / dir_add_count[i]))
else:
logger.info(
'Dir {0} Object {1} add:{2} with 0.02: {3}'.format(
lastdatafolder, i,
dir_add_count[i] / dir_test_count[i], 0))
if dir_add_pure_count[i] != -0:
logger.info(
'Dir {0} Object {1} add_pure:{2} with 0.02: {3}'.
format(
lastdatafolder, i,
dir_add_pure_count[i] / dir_test_count[i],
dir_add_pure_02_count[i] / dir_add_pure_count[i]))
else:
logger.info(
'Dir {0} Object {1} add_pure:{2} with 0.02: {3}'.
format(lastdatafolder, i,
dir_add_pure_count[i] / dir_test_count[i], 0))
if dir_add_s_count[i] != 0:
logger.info(
'Dir {0} Object {1} add_s:{2} with 0.02: {3}'.format(
lastdatafolder, i,
dir_add_s_count[i] / dir_test_count[i],
dir_add_s_02_count[i] / dir_add_s_count[i]))
else:
logger.info(
'Dir {0} Object {1} add_s:{2} with 0.02: {3}'.format(
lastdatafolder, i,
dir_add_s_count[i] / dir_test_count[i], 0))
logger.info('Dir {0} Object {1} dbd:{2}'.format(
lastdatafolder, i, dir_dbd_count[i] / dir_test_count[i]))
logger.info('Dir {0} Object {1} drr:{2}'.format(
lastdatafolder, i, dir_drr_count[i] / dir_test_count[i]))
logger.info('Dir {0} Object {1} ada:{2}'.format(
lastdatafolder, i, dir_ada_count[i] / dir_test_count[i]))
logger.info('Dir {0} Object {1} distance_1:{2}'.format(
lastdatafolder, i,
dir_distance_1_count[i] / dir_test_count[i]))
dir_dbd = 0.
dir_drr = 0.
dir_ada = 0.
dir_distance_1 = 0.
dir_dis = 0.
dir_add = 0
dir_add_s = 0
dir_add_pure = 0
dir_add_02 = 0
dir_add_s_02 = 0
dir_add_pure_02 = 0
dir_count = 0
for i in range(object_max):
if total_test_count[i] != 0:
dir_count += dir_test_count[i]
dir_dis += dir_test_dis[i]
dir_add += dir_add_count[i]
dir_add_pure += dir_add_pure_count[i]
dir_add_s += dir_add_s_count[i]
dir_add_02 += dir_add_02_count[i]
dir_add_pure_02 += dir_add_pure_02_count[i]
dir_add_s_02 += dir_add_s_02_count[i]
dir_dbd += dir_dbd_count[i]
dir_drr += dir_drr_count[i]
dir_ada += dir_ada_count[i]
dir_distance_1 += dir_distance_1_count[i]
dir_test_dis[i] = 0
dir_test_count[i] = 0
dir_add_count[i] = 0
dir_add_pure_count[i] = 0
dir_add_s_count[i] = 0
dir_add_02_count[i] = 0
dir_add_pure_02_count[i] = 0
dir_add_s_02_count[i] = 0
dir_dbd_count[i] = 0
dir_drr_count[i] = 0
dir_ada_count[i] = 0
dir_distance_1_count[i] = 0
logger.info('Dir {0} \'s total dis:{1} with {2} samples'.format(
lastdatafolder, dir_dis / dir_count, dir_count))
logger.info('Dir {0} \'s total add:{1} with 0.02: {2}'.format(
lastdatafolder, dir_add / dir_count, dir_add_02 / dir_add))
logger.info('Dir {0} \'s total add_s:{1} with 0.02: {2}'.format(
lastdatafolder, dir_add_s / dir_count, dir_add_s_02 / dir_add_s))
logger.info('Dir {0} \'s total add_pure:{1} with 0.02: {2}'.format(
lastdatafolder, dir_add_pure / dir_count,
dir_add_pure_02 / dir_add_pure))
logger.info('Dir {0} \'s total dbd:{1}'.format(lastdatafolder,
dir_dbd / dir_count))
logger.info('Dir {0} \'s total drr:{1}'.format(lastdatafolder,
dir_drr / dir_count))
logger.info('Dir {0} \'s total ada:{1}'.format(lastdatafolder,
dir_ada / dir_count))
logger.info('Dir {0} \'s total distance_1:{1}'.format(
lastdatafolder, dir_distance_1 / dir_count))
# end of handle dir output
# handle global output
total_unseen_count = 0
total_without_pose_count = 0
total_add_count_unseen_count = 0
total_add_02_count_unseen_count = 0
total_drr = 0.
total_dbd = 0.
total_ada = 0.
total_distance_1 = 0.
total_dis = 0.
total_add = 0
total_add_s = 0
total_add_pure = 0
total_add_02 = 0
total_add_s_02 = 0
total_add_pure_02 = 0
total_count = 0
for i in range(object_max):
if total_test_count[i] != 0:
logger.info(
'Total: Object {0} dis:{1} with {2} samples'.format(
i, total_test_dis[i] / total_test_count[i],
total_test_count[i]))
logger.info('Total: Object {0} add:{1} with 0.02: {2}'.format(
i, total_add_count[i] / total_test_count[i],
total_add_02_count[i] / total_add_count[i]))
logger.info('Total: Object {0} drr:{1}'.format(
i, total_drr_count[i] / total_test_count[i]))
logger.info('Total: Object {0} ada:{1}'.format(
i, total_ada_count[i] / total_test_count[i]))
logger.info('Total: Object {0} distance_1:{1}'.format(
i, total_distance_1_count[i] / total_test_count[i]))
if total_unseen_objects[i] != 0:
if total_unseen_objects[i] - total_object_without_pose[
i] != 0:
logger.info(
'Total: Unseen Object {0} add:{1} with 0.02: {2} with {3} samples '
.format(
i, total_add_count_unseen[i] /
(total_unseen_objects[i] -
total_object_without_pose[i]),
total_add_02_count_unseen[i] /
total_add_count_unseen[i],
(total_unseen_objects[i] -
total_object_without_pose[i])))
logger.info(
'Total: Object {0} unseen :{1} times, {2} of them without poses, success rate:{3}'
.format(i, total_unseen_objects[i],
total_object_without_pose[i],
(total_unseen_objects[i] -
total_object_without_pose[i]) /
total_unseen_objects[i]))
total_unseen_count += total_unseen_objects[i]
total_without_pose_count += total_object_without_pose[i]
total_count += total_test_count[i]
total_dis += total_test_dis[i]
total_add += total_add_count[i]
total_add_count_unseen_count += total_add_count_unseen[i]
total_add_02_count_unseen_count += total_add_02_count_unseen[i]
total_add_s += total_add_s_count[i]
total_add_pure += total_add_pure_count[i]
total_add_02 += total_add_02_count[i]
total_add_s_02 += total_add_s_02_count[i]
total_add_pure_02 += total_add_pure_02_count[i]
total_dbd += total_dbd_count[i]
total_drr += total_drr_count[i]
total_ada += total_ada_count[i]
total_distance_1 += total_distance_1_count[i]
logger.info('total dis:{0} with {1} samples'.format(
total_dis / total_count, total_count))
logger.info('total add:{0} with 0.02: {1}'.format(
total_add / total_count, total_add_02 / total_add))
logger.info('total unseen add:{0} with 0.02: {1}'.format(
total_add_count_unseen_count /
(total_unseen_count - total_without_pose_count),
total_add_02_count_unseen_count / total_add_count_unseen_count))
logger.info('total add_pure:{0} with 0.02: {1}'.format(
total_add_pure / total_count, total_add_pure_02 / total_add_pure))
logger.info('total add_s:{0} with 0.02: {1}'.format(
total_add_s / total_count, total_add_s_02 / total_add_s))
logger.info(
'detected unseen object :{0}, failed calculate {1} poses with success rate: {2}'
.format(total_unseen_count, total_without_pose_count,
(total_unseen_count - total_without_pose_count) /
total_unseen_count))
logger.info('Total drr:{0}'.format(total_drr / total_count))
logger.info('Total ada:{0}'.format(total_ada / total_count))
logger.info('Total distance_1:{0}'.format(total_distance_1 /
total_count))
def get_angles(self, cosine_of_target_classes):
angles = torch.acos(torch.clamp(cosine_of_target_classes, -1, 1))
if self.collect_stats:
with torch.no_grad():
self.avg_angle = np.degrees(torch.mean(angles).item())
return angles
def slerp(val, low, high):
omega = torch.acos(torch.dot(low / torch.norm(low), high / torch.norm(high)))
so = torch.sin(omega)
if so == 0:
return (1.0 - val) * low + val * high # L'Hopital's rule/LERP
return torch.sin((1.0 - val) * omega) / so * low + torch.sin(val * omega) / so * high
def error(self):
return torch.acos(meps * 0.5 * (torch.sum(
torch.sum(self.o[:, :, None, :] * identity[:, None, :, :],
3)[:, m], 1) - 1.0))
def arccos(val):
return torch.acos(val)
def get_nn_pose_from_desc(variable, **kwargs):
feat = kwargs.pop('feat', None)
db = kwargs.pop('db', None)
metric = kwargs.pop('metric', 'cosine')
k_nn = kwargs.pop('k_nn', 1)
step_k_nn = kwargs.pop('step_k_nn', 1)
angle_threshold = kwargs.pop('angle_threshold', None) # degree
pos_threshold = kwargs.pop('pos_threshold', None) # m
return_only_T = kwargs.pop('return_only_T', False) # m
if kwargs:
raise TypeError('Unexpected **kwargs: %r' % kwargs)
feats = recc_acces(variable, feat)
db = recc_acces(variable, db)
if not hasattr(get_nn_pose_from_desc, 'nn_computor'):
logger.info('New nn indexing, fitting the db...')
get_nn_pose_from_desc.nn_computor = skn.NearestNeighbors(n_neighbors=1,
metric=metric)
get_nn_pose_from_desc.nn_computor.fit(
torch.stack(db['feat']).cpu().numpy())
idx = get_nn_pose_from_desc.nn_computor.kneighbors(feats.cpu().numpy(),
return_distance=False,
n_neighbors=k_nn *
step_k_nn)
if k_nn > 0:
if angle_threshold:
poses = [db['pose'][idx[0, 0]]]
idx_next = 1
curr_d_angle = float('inf')
for i in range(1, k_nn * step_k_nn):
d_angle = 180 / 3.14159265 * 2 * torch.acos(
torch.abs(
torch.dot(db['pose'][idx[0, 0]]['q'][0],
db['pose'][idx[0, i]]['q'][0])))
if d_angle < angle_threshold and (d_angle > curr_d_angle
or curr_d_angle
== float('inf')):
curr_d_angle = d_angle
idx_next = i
elif d_angle > angle_threshold and (
d_angle < curr_d_angle
or curr_d_angle < angle_threshold):
curr_d_angle = d_angle
idx_next = i
poses.append(db['pose'][idx[0, idx_next]])
return poses
elif pos_threshold:
poses = [db['pose'][idx[0, 0]]]
idx_next = 1
curr_d_pos = float('inf')
for i in range(1, k_nn * step_k_nn):
d_pos = torch.sqrt(
torch.sum((db['pose'][idx[0, 0]]['p'][0] -
db['pose'][idx[0, i]]['p'][0])**2))
if d_pos < pos_threshold and (d_pos > curr_d_pos
or curr_d_pos == float('inf')):
curr_d_pos = d_pos
idx_next = i
elif d_pos > pos_threshold and (d_pos < curr_d_pos
or curr_d_pos < pos_threshold):
curr_d_pos = d_pos
idx_next = i
poses.append(db['pose'][idx[0, idx_next]])
return poses
else:
if return_only_T:
return [
db['pose'][idx[0, i]]['T']
for i in range(0, k_nn * step_k_nn, step_k_nn)
]
else:
return [
db['pose'][idx[0, i]]
for i in range(0, k_nn * step_k_nn, step_k_nn)
]
else:
if return_only_T:
return db['pose'][idx[0, 0]]['T']
else:
return db['pose'][idx[0, 0]]
def _get_psi_i(self, theta_i, joint_id):
if joint_id in [0, 2, 4, 6]:
a_p = self.gc[joint_id] * (
(self.c[joint_id, 1] - self.b[joint_id, 1]) *
torch.tan(theta_i) +
(self.b[joint_id, 0] - self.c[joint_id, 0]))
b_p = 2 * self.gc[joint_id] * (
self.a[joint_id, 1] * torch.tan(theta_i) - self.a[joint_id, 0])
c_p = self.gc[joint_id] * (
(self.c[joint_id, 1] + self.b[joint_id, 1]) *
torch.tan(theta_i) -
(self.b[joint_id, 0] + self.c[joint_id, 0]))
square_p = b_p**2 - 4 * a_p * c_p
psi_ = []
if square_p < 0:
return psi_
else:
psi_1 = 2 * torch.atan(
(-b_p - torch.sqrt(square_p)) / (2 * a_p))
theta_1 = torch.atan2(
self.gc[joint_id] *
(self.a[joint_id, 0] * torch.sin(psi_1) +
self.b[joint_id, 0] * torch.cos(psi_1) +
self.c[joint_id, 0]), self.gc[joint_id] *
(self.a[joint_id, 1] * torch.sin(psi_1) +
self.b[joint_id, 1] * torch.cos(psi_1) +
self.c[joint_id, 1]))
psi_2 = 2 * torch.atan(
(-b_p + torch.sqrt(square_p)) / (2 * a_p))
theta_2 = torch.atan2(
self.gc[joint_id] *
(self.a[joint_id, 0] * torch.sin(psi_2) +
self.b[joint_id, 0] * torch.cos(psi_2) +
self.c[joint_id, 0]), self.gc[joint_id] *
(self.a[joint_id, 1] * torch.sin(psi_2) +
self.b[joint_id, 1] * torch.cos(psi_2) +
self.c[joint_id, 1]))
if torch.isclose(theta_i, theta_1) and torch.isclose(
theta_i, theta_2):
psi_.append(psi_1)
psi_.append(psi_2)
return psi_
elif joint_id in [1, 5]:
square_p = self.a[joint_id, 0] ** 2 + self.b[joint_id, 0] ** 2 - \
(self.c[joint_id, 0] - torch.cos(theta_i)) ** 2
psi_ = []
if square_p < 0:
return psi_
else:
psi_1 = 2 * torch.atan(
(self.a[joint_id, 0] - torch.sqrt(square_p)) /
(torch.cos(theta_i) + self.b[joint_id, 0] -
self.c[joint_id, 0]))
theta_1 = torch.acos(self.a[joint_id, 0] * torch.sin(psi_1) +
self.b[joint_id, 0] * torch.cos(psi_1) +
self.c[joint_id, 0])
psi_2 = 2 * torch.atan(
(self.a[joint_id, 0] + torch.sqrt(square_p)) /
(torch.cos(theta_i) + self.b[joint_id, 0] -
self.c[joint_id, 0]))
theta_2 = torch.acos(self.a[joint_id, 0] * torch.sin(psi_2) +
self.b[joint_id, 0] * torch.cos(psi_2) +
self.c[joint_id, 0])
if torch.isclose(theta_i, theta_1) and torch.isclose(
theta_i, theta_2):
psi_.append(psi_1)
psi_.append(psi_2)
return psi_
else:
raise NotImplementedError()
def forward(self, inputs, targets):
"""
@inputs: a 3-dimensional tensor (a batch), including [samples-index, frames-dim-index, frames-index]
"""
assert len(inputs.shape) == 3
assert inputs.shape[2] == 1
# Normalize
normalized_x = F.normalize(inputs.squeeze(dim=2), dim=1)
normalized_weight = F.normalize(self.weight.squeeze(dim=2), dim=1)
cosine_theta = F.linear(normalized_x, normalized_weight) # Y = W*X
if not self.feature_normalize:
self.s = inputs.norm(2, dim=1) # [batch-size, l2-norm]
# The accuracy must be reported before margin penalty added
self.posterior = (self.s.detach() *
cosine_theta.detach()).unsqueeze(2)
else:
self.posterior = (self.s * cosine_theta.detach()).unsqueeze(2)
if not self.training:
# For valid set.
outputs = self.s * cosine_theta
return self.loss_function(outputs, targets)
# Margin Penalty
# cosine_theta [batch_size, num_class]
# targets.unsqueeze(1) [batch_size, 1]
cosine_theta_target = cosine_theta.gather(1, targets.unsqueeze(1))
if self.inter_loss > 0:
inter_cosine_theta = torch.softmax(self.s * cosine_theta, dim=1)
inter_cosine_theta_target = inter_cosine_theta.gather(
1, targets.unsqueeze(1))
# 1/N * \log (\frac{1 - logit_y}{C-1})
inter_loss = torch.log(
(inter_cosine_theta.sum(dim=1) - inter_cosine_theta_target) /
(self.num_targets - 1) + self.eps).mean()
if self.method == "am":
penalty_cosine_theta = cosine_theta_target - self.m
if self.double:
double_cosine_theta = cosine_theta + self.m
elif self.method == "aam":
# Another implementation w.r.t cosine(theta+m) = cosine_theta * cos_m - sin_theta * sin_m
# penalty_cosine_theta = self.cos_m * cosine_theta_target - self.sin_m * torch.sqrt((1-cosine_theta_target**2).clamp(min=0.))
penalty_cosine_theta = torch.cos(
torch.acos(cosine_theta_target) + self.m)
if self.double:
double_cosine_theta = torch.cos(
torch.acos(cosine_theta).add(-self.m))
elif self.method == "sm1":
# 可以自动调节惩罚项的大小，惩罚项线性减小
# penalty_cosine_theta = cosine_theta_target - (1 - cosine_theta_target) * self.m
penalty_cosine_theta = (1 + self.m) * cosine_theta_target - self.m
elif self.method == "sm2":
# 惩罚项指数下降
penalty_cosine_theta = cosine_theta_target - (
1 - cosine_theta_target**2) * self.m
elif self.method == "sm3":
penalty_cosine_theta = cosine_theta_target - (
1 - cosine_theta_target)**2 * self.m
else:
raise ValueError(
"Do not support this {0} margin w.r.t [ am | aam | sm1 | sm2 | sm3 ]"
.format(self.method))
# 模拟退火
penalty_cosine_theta = 1 / (1 + self.lambda_factor) * penalty_cosine_theta + \
self.lambda_factor / (1 + self.lambda_factor) * cosine_theta_target
if self.double:
cosine_theta = 1 / (1+self.lambda_factor) * double_cosine_theta + \
self.lambda_factor / (1+self.lambda_factor) * cosine_theta
if self.curricular is not None:
cosine_theta = self.curricular(cosine_theta, cosine_theta_target,
penalty_cosine_theta)
outputs = self.s * cosine_theta.scatter(1, targets.unsqueeze(1),
penalty_cosine_theta)
# Other extra loss
# Final reported loss will be always higher than softmax loss for the absolute margin penalty and
# it is a lie about why we can not decrease the loss to a mininum value. We should not report the
# loss after margin penalty did but we really report this invalid loss to avoid computing the
# training loss twice.
if self.ring_loss > 0:
ring_loss = torch.mean((self.s - self.r)**2) / 2
else:
ring_loss = 0.
if self.mhe_loss:
sub_weight = normalized_weight - torch.index_select(
normalized_weight, 0, targets).unsqueeze(dim=1)
# [N, C]
normed_sub_weight = sub_weight.norm(2, dim=2)
mask = torch.full_like(normed_sub_weight, True,
dtype=torch.bool).scatter_(
1, targets.unsqueeze(dim=1), False)
# [N, C-1]
normed_sub_weight_clean = torch.masked_select(
normed_sub_weight, mask).reshape(targets.size()[0], -1)
# torch.mean means 1/(N*(C-1))
the_mhe_loss = self.mhe_w * \
torch.mean((normed_sub_weight_clean **
2).clamp(min=self.eps)**-1)
return self.loss_function(
outputs / self.t,
targets) + the_mhe_loss + self.ring_loss * ring_loss
elif self.inter_loss > 0:
return self.loss_function(
outputs / self.t, targets
) + self.inter_loss * inter_loss + self.ring_loss * ring_loss
else:
return self.loss_function(outputs / self.t,
targets) + self.ring_loss * ring_loss
def backward(ctx, grad_output):
featureH, featureL, label, centers, margins, gamma = ctx.saved_tensors
num_pair = featureH.shape[0]
num_class = centers.shape[0]
centers_normed = F.normalize(centers, dim=1) # Cx1024
featureH_normed = F.normalize(featureH, dim=1) # N'x1024
distmatH = torch.mm(featureH_normed, centers_normed.t()) # N'xC
featureL_normed = F.normalize(featureL, dim=1)
distmatL = torch.mm(featureL_normed, centers_normed.t())
mask = distmatH.new_zeros(distmatH.size(), dtype=torch.long)
mask.scatter_add_(
1,
label.long().unsqueeze(1),
torch.ones(num_pair, 1, device=mask.device, dtype=torch.long))
distHic = distmatH[mask == 1]
distLic = distmatL[mask == 1]
distHicL, hard_index_batch = torch.max(distmatH[mask == 0].view(
num_pair, num_class - 1),
dim=1)
hard_index_batch[hard_index_batch >= label] += 1
li1 = torch.acos(distHic) - torch.acos(distLic) + margins[0]
li2 = torch.acos(distHic) - torch.acos(distHicL) + margins[1]
centers_normed_batch = centers_normed.index_select(0, label.long())
hard_normed_batch = centers_normed.index_select(0, hard_index_batch)
d = -(1 - distHic.pow(2)).pow(-0.5)
e = -(1 - distHicL.pow(2)).pow(-0.5)
f = -(1 - distLic.pow(2)).pow(-0.5)
I = torch.eye(featureH.shape[1], device=d.device)
xcH = (I - torch.einsum('bi,bj->bij',
(featureH_normed, featureH_normed))
) / featureH.norm(dim=1, keepdim=True).unsqueeze(-1)
xcL = (I - torch.einsum('bi,bj->bij',
(featureL_normed, featureL_normed))
) / featureL.norm(dim=1, keepdim=True).unsqueeze(-1)
cc = (I - torch.einsum('bi,bj->bij', (centers_normed, centers_normed))
) / centers.norm(dim=1, keepdim=True).unsqueeze(-1)
d = d.unsqueeze(1)
e = e.unsqueeze(1)
f = f.unsqueeze(1)
counts_h = centers.new_ones(num_class) # (C,)
counts_hl = centers.new_ones(num_class) # (C,)
counts_c = centers.new_ones(num_class) # (C,)
ones_h = centers.new_ones(num_pair) # (N',)
ones_h[li2 <= 0] = 0
ones_c = centers.new_ones(num_pair) # (N',)
ones_c[li1 <= 0] = 0
grad_centers = centers.new_zeros(centers.size()) # Cx1024
counts_h.scatter_add_(0, label.long(), ones_h)
counts_hl.scatter_add_(0, hard_index_batch, ones_h)
counts_c.scatter_add_(0, label.long(), ones_c)
grad_centers_h = featureH_normed * d
grad_centers_h[li2 <= 0] = 0
grad_centers += torch.scatter_add(
centers.new_zeros(centers.size()), 0,
label.unsqueeze(1).expand(featureH_normed.size()).long(),
grad_centers_h) / counts_h.unsqueeze(-1)
grad_centers_hl = -featureH_normed * e
grad_centers_hl[li2 <= 0] = 0
grad_centers += torch.scatter_add(
centers.new_zeros(centers.size()), 0,
hard_index_batch.unsqueeze(1).expand(featureH_normed.size()),
grad_centers_hl) / counts_hl.unsqueeze(-1)
grad_centers_c = featureH_normed * d - featureL_normed * f
grad_centers_c[li1 <= 0] = 0
grad_centers += torch.scatter_add(
centers.new_zeros(centers.size()), 0,
label.unsqueeze(1).expand(featureH_normed.size()).long(),
grad_centers_c * gamma) / counts_c.unsqueeze(-1)
grad_centers /= num_pair
grad = centers_normed_batch * d - hard_normed_batch * e
grad[li2 <= 0] = 0
grad_h = centers_normed_batch * d
grad_h[li1 <= 0] = 0
grad_h = grad_output * (grad + grad_h * gamma) / num_pair
grad_l = -centers_normed_batch * f
grad_l[li1 <= 0] = 0
grad_l = grad_output * grad_l * gamma / num_pair
return torch.bmm(xcH, grad_h.unsqueeze(-1)).squeeze(-1), torch.bmm(
xcL, grad_l.unsqueeze(-1)).squeeze(-1), None, torch.bmm(
cc, grad_centers.unsqueeze(-1)).squeeze(-1), None, None
def dist(self, x, y, *, keepdim=False):
inner = self.inner(None, x, y, keepdim=keepdim).clamp(-1, 1)
return torch.acos(inner)
pointErrs = []
normalErrs = []
meanAngleErrs = []
medianAngleErrs = []
if opt.lossMode == 0 or opt.lossMode == 1:
for m in range(0, len(pointPreds)):
pointErrs.append(torch.mean(torch.pow(pointPreds[m] - gtPoint, 2)))
for m in range(0, len(normalPreds)):
normalErrs.append(
torch.mean(torch.pow(normalPreds[m] - gtNormal, 2)))
meanAngleErrs.append(180.0 / np.pi * torch.mean(
torch.acos(
torch.clamp(torch.sum(normalPreds[m] * gtNormal, dim=1),
-1, 1))))
medianAngleErrs.append(180.0 / np.pi * torch.median(
torch.acos(
torch.clamp(torch.sum(normalPreds[m] * gtNormal, dim=1),
-1, 1))))
elif opt.lossMode == 2:
assert (len(pointPreds) == len(normalPreds))
for m in range(0, len(pointPreds)):
dist1, id1, dist2, id2 = chamferDist(gtPoint.unsqueeze(0),
pointPreds[m].unsqueeze(0))
pointErrs.append(0.5 * (torch.mean(dist1) + torch.mean(dist2)))
id1, id2 = id1.long().squeeze(0).detach(), id2.long().squeeze(
0).detach()
gtToPredNormal = torch.index_select(normalPreds[m],
def theta(v, eps=1e-5):
v_cos = cos_sim(v).clamp(-1. + eps, 1. - eps)
x = torch.acos(v_cos) + math.radians(10)
return x
def get_hard_negatives(self, queries, samples):
"""
Generate hard negatives candidates
"""
samples = int(min(samples, self.max_hard_negatives))
negatives_qids = random.sample(self.dataset.qid2question.keys(),
samples)
negatives = self.dataset.prepare_batch(negatives_qids)
query_vactorized = self.model.text_embedding(
x=torch.LongTensor(queries).to(self.device))
negatives_vectorized = self.model.text_embedding(
x=torch.LongTensor(negatives).to(self.device),
model_type='candidate')
similarity = torch.matmul(query_vactorized,
negatives_vectorized.transpose(0, 1))
query_norms = torch.matmul(query_vactorized,
query_vactorized.transpose(
0, 1)).pow(0.5).diag().unsqueeze(1)
negatives_norms = torch.matmul(negatives_vectorized,
negatives_vectorized.transpose(
0, 1)).pow(0.5).diag().unsqueeze(1)
norms = torch.matmul(query_norms, negatives_norms.transpose(0, 1))
similarity = similarity / norms
similarity = F.relu(similarity - self.model.eps)
if self.sim_func_from_use:
similarity = 1 - (torch.acos(similarity) / math.pi)
if self.threshold_similarity is not None:
similarity = similarity - (
similarity > self.threshold_similarity).type(
torch.FloatTensor).to(self.model.device)
similarity = F.relu(similarity)
if self.hard_k_next:
# if we wont get top-2
# if we take too many samples we can choose select sample existing in quieries
max_similar_matrix = torch.zeros_like(similarity)
values, args = similarity.max(dim=1)
for n, i in enumerate(range(max_similar_matrix.size(0))):
max_similar_matrix[i, args[n]] = values[n]
similarity -= max_similar_matrix
max_similar_indexes = list(similarity.argmax(dim=1).cpu().numpy())
max_similar_qids = [negatives_qids[n] for n in max_similar_indexes]
return self.dataset.prepare_batch(max_similar_qids)
def bboxes_iou_test(bboxes_a, bboxes_b, fmt='voc', iou_type='iou'):
"""
test function for the bboxes_iou function in `train_acne.py`,
with message printing and plot
"""
if 'plt' not in dir():
import matplotlib.pyplot as plt
if 'cv2' not in dir():
try:
import cv2
except ModuleNotFoundError:
cv2 = None
from PIL import Image, ImageDraw
assert iou_type.lower() in ['iou', 'giou', 'diou', 'ciou']
if isinstance(bboxes_a, np.ndarray):
bboxes_a = torch.Tensor(bboxes_a)
if isinstance(bboxes_b, np.ndarray):
bboxes_b = torch.Tensor(bboxes_b)
if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4:
raise IndexError
N, K = bboxes_a.shape[0], bboxes_b.shape[0]
# if N, K all equal 1, then plot
# top left
if fmt.lower() == 'voc': # xmin, ymin, xmax, ymax
# top left
tl_intersect = torch.max(bboxes_a[:, np.newaxis, :2], bboxes_b[:, :2]) # of shape `(N,K,2)`
# bottom right
br_intersect = torch.min(bboxes_a[:, np.newaxis, 2:], bboxes_b[:, 2:])
bb_a = bboxes_a[:, 2:] - bboxes_a[:, :2] # w, h
bb_b = bboxes_b[:, 2:] - bboxes_b[:, :2] # w, h
elif fmt.lower() == 'yolo': # xcen, ycen, w, h
tl_intersect = torch.max((bboxes_a[:, np.newaxis, :2] - bboxes_a[:, np.newaxis, 2:] / 2),
(bboxes_b[:, :2] - bboxes_b[:, 2:] / 2))
# bottom right
br_intersect = torch.min((bboxes_a[:, np.newaxis, :2] + bboxes_a[:, np.newaxis, 2:] / 2),
(bboxes_b[:, :2] + bboxes_b[:, 2:] / 2))
bb_a = bboxes_a[:, 2:]
bb_b = bboxes_b[:, 2:]
elif fmt.lower() == 'coco': # xmin, ymin, w, h
# top left
tl_intersect = torch.max(bboxes_a[:, np.newaxis, :2], bboxes_b[:, :2])
# bottom right
br_intersect = torch.min((bboxes_a[:, np.newaxis, :2] + bboxes_a[:, np.newaxis, 2:]),
(bboxes_b[:, :2] + bboxes_b[:, 2:]))
bb_a = bboxes_a[:, 2:]
bb_b = bboxes_b[:, 2:]
area_a = torch.prod(bb_a, 1)
area_b = torch.prod(bb_b, 1)
# torch.prod(input, dim, keepdim=False, dtype=None) → Tensor
# Returns the product of each row of the input tensor in the given dimension dim
# if tl, br does not form a nondegenerate squre, then the corr. element in the `prod` would be 0
en = (tl_intersect < br_intersect).type(tl_intersect.type()).prod(dim=2) # shape `(N,K,2)` ---> shape `(N,K)`
area_intersect = torch.prod(br_intersect - tl_intersect, 2) * en # * ((tl < br).all())
area_union = (area_a[:, np.newaxis] + area_b - area_intersect)
iou = _true_divide(area_intersect, area_union)
# if iou_type.lower() == 'iou':
# return iou
if fmt.lower() == 'voc': # xmin, ymin, xmax, ymax
# top left
tl_union = torch.min(bboxes_a[:, np.newaxis, :2], bboxes_b[:, :2]) # of shape `(N,K,2)`
# bottom right
br_union = torch.max(bboxes_a[:, np.newaxis, 2:], bboxes_b[:, 2:])
elif fmt.lower() == 'yolo': # xcen, ycen, w, h
tl_union = torch.min((bboxes_a[:, np.newaxis, :2] - bboxes_a[:, np.newaxis, 2:] / 2),
(bboxes_b[:, :2] - bboxes_b[:, 2:] / 2))
# bottom right
br_union = torch.max((bboxes_a[:, np.newaxis, :2] + bboxes_a[:, np.newaxis, 2:] / 2),
(bboxes_b[:, :2] + bboxes_b[:, 2:] / 2))
elif fmt.lower() == 'coco': # xmin, ymin, w, h
# top left
tl_union = torch.min(bboxes_a[:, np.newaxis, :2], bboxes_b[:, :2])
# bottom right
br_union = torch.max((bboxes_a[:, np.newaxis, :2] + bboxes_a[:, np.newaxis, 2:]),
(bboxes_b[:, :2] + bboxes_b[:, 2:]))
# c for covering, of shape `(N,K,2)`
# the last dim is box width, box hight
bboxes_c = br_union - tl_union
area_covering = torch.prod(bboxes_c, 2) # shape `(N,K)`
giou = iou - (area_covering - area_union) / area_covering
print(f"tl_union.shape = {tl_union.shape}")
print(f"br_union.shape = {br_union.shape}")
print(f"bboxes_c.shape = {bboxes_c.shape}")
# if iou_type.lower() == 'giou':
# return giou
if fmt.lower() == 'voc': # xmin, ymin, xmax, ymax
centre_a = (bboxes_a[..., 2 :] + bboxes_a[..., : 2]) / 2
centre_b = (bboxes_b[..., 2 :] + bboxes_b[..., : 2]) / 2
elif fmt.lower() == 'yolo': # xcen, ycen, w, h
centre_a = (bboxes_a[..., : 2] + bboxes_a[..., 2 :]) / 2
centre_b = (bboxes_b[..., : 2] + bboxes_b[..., 2 :]) / 2
elif fmt.lower() == 'coco': # xmin, ymin, w, h
centre_a = bboxes_a[..., 2 :] + bboxes_a[..., : 2]/2
centre_b = bboxes_b[..., 2 :] + bboxes_b[..., : 2]/2
centre_dist = torch.norm(centre_a[:, np.newaxis] - centre_b, p='fro', dim=2)
diag_len = torch.norm(bboxes_c, p='fro', dim=2)
diou = iou - centre_dist.pow(2) / diag_len.pow(2)
# if iou_type.lower() == 'diou':
# return diou
""" the legacy custom cosine similarity:
# bb_a of shape `(N,2)`, bb_b of shape `(K,2)`
v = torch.einsum('nm,km->nk', bb_a, bb_b)
v = _true_divide(v, (torch.norm(bb_a, p='fro', dim=1)[:,np.newaxis] * torch.norm(bb_b, p='fro', dim=1)))
# avoid nan for torch.acos near \pm 1
# https://github.com/pytorch/pytorch/issues/8069
eps = 1e-7
v = torch.clamp(v, -1+eps, 1-eps)
"""
v = F.cosine_similarity(bb_a[:,np.newaxis,:], bb_b, dim=-1)
v = (_true_divide(2*torch.acos(v), np.pi)).pow(2)
alpha = (_true_divide(v, 1-iou+v))*((iou>=0.5).type(iou.type()))
ciou = diou - alpha * v
if N==K==1:
print("\n"+"*"*50)
print(f"bboxes_a = {bboxes_a}")
print(f"bboxes_b = {bboxes_b}")
print(f"area_a = {area_a}")
print(f"area_b = {area_b}")
print(f"area_intersect = {area_intersect}")
print(f"area_union = {area_union}")
print(f"tl_intersect = {tl_intersect}")
print(f"br_intersect = {br_intersect}")
print(f"tl_union = {tl_union}")
print(f"br_union = {br_union}")
print(f"area_covering (area of bboxes_c) = {area_covering}")
print(f"centre_dist = {centre_dist}")
print(f"diag_len = {diag_len}")
print("for computing ciou")
inner_product = torch.einsum('nm,km->nk', bb_a, bb_b)
product_of_lengths = torch.norm(bb_a, p='fro', dim=1)[:,np.newaxis] * torch.norm(bb_b, p='fro', dim=1)
print(f"inner product of bb_a and bb_b is {inner_product}")
print(f"product of lengths of bb_a and bb_b is {product_of_lengths}")
print(f"inner product divided by product of lengths equals {_true_divide(inner_product, product_of_lengths)}")
print(f"normalized angle distance = {v}")
print(f"alpha = {alpha}")
print(f"v = {v}")
print(f"alpha = {alpha}")
bc = ED({"xmin":tl_union.numpy().astype(int)[0][0][0], "ymin":tl_union.numpy().astype(int)[0][0][1], "xmax":br_union.numpy().astype(int)[0][0][0], "ymax":br_union.numpy().astype(int)[0][0][1]})
adjust_x = bc.xmin - int(0.25*(bc.xmax-bc.xmin))
adjust_y = bc.ymin - int(0.25*(bc.ymax-bc.ymin))
print(f"adjust_x = {adjust_x}")
print(f"adjust_y = {adjust_y}")
bc.xmin, bc.ymin, bc.xmax, bc.ymax = bc.xmin-adjust_x, bc.ymin-adjust_y, bc.xmax-adjust_x, bc.ymax-adjust_y
ba, bb = bboxes_a.numpy().astype(int)[0], bboxes_b.numpy().astype(int)[0]
if fmt.lower() == 'voc': # xmin, ymin, xmax, ymax
ba = ED({"xmin":ba[0]-adjust_x, "ymin":ba[1]-adjust_y, "xmax":ba[2]-adjust_x, "ymax":ba[3]-adjust_y})
bb = ED({"xmin":bb[0]-adjust_x, "ymin":bb[1]-adjust_y, "xmax":bb[2]-adjust_x, "ymax":bb[3]-adjust_y})
elif fmt.lower() == 'yolo': # xcen, ycen, w, h
ba = ED({"xmin":ba[0]-ba[2]//2-adjust_x, "ymin":ba[1]-ba[3]//2-adjust_y, "xmax":ba[0]+ba[2]//2-adjust_x, "ymax":ba[1]+ba[3]//2-adjust_y})
bb = ED({"xmin":bb[0]-bb[2]//2-adjust_x, "ymin":bb[1]-bb[3]//2-adjust_y, "xmax":bb[0]+bb[2]//2-adjust_x, "ymax":bb[1]+bb[3]//2-adjust_y})
elif fmt.lower() == 'coco': # xmin, ymin, w, h
ba = ED({"xmin":ba[0]-adjust_x, "ymin":ba[1]-adjust_y, "xmax":ba[0]+ba[2]-adjust_x, "ymax":ba[1]+ba[3]-adjust_y})
bb = ED({"xmin":bb[0]-adjust_x, "ymin":bb[1]-adjust_y, "xmax":bb[0]+bb[2]-adjust_x, "ymax":bb[1]+bb[3]-adjust_y})
print(f"ba = {ba}")
print(f"bb = {bb}")
print(f"bc = {bc}")
plane = np.full(shape=(int(1.5*(bc.ymax-bc.ymin)),int(1.5*(bc.xmax-bc.xmin)),3), fill_value=255, dtype=np.uint8)
img_with_boxes = plane.copy()
line_size = 1
if cv2:
cv2.rectangle(img_with_boxes, (ba.xmin, ba.ymin), (ba.xmax, ba.ymax), (0, 255, 0), line_size)
cv2.rectangle(img_with_boxes, (bb.xmin, bb.ymin), (bb.xmax, bb.ymax), (0, 0, 255), line_size)
cv2.rectangle(img_with_boxes, (max(0,bc.ymin-1), max(0,bc.ymin-1)), (bc.xmax, bc.ymax), (255, 0, 0), line_size)
else:
img_with_boxes = Image.fromarray(img_with_boxes)
drawer = ImageDraw.Draw(img_with_boxes)
# drawer.line([(ba.xmin, ba.ymin), (ba.xmin, ba.ymax), (ba.xmax, ba.ymax), (ba.xmax, ba.ymin), (ba.xmin, ba.ymin)], fill='green', width=line_size)
# drawer.line([(bb.xmin, bb.ymin), (bb.xmin, bb.ymax), (bb.xmax, bb.ymax), (bb.xmax, bb.ymin), (bb.xmin, bb.ymin)], fill='blue', width=line_size)
# drawer.line([((max(0,bc.xmin-1), max(0,bc.ymin-1)), ((max(0,bc.xmin-1), bc.ymax), (bc.xmax, bc.ymax), (bc.xmax, max(0,bc.ymin-1)), ((max(0,bc.xmin-1), max(0,bc.ymin-1))], fill='red', width=line_size)
drawer.rectangle([(ba.xmin, ba.ymin), (ba.xmax, ba.ymax)], outline='green', width=line_size)
drawer.rectangle([(bb.xmin, bb.ymin), (bb.xmax, bb.ymax)], outline='blue', width=line_size)
drawer.rectangle([(max(0,bc.xmin-1), max(0,bc.ymin-1)), (bc.xmax+1, bc.ymax+1)], outline='red', width=line_size)
img_with_boxes = np.array(img_with_boxes)
del drawer
plt.figure(figsize=(7,7))
plt.imshow(img_with_boxes)
plt.show()
print(f"iou = {iou}")
print(f"giou = {giou}")
print(f"diou = {diou}")
print(f"ciou = {ciou}")
if iou_type.lower() == 'ciou':
return ciou
elif iou_type.lower() == 'diou':
return diou
elif iou_type.lower() == 'giou':
return giou
elif iou_type.lower() == 'iou':
return iou