def forward(self, x,segments):
inp_shape = x.shape[2:]
x = self.conv1(x)
x = self.gn1(x)
x = self.relu(x)
x = self.layer1(x)
x=self.full1(x)
x1=self.full2(x)
#print(x1.shape)
x = self.layer2(x)
#print(x.shape)
#x = self.layer3(x)
#print(x.shape)
x = self.layer4(x)
#print(x.shape)
# H, W -> H/2, W/2
x = self.pyramid_pooling(x)
#x = self.cbr_final(x)
#x = self.dropout(x)
x = self.deconv0(x)
x = self.deconv1(x)
x = self.deconv2(x)
#128+128
x=self.regress1(x)
#print(x.shape)
#print(x1.shape)
x_f=torch.cat((x,x1),1)
# x=self.regress2(x_f)
# x=self.regress3(x)
# x=self.regress4(x)
# x=self.final(x)
# x=self.final2(x)
y=self.class1(x_f)
y=self.class2(y)
y=self.class3(y)
y=self.class4(y)
#y=self.class5(y)
loss_var,loss_dis,loss_reg = cluster_loss(y,segments)
loss_var=loss_var.reshape((y.shape[0],1))
loss_dis=loss_dis.reshape((y.shape[0],1))
loss_reg=loss_reg.reshape((y.shape[0],1))
return y,loss_var,loss_dis,loss_reg
def forward(self, x, segments, labels, flag, task):
#print(x.shape)
location_map=torch.cat([(torch.arange(x.shape[-1])/x.shape[-1]).unsqueeze(0).expand(x.shape[-2],x.shape[-1]).unsqueeze(0), \
(torch.arange(x.shape[-2])/x.shape[-2]).unsqueeze(0).transpose(1,0).expand(x.shape[-2],x.shape[-1]).unsqueeze(0)],0).unsqueeze(0).float().cuda()
#x=torch.cat([x,location_map],1)
zero = torch.zeros(1).cuda()
one = torch.ones(1).cuda()
x = self.conv1(x)
x = self.conv2(x)
x1 = self.layer1(x)
#half resolution
x2 = self.layer2(x1)
#print(x.shape)
x = self.layer3(x2)
#print(x.shape)
x = self.layer4(x)
#print(x.shape)
# H, W -> H/2, W/2
x = self.pyramid_pooling(x)
#x = self.cbr_final(x)
#x = self.dropout(x)
x = self.fuse0(x)
x = self.fuse1(x)
#print(x.shape)
x = self.deconv1(x)
#print(x.shape,x2.shape)
x = self.fuse2(torch.cat((x, x2), 1))
x = self.deconv2(x)
#print(x.shape)
x_share = self.fuse3(torch.cat((x, x1), 1))
# x=self.regress1(x_share)
# #print(x.shape)
# x=self.regress2(x)
# x=self.regress3(x)
# depth=self.regress4(x)
# accurate_depth=depth
# return depth,accurate_depth
#clustering feature
#accurate_depth=depth*reliable
if flag == 0:
x_fuse = torch.cat([x_share, location_map], 1)
y = self.class0(x_fuse)
y = self.class1(y)
y = self.class2(y)
y = self.class3(y)
y = self.class4(y)
with torch.no_grad():
#masks=fast_cluster(y).view(1,1,x_share.shape[-2],x_share.shape[-1])
masks = segments.view(1, 1, x_share.shape[-2],
x_share.shape[-1])
mask_volume = torch.zeros(1, 80, x_share.shape[-2],
x_share.shape[-1]).cuda()
for i in range(1, torch.max(masks).int() + 1):
mask_volume[:, i - 1, :, :] = torch.where(
masks == i, one,
zero).view_as(mask_volume[:, i - 1, :, :])
#x=self.regress1(x_share)
#x=self.regress1(torch.cat([x_share,masks/torch.max(masks)],1))
x = self.regress1(torch.cat([x_share, mask_volume], 1))
#print(x.shape)
x = self.regress2(x)
x = self.regress3(x)
depth = self.regress4(x)
initial_depth = depth + 0
with torch.no_grad():
#masks=fast_cluster(y).view_as(depth)
#masks=segments.view_as(depth)
#coarse depth
coarse_depth = depth + 0
coarse_feature = x_share + 0
mean_features = torch.zeros(1, x_share.shape[1],
torch.max(masks).long() +
1).cuda()
mean_depth = torch.zeros(torch.max(masks).long() + 1).cuda()
#print(torch.max(masks))
for i in range(1, torch.max(masks).int() + 1):
index_r = torch.where(masks == i, one, zero)
mean_d = torch.sum(index_r * depth) / torch.sum(index_r)
mean_depth[i] = mean_d
coarse_depth = torch.where(masks == i, mean_d,
coarse_depth)
mean_f = torch.sum((index_r * x_share).view(
x_share.shape[0], x_share.shape[1], -1),
dim=-1) / torch.sum(index_r)
#print(mean_f.shape,mean_features[...,i].shape)
mean_features[..., i] = mean_f
coarse_feature = torch.where(
masks == i,
mean_f.view(x_share.shape[0], x_share.shape[1], 1, 1),
coarse_feature)
# #refine outer
# outer_feature=torch.zeros(1,2*x_share.shape[1]+2,torch.max(masks).long()+1,torch.max(masks).long()+1).cuda()
# for i in range(torch.min(masks).int(),torch.max(masks).int()+1):
# for j in range(torch.min(masks).int(),torch.max(masks).int()+1):
# if i!=j:
# #print(outer_feature[...,i,j].shape,mean_depth[i].view(1,1).shape,mean_features[...,i].shape)
# outer_feature[...,i,j]=torch.cat([mean_depth[i].view(1,1),mean_features[...,i],mean_depth[j].view(1,1),mean_features[...,j]],dim=-1)
# outer=self.outrefine1(outer_feature)
# outer=self.outrefine2(outer)
# outer=self.outrefine3(outer)
# outer_variance=self.outrefine4(outer)
# outer_depth=torch.zeros(torch.max(masks).long()+1).cuda()
# # #mean_depth_map=coarse_depth+0
# #with torch.no_grad():
# for i in range(torch.min(masks).int(),torch.max(masks).int()+1):
# outer_depth[i]=(torch.sum(mean_depth*outer_variance[...,i,:])+mean_depth[i])/torch.sum(outer_variance[...,i,:]+1)
# #outer_depth[i]=(torch.sum(mean_depth*outer_variance[...,i,:])+mean_depth[i])
# coarse_depth=torch.where(masks==i,outer_depth[i],coarse_depth)+0
#refine inner
#coarse_depth=self.output(coarse_depth)
inner_feature = torch.cat([coarse_depth, x_share, coarse_feature],
1)
inner = self.inrefine1(inner_feature)
inner = self.inrefine2(inner)
inner = self.inrefine3(inner)
inner = self.inrefine4(inner)
inner_variance = self.inrefine5(inner)
reliable_feature = torch.cat([depth, x_share, coarse_feature], 1)
reliable = self.inrefine1(reliable_feature)
reliable = self.inrefine2(reliable)
reliable = self.inrefine3(reliable)
reliable = self.inrefine4(reliable)
reliable_variance = self.inrefine5(reliable)
# #inner_variance[:,0,...]=inner_variance[:,0,...]/torch.max(inner_variance[:,0,...])
# reliable_to_depth=(inner_variance[:,0,...]/torch.max(inner_variance[:,0,...])).unsqueeze(1)
# variance_on_cosrse=inner_variance[:,1,...].unsqueeze(1)
# #print(inner_variance.shape)
# accurate_depth=depth*reliable_to_depth+(coarse_depth*variance_on_cosrse)*(1-reliable_to_depth)
loss_var, loss_dis, loss_reg = cluster_loss(y,
segments.long(),
device_id=cuda_id)
loss_var = loss_var.reshape((y.shape[0], 1))
loss_dis = loss_dis.reshape((y.shape[0], 1))
loss_reg = loss_reg.reshape((y.shape[0], 1))
accurate_depth = self.output(inner_variance + coarse_depth)
depth = self.output(reliable_variance + depth)
accurate_depth = torch.where(masks > 0,
(depth + accurate_depth) / 2,
initial_depth)
#print(torch.mean(depth).item(),torch.mean(coarse_depth).item())
return initial_depth, accurate_depth, loss_var, loss_dis, loss_reg
else:
if task == 'train':
with torch.no_grad():
masks = fast_cluster(y).view_as(depth)
print(torch.max(masks))
loss_var, loss_dis, loss_reg = cluster_loss(y, segments.long())
loss_var = loss_var.reshape((y.shape[0], 1))
loss_dis = loss_dis.reshape((y.shape[0], 1))
loss_reg = loss_reg.reshape((y.shape[0], 1))
return depth, masks, loss_var, loss_dis, loss_reg
elif task == 'test':
loss_var, loss_dis, loss_reg = cluster_loss(y, segments.long())
loss_var = loss_var.reshape((y.shape[0], 1))
loss_dis = loss_dis.reshape((y.shape[0], 1))
loss_reg = loss_reg.reshape((y.shape[0], 1))
return depth, loss_var, loss_dis, loss_reg
elif task == 'eval':
x_fuse = torch.cat([x_share, location_map], 1)
masks = segments.view_as(depth)
#coarse depth
coarse_depth = depth + 0
coarse_feature = x_fuse + 0
mean_features = torch.zeros(1, x_fuse.shape[1],
torch.max(masks).long() +
1).cuda()
mean_depth = torch.zeros(torch.max(masks).long() + 1).cuda()
for i in range(
torch.min(masks).int(),
torch.max(masks).int() + 1):
index_r = torch.where(masks == i, one, zero)
mean_d = torch.sum(index_r * depth) / torch.sum(index_r)
mean_depth[i] = mean_d + 0
coarse_depth = torch.where(masks == i, mean_depth[i],
coarse_depth)
mean_f = torch.sum((index_r * x_fuse).view(
x_fuse.shape[0], x_fuse.shape[1], -1),
dim=-1) / torch.sum(index_r)
#print(mean_f.shape,mean_features[...,i].shape)
mean_features[..., i] = mean_f
coarse_feature = torch.where(
masks == i,
mean_f.view(x_fuse.shape[0], x_fuse.shape[1], 1, 1),
coarse_feature)
#refine outer
# outer_feature=torch.zeros(1,2*x_fuse.shape[1]+2,torch.max(masks).long()-torch.min(masks).long()+1,torch.max(masks).long()-torch.min(masks).long()+1).cuda()
# for i in range(torch.min(masks).int(),torch.max(masks).int()+1):
# for j in range(torch.min(masks).int(),torch.max(masks).int()+1):
# if i!=j:
# #print(outer_feature[...,i,j].shape,mean_depth[i].view(1,1).shape,mean_features[...,i].shape)
# outer_feature[...,i,j]=torch.cat([mean_depth[i].view(1,1),mean_features[...,i],mean_depth[j].view(1,1),mean_features[...,j]],dim=-1)
# outer=self.outrefine1(outer_feature)
# outer=self.outrefine2(outer)
# outer=self.outrefine3(outer)
# outer_variance=self.outrefine4(outer)
# outer_depth=torch.zeros(torch.max(masks).long()-torch.min(masks).long()+1).cuda()
# #mean_depth_map=coarse_depth+0
# # print(torch.min(masks))
# # print(torch.sum(torch.where(masks==0,torch.ones(1).cuda(),torch.zeros(1).cuda())))
# for i in range(torch.min(masks).int(),torch.max(masks).int()+1):
# outer_depth[i]=(torch.sum(mean_depth*outer_variance[...,i,:])+mean_depth[i])/(torch.sum(outer_variance[...,i,:])+1)
# #outer_depth[i]=(torch.sum(mean_depth*outer_variance[...,i,:])+mean_depth[i])
# coarse_depth=torch.where(masks==i,outer_depth[i],coarse_depth)+0
#print(torch.max(coarse_depth),torch.mean(mean_depth),torch.mean(outer_depth),torch.max(outer_variance))
#mean_depth_map=coarse_depth+0
#refine inner
inner_feature = torch.cat(
[coarse_depth, x_fuse - coarse_feature], 1)
#print('inner_feature',torch.max(inner_feature).item())
inner = self.inrefine1(inner_feature)
#print('inner_1',torch.max(inner).item())
inner = self.inrefine2(inner)
#print('inner_2',torch.max(inner).item())
inner = self.inrefine3(inner)
#print('inner_3',torch.max(inner).item())
inner = self.inrefine4(inner)
inner_variance = self.inrefine5(inner)
accurate_depth = inner_variance
# inner_feature= torch.cat([depth,x_share],1)
# relialbe=self.reliable1(inner_feature)
# relialbe=self.reliable2(relialbe)
# relialbe=self.reliable3(relialbe)
# relialbe=self.reliable4(relialbe)
# relialbe=self.reliable5(relialbe)
# accurate_depth=relialbe
# print('inner_variance',torch.max(inner_variance).item())
# inner_variance[:,0,...]=inner_variance[:,0,...]/torch.max(inner_variance[:,0,...])
# reliable_to_depth=(torch.exp(-relialbe[:,0,...])).unsqueeze(1)
# reliable_to_coarse=(torch.exp(-inner_variance[:,0,...])).unsqueeze(1)
# variance_on_depth=relialbe[:,1,...].unsqueeze(1)
# variance_on_cosrse=inner_variance[:,1,...].unsqueeze(1)
# print('reliable_depth: %.2f reliable_coarse: %.2f variance_depth %.2f variance_coarse %.2f'%(torch.mean(reliable_to_depth).item(), \
# torch.mean(reliable_to_coarse).item(),torch.mean(variance_on_depth).item(),torch.mean(variance_on_cosrse).item()))
# #print('variance %.2f'%(torch.mean(inner_variance).item()))
# relialbe_weights=reliable_to_coarse+reliable_to_depth
# # #print(inner_variance.shape)
# accurate_depth=(depth*variance_on_depth*reliable_to_coarse+coarse_depth*variance_on_cosrse*reliable_to_coarse)/ \
# (torch.where(relialbe_weights==0,torch.ones(1).cuda(),relialbe_weights))
# refined_depth=depth*variance_on_depth
# coarse_depth=coarse_depth*variance_on_cosrse
# accurate_depth=(coarse_depth*reliable_to_coarse+refined_depth*(1-reliable_to_coarse))
# accurate_depth=refined_depth*reliable_to_depth
# print('depth',torch.max(depth).item())
# print('coarse',torch.max(coarse_depth).item())
# print('accurate',torch.max(accurate_depth).item())
# loss_var,loss_dis,loss_reg = cluster_loss(y,segments.long())
# loss_var=loss_var.reshape((y.shape[0],1))
# loss_dis=loss_dis.reshape((y.shape[0],1))
# loss_reg=loss_reg.reshape((y.shape[0],1))
# accurate_depth=inner_variance
# simple refinement
# x_fuse=x_share+depth.expand_as(x_share)
# inner=self.inrefine1(x_fuse)
# inner=self.inrefine2(inner)
# inner=self.inrefine3(inner)
# inner=self.inrefine4(inner)
# accurate_depth=self.inrefine5(inner)
accurate_depth = depth
return depth, accurate_depth
