def forward(self, x, points_mean, is_test=False):
x = self.conv0(x)
x = self.down0(x) # sp
x = self.conv1(x) # 2x sub
if not is_test:
vx_feat, vx_nxyz = tensor2points(x, voxel_size=(.1, .1, .2))
p1 = nearest_neighbor_interpolate(points_mean, vx_nxyz, vx_feat)
x = self.down1(x)
x = self.conv2(x)
if not is_test:
vx_feat, vx_nxyz = tensor2points(x, voxel_size=(.2, .2, .4))
p2 = nearest_neighbor_interpolate(points_mean, vx_nxyz, vx_feat)
x = self.down2(x)
x = self.conv3(x)
if not is_test:
vx_feat, vx_nxyz = tensor2points(x, voxel_size=(.4, .4, .8))
p3 = nearest_neighbor_interpolate(points_mean, vx_nxyz, vx_feat)
out = self.extra_conv(x)
if is_test:
return out, None
pointwise = self.point_fc(torch.cat([p1, p2, p3], dim=-1))
point_cls = self.point_cls(pointwise)
point_reg = self.point_reg(pointwise)
return out, (points_mean, point_cls, point_reg)
def forward(self,
voxel_features,
coors,
batch_size,
input_shape,
is_train=True):
# create point cloud
points_mean = torch.zeros_like(voxel_features)
points_mean[:, 0] = coors[:, 0]
points_mean[:, 1:] = voxel_features[:, :4]
# input: # [41, 1600, 1408]
sparse_shape = np.array(input_shape[::-1]) + [1, 0, 0]
coors = coors.int()
ret = spconv.SparseConvTensor(voxel_features, coors, sparse_shape,
batch_size)
ret1 = self.middle_conv1(ret)
ret2 = self.middle_conv2(ret1)
ret3 = self.middle_conv3(ret2)
ret4 = self.end_conv(ret3)
if is_train: #auxillary loss
vx_feat, vx_nxyz = tensor2points(ret1,
offset=(-50.4, -50.4, -5.0),
voxel_size=(.2, .2, .4))
p1 = self.nearest_neighbor_interpolate(
points_mean[:, :-1].contiguous(), vx_nxyz, vx_feat)
vx_feat, vx_nxyz = tensor2points(ret2,
offset=(-50.4, -50.4, -5.0),
voxel_size=(.4, .4, .8))
p2 = self.nearest_neighbor_interpolate(
points_mean[:, :-1].contiguous(), vx_nxyz, vx_feat)
vx_feat, vx_nxyz = tensor2points(ret3,
offset=(-50.4, -50.4, -5.0),
voxel_size=(.8, .8, 1.6))
p3 = self.nearest_neighbor_interpolate(
points_mean[:, :-1].contiguous(), vx_nxyz, vx_feat)
pointwise = self.point_fc(torch.cat([p1, p2, p3], dim=-1))
point_cls = self.point_cls(pointwise)
point_reg = self.point_reg(pointwise)
ret4 = ret4.dense()
N, C, D, H, W = ret4.shape
ret4 = ret4.view(N, C * D, H, W)
if is_train:
return ret4, (points_mean, point_cls, point_reg)
else:
return ret4, None
def forward(self, voxel_features, coors, batch_size, is_test=False):
# voxel_features [15470, 4] coors [15470, 4] 其中voxel_features中四个特征分别是3个坐标+1个反射；coors的四个特征分别是batch id+体素中点坐标
points_mean = torch.zeros_like(
voxel_features) # points_mean 记录了batch id + 3维平均坐标
points_mean[:, 0] = coors[:, 0] # 获得batch id
points_mean[:, 1:] = voxel_features[:, :3] # 获得每个体素的前三个特征：3个点云的平均坐标
coors = coors.int()
x = spconv.SparseConvTensor(voxel_features, coors, self.sparse_shape,
batch_size) # 初始化SparseConvTensor
x, middle = self.backbone(
x
) # x: [5, 200, 176] middle: [20, 800, 704] [10, 400, 352] [5, 200, 176]
x = x.dense() # [1, 64, 5, 200, 176] 第一个维度是batch size
N, C, D, H, W = x.shape
x = x.view(N, C * D, H, W) # [1, 320, 200, 176]
x, conv6 = self.fcn(
x) # x: [1, 256, 200, 176], conv6: [1, 256, 200, 176]
if is_test:
return x, conv6
else:
# auxiliary network
vx_feat, vx_nxyz = tensor2points(
middle[0], (0, -40., -3.),
voxel_size=(.1, .1,
.2)) # vx_feat [52914, 32] vx_nxyz [52913, 4]
p0 = nearest_neighbor_interpolate(points_mean, vx_nxyz,
vx_feat) # p0 [34114, 32]
vx_feat, vx_nxyz = tensor2points(middle[1], (0, -40., -3.),
voxel_size=(.2, .2, .4))
p1 = nearest_neighbor_interpolate(points_mean, vx_nxyz,
vx_feat) # p1 [34114, 64]
vx_feat, vx_nxyz = tensor2points(middle[2], (0, -40., -3.),
voxel_size=(.4, .4, .8))
p2 = nearest_neighbor_interpolate(points_mean, vx_nxyz,
vx_feat) # p2 [34114, 64]
# 均为全连接层
pointwise = self.point_fc(torch.cat(
[p0, p1, p2], dim=-1)) # pointwise [34114, 64]
point_cls = self.point_cls(pointwise) # point_cls [34114, 1]
point_reg = self.point_reg(pointwise) # point_cls [34114, 3]
return x, conv6, (points_mean, point_cls, point_reg)
def forward(self, voxel_features, coors, batch_size, is_test=False):
points_mean = torch.zeros_like(voxel_features)
points_mean[:, 0] = coors[:, 0]
points_mean[:, 1:] = voxel_features[:, :3]
coors = coors.int()
x = spconv.SparseConvTensor(voxel_features, coors, self.sparse_shape,
batch_size)
x, middle = self.backbone(x)
x = x.dense()
N, C, D, H, W = x.shape
x = x.view(N, C * D, H, W)
x, conv6 = self.fcn(x)
if is_test:
return x, conv6
else:
# auxiliary network
vx_feat, vx_nxyz = tensor2points(middle[0], (0, -40., -3.),
voxel_size=(.1, .1, .2))
p0 = nearest_neighbor_interpolate(points_mean, vx_nxyz, vx_feat)
vx_feat, vx_nxyz = tensor2points(middle[1], (0, -40., -3.),
voxel_size=(.2, .2, .4))
p1 = nearest_neighbor_interpolate(points_mean, vx_nxyz, vx_feat)
vx_feat, vx_nxyz = tensor2points(middle[2], (0, -40., -3.),
voxel_size=(.4, .4, .8))
p2 = nearest_neighbor_interpolate(points_mean, vx_nxyz, vx_feat)
pointwise = self.point_fc(torch.cat([p0, p1, p2], dim=-1))
point_cls = self.point_cls(pointwise)
point_reg = self.point_reg(pointwise)
return x, conv6, (points_mean, point_cls, point_reg)