def get_dataset(conv_type, task):
num_points = 1024
features = 2
batch_size = 2
if task == "object_detection":
include_box = True
else:
include_box = False
if conv_type.lower() == "dense":
num_points = 2048
batch_size = 1
if task == "registration":
if conv_type.lower() == "dense":
return PairMockDataset(features, num_points=num_points, batch_size=batch_size)
if conv_type.lower() == "sparse":
tr = Compose([XYZFeature(True, True, True), GridSampling3D(size=0.01, quantize_coords=True, mode="last")])
return PairMockDatasetGeometric(features, transform=tr, num_points=num_points, batch_size=batch_size)
return PairMockDatasetGeometric(features, batch_size=batch_size)
else:
if conv_type.lower() == "dense":
num_points = 2048
return MockDataset(features, num_points=num_points, include_box=include_box, batch_size=batch_size)
if conv_type.lower() == "sparse":
return MockDatasetGeometric(
features,
include_box=include_box,
transform=GridSampling3D(size=0.01, quantize_coords=True, mode="last"),
num_points=num_points,
batch_size=batch_size,
)
return MockDatasetGeometric(features, batch_size=batch_size)
def test_minkowski(self):
from torch_points3d.applications.minkowski import Minkowski
input_nc = 3
num_layers = 4
in_feat = 16
model = Minkowski(
architecture="encoder",
input_nc=input_nc,
in_feat=in_feat,
num_layers=num_layers,
config=None,
)
dataset = MockDatasetGeometric(input_nc,
transform=GridSampling3D(
0.01, quantize_coords=True),
num_points=128)
self.assertEqual(len(model._modules["down_modules"]), num_layers)
self.assertEqual(len(model._modules["inner_modules"]), 1)
self.assertFalse(model.has_mlp_head)
self.assertEqual(model.output_nc, 8 * in_feat)
try:
data_out = model.forward(dataset[0])
# self.assertEqual(data_out.x.shape[1], 8 * in_feat)
except Exception as e:
print("Model failing:")
print(model)
raise e
input_nc = 3
num_layers = 4
grid_sampling = 0.02
in_feat = 32
output_nc = 5
model = Minkowski(
architecture="encoder",
input_nc=input_nc,
output_nc=output_nc,
in_feat=in_feat,
in_grid_size=grid_sampling,
num_layers=num_layers,
config=None,
)
dataset = MockDatasetGeometric(input_nc,
transform=GridSampling3D(
0.01, quantize_coords=True),
num_points=128)
self.assertEqual(len(model._modules["down_modules"]), num_layers)
self.assertEqual(len(model._modules["inner_modules"]), 1)
self.assertTrue(model.has_mlp_head)
self.assertEqual(model.output_nc, output_nc)
try:
data_out = model.forward(dataset[0])
self.assertEqual(data_out.x.shape[1], output_nc)
except Exception as e:
print("Model failing:")
print(model)
raise e
def test_kpconv(self):
from torch_points3d.applications.kpconv import KPConv
input_nc = 3
num_layers = 4
grid_sampling = 0.02
in_feat = 32
model = KPConv(
architecture="unet",
input_nc=input_nc,
in_feat=in_feat,
in_grid_size=grid_sampling,
num_layers=num_layers,
config=None,
)
dataset = MockDatasetGeometric(input_nc + 1, transform=GridSampling3D(0.01), num_points=128)
self.assertEqual(len(model._modules["down_modules"]), num_layers + 1)
self.assertEqual(len(model._modules["inner_modules"]), 1)
self.assertEqual(len(model._modules["up_modules"]), 4)
self.assertFalse(model.has_mlp_head)
self.assertEqual(model.output_nc, in_feat)
try:
data_out = model.forward(dataset[0])
self.assertEqual(data_out.x.shape[1], in_feat)
except Exception as e:
print("Model failing:")
print(model)
raise e
input_nc = 3
num_layers = 4
grid_sampling = 0.02
in_feat = 32
output_nc = 5
model = KPConv(
architecture="unet",
input_nc=input_nc,
output_nc=output_nc,
in_feat=in_feat,
in_grid_size=grid_sampling,
num_layers=num_layers,
config=None,
)
dataset = MockDatasetGeometric(input_nc + 1, transform=GridSampling3D(0.01), num_points=128)
self.assertEqual(len(model._modules["down_modules"]), num_layers + 1)
self.assertEqual(len(model._modules["inner_modules"]), 1)
self.assertEqual(len(model._modules["up_modules"]), 4)
self.assertTrue(model.has_mlp_head)
self.assertEqual(model.output_nc, output_nc)
try:
data_out = model.forward(dataset[0])
self.assertEqual(data_out.x.shape[1], output_nc)
except Exception as e:
print("Model failing:")
print(model)
raise e
def test_multiscaleTransforms(self):
samplers = [GridSampling3D(0.25), None, GridSampling3D(0.5)]
search = [
RadiusNeighbourFinder(0.5, 100,
ConvolutionFormat.PARTIAL_DENSE.value),
RadiusNeighbourFinder(0.5, 150,
ConvolutionFormat.PARTIAL_DENSE.value),
RadiusNeighbourFinder(1, 200,
ConvolutionFormat.PARTIAL_DENSE.value),
]
upsampler = [KNNInterpolate(1), KNNInterpolate(1)]
N = 10
x = np.linspace(0, 1, N)
y = np.linspace(0, 1, N)
xv, yv = np.meshgrid(x, y)
pos = torch.tensor([xv.flatten(), yv.flatten(), np.zeros(N * N)]).T
x = torch.ones_like(pos)
d = Data(pos=pos, x=x).contiguous()
ms_transform = MultiScaleTransform({
"sampler": samplers,
"neighbour_finder": search,
"upsample_op": upsampler
})
transformed = ms_transform(d.clone())
npt.assert_almost_equal(transformed.x.numpy(), x.numpy())
npt.assert_almost_equal(transformed.pos.numpy(), pos.numpy())
ms = transformed.multiscale
npt.assert_almost_equal(ms[0].pos.numpy(), ms[1].pos.numpy())
npt.assert_almost_equal(ms[0].pos.numpy(),
samplers[0](d.clone()).pos.numpy())
npt.assert_almost_equal(ms[2].pos.numpy(),
samplers[2](ms[0].clone()).pos.numpy())
self.assertEqual(ms[0].__inc__("idx_neighboors", 0), pos.shape[0])
idx = search[0](
d.pos,
ms[0].pos,
torch.zeros((d.pos.shape[0]), dtype=torch.long),
torch.zeros((ms[0].pos.shape[0]), dtype=torch.long),
)
for i in range(len(ms[0].idx_neighboors)):
self.assertEqual(set(ms[0].idx_neighboors[i].tolist()),
set(idx[i].tolist()))
self.assertEqual(ms[1].idx_neighboors.shape[1], 150)
self.assertEqual(ms[2].idx_neighboors.shape[1], 200)
upsample = transformed.upsample
self.assertEqual(upsample[0].num_nodes, ms[1].num_nodes)
self.assertEqual(upsample[1].num_nodes, pos.shape[0])
self.assertEqual(upsample[1].x_idx.max(), ms[0].num_nodes - 1)
self.assertEqual(upsample[1].y_idx.max(), pos.shape[0] - 1)
self.assertEqual(upsample[1].__inc__("x_idx", 0), ms[0].num_nodes)
self.assertEqual(upsample[1].__inc__("y_idx", 0), pos.shape[0])
def test_sparseconv3d(self):
from torch_points3d.applications.sparseconv3d import SparseConv3d
input_nc = 3
num_layers = 4
in_feat = 32
out_feat = in_feat * 3
model = SparseConv3d(
architecture="unet", input_nc=input_nc, in_feat=in_feat, num_layers=num_layers, config=None,
)
dataset = MockDatasetGeometric(input_nc, transform=GridSampling3D(0.01, quantize_coords=True), num_points=128)
self.assertEqual(len(model._modules["down_modules"]), num_layers + 1)
self.assertEqual(len(model._modules["inner_modules"]), 1)
self.assertEqual(len(model._modules["up_modules"]), 4 + 1)
self.assertFalse(model.has_mlp_head)
self.assertEqual(model.output_nc, out_feat)
try:
data_out = model.forward(dataset[0])
self.assertEqual(data_out.x.shape[1], out_feat)
except Exception as e:
print("Model failing:")
print(model)
print(e)
input_nc = 3
num_layers = 4
output_nc = 5
model = SparseConv3d(
architecture="unet", input_nc=input_nc, output_nc=output_nc, num_layers=num_layers, config=None,
)
dataset = MockDatasetGeometric(input_nc, transform=GridSampling3D(0.01, quantize_coords=True), num_points=128)
self.assertEqual(len(model._modules["down_modules"]), num_layers + 1)
self.assertEqual(len(model._modules["inner_modules"]), 1)
self.assertEqual(len(model._modules["up_modules"]), 4 + 1)
self.assertTrue(model.has_mlp_head)
self.assertEqual(model.output_nc, output_nc)
try:
data_out = model.forward(dataset[0])
self.assertEqual(data_out.x.shape[1], output_nc)
except Exception as e:
print("Model failing:")
print(model)
raise e
def test_votenet_backbones(self):
from torch_points3d.applications.votenet import VoteNet
cfg = OmegaConf.load(
os.path.join(DIR_PATH,
"data/scannet-fixed/config_object_detection.yaml"))
config_data = cfg.data
config_data.is_test = True
dataset = ScannetDataset(config_data)
model = VoteNet(
original=False,
backbone="kpconv",
input_nc=dataset.feature_dimension,
num_classes=dataset.num_classes,
mean_size_arr=dataset.mean_size_arr,
compute_loss=True,
in_feat=4,
)
dataset.create_dataloaders(model,
batch_size=2,
shuffle=True,
num_workers=0,
precompute_multi_scale=False)
train_loader = dataset.train_dataloader
data = next(iter(train_loader))
data = GridSampling3D(0.1)(data)
# for key in data.keys:
# print(key, data[key].shape, data[key].dtype)
model.verify_data(data)
model.forward(data)
self.assertEqual(hasattr(model, "loss"), True)
attrs_test = {
"center": [2, 256, 3],
"heading_residuals": [2, 256, 1],
"heading_residuals_normalized": [2, 256, 1],
"heading_scores": [2, 256, 1],
"object_assignment": [2, 256],
"objectness_label": [2, 256],
"objectness_mask": [2, 256],
"objectness_scores": [2, 256, 2],
"sampled_votes": [2, 256, 3],
"seed_inds": [2048],
"seed_pos": [2, 1024, 3],
"seed_votes": [2, 1024, 3],
"sem_cls_scores": [2, 256, 20],
"size_residuals_normalized": [2, 256, 18, 3],
"size_scores": [2, 256, 18],
}
output = model.output
for k, v in attrs_test.items():
self.assertEqual(hasattr(output, k), True)
self.assertEqual(getattr(output, k).shape, torch.Size(v))
def test_registration_from_pretrained(self):
model = PretainedRegistry.from_pretrained(
"minkowski-registration-3dmatch", download=True)
input_nc = 1
dataset = MockDatasetGeometric(input_nc,
transform=GridSampling3D(
0.01, quantize_coords=True),
num_points=128)
model.set_input(dataset[0], device="cpu")
model.forward(dataset[0])
def __init__(self, option, model_type, dataset, modules):
super(PointGroup, self).__init__(option)
backbone_options = option.get("backbone", {"architecture": "unet"})
self.Backbone = Minkowski(
backbone_options.get("architecture", "unet"),
input_nc=dataset.feature_dimension,
num_layers=4,
config=backbone_options.get("config", {}),
)
self._scorer_type = option.get("scorer_type", "encoder")
cluster_voxel_size = option.get("cluster_voxel_size", 0.05)
if cluster_voxel_size:
self._voxelizer = GridSampling3D(cluster_voxel_size,
quantize_coords=True,
mode="mean")
else:
self._voxelizer = None
self.ScorerUnet = Minkowski("unet",
input_nc=self.Backbone.output_nc,
num_layers=4,
config=option.scorer_unet)
self.ScorerEncoder = Minkowski("encoder",
input_nc=self.Backbone.output_nc,
num_layers=4,
config=option.scorer_encoder)
self.ScorerMLP = MLP([
self.Backbone.output_nc, self.Backbone.output_nc,
self.ScorerUnet.output_nc
])
self.ScorerHead = Seq().append(
torch.nn.Linear(self.ScorerUnet.output_nc,
1)).append(torch.nn.Sigmoid())
self.Offset = Seq().append(
MLP([self.Backbone.output_nc, self.Backbone.output_nc],
bias=False))
self.Offset.append(torch.nn.Linear(self.Backbone.output_nc, 3))
self.Semantic = (Seq().append(
MLP([self.Backbone.output_nc, self.Backbone.output_nc],
bias=False)).append(
torch.nn.Linear(self.Backbone.output_nc,
dataset.num_classes)).append(
torch.nn.LogSoftmax(dim=-1)))
self.loss_names = [
"loss", "offset_norm_loss", "offset_dir_loss", "semantic_loss",
"score_loss"
]
stuff_classes = dataset.stuff_classes
if is_list(stuff_classes):
stuff_classes = torch.Tensor(stuff_classes).long()
self._stuff_classes = torch.cat(
[torch.tensor([IGNORE_LABEL]), stuff_classes])
def test_precompute(self):
pos = torch.tensor([[1, 0, 0], [0, 1, 0], [1, 1, 0], [0, 0, 0], [0.1, 0, 0]])
x = torch.tensor([0, 0, 0, 0, 1]).unsqueeze(-1)
support = Data(x=x, pos=pos)
query = GridSampling3D(1)(support.clone())
interpolate = KNNInterpolate(1)
up = interpolate.precompute(query, support)
self.assertEqual(up.num_nodes, 5)
self.assertEqual(up.x_idx[4], up.x_idx[3])
def get_dataset(conv_type, task):
features = 2
if task == "registration":
if conv_type.lower() == "dense":
return PairMockDataset(features, num_points=2048)
if conv_type.lower() == "sparse":
tr = Compose(
[XYZFeature(True, True, True), GridSampling3D(size=0.01, quantize_coords=True, mode="last")]
)
return PairMockDatasetGeometric(features, transform=tr, num_points=1024)
return PairMockDatasetGeometric(features)
else:
if conv_type.lower() == "dense":
return MockDataset(features, num_points=2048)
if conv_type.lower() == "sparse":
return MockDatasetGeometric(
features,
transform=GridSampling3D(size=0.01, quantize_coords=True, mode="last"),
num_points=1024,
)
return MockDatasetGeometric(features)
def test_siamese_minkowski(self):
params = load_model_config("registration", "minkowski", "MinkUNet_Fragment")
transform = Compose(
[XYZFeature(True, True, True), GridSampling3D(size=0.01, quantize_coords=True, mode="last")]
)
dataset = PairMockDatasetGeometric(5, transform=transform, num_points=1024, is_pair_ind=True)
model = instantiate_model(params, dataset)
d = dataset[0]
model.set_input(d, device)
model.forward()
model.backward()
ratio = test_hasgrad(model)
if ratio < 1:
print(
"Model registration.minkowski.MinkUNet_Fragment has %i%% of parameters with 0 gradient" % (100 * ratio)
)
def compute_subsampled_matches(data1,
data2,
voxel_size=0.1,
max_distance_overlap=0.02):
"""
compute matches on subsampled version of data and track ind
"""
grid_sampling = Compose(
[SaveOriginalPosId(),
GridSampling3D(voxel_size, mode='last')])
subsampled_data = grid_sampling(data1.clone())
origin_id = subsampled_data.origin_id.numpy()
pair = compute_overlap_and_matches(subsampled_data, data2,
max_distance_overlap)['pair']
pair[:, 0] = origin_id[pair[:, 0]]
return torch.from_numpy(pair.copy())
def __init__(
self,
down_conv_nn=None,
grid_size=None,
prev_grid_size=None,
sigma=1.0,
max_num_neighbors=16,
activation=torch.nn.LeakyReLU(negative_slope=0.1),
bn_momentum=0.02,
bn=FastBatchNorm1d,
deformable=False,
add_one=False,
**kwargs,
):
super(SimpleBlock, self).__init__()
assert len(down_conv_nn) == 2
num_inputs, num_outputs = down_conv_nn
if deformable:
density_parameter = self.DEFORMABLE_DENSITY
self.kp_conv = KPConvDeformableLayer(
num_inputs,
num_outputs,
point_influence=prev_grid_size * sigma,
add_one=add_one)
else:
density_parameter = self.RIGID_DENSITY
self.kp_conv = KPConvLayer(num_inputs,
num_outputs,
point_influence=prev_grid_size * sigma,
add_one=add_one)
search_radius = density_parameter * sigma * prev_grid_size
self.neighbour_finder = RadiusNeighbourFinder(search_radius,
max_num_neighbors,
conv_type=self.CONV_TYPE)
if bn:
self.bn = bn(num_outputs, momentum=bn_momentum)
else:
self.bn = None
self.activation = activation
is_strided = prev_grid_size != grid_size
if is_strided:
self.sampler = GridSampling3D(grid_size)
else:
self.sampler = None
[
-6.481465826011e-03, 8.051860151134e-03, -9.999466081774e-01,
-7.337429464231e-02
],
[
9.999773098287e-01, -1.805528627661e-03, -6.496203536139e-03,
-3.339968064433e-01
],
])
pcd_s = np.fromfile(path_s, dtype=np.float32).reshape(-1, 4)[:, :3].dot(
R_calib[:3, :3].T)
pcd_t = np.fromfile(path_t, dtype=np.float32).reshape(-1, 4)[:, :3].dot(
R_calib[:3, :3].T)
transform = Compose([
GridSampling3D(mode="last", size=0.3, quantize_coords=True),
AddOnes(),
AddFeatByKey(add_to_x=True, feat_name="ones"),
])
data_s = transform(
Batch(pos=torch.from_numpy(pcd_s).float(),
batch=torch.zeros(pcd_s.shape[0]).long()))
data_t = transform(
Batch(pos=torch.from_numpy(pcd_t).float(),
batch=torch.zeros(pcd_t.shape[0]).long()))
model = PretainedRegistry.from_pretrained(
"minkowski-registration-kitti").cuda()
with torch.no_grad():