# parse the camera intrinsic file
## WHY the parameters from intrinsic_color.txt are different from those in scene_id.txt file??
## The one in scene_id.txt should be after camera calibration (undistortion)
# ### Load from intrinsic_color.txt
# camera_intrinsic_path = os.path.join(scan_path, 'intrinsic', 'intrinsic_color.txt')
# camera_intrinsic = np.loadtxt(camera_intrinsic_path)[:3,:]
### Load from scene_id.txt
meta_file = os.path.join(scan_path, '{0}.txt'.format(scan_name))
camera_intrinsic = scannet_utils.read_camera_intrinsic(meta_file)
# parse the mesh file
mesh_file = os.path.join(scan_path,
'{0}_vh_clean_2.ply'.format(scan_name))
mesh_vertices = scannet_utils.read_mesh_vertices_rgb(mesh_file)
## visualize-1
# plotxyzrgb(mesh_vertices, block=True)
## visualize-2
fig = mlab.figure(figure=None,
bgcolor=(0, 0, 0),
fgcolor=None,
engine=None,
size=(1000, 1000))
mlab.points3d(mesh_vertices[:, 0],
mesh_vertices[:, 1],
mesh_vertices[:, 2],
mesh_vertices[:, 2],
mode='point',
def export(mesh_file,
agg_file,
seg_file,
meta_file,
label_map_file,
output_file=None):
"""Export original files to vert, ins_label, sem_label and bbox file.
Args:
mesh_file (str): Path of the mesh_file.
agg_file (str): Path of the agg_file.
seg_file (str): Path of the seg_file.
meta_file (str): Path of the meta_file.
label_map_file (str): Path of the label_map_file.
output_file (str): Path of the output folder.
Default: None.
It returns a tuple, which containts the the following things:
np.ndarray: Vertices of points data.
np.ndarray: Indexes of label.
np.ndarray: Indexes of instance.
np.ndarray: Instance bboxes.
dict: Map from object_id to label_id.
"""
label_map = scannet_utils.read_label_mapping(label_map_file,
label_from='raw_category',
label_to='nyu40id')
mesh_vertices = scannet_utils.read_mesh_vertices_rgb(mesh_file)
# Load scene axis alignment matrix
lines = open(meta_file).readlines()
for line in lines:
if 'axisAlignment' in line:
axis_align_matrix = [
float(x)
for x in line.rstrip().strip('axisAlignment = ').split(' ')
]
break
axis_align_matrix = np.array(axis_align_matrix).reshape((4, 4))
pts = np.ones((mesh_vertices.shape[0], 4))
pts[:, 0:3] = mesh_vertices[:, 0:3]
pts = np.dot(pts, axis_align_matrix.transpose()) # Nx4
mesh_vertices[:, 0:3] = pts[:, 0:3]
# Load semantic and instance labels
object_id_to_segs, label_to_segs = read_aggregation(agg_file)
seg_to_verts, num_verts = read_segmentation(seg_file)
label_ids = np.zeros(shape=(num_verts), dtype=np.uint32)
object_id_to_label_id = {}
for label, segs in label_to_segs.items():
label_id = label_map[label]
for seg in segs:
verts = seg_to_verts[seg]
label_ids[verts] = label_id
instance_ids = np.zeros(shape=(num_verts),
dtype=np.uint32) # 0: unannotated
num_instances = len(np.unique(list(object_id_to_segs.keys())))
for object_id, segs in object_id_to_segs.items():
for seg in segs:
verts = seg_to_verts[seg]
instance_ids[verts] = object_id
if object_id not in object_id_to_label_id:
object_id_to_label_id[object_id] = label_ids[verts][0]
instance_bboxes = np.zeros((num_instances, 7))
for obj_id in object_id_to_segs:
label_id = object_id_to_label_id[obj_id]
obj_pc = mesh_vertices[instance_ids == obj_id, 0:3]
if len(obj_pc) == 0:
continue
xmin = np.min(obj_pc[:, 0])
ymin = np.min(obj_pc[:, 1])
zmin = np.min(obj_pc[:, 2])
xmax = np.max(obj_pc[:, 0])
ymax = np.max(obj_pc[:, 1])
zmax = np.max(obj_pc[:, 2])
bbox = np.array([(xmin + xmax) / 2, (ymin + ymax) / 2,
(zmin + zmax) / 2, xmax - xmin, ymax - ymin,
zmax - zmin, label_id])
# NOTE: this assumes obj_id is in 1,2,3,.,,,.NUM_INSTANCES
instance_bboxes[obj_id - 1, :] = bbox
if output_file is not None:
np.save(output_file + '_vert.npy', mesh_vertices)
np.save(output_file + '_sem_label.npy', label_ids)
np.save(output_file + '_ins_label.npy', instance_ids)
np.save(output_file + '_bbox.npy', instance_bboxes)
return mesh_vertices, label_ids, instance_ids,\
instance_bboxes, object_id_to_label_id
def export(mesh_file,
agg_file,
seg_file,
meta_file,
label_map_file,
output_file=None,
test_mode=False):
"""Export original files to vert, ins_label, sem_label and bbox file.
Args:
mesh_file (str): Path of the mesh_file.
agg_file (str): Path of the agg_file.
seg_file (str): Path of the seg_file.
meta_file (str): Path of the meta_file.
label_map_file (str): Path of the label_map_file.
output_file (str): Path of the output folder.
Default: None.
test_mode (bool): Whether is generating test data without labels.
Default: False.
It returns a tuple, which containts the the following things:
np.ndarray: Vertices of points data.
np.ndarray: Indexes of label.
np.ndarray: Indexes of instance.
np.ndarray: Instance bboxes.
dict: Map from object_id to label_id.
"""
label_map = scannet_utils.read_label_mapping(label_map_file,
label_from='raw_category',
label_to='nyu40id')
mesh_vertices = scannet_utils.read_mesh_vertices_rgb(mesh_file)
# Load scene axis alignment matrix
lines = open(meta_file).readlines()
# test set data doesn't have align_matrix
axis_align_matrix = np.eye(4)
for line in lines:
if 'axisAlignment' in line:
axis_align_matrix = [
float(x)
for x in line.rstrip().strip('axisAlignment = ').split(' ')
]
break
axis_align_matrix = np.array(axis_align_matrix).reshape((4, 4))
# perform global alignment of mesh vertices
pts = np.ones((mesh_vertices.shape[0], 4))
pts[:, 0:3] = mesh_vertices[:, 0:3]
pts = np.dot(pts, axis_align_matrix.transpose()) # Nx4
aligned_mesh_vertices = np.concatenate([pts[:, 0:3], mesh_vertices[:, 3:]],
axis=1)
# Load semantic and instance labels
if not test_mode:
object_id_to_segs, label_to_segs = read_aggregation(agg_file)
seg_to_verts, num_verts = read_segmentation(seg_file)
label_ids = np.zeros(shape=(num_verts), dtype=np.uint32)
object_id_to_label_id = {}
for label, segs in label_to_segs.items():
label_id = label_map[label]
for seg in segs:
verts = seg_to_verts[seg]
label_ids[verts] = label_id
instance_ids = np.zeros(shape=(num_verts),
dtype=np.uint32) # 0: unannotated
for object_id, segs in object_id_to_segs.items():
for seg in segs:
verts = seg_to_verts[seg]
instance_ids[verts] = object_id
if object_id not in object_id_to_label_id:
object_id_to_label_id[object_id] = label_ids[verts][0]
unaligned_bboxes = extract_bbox(mesh_vertices, object_id_to_segs,
object_id_to_label_id, instance_ids)
aligned_bboxes = extract_bbox(aligned_mesh_vertices, object_id_to_segs,
object_id_to_label_id, instance_ids)
else:
label_ids = None
instance_ids = None
unaligned_bboxes = None
aligned_bboxes = None
object_id_to_label_id = None
if output_file is not None:
np.save(output_file + '_vert.npy', mesh_vertices)
if not test_mode:
np.save(output_file + '_sem_label.npy', label_ids)
np.save(output_file + '_ins_label.npy', instance_ids)
np.save(output_file + '_unaligned_bbox.npy', unaligned_bboxes)
np.save(output_file + '_aligned_bbox.npy', aligned_bboxes)
np.save(output_file + '_axis_align_matrix.npy', axis_align_matrix)
return mesh_vertices, label_ids, instance_ids, unaligned_bboxes, \
aligned_bboxes, object_id_to_label_id, axis_align_matrix
def export_one_scan(model, scan_name, output_filename_prefix):
mesh_file = os.path.join(SCANNET_DIR, scan_name,
scan_name + '_vh_clean_2.ply')
seg_file = os.path.join(SCANNET_DIR, scan_name,
scan_name + '_vh_clean_2.0.010000.segs.json')
if (model == 'votenet'):
meta_file = os.path.join(
SCANNET_DIR, scan_name, scan_name +
'.txt') # includes axisAlignment info for the train set scans.
agg_file = os.path.join(SCANNET_DIR, scan_name,
scan_name + '_vh_clean.aggregation.json')
mesh_vertices, aligned_vertices, semantic_labels, instance_labels, instance_bboxes, aligned_instance_bboxes = export(
mesh_file, agg_file, seg_file, meta_file, LABEL_MAP_FILE, None)
mask = np.logical_not(np.in1d(semantic_labels, DONOTCARE_CLASS_IDS))
mesh_vertices = mesh_vertices[mask, :]
aligned_vertices = aligned_vertices[mask, :]
semantic_labels = semantic_labels[mask]
instance_labels = instance_labels[mask]
if instance_bboxes.shape[0] > 1:
num_instances = len(np.unique(instance_labels))
print('Num of instances: ', num_instances)
bbox_mask = np.in1d(instance_bboxes[:, -2],
OBJ_CLASS_IDS) # match the mesh2cap
instance_bboxes = instance_bboxes[bbox_mask, :]
aligned_instance_bboxes = aligned_instance_bboxes[bbox_mask, :]
print('Num of care instances: ', instance_bboxes.shape[0])
else:
print("No semantic/instance annotation for test scenes")
N = mesh_vertices.shape[0]
if N > MAX_NUM_POINT:
choices = np.random.choice(N, MAX_NUM_POINT, replace=False)
mesh_vertices = mesh_vertices[choices, :]
aligned_vertices = aligned_vertices[choices, :]
semantic_labels = semantic_labels[choices]
instance_labels = instance_labels[choices]
print("Shape of points: {}".format(mesh_vertices.shape))
np.save(output_filename_prefix + '_vert.npy', mesh_vertices)
np.save(output_filename_prefix + '_aligned_vert.npy', aligned_vertices)
np.save(output_filename_prefix + '_sem_label.npy', semantic_labels)
np.save(output_filename_prefix + '_ins_label.npy', instance_labels)
np.save(output_filename_prefix + '_bbox.npy', instance_bboxes)
np.save(output_filename_prefix + '_aligned_bbox.npy',
aligned_instance_bboxes)
elif (model == 'pointgroup'):
TEST_SCAN_NAMES = sorted(
[line.rstrip() for line in open('meta_data/scannetv2_test.txt')])
if (scan_name in TEST_SCAN_NAMES):
vertices = scannet_utils.read_mesh_vertices_rgb(mesh_file)
coords = np.ascontiguousarray(vertices[:, :3] -
vertices[:, :3].mean(0))
colors = np.ascontiguousarray(vertices[:, 3:6]) / 127.5 - 1
torch.save((coords, colors),
output_filename_prefix + '_pointgroup.pth')
else:
labels_file = os.path.join(SCANNET_DIR, scan_name,
scan_name + '_vh_clean_2.labels.ply')
agg_file = os.path.join(SCANNET_DIR, scan_name,
scan_name + '.aggregation.json')
vertices = scannet_utils.read_mesh_vertices_rgb(mesh_file)
coords = np.ascontiguousarray(vertices[:, :3] -
vertices[:, :3].mean(0))
colors = np.ascontiguousarray(vertices[:, 3:6]) / 127.5 - 1
sem_labels = scannet_utils.get_labels(labels_file, OBJ_CLASS_IDS)
segid_to_pointid, _ = read_segmentation(seg_file)
instance_segids = scannet_utils.get_instance_segids(
scan_name, agg_file)
instance_labels = np.ones(sem_labels.shape[0]) * -100
for i in range(len(instance_segids)):
segids = instance_segids[i]
pointids = []
for segid in segids:
pointids += segid_to_pointid[segid]
instance_labels[pointids] = i
assert (len(np.unique(sem_labels[pointids])) == 1)
N = len(sem_labels)
if N > MAX_NUM_POINT:
choices = np.random.choice(N, MAX_NUM_POINT, replace=False)
coords = coords[choices, :]
colors = colors[choices, :]
sem_labels = sem_labels[choices]
instance_labels = instance_labels[choices]
torch.save((coords, colors, sem_labels, instance_labels),
output_filename_prefix + '_pointgroup.pth')
def export(mesh_file,
agg_file,
seg_file,
meta_file,
label_map_file,
output_file=None):
""" points are XYZ RGB (RGB in 0-255),
semantic label as nyu40 ids,
instance label as 1-#instance,
box as (cx,cy,cz,dx,dy,dz,semantic_label)
"""
label_map = scannet_utils.read_label_mapping(label_map_file,
label_from='raw_category',
label_to='nyu40id')
mesh_vertices = scannet_utils.read_mesh_vertices_rgb(mesh_file)
# Load scene axis alignment matrix
lines = open(meta_file).readlines()
for line in lines:
if 'axisAlignment' in line:
axis_align_matrix = [float(x) \
for x in line.rstrip().strip('axisAlignment = ').split(' ')]
break
axis_align_matrix = np.array(axis_align_matrix).reshape((4, 4))
pts = np.ones((mesh_vertices.shape[0], 4))
pts[:, 0:3] = mesh_vertices[:, 0:3]
pts = np.dot(pts, axis_align_matrix.transpose()) # Nx4
mesh_vertices[:, 0:3] = pts[:, 0:3]
# Load semantic and instance labels
object_id_to_segs, label_to_segs = read_aggregation(agg_file)
seg_to_verts, num_verts = read_segmentation(seg_file)
label_ids = np.zeros(shape=(num_verts), dtype=np.uint32) # 0: unannotated
object_id_to_label_id = {}
for label, segs in label_to_segs.items():
label_id = label_map[label]
for seg in segs:
verts = seg_to_verts[seg]
label_ids[verts] = label_id
instance_ids = np.zeros(shape=(num_verts),
dtype=np.uint32) # 0: unannotated
num_instances = len(np.unique(list(object_id_to_segs.keys())))
for object_id, segs in object_id_to_segs.items():
for seg in segs:
verts = seg_to_verts[seg]
instance_ids[verts] = object_id
if object_id not in object_id_to_label_id:
object_id_to_label_id[object_id] = label_ids[verts][0]
instance_bboxes = np.zeros((num_instances, 7))
for obj_id in object_id_to_segs:
label_id = object_id_to_label_id[obj_id]
obj_pc = mesh_vertices[instance_ids == obj_id, 0:3]
if len(obj_pc) == 0: continue
# Compute axis aligned box
# An axis aligned bounding box is parameterized by
# (cx,cy,cz) and (dx,dy,dz) and label id
# where (cx,cy,cz) is the center point of the box,
# dx is the x-axis length of the box.
xmin = np.min(obj_pc[:, 0])
ymin = np.min(obj_pc[:, 1])
zmin = np.min(obj_pc[:, 2])
xmax = np.max(obj_pc[:, 0])
ymax = np.max(obj_pc[:, 1])
zmax = np.max(obj_pc[:, 2])
bbox = np.array([(xmin + xmax) / 2, (ymin + ymax) / 2,
(zmin + zmax) / 2, xmax - xmin, ymax - ymin,
zmax - zmin, label_id])
# NOTE: this assumes obj_id is in 1,2,3,.,,,.NUM_INSTANCES
instance_bboxes[obj_id - 1, :] = bbox
if output_file is not None:
np.save(output_file + '_vert.npy', mesh_vertices)
np.save(output_file + '_sem_label.npy', label_ids)
np.save(output_file + '_ins_label.npy', instance_ids)
np.save(output_file + '_bbox.npy', instance_bboxes)
return mesh_vertices, label_ids, instance_ids,\
instance_bboxes, object_id_to_label_id
