def channelInfo(self, n):
triangle = self._triangle_at(n)
echo_i = clouds.triangle_to_echo_index(triangle)
sample = self._get_sample().masked
data = sample['data']
channel_i = data['indices'][echo_i]
h = sample['h']
try:
coeff = sample['timestamps_to_us_coeff']
except:
coeff = 1
rv = {
'v': int(channel_i // h),
'h': int(channel_i % h),
'i': int(channel_i),
'distance': float(data['distances'][echo_i]),
'amplitude': float(data['amplitudes'][echo_i]),
'timestamp': int(data['timestamps'][echo_i] * coeff),
'flag': int(data['flags'][echo_i]),
'category': ''
}
if self._seg3DSample._variant is not None:
seg_source = categories.get_source(
self._seg3DSample._variant.datasource.ds_type)
category_number = self._seg3DSample._variant.raw['data'][int(
echo_i)]['classes']
category_name, _ = categories.get_name_color(
seg_source, category_number)
rv['category'] = category_name
return QVariant(rv)
def label_names(self):
'''Converts the category numbers in their corresponding names (e.g. 0 -> 'pedestrian') and returns the list of names for all boxes in the sample'''
label_source_name = categories.get_source(platform_utils.parse_datasource_name(self.datasource.label)[2])
try:
return [categories.CATEGORIES[label_source_name][str(category_number)]['name'] for category_number in self.raw['data']['classes']]
except:
LoggingManager.instance().warning(f"Can not find the CATEGORIES and NAMES of {label_source_name}.")
def __update_seg_2d(self, sample, image):
datasources = [
ds_name for ds_name, show in self.show_seg_2d.items() if show
]
for ds_name in datasources:
seg_sample = self.platform[ds_name].get_at_timestamp(
sample.timestamp)
_, _, ds_type = parse_datasource_name(ds_name)
annotation_source = categories.get_source(ds_type)
if np.abs(np.int64(sample.timestamp) -
seg_sample.timestamp) <= 1e6:
if 'poly2d' in ds_name:
raw = seg_sample.raw
poly2d = raw['data']
if 'confidence' in raw:
mask = raw['confidence'] > self.conf_threshold
poly2d = poly2d[mask]
elif 'seg2d' in ds_name:
poly2d = seg_sample.poly2d(self.conf_threshold)
for poly in poly2d:
name, color = categories.get_name_color(
annotation_source, poly['classes'])
if self.category_filter is not '':
if name not in self.category_filter:
break
color = np.array(color) / 255
patch = Polygon(poly['polygon'],
closed=True,
linewidth=1,
edgecolor=color,
facecolor=list(color) + [0.15])
self.ax.add_patch(patch)
def __update_bbox_3d(self, sample:Image):
for ds_name, show in self.show_bbox_3d.items():
if not show: continue
box_source = categories.get_source(parse_datasource_name(ds_name)[2])
box3d_sample:Box3d = self.platform[ds_name].get_at_timestamp(sample.timestamp)
if np.abs(float(box3d_sample.timestamp) - float(sample.timestamp)) > 1e6: continue
box3d = box3d_sample.set_referential(self.datasource, ignore_orientation=True)
category_numbers = box3d.get_category_numbers()
poly_collection = []
color_collection = []
for box_index in range(len(box3d)):
center = box3d.get_centers()[box_index]
dimension = box3d.get_dimensions()[box_index]
rotation = box3d.get_rotations()[box_index]
confidence = box3d.get_confidences()[box_index]
category_name, color = categories.get_name_color(box_source, category_numbers[box_index])
id = box3d.get_ids()[box_index]
if confidence:
if confidence < self.conf_threshold: continue
if self.category_filter is not '':
if category_name not in self.category_filter: continue
color = np.array(color)/255
if self.use_box_colors:
color = utils.to_numpy(QColor(self.box_3d_colors[ds_name]))[:3]
vertices = linalg.bbox_to_8coordinates(center, dimension, rotation)
p, mask_fov = sample.project_pts(vertices, mask_fov=False, output_mask=True, undistorted=self.undistortimage, margin=1000)
if p[mask_fov].shape[0] < 8: continue
faces = [[0,1,3,2],[0,1,5,4],[0,2,6,4],[7,3,1,5],[7,5,4,6],[7,6,2,3]]
for face in faces:
poly = np.vstack([p[face[0]],p[face[1]],p[face[2]],p[face[3]],p[face[0]]])
poly_collection.append(poly)
color_collection.append(color)
if self.box_labels_size > 0:
text_label = category_name
if id: text_label += f" {id}"
if confidence: text_label += f" ({int(confidence*100)}%)"
txt = self.ax.text(p[:,0].min(),p[:,1].min(), text_label, color='w', fontweight='bold', fontsize=self.box_labels_size, clip_on=True)
txt.set_path_effects([PathEffects.withStroke(linewidth=1, foreground='k')])
alpha = 0.05
facecolors = [list(c)+[alpha] for c in color_collection]
poly_collection = PolyCollection(poly_collection, linewidths=0.5, edgecolors=color_collection, facecolors=facecolors)
self.ax.add_collection(poly_collection)
def __update_box2D(self, sample, image, box):
datasources = [
ds_name for ds_name, show in self.show_bbox_2d.items() if show
]
for ds_name in datasources:
_, _, ds_type = parse_datasource_name(ds_name)
box_source = categories.get_source(ds_type)
box2d_sample = self.platform[ds_name].get_at_timestamp(
sample.timestamp)
if np.abs(np.int64(sample.timestamp) -
box2d_sample.timestamp) <= 1e6:
raw = box2d_sample.raw
if 'confidence' in raw:
mask = (raw['confidence'] > self.conf_threshold)
box2d = raw['data'][mask]
else:
box2d = raw['data']
if len(box2d) > 0:
for i, box in enumerate(box2d):
top = (box['x'] - box['h'] / 2) * image.shape[0]
left = (box['y'] - box['w'] / 2) * image.shape[1]
name, color = categories.get_name_color(
box_source, box['classes'])
if self.category_filter is not '':
if name not in self.category_filter:
continue
color = np.array(color) / 255
if self.use_box_colors:
color = utils.to_numpy(
QColor(self.box_2d_colors[ds_name]))[:3]
if 'confidence' in raw:
conf = raw['confidence'][mask][i]
name = f"{name}({conf:.3f})"
rect = Rectangle((left, top),
box['w'] * image.shape[1],
box['h'] * image.shape[0],
linewidth=1,
edgecolor=color,
facecolor=list(color) + [0.15])
self.ax.add_patch(rect)
if self.box_labels_size > 0:
txt = self.ax.text(left,
top,
name + ':' + str(box['id']),
color='w',
fontweight='bold',
fontsize=self.box_labels_size,
clip_on=True)
txt.set_path_effects([
PathEffects.withStroke(linewidth=1,
foreground='k')
])
def colored_image(self, resolution: tuple = None):
polygons = self.raw
_, _, ds_type = platform.parse_datasource_name(self.datasource.label)
poly_source = categories.get_source(ds_type)
image = np.zeros(
(polygons['resolution'][0], polygons['resolution'][1], 3),
dtype=np.uint8)
for poly in polygons['data']:
name, color = categories.get_name_color(poly_source,
poly['classes'])
color = np.array(color) / 255
cv2.fillPoly(image, [poly['polygon']], color)
return self.resize_mask(
image, resolution) if resolution is not None else image
def mask_category(self, category):
classes = self.raw['data']['classes']
if type(category) is int:
return classes == category
elif type(category) is str:
_,_,ds_type = platform.parse_datasource_name(self.datasource.label)
seg_source = categories.get_source(ds_type)
try:
category_number = categories.get_category_number(seg_source, category)
except:
category_number = -1
return classes == category_number
else:
raise ValueError('The category must be either an integer or a string.')
def colors(self, mode=None):
_,_,ds_type = platform.parse_datasource_name(self.datasource.label)
seg_source = categories.get_source(ds_type)
classes = self.raw['data']['classes']
colors = np.zeros((len(classes),4))
for c in np.unique(classes):
name, color = categories.get_name_color(seg_source, c)
color = np.array(color)/255. # color channels between 0 and 1
color = np.append(color, 1) # alpha (opacity) = 1
ind = np.where(classes == c)[0]
colors[ind] = color
if mode=='quad_cloud':
colors = clouds.quad_stack(colors)
return colors
def __update_box2D(self, sample, image):
for ds_name, show in self.show_bbox_2d.items():
if not show: continue
box_source = categories.get_source(parse_datasource_name(ds_name)[2])
box2d:Box2d = self.platform[ds_name].get_at_timestamp(sample.timestamp)
if np.abs(float(box2d.timestamp) - float(sample.timestamp)) > 1e6: continue
category_numbers = box2d.get_category_numbers()
for box_index in range(len(box2d)):
center = box2d.get_centers()[box_index]
dimension = box2d.get_dimensions()[box_index]
confidence = box2d.get_confidences()[box_index]
category_name, color = categories.get_name_color(box_source, category_numbers[box_index])
id = box2d.get_ids()[box_index]
if confidence:
if confidence < self.conf_threshold: continue
if self.category_filter is not '':
if category_name not in self.category_filter: continue
color = np.array(color)/255
if self.use_box_colors:
color = utils.to_numpy(QColor(self.box_3d_colors[ds_name]))[:3]
top = (center[0]-dimension[0]/2)*image.shape[0]
left = (center[1]-dimension[1]/2)*image.shape[1]
rect = Rectangle((left,top), dimension[1]*image.shape[1], dimension[0]*image.shape[0], linewidth=1, edgecolor=color, facecolor=list(color)+[0.15])
self.ax.add_patch(rect)
if self.box_labels_size > 0:
text_label = category_name
if id: text_label += f" {id}"
if confidence: text_label += f" ({int(confidence*100)}%)"
txt = self.ax.text(left, top, text_label, color='w', fontweight='bold', fontsize=self.box_labels_size, clip_on=True)
txt.set_path_effects([PathEffects.withStroke(linewidth=1, foreground='k')])
def mask_category(self,
category: str,
resolution: tuple = None,
confidence_threshold: float = 0.5):
polygons = self.raw
_, _, ds_type = platform.parse_datasource_name(self.datasource.label)
poly_source = categories.get_source(ds_type)
mask = np.zeros((polygons['resolution'][0], polygons['resolution'][1]),
dtype=np.uint8)
for i, poly in enumerate(polygons['data']):
if 'confidence' in polygons:
if polygons['confidence'][i] < confidence_threshold:
continue
name, color = categories.get_name_color(poly_source,
poly['classes'])
if name == category:
cv2.fillPoly(mask, [poly['polygon']], 1)
return self.resize_mask(mask,
resolution) if resolution is not None else mask
def _draw_bounding_box_actors(self):
for ds_name, actors in self.viewport.bboxActors.items():
actors['actor'].clearActors()
if not self.controls.showBBox3D[ds_name]: continue
box_source = categories.get_source(
platform_utils.parse_datasource_name(ds_name)[2])
box3d_sample: Box3d = self.platform[ds_name].get_at_timestamp(
self.sample.timestamp)
if np.abs(
float(box3d_sample.timestamp) -
float(self.sample.timestamp)) > 1e6:
continue
box3d = box3d_sample.set_referential(self.ds_name,
ignore_orientation=True)
category_numbers = box3d.get_category_numbers()
for box_index in range(len(box3d)):
center = box3d.get_centers()[box_index]
dimension = box3d.get_dimensions()[box_index]
rotation = box3d.get_rotations()[box_index]
confidence = box3d.get_confidences()[box_index]
category_name, color = categories.get_name_color(
box_source, category_numbers[box_index])
id = box3d.get_ids()[box_index]
if confidence:
if confidence < int(self.controls.confThreshold) / 100.0:
continue
if self.controls.categoryFilter is not '':
if category_name not in self.controls.categoryFilter:
continue
color = QColor.fromRgb(*color)
text_color = QColor('white')
if self.controls.useBoxColors:
text_color = color = QColor(
self.controls.box3DColors[ds_name])
bbox_actor, text_anchor = CustomActors.bbox(center,
dimension,
rotation,
color=color,
return_anchor=True)
bbox_actor.effect.lineWidth = 2
tf = linalg.tf_from_pos_euler(text_anchor)
actors['actor'].addActor(bbox_actor)
if self.controls.boxLabelsSize > 0:
text_label = category_name
if id: text_label += f" {id}"
if confidence: text_label += f" ({int(confidence)}%)"
text_actor = CustomActors.text(
text_label,
color=text_color,
origin=[0, 0, 0],
v=[0, -1, 0],
matrix=utils.from_numpy(tf),
scale=0.1,
font_size=self.controls.boxLabelsSize,
line_width=3,
is_billboard=True,
)
actors['actor'].addActor(text_actor)
def __update_bbox_3d(self, sample, image, box):
datasources = [
ds_name for ds_name, show in self.show_bbox_3d.items() if show
]
for ds_name in datasources:
_, _, ds_type = parse_datasource_name(ds_name)
box_source = categories.get_source(ds_type)
if box_source not in categories.CATEGORIES: #FIXME: this should not be here
box_source = 'deepen'
box3d_sample = self.platform[ds_name].get_at_timestamp(
sample.timestamp)
if np.abs(np.int64(sample.timestamp) -
box3d_sample.timestamp) <= 1e6:
raw = box3d_sample.raw
box3d = box3d_sample.mapto(self.datasource,
ignore_orientation=True)
mask = (box3d['flags'] >= 0)
if 'confidence' in raw:
mask = mask & (raw['confidence'] > self.conf_threshold)
if len(box3d[mask]) > 0:
poly_collection = []
color_collection = []
for i, box in enumerate(box3d[mask]):
name, color = categories.get_name_color(
box_source, box['classes'])
if self.category_filter is not '' and name not in self.category_filter:
break
color = np.array(color) / 255
if self.use_box_colors:
color = utils.to_numpy(
QColor(self.box_3d_colors[ds_name]))[:3]
if 'confidence' in raw:
conf = raw['confidence'][mask][i]
name = f"{name}({conf:.3f})"
vertices = linalg.bbox_to_8coordinates(
box['c'], box['d'], box['r'])
p, mask_fov = sample.project_pts(
vertices,
mask_fov=False,
output_mask=True,
undistorted=self.undistortimage,
margin=1000)
if p[mask_fov].shape[0] < 8:
continue
faces = [[0, 1, 3, 2], [0, 1, 5, 4], [0, 2, 6, 4],
[7, 3, 1, 5], [7, 5, 4, 6], [7, 6, 2, 3]]
for face in faces:
poly = np.vstack([
p[face[0]], p[face[1]], p[face[2]], p[face[3]],
p[face[0]]
])
poly_collection.append(poly)
color_collection.append(color)
if self.box_labels_size > 0:
txt = self.ax.text(p[:, 0].min(),
p[:, 1].min(),
name + ':' + str(box['id']),
color='w',
fontweight='bold',
fontsize=self.box_labels_size,
clip_on=True)
txt.set_path_effects([
PathEffects.withStroke(linewidth=1,
foreground='k')
])
alpha = 0.05
facecolors = [list(c) + [alpha] for c in color_collection]
poly_collection = PolyCollection(
poly_collection,
linewidths=0.5,
edgecolors=color_collection,
facecolors=facecolors)
self.ax.add_collection(poly_collection)
def _draw_bounding_box_actors(self):
## Attemp to add Box in Viewer API
## WIP to have a better implementation
for datasource, actors in self.viewport.bboxActors.items():
actors['actor'].clearActors()
if not self.controls.showBBox3D[datasource]:
continue
sample = self._get_sample(datasource)
if np.abs(
np.int64(self._ds_name_sample.timestamp) -
sample.timestamp) <= 1e6:
_, _, ds_type = platform_utils.parse_datasource_name(
datasource)
box_source = categories.get_source(ds_type)
raw = sample.raw
bbox = sample.mapto(self.ds_name, ignore_orientation=True)
mask = (bbox['flags'] >= 0)
if 'confidence' in raw:
mask = mask & (sample.raw['confidence'] >
int(self.controls.confThreshold) / 100.0)
if (self.controls.showIoU) and (self.controls.refdsIoU
is not ''):
scores_iou = sample.compute_iou(box=self._get_sample(
self.controls.refdsIoU),
return_max=True,
map2yaw=None)
if len(bbox[mask]) > 0:
for i, box in enumerate(bbox[mask]):
c = box['c']
d = box['d']
r = box['r']
has_attributes = 'attributes' in raw
if has_attributes:
attributes = raw['attributes'][mask][i]
name, color = categories.get_name_color(
box_source, box['classes'])
if self.controls.categoryFilter is not '':
if name not in self.controls.categoryFilter:
continue
color = QColor.fromRgb(*color)
text_color = QColor('white')
if self.controls.useBoxColors:
text_color = color = QColor(
self.controls.box3DColors[datasource])
if 'confidence' in raw:
conf = raw['confidence'][mask][i]
name = f'{name}({conf:.3f})'
if (self.controls.showIoU) and (
self.controls.refdsIoU is not ''):
name = f'{name}[IoU={scores_iou[mask][i]:.3f}]'
bbox_actor, text_anchor = CustomActors.bbox(
c, d, r, color=color, return_anchor=True)
bbox_actor.effect.lineWidth = 2
if has_attributes:
bbox_actor.hovered.connect(
self._update_cursor(actors['cursor'],
attributes))
tf = linalg.tf_from_pos_euler(text_anchor)
actors['actor'].addActor(bbox_actor)
if self.controls.boxLabelsSize > 0:
text_actor = CustomActors.text(
name,
color=text_color,
origin=[0, 0, 0],
v=[0, -1, 0],
matrix=utils.from_numpy(tf),
scale=0.1,
font_size=self.controls.boxLabelsSize,
line_width=3,
is_billboard=True)
actors['actor'].addActor(text_actor)
