def test_get_set_detector_name(self):
# Check default
do = DetectedObject(self.bbox)
nt.assert_equal(do.detector_name, "")
do.detector_name = self.detector_name
nt.assert_equal(do.detector_name, self.detector_name)
def test_nice_format(self):
# Test default
do = DetectedObject(self.bbox)
nt.assert_equal(do.__nice__(), "conf=1.0")
do.confidence = -0.5
nt.assert_equal(do.__nice__(), "conf=-0.5")
def test_get_set_index(self):
# Check default
do = DetectedObject(self.bbox)
nt.assert_equal(do.index, 0)
do.index = 5
nt.assert_equal(do.index, 5)
def test_get_set_geo_point(self):
do = DetectedObject(self.bbox)
# Check default
nt.ok_(self.check_geo_points_equal(do.geo_point, GeoPoint()))
# Setting to different value
do.geo_point = self.geo_point
nt.ok_(self.check_geo_points_equal(do.geo_point, self.geo_point))
def test_init(self):
DetectedObject(self.bbox)
DetectedObject(self.bbox, self.conf, self.dot, self.mask)
# Referencing keyword arguments
DetectedObject(self.bbox,
confidence=self.conf,
classifications=self.dot,
mask=self.mask)
def test_get_set_bbox(self):
do = DetectedObject(self.bbox)
# Check default
nt.ok_(do.bounding_box, self.bbox)
# Setting to different value
new_bbox = BoundingBox(20, 20, 40, 40)
do.bounding_box = new_bbox
nt.ok_(do.bounding_box == new_bbox)
def _create_detected_object_set():
from kwiver.vital.types import DetectedObject, DetectedObjectSet, BoundingBox
dos = DetectedObjectSet()
bbox = BoundingBox(0, 10, 100, 50)
dos.add(DetectedObject(bbox, 0.2))
dos.add(DetectedObject(bbox, 0.5))
dos.add(DetectedObject(bbox, 0.4))
return dos
def test_get_set_confidence(self):
# Check default
do = DetectedObject(self.bbox)
np.testing.assert_almost_equal(do.confidence, 1.0)
# Check setting through constructor
do = DetectedObject(self.bbox, confidence=-1.5)
np.testing.assert_almost_equal(do.confidence, -1.5)
# Check setting through setter
do.confidence = 2.5
np.testing.assert_almost_equal(do.confidence, 2.5)
def test_get_set_mask(self):
# Check default
do = DetectedObject(self.bbox)
nt.ok_(self.check_img_containers_equal(do.mask, None))
# Check setting through setter
do.mask = self.mask
nt.ok_(self.check_img_containers_equal(do.mask, self.mask))
# Check setting through constructor
new_mask = ImageContainer(Image(2048, 1080))
do = DetectedObject(self.bbox, mask=new_mask)
nt.ok_(self.check_img_containers_equal(do.mask, new_mask))
def test_get_set_type(self):
# Check default
do = DetectedObject(self.bbox)
nt.ok_(self.check_det_obj_types_equal(do.type, None))
# Check setting through setter
do.type = self.dot
nt.ok_(self.check_det_obj_types_equal(do.type, self.dot))
# Check setting through constructor
new_dot = DetectedObjectType("other_example_class", -3.14)
do = DetectedObject(self.bbox, classifications=new_dot)
nt.ok_(self.check_det_obj_types_equal(do.type, new_dot))
def _classification_to_kwiver_detections(classification, w, h):
"""
Convert kwarray classifications to kwiver deteted object sets
Args:
classification (bioharn.clf_predict.Classification)
w (int): width of image
h (int): height of image
Returns:
kwiver.vital.types.DetectedObjectSet
"""
detected_objects = DetectedObjectSet()
if classification.data.get('prob', None) is not None:
# If we have a probability for each class, uses that
class_names = list(classification.classes)
class_prob = classification.prob
detected_object_type = DetectedObjectType(class_names, class_prob)
else:
# Otherwise we only have the score for the predicted calss
class_name = classification.classes[classification.cidx]
class_score = classification.conf
detected_object_type = DetectedObjectType(class_name, class_score)
bounding_box = BoundingBoxD(0, 0, w, h)
detected_object = DetectedObject(bounding_box, classification.conf,
detected_object_type)
detected_objects.add(detected_object)
return detected_objects
def _kwimage_to_kwiver_detections(detections):
"""
Convert kwimage detections to kwiver deteted object sets
Args:
detected_objects (kwimage.Detections)
Returns:
kwiver.vital.types.DetectedObjectSet
"""
# convert segmentation masks
if 'segmentations' in detections.data:
print("Warning: segmentations not implemented")
boxes = detections.boxes.to_tlbr()
scores = detections.scores
class_idxs = detections.class_idxs
# convert to kwiver format, apply threshold
detected_objects = DetectedObjectSet()
for tlbr, score, cidx in zip(boxes.data, scores, class_idxs):
class_name = detections.classes[cidx]
bbox_int = np.round(tlbr).astype(np.int32)
bounding_box = BoundingBox(bbox_int[0], bbox_int[1], bbox_int[2],
bbox_int[3])
detected_object_type = DetectedObjectType(class_name, score)
detected_object = DetectedObject(bounding_box, score,
detected_object_type)
detected_objects.add(detected_object)
return detected_objects
def _create_object_track_set(self):
bbox = BoundingBox(10, 10, 20, 20)
dot = DetectedObjectType("test", 0.4)
do = DetectedObject(bbox, 0.4, dot)
track = Track()
for i in range(10):
track.append(ObjectTrackState(i, i, do))
return ObjectTrackSet([track])
def _kwimage_to_kwiver_detections(detections):
"""
Convert kwimage detections to kwiver deteted object sets
Args:
detected_objects (kwimage.Detections)
Returns:
kwiver.vital.types.DetectedObjectSet
"""
from kwiver.vital.types.types import ImageContainer, Image
segmentations = None
# convert segmentation masks
if 'segmentations' in detections.data:
segmentations = detections.data['segmentations']
boxes = detections.boxes.to_tlbr()
scores = detections.scores
class_idxs = detections.class_idxs
if not segmentations:
# Placeholders
segmentations = (None, ) * len(boxes)
# convert to kwiver format, apply threshold
detected_objects = DetectedObjectSet()
for tlbr, score, cidx, seg in zip(boxes.data, scores, class_idxs,
segmentations):
class_name = detections.classes[cidx]
bbox_int = np.round(tlbr).astype(np.int32)
bounding_box = BoundingBoxD(bbox_int[0], bbox_int[1], bbox_int[2],
bbox_int[3])
detected_object_type = DetectedObjectType(class_name, score)
detected_object = DetectedObject(bounding_box, score,
detected_object_type)
if seg:
mask = seg.to_relative_mask().numpy().data
detected_object.mask = ImageContainer(Image(mask))
detected_objects.add(detected_object)
return detected_objects
def _create_detected_object(self):
"""
Helper function to generate a detected object for the track state
:return: Detected object with bounding box coordinates of
(10, 10, 20, 20), confidence of 0.4 and "test" label
"""
bbox = bbD(10, 10, 20, 20)
cm = DOT("test", 0.4)
do = DetectedObject(bbox, 0.4, cm)
return do
def detect(self, in_img_c):
import tensorflow as tf
import humanfriendly
image_height = in_img_c.height()
image_width = in_img_c.width()
if (self.norm_image_type and self.norm_image_type != "none"):
print("Normalizing input image")
in_img = in_img_c.image().asarray().astype("uint16")
bottom, top = self.get_scaling_values(self.norm_image_type, in_img,
image_height)
in_img = self.lin_normalize_image(in_img, bottom, top)
in_img = np.tile(in_img, (1, 1, 3))
else:
in_img = np.array(get_pil_image(in_img_c.image()).convert("RGB"))
start_time = time.time()
boxes, scores, classes = self.generate_detection(
self.detection_graph, in_img)
elapsed = time.time() - start_time
print("Done running detector in {}".format(
humanfriendly.format_timespan(elapsed)))
good_boxes = []
detections = DetectedObjectSet()
for i in range(0, len(scores)):
if (scores[i] >= self.confidence_thresh):
bbox = boxes[i]
good_boxes.append(bbox)
top_rel = bbox[0]
left_rel = bbox[1]
bottom_rel = bbox[2]
right_rel = bbox[3]
xmin = left_rel * image_width
ymin = top_rel * image_height
xmax = right_rel * image_width
ymax = bottom_rel * image_height
dot = DetectedObjectType(self.category_name, scores[i])
obj = DetectedObject(BoundingBoxD(xmin, ymin, xmax, ymax),
scores[i], dot)
detections.add(obj)
print("Detected {}".format(len(good_boxes)))
return detections
def test_str_format(self):
# Test default
do = DetectedObject(self.bbox)
nt.assert_equal(do.__str__(), "<DetectedObject(conf=1.0)>")
do = DetectedObject(self.bbox, confidence=-0.5)
nt.assert_equal(do.__str__(), "<DetectedObject(conf=-0.5)>")
def test_notes(self):
# Check default
do = DetectedObject(self.bbox)
nt.assert_equal(do.notes, [])
# Clearing empty is OK
do.clear_notes()
nt.assert_equal(do.notes, [])
# Add a few values
do.add_note(self.note_to_add)
exp_notes = [self.note_to_add]
nt.assert_equal(do.notes, exp_notes)
new_note = "other_example_note"
do.add_note(new_note)
exp_notes.append(new_note)
nt.assert_equal(do.notes, exp_notes)
# Clearing works as expected
do.clear_notes()
nt.assert_equal(do.notes, [])
def detect(self, image_data):
dot = DetectedObjectSet([
DetectedObject(
BoundingBox(
self.m_center_x + self.frame_ct * self.m_dx -
self.m_width / 2.0, self.m_center_y +
self.frame_ct * self.m_dy - self.m_height / 2.0,
self.m_center_x + self.frame_ct * self.m_dx +
self.m_width / 2.0, self.m_center_y +
self.frame_ct * self.m_dy + self.m_height / 2.0))
])
self.frame_ct += 1
return dot
def _create_track(self):
"""
Helper function to create a track
:return: Track with 10 object track state. Every track state has same
detected object however the fram number and time varies from
[0, 10)
"""
bbox = BoundingBox(10, 10, 20, 20)
cm = ClassMap("test", 0.4)
do = DetectedObject(bbox, 0.4, cm)
track = Track()
for i in range(10):
track.append(ObjectTrackState(i, i, do))
return track
def test_descriptor(self):
# Check default
do = DetectedObject(self.bbox)
nt.ok_(self.check_descriptors_equal(do.descriptor_copy(), None))
do.set_descriptor(self.descriptor)
nt.ok_(
self.check_descriptors_equal(do.descriptor_copy(),
self.descriptor))
def test_descriptor_modify(self):
do = DetectedObject(self.bbox)
do.set_descriptor(self.descriptor)
# Attempts to modify the descriptor don't work
do.descriptor_copy()[0] += 1
# print(do.descriptor_copy().todoublearray(), self.descriptor.todoublearray())
nt.ok_(
self.check_descriptors_equal(do.descriptor_copy(),
self.descriptor))
# Modify the object copied from. Changes should not be reflected in
# the detected_objects reference
self.descriptor[0] += 1
# print(do.descriptor_copy().todoublearray(), self.descriptor.todoublearray())
nt.assert_false(
self.check_descriptors_equal(do.descriptor_copy(),
self.descriptor))
# Storing the copy in a new variable, obviously, allows for modification
desc = do.descriptor_copy()
desc[0] += 1
nt.assert_false(
self.check_descriptors_equal(do.descriptor_copy(), desc))
def test_keypoints(self):
# Check default
do = DetectedObject(self.bbox)
self.check_keypoints_equal(do.keypoints, dict())
# Clearing empty is OK
do.clear_keypoints()
self.check_keypoints_equal(do.keypoints, dict())
# Add a few values
do.add_keypoint(self.keypoint_id, self.keypoint_to_add)
exp_keypoints = {self.keypoint_id: self.keypoint_to_add}
self.check_keypoints_equal(do.keypoints, exp_keypoints)
new_keypoint_id = "other_example_keypoint_id"
new_keypoint = Point2d()
new_keypoint.value = self.loc1
do.add_keypoint(new_keypoint_id, new_keypoint)
exp_keypoints[new_keypoint_id] = new_keypoint
self.check_keypoints_equal(do.keypoints, exp_keypoints)
# Clearing works as expected
do.clear_keypoints()
self.check_keypoints_equal(do.keypoints, dict())
def test_repr_format(self):
# Test default
do = DetectedObject(self.bbox)
nt.assert_equal(do.__repr__(),
"<DetectedObject(conf=1.0) at {}>".format(hex(id(do))))
do = DetectedObject(self.bbox, confidence=-0.5)
nt.assert_equal(
do.__repr__(),
"<DetectedObject(conf=-0.5) at {}>".format(hex(id(do))))
def setUp(self):
bbox = BoundingBox(10, 10, 20, 20)
dot = DetectedObjectType("test", 0.4)
do = DetectedObject(bbox, 0.4, dot)
track = Track()
for i in range(10):
track.append(ObjectTrackState(i, i, do))
self.track_ = track
self.time_1 = Timestamp()
self.time_1.set_time_seconds(1234)
self.time_2 = Timestamp()
self.time_2.set_time_seconds(4321)
self.obj_ts = ObjectTrackSet([self.track_])
self.act_type = ActivityType("self_act", 0.87)
self.act = Activity(1, "self_act", 0.87, self.act_type, self.time_1,
self.time_2, self.obj_ts)
def _dowork(self, img_container):
"""
Helper to decouple the algorithm and pipeline logic
CommandLine:
xdoctest viame.processes.camtrawl.processes CamtrawlDetectFishProcess._dowork
Example:
>>> from viame.processes.camtrawl.processes import *
>>> from kwiver.vital.types import ImageContainer
>>> import kwiver.sprokit.pipeline.config
>>> # construct dummy process instance
>>> conf = kwiver.sprokit.pipeline.config.empty_config()
>>> self = CamtrawlDetectFishProcess(conf)
>>> self._configure()
>>> # construct test data
>>> from vital.util import VitalPIL
>>> from PIL import Image as PILImage
>>> pil_img = PILImage.open(ub.grabdata('https://i.imgur.com/Jno2da3.png'))
>>> pil_img = PILImage.fromarray(np.zeros((512, 512, 3), dtype=np.uint8))
>>> img_container = ImageContainer(VitalPIL.from_pil(pil_img))
>>> # Initialize the background detector by sending 10 black frames
>>> for i in range(10):
>>> empty_set = self._dowork(img_container)
>>> # now add a white box that should be detected
>>> np_img = np.zeros((512, 512, 3), dtype=np.uint8)
>>> np_img[300:340, 220:380] = 255
>>> img_container = ImageContainer.fromarray(np_img)
>>> detection_set = self._dowork(img_container)
>>> assert len(detection_set) == 1
>>> obj = detection_set[0]
"""
# This should be read as np.uint8
np_img = img_container.asarray()
detection_set = DetectedObjectSet()
ct_detections = self.detector.detect(np_img)
for detection in ct_detections:
bbox = BoundingBoxD(*detection.bbox.coords)
mask = detection.mask.astype(np.uint8)
vital_mask = ImageContainer.fromarray(mask)
dot = DetectedObjectType("Motion", 1.0)
obj = DetectedObject(bbox, 1.0, dot, mask=vital_mask)
detection_set.add(obj)
return detection_set
def _step(self):
image_container = self.grab_input_using_trait("image")
timestamp = self.grab_input_using_trait("timestamp")
file_name = self.grab_input_using_trait("file_name")
image = image_container.asarray()
h, w, _ = image.shape
bbox_x = w//2
bbox_y = h//2
bbox = BoundingBox( bbox_x - int(self.config_value("bbox_width"))//2,
bbox_y - int(self.config_value("bbox_height"))//2,
bbox_x + int(self.config_value("bbox_width"))//2,
bbox_y + int(self.config_value("bbox_height"))//2 )
dot = DetectedObjectType("Test", 1.0)
do = DetectedObject(bbox, 1.0, dot)
dos = DetectedObjectSet()
dos.add(do)
self.push_to_port_using_trait("detected_object_set", dos)
def merge(self, det_sets):
# Get detection HL info in a list
pred_sets = []
for det_set in det_sets:
pred_set = []
for det in det_set:
# Extract box info for this det
bbox = det.bounding_box
bbox_min_x = int(bbox.min_x())
bbox_max_x = int(bbox.max_x())
bbox_min_y = int(bbox.min_y())
bbox_max_y = int(bbox.max_y())
# Extract type info for this det
if det.type is None:
continue
#class_names = list( det.type.class_names() )
#class_scores = [ det.type.score( n ) for n in class_names ]
class_name = det.type.get_most_likely_class()
class_score = det.type.score(class_name)
pred_set.append([
bbox_min_x, bbox_min_y, bbox_max_x, bbox_max_y, class_name,
class_score
])
pred_sets.append(pred_set)
# Run merging algorithm
#ensemble_preds = ensemble_box( preds_set, self._fusion_weights,
# self._iou_thr, self._skip_box_thr, self._sigma, self._fusion_type )
ensemble_preds = []
# Compile output detections
output = DetectedObjectSet()
for pred in ensemble_preds:
score = pred[5]
bbox = BoundingBoxD(pred[0], pred[1], pred[2], pred[3])
dot = DetectedObjectType(pred[4], score)
det = DetectedObject(bbox, score, dot)
output.add(det)
return output
def test_id(self):
a = self.act
self.assertEqual(a.id, 1)
a.id = 10
self.assertEqual(a.id, 10)
self.assertEqual(a.label, "self_act")
a.label = "second_act"
self.assertEqual(a.label, "second_act")
self.assertEqual(a.activity_type.score("self_act"), 0.87)
a.activity_type = ActivityType()
self.assertEqual(a.confidence, 0.87)
a.confidence = 1
self.assertEqual(a.confidence, 1)
self.assertEqual(a.start_time.get_time_seconds(), 1234)
tmp_time = Timestamp().set_time_seconds(1237)
a.start_time = tmp_time
self.assertEqual(a.start_time.get_time_seconds(), 1237)
self.assertEqual(a.end_time.get_time_seconds(), 4321)
tmp_time = Timestamp()
tmp_time.set_time_seconds(4322)
a.end_time = tmp_time
self.assertEqual(a.end_time.get_time_seconds(), 4322)
self.assertEqual(a.participants.all_frame_ids(), set(range(10)))
bbox = BoundingBox(10, 10, 20, 20)
dot = DetectedObjectType("test", 0.4)
do = DetectedObject(bbox, 0.4, dot)
track = Track()
for i in range(5):
track.append(ObjectTrackState(i, i, do))
new_t = track
new_ots = ObjectTrackSet([new_t])
a.participants = new_ots
self.assertEqual(a.participants.all_frame_ids(), set(range(5)))
self.assertEqual(a.duration[0].get_time_seconds(),
a.start_time.get_time_seconds())
self.assertEqual(a.duration[1].get_time_seconds(),
a.end_time.get_time_seconds())
def detect(self, image_data):
input_image = image_data.asarray().astype('uint8')
if self._rgb_to_bgr:
input_image = cv2.cvtColor(input_image, cv2.COLOR_RGB2BGR)
from mmdet.apis import inference_detector
detections = inference_detector(self._model, input_image)
if isinstance(detections, tuple):
bbox_result, segm_result = detections
else:
bbox_result, segm_result = detections, None
if np.size(bbox_result) > 0:
bboxes = np.vstack(bbox_result)
else:
bboxes = []
# convert segmentation masks
masks = []
if segm_result is not None:
segms = mmcv.concat_list(segm_result)
inds = np.where(bboxes[:, -1] > score_thr)[0]
for i in inds:
masks.append(maskUtils.decode(segms[i]).astype(np.bool))
# collect labels
labels = [
np.full(bbox.shape[0], i, dtype=np.int32)
for i, bbox in enumerate(bbox_result)
]
if np.size(labels) > 0:
labels = np.concatenate(labels)
else:
labels = []
# convert to kwiver format, apply threshold
output = DetectedObjectSet()
for bbox, label in zip(bboxes, labels):
class_confidence = float(bbox[-1])
if class_confidence < self._thresh:
continue
bbox_int = bbox.astype(np.int32)
bounding_box = BoundingBoxD(bbox_int[0], bbox_int[1], bbox_int[2],
bbox_int[3])
class_name = self._labels[label]
detected_object_type = DetectedObjectType(class_name,
class_confidence)
detected_object = DetectedObject(bounding_box,
np.max(class_confidence),
detected_object_type)
output.add(detected_object)
if np.size(labels) > 0 and self._display_detections:
mmcv.imshow_det_bboxes(input_image,
bboxes,
labels,
class_names=self._labels,
score_thr=self._thresh,
show=True)
return output
