def get_masks_with_features(self, debug=False):
image = self.camera.get_image()
objects: List[ObjectInfo] = self.find_objects.find_objects(image)
objects_with_features = []
cropped_images = [(i, cv.cvtColor(x.object_img_cutout_cropped, cv.COLOR_BGR2RGB)) for i, x in enumerate(objects)]
pil_cropped_images = [Image.fromarray(x) for (i, x) in cropped_images]
indexes = [i for (i, x) in cropped_images]
all_features = self.feature_extractor.get_features(pil_cropped_images)
features_with_index = [(i, features) for (i, features) in zip(indexes, all_features)]
for i, features in features_with_index:
object_with_features = ObjectInfoWithFeatures(objects[i], features)
objects_with_features.append(object_with_features)
if debug:
cv.imshow("Image", image)
for i, (obj_info_with_features) in enumerate(objects_with_features):
cv.imshow(f"obj-{i+1}", obj_info_with_features.object_img_cutout_cropped)
print(f"obj-{i+1} features: {','.join([str(x) for x in obj_info_with_features.features])}")
cv.waitKey(0)
return objects_with_features
def test_spatialPart(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "above"),
(EntityType.OBJECT, "bottom cover")]
features = []
feature = [
np.array([1, 1, 1, 1, 1]),
np.array([1, 1, 1, 1, 1]),
np.array([1, 1, 1, 1, 1])
]
bbox = [
np.array([1, 2, 3, 4]),
np.array([1, 2, 3, 4]),
np.array([1, 2, 3, 4])
]
mask = [
np.array([4, 3, 2, 1]),
np.array([4, 3, 2, 1]),
np.array([4, 3, 2, 1])
]
cropped_rbg = [
np.array([5, 5, 5, 5]),
np.array([5, 5, 5, 5]),
np.array([5, 5, 5, 5])
]
for i in range(3):
object_info = ObjectInfoWithFeatures()
object_info.features = feature[i]
object_info.bbox_xxyy = bbox[i]
object_info.mask_full = mask[i]
object_info.mask_cropped = cropped_rbg[i]
features.append(object_info)
db_features = [("dummy name", np.array([1, 1, 1, 1, 1])),
("dummy name", np.array([1, 1, 1, 1, 1])),
("dummy name", np.array([1, 1, 1, 1, 1]))]
self.vision_mock.get_masks_with_features = Mock(return_value=features)
self.db_mock.get_feature = Mock(return_value=np.array([1, 1, 1, 1, 1]))
self.db_mock.get_all_features = Mock(return_value=db_features)
self.ner_mock.get_entities = Mock(return_value=entities)
self.spatial_mock.locate_specific_object = Mock(
return_value=([1], StatusEnum.SUCCESS))
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
returned = self.grounding.find_object(object_entity)
self.assertTrue(returned.is_success)
self.assertIsNotNone(returned.object_infos)
def test_unknownObject(self):
object_info = ObjectInfoWithFeatures()
features = []
object_info.features = np.array([1, 1, 1, 1, 1])
object_info.bbox_xxyy = np.array([1, 2, 3, 4])
object_info.mask_full = np.array([4, 3, 2, 1])
object_info.mask_cropped = np.array([5, 5, 5, 5])
features.append(object_info)
self.vision_mock.get_masks_with_features = Mock(return_value=features)
object_name = "albert is cool"
object_spatial_desc = None
object_entity = ObjectEntity()
object_entity.name = object_name
self.returned = self.grounding.find_object(object_entity)
self.assertEqual(self.returned.error_code, GroundingErrorType.UNKNOWN)
def test_updateCorrectObject(self):
object_info = ObjectInfoWithFeatures()
features = []
object_info.features = np.array([1, 1, 1, 1, 1])
object_info.bbox_xxyy = np.array([1, 2, 3, 4])
object_info.mask_full = np.array([4, 3, 2, 1])
object_info.mask_cropped = np.array([5, 5, 5, 5])
features.append(object_info)
self.vision_mock.get_masks_with_features = Mock(return_value=features)
object_name = "black cover"
object_spatial_desc = None
object_entity = ObjectEntity()
object_entity.name = object_name
object_entity.spatial_descriptions = object_spatial_desc
self.returned = self.grounding.update_features(object_entity)
self.assertTrue(self.returned.is_success)
def test_newObject(self):
object_info = ObjectInfoWithFeatures()
features = []
object_info.features = np.array([1, 1, 1, 1, 1])
object_info.bbox_xxyy = np.array([1, 2, 3, 4])
object_info.mask_full = np.array([4, 3, 2, 1])
object_info.mask_cropped = np.array([5, 5, 5, 5])
features.append(object_info)
self.vision_mock.get_masks_with_features = Mock(return_value=features)
entity_name = "green cover" # Make sure this is a new object.
spatial_desc = None
object_entity = ObjectEntity()
object_entity.name = entity_name
object_entity.spatial_descriptions = spatial_desc
self.returned = self.grounding.learn_new_object(object_entity)
self.assertTrue(self.returned.is_success)
def test_learnKnownObject(self):
object_info = ObjectInfoWithFeatures()
features = []
object_info.features = np.array([1, 1, 1, 1, 1])
object_info.bbox_xxyy = np.array([1, 2, 3, 4])
object_info.mask_full = np.array([4, 3, 2, 1])
object_info.mask_cropped = np.array([5, 5, 5, 5])
features.append(object_info)
self.vision_mock.get_masks_with_features = Mock(return_value=features)
object_name = "black cover"
object_spatial_desc = None
object_entity = ObjectEntity()
object_entity.name = object_name
object_entity.spatial_descriptions = object_spatial_desc
self.returned = self.grounding.learn_new_object(object_entity)
self.assertEqual(self.returned.error_code,
GroundingErrorType.ALREADY_KNOWN)