def get_masks_with_features(self, debug=False): image = self.camera.get_image() objects: List[ObjectInfo] = self.find_objects.find_objects(image) objects_with_features = [] cropped_images = [(i, cv.cvtColor(x.object_img_cutout_cropped, cv.COLOR_BGR2RGB)) for i, x in enumerate(objects)] pil_cropped_images = [Image.fromarray(x) for (i, x) in cropped_images] indexes = [i for (i, x) in cropped_images] all_features = self.feature_extractor.get_features(pil_cropped_images) features_with_index = [(i, features) for (i, features) in zip(indexes, all_features)] for i, features in features_with_index: object_with_features = ObjectInfoWithFeatures(objects[i], features) objects_with_features.append(object_with_features) if debug: cv.imshow("Image", image) for i, (obj_info_with_features) in enumerate(objects_with_features): cv.imshow(f"obj-{i+1}", obj_info_with_features.object_img_cutout_cropped) print(f"obj-{i+1} features: {','.join([str(x) for x in obj_info_with_features.features])}") cv.waitKey(0) return objects_with_features
def test_spatialPart(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "above"), (EntityType.OBJECT, "bottom cover")] features = [] feature = [ np.array([1, 1, 1, 1, 1]), np.array([1, 1, 1, 1, 1]), np.array([1, 1, 1, 1, 1]) ] bbox = [ np.array([1, 2, 3, 4]), np.array([1, 2, 3, 4]), np.array([1, 2, 3, 4]) ] mask = [ np.array([4, 3, 2, 1]), np.array([4, 3, 2, 1]), np.array([4, 3, 2, 1]) ] cropped_rbg = [ np.array([5, 5, 5, 5]), np.array([5, 5, 5, 5]), np.array([5, 5, 5, 5]) ] for i in range(3): object_info = ObjectInfoWithFeatures() object_info.features = feature[i] object_info.bbox_xxyy = bbox[i] object_info.mask_full = mask[i] object_info.mask_cropped = cropped_rbg[i] features.append(object_info) db_features = [("dummy name", np.array([1, 1, 1, 1, 1])), ("dummy name", np.array([1, 1, 1, 1, 1])), ("dummy name", np.array([1, 1, 1, 1, 1]))] self.vision_mock.get_masks_with_features = Mock(return_value=features) self.db_mock.get_feature = Mock(return_value=np.array([1, 1, 1, 1, 1])) self.db_mock.get_all_features = Mock(return_value=db_features) self.ner_mock.get_entities = Mock(return_value=entities) self.spatial_mock.locate_specific_object = Mock( return_value=([1], StatusEnum.SUCCESS)) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] returned = self.grounding.find_object(object_entity) self.assertTrue(returned.is_success) self.assertIsNotNone(returned.object_infos)
def test_unknownObject(self): object_info = ObjectInfoWithFeatures() features = [] object_info.features = np.array([1, 1, 1, 1, 1]) object_info.bbox_xxyy = np.array([1, 2, 3, 4]) object_info.mask_full = np.array([4, 3, 2, 1]) object_info.mask_cropped = np.array([5, 5, 5, 5]) features.append(object_info) self.vision_mock.get_masks_with_features = Mock(return_value=features) object_name = "albert is cool" object_spatial_desc = None object_entity = ObjectEntity() object_entity.name = object_name self.returned = self.grounding.find_object(object_entity) self.assertEqual(self.returned.error_code, GroundingErrorType.UNKNOWN)
def test_updateCorrectObject(self): object_info = ObjectInfoWithFeatures() features = [] object_info.features = np.array([1, 1, 1, 1, 1]) object_info.bbox_xxyy = np.array([1, 2, 3, 4]) object_info.mask_full = np.array([4, 3, 2, 1]) object_info.mask_cropped = np.array([5, 5, 5, 5]) features.append(object_info) self.vision_mock.get_masks_with_features = Mock(return_value=features) object_name = "black cover" object_spatial_desc = None object_entity = ObjectEntity() object_entity.name = object_name object_entity.spatial_descriptions = object_spatial_desc self.returned = self.grounding.update_features(object_entity) self.assertTrue(self.returned.is_success)
def test_newObject(self): object_info = ObjectInfoWithFeatures() features = [] object_info.features = np.array([1, 1, 1, 1, 1]) object_info.bbox_xxyy = np.array([1, 2, 3, 4]) object_info.mask_full = np.array([4, 3, 2, 1]) object_info.mask_cropped = np.array([5, 5, 5, 5]) features.append(object_info) self.vision_mock.get_masks_with_features = Mock(return_value=features) entity_name = "green cover" # Make sure this is a new object. spatial_desc = None object_entity = ObjectEntity() object_entity.name = entity_name object_entity.spatial_descriptions = spatial_desc self.returned = self.grounding.learn_new_object(object_entity) self.assertTrue(self.returned.is_success)
def test_learnKnownObject(self): object_info = ObjectInfoWithFeatures() features = [] object_info.features = np.array([1, 1, 1, 1, 1]) object_info.bbox_xxyy = np.array([1, 2, 3, 4]) object_info.mask_full = np.array([4, 3, 2, 1]) object_info.mask_cropped = np.array([5, 5, 5, 5]) features.append(object_info) self.vision_mock.get_masks_with_features = Mock(return_value=features) object_name = "black cover" object_spatial_desc = None object_entity = ObjectEntity() object_entity.name = object_name object_entity.spatial_descriptions = object_spatial_desc self.returned = self.grounding.learn_new_object(object_entity) self.assertEqual(self.returned.error_code, GroundingErrorType.ALREADY_KNOWN)