def test_ner(model_path, tag_path): ner = NER(model_path, tag_path) cmd_builder = CommandBuilder(ner) while True: print("Input sentence to test:") text = input("> ") task = cmd_builder.get_task(text) print(task)
def setUp(self): self.ner_mock = Mock() self.db_mock = Mock() self.spatial = SpatialRelation(database_handler=self.db_mock) self.cmd_builder = CommandBuilder(self.ner_mock) self.objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000 (0, "black cover", [100, 400, 100, 300]), (1, "blue cover", [700, 1100, 100, 300]), (2, "fuse", [100, 400, 800, 1000]), (3, "bottom cover", [700, 1100, 800, 1000]), (4, "white cover", [100, 400, 1500, 1600]), (5, "green cover", [700, 700, 1500, 1600]) ]
def setUp(self): self.ner_mock = Mock() self.db_mock = Mock() self.spatial_mock = Mock() self.vision_mock = Mock() self.returned = GroundingReturn() self.cmd_builder = CommandBuilder(self.ner_mock) self.grounding = Grounding(db=self.db_mock, vision_controller=self.vision_mock, spatial=self.spatial_mock) self.objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000 (0, "black cover", [100, 400, 100, 300]), (1, "blue cover", [700, 1100, 100, 300]), (2, "fuse", [100, 400, 800, 1000]), (3, "bottom cover", [700, 1100, 800, 1000]), (4, "white cover", [100, 400, 1500, 1600]), (5, "blue cover", [700, 700, 1500, 1600]) ]
class NERService: def __init__(self, model_path, tags_path): self.cmd_builder = CommandBuilder(model_path, tags_path) def handle_ner_request(self, request: NERRequest): task = self.cmd_builder.get_task(request.sentence) ros_task = create_ros_task(task) ros_outer_task = OuterTask(type=ros_task.type, object1=ros_task.object1, object2=ros_task.object2, child_tasks=[]) if len(task.child_tasks) > 0: for child_task in task.child_tasks: ros_task = create_ros_task(child_task) ros_outer_task.child_tasks.append(ros_task) return NERResponse(task=ros_outer_task) def ner_server(self): rospy.init_node("ner_server") s = rospy.Service("ner", NER, self.handle_ner_request) rospy.spin()
def setUp(self): self.ner = NER("ner_model.bin", "../scripts/NER_4/tags.txt") self.cmd_builder = CommandBuilder(self.ner)
class NERIntegrationTestCase(unittest.TestCase): def setUp(self): self.ner = NER("ner_model.bin", "../scripts/NER_4/tags.txt") self.cmd_builder = CommandBuilder(self.ner) def test_get_task__sentence_with_pick_up_task__returns_pick_up_task_and_locations(self): task = self.cmd_builder.get_task("Please pick up the blue cover that is next to the black bottom cover which is above a bottom cover") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "blue cover") self.assertEqual(len(task.objects_to_execute_on[0].spatial_descriptions), 2) self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type, SpatialType.NEXT_TO) self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity) self.assertEqual("black bottom cover", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name) self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[1].spatial_type, SpatialType.ABOVE) self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[1].object_entity) self.assertEqual("bottom cover", task.objects_to_execute_on[0].spatial_descriptions[1].object_entity.name) def test_get_task__sentence_with_find_task__returns_pick_up_task_and_locations(self): task = self.cmd_builder.get_task("Please find the blue cover that is next to the yellow bottom cover") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "blue cover") self.assertEqual(len(task.objects_to_execute_on[0].spatial_descriptions), 1) self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type, SpatialType.NEXT_TO) self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity) self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name, "yellow bottom cover") def test_get_task__sentence_with_pick_up_task__returns_pick_up_task_and_locations(self): task = self.cmd_builder.get_task( "Please pick up the blue cover that is next to the black bottom cover which is above a bottom cover") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "blue cover") self.assertEqual(len(task.objects_to_execute_on[0].spatial_descriptions), 2) self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type, SpatialType.NEXT_TO) self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity) self.assertEqual("black bottom cover", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name) self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[1].spatial_type, SpatialType.ABOVE) self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[1].object_entity) self.assertEqual("bottom cover", task.objects_to_execute_on[0].spatial_descriptions[1].object_entity.name) def test_get_task__sentence_with_pick_up_task__returns_pick_up_test1(self): task = self.cmd_builder.get_task( "Please pick up the black bottom cover") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "black bottom cover") self.assertTrue(task.name == "pick up") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test2(self): task = self.cmd_builder.get_task( "Please pick up the screwdriver") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "screwdriver") self.assertEqual(task.name,"get") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test3(self): task = self.cmd_builder.get_task( "Please move the hammer") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "hammer") self.assertEqual(task.name,"move") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test4(self): task = self.cmd_builder.get_task( "Please pick up the fuse next to the book") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "fuse") self.assertEqual(task.name, "pick up") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test5(self): task = self.cmd_builder.get_task( "Please pick up the green cover on top of the table") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "green cover") self.assertEqual(task.name, "pick up") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test6(self): task = self.cmd_builder.get_task( "Please place the blue cover on top of the black bottom cover") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "blue cover") self.assertEqual(task.name, "place") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test7(self): task = self.cmd_builder.get_task( "Please find the white bottom cover") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "white bottom cover") self.assertEqual(task.name, "find") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test8(self): task = self.cmd_builder.get_task( "Please find the fuse") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "fuse") self.assertEqual(task.name, "find") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test9(self): task = self.cmd_builder.get_task( "Please weld the cover") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "cover") self.assertEqual(task.name, "weld") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test10(self): task = self.cmd_builder.get_task( "Please put the fuse on top of the white cover") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "fuse") self.assertEqual(task.name, "put") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test11(self): task = self.cmd_builder.get_task( "Please place the black bottom cover on top of the white cover") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "black bottom cover") self.assertEqual(task.name, "place") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test12(self): task = self.cmd_builder.get_task( "Please place the white cover on top of the black bottom cover") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "white cover") self.assertEqual(task.name, "place") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test13(self): task = self.cmd_builder.get_task( "Hey robot, could you destroy the car for me?") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "car") self.assertEqual(task.name, "destroy") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test14(self): task = self.cmd_builder.get_task( "Hey robot, could you fly to the moon?") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "moon") self.assertEqual(task.name, "fly") def test_get_task__sentence_with_pick_up_task__returns_pick_up_test15(self): task = self.cmd_builder.get_task( "Hey robot, could you turn the tool?") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual(task.objects_to_execute_on[0].name, "tool") self.assertEqual(task.name, "turn")
def setUp(self): self.ner_mock = Mock() self.cmd_builder = CommandBuilder(self.ner_mock)
class CommandBuilderTestCase(unittest.TestCase): def setUp(self): self.ner_mock = Mock() self.cmd_builder = CommandBuilder(self.ner_mock) def test_get_task__entities_with_pick_up_task__returns_pick_up_task_and_locations(self): entities = [ (EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "next"), (EntityType.COLOUR, "black"), (EntityType.OBJECT, "bottom cover"), (EntityType.LOCATION, "above"), (EntityType.OBJECT, "bottom cover") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual("blue cover", task.objects_to_execute_on[0].name) self.assertEqual(2, len(task.objects_to_execute_on[0].spatial_descriptions)) self.assertEqual(SpatialType.NEXT_TO, task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type) self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity) self.assertEqual("black bottom cover", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name) self.assertEqual(SpatialType.ABOVE, task.objects_to_execute_on[0].spatial_descriptions[1].spatial_type) self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[1].object_entity) self.assertEqual("bottom cover", task.objects_to_execute_on[0].spatial_descriptions[1].object_entity.name) def test_get_task__entities_with_task__returns_task_with_name(self): entities = [ (EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertEqual("pick up", task.name) def test_get_task__entities_with_pick_up_task__returns_pick_up_task_with_two_objects(self): entities = [ (EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.COLOUR, "black"), (EntityType.OBJECT, "bottom cover") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertEqual("blue cover", task.objects_to_execute_on[0].name) self.assertEqual("black bottom cover", task.objects_to_execute_on[1].name) def test_get_task__entities_with_pick_up_task__returns_pick_up_task_with_multiple_objects(self): entities = [ (EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.COLOUR, "green"), (EntityType.OBJECT, "fuse"), (EntityType.COLOUR, "black"), (EntityType.OBJECT, "bottom cover"), (EntityType.COLOUR, "red"), (EntityType.OBJECT, "fuse") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertEqual("blue cover", task.objects_to_execute_on[0].name) self.assertEqual("green fuse", task.objects_to_execute_on[1].name) self.assertEqual("black bottom cover", task.objects_to_execute_on[2].name) self.assertEqual("red fuse", task.objects_to_execute_on[3].name) def test_get_task__entities_with_pick_up_task__returns_pick_up_task_with_multiple_objects_with_spatial_description(self): entities = [ (EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "next"), (EntityType.COLOUR, "green"), (EntityType.OBJECT, "fuse"), (EntityType.COLOUR, "black"), (EntityType.OBJECT, "bottom cover"), (EntityType.COLOUR, "red"), (EntityType.OBJECT, "fuse") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertEqual("blue cover", task.objects_to_execute_on[0].name) self.assertEqual("green fuse", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name) self.assertEqual("black bottom cover", task.objects_to_execute_on[1].name) self.assertEqual("red fuse", task.objects_to_execute_on[2].name) def test_get_task__entities_with_pick_up_task__returns_pick_up_task_with_multiple_objects_with_multiple_spatial_descriptions(self): entities = [ (EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "next"), (EntityType.COLOUR, "green"), (EntityType.OBJECT, "fuse"), (EntityType.COLOUR, "black"), (EntityType.OBJECT, "bottom cover"), (EntityType.LOCATION, "next"), (EntityType.COLOUR, "red"), (EntityType.OBJECT, "fuse") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertEqual("blue cover", task.objects_to_execute_on[0].name) self.assertEqual("green fuse", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name) self.assertEqual("black bottom cover", task.objects_to_execute_on[1].name) self.assertEqual("red fuse", task.objects_to_execute_on[1].spatial_descriptions[0].object_entity.name) def test_get_task__entities_with_pick_up_task_with_no_spatial_relations__returns_pick_up_task(self): entities = [ (EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual("blue cover", task.objects_to_execute_on[0].name) self.assertEqual(0, len(task.objects_to_execute_on[0].spatial_descriptions)) def test_get_task__entities_with_find_task__returns_find_task_and_locations(self): entities = [ (EntityType.TASK, "find"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "next"), (EntityType.COLOUR, "yellow"), (EntityType.OBJECT, "bottom cover") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertIsNotNone(task.objects_to_execute_on[0]) self.assertEqual("blue cover", task.objects_to_execute_on[0].name) self.assertEqual(1, len(task.objects_to_execute_on[0].spatial_descriptions)) self.assertEqual(SpatialType.NEXT_TO, task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type) self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity) self.assertEqual("yellow bottom cover", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name) def test_get_task__entites_with_static_location__returns_task_with_correct_spatial_type(self): entities = [ (EntityType.TASK, "place"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "top left corner"), (EntityType.OBJECT, "table") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertEqual(SpatialType.OTHER, task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type) def test_get_task__entites_with_static_location__returns_task_with_correct_spatial_name(self): entities = [ (EntityType.TASK, "place"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "top left corner"), (EntityType.OBJECT, "table") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertEqual("top left corner", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name) def test_get_task__entites_with_relative_and_static_location__returns_task_with_correct_spatial_types(self): entities = [ (EntityType.TASK, "place"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "next"), (EntityType.COLOUR, "white"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "top left corner"), (EntityType.OBJECT, "table") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertEqual(SpatialType.NEXT_TO, task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type) self.assertEqual(SpatialType.OTHER, task.objects_to_execute_on[0].spatial_descriptions[1].spatial_type) def test_get_task__entites_with_relative_and_static_location__returns_task_with_correct_spatial_names(self): entities = [ (EntityType.TASK, "place"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "next"), (EntityType.COLOUR, "white"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "top left corner"), (EntityType.OBJECT, "table") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertEqual("white cover", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name) self.assertEqual("top left corner", task.objects_to_execute_on[0].spatial_descriptions[1].object_entity.name) def test_get_task_entities__no_main_object__returns_object_with_spatial_descriptions(self): # example Place it in the top left corner entities = [ (EntityType.TASK, "place"), (EntityType.LOCATION, "top left corner"), (EntityType.OBJECT, "table") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") self.assertEqual("", task.objects_to_execute_on[0].name) self.assertEqual("top left corner", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name) def test_add_entities_to_task__objects_in_entity_list__adds_object_to_task(self): entities = [ (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover") ] self.ner_mock.get_entities = Mock(return_value=entities) task = Task("pick") task.task_type = TaskType.PICK self.cmd_builder.add_entities_to_task(task, "Dummy sentence") self.assertEqual("blue cover", task.objects_to_execute_on[0].name) def test_add_entities_to_task__sub_task_in_entity_list__adds_subtask_to_task(self): entities = [ (EntityType.TASK, "place"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "next"), (EntityType.COLOUR, "white"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "top left corner"), (EntityType.OBJECT, "table") ] self.ner_mock.get_entities = Mock(return_value=entities) task = Task("pick") task.task_type = TaskType.PICK self.cmd_builder.add_entities_to_task(task, "Dummy sentence") self.assertEqual("place", task.child_tasks[0].name) self.assertEqual("white cover", task.child_tasks[0].objects_to_execute_on[0].spatial_descriptions[0].object_entity.name) self.assertEqual("top left corner", task.child_tasks[0].objects_to_execute_on[0].spatial_descriptions[1].object_entity.name)
class SpatialModuleTwoOfEach(unittest.TestCase): def setUp(self): self.ner_mock = Mock() self.db_mock = Mock() self.spatial = SpatialRelation(database_handler=self.db_mock) self.cmd_builder = CommandBuilder(self.ner_mock) self.objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000 (0, "blue cover", [100, 400, 100, 300]), (1, "blue cover", [700, 1100, 100, 300]), (2, "fuse", [100, 400, 800, 1000]), (3, "bottom cover", [700, 1100, 800, 1000]), (4, "white cover", [100, 400, 1500, 1600]), (5, "white cover", [700, 700, 1500, 1600]), (6, "blue cover", [0, 100, 1550, 1600]) ] def test_above(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "above"), (EntityType.OBJECT, "bottom cover")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[1][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_right(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "right"), (EntityType.OBJECT, "fuse")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[1][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_left(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "white"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "left"), (EntityType.OBJECT, "bottom cover")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[4][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_below(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "white"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "below"), (EntityType.OBJECT, "fuse")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[4][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_diagonalRightUp(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "right"), (EntityType.OBJECT, "fuse")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[1][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_serial(self): entities = [ (EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "right"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "above"), (EntityType.OBJECT, "fuse"), (EntityType.LOCATION, "left"), (EntityType.OBJECT, "bottom cover") ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[1][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_twoOfReference(self): self.skipTest("Contradicts twoOfReference test") entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "above"), (EntityType.COLOUR, "white"), (EntityType.OBJECT, "cover")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] actual_indices, status = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(StatusEnum.NO_VALID_OBJECTS, status) def test_locate_specific_object__two_valid_results__returns_list_with_correct_length( self): #self.skipTest("Contradicts twoOfReference test") entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "above"), (EntityType.COLOUR, "white"), (EntityType.OBJECT, "cover")] objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000 (0, "white cover", [600, 800, 600, 700]), (1, "blue cover", [400, 600, 500, 550]), (2, "blue cover", [600, 800, 800, 900]), (3, "bottom cover", [700, 1100, 800, 1000]), (4, "blue cover", [0, 100, 1550, 1600]), (5, "white cover", [600, 800, 1000, 1100]) ] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] self.assertEqual( 2, len(self.spatial.locate_specific_object(object_entity, objects))) def test_locate_specific_object__static_location__returns_closest_object( self): object_entity = ObjectEntity(name="blue cover") spatial_description = SpatialDescription( spatial_type=SpatialType.OTHER) spatial_description.object_entity.name = "top left corner" object_entity.spatial_descriptions.append(spatial_description) self.db_mock.get_location_by_name = Mock(return_value=(100, 100, 0)) expected_index = self.objects[0][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_locate_specific_object__static_location_2__returns_closest_object( self): object_entity = ObjectEntity(name="blue cover") spatial_description = SpatialDescription( spatial_type=SpatialType.OTHER) spatial_description.object_entity.name = "top left corner" object_entity.spatial_descriptions.append(spatial_description) self.db_mock.get_location_by_name = Mock(return_value=(700, 100, 0)) expected_index = self.objects[1][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_locate_specific_object__relative_to_object_at_static_location__returns_closest_object( self): object_entity = ObjectEntity(name="blue cover") spatial_description = SpatialDescription( spatial_type=SpatialType.BELOW) spatial_description.object_entity.name = "white cover" spatial_description_2 = SpatialDescription( spatial_type=SpatialType.OTHER) spatial_description_2.object_entity.name = "bottom left corner" object_entity.spatial_descriptions.append(spatial_description) object_entity.spatial_descriptions.append(spatial_description_2) self.db_mock.get_location_by_name = Mock(return_value=(100, 1500, 0)) expected_index = self.objects[6][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_get_location__static_location__returns_static_location(self): spatial_description = SpatialDescription( spatial_type=SpatialType.OTHER) spatial_description.object_entity.name = "top left corner" spatial_descriptions = [spatial_description] expected_x, expected_y = 100, 100 self.db_mock.get_location_by_name = Mock(return_value=(expected_x, expected_y, 0)) locations, status = self.spatial.get_location(spatial_descriptions, self.objects) x, y = locations[0] self.assertEqual(expected_x, x) self.assertEqual(expected_y, y) def test_get_location__object_at_static_location__returns_error(self): objects = [ # copy to make sure setUp changes dont mess up this test (0, "blue cover", [100, 400, 100, 300]), (1, "blue cover", [700, 1100, 100, 300]), (2, "fuse", [100, 400, 800, 1000]), (3, "bottom cover", [700, 1100, 800, 1000]), (4, "white cover", [100, 400, 1500, 1600]), (5, "white cover", [700, 700, 1500, 1600]), (6, "blue cover", [0, 100, 1550, 1600]) ] spatial_description = SpatialDescription( spatial_type=SpatialType.NEXT_TO) spatial_description.object_entity.name = "white cover" spatial_description_2 = SpatialDescription( spatial_type=SpatialType.OTHER) spatial_description_2.object_entity.name = "top left corner" spatial_descriptions = [] spatial_descriptions.append(spatial_description) spatial_descriptions.append(spatial_description_2) expected_bbox = objects[4][2] expected_x, expected_y = expected_bbox[1] - ( expected_bbox[1] - expected_bbox[0]) / 2, expected_bbox[3] - ( expected_bbox[3] - expected_bbox[2]) / 2 self.db_mock.get_location_by_name = Mock(return_value=(100, 100, 0)) locations, status = self.spatial.get_location(spatial_descriptions, objects) self.assertEqual(StatusEnum.NO_VALID_OBJECTS, status) def test_get_location__object_at_static_location__return_object_center( self): objects = [ # copy to make sure setUp changes dont mess up this test (0, "blue cover", [100, 400, 100, 300]), (1, "blue cover", [700, 1100, 100, 300]), (2, "fuse", [100, 400, 800, 1000]), (3, "bottom cover", [700, 1100, 800, 1000]), (4, "white cover", [100, 400, 1500, 1600]), (5, "white cover", [700, 700, 1500, 1600]), (6, "blue cover", [0, 100, 1550, 1600]) ] spatial_description = SpatialDescription( spatial_type=SpatialType.NEXT_TO) spatial_description.object_entity.name = "white cover" spatial_description_2 = SpatialDescription( spatial_type=SpatialType.OTHER) spatial_description_2.object_entity.name = "bottom left corner" spatial_descriptions = [] spatial_descriptions.append(spatial_description) spatial_descriptions.append(spatial_description_2) expected_bbox = objects[4][2] expected_x, expected_y = expected_bbox[1] - ( expected_bbox[1] - expected_bbox[0]) / 2, expected_bbox[3] - ( expected_bbox[3] - expected_bbox[2]) / 2 self.db_mock.get_location_by_name = Mock(return_value=(100, 1500, 0)) locations, status = self.spatial.get_location(spatial_descriptions, objects) x, y = locations[0] self.assertAlmostEqual(expected_x, x) self.assertAlmostEqual(expected_y, y)
class SpatialModuleOneOfEach(unittest.TestCase): def setUp(self): self.ner_mock = Mock() self.db_mock = Mock() self.spatial = SpatialRelation(database_handler=self.db_mock) self.cmd_builder = CommandBuilder(self.ner_mock) self.objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000 (0, "black cover", [100, 400, 100, 300]), (1, "blue cover", [700, 1100, 100, 300]), (2, "fuse", [100, 400, 800, 1000]), (3, "bottom cover", [700, 1100, 800, 1000]), (4, "white cover", [100, 400, 1500, 1600]), (5, "green cover", [700, 700, 1500, 1600]) ] def test_above(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "above"), (EntityType.OBJECT, "bottom cover")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[1][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_right(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "right"), (EntityType.COLOUR, "black"), (EntityType.OBJECT, "cover")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[1][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_left(self): entities = [(EntityType.TASK, "pick up"), (EntityType.OBJECT, "fuse"), (EntityType.LOCATION, "left"), (EntityType.OBJECT, "bottom cover")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[2][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_below(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "white"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "below"), (EntityType.OBJECT, "fuse")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[4][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_diagonalRightUp1(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "above"), (EntityType.OBJECT, "fuse")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[1][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_diagonalRightUp2(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "right"), (EntityType.OBJECT, "fuse")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[1][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_serial(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "black"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "left"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "above"), (EntityType.OBJECT, "bottom cover"), (EntityType.LOCATION, "right"), (EntityType.OBJECT, "fuse")] self.ner_mock.get_entities = Mock(return_value=entities) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] expected_index = self.objects[0][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0]) def test_locate_specific_object__static_location__returns_closest_object( self): object_entity = ObjectEntity(name="blue cover") spatial_description = SpatialDescription( spatial_type=SpatialType.OTHER) spatial_description.object_entity.name = "top left corner" object_entity.spatial_descriptions.append(spatial_description) self.db_mock.get_location_by_name = Mock(return_value=(100, 100, 0)) expected_index = self.objects[1][0] actual_indices, success_enum = self.spatial.locate_specific_object( object_entity, self.objects) self.assertEqual(expected_index, actual_indices[0])
class FindObjectIsolatedTest(unittest.TestCase): def setUp(self): self.ner_mock = Mock() self.db_mock = Mock() self.spatial_mock = Mock() self.vision_mock = Mock() self.returned = GroundingReturn() self.cmd_builder = CommandBuilder(self.ner_mock) self.grounding = Grounding(db=self.db_mock, vision_controller=self.vision_mock, spatial=self.spatial_mock) self.objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000 (0, "black cover", [100, 400, 100, 300]), (1, "blue cover", [700, 1100, 100, 300]), (2, "fuse", [100, 400, 800, 1000]), (3, "bottom cover", [700, 1100, 800, 1000]), (4, "white cover", [100, 400, 1500, 1600]), (5, "blue cover", [700, 700, 1500, 1600]) ] def test_failFindObject(self): object_info = ObjectInfoWithFeatures() features = [] object_info.features = np.array([1, 1, 1, 1, 1]) object_info.bbox_xxyy = np.array([1, 2, 3, 4]) object_info.mask_full = np.array([4, 3, 2, 1]) object_info.mask_cropped = np.array([5, 5, 5, 5]) features.append(object_info) self.vision_mock.get_masks_with_features = Mock(return_value=features) self.db_mock.get_feature = Mock(return_value=np.array([5, 5, 5, 5, 5])) object_name = "black cover" object_spatial_desc = None object_entity = ObjectEntity() object_entity.name = object_name object_entity.spatial_descriptions = object_spatial_desc returned = self.grounding.find_object(object_entity) self.assertEqual(returned.error_code, GroundingErrorType.CANT_FIND) def test_bestMatch(self): indexes = [1, 2, 3, 4, 5] distances = [0.78, 2, 1.5, 0.23, 0.1] best_index = self.grounding.find_best_match(indexes, distances) self.assertEqual(5, best_index) def test_sameObject(self): features_1 = np.array([0.35, 0.88, 0.20, 0.30]) features_2 = np.array([0.30, 0.85, 0.17, 0.32]) self.assertTrue(self.grounding.is_same_object(features_1, features_2)) def test_unknownObject(self): object_info = ObjectInfoWithFeatures() features = [] object_info.features = np.array([1, 1, 1, 1, 1]) object_info.bbox_xxyy = np.array([1, 2, 3, 4]) object_info.mask_full = np.array([4, 3, 2, 1]) object_info.mask_cropped = np.array([5, 5, 5, 5]) features.append(object_info) self.vision_mock.get_masks_with_features = Mock(return_value=features) self.db_mock.get_feature = Mock(return_value=None) object_name = "albert is cool" object_spatial_desc = None object_entity = ObjectEntity() object_entity.name = object_name object_entity.spatial_descriptions = object_spatial_desc returned = self.grounding.find_object(object_entity) self.assertEqual(returned.error_code, GroundingErrorType.UNKNOWN ) # Should return false if the object is unknown def test_spatialPart(self): entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"), (EntityType.LOCATION, "above"), (EntityType.OBJECT, "bottom cover")] features = [] feature = [ np.array([1, 1, 1, 1, 1]), np.array([1, 1, 1, 1, 1]), np.array([1, 1, 1, 1, 1]) ] bbox = [ np.array([1, 2, 3, 4]), np.array([1, 2, 3, 4]), np.array([1, 2, 3, 4]) ] mask = [ np.array([4, 3, 2, 1]), np.array([4, 3, 2, 1]), np.array([4, 3, 2, 1]) ] cropped_rbg = [ np.array([5, 5, 5, 5]), np.array([5, 5, 5, 5]), np.array([5, 5, 5, 5]) ] for i in range(3): object_info = ObjectInfoWithFeatures() object_info.features = feature[i] object_info.bbox_xxyy = bbox[i] object_info.mask_full = mask[i] object_info.mask_cropped = cropped_rbg[i] features.append(object_info) db_features = [("dummy name", np.array([1, 1, 1, 1, 1])), ("dummy name", np.array([1, 1, 1, 1, 1])), ("dummy name", np.array([1, 1, 1, 1, 1]))] self.vision_mock.get_masks_with_features = Mock(return_value=features) self.db_mock.get_feature = Mock(return_value=np.array([1, 1, 1, 1, 1])) self.db_mock.get_all_features = Mock(return_value=db_features) self.ner_mock.get_entities = Mock(return_value=entities) self.spatial_mock.locate_specific_object = Mock( return_value=([1], StatusEnum.SUCCESS)) task = self.cmd_builder.get_task("Dummy sentence") object_entity = task.objects_to_execute_on[0] returned = self.grounding.find_object(object_entity) self.assertTrue(returned.is_success) self.assertIsNotNone(returned.object_infos)
def __init__(self, model_path, tags_path): self.cmd_builder = CommandBuilder(model_path, tags_path)