def test_ner(model_path, tag_path):
ner = NER(model_path, tag_path)
cmd_builder = CommandBuilder(ner)
while True:
print("Input sentence to test:")
text = input("> ")
task = cmd_builder.get_task(text)
print(task)
def setUp(self):
self.ner_mock = Mock()
self.db_mock = Mock()
self.spatial = SpatialRelation(database_handler=self.db_mock)
self.cmd_builder = CommandBuilder(self.ner_mock)
self.objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000
(0, "black cover", [100, 400, 100, 300]),
(1, "blue cover", [700, 1100, 100, 300]),
(2, "fuse", [100, 400, 800, 1000]),
(3, "bottom cover", [700, 1100, 800, 1000]),
(4, "white cover", [100, 400, 1500, 1600]),
(5, "green cover", [700, 700, 1500, 1600])
]
def setUp(self):
self.ner_mock = Mock()
self.db_mock = Mock()
self.spatial_mock = Mock()
self.vision_mock = Mock()
self.returned = GroundingReturn()
self.cmd_builder = CommandBuilder(self.ner_mock)
self.grounding = Grounding(db=self.db_mock,
vision_controller=self.vision_mock,
spatial=self.spatial_mock)
self.objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000
(0, "black cover", [100, 400, 100, 300]),
(1, "blue cover", [700, 1100, 100, 300]),
(2, "fuse", [100, 400, 800, 1000]),
(3, "bottom cover", [700, 1100, 800, 1000]),
(4, "white cover", [100, 400, 1500, 1600]),
(5, "blue cover", [700, 700, 1500, 1600])
]
class NERService:
def __init__(self, model_path, tags_path):
self.cmd_builder = CommandBuilder(model_path, tags_path)
def handle_ner_request(self, request: NERRequest):
task = self.cmd_builder.get_task(request.sentence)
ros_task = create_ros_task(task)
ros_outer_task = OuterTask(type=ros_task.type,
object1=ros_task.object1,
object2=ros_task.object2,
child_tasks=[])
if len(task.child_tasks) > 0:
for child_task in task.child_tasks:
ros_task = create_ros_task(child_task)
ros_outer_task.child_tasks.append(ros_task)
return NERResponse(task=ros_outer_task)
def ner_server(self):
rospy.init_node("ner_server")
s = rospy.Service("ner", NER, self.handle_ner_request)
rospy.spin()
def setUp(self):
self.ner = NER("ner_model.bin", "../scripts/NER_4/tags.txt")
self.cmd_builder = CommandBuilder(self.ner)
class NERIntegrationTestCase(unittest.TestCase):
def setUp(self):
self.ner = NER("ner_model.bin", "../scripts/NER_4/tags.txt")
self.cmd_builder = CommandBuilder(self.ner)
def test_get_task__sentence_with_pick_up_task__returns_pick_up_task_and_locations(self):
task = self.cmd_builder.get_task("Please pick up the blue cover that is next to the black bottom cover which is above a bottom cover")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "blue cover")
self.assertEqual(len(task.objects_to_execute_on[0].spatial_descriptions), 2)
self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type, SpatialType.NEXT_TO)
self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity)
self.assertEqual("black bottom cover", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name)
self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[1].spatial_type, SpatialType.ABOVE)
self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[1].object_entity)
self.assertEqual("bottom cover", task.objects_to_execute_on[0].spatial_descriptions[1].object_entity.name)
def test_get_task__sentence_with_find_task__returns_pick_up_task_and_locations(self):
task = self.cmd_builder.get_task("Please find the blue cover that is next to the yellow bottom cover")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "blue cover")
self.assertEqual(len(task.objects_to_execute_on[0].spatial_descriptions), 1)
self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type, SpatialType.NEXT_TO)
self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity)
self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name, "yellow bottom cover")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_task_and_locations(self):
task = self.cmd_builder.get_task(
"Please pick up the blue cover that is next to the black bottom cover which is above a bottom cover")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "blue cover")
self.assertEqual(len(task.objects_to_execute_on[0].spatial_descriptions), 2)
self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type, SpatialType.NEXT_TO)
self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity)
self.assertEqual("black bottom cover",
task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name)
self.assertEqual(task.objects_to_execute_on[0].spatial_descriptions[1].spatial_type, SpatialType.ABOVE)
self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[1].object_entity)
self.assertEqual("bottom cover", task.objects_to_execute_on[0].spatial_descriptions[1].object_entity.name)
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test1(self):
task = self.cmd_builder.get_task(
"Please pick up the black bottom cover")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "black bottom cover")
self.assertTrue(task.name == "pick up")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test2(self):
task = self.cmd_builder.get_task(
"Please pick up the screwdriver")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "screwdriver")
self.assertEqual(task.name,"get")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test3(self):
task = self.cmd_builder.get_task(
"Please move the hammer")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "hammer")
self.assertEqual(task.name,"move")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test4(self):
task = self.cmd_builder.get_task(
"Please pick up the fuse next to the book")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "fuse")
self.assertEqual(task.name, "pick up")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test5(self):
task = self.cmd_builder.get_task(
"Please pick up the green cover on top of the table")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "green cover")
self.assertEqual(task.name, "pick up")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test6(self):
task = self.cmd_builder.get_task(
"Please place the blue cover on top of the black bottom cover")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "blue cover")
self.assertEqual(task.name, "place")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test7(self):
task = self.cmd_builder.get_task(
"Please find the white bottom cover")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "white bottom cover")
self.assertEqual(task.name, "find")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test8(self):
task = self.cmd_builder.get_task(
"Please find the fuse")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "fuse")
self.assertEqual(task.name, "find")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test9(self):
task = self.cmd_builder.get_task(
"Please weld the cover")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "cover")
self.assertEqual(task.name, "weld")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test10(self):
task = self.cmd_builder.get_task(
"Please put the fuse on top of the white cover")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "fuse")
self.assertEqual(task.name, "put")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test11(self):
task = self.cmd_builder.get_task(
"Please place the black bottom cover on top of the white cover")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "black bottom cover")
self.assertEqual(task.name, "place")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test12(self):
task = self.cmd_builder.get_task(
"Please place the white cover on top of the black bottom cover")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "white cover")
self.assertEqual(task.name, "place")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test13(self):
task = self.cmd_builder.get_task(
"Hey robot, could you destroy the car for me?")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "car")
self.assertEqual(task.name, "destroy")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test14(self):
task = self.cmd_builder.get_task(
"Hey robot, could you fly to the moon?")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "moon")
self.assertEqual(task.name, "fly")
def test_get_task__sentence_with_pick_up_task__returns_pick_up_test15(self):
task = self.cmd_builder.get_task(
"Hey robot, could you turn the tool?")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual(task.objects_to_execute_on[0].name, "tool")
self.assertEqual(task.name, "turn")
def setUp(self):
self.ner_mock = Mock()
self.cmd_builder = CommandBuilder(self.ner_mock)
class CommandBuilderTestCase(unittest.TestCase):
def setUp(self):
self.ner_mock = Mock()
self.cmd_builder = CommandBuilder(self.ner_mock)
def test_get_task__entities_with_pick_up_task__returns_pick_up_task_and_locations(self):
entities = [
(EntityType.TASK, "pick up"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "next"),
(EntityType.COLOUR, "black"),
(EntityType.OBJECT, "bottom cover"),
(EntityType.LOCATION, "above"),
(EntityType.OBJECT, "bottom cover")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual("blue cover", task.objects_to_execute_on[0].name)
self.assertEqual(2, len(task.objects_to_execute_on[0].spatial_descriptions))
self.assertEqual(SpatialType.NEXT_TO, task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type)
self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity)
self.assertEqual("black bottom cover", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name)
self.assertEqual(SpatialType.ABOVE, task.objects_to_execute_on[0].spatial_descriptions[1].spatial_type)
self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[1].object_entity)
self.assertEqual("bottom cover", task.objects_to_execute_on[0].spatial_descriptions[1].object_entity.name)
def test_get_task__entities_with_task__returns_task_with_name(self):
entities = [
(EntityType.TASK, "pick up"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertEqual("pick up", task.name)
def test_get_task__entities_with_pick_up_task__returns_pick_up_task_with_two_objects(self):
entities = [
(EntityType.TASK, "pick up"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.COLOUR, "black"),
(EntityType.OBJECT, "bottom cover")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertEqual("blue cover", task.objects_to_execute_on[0].name)
self.assertEqual("black bottom cover", task.objects_to_execute_on[1].name)
def test_get_task__entities_with_pick_up_task__returns_pick_up_task_with_multiple_objects(self):
entities = [
(EntityType.TASK, "pick up"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.COLOUR, "green"),
(EntityType.OBJECT, "fuse"),
(EntityType.COLOUR, "black"),
(EntityType.OBJECT, "bottom cover"),
(EntityType.COLOUR, "red"),
(EntityType.OBJECT, "fuse")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertEqual("blue cover", task.objects_to_execute_on[0].name)
self.assertEqual("green fuse", task.objects_to_execute_on[1].name)
self.assertEqual("black bottom cover", task.objects_to_execute_on[2].name)
self.assertEqual("red fuse", task.objects_to_execute_on[3].name)
def test_get_task__entities_with_pick_up_task__returns_pick_up_task_with_multiple_objects_with_spatial_description(self):
entities = [
(EntityType.TASK, "pick up"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "next"),
(EntityType.COLOUR, "green"),
(EntityType.OBJECT, "fuse"),
(EntityType.COLOUR, "black"),
(EntityType.OBJECT, "bottom cover"),
(EntityType.COLOUR, "red"),
(EntityType.OBJECT, "fuse")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertEqual("blue cover", task.objects_to_execute_on[0].name)
self.assertEqual("green fuse", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name)
self.assertEqual("black bottom cover", task.objects_to_execute_on[1].name)
self.assertEqual("red fuse", task.objects_to_execute_on[2].name)
def test_get_task__entities_with_pick_up_task__returns_pick_up_task_with_multiple_objects_with_multiple_spatial_descriptions(self):
entities = [
(EntityType.TASK, "pick up"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "next"),
(EntityType.COLOUR, "green"),
(EntityType.OBJECT, "fuse"),
(EntityType.COLOUR, "black"),
(EntityType.OBJECT, "bottom cover"),
(EntityType.LOCATION, "next"),
(EntityType.COLOUR, "red"),
(EntityType.OBJECT, "fuse")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertEqual("blue cover", task.objects_to_execute_on[0].name)
self.assertEqual("green fuse", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name)
self.assertEqual("black bottom cover", task.objects_to_execute_on[1].name)
self.assertEqual("red fuse", task.objects_to_execute_on[1].spatial_descriptions[0].object_entity.name)
def test_get_task__entities_with_pick_up_task_with_no_spatial_relations__returns_pick_up_task(self):
entities = [
(EntityType.TASK, "pick up"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual("blue cover", task.objects_to_execute_on[0].name)
self.assertEqual(0, len(task.objects_to_execute_on[0].spatial_descriptions))
def test_get_task__entities_with_find_task__returns_find_task_and_locations(self):
entities = [
(EntityType.TASK, "find"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "next"),
(EntityType.COLOUR, "yellow"),
(EntityType.OBJECT, "bottom cover")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertIsNotNone(task.objects_to_execute_on[0])
self.assertEqual("blue cover", task.objects_to_execute_on[0].name)
self.assertEqual(1, len(task.objects_to_execute_on[0].spatial_descriptions))
self.assertEqual(SpatialType.NEXT_TO, task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type)
self.assertIsNotNone(task.objects_to_execute_on[0].spatial_descriptions[0].object_entity)
self.assertEqual("yellow bottom cover", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name)
def test_get_task__entites_with_static_location__returns_task_with_correct_spatial_type(self):
entities = [
(EntityType.TASK, "place"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "top left corner"),
(EntityType.OBJECT, "table")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertEqual(SpatialType.OTHER, task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type)
def test_get_task__entites_with_static_location__returns_task_with_correct_spatial_name(self):
entities = [
(EntityType.TASK, "place"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "top left corner"),
(EntityType.OBJECT, "table")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertEqual("top left corner", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name)
def test_get_task__entites_with_relative_and_static_location__returns_task_with_correct_spatial_types(self):
entities = [
(EntityType.TASK, "place"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "next"),
(EntityType.COLOUR, "white"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "top left corner"),
(EntityType.OBJECT, "table")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertEqual(SpatialType.NEXT_TO, task.objects_to_execute_on[0].spatial_descriptions[0].spatial_type)
self.assertEqual(SpatialType.OTHER, task.objects_to_execute_on[0].spatial_descriptions[1].spatial_type)
def test_get_task__entites_with_relative_and_static_location__returns_task_with_correct_spatial_names(self):
entities = [
(EntityType.TASK, "place"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "next"),
(EntityType.COLOUR, "white"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "top left corner"),
(EntityType.OBJECT, "table")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertEqual("white cover", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name)
self.assertEqual("top left corner", task.objects_to_execute_on[0].spatial_descriptions[1].object_entity.name)
def test_get_task_entities__no_main_object__returns_object_with_spatial_descriptions(self):
# example Place it in the top left corner
entities = [
(EntityType.TASK, "place"),
(EntityType.LOCATION, "top left corner"),
(EntityType.OBJECT, "table")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
self.assertEqual("", task.objects_to_execute_on[0].name)
self.assertEqual("top left corner", task.objects_to_execute_on[0].spatial_descriptions[0].object_entity.name)
def test_add_entities_to_task__objects_in_entity_list__adds_object_to_task(self):
entities = [
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = Task("pick")
task.task_type = TaskType.PICK
self.cmd_builder.add_entities_to_task(task, "Dummy sentence")
self.assertEqual("blue cover", task.objects_to_execute_on[0].name)
def test_add_entities_to_task__sub_task_in_entity_list__adds_subtask_to_task(self):
entities = [
(EntityType.TASK, "place"),
(EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "next"),
(EntityType.COLOUR, "white"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "top left corner"),
(EntityType.OBJECT, "table")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = Task("pick")
task.task_type = TaskType.PICK
self.cmd_builder.add_entities_to_task(task, "Dummy sentence")
self.assertEqual("place", task.child_tasks[0].name)
self.assertEqual("white cover", task.child_tasks[0].objects_to_execute_on[0].spatial_descriptions[0].object_entity.name)
self.assertEqual("top left corner", task.child_tasks[0].objects_to_execute_on[0].spatial_descriptions[1].object_entity.name)
class SpatialModuleTwoOfEach(unittest.TestCase):
def setUp(self):
self.ner_mock = Mock()
self.db_mock = Mock()
self.spatial = SpatialRelation(database_handler=self.db_mock)
self.cmd_builder = CommandBuilder(self.ner_mock)
self.objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000
(0, "blue cover", [100, 400, 100, 300]),
(1, "blue cover", [700, 1100, 100, 300]),
(2, "fuse", [100, 400, 800, 1000]),
(3, "bottom cover", [700, 1100, 800, 1000]),
(4, "white cover", [100, 400, 1500, 1600]),
(5, "white cover", [700, 700, 1500, 1600]),
(6, "blue cover", [0, 100, 1550, 1600])
]
def test_above(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "above"),
(EntityType.OBJECT, "bottom cover")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[1][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_right(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "right"),
(EntityType.OBJECT, "fuse")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[1][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_left(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "white"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "left"),
(EntityType.OBJECT, "bottom cover")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[4][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_below(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "white"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "below"),
(EntityType.OBJECT, "fuse")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[4][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_diagonalRightUp(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "right"),
(EntityType.OBJECT, "fuse")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[1][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_serial(self):
entities = [
(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"), (EntityType.LOCATION, "right"),
(EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "above"), (EntityType.OBJECT, "fuse"),
(EntityType.LOCATION, "left"), (EntityType.OBJECT, "bottom cover")
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[1][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_twoOfReference(self):
self.skipTest("Contradicts twoOfReference test")
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "above"),
(EntityType.COLOUR, "white"), (EntityType.OBJECT, "cover")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
actual_indices, status = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(StatusEnum.NO_VALID_OBJECTS, status)
def test_locate_specific_object__two_valid_results__returns_list_with_correct_length(
self):
#self.skipTest("Contradicts twoOfReference test")
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "above"),
(EntityType.COLOUR, "white"), (EntityType.OBJECT, "cover")]
objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000
(0, "white cover", [600, 800, 600, 700]),
(1, "blue cover", [400, 600, 500, 550]),
(2, "blue cover", [600, 800, 800, 900]),
(3, "bottom cover", [700, 1100, 800, 1000]),
(4, "blue cover", [0, 100, 1550, 1600]),
(5, "white cover", [600, 800, 1000, 1100])
]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
self.assertEqual(
2, len(self.spatial.locate_specific_object(object_entity,
objects)))
def test_locate_specific_object__static_location__returns_closest_object(
self):
object_entity = ObjectEntity(name="blue cover")
spatial_description = SpatialDescription(
spatial_type=SpatialType.OTHER)
spatial_description.object_entity.name = "top left corner"
object_entity.spatial_descriptions.append(spatial_description)
self.db_mock.get_location_by_name = Mock(return_value=(100, 100, 0))
expected_index = self.objects[0][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_locate_specific_object__static_location_2__returns_closest_object(
self):
object_entity = ObjectEntity(name="blue cover")
spatial_description = SpatialDescription(
spatial_type=SpatialType.OTHER)
spatial_description.object_entity.name = "top left corner"
object_entity.spatial_descriptions.append(spatial_description)
self.db_mock.get_location_by_name = Mock(return_value=(700, 100, 0))
expected_index = self.objects[1][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_locate_specific_object__relative_to_object_at_static_location__returns_closest_object(
self):
object_entity = ObjectEntity(name="blue cover")
spatial_description = SpatialDescription(
spatial_type=SpatialType.BELOW)
spatial_description.object_entity.name = "white cover"
spatial_description_2 = SpatialDescription(
spatial_type=SpatialType.OTHER)
spatial_description_2.object_entity.name = "bottom left corner"
object_entity.spatial_descriptions.append(spatial_description)
object_entity.spatial_descriptions.append(spatial_description_2)
self.db_mock.get_location_by_name = Mock(return_value=(100, 1500, 0))
expected_index = self.objects[6][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_get_location__static_location__returns_static_location(self):
spatial_description = SpatialDescription(
spatial_type=SpatialType.OTHER)
spatial_description.object_entity.name = "top left corner"
spatial_descriptions = [spatial_description]
expected_x, expected_y = 100, 100
self.db_mock.get_location_by_name = Mock(return_value=(expected_x,
expected_y, 0))
locations, status = self.spatial.get_location(spatial_descriptions,
self.objects)
x, y = locations[0]
self.assertEqual(expected_x, x)
self.assertEqual(expected_y, y)
def test_get_location__object_at_static_location__returns_error(self):
objects = [ # copy to make sure setUp changes dont mess up this test
(0, "blue cover", [100, 400, 100, 300]),
(1, "blue cover", [700, 1100, 100, 300]),
(2, "fuse", [100, 400, 800, 1000]),
(3, "bottom cover", [700, 1100, 800, 1000]),
(4, "white cover", [100, 400, 1500, 1600]),
(5, "white cover", [700, 700, 1500, 1600]),
(6, "blue cover", [0, 100, 1550, 1600])
]
spatial_description = SpatialDescription(
spatial_type=SpatialType.NEXT_TO)
spatial_description.object_entity.name = "white cover"
spatial_description_2 = SpatialDescription(
spatial_type=SpatialType.OTHER)
spatial_description_2.object_entity.name = "top left corner"
spatial_descriptions = []
spatial_descriptions.append(spatial_description)
spatial_descriptions.append(spatial_description_2)
expected_bbox = objects[4][2]
expected_x, expected_y = expected_bbox[1] - (
expected_bbox[1] - expected_bbox[0]) / 2, expected_bbox[3] - (
expected_bbox[3] - expected_bbox[2]) / 2
self.db_mock.get_location_by_name = Mock(return_value=(100, 100, 0))
locations, status = self.spatial.get_location(spatial_descriptions,
objects)
self.assertEqual(StatusEnum.NO_VALID_OBJECTS, status)
def test_get_location__object_at_static_location__return_object_center(
self):
objects = [ # copy to make sure setUp changes dont mess up this test
(0, "blue cover", [100, 400, 100, 300]),
(1, "blue cover", [700, 1100, 100, 300]),
(2, "fuse", [100, 400, 800, 1000]),
(3, "bottom cover", [700, 1100, 800, 1000]),
(4, "white cover", [100, 400, 1500, 1600]),
(5, "white cover", [700, 700, 1500, 1600]),
(6, "blue cover", [0, 100, 1550, 1600])
]
spatial_description = SpatialDescription(
spatial_type=SpatialType.NEXT_TO)
spatial_description.object_entity.name = "white cover"
spatial_description_2 = SpatialDescription(
spatial_type=SpatialType.OTHER)
spatial_description_2.object_entity.name = "bottom left corner"
spatial_descriptions = []
spatial_descriptions.append(spatial_description)
spatial_descriptions.append(spatial_description_2)
expected_bbox = objects[4][2]
expected_x, expected_y = expected_bbox[1] - (
expected_bbox[1] - expected_bbox[0]) / 2, expected_bbox[3] - (
expected_bbox[3] - expected_bbox[2]) / 2
self.db_mock.get_location_by_name = Mock(return_value=(100, 1500, 0))
locations, status = self.spatial.get_location(spatial_descriptions,
objects)
x, y = locations[0]
self.assertAlmostEqual(expected_x, x)
self.assertAlmostEqual(expected_y, y)
class SpatialModuleOneOfEach(unittest.TestCase):
def setUp(self):
self.ner_mock = Mock()
self.db_mock = Mock()
self.spatial = SpatialRelation(database_handler=self.db_mock)
self.cmd_builder = CommandBuilder(self.ner_mock)
self.objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000
(0, "black cover", [100, 400, 100, 300]),
(1, "blue cover", [700, 1100, 100, 300]),
(2, "fuse", [100, 400, 800, 1000]),
(3, "bottom cover", [700, 1100, 800, 1000]),
(4, "white cover", [100, 400, 1500, 1600]),
(5, "green cover", [700, 700, 1500, 1600])
]
def test_above(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "above"),
(EntityType.OBJECT, "bottom cover")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[1][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_right(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "right"),
(EntityType.COLOUR, "black"), (EntityType.OBJECT, "cover")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[1][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_left(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.OBJECT, "fuse"),
(EntityType.LOCATION, "left"),
(EntityType.OBJECT, "bottom cover")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[2][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_below(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "white"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "below"),
(EntityType.OBJECT, "fuse")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[4][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_diagonalRightUp1(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "above"),
(EntityType.OBJECT, "fuse")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[1][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_diagonalRightUp2(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "right"),
(EntityType.OBJECT, "fuse")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[1][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_serial(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "black"),
(EntityType.OBJECT, "cover"), (EntityType.LOCATION,
"left"),
(EntityType.COLOUR, "blue"), (EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "above"),
(EntityType.OBJECT, "bottom cover"),
(EntityType.LOCATION, "right"),
(EntityType.OBJECT, "fuse")]
self.ner_mock.get_entities = Mock(return_value=entities)
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
expected_index = self.objects[0][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
def test_locate_specific_object__static_location__returns_closest_object(
self):
object_entity = ObjectEntity(name="blue cover")
spatial_description = SpatialDescription(
spatial_type=SpatialType.OTHER)
spatial_description.object_entity.name = "top left corner"
object_entity.spatial_descriptions.append(spatial_description)
self.db_mock.get_location_by_name = Mock(return_value=(100, 100, 0))
expected_index = self.objects[1][0]
actual_indices, success_enum = self.spatial.locate_specific_object(
object_entity, self.objects)
self.assertEqual(expected_index, actual_indices[0])
class FindObjectIsolatedTest(unittest.TestCase):
def setUp(self):
self.ner_mock = Mock()
self.db_mock = Mock()
self.spatial_mock = Mock()
self.vision_mock = Mock()
self.returned = GroundingReturn()
self.cmd_builder = CommandBuilder(self.ner_mock)
self.grounding = Grounding(db=self.db_mock,
vision_controller=self.vision_mock,
spatial=self.spatial_mock)
self.objects = [ # bbox = [x1, x2, y1, y2] and images spans from 0,0 to 1500,2000
(0, "black cover", [100, 400, 100, 300]),
(1, "blue cover", [700, 1100, 100, 300]),
(2, "fuse", [100, 400, 800, 1000]),
(3, "bottom cover", [700, 1100, 800, 1000]),
(4, "white cover", [100, 400, 1500, 1600]),
(5, "blue cover", [700, 700, 1500, 1600])
]
def test_failFindObject(self):
object_info = ObjectInfoWithFeatures()
features = []
object_info.features = np.array([1, 1, 1, 1, 1])
object_info.bbox_xxyy = np.array([1, 2, 3, 4])
object_info.mask_full = np.array([4, 3, 2, 1])
object_info.mask_cropped = np.array([5, 5, 5, 5])
features.append(object_info)
self.vision_mock.get_masks_with_features = Mock(return_value=features)
self.db_mock.get_feature = Mock(return_value=np.array([5, 5, 5, 5, 5]))
object_name = "black cover"
object_spatial_desc = None
object_entity = ObjectEntity()
object_entity.name = object_name
object_entity.spatial_descriptions = object_spatial_desc
returned = self.grounding.find_object(object_entity)
self.assertEqual(returned.error_code, GroundingErrorType.CANT_FIND)
def test_bestMatch(self):
indexes = [1, 2, 3, 4, 5]
distances = [0.78, 2, 1.5, 0.23, 0.1]
best_index = self.grounding.find_best_match(indexes, distances)
self.assertEqual(5, best_index)
def test_sameObject(self):
features_1 = np.array([0.35, 0.88, 0.20, 0.30])
features_2 = np.array([0.30, 0.85, 0.17, 0.32])
self.assertTrue(self.grounding.is_same_object(features_1, features_2))
def test_unknownObject(self):
object_info = ObjectInfoWithFeatures()
features = []
object_info.features = np.array([1, 1, 1, 1, 1])
object_info.bbox_xxyy = np.array([1, 2, 3, 4])
object_info.mask_full = np.array([4, 3, 2, 1])
object_info.mask_cropped = np.array([5, 5, 5, 5])
features.append(object_info)
self.vision_mock.get_masks_with_features = Mock(return_value=features)
self.db_mock.get_feature = Mock(return_value=None)
object_name = "albert is cool"
object_spatial_desc = None
object_entity = ObjectEntity()
object_entity.name = object_name
object_entity.spatial_descriptions = object_spatial_desc
returned = self.grounding.find_object(object_entity)
self.assertEqual(returned.error_code, GroundingErrorType.UNKNOWN
) # Should return false if the object is unknown
def test_spatialPart(self):
entities = [(EntityType.TASK, "pick up"), (EntityType.COLOUR, "blue"),
(EntityType.OBJECT, "cover"),
(EntityType.LOCATION, "above"),
(EntityType.OBJECT, "bottom cover")]
features = []
feature = [
np.array([1, 1, 1, 1, 1]),
np.array([1, 1, 1, 1, 1]),
np.array([1, 1, 1, 1, 1])
]
bbox = [
np.array([1, 2, 3, 4]),
np.array([1, 2, 3, 4]),
np.array([1, 2, 3, 4])
]
mask = [
np.array([4, 3, 2, 1]),
np.array([4, 3, 2, 1]),
np.array([4, 3, 2, 1])
]
cropped_rbg = [
np.array([5, 5, 5, 5]),
np.array([5, 5, 5, 5]),
np.array([5, 5, 5, 5])
]
for i in range(3):
object_info = ObjectInfoWithFeatures()
object_info.features = feature[i]
object_info.bbox_xxyy = bbox[i]
object_info.mask_full = mask[i]
object_info.mask_cropped = cropped_rbg[i]
features.append(object_info)
db_features = [("dummy name", np.array([1, 1, 1, 1, 1])),
("dummy name", np.array([1, 1, 1, 1, 1])),
("dummy name", np.array([1, 1, 1, 1, 1]))]
self.vision_mock.get_masks_with_features = Mock(return_value=features)
self.db_mock.get_feature = Mock(return_value=np.array([1, 1, 1, 1, 1]))
self.db_mock.get_all_features = Mock(return_value=db_features)
self.ner_mock.get_entities = Mock(return_value=entities)
self.spatial_mock.locate_specific_object = Mock(
return_value=([1], StatusEnum.SUCCESS))
task = self.cmd_builder.get_task("Dummy sentence")
object_entity = task.objects_to_execute_on[0]
returned = self.grounding.find_object(object_entity)
self.assertTrue(returned.is_success)
self.assertIsNotNone(returned.object_infos)
def __init__(self, model_path, tags_path):
self.cmd_builder = CommandBuilder(model_path, tags_path)