def test_dynamic_perception_graph_instantiation():
ball = ObjectPerception("ball", _BALL_SCHEMA.geon.copy())
table = ObjectPerception("table", axes=_TABLE_SCHEMA.axes.copy())
first_frame = DevelopmentalPrimitivePerceptionFrame(
perceived_objects=[ball, table],
relations=[
above(ball, table),
Relation(IN_REGION, ball,
Region(table, distance=EXTERIOR_BUT_IN_CONTACT)),
Relation(IN_REGION, table,
Region(ball, distance=EXTERIOR_BUT_IN_CONTACT)),
],
)
second_frame = DevelopmentalPrimitivePerceptionFrame(
perceived_objects=[ball, table],
relations=[Relation(IN_REGION, ball, Region(table, distance=DISTAL))],
)
perception_graph = PerceptionGraph.from_dynamic_perceptual_representation(
PerceptualRepresentation(frames=[first_frame, second_frame]))
assert perception_graph.dynamic
# Ensure we don't attempt to handle more than two frames yet.
with pytest.raises(ValueError):
PerceptionGraph.from_dynamic_perceptual_representation(
PerceptualRepresentation(
frames=[first_frame, second_frame, second_frame]))
def test_recognized_particular():
# create a simple situation consisting of only Mom and Dad
mom = ObjectPerception("mom", axes=_PERSON_SCHEMA.axes.copy())
dad = ObjectPerception("dad", axes=_PERSON_SCHEMA.axes.copy())
PerceptualRepresentation.single_frame(
DevelopmentalPrimitivePerceptionFrame(
perceived_objects=[mom, dad],
property_assertions=[
HasBinaryProperty(mom, SENTIENT),
HasBinaryProperty(dad, SENTIENT),
HasBinaryProperty(mom, IS_MOM),
HasBinaryProperty(dad, IS_DAD),
],
))
def test_relations():
# ball on a table
ball = ObjectPerception("ball", _BALL_SCHEMA.geon.copy())
table = ObjectPerception("table", axes=_TABLE_SCHEMA.axes.copy())
PerceptualRepresentation.single_frame(
DevelopmentalPrimitivePerceptionFrame(
perceived_objects=[ball, table],
relations=[
above(ball, table),
Relation(IN_REGION, ball,
Region(table, distance=EXTERIOR_BUT_IN_CONTACT)),
Relation(IN_REGION, table,
Region(ball, distance=EXTERIOR_BUT_IN_CONTACT)),
],
))
def test_color():
# create a situation with a red ball
red = RgbColorPerception(255, 0, 0)
ball = ObjectPerception("ball", _BALL_SCHEMA.geon.copy())
PerceptualRepresentation.single_frame(
DevelopmentalPrimitivePerceptionFrame(
perceived_objects=[ball],
property_assertions=[HasColor(ball, red)]))
def test_difference():
ball = ObjectPerception("ball", _BALL_SCHEMA.geon.copy())
cup = ObjectPerception("cup", _make_cup_schema().geon.copy())
table = ObjectPerception("table", axes=_TABLE_SCHEMA.axes.copy())
first_frame = DevelopmentalPrimitivePerceptionFrame(
perceived_objects=[ball, table],
relations=[
above(ball, table),
Relation(IN_REGION, ball,
Region(table, distance=EXTERIOR_BUT_IN_CONTACT)),
Relation(IN_REGION, table,
Region(ball, distance=EXTERIOR_BUT_IN_CONTACT)),
],
)
second_frame = DevelopmentalPrimitivePerceptionFrame(
perceived_objects=[ball, table, cup], relations=[above(ball, table)])
diff = diff_primitive_perception_frames(before=first_frame,
after=second_frame)
assert len(diff.removed_relations) == 2
assert not diff.added_relations
assert len(diff.added_objects) == 1
assert not diff.removed_objects
assert diff.before_axis_info == first_frame.axis_info
assert diff.after_axis_info == second_frame.axis_info
assert not diff.added_property_assertions
assert not diff.removed_property_assertions
# Reversed
diff_2 = diff_primitive_perception_frames(before=second_frame,
after=first_frame)
assert len(diff_2.added_relations) == 2
assert not diff_2.removed_relations
assert not diff_2.added_objects
assert len(diff_2.removed_objects) == 1
assert diff_2.before_axis_info == second_frame.axis_info
assert diff_2.after_axis_info == first_frame.axis_info
assert not diff_2.added_property_assertions
assert not diff_2.removed_property_assertions
def test_gravity_constraint() -> None:
aabb_floating = AxisAlignedBoundingBox.create_at_center_point(
center=np.array([0, 0, 5])
)
# boxes are 2 units tall by default, so this one is resting on the ground
aabb_grounded = AxisAlignedBoundingBox.create_at_center_point(
center=np.array([0, 0, 1])
)
ground_region = Region(GROUND_PERCEPTION, EXTERIOR_BUT_IN_CONTACT, GRAVITATIONAL_UP)
floating_perception = ObjectPerception(
"floating_thing",
axes=Axes(
primary_axis=symmetric_vertical("floating-thing-generating"),
orienting_axes=immutableset(
[symmetric("side-to-side0"), symmetric("side-to-side1")]
),
),
)
grounded_perception = ObjectPerception(
"grounded_thing",
axes=Axes(
primary_axis=symmetric_vertical("grounded-thing-generating"),
orienting_axes=immutableset(
[symmetric("side-to-side0"), symmetric("side-to-side1")]
),
),
)
gravity_penalty = WeakGravityPenalty(
{floating_perception: aabb_floating, grounded_perception: aabb_grounded},
{floating_perception: [ground_region], grounded_perception: [ground_region]},
)
floating_result = gravity_penalty(aabb_floating, immutableset([ground_region]))
assert floating_result > 0
grounded_result = gravity_penalty(aabb_grounded, immutableset([ground_region]))
assert grounded_result <= 0
def test_running_model() -> None:
# for code coverage purposes
ball = ObjectPerception(
"ball",
axes=Axes(
primary_axis=symmetric_vertical("ball-generating"),
orienting_axes=immutableset(
[symmetric("side-to-side0"), symmetric("side-to-side1")]
),
),
)
box = ObjectPerception(
"box",
axes=Axes(
primary_axis=straight_up("top_to_bottom"),
orienting_axes=immutableset(
[symmetric("side-to-side0"), symmetric("side-to-side1")]
),
),
)
objs = immutableset([ball, box])
relations: Mapping[ObjectPerception, List[Region[ObjectPerception]]] = {}
scales: Mapping[str, Tuple[float, float, float]] = {
"box": (1.0, 1.0, 1.0),
"ball": (2.0, 1.0, 1.0),
}
run_model(
objs,
{},
relations,
scales,
num_iterations=10,
yield_steps=10,
frozen_objects=immutableset([]),
)
def main() -> None:
top_to_bottom = straight_up("top-to-bottom")
side_to_side_0 = symmetric("side-to-side-0")
side_to_side_1 = symmetric("side-to-side-1")
box = ObjectPerception(
"box",
geon=None,
axes=Axes(
primary_axis=top_to_bottom,
orienting_axes=immutableset([side_to_side_0, side_to_side_1]),
),
)
generating_axis = symmetric_vertical("ball-generating")
orienting_axis_0 = symmetric("ball-orienting-0")
orienting_axis_1 = symmetric("ball-orienting-1")
# ball situated on top of box
ball = ObjectPerception(
"ball",
geon=None,
axes=Axes(
primary_axis=generating_axis,
orienting_axes=immutableset([orienting_axis_0, orienting_axis_1]),
),
)
in_region_relations: Mapping[ObjectPerception,
List[Region[ObjectPerception]]] = {
ball: [
Region[ObjectPerception](
box, EXTERIOR_BUT_IN_CONTACT,
GRAVITATIONAL_UP)
]
}
# other objects have no particular constraints:
positioning_model = PositioningModel.for_objects_random_positions(
object_perceptions=immutableset([ball, box]),
sub_objects={},
in_region_relations=in_region_relations,
)
# we will start with an aggressive learning rate
optimizer = optim.SGD(positioning_model.parameters(), lr=1.0)
# but will decrease it whenever the loss plateaus
learning_rate_schedule = ReduceLROnPlateau(
optimizer,
"min",
# decrease the rate if the loss hasn't improved in
# 3 epochs
patience=3,
)
iterations = 100
for iteration in range(iterations):
print(f"====== Iteration {iteration} ======")
positioning_model.dump_object_positions(prefix="\t")
loss = positioning_model()
print(f"\tLoss: {loss.item()}")
loss.backward()
optimizer.step()
optimizer.zero_grad()
learning_rate_schedule.step(loss)
print("========= Final Positions ========")
positioning_model.dump_object_positions(prefix="\t")
def test_in_region_constraint() -> None:
ball = ObjectPerception(
"ball",
axes=Axes(
primary_axis=symmetric_vertical("ball-generating"),
orienting_axes=immutableset(
[symmetric("side-to-side0"), symmetric("side-to-side1")]
),
),
)
box = ObjectPerception(
"box",
axes=Axes(
primary_axis=straight_up("top_to_bottom"),
orienting_axes=immutableset(
[symmetric("side-to-side0"), symmetric("side-to-side1")]
),
),
)
aabb_ball = AxisAlignedBoundingBox.create_at_center_point(center=np.array([0, 0, 1]))
aabb_box = AxisAlignedBoundingBox.create_at_center_point(center=np.array([0, -2, 1]))
# specifying that the box should be to the right of the ball
direction = Direction(
positive=True, relative_to_axis=HorizontalAxisOfObject(ball, index=0)
)
region = Region(ball, PROXIMAL, direction)
obj_percept_to_aabb = {ball: aabb_ball, box: aabb_box}
in_region_relations = {box: [region]}
in_region_penalty = InRegionPenalty(obj_percept_to_aabb, {}, {}, in_region_relations)
box_penalty = in_region_penalty(box, immutableset([region]))
assert box_penalty > 0
# now with a box that *is* to the right of the ball
box2 = ObjectPerception(
"box2",
axes=Axes(
primary_axis=straight_up("top_to_bottom"),
orienting_axes=immutableset(
[symmetric("side-to-side0"), symmetric("side-to-side1")]
),
),
)
aabb_box2 = AxisAlignedBoundingBox.create_at_center_point(center=np.array([2, 0, 1]))
obj_percept_to_aabb = {ball: aabb_ball, box2: aabb_box2}
in_region_relations = {box2: [region]}
in_region_penalty = InRegionPenalty(obj_percept_to_aabb, {}, {}, in_region_relations)
box_penalty = in_region_penalty(box2, immutableset([region]))
assert box_penalty == 0
def perception_text(
self, perception: PerceptualRepresentation[
DevelopmentalPrimitivePerceptionFrame]
) -> str:
"""
Turns a perception into a list of items in the perceptions frames.
"""
output_text: List[str] = []
check_state(
len(perception.frames) in (1, 2),
"Only know how to handle 1 or 2 frame "
"perceptions for now",
)
perception_is_dynamic = len(perception.frames) > 1
# first, we build an index of objects to their properties.
# This will be used so that when we list the objects,
# we can easily list their properties in brackets right after them.
def extract_subject(prop: PropertyPerception) -> ObjectPerception:
return prop.perceived_object
first_frame_properties = _index_to_setmultidict(
perception.frames[0].property_assertions, extract_subject)
second_frame_properties = (_index_to_setmultidict(
perception.frames[1].property_assertions, extract_subject)
if perception_is_dynamic else
immutablesetmultidict())
# Next, we determine what objects persist between both frames
# and which do not.
first_frame_objects = perception.frames[0].perceived_objects
second_frame_objects = (perception.frames[1].perceived_objects
if perception_is_dynamic else immutableset())
static_objects = (
first_frame_objects.intersection(second_frame_objects)
if perception_is_dynamic else first_frame_objects)
all_objects = first_frame_objects.union(second_frame_objects)
# For objects, properties, and relations we will use arrows to indicate
# when something beings or ceased to exist between frames.
# Since the logic will be the same for all three types,
# we pull it out into a function.
def compute_arrow(
item: Any, static_items: AbstractSet[Any],
first_frame_items: AbstractSet[Any]) -> Tuple[str, str]:
if item in static_items:
# item doesn't change - no arrow
return ("", "")
elif item in first_frame_items:
# item ceases to exist
return ("", " ---> Ø")
else:
# item beings to exist in the second frame
return ("Ø ---> ", "")
# the logic for rendering objects, which will be used in the loop below.
# This needs to be an inner function so it can access the frame property maps, etc.
def render_object(obj: ObjectPerception) -> str:
obj_text = f"<i>{obj.debug_handle}</i>"
first_frame_obj_properties = first_frame_properties[obj]
second_frame_obj_properties = second_frame_properties[obj]
static_properties = (second_frame_obj_properties.intersection(
first_frame_obj_properties) if second_frame_obj_properties else
first_frame_obj_properties)
# logic for rendering properties, for use in the loop below.
def render_property(prop: PropertyPerception) -> str:
(prop_prefix,
prop_suffix) = compute_arrow(prop, static_properties,
first_frame_obj_properties)
prop_string: str
if isinstance(prop, HasColor):
prop_string = (
f'<span style="background-color: {prop.color}; '
f'color: {prop.color.inverse()}; border: 1px solid black;">'
f"color={prop.color.hex}</span>")
elif isinstance(prop, HasBinaryProperty):
prop_string = prop.binary_property.handle
else:
raise RuntimeError(f"Cannot render property: {prop}")
return f"{prop_prefix}{prop_string}{prop_suffix}"
all_properties: ImmutableSet[PropertyPerception] = immutableset(
flatten(
[first_frame_obj_properties, second_frame_obj_properties]))
prop_strings = [render_property(prop) for prop in all_properties]
if prop_strings:
return f"{obj_text}[{'; '.join(prop_strings)}]"
else:
return obj_text
# Here we process the relations between the two scenes to determine all relations.
# This has to be done before rending objects so we can use the PART_OF relation to order
# the objects.
first_frame_relations = perception.frames[0].relations
second_frame_relations = (perception.frames[1].relations
if perception_is_dynamic else immutableset())
static_relations = (
second_frame_relations.intersection(first_frame_relations)
if perception_is_dynamic else first_frame_relations)
all_relations = first_frame_relations.union(second_frame_relations)
# Here we add the perceived objects to a NetworkX DiGraph with PART_OF relations being the
# edges between objects. This allows us to do pre-order traversal of the Graph to make a
# nested <ul></ul> for the objects rather than a flat list.
graph = DiGraph()
root = ObjectPerception("root", axes=WORLD_AXES)
graph.add_node(root)
expressed_relations = set()
axis_to_object: Dict[GeonAxis, ObjectPerception] = {}
for object_ in all_objects:
graph.add_node(object_)
graph.add_edge(root, object_)
for axis in object_.axes.all_axes:
axis_to_object[axis] = object_
for relation_ in all_relations:
if relation_.relation_type == PART_OF:
graph.add_edge(relation_.second_slot, relation_.first_slot)
if graph.has_edge(root, relation_.first_slot):
graph.remove_edge(root, relation_.first_slot)
expressed_relations.add(relation_)
# Next, we render objects, together with their properties, using preorder DFS Traversal
# We also add in `In Region` relationships at this step for objects which have them.
output_text.append(
"\n\t\t\t\t\t<h5>Perceived Objects</h5>\n\t\t\t\t\t<ul>")
visited = set()
region_relations = immutableset(region for region in all_relations
if region.relation_type == IN_REGION)
# This loop doesn't quite get the tab spacing right. It could at the cost of increased
# complexity. Would need to track the "depth" we are currently at.
axis_info = perception.frames[0].axis_info
def dfs_walk(node, depth=0):
visited.add(node)
if not node == root:
(obj_prefix,
obj_suffix) = compute_arrow(node, static_objects,
first_frame_objects)
output_text.append(
f"\t" * (6 + depth) +
f"<li>{obj_prefix}{render_object(node)}{obj_suffix}<ul>")
if node.geon:
output_text.append(
f"\t\t\t\t\t\t<li>Geon: {self._render_geon(node.geon, indent_dept=7)}</li>"
)
# Handle Region Relations
for region_relation in region_relations:
if region_relation.first_slot == node:
(relation_prefix, relation_suffix) = compute_arrow(
region_relation, static_relations,
first_frame_relations)
relation_str = self._render_relation(
axis_info, region_relation)
output_text.append(
f"\t\t\t\t\t\t<li>{relation_prefix}"
f"{relation_str}{relation_suffix}</li>")
expressed_relations.add(region_relation)
for succ in graph.successors(node):
if succ not in visited:
depth = depth + 6
dfs_walk(succ, depth)
depth = depth - 6
output_text.append("\t" * (6 + depth) + f"</ul></li>")
dfs_walk(root)
output_text.append("\t\t\t\t\t</ul>")
# Finally we render remaining relations between objects
remaining_relations = immutableset(
relation for relation in all_relations
if relation not in expressed_relations)
if remaining_relations:
output_text.append(
"\t\t\t\t\t<h5>Other Relations</h5>\n\t\t\t\t\t<ul>")
for relation in remaining_relations:
(relation_prefix,
relation_suffix) = compute_arrow(relation, static_relations,
first_frame_relations)
single_size_relation: Optional[Tuple[
Any, str, Any]] = self._get_single_size_relation(
relation, all_relations)
if single_size_relation:
relation_text = f"{single_size_relation[0]} {single_size_relation[1]} {single_size_relation[2]}"
size_output = f"\t\t\t\t\t\t<li>{relation_prefix}{relation_text}{relation_suffix}</li>"
if size_output not in output_text:
output_text.append(size_output)
else:
output_text.append(
f"\t\t\t\t\t\t<li>{relation_prefix}{relation}{relation_suffix}</li>"
)
output_text.append("\t\t\t\t\t</ul>")
if perception.during:
output_text.append("\t\t\t\t\t<h5>During the action</h5>")
output_text.append(
self._render_during(perception.during, indent_depth=5))
if axis_info and axis_info.axes_facing:
output_text.append(("\t\t\t\t\t<h5>Axis Facings</h5>"))
output_text.append(("\t\t\t\t\t<ul>"))
for object_ in axis_info.axes_facing:
output_text.append(
f"\t\t\t\t\t\t<li>{object_.debug_handle} faced by:\n\t\t\t\t\t\t<ul>"
)
for axis in axis_info.axes_facing[object_]:
output_text.append(
f"\t\t\t\t\t\t\t<li>{axis} possessed by {axis_to_object[axis]}</li>"
)
output_text.append("\t\t\t\t\t\t</ul>")
output_text.append("\t\t\t\t\t</ul>")
return "\n".join(output_text)
def test_primary_axis_function():
hand_axes = first(GAILA_PHASE_1_ONTOLOGY.structural_schemata(HAND)).axes
hand = ObjectPerception("hand", axes=hand_axes)
hand_primary_axis = PrimaryAxisOfObject(hand).to_concrete_axis(AxesInfo())
assert hand_primary_axis == hand_axes.primary_axis