def test_recognizes_ontology_objects(object_type, language_mode):
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[
SituationObject.instantiate_ontology_node(
ontology_node=object_type, ontology=GAILA_PHASE_1_ONTOLOGY)
],
)
perception_generator = HighLevelSemanticsSituationToDevelopmentalPrimitivePerceptionGenerator(
GAILA_PHASE_1_ONTOLOGY)
perception = perception_generator.generate_perception(
situation, chooser=RandomChooser.for_seed(0), include_ground=False)
learner = IntegratedTemplateLearner(
object_learner=LANGUAGE_MODE_TO_TEMPLATE_LEARNER_OBJECT_RECOGNIZER[
language_mode])
descriptions = learner.describe(perception)
assert descriptions
if language_mode == LanguageMode.ENGLISH:
assert object_type.handle in one(
descriptions.items())[0].as_token_sequence()
else:
mappings = (
GAILA_PHASE_1_CHINESE_LEXICON._ontology_node_to_word # pylint:disable=protected-access
)
for k, v in mappings.items():
if k.handle == object_type.handle:
assert v.base_form in one(
descriptions.items())[0].as_token_sequence()
def build_object_multiples_situations(
ontology: Ontology,
*,
samples_per_object: int = 3,
chooser: RandomChooser) -> Iterable[HighLevelSemanticsSituation]:
for object_type in PHASE_1_CURRICULUM_OBJECTS:
# Exclude slow objects for now
if object_type.handle in ["bird", "dog", "truck"]:
continue
is_liquid = ontology.has_all_properties(object_type, [LIQUID])
# don't want multiples of named people
if not is_recognized_particular(ontology,
object_type) and not is_liquid:
for _ in range(samples_per_object):
num_objects = chooser.choice(range(2, 4))
yield HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[
SituationObject.instantiate_ontology_node(
ontology_node=object_type,
debug_handle=object_type.handle + f"_{idx}",
ontology=GAILA_PHASE_1_ONTOLOGY,
) for idx in range(num_objects)
],
axis_info=AxesInfo(),
)
def test_dynamic_prepositions_implicit_grounding():
truck = situation_object(ontology_node=TRUCK)
baby = situation_object(ontology_node=BABY)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[baby, truck],
actions=[Action(FALL, argument_roles_to_fillers=[(THEME, baby)])],
after_action_relations=[on(baby, truck)],
)
perception = _PERCEPTION_GENERATOR.generate_perception(
situation, chooser=RandomChooser.for_seed(0)
)
first_frame = perception.frames[0]
baby_perception = perception_with_handle(first_frame, "**baby_0")
ground_perception = perception_with_handle(first_frame, "the ground")
truck_perception = perception_with_handle(first_frame, "**truck_0")
first_frame_relations = first_frame.relations
second_frame_relations = perception.frames[1].relations
assert on(baby_perception, truck_perception)[0] not in first_frame_relations
assert on(baby_perception, truck_perception)[0] in second_frame_relations
assert on(baby_perception, ground_perception)[0] not in second_frame_relations
assert on(truck_perception, ground_perception)[0] in first_frame_relations
assert on(truck_perception, ground_perception)[0] in second_frame_relations
def test_object_not_on_ground():
"""
Intended to test that one can specify an object is not on the ground
"""
table = situation_object(TABLE)
ground = situation_object(GROUND)
perception = _PERCEPTION_GENERATOR.generate_perception(
HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=immutableset([table]),
other_objects=immutableset([ground]),
always_relations=flatten_relations(negate(on(table, ground))),
),
chooser=RandomChooser.for_seed(0),
)
frame = perception.frames[0]
relations = frame.relations
table_perception = perception_with_handle(frame, "**table_0")
ground_perception = perception_with_handle(frame, "the ground")
assert not any(
relation.relation_type == IN_REGION
and relation.first_slot == table_perception
and isinstance(relation.second_slot, Region)
and relation.region.reference_object == ground_perception
and relation.region.distance == EXTERIOR_BUT_IN_CONTACT
for relation in relations
)
def test_gaze_specified():
cookie = situation_object(COOKIE)
table = situation_object(TABLE)
dad = situation_object(DAD)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[cookie, table, dad],
actions=[
Action(EAT,
argument_roles_to_fillers=[(AGENT, dad), (PATIENT, cookie)])
],
gazed_objects=[cookie],
)
perception = _PERCEPTION_GENERATOR.generate_perception(
situation, chooser=RandomChooser.for_seed(0))
frame = perception.frames[0]
cookie_perception = perception_with_handle(frame, "**cookie_0")
dad_perception = perception_with_handle(frame, "**Dad_0")
table_perception = perception_with_handle(frame, "**table_0")
# only the cookie is gazed at, because the user said so.
assert HasBinaryProperty(cookie_perception,
GAZED_AT) in frame.property_assertions
assert HasBinaryProperty(dad_perception,
GAZED_AT) not in frame.property_assertions
assert HasBinaryProperty(table_perception,
GAZED_AT) not in frame.property_assertions
def test_person_and_ball_color():
person = situation_object(PERSON)
ball = situation_object(BALL, properties=[RED])
person_and_ball_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[person, ball]
)
person_and_ball_perception = _PERCEPTION_GENERATOR.generate_perception(
person_and_ball_situation, chooser=RandomChooser.for_seed(0)
)
assert len(person_and_ball_perception.frames) == 1
frame = person_and_ball_perception.frames[0]
assert (
prop_assertion is PropertyPerception
for prop_assertion in frame.property_assertions
)
person_perception = perception_with_handle(frame, "**person_0")
ball_perception = perception_with_handle(frame, "**ball_0")
assert HasBinaryProperty(person_perception, ANIMATE) in frame.property_assertions
assert HasBinaryProperty(person_perception, SELF_MOVING) in frame.property_assertions
assert HasBinaryProperty(ball_perception, INANIMATE) in frame.property_assertions
assert any(
isinstance(property_, HasColor) and property_.perceived_object == ball_perception
for property_ in frame.property_assertions
)
def test_trivial_dynamic_situation_with_schemaless_object(language_mode):
dad_situation_object = SituationObject.instantiate_ontology_node(
ontology_node=DAD, ontology=GAILA_PHASE_1_ONTOLOGY)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[dad_situation_object])
perception_generator = HighLevelSemanticsSituationToDevelopmentalPrimitivePerceptionGenerator(
GAILA_PHASE_1_ONTOLOGY)
# We explicitly exclude ground in perception generation
# this generates a static perception...
perception = perception_generator.generate_perception(
situation, chooser=RandomChooser.for_seed(0), include_ground=False)
# so we need to construct a dynamic one by hand from two identical scenes
dynamic_perception = PerceptualRepresentation(
frames=[perception.frames[0], perception.frames[0]])
perception_graph = PerceptionGraph.from_dynamic_perceptual_representation(
dynamic_perception)
perception_semantic_alignment = PerceptionSemanticAlignment.create_unaligned(
perception_graph)
(_, description_to_matched_semantic_node
) = LANGUAGE_MODE_TO_OBJECT_RECOGNIZER[language_mode].match_objects(
perception_semantic_alignment)
assert len(description_to_matched_semantic_node) == 1
assert (language_mode == LanguageMode.ENGLISH and
("Dad", ) in description_to_matched_semantic_node) or (
language_mode == LanguageMode.CHINESE and
("ba4 ba4", ) in description_to_matched_semantic_node)
def make_bird_flies_over_a_house():
bird = situation_object(BIRD)
house = situation_object(HOUSE)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[bird, house],
actions=[
Action(
FLY,
argument_roles_to_fillers=[(AGENT, bird)],
during=DuringAction(
at_some_point=[
Relation(
IN_REGION,
bird,
Region(
reference_object=house,
distance=DISTAL,
direction=Direction(
positive=True,
relative_to_axis=GRAVITATIONAL_AXIS_FUNCTION,
),
),
)
]
),
)
],
)
return situation
def test_no_region_in_gazed_objects():
putter = situation_object(MOM)
object_put = situation_object(BALL)
on_region_object = situation_object(TABLE)
situation = HighLevelSemanticsSituation(
salient_objects=[putter, object_put, on_region_object],
actions=[
Action(
PUT,
argument_roles_to_fillers=[
(AGENT, putter),
(THEME, object_put),
(
GOAL,
Region(
on_region_object,
distance=EXTERIOR_BUT_IN_CONTACT,
direction=GRAVITATIONAL_UP,
),
),
],
)
],
ontology=GAILA_PHASE_1_ONTOLOGY,
)
assert situation.gazed_objects
for object_ in situation.gazed_objects:
assert not isinstance(object_, Region)
def make_mom_put_ball_on_table():
mom = situation_object(ontology_node=MOM)
ball = situation_object(ontology_node=BALL)
table = situation_object(ontology_node=TABLE)
return HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[mom, ball, table],
actions=[
Action(
PUT,
(
(AGENT, mom),
(THEME, ball),
(
GOAL,
Region(
reference_object=table,
distance=EXTERIOR_BUT_IN_CONTACT,
direction=GRAVITATIONAL_UP,
),
),
),
)
],
)
def test_liquid_perceivable():
juice_perception = _PERCEPTION_GENERATOR.generate_perception(
HighLevelSemanticsSituation(ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[situation_object(JUICE)]),
chooser=RandomChooser.for_seed(0),
).frames[0]
assert _some_object_has_binary_property(juice_perception, LIQUID)
def test_objects_in_something_not_implcitly_grounded():
box = situation_object(ontology_node=BOX)
ball = situation_object(ontology_node=BALL)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[box, ball],
always_relations=[
Relation(
IN_REGION,
ball,
Region(
box,
distance=PROXIMAL,
direction=Direction(
positive=True,
relative_to_axis=HorizontalAxisOfObject(box, index=0),
),
),
)
],
)
perception = _PERCEPTION_GENERATOR.generate_perception(
situation, chooser=RandomChooser.for_seed(0)
)
first_frame = perception.frames[0]
ball_perception = perception_with_handle(first_frame, "**ball_0")
ground_perception = perception_with_handle(first_frame, "the ground")
box_perception = perception_with_handle(first_frame, "**box_0")
first_frame_relations = first_frame.relations
assert on(ball_perception, ground_perception)[0] in first_frame_relations
assert on(box_perception, ground_perception)[0] in first_frame_relations
def test_colors_across_part_of_relations():
"""
Intended to test color inheritance across part-of relations
with objects that have a prototypical color
"""
learner_perception = _PERCEPTION_GENERATOR.generate_perception(
HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[situation_object(TRUCK)]
),
chooser=RandomChooser.for_seed(0),
)
frame = learner_perception.frames[0]
property_assertions = frame.property_assertions
truck_perception = perception_with_handle(frame, "**truck_0")
flatbed_perception = perception_with_handle(frame, "**flatbed_0")
tire0_perception = perception_with_handle(frame, "**tire_0")
tire1_perception = perception_with_handle(frame, "**tire_1")
tire2_perception = perception_with_handle(frame, "**tire_2")
tire3_perception = perception_with_handle(frame, "**tire_3")
tire4_perception = perception_with_handle(frame, "**tire_4")
tire5_perception = perception_with_handle(frame, "**tire_5")
tire6_perception = perception_with_handle(frame, "**tire_6")
tire7_perception = perception_with_handle(frame, "**tire_7")
blue_options = COLORS_TO_RGBS[BLUE]
blue_perceptions = [RgbColorPerception(r, g, b) for r, g, b in blue_options]
red_options = COLORS_TO_RGBS[RED]
red_perceptions = [RgbColorPerception(r, g, b) for r, g, b in red_options]
black_options = COLORS_TO_RGBS[BLACK]
black_perceptions = [RgbColorPerception(r, g, b) for r, g, b in black_options]
assert any(
HasColor(truck_perception, blue_perception) in property_assertions
for blue_perception in blue_perceptions
) or any(
HasColor(truck_perception, red_perception) in property_assertions
for red_perception in red_perceptions
)
assert any(
HasColor(flatbed_perception, blue_perception) in property_assertions
for blue_perception in blue_perceptions
) or any(
HasColor(flatbed_perception, red_perception) in property_assertions
for red_perception in red_perceptions
)
assert any(
(
HasColor(tire0_perception, black_perception)
and HasColor(tire1_perception, black_perception)
and HasColor(tire2_perception, black_perception)
and HasColor(tire3_perception, black_perception)
and HasColor(tire4_perception, black_perception)
and HasColor(tire5_perception, black_perception)
and HasColor(tire6_perception, black_perception)
and HasColor(tire7_perception, black_perception)
)
in property_assertions
for black_perception in black_perceptions
)
def test_perceive_explicit_relations():
# we want to test that relations explicitly called out in the situation are perceived.
# Such relations fall into three buckets:
# those which hold before an action,
# those which hold after an action,
# and those which hold both before and after an action.
# To test all three of these at once, we use a situation where
# (a) Mom is putting a ball on a table
# (b) before the action the ball is far from a box,
# (c) but after the action the ball is near the box,
# (d) throughout the action the box is on the table.
mom = situation_object(MOM)
ball = situation_object(BALL)
box = situation_object(BOX)
table = situation_object(TABLE)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[mom, box, ball, table],
always_relations=[on(ball, table)],
before_action_relations=[far(ball, box)],
after_action_relations=[near(ball, box)],
actions=[
Action(
PUT,
argument_roles_to_fillers=[
(AGENT, mom),
(THEME, ball),
(
GOAL,
Region(
reference_object=table,
distance=EXTERIOR_BUT_IN_CONTACT,
direction=GRAVITATIONAL_UP,
),
),
],
)
],
)
perception = _PERCEPTION_GENERATOR.generate_perception(
situation, chooser=RandomChooser.for_seed(0)
)
ball_perception = perception_with_handle(perception.frames[0], "**ball_0")
box_perception = perception_with_handle(perception.frames[0], "**box_0")
table_perception = perception_with_handle(perception.frames[0], "**table_0")
assert only(on(ball_perception, table_perception)) in perception.frames[0].relations
assert only(on(ball_perception, table_perception)) in perception.frames[0].relations
assert only(far(ball_perception, box_perception)) in perception.frames[0].relations
assert (
only(far(ball_perception, box_perception)) not in perception.frames[1].relations
)
assert (
only(near(ball_perception, box_perception)) not in perception.frames[0].relations
)
assert only(near(ball_perception, box_perception)) in perception.frames[1].relations
def test_liquid_in_and_out_of_container():
juice = situation_object(JUICE)
box = situation_object(ontology_node=BOX)
table = situation_object(ontology_node=TABLE)
two_d_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[juice, table],
always_relations=[on(juice, table)],
)
two_d_perception = _PERCEPTION_GENERATOR.generate_perception(
two_d_situation, chooser=RandomChooser.for_seed(0)
)
three_d_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[juice, box],
always_relations=[
Relation(IN_REGION, juice, Region(box, distance=INTERIOR, direction=None))
],
)
three_d_perception = _PERCEPTION_GENERATOR.generate_perception(
three_d_situation, chooser=RandomChooser.for_seed(0)
)
two_perceived_objects = two_d_perception.frames[0].perceived_objects
two_object_handles = set(obj.debug_handle for obj in two_perceived_objects)
assert all(handle in two_object_handles for handle in {"**juice_0", "**table_0"})
three_perceived_objects = three_d_perception.frames[0].perceived_objects
three_object_handles = set(obj.debug_handle for obj in three_perceived_objects)
assert all(handle in three_object_handles for handle in {"**juice_0", "**box_0"})
assert any(
isinstance(p, HasBinaryProperty)
and perception_with_handle(two_d_perception.frames[0], "**juice_0")
== p.perceived_object
and p.binary_property == TWO_DIMENSIONAL
for p in two_d_perception.frames[0].property_assertions
)
assert not any(
isinstance(p, HasBinaryProperty)
and perception_with_handle(three_d_perception.frames[0], "**juice_0")
== p.perceived_object
and p.binary_property == TWO_DIMENSIONAL
for p in three_d_perception.frames[0].property_assertions
)
def test_person_and_ball():
person = situation_object(PERSON)
ball = situation_object(BALL)
person_and_ball_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[person, ball]
)
person_and_ball_perception = _PERCEPTION_GENERATOR.generate_perception(
person_and_ball_situation, chooser=RandomChooser.for_seed(0)
)
perceived_objects = person_and_ball_perception.frames[0].perceived_objects
object_handles = set(obj.debug_handle for obj in perceived_objects)
assert len(person_and_ball_perception.frames) == 1
assert object_handles == {
"**ball_0",
"**person_0",
"**head_0",
"**torso_0",
"**arm_0",
"**armsegment_0",
"**armsegment_1",
"**hand_0",
"**arm_1",
"**armsegment_2",
"**armsegment_3",
"**hand_1",
"**(animal) leg_0",
"**leg-segment_0",
"**leg-segment_1",
"**foot_0",
"**(animal) leg_1",
"**leg-segment_2",
"**leg-segment_3",
"**foot_1",
"the ground",
}
assert person_and_ball_perception.frames[0].relations
person_perception = perception_with_handle(
person_and_ball_perception.frames[0], "**person_0"
)
ball_perception = perception_with_handle(
person_and_ball_perception.frames[0], "**ball_0"
)
assert set(person_and_ball_perception.frames[0].property_assertions).issuperset(
{
HasBinaryProperty(person_perception, ANIMATE),
HasBinaryProperty(person_perception, SELF_MOVING),
HasBinaryProperty(ball_perception, INANIMATE),
}
)
def test_speaker_perceivable():
speaker_situation_perception = _PERCEPTION_GENERATOR.generate_perception(
HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[situation_object(PERSON, properties=[IS_SPEAKER])],
),
chooser=RandomChooser.for_seed(0),
)
assert _some_object_has_binary_property(
speaker_situation_perception.frames[0], IS_SPEAKER
)
def test_implicit_ground():
table_perception = _PERCEPTION_GENERATOR.generate_perception(
HighLevelSemanticsSituation(ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[situation_object(TABLE)]),
chooser=RandomChooser.for_seed(0),
)
perceived_objects = table_perception.frames[0].perceived_objects
object_handles = set(obj.debug_handle for obj in perceived_objects)
# assert that a "ground" object is perceived
assert "the ground" in object_handles
def test_not_two_speakers():
with pytest.raises(RuntimeError):
_PERCEPTION_GENERATOR.generate_perception(
HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[
situation_object(PERSON, properties=[IS_SPEAKER]),
situation_object(PERSON, properties=[IS_SPEAKER]),
],
),
chooser=RandomChooser.for_seed(0),
)
def test_explicit_ground():
ground_perception = _PERCEPTION_GENERATOR.generate_perception(
HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[situation_object(GROUND)]
),
chooser=RandomChooser.for_seed(0),
)
perceived_objects = ground_perception.frames[0].perceived_objects
object_handles = set(obj.debug_handle for obj in perceived_objects)
# assert that a second "ground" object was not generated
assert object_handles == {"the ground"}
def test_path_from_action_description():
ball = situation_object(BALL)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[ball],
actions=[Action(FALL, argument_roles_to_fillers=[(THEME, ball)])],
)
perception = _PERCEPTION_GENERATOR.generate_perception(
situation, chooser=RandomChooser.for_seed(0)
)
ball_perception = perception_with_handle(perception.frames[0], "**ball_0")
ground_perception = perception_with_handle(perception.frames[0], "the ground")
assert perception.during
assert perception.during.objects_to_paths
assert len(perception.during.objects_to_paths) == 1
path = only(perception.during.objects_to_paths[ball_perception])
assert path.reference_object == ground_perception
assert path.operator == TOWARD
def test_relations_between_objects_and_ground():
# person_put_ball_on_table
person = situation_object(ontology_node=PERSON)
ball = situation_object(ontology_node=BALL)
table = situation_object(ontology_node=TABLE)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[person, ball, table],
actions=[
# What is the best way of representing the destination in the high-level semantics?
# Here we represent it as indicating a relation which should be true.
Action(
PUT,
(
(AGENT, person),
(THEME, ball),
(
GOAL,
Region(
reference_object=table,
distance=EXTERIOR_BUT_IN_CONTACT,
direction=GRAVITATIONAL_UP,
),
),
),
)
],
)
perception = _PERCEPTION_GENERATOR.generate_perception(
situation, chooser=RandomChooser.for_seed(0)
)
first_frame = perception.frames[0]
ball_perception = perception_with_handle(first_frame, "**ball_0")
ground_perception = perception_with_handle(first_frame, "the ground")
table_perception = perception_with_handle(first_frame, "**table_0")
first_frame_relations = first_frame.relations
second_frame_relations = perception.frames[1].relations
assert on(ball_perception, ground_perception)[0] in first_frame_relations
assert on(ball_perception, ground_perception)[0] not in second_frame_relations
assert on(table_perception, ground_perception)[0] in first_frame_relations
assert on(table_perception, ground_perception)[0] in second_frame_relations
def test_multiple_trecognized_particulars_banned():
with pytest.raises(RuntimeError):
dad_0 = situation_object(DAD)
dad_1 = situation_object(DAD)
box = situation_object(BOX)
HighLevelSemanticsSituation(
salient_objects=[dad_0, dad_1, box],
actions=[
Action(
GIVE,
argument_roles_to_fillers=[
(AGENT, dad_0),
(THEME, box),
(GOAL, dad_1),
],
)
],
ontology=GAILA_PHASE_1_ONTOLOGY,
)
def _instantiate_situation(
self,
template: Phase1SituationTemplate,
variable_assignment: "TemplateVariableAssignment",
object_var_to_instantiations,
) -> HighLevelSemanticsSituation:
return HighLevelSemanticsSituation(
from_template=template.name,
ontology=self.ontology,
salient_objects=[
object_var_to_instantiations[obj_var]
for obj_var in template.salient_object_variables
],
other_objects=[
object_var_to_instantiations[obj_var] # type: ignore
for obj_var in (immutableset(object_var_to_instantiations.keys(
)).difference(template.salient_object_variables))
# We use the keys of the mapping in case a default addressee was added
],
always_relations=[
relation.copy_remapping_objects(object_var_to_instantiations)
for relation in template.asserted_always_relations
],
actions=[
self._instantiate_action(
action,
object_var_to_instantiations,
variable_assignment.action_variables_to_fillers,
) for action in template.actions
],
before_action_relations=[
relation.copy_remapping_objects(object_var_to_instantiations)
for relation in template.before_action_relations
],
after_action_relations=[
relation.copy_remapping_objects(object_var_to_instantiations)
for relation in template.after_action_relations
],
syntax_hints=template.syntax_hints,
axis_info=self._compute_axis_info(object_var_to_instantiations),
gazed_objects=immutableset(object_var_to_instantiations[object_]
for object_ in template.gazed_objects),
)
def test_subobject_perception_has_appropriate_properties():
"""
Intended to test that an object's subobjects have their properties assigned to them properly
"""
learner_perception = _PERCEPTION_GENERATOR.generate_perception(
HighLevelSemanticsSituation(ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[situation_object(TABLE)]),
chooser=RandomChooser.for_seed(0),
)
frame = learner_perception.frames[0]
property_assertions = frame.property_assertions
leg_perceptions = [
perception_with_handle(frame, "**(furniture) leg_0"),
perception_with_handle(frame, "**(furniture) leg_1"),
perception_with_handle(frame, "**(furniture) leg_2"),
perception_with_handle(frame, "**(furniture) leg_3"),
]
assert all(
HasBinaryProperty(leg_perception, INANIMATE) in property_assertions
for leg_perception in leg_perceptions)
def test_grounding_of_unsupported_objects():
box = situation_object(ontology_node=BOX)
ball = situation_object(ontology_node=BALL)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[box, ball],
always_relations=[Relation(IN_REGION, ball, Region(box, distance=INTERIOR))],
)
perception = _PERCEPTION_GENERATOR.generate_perception(
situation, chooser=RandomChooser.for_seed(0)
)
first_frame = perception.frames[0]
ball_perception = perception_with_handle(first_frame, "**ball_0")
ground_perception = perception_with_handle(first_frame, "the ground")
box_perception = perception_with_handle(first_frame, "**box_0")
first_frame_relations = first_frame.relations
assert on(ball_perception, ground_perception)[0] not in first_frame_relations
assert on(box_perception, ground_perception)[0] in first_frame_relations
def test_with_object_recognizer(language_mode):
integrated_learner = IntegratedTemplateLearner(
object_learner=LANGUAGE_MODE_TO_TEMPLATE_LEARNER_OBJECT_RECOGNIZER[language_mode],
attribute_learner=None,
relation_learner=None,
action_learner=None,
)
dad_situation_object = SituationObject.instantiate_ontology_node(
ontology_node=DAD, ontology=GAILA_PHASE_1_ONTOLOGY
)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[dad_situation_object]
)
perception_generator = HighLevelSemanticsSituationToDevelopmentalPrimitivePerceptionGenerator(
GAILA_PHASE_1_ONTOLOGY
)
# We explicitly exclude ground in perception generation
# this generates a static perception...
perception = perception_generator.generate_perception(
situation, chooser=RandomChooser.for_seed(0), include_ground=False
)
# so we need to construct a dynamic one by hand from two identical scenes
dynamic_perception = PerceptualRepresentation(
frames=[perception.frames[0], perception.frames[0]]
)
descriptions = integrated_learner.describe(dynamic_perception)
assert len(descriptions) == 1
assert (
language_mode == LanguageMode.ENGLISH
and one(descriptions.keys()).as_token_sequence() == ("Dad",)
) or (
language_mode == LanguageMode.CHINESE
and one(descriptions.keys()).as_token_sequence() == ("ba4 ba4",)
)
def test_big_ball():
ball1 = situation_object(BALL, debug_handle="ball_0")
ball2 = situation_object(BALL, debug_handle="ball_1")
ball_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[ball2, ball1],
always_relations=[bigger_than(ball1, ball2)],
)
assert (ball_situation.always_relations[0].first_slot.ontology_node ==
ball_situation.always_relations[0].second_slot.ontology_node)
ball_perception = _PERCEPTION_GENERATOR.generate_perception(
ball_situation, chooser=RandomChooser.for_seed(0))
perceived_objects = ball_perception.frames[0].perceived_objects
object_handles = set(obj.debug_handle for obj in perceived_objects)
assert object_handles == {"**ball_0", "**ball_1", "the ground"}
assert any(relation.relation_type == BIGGER_THAN_SAME_TYPE
for relation in ball_perception.frames[0].relations)
assert any(relation.relation_type == SMALLER_THAN_SAME_TYPE
for relation in ball_perception.frames[0].relations)
def build_object_learner_experiment_curriculum_train(
num_samples: Optional[int],
num_noise_objects: Optional[int],
language_generator: LanguageGenerator[
HighLevelSemanticsSituation, LinearizedDependencyTree
],
*,
params: Parameters = Parameters.empty(),
) -> Sequence[Phase1InstanceGroup]:
situations = make_multiple_object_situation(
num_samples, num_noise_objects, language_generator
)
accurate_language_chance = params.floating_point(
"accurate_language_percentage", default=0.5
)
output_situations = []
random.seed(params.integer("random_seed", default=0))
rng = RandomChooser.for_seed(params.integer("language_random_seed", default=0))
for (situation, language, perception) in situations.instances():
if random.random() <= accurate_language_chance:
output_language = language
else:
# Make Invalid Language
if situation and isinstance(situation, HighLevelSemanticsSituation):
# First, gather all OntologyNodes which aren't already present in the situation
present_ontology_nodes = [
_object.ontology_node for _object in situation.all_objects
]
valid_other_objects = [
node
for node in PHASE_1_CURRICULUM_OBJECTS
if node not in present_ontology_nodes
]
# Then choose one at random
chosen_ontology_node = rng.choice(valid_other_objects)
# Make a fake situation with just this object in it, ignoring colors
wrong_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_2_ONTOLOGY,
salient_objects=[
SituationObject.instantiate_ontology_node(
chosen_ontology_node, ontology=GAILA_PHASE_2_ONTOLOGY
)
],
syntax_hints=[IGNORE_COLORS],
)
# Generate the language as if it came from this fake situation rather than the original one
fake_language = only(
language_generator.generate_language(wrong_situation, chooser=rng)
)
output_language = LinearizedDependencyTree(
dependency_tree=fake_language.dependency_tree,
surface_token_order=fake_language.surface_token_order,
accurate=False,
)
else:
raise RuntimeError(
f"Unable to make invalid language without a situation of type HighlevelSemanticsSituation. Got situation: {situation}"
)
output_situations.append((situation, output_language, perception))
return [
AblatedLanguageSituationsInstanceGroup(
name=f"{situations.name()}_ablated", instances=output_situations
)
]
def run_experiment(learner, curricula, experiment_id):
# Teach each pretraining curriculum
for curriculum in curricula:
print("Teaching", curriculum.name(), "curriculum")
for (
_,
linguistic_description,
perceptual_representation,
) in curriculum.instances():
# Get the object matches first - prepositison learner can't learn without already recognized objects
# print('Observation: ',' '.join(linguistic_description.as_token_sequence()))
learner.observe(
LearningExample(perceptual_representation,
linguistic_description))
# Teach each kind member
empty_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_2_ONTOLOGY,
salient_objects=immutableset([
SituationObject.instantiate_ontology_node(
ontology_node=GROUND,
debug_handle=GROUND.handle,
ontology=GAILA_PHASE_1_ONTOLOGY,
)
]),
)
empty_perception = GAILA_PHASE_2_PERCEPTION_GENERATOR.generate_perception(
empty_situation, PHASE1_CHOOSER_FACTORY())
pseudoword_to_kind = {"wug": "animal", "vonk": "food", "snarp": "people"}
print("Teaching new objects in known categories")
for word, kind in pseudoword_to_kind.items():
print("Observation: ", word, "s", "are", kind, "s")
learner.observe(
LearningExample(
empty_perception,
TokenSequenceLinguisticDescription(tokens=(word, "s", "are",
kind, "s"))
if kind != "people" else TokenSequenceLinguisticDescription(
tokens=(word, "s", "are", kind, "s")),
))
semantics_manager: SemanticsManager = SemanticsManager(
semantics_graph=learner.semantics_graph)
complete_results = []
print("\nResults for ", experiment_id)
for word, _ in pseudoword_to_kind.items():
results = [(kind,
semantics_manager.evaluate_kind_membership(word, kind))
for kind in pseudoword_to_kind.values()]
complete_results.append(results)
results_df = pd.DataFrame(
[[np.asscalar(i[1]) for i in l] for l in complete_results],
columns=["Animal", "Food", "People"],
)
results_df.insert(0, "Words", pseudoword_to_kind.keys())
# print(results_df.to_csv(index=False))
print(tabulate(results_df, headers="keys", tablefmt="psql"))
learner.log_hypotheses(Path(f"./renders/{experiment_id}"))
learner.render_semantics_to_file(
graph=learner.semantics_graph,
graph_name="semantics",
output_file=Path(f"./renders/{experiment_id}/semantics.png"),
)