def _put_on_body_part_template(
# X puts Y on body part
agent: TemplateObjectVariable,
theme: TemplateObjectVariable,
goal_reference: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
) -> Phase1SituationTemplate:
return Phase1SituationTemplate(
f"{agent.handle}-puts-{theme.handle}-on-{goal_reference.handle}",
salient_object_variables=[agent, theme, goal_reference],
background_object_variables=background,
actions=[
Action(
PUT,
argument_roles_to_fillers=[
(AGENT, agent),
(THEME, theme),
(
GOAL,
Region(
goal_reference,
distance=EXTERIOR_BUT_IN_CONTACT,
direction=GRAVITATIONAL_UP,
),
),
],
)
],
constraining_relations=flatten_relations(
bigger_than([agent, goal_reference], theme)
),
asserted_always_relations=flatten_relations(has(agent, goal_reference)),
)
def test_before_after_relations_asserted():
ball = object_variable("ball", root_node=BALL)
box = object_variable("box", root_node=BOX)
ground = object_variable("ground", root_node=GROUND)
template_action = Phase1SituationTemplate(
"Before/After Relation",
salient_object_variables=[ball, box],
background_object_variables=[ground],
actions=[
Action(
ROLL,
argument_roles_to_fillers=[(AGENT, ball)],
auxiliary_variable_bindings=[(ROLL_SURFACE_AUXILIARY, ground)],
)
],
before_action_relations=flatten_relations([on(ball, box)]),
after_action_relations=flatten_relations([far(ball, box)]),
)
situation_with_relations = tuple(
sampled(
template_action,
ontology=GAILA_PHASE_1_ONTOLOGY,
chooser=RandomChooser.for_seed(0),
max_to_sample=1,
))
assert situation_with_relations[0].before_action_relations
assert situation_with_relations[0].after_action_relations
def _go_under_template(
agent: TemplateObjectVariable,
goal_object: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
*,
is_distal: bool, # pylint:disable=unused-argument
) -> Phase1SituationTemplate:
return Phase1SituationTemplate(
f"go_under-{agent.handle}-under-{goal_object.handle}",
salient_object_variables=[agent, goal_object],
background_object_variables=background,
actions=[
Action(
GO,
argument_roles_to_fillers=[
(AGENT, agent),
(
GOAL,
Region(
goal_object, distance=PROXIMAL, direction=GRAVITATIONAL_DOWN
),
),
],
)
],
before_action_relations=[negate(on(goal_object, GROUND_OBJECT_TEMPLATE))],
asserted_always_relations=[negate(on(goal_object, GROUND_OBJECT_TEMPLATE))],
after_action_relations=[
negate(on(goal_object, GROUND_OBJECT_TEMPLATE)),
near(agent, goal_object),
],
constraining_relations=flatten_relations(bigger_than(goal_object, agent)),
)
def _behind_template(
figure: TemplateObjectVariable,
ground: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
*,
is_training: bool,
is_near: bool,
speaker_root_node: OntologyNode = PERSON,
background_relations: Iterable[Relation[Any]] = immutableset(),
) -> Phase1SituationTemplate:
handle = "training" if is_training else "testing"
direction = Direction(positive=False,
relative_to_axis=FacingAddresseeAxis(ground))
speaker = standard_object("speaker",
speaker_root_node,
added_properties=[IS_SPEAKER])
addressee = standard_object("addressee",
LEARNER,
added_properties=[IS_ADDRESSEE])
computed_background = [speaker, addressee]
computed_background.extend(background)
relations = [
near(figure, ground, direction=direction)
if is_near else far(figure, ground, direction=direction)
]
relations.extend(background_relations) # type: ignore
return Phase1SituationTemplate(
f"preposition-{handle}-{figure.handle}-behind-{ground.handle}",
salient_object_variables=[figure, ground],
background_object_variables=computed_background,
asserted_always_relations=flatten_relations(relations),
gazed_objects=[figure],
)
def _beside_template(
figure: TemplateObjectVariable,
ground: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
*,
is_right: bool,
is_training: bool,
background_relations: Iterable[Relation[Any]] = immutableset(),
) -> Phase1SituationTemplate:
direction_str = "right" if is_right else "left"
handle = "training" if is_training else "testing"
relations = [
near(
figure,
ground,
direction=Direction(
positive=is_right,
relative_to_axis=HorizontalAxisOfObject(ground, index=0),
),
)
]
relations.extend(background_relations) # type: ignore
return Phase1SituationTemplate(
f"preposition-{handle}-{figure.handle}-beside-{ground.handle}-{direction_str}",
salient_object_variables=[figure, ground],
background_object_variables=background,
asserted_always_relations=flatten_relations(relations),
gazed_objects=[figure],
)
def test_object_not_on_ground():
"""
Intended to test that one can specify an object is not on the ground
"""
table = situation_object(TABLE)
ground = situation_object(GROUND)
perception = _PERCEPTION_GENERATOR.generate_perception(
HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=immutableset([table]),
other_objects=immutableset([ground]),
always_relations=flatten_relations(negate(on(table, ground))),
),
chooser=RandomChooser.for_seed(0),
)
frame = perception.frames[0]
relations = frame.relations
table_perception = perception_with_handle(frame, "**table_0")
ground_perception = perception_with_handle(frame, "the ground")
assert not any(
relation.relation_type == IN_REGION
and relation.first_slot == table_perception
and isinstance(relation.second_slot, Region)
and relation.region.reference_object == ground_perception
and relation.region.distance == EXTERIOR_BUT_IN_CONTACT
for relation in relations
)
def _under_template(
figure: TemplateObjectVariable,
ground: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
*,
is_training: bool,
is_distal: bool,
syntax_hints: Iterable[str] = immutableset(),
background_relations: Iterable[TemplateObjectVariable] = immutableset(),
) -> Phase1SituationTemplate:
handle = "training" if is_training else "testing"
relations = [
negate(on(figure, GROUND_OBJECT_TEMPLATE)),
strictly_under(ground, figure, dist=DISTAL if is_distal else PROXIMAL),
]
relations.extend(background_relations) # type: ignore
return Phase1SituationTemplate(
f"preposition-{handle}-{figure.handle}-under-{ground.handle}",
salient_object_variables=[figure, ground],
background_object_variables=background,
asserted_always_relations=flatten_relations(relations),
constraining_relations=[bigger_than(ground, figure)],
gazed_objects=[figure],
syntax_hints=syntax_hints,
)
def _fly_under_template(
# A bird flies under a chair
agent: TemplateObjectVariable,
object_in_path: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
) -> Phase1SituationTemplate:
return Phase1SituationTemplate(
f"{agent.handle}-flies-under-{object_in_path.handle}",
salient_object_variables=[agent, object_in_path],
background_object_variables=background,
actions=[
Action(
FLY,
argument_roles_to_fillers=[(AGENT, agent)],
during=DuringAction(
at_some_point=flatten_relations(strictly_above(object_in_path, agent))
),
)
],
asserted_always_relations=[negate(on(object_in_path, GROUND_OBJECT_TEMPLATE))],
before_action_relations=[negate(on(object_in_path, GROUND_OBJECT_TEMPLATE))],
after_action_relations=[negate(on(object_in_path, GROUND_OBJECT_TEMPLATE))],
constraining_relations=flatten_relations(bigger_than(object_in_path, agent)),
)
def _on_template(
figure: TemplateObjectVariable,
ground: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
*,
is_training: bool,
background_relations: Iterable[Relation[Any]] = immutableset(),
) -> Phase1SituationTemplate:
handle = "training" if is_training else "testing"
relations = [on(figure, ground)]
relations.extend(background_relations) # type: ignore
return Phase1SituationTemplate(
f"preposition-{handle}-{figure.handle}-on-{ground.handle}",
salient_object_variables=[figure, ground],
background_object_variables=background,
asserted_always_relations=flatten_relations(relations),
gazed_objects=[figure],
)
def _fly_over_template(
# A bird flies over a ball
agent: TemplateObjectVariable,
object_in_path: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
) -> Phase1SituationTemplate:
return Phase1SituationTemplate(
f"{agent.handle}-flies-over-{object_in_path.handle}",
salient_object_variables=[agent, object_in_path],
background_object_variables=background,
actions=[
Action(
FLY,
argument_roles_to_fillers=[(AGENT, agent)],
during=DuringAction(
at_some_point=flatten_relations(strictly_above(agent, object_in_path))
),
)
],
)
def _go_in_template(
agent: TemplateObjectVariable,
goal_object: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
) -> Phase1SituationTemplate:
return Phase1SituationTemplate(
f"go_in-{agent.handle}-in-{goal_object.handle}",
salient_object_variables=[agent, goal_object],
background_object_variables=background,
actions=[
Action(
GO,
argument_roles_to_fillers=[
(AGENT, agent),
(GOAL, Region(goal_object, distance=INTERIOR)),
],
)
],
constraining_relations=flatten_relations(bigger_than(goal_object, agent)),
after_action_relations=[inside(agent, goal_object)],
)
def _jump_over_template(
# "Mom jumps over a ball"
agent: TemplateObjectVariable,
object_in_path: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
) -> Phase1SituationTemplate:
return Phase1SituationTemplate(
f"{agent.handle}-jumps-over-{object_in_path.handle}",
salient_object_variables=[agent, object_in_path],
background_object_variables=background,
actions=[
Action(
JUMP,
argument_roles_to_fillers=[(AGENT, agent)],
during=DuringAction(
at_some_point=flatten_relations(
strictly_above(agent, object_in_path)
),
objects_to_paths=[
(
agent,
SpatialPath(
operator=VIA,
reference_source_object=Region(
object_in_path,
direction=GRAVITATIONAL_UP,
distance=DISTAL,
),
reference_destination_object=GROUND_OBJECT_TEMPLATE,
),
)
],
),
auxiliary_variable_bindings=[
(JUMP_INITIAL_SUPPORTER_AUX, GROUND_OBJECT_TEMPLATE)
],
)
],
asserted_always_relations=[negate(on(agent, GROUND_OBJECT_TEMPLATE))],
)
def _put_in_template(
agent: TemplateObjectVariable,
theme: TemplateObjectVariable,
goal_reference: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
) -> Phase1SituationTemplate:
return Phase1SituationTemplate(
f"{agent.handle}-puts-{theme.handle}-in-{goal_reference.handle}",
salient_object_variables=[agent, theme, goal_reference],
background_object_variables=background,
actions=[
Action(
PUT,
argument_roles_to_fillers=[
(AGENT, agent),
(THEME, theme),
(GOAL, Region(goal_reference, distance=INTERIOR)),
],
)
],
constraining_relations=flatten_relations(bigger_than(goal_reference, theme)),
)
def _in_front_template(
figure: TemplateObjectVariable,
ground: TemplateObjectVariable,
background: Iterable[TemplateObjectVariable],
*,
is_training: bool,
is_near: bool,
background_relations: Iterable[Relation[Any]] = immutableset(),
) -> Phase1SituationTemplate:
handle = "training" if is_training else "testing"
direction = Direction(positive=True,
relative_to_axis=FacingAddresseeAxis(ground))
relations = [
near(figure, ground, direction=direction)
if is_near else far(figure, ground, direction=direction)
]
relations.extend(background_relations) # type: ignore
return Phase1SituationTemplate(
f"preposition-{handle}-{figure.handle}-behind-{ground.handle}",
salient_object_variables=[figure, ground],
background_object_variables=background,
asserted_always_relations=flatten_relations(relations),
gazed_objects=[figure],
)
def background_relations_builder(
background_objects: Iterable[TemplateObjectVariable],
num_relations: int,
*,
target: Optional[TemplateObjectVariable] = None,
target_2: Optional[TemplateObjectVariable] = None,
add_noise: bool = True,
include_targets_in_noise: bool = False,
chooser: RandomChooser = RandomChooser.for_seed(0),
) -> Iterable[Relation[Any]]:
if add_noise:
potential_objects = list(background_objects)
if target and include_targets_in_noise:
potential_objects.append(target)
if target_2 and include_targets_in_noise:
potential_objects.append(target_2)
if len(potential_objects) < 2:
return immutableset()
relations = []
for _ in range(num_relations):
choice = chooser.choice(NOISE_RELATION_DSL_OPTIONS)
if choice == "on":
relations.append(
on(
chooser.choice(potential_objects),
chooser.choice(potential_objects),
))
elif choice == "beside":
obj_choice_2 = chooser.choice(potential_objects)
relations.append(
near(
chooser.choice(potential_objects),
obj_choice_2,
direction=Direction(
positive=chooser.choice(BOOL_SET),
relative_to_axis=HorizontalAxisOfObject(
obj_choice_2, index=0),
),
))
elif choice == "under":
relations.append(
strictly_under(
chooser.choice(potential_objects),
chooser.choice(potential_objects),
dist=DISTAL if chooser.choice(BOOL_SET) else PROXIMAL,
))
elif choice == "in_front":
obj_choice_2 = chooser.choice(potential_objects)
direction = Direction(
positive=chooser.choice(BOOL_SET),
relative_to_axis=FacingAddresseeAxis(obj_choice_2),
)
relations.append(
near(
chooser.choice(potential_objects),
obj_choice_2,
direction=direction,
) if chooser.choice(BOOL_SET) else far(
chooser.choice(potential_objects),
obj_choice_2,
direction=direction,
))
else:
raise RuntimeError(
"Invalid relation type in background relations")
return flatten_relations(relations)
else:
return immutableset()
def test_subset_learner_subobject():
mom = SituationObject.instantiate_ontology_node(
ontology_node=MOM, ontology=GAILA_PHASE_1_ONTOLOGY)
head = SituationObject.instantiate_ontology_node(
ontology_node=HEAD, ontology=GAILA_PHASE_1_ONTOLOGY)
hand = SituationObject.instantiate_ontology_node(
ontology_node=HAND, ontology=GAILA_PHASE_1_ONTOLOGY)
ball = SituationObject.instantiate_ontology_node(
ontology_node=BALL, ontology=GAILA_PHASE_1_ONTOLOGY)
house = SituationObject.instantiate_ontology_node(
ontology_node=HOUSE, ontology=GAILA_PHASE_1_ONTOLOGY)
ground = SituationObject.instantiate_ontology_node(
ontology_node=GROUND, ontology=GAILA_PHASE_1_ONTOLOGY)
mom_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=immutableset([mom]))
floating_head_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=immutableset([head]),
other_objects=immutableset([ground]),
always_relations=flatten_relations(negate(on(head, ground))),
)
# Need to include some extra situations so that the learner will prune its semantics for 'a'
# away and not recognize it as an object.
floating_hand_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=immutableset([hand]),
other_objects=immutableset([ground]),
always_relations=flatten_relations(negate(on(hand, ground))),
)
floating_ball_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=immutableset([ball]),
other_objects=immutableset([ground]),
always_relations=flatten_relations(negate(on(ball, ground))),
)
floating_house_situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=immutableset([house]),
other_objects=immutableset([ground]),
always_relations=flatten_relations(negate(on(house, ground))),
)
object_learner = SubsetObjectLearnerNew(ontology=GAILA_PHASE_1_ONTOLOGY,
beam_size=5,
language_mode=LanguageMode.ENGLISH)
for situation in [
mom_situation,
floating_head_situation,
floating_hand_situation,
floating_ball_situation,
floating_house_situation,
]:
perceptual_representation = GAILA_PHASE_1_PERCEPTION_GENERATOR.generate_perception(
situation, chooser=RandomChooser.for_seed(0))
for linguistic_description in GAILA_PHASE_1_LANGUAGE_GENERATOR.generate_language(
situation, chooser=RandomChooser.for_seed(0)):
perception_graph = PerceptionGraph.from_frame(
perceptual_representation.frames[0])
object_learner.learn_from(
LanguagePerceptionSemanticAlignment(
language_concept_alignment=LanguageConceptAlignment.
create_unaligned(language=linguistic_description),
perception_semantic_alignment=PerceptionSemanticAlignment(
perception_graph=perception_graph, semantic_nodes=[]),
))
mom_perceptual_representation = GAILA_PHASE_1_PERCEPTION_GENERATOR.generate_perception(
mom_situation, chooser=RandomChooser.for_seed(0))
perception_graph = PerceptionGraph.from_frame(
mom_perceptual_representation.frames[0])
enriched = object_learner.enrich_during_description(
PerceptionSemanticAlignment.create_unaligned(perception_graph))
semantic_node_types_and_debug_strings = {
(type(semantic_node), semantic_node.concept.debug_string)
for semantic_node in enriched.semantic_nodes
}
assert (ObjectSemanticNode, "Mom") in semantic_node_types_and_debug_strings
assert (ObjectSemanticNode,
"head") in semantic_node_types_and_debug_strings
assert (ObjectSemanticNode,
"hand") in semantic_node_types_and_debug_strings
# 2 with square seat
_CHAIR_2_SCHEMA = ObjectStructuralSchema(
CHAIR,
sub_objects=[
_CHAIR_SCHEMA_BACK,
_CHAIR_SCHEMA_SQUARE_SEAT,
_CHAIR_SCHEMA_FRONT_LEFT_LEG,
_CHAIR_SCHEMA_FRONT_RIGHT_LEG,
_CHAIR_SCHEMA_BACK_LEFT_LEG,
_CHAIR_SCHEMA_BACK_RIGHT_LEG,
],
sub_object_relations=flatten_relations(
[
contacts(_CHAIR_LEGS, _CHAIR_SCHEMA_SQUARE_SEAT),
above(_CHAIR_SCHEMA_SQUARE_SEAT, _CHAIR_LEGS),
contacts(_CHAIR_SCHEMA_BACK, _CHAIR_SCHEMA_SQUARE_SEAT),
above(_CHAIR_SCHEMA_BACK, _CHAIR_SCHEMA_SQUARE_SEAT),
]
),
axes=_CHAIR_SCHEMA_BACK.schema.axes.copy(),
)
# 3 with no back
_CHAIR_3_SCHEMA = ObjectStructuralSchema(
CHAIR,
sub_objects=[
_CHAIR_SCHEMA_SEAT,
_CHAIR_SCHEMA_FRONT_LEFT_LEG,
_CHAIR_SCHEMA_FRONT_RIGHT_LEG,
_CHAIR_SCHEMA_BACK_LEFT_LEG,
_CHAIR_SCHEMA_BACK_RIGHT_LEG,
],
