def _put_on_body_part_template( # X puts Y on body part agent: TemplateObjectVariable, theme: TemplateObjectVariable, goal_reference: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], ) -> Phase1SituationTemplate: return Phase1SituationTemplate( f"{agent.handle}-puts-{theme.handle}-on-{goal_reference.handle}", salient_object_variables=[agent, theme, goal_reference], background_object_variables=background, actions=[ Action( PUT, argument_roles_to_fillers=[ (AGENT, agent), (THEME, theme), ( GOAL, Region( goal_reference, distance=EXTERIOR_BUT_IN_CONTACT, direction=GRAVITATIONAL_UP, ), ), ], ) ], constraining_relations=flatten_relations( bigger_than([agent, goal_reference], theme) ), asserted_always_relations=flatten_relations(has(agent, goal_reference)), )
def test_before_after_relations_asserted(): ball = object_variable("ball", root_node=BALL) box = object_variable("box", root_node=BOX) ground = object_variable("ground", root_node=GROUND) template_action = Phase1SituationTemplate( "Before/After Relation", salient_object_variables=[ball, box], background_object_variables=[ground], actions=[ Action( ROLL, argument_roles_to_fillers=[(AGENT, ball)], auxiliary_variable_bindings=[(ROLL_SURFACE_AUXILIARY, ground)], ) ], before_action_relations=flatten_relations([on(ball, box)]), after_action_relations=flatten_relations([far(ball, box)]), ) situation_with_relations = tuple( sampled( template_action, ontology=GAILA_PHASE_1_ONTOLOGY, chooser=RandomChooser.for_seed(0), max_to_sample=1, )) assert situation_with_relations[0].before_action_relations assert situation_with_relations[0].after_action_relations
def _go_under_template( agent: TemplateObjectVariable, goal_object: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], *, is_distal: bool, # pylint:disable=unused-argument ) -> Phase1SituationTemplate: return Phase1SituationTemplate( f"go_under-{agent.handle}-under-{goal_object.handle}", salient_object_variables=[agent, goal_object], background_object_variables=background, actions=[ Action( GO, argument_roles_to_fillers=[ (AGENT, agent), ( GOAL, Region( goal_object, distance=PROXIMAL, direction=GRAVITATIONAL_DOWN ), ), ], ) ], before_action_relations=[negate(on(goal_object, GROUND_OBJECT_TEMPLATE))], asserted_always_relations=[negate(on(goal_object, GROUND_OBJECT_TEMPLATE))], after_action_relations=[ negate(on(goal_object, GROUND_OBJECT_TEMPLATE)), near(agent, goal_object), ], constraining_relations=flatten_relations(bigger_than(goal_object, agent)), )
def _behind_template( figure: TemplateObjectVariable, ground: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], *, is_training: bool, is_near: bool, speaker_root_node: OntologyNode = PERSON, background_relations: Iterable[Relation[Any]] = immutableset(), ) -> Phase1SituationTemplate: handle = "training" if is_training else "testing" direction = Direction(positive=False, relative_to_axis=FacingAddresseeAxis(ground)) speaker = standard_object("speaker", speaker_root_node, added_properties=[IS_SPEAKER]) addressee = standard_object("addressee", LEARNER, added_properties=[IS_ADDRESSEE]) computed_background = [speaker, addressee] computed_background.extend(background) relations = [ near(figure, ground, direction=direction) if is_near else far(figure, ground, direction=direction) ] relations.extend(background_relations) # type: ignore return Phase1SituationTemplate( f"preposition-{handle}-{figure.handle}-behind-{ground.handle}", salient_object_variables=[figure, ground], background_object_variables=computed_background, asserted_always_relations=flatten_relations(relations), gazed_objects=[figure], )
def _beside_template( figure: TemplateObjectVariable, ground: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], *, is_right: bool, is_training: bool, background_relations: Iterable[Relation[Any]] = immutableset(), ) -> Phase1SituationTemplate: direction_str = "right" if is_right else "left" handle = "training" if is_training else "testing" relations = [ near( figure, ground, direction=Direction( positive=is_right, relative_to_axis=HorizontalAxisOfObject(ground, index=0), ), ) ] relations.extend(background_relations) # type: ignore return Phase1SituationTemplate( f"preposition-{handle}-{figure.handle}-beside-{ground.handle}-{direction_str}", salient_object_variables=[figure, ground], background_object_variables=background, asserted_always_relations=flatten_relations(relations), gazed_objects=[figure], )
def test_object_not_on_ground(): """ Intended to test that one can specify an object is not on the ground """ table = situation_object(TABLE) ground = situation_object(GROUND) perception = _PERCEPTION_GENERATOR.generate_perception( HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=immutableset([table]), other_objects=immutableset([ground]), always_relations=flatten_relations(negate(on(table, ground))), ), chooser=RandomChooser.for_seed(0), ) frame = perception.frames[0] relations = frame.relations table_perception = perception_with_handle(frame, "**table_0") ground_perception = perception_with_handle(frame, "the ground") assert not any( relation.relation_type == IN_REGION and relation.first_slot == table_perception and isinstance(relation.second_slot, Region) and relation.region.reference_object == ground_perception and relation.region.distance == EXTERIOR_BUT_IN_CONTACT for relation in relations )
def _under_template( figure: TemplateObjectVariable, ground: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], *, is_training: bool, is_distal: bool, syntax_hints: Iterable[str] = immutableset(), background_relations: Iterable[TemplateObjectVariable] = immutableset(), ) -> Phase1SituationTemplate: handle = "training" if is_training else "testing" relations = [ negate(on(figure, GROUND_OBJECT_TEMPLATE)), strictly_under(ground, figure, dist=DISTAL if is_distal else PROXIMAL), ] relations.extend(background_relations) # type: ignore return Phase1SituationTemplate( f"preposition-{handle}-{figure.handle}-under-{ground.handle}", salient_object_variables=[figure, ground], background_object_variables=background, asserted_always_relations=flatten_relations(relations), constraining_relations=[bigger_than(ground, figure)], gazed_objects=[figure], syntax_hints=syntax_hints, )
def _fly_under_template( # A bird flies under a chair agent: TemplateObjectVariable, object_in_path: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], ) -> Phase1SituationTemplate: return Phase1SituationTemplate( f"{agent.handle}-flies-under-{object_in_path.handle}", salient_object_variables=[agent, object_in_path], background_object_variables=background, actions=[ Action( FLY, argument_roles_to_fillers=[(AGENT, agent)], during=DuringAction( at_some_point=flatten_relations(strictly_above(object_in_path, agent)) ), ) ], asserted_always_relations=[negate(on(object_in_path, GROUND_OBJECT_TEMPLATE))], before_action_relations=[negate(on(object_in_path, GROUND_OBJECT_TEMPLATE))], after_action_relations=[negate(on(object_in_path, GROUND_OBJECT_TEMPLATE))], constraining_relations=flatten_relations(bigger_than(object_in_path, agent)), )
def _on_template( figure: TemplateObjectVariable, ground: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], *, is_training: bool, background_relations: Iterable[Relation[Any]] = immutableset(), ) -> Phase1SituationTemplate: handle = "training" if is_training else "testing" relations = [on(figure, ground)] relations.extend(background_relations) # type: ignore return Phase1SituationTemplate( f"preposition-{handle}-{figure.handle}-on-{ground.handle}", salient_object_variables=[figure, ground], background_object_variables=background, asserted_always_relations=flatten_relations(relations), gazed_objects=[figure], )
def _fly_over_template( # A bird flies over a ball agent: TemplateObjectVariable, object_in_path: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], ) -> Phase1SituationTemplate: return Phase1SituationTemplate( f"{agent.handle}-flies-over-{object_in_path.handle}", salient_object_variables=[agent, object_in_path], background_object_variables=background, actions=[ Action( FLY, argument_roles_to_fillers=[(AGENT, agent)], during=DuringAction( at_some_point=flatten_relations(strictly_above(agent, object_in_path)) ), ) ], )
def _go_in_template( agent: TemplateObjectVariable, goal_object: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], ) -> Phase1SituationTemplate: return Phase1SituationTemplate( f"go_in-{agent.handle}-in-{goal_object.handle}", salient_object_variables=[agent, goal_object], background_object_variables=background, actions=[ Action( GO, argument_roles_to_fillers=[ (AGENT, agent), (GOAL, Region(goal_object, distance=INTERIOR)), ], ) ], constraining_relations=flatten_relations(bigger_than(goal_object, agent)), after_action_relations=[inside(agent, goal_object)], )
def _jump_over_template( # "Mom jumps over a ball" agent: TemplateObjectVariable, object_in_path: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], ) -> Phase1SituationTemplate: return Phase1SituationTemplate( f"{agent.handle}-jumps-over-{object_in_path.handle}", salient_object_variables=[agent, object_in_path], background_object_variables=background, actions=[ Action( JUMP, argument_roles_to_fillers=[(AGENT, agent)], during=DuringAction( at_some_point=flatten_relations( strictly_above(agent, object_in_path) ), objects_to_paths=[ ( agent, SpatialPath( operator=VIA, reference_source_object=Region( object_in_path, direction=GRAVITATIONAL_UP, distance=DISTAL, ), reference_destination_object=GROUND_OBJECT_TEMPLATE, ), ) ], ), auxiliary_variable_bindings=[ (JUMP_INITIAL_SUPPORTER_AUX, GROUND_OBJECT_TEMPLATE) ], ) ], asserted_always_relations=[negate(on(agent, GROUND_OBJECT_TEMPLATE))], )
def _put_in_template( agent: TemplateObjectVariable, theme: TemplateObjectVariable, goal_reference: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], ) -> Phase1SituationTemplate: return Phase1SituationTemplate( f"{agent.handle}-puts-{theme.handle}-in-{goal_reference.handle}", salient_object_variables=[agent, theme, goal_reference], background_object_variables=background, actions=[ Action( PUT, argument_roles_to_fillers=[ (AGENT, agent), (THEME, theme), (GOAL, Region(goal_reference, distance=INTERIOR)), ], ) ], constraining_relations=flatten_relations(bigger_than(goal_reference, theme)), )
def _in_front_template( figure: TemplateObjectVariable, ground: TemplateObjectVariable, background: Iterable[TemplateObjectVariable], *, is_training: bool, is_near: bool, background_relations: Iterable[Relation[Any]] = immutableset(), ) -> Phase1SituationTemplate: handle = "training" if is_training else "testing" direction = Direction(positive=True, relative_to_axis=FacingAddresseeAxis(ground)) relations = [ near(figure, ground, direction=direction) if is_near else far(figure, ground, direction=direction) ] relations.extend(background_relations) # type: ignore return Phase1SituationTemplate( f"preposition-{handle}-{figure.handle}-behind-{ground.handle}", salient_object_variables=[figure, ground], background_object_variables=background, asserted_always_relations=flatten_relations(relations), gazed_objects=[figure], )
def background_relations_builder( background_objects: Iterable[TemplateObjectVariable], num_relations: int, *, target: Optional[TemplateObjectVariable] = None, target_2: Optional[TemplateObjectVariable] = None, add_noise: bool = True, include_targets_in_noise: bool = False, chooser: RandomChooser = RandomChooser.for_seed(0), ) -> Iterable[Relation[Any]]: if add_noise: potential_objects = list(background_objects) if target and include_targets_in_noise: potential_objects.append(target) if target_2 and include_targets_in_noise: potential_objects.append(target_2) if len(potential_objects) < 2: return immutableset() relations = [] for _ in range(num_relations): choice = chooser.choice(NOISE_RELATION_DSL_OPTIONS) if choice == "on": relations.append( on( chooser.choice(potential_objects), chooser.choice(potential_objects), )) elif choice == "beside": obj_choice_2 = chooser.choice(potential_objects) relations.append( near( chooser.choice(potential_objects), obj_choice_2, direction=Direction( positive=chooser.choice(BOOL_SET), relative_to_axis=HorizontalAxisOfObject( obj_choice_2, index=0), ), )) elif choice == "under": relations.append( strictly_under( chooser.choice(potential_objects), chooser.choice(potential_objects), dist=DISTAL if chooser.choice(BOOL_SET) else PROXIMAL, )) elif choice == "in_front": obj_choice_2 = chooser.choice(potential_objects) direction = Direction( positive=chooser.choice(BOOL_SET), relative_to_axis=FacingAddresseeAxis(obj_choice_2), ) relations.append( near( chooser.choice(potential_objects), obj_choice_2, direction=direction, ) if chooser.choice(BOOL_SET) else far( chooser.choice(potential_objects), obj_choice_2, direction=direction, )) else: raise RuntimeError( "Invalid relation type in background relations") return flatten_relations(relations) else: return immutableset()
def test_subset_learner_subobject(): mom = SituationObject.instantiate_ontology_node( ontology_node=MOM, ontology=GAILA_PHASE_1_ONTOLOGY) head = SituationObject.instantiate_ontology_node( ontology_node=HEAD, ontology=GAILA_PHASE_1_ONTOLOGY) hand = SituationObject.instantiate_ontology_node( ontology_node=HAND, ontology=GAILA_PHASE_1_ONTOLOGY) ball = SituationObject.instantiate_ontology_node( ontology_node=BALL, ontology=GAILA_PHASE_1_ONTOLOGY) house = SituationObject.instantiate_ontology_node( ontology_node=HOUSE, ontology=GAILA_PHASE_1_ONTOLOGY) ground = SituationObject.instantiate_ontology_node( ontology_node=GROUND, ontology=GAILA_PHASE_1_ONTOLOGY) mom_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=immutableset([mom])) floating_head_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=immutableset([head]), other_objects=immutableset([ground]), always_relations=flatten_relations(negate(on(head, ground))), ) # Need to include some extra situations so that the learner will prune its semantics for 'a' # away and not recognize it as an object. floating_hand_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=immutableset([hand]), other_objects=immutableset([ground]), always_relations=flatten_relations(negate(on(hand, ground))), ) floating_ball_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=immutableset([ball]), other_objects=immutableset([ground]), always_relations=flatten_relations(negate(on(ball, ground))), ) floating_house_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=immutableset([house]), other_objects=immutableset([ground]), always_relations=flatten_relations(negate(on(house, ground))), ) object_learner = SubsetObjectLearnerNew(ontology=GAILA_PHASE_1_ONTOLOGY, beam_size=5, language_mode=LanguageMode.ENGLISH) for situation in [ mom_situation, floating_head_situation, floating_hand_situation, floating_ball_situation, floating_house_situation, ]: perceptual_representation = GAILA_PHASE_1_PERCEPTION_GENERATOR.generate_perception( situation, chooser=RandomChooser.for_seed(0)) for linguistic_description in GAILA_PHASE_1_LANGUAGE_GENERATOR.generate_language( situation, chooser=RandomChooser.for_seed(0)): perception_graph = PerceptionGraph.from_frame( perceptual_representation.frames[0]) object_learner.learn_from( LanguagePerceptionSemanticAlignment( language_concept_alignment=LanguageConceptAlignment. create_unaligned(language=linguistic_description), perception_semantic_alignment=PerceptionSemanticAlignment( perception_graph=perception_graph, semantic_nodes=[]), )) mom_perceptual_representation = GAILA_PHASE_1_PERCEPTION_GENERATOR.generate_perception( mom_situation, chooser=RandomChooser.for_seed(0)) perception_graph = PerceptionGraph.from_frame( mom_perceptual_representation.frames[0]) enriched = object_learner.enrich_during_description( PerceptionSemanticAlignment.create_unaligned(perception_graph)) semantic_node_types_and_debug_strings = { (type(semantic_node), semantic_node.concept.debug_string) for semantic_node in enriched.semantic_nodes } assert (ObjectSemanticNode, "Mom") in semantic_node_types_and_debug_strings assert (ObjectSemanticNode, "head") in semantic_node_types_and_debug_strings assert (ObjectSemanticNode, "hand") in semantic_node_types_and_debug_strings
# 2 with square seat _CHAIR_2_SCHEMA = ObjectStructuralSchema( CHAIR, sub_objects=[ _CHAIR_SCHEMA_BACK, _CHAIR_SCHEMA_SQUARE_SEAT, _CHAIR_SCHEMA_FRONT_LEFT_LEG, _CHAIR_SCHEMA_FRONT_RIGHT_LEG, _CHAIR_SCHEMA_BACK_LEFT_LEG, _CHAIR_SCHEMA_BACK_RIGHT_LEG, ], sub_object_relations=flatten_relations( [ contacts(_CHAIR_LEGS, _CHAIR_SCHEMA_SQUARE_SEAT), above(_CHAIR_SCHEMA_SQUARE_SEAT, _CHAIR_LEGS), contacts(_CHAIR_SCHEMA_BACK, _CHAIR_SCHEMA_SQUARE_SEAT), above(_CHAIR_SCHEMA_BACK, _CHAIR_SCHEMA_SQUARE_SEAT), ] ), axes=_CHAIR_SCHEMA_BACK.schema.axes.copy(), ) # 3 with no back _CHAIR_3_SCHEMA = ObjectStructuralSchema( CHAIR, sub_objects=[ _CHAIR_SCHEMA_SEAT, _CHAIR_SCHEMA_FRONT_LEFT_LEG, _CHAIR_SCHEMA_FRONT_RIGHT_LEG, _CHAIR_SCHEMA_BACK_LEFT_LEG, _CHAIR_SCHEMA_BACK_RIGHT_LEG, ],