def test_recognizes_ontology_objects(object_type, language_mode): situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[ SituationObject.instantiate_ontology_node( ontology_node=object_type, ontology=GAILA_PHASE_1_ONTOLOGY) ], ) perception_generator = HighLevelSemanticsSituationToDevelopmentalPrimitivePerceptionGenerator( GAILA_PHASE_1_ONTOLOGY) perception = perception_generator.generate_perception( situation, chooser=RandomChooser.for_seed(0), include_ground=False) learner = IntegratedTemplateLearner( object_learner=LANGUAGE_MODE_TO_TEMPLATE_LEARNER_OBJECT_RECOGNIZER[ language_mode]) descriptions = learner.describe(perception) assert descriptions if language_mode == LanguageMode.ENGLISH: assert object_type.handle in one( descriptions.items())[0].as_token_sequence() else: mappings = ( GAILA_PHASE_1_CHINESE_LEXICON._ontology_node_to_word # pylint:disable=protected-access ) for k, v in mappings.items(): if k.handle == object_type.handle: assert v.base_form in one( descriptions.items())[0].as_token_sequence()
def build_object_multiples_situations( ontology: Ontology, *, samples_per_object: int = 3, chooser: RandomChooser) -> Iterable[HighLevelSemanticsSituation]: for object_type in PHASE_1_CURRICULUM_OBJECTS: # Exclude slow objects for now if object_type.handle in ["bird", "dog", "truck"]: continue is_liquid = ontology.has_all_properties(object_type, [LIQUID]) # don't want multiples of named people if not is_recognized_particular(ontology, object_type) and not is_liquid: for _ in range(samples_per_object): num_objects = chooser.choice(range(2, 4)) yield HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[ SituationObject.instantiate_ontology_node( ontology_node=object_type, debug_handle=object_type.handle + f"_{idx}", ontology=GAILA_PHASE_1_ONTOLOGY, ) for idx in range(num_objects) ], axis_info=AxesInfo(), )
def test_dynamic_prepositions_implicit_grounding(): truck = situation_object(ontology_node=TRUCK) baby = situation_object(ontology_node=BABY) situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[baby, truck], actions=[Action(FALL, argument_roles_to_fillers=[(THEME, baby)])], after_action_relations=[on(baby, truck)], ) perception = _PERCEPTION_GENERATOR.generate_perception( situation, chooser=RandomChooser.for_seed(0) ) first_frame = perception.frames[0] baby_perception = perception_with_handle(first_frame, "**baby_0") ground_perception = perception_with_handle(first_frame, "the ground") truck_perception = perception_with_handle(first_frame, "**truck_0") first_frame_relations = first_frame.relations second_frame_relations = perception.frames[1].relations assert on(baby_perception, truck_perception)[0] not in first_frame_relations assert on(baby_perception, truck_perception)[0] in second_frame_relations assert on(baby_perception, ground_perception)[0] not in second_frame_relations assert on(truck_perception, ground_perception)[0] in first_frame_relations assert on(truck_perception, ground_perception)[0] in second_frame_relations
def test_object_not_on_ground(): """ Intended to test that one can specify an object is not on the ground """ table = situation_object(TABLE) ground = situation_object(GROUND) perception = _PERCEPTION_GENERATOR.generate_perception( HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=immutableset([table]), other_objects=immutableset([ground]), always_relations=flatten_relations(negate(on(table, ground))), ), chooser=RandomChooser.for_seed(0), ) frame = perception.frames[0] relations = frame.relations table_perception = perception_with_handle(frame, "**table_0") ground_perception = perception_with_handle(frame, "the ground") assert not any( relation.relation_type == IN_REGION and relation.first_slot == table_perception and isinstance(relation.second_slot, Region) and relation.region.reference_object == ground_perception and relation.region.distance == EXTERIOR_BUT_IN_CONTACT for relation in relations )
def test_gaze_specified(): cookie = situation_object(COOKIE) table = situation_object(TABLE) dad = situation_object(DAD) situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[cookie, table, dad], actions=[ Action(EAT, argument_roles_to_fillers=[(AGENT, dad), (PATIENT, cookie)]) ], gazed_objects=[cookie], ) perception = _PERCEPTION_GENERATOR.generate_perception( situation, chooser=RandomChooser.for_seed(0)) frame = perception.frames[0] cookie_perception = perception_with_handle(frame, "**cookie_0") dad_perception = perception_with_handle(frame, "**Dad_0") table_perception = perception_with_handle(frame, "**table_0") # only the cookie is gazed at, because the user said so. assert HasBinaryProperty(cookie_perception, GAZED_AT) in frame.property_assertions assert HasBinaryProperty(dad_perception, GAZED_AT) not in frame.property_assertions assert HasBinaryProperty(table_perception, GAZED_AT) not in frame.property_assertions
def test_person_and_ball_color(): person = situation_object(PERSON) ball = situation_object(BALL, properties=[RED]) person_and_ball_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[person, ball] ) person_and_ball_perception = _PERCEPTION_GENERATOR.generate_perception( person_and_ball_situation, chooser=RandomChooser.for_seed(0) ) assert len(person_and_ball_perception.frames) == 1 frame = person_and_ball_perception.frames[0] assert ( prop_assertion is PropertyPerception for prop_assertion in frame.property_assertions ) person_perception = perception_with_handle(frame, "**person_0") ball_perception = perception_with_handle(frame, "**ball_0") assert HasBinaryProperty(person_perception, ANIMATE) in frame.property_assertions assert HasBinaryProperty(person_perception, SELF_MOVING) in frame.property_assertions assert HasBinaryProperty(ball_perception, INANIMATE) in frame.property_assertions assert any( isinstance(property_, HasColor) and property_.perceived_object == ball_perception for property_ in frame.property_assertions )
def test_trivial_dynamic_situation_with_schemaless_object(language_mode): dad_situation_object = SituationObject.instantiate_ontology_node( ontology_node=DAD, ontology=GAILA_PHASE_1_ONTOLOGY) situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[dad_situation_object]) perception_generator = HighLevelSemanticsSituationToDevelopmentalPrimitivePerceptionGenerator( GAILA_PHASE_1_ONTOLOGY) # We explicitly exclude ground in perception generation # this generates a static perception... perception = perception_generator.generate_perception( situation, chooser=RandomChooser.for_seed(0), include_ground=False) # so we need to construct a dynamic one by hand from two identical scenes dynamic_perception = PerceptualRepresentation( frames=[perception.frames[0], perception.frames[0]]) perception_graph = PerceptionGraph.from_dynamic_perceptual_representation( dynamic_perception) perception_semantic_alignment = PerceptionSemanticAlignment.create_unaligned( perception_graph) (_, description_to_matched_semantic_node ) = LANGUAGE_MODE_TO_OBJECT_RECOGNIZER[language_mode].match_objects( perception_semantic_alignment) assert len(description_to_matched_semantic_node) == 1 assert (language_mode == LanguageMode.ENGLISH and ("Dad", ) in description_to_matched_semantic_node) or ( language_mode == LanguageMode.CHINESE and ("ba4 ba4", ) in description_to_matched_semantic_node)
def make_bird_flies_over_a_house(): bird = situation_object(BIRD) house = situation_object(HOUSE) situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[bird, house], actions=[ Action( FLY, argument_roles_to_fillers=[(AGENT, bird)], during=DuringAction( at_some_point=[ Relation( IN_REGION, bird, Region( reference_object=house, distance=DISTAL, direction=Direction( positive=True, relative_to_axis=GRAVITATIONAL_AXIS_FUNCTION, ), ), ) ] ), ) ], ) return situation
def test_no_region_in_gazed_objects(): putter = situation_object(MOM) object_put = situation_object(BALL) on_region_object = situation_object(TABLE) situation = HighLevelSemanticsSituation( salient_objects=[putter, object_put, on_region_object], actions=[ Action( PUT, argument_roles_to_fillers=[ (AGENT, putter), (THEME, object_put), ( GOAL, Region( on_region_object, distance=EXTERIOR_BUT_IN_CONTACT, direction=GRAVITATIONAL_UP, ), ), ], ) ], ontology=GAILA_PHASE_1_ONTOLOGY, ) assert situation.gazed_objects for object_ in situation.gazed_objects: assert not isinstance(object_, Region)
def make_mom_put_ball_on_table(): mom = situation_object(ontology_node=MOM) ball = situation_object(ontology_node=BALL) table = situation_object(ontology_node=TABLE) return HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[mom, ball, table], actions=[ Action( PUT, ( (AGENT, mom), (THEME, ball), ( GOAL, Region( reference_object=table, distance=EXTERIOR_BUT_IN_CONTACT, direction=GRAVITATIONAL_UP, ), ), ), ) ], )
def test_liquid_perceivable(): juice_perception = _PERCEPTION_GENERATOR.generate_perception( HighLevelSemanticsSituation(ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[situation_object(JUICE)]), chooser=RandomChooser.for_seed(0), ).frames[0] assert _some_object_has_binary_property(juice_perception, LIQUID)
def test_objects_in_something_not_implcitly_grounded(): box = situation_object(ontology_node=BOX) ball = situation_object(ontology_node=BALL) situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[box, ball], always_relations=[ Relation( IN_REGION, ball, Region( box, distance=PROXIMAL, direction=Direction( positive=True, relative_to_axis=HorizontalAxisOfObject(box, index=0), ), ), ) ], ) perception = _PERCEPTION_GENERATOR.generate_perception( situation, chooser=RandomChooser.for_seed(0) ) first_frame = perception.frames[0] ball_perception = perception_with_handle(first_frame, "**ball_0") ground_perception = perception_with_handle(first_frame, "the ground") box_perception = perception_with_handle(first_frame, "**box_0") first_frame_relations = first_frame.relations assert on(ball_perception, ground_perception)[0] in first_frame_relations assert on(box_perception, ground_perception)[0] in first_frame_relations
def test_colors_across_part_of_relations(): """ Intended to test color inheritance across part-of relations with objects that have a prototypical color """ learner_perception = _PERCEPTION_GENERATOR.generate_perception( HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[situation_object(TRUCK)] ), chooser=RandomChooser.for_seed(0), ) frame = learner_perception.frames[0] property_assertions = frame.property_assertions truck_perception = perception_with_handle(frame, "**truck_0") flatbed_perception = perception_with_handle(frame, "**flatbed_0") tire0_perception = perception_with_handle(frame, "**tire_0") tire1_perception = perception_with_handle(frame, "**tire_1") tire2_perception = perception_with_handle(frame, "**tire_2") tire3_perception = perception_with_handle(frame, "**tire_3") tire4_perception = perception_with_handle(frame, "**tire_4") tire5_perception = perception_with_handle(frame, "**tire_5") tire6_perception = perception_with_handle(frame, "**tire_6") tire7_perception = perception_with_handle(frame, "**tire_7") blue_options = COLORS_TO_RGBS[BLUE] blue_perceptions = [RgbColorPerception(r, g, b) for r, g, b in blue_options] red_options = COLORS_TO_RGBS[RED] red_perceptions = [RgbColorPerception(r, g, b) for r, g, b in red_options] black_options = COLORS_TO_RGBS[BLACK] black_perceptions = [RgbColorPerception(r, g, b) for r, g, b in black_options] assert any( HasColor(truck_perception, blue_perception) in property_assertions for blue_perception in blue_perceptions ) or any( HasColor(truck_perception, red_perception) in property_assertions for red_perception in red_perceptions ) assert any( HasColor(flatbed_perception, blue_perception) in property_assertions for blue_perception in blue_perceptions ) or any( HasColor(flatbed_perception, red_perception) in property_assertions for red_perception in red_perceptions ) assert any( ( HasColor(tire0_perception, black_perception) and HasColor(tire1_perception, black_perception) and HasColor(tire2_perception, black_perception) and HasColor(tire3_perception, black_perception) and HasColor(tire4_perception, black_perception) and HasColor(tire5_perception, black_perception) and HasColor(tire6_perception, black_perception) and HasColor(tire7_perception, black_perception) ) in property_assertions for black_perception in black_perceptions )
def test_perceive_explicit_relations(): # we want to test that relations explicitly called out in the situation are perceived. # Such relations fall into three buckets: # those which hold before an action, # those which hold after an action, # and those which hold both before and after an action. # To test all three of these at once, we use a situation where # (a) Mom is putting a ball on a table # (b) before the action the ball is far from a box, # (c) but after the action the ball is near the box, # (d) throughout the action the box is on the table. mom = situation_object(MOM) ball = situation_object(BALL) box = situation_object(BOX) table = situation_object(TABLE) situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[mom, box, ball, table], always_relations=[on(ball, table)], before_action_relations=[far(ball, box)], after_action_relations=[near(ball, box)], actions=[ Action( PUT, argument_roles_to_fillers=[ (AGENT, mom), (THEME, ball), ( GOAL, Region( reference_object=table, distance=EXTERIOR_BUT_IN_CONTACT, direction=GRAVITATIONAL_UP, ), ), ], ) ], ) perception = _PERCEPTION_GENERATOR.generate_perception( situation, chooser=RandomChooser.for_seed(0) ) ball_perception = perception_with_handle(perception.frames[0], "**ball_0") box_perception = perception_with_handle(perception.frames[0], "**box_0") table_perception = perception_with_handle(perception.frames[0], "**table_0") assert only(on(ball_perception, table_perception)) in perception.frames[0].relations assert only(on(ball_perception, table_perception)) in perception.frames[0].relations assert only(far(ball_perception, box_perception)) in perception.frames[0].relations assert ( only(far(ball_perception, box_perception)) not in perception.frames[1].relations ) assert ( only(near(ball_perception, box_perception)) not in perception.frames[0].relations ) assert only(near(ball_perception, box_perception)) in perception.frames[1].relations
def test_liquid_in_and_out_of_container(): juice = situation_object(JUICE) box = situation_object(ontology_node=BOX) table = situation_object(ontology_node=TABLE) two_d_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[juice, table], always_relations=[on(juice, table)], ) two_d_perception = _PERCEPTION_GENERATOR.generate_perception( two_d_situation, chooser=RandomChooser.for_seed(0) ) three_d_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[juice, box], always_relations=[ Relation(IN_REGION, juice, Region(box, distance=INTERIOR, direction=None)) ], ) three_d_perception = _PERCEPTION_GENERATOR.generate_perception( three_d_situation, chooser=RandomChooser.for_seed(0) ) two_perceived_objects = two_d_perception.frames[0].perceived_objects two_object_handles = set(obj.debug_handle for obj in two_perceived_objects) assert all(handle in two_object_handles for handle in {"**juice_0", "**table_0"}) three_perceived_objects = three_d_perception.frames[0].perceived_objects three_object_handles = set(obj.debug_handle for obj in three_perceived_objects) assert all(handle in three_object_handles for handle in {"**juice_0", "**box_0"}) assert any( isinstance(p, HasBinaryProperty) and perception_with_handle(two_d_perception.frames[0], "**juice_0") == p.perceived_object and p.binary_property == TWO_DIMENSIONAL for p in two_d_perception.frames[0].property_assertions ) assert not any( isinstance(p, HasBinaryProperty) and perception_with_handle(three_d_perception.frames[0], "**juice_0") == p.perceived_object and p.binary_property == TWO_DIMENSIONAL for p in three_d_perception.frames[0].property_assertions )
def test_person_and_ball(): person = situation_object(PERSON) ball = situation_object(BALL) person_and_ball_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[person, ball] ) person_and_ball_perception = _PERCEPTION_GENERATOR.generate_perception( person_and_ball_situation, chooser=RandomChooser.for_seed(0) ) perceived_objects = person_and_ball_perception.frames[0].perceived_objects object_handles = set(obj.debug_handle for obj in perceived_objects) assert len(person_and_ball_perception.frames) == 1 assert object_handles == { "**ball_0", "**person_0", "**head_0", "**torso_0", "**arm_0", "**armsegment_0", "**armsegment_1", "**hand_0", "**arm_1", "**armsegment_2", "**armsegment_3", "**hand_1", "**(animal) leg_0", "**leg-segment_0", "**leg-segment_1", "**foot_0", "**(animal) leg_1", "**leg-segment_2", "**leg-segment_3", "**foot_1", "the ground", } assert person_and_ball_perception.frames[0].relations person_perception = perception_with_handle( person_and_ball_perception.frames[0], "**person_0" ) ball_perception = perception_with_handle( person_and_ball_perception.frames[0], "**ball_0" ) assert set(person_and_ball_perception.frames[0].property_assertions).issuperset( { HasBinaryProperty(person_perception, ANIMATE), HasBinaryProperty(person_perception, SELF_MOVING), HasBinaryProperty(ball_perception, INANIMATE), } )
def test_speaker_perceivable(): speaker_situation_perception = _PERCEPTION_GENERATOR.generate_perception( HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[situation_object(PERSON, properties=[IS_SPEAKER])], ), chooser=RandomChooser.for_seed(0), ) assert _some_object_has_binary_property( speaker_situation_perception.frames[0], IS_SPEAKER )
def test_implicit_ground(): table_perception = _PERCEPTION_GENERATOR.generate_perception( HighLevelSemanticsSituation(ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[situation_object(TABLE)]), chooser=RandomChooser.for_seed(0), ) perceived_objects = table_perception.frames[0].perceived_objects object_handles = set(obj.debug_handle for obj in perceived_objects) # assert that a "ground" object is perceived assert "the ground" in object_handles
def test_not_two_speakers(): with pytest.raises(RuntimeError): _PERCEPTION_GENERATOR.generate_perception( HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[ situation_object(PERSON, properties=[IS_SPEAKER]), situation_object(PERSON, properties=[IS_SPEAKER]), ], ), chooser=RandomChooser.for_seed(0), )
def test_explicit_ground(): ground_perception = _PERCEPTION_GENERATOR.generate_perception( HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[situation_object(GROUND)] ), chooser=RandomChooser.for_seed(0), ) perceived_objects = ground_perception.frames[0].perceived_objects object_handles = set(obj.debug_handle for obj in perceived_objects) # assert that a second "ground" object was not generated assert object_handles == {"the ground"}
def test_path_from_action_description(): ball = situation_object(BALL) situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[ball], actions=[Action(FALL, argument_roles_to_fillers=[(THEME, ball)])], ) perception = _PERCEPTION_GENERATOR.generate_perception( situation, chooser=RandomChooser.for_seed(0) ) ball_perception = perception_with_handle(perception.frames[0], "**ball_0") ground_perception = perception_with_handle(perception.frames[0], "the ground") assert perception.during assert perception.during.objects_to_paths assert len(perception.during.objects_to_paths) == 1 path = only(perception.during.objects_to_paths[ball_perception]) assert path.reference_object == ground_perception assert path.operator == TOWARD
def test_relations_between_objects_and_ground(): # person_put_ball_on_table person = situation_object(ontology_node=PERSON) ball = situation_object(ontology_node=BALL) table = situation_object(ontology_node=TABLE) situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[person, ball, table], actions=[ # What is the best way of representing the destination in the high-level semantics? # Here we represent it as indicating a relation which should be true. Action( PUT, ( (AGENT, person), (THEME, ball), ( GOAL, Region( reference_object=table, distance=EXTERIOR_BUT_IN_CONTACT, direction=GRAVITATIONAL_UP, ), ), ), ) ], ) perception = _PERCEPTION_GENERATOR.generate_perception( situation, chooser=RandomChooser.for_seed(0) ) first_frame = perception.frames[0] ball_perception = perception_with_handle(first_frame, "**ball_0") ground_perception = perception_with_handle(first_frame, "the ground") table_perception = perception_with_handle(first_frame, "**table_0") first_frame_relations = first_frame.relations second_frame_relations = perception.frames[1].relations assert on(ball_perception, ground_perception)[0] in first_frame_relations assert on(ball_perception, ground_perception)[0] not in second_frame_relations assert on(table_perception, ground_perception)[0] in first_frame_relations assert on(table_perception, ground_perception)[0] in second_frame_relations
def test_multiple_trecognized_particulars_banned(): with pytest.raises(RuntimeError): dad_0 = situation_object(DAD) dad_1 = situation_object(DAD) box = situation_object(BOX) HighLevelSemanticsSituation( salient_objects=[dad_0, dad_1, box], actions=[ Action( GIVE, argument_roles_to_fillers=[ (AGENT, dad_0), (THEME, box), (GOAL, dad_1), ], ) ], ontology=GAILA_PHASE_1_ONTOLOGY, )
def _instantiate_situation( self, template: Phase1SituationTemplate, variable_assignment: "TemplateVariableAssignment", object_var_to_instantiations, ) -> HighLevelSemanticsSituation: return HighLevelSemanticsSituation( from_template=template.name, ontology=self.ontology, salient_objects=[ object_var_to_instantiations[obj_var] for obj_var in template.salient_object_variables ], other_objects=[ object_var_to_instantiations[obj_var] # type: ignore for obj_var in (immutableset(object_var_to_instantiations.keys( )).difference(template.salient_object_variables)) # We use the keys of the mapping in case a default addressee was added ], always_relations=[ relation.copy_remapping_objects(object_var_to_instantiations) for relation in template.asserted_always_relations ], actions=[ self._instantiate_action( action, object_var_to_instantiations, variable_assignment.action_variables_to_fillers, ) for action in template.actions ], before_action_relations=[ relation.copy_remapping_objects(object_var_to_instantiations) for relation in template.before_action_relations ], after_action_relations=[ relation.copy_remapping_objects(object_var_to_instantiations) for relation in template.after_action_relations ], syntax_hints=template.syntax_hints, axis_info=self._compute_axis_info(object_var_to_instantiations), gazed_objects=immutableset(object_var_to_instantiations[object_] for object_ in template.gazed_objects), )
def test_subobject_perception_has_appropriate_properties(): """ Intended to test that an object's subobjects have their properties assigned to them properly """ learner_perception = _PERCEPTION_GENERATOR.generate_perception( HighLevelSemanticsSituation(ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[situation_object(TABLE)]), chooser=RandomChooser.for_seed(0), ) frame = learner_perception.frames[0] property_assertions = frame.property_assertions leg_perceptions = [ perception_with_handle(frame, "**(furniture) leg_0"), perception_with_handle(frame, "**(furniture) leg_1"), perception_with_handle(frame, "**(furniture) leg_2"), perception_with_handle(frame, "**(furniture) leg_3"), ] assert all( HasBinaryProperty(leg_perception, INANIMATE) in property_assertions for leg_perception in leg_perceptions)
def test_grounding_of_unsupported_objects(): box = situation_object(ontology_node=BOX) ball = situation_object(ontology_node=BALL) situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[box, ball], always_relations=[Relation(IN_REGION, ball, Region(box, distance=INTERIOR))], ) perception = _PERCEPTION_GENERATOR.generate_perception( situation, chooser=RandomChooser.for_seed(0) ) first_frame = perception.frames[0] ball_perception = perception_with_handle(first_frame, "**ball_0") ground_perception = perception_with_handle(first_frame, "the ground") box_perception = perception_with_handle(first_frame, "**box_0") first_frame_relations = first_frame.relations assert on(ball_perception, ground_perception)[0] not in first_frame_relations assert on(box_perception, ground_perception)[0] in first_frame_relations
def test_with_object_recognizer(language_mode): integrated_learner = IntegratedTemplateLearner( object_learner=LANGUAGE_MODE_TO_TEMPLATE_LEARNER_OBJECT_RECOGNIZER[language_mode], attribute_learner=None, relation_learner=None, action_learner=None, ) dad_situation_object = SituationObject.instantiate_ontology_node( ontology_node=DAD, ontology=GAILA_PHASE_1_ONTOLOGY ) situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[dad_situation_object] ) perception_generator = HighLevelSemanticsSituationToDevelopmentalPrimitivePerceptionGenerator( GAILA_PHASE_1_ONTOLOGY ) # We explicitly exclude ground in perception generation # this generates a static perception... perception = perception_generator.generate_perception( situation, chooser=RandomChooser.for_seed(0), include_ground=False ) # so we need to construct a dynamic one by hand from two identical scenes dynamic_perception = PerceptualRepresentation( frames=[perception.frames[0], perception.frames[0]] ) descriptions = integrated_learner.describe(dynamic_perception) assert len(descriptions) == 1 assert ( language_mode == LanguageMode.ENGLISH and one(descriptions.keys()).as_token_sequence() == ("Dad",) ) or ( language_mode == LanguageMode.CHINESE and one(descriptions.keys()).as_token_sequence() == ("ba4 ba4",) )
def _satisfies_constraints( self, template: Phase1SituationTemplate, instantiated_situation: HighLevelSemanticsSituation, variable_binding: Mapping["TemplateObjectVariable", SituationObject], ) -> bool: for constraining_relation in template.constraining_relations: # the constraint is satisfied if it is explicitly-specified as true relation_bound_to_situation_objects = constraining_relation.copy_remapping_objects( variable_binding) relation_explicitly_specified = instantiated_situation.relation_always_holds( relation_bound_to_situation_objects) # or if we can deduce it is true from general relations in the ontology # (e.g. general tendencies like people being larger than balls) # Note we do not currently allow overriding relations derived from the ontology. # See https://github.com/isi-vista/adam/issues/229 relation_implied_by_ontology_relations: bool if isinstance(relation_bound_to_situation_objects.second_slot, SituationObject): # second slot could have been a region, in which case ontological relations # do not apply relation_in_terms_of_object_types = relation_bound_to_situation_objects.copy_remapping_objects( { relation_bound_to_situation_objects.first_slot: relation_bound_to_situation_objects.first_slot. ontology_node, relation_bound_to_situation_objects.second_slot: relation_bound_to_situation_objects.second_slot. ontology_node, }) relation_implied_by_ontology_relations = ( relation_in_terms_of_object_types in self.ontology.relations) else: relation_implied_by_ontology_relations = False if not (relation_explicitly_specified or relation_implied_by_ontology_relations): return False return True
def test_big_ball(): ball1 = situation_object(BALL, debug_handle="ball_0") ball2 = situation_object(BALL, debug_handle="ball_1") ball_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[ball2, ball1], always_relations=[bigger_than(ball1, ball2)], ) assert (ball_situation.always_relations[0].first_slot.ontology_node == ball_situation.always_relations[0].second_slot.ontology_node) ball_perception = _PERCEPTION_GENERATOR.generate_perception( ball_situation, chooser=RandomChooser.for_seed(0)) perceived_objects = ball_perception.frames[0].perceived_objects object_handles = set(obj.debug_handle for obj in perceived_objects) assert object_handles == {"**ball_0", "**ball_1", "the ground"} assert any(relation.relation_type == BIGGER_THAN_SAME_TYPE for relation in ball_perception.frames[0].relations) assert any(relation.relation_type == SMALLER_THAN_SAME_TYPE for relation in ball_perception.frames[0].relations)
def build_object_learner_experiment_curriculum_train( num_samples: Optional[int], num_noise_objects: Optional[int], language_generator: LanguageGenerator[ HighLevelSemanticsSituation, LinearizedDependencyTree ], *, params: Parameters = Parameters.empty(), ) -> Sequence[Phase1InstanceGroup]: situations = make_multiple_object_situation( num_samples, num_noise_objects, language_generator ) accurate_language_chance = params.floating_point( "accurate_language_percentage", default=0.5 ) output_situations = [] random.seed(params.integer("random_seed", default=0)) rng = RandomChooser.for_seed(params.integer("language_random_seed", default=0)) for (situation, language, perception) in situations.instances(): if random.random() <= accurate_language_chance: output_language = language else: # Make Invalid Language if situation and isinstance(situation, HighLevelSemanticsSituation): # First, gather all OntologyNodes which aren't already present in the situation present_ontology_nodes = [ _object.ontology_node for _object in situation.all_objects ] valid_other_objects = [ node for node in PHASE_1_CURRICULUM_OBJECTS if node not in present_ontology_nodes ] # Then choose one at random chosen_ontology_node = rng.choice(valid_other_objects) # Make a fake situation with just this object in it, ignoring colors wrong_situation = HighLevelSemanticsSituation( ontology=GAILA_PHASE_2_ONTOLOGY, salient_objects=[ SituationObject.instantiate_ontology_node( chosen_ontology_node, ontology=GAILA_PHASE_2_ONTOLOGY ) ], syntax_hints=[IGNORE_COLORS], ) # Generate the language as if it came from this fake situation rather than the original one fake_language = only( language_generator.generate_language(wrong_situation, chooser=rng) ) output_language = LinearizedDependencyTree( dependency_tree=fake_language.dependency_tree, surface_token_order=fake_language.surface_token_order, accurate=False, ) else: raise RuntimeError( f"Unable to make invalid language without a situation of type HighlevelSemanticsSituation. Got situation: {situation}" ) output_situations.append((situation, output_language, perception)) return [ AblatedLanguageSituationsInstanceGroup( name=f"{situations.name()}_ablated", instances=output_situations ) ]