def test_object_recognition_with_drink_perception():
"""
Regression test to confirm we can perform object recognition on a pickled and unpickled "drink"
perception. If we do this using the normal pickling interface we get an error. This test checks
that we don't run into such an error when we instead pickle and unpickle the perception using
the AdamPickler and AdamUnpickler.
See https://github.com/isi-vista/adam/issues/958.
"""
language_mode = LanguageMode.ENGLISH
template = drink_test_template()
curriculum = phase1_instances(
"train",
sampled(
template,
max_to_sample=3,
ontology=GAILA_PHASE_1_ONTOLOGY,
chooser=PHASE1_CHOOSER_FACTORY(),
block_multiple_of_the_same_type=True,
),
language_generator=phase1_language_generator(language_mode),
)
object_recognizer = LANGUAGE_MODE_TO_TEMPLATE_LEARNER_OBJECT_RECOGNIZER[
language_mode]
learner = IntegratedTemplateLearner(object_learner=object_recognizer)
for (_, linguistic_description,
perceptual_representation) in curriculum.instances():
new_perceptual_representation = _pickle_and_unpickle_object(
perceptual_representation)
learner.observe(
LearningExample(new_perceptual_representation,
linguistic_description))
def test_recognizes_ontology_objects(object_type, language_mode):
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY,
salient_objects=[
SituationObject.instantiate_ontology_node(
ontology_node=object_type, ontology=GAILA_PHASE_1_ONTOLOGY)
],
)
perception_generator = HighLevelSemanticsSituationToDevelopmentalPrimitivePerceptionGenerator(
GAILA_PHASE_1_ONTOLOGY)
perception = perception_generator.generate_perception(
situation, chooser=RandomChooser.for_seed(0), include_ground=False)
learner = IntegratedTemplateLearner(
object_learner=LANGUAGE_MODE_TO_TEMPLATE_LEARNER_OBJECT_RECOGNIZER[
language_mode])
descriptions = learner.describe(perception)
assert descriptions
if language_mode == LanguageMode.ENGLISH:
assert object_type.handle in one(
descriptions.items())[0].as_token_sequence()
else:
mappings = (
GAILA_PHASE_1_CHINESE_LEXICON._ontology_node_to_word # pylint:disable=protected-access
)
for k, v in mappings.items():
if k.handle == object_type.handle:
assert v.base_form in one(
descriptions.items())[0].as_token_sequence()
def integrated_learner_factory(language_mode: LanguageMode):
return IntegratedTemplateLearner(
object_learner=LANGUAGE_MODE_TO_TEMPLATE_LEARNER_OBJECT_RECOGNIZER[language_mode],
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_1_ONTOLOGY, beam_size=5, language_mode=language_mode
),
)
def test_with_object_recognizer(language_mode):
integrated_learner = IntegratedTemplateLearner(
object_learner=LANGUAGE_MODE_TO_TEMPLATE_LEARNER_OBJECT_RECOGNIZER[language_mode],
attribute_learner=None,
relation_learner=None,
action_learner=None,
)
dad_situation_object = SituationObject.instantiate_ontology_node(
ontology_node=DAD, ontology=GAILA_PHASE_1_ONTOLOGY
)
situation = HighLevelSemanticsSituation(
ontology=GAILA_PHASE_1_ONTOLOGY, salient_objects=[dad_situation_object]
)
perception_generator = HighLevelSemanticsSituationToDevelopmentalPrimitivePerceptionGenerator(
GAILA_PHASE_1_ONTOLOGY
)
# We explicitly exclude ground in perception generation
# this generates a static perception...
perception = perception_generator.generate_perception(
situation, chooser=RandomChooser.for_seed(0), include_ground=False
)
# so we need to construct a dynamic one by hand from two identical scenes
dynamic_perception = PerceptualRepresentation(
frames=[perception.frames[0], perception.frames[0]]
)
descriptions = integrated_learner.describe(dynamic_perception)
assert len(descriptions) == 1
assert (
language_mode == LanguageMode.ENGLISH
and one(descriptions.keys()).as_token_sequence() == ("Dad",)
) or (
language_mode == LanguageMode.CHINESE
and one(descriptions.keys()).as_token_sequence() == ("ba4 ba4",)
)
def integrated_learner_factory(language_mode: LanguageMode):
return IntegratedTemplateLearner(
object_learner=ObjectRecognizerAsTemplateLearner(
object_recognizer=object_recognizer_factory(language_mode),
language_mode=language_mode,
),
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_1_ONTOLOGY,
beam_size=5,
language_mode=language_mode),
action_learner=SubsetVerbLearnerNew(ontology=GAILA_PHASE_1_ONTOLOGY,
beam_size=5,
language_mode=language_mode),
)
def pursuit_learner_factory(language_mode: LanguageMode):
rng = random.Random()
rng.seed(0)
return IntegratedTemplateLearner(
object_learner=LANGUAGE_MODE_TO_TEMPLATE_LEARNER_OBJECT_RECOGNIZER[
language_mode],
attribute_learner=PursuitAttributeLearnerNew(
learning_factor=0.05,
graph_match_confirmation_threshold=0.7,
lexicon_entry_threshold=0.7,
rng=rng,
smoothing_parameter=0.002,
ontology=GAILA_PHASE_1_ONTOLOGY,
language_mode=language_mode,
rank_gaze_higher=False,
),
)
def integrated_learner_factory(language_mode: LanguageMode):
rng = random.Random()
rng.seed(0)
return IntegratedTemplateLearner(
object_learner=LANGUAGE_MODE_TO_TEMPLATE_LEARNER_OBJECT_RECOGNIZER[
language_mode],
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_1_ONTOLOGY,
beam_size=5,
language_mode=language_mode),
plural_learner=SubsetPluralLearnerNew(ontology=GAILA_PHASE_1_ONTOLOGY,
beam_size=5,
language_mode=language_mode),
action_learner=SubsetVerbLearnerNew(ontology=GAILA_PHASE_1_ONTOLOGY,
beam_size=5,
language_mode=language_mode),
generics_learner=SimpleGenericsLearner(),
)
def integrated_learner_factory(language_mode: LanguageMode):
rng = random.Random()
rng.seed(0)
return IntegratedTemplateLearner(
object_learner=SubsetObjectLearnerNew(ontology=GAILA_PHASE_1_ONTOLOGY,
beam_size=5,
language_mode=language_mode),
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_1_ONTOLOGY,
beam_size=5,
language_mode=language_mode),
plural_learner=SubsetPluralLearnerNew(ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=5,
language_mode=language_mode),
action_learner=SubsetVerbLearnerNew(ontology=GAILA_PHASE_1_ONTOLOGY,
beam_size=5,
language_mode=language_mode),
generics_learner=SimpleGenericsLearner(),
)
def integrated_learner_factory(language_mode: LanguageMode):
return IntegratedTemplateLearner(
object_learner=SubsetObjectLearnerNew(ontology=GAILA_PHASE_1_ONTOLOGY,
beam_size=10,
language_mode=language_mode))
def test_pursuit_object_learner_with_gaze(language_mode):
target_objects = [
BALL,
# PERSON,
# CHAIR,
# TABLE,
DOG,
# BIRD,
BOX,
]
language_generator = phase1_language_generator(language_mode)
target_test_templates = []
for obj in target_objects:
# Create train and test templates for the target objects
test_obj_object = object_variable("obj-with-color", obj)
test_template = Phase1SituationTemplate(
"colored-obj-object",
salient_object_variables=[test_obj_object],
syntax_hints=[IGNORE_COLORS],
gazed_objects=[test_obj_object],
)
target_test_templates.extend(
all_possible(
test_template,
chooser=PHASE1_CHOOSER_FACTORY(),
ontology=GAILA_PHASE_1_ONTOLOGY,
))
rng = random.Random()
rng.seed(0)
# We can use this to generate the actual pursuit curriculum
train_curriculum = make_simple_pursuit_curriculum(
target_objects=target_objects,
num_instances=30,
num_objects_in_instance=3,
num_noise_instances=0,
language_generator=language_generator,
add_gaze=True,
)
test_obj_curriculum = phase1_instances(
"obj test",
situations=target_test_templates,
language_generator=language_generator,
)
# All parameters should be in the range 0-1.
# Learning factor works better when kept < 0.5
# Graph matching threshold doesn't seem to matter that much, as often seems to be either a
# complete or a very small match.
# The lexicon threshold works better between 0.07-0.3, but we need to play around with it because we end up not
# lexicalize items sufficiently because of diminishing lexicon probability through training
rng = random.Random()
rng.seed(0)
learner = IntegratedTemplateLearner(object_learner=PursuitObjectLearnerNew(
learning_factor=0.05,
graph_match_confirmation_threshold=0.7,
lexicon_entry_threshold=0.7,
rng=rng,
smoothing_parameter=0.002,
ontology=GAILA_PHASE_1_ONTOLOGY,
language_mode=language_mode,
rank_gaze_higher=True,
))
for training_stage in [train_curriculum]:
for (
_,
linguistic_description,
perceptual_representation,
) in training_stage.instances():
learner.observe(
LearningExample(perceptual_representation,
linguistic_description))
for test_instance_group in [test_obj_curriculum]:
for (
_,
test_instance_language,
test_instance_perception,
) in test_instance_group.instances():
logging.info("lang: %s", test_instance_language)
descriptions_from_learner = learner.describe(
test_instance_perception)
gold = test_instance_language.as_token_sequence()
assert gold in [
desc.as_token_sequence() for desc in descriptions_from_learner
]
def learner_factory_from_params(
params: Parameters,
graph_logger: Optional[HypothesisLogger],
language_mode: LanguageMode = LanguageMode.ENGLISH,
) -> Callable[[], TopLevelLanguageLearner]: # type: ignore
learner_type = params.string(
"learner",
[
"pursuit",
"object-subset",
"preposition-subset",
"attribute-subset",
"verb-subset",
"integrated-learner",
"integrated-learner-recognizer-without-generics",
"integrated-learner-recognizer",
"pursuit-gaze",
"integrated-object-only",
"integrated-learner-params",
"integrated-pursuit-attribute-only",
],
)
beam_size = params.positive_integer("beam_size", default=10)
rng = random.Random()
rng.seed(0)
perception_generator = GAILA_PHASE_1_PERCEPTION_GENERATOR
objects = [YOU_HACK, ME_HACK]
objects.extend(PHASE_1_CURRICULUM_OBJECTS)
# Eval hack! This is specific to the Phase 1 ontology
object_recognizer = ObjectRecognizer.for_ontology_types(
objects,
determiners=ENGLISH_DETERMINERS,
ontology=GAILA_PHASE_1_ONTOLOGY,
language_mode=language_mode,
perception_generator=perception_generator,
)
if learner_type == "pursuit":
return lambda: ObjectPursuitLearner.from_parameters(
params.namespace("pursuit"), graph_logger=graph_logger)
elif learner_type == "pursuit-gaze":
return lambda: IntegratedTemplateLearner(
object_learner=PursuitObjectLearnerNew(
learning_factor=0.05,
graph_match_confirmation_threshold=0.7,
lexicon_entry_threshold=0.7,
rng=rng,
smoothing_parameter=0.002,
ontology=GAILA_PHASE_2_ONTOLOGY,
language_mode=language_mode,
rank_gaze_higher=True,
),
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
relation_learner=SubsetRelationLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
action_learner=SubsetVerbLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
)
elif learner_type == "object-subset":
return lambda: SubsetObjectLearner(ontology=GAILA_PHASE_1_ONTOLOGY,
language_mode=LanguageMode.ENGLISH)
elif learner_type == "attribute-subset":
return lambda: SubsetAttributeLearner(
ontology=GAILA_PHASE_1_ONTOLOGY,
object_recognizer=object_recognizer,
language_mode=LanguageMode.ENGLISH,
)
elif learner_type == "preposition-subset":
return lambda: SubsetPrepositionLearner(
# graph_logger=graph_logger,
object_recognizer=object_recognizer,
ontology=GAILA_PHASE_1_ONTOLOGY,
language_mode=LanguageMode.ENGLISH,
)
elif learner_type == "verb-subset":
return lambda: SubsetVerbLearner(
ontology=GAILA_PHASE_1_ONTOLOGY,
object_recognizer=object_recognizer,
language_mode=LanguageMode.ENGLISH,
)
elif learner_type == "integrated-learner":
return lambda: IntegratedTemplateLearner(
object_learner=SubsetObjectLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
relation_learner=SubsetRelationLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
action_learner=SubsetVerbLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
functional_learner=FunctionalLearner(language_mode=language_mode),
)
elif learner_type == "integrated-learner-recognizer":
return lambda: IntegratedTemplateLearner(
object_learner=ObjectRecognizerAsTemplateLearner(
object_recognizer=object_recognizer,
language_mode=language_mode),
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
relation_learner=SubsetRelationLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
action_learner=SubsetVerbLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
functional_learner=FunctionalLearner(language_mode=language_mode),
generics_learner=SimpleGenericsLearner(),
)
elif learner_type == "ic":
return lambda: IntegratedTemplateLearner(
object_learner=ObjectRecognizerAsTemplateLearner(
object_recognizer=object_recognizer,
language_mode=language_mode),
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
relation_learner=SubsetRelationLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
action_learner=SubsetVerbLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
functional_learner=FunctionalLearner(language_mode=language_mode),
)
elif learner_type == "integrated-object-only":
object_learner_type = params.string(
"object_learner_type",
valid_options=["subset", "pbv", "pursuit"],
default="subset",
)
if params.has_namespace("learner_params"):
learner_params = params.namespace("learner_params")
else:
learner_params = params.empty(namespace_prefix="learner_params")
object_learner_factory: Callable[[], TemplateLearner]
if object_learner_type == "subset":
def subset_factory() -> SubsetObjectLearnerNew:
return SubsetObjectLearnerNew( # type: ignore
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
)
object_learner_factory = subset_factory
elif object_learner_type == "pbv":
def pbv_factory() -> ProposeButVerifyObjectLearner:
return ProposeButVerifyObjectLearner.from_params( # type: ignore
learner_params)
object_learner_factory = pbv_factory
elif object_learner_type == "pursuit":
def pursuit_factory() -> PursuitObjectLearnerNew:
return PursuitObjectLearnerNew( # type: ignore
learning_factor=learner_params.floating_point(
"learning_factor"),
graph_match_confirmation_threshold=learner_params.
floating_point("graph_match_confirmation_threshold"),
lexicon_entry_threshold=learner_params.floating_point(
"lexicon_entry_threshold"),
rng=rng,
smoothing_parameter=learner_params.floating_point(
"smoothing_parameter"),
ontology=GAILA_PHASE_2_ONTOLOGY,
language_mode=language_mode,
)
object_learner_factory = pursuit_factory
else:
raise RuntimeError(
f"Invalid Object Learner Type Selected: {learner_type}")
return lambda: IntegratedTemplateLearner(object_learner=
object_learner_factory())
elif learner_type == "integrated-learner-params":
object_learner = build_object_learner_factory( # type:ignore
params.namespace_or_empty("object_learner"), beam_size,
language_mode)
attribute_learner = build_attribute_learner_factory( # type:ignore
params.namespace_or_empty("attribute_learner"), beam_size,
language_mode)
relation_learner = build_relation_learner_factory( # type:ignore
params.namespace_or_empty("relation_learner"), beam_size,
language_mode)
action_learner = build_action_learner_factory( # type:ignore
params.namespace_or_empty("action_learner"), beam_size,
language_mode)
plural_learner = build_plural_learner_factory( # type:ignore
params.namespace_or_empty("plural_learner"), beam_size,
language_mode)
return lambda: IntegratedTemplateLearner(
object_learner=object_learner,
attribute_learner=attribute_learner,
relation_learner=relation_learner,
action_learner=action_learner,
functional_learner=FunctionalLearner(language_mode=language_mode)
if params.boolean("include_functional_learner", default=True) else
None,
generics_learner=SimpleGenericsLearner() if params.boolean(
"include_generics_learner", default=True) else None,
plural_learner=plural_learner,
suppress_error=params.boolean("suppress_error", default=True),
)
elif learner_type == "integrated-pursuit-attribute-only":
return lambda: IntegratedTemplateLearner(
object_learner=ObjectRecognizerAsTemplateLearner(
object_recognizer=object_recognizer,
language_mode=language_mode),
attribute_learner=PursuitAttributeLearnerNew(
learning_factor=0.05,
graph_match_confirmation_threshold=0.7,
lexicon_entry_threshold=0.7,
rng=rng,
smoothing_parameter=0.002,
rank_gaze_higher=False,
ontology=GAILA_PHASE_1_ONTOLOGY,
language_mode=language_mode,
),
)
else:
raise RuntimeError("can't happen")
def learner_factory_from_params(
params: Parameters,
graph_logger: Optional[HypothesisLogger],
language_mode: LanguageMode = LanguageMode.ENGLISH,
) -> Callable[[], TopLevelLanguageLearner]: # type: ignore
learner_type = params.string(
"learner",
[
"pursuit",
"object-subset",
"preposition-subset",
"attribute-subset",
"verb-subset",
"integrated-learner",
"integrated-learner-recognizer",
"pursuit-gaze",
],
)
beam_size = params.positive_integer("beam_size", default=10)
if language_mode == LanguageMode.CHINESE and learner_type not in [
"integrated-learner",
"integrated-learner-recognizer",
]:
raise RuntimeError(
"Only able to test Chinese with integrated learner.")
rng = random.Random()
rng.seed(0)
perception_generator = GAILA_PHASE_1_PERCEPTION_GENERATOR
objects = [YOU_HACK, ME_HACK]
objects.extend(PHASE_1_CURRICULUM_OBJECTS)
# Eval hack! This is specific to the Phase 1 ontology
object_recognizer = ObjectRecognizer.for_ontology_types(
objects,
determiners=ENGLISH_DETERMINERS,
ontology=GAILA_PHASE_1_ONTOLOGY,
language_mode=language_mode,
perception_generator=perception_generator,
)
if learner_type == "pursuit":
return lambda: ObjectPursuitLearner.from_parameters(
params.namespace("pursuit"), graph_logger=graph_logger)
elif learner_type == "pursuit-gaze":
return lambda: IntegratedTemplateLearner(
object_learner=PursuitObjectLearnerNew(
learning_factor=0.05,
graph_match_confirmation_threshold=0.7,
lexicon_entry_threshold=0.7,
rng=rng,
smoothing_parameter=0.002,
ontology=GAILA_PHASE_2_ONTOLOGY,
language_mode=language_mode,
rank_gaze_higher=True,
),
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
relation_learner=SubsetRelationLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
action_learner=SubsetVerbLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
)
elif learner_type == "object-subset":
return lambda: SubsetObjectLearner(ontology=GAILA_PHASE_1_ONTOLOGY,
language_mode=LanguageMode.ENGLISH)
elif learner_type == "attribute-subset":
return lambda: SubsetAttributeLearner(
ontology=GAILA_PHASE_1_ONTOLOGY,
object_recognizer=object_recognizer,
language_mode=LanguageMode.ENGLISH,
)
elif learner_type == "preposition-subset":
return lambda: SubsetPrepositionLearner(
# graph_logger=graph_logger,
object_recognizer=object_recognizer,
ontology=GAILA_PHASE_1_ONTOLOGY,
language_mode=LanguageMode.ENGLISH,
)
elif learner_type == "verb-subset":
return lambda: SubsetVerbLearner(
ontology=GAILA_PHASE_1_ONTOLOGY,
object_recognizer=object_recognizer,
language_mode=LanguageMode.ENGLISH,
)
elif learner_type == "integrated-learner":
return lambda: IntegratedTemplateLearner(
object_learner=SubsetObjectLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
relation_learner=SubsetRelationLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
action_learner=SubsetVerbLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
functional_learner=FunctionalLearner(language_mode=language_mode),
)
elif learner_type == "integrated-learner-recognizer":
return lambda: IntegratedTemplateLearner(
object_learner=ObjectRecognizerAsTemplateLearner(
object_recognizer=object_recognizer,
language_mode=language_mode),
attribute_learner=SubsetAttributeLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
relation_learner=SubsetRelationLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
action_learner=SubsetVerbLearnerNew(
ontology=GAILA_PHASE_2_ONTOLOGY,
beam_size=beam_size,
language_mode=language_mode,
),
functional_learner=FunctionalLearner(language_mode=language_mode),
)
else:
raise RuntimeError("can't happen")
