def for_ontology_types(
ontology_types: Iterable[OntologyNode],
determiners: Iterable[str],
ontology: Ontology,
language_mode: LanguageMode,
*,
perception_generator:
HighLevelSemanticsSituationToDevelopmentalPrimitivePerceptionGenerator,
) -> "ObjectRecognizer":
ontology_types_to_concepts = {
obj_type: ObjectConcept(obj_type.handle)
for obj_type in ontology_types
}
return ObjectRecognizer(
concepts_to_static_patterns=_sort_mapping_by_pattern_complexity(
immutabledict((
concept,
PerceptionGraphPattern.from_ontology_node(
obj_type,
ontology,
perception_generator=perception_generator),
) for (obj_type,
concept) in ontology_types_to_concepts.items())),
determiners=determiners,
concepts_to_names={
concept: obj_type.handle
for obj_type, concept in ontology_types_to_concepts.items()
},
language_mode=language_mode,
)
def evaluate_kind_membership(self, word: str, kind: str) -> float:
word_node = self.concept_as_str_node(ObjectConcept(word))
kind_node = self.concept_as_str_node(KindConcept(kind))
if kind_node not in self.nodes or word_node not in self.nodes:
return 0
return cos_sim(
self.object_concept_embedding(word_node),
self.kind_concept_embedding(kind_node),
)
def learn_from(
self,
language_perception_semantic_alignment:
LanguagePerceptionSemanticAlignment,
offset: int = 0,
) -> None:
sequence = (language_perception_semantic_alignment.
language_concept_alignment.language.as_token_sequence())
recognized_semantic_nodes = list(
language_perception_semantic_alignment.
perception_semantic_alignment.semantic_nodes)
span = (language_perception_semantic_alignment.
language_concept_alignment.node_to_language_span)
# Get actions and attributes that are recognized in the scene
action_nodes = [
n for n in recognized_semantic_nodes
if isinstance(n, ActionSemanticNode)
]
attribute_nodes = [
n for n in recognized_semantic_nodes
if isinstance(n, AttributeSemanticNode)
]
# Check if a recognized object matches the heard utterance
significant_object_concept: Optional[Concept] = None
for node in recognized_semantic_nodes:
if isinstance(node, ObjectSemanticNode) and node in span:
significant_object_concept = node.concept
# Actions: E.g dog s walk
# Attributes: E.g cookies are brown
# Kinds: E.g cookie is a food
# Both of these statements are regular generics as the noun is indefinite. We just need to
# check attributes in addition to actions. We hardwire "are" in order to recognize kinds.
# If there is a recognized object node that matches the scene, and a generic action OR attribute, learn!
for nodes in [action_nodes, attribute_nodes]:
if significant_object_concept and nodes:
# Get the slot of object for each recognized action in the scene
other_concepts_and_object_slots = []
for node in nodes: # type: ignore
slot = get_slot_from_semantic_node(
significant_object_concept, node)
if slot != "":
other_concepts_and_object_slots.append(
(node.concept, slot))
# Update the representation for learned generics for the sequence
if sequence in self.learned_representations:
known_representation = self.learned_representations[
sequence]
known_representation[1].update(
other_concepts_and_object_slots)
else:
self.learned_representations[sequence] = (
significant_object_concept,
set(other_concepts_and_object_slots),
)
# Kinds: E.g cookie is a food
elif significant_object_concept and ("are" in sequence
or "shr4" in sequence):
# Filter out the kind words and use that for semantic encoding.
pred = "shr4" if "shr4" in sequence else "are"
kind = sequence[sequence.index(pred) + 1]
# Create a concept for the kind
if kind not in self.learned_concepts:
kind_concept = KindConcept(kind)
self.learned_concepts[kind] = kind_concept
self.learned_representations[sequence] = (
significant_object_concept,
{(self.learned_concepts[kind], "is")},
)
# Potential "Wugs are animals"
elif not significant_object_concept and ("are" in sequence
or "shr4" in sequence):
# Filter out the kind words and use that for semantic encoding.
pred = "shr4" if "shr4" in sequence else "are"
potential_kind = sequence[sequence.index(pred) + 1]
new_object_text = sequence[0]
# If we know the kind, generalize wug as an animal
if potential_kind in self.learned_concepts:
if new_object_text not in self.learned_concepts:
new_object_concept = ObjectConcept(new_object_text)
self.learned_concepts[
new_object_text] = new_object_concept
self.learned_representations[sequence] = (
self.learned_concepts[new_object_text],
{(self.learned_concepts[potential_kind], "is")},
)
def _new_concept(self, debug_string: str) -> ObjectConcept:
return ObjectConcept(debug_string)