def _preprocess_scene(
self, perception_semantic_alignment: PerceptionSemanticAlignment
) -> PerceptionSemanticAlignment:
# Avoid accidentally identifying a word with the learner itself.
return perception_semantic_alignment.copy_with_updated_graph_and_added_nodes(
new_graph=graph_without_learner(
perception_semantic_alignment.perception_graph),
new_nodes=[],
)
def _enrich_common(
self, perception_semantic_alignment: PerceptionSemanticAlignment
) -> Tuple[PerceptionSemanticAlignment, AbstractSet[SemanticNode]]:
"""
Shared code between `enrich_during_learning` and `enrich_during_description`.
"""
preprocessing_result = self._preprocess_scene(perception_semantic_alignment)
preprocessed_perception_graph = preprocessing_result.perception_graph
# This accumulates our output.
match_to_score: List[Tuple[SemanticNode, float]] = []
# In the case of objects only, we alter the perception graph once they
# are recognized by replacing the matched portion of the graph with the
# ObjectSemanticNodes. We gather them as we match and do the replacement below.
matched_objects: List[Tuple[SemanticNode, PerceptionGraphPatternMatch]] = []
# We pull this out into a function because we do matching in two passes:
# first against templates whose meanings we are sure of (=have lexicalized)
# and then, if no match has been found, against those we are still learning.
def match_template(
*, concept: Concept, pattern: PerceptionGraphTemplate, score: float
) -> None:
# try to see if (our model of) its semantics is present in the situation.
matcher = pattern.graph_pattern.matcher(
preprocessed_perception_graph,
match_mode=MatchMode.NON_OBJECT,
# debug_callback=self._debug_callback,
)
for match in matcher.matches(use_lookahead_pruning=True):
# if there is a match, use that match to describe the situation.
semantic_node_for_match = pattern_match_to_semantic_node(
concept=concept, pattern=pattern, match=match
)
match_to_score.append((semantic_node_for_match, score))
# We want to replace object matches with their semantic nodes,
# but we don't want to alter the graph while matching it,
# so we accumulate these to replace later.
if isinstance(concept, ObjectConcept):
matched_objects.append((semantic_node_for_match, match))
# A template only has to match once; we don't care about finding additional matches.
return
# For each template whose semantics we are certain of (=have been added to the lexicon)
for (concept, graph_pattern, score) in self._primary_templates():
check_state(isinstance(graph_pattern, PerceptionGraphTemplate))
if (
preprocessed_perception_graph.dynamic
== graph_pattern.graph_pattern.dynamic
):
match_template(concept=concept, pattern=graph_pattern, score=score)
else:
logging.debug(
f"Unable to try and match {concept} to {preprocessed_perception_graph} "
f"because both patterns must be static or dynamic"
)
if not match_to_score:
# Try to match against patterns being learned
# only if no lexicalized pattern was matched.
for (concept, graph_pattern, score) in self._fallback_templates():
# we may have multiple pattern hypotheses for a single concept, in which case we only want to identify the concept once
if not any(m[0].concept == concept for m in match_to_score):
match_template(concept=concept, pattern=graph_pattern, score=score)
perception_graph_after_matching = perception_semantic_alignment.perception_graph
# Replace any objects found
def by_pattern_complexity(pair):
_, pattern_match = pair
return len(pattern_match.matched_pattern)
matched_objects.sort(key=by_pattern_complexity, reverse=True)
already_replaced: Set[ObjectPerception] = set()
new_nodes: List[SemanticNode] = []
for (matched_object_node, pattern_match) in matched_objects:
root: ObjectPerception = _get_root_object_perception(
pattern_match.matched_sub_graph._graph, # pylint:disable=protected-access
immutableset(
pattern_match.matched_sub_graph._graph.nodes, # pylint:disable=protected-access
disable_order_check=True,
),
)
if root not in already_replaced:
perception_graph_after_matching = replace_match_root_with_object_semantic_node(
object_semantic_node=cast(ObjectSemanticNode, matched_object_node),
current_perception=perception_graph_after_matching,
pattern_match=pattern_match,
)
already_replaced.add(root)
new_nodes.append(matched_object_node)
else:
logging.info(
f"Matched pattern for {matched_object_node} "
f"but root object {root} already replaced."
)
if matched_objects:
immutable_new_nodes = immutableset(new_nodes)
else:
immutable_new_nodes = immutableset(node for (node, _) in match_to_score)
(
perception_graph_after_post_processing,
nodes_after_post_processing,
) = self._enrich_post_process(
perception_graph_after_matching, immutable_new_nodes
)
return (
perception_semantic_alignment.copy_with_updated_graph_and_added_nodes(
new_graph=perception_graph_after_post_processing,
new_nodes=nodes_after_post_processing,
),
nodes_after_post_processing,
)