def _find_partial_match(
self, hypothesis: PerceptionGraphTemplate, graph: PerceptionGraph
) -> "ObjectPursuitLearner.ObjectHypothesisPartialMatch":
pattern = hypothesis.graph_pattern
hypothesis_pattern_common_subgraph = get_largest_matching_pattern(
pattern,
graph,
debug_callback=self._debug_callback,
graph_logger=self._hypothesis_logger,
ontology=self._ontology,
match_mode=MatchMode.OBJECT,
)
self.debug_counter += 1
leading_hypothesis_num_nodes = len(pattern)
num_nodes_matched = (len(
hypothesis_pattern_common_subgraph.copy_as_digraph().nodes)
if hypothesis_pattern_common_subgraph else 0)
return ObjectPursuitLearner.ObjectHypothesisPartialMatch(
PerceptionGraphTemplate(
graph_pattern=hypothesis_pattern_common_subgraph)
if hypothesis_pattern_common_subgraph else None,
num_nodes_matched=num_nodes_matched,
num_nodes_in_pattern=leading_hypothesis_num_nodes,
)
def _find_partial_match(
self,
hypothesis: PerceptionGraphTemplate,
graph: PerceptionGraph,
*,
required_alignments: Mapping[SyntaxSemanticsVariable,
ObjectSemanticNode], # pylint:disable=unused-argument
) -> "PursuitObjectLearnerNew.ObjectHypothesisPartialMatch":
pattern = hypothesis.graph_pattern
hypothesis_pattern_common_subgraph = get_largest_matching_pattern(
pattern,
graph,
debug_callback=self._debug_callback,
graph_logger=self._hypothesis_logger,
ontology=self._ontology,
match_mode=MatchMode.OBJECT,
)
self.debug_counter += 1
leading_hypothesis_num_nodes = len(pattern)
num_nodes_matched = (len(
hypothesis_pattern_common_subgraph.copy_as_digraph().nodes)
if hypothesis_pattern_common_subgraph else 0)
return PursuitObjectLearnerNew.ObjectHypothesisPartialMatch(
PerceptionGraphTemplate(
graph_pattern=hypothesis_pattern_common_subgraph)
if hypothesis_pattern_common_subgraph else None,
num_nodes_matched=num_nodes_matched,
num_nodes_in_pattern=leading_hypothesis_num_nodes,
)
def _find_partial_match(
self, hypothesis: PerceptionGraphTemplate, graph: PerceptionGraph
) -> "PrepositionPursuitLearner.PrepositionHypothesisPartialMatch":
pattern = hypothesis.graph_pattern
hypothesis_pattern_common_subgraph = get_largest_matching_pattern(
pattern,
graph,
debug_callback=self._debug_callback,
graph_logger=self._hypothesis_logger,
ontology=self._ontology,
match_mode=MatchMode.OBJECT,
)
self.debug_counter += 1
leading_hypothesis_num_nodes = len(pattern)
num_nodes_matched = (len(
hypothesis_pattern_common_subgraph.copy_as_digraph().nodes)
if hypothesis_pattern_common_subgraph else 0)
if hypothesis_pattern_common_subgraph:
partial_hypothesis: Optional[
PerceptionGraphTemplate] = PerceptionGraphTemplate(
graph_pattern=hypothesis_pattern_common_subgraph,
template_variable_to_pattern_node=hypothesis.
template_variable_to_pattern_node,
)
else:
partial_hypothesis = None
return PrepositionPursuitLearner.PrepositionHypothesisPartialMatch(
partial_hypothesis,
num_nodes_matched=num_nodes_matched,
num_nodes_in_pattern=leading_hypothesis_num_nodes,
)
def _hypothesis_from_perception(
self, preprocessed_input: LanguageAlignedPerception
) -> PerceptionGraphTemplate:
return PerceptionGraphTemplate.from_graph(
preprocessed_input.perception_graph,
template_variable_to_matched_object_node=immutabledict(
zip(STANDARD_SLOT_VARIABLES,
preprocessed_input.aligned_nodes)),
)
def _hypothesis_from_perception(
self, preprocessed_input: LanguageAlignedPerception
) -> PerceptionGraphTemplate:
new_hypothesis = PerceptionGraphPattern.from_graph(
preprocessed_input.perception_graph).perception_graph_pattern
return PerceptionGraphTemplate(
graph_pattern=new_hypothesis,
template_variable_to_pattern_node=immutabledict(),
)
def _update_hypothesis(
self,
previous_pattern_hypothesis: PerceptionGraphTemplate,
current_pattern_hypothesis: PerceptionGraphTemplate,
) -> Optional[PerceptionGraphTemplate]:
return previous_pattern_hypothesis.intersection(
current_pattern_hypothesis,
ontology=self._ontology,
match_mode=MatchMode.NON_OBJECT,
)
def _hypotheses_from_perception(
self,
learning_state: LanguagePerceptionSemanticAlignment,
bound_surface_template: SurfaceTemplateBoundToSemanticNodes,
) -> AbstractSet[PerceptionGraphTemplate]:
# For the subset learner, our hypothesis is the entire graph.
return immutableset([
PerceptionGraphTemplate.from_graph(
learning_state.perception_semantic_alignment.perception_graph,
template_variable_to_matched_object_node=bound_surface_template
.slot_to_semantic_node,
)
])
def _hypotheses_from_perception(
self,
learning_state: LanguagePerceptionSemanticAlignment,
bound_surface_template: SurfaceTemplateBoundToSemanticNodes,
) -> AbstractSet[PerceptionGraphTemplate]:
# This makes a hypothesis for the whole graph, with the wildcard slot
# at each recognized object.
return immutableset([
PerceptionGraphTemplate.from_graph(
learning_state.perception_semantic_alignment.perception_graph,
template_variable_to_matched_object_node=bound_surface_template
.slot_to_semantic_node,
)
])
def _hypotheses_from_perception(
self,
learning_state: LanguagePerceptionSemanticAlignment,
bound_surface_template: SurfaceTemplateBoundToSemanticNodes,
) -> AbstractSet[PerceptionGraphTemplate]:
return immutableset(
PerceptionGraphTemplate.from_graph(
perception_graph=candidate_relation_meaning,
template_variable_to_matched_object_node=bound_surface_template
.slot_to_semantic_node,
) for candidate_relation_meaning in _extract_candidate_relations(
learning_state.perception_semantic_alignment.perception_graph,
bound_surface_template.slot_to_semantic_node[SLOT1],
bound_surface_template.slot_to_semantic_node[SLOT2],
))
def candidate_object_hypotheses(
language_perception_semantic_alignment: LanguagePerceptionSemanticAlignment
) -> Sequence[PerceptionGraphTemplate]:
"""
Given a learning input, returns all possible meaning hypotheses.
"""
return [
PerceptionGraphTemplate(
graph_pattern=PerceptionGraphPattern.from_graph(
object_
).perception_graph_pattern
)
for object_ in get_objects_from_perception(
language_perception_semantic_alignment.perception_semantic_alignment.perception_graph
)
]
def _hypothesis_from_perception(
self, preprocessed_input: LanguageAlignedPerception
) -> PerceptionGraphTemplate:
num_nodes_aligned_to_language = len(preprocessed_input.aligned_nodes)
if num_nodes_aligned_to_language != 1:
raise RuntimeError(
f"Attribute learner can work only with a single aligned node,"
f"but got {num_nodes_aligned_to_language}. Language is "
f"{preprocessed_input.language.as_token_string()}"
)
return PerceptionGraphTemplate.from_graph(
preprocessed_input.perception_graph,
template_variable_to_matched_object_node=immutabledict(
zip(STANDARD_SLOT_VARIABLES, preprocessed_input.aligned_nodes)
),
)
def _hypotheses_from_perception(
self,
learning_state: LanguagePerceptionSemanticAlignment,
bound_surface_template: SurfaceTemplateBoundToSemanticNodes,
) -> AbstractSet[PerceptionGraphTemplate]:
if bound_surface_template.slot_to_semantic_node:
raise RuntimeError(
"Object learner should not have slot to semantic node alignments!"
)
return immutableset(
PerceptionGraphTemplate(
graph_pattern=PerceptionGraphPattern.from_graph(
candidate_object).perception_graph_pattern,
template_variable_to_pattern_node=immutabledict(),
) for candidate_object in extract_candidate_objects(
learning_state.perception_semantic_alignment.perception_graph))
def _find_partial_match(
self,
hypothesis: PerceptionGraphTemplate,
graph: PerceptionGraph,
*,
required_alignments: Mapping[SyntaxSemanticsVariable, ObjectSemanticNode],
) -> "AbstractPursuitLearnerNew.PartialMatch":
pattern = hypothesis.graph_pattern
hypothesis_pattern_common_subgraph = get_largest_matching_pattern(
pattern,
graph,
debug_callback=self._debug_callback,
graph_logger=self._hypothesis_logger,
ontology=self._ontology,
match_mode=MatchMode.NON_OBJECT,
allowed_matches=immutablesetmultidict(
[
(hypothesis.template_variable_to_pattern_node[variable], object_node)
for variable, object_node in required_alignments.items()
]
),
)
self.debug_counter += 1
leading_hypothesis_num_nodes = len(pattern)
num_nodes_matched = (
len(hypothesis_pattern_common_subgraph.copy_as_digraph().nodes)
if hypothesis_pattern_common_subgraph
else 0
)
if hypothesis_pattern_common_subgraph:
partial_hypothesis: Optional[
PerceptionGraphTemplate
] = PerceptionGraphTemplate(
graph_pattern=hypothesis_pattern_common_subgraph,
template_variable_to_pattern_node=hypothesis.template_variable_to_pattern_node,
)
else:
partial_hypothesis = None
return PursuitAttributeLearnerNew.AttributeHypothesisPartialMatch(
partial_hypothesis,
num_nodes_matched=num_nodes_matched,
num_nodes_in_pattern=leading_hypothesis_num_nodes,
)
def _update_hypothesis(
self,
previous_pattern_hypothesis: PerceptionGraphTemplate,
current_pattern_hypothesis: PerceptionGraphTemplate,
) -> Optional[PerceptionGraphTemplate]:
return previous_pattern_hypothesis.intersection(
current_pattern_hypothesis,
ontology=self._ontology,
match_mode=MatchMode.NON_OBJECT,
allowed_matches=immutablesetmultidict([
(node2, node1)
for previous_slot, node1 in previous_pattern_hypothesis.
template_variable_to_pattern_node.items()
for new_slot, node2 in current_pattern_hypothesis.
template_variable_to_pattern_node.items()
if previous_slot == new_slot
]),
)
def match_template(*, concept: Concept,
pattern: PerceptionGraphTemplate,
score: float) -> None:
matches_with_nodes = self._match_template(
concept=concept,
pattern=pattern.copy_with_temporal_scopes(ENTIRE_SCENE)
if preprocessed_perception_graph.dynamic
and not pattern.graph_pattern.dynamic else pattern,
perception_graph=preprocessed_perception_graph,
)
# The template may have zero, one, or many matches, so we loop over the matches found
# Note that, with the exception of the object learners,
# some matches may be essentially identical to each other.
# This is fine because the corresponding semantic nodes will be equal,
# so they'll get thrown out when we take the immutable set of new nodes.
for match_with_node in matches_with_nodes:
(match, semantic_node_for_match) = match_with_node
match_to_score.append((semantic_node_for_match, score))
# We want to replace object matches with their semantic nodes,
# but we don't want to alter the graph while matching it,
# so we accumulate these to replace later.
if isinstance(concept, ObjectConcept):
matched_objects.append((semantic_node_for_match, match))
def _learning_step(
self,
language_perception_semantic_alignment:
LanguagePerceptionSemanticAlignment,
bound_surface_template: SurfaceTemplateBoundToSemanticNodes,
) -> None:
"""
Try to learn the semantics of a `SurfaceTemplate` given the assumption
that its argument slots (if any) are bound to objects according to
*bound_surface_template*.
For example, "try to learn the meaning of 'red' given the language 'red car'
and an alignment of 'car' to particular perceptions in the perception graph.
"""
# Generate all possible meanings from the Graph
meanings_from_perception = immutableset(
self._hypotheses_from_perception(
language_perception_semantic_alignment,
bound_surface_template))
meanings_to_pattern_template: Mapping[
PerceptionGraph, PerceptionGraphTemplate] = immutabledict(
(meaning,
PerceptionGraphTemplate.from_graph(meaning, immutabledict()))
for meaning in meanings_from_perception)
# We check for meanings that are described by lexicalized concepts
# and don't try to learn those lexicalized concepts further.
# jac: Not mentioned in the part of the paper I read. New?
concepts_to_remove: Set[Concept] = set()
def check_and_remove_meaning(
other_concept: Concept,
hypothesis: "AbstractCrossSituationalLearner.Hypothesis",
*,
ontology: Ontology,
) -> None:
match = compute_match_ratio(
hypothesis.pattern_template,
language_perception_semantic_alignment.
perception_semantic_alignment.perception_graph,
ontology=ontology,
)
if match and match.matching_subgraph:
for meaning in meanings_from_perception:
if match.matching_subgraph.check_isomorphism(
meanings_to_pattern_template[meaning].graph_pattern
):
concepts_to_remove.add(other_concept)
for (other_concept, hypotheses) in self._concept_to_hypotheses.items():
for hypothesis in hypotheses:
if hypothesis.probability > self._lexicon_entry_threshold:
check_and_remove_meaning(other_concept,
hypothesis,
ontology=self._ontology)
# We have seen this template before and already have a concept for it
# So we attempt to verify our already picked concept
if bound_surface_template.surface_template in self._surface_template_to_concept:
# We don't directly associate surface templates with perceptions.
# Instead we mediate the relationship with "concept" objects.
# These don't matter now, but the split might be helpful in the future
# when we might have multiple ways of expressing the same idea.
concept = self._surface_template_to_concept[
bound_surface_template.surface_template]
else:
concept = self._new_concept(debug_string=bound_surface_template.
surface_template.to_short_string())
self._surface_template_to_concept[
bound_surface_template.surface_template] = concept
self._concept_to_surface_template[
concept] = bound_surface_template.surface_template
concepts_after_preprocessing = immutableset([
concept for concept in self._concepts_in_utterance
if concept not in concepts_to_remove
# TODO Does it make sense to include a dummy concept/"word"? The paper has one so I
# am including it for now.
] + [self._dummy_concept])
# Step 0. Update priors for any meanings as-yet unobserved.
# Step 1. Compute alignment probabilities (pp. 1029)
# We have an identified "word" (concept) from U(t)
# and a collection of meanings from the scene S(t).
# We now want to calculate the alignment probabilities,
# which will be used to update this concept's association scores, assoc(w|m, U(t), S(t)),
# and meaning probabilities, p(m|w).
alignment_probabilities = self._get_alignment_probabilities(
concepts_after_preprocessing, meanings_from_perception)
# We have an identified "word" (concept) from U(t)
# and a collection of meanings from the scene S(t).
# We now want to update p(.|w), which means calculating the probabilities.
new_hypotheses = self._updated_meaning_probabilities(
concept,
meanings_from_perception,
meanings_to_pattern_template,
alignment_probabilities,
)
# Finally, update our hypotheses for this concept
self._updated_hypotheses[concept] = new_hypotheses
def preposition_hypothesis_from_perception(
scene_aligned_perception: LanguageAlignedPerception,
template_variables_to_object_match_nodes: Mapping[SyntaxSemanticsVariable,
ObjectSemanticNode],
) -> PerceptionGraphTemplate:
"""
Create a hypothesis for the semantics of a preposition based on the observed scene.
Our current implementation is to just include the content
on the path between the recognized object nodes
and one hop away from that path.
"""
# The directions of edges in the perception graph are not necessarily meaningful
# from the point-of-view of hypothesis generation, so we need an undirected copy
# of the graph.
perception_digraph = scene_aligned_perception.perception_graph.copy_as_digraph(
)
perception_graph_undirected = perception_digraph.to_undirected(
# as_view=True loses determinism
as_view=False)
if {SLOT1, SLOT2} != set(template_variables_to_object_match_nodes.keys()):
raise RuntimeError(
"Can only make a preposition hypothesis if the recognized "
"objects are aligned to SurfaceTemplateVariables SLOT1 and SLOT2")
slot1_object = template_variables_to_object_match_nodes[SLOT1]
slot2_object = template_variables_to_object_match_nodes[SLOT2]
# The core of our hypothesis for the semantics of a preposition is all nodes
# along the shortest path between the two objects involved in the perception graph.
hypothesis_spine_nodes: ImmutableSet[PerceptionGraphNode] = immutableset(
flatten(
# if there are multiple paths between the object match nodes,
# we aren't sure which are relevant, so we include them all in our hypothesis
# and figure we can trim out irrelevant stuff as we make more observations.
all_shortest_paths(perception_graph_undirected, slot2_object,
slot1_object)))
# Along the core of our hypothesis we also want to collect the predecessors and successors
hypothesis_nodes_mutable = []
for node in hypothesis_spine_nodes:
if node not in {slot1_object, slot2_object}:
for successor in perception_digraph.successors(node):
if not isinstance(successor, ObjectPerception):
hypothesis_nodes_mutable.append(successor)
for predecessor in perception_digraph.predecessors(node):
if not isinstance(predecessor, ObjectPerception):
hypothesis_nodes_mutable.append(predecessor)
hypothesis_nodes_mutable.extend(hypothesis_spine_nodes)
# We wrap the nodes in an immutable set to remove duplicates
# while preserving iteration determinism.
hypothesis_nodes = immutableset(hypothesis_nodes_mutable)
preposition_sub_graph = PerceptionGraph(
digraph_with_nodes_sorted_by(
subgraph(perception_digraph, hypothesis_nodes), _graph_node_order))
return PerceptionGraphTemplate.from_graph(
preposition_sub_graph, template_variables_to_object_match_nodes)
def _hypothesis_from_perception(
self, perception: PerceptionGraph) -> PerceptionGraphTemplate:
return PerceptionGraphTemplate(
graph_pattern=PerceptionGraphPattern.from_graph(
perception).perception_graph_pattern)
