def _extract_candidate_attributes(
whole_scene_perception_graph: PerceptionGraph,
object_with_attribute: ObjectSemanticNode,
) -> Sequence[PerceptionGraph]:
perception_digraph = whole_scene_perception_graph.copy_as_digraph()
# For now, we assume all attributes are based on properties.
properties = immutableset(
[
node
for _, node, label in perception_digraph.out_edges(
object_with_attribute, data="label"
)
if label == HAS_PROPERTY_LABEL
]
)
# Furthermore, we limit the search space to the even smaller set of hypotheses
# where we consider only single properties as possible attributes.
# Otherwise there are too many hypotheses for the pursuit learner to search through
# and it's unlikely to converge on the correct hypothesis
# in any reasonable amount of time or number of examples.
candidate_attribute_subgraph_node_sets = [
immutableset([object_with_attribute, property]) for property in properties
]
return immutableset(
[
whole_scene_perception_graph.subgraph_by_nodes(
candidate_attribute_subgraph_nodes
)
for candidate_attribute_subgraph_nodes in candidate_attribute_subgraph_node_sets
]
)
def _extract_candidate_relations(
whole_scene_perception_graph: PerceptionGraph,
relation_object_1: ObjectSemanticNode,
relation_object_2: ObjectSemanticNode,
) -> Sequence[PerceptionGraph]:
# The directions of edges in the perception graph are not necessarily meaningful
# from the point-of-view of hypothesis generation, so we need an undirected copy
# of the graph.
perception_digraph = whole_scene_perception_graph.copy_as_digraph()
perception_graph_undirected = perception_digraph.to_undirected(
# as_view=True loses determinism
as_view=False)
output_graphs = []
# The core of our hypothesis for the semantics of a preposition is all nodes
# along the shortest path between the two objects involved in the perception graph.
for hypothesis_spine_nodes in all_shortest_paths(
perception_graph_undirected, relation_object_2, relation_object_1):
# Along the core of our hypothesis we also want to collect the predecessors and successors
hypothesis_nodes_mutable = []
for node in hypothesis_spine_nodes:
if node not in {relation_object_1, relation_object_2}:
for successor in perception_digraph.successors(node):
if not (isinstance(successor, ObjectPerception)
or isinstance(successor, ObjectSemanticNode)):
hypothesis_nodes_mutable.append(successor)
for predecessor in perception_digraph.predecessors(node):
if not (isinstance(predecessor, ObjectPerception)
or isinstance(predecessor, ObjectSemanticNode)):
hypothesis_nodes_mutable.append(predecessor)
hypothesis_nodes_mutable.extend(hypothesis_spine_nodes)
# We wrap the nodes in an immutable set to remove duplicates
# while preserving iteration determinism.
hypothesis_nodes = immutableset(hypothesis_nodes_mutable)
output_graphs.append(
PerceptionGraph(
digraph_with_nodes_sorted_by(
subgraph(perception_digraph, hypothesis_nodes),
_graph_node_order)))
return output_graphs
def graph_without_learner(
perception_graph: PerceptionGraph) -> PerceptionGraph:
""" Helper function to return a `PerceptionGraph`
without a ground object and its related nodes."""
graph = perception_graph.copy_as_digraph()
# Get the learner node
learner_node_candidates = [
node for node in graph.nodes() if isinstance(node, ObjectPerception)
and node.debug_handle == LEARNER.handle
]
if len(learner_node_candidates) > 1:
raise RuntimeError("More than one learners in perception.")
elif len(learner_node_candidates) == 1:
learner_node = first(learner_node_candidates)
# Remove learner
graph.remove_node(learner_node)
# remove remaining islands
islands = list(isolates(graph))
graph.remove_nodes_from(islands)
return PerceptionGraph(graph, dynamic=perception_graph.dynamic)
def get_objects_from_perception(
observed_perception_graph: PerceptionGraph
) -> List[PerceptionGraph]:
"""
Utility function to get a list of `PerceptionGraphs` which are independent objects in the scene
"""
perception_as_digraph = observed_perception_graph.copy_as_digraph()
perception_as_graph = perception_as_digraph.to_undirected()
meanings = []
# 1) Take all of the obj perc that dont have part of relationships with anything else
root_object_percetion_nodes = []
for node in perception_as_graph.nodes:
if isinstance(node, ObjectPerception) and node.debug_handle != "the ground":
if not any(
[
u == node and str(data["label"]) == "partOf"
for u, v, data in perception_as_digraph.edges.data()
]
):
root_object_percetion_nodes.append(node)
# 2) for each of these, walk along the part of relationships backwards,
# i.e find all of the subparts of the root object
for root_object_perception_node in root_object_percetion_nodes:
# Iteratively get all other object perceptions that connect to a root with a part of
# relation
all_object_perception_nodes = [root_object_perception_node]
frontier = [root_object_perception_node]
updated = True
while updated:
updated = False
new_frontier = []
for frontier_node in frontier:
for node in perception_as_graph.neighbors(frontier_node):
edge_data = perception_as_digraph.get_edge_data(
node, frontier_node, default=-1
)
if edge_data != -1 and str(edge_data["label"]) == "partOf":
new_frontier.append(node)
if new_frontier:
all_object_perception_nodes.extend(new_frontier)
updated = True
frontier = new_frontier
# Now we have a list of all perceptions that are connected
# 3) For each of these objects including root object, get axes, properties,
# and relations and regions which are between these internal object perceptions
other_nodes = []
for node in all_object_perception_nodes:
for neighbor in perception_as_graph.neighbors(node):
# Filter out regions that don't have a reference in all object perception nodes
# TODO: We currently remove colors to achieve a match - otherwise finding
# patterns fails.
if (
isinstance(neighbor, Region)
and neighbor.reference_object not in all_object_perception_nodes
or isinstance(neighbor, RgbColorPerception)
):
continue
# Append all other none-object nodes to be kept in the subgraph
if not isinstance(neighbor, ObjectPerception):
other_nodes.append(neighbor)
generated_subgraph = networkx_utils.subgraph(
perception_as_digraph, all_object_perception_nodes + other_nodes
)
meanings.append(PerceptionGraph(generated_subgraph))
logging.info(f"Got {len(meanings)} candidate meanings")
return meanings
def replace_match_root_with_object_semantic_node(
object_semantic_node: ObjectSemanticNode,
current_perception: PerceptionGraph,
pattern_match: PerceptionGraphPatternMatch,
) -> PerceptionGraph:
"""
Internal function to replace the root node of the perception matched by the object pattern
with an `ObjectSemanticNode`.
The `ObjectSemanticNode` inherits both the root node's internal relationships and all external
relationships involving either the root node or its children.
"""
perception_digraph = current_perception.copy_as_digraph()
perception_digraph.add_node(object_semantic_node)
matched_subgraph_nodes: ImmutableSet[PerceptionGraphNode] = immutableset(
pattern_match.matched_sub_graph._graph.nodes, # pylint:disable=protected-access
disable_order_check=True,
)
root = _get_root_object_perception(perception_digraph,
matched_subgraph_nodes)
# Multiple sub-objects of a matched object may link to the same property
# (for example, to a color shared by all the parts).
# In this case, we want the shared object node to link to this property only once.
external_properties: Set[Union[OntologyNode, ObjectSemanticNode]] = set()
for matched_subgraph_node in matched_subgraph_nodes:
if isinstance(matched_subgraph_node, ObjectSemanticNode):
raise RuntimeError(
f"We do not currently allow object recognitions to themselves "
f"operate over other object recognitions, but got match "
f"{pattern_match.matched_sub_graph}")
# A pattern might refer to shared parts of the world like the learner
# or the ground, and we don't want the replacement root to inherit the
# shared world item's relationships.
if matched_subgraph_node in SHARED_WORLD_ITEMS:
continue
# If there is an edge from the matched sub-graph to a node outside it,
# also add an edge from the object match node to that node.
for matched_subgraph_node_successor in perception_digraph.successors(
matched_subgraph_node):
edge_label = _get_edge_label(perception_digraph,
matched_subgraph_node,
matched_subgraph_node_successor)
# don't want to add edges which are internal to the matched sub-graph
if matched_subgraph_node_successor not in matched_subgraph_nodes:
if edge_equals_ignoring_temporal_scope(edge_label,
HAS_PROPERTY_LABEL):
# Prevent multiple `has-property` assertions to the same color node
# On a recognized object
if matched_subgraph_node_successor[
0] in external_properties or matched_subgraph_node_successor[
0] in {
SMALLER_THAN,
BIGGER_THAN,
}:
if (perception_digraph.degree(
matched_subgraph_node_successor) != 1):
raise_graph_exception(
f"Node {matched_subgraph_node_successor} "
f"appears to be a duplicate property node, "
f"but has degree != 1",
current_perception,
)
continue
else:
external_properties.add(
matched_subgraph_node_successor[0])
perception_digraph.add_edge(
object_semantic_node,
matched_subgraph_node_successor,
label=edge_label,
)
# If there is an edge to the matched sub-graph from a node outside it,
# also add an edge to the object match node from that node.
for matched_subgraph_node_predecessor in perception_digraph.predecessors(
matched_subgraph_node):
edge_label = _get_edge_label(
perception_digraph,
matched_subgraph_node_predecessor,
matched_subgraph_node,
)
# don't want to add edges which are internal to the matched sub-graph
if matched_subgraph_node_predecessor not in matched_subgraph_nodes:
if edge_equals_ignoring_temporal_scope(edge_label,
HAS_PROPERTY_LABEL):
# Prevent multiple `has-property` assertions to the same color node
# On a recognized object
if isinstance(matched_subgraph_node_predecessor,
ObjectSemanticNode):
prop = matched_subgraph_node_predecessor
else:
prop = matched_subgraph_node_predecessor[0]
if prop in external_properties:
if (perception_digraph.degree(
matched_subgraph_node_predecessor) != 1):
raise_graph_exception(
f"Node {matched_subgraph_node_predecessor} "
f"appears to be a duplicate property node, "
f"but has degree != 1",
current_perception,
)
continue
else:
external_properties.add(prop)
perception_digraph.add_edge(
matched_subgraph_node_predecessor,
object_semantic_node,
label=edge_label,
)
if root in SHARED_WORLD_ITEMS:
raise RuntimeError(
f"Pattern match root cannot be a shared world item, "
f"but got match {pattern_match}")
perception_digraph.remove_node(root)
# We want to re-add any relationships linked directly to the root node of an object.
# Example: water is a liquid
# Example: this hand is a part of this person
# These may be relevant to learning verb semantics
# (e.g. you can only drink a liquid)
_add_relationships_linked_to_root_object_perception(
original_graph=current_perception.copy_as_digraph(),
output_graph=perception_digraph,
matched_nodes=matched_subgraph_nodes,
matched_object_node=object_semantic_node,
)
return PerceptionGraph(perception_digraph,
dynamic=current_perception.dynamic)
def extract_candidate_objects(
whole_scene_perception_graph: PerceptionGraph
) -> Sequence[PerceptionGraph]:
"""
Pulls out distinct objects from a scene.
We will attempt to recognize only these and will ignore other parts of the scene.
"""
scene_digraph = whole_scene_perception_graph.copy_as_digraph()
def is_part_of_label(label) -> bool:
return label == PART_OF or (isinstance(
label, TemporallyScopedEdgeLabel) and label.attribute == PART_OF)
# We first identify root object nodes, which are object nodes with no part-of
# relationship with other object nodes.
def is_root_object_node(node) -> bool:
if isinstance(node, ObjectPerception):
for (_, _, edge_label) in scene_digraph.out_edges(node,
data="label"):
if is_part_of_label(edge_label):
# This object node is part of another object and cannot be a root.
return False
return True
return False
candidate_object_root_nodes = [
node for node in scene_digraph.nodes
if is_root_object_node(node) and node not in (GROUND_PERCEPTION,
LEARNER_PERCEPTION)
]
candidate_objects: List[PerceptionGraph] = []
for root_object_node in candidate_object_root_nodes:
# Having identified the root nodes of the candidate objects,
# we now gather all sub-object nodes.
object_nodes_in_object_list = []
nodes_to_examine = [root_object_node]
# This would be clearer recursively
# but I'm betting this implementation is a bit faster in Python.
nodes_visited: Set[PerceptionGraphNode] = set()
while nodes_to_examine:
node_to_examine = nodes_to_examine.pop()
if node_to_examine in nodes_visited:
continue
nodes_visited.add(node_to_examine)
object_nodes_in_object_list.append(node_to_examine)
for (next_node, _,
edge_label) in scene_digraph.in_edges(node_to_examine,
data="label"):
if is_part_of_label(edge_label):
nodes_to_examine.append(next_node)
object_nodes_in_object = immutableset(object_nodes_in_object_list)
# Now we know all object nodes for this candidate object.
# Finally, we find the sub-graph to match against which could possibly correspond
# to this candidate object
# by performing a BFS over the graph
# but *stopping whenever we encounter an object node
# which is not part of this candidate object*.
# This is a little more generous than we need to be, but it's simple.
nodes_to_examine = [root_object_node]
candidate_subgraph_nodes = []
nodes_visited.clear()
while nodes_to_examine:
node_to_examine = nodes_to_examine.pop()
is_allowable_node = (
not isinstance(node_to_examine,
(ObjectPerception, ObjectSemanticNode))
or node_to_examine in object_nodes_in_object)
if node_to_examine not in nodes_visited and is_allowable_node:
nodes_visited.add(node_to_examine)
candidate_subgraph_nodes.append(node_to_examine)
nodes_to_examine.extend(
out_neighbor
for (_, out_neighbor
) in scene_digraph.out_edges(node_to_examine)
if not (isinstance(out_neighbor, tuple) and isinstance(
out_neighbor[0], Region) and out_neighbor[0].
reference_object == GROUND_PERCEPTION))
nodes_to_examine.extend(
in_neighbor
for (in_neighbor,
_) in scene_digraph.in_edges(node_to_examine)
# Avoid in-edges from Regions as they can come from other objects regions (e.g ground).
if not (
# isinstance(node_to_examine, GeonAxis)
isinstance(in_neighbor, tuple)
and isinstance(in_neighbor[0], Region)))
candidate_objects.append(
whole_scene_perception_graph.subgraph_by_nodes(
immutableset(candidate_subgraph_nodes)))
return candidate_objects
def replace_match_with_object_graph_node(
matched_object_node: ObjectSemanticNode,
current_perception: PerceptionGraph,
pattern_match: PerceptionGraphPatternMatch,
) -> PerceptionGraphWithReplacedObjectResult:
"""
Internal function to replace the nodes of the perception matched by the object pattern
with an `ObjectSemanticNode`.
Any external relationships those nodes had is inherited by the `ObjectSemanticNode`.
"""
perception_digraph = current_perception.copy_as_digraph()
perception_digraph.add_node(matched_object_node)
matched_subgraph_nodes: ImmutableSet[PerceptionGraphNode] = immutableset(
[
node for node in pattern_match.matched_sub_graph._graph.nodes # pylint:disable=protected-access
if node in perception_digraph.nodes
],
disable_order_check=True,
)
# Multiple sub-objects of a matched object may link to the same property
# (for example, to a color shared by all the parts).
# In this case, we want the shared object node to link to this property only once.
external_properties: Set[Union[OntologyNode, ObjectSemanticNode]] = set()
duplicate_nodes_to_remove: List[PerceptionGraphNode] = []
for matched_subgraph_node in matched_subgraph_nodes:
if isinstance(matched_subgraph_node, ObjectSemanticNode):
raise RuntimeError(
f"We do not currently allow object recognitions to themselves "
f"operate over other object recognitions, but got match "
f"{pattern_match.matched_sub_graph}")
# A pattern might refer to shared parts of the world like the learner
# or the ground, and we don't want the replacement root to inherit the
# shared world item's relationships.
if matched_subgraph_node in SHARED_WORLD_ITEMS:
continue
# If there is an edge from the matched sub-graph to a node outside it,
# also add an edge from the object match node to that node.
for matched_subgraph_node_successor in perception_digraph.successors(
matched_subgraph_node):
edge_label = _get_edge_label(perception_digraph,
matched_subgraph_node,
matched_subgraph_node_successor)
# don't want to add edges which are internal to the matched sub-graph
if matched_subgraph_node_successor not in matched_subgraph_nodes:
if edge_equals_ignoring_temporal_scope(edge_label,
HAS_PROPERTY_LABEL):
# Prevent multiple `has-property` assertions to the same color node
# On a recognized object
# Also prevent size relations from being inherited on root object.
if matched_subgraph_node_successor[
0] in external_properties or matched_subgraph_node_successor[
0] in {
SMALLER_THAN,
BIGGER_THAN,
ABOUT_THE_SAME_SIZE_AS_LEARNER,
}:
if (perception_digraph.degree(
matched_subgraph_node_successor) != 1):
raise_graph_exception(
f"Node {matched_subgraph_node_successor} "
f"appears to be a duplicate property node, "
f"but has degree != 1",
current_perception,
)
duplicate_nodes_to_remove.append(
matched_subgraph_node_successor)
continue
else:
external_properties.add(
matched_subgraph_node_successor[0])
perception_digraph.add_edge(matched_object_node,
matched_subgraph_node_successor,
label=edge_label)
# If there is an edge to the matched sub-graph from a node outside it,
# also add an edge to the object match node from that node.
for matched_subgraph_node_predecessor in perception_digraph.predecessors(
matched_subgraph_node):
edge_label = _get_edge_label(
perception_digraph,
matched_subgraph_node_predecessor,
matched_subgraph_node,
)
# don't want to add edges which are internal to the matched sub-graph
if matched_subgraph_node_predecessor not in matched_subgraph_nodes:
if edge_equals_ignoring_temporal_scope(edge_label,
HAS_PROPERTY_LABEL):
# Prevent multiple `has-property` assertions to the same color node
# On a recognized object
if isinstance(matched_subgraph_node_predecessor,
ObjectSemanticNode):
prop = matched_subgraph_node_predecessor
else:
prop = matched_subgraph_node_predecessor[0]
if prop in external_properties:
if (perception_digraph.degree(
matched_subgraph_node_predecessor) != 1):
raise_graph_exception(
f"Node {matched_subgraph_node_predecessor} "
f"appears to be a duplicate property node, "
f"but has degree != 1",
current_perception,
)
duplicate_nodes_to_remove.append(
matched_subgraph_node_predecessor)
continue
else:
external_properties.add(prop)
perception_digraph.add_edge(
matched_subgraph_node_predecessor,
matched_object_node,
label=edge_label,
)
# Remove all matched nodes which are not shared world items (e.g. gravity, the learner)
to_remove = immutableset([
matched_node for matched_node in matched_subgraph_nodes
if matched_node not in SHARED_WORLD_ITEMS
] + duplicate_nodes_to_remove)
perception_digraph.remove_nodes_from(to_remove)
# We want to re-add any properties linked directly to the root node of an object.
# Example: water is a liquid
# These may be relevant to learning verb semantics
# (e.g. you can only drink a liquid)
_add_external_properties_linked_to_root_object_perception(
original_graph=current_perception.copy_as_digraph(),
output_graph=perception_digraph,
matched_nodes=matched_subgraph_nodes,
matched_object_node=matched_object_node,
)
# We also want to re-add any relationships directly linked to the root node of an object.
# These may have been overwritten when iterating over the matched nodes.
# Example: ball is behind table
_add_external_relationships_linked_to_root_object_perception(
original_graph=current_perception.copy_as_digraph(),
output_graph=perception_digraph,
matched_nodes=matched_subgraph_nodes,
matched_object_node=matched_object_node,
)
return PerceptionGraphWithReplacedObjectResult(
PerceptionGraph(perception_digraph,
dynamic=current_perception.dynamic), to_remove)