def _extract_candidate_attributes( whole_scene_perception_graph: PerceptionGraph, object_with_attribute: ObjectSemanticNode, ) -> Sequence[PerceptionGraph]: perception_digraph = whole_scene_perception_graph.copy_as_digraph() # For now, we assume all attributes are based on properties. properties = immutableset( [ node for _, node, label in perception_digraph.out_edges( object_with_attribute, data="label" ) if label == HAS_PROPERTY_LABEL ] ) # Furthermore, we limit the search space to the even smaller set of hypotheses # where we consider only single properties as possible attributes. # Otherwise there are too many hypotheses for the pursuit learner to search through # and it's unlikely to converge on the correct hypothesis # in any reasonable amount of time or number of examples. candidate_attribute_subgraph_node_sets = [ immutableset([object_with_attribute, property]) for property in properties ] return immutableset( [ whole_scene_perception_graph.subgraph_by_nodes( candidate_attribute_subgraph_nodes ) for candidate_attribute_subgraph_nodes in candidate_attribute_subgraph_node_sets ] )
def _extract_candidate_relations( whole_scene_perception_graph: PerceptionGraph, relation_object_1: ObjectSemanticNode, relation_object_2: ObjectSemanticNode, ) -> Sequence[PerceptionGraph]: # The directions of edges in the perception graph are not necessarily meaningful # from the point-of-view of hypothesis generation, so we need an undirected copy # of the graph. perception_digraph = whole_scene_perception_graph.copy_as_digraph() perception_graph_undirected = perception_digraph.to_undirected( # as_view=True loses determinism as_view=False) output_graphs = [] # The core of our hypothesis for the semantics of a preposition is all nodes # along the shortest path between the two objects involved in the perception graph. for hypothesis_spine_nodes in all_shortest_paths( perception_graph_undirected, relation_object_2, relation_object_1): # Along the core of our hypothesis we also want to collect the predecessors and successors hypothesis_nodes_mutable = [] for node in hypothesis_spine_nodes: if node not in {relation_object_1, relation_object_2}: for successor in perception_digraph.successors(node): if not (isinstance(successor, ObjectPerception) or isinstance(successor, ObjectSemanticNode)): hypothesis_nodes_mutable.append(successor) for predecessor in perception_digraph.predecessors(node): if not (isinstance(predecessor, ObjectPerception) or isinstance(predecessor, ObjectSemanticNode)): hypothesis_nodes_mutable.append(predecessor) hypothesis_nodes_mutable.extend(hypothesis_spine_nodes) # We wrap the nodes in an immutable set to remove duplicates # while preserving iteration determinism. hypothesis_nodes = immutableset(hypothesis_nodes_mutable) output_graphs.append( PerceptionGraph( digraph_with_nodes_sorted_by( subgraph(perception_digraph, hypothesis_nodes), _graph_node_order))) return output_graphs
def graph_without_learner( perception_graph: PerceptionGraph) -> PerceptionGraph: """ Helper function to return a `PerceptionGraph` without a ground object and its related nodes.""" graph = perception_graph.copy_as_digraph() # Get the learner node learner_node_candidates = [ node for node in graph.nodes() if isinstance(node, ObjectPerception) and node.debug_handle == LEARNER.handle ] if len(learner_node_candidates) > 1: raise RuntimeError("More than one learners in perception.") elif len(learner_node_candidates) == 1: learner_node = first(learner_node_candidates) # Remove learner graph.remove_node(learner_node) # remove remaining islands islands = list(isolates(graph)) graph.remove_nodes_from(islands) return PerceptionGraph(graph, dynamic=perception_graph.dynamic)
def get_objects_from_perception( observed_perception_graph: PerceptionGraph ) -> List[PerceptionGraph]: """ Utility function to get a list of `PerceptionGraphs` which are independent objects in the scene """ perception_as_digraph = observed_perception_graph.copy_as_digraph() perception_as_graph = perception_as_digraph.to_undirected() meanings = [] # 1) Take all of the obj perc that dont have part of relationships with anything else root_object_percetion_nodes = [] for node in perception_as_graph.nodes: if isinstance(node, ObjectPerception) and node.debug_handle != "the ground": if not any( [ u == node and str(data["label"]) == "partOf" for u, v, data in perception_as_digraph.edges.data() ] ): root_object_percetion_nodes.append(node) # 2) for each of these, walk along the part of relationships backwards, # i.e find all of the subparts of the root object for root_object_perception_node in root_object_percetion_nodes: # Iteratively get all other object perceptions that connect to a root with a part of # relation all_object_perception_nodes = [root_object_perception_node] frontier = [root_object_perception_node] updated = True while updated: updated = False new_frontier = [] for frontier_node in frontier: for node in perception_as_graph.neighbors(frontier_node): edge_data = perception_as_digraph.get_edge_data( node, frontier_node, default=-1 ) if edge_data != -1 and str(edge_data["label"]) == "partOf": new_frontier.append(node) if new_frontier: all_object_perception_nodes.extend(new_frontier) updated = True frontier = new_frontier # Now we have a list of all perceptions that are connected # 3) For each of these objects including root object, get axes, properties, # and relations and regions which are between these internal object perceptions other_nodes = [] for node in all_object_perception_nodes: for neighbor in perception_as_graph.neighbors(node): # Filter out regions that don't have a reference in all object perception nodes # TODO: We currently remove colors to achieve a match - otherwise finding # patterns fails. if ( isinstance(neighbor, Region) and neighbor.reference_object not in all_object_perception_nodes or isinstance(neighbor, RgbColorPerception) ): continue # Append all other none-object nodes to be kept in the subgraph if not isinstance(neighbor, ObjectPerception): other_nodes.append(neighbor) generated_subgraph = networkx_utils.subgraph( perception_as_digraph, all_object_perception_nodes + other_nodes ) meanings.append(PerceptionGraph(generated_subgraph)) logging.info(f"Got {len(meanings)} candidate meanings") return meanings
def replace_match_root_with_object_semantic_node( object_semantic_node: ObjectSemanticNode, current_perception: PerceptionGraph, pattern_match: PerceptionGraphPatternMatch, ) -> PerceptionGraph: """ Internal function to replace the root node of the perception matched by the object pattern with an `ObjectSemanticNode`. The `ObjectSemanticNode` inherits both the root node's internal relationships and all external relationships involving either the root node or its children. """ perception_digraph = current_perception.copy_as_digraph() perception_digraph.add_node(object_semantic_node) matched_subgraph_nodes: ImmutableSet[PerceptionGraphNode] = immutableset( pattern_match.matched_sub_graph._graph.nodes, # pylint:disable=protected-access disable_order_check=True, ) root = _get_root_object_perception(perception_digraph, matched_subgraph_nodes) # Multiple sub-objects of a matched object may link to the same property # (for example, to a color shared by all the parts). # In this case, we want the shared object node to link to this property only once. external_properties: Set[Union[OntologyNode, ObjectSemanticNode]] = set() for matched_subgraph_node in matched_subgraph_nodes: if isinstance(matched_subgraph_node, ObjectSemanticNode): raise RuntimeError( f"We do not currently allow object recognitions to themselves " f"operate over other object recognitions, but got match " f"{pattern_match.matched_sub_graph}") # A pattern might refer to shared parts of the world like the learner # or the ground, and we don't want the replacement root to inherit the # shared world item's relationships. if matched_subgraph_node in SHARED_WORLD_ITEMS: continue # If there is an edge from the matched sub-graph to a node outside it, # also add an edge from the object match node to that node. for matched_subgraph_node_successor in perception_digraph.successors( matched_subgraph_node): edge_label = _get_edge_label(perception_digraph, matched_subgraph_node, matched_subgraph_node_successor) # don't want to add edges which are internal to the matched sub-graph if matched_subgraph_node_successor not in matched_subgraph_nodes: if edge_equals_ignoring_temporal_scope(edge_label, HAS_PROPERTY_LABEL): # Prevent multiple `has-property` assertions to the same color node # On a recognized object if matched_subgraph_node_successor[ 0] in external_properties or matched_subgraph_node_successor[ 0] in { SMALLER_THAN, BIGGER_THAN, }: if (perception_digraph.degree( matched_subgraph_node_successor) != 1): raise_graph_exception( f"Node {matched_subgraph_node_successor} " f"appears to be a duplicate property node, " f"but has degree != 1", current_perception, ) continue else: external_properties.add( matched_subgraph_node_successor[0]) perception_digraph.add_edge( object_semantic_node, matched_subgraph_node_successor, label=edge_label, ) # If there is an edge to the matched sub-graph from a node outside it, # also add an edge to the object match node from that node. for matched_subgraph_node_predecessor in perception_digraph.predecessors( matched_subgraph_node): edge_label = _get_edge_label( perception_digraph, matched_subgraph_node_predecessor, matched_subgraph_node, ) # don't want to add edges which are internal to the matched sub-graph if matched_subgraph_node_predecessor not in matched_subgraph_nodes: if edge_equals_ignoring_temporal_scope(edge_label, HAS_PROPERTY_LABEL): # Prevent multiple `has-property` assertions to the same color node # On a recognized object if isinstance(matched_subgraph_node_predecessor, ObjectSemanticNode): prop = matched_subgraph_node_predecessor else: prop = matched_subgraph_node_predecessor[0] if prop in external_properties: if (perception_digraph.degree( matched_subgraph_node_predecessor) != 1): raise_graph_exception( f"Node {matched_subgraph_node_predecessor} " f"appears to be a duplicate property node, " f"but has degree != 1", current_perception, ) continue else: external_properties.add(prop) perception_digraph.add_edge( matched_subgraph_node_predecessor, object_semantic_node, label=edge_label, ) if root in SHARED_WORLD_ITEMS: raise RuntimeError( f"Pattern match root cannot be a shared world item, " f"but got match {pattern_match}") perception_digraph.remove_node(root) # We want to re-add any relationships linked directly to the root node of an object. # Example: water is a liquid # Example: this hand is a part of this person # These may be relevant to learning verb semantics # (e.g. you can only drink a liquid) _add_relationships_linked_to_root_object_perception( original_graph=current_perception.copy_as_digraph(), output_graph=perception_digraph, matched_nodes=matched_subgraph_nodes, matched_object_node=object_semantic_node, ) return PerceptionGraph(perception_digraph, dynamic=current_perception.dynamic)
def extract_candidate_objects( whole_scene_perception_graph: PerceptionGraph ) -> Sequence[PerceptionGraph]: """ Pulls out distinct objects from a scene. We will attempt to recognize only these and will ignore other parts of the scene. """ scene_digraph = whole_scene_perception_graph.copy_as_digraph() def is_part_of_label(label) -> bool: return label == PART_OF or (isinstance( label, TemporallyScopedEdgeLabel) and label.attribute == PART_OF) # We first identify root object nodes, which are object nodes with no part-of # relationship with other object nodes. def is_root_object_node(node) -> bool: if isinstance(node, ObjectPerception): for (_, _, edge_label) in scene_digraph.out_edges(node, data="label"): if is_part_of_label(edge_label): # This object node is part of another object and cannot be a root. return False return True return False candidate_object_root_nodes = [ node for node in scene_digraph.nodes if is_root_object_node(node) and node not in (GROUND_PERCEPTION, LEARNER_PERCEPTION) ] candidate_objects: List[PerceptionGraph] = [] for root_object_node in candidate_object_root_nodes: # Having identified the root nodes of the candidate objects, # we now gather all sub-object nodes. object_nodes_in_object_list = [] nodes_to_examine = [root_object_node] # This would be clearer recursively # but I'm betting this implementation is a bit faster in Python. nodes_visited: Set[PerceptionGraphNode] = set() while nodes_to_examine: node_to_examine = nodes_to_examine.pop() if node_to_examine in nodes_visited: continue nodes_visited.add(node_to_examine) object_nodes_in_object_list.append(node_to_examine) for (next_node, _, edge_label) in scene_digraph.in_edges(node_to_examine, data="label"): if is_part_of_label(edge_label): nodes_to_examine.append(next_node) object_nodes_in_object = immutableset(object_nodes_in_object_list) # Now we know all object nodes for this candidate object. # Finally, we find the sub-graph to match against which could possibly correspond # to this candidate object # by performing a BFS over the graph # but *stopping whenever we encounter an object node # which is not part of this candidate object*. # This is a little more generous than we need to be, but it's simple. nodes_to_examine = [root_object_node] candidate_subgraph_nodes = [] nodes_visited.clear() while nodes_to_examine: node_to_examine = nodes_to_examine.pop() is_allowable_node = ( not isinstance(node_to_examine, (ObjectPerception, ObjectSemanticNode)) or node_to_examine in object_nodes_in_object) if node_to_examine not in nodes_visited and is_allowable_node: nodes_visited.add(node_to_examine) candidate_subgraph_nodes.append(node_to_examine) nodes_to_examine.extend( out_neighbor for (_, out_neighbor ) in scene_digraph.out_edges(node_to_examine) if not (isinstance(out_neighbor, tuple) and isinstance( out_neighbor[0], Region) and out_neighbor[0]. reference_object == GROUND_PERCEPTION)) nodes_to_examine.extend( in_neighbor for (in_neighbor, _) in scene_digraph.in_edges(node_to_examine) # Avoid in-edges from Regions as they can come from other objects regions (e.g ground). if not ( # isinstance(node_to_examine, GeonAxis) isinstance(in_neighbor, tuple) and isinstance(in_neighbor[0], Region))) candidate_objects.append( whole_scene_perception_graph.subgraph_by_nodes( immutableset(candidate_subgraph_nodes))) return candidate_objects
def replace_match_with_object_graph_node( matched_object_node: ObjectSemanticNode, current_perception: PerceptionGraph, pattern_match: PerceptionGraphPatternMatch, ) -> PerceptionGraphWithReplacedObjectResult: """ Internal function to replace the nodes of the perception matched by the object pattern with an `ObjectSemanticNode`. Any external relationships those nodes had is inherited by the `ObjectSemanticNode`. """ perception_digraph = current_perception.copy_as_digraph() perception_digraph.add_node(matched_object_node) matched_subgraph_nodes: ImmutableSet[PerceptionGraphNode] = immutableset( [ node for node in pattern_match.matched_sub_graph._graph.nodes # pylint:disable=protected-access if node in perception_digraph.nodes ], disable_order_check=True, ) # Multiple sub-objects of a matched object may link to the same property # (for example, to a color shared by all the parts). # In this case, we want the shared object node to link to this property only once. external_properties: Set[Union[OntologyNode, ObjectSemanticNode]] = set() duplicate_nodes_to_remove: List[PerceptionGraphNode] = [] for matched_subgraph_node in matched_subgraph_nodes: if isinstance(matched_subgraph_node, ObjectSemanticNode): raise RuntimeError( f"We do not currently allow object recognitions to themselves " f"operate over other object recognitions, but got match " f"{pattern_match.matched_sub_graph}") # A pattern might refer to shared parts of the world like the learner # or the ground, and we don't want the replacement root to inherit the # shared world item's relationships. if matched_subgraph_node in SHARED_WORLD_ITEMS: continue # If there is an edge from the matched sub-graph to a node outside it, # also add an edge from the object match node to that node. for matched_subgraph_node_successor in perception_digraph.successors( matched_subgraph_node): edge_label = _get_edge_label(perception_digraph, matched_subgraph_node, matched_subgraph_node_successor) # don't want to add edges which are internal to the matched sub-graph if matched_subgraph_node_successor not in matched_subgraph_nodes: if edge_equals_ignoring_temporal_scope(edge_label, HAS_PROPERTY_LABEL): # Prevent multiple `has-property` assertions to the same color node # On a recognized object # Also prevent size relations from being inherited on root object. if matched_subgraph_node_successor[ 0] in external_properties or matched_subgraph_node_successor[ 0] in { SMALLER_THAN, BIGGER_THAN, ABOUT_THE_SAME_SIZE_AS_LEARNER, }: if (perception_digraph.degree( matched_subgraph_node_successor) != 1): raise_graph_exception( f"Node {matched_subgraph_node_successor} " f"appears to be a duplicate property node, " f"but has degree != 1", current_perception, ) duplicate_nodes_to_remove.append( matched_subgraph_node_successor) continue else: external_properties.add( matched_subgraph_node_successor[0]) perception_digraph.add_edge(matched_object_node, matched_subgraph_node_successor, label=edge_label) # If there is an edge to the matched sub-graph from a node outside it, # also add an edge to the object match node from that node. for matched_subgraph_node_predecessor in perception_digraph.predecessors( matched_subgraph_node): edge_label = _get_edge_label( perception_digraph, matched_subgraph_node_predecessor, matched_subgraph_node, ) # don't want to add edges which are internal to the matched sub-graph if matched_subgraph_node_predecessor not in matched_subgraph_nodes: if edge_equals_ignoring_temporal_scope(edge_label, HAS_PROPERTY_LABEL): # Prevent multiple `has-property` assertions to the same color node # On a recognized object if isinstance(matched_subgraph_node_predecessor, ObjectSemanticNode): prop = matched_subgraph_node_predecessor else: prop = matched_subgraph_node_predecessor[0] if prop in external_properties: if (perception_digraph.degree( matched_subgraph_node_predecessor) != 1): raise_graph_exception( f"Node {matched_subgraph_node_predecessor} " f"appears to be a duplicate property node, " f"but has degree != 1", current_perception, ) duplicate_nodes_to_remove.append( matched_subgraph_node_predecessor) continue else: external_properties.add(prop) perception_digraph.add_edge( matched_subgraph_node_predecessor, matched_object_node, label=edge_label, ) # Remove all matched nodes which are not shared world items (e.g. gravity, the learner) to_remove = immutableset([ matched_node for matched_node in matched_subgraph_nodes if matched_node not in SHARED_WORLD_ITEMS ] + duplicate_nodes_to_remove) perception_digraph.remove_nodes_from(to_remove) # We want to re-add any properties linked directly to the root node of an object. # Example: water is a liquid # These may be relevant to learning verb semantics # (e.g. you can only drink a liquid) _add_external_properties_linked_to_root_object_perception( original_graph=current_perception.copy_as_digraph(), output_graph=perception_digraph, matched_nodes=matched_subgraph_nodes, matched_object_node=matched_object_node, ) # We also want to re-add any relationships directly linked to the root node of an object. # These may have been overwritten when iterating over the matched nodes. # Example: ball is behind table _add_external_relationships_linked_to_root_object_perception( original_graph=current_perception.copy_as_digraph(), output_graph=perception_digraph, matched_nodes=matched_subgraph_nodes, matched_object_node=matched_object_node, ) return PerceptionGraphWithReplacedObjectResult( PerceptionGraph(perception_digraph, dynamic=current_perception.dynamic), to_remove)