def pattern_remove_incomplete_region_or_spatial_path(
perception_graph: PerceptionGraphPattern
) -> PerceptionGraphPattern:
"""
Helper function to return a `PerceptionGraphPattern` verifying
that region and spatial path perceptions contain a reference object.
"""
graph = perception_graph.copy_as_digraph()
region_and_path_nodes: ImmutableSet[NodePredicate] = immutableset(
node
for node in graph.nodes
if isinstance(node, IsPathPredicate) or isinstance(node, RegionPredicate)
)
nodes_without_reference: List[NodePredicate] = []
for node in region_and_path_nodes:
has_reference_edge: bool = False
for successor in graph.successors(node):
predicate = graph.edges[node, successor]["predicate"]
if isinstance(predicate, RelationTypeIsPredicate):
if predicate.relation_type in [
REFERENCE_OBJECT_LABEL,
REFERENCE_OBJECT_DESTINATION_LABEL,
REFERENCE_OBJECT_SOURCE_LABEL,
]:
has_reference_edge = True
break
if not has_reference_edge:
nodes_without_reference.append(node)
logging.info(
f"Removing incomplete regions and paths. "
f"Removing nodes: {nodes_without_reference}"
)
graph.remove_nodes_from(nodes_without_reference)
def sort_by_num_nodes(g: DiGraph) -> int:
return len(g.nodes)
# We should maybe consider doing this a different way
# As this approach just brute force solves the problem rather than being methodical about it
if number_weakly_connected_components(graph) > 1:
components = [
component
for component in [
subgraph(graph, comp) for comp in weakly_connected_components(graph)
]
]
components.sort(key=sort_by_num_nodes, reverse=True)
computed_graph = subgraph(graph, components[0].nodes)
removed_nodes: List[NodePredicate] = []
for i in range(1, len(components)):
removed_nodes.extend(components[i].nodes)
logging.info(f"Cleanup disconnected elements. Removing: {removed_nodes}")
else:
computed_graph = graph
return PerceptionGraphPattern(computed_graph, dynamic=perception_graph.dynamic)
def test_successfully_extending_partial_match():
"""
Tests whether we can match a perception pattern against a perception graph
when initializing the search from a partial match.
"""
target_object = BOX
# Create train and test templates for the target objects
train_obj_object = object_variable("obj-with-color", target_object)
obj_template = Phase1SituationTemplate(
"colored-obj-object", salient_object_variables=[train_obj_object])
template = all_possible(obj_template,
chooser=PHASE1_CHOOSER_FACTORY(),
ontology=GAILA_PHASE_1_ONTOLOGY)
train_curriculum = phase1_instances("all obj situations",
situations=template)
perceptual_representation = only(train_curriculum.instances())[2]
# Original perception graph
perception = PerceptionGraph.from_frame(
perceptual_representation.frames[0])
# Create a perception pattern for the whole thing
# and also a perception pattern for a subset of the whole pattern
whole_perception_pattern = PerceptionGraphPattern.from_graph(
perception).perception_graph_pattern
partial_digraph = whole_perception_pattern.copy_as_digraph()
partial_digraph.remove_nodes_from([
node for node in partial_digraph.nodes
if isinstance(node, IsColorNodePredicate)
])
partial_perception_pattern = PerceptionGraphPattern(partial_digraph)
# get our initial match by matching the partial pattern
matcher = partial_perception_pattern.matcher(
perception, match_mode=MatchMode.NON_OBJECT)
partial_match: PerceptionGraphPatternMatch = first(
matcher.matches(use_lookahead_pruning=True))
partial_mapping = partial_match.pattern_node_to_matched_graph_node
# Try to extend the partial mapping, to create a complete mapping
matcher_2 = whole_perception_pattern.matcher(
perception, match_mode=MatchMode.NON_OBJECT)
complete_match: PerceptionGraphPatternMatch = first(
matcher_2.matches(initial_partial_match=partial_mapping,
use_lookahead_pruning=True),
None,
)
complete_mapping = complete_match.pattern_node_to_matched_graph_node
assert len(complete_mapping) == len(perception.copy_as_digraph().nodes)
assert len(complete_mapping) == len(
whole_perception_pattern.copy_as_digraph().nodes)
def get_largest_matching_pattern(
pattern: PerceptionGraphPattern,
graph: PerceptionGraph,
*,
debug_callback: Optional[DebugCallableType] = None,
graph_logger: Optional[GraphLogger] = None,
ontology: Ontology,
match_ratio: Optional[float] = None,
match_mode: MatchMode,
trim_after_match: Optional[Callable[[PerceptionGraphPattern],
PerceptionGraphPattern]] = None,
allowed_matches: ImmutableSetMultiDict[Any, Any] = immutablesetmultidict(),
) -> Optional[PerceptionGraphPattern]:
""" Helper function to return the largest matching `PerceptionGraphPattern`
for learner from a perception pattern and graph pair."""
matching = pattern.matcher(
graph,
debug_callback=debug_callback,
match_mode=match_mode,
allowed_matches=allowed_matches,
)
return matching.relax_pattern_until_it_matches(
graph_logger=graph_logger,
ontology=ontology,
min_ratio=match_ratio,
trim_after_match=trim_after_match,
)
def for_ontology_types(
ontology_types: Iterable[OntologyNode],
determiners: Iterable[str],
ontology: Ontology,
language_mode: LanguageMode,
*,
perception_generator:
HighLevelSemanticsSituationToDevelopmentalPrimitivePerceptionGenerator,
) -> "ObjectRecognizer":
ontology_types_to_concepts = {
obj_type: ObjectConcept(obj_type.handle)
for obj_type in ontology_types
}
return ObjectRecognizer(
concepts_to_static_patterns=_sort_mapping_by_pattern_complexity(
immutabledict((
concept,
PerceptionGraphPattern.from_ontology_node(
obj_type,
ontology,
perception_generator=perception_generator),
) for (obj_type,
concept) in ontology_types_to_concepts.items())),
determiners=determiners,
concepts_to_names={
concept: obj_type.handle
for obj_type, concept in ontology_types_to_concepts.items()
},
language_mode=language_mode,
)
def from_graph(
perception_graph: PerceptionGraph,
template_variable_to_matched_object_node: Mapping[
SyntaxSemanticsVariable, ObjectSemanticNode
],
) -> "PerceptionGraphTemplate":
# It is possible the perception graph has additional recognized objects
# which are not aligned to surface template slots.
# We assume these are not arguments of the verb and remove them from the perception
# before creating a pattern.
pattern_from_graph = PerceptionGraphPattern.from_graph(perception_graph)
pattern_graph = pattern_from_graph.perception_graph_pattern
matched_object_to_matched_predicate = (
pattern_from_graph.perception_graph_node_to_pattern_node
)
template_variable_to_pattern_node: List[Any] = []
for (
template_variable,
object_node,
) in template_variable_to_matched_object_node.items():
if object_node in matched_object_to_matched_predicate:
template_variable_to_pattern_node.append(
(template_variable, matched_object_to_matched_predicate[object_node])
)
return PerceptionGraphTemplate(
graph_pattern=pattern_graph,
template_variable_to_pattern_node=template_variable_to_pattern_node,
)
def _enrich_post_process(
perception_graph_after_matching: PerceptionGraph,
immutable_new_nodes: AbstractSet[SemanticNode],
) -> Tuple[PerceptionGraph, AbstractSet[SemanticNode]]:
new_nodes = []
perception_graph_after_processing = perception_graph_after_matching
for candiate_object_graph in extract_candidate_objects(
perception_graph_after_matching,
sort_by_increasing_size=False):
fake_pattern_graph = PerceptionGraphPattern.from_graph(
candiate_object_graph)
fake_object_semantic_node = ObjectSemanticNode(
concept=FunctionalObjectConcept("unknown_object"))
perception_graph_after_processing = replace_match_with_object_graph_node(
matched_object_node=fake_object_semantic_node,
current_perception=perception_graph_after_processing,
pattern_match=PerceptionGraphPatternMatch(
matched_pattern=fake_pattern_graph.
perception_graph_pattern,
graph_matched_against=perception_graph_after_processing,
matched_sub_graph=candiate_object_graph,
pattern_node_to_matched_graph_node=fake_pattern_graph.
perception_graph_node_to_pattern_node,
),
).perception_graph_after_replacement
new_nodes.append(fake_object_semantic_node)
return (
perception_graph_after_processing,
immutableset(chain(immutable_new_nodes, new_nodes)),
)
def test_matching_static_vs_dynamic_graphs():
target_object = BOX
train_obj_object = object_variable("obj-with-color", target_object)
obj_template = Phase1SituationTemplate(
"colored-obj-object", salient_object_variables=[train_obj_object])
template = all_possible(obj_template,
chooser=PHASE1_CHOOSER_FACTORY(),
ontology=GAILA_PHASE_1_ONTOLOGY)
train_curriculum = phase1_instances("all obj situations",
situations=template)
perceptual_representation = only(train_curriculum.instances())[2]
perception_graph = graph_without_learner(
PerceptionGraph.from_frame(perceptual_representation.frames[0]))
temporal_perception_graph = perception_graph.copy_with_temporal_scopes(
temporal_scopes=[TemporalScope.AFTER])
perception_pattern = PerceptionGraphPattern.from_graph(
perception_graph).perception_graph_pattern
temporal_perception_pattern = perception_pattern.copy_with_temporal_scopes(
required_temporal_scopes=[TemporalScope.AFTER])
# Test runtime error for matching static pattern against dynamic graph and vice versa
with pytest.raises(RuntimeError):
perception_pattern.matcher(temporal_perception_graph,
match_mode=MatchMode.NON_OBJECT)
with pytest.raises(RuntimeError):
temporal_perception_pattern.matcher(perception_graph,
match_mode=MatchMode.NON_OBJECT)
def _hypothesis_from_perception(
self, preprocessed_input: LanguageAlignedPerception
) -> PerceptionGraphTemplate:
new_hypothesis = PerceptionGraphPattern.from_graph(
preprocessed_input.perception_graph).perception_graph_pattern
return PerceptionGraphTemplate(
graph_pattern=new_hypothesis,
template_variable_to_pattern_node=immutabledict(),
)
def candidate_object_hypotheses(
language_perception_semantic_alignment: LanguagePerceptionSemanticAlignment
) -> Sequence[PerceptionGraphTemplate]:
"""
Given a learning input, returns all possible meaning hypotheses.
"""
return [
PerceptionGraphTemplate(
graph_pattern=PerceptionGraphPattern.from_graph(
object_
).perception_graph_pattern
)
for object_ in get_objects_from_perception(
language_perception_semantic_alignment.perception_semantic_alignment.perception_graph
)
]
def _hypotheses_from_perception(
self,
learning_state: LanguagePerceptionSemanticAlignment,
bound_surface_template: SurfaceTemplateBoundToSemanticNodes,
) -> AbstractSet[PerceptionGraphTemplate]:
if bound_surface_template.slot_to_semantic_node:
raise RuntimeError(
"Object learner should not have slot to semantic node alignments!"
)
return immutableset(
PerceptionGraphTemplate(
graph_pattern=PerceptionGraphPattern.from_graph(
candidate_object).perception_graph_pattern,
template_variable_to_pattern_node=immutabledict(),
) for candidate_object in extract_candidate_objects(
learning_state.perception_semantic_alignment.perception_graph))
def _enrich_post_process(
self,
perception_graph_after_matching: PerceptionGraph,
immutable_new_nodes: AbstractSet[SemanticNode],
) -> Tuple[PerceptionGraph, AbstractSet[SemanticNode]]:
object_root_nodes = immutableset( # pylint:disable=protected-access
node for node in perception_graph_after_matching._graph.nodes # pylint:disable=protected-access
if isinstance(node, ObjectPerception))
new_nodes = []
perception_graph_after_processing = perception_graph_after_matching
for object_root_node in object_root_nodes:
fake_subgraph = subgraph( # pylint:disable=protected-access
perception_graph_after_matching._graph, # pylint:disable=protected-access
[object_root_node],
)
fake_perception_graph = PerceptionGraph(
graph=fake_subgraph,
dynamic=perception_graph_after_matching.dynamic)
fake_pattern_graph = PerceptionGraphPattern.from_graph(
fake_perception_graph)
fake_object_semantic_node = ObjectSemanticNode(
concept=FunctionalObjectConcept("unknown_object"))
# perception_graph_after_processing = replace_match_root_with_object_semantic_node(
# object_semantic_node=fake_object_semantic_node,
perception_graph_after_processing = replace_match_with_object_graph_node(
matched_object_node=fake_object_semantic_node,
current_perception=perception_graph_after_processing,
pattern_match=PerceptionGraphPatternMatch(
matched_pattern=fake_pattern_graph.
perception_graph_pattern,
graph_matched_against=perception_graph_after_matching,
matched_sub_graph=fake_perception_graph,
pattern_node_to_matched_graph_node=fake_pattern_graph.
perception_graph_node_to_pattern_node,
),
).perception_graph_after_replacement
new_nodes.append(fake_object_semantic_node)
return (
perception_graph_after_processing,
immutableset(chain(immutable_new_nodes, new_nodes)),
)
def test_syntactically_infeasible_partial_match():
"""
Tests whether syntactic feasibility works as intended
"""
# We use a situation to generate the perceptual representation
# for a box with color.
target_object = BOX
train_obj_object = object_variable("obj-with-color", target_object)
obj_template = Phase1SituationTemplate(
"colored-obj-object", salient_object_variables=[train_obj_object])
template = all_possible(obj_template,
chooser=PHASE1_CHOOSER_FACTORY(),
ontology=GAILA_PHASE_1_ONTOLOGY)
train_curriculum = phase1_instances("all obj situations",
situations=template)
perceptual_representation = only(train_curriculum.instances())[2]
# Original perception graph
perception = graph_without_learner(
PerceptionGraph.from_frame(perceptual_representation.frames[0]))
# Create an altered perception graph we remove the color node
altered_perception_digraph = perception.copy_as_digraph()
nodes = []
for node in perception.copy_as_digraph().nodes:
# If we find a color node, we add an extra edge to it
if isinstance(node, tuple) and isinstance(node[0], RgbColorPerception):
nodes.append(node)
# change edge information
for node in nodes:
random_node = r.choice(list(altered_perception_digraph.nodes))
altered_perception_digraph.add_edge(node, random_node, label=PART_OF)
random_node_2 = r.choice(list(altered_perception_digraph.nodes))
altered_perception_digraph.add_edge(random_node_2, node, label=PART_OF)
altered_perception_perception_graph = PerceptionGraph(
altered_perception_digraph)
altered_perception_pattern = PerceptionGraphPattern.from_graph(
altered_perception_perception_graph).perception_graph_pattern
# Start the matching process, get a partial match
first_matcher = altered_perception_pattern.matcher(
altered_perception_perception_graph, match_mode=MatchMode.OBJECT)
partial_match: PerceptionGraphPatternMatch = first(
first_matcher.matches(use_lookahead_pruning=True), None)
partial_mapping = partial_match.pattern_node_to_matched_graph_node
# Try to extend the partial mapping, we expect a semantic infeasibility runtime error
second_matcher = altered_perception_pattern.matcher(
perception, match_mode=MatchMode.OBJECT)
# The partial mapping (obtained from first matcher with original perception graph)
# syntactically doesn't match the one in the altered version (second matcher with altered graph)
with pytest.raises(RuntimeError):
first(
second_matcher.matches(initial_partial_match=partial_mapping,
use_lookahead_pruning=True),
None,
)
def _hypothesis_from_perception(
self, perception: PerceptionGraph) -> PerceptionGraphTemplate:
return PerceptionGraphTemplate(
graph_pattern=PerceptionGraphPattern.from_graph(
perception).perception_graph_pattern)
def match_objects(
self,
perception_semantic_alignment: PerceptionSemanticAlignment,
*,
post_process: Callable[[PerceptionGraph, AbstractSet[SemanticNode]],
Tuple[PerceptionGraph,
AbstractSet[SemanticNode]],
] = default_post_process_enrichment,
) -> Tuple[PerceptionSemanticAlignment, Mapping[Tuple[str, ...],
ObjectSemanticNode]]:
r"""
Recognize known objects in a `PerceptionGraph`.
The matched portion of the graph will be replaced with an `ObjectSemanticNode`\ s
which will inherit all relationships of any nodes internal to the matched portion
with any external nodes.
This is useful as a pre-processing step
before prepositional and verbal learning experiments.
"""
# pylint: disable=global-statement,invalid-name
global cumulative_millis_in_successful_matches_ms
global cumulative_millis_in_failed_matches_ms
object_nodes: List[Tuple[Tuple[str, ...], ObjectSemanticNode]] = []
perception_graph = perception_semantic_alignment.perception_graph
is_dynamic = perception_semantic_alignment.perception_graph.dynamic
if is_dynamic:
concepts_to_patterns = self._concepts_to_dynamic_patterns
else:
concepts_to_patterns = self._concepts_to_static_patterns
# We special case handling the ground perception
# Because we don't want to remove it from the graph, we just want to use it's
# Object node as a recognized object. The situation "a box on the ground"
# Prompted the need to recognize the ground
graph_to_return = perception_graph
for node in graph_to_return._graph.nodes: # pylint:disable=protected-access
if node == GROUND_PERCEPTION:
matched_object_node = ObjectSemanticNode(GROUND_OBJECT_CONCEPT)
if LanguageMode.ENGLISH == self._language_mode:
object_nodes.append(
((f"{GROUND_OBJECT_CONCEPT.debug_string}", ),
matched_object_node))
elif LanguageMode.CHINESE == self._language_mode:
object_nodes.append((("di4 myan4", ), matched_object_node))
else:
raise RuntimeError("Invalid language_generator")
# We construct a fake match which is only the ground perception node
subgraph_of_root = subgraph(perception_graph.copy_as_digraph(),
[node])
pattern_match = PerceptionGraphPatternMatch(
matched_pattern=PerceptionGraphPattern(
graph=subgraph_of_root,
dynamic=perception_graph.dynamic),
graph_matched_against=perception_graph,
matched_sub_graph=PerceptionGraph(
graph=subgraph_of_root,
dynamic=perception_graph.dynamic),
pattern_node_to_matched_graph_node=immutabledict(),
)
graph_to_return = replace_match_with_object_graph_node(
matched_object_node, graph_to_return, pattern_match)
candidate_object_subgraphs = extract_candidate_objects(
perception_graph)
for candidate_object_graph in candidate_object_subgraphs:
num_object_nodes = candidate_object_graph.count_nodes_matching(
lambda node: isinstance(node, ObjectPerception))
for (concept, pattern) in concepts_to_patterns.items():
# As an optimization, we count how many sub-object nodes
# are in the graph and the pattern.
# If they aren't the same, the match is impossible
# and we can bail out early.
if num_object_nodes != self._concept_to_num_subobjects[concept]:
continue
with Timer(factor=1000) as t:
matcher = pattern.matcher(candidate_object_graph,
match_mode=MatchMode.OBJECT)
pattern_match = first(
matcher.matches(use_lookahead_pruning=True), None)
if pattern_match:
cumulative_millis_in_successful_matches_ms += t.elapsed
matched_object_node = ObjectSemanticNode(concept)
# We wrap the concept in a tuple because it could in theory be multiple
# tokens,
# even though currently it never is.
if self._language_mode == LanguageMode.ENGLISH:
object_nodes.append(
((concept.debug_string, ), matched_object_node))
elif self._language_mode == LanguageMode.CHINESE:
if concept.debug_string == "me":
object_nodes.append(
(("wo3", ), matched_object_node))
elif concept.debug_string == "you":
object_nodes.append(
(("ni3", ), matched_object_node))
mappings = (
GAILA_PHASE_1_CHINESE_LEXICON.
_ontology_node_to_word # pylint:disable=protected-access
)
for k, v in mappings.items():
if k.handle == concept.debug_string:
debug_string = str(v.base_form)
object_nodes.append(
((debug_string, ), matched_object_node))
graph_to_return = replace_match_with_object_graph_node(
matched_object_node, graph_to_return, pattern_match)
# We match each candidate objects against only one object type.
# See https://github.com/isi-vista/adam/issues/627
break
else:
cumulative_millis_in_failed_matches_ms += t.elapsed
if object_nodes:
logging.info(
"Object recognizer recognized: %s",
[concept for (concept, _) in object_nodes],
)
logging.info(
"object matching: ms in success: %s, ms in failed: %s",
cumulative_millis_in_successful_matches_ms,
cumulative_millis_in_failed_matches_ms,
)
semantic_object_nodes = immutableset(node
for (_, node) in object_nodes)
post_process_graph, post_process_nodes = post_process(
graph_to_return, semantic_object_nodes)
return (
perception_semantic_alignment.
copy_with_updated_graph_and_added_nodes(
new_graph=post_process_graph, new_nodes=post_process_nodes),
immutabledict(object_nodes),
)
def do_object_on_table_test(
object_type_to_match: OntologyNode,
object_schema: ObjectStructuralSchema,
negative_object_ontology_node: OntologyNode,
):
"""
Tests the `PerceptionGraphMatcher` can match simple objects.
"""
# we create four situations:
# a object_to_match above or under a table with color red or blue
color = color_variable("color")
object_to_match = object_variable(
debug_handle=object_type_to_match.handle,
root_node=object_type_to_match,
added_properties=[color],
)
table = standard_object("table_0", TABLE)
object_on_table_template = Phase1SituationTemplate(
"object_to_match-on-table",
salient_object_variables=[object_to_match, table],
asserted_always_relations=[
bigger_than(table, object_to_match),
on(object_to_match, table),
],
)
object_under_table_template = Phase1SituationTemplate(
"object_to_match-under-table",
salient_object_variables=[object_to_match, table],
asserted_always_relations=[
bigger_than(table, object_to_match),
above(table, object_to_match),
],
)
# We test that a perceptual pattern for "object_to_match" matches in all four cases.
object_to_match_pattern = PerceptionGraphPattern.from_schema(
object_schema, perception_generator=GAILA_PHASE_1_PERCEPTION_GENERATOR)
situations_with_object_to_match = chain(
all_possible_test(object_on_table_template),
all_possible_test(object_under_table_template),
)
for (_,
situation_with_object) in enumerate(situations_with_object_to_match):
perception = GAILA_PHASE_1_PERCEPTION_GENERATOR.generate_perception(
situation_with_object, chooser=RandomChooser.for_seed(0))
perception_graph = PerceptionGraph.from_frame(perception.frames[0])
# perception_graph.render_to_file(f"object_to_match {idx}", out_dir / f"object_to_match
# -{idx}.pdf")
# object_to_match_pattern.render_to_file(f"object_to_match pattern", out_dir /
# "object_to_match_pattern.pdf")
matcher = object_to_match_pattern.matcher(perception_graph,
match_mode=MatchMode.OBJECT)
# debug_matching = matcher.debug_matching(
# use_lookahead_pruning=False, render_match_to=Path("/Users/gabbard/tmp")
# )
result = any(matcher.matches(use_lookahead_pruning=False))
if not result:
return False
# Now let's create the same situations, but substitute a negative_object for a object_to_match.
negative_object = object_variable(
debug_handle=negative_object_ontology_node.handle,
root_node=negative_object_ontology_node,
added_properties=[color],
)
negative_object_on_table_template = Phase1SituationTemplate(
"negative_object-on-table",
salient_object_variables=[negative_object, table],
asserted_always_relations=[
bigger_than(table, negative_object),
on(negative_object, table),
],
)
negative_object_under_table_template = Phase1SituationTemplate(
"negative_object-under-table",
salient_object_variables=[negative_object, table],
asserted_always_relations=[
bigger_than(table, negative_object),
above(table, negative_object),
],
)
situations_with_negative_object = chain(
all_possible_test(negative_object_on_table_template),
all_possible_test(negative_object_under_table_template),
)
# The pattern should now fail to match.
for situation_with_negative_object in situations_with_negative_object:
perception = GAILA_PHASE_1_PERCEPTION_GENERATOR.generate_perception(
situation_with_negative_object, chooser=RandomChooser.for_seed(0))
perception_graph = PerceptionGraph.from_frame(perception.frames[0])
if any(
object_to_match_pattern.matcher(
perception_graph, match_mode=MatchMode.OBJECT).matches(
use_lookahead_pruning=True)):
return False
return True
ObjectSemanticNode,
GROUND_OBJECT_CONCEPT,
SemanticNode,
)
from adam.utils.networkx_utils import subgraph
from attr import attrib, attrs
from attr.validators import deep_iterable, deep_mapping, instance_of
from immutablecollections import ImmutableDict, ImmutableSet, immutabledict, immutableset
from immutablecollections.converter_utils import _to_immutabledict, _to_immutableset
from vistautils.span import Span
_LIST_OF_PERCEIVED_PATTERNS = immutableset(
(
node.handle,
PerceptionGraphPattern.from_schema(
first(GAILA_PHASE_1_ONTOLOGY.structural_schemata(node)),
perception_generator=GAILA_PHASE_1_PERCEPTION_GENERATOR,
),
) for node in PHASE_1_CURRICULUM_OBJECTS
if node in GAILA_PHASE_1_ONTOLOGY._structural_schemata.keys() # pylint:disable=protected-access
)
@attrs(frozen=True, slots=True, auto_attribs=True)
class PerceptionGraphFromObjectRecognizer:
"""
See `ObjectRecognizer.match_objects`
"""
perception_graph: PerceptionGraph
description_to_matched_object_node: ImmutableDict[Tuple[
str, ...], ObjectSemanticNode] = attrib(converter=_to_immutabledict)
def test_last_failed_pattern_node():
"""
Tests whether `MatchFailure` can find the correct node.
"""
target_object = BOX
# Create train and test templates for the target objects
train_obj_object = object_variable("obj-with-color", target_object)
obj_template = Phase1SituationTemplate(
"colored-obj-object", salient_object_variables=[train_obj_object])
template = all_possible(obj_template,
chooser=PHASE1_CHOOSER_FACTORY(),
ontology=GAILA_PHASE_1_ONTOLOGY)
train_curriculum = phase1_instances("all obj situations",
situations=template)
for (_, _, perceptual_representation) in train_curriculum.instances():
# Original perception graph
perception = graph_without_learner(
PerceptionGraph.from_frame(perceptual_representation.frames[0]))
# Original perception pattern
whole_perception_pattern = PerceptionGraphPattern.from_graph(
perception).perception_graph_pattern
# Create an altered perception graph we replace the color node
altered_perception_digraph = perception.copy_as_digraph()
nodes_to_remove = []
edges = []
different_nodes = []
for node in perception.copy_as_digraph().nodes:
# If we find a color node, we make it black
if isinstance(node, tuple) and isinstance(node[0],
RgbColorPerception):
new_node = (RgbColorPerception(0, 0, 0), 42)
# Get edge information
for edge in perception.copy_as_digraph().edges(data=True):
if edge[0] == node:
edges.append((new_node, edge[1], edge[2]))
if edge[1] == node:
edges.append((edge[0], new_node, edge[2]))
nodes_to_remove.append(node)
different_nodes.append(new_node)
# add new nodes
for node in different_nodes:
altered_perception_digraph.add_node(node)
# add edge information
for edge in edges:
altered_perception_digraph.add_edge(edge[0], edge[1])
for k, v in edge[2].items():
altered_perception_digraph[edge[0]][edge[1]][k] = v
# remove original node
altered_perception_digraph.remove_nodes_from(nodes_to_remove)
# Start the matching process
matcher = whole_perception_pattern.matcher(
PerceptionGraph(altered_perception_digraph),
match_mode=MatchMode.NON_OBJECT)
match_or_failure = matcher.first_match_or_failure_info()
assert isinstance(match_or_failure, PatternMatching.MatchFailure)
assert isinstance(match_or_failure.last_failed_pattern_node,
IsColorNodePredicate)
def test_copy_with_temporal_scope_pattern_content():
"""
Tests whether copy_with_temporal_scope converts patterns to be dynamic as intended
"""
# We use a situation to generate the perceptual representation
# for a box with color.
target_object = BOX
train_obj_object = object_variable("obj-with-color", target_object)
obj_template = Phase1SituationTemplate(
"colored-obj-object", salient_object_variables=[train_obj_object])
template = all_possible(obj_template,
chooser=PHASE1_CHOOSER_FACTORY(),
ontology=GAILA_PHASE_1_ONTOLOGY)
train_curriculum = phase1_instances("all obj situations",
situations=template)
perceptual_representation = only(train_curriculum.instances())[2]
perception_graph = graph_without_learner(
PerceptionGraph.from_frame(perceptual_representation.frames[0]))
perception_pattern = PerceptionGraphPattern.from_graph(
perception_graph).perception_graph_pattern
temporal_perception_graph = perception_graph.copy_with_temporal_scopes(
temporal_scopes=[TemporalScope.AFTER])
temporal_perception_pattern = perception_pattern.copy_with_temporal_scopes(
required_temporal_scopes=TemporalScope.AFTER)
# Exception while applying to dynamic pattern
with pytest.raises(RuntimeError):
temporal_perception_pattern.copy_with_temporal_scopes(
required_temporal_scopes=TemporalScope.AFTER)
for (source, target) in perception_pattern.copy_as_digraph().edges():
assert not isinstance(
perception_pattern.copy_as_digraph()[source][target]["predicate"],
HoldsAtTemporalScopePredicate,
)
for (source,
target) in temporal_perception_pattern.copy_as_digraph().edges():
# Check type, and then the content
predicate = temporal_perception_pattern.copy_as_digraph(
)[source][target]["predicate"]
# Test HoldsAtTemporalScope dot label, matches predicate
assert isinstance(predicate.dot_label(), str)
assert predicate.matches_predicate(
HoldsAtTemporalScopePredicate(predicate.wrapped_edge_predicate,
predicate.temporal_scopes))
assert not predicate.matches_predicate(
HoldsAtTemporalScopePredicate(predicate.wrapped_edge_predicate,
[TemporalScope.BEFORE]))
assert isinstance(predicate, HoldsAtTemporalScopePredicate)
assert (predicate.wrapped_edge_predicate == perception_pattern.
copy_as_digraph()[source][target]["predicate"])
assert len(predicate.temporal_scopes) == 1
assert only(predicate.temporal_scopes) == TemporalScope.AFTER
# Test normal matching behavior
temporal_matcher = temporal_perception_pattern.matcher(
temporal_perception_graph, match_mode=MatchMode.NON_OBJECT)
first(temporal_matcher.matches(use_lookahead_pruning=True))
# Test HoldsAtTemporalScopePredicate
for (source, target) in perception_graph.copy_as_digraph().edges():
label = "test edge label"
edge_predicate = AnyEdgePredicate()
temporal_predicate = HoldsAtTemporalScopePredicate(
edge_predicate, [TemporalScope.AFTER])
temporal_edge_label = TemporallyScopedEdgeLabel(
label, [TemporalScope.AFTER])
assert temporal_predicate(source, temporal_edge_label, target)
# Non temporal edge exception
with pytest.raises(RuntimeError):
temporal_predicate(source, label, target)
def test_semantically_infeasible_partial_match():
"""
Tests whether semantic feasibility works as intended
"""
target_object = BOX
# Create train and test templates for the target objects
train_obj_object = object_variable("obj-with-color", target_object)
obj_template = Phase1SituationTemplate(
"colored-obj-object", salient_object_variables=[train_obj_object])
template = all_possible(obj_template,
chooser=PHASE1_CHOOSER_FACTORY(),
ontology=GAILA_PHASE_1_ONTOLOGY)
train_curriculum = phase1_instances("all obj situations",
situations=template)
perceptual_representation = only(train_curriculum.instances())[2]
# Original perception graph
perception = graph_without_learner(
PerceptionGraph.from_frame(perceptual_representation.frames[0]))
whole_perception_pattern = PerceptionGraphPattern.from_graph(
perception).perception_graph_pattern
# Create an altered perception graph we remove the color node
altered_perception_digraph = perception.copy_as_digraph()
nodes_to_remove = []
edges = []
different_nodes = []
for node in perception.copy_as_digraph().nodes:
# If we find a color node, we make it black
if isinstance(node, tuple) and isinstance(node[0], RgbColorPerception):
new_node = (RgbColorPerception(0, 0, 0), node[1])
# Get edge information
for edge in perception.copy_as_digraph().edges(data=True):
if edge[0] == node:
edges.append((new_node, edge[1], edge[2]))
if edge[1] == node:
edges.append((edge[0], new_node, edge[2]))
nodes_to_remove.append(node)
different_nodes.append(new_node)
# remove original node
altered_perception_digraph.remove_nodes_from(nodes_to_remove)
# add new nodes
for node in different_nodes:
altered_perception_digraph.add_node(node)
# add edge information
for edge in edges:
altered_perception_digraph.add_edge(edge[0], edge[1])
for k, v in edge[2].items():
altered_perception_digraph[edge[0]][edge[1]][k] = v
altered_perception_pattern = PerceptionGraphPattern.from_graph(
PerceptionGraph(altered_perception_digraph)).perception_graph_pattern
partial_digraph = altered_perception_pattern.copy_as_digraph()
partial_digraph.remove_nodes_from([
node for node in partial_digraph.nodes
if isinstance(node, IsColorNodePredicate)
])
# Start the matching process, get a partial match
matcher = whole_perception_pattern.matcher(perception,
match_mode=MatchMode.OBJECT)
partial_match: PerceptionGraphPatternMatch = first(
matcher.matches(use_lookahead_pruning=True))
partial_mapping = partial_match.pattern_node_to_matched_graph_node
# Try to extend the partial mapping, we expect a semantic infeasibility runtime error
second_matcher = whole_perception_pattern.matcher(
PerceptionGraph(altered_perception_digraph),
match_mode=MatchMode.OBJECT)
# The partial mapping (obtained from first matcher with original perception graph)
# semantically doesn't match the one in the altered version (second matcher with altered graph)
with pytest.raises(RuntimeError):
first(
second_matcher.matches(initial_partial_match=partial_mapping,
use_lookahead_pruning=True),
None,
)
def test_allowed_matches_with_bad_partial_match():
"""
Tests whether PatternMarching's allowed_matches functionality works as intended when a bad
partial match is specified.
"""
target_object = BOX
train_obj_object = object_variable("obj-with-color", target_object)
obj_template = Phase1SituationTemplate(
"colored-obj-object", salient_object_variables=[train_obj_object])
template = all_possible(obj_template,
chooser=PHASE1_CHOOSER_FACTORY(),
ontology=GAILA_PHASE_1_ONTOLOGY)
train_curriculum = phase1_instances("all obj situations",
situations=template)
perceptual_representation = only(train_curriculum.instances())[2]
perception = graph_without_learner(
PerceptionGraph.from_frame(perceptual_representation.frames[0]))
pattern1: PerceptionGraphPattern = PerceptionGraphPattern.from_graph(
perception.subgraph_by_nodes({
cast(PerceptionGraphNode, node)
for node in perception._graph.nodes # pylint: disable=protected-access
if getattr(node, "debug_handle", None) == "box_0"
})).perception_graph_pattern
pattern2: PerceptionGraphPattern = PerceptionGraphPattern.from_graph(
perception.subgraph_by_nodes({
cast(PerceptionGraphNode, node)
for node in perception._graph.nodes # pylint: disable=protected-access
if getattr(node, "debug_handle", None) in {"box_0", "the ground"}
})).perception_graph_pattern
pattern1_box: AnyObjectPerception = cast(
AnyObjectPerception,
only(node for node in pattern1._graph # pylint: disable=protected-access
if getattr(node, "debug_handle", None) == "box_0"),
)
pattern2_box: AnyObjectPerception = cast(
AnyObjectPerception,
only(node for node in pattern2._graph # pylint: disable=protected-access
if getattr(node, "debug_handle", None) == "box_0"),
)
pattern2_ground: AnyObjectPerception = cast(
AnyObjectPerception,
only(node for node in pattern2._graph # pylint: disable=protected-access
if getattr(node, "debug_handle", None) == "the ground"),
)
matcher = PatternMatching(
pattern=pattern1,
graph_to_match_against=pattern2,
matching_pattern_against_pattern=True,
match_mode=MatchMode.OBJECT,
allowed_matches=immutablesetmultidict([(pattern1_box, pattern2_box)]),
)
with pytest.raises(RuntimeError):
first(
matcher.matches(
initial_partial_match={pattern1_box: pattern2_ground},
use_lookahead_pruning=True,
),
None,
)