def put(self, key: str, value: V) -> None:
check_state(self._zip_file, "Must use zip key-value sink as a context manager")
check_not_none(key)
check_not_none(value)
if key in self._keys:
raise ValueError(
"Zip-backed key-value sinks do not support duplicate puts on the "
"same key"
)
self._keys.add(key)
filename = self._filename_function(key)
check_arg(isinstance(value, str) or isinstance(value, bytes))
self._zip_file.writestr(filename, value) # type: ignore
def _internal_get(
self, key: str, *, has_default_val: bool, default_val: Optional[V]
) -> Optional[V]:
check_state(self._zip_file, "Must use zip key-value source as a context manager")
check_not_none(key)
filename = self._filename_function(key)
try:
# safe by check_state above
zip_bytes = self._zip_file.read(filename) # type: ignore
except KeyError as e:
if has_default_val:
return default_val
raise KeyError(
f"Key '{key}' not found in zip key-value source backed by " f"{self.path}"
) from e
return self._process_bytes(zip_bytes)
def items(
self, key_filter: Callable[[str], bool] = lambda x: True
) -> Iterator[Tuple[str, bytes]]:
check_state(
self.inp,
"Need to enter TarGZipBytesLinearKeyValueSource as context "
"manager before using it.",
)
def generator_function() -> Iterator[Tuple[str, bytes]]:
# safe by check_state above
for member in self.inp: # type: ignore
if member.isfile() and self.name_filter(member.name):
key = self.key_function(member.name)
if key and key_filter(key):
data = self.inp.extractfile(member) # type: ignore
if data:
with data:
yield (key, data.read())
else:
raise IOError(f"Cannot read member {member} of {self}")
return generator_function()
def _enrich_common(
self, perception_semantic_alignment: PerceptionSemanticAlignment
) -> Tuple[PerceptionSemanticAlignment, AbstractSet[SemanticNode]]:
"""
Shared code between `enrich_during_learning` and `enrich_during_description`.
"""
preprocessing_result = self._preprocess_scene(perception_semantic_alignment)
preprocessed_perception_graph = preprocessing_result.perception_graph
# This accumulates our output.
match_to_score: List[Tuple[SemanticNode, float]] = []
# In the case of objects only, we alter the perception graph once they
# are recognized by replacing the matched portion of the graph with the
# ObjectSemanticNodes. We gather them as we match and do the replacement below.
matched_objects: List[Tuple[SemanticNode, PerceptionGraphPatternMatch]] = []
# We pull this out into a function because we do matching in two passes:
# first against templates whose meanings we are sure of (=have lexicalized)
# and then, if no match has been found, against those we are still learning.
def match_template(
*, concept: Concept, pattern: PerceptionGraphTemplate, score: float
) -> None:
# try to see if (our model of) its semantics is present in the situation.
matcher = pattern.graph_pattern.matcher(
preprocessed_perception_graph,
match_mode=MatchMode.NON_OBJECT,
# debug_callback=self._debug_callback,
)
for match in matcher.matches(use_lookahead_pruning=True):
# if there is a match, use that match to describe the situation.
semantic_node_for_match = pattern_match_to_semantic_node(
concept=concept, pattern=pattern, match=match
)
match_to_score.append((semantic_node_for_match, score))
# We want to replace object matches with their semantic nodes,
# but we don't want to alter the graph while matching it,
# so we accumulate these to replace later.
if isinstance(concept, ObjectConcept):
matched_objects.append((semantic_node_for_match, match))
# A template only has to match once; we don't care about finding additional matches.
return
# For each template whose semantics we are certain of (=have been added to the lexicon)
for (concept, graph_pattern, score) in self._primary_templates():
check_state(isinstance(graph_pattern, PerceptionGraphTemplate))
if (
preprocessed_perception_graph.dynamic
== graph_pattern.graph_pattern.dynamic
):
match_template(concept=concept, pattern=graph_pattern, score=score)
else:
logging.debug(
f"Unable to try and match {concept} to {preprocessed_perception_graph} "
f"because both patterns must be static or dynamic"
)
if not match_to_score:
# Try to match against patterns being learned
# only if no lexicalized pattern was matched.
for (concept, graph_pattern, score) in self._fallback_templates():
# we may have multiple pattern hypotheses for a single concept, in which case we only want to identify the concept once
if not any(m[0].concept == concept for m in match_to_score):
match_template(concept=concept, pattern=graph_pattern, score=score)
perception_graph_after_matching = perception_semantic_alignment.perception_graph
# Replace any objects found
def by_pattern_complexity(pair):
_, pattern_match = pair
return len(pattern_match.matched_pattern)
matched_objects.sort(key=by_pattern_complexity, reverse=True)
already_replaced: Set[ObjectPerception] = set()
new_nodes: List[SemanticNode] = []
for (matched_object_node, pattern_match) in matched_objects:
root: ObjectPerception = _get_root_object_perception(
pattern_match.matched_sub_graph._graph, # pylint:disable=protected-access
immutableset(
pattern_match.matched_sub_graph._graph.nodes, # pylint:disable=protected-access
disable_order_check=True,
),
)
if root not in already_replaced:
perception_graph_after_matching = replace_match_root_with_object_semantic_node(
object_semantic_node=cast(ObjectSemanticNode, matched_object_node),
current_perception=perception_graph_after_matching,
pattern_match=pattern_match,
)
already_replaced.add(root)
new_nodes.append(matched_object_node)
else:
logging.info(
f"Matched pattern for {matched_object_node} "
f"but root object {root} already replaced."
)
if matched_objects:
immutable_new_nodes = immutableset(new_nodes)
else:
immutable_new_nodes = immutableset(node for (node, _) in match_to_score)
(
perception_graph_after_post_processing,
nodes_after_post_processing,
) = self._enrich_post_process(
perception_graph_after_matching, immutable_new_nodes
)
return (
perception_semantic_alignment.copy_with_updated_graph_and_added_nodes(
new_graph=perception_graph_after_post_processing,
new_nodes=nodes_after_post_processing,
),
nodes_after_post_processing,
)
def perception_text(
self, perception: PerceptualRepresentation[
DevelopmentalPrimitivePerceptionFrame]
) -> str:
"""
Turns a perception into a list of items in the perceptions frames.
"""
output_text: List[str] = []
check_state(
len(perception.frames) in (1, 2),
"Only know how to handle 1 or 2 frame "
"perceptions for now",
)
perception_is_dynamic = len(perception.frames) > 1
# first, we build an index of objects to their properties.
# This will be used so that when we list the objects,
# we can easily list their properties in brackets right after them.
def extract_subject(prop: PropertyPerception) -> ObjectPerception:
return prop.perceived_object
first_frame_properties = _index_to_setmultidict(
perception.frames[0].property_assertions, extract_subject)
second_frame_properties = (_index_to_setmultidict(
perception.frames[1].property_assertions, extract_subject)
if perception_is_dynamic else
immutablesetmultidict())
# Next, we determine what objects persist between both frames
# and which do not.
first_frame_objects = perception.frames[0].perceived_objects
second_frame_objects = (perception.frames[1].perceived_objects
if perception_is_dynamic else immutableset())
static_objects = (
first_frame_objects.intersection(second_frame_objects)
if perception_is_dynamic else first_frame_objects)
all_objects = first_frame_objects.union(second_frame_objects)
# For objects, properties, and relations we will use arrows to indicate
# when something beings or ceased to exist between frames.
# Since the logic will be the same for all three types,
# we pull it out into a function.
def compute_arrow(
item: Any, static_items: AbstractSet[Any],
first_frame_items: AbstractSet[Any]) -> Tuple[str, str]:
if item in static_items:
# item doesn't change - no arrow
return ("", "")
elif item in first_frame_items:
# item ceases to exist
return ("", " ---> Ø")
else:
# item beings to exist in the second frame
return ("Ø ---> ", "")
# the logic for rendering objects, which will be used in the loop below.
# This needs to be an inner function so it can access the frame property maps, etc.
def render_object(obj: ObjectPerception) -> str:
obj_text = f"<i>{obj.debug_handle}</i>"
first_frame_obj_properties = first_frame_properties[obj]
second_frame_obj_properties = second_frame_properties[obj]
static_properties = (second_frame_obj_properties.intersection(
first_frame_obj_properties) if second_frame_obj_properties else
first_frame_obj_properties)
# logic for rendering properties, for use in the loop below.
def render_property(prop: PropertyPerception) -> str:
(prop_prefix,
prop_suffix) = compute_arrow(prop, static_properties,
first_frame_obj_properties)
prop_string: str
if isinstance(prop, HasColor):
prop_string = (
f'<span style="background-color: {prop.color}; '
f'color: {prop.color.inverse()}; border: 1px solid black;">'
f"color={prop.color.hex}</span>")
elif isinstance(prop, HasBinaryProperty):
prop_string = prop.binary_property.handle
else:
raise RuntimeError(f"Cannot render property: {prop}")
return f"{prop_prefix}{prop_string}{prop_suffix}"
all_properties: ImmutableSet[PropertyPerception] = immutableset(
flatten(
[first_frame_obj_properties, second_frame_obj_properties]))
prop_strings = [render_property(prop) for prop in all_properties]
if prop_strings:
return f"{obj_text}[{'; '.join(prop_strings)}]"
else:
return obj_text
# Here we process the relations between the two scenes to determine all relations.
# This has to be done before rending objects so we can use the PART_OF relation to order
# the objects.
first_frame_relations = perception.frames[0].relations
second_frame_relations = (perception.frames[1].relations
if perception_is_dynamic else immutableset())
static_relations = (
second_frame_relations.intersection(first_frame_relations)
if perception_is_dynamic else first_frame_relations)
all_relations = first_frame_relations.union(second_frame_relations)
# Here we add the perceived objects to a NetworkX DiGraph with PART_OF relations being the
# edges between objects. This allows us to do pre-order traversal of the Graph to make a
# nested <ul></ul> for the objects rather than a flat list.
graph = DiGraph()
root = ObjectPerception("root", axes=WORLD_AXES)
graph.add_node(root)
expressed_relations = set()
axis_to_object: Dict[GeonAxis, ObjectPerception] = {}
for object_ in all_objects:
graph.add_node(object_)
graph.add_edge(root, object_)
for axis in object_.axes.all_axes:
axis_to_object[axis] = object_
for relation_ in all_relations:
if relation_.relation_type == PART_OF:
graph.add_edge(relation_.second_slot, relation_.first_slot)
if graph.has_edge(root, relation_.first_slot):
graph.remove_edge(root, relation_.first_slot)
expressed_relations.add(relation_)
# Next, we render objects, together with their properties, using preorder DFS Traversal
# We also add in `In Region` relationships at this step for objects which have them.
output_text.append(
"\n\t\t\t\t\t<h5>Perceived Objects</h5>\n\t\t\t\t\t<ul>")
visited = set()
region_relations = immutableset(region for region in all_relations
if region.relation_type == IN_REGION)
# This loop doesn't quite get the tab spacing right. It could at the cost of increased
# complexity. Would need to track the "depth" we are currently at.
axis_info = perception.frames[0].axis_info
def dfs_walk(node, depth=0):
visited.add(node)
if not node == root:
(obj_prefix,
obj_suffix) = compute_arrow(node, static_objects,
first_frame_objects)
output_text.append(
f"\t" * (6 + depth) +
f"<li>{obj_prefix}{render_object(node)}{obj_suffix}<ul>")
if node.geon:
output_text.append(
f"\t\t\t\t\t\t<li>Geon: {self._render_geon(node.geon, indent_dept=7)}</li>"
)
# Handle Region Relations
for region_relation in region_relations:
if region_relation.first_slot == node:
(relation_prefix, relation_suffix) = compute_arrow(
region_relation, static_relations,
first_frame_relations)
relation_str = self._render_relation(
axis_info, region_relation)
output_text.append(
f"\t\t\t\t\t\t<li>{relation_prefix}"
f"{relation_str}{relation_suffix}</li>")
expressed_relations.add(region_relation)
for succ in graph.successors(node):
if succ not in visited:
depth = depth + 6
dfs_walk(succ, depth)
depth = depth - 6
output_text.append("\t" * (6 + depth) + f"</ul></li>")
dfs_walk(root)
output_text.append("\t\t\t\t\t</ul>")
# Finally we render remaining relations between objects
remaining_relations = immutableset(
relation for relation in all_relations
if relation not in expressed_relations)
if remaining_relations:
output_text.append(
"\t\t\t\t\t<h5>Other Relations</h5>\n\t\t\t\t\t<ul>")
for relation in remaining_relations:
(relation_prefix,
relation_suffix) = compute_arrow(relation, static_relations,
first_frame_relations)
single_size_relation: Optional[Tuple[
Any, str, Any]] = self._get_single_size_relation(
relation, all_relations)
if single_size_relation:
relation_text = f"{single_size_relation[0]} {single_size_relation[1]} {single_size_relation[2]}"
size_output = f"\t\t\t\t\t\t<li>{relation_prefix}{relation_text}{relation_suffix}</li>"
if size_output not in output_text:
output_text.append(size_output)
else:
output_text.append(
f"\t\t\t\t\t\t<li>{relation_prefix}{relation}{relation_suffix}</li>"
)
output_text.append("\t\t\t\t\t</ul>")
if perception.during:
output_text.append("\t\t\t\t\t<h5>During the action</h5>")
output_text.append(
self._render_during(perception.during, indent_depth=5))
if axis_info and axis_info.axes_facing:
output_text.append(("\t\t\t\t\t<h5>Axis Facings</h5>"))
output_text.append(("\t\t\t\t\t<ul>"))
for object_ in axis_info.axes_facing:
output_text.append(
f"\t\t\t\t\t\t<li>{object_.debug_handle} faced by:\n\t\t\t\t\t\t<ul>"
)
for axis in axis_info.axes_facing[object_]:
output_text.append(
f"\t\t\t\t\t\t\t<li>{axis} possessed by {axis_to_object[axis]}</li>"
)
output_text.append("\t\t\t\t\t\t</ul>")
output_text.append("\t\t\t\t\t</ul>")
return "\n".join(output_text)
def _enrich_common(
self, perception_semantic_alignment: PerceptionSemanticAlignment
) -> Tuple[PerceptionSemanticAlignment, AbstractSet[SemanticNode]]:
"""
Shared code between `enrich_during_learning` and `enrich_during_description`.
"""
preprocessing_result = self._preprocess_scene(
perception_semantic_alignment)
preprocessed_perception_graph = preprocessing_result.perception_graph
# This accumulates our output.
match_to_score: List[Tuple[SemanticNode, float]] = []
# In the case of objects only, we alter the perception graph once they
# are recognized by replacing the matched portion of the graph with the
# ObjectSemanticNodes. We gather them as we match and do the replacement below.
matched_objects: List[Tuple[SemanticNode,
PerceptionGraphPatternMatch]] = []
# We pull this out into a function because we do matching in two passes:
# first against templates whose meanings we are sure of (=have lexicalized)
# and then, if no match has been found, against those we are still learning.
def match_template(*, concept: Concept,
pattern: PerceptionGraphTemplate,
score: float) -> None:
matches_with_nodes = self._match_template(
concept=concept,
pattern=pattern.copy_with_temporal_scopes(ENTIRE_SCENE)
if preprocessed_perception_graph.dynamic
and not pattern.graph_pattern.dynamic else pattern,
perception_graph=preprocessed_perception_graph,
)
# The template may have zero, one, or many matches, so we loop over the matches found
# Note that, with the exception of the object learners,
# some matches may be essentially identical to each other.
# This is fine because the corresponding semantic nodes will be equal,
# so they'll get thrown out when we take the immutable set of new nodes.
for match_with_node in matches_with_nodes:
(match, semantic_node_for_match) = match_with_node
match_to_score.append((semantic_node_for_match, score))
# We want to replace object matches with their semantic nodes,
# but we don't want to alter the graph while matching it,
# so we accumulate these to replace later.
if isinstance(concept, ObjectConcept):
matched_objects.append((semantic_node_for_match, match))
# For each template whose semantics we are certain of (=have been added to the lexicon)
for (concept, graph_pattern, score) in self._primary_templates():
check_state(isinstance(graph_pattern, PerceptionGraphTemplate))
if (preprocessed_perception_graph.dynamic ==
graph_pattern.graph_pattern.dynamic):
match_template(concept=concept,
pattern=graph_pattern,
score=score)
elif (preprocessed_perception_graph.dynamic
and not graph_pattern.graph_pattern.dynamic):
match_template(
concept=concept,
pattern=graph_pattern.copy_with_temporal_scopes(
ENTIRE_SCENE),
score=score,
)
else:
logging.debug(
f"Unable to try and match {concept} to {preprocessed_perception_graph} "
f"because both patterns must be static or dynamic")
if not match_to_score:
# Try to match against patterns being learned
# only if no lexicalized pattern was matched.
for (concept, graph_pattern, score) in self._fallback_templates():
# we may have multiple pattern hypotheses for a single concept, in which case we only want to identify the concept once
if not any(m[0].concept == concept for m in match_to_score):
match_template(concept=concept,
pattern=graph_pattern,
score=score)
perception_graph_after_matching = perception_semantic_alignment.perception_graph
# Replace any objects found
def by_pattern_complexity(pair):
_, pattern_match = pair
return pattern_match.matched_pattern.pattern_complexity()
matched_objects.sort(key=by_pattern_complexity, reverse=True)
already_replaced: Set[ObjectPerception] = set()
new_nodes: List[SemanticNode] = []
for (matched_object_node, pattern_match) in matched_objects:
root: ObjectPerception = _get_root_object_perception(
pattern_match.matched_sub_graph._graph, # pylint:disable=protected-access
immutableset(
pattern_match.matched_sub_graph._graph.nodes, # pylint:disable=protected-access
disable_order_check=True,
),
)
if root not in already_replaced:
try:
replacement_result = replace_match_with_object_graph_node(
matched_object_node=cast(ObjectSemanticNode,
matched_object_node),
current_perception=perception_graph_after_matching,
pattern_match=pattern_match,
)
perception_graph_after_matching = (
replacement_result.perception_graph_after_replacement)
already_replaced.update( # type: ignore
replacement_result.removed_nodes)
new_nodes.append(matched_object_node)
except networkx.exception.NetworkXError:
logging.info(f"Matched pattern for {matched_object_node} "
f"contains nodes that are already replaced")
else:
logging.info(f"Matched pattern for {matched_object_node} "
f"but root object {root} already replaced.")
# If objects, only include the replaced graph nodes in the enrichment
if new_nodes and isinstance(new_nodes[0], ObjectSemanticNode):
immutable_new_nodes = immutableset(new_nodes)
else:
immutable_new_nodes = immutableset(node
for (node, _) in match_to_score)
# Keep recursively enriching so we can capture plurals. Do it only if we matched objects in the scene.
if new_nodes and isinstance(new_nodes[0], ObjectSemanticNode):
rec = self._enrich_common(
perception_semantic_alignment.
copy_with_updated_graph_and_added_nodes(
new_graph=perception_graph_after_matching,
new_nodes=immutable_new_nodes,
))
return rec[0], set(immutable_new_nodes).union(rec[1])
(
perception_graph_after_post_processing,
nodes_after_post_processing,
) = self._enrich_post_process(perception_graph_after_matching,
immutable_new_nodes)
return (
perception_semantic_alignment.
copy_with_updated_graph_and_added_nodes(
new_graph=perception_graph_after_post_processing,
new_nodes=nodes_after_post_processing,
),
nodes_after_post_processing,
)
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
check_state(self._zip_file)
self._zip_file.close() # type: ignore
self._zip_file = None
def keys(self) -> Optional[AbstractSet[str]]:
check_state(self._zip_file, "Must use zip key-value source as a context manager")
return self._keys