def describe( self, perception: PerceptualRepresentation[DevelopmentalPrimitivePerceptionFrame] ) -> Mapping[LinguisticDescription, float]: perception_graph = self._extract_perception_graph(perception) cur_description_state = PerceptionSemanticAlignment.create_unaligned( perception_graph ) for sub_learner in [ self.object_learner, self.attribute_learner, self.relation_learner, ]: if sub_learner: cur_description_state = sub_learner.enrich_during_description( cur_description_state ) if perception.is_dynamic() and self.action_learner: cur_description_state = self.action_learner.enrich_during_description( cur_description_state ) if self.functional_learner: cur_description_state = self.functional_learner.enrich_during_description( cur_description_state ) return self._linguistic_descriptions_from_semantics(cur_description_state)
def _preprocess_scene( self, perception_semantic_alignment: PerceptionSemanticAlignment ) -> PerceptionSemanticAlignment: # Avoid accidentally identifying a word with the learner itself. return perception_semantic_alignment.copy_with_updated_graph_and_added_nodes( new_graph=graph_without_learner( perception_semantic_alignment.perception_graph), new_nodes=[], )
def match_objects_old( self, perception_graph: PerceptionGraph ) -> PerceptionGraphFromObjectRecognizer: new_style_input = PerceptionSemanticAlignment( perception_graph=perception_graph, semantic_nodes=[]) new_style_output = self.match_objects(new_style_input) return PerceptionGraphFromObjectRecognizer( perception_graph=new_style_output[0].perception_graph, description_to_matched_object_node=new_style_output[1], )
def match_objects_with_language_old( self, language_aligned_perception: LanguageAlignedPerception ) -> LanguageAlignedPerception: if language_aligned_perception.node_to_language_span: raise RuntimeError( "Don't know how to handle a non-empty node-to-language-span") new_style_input = LanguagePerceptionSemanticAlignment( language_concept_alignment=LanguageConceptAlignment( language_aligned_perception.language, node_to_language_span=[]), perception_semantic_alignment=PerceptionSemanticAlignment( perception_graph=language_aligned_perception.perception_graph, semantic_nodes=[], ), ) new_style_output = self.match_objects_with_language(new_style_input) return LanguageAlignedPerception( language=new_style_output.language_concept_alignment.language, perception_graph=new_style_output.perception_semantic_alignment. perception_graph, node_to_language_span=new_style_output.language_concept_alignment. node_to_language_span, )
def _preprocess_scene( self, perception_semantic_alignment: PerceptionSemanticAlignment ) -> PerceptionSemanticAlignment: nodes = [ s for s in perception_semantic_alignment.semantic_nodes if isinstance(s, ObjectSemanticNode) ] counts = collections.Counter([s.concept for s in nodes]) digraph = perception_semantic_alignment.perception_graph.copy_as_digraph( ) for node in nodes: count = counts[node.concept] if count > 1: if count == 2: count_node = TWO else: count_node = MANY digraph.add_node(count_node) digraph.add_edge(node, count_node, label=HAS_COUNT) graph_with_counts = PerceptionGraph( digraph, dynamic=perception_semantic_alignment.perception_graph.dynamic) return PerceptionSemanticAlignment( graph_with_counts, perception_semantic_alignment.semantic_nodes)
def _enrich_common( self, perception_semantic_alignment: PerceptionSemanticAlignment ) -> Tuple[PerceptionSemanticAlignment, AbstractSet[SemanticNode]]: """ Shared code between `enrich_during_learning` and `enrich_during_description`. """ preprocessing_result = self._preprocess_scene(perception_semantic_alignment) preprocessed_perception_graph = preprocessing_result.perception_graph # This accumulates our output. match_to_score: List[Tuple[SemanticNode, float]] = [] # In the case of objects only, we alter the perception graph once they # are recognized by replacing the matched portion of the graph with the # ObjectSemanticNodes. We gather them as we match and do the replacement below. matched_objects: List[Tuple[SemanticNode, PerceptionGraphPatternMatch]] = [] # We pull this out into a function because we do matching in two passes: # first against templates whose meanings we are sure of (=have lexicalized) # and then, if no match has been found, against those we are still learning. def match_template( *, concept: Concept, pattern: PerceptionGraphTemplate, score: float ) -> None: # try to see if (our model of) its semantics is present in the situation. matcher = pattern.graph_pattern.matcher( preprocessed_perception_graph, match_mode=MatchMode.NON_OBJECT, # debug_callback=self._debug_callback, ) for match in matcher.matches(use_lookahead_pruning=True): # if there is a match, use that match to describe the situation. semantic_node_for_match = pattern_match_to_semantic_node( concept=concept, pattern=pattern, match=match ) match_to_score.append((semantic_node_for_match, score)) # We want to replace object matches with their semantic nodes, # but we don't want to alter the graph while matching it, # so we accumulate these to replace later. if isinstance(concept, ObjectConcept): matched_objects.append((semantic_node_for_match, match)) # A template only has to match once; we don't care about finding additional matches. return # For each template whose semantics we are certain of (=have been added to the lexicon) for (concept, graph_pattern, score) in self._primary_templates(): check_state(isinstance(graph_pattern, PerceptionGraphTemplate)) if ( preprocessed_perception_graph.dynamic == graph_pattern.graph_pattern.dynamic ): match_template(concept=concept, pattern=graph_pattern, score=score) else: logging.debug( f"Unable to try and match {concept} to {preprocessed_perception_graph} " f"because both patterns must be static or dynamic" ) if not match_to_score: # Try to match against patterns being learned # only if no lexicalized pattern was matched. for (concept, graph_pattern, score) in self._fallback_templates(): # we may have multiple pattern hypotheses for a single concept, in which case we only want to identify the concept once if not any(m[0].concept == concept for m in match_to_score): match_template(concept=concept, pattern=graph_pattern, score=score) perception_graph_after_matching = perception_semantic_alignment.perception_graph # Replace any objects found def by_pattern_complexity(pair): _, pattern_match = pair return len(pattern_match.matched_pattern) matched_objects.sort(key=by_pattern_complexity, reverse=True) already_replaced: Set[ObjectPerception] = set() new_nodes: List[SemanticNode] = [] for (matched_object_node, pattern_match) in matched_objects: root: ObjectPerception = _get_root_object_perception( pattern_match.matched_sub_graph._graph, # pylint:disable=protected-access immutableset( pattern_match.matched_sub_graph._graph.nodes, # pylint:disable=protected-access disable_order_check=True, ), ) if root not in already_replaced: perception_graph_after_matching = replace_match_root_with_object_semantic_node( object_semantic_node=cast(ObjectSemanticNode, matched_object_node), current_perception=perception_graph_after_matching, pattern_match=pattern_match, ) already_replaced.add(root) new_nodes.append(matched_object_node) else: logging.info( f"Matched pattern for {matched_object_node} " f"but root object {root} already replaced." ) if matched_objects: immutable_new_nodes = immutableset(new_nodes) else: immutable_new_nodes = immutableset(node for (node, _) in match_to_score) ( perception_graph_after_post_processing, nodes_after_post_processing, ) = self._enrich_post_process( perception_graph_after_matching, immutable_new_nodes ) return ( perception_semantic_alignment.copy_with_updated_graph_and_added_nodes( new_graph=perception_graph_after_post_processing, new_nodes=nodes_after_post_processing, ), nodes_after_post_processing, )
def observe( self, learning_example: LearningExample[ DevelopmentalPrimitivePerceptionFrame, LinguisticDescription ], observation_num: int = -1, ) -> None: if observation_num >= 0: logging.info( "Observation %s: %s", observation_num, learning_example.linguistic_description.as_token_string(), ) else: logging.info( "Observation %s: %s", self._observation_num, learning_example.linguistic_description.as_token_string(), ) self._observation_num += 1 # We need to track the alignment between perceived objects # and portions of the input language, so internally we operate over # LanguageAlignedPerceptions. current_learner_state = LanguagePerceptionSemanticAlignment( language_concept_alignment=LanguageConceptAlignment.create_unaligned( language=learning_example.linguistic_description ), perception_semantic_alignment=PerceptionSemanticAlignment( perception_graph=self._extract_perception_graph( learning_example.perception ), semantic_nodes=[], ), ) # We iteratively let each "layer" of semantic analysis attempt # to learn from the perception, # and then to annotate the perception with any semantic alignments it knows. for sub_learner in [ self.object_learner, self.attribute_learner, self.relation_learner, ]: if sub_learner: # Currently we do not attempt to learn static things from dynamic situations # because the static learners do not know how to deal with the temporal # perception graph edge wrappers. # See https://github.com/isi-vista/adam/issues/792 . if not learning_example.perception.is_dynamic(): sub_learner.learn_from( current_learner_state, observation_num=observation_num ) current_learner_state = sub_learner.enrich_during_learning( current_learner_state ) if learning_example.perception.is_dynamic() and self.action_learner: self.action_learner.learn_from(current_learner_state) current_learner_state = self.action_learner.enrich_during_learning( current_learner_state ) if self.functional_learner: self.functional_learner.learn_from( current_learner_state, observation_num=observation_num )
def observe( self, learning_example: LearningExample[ DevelopmentalPrimitivePerceptionFrame, LinguisticDescription], offset: int = 0, ) -> None: logging.info( "Observation %s: %s", self._observation_num + offset, learning_example.linguistic_description.as_token_string(), ) self._observation_num += 1 # We need to track the alignment between perceived objects # and portions of the input language, so internally we operate over # LanguageAlignedPerceptions. current_learner_state = LanguagePerceptionSemanticAlignment( language_concept_alignment=LanguageConceptAlignment. create_unaligned(language=learning_example.linguistic_description), perception_semantic_alignment=PerceptionSemanticAlignment( perception_graph=self._extract_perception_graph( learning_example.perception), semantic_nodes=[], ), ) # We iteratively let each "layer" of semantic analysis attempt # to learn from the perception, # and then to annotate the perception with any semantic alignments it knows. for sub_learner in [ self.object_learner, self.attribute_learner, self.plural_learner, self.relation_learner, ]: if sub_learner: # Currently we do not attempt to learn static things from dynamic situations # because the static learners do not know how to deal with the temporal # perception graph edge wrappers. # See https://github.com/isi-vista/adam/issues/792 . if not learning_example.perception.is_dynamic(): # For more details on the try/excepts below # See: https://github.com/isi-vista/adam/issues/1008 try: sub_learner.learn_from(current_learner_state, offset=offset) except (RuntimeError, KeyError) as e: logging.warning( f"Sub_learner ({sub_learner}) was unable to learn from instance number {self._observation_num}.\n" f"Instance: {current_learner_state}.\n" f"Full Error Information: {e}") if not self._suppress_error: raise e current_learner_state = sub_learner.enrich_during_learning( current_learner_state) # Check definiteness after recognizing objects if sub_learner == self.object_learner: self.learn_definiteness_markers(current_learner_state) if learning_example.perception.is_dynamic() and self.action_learner: try: self.action_learner.learn_from(current_learner_state) except (RuntimeError, KeyError) as e: logging.warning( f"Action Learner ({self.action_learner}) was unable to learn from instance number {self._observation_num}.\n" f"Instance: {current_learner_state}.\n" f"Full Error Information: {e}") if not self._suppress_error: raise e current_learner_state = self.action_learner.enrich_during_learning( current_learner_state) if self.functional_learner: self.functional_learner.learn_from(current_learner_state, offset=offset) # Engage generics learner if the utterance is indefinite if self.generics_learner and not self.is_definite( current_learner_state): # Lack of definiteness could be marking a generic statement # Check if the known descriptions match the utterance descs = self._linguistic_descriptions_from_semantics( current_learner_state.perception_semantic_alignment) # If the statement isn't a recognized sentence, run learner if not learning_example.linguistic_description.as_token_sequence( ) in [desc.as_token_sequence() for desc in descs]: # Pass plural markers to generics before learning from a statement if isinstance(self.plural_learner, SubsetPluralLearnerNew): self.generics_learner.plural_markers = list( # pylint: disable=assigning-non-slot self.plural_learner.potential_plural_markers.keys()) self.generics_learner.learn_from(current_learner_state) # Update concept semantics self.update_concept_semantics(current_learner_state)