def _sliced_states_iterator( trackers: List[TrackerWithCachedStates], domain: Domain, max_history: Optional[int], tokenizer: Optional[Tokenizer], ) -> Generator[TrackerEventStateTuple, None, None]: """Creates an iterator over sliced states. Iterate over all given trackers and all sliced states within each tracker, where the slicing is based on `max_history`. Args: trackers: List of trackers. domain: Domain (used for tracker.past_states). max_history: Assumed `max_history` value for slicing. tokenizer: A tokenizer to tokenize the user messages. Yields: A (tracker, event, sliced_states) triplet. """ for tracker in trackers: states = tracker.past_states(domain) idx = 0 for event in tracker.events: if isinstance(event, ActionExecuted): sliced_states = MaxHistoryTrackerFeaturizer.slice_state_history( states[:idx + 1], max_history) if tokenizer: _apply_tokenizer_to_states(tokenizer, sliced_states) # TODO: deal with oov (different tokens can lead to identical features # if some of those tokens are out of vocabulary for all featurizers) yield TrackerEventStateTuple(tracker, event, sliced_states) idx += 1
def _sliced_states_iterator( trackers: List[TrackerWithCachedStates], domain: Domain, max_history: int ) -> Generator[TrackerEventStateTuple, None, None]: """Creates an iterator over sliced states. Iterate over all given trackers and all sliced states within each tracker, where the slicing is based on `max_history`. Args: trackers: List of trackers. domain: Domain (used for tracker.past_states). max_history: Assumed `max_history` value for slicing. Yields: A (tracker, event, sliced_states) triplet. """ for tracker in trackers: states = tracker.past_states(domain) idx = 0 for event in tracker.events: if isinstance(event, ActionExecuted): sliced_states = MaxHistoryTrackerFeaturizer.slice_state_history( states[: idx + 1], max_history ) yield TrackerEventStateTuple(tracker, event, sliced_states) idx += 1