async def prepare_token_serialisation( tracker_store: TrackerStore, response_selector_agent: Agent, sender_id: Text ): text = "Good morning" tokenizer = WhitespaceTokenizer(WhitespaceTokenizer.get_default_config()) tokens = tokenizer.tokenize(Message(data={"text": text}), "text") indices = [[t.start, t.end] for t in tokens] tracker = tracker_store.get_or_create_tracker(sender_id=sender_id) parse_data = await response_selector_agent.parse_message(text) event = UserUttered( "Good morning", parse_data.get("intent"), parse_data.get("entities", []), parse_data, ) tracker.update(event) tracker_store.save(tracker) retrieved_tracker = tracker_store.retrieve(sender_id=sender_id) event = retrieved_tracker.get_last_event_for(event_type=UserUttered) event_tokens = event.as_dict().get("parse_data").get("text_tokens") assert event_tokens == indices
def create_whitespace_tokenizer( config: Optional[Dict] = None) -> WhitespaceTokenizer: config = config if config else {} return WhitespaceTokenizer({ **WhitespaceTokenizer.get_default_config(), **config })
def test_validate_after_adding_adding_default_parameter( get_validation_method: Callable[..., ValidationMethodType], nlu: bool, core: bool, ): # create a schema and rely on rasa to fill in defaults later schema1 = _get_example_schema() schema1.nodes["nlu-node"] = SchemaNode(needs={}, uses=WhitespaceTokenizer, constructor_name="", fn="", config={}) schema1.nodes["core-node"] = SchemaNode(needs={}, uses=RulePolicy, constructor_name="", fn="", config={}) # training validate = get_validation_method(finetuning=False, load=False, nlu=nlu, core=core, graph_schema=schema1) validate(importer=EmptyDataImporter()) # same schema -- we just explicitly pass default values schema2 = copy.deepcopy(schema1) schema2.nodes["nlu-node"] = SchemaNode( needs={}, uses=WhitespaceTokenizer, constructor_name="", fn="", config=WhitespaceTokenizer.get_default_config(), ) schema2.nodes["core-node"] = SchemaNode( needs={}, uses=RulePolicy, constructor_name="", fn="", config=RulePolicy.get_default_config(), ) # finetuning *does not raise* loaded_validate = get_validation_method(finetuning=True, load=True, nlu=nlu, core=core, graph_schema=schema2) loaded_validate(importer=EmptyDataImporter())
def create_whitespace_tokenizer( config: Optional[Dict[Text, Any]] = None ) -> WhitespaceTokenizer: return WhitespaceTokenizer( {**WhitespaceTokenizer.get_default_config(), **(config if config else {})} )
def whitespace_tokenizer() -> WhitespaceTokenizer: return WhitespaceTokenizer(WhitespaceTokenizer.get_default_config())