async def prepare_token_serialisation(
tracker_store: TrackerStore, response_selector_agent: Agent, sender_id: Text
):
text = "Good morning"
tokenizer = WhitespaceTokenizer(WhitespaceTokenizer.get_default_config())
tokens = tokenizer.tokenize(Message(data={"text": text}), "text")
indices = [[t.start, t.end] for t in tokens]
tracker = tracker_store.get_or_create_tracker(sender_id=sender_id)
parse_data = await response_selector_agent.parse_message(text)
event = UserUttered(
"Good morning",
parse_data.get("intent"),
parse_data.get("entities", []),
parse_data,
)
tracker.update(event)
tracker_store.save(tracker)
retrieved_tracker = tracker_store.retrieve(sender_id=sender_id)
event = retrieved_tracker.get_last_event_for(event_type=UserUttered)
event_tokens = event.as_dict().get("parse_data").get("text_tokens")
assert event_tokens == indices
def create_whitespace_tokenizer(
config: Optional[Dict] = None) -> WhitespaceTokenizer:
config = config if config else {}
return WhitespaceTokenizer({
**WhitespaceTokenizer.get_default_config(),
**config
})
def test_validate_after_adding_adding_default_parameter(
get_validation_method: Callable[..., ValidationMethodType],
nlu: bool,
core: bool,
):
# create a schema and rely on rasa to fill in defaults later
schema1 = _get_example_schema()
schema1.nodes["nlu-node"] = SchemaNode(needs={},
uses=WhitespaceTokenizer,
constructor_name="",
fn="",
config={})
schema1.nodes["core-node"] = SchemaNode(needs={},
uses=RulePolicy,
constructor_name="",
fn="",
config={})
# training
validate = get_validation_method(finetuning=False,
load=False,
nlu=nlu,
core=core,
graph_schema=schema1)
validate(importer=EmptyDataImporter())
# same schema -- we just explicitly pass default values
schema2 = copy.deepcopy(schema1)
schema2.nodes["nlu-node"] = SchemaNode(
needs={},
uses=WhitespaceTokenizer,
constructor_name="",
fn="",
config=WhitespaceTokenizer.get_default_config(),
)
schema2.nodes["core-node"] = SchemaNode(
needs={},
uses=RulePolicy,
constructor_name="",
fn="",
config=RulePolicy.get_default_config(),
)
# finetuning *does not raise*
loaded_validate = get_validation_method(finetuning=True,
load=True,
nlu=nlu,
core=core,
graph_schema=schema2)
loaded_validate(importer=EmptyDataImporter())
def create_whitespace_tokenizer(
config: Optional[Dict[Text, Any]] = None
) -> WhitespaceTokenizer:
return WhitespaceTokenizer(
{**WhitespaceTokenizer.get_default_config(), **(config if config else {})}
)
def whitespace_tokenizer() -> WhitespaceTokenizer:
return WhitespaceTokenizer(WhitespaceTokenizer.get_default_config())
