def test_split_intent(text: Text, expected_tokens: List[Text]):
component_config = {
"intent_tokenization_flag": True,
"intent_split_symbol": "+"
}
tk = WhitespaceTokenizer(component_config)
message = Message.build(text=text)
message.set(INTENT, text)
assert [t.text for t in tk._split_name(message, INTENT)] == expected_tokens
def test_split_intent_response_key(text, expected_tokens):
component_config = {
"intent_tokenization_flag": True,
"intent_split_symbol": "+"
}
tk = WhitespaceTokenizer(component_config)
message = Message.build(text=text)
message.set(INTENT_RESPONSE_KEY, text)
assert [
t.text for t in tk._split_name(message, attribute=INTENT_RESPONSE_KEY)
] == expected_tokens