def test_split_intent(text, expected_tokens):
component_config = {
"intent_tokenization_flag": True,
"intent_split_symbol": "+"
}
tk = WhitespaceTokenizer(component_config)
message = Message(text)
message.set(INTENT_ATTRIBUTE, text)
assert [t.text for t in tk._split_intent(message)] == expected_tokens
def test_split_intent_response_key(text, expected_tokens):
component_config = {
"intent_tokenization_flag": True,
"intent_split_symbol": "+"
}
tk = WhitespaceTokenizer(component_config)
message = Message(text)
message.set(INTENT_RESPONSE_KEY, text)
assert [
t.text
for t in tk._split_intent(message, attribute=INTENT_RESPONSE_KEY)
] == expected_tokens
