def test_split_intent(text: Text, expected_tokens: List[Text]): component_config = { "intent_tokenization_flag": True, "intent_split_symbol": "+" } tk = WhitespaceTokenizer(component_config) message = Message.build(text=text) message.set(INTENT, text) assert [t.text for t in tk._split_name(message, INTENT)] == expected_tokens
def test_split_intent_response_key(text, expected_tokens): component_config = { "intent_tokenization_flag": True, "intent_split_symbol": "+" } tk = WhitespaceTokenizer(component_config) message = Message.build(text=text) message.set(INTENT_RESPONSE_KEY, text) assert [ t.text for t in tk._split_name(message, attribute=INTENT_RESPONSE_KEY) ] == expected_tokens