def test_convert_featurizer_tokens_to_text(component_builder, sentence, expected_text):
tokenizer = component_builder.create_component_from_class(ConveRTTokenizer)
tokens = tokenizer.tokenize(Message(sentence), attribute=TEXT)
actual_text = ConveRTFeaturizer._tokens_to_text([tokens])[0]
assert expected_text == actual_text
def test_convert_featurizer_tokens_to_text(
create_or_load_convert_featurizer: Callable[[Dict[Text, Any]],
ConveRTFeaturizer],
sentence: Text,
expected_text: Text,
monkeypatch: MonkeyPatch,
whitespace_tokenizer: WhitespaceTokenizer,
):
monkeypatch.setattr(
ConveRTFeaturizer,
"_validate_model_url",
lambda _: None,
)
component_config = {
FEATURIZER_CLASS_ALIAS: "alias",
"model_url": RESTRICTED_ACCESS_URL,
}
featurizer = create_or_load_convert_featurizer(component_config)
message = Message.build(text=sentence)
td = TrainingData([message])
whitespace_tokenizer.process_training_data(td)
tokens = featurizer.tokenize(message, attribute=TEXT)
actual_text = ConveRTFeaturizer._tokens_to_text([tokens])[0]
assert expected_text == actual_text
def test_convert_featurizer_tokens_to_text(sentence, expected_text):
tokens = ConveRTTokenizer().tokenize(Message(sentence),
attribute=TEXT_ATTRIBUTE)
actual_text = ConveRTFeaturizer._tokens_to_text([tokens])[0]
assert expected_text == actual_text
def test_convert_featurizer_tokens_to_text(
sentence: Text, expected_text: Text, monkeypatch: MonkeyPatch
):
monkeypatch.setattr(
ConveRTTokenizer, "_get_validated_model_url", lambda x: RESTRICTED_ACCESS_URL
)
component_config = {"name": "ConveRTTokenizer", "model_url": RESTRICTED_ACCESS_URL}
tokenizer = ConveRTTokenizer(component_config)
tokens = tokenizer.tokenize(Message(data={TEXT: sentence}), attribute=TEXT)
actual_text = ConveRTFeaturizer._tokens_to_text([tokens])[0]
assert expected_text == actual_text
def test_convert_featurizer_tokens_to_text(sentence: Text, expected_text: Text,
monkeypatch: MonkeyPatch):
tokenizer = WhitespaceTokenizer()
monkeypatch.setattr(ConveRTFeaturizer, "_get_validated_model_url",
lambda x: RESTRICTED_ACCESS_URL)
component_config = {
"name": "ConveRTFeaturizer",
"model_url": RESTRICTED_ACCESS_URL
}
featurizer = ConveRTFeaturizer(component_config)
message = Message.build(text=sentence)
td = TrainingData([message])
tokenizer.train(td)
tokens = featurizer.tokenize(message, attribute=TEXT)
actual_text = ConveRTFeaturizer._tokens_to_text([tokens])[0]
assert expected_text == actual_text