def test_crf_use_dense_features(ner_crf_pos_feature_config, spacy_nlp):
from rasa.nlu.extractors.crf_entity_extractor import CRFEntityExtractor
from rasa.nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer
from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer
ner_crf_pos_feature_config["features"][1].append("text_dense_features")
crf_extractor = CRFEntityExtractor(component_config=ner_crf_pos_feature_config)
spacy_featurizer = SpacyFeaturizer()
white_space_tokenizer = WhitespaceTokenizer({"use_cls_token": False})
text = "Rasa is a company in Berlin"
message = Message(text)
message.set("spacy_doc", spacy_nlp(text))
white_space_tokenizer.process(message)
spacy_featurizer.process(message)
text_data = crf_extractor._from_text_to_crf(message)
features = crf_extractor._sentence_to_features(text_data)
assert "0:text_dense_features" in features[0]
for i in range(0, len(message.data.get("text_dense_features")[0])):
assert (
features[0]["0:text_dense_features"]["text_dense_features"][str(i)]
== message.data.get("text_dense_features")[0][i]
)
def test_convert_training_examples(
spacy_nlp: Any,
text: Text,
intent: Optional[Text],
entities: Optional[List[Dict[Text, Any]]],
attributes: List[Text],
real_sparse_feature_sizes: Dict[Text, Dict[Text, List[int]]],
):
message = Message(data={TEXT: text, INTENT: intent, ENTITIES: entities})
tokenizer = SpacyTokenizer()
count_vectors_featurizer = CountVectorsFeaturizer()
spacy_featurizer = SpacyFeaturizer()
message.set(SPACY_DOCS[TEXT], spacy_nlp(text))
training_data = TrainingData([message])
tokenizer.train(training_data)
count_vectors_featurizer.train(training_data)
spacy_featurizer.train(training_data)
entity_tag_spec = [
EntityTagSpec(
"entity",
{
0: "O",
1: "name",
2: "location"
},
{
"O": 0,
"name": 1,
"location": 2
},
3,
)
]
output, sparse_feature_sizes = model_data_utils.featurize_training_examples(
[message],
attributes=attributes,
entity_tag_specs=entity_tag_spec,
)
assert len(output) == 1
for attribute in attributes:
assert attribute in output[0]
for attribute in {INTENT, TEXT, ENTITIES} - set(attributes):
assert attribute not in output[0]
# we have sparse sentence, sparse sequence, dense sentence, and dense sequence
# features in the list
assert len(output[0][TEXT]) == 4
if INTENT in attributes:
# we will just have space sentence features
assert len(output[0][INTENT]) == 1
if ENTITIES in attributes:
# we will just have space sentence features
assert len(output[0][ENTITIES]) == len(entity_tag_spec)
# check that it calculates sparse_feature_sizes correctly
assert sparse_feature_sizes == real_sparse_feature_sizes
def create_spacy_featurizer(config: Dict[Text, Any]) -> SpacyFeaturizer:
return SpacyFeaturizer(
{
**SpacyFeaturizer.get_default_config(),
**config
},
"spacy_featurizer",
)
async def test_train_persist_with_different_configurations(
crf_entity_extractor: Callable[[Dict[Text, Any]], CRFEntityExtractor],
config_params: Dict[Text, Any],
default_model_storage: ModelStorage,
default_execution_context: ExecutionContext,
spacy_tokenizer: SpacyTokenizer,
spacy_featurizer: SpacyFeaturizer,
spacy_nlp_component: SpacyNLP,
spacy_model: SpacyModel,
):
crf_extractor = crf_entity_extractor(config_params)
importer = RasaFileImporter(training_data_paths=["data/examples/rasa"])
training_data = importer.get_nlu_data()
training_data = spacy_nlp_component.process_training_data(
training_data, spacy_model)
training_data = spacy_tokenizer.process_training_data(training_data)
training_data = spacy_featurizer.process_training_data(training_data)
crf_extractor.train(training_data)
message = Message(data={TEXT: "I am looking for an italian restaurant"})
messages = spacy_nlp_component.process([message], spacy_model)
messages = spacy_tokenizer.process(messages)
message = spacy_featurizer.process(messages)[0]
message2 = copy.deepcopy(message)
processed_message = crf_extractor.process([message])[0]
loaded_extractor = CRFEntityExtractor.load(
{
**CRFEntityExtractor.get_default_config(),
**config_params
},
default_model_storage,
Resource("CRFEntityExtractor"),
default_execution_context,
)
processed_message2 = loaded_extractor.process([message2])[0]
assert processed_message2.fingerprint() == processed_message.fingerprint()
detected_entities = processed_message2.get(ENTITIES)
assert len(detected_entities) == 1
assert detected_entities[0]["entity"] == "cuisine"
assert detected_entities[0]["value"] == "italian"
def test_spacy_intent_featurizer(spacy_nlp_component):
from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer
td = training_data.load_data("data/examples/rasa/demo-rasa.json")
spacy_nlp_component.train(td, config=None)
spacy_featurizer = SpacyFeaturizer()
spacy_featurizer.train(td, config=None)
intent_features_exist = np.array([
True if example.get("intent_features") is not None else False
for example in td.intent_examples
])
# no intent features should have been set
assert not any(intent_features_exist)
def test_spacy_featurizer_train(spacy_nlp):
featurizer = SpacyFeaturizer.create({}, RasaNLUModelConfig())
sentence = "Hey how are you today"
message = Message(sentence)
message.set(RESPONSE_ATTRIBUTE, sentence)
message.set(INTENT_ATTRIBUTE, "intent")
message.set(SPACY_DOCS[TEXT_ATTRIBUTE], spacy_nlp(sentence))
message.set(SPACY_DOCS[RESPONSE_ATTRIBUTE], spacy_nlp(sentence))
featurizer.train(TrainingData([message]), RasaNLUModelConfig())
expected = np.array([-0.28451, 0.31007, -0.57039, -0.073056, -0.17322])
expected_cls = np.array(
[-0.196496, 0.3249364, -0.37408298, -0.10622784, 0.062756])
vecs = message.get(DENSE_FEATURE_NAMES[TEXT_ATTRIBUTE])
assert 6 == len(vecs)
assert np.allclose(vecs[0][:5], expected, atol=1e-5)
assert np.allclose(vecs[-1][:5], expected_cls, atol=1e-5)
vecs = message.get(DENSE_FEATURE_NAMES[RESPONSE_ATTRIBUTE])
assert 6 == len(vecs)
assert np.allclose(vecs[0][:5], expected, atol=1e-5)
assert np.allclose(vecs[-1][:5], expected_cls, atol=1e-5)
vecs = message.get(DENSE_FEATURE_NAMES[INTENT_ATTRIBUTE])
assert vecs is None
def test_spacy_featurizer_sequence(sentence, expected, spacy_nlp):
from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer
doc = spacy_nlp(sentence)
token_vectors = [t.vector for t in doc]
ftr = SpacyFeaturizer.create({}, RasaNLUModelConfig())
greet = {TEXT: sentence, "intent": "greet", "text_features": [0.5]}
message = Message(data=greet)
message.set(SPACY_DOCS[TEXT], doc)
ftr._set_spacy_features(message)
seq_vecs, sen_vecs = message.get_dense_features(TEXT, [])
if seq_vecs:
seq_vecs = seq_vecs.features
if sen_vecs:
sen_vecs = sen_vecs.features
vecs = seq_vecs[0][:5]
assert np.allclose(token_vectors[0][:5], vecs, atol=1e-4)
assert np.allclose(vecs, expected, atol=1e-4)
assert sen_vecs is not None
def test_spacy_featurizer_train(spacy_nlp):
featurizer = SpacyFeaturizer.create({}, RasaNLUModelConfig())
sentence = "Hey how are you today"
message = Message(sentence)
message.set(RESPONSE, sentence)
message.set(INTENT, "intent")
message.set(SPACY_DOCS[TEXT], spacy_nlp(sentence))
message.set(SPACY_DOCS[RESPONSE], spacy_nlp(sentence))
featurizer.train(TrainingData([message]), RasaNLUModelConfig())
expected = np.array([-0.28451, 0.31007, -0.57039, -0.073056, -0.17322])
expected_cls = np.array([-0.196496, 0.3249364, -0.37408298, -0.10622784, 0.062756])
seq_vecs, sen_vecs = message.get_dense_features(TEXT, [])
assert 5 == len(seq_vecs)
assert 1 == len(sen_vecs)
assert np.allclose(seq_vecs[0][:5], expected, atol=1e-5)
assert np.allclose(sen_vecs[-1][:5], expected_cls, atol=1e-5)
seq_vecs, sen_vecs = message.get_dense_features(RESPONSE, [])
assert 5 == len(seq_vecs)
assert 1 == len(sen_vecs)
assert np.allclose(seq_vecs[0][:5], expected, atol=1e-5)
assert np.allclose(sen_vecs[-1][:5], expected_cls, atol=1e-5)
seq_vecs, sen_vecs = message.get_dense_features(INTENT, [])
assert seq_vecs is None
assert sen_vecs is None
def test_crf_use_dense_features(
crf_entity_extractor: Callable[[Dict[Text, Any]],
CRFEntityExtractorGraphComponent],
spacy_nlp: Any,
):
component_config = {
"features": [
["low", "title", "upper", "pos", "pos2"],
[
"low",
"suffix3",
"suffix2",
"upper",
"title",
"digit",
"pos",
"pos2",
"text_dense_features",
],
["low", "title", "upper", "pos", "pos2"],
]
}
crf_extractor = crf_entity_extractor(component_config)
spacy_featurizer = SpacyFeaturizer()
spacy_tokenizer = SpacyTokenizer()
text = "Rasa is a company in Berlin"
message = Message(data={TEXT: text})
message.set(SPACY_DOCS[TEXT], spacy_nlp(text))
spacy_tokenizer.process(message)
spacy_featurizer.process(message)
text_data = crf_extractor._convert_to_crf_tokens(message)
features = crf_extractor._crf_tokens_to_features(text_data)
assert "0:text_dense_features" in features[0]
dense_features, _ = message.get_dense_features(TEXT, [])
if dense_features:
dense_features = dense_features.features
for i in range(0, len(dense_features[0])):
assert (features[0]["0:text_dense_features"]["text_dense_features"][
str(i)] == dense_features[0][i])
def test_spacy_featurizer(sentence, spacy_nlp):
ftr = SpacyFeaturizer.create({}, RasaNLUModelConfig())
doc = spacy_nlp(sentence)
vecs = ftr._features_for_doc(doc)
expected = [t.vector for t in doc]
assert np.allclose(vecs, expected, atol=1e-5)
def test_spacy_featurizer(sentence, spacy_nlp):
from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer
ftr = SpacyFeaturizer.create({"return_sequence": True},
RasaNLUModelConfig())
doc = spacy_nlp(sentence)
vecs = ftr._features_for_doc(doc)
expected = [t.vector for t in doc]
assert np.allclose(vecs, expected, atol=1e-5)
def test_crf_use_dense_features(spacy_nlp: Any):
crf_extractor = CRFEntityExtractor(
component_config={
"features": [
["low", "title", "upper", "pos", "pos2"],
[
"low",
"suffix3",
"suffix2",
"upper",
"title",
"digit",
"pos",
"pos2",
"text_dense_features",
],
["low", "title", "upper", "pos", "pos2"],
]
}
)
spacy_featurizer = SpacyFeaturizer()
spacy_tokenizer = SpacyTokenizer()
text = "Rasa is a company in Berlin"
message = Message(text)
message.set(SPACY_DOCS[TEXT], spacy_nlp(text))
spacy_tokenizer.process(message)
spacy_featurizer.process(message)
text_data = crf_extractor._convert_to_crf_tokens(message)
features = crf_extractor._crf_tokens_to_features(text_data)
assert "0:text_dense_features" in features[0]
dense_features = message.get_dense_features(TEXT, [])
for i in range(0, len(dense_features[0])):
assert (
features[0]["0:text_dense_features"]["text_dense_features"][str(i)]
== dense_features[0][i]
)
def test_spacy_featurizer_cls_vector(spacy_nlp):
featurizer = SpacyFeaturizer.create({}, RasaNLUModelConfig())
sentence = "Hey how are you today"
message = Message(sentence)
message.set(SPACY_DOCS[TEXT_ATTRIBUTE], spacy_nlp(sentence))
featurizer._set_spacy_features(message)
vecs = message.get(DENSE_FEATURE_NAMES[TEXT_ATTRIBUTE])
expected = np.array([-0.28451, 0.31007, -0.57039, -0.073056, -0.17322])
expected_cls = np.array(
[-0.196496, 0.3249364, -0.37408298, -0.10622784, 0.062756])
assert 6 == len(vecs)
assert np.allclose(vecs[0][:5], expected, atol=1e-5)
assert np.allclose(vecs[-1][:5], expected_cls, atol=1e-5)
def test_spacy_featurizer_sequence(sentence, expected, spacy_nlp):
from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer
doc = spacy_nlp(sentence)
token_vectors = [t.vector for t in doc]
ftr = SpacyFeaturizer.create({}, RasaNLUModelConfig())
greet = {"intent": "greet", "text_features": [0.5]}
message = Message(sentence, greet)
message.set("spacy_doc", doc)
ftr._set_spacy_features(message)
vecs = message.get("text_dense_features")[0][:5]
assert np.allclose(token_vectors[0][:5], vecs, atol=1e-4)
assert np.allclose(vecs, expected, atol=1e-4)
def test_spacy_featurizer_using_empty_model():
from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer
import spacy
sentence = "This test is using an empty spaCy model"
model = spacy.blank("en")
doc = model(sentence)
ftr = SpacyFeaturizer.create({}, RasaNLUModelConfig())
message = Message(sentence)
message.set(SPACY_DOCS[TEXT], doc)
ftr._set_spacy_features(message)
vecs = message.get(DENSE_FEATURE_NAMES[TEXT])
assert vecs is None
def test_spacy_featurizer_no_sequence(sentence, expected, spacy_nlp):
from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer
doc = spacy_nlp(sentence)
spacy_config = {"return_sequence": False}
ftr = SpacyFeaturizer.create(spacy_config, RasaNLUModelConfig())
greet = {"intent": "greet", "text_features": [0.5]}
message = Message(sentence, greet)
message.set("spacy_doc", doc)
ftr._set_spacy_features(message)
vecs = message.get("text_dense_features")[0]
assert np.allclose(doc.vector, vecs, atol=1e-4)
assert np.allclose(expected, vecs[:5], atol=1e-4)
def test_spacy_featurizer_casing(spacy_nlp):
from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer
# if this starts failing for the default model, we should think about
# removing the lower casing the spacy nlp component does when it
# retrieves vectors. For compressed spacy models (e.g. models
# ending in _sm) this test will most likely fail.
ftr = SpacyFeaturizer.create({}, RasaNLUModelConfig())
td = training_data.load_data("data/examples/rasa/demo-rasa.json")
for e in td.intent_examples:
doc = spacy_nlp(e.text)
doc_capitalized = spacy_nlp(e.text.capitalize())
vecs = ftr._features_for_doc(doc)
vecs_capitalized = ftr._features_for_doc(doc_capitalized)
assert np.allclose(
vecs, vecs_capitalized,
atol=1e-5), "Vectors are unequal for texts '{}' and '{}'".format(
e.text, e.text.capitalize())
def test_spacy_featurizer_cls_vector(spacy_nlp):
featurizer = SpacyFeaturizer.create({}, RasaNLUModelConfig())
sentence = "Hey how are you today"
message = Message(data={TEXT: sentence})
message.set(SPACY_DOCS[TEXT], spacy_nlp(sentence))
featurizer._set_spacy_features(message)
seq_vecs, sen_vecs = message.get_dense_features(TEXT, [])
if seq_vecs:
seq_vecs = seq_vecs.features
if sen_vecs:
sen_vecs = sen_vecs.features
expected = np.array([-0.28451, 0.31007, -0.57039, -0.073056, -0.17322])
expected_cls = np.array([-0.196496, 0.3249364, -0.37408298, -0.10622784, 0.062756])
assert 5 == len(seq_vecs)
assert 1 == len(sen_vecs)
assert np.allclose(seq_vecs[0][:5], expected, atol=1e-5)
assert np.allclose(sen_vecs[-1][:5], expected_cls, atol=1e-5)
def test_spacy_featurizer_using_empty_model():
from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer
import spacy
sentence = "This test is using an empty spaCy model"
model = spacy.blank("en")
doc = model(sentence)
ftr = SpacyFeaturizer.create({}, RasaNLUModelConfig())
message = Message(data={TEXT: sentence})
message.set(SPACY_DOCS[TEXT], doc)
ftr._set_spacy_features(message)
seq_vecs, sen_vecs = message.get_dense_features(TEXT, [])
if seq_vecs:
seq_vecs = seq_vecs.features
if sen_vecs:
sen_vecs = sen_vecs.features
assert seq_vecs is None
assert sen_vecs is None
from rasa.nlu.tokenizers.mitie_tokenizer import MitieTokenizer
from rasa.nlu.featurizers.dense_featurizer.mitie_featurizer import MitieFeaturizer
from rasa.nlu.tokenizers.tokenizer import Tokenizer
from rasa.nlu.tokenizers.convert_tokenizer import ConveRTTokenizer
from rasa.nlu.featurizers.dense_featurizer.convert_featurizer import ConveRTFeaturizer
from rasa.nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer
from rasa.nlu.featurizers.sparse_featurizer.count_vectors_featurizer import CountVectorsFeaturizer
from rasa.nlu.constants import SPACY_DOCS, TEXT, DENSE_FEATURE_NAMES, TOKENS_NAMES, SPARSE_FEATURE_NAMES
logger = logging_setup()
featurizer = SpacyFeaturizer.create({}, RasaNLUModelConfig())
test_input = "Okay, pick up this yellow banana for me."
message = Message(test_input)
message.set(SPACY_DOCS[TEXT], spacy_nlp(test_input))
featurizer._set_spacy_features(message)
vecs = message.get(DENSE_FEATURE_NAMES[TEXT])
logger.info("SpaCy: {}".format(vecs.shape))
message = Message(test_input)
featurizer = MitieFeaturizer.create({}, RasaNLUModelConfig())
MitieTokenizer().process(message)
tokens = message.get(TOKENS_NAMES[TEXT])
vecs = featurizer.features_for_tokens(tokens, mitie_feature_extractor)
logger.info("Mitie: {}".format(vecs.shape))
def spacy_featurizer() -> SpacyFeaturizer:
return SpacyFeaturizer(SpacyFeaturizer.get_default_config(),
name="SpacyFeaturizer")