def inner(config: Optional[Dict[Text, Any]] = None) -> CountVectorsFeaturizer:
config = config or {}
return CountVectorsFeaturizer.create(
{**CountVectorsFeaturizer.get_default_config(), **config},
default_model_storage,
Resource("count_vectors_featurizer"),
default_execution_context,
)
def inner(
config: Optional[Dict[Text, Any]] = None, is_finetuning: bool = False
) -> CountVectorsFeaturizer:
config = config or {}
return CountVectorsFeaturizer.load(
{**CountVectorsFeaturizer.get_default_config(), **config},
default_model_storage,
Resource("count_vectors_featurizer"),
dataclasses.replace(default_execution_context, is_finetuning=is_finetuning),
)
def featurizer_sparse(tmpdir):
"""Generate a featurizer for tests."""
node_storage = LocalModelStorage(pathlib.Path(tmpdir))
node_resource = Resource("sparse_feat")
context = ExecutionContext(node_storage, node_resource)
return CountVectorsFeaturizer(
config=CountVectorsFeaturizer.get_default_config(),
resource=node_resource,
model_storage=node_storage,
execution_context=context,
)
def test_convert_training_examples(
spacy_nlp: Language,
text: Text,
intent: Optional[Text],
entities: Optional[List[Dict[Text, Any]]],
attributes: List[Text],
real_sparse_feature_sizes: Dict[Text, Dict[Text, List[int]]],
default_model_storage: ModelStorage,
default_execution_context: ExecutionContext,
):
message = Message(data={TEXT: text, INTENT: intent, ENTITIES: entities})
tokenizer = SpacyTokenizer.create(
SpacyTokenizer.get_default_config(),
default_model_storage,
Resource("tokenizer"),
default_execution_context,
)
count_vectors_featurizer = CountVectorsFeaturizer.create(
CountVectorsFeaturizer.get_default_config(),
default_model_storage,
Resource("count_featurizer"),
default_execution_context,
)
spacy_featurizer = SpacyFeaturizer.create(
SpacyFeaturizer.get_default_config(),
default_model_storage,
Resource("spacy_featurizer"),
default_execution_context,
)
message.set(SPACY_DOCS[TEXT], spacy_nlp(text))
training_data = TrainingData([message])
tokenizer.process_training_data(training_data)
count_vectors_featurizer.train(training_data)
count_vectors_featurizer.process_training_data(training_data)
spacy_featurizer.process_training_data(training_data)
entity_tag_spec = [
EntityTagSpec(
"entity",
{
0: "O",
1: "name",
2: "location"
},
{
"O": 0,
"name": 1,
"location": 2
},
3,
)
]
output, sparse_feature_sizes = model_data_utils.featurize_training_examples(
[message], attributes=attributes, entity_tag_specs=entity_tag_spec)
assert len(output) == 1
for attribute in attributes:
assert attribute in output[0]
for attribute in {INTENT, TEXT, ENTITIES} - set(attributes):
assert attribute not in output[0]
# we have sparse sentence, sparse sequence, dense sentence, and dense sequence
# features in the list
assert len(output[0][TEXT]) == 4
if INTENT in attributes:
# we will just have space sentence features
assert len(output[0][INTENT]) == 1
if ENTITIES in attributes:
# we will just have space sentence features
assert len(output[0][ENTITIES]) == len(entity_tag_spec)
# check that it calculates sparse_feature_sizes correctly
assert sparse_feature_sizes == real_sparse_feature_sizes