async def test_softmax_ranking(
component_builder: ComponentBuilder,
tmp_path: Path,
classifier_params: Dict[Text, int],
data_path: Text,
output_length: int,
):
pipeline = as_pipeline(
"WhitespaceTokenizer", "CountVectorsFeaturizer", "ResponseSelector"
)
assert pipeline[2]["name"] == "ResponseSelector"
pipeline[2].update(classifier_params)
_config = RasaNLUModelConfig({"pipeline": pipeline})
(trained_model, _, persisted_path) = await rasa.nlu.train.train(
_config,
path=str(tmp_path),
data=data_path,
component_builder=component_builder,
)
loaded = Interpreter.load(persisted_path, component_builder)
parse_data = loaded.parse("hello")
response_ranking = parse_data.get("response_selector").get("default").get("ranking")
# check that the output was correctly truncated after normalization
assert len(response_ranking) == output_length
async def _train_persist_load_with_different_settings(
pipeline: List[Dict[Text, Any]],
component_builder: ComponentBuilder,
tmp_path: Path,
should_finetune: bool,
):
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": "en"})
(trainer, trained, persisted_path) = await rasa.nlu.train.train(
_config,
path=str(tmp_path),
data="data/examples/rasa/demo-rasa.yml",
component_builder=component_builder,
)
assert trainer.pipeline
assert trained.pipeline
loaded = Interpreter.load(
persisted_path,
component_builder,
new_config=_config if should_finetune else None,
)
assert loaded.pipeline
assert loaded.parse("Rasa is great!") == trained.parse("Rasa is great!")
async def test_margin_loss_is_not_normalized(
monkeypatch: MonkeyPatch,
component_builder: ComponentBuilder,
tmp_path: Path,
classifier_params: Dict[Text, int],
):
pipeline = as_pipeline(
"WhitespaceTokenizer", "CountVectorsFeaturizer", "ResponseSelector"
)
assert pipeline[2]["name"] == "ResponseSelector"
pipeline[2].update(classifier_params)
mock = Mock()
monkeypatch.setattr(train_utils, "normalize", mock.normalize)
_config = RasaNLUModelConfig({"pipeline": pipeline})
(trained_model, _, persisted_path) = await rasa.nlu.train.train(
_config,
path=str(tmp_path),
data="data/test_selectors",
component_builder=component_builder,
)
loaded = Interpreter.load(persisted_path, component_builder)
parse_data = loaded.parse("hello")
response_ranking = parse_data.get("response_selector").get("default").get("ranking")
# check that the output was not normalized
mock.normalize.assert_not_called()
# check that the output was correctly truncated
assert len(response_ranking) == 9
def test_train_selector(pipeline, component_builder, tmpdir):
# use data that include some responses
td = load_data("data/examples/rasa/demo-rasa.md")
td_responses = load_data("data/examples/rasa/demo-rasa-responses.md")
td = td.merge(td_responses)
nlu_config = RasaNLUModelConfig({"language": "en", "pipeline": pipeline})
trainer = Trainer(nlu_config)
trainer.train(td)
persisted_path = trainer.persist(tmpdir)
assert trainer.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
parsed = loaded.parse("hello")
assert loaded.pipeline
assert parsed is not None
assert (parsed.get(RESPONSE_SELECTOR_PROPERTY_NAME).get("default").get(
"full_retrieval_intent")) is not None
ranking = parsed.get(RESPONSE_SELECTOR_PROPERTY_NAME).get("default").get(
"ranking")
assert ranking is not None
for rank in ranking:
assert rank.get("name") is not None
assert rank.get("confidence") is not None
assert rank.get("full_retrieval_intent") is not None
def test_train_selector(pipeline, component_builder, tmpdir):
# use data that include some responses
training_data = load_data("data/examples/rasa/demo-rasa.md")
training_data_responses = load_data(
"data/examples/rasa/demo-rasa-responses.md")
training_data = training_data.merge(training_data_responses)
nlu_config = RasaNLUModelConfig({"language": "en", "pipeline": pipeline})
trainer = Trainer(nlu_config)
trainer.train(training_data)
persisted_path = trainer.persist(tmpdir)
assert trainer.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
parsed = loaded.parse("hello")
assert loaded.pipeline
assert parsed is not None
assert (parsed.get("response_selector").get("all_retrieval_intents")) == [
"chitchat"
]
assert (parsed.get("response_selector").get("default").get("response").get(
"intent_response_key")) is not None
assert (parsed.get("response_selector").get("default").get("response").get(
"response_templates")) is not None
ranking = parsed.get("response_selector").get("default").get("ranking")
assert ranking is not None
for rank in ranking:
assert rank.get("confidence") is not None
assert rank.get("intent_response_key") is not None
async def test_process_gives_diagnostic_data(
trained_response_selector_bot: Path):
"""Tests if processing a message returns attention weights as numpy array."""
with rasa.model.unpack_model(
trained_response_selector_bot) as unpacked_model_directory:
_, nlu_model_directory = rasa.model.get_model_subdirectories(
unpacked_model_directory)
interpreter = Interpreter.load(nlu_model_directory)
message = Message(data={TEXT: "hello"})
for component in interpreter.pipeline:
component.process(message)
diagnostic_data = message.get(DIAGNOSTIC_DATA)
# The last component is ResponseSelector, which should add diagnostic data
name = f"component_{len(interpreter.pipeline) - 1}_ResponseSelector"
assert isinstance(diagnostic_data, dict)
assert name in diagnostic_data
assert "text_transformed" in diagnostic_data[name]
assert isinstance(diagnostic_data[name].get("text_transformed"),
np.ndarray)
# The `attention_weights` key should exist, regardless of there being a transformer
assert "attention_weights" in diagnostic_data[name]
# By default, ResponseSelector has `number_of_transformer_layers = 0`
assert diagnostic_data[name].get("attention_weights") is None
async def test_cross_entropy_without_normalization(
component_builder: ComponentBuilder,
tmp_path: Path,
classifier_params: Dict[Text, Any],
prediction_min: float,
prediction_max: float,
output_length: int,
monkeypatch: MonkeyPatch,
):
pipeline = as_pipeline("WhitespaceTokenizer", "CountVectorsFeaturizer",
"ResponseSelector")
assert pipeline[2]["name"] == "ResponseSelector"
pipeline[2].update(classifier_params)
_config = RasaNLUModelConfig({"pipeline": pipeline})
(trained_model, _, persisted_path) = await train(
_config,
path=str(tmp_path),
data="data/test_selectors",
component_builder=component_builder,
)
loaded = Interpreter.load(persisted_path, component_builder)
mock = Mock()
monkeypatch.setattr(train_utils, "normalize", mock.normalize)
parse_data = loaded.parse("hello")
response_ranking = parse_data.get("response_selector").get("default").get(
"ranking")
# check that the output was correctly truncated
assert len(response_ranking) == output_length
response_confidences = [
response.get("confidence") for response in response_ranking
]
# check each confidence is in range
confidence_in_range = [
prediction_min <= confidence <= prediction_max
for confidence in response_confidences
]
assert all(confidence_in_range)
# normalize shouldn't have been called
mock.normalize.assert_not_called()
def test_train_response_selector(component_builder, tmpdir):
td = load_data("data/examples/rasa/demo-rasa.md")
td_responses = load_data("data/examples/rasa/demo-rasa-responses.md")
td = td.merge(td_responses)
td.fill_response_phrases()
nlu_config = config.load(
"sample_configs/config_embedding_intent_response_selector.yml"
)
trainer = Trainer(nlu_config)
trainer.train(td)
persisted_path = trainer.persist(tmpdir)
assert trainer.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_selector(pipeline, component_builder, tmpdir):
# use data that include some responses
td = load_data("data/examples/rasa/demo-rasa.md")
td_responses = load_data("data/examples/rasa/demo-rasa-responses.md")
td = td.merge(td_responses)
td.fill_response_phrases()
nlu_config = RasaNLUModelConfig({"language": "en", "pipeline": pipeline})
trainer = Trainer(nlu_config)
trainer.train(td)
persisted_path = trainer.persist(tmpdir)
assert trainer.pipeline
loaded = Interpreter.load(persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
async def test_cross_entropy_with_linear_norm(
component_builder: ComponentBuilder,
tmp_path: Path,
classifier_params: Dict[Text, Any],
output_length: int,
monkeypatch: MonkeyPatch,
):
pipeline = as_pipeline("WhitespaceTokenizer", "CountVectorsFeaturizer",
"ResponseSelector")
assert pipeline[2]["name"] == "ResponseSelector"
pipeline[2].update(classifier_params)
_config = RasaNLUModelConfig({"pipeline": pipeline})
(trained_model, _, persisted_path) = await rasa.nlu.train.train(
_config,
path=str(tmp_path),
data="data/test_selectors",
component_builder=component_builder,
)
loaded = Interpreter.load(persisted_path, component_builder)
mock = Mock()
monkeypatch.setattr(train_utils, "normalize", mock.normalize)
parse_data = loaded.parse("hello")
response_ranking = parse_data.get("response_selector").get("default").get(
"ranking")
# check that the output was correctly truncated
assert len(response_ranking) == output_length
response_confidences = [
response.get("confidence") for response in response_ranking
]
# check whether normalization had the expected effect
output_sums_to_1 = sum(response_confidences) == pytest.approx(1)
assert output_sums_to_1
# normalize shouldn't have been called
mock.normalize.assert_not_called()
async def test_sparse_feature_sizes_decreased_incremental_training(
iter1_path: Text,
iter2_path: Text,
should_raise_exception: bool,
component_builder: ComponentBuilder,
tmpdir: Path,
):
pipeline = [
{
"name": "WhitespaceTokenizer"
},
{
"name": "LexicalSyntacticFeaturizer"
},
{
"name": "RegexFeaturizer"
},
{
"name": "CountVectorsFeaturizer"
},
{
"name": "CountVectorsFeaturizer",
"analyzer": "char_wb",
"min_ngram": 1,
"max_ngram": 4,
},
{
"name": "ResponseSelector",
EPOCHS: 1
},
]
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": "en"})
(_, trained, persisted_path) = await rasa.nlu.train.train(
_config,
path=str(tmpdir),
data=iter1_path,
component_builder=component_builder,
)
assert trained.pipeline
loaded = Interpreter.load(
persisted_path,
component_builder,
new_config=_config,
)
assert loaded.pipeline
assert loaded.parse("Rasa is great!") == trained.parse("Rasa is great!")
if should_raise_exception:
with pytest.raises(Exception) as exec_info:
(_, trained, _) = await rasa.nlu.train.train(
_config,
path=str(tmpdir),
data=iter2_path,
component_builder=component_builder,
model_to_finetune=loaded,
)
assert "Sparse feature sizes have decreased" in str(exec_info.value)
else:
(_, trained, _) = await rasa.nlu.train.train(
_config,
path=str(tmpdir),
data=iter2_path,
component_builder=component_builder,
model_to_finetune=loaded,
)
assert trained.pipeline
async def test_adjusting_layers_incremental_training(
component_builder: ComponentBuilder, tmpdir: Path):
"""Tests adjusting sparse layers of `ResponseSelector` to increased sparse
feature sizes during incremental training.
Testing is done by checking the layer sizes.
Checking if they were replaced correctly is also important
and is done in `test_replace_dense_for_sparse_layers`
in `test_rasa_layers.py`.
"""
iter1_data_path = "data/test_incremental_training/iter1/"
iter2_data_path = "data/test_incremental_training/"
pipeline = [
{
"name": "WhitespaceTokenizer"
},
{
"name": "LexicalSyntacticFeaturizer"
},
{
"name": "RegexFeaturizer"
},
{
"name": "CountVectorsFeaturizer"
},
{
"name": "CountVectorsFeaturizer",
"analyzer": "char_wb",
"min_ngram": 1,
"max_ngram": 4,
},
{
"name": "ResponseSelector",
EPOCHS: 1
},
]
_config = RasaNLUModelConfig({"pipeline": pipeline, "language": "en"})
(_, trained, persisted_path) = await rasa.nlu.train.train(
_config,
path=str(tmpdir),
data=iter1_data_path,
component_builder=component_builder,
)
assert trained.pipeline
old_data_signature = trained.pipeline[-1].model.data_signature
old_predict_data_signature = trained.pipeline[
-1].model.predict_data_signature
message = Message.build(text="Rasa is great!")
trained.featurize_message(message)
old_sparse_feature_sizes = message.get_sparse_feature_sizes(attribute=TEXT)
initial_rs_layers = (
trained.pipeline[-1].model._tf_layers["sequence_layer.text"].
_tf_layers["feature_combining"])
initial_rs_sequence_layer = initial_rs_layers._tf_layers[
"sparse_dense.sequence"]._tf_layers["sparse_to_dense"]
initial_rs_sentence_layer = initial_rs_layers._tf_layers[
"sparse_dense.sentence"]._tf_layers["sparse_to_dense"]
initial_rs_sequence_size = initial_rs_sequence_layer.get_kernel().shape[0]
initial_rs_sentence_size = initial_rs_sentence_layer.get_kernel().shape[0]
assert initial_rs_sequence_size == sum(
old_sparse_feature_sizes[FEATURE_TYPE_SEQUENCE])
assert initial_rs_sentence_size == sum(
old_sparse_feature_sizes[FEATURE_TYPE_SENTENCE])
loaded = Interpreter.load(
persisted_path,
component_builder,
new_config=_config,
)
assert loaded.pipeline
assert loaded.parse("Rasa is great!") == trained.parse("Rasa is great!")
(_, trained, _) = await rasa.nlu.train.train(
_config,
path=str(tmpdir),
data=iter2_data_path,
component_builder=component_builder,
model_to_finetune=loaded,
)
assert trained.pipeline
message = Message.build(text="Rasa is great!")
trained.featurize_message(message)
new_sparse_feature_sizes = message.get_sparse_feature_sizes(attribute=TEXT)
final_rs_layers = (
trained.pipeline[-1].model._tf_layers["sequence_layer.text"].
_tf_layers["feature_combining"])
final_rs_sequence_layer = final_rs_layers._tf_layers[
"sparse_dense.sequence"]._tf_layers["sparse_to_dense"]
final_rs_sentence_layer = final_rs_layers._tf_layers[
"sparse_dense.sentence"]._tf_layers["sparse_to_dense"]
final_rs_sequence_size = final_rs_sequence_layer.get_kernel().shape[0]
final_rs_sentence_size = final_rs_sentence_layer.get_kernel().shape[0]
assert final_rs_sequence_size == sum(
new_sparse_feature_sizes[FEATURE_TYPE_SEQUENCE])
assert final_rs_sentence_size == sum(
new_sparse_feature_sizes[FEATURE_TYPE_SENTENCE])
# check if the data signatures were correctly updated
new_data_signature = trained.pipeline[-1].model.data_signature
new_predict_data_signature = trained.pipeline[
-1].model.predict_data_signature
iter2_data = load_data(iter2_data_path)
expected_sequence_lengths = len([
message for message in iter2_data.training_examples
if message.get(INTENT_RESPONSE_KEY)
])
def test_data_signatures(
new_signature: Dict[Text, Dict[Text, List[FeatureArray]]],
old_signature: Dict[Text, Dict[Text, List[FeatureArray]]],
):
# Wherever attribute / feature_type signature is not
# expected to change, directly compare it to old data signature.
# Else compute its expected signature and compare
attributes_expected_to_change = [TEXT]
feature_types_expected_to_change = [
FEATURE_TYPE_SEQUENCE,
FEATURE_TYPE_SENTENCE,
]
for attribute, signatures in new_signature.items():
for feature_type, feature_signatures in signatures.items():
if feature_type == "sequence_lengths":
assert feature_signatures[
0].units == expected_sequence_lengths
elif feature_type not in feature_types_expected_to_change:
assert feature_signatures == old_signature.get(
attribute).get(feature_type)
else:
for index, feature_signature in enumerate(
feature_signatures):
if (feature_signature.is_sparse and attribute
in attributes_expected_to_change):
assert feature_signature.units == sum(
new_sparse_feature_sizes.get(feature_type))
else:
# dense signature or attributes that are not
# expected to change can be compared directly
assert (
feature_signature.units == old_signature.get(
attribute).get(feature_type)[index].units)
test_data_signatures(new_data_signature, old_data_signature)
test_data_signatures(new_predict_data_signature,
old_predict_data_signature)