def test_samples(backend_name, spacy_nlp_en):
interpreter = utilities.interpreter_for(
spacy_nlp_en, utilities.base_test_conf(backend_name))
available_intents = ["greet", "restaurant_search", "affirm", "goodbye"]
samples = [(u"good bye", {
'intent': 'goodbye',
'entities': []
}),
(u"i am looking for an indian spot", {
'intent':
'restaurant_search',
'entities': [{
"start": 20,
"end": 26,
"value": "indian",
"entity": "cuisine"
}]
})]
for text, gold in samples:
result = interpreter.parse(text)
assert result['text'] == text, \
"Wrong text for sample '{}'".format(text)
assert result['intent'] in available_intents, \
"Wrong intent for sample '{}'".format(text)
assert result['confidence'] >= 0, \
"Low confidence for sample '{}'".format(text)
# This ensures the model doesn't detect entities that are not present
# Models on our test data set are not stable enough to require the entities to be found
for entity in result['entities']:
assert entity in gold['entities'], \
"Wrong entities for sample '{}'".format(text)
def test_train_model(pipeline_template, component_builder):
_config = utilities.base_test_conf(pipeline_template)
(trained, persisted_path) = utilities.run_train(_config, component_builder)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path,
component_builder)
assert loaded.pipeline
def test_train_model(pipeline_template, component_builder):
_config = utilities.base_test_conf(pipeline_template)
(trained, persisted_path) = utilities.run_train(_config, component_builder)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello", time=None) is not None
def test_train_named_model(component_builder):
_config = utilities.base_test_conf("keyword")
_config['name'] = "my_keyword_model"
(trained, persisted_path) = utilities.run_train(_config, component_builder)
assert trained.pipeline
assert persisted_path.strip("/\\").endswith(
"my_keyword_model") # should be saved in a dir named after model
def test_samples(backend_name):
interpreter = utilities.interpreter_for(
utilities.base_test_conf(backend_name))
available_intents = ["greet", "restaurant_search", "affirm", "goodbye"]
samples = [(u"good bye", {
'intent': 'goodbye',
'entities': []
}),
(u"i am looking for an indian spot", {
'intent':
'restaurant_search',
'entities': [{
"start": 20,
"end": 26,
"value": "indian",
"entity": "cuisine"
}]
})]
for text, gold in samples:
result = interpreter.parse(text)
assert result['text'] == text, \
"Wrong text for sample '{}'".format(text)
assert result['intent'] in available_intents, \
"Wrong intent for sample '{}'".format(text)
assert result['confidence'] >= 0, \
"Low confidence for sample '{}'".format(text)
assert result['entities'] == gold['entities'], \
"Wrong entities for sample '{}'".format(text)
def test_train_model_noents(pipeline_template, interpreter_builder):
_config = utilities.base_test_conf(pipeline_template)
_config['data'] = "./data/examples/rasa/demo-rasa-noents.json"
(trained, persisted_path) = utilities.run_train(_config)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path,
interpreter_builder)
assert loaded.pipeline
def test_load_and_persist_without_train(component_builder):
_config = utilities.base_test_conf("all_components")
trainer = Trainer(_config, component_builder)
persistor = create_persistor(_config)
persisted_path = trainer.persist(_config['path'], persistor, model_name=_config['name'])
loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello", time=None) is not None
def test_duckling_entity_extractor(component_builder):
_config = utilities.base_test_conf("all_components")
_config["duckling_dimensions"] = ["time"]
duckling = component_builder.create_component("ner_duckling", _config)
message = Message("Today is the 5th of May. Let us meet tomorrow.")
duckling.process(message)
entities = message.get("entities")
assert len(entities) == 3
def test_train_model_multithread(component_builder):
_config = utilities.base_test_conf("all_components")
_config['num_threads'] = 2
(trained, persisted_path) = utilities.run_train(_config, component_builder)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello", time=None) is not None
def test_train_model_noents(component_builder):
_config = utilities.base_test_conf("all_components")
_config['data'] = "./data/examples/rasa/demo-rasa-noents.json"
(trained, persisted_path) = utilities.run_train(_config, component_builder)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello", time=None) is not None
def test_train_model(pipeline_template, component_builder):
_config = utilities.base_test_conf(pipeline_template)
(trained, persisted_path) = utilities.run_train(_config, component_builder)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model_multithread(pipeline_template, interpreter_builder):
_config = utilities.base_test_conf(pipeline_template)
_config['num_threads'] = 2
(trained, persisted_path) = utilities.run_train(_config)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path,
interpreter_builder)
assert loaded.pipeline
def test_train_model_noents(component_builder):
_config = utilities.base_test_conf("all_components")
_config['data'] = "./data/examples/rasa/demo-rasa-noents.json"
(trained, persisted_path) = utilities.run_train(_config, component_builder)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path,
component_builder)
assert loaded.pipeline
assert loaded.parse("hello", time=None) is not None
def test_load_and_persist_without_train(component_builder):
_config = utilities.base_test_conf("all_components")
trainer = Trainer(_config, component_builder)
persistor = create_persistor(_config)
persisted_path = trainer.persist(_config['path'], persistor, model_name=_config['name'])
loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model_noents(component_builder):
_config = utilities.base_test_conf("all_components")
_config['data'] = "./data/examples/mynlu/demo-mynlu-noents.json"
(trained, persisted_path) = utilities.run_train(_config, component_builder)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model_multithread(component_builder):
_config = utilities.base_test_conf("all_components")
_config['num_threads'] = 2
(trained, persisted_path) = utilities.run_train(_config, component_builder)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path,
component_builder)
assert loaded.pipeline
assert loaded.parse("hello", time=None) is not None
def test_train_with_empty_data(component_builder):
_config = utilities.base_test_conf("all_components")
trainer = Trainer(_config, component_builder)
trainer.train(TrainingData())
persistor = create_persistor(_config)
persisted_path = trainer.persist(_config['path'], persistor, model_name=_config['name'])
loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
assert loaded.pipeline
assert loaded.parse("hello") is not None
assert loaded.parse("Hello today is Monday, again!") is not None
def test_duckling_entity_extractor_and_synonyms(component_builder):
_config = utilities.base_test_conf("all_components")
_config["duckling_dimensions"] = ["number"]
duckling = component_builder.create_component("ner_duckling", _config)
synonyms = component_builder.create_component("ner_synonyms", _config)
message = Message("He was 6 feet away")
duckling.process(message)
synonyms.process(
message
) # checks that the synonym processor can handle entities that have int values
assert message is not None
def test_train_with_empty_data(component_builder):
_config = utilities.base_test_conf("all_components")
trainer = Trainer(_config, component_builder)
trainer.train(TrainingData())
persistor = create_persistor(_config)
persisted_path = trainer.persist(_config['path'],
persistor,
model_name=_config['name'])
loaded = utilities.load_interpreter_for_model(_config, persisted_path,
component_builder)
assert loaded.pipeline
assert loaded.parse("hello", time=None) is not None
def test_train_spacy_sklearn_finetune_ner(interpreter_builder):
_config = utilities.base_test_conf("spacy_sklearn")
_config['fine_tune_spacy_ner'] = True
(trained, persisted_path) = utilities.run_train(_config)
assert trained.pipeline
loaded = utilities.load_interpreter_for_model(_config, persisted_path,
interpreter_builder)
result = loaded.parse(u"I am living in New York now.")
entities = result['entities']
# Although the model is trained on restaurant entities, we can use the entities (`GPE`, `DATE`)
# from spacy since we are fine tuning. This should even be the case if the rasa-entity training data changes!
assert {
u'start': 15,
u'end': 23,
u'value': u'New York',
u'entity': u'GPE'
} in entities
def test_duckling_entity_extractor(component_builder):
_config = utilities.base_test_conf("all_components")
_config["duckling_dimensions"] = ["time"]
duckling = component_builder.create_component("ner_duckling", _config)
message = Message("Today is the 5th of May. Let us meet tomorrow.")
duckling.process(message)
entities = message.get("entities")
assert len(entities) == 3
# Test duckling with a defined date
message = Message(
"Let us meet tomorrow.",
time="1381536182000") # 1381536182000 == 2013/10/12 02:03:02
duckling.process(message)
entities = message.get("entities")
assert len(entities) == 1
assert entities[0]["text"] == "tomorrow"
assert entities[0]["value"] == "2013-10-13T00:00:00.000Z"
def test_samples(pipeline_template, component_builder):
_conf = utilities.base_test_conf(pipeline_template)
_conf["data"] = "./data/examples/rasa/demo-rasa.json"
interpreter = utilities.interpreter_for(component_builder, _conf)
available_intents = [
"greet", "restaurant_search", "affirm", "goodbye", "None"
]
samples = [("good bye", {
'intent': 'goodbye',
'entities': []
}),
("i am looking for an indian spot", {
'intent':
'restaurant_search',
'entities': [{
"start": 20,
"end": 26,
"value": "indian",
"entity": "cuisine"
}]
})]
for text, gold in samples:
result = interpreter.parse(text)
assert result['text'] == text, \
"Wrong text for sample '{}'".format(text)
assert result['intent']['name'] in available_intents, \
"Wrong intent for sample '{}'".format(text)
assert result['intent']['confidence'] >= 0, \
"Low confidence for sample '{}'".format(text)
# This ensures the model doesn't detect entities that are not present
# Models on our test data set are not stable enough to require the entities to be found
for entity in result['entities']:
del entity["extractor"]
assert entity in gold['entities'], \
"Wrong entities for sample '{}'".format(text)
def test_handles_pipeline_with_non_existing_component(component_builder):
_config = utilities.base_test_conf("spacy_sklearn")
_config['pipeline'].append("my_made_up_component")
with pytest.raises(Exception) as execinfo:
utilities.run_train(_config, component_builder)
assert "Failed to find component" in str(execinfo.value)
def test_handles_pipeline_with_non_existing_component(component_builder):
_config = utilities.base_test_conf("spacy_sklearn")
_config['pipeline'].append("my_made_up_component")
with pytest.raises(Exception) as execinfo:
utilities.run_train(_config, component_builder)
assert "Failed to find component" in str(execinfo.value)
def test_train_model_empty_pipeline(component_builder):
_config = utilities.base_test_conf(
pipeline_template=None) # Should return an empty pipeline
with pytest.raises(ValueError):
utilities.run_train(_config, component_builder)
def test_train_model_empty_pipeline(component_builder):
_config = utilities.base_test_conf(pipeline_template=None) # Should return an empty pipeline
with pytest.raises(ValueError):
utilities.run_train(_config, component_builder)
def test_train_named_model(component_builder):
_config = utilities.base_test_conf("keyword")
_config['name'] = "my_keyword_model"
(trained, persisted_path) = utilities.run_train(_config, component_builder)
assert trained.pipeline
assert persisted_path.strip("/\\").endswith("my_keyword_model") # should be saved in a dir named after model
