def test_entity_type(payload) -> None:
"""
Evaluate a set of cases from a file.
"""
body = payload["input"]
mock_entity_json = payload["mock_entity_json"]
expected = payload.get("expected")
exception = payload.get("exception")
duckling_plugin = DucklingPlugin(
dest="output.entities",
dimensions=["people", "time", "date", "duration"],
locale="en_IN",
timezone="Asia/Kolkata",
)
request_callback = request_builder(mock_entity_json)
httpretty.register_uri(httpretty.POST,
"http://0.0.0.0:8000/parse",
body=request_callback)
workflow = Workflow([duckling_plugin])
if expected:
_, output = workflow.run(Input(utterances=body))
entities = output["entities"]
for i, entity in enumerate(entities):
assert entity["entity_type"] == expected[i]["entity_type"]
elif exception:
with pytest.raises(EXCEPTIONS[exception]):
workflow.run(Input(utterances=body))
def test_input_extension():
instance = Input(utterances="test", reference_time=1644238676772)
extended = Input.from_dict({
"utterances": "test",
"reference_time": 1644238676772
})
assert instance == extended
def test_get_list_entities(payload):
input_ = payload.get("input")
lang_ = payload.get("lang")
expected = payload.get("expected")
exception = payload.get("exception")
config = payload.get("config")
transcripts = [expectation["text"] for expectation in input_]
if expected:
list_entity_plugin = ListSearchPlugin(dest="output.entities", **config)
workflow = Workflow([list_entity_plugin])
_, output = workflow.run(Input(utterances=transcripts, lang=lang_))
entities = output["entities"]
if not entities and expected:
pytest.fail("No entities found!")
for i, entity in enumerate(entities):
assert entity["value"] == expected[i]["value"]
assert entity["type"] == expected[i]["type"]
if "score" in expected[i]:
assert entity["score"] == expected[i]["score"]
else:
with pytest.raises(EXCEPTIONS.get(exception)):
list_entity_plugin = ListSearchPlugin(dest="output.entities",
**config)
workflow = Workflow([list_entity_plugin])
workflow.run(Input(utterances=transcripts, lang=lang_))
def test_plugin_working_cases(payload) -> None:
"""
An end-to-end example showing how to use `DucklingPlugin` with a `Workflow`.
"""
body = payload["input"]
mock_entity_json = payload["mock_entity_json"]
expected_types = payload.get("expected")
exception = payload.get("exception")
duckling_args = payload.get("duckling")
response_code = payload.get("response_code", 200)
locale = payload.get("locale")
reference_time = payload.get("reference_time")
use_latent = payload.get("use_latent")
duckling_plugin = DucklingPlugin(dest="output.entities", **duckling_args)
request_callback = request_builder(mock_entity_json,
response_code=response_code)
httpretty.register_uri(httpretty.POST,
"http://0.0.0.0:8000/parse",
body=request_callback)
workflow = Workflow([duckling_plugin])
if isinstance(reference_time, str):
reference_time = make_unix_ts("Asia/Kolkata")(reference_time)
if expected_types is not None:
input_ = Input(
utterances=body,
locale=locale,
reference_time=reference_time,
latent_entities=use_latent,
)
_, output = workflow.run(input_)
if not output["entities"]:
assert output["entities"] == []
for i, entity in enumerate(output["entities"]):
expected_entity_type = expected_types[i]["entity_type"]
assert entity["entity_type"] == expected_entity_type
else:
with pytest.raises(EXCEPTIONS[exception]):
input_ = Input(
utterances=body,
locale=locale,
reference_time=reference_time,
latent_entities=use_latent,
)
workflow.run(input_)
def test_calibration_model_utility():
input_ = Input(utterances=[[{
"transcript": "hello",
"am_score": -100,
"lm_score": -200
}]])
assert calibration_model.utility(input_, Output()) == ["hello"]
calibration_model.threshold = float("inf")
input_ = Input(utterances=[[{
"transcript": "hello world hello world",
"am_score": -100,
"lm_score": -200,
}]])
assert calibration_model.utility(input_,
Output()) == ["hello world hello world"]
def test_plugin_exit_at_missing_tracker():
combine_date_time_plugin = CombineDateTimeOverSlots(
trigger_intents=["_callback_"], dest="output.entities")
workflow = Workflow(plugins=[combine_date_time_plugin])
_, output = workflow.run(Input(utterances=[""]))
assert output[const.ENTITIES] == []
def test_duckling_timeout() -> None:
"""
[summary]
:return: [description]
:rtype: [type]
"""
locale = "en_IN"
wait_time = 0.1
def raise_timeout(_, __, headers):
time.sleep(wait_time)
return 200, headers, "received"
httpretty.register_uri(httpretty.POST,
"http://0.0.0.0:8000/parse",
body=raise_timeout)
duckling_plugin = DucklingPlugin(
locale=locale,
dimensions=["time"],
timezone="Asia/Kolkata",
threshold=0.2,
timeout=0.01,
dest="output.entities",
)
workflow = Workflow([duckling_plugin])
_, output = workflow.run(Input(utterances="test"))
assert output["entities"] == []
def test_max_workers_greater_than_zero() -> None:
"""Checks that "ValueError: max_workers must be greater than 0" is not raised when there are no transcriptions
When we get an empty transcription from ASR in a production setup, FSM does not send the empty transcription to the SLU service.
Whereas in a development setup, when one tries to run `slu test` with atleast one data point that does not have any transcriptions(`[]`)
it will raise a `ValueError: max_workers must be greater than 0` exception.
The corresponding fix has been done and this test ensures that the exception is not raised when there are no transcriptions even in development setup
:return: None
:rtype: None
"""
locale = "en_IN"
duckling_plugin = DucklingPlugin(
dest="output.entities",
dimensions=["time"],
timezone="Asia/Kolkata",
url="https://duckling/parse",
)
workflow = Workflow([duckling_plugin])
alternatives = [] # When ASR returns empty transcriptions.
try:
workflow.run(Input(utterances=alternatives, locale=locale))
except ValueError as exc:
pytest.fail(f"{exc}")
def test_plugin_cases(payload) -> None:
"""
Test cases where the plugin should work.
"""
entities = payload.get("inputs", {}).get("entities", [])
tracker = payload.get("inputs", {}).get("tracker", [])
expected = payload.get("expected", {})
duckling_plugin = DucklingPlugin(dimensions=["date", "time"],
timezone="Asia/Kolkata",
dest="output.entities")
for i, entity in enumerate(entities):
current_turn_entities = duckling_plugin._reshape(entity, i)
combine_date_time_plugin = CombineDateTimeOverSlots(
trigger_intents=["_callback_"],
dest="output.entities",
)
workflow = Workflow(plugins=[combine_date_time_plugin])
workflow.output = Output(entities=current_turn_entities)
_, output = workflow.run(Input(utterances=[""], slot_tracker=tracker))
entity_values = [entity["value"] for entity in output[const.ENTITIES]]
if len(entity_values) != len(expected):
pytest.fail("Expected {} entities but got {}".format(
len(expected), len(entity_values)))
for entity_value, expected_value in zip(entity_values, expected):
try:
expected = datetime.fromisoformat(expected_value)
generated = datetime.fromisoformat(entity_value)
assert generated == expected, f"Expected {expected} but got {generated}"
except (ValueError, TypeError):
assert entity_value == expected_value
def test_slot_invalid_intent() -> None:
"""
Here, we will see that an entity will not fill an intent unless the intent has a slot for it.
`intent_1` doesn't have a slot for an entity of type `entity_2`.
"""
intent_name = "intent_1"
# ... a mock `Intent`
intent = Intent(name=intent_name, score=0.8)
# Setting up the slot-filler, both instantiation and plugin is created. (notice two calls).
slot_filler = RuleBasedSlotFillerPlugin(rules=rules, dest="output.intents")
# Create a mock `workflow`
workflow = Workflow([slot_filler])
# and a mock `Entity`.
body = "12th december"
entity = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_1",
values=[{
"key": "value"
}],
)
# The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`.
workflow.set("output.intents", [1]).set("output.entities", [entity])
with pytest.raises(AttributeError):
workflow.run(Input(utterances=body))
def test_slot_filling_multiple() -> None:
"""
Let's try filling both the slots this time with fill_multiple=True!
`intent_2` supports both `entity_1` and `entity_2`.
"""
intent_name = "intent_2"
# Setting up the slot-filler, both instantiation and plugin is created. (notice two calls).
slot_filler = RuleBasedSlotFillerPlugin(rules=rules,
dest="output.intents",
fill_multiple=True)
# Create a mock `workflow`
workflow = Workflow([slot_filler])
# ... a mock `Intent`
intent = Intent(name=intent_name, score=0.8)
# and mock `Entity`-ies.
body = "12th december"
entity_1 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_1",
values=[{
"key": "value"
}],
)
entity_2 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_2",
values=[{
"key": "value"
}],
)
# The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`.
workflow.set("output.intents", [intent]).set("output.entities",
[entity_1, entity_2])
_, output = workflow.run(Input(utterances=body))
# `workflow.output[0]` is the `Intent` we created.
# The `entity_1_slot` and `entity_2_slot` are filled.
assert output[const.INTENTS][0]["slots"]["entity_1_slot"]["values"] == [
entity_1.json()
]
assert output[const.INTENTS][0]["slots"]["entity_2_slot"]["values"] == [
entity_2.json()
]
def test_plugin_no_set_on_invalid_output():
arbitrary_plugin = ArbitraryPlugin(
dest="output.intents",
guards=[lambda i, _: i.current_state == "COF"],
)
workflow = Workflow()
workflow.input = Input(utterances="hello")
workflow.output = None
assert arbitrary_plugin(workflow) is None
def test_slot_competition_fill_multiple() -> None:
"""
What happens when we have two entities of the same type but different value?
"""
intent_name = "intent_1"
# Setting up the slot-filler, both instantiation and plugin is created. (notice two calls).
slot_filler = RuleBasedSlotFillerPlugin(rules=rules,
dest="output.intents",
fill_multiple=True)
# Create a mock `workflow`
workflow = Workflow([slot_filler])
# ... a mock `Intent`
intent = Intent(name=intent_name, score=0.8)
# Here we have two entities which compete for the same slot but have different values.
body = "12th december"
entity_1 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_1",
values=[{
"key": "value_1"
}],
)
entity_2 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_1",
values=[{
"key": "value_2"
}],
)
workflow.set("output.intents", [intent]).set("output.entities",
[entity_1, entity_2])
_, output = workflow.run(Input(utterances=body))
# `workflow.output[0]` is the `Intent` we created.
# The `entity_1_slot` and `entity_2_slot` are filled.
assert output[const.INTENTS][0]["slots"]["entity_1_slot"]["values"] == [
entity_1.json(),
entity_2.json(),
]
def test_voting_0_intents():
"""
The code uses division. So its always good to
have a test to see if it takes care of division 0.
"""
intents: List[Intent] = []
vote_plugin = VotePlugin(dest="output.intents")
workflow = Workflow([vote_plugin])
workflow.output = Output(intents=intents)
_, output = workflow.run(Input(utterances=["some text"]))
assert output["intents"][0]["name"] == const.S_INTENT_OOS
def test_workflow_history_logs() -> None:
"""
We can execute the workflow.
"""
workflow = Workflow(
[MergeASROutputPlugin(dest="input.clf_feature", debug=True)],
debug=True,
)
input_, _ = workflow.run(Input(utterances=["apples"]))
assert input_["clf_feature"] == ["<s> apples </s>"]
assert workflow.input == None
assert workflow.output == Output()
def test_representation_oos():
intents = [
Intent(name="a", score=0.99),
Intent(name="b", score=0.1),
Intent(name="b", score=0.4),
Intent(name="b", score=0.31),
Intent(name="d", score=0.44),
]
vote_plugin = VotePlugin(dest="output.intents")
workflow = Workflow([vote_plugin])
workflow.output = Output(intents=intents)
_, output = workflow.run(Input(utterances=["some text"]))
assert output["intents"][0]["name"] == "_oos_"
def test_get_list_entities(payload):
input_ = payload.get("input")
expected = payload.get("expected")
exception = payload.get("exception")
config = payload.get("config")
spacy_mocker = None
transcripts = [expectation["text"] for expectation in input_]
if config["style"] == "spacy":
spacy_mocker = SpacyMocker(input_)
if expected:
list_entity_plugin = ListEntityPlugin(dest="output.entities",
spacy_nlp=spacy_mocker,
**config)
workflow = Workflow([list_entity_plugin])
print(transcripts)
_, output = workflow.run(input_=Input(utterances=transcripts))
entities = output["entities"]
if not entities and expected:
pytest.fail("No entities found!")
for i, entity in enumerate(entities):
assert entity["value"] == expected[i]["value"]
assert entity["type"] == expected[i]["type"]
if "score" in expected[i]:
assert entity["score"] == expected[i]["score"]
else:
with pytest.raises(EXCEPTIONS.get(exception)):
list_entity_plugin = ListEntityPlugin(dest="output.entities",
spacy_nlp=spacy_mocker,
**config)
workflow = Workflow([list_entity_plugin])
_, output = workflow.run(input_=Input(utterances=transcripts))
def test_voting_on_weak_signals():
"""
Testing all weak intents.
"""
intents = [
Intent(name="a", score=0.3),
Intent(name="a", score=0.2),
Intent(name="b", score=0.1),
Intent(name="b", score=0.1),
]
vote_plugin = VotePlugin(dest="output.intents")
workflow = Workflow([vote_plugin])
workflow.output = Output(intents=intents)
_, output = workflow.run(Input(utterances=["some text"]))
assert output["intents"][0]["name"] == "_oos_"
def test_voting_on_conflicts():
"""
Testing the case with conflicts.
"""
intents = [
Intent(name="a", score=1),
Intent(name="a", score=1),
Intent(name="b", score=1),
Intent(name="b", score=1),
]
vote_plugin = VotePlugin(dest="output.intents")
workflow = Workflow([vote_plugin])
workflow.output = Output(intents=intents)
_, output = workflow.run(Input(utterances=["some text"]))
assert output["intents"][0]["name"] == "_oos_"
def test_voting_n_intents():
"""
Testing the usual case.
"""
intents = [
Intent(name="a", score=1),
Intent(name="a", score=1),
Intent(name="b", score=0.13),
Intent(name="a", score=1),
]
vote_plugin = VotePlugin(
debug=False,
dest="output.intents",
)
workflow = Workflow([vote_plugin])
workflow.output = Output(intents=intents)
_, output = workflow.run(Input(utterances=["some text"]))
assert output["intents"][0]["name"] == "a"
def test_arbitrary_plugin() -> None:
"""
We will test how an arbitrary-class-based plugin works with a workflow.
"""
# create an instance of `ArbitraryPlugin`.
arbitrary_plugin = ArbitraryPlugin(dest="output.intents")
# create an instance of a `Workflow`.
# we are calling the `arbitrary_plugin` to get the `plugin` de method.
workflow = Workflow([arbitrary_plugin])
input_ = Input(utterances=[[{"transcript": "hello"}]])
# This runs all the `preprocessors` and `postprocessors` provided previously.
# we can expect our `arbitrary_plugin` will also be used.
_, output = workflow.run(input_)
first_intent, *rest = output["intents"]
# This test would pass only if our plugin works correctly!
assert first_intent["name"] == "_greeting_"
assert rest == []
def test_aggregate_fn_incorrect():
intents = [
Intent(name="a", score=0.99),
Intent(name="a", score=0.99),
Intent(name="a", score=0.91),
Intent(name="b", score=0.1),
Intent(name="c", score=0.31),
Intent(name="d", score=0.44),
]
vote_plugin = VotePlugin(
dest="output.intents",
aggregate_fn=5,
)
workflow = Workflow([vote_plugin])
workflow.output = Output(intents=intents)
with pytest.raises(TypeError):
_, output = workflow.run(Input(utterances=[""]))
assert output["intents"][0]["name"] == "a"
def test_duckling_connection_error() -> None:
"""
[summary]
:return: [description]
:rtype: [type]
"""
locale = "en_IN"
duckling_plugin = DucklingPlugin(
locale=locale,
dimensions=["time"],
timezone="Asia/Kolkata",
dest="output.entities",
threshold=0.2,
timeout=0.01,
url="https://duckling/parse",
)
workflow = Workflow([duckling_plugin])
_, output = workflow.run(Input(utterances="test", locale=locale))
assert output["entities"] == []
def test_slot_no_fill() -> None:
"""
Here, we will see that an entity will not fill an intent unless the intent has a slot for it.
`intent_1` doesn't have a slot for an entity of type `entity_2`.
"""
intent_name = "intent_1"
# Setting up the slot-filler, both instantiation and plugin is created. (notice two calls).
slot_filler = RuleBasedSlotFillerPlugin(rules=rules, dest="output.intents")
# Create a mock `workflow`
workflow = Workflow([slot_filler])
# ... a mock `Intent`
intent = Intent(name=intent_name, score=0.8)
# and a mock `Entity`.
body = "12th december"
entity = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_2",
values=[{
"key": "value"
}],
)
# The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`.
workflow.set("output.intents", [intent]).set("output.entities", [entity])
_, output = workflow.run(Input(utterances=body))
# `workflow.output[0]` is the `Intent` we created.
# we can see that the `entity_2_slot` is not filled by our mock entity.
assert "entity_1_slot" not in output[const.INTENTS][0]["slots"]
def test_slot_filling() -> None:
"""
This test case covers a trivial usage of a slot-filler.
We have `rules` that demonstrate association of intents with entities and their respective slot-configuration.
"""
intent_name = "intent_1"
slot_filler = RuleBasedSlotFillerPlugin(rules=rules, dest="output.intents")
# Create a mock `workflow`
workflow = Workflow([slot_filler])
# ... a mock `Intent`
intent = Intent(name=intent_name, score=0.8)
# and a mock `Entity`.
body = "12th december"
entity = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_1",
values=[{
"key": "value"
}],
)
# The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`.
workflow.set("output.intents", [intent]).set("output.entities", [entity])
_, output = workflow.run(Input(utterances=body))
intent, *_ = output[const.INTENTS]
# `workflow.output[0]` is the `Intent` we created.
# so we are checking if the `entity_1_slot` is filled by our mock entity.
assert intent["slots"]["entity_1_slot"]["values"][0] == entity.json()
def test_inference(payload):
save_module_name = const.XLMR_MODULE
save_model_name = const.XLMR_MULTI_CLASS_MODEL
const.XLMR_MODULE = "tests.plugin.text.classification.test_xlmr"
const.XLMR_MULTI_CLASS_MODEL = "MockClassifier"
directory = "/tmp"
file_path = os.path.join(directory, const.LABELENCODER_FILE)
if os.path.exists(file_path):
os.remove(file_path)
transcripts = payload.get("input")
intent = payload["expected"]["label"]
xlmr_clf = XLMRMultiClass(
model_dir=directory,
dest="output.intents",
debug=False,
)
merge_asr_output_plugin = MergeASROutputPlugin(dest="input.clf_feature",
debug=False)
workflow = Workflow([merge_asr_output_plugin, xlmr_clf])
train_df = pd.DataFrame([
{
"data": json.dumps([[{
"transcript": "yes"
}]]),
"labels": "_confirm_",
},
{
"data": json.dumps([[{
"transcript": "yea"
}]]),
"labels": "_confirm_",
},
{
"data": json.dumps([[{
"transcript": "no"
}]]),
"labels": "_cancel_",
},
{
"data": json.dumps([[{
"transcript": "nope"
}]]),
"labels": "_cancel_",
},
])
workflow.train(train_df)
assert isinstance(
xlmr_clf.model,
MockClassifier), "model should be a MockClassifier after training."
_, output = workflow.run(input_=Input(utterances=[[{
"transcript": transcript
} for transcript in transcripts]]))
assert output[const.INTENTS][0]["name"] == intent
assert output[const.INTENTS][0]["score"] > 0.9
if os.path.exists(file_path):
os.remove(file_path)
const.XLMR_MODULE = save_module_name
const.XLMR_MULTI_CLASS_MODEL = save_model_name
def test_inference(payload):
directory = "/tmp"
file_path = os.path.join(directory, const.MLPMODEL_FILE)
if os.path.exists(file_path):
os.remove(file_path)
USE = "use"
fake_args = {
const.TRAIN: {
const.NUM_TRAIN_EPOCHS: 5,
const.USE_GRIDSEARCH: {
USE: False,
const.CV: 2,
const.VERBOSE_LEVEL: 2,
const.PARAMS: {
"activation": ["relu", "tanh"],
"hidden_layer_sizes": [(10, ), (2, 2)],
"ngram_range": [(1, 1), (1, 2)],
"max_iter": [20, 2],
},
},
},
const.TEST: {},
const.PRODUCTION: {},
}
transcripts = payload.get("input")
intent = payload["expected"]["label"]
mlp_clf = MLPMultiClass(
model_dir=directory,
dest="output.intents",
args_map=fake_args,
debug=False,
)
merge_asr_output_plugin = MergeASROutputPlugin(
dest="input.clf_feature",
debug=False,
)
workflow = Workflow([merge_asr_output_plugin, mlp_clf])
train_df = pd.DataFrame([
{
"data": json.dumps([[{
"transcript": "yes"
}]]),
"labels": "_confirm_",
},
{
"data": json.dumps([[{
"transcript": "ye"
}]]),
"labels": "_confirm_",
},
{
"data": json.dumps([[{
"transcript": "<s> yes </s> <s> ye </s>"
}]]),
"labels": "_confirm_",
},
{
"data": json.dumps([[{
"transcript": "no"
}]]),
"labels": "_cancel_",
},
{
"data": json.dumps([[{
"transcript": "new"
}]]),
"labels": "_cancel_",
},
{
"data": json.dumps([[{
"transcript": "<s> new </s> <s> no </s>"
}]]),
"labels": "_cancel_",
},
])
workflow.train(train_df)
_, output = workflow.run(
Input(utterances=[[{
"transcript": transcript
} for transcript in transcripts]]))
assert output[const.INTENTS][0]["name"] == intent
assert output[const.INTENTS][0]["score"] > 0.5
if os.path.exists(file_path):
os.remove(file_path)
def test_canonicalization_utility():
input_ = Input(utterances=[[{"transcript": "hello apple"}]])
input_, _ = workflow.run(input_)
assert input_["clf_feature"] == ["MASK <fruits>"]
def test_invalid_reftime():
with pytest.raises(ValueError):
Input(utterances="test", reference_time=18**15)
