def test_missing_access_fn() -> None:
"""
This test shows that the plugin needs an `access` provided or else it raises a type error.
"""
slot_filler = RuleBasedSlotFillerPlugin(rules=rules)()
workflow = Workflow(preprocessors=[], postprocessors=[slot_filler])
intent = Intent(name="intent", score=0.8)
body = "12th december"
entity = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
type="basic",
values=[{
"key": "value"
}],
slot_names=["basic_slot"],
)
workflow.output = (intent, [entity])
with pytest.raises(TypeError):
workflow.run("")
def test_workflow_set_output() -> None:
"""
We can set output and input as anything.
"""
workflow = Workflow(preprocessors=[], postprocessors=[])
workflow.output = 10
assert workflow.output == 10, "workflow.get_output() should == 10."
def test_duckling_timeout() -> None:
"""
[summary]
:return: [description]
:rtype: [type]
"""
locale = "en_IN"
wait_time = 0.1
def raise_timeout(_, __, headers):
time.sleep(wait_time)
return 200, headers, "received"
httpretty.register_uri(httpretty.POST,
"http://0.0.0.0:8000/parse",
body=raise_timeout)
duckling_plugin = DucklingPlugin(
locale=locale,
dimensions=["time"],
timezone="Asia/Kolkata",
threshold=0.2,
timeout=0.01,
dest="output.entities",
)
workflow = Workflow([duckling_plugin])
_, output = workflow.run(Input(utterances="test"))
assert output["entities"] == []
def test_workflow_load_model_error() -> None:
"""
`load_model` has to be defined by the subclass.
"""
workflow = Workflow(preprocessors=[], postprocessors=[])
with pytest.raises(NotImplementedError):
workflow.load_model()
def test_workflow_invalid_set_path():
"""
We can't set invalid values in workflow.
"""
workflow = Workflow()
with pytest.raises(ValueError):
workflow.set("invalid.path", [])
def test_workflow_invalid_set_value():
"""
We can't set invalid values in workflow.
"""
workflow = Workflow()
with pytest.raises(ValueError):
workflow.set("output.intents", 10)
def test_plugin_no_entities() -> None:
"""
An end-to-end example showing how `DucklingParser` works in case
the input has no entities.
"""
body = "i need it"
expected = []
def access(workflow):
return workflow.input, None
def mutate(workflow, entities):
workflow.output = {"entities": entities}
parser = DucklingParser(access=access,
mutate=mutate,
dimensions=["people"],
locale="en_IN")
request_callback = request_builder(expected)
httpretty.register_uri(httpretty.POST,
"http://0.0.0.0:8000/parse",
body=request_callback)
workflow = Workflow(preprocessors=[parser()], postprocessors=[])
workflow.run(body)
assert workflow.output["entities"] == []
def test_max_workers_greater_than_zero() -> None:
"""Checks that "ValueError: max_workers must be greater than 0" is not raised when there are no transcriptions
When we get an empty transcription from ASR in a production setup, FSM does not send the empty transcription to the SLU service.
Whereas in a development setup, when one tries to run `slu test` with atleast one data point that does not have any transcriptions(`[]`)
it will raise a `ValueError: max_workers must be greater than 0` exception.
The corresponding fix has been done and this test ensures that the exception is not raised when there are no transcriptions even in development setup
:return: None
:rtype: None
"""
locale = "en_IN"
duckling_plugin = DucklingPlugin(
dest="output.entities",
dimensions=["time"],
timezone="Asia/Kolkata",
url="https://duckling/parse",
)
workflow = Workflow([duckling_plugin])
alternatives = [] # When ASR returns empty transcriptions.
try:
workflow.run(Input(utterances=alternatives, locale=locale))
except ValueError as exc:
pytest.fail(f"{exc}")
def test_entity_type(payload) -> None:
"""
Evaluate a set of cases from a file.
"""
body = payload["input"]
mock_entity_json = payload["mock_entity_json"]
expected = payload.get("expected")
exception = payload.get("exception")
duckling_plugin = DucklingPlugin(
dest="output.entities",
dimensions=["people", "time", "date", "duration"],
locale="en_IN",
timezone="Asia/Kolkata",
)
request_callback = request_builder(mock_entity_json)
httpretty.register_uri(httpretty.POST,
"http://0.0.0.0:8000/parse",
body=request_callback)
workflow = Workflow([duckling_plugin])
if expected:
_, output = workflow.run(Input(utterances=body))
entities = output["entities"]
for i, entity in enumerate(entities):
assert entity["entity_type"] == expected[i]["entity_type"]
elif exception:
with pytest.raises(EXCEPTIONS[exception]):
workflow.run(Input(utterances=body))
def test_plugin_cases(payload) -> None:
"""
Test cases where the plugin should work.
"""
entities = payload.get("inputs", {}).get("entities", [])
tracker = payload.get("inputs", {}).get("tracker", [])
expected = payload.get("expected", {})
duckling_plugin = DucklingPlugin(dimensions=["date", "time"],
timezone="Asia/Kolkata",
dest="output.entities")
for i, entity in enumerate(entities):
current_turn_entities = duckling_plugin._reshape(entity, i)
combine_date_time_plugin = CombineDateTimeOverSlots(
trigger_intents=["_callback_"],
dest="output.entities",
)
workflow = Workflow(plugins=[combine_date_time_plugin])
workflow.output = Output(entities=current_turn_entities)
_, output = workflow.run(Input(utterances=[""], slot_tracker=tracker))
entity_values = [entity["value"] for entity in output[const.ENTITIES]]
if len(entity_values) != len(expected):
pytest.fail("Expected {} entities but got {}".format(
len(expected), len(entity_values)))
for entity_value, expected_value in zip(entity_values, expected):
try:
expected = datetime.fromisoformat(expected_value)
generated = datetime.fromisoformat(entity_value)
assert generated == expected, f"Expected {expected} but got {generated}"
except (ValueError, TypeError):
assert entity_value == expected_value
def test_incorrect_access_fn() -> None:
"""
This test shows that the plugin needs `access` function to be a `PluginFn`,
or else it throws a `TypeError`.
"""
rules = {"basic": {"slot_name": "basic_slot", "entity_type": "basic"}}
access = 5
slot_filler = RuleBasedSlotFillerPlugin(rules=rules, access=access)()
workflow = Workflow(preprocessors=[], postprocessors=[slot_filler])
intent = Intent(name="intent", score=0.8)
body = "12th december"
entity = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
type="basic",
values=[{
"key": "value"
}],
slot_names=["basic_slot"],
)
workflow.output = (intent, [entity])
with pytest.raises(TypeError):
workflow.run("")
def test_plugin_exit_at_missing_tracker():
combine_date_time_plugin = CombineDateTimeOverSlots(
trigger_intents=["_callback_"], dest="output.entities")
workflow = Workflow(plugins=[combine_date_time_plugin])
_, output = workflow.run(Input(utterances=[""]))
assert output[const.ENTITIES] == []
def test_slot_filling_multiple() -> None:
"""
Let's try filling both the slots this time with fill_multiple=True!
`intent_2` supports both `entity_1` and `entity_2`.
"""
def access(workflow: Workflow) -> Any:
return workflow.output
intent_name = "intent_2"
# Setting up the slot-filler, both instantiation and plugin is created. (notice two calls).
slot_filler = RuleBasedSlotFillerPlugin(rules=rules,
access=access,
fill_multiple=True)()
# Create a mock `workflow`
workflow = Workflow(preprocessors=[], postprocessors=[slot_filler])
# ... a mock `Intent`
intent = Intent(name=intent_name, score=0.8)
# and mock `Entity`-ies.
body = "12th december"
entity_1 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
type="entity_1",
values=[{
"key": "value"
}],
slot_names=["entity_1_slot"],
)
entity_2 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
type="entity_2",
values=[{
"key": "value"
}],
slot_names=["entity_2_slot"],
)
# The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`.
workflow.output = (intent, [entity_1, entity_2])
workflow.run(body)
# `workflow.output[0]` is the `Intent` we created.
# The `entity_1_slot` and `entity_2_slot` are filled.
assert workflow.output[0].slots["entity_1_slot"].values == [entity_1]
assert workflow.output[0].slots["entity_2_slot"].values == [entity_2]
def test_slot_filling_multiple() -> None:
"""
Let's try filling both the slots this time with fill_multiple=True!
`intent_2` supports both `entity_1` and `entity_2`.
"""
intent_name = "intent_2"
# Setting up the slot-filler, both instantiation and plugin is created. (notice two calls).
slot_filler = RuleBasedSlotFillerPlugin(rules=rules,
dest="output.intents",
fill_multiple=True)
# Create a mock `workflow`
workflow = Workflow([slot_filler])
# ... a mock `Intent`
intent = Intent(name=intent_name, score=0.8)
# and mock `Entity`-ies.
body = "12th december"
entity_1 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_1",
values=[{
"key": "value"
}],
)
entity_2 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_2",
values=[{
"key": "value"
}],
)
# The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`.
workflow.set("output.intents", [intent]).set("output.entities",
[entity_1, entity_2])
_, output = workflow.run(Input(utterances=body))
# `workflow.output[0]` is the `Intent` we created.
# The `entity_1_slot` and `entity_2_slot` are filled.
assert output[const.INTENTS][0]["slots"]["entity_1_slot"]["values"] == [
entity_1.json()
]
assert output[const.INTENTS][0]["slots"]["entity_2_slot"]["values"] == [
entity_2.json()
]
def test_plugin_no_set_on_invalid_output():
arbitrary_plugin = ArbitraryPlugin(
dest="output.intents",
guards=[lambda i, _: i.current_state == "COF"],
)
workflow = Workflow()
workflow.input = Input(utterances="hello")
workflow.output = None
assert arbitrary_plugin(workflow) is None
def test_slot_competition_fill_multiple() -> None:
"""
What happens when we have two entities of the same type but different value?
"""
intent_name = "intent_1"
# Setting up the slot-filler, both instantiation and plugin is created. (notice two calls).
slot_filler = RuleBasedSlotFillerPlugin(rules=rules,
dest="output.intents",
fill_multiple=True)
# Create a mock `workflow`
workflow = Workflow([slot_filler])
# ... a mock `Intent`
intent = Intent(name=intent_name, score=0.8)
# Here we have two entities which compete for the same slot but have different values.
body = "12th december"
entity_1 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_1",
values=[{
"key": "value_1"
}],
)
entity_2 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
entity_type="entity_1",
values=[{
"key": "value_2"
}],
)
workflow.set("output.intents", [intent]).set("output.entities",
[entity_1, entity_2])
_, output = workflow.run(Input(utterances=body))
# `workflow.output[0]` is the `Intent` we created.
# The `entity_1_slot` and `entity_2_slot` are filled.
assert output[const.INTENTS][0]["slots"]["entity_1_slot"]["values"] == [
entity_1.json(),
entity_2.json(),
]
def test_slot_competition() -> None:
"""
What happens when we have two entities of the same type but different value?
"""
def access(workflow: Workflow) -> Any:
return workflow.output
intent_name = "intent_1"
# Setting up the slot-filler, both instantiation and plugin is created. (notice two calls).
slot_filler = RuleBasedSlotFillerPlugin(rules=rules, access=access)()
# Create a mock `workflow`
workflow = Workflow(preprocessors=[], postprocessors=[slot_filler])
# ... a mock `Intent`
intent = Intent(name=intent_name, score=0.8)
# Here we have two entities which compete for the same slot but have different values.
body = "12th december"
entity_1 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
type="entity_1",
values=[{
"key": "value_1"
}],
slot_names=["entity_1_slot"],
)
entity_2 = BaseEntity(
range={
"from": 0,
"to": len(body)
},
body=body,
dim="default",
type="entity_1",
values=[{
"key": "value_2"
}],
slot_names=["entity_1_slot"],
)
# The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`.
workflow.output = (intent, [entity_1, entity_2])
workflow.run(body)
# `workflow.output[0]` is the `Intent` we created.
# The `entity_1_slot` and `entity_2_slot` are filled.
assert "entity_1_slot" not in workflow.output[0].slots
def test_voting_0_intents():
"""
The code uses division. So its always good to
have a test to see if it takes care of division 0.
"""
intents: List[Intent] = []
vote_plugin = VotePlugin(access=lambda w: (w.output[0], 0), mutate=update_intent)()
workflow = Workflow(preprocessors=[], postprocessors=[vote_plugin])
workflow.output = intents, []
intent, _ = workflow.run(input_="")
assert intent.name == const.S_INTENT_OOS
def test_voting_0_intents():
"""
The code uses division. So its always good to
have a test to see if it takes care of division 0.
"""
intents: List[Intent] = []
vote_plugin = VotePlugin(dest="output.intents")
workflow = Workflow([vote_plugin])
workflow.output = Output(intents=intents)
_, output = workflow.run(Input(utterances=["some text"]))
assert output["intents"][0]["name"] == const.S_INTENT_OOS
def test_merge_asr_output() -> None:
"""
This case shows the merge in case there is only one option.
"""
workflow = Workflow(
preprocessors=[merge_asr_output_plugin(access, mutate)], postprocessors=[]
)
workflow.run([[{"transcript": "hello world", "confidence": None}]])
assert workflow.input == "<s> hello world </s>"
def test_merge_keyerror_on_missing_transcript() -> None:
"""
This test, shows that `transcript` is an important key. If the asr has a different key, than `transcript`
then this plugin would not work for you.
"""
workflow = Workflow(
preprocessors=[merge_asr_output_plugin(access, mutate)], postprocessors=[]
)
with pytest.raises(TypeError):
workflow.run([[{"not_transcript": "hello world", "confidence": None}]])
def test_workflow_history_logs() -> None:
"""
We can execute the workflow.
"""
workflow = Workflow(
[MergeASROutputPlugin(dest="input.clf_feature", debug=True)],
debug=True,
)
input_, _ = workflow.run(Input(utterances=["apples"]))
assert input_["clf_feature"] == ["<s> apples </s>"]
assert workflow.input == None
assert workflow.output == Output()
def test_plugin_io_type_mismatch(access, mutate) -> None:
"""
Here we are chcking if the plugin has access to workflow.
Since we have provided `access`, `mutate` of incorrect types to `DucklingParser`
we will receive a `TypeError`.
"""
parser = DucklingParser(access=access, mutate=mutate, locale="en_IN")
plugin = parser()
workflow = Workflow(preprocessors=[plugin], postprocessors=[])
with pytest.raises(TypeError):
workflow.run("")
def test_workflow_as_dict():
"""
We can serialize a workflow.
"""
workflow = Workflow()
assert workflow.json() == {
"input": None,
"output": {
const.INTENTS: [],
const.ENTITIES: []
},
}
def test_tokenizer_plugin() -> None:
"""
We will test the tokenizer plugin.
"""
# create an instance of a `Workflow`.
# we are calling the `arbitrary_plugin` to get the `plugin` de method.
workflow = Workflow(
preprocessors=[tokenizer_plugin(access=access, mutate=mutate)],
postprocessors=[],
)
workflow.run("Mary had a little lambda")
assert workflow.output == ["Mary", "had", "a", "little", "lambda"]
def test_missing_access():
intents = [
Intent(name="a", score=0.3),
Intent(name="a", score=0.2),
Intent(name="b", score=0.1),
Intent(name="b", score=0.1),
]
vote_plugin = VotePlugin(mutate=update_intent)()
workflow = Workflow(preprocessors=[], postprocessors=[vote_plugin])
workflow.output = intents, []
with pytest.raises(TypeError):
intent, _ = workflow.run(input_="")
def test_plugin_working_cases(payload) -> None:
"""
An end-to-end example showing how to use `DucklingPlugin` with a `Workflow`.
"""
body = payload["input"]
mock_entity_json = payload["mock_entity_json"]
expected_types = payload.get("expected")
exception = payload.get("exception")
duckling_args = payload.get("duckling")
response_code = payload.get("response_code", 200)
locale = payload.get("locale")
reference_time = payload.get("reference_time")
use_latent = payload.get("use_latent")
duckling_plugin = DucklingPlugin(dest="output.entities", **duckling_args)
request_callback = request_builder(mock_entity_json,
response_code=response_code)
httpretty.register_uri(httpretty.POST,
"http://0.0.0.0:8000/parse",
body=request_callback)
workflow = Workflow([duckling_plugin])
if isinstance(reference_time, str):
reference_time = make_unix_ts("Asia/Kolkata")(reference_time)
if expected_types is not None:
input_ = Input(
utterances=body,
locale=locale,
reference_time=reference_time,
latent_entities=use_latent,
)
_, output = workflow.run(input_)
if not output["entities"]:
assert output["entities"] == []
for i, entity in enumerate(output["entities"]):
expected_entity_type = expected_types[i]["entity_type"]
assert entity["entity_type"] == expected_entity_type
else:
with pytest.raises(EXCEPTIONS[exception]):
input_ = Input(
utterances=body,
locale=locale,
reference_time=reference_time,
latent_entities=use_latent,
)
workflow.run(input_)
def test_representation_oos():
intents = [
Intent(name="a", score=0.99),
Intent(name="b", score=0.1),
Intent(name="b", score=0.4),
Intent(name="b", score=0.31),
Intent(name="d", score=0.44),
]
vote_plugin = VotePlugin(dest="output.intents")
workflow = Workflow([vote_plugin])
workflow.output = Output(intents=intents)
_, output = workflow.run(Input(utterances=["some text"]))
assert output["intents"][0]["name"] == "_oos_"
def test_representation_oos():
intents = [
Intent(name="a", score=0.99),
Intent(name="b", score=0.1),
Intent(name="c", score=0.31),
Intent(name="d", score=0.44),
]
vote_plugin = VotePlugin(
access=lambda w: (w.output[0], len(intents)), mutate=update_intent
)()
workflow = Workflow(preprocessors=[], postprocessors=[vote_plugin])
workflow.output = intents, []
intent, _ = workflow.run(input_="")
assert intent.name == "_oos_"
def test_voting_on_weak_signals():
"""
Testing all weak intents.
"""
intents = [
Intent(name="a", score=0.3),
Intent(name="a", score=0.2),
Intent(name="b", score=0.1),
Intent(name="b", score=0.1),
]
vote_plugin = VotePlugin(dest="output.intents")
workflow = Workflow([vote_plugin])
workflow.output = Output(intents=intents)
_, output = workflow.run(Input(utterances=["some text"]))
assert output["intents"][0]["name"] == "_oos_"
