def test_get_list_entities(payload): input_ = payload.get("input") lang_ = payload.get("lang") expected = payload.get("expected") exception = payload.get("exception") config = payload.get("config") transcripts = [expectation["text"] for expectation in input_] if expected: list_entity_plugin = ListSearchPlugin(dest="output.entities", **config) workflow = Workflow([list_entity_plugin]) _, output = workflow.run(Input(utterances=transcripts, lang=lang_)) entities = output["entities"] if not entities and expected: pytest.fail("No entities found!") for i, entity in enumerate(entities): assert entity["value"] == expected[i]["value"] assert entity["type"] == expected[i]["type"] if "score" in expected[i]: assert entity["score"] == expected[i]["score"] else: with pytest.raises(EXCEPTIONS.get(exception)): list_entity_plugin = ListSearchPlugin(dest="output.entities", **config) workflow = Workflow([list_entity_plugin]) workflow.run(Input(utterances=transcripts, lang=lang_))
def test_plugin_exit_at_missing_tracker(): combine_date_time_plugin = CombineDateTimeOverSlots( trigger_intents=["_callback_"], dest="output.entities") workflow = Workflow(plugins=[combine_date_time_plugin]) _, output = workflow.run(Input(utterances=[""])) assert output[const.ENTITIES] == []
def test_plugin_no_set_on_invalid_input(): arbitrary_plugin = ArbitraryPlugin( dest="output.intents", guards=[lambda i, _: i.current_state == "COF"], ) workflow = Workflow() assert arbitrary_plugin(workflow) is None
def test_max_workers_greater_than_zero() -> None: """Checks that "ValueError: max_workers must be greater than 0" is not raised when there are no transcriptions When we get an empty transcription from ASR in a production setup, FSM does not send the empty transcription to the SLU service. Whereas in a development setup, when one tries to run `slu test` with atleast one data point that does not have any transcriptions(`[]`) it will raise a `ValueError: max_workers must be greater than 0` exception. The corresponding fix has been done and this test ensures that the exception is not raised when there are no transcriptions even in development setup :return: None :rtype: None """ locale = "en_IN" duckling_plugin = DucklingPlugin( dest="output.entities", dimensions=["time"], timezone="Asia/Kolkata", url="https://duckling/parse", ) workflow = Workflow([duckling_plugin]) alternatives = [] # When ASR returns empty transcriptions. try: workflow.run(Input(utterances=alternatives, locale=locale)) except ValueError as exc: pytest.fail(f"{exc}")
def test_workflow_load_model_error() -> None: """ `load_model` has to be defined by the subclass. """ workflow = Workflow(preprocessors=[], postprocessors=[]) with pytest.raises(NotImplementedError): workflow.load_model()
def test_missing_access_fn() -> None: """ This test shows that the plugin needs an `access` provided or else it raises a type error. """ slot_filler = RuleBasedSlotFillerPlugin(rules=rules)() workflow = Workflow(preprocessors=[], postprocessors=[slot_filler]) intent = Intent(name="intent", score=0.8) body = "12th december" entity = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", type="basic", values=[{ "key": "value" }], slot_names=["basic_slot"], ) workflow.output = (intent, [entity]) with pytest.raises(TypeError): workflow.run("")
def test_workflow_set_output() -> None: """ We can set output and input as anything. """ workflow = Workflow(preprocessors=[], postprocessors=[]) workflow.output = 10 assert workflow.output == 10, "workflow.get_output() should == 10."
def test_slot_invalid_intent() -> None: """ Here, we will see that an entity will not fill an intent unless the intent has a slot for it. `intent_1` doesn't have a slot for an entity of type `entity_2`. """ intent_name = "intent_1" # ... a mock `Intent` intent = Intent(name=intent_name, score=0.8) # Setting up the slot-filler, both instantiation and plugin is created. (notice two calls). slot_filler = RuleBasedSlotFillerPlugin(rules=rules, dest="output.intents") # Create a mock `workflow` workflow = Workflow([slot_filler]) # and a mock `Entity`. body = "12th december" entity = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", entity_type="entity_1", values=[{ "key": "value" }], ) # The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`. workflow.set("output.intents", [1]).set("output.entities", [entity]) with pytest.raises(AttributeError): workflow.run(Input(utterances=body))
def test_incorrect_access_fn() -> None: """ This test shows that the plugin needs `access` function to be a `PluginFn`, or else it throws a `TypeError`. """ rules = {"basic": {"slot_name": "basic_slot", "entity_type": "basic"}} access = 5 slot_filler = RuleBasedSlotFillerPlugin(rules=rules, access=access)() workflow = Workflow(preprocessors=[], postprocessors=[slot_filler]) intent = Intent(name="intent", score=0.8) body = "12th december" entity = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", type="basic", values=[{ "key": "value" }], slot_names=["basic_slot"], ) workflow.output = (intent, [entity]) with pytest.raises(TypeError): workflow.run("")
def test_duckling_timeout() -> None: """ [summary] :return: [description] :rtype: [type] """ locale = "en_IN" wait_time = 0.1 def raise_timeout(_, __, headers): time.sleep(wait_time) return 200, headers, "received" httpretty.register_uri(httpretty.POST, "http://0.0.0.0:8000/parse", body=raise_timeout) duckling_plugin = DucklingPlugin( locale=locale, dimensions=["time"], timezone="Asia/Kolkata", threshold=0.2, timeout=0.01, dest="output.entities", ) workflow = Workflow([duckling_plugin]) _, output = workflow.run(Input(utterances="test")) assert output["entities"] == []
def test_workflow_invalid_set_value(): """ We can't set invalid values in workflow. """ workflow = Workflow() with pytest.raises(ValueError): workflow.set("output.intents", 10)
def test_workflow_invalid_set_path(): """ We can't set invalid values in workflow. """ workflow = Workflow() with pytest.raises(ValueError): workflow.set("invalid.path", [])
def test_plugin_no_entities() -> None: """ An end-to-end example showing how `DucklingParser` works in case the input has no entities. """ body = "i need it" expected = [] def access(workflow): return workflow.input, None def mutate(workflow, entities): workflow.output = {"entities": entities} parser = DucklingParser(access=access, mutate=mutate, dimensions=["people"], locale="en_IN") request_callback = request_builder(expected) httpretty.register_uri(httpretty.POST, "http://0.0.0.0:8000/parse", body=request_callback) workflow = Workflow(preprocessors=[parser()], postprocessors=[]) workflow.run(body) assert workflow.output["entities"] == []
def test_plugin_cases(payload) -> None: """ Test cases where the plugin should work. """ entities = payload.get("inputs", {}).get("entities", []) tracker = payload.get("inputs", {}).get("tracker", []) expected = payload.get("expected", {}) duckling_plugin = DucklingPlugin(dimensions=["date", "time"], timezone="Asia/Kolkata", dest="output.entities") for i, entity in enumerate(entities): current_turn_entities = duckling_plugin._reshape(entity, i) combine_date_time_plugin = CombineDateTimeOverSlots( trigger_intents=["_callback_"], dest="output.entities", ) workflow = Workflow(plugins=[combine_date_time_plugin]) workflow.output = Output(entities=current_turn_entities) _, output = workflow.run(Input(utterances=[""], slot_tracker=tracker)) entity_values = [entity["value"] for entity in output[const.ENTITIES]] if len(entity_values) != len(expected): pytest.fail("Expected {} entities but got {}".format( len(expected), len(entity_values))) for entity_value, expected_value in zip(entity_values, expected): try: expected = datetime.fromisoformat(expected_value) generated = datetime.fromisoformat(entity_value) assert generated == expected, f"Expected {expected} but got {generated}" except (ValueError, TypeError): assert entity_value == expected_value
def test_entity_type(payload) -> None: """ Evaluate a set of cases from a file. """ body = payload["input"] mock_entity_json = payload["mock_entity_json"] expected = payload.get("expected") exception = payload.get("exception") duckling_plugin = DucklingPlugin( dest="output.entities", dimensions=["people", "time", "date", "duration"], locale="en_IN", timezone="Asia/Kolkata", ) request_callback = request_builder(mock_entity_json) httpretty.register_uri(httpretty.POST, "http://0.0.0.0:8000/parse", body=request_callback) workflow = Workflow([duckling_plugin]) if expected: _, output = workflow.run(Input(utterances=body)) entities = output["entities"] for i, entity in enumerate(entities): assert entity["entity_type"] == expected[i]["entity_type"] elif exception: with pytest.raises(EXCEPTIONS[exception]): workflow.run(Input(utterances=body))
def test_slot_filling_multiple() -> None: """ Let's try filling both the slots this time with fill_multiple=True! `intent_2` supports both `entity_1` and `entity_2`. """ def access(workflow: Workflow) -> Any: return workflow.output intent_name = "intent_2" # Setting up the slot-filler, both instantiation and plugin is created. (notice two calls). slot_filler = RuleBasedSlotFillerPlugin(rules=rules, access=access, fill_multiple=True)() # Create a mock `workflow` workflow = Workflow(preprocessors=[], postprocessors=[slot_filler]) # ... a mock `Intent` intent = Intent(name=intent_name, score=0.8) # and mock `Entity`-ies. body = "12th december" entity_1 = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", type="entity_1", values=[{ "key": "value" }], slot_names=["entity_1_slot"], ) entity_2 = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", type="entity_2", values=[{ "key": "value" }], slot_names=["entity_2_slot"], ) # The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`. workflow.output = (intent, [entity_1, entity_2]) workflow.run(body) # `workflow.output[0]` is the `Intent` we created. # The `entity_1_slot` and `entity_2_slot` are filled. assert workflow.output[0].slots["entity_1_slot"].values == [entity_1] assert workflow.output[0].slots["entity_2_slot"].values == [entity_2]
def test_slot_filling_multiple() -> None: """ Let's try filling both the slots this time with fill_multiple=True! `intent_2` supports both `entity_1` and `entity_2`. """ intent_name = "intent_2" # Setting up the slot-filler, both instantiation and plugin is created. (notice two calls). slot_filler = RuleBasedSlotFillerPlugin(rules=rules, dest="output.intents", fill_multiple=True) # Create a mock `workflow` workflow = Workflow([slot_filler]) # ... a mock `Intent` intent = Intent(name=intent_name, score=0.8) # and mock `Entity`-ies. body = "12th december" entity_1 = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", entity_type="entity_1", values=[{ "key": "value" }], ) entity_2 = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", entity_type="entity_2", values=[{ "key": "value" }], ) # The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`. workflow.set("output.intents", [intent]).set("output.entities", [entity_1, entity_2]) _, output = workflow.run(Input(utterances=body)) # `workflow.output[0]` is the `Intent` we created. # The `entity_1_slot` and `entity_2_slot` are filled. assert output[const.INTENTS][0]["slots"]["entity_1_slot"]["values"] == [ entity_1.json() ] assert output[const.INTENTS][0]["slots"]["entity_2_slot"]["values"] == [ entity_2.json() ]
def test_plugin_guards() -> None: arbitrary_plugin = ArbitraryPlugin( dest="output.intents", guards=[lambda i, _: i.current_state == "COF"], ) workflow = (Workflow().set("input.utterances", ["hello"]).set("input.current_state", "COF")) assert arbitrary_plugin.prevent(workflow.input, workflow.output) is True assert arbitrary_plugin(workflow) is None
def test_not_callable_pre_post_processors_type_error() -> None: """""" workflow = Workflow(preprocessors=[2], postprocessors=[3]) # type: ignore with pytest.raises(TypeError): workflow.preprocess() with pytest.raises(TypeError): workflow.postprocess()
def test_plugin_no_set_on_invalid_output(): arbitrary_plugin = ArbitraryPlugin( dest="output.intents", guards=[lambda i, _: i.current_state == "COF"], ) workflow = Workflow() workflow.input = Input(utterances="hello") workflow.output = None assert arbitrary_plugin(workflow) is None
def test_slot_competition_fill_multiple() -> None: """ What happens when we have two entities of the same type but different value? """ intent_name = "intent_1" # Setting up the slot-filler, both instantiation and plugin is created. (notice two calls). slot_filler = RuleBasedSlotFillerPlugin(rules=rules, dest="output.intents", fill_multiple=True) # Create a mock `workflow` workflow = Workflow([slot_filler]) # ... a mock `Intent` intent = Intent(name=intent_name, score=0.8) # Here we have two entities which compete for the same slot but have different values. body = "12th december" entity_1 = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", entity_type="entity_1", values=[{ "key": "value_1" }], ) entity_2 = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", entity_type="entity_1", values=[{ "key": "value_2" }], ) workflow.set("output.intents", [intent]).set("output.entities", [entity_1, entity_2]) _, output = workflow.run(Input(utterances=body)) # `workflow.output[0]` is the `Intent` we created. # The `entity_1_slot` and `entity_2_slot` are filled. assert output[const.INTENTS][0]["slots"]["entity_1_slot"]["values"] == [ entity_1.json(), entity_2.json(), ]
def test_slot_competition() -> None: """ What happens when we have two entities of the same type but different value? """ def access(workflow: Workflow) -> Any: return workflow.output intent_name = "intent_1" # Setting up the slot-filler, both instantiation and plugin is created. (notice two calls). slot_filler = RuleBasedSlotFillerPlugin(rules=rules, access=access)() # Create a mock `workflow` workflow = Workflow(preprocessors=[], postprocessors=[slot_filler]) # ... a mock `Intent` intent = Intent(name=intent_name, score=0.8) # Here we have two entities which compete for the same slot but have different values. body = "12th december" entity_1 = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", type="entity_1", values=[{ "key": "value_1" }], slot_names=["entity_1_slot"], ) entity_2 = BaseEntity( range={ "from": 0, "to": len(body) }, body=body, dim="default", type="entity_1", values=[{ "key": "value_2" }], slot_names=["entity_1_slot"], ) # The RuleBasedSlotFillerPlugin specifies that it expects `Tuple[Intent, List[Entity])` on `access(workflow)`. workflow.output = (intent, [entity_1, entity_2]) workflow.run(body) # `workflow.output[0]` is the `Intent` we created. # The `entity_1_slot` and `entity_2_slot` are filled. assert "entity_1_slot" not in workflow.output[0].slots
def test_voting_0_intents(): """ The code uses division. So its always good to have a test to see if it takes care of division 0. """ intents: List[Intent] = [] vote_plugin = VotePlugin(access=lambda w: (w.output[0], 0), mutate=update_intent)() workflow = Workflow(preprocessors=[], postprocessors=[vote_plugin]) workflow.output = intents, [] intent, _ = workflow.run(input_="") assert intent.name == const.S_INTENT_OOS
def test_voting_0_intents(): """ The code uses division. So its always good to have a test to see if it takes care of division 0. """ intents: List[Intent] = [] vote_plugin = VotePlugin(dest="output.intents") workflow = Workflow([vote_plugin]) workflow.output = Output(intents=intents) _, output = workflow.run(Input(utterances=["some text"])) assert output["intents"][0]["name"] == const.S_INTENT_OOS
def test_merge_asr_output() -> None: """ This case shows the merge in case there is only one option. """ workflow = Workflow( preprocessors=[merge_asr_output_plugin(access, mutate)], postprocessors=[] ) workflow.run([[{"transcript": "hello world", "confidence": None}]]) assert workflow.input == "<s> hello world </s>"
def test_merge_keyerror_on_missing_transcript() -> None: """ This test, shows that `transcript` is an important key. If the asr has a different key, than `transcript` then this plugin would not work for you. """ workflow = Workflow( preprocessors=[merge_asr_output_plugin(access, mutate)], postprocessors=[] ) with pytest.raises(TypeError): workflow.run([[{"not_transcript": "hello world", "confidence": None}]])
def test_workflow_history_logs() -> None: """ We can execute the workflow. """ workflow = Workflow( [MergeASROutputPlugin(dest="input.clf_feature", debug=True)], debug=True, ) input_, _ = workflow.run(Input(utterances=["apples"])) assert input_["clf_feature"] == ["<s> apples </s>"] assert workflow.input == None assert workflow.output == Output()
def test_workflow_as_dict(): """ We can serialize a workflow. """ workflow = Workflow() assert workflow.json() == { "input": None, "output": { const.INTENTS: [], const.ENTITIES: [] }, }
def test_plugin_io_type_mismatch(access, mutate) -> None: """ Here we are chcking if the plugin has access to workflow. Since we have provided `access`, `mutate` of incorrect types to `DucklingParser` we will receive a `TypeError`. """ parser = DucklingParser(access=access, mutate=mutate, locale="en_IN") plugin = parser() workflow = Workflow(preprocessors=[plugin], postprocessors=[]) with pytest.raises(TypeError): workflow.run("")
def test_missing_access(): intents = [ Intent(name="a", score=0.3), Intent(name="a", score=0.2), Intent(name="b", score=0.1), Intent(name="b", score=0.1), ] vote_plugin = VotePlugin(mutate=update_intent)() workflow = Workflow(preprocessors=[], postprocessors=[vote_plugin]) workflow.output = intents, [] with pytest.raises(TypeError): intent, _ = workflow.run(input_="")