def test_marker_from_config():
config = {
AndMarker.positive_tag(): [
{
SlotSetMarker.positive_tag(): "s1"
},
{
OrMarker.positive_tag(): [
{
IntentDetectedMarker.positive_tag(): "4"
},
{
IntentDetectedMarker.negated_tag(): "6"
},
]
},
]
}
marker = Marker.from_config(config)
assert isinstance(marker, AndMarker)
assert isinstance(marker.sub_markers[0], SlotSetMarker)
or_marker = marker.sub_markers[1]
assert isinstance(or_marker, OrMarker)
for sub_marker in or_marker.sub_markers:
assert isinstance(sub_marker, ConditionMarker)
def test_operator_occur(negated: bool):
events_expected = [
(UserUttered(intent={INTENT_NAME_KEY: "0"}), False),
(SlotSet("2", value=None), False),
(UserUttered(intent={INTENT_NAME_KEY: "1"}), True),
(SlotSet("2", value=None), True),
(UserUttered(intent={INTENT_NAME_KEY: "2"}), True),
(SlotSet("2", value="bla"), True),
(UserUttered(intent={INTENT_NAME_KEY: "2"}), True),
]
events, expected = zip(*events_expected)
sub_marker = OrMarker([IntentDetectedMarker("1"),
SlotSetMarker("2")],
name="or marker",
negated=False)
marker = OccurrenceMarker([sub_marker],
name="marker_name",
negated=negated)
for event in events:
marker.track(event)
expected = list(expected)
if negated:
expected = [not applies for applies in expected]
assert marker.history == expected
assert marker.relevant_events() == [expected.index(True)]
def test_operators_nested_simple():
events = [
UserUttered(intent={"name": "1"}),
UserUttered(intent={"name": "2"}),
UserUttered(intent={"name": "3"}),
SlotSet("s1", value="any"),
UserUttered(intent={"name": "4"}),
UserUttered(intent={"name": "5"}),
UserUttered(intent={"name": "6"}),
]
marker = AndMarker(
markers=[
SlotSetMarker("s1"),
OrMarker([
IntentDetectedMarker("4"),
IntentDetectedMarker("6"),
]),
],
name="marker_name",
)
evaluation = marker.evaluate_events(events)
assert len(evaluation[0]["marker_name"]) == 2
assert evaluation[0]["marker_name"][0].preceding_user_turns == 3
assert evaluation[0]["marker_name"][1].preceding_user_turns == 5
async def test_markers_cli_results_save_correctly(tmp_path: Path):
domain = Domain.empty()
store = InMemoryTrackerStore(domain)
for i in range(5):
tracker = DialogueStateTracker(str(i), None)
tracker.update_with_events([SlotSet(str(j), "slot") for j in range(5)],
domain)
tracker.update(ActionExecuted(ACTION_SESSION_START_NAME))
tracker.update(UserUttered("hello"))
tracker.update_with_events(
[SlotSet(str(5 + j), "slot") for j in range(5)], domain)
await store.save(tracker)
tracker_loader = MarkerTrackerLoader(store, "all")
results_path = tmp_path / "results.csv"
markers = OrMarker(markers=[
SlotSetMarker("2", name="marker1"),
SlotSetMarker("7", name="marker2")
])
await markers.evaluate_trackers(tracker_loader.load(), results_path)
with open(results_path, "r") as results:
result_reader = csv.DictReader(results)
senders = set()
for row in result_reader:
senders.add(row["sender_id"])
if row["marker"] == "marker1":
assert row["session_idx"] == "0"
assert int(row["event_idx"]) >= 2
assert row["num_preceding_user_turns"] == "0"
if row["marker"] == "marker2":
assert row["session_idx"] == "1"
assert int(row["event_idx"]) >= 3
assert row["num_preceding_user_turns"] == "1"
assert len(senders) == 5
def test_sessions_evaluated_returns_event_indices_wrt_tracker_not_dialogue():
events = [
ActionExecuted(action_name=ACTION_SESSION_START_NAME),
UserUttered(intent={INTENT_NAME_KEY: "ignored"}),
UserUttered(intent={INTENT_NAME_KEY: "ignored"}),
UserUttered(intent={INTENT_NAME_KEY: "ignored"}),
SlotSet("same-text", value="any"),
ActionExecuted(action_name=ACTION_SESSION_START_NAME),
UserUttered(intent={INTENT_NAME_KEY: "no-slot-set-here"}),
UserUttered(intent={INTENT_NAME_KEY: "no-slot-set-here"}),
SlotSet("same-text", value="any"),
]
marker = SlotSetMarker(text="same-text", name="my-marker")
evaluation = marker.evaluate_events(events)
assert len(evaluation) == 2
assert len(evaluation[0]["my-marker"]) == 1
assert evaluation[0]["my-marker"][0].preceding_user_turns == 3
assert evaluation[0]["my-marker"][0].idx == 4
assert len(evaluation[1]["my-marker"]) == 1
assert evaluation[1]["my-marker"][0].preceding_user_turns == 2
assert evaluation[1]["my-marker"][
0].idx == 8 # i.e. NOT the index in the dialogue
def test_sessions_evaluated_separately():
"""Each marker applies an exact number of times (slots are immediately un-set)."""
events = [
ActionExecuted(ACTION_SESSION_START_NAME),
UserUttered(intent={INTENT_NAME_KEY: "ignored"}),
UserUttered(intent={INTENT_NAME_KEY: "ignored"}),
UserUttered(intent={INTENT_NAME_KEY: "ignored"}),
SlotSet("same-text", value="any"),
ActionExecuted(action_name=ACTION_SESSION_START_NAME),
UserUttered(intent={INTENT_NAME_KEY: "no-slot-set-here"}),
UserUttered(intent={INTENT_NAME_KEY: "no-slot-set-here"}),
]
marker = SlotSetMarker(text="same-text", name="my-marker")
evaluation = marker.evaluate_events(events)
assert len(evaluation) == 2
assert len(evaluation[0]["my-marker"]) == 1
assert evaluation[0]["my-marker"][0].preceding_user_turns == 3
assert len(
evaluation[1]["my-marker"]) == 0 # i.e. slot set does not "leak"
def test_operator_occur_never_applied_negated():
events_expected = [
(UserUttered(intent={INTENT_NAME_KEY: "1"}), False),
(SlotSet("2", value=None), False),
(UserUttered(intent={INTENT_NAME_KEY: "0"}), False),
(SlotSet("1", value="test"), False),
]
events, expected = zip(*events_expected)
sub_marker = OrMarker([IntentDetectedMarker("1"),
SlotSetMarker("2")],
name="or marker",
negated=False)
marker = OccurrenceMarker([sub_marker],
name="or never occurred",
negated=True)
for event in events:
marker.track(event)
assert marker.relevant_events() == []
def test_operator_and(negated: bool):
events_expected = [
(UserUttered(intent={INTENT_NAME_KEY: "1"}), False),
(SlotSet("2", value="bla"), False),
(UserUttered(intent={INTENT_NAME_KEY: "1"}), True),
(SlotSet("2", value=None), False),
(UserUttered(intent={INTENT_NAME_KEY: "1"}), False),
(SlotSet("2", value="bla"), False),
(UserUttered(intent={INTENT_NAME_KEY: "2"}), False),
]
events, expected = zip(*events_expected)
sub_markers = [IntentDetectedMarker("1"), SlotSetMarker("2")]
marker = AndMarker(sub_markers, name="marker_name", negated=negated)
for event in events:
marker.track(event)
expected = list(expected)
if negated:
expected = [not applies for applies in expected]
assert marker.history == expected
constant_condition_text=None,
constant_negated=None,
)
domain = Domain.empty()
with pytest.warns(None):
is_valid = marker.validate_against_domain(domain)
assert not is_valid
@pytest.mark.parametrize(
"config",
[
# (1) top level configuration
# - config is list
[SlotSetMarker.positive_tag()],
# - config is list with nested dict
[{
SlotSetMarker.positive_tag(): "s1"
}],
# - config is just a str
# (2) sub-markers
SlotSetMarker.positive_tag(),
# - sub-marker config is dict not list
{
AndMarker.positive_tag(): {
SlotSetMarker.positive_tag(): "s1"
}
},
# - sub-marker under condition
{