def test_operator_or(negated: bool):
events = [
UserUttered(intent={INTENT_NAME_KEY: "1"}),
UserUttered(intent={INTENT_NAME_KEY: "unknown"}),
UserUttered(intent={INTENT_NAME_KEY: "2"}),
UserUttered(intent={INTENT_NAME_KEY: "unknown"}),
]
sub_markers = [IntentDetectedMarker("1"), IntentDetectedMarker("2")]
marker = OrMarker(sub_markers, name="marker_name", negated=negated)
for event in events:
marker.track(event)
expected = [True, False, True, False]
if negated:
expected = [not applies for applies in expected]
assert marker.history == expected
def test_marker_from_config():
config = {
AndMarker.positive_tag(): [
{
SlotSetMarker.positive_tag(): "s1"
},
{
OrMarker.positive_tag(): [
{
IntentDetectedMarker.positive_tag(): "4"
},
{
IntentDetectedMarker.negated_tag(): "6"
},
]
},
]
}
marker = Marker.from_config(config)
assert isinstance(marker, AndMarker)
assert isinstance(marker.sub_markers[0], SlotSetMarker)
or_marker = marker.sub_markers[1]
assert isinstance(or_marker, OrMarker)
for sub_marker in or_marker.sub_markers:
assert isinstance(sub_marker, ConditionMarker)
def test_operator_occur(negated: bool):
events_expected = [
(UserUttered(intent={INTENT_NAME_KEY: "0"}), False),
(SlotSet("2", value=None), False),
(UserUttered(intent={INTENT_NAME_KEY: "1"}), True),
(SlotSet("2", value=None), True),
(UserUttered(intent={INTENT_NAME_KEY: "2"}), True),
(SlotSet("2", value="bla"), True),
(UserUttered(intent={INTENT_NAME_KEY: "2"}), True),
]
events, expected = zip(*events_expected)
sub_marker = OrMarker([IntentDetectedMarker("1"),
SlotSetMarker("2")],
name="or marker",
negated=False)
marker = OccurrenceMarker([sub_marker],
name="marker_name",
negated=negated)
for event in events:
marker.track(event)
expected = list(expected)
if negated:
expected = [not applies for applies in expected]
assert marker.history == expected
assert marker.relevant_events() == [expected.index(True)]
def test_operators_nested_simple():
events = [
UserUttered(intent={"name": "1"}),
UserUttered(intent={"name": "2"}),
UserUttered(intent={"name": "3"}),
SlotSet("s1", value="any"),
UserUttered(intent={"name": "4"}),
UserUttered(intent={"name": "5"}),
UserUttered(intent={"name": "6"}),
]
marker = AndMarker(
markers=[
SlotSetMarker("s1"),
OrMarker([
IntentDetectedMarker("4"),
IntentDetectedMarker("6"),
]),
],
name="marker_name",
)
evaluation = marker.evaluate_events(events)
assert len(evaluation[0]["marker_name"]) == 2
assert evaluation[0]["marker_name"][0].preceding_user_turns == 3
assert evaluation[0]["marker_name"][1].preceding_user_turns == 5
def test_markers_cli_results_save_correctly(tmp_path: Path):
domain = Domain.empty()
store = InMemoryTrackerStore(domain)
for i in range(5):
tracker = DialogueStateTracker(str(i), None)
tracker.update_with_events([SlotSet(str(j), "slot") for j in range(5)],
domain)
tracker.update(ActionExecuted(ACTION_SESSION_START_NAME))
tracker.update(UserUttered("hello"))
tracker.update_with_events(
[SlotSet(str(5 + j), "slot") for j in range(5)], domain)
store.save(tracker)
tracker_loader = MarkerTrackerLoader(store, "all")
results_path = tmp_path / "results.csv"
markers = OrMarker(markers=[
SlotSetMarker("2", name="marker1"),
SlotSetMarker("7", name="marker2")
])
markers.evaluate_trackers(tracker_loader.load(), results_path)
with open(results_path, "r") as results:
result_reader = csv.DictReader(results)
senders = set()
for row in result_reader:
senders.add(row["sender_id"])
if row["marker"] == "marker1":
assert row["session_idx"] == "0"
assert int(row["event_idx"]) >= 2
assert row["num_preceding_user_turns"] == "0"
if row["marker"] == "marker2":
assert row["session_idx"] == "1"
assert int(row["event_idx"]) >= 3
assert row["num_preceding_user_turns"] == "1"
assert len(senders) == 5
def test_operator_occur_never_applied_negated():
events_expected = [
(UserUttered(intent={INTENT_NAME_KEY: "1"}), False),
(SlotSet("2", value=None), False),
(UserUttered(intent={INTENT_NAME_KEY: "0"}), False),
(SlotSet("1", value="test"), False),
]
events, expected = zip(*events_expected)
sub_marker = OrMarker([IntentDetectedMarker("1"),
SlotSetMarker("2")],
name="or marker",
negated=False)
marker = OccurrenceMarker([sub_marker],
name="or never occurred",
negated=True)
for event in events:
marker.track(event)
assert marker.relevant_events() == []
def from_path(cls, path: Union[Path, Text]) -> Marker:
"""Loads markers from one config file or all config files in a directory tree.
Each config file should contain a dictionary mapping marker names to the
respective marker configuration.
To avoid confusion, the marker names must not coincide with the tag of
any pre-defined markers. Moreover, marker names must be unique. This means,
if you you load the markers from multiple files, then you have to make sure
that the names of the markers defined in these files are unique across all
loaded files.
Note that all loaded markers will be combined into one marker via one
artificial OR-operator. When evaluating the resulting marker, then the
artificial OR-operator will be ignored and results will be reported for
every sub-marker.
For more details how a single marker configuration looks like, have a look
at `Marker.from_config`.
Args:
path: either the path to a single config file or the root of the directory
tree that contains marker config files
Returns:
all configured markers, combined into one marker object
"""
MarkerRegistry.register_builtin_markers()
from rasa.core.evaluation.marker import OrMarker
# collect all the files from which we want to load configs (i.e. either just
# the given path if points to a yaml or all yaml-files in the directory tree)
yaml_files = cls._collect_yaml_files_from_path(path)
# collect all the configs from those yaml files (assure it's dictionaries
# mapping marker names to something) -- keep track of which config came
# from which file to be able to raise meaningful errors later
loaded_configs: Dict[
Text, Dict] = cls._collect_configs_from_yaml_files(yaml_files)
# create markers from all loaded configurations
loaded_markers: List[Marker] = []
for yaml_file, config in loaded_configs.items():
for marker_name, marker_config in config.items():
try:
marker = Marker.from_config(marker_config,
name=marker_name)
except InvalidMarkerConfig as e:
# we don't re-raise here because the stack trace would only be
# printed when we run rasa evaluate with --debug flag
raise InvalidMarkerConfig(
f"Could not load marker {marker_name} from {yaml_file}. "
f"Reason: {str(e)}. ")
loaded_markers.append(marker)
# Reminder: We could also just create a dictionary of markers from this.
# However, if we want to allow re-using top-level markers (e.g.
# "custom_marker1 or custom_marker2" and/or optimize the marker evaluation such
# that e.g. the same condition is not instantiated (and evaluated) twice, then
# the current approach might be better (e.g. with a dictionary of markers one
# might expect the markers to be independent objects).
# combine the markers
marker = OrMarker(markers=loaded_markers)
marker.name = Marker.ANY_MARKER # cannot be set via name parameter
return marker
folder = full_path.parents[0]
if folder != tmp_path:
Path.mkdir(folder, exist_ok=False)
rasa.shared.utils.io.write_yaml(data=config, target=full_path)
with pytest.raises(InvalidMarkerConfig):
Marker.from_path(tmp_path)
@pytest.mark.parametrize(
"marker,expected_depth",
[
(
AndMarker(markers=[
SlotSetMarker("s1"),
OrMarker(
[IntentDetectedMarker("4"),
IntentDetectedMarker("6")]),
]),
3,
),
(SlotSetMarker("s1"), 1),
(AndMarker(markers=[SlotSetMarker("s1"),
IntentDetectedMarker("6")]), 2),
],
)
def test_marker_depth(marker: Marker, expected_depth: int):
assert marker.max_depth() == expected_depth
def test_split_sessions(tmp_path):
"""Tests loading a tracker with multiple sessions."""