def create_frame_prediction_pairs(self):
return [
FramePredictionPair(
CompositionalMetricReporter.tree_to_metric_node(pred_tree),
CompositionalMetricReporter.tree_to_metric_node(target_tree),
) for pred_tree, target_tree in self.all_context[PRED_TARGET_TREES]
]
def create_frame_prediction_pairs(self):
return [
FramePredictionPair(
CompositionalMetricReporter.tree_to_metric_node(pred_tree),
CompositionalMetricReporter.tree_to_metric_node(target_tree),
) for pred_tree, target_tree in self.pred_target_trees
]
def calculate_metric(self):
return compute_all_metrics(
[
FramePredictionPair(pred_frame, target_frame) for pred_frame,
target_frame in zip(self.all_preds, self.all_targets)
],
frame_accuracy=True,
)
def calculate_metric(self):
return compute_all_metrics(
[
FramePredictionPair(
CompositionalMetricReporter.tree_to_metric_node(pred_tree),
CompositionalMetricReporter.tree_to_metric_node(target_tree),
)
for pred_tree, target_tree in self.all_context[PRED_TARGET_TREES]
],
overall_metrics=True,
)
def calculate_metric(self):
return compute_all_metrics(
[
FramePredictionPair(
create_frame(self.doc_label_names[intent_pred], slots_pred,
utterance),
create_frame(self.doc_label_names[intent_target],
slots_label, utterance),
) for intent_pred, intent_target, slots_pred, slots_label,
utterance in zip(
self.all_doc_preds,
self.all_doc_targets,
self.all_context["slots_prediction"],
self.all_context[DatasetFieldName.RAW_WORD_LABEL],
self.all_context[DatasetFieldName.UTTERANCE_FIELD],
)
],
frame_accuracy=True,
)
},
),
"frames_match": False,
"bracket_confusions": {
"intent_confusion": {"TP": 1, "FP": 1, "FN": 1},
"slot_confusion": {"TP": 2, "FP": 2, "FN": 2},
},
"tree_confusions": {
"intent_confusion": {"TP": 0, "FP": 2, "FN": 2},
"slot_confusion": {"TP": 2, "FP": 2, "FN": 2},
},
},
]
FRAME_PAIRS = [
FramePredictionPair(example["predicted"], example["expected"])
for example in TEST_EXAMPLES
]
TEST_EXAMPLES_MORE: List[Dict[str, Any]] = [
# Invalid tree
{
"predicted":
Node(
label=INVALID_TREE_INTENT,
span=Span(start=0, end=20),
children={Node(label="slot1", span=Span(start=1, end=2))},
),
"expected":
Node(
label="intent1",