def calculate_metric(self):
# to support top k predictions
try:
top_k_exists = self.all_preds[0][0][0][0]
# convert to frames
if top_k_exists:
batch_size = len(self.all_preds)
all_predicted_frames: List[List[Node]] = [[]] * batch_size
for i, top_k_preds in enumerate(
self.all_context[ALL_PRED_TREES]):
for pred_tree in top_k_preds:
all_predicted_frames[i].append(
CompositionalMetricReporter.tree_to_metric_node(
pred_tree))
return compute_all_metrics(
self.create_frame_prediction_pairs(),
overall_metrics=True,
all_predicted_frames=all_predicted_frames,
)
except TypeError:
return compute_all_metrics(self.create_frame_prediction_pairs(),
overall_metrics=True)
def calculate_metric(self):
all_metrics = compute_all_metrics(
self.create_frame_prediction_pairs(),
overall_metrics=True,
calculated_loss=self.calculate_loss(),
)
return all_metrics
def calculate_metric(self):
return compute_all_metrics(
[
FramePredictionPair(pred_frame, target_frame) for pred_frame,
target_frame in zip(self.all_preds, self.all_targets)
],
frame_accuracy=True,
)
def calculate_metric(self):
return compute_all_metrics(
[
FramePredictionPair(
CompositionalMetricReporter.tree_to_metric_node(pred_tree),
CompositionalMetricReporter.tree_to_metric_node(target_tree),
)
for pred_tree, target_tree in self.all_context[PRED_TARGET_TREES]
],
overall_metrics=True,
)
def compute_masked_metrics(
frame_pairs: Sequence[FramePredictionPair],
all_target_lens: List[int],
all_target_length_preds: List[List[int]],
select_length_beam: int,
top_intent_accuracy: bool = True,
frame_accuracy: bool = True,
frame_accuracies_by_depth: bool = True,
bracket_metrics: bool = True,
tree_metrics: bool = True,
overall_metrics: bool = False,
all_predicted_frames: List[List[Node]] = None,
calculated_loss: float = None,
length_metrics: Dict = None,
non_invalid_frame_pairs: Optional[Sequence[FramePredictionPair]] = None,
extracted_frame_pairs: Optional[Sequence[FramePredictionPair]] = None,
print_length_metrics: bool = True,
) -> MaskedSeq2SeqJointMetrics:
all_metrics = compute_all_metrics(
frame_pairs,
top_intent_accuracy,
frame_accuracy,
frame_accuracies_by_depth,
bracket_metrics,
tree_metrics,
overall_metrics,
all_predicted_frames,
calculated_loss,
)
length_metrics, length_reports = compute_length_metrics(
all_target_lens, all_target_length_preds, select_length_beam)
return MaskedSeq2SeqJointMetrics(
top_intent_accuracy=all_metrics.top_intent_accuracy,
frame_accuracy=all_metrics.frame_accuracy,
frame_accuracy_top_k=all_metrics.frame_accuracy_top_k,
frame_accuracies_by_depth=all_metrics.frame_accuracies_by_depth,
bracket_metrics=all_metrics.bracket_metrics,
tree_metrics=all_metrics.tree_metrics,
loss=all_metrics.loss,
length_metrics=length_metrics,
length_reports=length_reports,
non_invalid_fa=compute_frame_accuracy(non_invalid_frame_pairs),
extracted_fa=compute_frame_accuracy(extracted_frame_pairs),
print_length_metrics=print_length_metrics,
)
def calculate_metric(self):
return compute_all_metrics(
[
FramePredictionPair(
create_frame(self.doc_label_names[intent_pred], slots_pred,
utterance),
create_frame(self.doc_label_names[intent_target],
slots_label, utterance),
) for intent_pred, intent_target, slots_pred, slots_label,
utterance in zip(
self.all_doc_preds,
self.all_doc_targets,
self.all_context["slots_prediction"],
self.all_context[DatasetFieldName.RAW_WORD_LABEL],
self.all_context[DatasetFieldName.UTTERANCE_FIELD],
)
],
frame_accuracy=True,
)
def compute_masked_metrics(
frame_pairs: Sequence[FramePredictionPair],
all_target_lens: List[int],
all_target_length_preds: List[List[int]],
select_length_beam: int,
top_intent_accuracy: bool = True,
frame_accuracy: bool = True,
frame_accuracies_by_depth: bool = True,
bracket_metrics: bool = True,
tree_metrics: bool = True,
overall_metrics: bool = False,
all_predicted_frames: List[List[Node]] = None,
calculated_loss: float = None,
length_metrics: Dict = None,
non_invalid_frame_pairs: Optional[Sequence[FramePredictionPair]] = None,
extracted_frame_pairs: Optional[Sequence[FramePredictionPair]] = None,
print_length_metrics: bool = True,
num_weights=None,
current_model_parameter_size=None,
) -> MaskedSeq2SeqJointMetrics:
all_metrics = compute_all_metrics(
frame_pairs,
top_intent_accuracy,
frame_accuracy,
frame_accuracies_by_depth,
bracket_metrics,
tree_metrics,
overall_metrics,
all_predicted_frames,
calculated_loss,
)
length_metrics, length_reports = compute_length_metrics(
all_target_lens,
all_target_length_preds,
select_length_beam,
log_per_label_metrics=print_length_metrics,
)
# Compute sizes of 32bit, 8bit, and 4bit versions of the model
# in MB (hence the conversion factor), if the number of weights is provided.
# Default to -1 if unavailable.
size_32bit_model = -1
size_8bit_model = -1
size_4bit_model = -1
if num_weights:
size_32bit_model = 32 * num_weights / (BIT2BYTE_CONSTANT)
size_8bit_model = 8 * num_weights / (BIT2BYTE_CONSTANT)
size_4bit_model = 4 * num_weights / (BIT2BYTE_CONSTANT)
if not current_model_parameter_size:
current_model_parameter_size = -1
return MaskedSeq2SeqJointMetrics(
top_intent_accuracy=all_metrics.top_intent_accuracy,
current_model_parameter_size=current_model_parameter_size,
size_32bit_model=size_32bit_model,
size_8bit_model=size_8bit_model,
size_4bit_model=size_4bit_model,
frame_accuracy=all_metrics.frame_accuracy,
frame_accuracy_top_k=all_metrics.frame_accuracy_top_k,
frame_accuracies_by_depth=all_metrics.frame_accuracies_by_depth,
bracket_metrics=all_metrics.bracket_metrics,
tree_metrics=all_metrics.tree_metrics,
percent_invalid_trees=all_metrics.percent_invalid_trees,
percent_trees_wrong_label=all_metrics.percent_trees_wrong_label,
loss=all_metrics.loss,
length_metrics=length_metrics,
length_reports=length_reports,
non_invalid_fa=compute_frame_accuracy(non_invalid_frame_pairs),
extracted_fa=compute_frame_accuracy(extracted_frame_pairs),
print_length_metrics=print_length_metrics,
)
def calculate_metric(self):
return compute_all_metrics(self.create_frame_prediction_pairs(),
overall_metrics=True)
def test_print_compute_all_metrics(self) -> None:
compute_all_metrics(FRAME_PAIRS, overall_metrics=True).print_metrics()
def calculate_metric(self):
return compute_all_metrics(
self.create_frame_prediction_pairs(),
overall_metrics=True,
all_predicted_frames=self.all_context[ALL_PRED_FRAMES],
)
