def calculate_metric(self): # to support top k predictions try: top_k_exists = self.all_preds[0][0][0][0] # convert to frames if top_k_exists: batch_size = len(self.all_preds) all_predicted_frames: List[List[Node]] = [[]] * batch_size for i, top_k_preds in enumerate( self.all_context[ALL_PRED_TREES]): for pred_tree in top_k_preds: all_predicted_frames[i].append( CompositionalMetricReporter.tree_to_metric_node( pred_tree)) return compute_all_metrics( self.create_frame_prediction_pairs(), overall_metrics=True, all_predicted_frames=all_predicted_frames, ) except TypeError: return compute_all_metrics(self.create_frame_prediction_pairs(), overall_metrics=True)
def calculate_metric(self): all_metrics = compute_all_metrics( self.create_frame_prediction_pairs(), overall_metrics=True, calculated_loss=self.calculate_loss(), ) return all_metrics
def calculate_metric(self): return compute_all_metrics( [ FramePredictionPair(pred_frame, target_frame) for pred_frame, target_frame in zip(self.all_preds, self.all_targets) ], frame_accuracy=True, )
def calculate_metric(self): return compute_all_metrics( [ FramePredictionPair( CompositionalMetricReporter.tree_to_metric_node(pred_tree), CompositionalMetricReporter.tree_to_metric_node(target_tree), ) for pred_tree, target_tree in self.all_context[PRED_TARGET_TREES] ], overall_metrics=True, )
def compute_masked_metrics( frame_pairs: Sequence[FramePredictionPair], all_target_lens: List[int], all_target_length_preds: List[List[int]], select_length_beam: int, top_intent_accuracy: bool = True, frame_accuracy: bool = True, frame_accuracies_by_depth: bool = True, bracket_metrics: bool = True, tree_metrics: bool = True, overall_metrics: bool = False, all_predicted_frames: List[List[Node]] = None, calculated_loss: float = None, length_metrics: Dict = None, non_invalid_frame_pairs: Optional[Sequence[FramePredictionPair]] = None, extracted_frame_pairs: Optional[Sequence[FramePredictionPair]] = None, print_length_metrics: bool = True, ) -> MaskedSeq2SeqJointMetrics: all_metrics = compute_all_metrics( frame_pairs, top_intent_accuracy, frame_accuracy, frame_accuracies_by_depth, bracket_metrics, tree_metrics, overall_metrics, all_predicted_frames, calculated_loss, ) length_metrics, length_reports = compute_length_metrics( all_target_lens, all_target_length_preds, select_length_beam) return MaskedSeq2SeqJointMetrics( top_intent_accuracy=all_metrics.top_intent_accuracy, frame_accuracy=all_metrics.frame_accuracy, frame_accuracy_top_k=all_metrics.frame_accuracy_top_k, frame_accuracies_by_depth=all_metrics.frame_accuracies_by_depth, bracket_metrics=all_metrics.bracket_metrics, tree_metrics=all_metrics.tree_metrics, loss=all_metrics.loss, length_metrics=length_metrics, length_reports=length_reports, non_invalid_fa=compute_frame_accuracy(non_invalid_frame_pairs), extracted_fa=compute_frame_accuracy(extracted_frame_pairs), print_length_metrics=print_length_metrics, )
def calculate_metric(self): return compute_all_metrics( [ FramePredictionPair( create_frame(self.doc_label_names[intent_pred], slots_pred, utterance), create_frame(self.doc_label_names[intent_target], slots_label, utterance), ) for intent_pred, intent_target, slots_pred, slots_label, utterance in zip( self.all_doc_preds, self.all_doc_targets, self.all_context["slots_prediction"], self.all_context[DatasetFieldName.RAW_WORD_LABEL], self.all_context[DatasetFieldName.UTTERANCE_FIELD], ) ], frame_accuracy=True, )
def compute_masked_metrics( frame_pairs: Sequence[FramePredictionPair], all_target_lens: List[int], all_target_length_preds: List[List[int]], select_length_beam: int, top_intent_accuracy: bool = True, frame_accuracy: bool = True, frame_accuracies_by_depth: bool = True, bracket_metrics: bool = True, tree_metrics: bool = True, overall_metrics: bool = False, all_predicted_frames: List[List[Node]] = None, calculated_loss: float = None, length_metrics: Dict = None, non_invalid_frame_pairs: Optional[Sequence[FramePredictionPair]] = None, extracted_frame_pairs: Optional[Sequence[FramePredictionPair]] = None, print_length_metrics: bool = True, num_weights=None, current_model_parameter_size=None, ) -> MaskedSeq2SeqJointMetrics: all_metrics = compute_all_metrics( frame_pairs, top_intent_accuracy, frame_accuracy, frame_accuracies_by_depth, bracket_metrics, tree_metrics, overall_metrics, all_predicted_frames, calculated_loss, ) length_metrics, length_reports = compute_length_metrics( all_target_lens, all_target_length_preds, select_length_beam, log_per_label_metrics=print_length_metrics, ) # Compute sizes of 32bit, 8bit, and 4bit versions of the model # in MB (hence the conversion factor), if the number of weights is provided. # Default to -1 if unavailable. size_32bit_model = -1 size_8bit_model = -1 size_4bit_model = -1 if num_weights: size_32bit_model = 32 * num_weights / (BIT2BYTE_CONSTANT) size_8bit_model = 8 * num_weights / (BIT2BYTE_CONSTANT) size_4bit_model = 4 * num_weights / (BIT2BYTE_CONSTANT) if not current_model_parameter_size: current_model_parameter_size = -1 return MaskedSeq2SeqJointMetrics( top_intent_accuracy=all_metrics.top_intent_accuracy, current_model_parameter_size=current_model_parameter_size, size_32bit_model=size_32bit_model, size_8bit_model=size_8bit_model, size_4bit_model=size_4bit_model, frame_accuracy=all_metrics.frame_accuracy, frame_accuracy_top_k=all_metrics.frame_accuracy_top_k, frame_accuracies_by_depth=all_metrics.frame_accuracies_by_depth, bracket_metrics=all_metrics.bracket_metrics, tree_metrics=all_metrics.tree_metrics, percent_invalid_trees=all_metrics.percent_invalid_trees, percent_trees_wrong_label=all_metrics.percent_trees_wrong_label, loss=all_metrics.loss, length_metrics=length_metrics, length_reports=length_reports, non_invalid_fa=compute_frame_accuracy(non_invalid_frame_pairs), extracted_fa=compute_frame_accuracy(extracted_frame_pairs), print_length_metrics=print_length_metrics, )
def calculate_metric(self): return compute_all_metrics(self.create_frame_prediction_pairs(), overall_metrics=True)
def test_print_compute_all_metrics(self) -> None: compute_all_metrics(FRAME_PAIRS, overall_metrics=True).print_metrics()
def calculate_metric(self): return compute_all_metrics( self.create_frame_prediction_pairs(), overall_metrics=True, all_predicted_frames=self.all_context[ALL_PRED_FRAMES], )