def test_accuracy(caplog): """Unit test of accuracy_scorer""" caplog.set_level(logging.INFO) golds = np.array([0, 1, 0, 1, 0, 1]) gold_probs = np.array([[0.6, 0.4], [0.1, 0.9], [0.7, 0.3], [0.2, 0.8], [0.9, 0.1], [0.4, 0.6]]) probs = np.array([[0.9, 0.1], [0.6, 0.4], [1.0, 0.0], [0.8, 0.2], [0.6, 0.4], [0.05, 0.95]]) preds = np.array([0, 0, 0, 0, 0, 1]) metric_dict = accuracy_scorer(golds, None, preds) assert isequal(metric_dict, {"accuracy": 0.6666666666666666}) metric_dict = accuracy_scorer(golds, probs, None) assert isequal(metric_dict, {"accuracy": 0.6666666666666666}) metric_dict = accuracy_scorer(golds, probs, preds, topk=2) assert isequal(metric_dict, {"accuracy@2": 1.0}) metric_dict = accuracy_scorer(gold_probs, None, preds) assert isequal(metric_dict, {"accuracy": 0.6666666666666666}) metric_dict = accuracy_scorer(gold_probs, probs, preds, topk=2) assert isequal(metric_dict, {"accuracy@2": 1.0})
def accuracy_f1_scorer( golds: ndarray, probs: Optional[ndarray], preds: ndarray, uids: Optional[List[str]] = None, pos_label: int = 1, ) -> Dict[str, float]: """Average of accuracy and f1 score. Args: golds: Ground truth values. probs: Predicted probabilities. preds: Predicted values. uids: Unique ids, defaults to None. pos_label: The positive class label, defaults to 1. Returns: Average of accuracy and f1. """ metrics = dict() accuracy = accuracy_scorer(golds, probs, preds, uids) f1 = f1_scorer(golds, probs, preds, uids, pos_label=pos_label) metrics["accuracy_f1"] = mean([accuracy["accuracy"], f1["f1"]]) return metrics
def accuracy_f1_scorer( golds: ndarray, probs: Optional[ndarray], preds: ndarray, uids: Optional[List[str]] = None, ) -> Dict[str, float]: r"""Average of accuracy and f1 score. Args: golds(ndarray): Ground truth values. probs(ndarray or None): Predicted probabilities. preds(ndarray): Predicted values. uids(list, optional): Unique ids, defaults to None. Returns: dict: Average of accuracy and f1. """ metrics = dict() accuracy = accuracy_scorer(golds, probs, preds, uids) f1 = f1_scorer(golds, probs, preds, uids) metrics["accuracy_f1"] = np.mean([accuracy["accuracy"], f1["f1"]]) return metrics
def test_accuracy(caplog): """Unit test of accuracy_scorer""" caplog.set_level(logging.INFO) golds = np.array([0, 1, 0, 1, 0, 1]) preds = np.array([0, 0, 0, 0, 0, 1]) metric_dict = accuracy_scorer(golds, None, preds) assert isequal(metric_dict, {"accuracy": 0.6666666666666666})
def test_accuracy(caplog): """Unit test of accuracy_scorer.""" caplog.set_level(logging.INFO) metric_dict = accuracy_scorer(GOLDS, PROBS, PREDS) assert isequal(metric_dict, {"accuracy": 0.6666666666666666}) metric_dict = accuracy_scorer(GOLDS, None, PREDS) assert isequal(metric_dict, {"accuracy": 0.6666666666666666}) metric_dict = accuracy_scorer(GOLDS, PROBS, None) assert isequal(metric_dict, {"accuracy": 0.6666666666666666}) metric_dict = accuracy_scorer(GOLDS, None, PREDS, normalize=False) assert isequal(metric_dict, {"accuracy": 4}) metric_dict = accuracy_scorer(GOLDS, PROBS, PREDS, topk=2) assert isequal(metric_dict, {"accuracy@2": 1.0}) metric_dict = accuracy_scorer(PROB_GOLDS, None, PREDS) assert isequal(metric_dict, {"accuracy": 0.6666666666666666}) metric_dict = accuracy_scorer(PROB_GOLDS, PROBS, PREDS, topk=2) assert isequal(metric_dict, {"accuracy@2": 1.0}) metric_dict = accuracy_scorer(PROB_GOLDS, PROBS, PREDS, topk=2, normalize=False) assert isequal(metric_dict, {"accuracy@2": 6})
def accuracy_f1_scorer(golds, probs, preds, uids=None): """Average of accuracy and f1 score. :param golds: Ground truth (correct) target values. :type golds: 1-d np.array :param probs: Predicted target probabilities. (Not used!) :type probs: k-d np.array :param preds: Predicted target values. :type preds: 1-d np.array :param uids: Unique ids. :type uids: list :return: Average of accuracy and f1. :rtype: dict """ metrics = dict() accuracy = accuracy_scorer(golds, probs, preds, uids) f1 = f1_scorer(golds, probs, preds, uids) metrics["accuracy_f1"] = np.mean([accuracy["accuracy"], f1["f1"]]) return metrics
def accuracy_macro_f1(golds, probs, preds, uids): f1 = macro_f1(golds, probs, preds, uids) accuracy = accuracy_scorer(golds, probs, preds, uids) return (f1 + accuracy["accuracy"]) / 2