def multiclass_classification(target, predictions): all_labels = [i[11:] for i in predictions.columns.tolist()[:-1]] ll = logloss(target, predictions[predictions.columns[:-1]]) labels = {i: l for i, l in enumerate(all_labels)} predictions = predictions["label"] target = target["target"].map(labels) # Print the confusion matrix conf_matrix = confusion_matrix(target, predictions, labels=all_labels) rows = [f"Predicted as {a}" for a in all_labels] cols = [f"Labeled as {a}" for a in all_labels] conf_matrix = pd.DataFrame(conf_matrix, columns=rows, index=cols) max_metrics = classification_report(target, predictions, digits=6, labels=all_labels, output_dict=True) max_metrics["logloss"] = ll return { "max_metrics": pd.DataFrame(max_metrics).transpose(), "confusion_matrix": conf_matrix, }
def multiclass_classification(target, predictions, sample_weight=None): all_labels = [i[11:] for i in predictions.columns.tolist()[:-1]] predicted_probas = predictions[predictions.columns[:-1]] ll = logloss(target, predictions[predictions.columns[:-1]], sample_weight=sample_weight) if "target" in target.columns.tolist(): # multiclass coding with integer labels = {i: l for i, l in enumerate(all_labels)} target = target["target"].map(labels) else: # multiclass coding with one-hot encoding old_columns = target.columns t = target[old_columns[0]] for l in all_labels: t[target[f"target_{l}"] == 1] = l target = pd.DataFrame({"target": t}) # Print the confusion matrix predicted_labels = predictions["label"] predictions = predictions["label"] if not pd.api.types.is_string_dtype(predictions): predictions = predictions.astype(str) if not pd.api.types.is_string_dtype(target): target = target.astype(str) conf_matrix = confusion_matrix(target, predictions, labels=all_labels, sample_weight=sample_weight) rows = [f"Predicted as {a}" for a in all_labels] cols = [f"Labeled as {a}" for a in all_labels] conf_matrix = pd.DataFrame(conf_matrix, columns=rows, index=cols) max_metrics = classification_report( target, predictions, digits=6, labels=all_labels, output_dict=True, sample_weight=sample_weight, ) max_metrics["logloss"] = ll return { "max_metrics": pd.DataFrame(max_metrics).transpose(), "confusion_matrix": conf_matrix, "additional_plots": AdditionalPlots.plots_multiclass(target, predicted_labels, predicted_probas), }
def multiclass_classification(target, predictions): all_labels = [i[11:] for i in predictions.columns.tolist()[:-1]] ll = logloss(target, predictions[predictions.columns[:-1]]) if "target" in target.columns.tolist(): # multiclass coding with integer labels = {i: l for i, l in enumerate(all_labels)} target = target["target"].map(labels) else: # multiclass coding with one-hot encoding old_columns = target.columns print(old_columns) t = target[old_columns[0]] for l in all_labels: t[target[f"target_{l}"] == 1] = l print(l, t) target = pd.DataFrame({"target": t}) # Print the confusion matrix predictions = predictions["label"] conf_matrix = confusion_matrix(target, predictions, labels=all_labels) rows = [f"Predicted as {a}" for a in all_labels] cols = [f"Labeled as {a}" for a in all_labels] conf_matrix = pd.DataFrame(conf_matrix, columns=rows, index=cols) max_metrics = classification_report(target, predictions, digits=6, labels=all_labels, output_dict=True) max_metrics["logloss"] = ll return { "max_metrics": pd.DataFrame(max_metrics).transpose(), "confusion_matrix": conf_matrix, }
def binary_classification(target, predictions, sample_weight=None): negative_label, positive_label = "0", "1" mapping = None try: pred_col = predictions.columns[0] if "_0_for_" in pred_col and "_1_for_" in pred_col: t = pred_col.split("_0_for_")[1] t = t.split("_1_for_") negative_label, positive_label = t[0], t[1] mapping = {0: negative_label, 1: positive_label} except Exception as e: pass predictions = np.array(predictions) sorted_predictions = np.sort(predictions) STEPS = 100 # can go lower for speed increase ??? details = { "threshold": [], "f1": [], "accuracy": [], "precision": [], "recall": [], "mcc": [], } samples_per_step = max(1, np.floor(predictions.shape[0] / STEPS)) for i in range(STEPS): idx = int(i * samples_per_step) if idx + 1 >= predictions.shape[0]: break if i == 0: th = 0.9 * np.min(sorted_predictions) else: th = float( 0.5 * (sorted_predictions[idx] + sorted_predictions[idx + 1])) if np.sum(predictions > th) < 1: break response = (predictions > th).astype(int) details["threshold"] += [th] details["f1"] += [ f1_score(target, response, sample_weight=sample_weight) ] details["accuracy"] += [ accuracy_score(target, response, sample_weight=sample_weight) ] details["precision"] += [ precision_score(target, response, sample_weight=sample_weight) ] details["recall"] += [ recall_score(target, response, sample_weight=sample_weight) ] if i == 0: details["mcc"] += [0.0] else: details["mcc"] += [ matthews_corrcoef(target, response, sample_weight=sample_weight) ] # max metrics max_metrics = { "logloss": { "score": logloss(target, predictions, sample_weight=sample_weight), "threshold": None, }, # there is no threshold for LogLoss "auc": { "score": roc_auc_score(target, predictions, sample_weight=sample_weight), "threshold": None, }, # there is no threshold for AUC "f1": { "score": np.max(details["f1"]), "threshold": details["threshold"][np.argmax(details["f1"])], }, "accuracy": { "score": np.max(details["accuracy"]), "threshold": details["threshold"][np.argmax(details["accuracy"])], }, "precision": { "score": np.max(details["precision"]), "threshold": details["threshold"][np.argmax(details["precision"])], }, "recall": { "score": np.max(details["recall"]), "threshold": details["threshold"][np.argmax(details["recall"])], }, "mcc": { "score": np.max(details["mcc"]), "threshold": details["threshold"][np.argmax(details["mcc"])], }, } threshold = float(max_metrics["accuracy"]["threshold"]) # if sample_weight is not None: # new_max_metrics = {} # for k, v in max_metrics.items(): # new_max_metrics["weighted_" + k] = v # max_metrics = new_max_metrics # confusion matrix conf_matrix = confusion_matrix(target, predictions > threshold, sample_weight=sample_weight) conf_matrix = pd.DataFrame( conf_matrix, columns=[ f"Predicted as {negative_label}", f"Predicted as {positive_label}", ], index=[ f"Labeled as {negative_label}", f"Labeled as {positive_label}" ], ) predicted_labels = pd.Series( (predictions.ravel() > threshold).astype(int)) predicted_probas = pd.DataFrame({ "proba_0": 1 - predictions.ravel(), "proba_1": predictions.ravel(), }) if mapping is not None: labeled_target = target["target"].map(mapping) predicted_labels = predicted_labels.map(mapping) else: labeled_target = target return { "metric_details": pd.DataFrame(details), "max_metrics": pd.DataFrame(max_metrics), "confusion_matrix": conf_matrix, "threshold": threshold, "additional_plots": AdditionalPlots.plots_binary(labeled_target, predicted_labels, predicted_probas), }
def binary_classification(target, predictions): predictions = np.array(predictions) sorted_predictions = np.sort(predictions) STEPS = 100 # can go lower for speed increase ??? details = { "threshold": [], "f1": [], "accuracy": [], "precision": [], "recall": [], "mcc": [], } samples_per_step = max(1, np.floor(predictions.shape[0] / STEPS)) for i in range(STEPS): idx = int(i * samples_per_step) if idx + 1 >= predictions.shape[0]: break if i == 0: th = 0.9 * np.min(sorted_predictions) else: th = float( 0.5 * (sorted_predictions[idx] + sorted_predictions[idx + 1])) if np.sum(predictions > th) < 1: break response = (predictions > th).astype(int) details["threshold"] += [th] details["f1"] += [f1_score(target, response)] details["accuracy"] += [accuracy_score(target, response)] details["precision"] += [precision_score(target, response)] details["recall"] += [recall_score(target, response)] if i == 0: details["mcc"] += [0.0] else: details["mcc"] += [matthews_corrcoef(target, response)] # max metrics max_metrics = { "logloss": { "score": logloss(target, predictions), "threshold": None, }, # there is no threshold for LogLoss "auc": { "score": roc_auc_score(target, predictions), "threshold": None, }, # there is no threshold for AUC "f1": { "score": np.max(details["f1"]), "threshold": details["threshold"][np.argmax(details["f1"])], }, "accuracy": { "score": np.max(details["accuracy"]), "threshold": details["threshold"][np.argmax(details["accuracy"])], }, "precision": { "score": np.max(details["precision"]), "threshold": details["threshold"][np.argmax(details["precision"])], }, "recall": { "score": np.max(details["recall"]), "threshold": details["threshold"][np.argmax(details["recall"])], }, "mcc": { "score": np.max(details["mcc"]), "threshold": details["threshold"][np.argmax(details["mcc"])], }, } # confusion matrix conf_matrix = confusion_matrix( target, predictions > max_metrics["f1"]["threshold"]) conf_matrix = pd.DataFrame( conf_matrix, columns=["Predicted as negative", "Predicted as positive"], index=["Labeled as negative", "Labeled as positive"], ) return { "metric_details": pd.DataFrame(details), "max_metrics": pd.DataFrame(max_metrics), "confusion_matrix": conf_matrix, "threshold": float(max_metrics["f1"]["threshold"]), }