def calc_performance_metrics( labels: np.ndarray, predictions: np.ndarray, decimal_points: Optional[int] = 4) -> _PerformanceMetrics: """Calculates performance metrics related to a regression model. Args: labels: An array of true labels containing numeric values. predictions: An array of predictions containing numeric values. decimal_points: Number of decimal points to use when outputting the calculated performance metrics. Returns: Object of _PerformanceMetrics class containing the regression diagnostics metrics. """ utils.assert_label_and_prediction_length_match(labels, predictions) mse = metrics.mean_squared_error(labels, predictions) rmse = np.sqrt(mse) mae = metrics.mean_absolute_error(labels, predictions) mape = metrics.mean_absolute_percentage_error(labels, predictions) r2 = metrics.r2_score(labels, predictions) corr = sp.stats.pearsonr(labels, predictions)[0] return _PerformanceMetrics( mean_squared_error=round(mse, decimal_points), root_mean_squared_error=round(rmse, decimal_points), mean_absolute_error=round(mae, decimal_points), mean_absolute_percentage_error=round(mape, decimal_points), r_squared=round(r2, decimal_points), pearson_correlation=round(corr, decimal_points))
def plot_prediction_residuals(labels: np.ndarray, predictions: np.ndarray, fig_width: Optional[int] = 12, fig_height: Optional[int] = 12, title_fontsize: Optional[int] = 12, axis_label_fontsize: Optional[int] = 10, use_log: Optional[bool] = False) -> axes.Axes: """Plots scatter plots of true labels and residuals versus the predicted values. Args: labels: An array of true labels containing numeric values. predictions: An array of predictions containing numeric values. fig_width: Width of the figure. fig_height: Height of the figure. title_fontsize: Title font size of the plots. axis_label_fontsize: Axis label font size of the plots. use_log: Boolean value indicating taking logarithm of the actual and predicted values. Returns: plots: Scatter plots of true values and residuals versus the predicted values. """ utils.assert_label_and_prediction_length_match(labels, predictions) _, plots = pyplot.subplots(nrows=2, figsize=(fig_width, fig_height)) if use_log: plots[0].scatter(x=np.log1p(predictions), y=np.log1p(labels)) plots[0].set_title( 'Scatter plot of true label values versus predicted values with log transformation', fontsize=title_fontsize) plots[0].set_xlabel('Logarithm of predicted values', fontsize=axis_label_fontsize) plots[0].set_ylabel('Logarithm of label values', fontsize=axis_label_fontsize) else: plots[0].scatter(x=predictions, y=labels) plots[0].set_title( 'Scatter plot of true label values versus predicted values', fontsize=title_fontsize) plots[0].set_xlabel('Predicted values', fontsize=axis_label_fontsize) plots[0].set_ylabel('Label values', fontsize=axis_label_fontsize) plots[1].scatter(x=predictions, y=labels - predictions) plots[1].set_title('Scatter plot of residuals versus predicted values', fontsize=title_fontsize) plots[1].set_xlabel('Predicted values', fontsize=axis_label_fontsize) plots[1].set_ylabel('Residuals', fontsize=axis_label_fontsize) plots[1].axhline(0, linestyle='--') return plots
def plot_predicted_probabilities(labels: np.ndarray, probability_predictions: np.ndarray, colors: Optional[Sequence[str]] = ('b', 'g'), print_stats: bool = True, fig_width: Optional[int] = 20, fig_height: Optional[int] = 15) -> axes.Axes: """Plots the distributions of predicted probabilities for each class. Args: labels: An array of true binary labels represented by 1.0 and 0.0. probability_predictions: An array of predicted probabilities between 0.0 and 1.0. colors: Colors for the probability plots. print_stats: Flag that whether the stats of probabilities are plotted fig_width: Width of the figure. fig_height: Height of the figure. Returns: plots: Class density plots of the predicted probabilities. """ utils.assert_label_values_are_valid(labels) utils.assert_prediction_values_are_valid(probability_predictions) utils.assert_label_and_prediction_length_match(labels, probability_predictions) assert len(np.unique(labels)) == len(colors),\ 'number of colors should be the same as number of unique labels.' unique_labels = np.sort(np.unique(labels)) _, plots = pyplot.subplots(figsize=(fig_width, fig_height)) for color, label in zip(colors, unique_labels): index_plot = np.where(labels == label)[0] preds_plot = probability_predictions[index_plot] label_plot = 'class[%s]' % (str(label)) if print_stats: label_plot += ': mean=%.4f, std=%.4f, median=%.4f' % ( np.mean(preds_plot), np.std(preds_plot), np.median(preds_plot)) sns.kdeplot(preds_plot, shade=True, color=color, label=label_plot, ax=plots) pyplot.title('Distribution of predicted probabilities') pyplot.legend() pyplot.xlabel('Probability') pyplot.ylabel('Density') pyplot.xlim([0, 1]) return plots
def plot_confusion_matrix(labels: np.ndarray, predictions: np.ndarray, class_label_names: Optional[Dict[Union[str, int], Union[str, int]]] = None, normalize: Optional[str] = None, title_fontsize: Optional[int] = 12, x_label_fontsize: Optional[int] = 12, y_label_fontsize: Optional[int] = 12, heatmap_color: Optional[str] = 'Greens') -> None: """Plot confusion matrix for a multiclass classification model. Args: labels: An array of true labels containing multiclass labels. predictions: An array of predictions containing multiclass labels. class_label_names: Dictionary of multiclass labels and corresponding target names. The type of both class lable and target names can be either 'int' or 'str'. E.g. {0: 'low_value', 1: 'mid_value', 2: 'high_value'}. normalize: A parameter controlling whether to normalize the counts in the matrix. title_fontsize: Font size of the figure title. x_label_fontsize: Font size of the x axis labels. y_label_fontsize: Font size of the y axis labels. heatmap_color: Color of the heatmap plot. Returns: Heatmap of confusion matrix. """ utils.assert_label_and_prediction_length_match(labels, predictions) if class_label_names is None: class_labels = list(set(labels)) target_names = ['%s' % l for l in class_labels] else: class_labels = list(class_label_names.keys()) target_names = list(class_label_names.values()) plot = ConfusionMatrixDisplay.from_predictions(y_true=labels, y_pred=predictions, labels=np.unique(labels), display_labels=target_names, normalize=normalize, include_values=True, cmap=heatmap_color) plot.ax_.set_title('Confusion matrix', fontsize=title_fontsize) plot.ax_.set_xlabel('Predicted label', fontsize=x_label_fontsize) plot.ax_.set_ylabel('Actual label', fontsize=y_label_fontsize) plt.show()
def plot_precision_recall_curve( labels: np.ndarray, probability_predictions: np.ndarray, print_stats: bool = True, fig_width: Optional[int] = 8, fig_height: Optional[int] = 8, curve_color: Optional[str] = 'blue') -> axes.Axes: """Plots the Precision-Recall curve for the predictions. Args: labels: An array of true binary labels represented by 1.0 and 0.0. probability_predictions: An array of predicted probabilities between 0.0 and 1.0. print_stats: Flag that whether the Average Precision is plotted. fig_width: Width of the figure. fig_height: Height of the figure. curve_color: Color of the Precision-Recall curve. Returns: plots: Class density plots of the ROC curve. """ utils.assert_label_values_are_valid(labels) utils.assert_prediction_values_are_valid(probability_predictions) utils.assert_label_and_prediction_length_match(labels, probability_predictions) precision, recall, _ = sklearn.metrics.precision_recall_curve( labels, probability_predictions) _, plots = pyplot.subplots(figsize=(fig_width, fig_height)) pyplot.plot(recall, precision, marker='.', label='Precision-Recall', color=curve_color) pyplot.xlabel('Recall') pyplot.ylabel('Precision') pyplot.legend() if print_stats: pyplot.title('Average Precision=%.4f' % sklearn.metrics.average_precision_score( labels, probability_predictions)) return plots
def plot_binned_preds_labels(labels: np.ndarray, predictions: np.ndarray, number_bins: Optional[int] = 10, fig_width: Optional[int] = 10, fig_height: Optional[int] = 7) -> axes.Axes: """Plots the actual label distributions (box plots) for prediction bins. Args: labels: An array of true labels containing numeric values. predictions: An array of predictions containing numeric values. number_bins: Number of bins that we want to divide the ranked predictions into. Default is deciles (10 bins) such that the 1st bin contains the highest 10% of the predictions, the 2nd bin contains the next 10% of the predictions and so on. fig_width: Width of the figure. fig_height: Height of the figure. Returns: plot: Box plots of the actual label distributions for prediction bins. """ utils.assert_label_and_prediction_length_match(labels, predictions) # Separate the predictions into bins. data = pd.DataFrame(list(zip(labels, predictions)), columns=['labels', 'predictions']) data = data.sort_values('predictions', ascending=False) data['prediction_rank'] = range(data.shape[0]) # To avoid duplicate edges of bins use the index in the qcat function below. data['prediction_bin_number'] = pd.qcut( data['prediction_rank'], q=number_bins, labels=False) + 1 _, plot = pyplot.subplots(figsize=(fig_width, fig_height)) data.pivot(columns='prediction_bin_number', values='labels').boxplot() plot.set_title('Distribution of actual labels over prediction bins') plot.set_ylabel('Actual label distribution') plot.set_xlabel('Prediction bin [Highest to Lowest]') return plot
def plot_confusion_matrix_bin_heatmap( labels: np.ndarray, predictions: np.ndarray, number_bins: Optional[int] = 10, normalize: Optional[str] = 'true', fig_width: Optional[int] = 12, fig_height: Optional[int] = 12, title_fontsize: Optional[int] = 12, axis_label_fontsize: Optional[int] = 10, heatmap_color: Optional[str] = 'YlGnBu') -> axes.Axes: """Plots the heatmap of the bins of the actual and predicted values. Args: labels: An array of true labels containing numeric values. predictions: An array of predictions containing numeric values. number_bins: Number of bins that we want to divide the ranked predictions into. Default is deciles (10 bins) such that the 1st bin contains the highest 10% of the predictions, the 2nd bin contains the next 10% of the predictions and so on. normalize: Normalizes confusion matrix over the true labels (rows), predictions (columns) conditions or all the population. Takes the values 'true', 'pred' and 'all' respectively. fig_width: Width of the figure. fig_height: Height of the figure. title_fontsize: Title font size of the plots. axis_label_fontsize: Axis label font size of the plots. heatmap_color: Color of the heatmap plot. Returns: plot: Heatmap of the bins of the actual and predicted values. """ utils.assert_label_and_prediction_length_match(labels, predictions) assert str(normalize).lower() in [ 'true', 'pred', 'all' ], ("normalize parameter value should be either 'true', 'pred' or 'all'") data = pd.DataFrame(list(zip(labels, predictions)), columns=['labels', 'predictions']) data = data.sort_values('labels', ascending=False) data['labels_rank'] = range(data.shape[0]) data = data.sort_values('predictions', ascending=False) data['prediction_rank'] = range(data.shape[0]) data['bin_number_label'] = pd.qcut(data['labels_rank'], q=number_bins, labels=False) data['bin_number_predictions'] = pd.qcut(data['prediction_rank'], q=number_bins, labels=False) conf_matrix = metrics.confusion_matrix( y_true=data['bin_number_label'], y_pred=data['bin_number_predictions'], normalize=normalize) tick_labels = range(1, number_bins + 1) _, plot = pyplot.subplots(figsize=(fig_width, fig_height)) plot = sns.heatmap(conf_matrix, cbar=False, cmap=heatmap_color, annot=True, xticklabels=tick_labels, yticklabels=tick_labels) plot.set_title('Heatmap of the bins of the actual and predicted values', fontsize=title_fontsize) plot.set_xlabel('Prediction value bins [Highest to Lowest]', fontsize=axis_label_fontsize) plot.set_ylabel('Actual value bins [Highest to Lowest]', fontsize=axis_label_fontsize) return plot
def calc_reg_bin_metrics(labels: np.ndarray, predictions: np.ndarray, number_bins: Optional[int] = 10, decimal_points: Optional[int] = 4) -> pd.DataFrame: """Calculates performance metrics for each bin of the predictions. Args: labels: An array of true labels containing numeric values. predictions: An array of predictions containing numeric values. number_bins: Number of bins that we want to divide the ranked predictions into. Default is deciles (10 bins) such that the 1st bin contains the highest 10% of the predictions, the 2nd bin contains the next 10% of the predictions and so on. decimal_points: Number of decimal points to use when outputting the calculated performance metrics. Returns: bin_metrics: Following metrics calculated for each bin: mean_label: Mean of actual values in the bin. mean_prediction: Mean of predictions in the bin. rmse: Root mean squared error. mape: Mean absolute percentage error. corr: pearson_correlation coefficient. """ utils.assert_label_and_prediction_length_match(labels, predictions) # Separate the predictions into bins. binned_data = pd.DataFrame(list(zip(labels, predictions)), columns=['label', 'prediction']) binned_data = binned_data.sort_values('prediction').reset_index() # To avoid duplicate edges of bins use the index in the qcat function below. binned_data['bin_number'] = pd.qcut(binned_data.index, q=number_bins, labels=False) bin_metrics = binned_data.groupby('bin_number', as_index=False).agg({ 'label': 'mean', 'prediction': 'mean' }).rename(columns={ 'label': 'mean_label', 'prediction': 'mean_prediction' }) bin_metrics['mean_label'] = round(bin_metrics['mean_label'], decimal_points) bin_metrics['mean_prediction'] = round(bin_metrics['mean_prediction'], decimal_points) bin_metrics['rmse'] = 0 bin_metrics['mape'] = 0 bin_metrics['corr'] = 0 for i in range(number_bins): (bin_labels, bin_predictions) = ( binned_data[binned_data.bin_number == i]['label'].values, binned_data[binned_data.bin_number == i]['prediction'].values) bin_perf_metrics = calc_performance_metrics(bin_labels, bin_predictions, decimal_points) bin_metrics.loc[i, 'rmse'] = bin_perf_metrics.root_mean_squared_error bin_metrics.loc[ i, 'mape'] = bin_perf_metrics.mean_absolute_percentage_error bin_metrics.loc[i, 'corr'] = bin_perf_metrics.pearson_correlation bin_metrics['bin_number'] = number_bins - bin_metrics['bin_number'] bin_metrics = bin_metrics.sort_values(['bin_number']) return bin_metrics
def calc_chisquared_sample_sizes_for_bins( labels: np.ndarray, probability_predictions: np.ndarray, number_bins: int = 3, uplift_percentages: Sequence[np.float64] = (10, 20), power_percentages: Sequence[np.float64] = (80, 90), confidence_level_percentages: Sequence[np.float64] = (90, 95) ) -> pd.DataFrame: """Calculates statistical sample sizes for the bins defined on predictions. These sample sizes for the bins defined on the predicted probabilities are estimated using the Chi-squared test of proportions for each combination of uplift_percentage, power_percentage and confidence_level_percentage. These sizes could be used as the minimum required size for each Test or Control group when designing an experiment to target users from each of these bins of predictions. Args: labels: An array of true binary labels represented by 1.0 and 0.0. probability_predictions: An array of predicted probabilities between 0.0 and 1.0. number_bins: Number of bins that we want to divide the ranked predictions into. Default is 3 bins such that the 1st bin contains the highest 1/3rd of the predictions (High Propensity group), the 2nd bin contains the next 1/3rd of the predictions (Medium Propensity group) and the last bin contains the lowest 1/3rd of the predictions (Lowest Propensity group). uplift_percentages: Sequence of different expected uplift percentages. power_percentages: Sequence of different statistical powers for the test. confidence_level_percentages: Sequence of different statistical confidence levels for the test. Returns: bin_metrics: Following metrics calculated for each bin of the predictions. bin_number: Bin number starting from 1. bin_size: Total numbers of instances in the bin. min_probability: Minimum predicted probability within the bin. conversion_rate: Proportion of positive instances out of all the instances in the bin (precision). expected_uplift: Expected uplift_percentage. power_percentage: Statistical power of the test. confidence_level_percentage: Statistical confidence level of the test. required_sample_size: Statistical sample size required. """ utils.assert_label_values_are_valid(labels) utils.assert_prediction_values_are_valid(probability_predictions) utils.assert_label_and_prediction_length_match(labels, probability_predictions) # separate the probability_predictions into bins of equal size binned_data = pd.DataFrame(list(zip(labels, probability_predictions)), columns=['label', 'prediction']) binned_data = binned_data.sort_values('prediction').reset_index() # to avoid duplicate edges of bins use the index in the qcat function below binned_data['bin_number'] = pd.qcut(binned_data.index, q=number_bins, labels=False) # calculate the conversion rate for each bin total_instances = (binned_data[['bin_number', 'label']].groupby('bin_number').count()) total_instances.columns = ['bin_size'] total_instances = total_instances.reset_index() positive_instances = (binned_data.loc[binned_data['label'] > 0][[ 'bin_number', 'label' ]].groupby('bin_number').count()) positive_instances.columns = ['positive_instances'] positive_instances = positive_instances.reset_index() bin_conv_rate = pd.merge(total_instances, positive_instances, on='bin_number', how='left') bin_conv_rate.fillna(0, inplace=True) bin_conv_rate['conversion_rate'] = round( (bin_conv_rate['positive_instances'] / bin_conv_rate['bin_size'] * 100), 2) bin_metrics_list = list() for bin_number in bin_conv_rate['bin_number']: conv_rate = bin_conv_rate['conversion_rate'][bin_number] bin_size = bin_conv_rate['bin_size'][bin_number] min_prob = min( binned_data[binned_data['bin_number'] == bin_number]['prediction']) for uplift_percentage in uplift_percentages: for power_percentage in power_percentages: for confidence_level_percentage in confidence_level_percentages: sample_size = calc_chisquared_sample_size( conv_rate, uplift_percentage, power_percentage, confidence_level_percentage) bin_metrics_list.append( (bin_number, bin_size, min_prob, conv_rate, uplift_percentage, power_percentage, confidence_level_percentage, sample_size)) bin_metrics = pd.DataFrame(bin_metrics_list, columns=[ 'bin_number', 'bin_size', 'min_probability', 'conv_rate_percentage', 'uplift_percentage', 'power_percentage', 'confidence_level_percentage', 'required_sample_size' ]) # reverse the order of bin numbers such that bin 1 has the highest # predicted probability bin_metrics['bin_number'] = number_bins - bin_metrics['bin_number'] bin_metrics = bin_metrics.sort_values(['bin_number' ]).reset_index(drop=True) return bin_metrics
def calc_t_sample_sizes_for_cumulative_bins( labels: np.ndarray, numeric_predictions: np.ndarray, number_bins: int = 10, uplift_percentages: Sequence[np.float64] = (10, 20), power_percentages: Sequence[np.float64] = (80, 90), confidence_level_percentages: Sequence[np.float64] = (90, 95) ) -> pd.DataFrame: """Calculates statistical sample sizes for the cumulative bins of predictions. These sample sizes are estimated using the T-test for each combination of uplift_percentage, power_percentage and confidence_level_percentage for the cumulative bins of numeric predictions from a regression model. These sizes could be used as the minimum required sizes for each Test or Control group when designing a media experiment to target users having the top X% of predicted values. Args: labels: An array of actual numeric label. numeric_predictions: An array of numeric predictions. number_bins: Number of cumulative bins that we want to divide the ranked predictions into. Default is deciles (10 bins) such that the 1st bin contains the highest 10% of the predictions, the 2nd bin contains the highest 20% of the predictions and so on. uplift_percentages: Sequence of different expected uplift percentages. power_percentages: Sequence of different statistical power percentages. confidence_level_percentages: Sequence of different statistical confidence level percentages. Returns: bin_metrics: Following metrics calculated for each cumulative bin. cumulative_bin_number: Bin number starting from 1 for the bin having largest predicted values. bin_size: Total numbers of instances in the bin. bin_size_percentage: Percentage of instances in the bin out of all the instances in the labels. min_predicted_val: Minimum predicted value of the bin. average_actual_val: Average actual label value of the bin. stdev_actual_val: Standard deviation of actual label value of the bin. expected_uplift: Expected uplift percentage of the test. power_percentage: Statistical power percentage of the test. confidence_level_percentage: Statistical confidence level percentage of the test. required_sample_size: Statistical sample size required. """ utils.assert_label_and_prediction_length_match(labels, numeric_predictions) # Separate the probability_predictions into bins label_predictions = pd.DataFrame(list(zip(labels, numeric_predictions)), columns=['label', 'prediction']) label_predictions = label_predictions.sort_values(by='prediction', ascending=False) number_total_instances = label_predictions.shape[0] equal_bin_size = number_total_instances / number_bins # Calculate the stats for cumulative bins cumulative_bin_metrics_list = [] for bin_number in range(1, (number_bins + 1)): current_bin_size = round(equal_bin_size * bin_number) bin_size_percentage = round( current_bin_size / number_total_instances * 100, 2) bin_instances = label_predictions.head(current_bin_size) mean_actual_val = round(np.mean(bin_instances['prediction']), 2) stdev_actual_val = round(np.std(bin_instances['prediction']), 2) min_predicted_val = min(bin_instances['prediction']) for uplift_percentage in uplift_percentages: for power_percentage in power_percentages: for confidence_level_percentage in confidence_level_percentages: sample_size = calc_t_sample_size( mean_actual_val, stdev_actual_val, uplift_percentage, power_percentage, confidence_level_percentage) cumulative_bin_metrics_list.append( (bin_number, current_bin_size, bin_size_percentage, min_predicted_val, mean_actual_val, stdev_actual_val, uplift_percentage, power_percentage, confidence_level_percentage, sample_size)) return pd.DataFrame(cumulative_bin_metrics_list, columns=[ 'cumulative_bin_number', 'bin_size', 'bin_size_percentage', 'min_predicted_val', 'mean_actual_val', 'stdev_actual_val', 'uplift_percentage', 'power_percentage', 'confidence_level_percentage', 'required_sample_size' ])
def calc_t_sample_sizes_for_bins( labels: np.ndarray, numeric_predictions: np.ndarray, number_bins: int = 3, uplift_percentages: Sequence[np.float64] = (10, 20), power_percentages: Sequence[np.float64] = (80, 90), confidence_level_percentages: Sequence[np.float64] = (90, 95) ) -> pd.DataFrame: """Calculates statistical sample sizes for the bins of numeric predictions. These sample sizes (for the bins defined on the numeric predictions from a regression model) are estimated using the T-test for each combination of uplift_percentage, power_percentage and confidence_level_percentage. These sizes could be used as the minimum required size for each Test or Control group when designing a media experiment to target users from each prediction bin. Args: labels: An array of actual numeric label. numeric_predictions: An array of numetric predictions. number_bins: Number of bins that we want to divide the ranked predictions into. Default is 3 bins such that the 1st bin contains the highest 1/3rd of the predictions (High value group), the 2nd bin contains the next 1/3rd of the predictions (Medium value group) and the last bin contains the lowest 1/3rd of the predictions (Low value group). uplift_percentages: Sequence of different expected uplift percentages. power_percentages: Sequence of different statistical power percenrtages. confidence_level_percentages: Sequence of different statistical confidence level percentages. Returns: bin_metrics: Following metrics calculated for each bin of the predictions. bin_number: Bin number starting from 1 for the bin with the highest values. bin_size: Total numbers of instances in the bin. min_predicted_val: Minimum predicted value within the bin. average_actual_val: Average actual label value of the bin. stdev_actual_val: Standard deviation of actual label value of the bin. expected_uplift: Expected uplift percentage of the test. power_percentage: Statistical power percentage of the test. confidence_level_percentage: Statistical confidence level percentage of the test. required_sample_size: Statistical sample size required. """ utils.assert_label_and_prediction_length_match(labels, numeric_predictions) # Separate the probability_predictions into bins of equal size binned_data = pd.DataFrame(list(zip(labels, numeric_predictions)), columns=['label', 'prediction']) binned_data = binned_data.sort_values( 'prediction', ascending=False, ).reset_index(drop=True) # To avoid duplicate edges of bins use the index in the qcut function below binned_data['bin_number'] = pd.qcut(binned_data.index, q=number_bins, labels=False) # Calculate the statistics for each bin bin_stats = (binned_data.groupby(['bin_number' ]).agg(['min', 'mean', 'std', 'count']).reset_index()) bin_stats.columns = [ 'bin_number', 'min_label', 'mean_label', 'stdev_label', 'bin_label_size', 'min_pred', 'mean_pred', 'stdev_pred', 'bin_pred_size' ] bin_metrics_list = list() for bin_number in bin_stats['bin_number']: mean_label = bin_stats['mean_label'][bin_number] stdev_label = bin_stats['stdev_label'][bin_number] bin_size = bin_stats['bin_label_size'][bin_number] min_pred_val = bin_stats['min_pred'][bin_number] for uplift_percentage in uplift_percentages: for power_percentage in power_percentages: for confidence_level_percentage in confidence_level_percentages: sample_size = calc_t_sample_size( mean_label, stdev_label, uplift_percentage, power_percentage, confidence_level_percentage) bin_metrics_list.append( (bin_number, bin_size, min_pred_val, mean_label, stdev_label, uplift_percentage, power_percentage, confidence_level_percentage, sample_size)) bin_metrics = pd.DataFrame(bin_metrics_list, columns=[ 'bin_number', 'bin_size', 'min_predicted_val', 'average_actual_val', 'stdev_actual_val', 'uplift_percentage', 'power_percentage', 'confidence_level_percentage', 'required_sample_size' ]) # Start the bin numbers from 1 bin_metrics['bin_number'] = bin_metrics['bin_number'] + 1 return bin_metrics
def calc_performance_metrics( labels: np.ndarray, probability_predictions: np.ndarray, binarize_threshold: Optional[float] = None, decimal_points: Optional[int] = 4, ) -> Dict[str, float]: """Calculates performance metrics related to a binary classification model. Args: labels: An array of true binary labels represented by 1.0 and 0.0. probability_predictions: An array of predicted probabilities between 0.0 and 1.0. binarize_threshold: Probability threshold to be used to binarize the predicted probabilities. By default the proportion of positive instances in the labels is used. decimal_points: Number of decimal points to use when outputting the calculated performance metrics. Returns: metrics: Dictionary of the following performance metric {prop_positives: Proportion of instances where label = 1.0, auc_roc: Area under the recall vs (1-specificity) (ROC) curve, auc_pr: Area under the recall vs precision (ROC) curve. Following metrics are calculated after binarizing the predicted probabilities based on the given binarize_threshold, accuracy: Total accuracy of the predictions, true_positive_rate (recall or sensitivity): True positive rate, true_negative_rate (specificity): True negative rate, precision: Precision (confidence) of the true positive predictions, f1_score: F1 score of sensitivity and specificity, precision_uplift: Uplift of the precision compared to random prediction} """ utils.assert_label_values_are_valid(labels) utils.assert_prediction_values_are_valid(probability_predictions) utils.assert_label_and_prediction_length_match(labels, probability_predictions) num_positives = labels.sum() prop_positives = float(num_positives) / len(labels) if binarize_threshold is None: binarize_threshold = prop_positives # Calculate auc metrics. auc_roc = round( sklearn.metrics.roc_auc_score(labels, probability_predictions), decimal_points) auc_pr = round( sklearn.metrics.average_precision_score(labels, probability_predictions), decimal_points) # Binarize the predictions. binarized_predictions = ((probability_predictions > binarize_threshold).astype(int)) # Calculate metrics based on binarized predictions. accuracy = sklearn.metrics.accuracy_score(labels, binarized_predictions) tp_rate = sklearn.metrics.recall_score(labels, binarized_predictions, pos_label=1) tn_rate = sklearn.metrics.recall_score(labels, binarized_predictions, pos_label=0) precision = sklearn.metrics.precision_score(labels, binarized_predictions) f1_score = sklearn.metrics.f1_score(labels, binarized_predictions) return { 'prop_positives': round(prop_positives, decimal_points), 'auc_roc': round(auc_roc, decimal_points), 'auc_pr': round(auc_pr, decimal_points), 'binarize_threshold': round(binarize_threshold, decimal_points), 'accuracy': round(accuracy, decimal_points), 'true_positive_rate': round(tp_rate, decimal_points), 'true_negative_rate': round(tn_rate, decimal_points), 'precision': round(precision, decimal_points), 'f1_score': round(f1_score, decimal_points) }
def calc_cumulative_bin_metrics( labels: np.ndarray, probability_predictions: np.ndarray, number_bins: int = 10, decimal_points: Optional[int] = 4) -> pd.DataFrame: """Calculates performance metrics for cumulative bins of the predictions. Args: labels: An array of true binary labels represented by 1.0 and 0.0. probability_predictions: An array of predicted probabilities between 0.0 and 1.0. number_bins: Number of cumulative bins that we want to divide the ranked predictions into. Default is 10 bins such that the 1st bin contains the highest 10% of the predictions, 2nd bin contains the highest 20% of the predictions and so on. decimal_points: Number of decimal points to use when outputting the calculated performance metrics. Returns: bin_metrics: Following metrics calculated for each cumulative bin. cumulative_bin_number: Bin number starting from 1. bin_size: Total numbers of instances in the bin, bin_size_proportion: Proportion of instances in the bin out of all the instances in the labels. positive_instances: Numbers of positive instances in the bin, precision: Proportion of positive instances out of all the instances in the bin, coverage (recall): Proportion of positives instances in the bin out of all the positive instances in the labels, prop_label_positives: Proportion of positive instances in the labels, precision_uplift: Uplift of precision of the bin compared to the precision of the random prediction (prop_label_positives). """ utils.assert_label_values_are_valid(labels) utils.assert_prediction_values_are_valid(probability_predictions) utils.assert_label_and_prediction_length_match(labels, probability_predictions) # Separate the probability_predictions into bins. label_predictions = pd.DataFrame(list(zip(labels, probability_predictions)), columns=['label', 'prediction']) label_predictions = label_predictions.sort_values(by='prediction', ascending=False) number_total_instances = label_predictions.shape[0] equal_bin_size = number_total_instances / number_bins number_total_positive_instances = label_predictions[ label_predictions['label'] > 0].shape[0] prop_label_positives = round( number_total_positive_instances / number_total_instances, decimal_points) cumulative_bin_metrics_list = list() for i in range(1, (number_bins + 1)): current_bin_size = round(equal_bin_size * i) bin_size_proportion = round(current_bin_size / number_total_instances, decimal_points) bin_instances = label_predictions.head(current_bin_size) number_bin_positive_instances = bin_instances[ bin_instances['label'] > 0].shape[0] bin_precision = round(number_bin_positive_instances / current_bin_size, decimal_points) bin_recall = round( number_bin_positive_instances / number_total_positive_instances, decimal_points) bin_precision_uplift = round(bin_precision / prop_label_positives, decimal_points) cumulative_bin_metrics_list.append( (i, current_bin_size, bin_size_proportion, number_bin_positive_instances, bin_precision, bin_recall, prop_label_positives, bin_precision_uplift)) return pd.DataFrame(cumulative_bin_metrics_list, columns=[ 'cumulative_bin_number', 'bin_size', 'bin_size_proportion', 'positive_instances', 'precision', 'coverage (recall)', 'prop_label_positives', 'precision_uplift' ])
def calc_bin_metrics(labels: np.ndarray, probability_predictions: np.ndarray, number_bins: Optional[int] = 10, decimal_points: Optional[int] = 4) -> pd.DataFrame: """Calculates performance metrics for each bin of the predictions. Args: labels: An array of true binary labels represented by 1.0 and 0.0. probability_predictions: An array of predicted probabilities between 0.0 and 1.0. number_bins: Number of bins that we want to divide the ranked predictions into. Default is deciles (10 bins) such that the 1st bin contains the highest 10% of the predictions, the 2nd bin contains the next 10% of the predictions and so on. decimal_points: Number of decimal points to use when outputting the calculated performance metrics. Returns: bin_metrics: Following metrics calculated for each bin. bin_number: Bin number starting from 1. bin_size: Total numbers of instances in the bin, positive_instances: Numbers of positive instances in the bin, precision: Proportion of positive instances out of all the instances in the bin, coverage: Proportion of positives instances out of all the positive instances in the dataset prop_positives: Proportion of positive instances in the label, precision_uplift: Uplift of precision compared to the precision of the random prediction (prop_positives). """ utils.assert_label_values_are_valid(labels) utils.assert_prediction_values_are_valid(probability_predictions) utils.assert_label_and_prediction_length_match(labels, probability_predictions) # Separate the probability_predictions into bins. bins = pd.qcut(probability_predictions, q=number_bins, labels=False) binned_data = pd.DataFrame(list(zip(labels, probability_predictions, bins)), columns=['label', 'prediction', 'bin_number']) # Calculate the metrics for each bin. total_instances = (binned_data[['bin_number', 'label']].groupby('bin_number').count()) total_instances.columns = ['bin_size'] total_instances = total_instances.reset_index() positive_instances = (binned_data.loc[binned_data['label'] > 0][[ 'bin_number', 'label' ]].groupby('bin_number').count()) positive_instances.columns = ['positive_instances'] positive_instances = positive_instances.reset_index() bin_metrics = pd.merge(total_instances, positive_instances, on='bin_number', how='left') bin_metrics.fillna(0, inplace=True) bin_metrics['precision'] = (bin_metrics['positive_instances'] / bin_metrics['bin_size']) bin_metrics['precision'] = [ round(val, decimal_points) for val in bin_metrics['precision'] ] prop_positives = round(labels[labels == 1.0].shape[0] / len(labels), decimal_points) bin_metrics['prop_positives'] = prop_positives # Convert bin_number from zero-based offset to 1-based offset. bin_metrics['bin_number'] = bin_metrics['bin_number'] + 1 bin_metrics['precision_uplift'] = bin_metrics['precision'] / prop_positives bin_metrics['precision_uplift'] = [ round(val, decimal_points) for val in bin_metrics['precision_uplift'] ] bin_metrics['coverage'] = (bin_metrics['positive_instances'] / sum(bin_metrics['positive_instances'])) bin_metrics['coverage'] = [ round(val, decimal_points) for val in bin_metrics['coverage'] ] # Reverse the order of bin numbers such that bin 1 has the highest # predicted probability. bin_metrics['bin_number'] = number_bins - bin_metrics['bin_number'] + 1 bin_metrics = bin_metrics.sort_values(['bin_number' ]).reset_index(drop=True) return bin_metrics
def test_assert_label_and_prediction_length_match_right_error(self): with self.assertRaises(AssertionError): utils.assert_label_and_prediction_length_match( np.array([0.0, 0.0, 1.0, 1.0]), np.array([0.0, 0.5, 0.33]))
def calc_chisquared_sample_sizes_for_cumulative_bins( labels: np.ndarray, probability_predictions: np.ndarray, number_bins: int = 10, uplift_percentages: Sequence[np.float64] = (10, 20), power_percentages: Sequence[np.float64] = (80, 90), confidence_level_percentages: Sequence[np.float64] = (90, 95) ) -> pd.DataFrame: """Calculates statistical sample sizes for the cumulative bins of predictions. These sample sizes for the cumulative bins of predicted probabilities are estimated using the Chi-squared test of proportions for each combination of uplift_percentage, power_percentage and confidence_level_percentage. These sizes could be used as the minimum required sizes for each Test or Control group when designing an experiment to target users having the top X% of predicted probabilities. Args: labels: An array of true binary labels represented by 1.0 and 0.0. probability_predictions: An array of predicted probabilities between 0.0 and 1.0. number_bins: Number of cumulative bins that we want to divide the ranked predictions into. Default is deciles (10 bins) such that the 1st bin contains the highest 10% of the predictions, the 2nd bin contains the highest 20% of the predictions and so on. uplift_percentages: Sequence of different expected uplift percentages. power_percentages: Sequence of different statistical powers for the test. confidence_level_percentages: Sequence of different statistical confidence levels for the test. Returns: bin_metrics: Following metrics calculated for each cumulative bin. cumulative_bin_number: Bin number starting from 1. bin_size: Total numbers of instances in the bin. bin_size_percentage: Percentage of instances in the bin out of all the instances in the labels. min_probability: Minimum predicted probability within the bin. conversion_rate: Proportion of positive instances out of all the instances in the bin (precision). expected_uplift: Expected uplift_percentage. power_percentage: Statistical power of the test. confidence_level_percentage: Statistical confidence level of the test. required_sample_size: Statistical sample size required. """ utils.assert_label_values_are_valid(labels) utils.assert_prediction_values_are_valid(probability_predictions) utils.assert_label_and_prediction_length_match(labels, probability_predictions) # Separate the probability_predictions into bins. label_predictions = pd.DataFrame(list(zip(labels, probability_predictions)), columns=['label', 'prediction']) label_predictions = label_predictions.sort_values(by='prediction', ascending=False) number_total_instances = label_predictions.shape[0] equal_bin_size = number_total_instances / number_bins cumulative_bin_metrics_list = [] for bin_number in range(1, (number_bins + 1)): current_bin_size = round(equal_bin_size * bin_number) bin_size_percentage = round( current_bin_size / number_total_instances * 100, 2) bin_instances = label_predictions.head(current_bin_size) positive_instance_indeces = bin_instances['label'] > 0.0 number_bin_positive_instances = bin_instances[ positive_instance_indeces].shape[0] conv_rate = round( number_bin_positive_instances / current_bin_size * 100, 2) min_prob = min(bin_instances['prediction']) for uplift_percentage in uplift_percentages: for power_percentage in power_percentages: for confidence_level_percentage in confidence_level_percentages: sample_size = calc_chisquared_sample_size( conv_rate, uplift_percentage, power_percentage, confidence_level_percentage) cumulative_bin_metrics_list.append( (bin_number, current_bin_size, bin_size_percentage, min_prob, conv_rate, uplift_percentage, power_percentage, confidence_level_percentage, sample_size)) return pd.DataFrame(cumulative_bin_metrics_list, columns=[ 'cumulative_bin_number', 'bin_size', 'bin_size_percentage', 'min_probability', 'conv_rate_percentage', 'uplift_percentage', 'power_percentage', 'confidence_level_percentage', 'required_sample_size' ])
def calc_performance_metrics( labels: np.ndarray, predictions: np.ndarray, pred_probs: np.ndarray, class_label_names: Optional[Dict[Union[str, int], Union[str, int]]] = None, decimal_points: Optional[int] = 3, average_type: Optional[str] = 'weighted', multi_class_type: Optional[str] = 'ovr') -> Dict[str, float]: """Calculates performance metrics for a multiclass classification model. Args: labels: An array of true labels containing multiclass labels. predictions: An array of predictions containing multiclass labels. pred_probs: An array of shape (n_samples, n_classes) of predicted probabilities. class_label_names: Optional. Dictionary of multiclass labels and corresponding target names. The type of both class lable and target names can be either 'int' or 'str'. E.g. {0: 'low_value', 1: 'mid_value', 2: 'high_value'}. decimal_points: Number of decimal points to use when outputting the calculated evaluation metrics. average_type: The averaging method applied to the data while calculating scores. multi_class_type: The method applied to AUC calculation. It can take 'ovr' or 'ovo'. 'ovr' stands for One-vs-rest. 'ovo' stands for One-vs-one. Returns: Dictionary of evaluation metrics of a multiclass classification model: {classification_report: Summary report of precision, recall, F1 score for each class. auc_roc_score: Area under the recall vs (1-specificity) (ROC) curve. confusion_matrix: Confusion matrix to evaluate the accuracy of each class. """ utils.assert_label_and_prediction_length_match(labels, predictions) assert len(labels) == pred_probs.shape[0], ( 'The true labels and prediction probability should have the same length.' ) assert len(set(labels)) == pred_probs.shape[1], ( 'The number of classes of labels and prediction probability should be ' 'the same.') if class_label_names is None: class_labels = list(set(labels)) target_names = ['%s' % l for l in class_labels] else: class_labels = list(class_label_names.keys()) target_names = list(class_label_names.values()) class_report = classification_report(y_true=labels, y_pred=predictions, labels=class_labels, target_names=target_names) auc_score = roc_auc_score(y_true=labels, y_score=pred_probs, average=average_type, multi_class=multi_class_type) conf_matrix = confusion_matrix(y_true=labels, y_pred=predictions, labels=class_labels) return { 'classification_report': class_report, 'auc_roc_score': round(auc_score, decimal_points), 'confusion_matrix': conf_matrix }