def test_get_success_ratio_all_zeros(self):
"""Ensures correct output from get_success_ratio.
In this case, input values are all zero.
"""
this_success_ratio = model_eval.get_success_ratio(
CONTINGENCY_TABLE_ALL_ZEROS)
self.assertTrue(numpy.isnan(this_success_ratio))
def test_get_success_ratio(self):
"""Ensures correct output from get_success_ratio.
In this case, input values are non-zero.
"""
this_success_ratio = model_eval.get_success_ratio(
CONTINGENCY_TABLE_THRESHOLD_HALF)
self.assertTrue(
numpy.isclose(this_success_ratio,
SUCCESS_RATIO_THRESHOLD_HALF,
atol=TOLERANCE))
def run_evaluation(class_probability_matrix, observed_labels, output_dir_name):
"""Evaluates a set of multiclass probabilistic predictions.
E = number of examples
K = number of classes
:param class_probability_matrix: E-by-K numpy array, where
class_probability_matrix[i, k] = probability that the [i]th example
belongs to the [k]th class. Classes should be mutually exclusive and
collectively exhaustive, so that the sum across each row is 1.0.
:param observed_labels: length-E numpy array of observed labels. Each label
must be an integer from 0...(K - 1).
:param output_dir_name: Name of output directory. Results will be saved
here.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
print 'Finding best binarization threshold (front vs. no front)...'
binarization_threshold, best_gerrity_score = (
eval_utils.find_best_binarization_threshold(
class_probability_matrix=class_probability_matrix,
observed_labels=observed_labels,
threshold_arg=model_eval.THRESHOLD_ARG_FOR_UNIQUE_FORECASTS,
criterion_function=eval_utils.get_gerrity_score,
optimization_direction=eval_utils.MAX_OPTIMIZATION_DIRECTION,
forecast_precision_for_thresholds=FORECAST_PRECISION_FOR_THRESHOLDS
))
print(
'Best binarization threshold = {0:.4f} ... corresponding Gerrity score '
'= {1:.4f}').format(binarization_threshold, best_gerrity_score)
print 'Determinizing multiclass probabilities...'
predicted_labels = eval_utils.determinize_probabilities(
class_probability_matrix=class_probability_matrix,
binarization_threshold=binarization_threshold)
contingency_matrix = eval_utils.get_contingency_table(
predicted_labels=predicted_labels,
observed_labels=observed_labels,
num_classes=class_probability_matrix.shape[1])
print 'Multiclass contingency table is shown below:\n{0:s}'.format(
str(contingency_matrix))
print SEPARATOR_STRING
accuracy = eval_utils.get_accuracy(contingency_matrix)
peirce_score = eval_utils.get_peirce_score(contingency_matrix)
heidke_score = eval_utils.get_heidke_score(contingency_matrix)
gerrity_score = eval_utils.get_gerrity_score(contingency_matrix)
print(
'Multiclass accuracy = {0:.4f} ... Peirce score = {1:.4f} ... '
'Heidke score = {2:.4f} ... Gerrity score = {3:.4f}\n').format(
accuracy, peirce_score, heidke_score, gerrity_score)
binary_contingency_dict = model_eval.get_contingency_table(
forecast_labels=(predicted_labels > 0).astype(int),
observed_labels=(observed_labels > 0).astype(int))
print 'Binary contingency table is shown below:\n{0:s}'.format(
str(binary_contingency_dict))
print SEPARATOR_STRING
binary_pod = model_eval.get_pod(binary_contingency_dict)
binary_pofd = model_eval.get_pofd(binary_contingency_dict)
binary_success_ratio = model_eval.get_success_ratio(
binary_contingency_dict)
binary_focn = model_eval.get_focn(binary_contingency_dict)
binary_accuracy = model_eval.get_accuracy(binary_contingency_dict)
binary_csi = model_eval.get_csi(binary_contingency_dict)
binary_frequency_bias = model_eval.get_frequency_bias(
binary_contingency_dict)
print(
'Binary POD = {0:.4f} ... POFD = {1:.4f} ... success ratio = {2:.4f} '
'... FOCN = {3:.4f} ... accuracy = {4:.4f} ... CSI = {5:.4f} ... '
'frequency bias = {6:.4f}\n').format(binary_pod, binary_pofd,
binary_success_ratio, binary_focn,
binary_accuracy, binary_csi,
binary_frequency_bias)
auc_by_class, sklearn_auc_by_class = _plot_roc_curves(
class_probability_matrix=class_probability_matrix,
observed_labels=observed_labels,
output_dir_name=output_dir_name)
print '\n'
aupd_by_class = _plot_performance_diagrams(
class_probability_matrix=class_probability_matrix,
observed_labels=observed_labels,
output_dir_name=output_dir_name)
print '\n'
reliability_by_class, bss_by_class = _plot_attributes_diagrams(
class_probability_matrix=class_probability_matrix,
observed_labels=observed_labels,
output_dir_name=output_dir_name)
print '\n'
evaluation_file_name = '{0:s}/model_evaluation.p'.format(output_dir_name)
print 'Writing results to: "{0:s}"...\n'.format(evaluation_file_name)
eval_utils.write_evaluation_results(
class_probability_matrix=class_probability_matrix,
observed_labels=observed_labels,
binarization_threshold=binarization_threshold,
accuracy=accuracy,
peirce_score=peirce_score,
heidke_score=heidke_score,
gerrity_score=gerrity_score,
binary_pod=binary_pod,
binary_pofd=binary_pofd,
binary_success_ratio=binary_success_ratio,
binary_focn=binary_focn,
binary_accuracy=binary_accuracy,
binary_csi=binary_csi,
binary_frequency_bias=binary_frequency_bias,
auc_by_class=auc_by_class,
scikit_learn_auc_by_class=sklearn_auc_by_class,
aupd_by_class=aupd_by_class,
reliability_by_class=reliability_by_class,
bss_by_class=bss_by_class,
pickle_file_name=evaluation_file_name)
def run_evaluation(forecast_probabilities, observed_labels, output_dir_name):
"""Evaluates forecast-observation pairs from any forecasting method.
Specifically, this method does the following:
- creates ROC (receiver operating characteristic) curve
- creates performance diagram
- creates attributes diagram
- saves each of the aforelisted figures to a .jpg file
- computes many performance metrics and saves them to a Pickle file
:param forecast_probabilities: length-N numpy array of forecast event
probabilities.
:param observed_labels: length-N numpy array of observed labels (1 for
"yes", 0 for "no").
:param output_dir_name: Name of output directory.
"""
file_system_utils.mkdir_recursive_if_necessary(
directory_name=output_dir_name)
# TODO(thunderhoser): Make binarization threshold an input argument to this
# method.
(binarization_threshold, best_csi
) = model_eval.find_best_binarization_threshold(
forecast_probabilities=forecast_probabilities,
observed_labels=observed_labels,
threshold_arg=model_eval.THRESHOLD_ARG_FOR_UNIQUE_FORECASTS,
criterion_function=model_eval.get_csi,
optimization_direction=model_eval.MAX_OPTIMIZATION_DIRECTION,
unique_forecast_precision=FORECAST_PRECISION_FOR_THRESHOLDS)
print (
'Best binarization threshold = {0:.4f} ... corresponding CSI = {1:.4f}'
).format(binarization_threshold, best_csi)
print 'Binarizing forecast probabilities...'
forecast_labels = model_eval.binarize_forecast_probs(
forecast_probabilities=forecast_probabilities,
binarization_threshold=binarization_threshold)
print 'Creating contingency table...'
contingency_table_as_dict = model_eval.get_contingency_table(
forecast_labels=forecast_labels, observed_labels=observed_labels)
print '{0:s}\n'.format(str(contingency_table_as_dict))
print 'Computing performance metrics...'
pod = model_eval.get_pod(contingency_table_as_dict)
pofd = model_eval.get_pofd(contingency_table_as_dict)
success_ratio = model_eval.get_success_ratio(contingency_table_as_dict)
focn = model_eval.get_focn(contingency_table_as_dict)
accuracy = model_eval.get_accuracy(contingency_table_as_dict)
csi = model_eval.get_csi(contingency_table_as_dict)
frequency_bias = model_eval.get_frequency_bias(contingency_table_as_dict)
peirce_score = model_eval.get_peirce_score(contingency_table_as_dict)
heidke_score = model_eval.get_heidke_score(contingency_table_as_dict)
print (
'POD = {0:.4f} ... POFD = {1:.4f} ... success ratio = {2:.4f} ... '
'FOCN = {3:.4f} ... accuracy = {4:.4f} ... CSI = {5:.4f} ... frequency '
'bias = {6:.4f} ... Peirce score = {7:.4f} ... Heidke score = {8:.4f}\n'
).format(pod, pofd, success_ratio, focn, accuracy, csi, frequency_bias,
peirce_score, heidke_score)
auc, scikit_learn_auc = _create_roc_curve(
forecast_probabilities=forecast_probabilities,
observed_labels=observed_labels, output_dir_name=output_dir_name)
print '\n'
bss_dict = _create_attributes_diagram(
forecast_probabilities=forecast_probabilities,
observed_labels=observed_labels, output_dir_name=output_dir_name)
print '\n'
aupd = _create_performance_diagram(
forecast_probabilities=forecast_probabilities,
observed_labels=observed_labels, output_dir_name=output_dir_name)
print '\n'
evaluation_file_name = '{0:s}/model_evaluation.p'.format(output_dir_name)
print 'Writing results to: "{0:s}"...'.format(evaluation_file_name)
model_eval.write_results(
forecast_probabilities=forecast_probabilities,
observed_labels=observed_labels,
binarization_threshold=binarization_threshold, pod=pod, pofd=pofd,
success_ratio=success_ratio, focn=focn, accuracy=accuracy, csi=csi,
frequency_bias=frequency_bias, peirce_score=peirce_score,
heidke_score=heidke_score, auc=auc, scikit_learn_auc=scikit_learn_auc,
aupd=aupd, bss_dict=bss_dict, pickle_file_name=evaluation_file_name)