def figures_mesa_three_class():
classifiers = utils.get_classifiers()
# Uncomment to just use MLP:
# classifiers = [AttributedClassifier(name='Neural Net', classifier=MLPClassifier(activation='relu', hidden_layer_sizes=(15, 15, 15),
# max_iter=1000, alpha=0.01, solver='lbfgs'))]
feature_sets = utils.get_base_feature_sets()
three_class_performance_summaries = []
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = ThreeClassClassifierSummaryBuilder.build_mesa_leave_one_out(
attributed_classifier, feature_sets)
PerformancePlotBuilder.make_bland_altman(classifier_summary, '_mesa')
PerformancePlotBuilder.make_single_threshold_histograms(
classifier_summary, '_mesa')
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = ThreeClassClassifierSummaryBuilder.build_mesa_all_combined(
attributed_classifier, feature_sets)
three_class_performance_dictionary = CurvePlotBuilder.make_three_class_roc(
classifier_summary, '_mesa')
classifier_summary.performance_dictionary = three_class_performance_dictionary
three_class_performance_summaries.append(classifier_summary)
CurvePlotBuilder.combine_sw_and_three_class_plots(
attributed_classifier, 1, 'mesa')
TableBuilder.print_table_three_class(three_class_performance_summaries)
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1,
'_mesa_three_class_roc')
def test_make_pr_plot(self, mock_build_pr, mock_tidy_plot, mock_set_labels,
mock_plt):
performance_dictionary = {
tuple([FeatureType.count, FeatureType.heart_rate]):
['placeholder', 'for', 'raw', 'performances'],
tuple([FeatureType.count]):
['placeholder', 'for', 'raw', 'performances']
}
attributed_classifier = AttributedClassifier(
name="Logistic Regression", classifier=LogisticRegression())
classifier_summary = ClassifierSummary(
attributed_classifier=attributed_classifier,
performance_dictionary=performance_dictionary)
CurvePlotBuilder.make_pr_sw(classifier_summary)
mock_build_pr.assert_called_once_with(classifier_summary)
mock_tidy_plot.assert_called_once_with()
mock_set_labels.assert_called_once_with(
attributed_classifier, 'Fraction of wake scored as wake',
'Fraction of predicted wake correct', (0.5, 1.0))
mock_plt.savefig.assert_called_once_with(
str(
Constants.FIGURE_FILE_PATH.joinpath(
attributed_classifier.name + '_' + str(4) +
'__sw_pr.png')))
mock_plt.close.assert_called_once_with()
def test_tidy_plot(self, mock_plt):
mock_axis = mock_plt.subplot.return_value
CurvePlotBuilder.tidy_plot()
mock_plt.subplot.assert_called_once_with(111)
mock_axis.spines[''].set_visible.assert_has_calls(
[call(False), call(False),
call(True), call(True)])
mock_axis.yaxis.set_ticks_position.assert_called_once_with('left')
mock_axis.xaxis.set_ticks_position.assert_called_once_with('bottom')
def figures_compare_time_based_features():
classifiers = utils.get_classifiers()
feature_sets = [
[FeatureType.count, FeatureType.heart_rate],
[FeatureType.count, FeatureType.heart_rate, FeatureType.time],
[FeatureType.count, FeatureType.heart_rate, FeatureType.cosine],
[
FeatureType.count, FeatureType.heart_rate,
FeatureType.circadian_model
]
]
trial_count = 50
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_sw(classifier_summary, '_time_only')
CurvePlotBuilder.make_pr_sw(classifier_summary, '_time_only')
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_time_only_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_time_only_sw_roc')
def test_set_labels(self, mock_font_manager, mock_plt):
attributed_classifier = AttributedClassifier(
name="Logistic Regression", classifier=LogisticRegression())
x_label = 'X Label Text'
y_label = 'Y Label Text'
legend_location = (1.0, 0.2)
mock_font_manager.FontProperties.return_value = font_placeholder = 'FontPlaceholder'
CurvePlotBuilder.set_labels(attributed_classifier, x_label, y_label,
legend_location)
font_name = "Arial"
font_size = 14
mock_font_manager.FontProperties.assert_called_once_with(
family=font_name, style='normal', size=font_size)
mock_plt.xlabel.assert_called_once_with(x_label,
fontsize=font_size,
fontname=font_name)
mock_plt.ylabel.assert_called_once_with(y_label,
fontsize=font_size,
fontname=font_name)
mock_plt.title.assert_called_once_with(attributed_classifier.name,
fontsize=18,
fontname=font_name,
fontweight='bold')
attributed_classifier = AttributedClassifier(
name="Neural Net", classifier=LogisticRegression())
CurvePlotBuilder.set_labels(attributed_classifier, x_label, y_label,
legend_location)
mock_plt.legend.assert_called_once_with(bbox_to_anchor=legend_location,
borderaxespad=0.,
prop=font_placeholder)
def figures_mc_sleep_wake():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
trial_count = 20
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_sw(classifier_summary)
CurvePlotBuilder.make_pr_sw(classifier_summary)
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_sw_roc')
def figure_leave_one_out_roc_and_pr():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_leave_one_out(
attributed_classifier, feature_sets)
CurvePlotBuilder.make_roc_sw(classifier_summary)
CurvePlotBuilder.make_pr_sw(classifier_summary)
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(
classifiers, len(SubjectBuilder.get_all_subject_ids()), '_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(
classifiers, len(SubjectBuilder.get_all_subject_ids()), '_sw_roc')
def figures_mesa_sleep_wake():
classifiers = utils.get_classifiers()
# Uncomment to just use MLP:
# classifiers = [AttributedClassifier(name='Neural Net',
# classifier=MLPClassifier(activation='relu', hidden_layer_sizes=(15, 15, 15),
# max_iter=1000, alpha=0.01, solver='lbfgs'))]
feature_sets = utils.get_base_feature_sets()
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_mesa(
attributed_classifier, feature_sets)
CurvePlotBuilder.make_roc_sw(classifier_summary, '_mesa')
CurvePlotBuilder.make_pr_sw(classifier_summary, '_mesa')
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_sw_roc')
def figures_mc_three_class():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
trial_count = 20
three_class_performance_summaries = []
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = ThreeClassClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_one_vs_rest(classifier_summary)
three_class_performance_dictionary = CurvePlotBuilder.make_three_class_roc(
classifier_summary)
classifier_summary.performance_dictionary = three_class_performance_dictionary
three_class_performance_summaries.append(classifier_summary)
TableBuilder.print_table_three_class(three_class_performance_summaries)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_three_class_roc')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_ovr_rem_roc')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_ovr_nrem_roc')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_ovr_wake_roc')
def test_plot_pr(self, mock_curve_performance_builder,
mock_feature_set_service, mock_plt):
first_raw_performances = [
RawPerformance(true_labels=np.array([0, 1]),
class_probabilities=np.array([[0, 1], [1, 0]])),
RawPerformance(true_labels=np.array([0, 1]),
class_probabilities=np.array([[0.2, 0.8],
[0.1, 0.9]]))
]
second_raw_performances = [
RawPerformance(true_labels=np.array([1, 1]),
class_probabilities=np.array([[0, 1], [1, 0]])),
RawPerformance(true_labels=np.array([0, 0]),
class_probabilities=np.array([[0.2, 0.8],
[0.1, 0.9]]))
]
performance_dictionary = {
tuple([FeatureType.count, FeatureType.heart_rate]):
first_raw_performances,
tuple([FeatureType.count]): second_raw_performances
}
attributed_classifier = AttributedClassifier(
name="Logistic Regression", classifier=LogisticRegression())
classifier_summary = ClassifierSummary(
attributed_classifier=attributed_classifier,
performance_dictionary=performance_dictionary)
first_pr_performance = PrecisionRecallPerformance(
recalls=np.array([1]), precisions=np.array([.2]))
second_pr_performance = PrecisionRecallPerformance(
recalls=np.array([.3]), precisions=np.array([1]))
first_label = 'Label 1'
second_label = 'Label 2'
first_color = '#ffffff'
second_color = '#123456'
mock_curve_performance_builder.build_precision_recall_from_raw.side_effect = [
first_pr_performance, second_pr_performance
]
mock_feature_set_service.get_label.side_effect = [
first_label, second_label
]
mock_feature_set_service.get_color.side_effect = [
first_color, second_color
]
CurvePlotBuilder.build_pr_plot(classifier_summary)
mock_curve_performance_builder.build_precision_recall_from_raw.assert_has_calls(
[call(first_raw_performances)])
mock_curve_performance_builder.build_precision_recall_from_raw.assert_has_calls(
[call(second_raw_performances)])
mock_feature_set_service.get_label.assert_has_calls(
[call([FeatureType.count, FeatureType.heart_rate])])
mock_feature_set_service.get_color.assert_has_calls(
[call([FeatureType.count, FeatureType.heart_rate])])
mock_feature_set_service.get_label.assert_has_calls(
[call([FeatureType.count])])
mock_feature_set_service.get_color.assert_has_calls(
[call([FeatureType.count])])
mock_plt.plot.assert_has_calls([
call(first_pr_performance.recalls,
first_pr_performance.precisions,
label=first_label,
color=first_color)
])
mock_plt.plot.assert_has_calls([
call(second_pr_performance.recalls,
second_pr_performance.precisions,
label=second_label,
color=second_color)
])
