def figures_compare_time_based_features():
classifiers = utils.get_classifiers()
feature_sets = [
[FeatureType.count, FeatureType.heart_rate],
[FeatureType.count, FeatureType.heart_rate, FeatureType.time],
[FeatureType.count, FeatureType.heart_rate, FeatureType.cosine],
[
FeatureType.count, FeatureType.heart_rate,
FeatureType.circadian_model
]
]
trial_count = 50
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_sw(classifier_summary, '_time_only')
CurvePlotBuilder.make_pr_sw(classifier_summary, '_time_only')
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_time_only_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_time_only_sw_roc')
def test_leave_one_out(self, mock_train_test_splitter,
mock_classifier_service, mock_subject_builder):
attributed_classifier = AttributedClassifier(
name="Logistic Regression", classifier=LogisticRegression())
feature_sets = [[FeatureType.cosine, FeatureType.circadian_model],
[FeatureType.count]]
mock_subject_builder.get_all_subject_ids.return_value = subject_ids = [
"subjectA", "subjectB"
]
mock_subject_builder.get_subject_dictionary.return_value = subject_dictionary = {
"subjectA": [],
"subjectB": []
}
mock_train_test_splitter.leave_one_out.return_value = expected_data_splits = [
DataSplit(training_set="subjectA", testing_set="subjectB")
]
mock_classifier_service.run_sw.side_effect = raw_performance_arrays = [
[
RawPerformance(true_labels=np.array([1, 2]),
class_probabilities=np.array([3, 4])),
RawPerformance(true_labels=np.array([0, 1]),
class_probabilities=np.array([2, 3]))
],
[
RawPerformance(true_labels=np.array([1, 1]),
class_probabilities=np.array([4, 4])),
RawPerformance(true_labels=np.array([0, 0]),
class_probabilities=np.array([2, 2]))
]
]
returned_summary = SleepWakeClassifierSummaryBuilder.build_leave_one_out(
attributed_classifier, feature_sets)
mock_subject_builder.get_all_subject_ids.assert_called_once_with()
mock_subject_builder.get_subject_dictionary.assert_called_once_with()
mock_train_test_splitter.leave_one_out.assert_called_once_with(
subject_ids)
mock_classifier_service.run_sw.assert_has_calls([
call(expected_data_splits, attributed_classifier,
subject_dictionary, feature_sets[0]),
call(expected_data_splits, attributed_classifier,
subject_dictionary, feature_sets[1])
])
self.assertEqual(returned_summary.attributed_classifier,
attributed_classifier)
self.assertEqual(
returned_summary.performance_dictionary[tuple(feature_sets[0])],
raw_performance_arrays[0])
self.assertEqual(
returned_summary.performance_dictionary[tuple(feature_sets[1])],
raw_performance_arrays[1])
def figures_mc_sleep_wake():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
trial_count = 20
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_sw(classifier_summary)
CurvePlotBuilder.make_pr_sw(classifier_summary)
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_sw_roc')
def figure_leave_one_out_roc_and_pr():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_leave_one_out(
attributed_classifier, feature_sets)
CurvePlotBuilder.make_roc_sw(classifier_summary)
CurvePlotBuilder.make_pr_sw(classifier_summary)
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(
classifiers, len(SubjectBuilder.get_all_subject_ids()), '_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(
classifiers, len(SubjectBuilder.get_all_subject_ids()), '_sw_roc')
def figures_leave_one_out_sleep_wake_performance():
attributed_classifier = AttributedClassifier(
name='Neural Net',
classifier=MLPClassifier(activation='relu',
hidden_layer_sizes=(15, 15, 15),
max_iter=1000,
alpha=0.01,
solver='lbfgs'))
feature_sets = [[
FeatureType.count, FeatureType.heart_rate, FeatureType.circadian_model
]]
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_leave_one_out(
attributed_classifier, feature_sets)
PerformancePlotBuilder.make_histogram_with_thresholds(classifier_summary)
PerformancePlotBuilder.make_single_threshold_histograms(classifier_summary)
def figures_mesa_sleep_wake():
classifiers = utils.get_classifiers()
# Uncomment to just use MLP:
# classifiers = [AttributedClassifier(name='Neural Net',
# classifier=MLPClassifier(activation='relu', hidden_layer_sizes=(15, 15, 15),
# max_iter=1000, alpha=0.01, solver='lbfgs'))]
feature_sets = utils.get_base_feature_sets()
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_mesa(
attributed_classifier, feature_sets)
CurvePlotBuilder.make_roc_sw(classifier_summary, '_mesa')
CurvePlotBuilder.make_pr_sw(classifier_summary, '_mesa')
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_sw_roc')