def figures_mc_three_class():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
trial_count = 20
three_class_performance_summaries = []
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = ThreeClassClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_one_vs_rest(classifier_summary)
three_class_performance_dictionary = CurvePlotBuilder.make_three_class_roc(
classifier_summary)
classifier_summary.performance_dictionary = three_class_performance_dictionary
three_class_performance_summaries.append(classifier_summary)
TableBuilder.print_table_three_class(three_class_performance_summaries)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_three_class_roc')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_ovr_rem_roc')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_ovr_nrem_roc')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_ovr_wake_roc')
def figures_compare_time_based_features():
classifiers = utils.get_classifiers()
feature_sets = [
[FeatureType.count, FeatureType.heart_rate],
[FeatureType.count, FeatureType.heart_rate, FeatureType.time],
[FeatureType.count, FeatureType.heart_rate, FeatureType.cosine],
[
FeatureType.count, FeatureType.heart_rate,
FeatureType.circadian_model
]
]
trial_count = 50
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_sw(classifier_summary, '_time_only')
CurvePlotBuilder.make_pr_sw(classifier_summary, '_time_only')
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_time_only_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_time_only_sw_roc')
def figures_mesa_three_class():
classifiers = utils.get_classifiers()
# Uncomment to just use MLP:
# classifiers = [AttributedClassifier(name='Neural Net', classifier=MLPClassifier(activation='relu', hidden_layer_sizes=(15, 15, 15),
# max_iter=1000, alpha=0.01, solver='lbfgs'))]
feature_sets = utils.get_base_feature_sets()
three_class_performance_summaries = []
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = ThreeClassClassifierSummaryBuilder.build_mesa_leave_one_out(
attributed_classifier, feature_sets)
PerformancePlotBuilder.make_bland_altman(classifier_summary, '_mesa')
PerformancePlotBuilder.make_single_threshold_histograms(
classifier_summary, '_mesa')
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = ThreeClassClassifierSummaryBuilder.build_mesa_all_combined(
attributed_classifier, feature_sets)
three_class_performance_dictionary = CurvePlotBuilder.make_three_class_roc(
classifier_summary, '_mesa')
classifier_summary.performance_dictionary = three_class_performance_dictionary
three_class_performance_summaries.append(classifier_summary)
CurvePlotBuilder.combine_sw_and_three_class_plots(
attributed_classifier, 1, 'mesa')
TableBuilder.print_table_three_class(three_class_performance_summaries)
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1,
'_mesa_three_class_roc')
def figures_mc_sleep_wake():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
trial_count = 20
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_sw(classifier_summary)
CurvePlotBuilder.make_pr_sw(classifier_summary)
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_sw_roc')
def figure_leave_one_out_roc_and_pr():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_leave_one_out(
attributed_classifier, feature_sets)
CurvePlotBuilder.make_roc_sw(classifier_summary)
CurvePlotBuilder.make_pr_sw(classifier_summary)
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(
classifiers, len(SubjectBuilder.get_all_subject_ids()), '_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(
classifiers, len(SubjectBuilder.get_all_subject_ids()), '_sw_roc')
def figures_mesa_sleep_wake():
classifiers = utils.get_classifiers()
# Uncomment to just use MLP:
# classifiers = [AttributedClassifier(name='Neural Net',
# classifier=MLPClassifier(activation='relu', hidden_layer_sizes=(15, 15, 15),
# max_iter=1000, alpha=0.01, solver='lbfgs'))]
feature_sets = utils.get_base_feature_sets()
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_mesa(
attributed_classifier, feature_sets)
CurvePlotBuilder.make_roc_sw(classifier_summary, '_mesa')
CurvePlotBuilder.make_pr_sw(classifier_summary, '_mesa')
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_sw_roc')
def test_print_table_sw(self, mock_print, mock_summarize_thresholds):
first_raw_performances = [
RawPerformance(true_labels=np.array([0, 1]),
class_probabilities=np.array([[0, 1], [1, 0]])),
RawPerformance(true_labels=np.array([0, 1]),
class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]]))]
second_raw_performances = [
RawPerformance(true_labels=np.array([1, 1]),
class_probabilities=np.array([[0, 1], [1, 0]])),
RawPerformance(true_labels=np.array([0, 0]),
class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]]))]
performance_dictionary = {tuple([FeatureType.count, FeatureType.heart_rate]): first_raw_performances,
tuple([FeatureType.count]): second_raw_performances}
attributed_classifier = AttributedClassifier(name="Logistic Regression", classifier=LogisticRegression())
classifier_summary = ClassifierSummary(attributed_classifier=attributed_classifier,
performance_dictionary=performance_dictionary)
first_performance = SleepWakePerformance(accuracy=0, wake_correct=0, sleep_correct=0, kappa=0, auc=0,
sleep_predictive_value=0,
wake_predictive_value=0)
second_performance = SleepWakePerformance(accuracy=1, wake_correct=1, sleep_correct=1, kappa=1, auc=1,
sleep_predictive_value=1,
wake_predictive_value=1)
third_performance = SleepWakePerformance(accuracy=0.5, wake_correct=0.5, sleep_correct=0.5, kappa=0.5, auc=0.5,
sleep_predictive_value=0.5,
wake_predictive_value=0.5)
fourth_performance = SleepWakePerformance(accuracy=0.2, wake_correct=0.2, sleep_correct=0.2, kappa=0.2, auc=0.2,
sleep_predictive_value=0.2,
wake_predictive_value=0.2)
mock_summarize_thresholds.side_effect = [([0.3, 0.7], [first_performance, second_performance]),
([0.2, 0.8], [third_performance, fourth_performance])]
TableBuilder.print_table_sw(classifier_summary)
frontmatter = '\\begin{table} \\caption{Sleep/wake differentiation performance by Logistic Regression ' \
+ 'across different feature inputs in the Apple Watch (PPG, MEMS) dataset} ' \
'\\begin{tabular}{l*{5}{c}} & Accuracy & Wake correct (specificity) ' \
'& Sleep correct (sensitivity) & $\\kappa$ & AUC \\\\ '
header_line_1 = '\\hline Motion, HR &'
header_line_2 = '\\hline Motion only &'
results_line_1 = '& ' + str(first_performance.accuracy) + ' & ' + str(
first_performance.wake_correct) + ' & ' + str(
first_performance.sleep_correct) + ' & ' + str(first_performance.kappa) + ' & \\\\'
results_line_2 = '& ' + str(second_performance.accuracy) + ' & ' + str(
second_performance.wake_correct) + ' & ' + str(
second_performance.sleep_correct) + ' & ' + str(second_performance.kappa) + ' & \\\\'
results_line_3 = '& ' + str(third_performance.accuracy) + ' & ' + str(
third_performance.wake_correct) + ' & ' + str(
third_performance.sleep_correct) + ' & ' + str(third_performance.kappa) + ' & \\\\'
results_line_4 = str(fourth_performance.accuracy) + ' & ' + str(
fourth_performance.wake_correct) + ' & ' + str(
fourth_performance.sleep_correct) + ' & ' + str(fourth_performance.kappa) + ' & ' + str(
fourth_performance.auc) + ' \\\\'
backmatter = '\\hline \\end{tabular} \\label{tab:' \
+ attributed_classifier.name[0:4] \
+ 'params} \\small \\vspace{.2cm} ' \
'\\caption*{Fraction of wake correct, fraction of sleep correct, accuracy, ' \
'$\\kappa$, and AUC for sleep-wake predictions of Logistic Regression' \
' with use of motion, HR, clock proxy, or combination of features. PPG, ' \
'photoplethysmography; MEMS, microelectromechanical systems; HR, heart rate; ' \
'AUC, area under the curve.} \\end{table}'
mock_print.assert_has_calls([call(frontmatter),
call(header_line_1),
call(results_line_1),
call(results_line_2),
call(header_line_2),
call(results_line_3),
call(results_line_4),
call(backmatter)])