def figures_mc_three_class(): classifiers = utils.get_classifiers() feature_sets = utils.get_base_feature_sets() trial_count = 20 three_class_performance_summaries = [] for attributed_classifier in classifiers: if Constants.VERBOSE: print('Running ' + attributed_classifier.name + '...') classifier_summary = ThreeClassClassifierSummaryBuilder.build_monte_carlo( attributed_classifier, feature_sets, trial_count) CurvePlotBuilder.make_roc_one_vs_rest(classifier_summary) three_class_performance_dictionary = CurvePlotBuilder.make_three_class_roc( classifier_summary) classifier_summary.performance_dictionary = three_class_performance_dictionary three_class_performance_summaries.append(classifier_summary) TableBuilder.print_table_three_class(three_class_performance_summaries) CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_three_class_roc') CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_ovr_rem_roc') CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_ovr_nrem_roc') CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_ovr_wake_roc')
def figures_compare_time_based_features(): classifiers = utils.get_classifiers() feature_sets = [ [FeatureType.count, FeatureType.heart_rate], [FeatureType.count, FeatureType.heart_rate, FeatureType.time], [FeatureType.count, FeatureType.heart_rate, FeatureType.cosine], [ FeatureType.count, FeatureType.heart_rate, FeatureType.circadian_model ] ] trial_count = 50 for attributed_classifier in classifiers: if Constants.VERBOSE: print('Running ' + attributed_classifier.name + '...') classifier_summary = SleepWakeClassifierSummaryBuilder.build_monte_carlo( attributed_classifier, feature_sets, trial_count) CurvePlotBuilder.make_roc_sw(classifier_summary, '_time_only') CurvePlotBuilder.make_pr_sw(classifier_summary, '_time_only') TableBuilder.print_table_sw(classifier_summary) CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_time_only_sw_pr') CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_time_only_sw_roc')
def figures_mesa_three_class(): classifiers = utils.get_classifiers() # Uncomment to just use MLP: # classifiers = [AttributedClassifier(name='Neural Net', classifier=MLPClassifier(activation='relu', hidden_layer_sizes=(15, 15, 15), # max_iter=1000, alpha=0.01, solver='lbfgs'))] feature_sets = utils.get_base_feature_sets() three_class_performance_summaries = [] for attributed_classifier in classifiers: if Constants.VERBOSE: print('Running ' + attributed_classifier.name + '...') classifier_summary = ThreeClassClassifierSummaryBuilder.build_mesa_leave_one_out( attributed_classifier, feature_sets) PerformancePlotBuilder.make_bland_altman(classifier_summary, '_mesa') PerformancePlotBuilder.make_single_threshold_histograms( classifier_summary, '_mesa') for attributed_classifier in classifiers: if Constants.VERBOSE: print('Running ' + attributed_classifier.name + '...') classifier_summary = ThreeClassClassifierSummaryBuilder.build_mesa_all_combined( attributed_classifier, feature_sets) three_class_performance_dictionary = CurvePlotBuilder.make_three_class_roc( classifier_summary, '_mesa') classifier_summary.performance_dictionary = three_class_performance_dictionary three_class_performance_summaries.append(classifier_summary) CurvePlotBuilder.combine_sw_and_three_class_plots( attributed_classifier, 1, 'mesa') TableBuilder.print_table_three_class(three_class_performance_summaries) CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_three_class_roc')
def figures_mc_sleep_wake(): classifiers = utils.get_classifiers() feature_sets = utils.get_base_feature_sets() trial_count = 20 for attributed_classifier in classifiers: if Constants.VERBOSE: print('Running ' + attributed_classifier.name + '...') classifier_summary = SleepWakeClassifierSummaryBuilder.build_monte_carlo( attributed_classifier, feature_sets, trial_count) CurvePlotBuilder.make_roc_sw(classifier_summary) CurvePlotBuilder.make_pr_sw(classifier_summary) TableBuilder.print_table_sw(classifier_summary) CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_sw_pr') CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_sw_roc')
def figure_leave_one_out_roc_and_pr(): classifiers = utils.get_classifiers() feature_sets = utils.get_base_feature_sets() for attributed_classifier in classifiers: if Constants.VERBOSE: print('Running ' + attributed_classifier.name + '...') classifier_summary = SleepWakeClassifierSummaryBuilder.build_leave_one_out( attributed_classifier, feature_sets) CurvePlotBuilder.make_roc_sw(classifier_summary) CurvePlotBuilder.make_pr_sw(classifier_summary) TableBuilder.print_table_sw(classifier_summary) CurvePlotBuilder.combine_plots_as_grid( classifiers, len(SubjectBuilder.get_all_subject_ids()), '_sw_pr') CurvePlotBuilder.combine_plots_as_grid( classifiers, len(SubjectBuilder.get_all_subject_ids()), '_sw_roc')
def figures_mesa_sleep_wake(): classifiers = utils.get_classifiers() # Uncomment to just use MLP: # classifiers = [AttributedClassifier(name='Neural Net', # classifier=MLPClassifier(activation='relu', hidden_layer_sizes=(15, 15, 15), # max_iter=1000, alpha=0.01, solver='lbfgs'))] feature_sets = utils.get_base_feature_sets() for attributed_classifier in classifiers: if Constants.VERBOSE: print('Running ' + attributed_classifier.name + '...') classifier_summary = SleepWakeClassifierSummaryBuilder.build_mesa( attributed_classifier, feature_sets) CurvePlotBuilder.make_roc_sw(classifier_summary, '_mesa') CurvePlotBuilder.make_pr_sw(classifier_summary, '_mesa') TableBuilder.print_table_sw(classifier_summary) CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_sw_pr') CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_sw_roc')
def test_print_table_sw(self, mock_print, mock_summarize_thresholds): first_raw_performances = [ RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0, 1], [1, 0]])), RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]]))] second_raw_performances = [ RawPerformance(true_labels=np.array([1, 1]), class_probabilities=np.array([[0, 1], [1, 0]])), RawPerformance(true_labels=np.array([0, 0]), class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]]))] performance_dictionary = {tuple([FeatureType.count, FeatureType.heart_rate]): first_raw_performances, tuple([FeatureType.count]): second_raw_performances} attributed_classifier = AttributedClassifier(name="Logistic Regression", classifier=LogisticRegression()) classifier_summary = ClassifierSummary(attributed_classifier=attributed_classifier, performance_dictionary=performance_dictionary) first_performance = SleepWakePerformance(accuracy=0, wake_correct=0, sleep_correct=0, kappa=0, auc=0, sleep_predictive_value=0, wake_predictive_value=0) second_performance = SleepWakePerformance(accuracy=1, wake_correct=1, sleep_correct=1, kappa=1, auc=1, sleep_predictive_value=1, wake_predictive_value=1) third_performance = SleepWakePerformance(accuracy=0.5, wake_correct=0.5, sleep_correct=0.5, kappa=0.5, auc=0.5, sleep_predictive_value=0.5, wake_predictive_value=0.5) fourth_performance = SleepWakePerformance(accuracy=0.2, wake_correct=0.2, sleep_correct=0.2, kappa=0.2, auc=0.2, sleep_predictive_value=0.2, wake_predictive_value=0.2) mock_summarize_thresholds.side_effect = [([0.3, 0.7], [first_performance, second_performance]), ([0.2, 0.8], [third_performance, fourth_performance])] TableBuilder.print_table_sw(classifier_summary) frontmatter = '\\begin{table} \\caption{Sleep/wake differentiation performance by Logistic Regression ' \ + 'across different feature inputs in the Apple Watch (PPG, MEMS) dataset} ' \ '\\begin{tabular}{l*{5}{c}} & Accuracy & Wake correct (specificity) ' \ '& Sleep correct (sensitivity) & $\\kappa$ & AUC \\\\ ' header_line_1 = '\\hline Motion, HR &' header_line_2 = '\\hline Motion only &' results_line_1 = '& ' + str(first_performance.accuracy) + ' & ' + str( first_performance.wake_correct) + ' & ' + str( first_performance.sleep_correct) + ' & ' + str(first_performance.kappa) + ' & \\\\' results_line_2 = '& ' + str(second_performance.accuracy) + ' & ' + str( second_performance.wake_correct) + ' & ' + str( second_performance.sleep_correct) + ' & ' + str(second_performance.kappa) + ' & \\\\' results_line_3 = '& ' + str(third_performance.accuracy) + ' & ' + str( third_performance.wake_correct) + ' & ' + str( third_performance.sleep_correct) + ' & ' + str(third_performance.kappa) + ' & \\\\' results_line_4 = str(fourth_performance.accuracy) + ' & ' + str( fourth_performance.wake_correct) + ' & ' + str( fourth_performance.sleep_correct) + ' & ' + str(fourth_performance.kappa) + ' & ' + str( fourth_performance.auc) + ' \\\\' backmatter = '\\hline \\end{tabular} \\label{tab:' \ + attributed_classifier.name[0:4] \ + 'params} \\small \\vspace{.2cm} ' \ '\\caption*{Fraction of wake correct, fraction of sleep correct, accuracy, ' \ '$\\kappa$, and AUC for sleep-wake predictions of Logistic Regression' \ ' with use of motion, HR, clock proxy, or combination of features. PPG, ' \ 'photoplethysmography; MEMS, microelectromechanical systems; HR, heart rate; ' \ 'AUC, area under the curve.} \\end{table}' mock_print.assert_has_calls([call(frontmatter), call(header_line_1), call(results_line_1), call(results_line_2), call(header_line_2), call(results_line_3), call(results_line_4), call(backmatter)])