def test_leave_one_out(self, mock_train_test_splitter, mock_classifier_service, mock_subject_builder): attributed_classifier = AttributedClassifier( name="Logistic Regression", classifier=LogisticRegression()) feature_sets = [[FeatureType.cosine, FeatureType.circadian_model], [FeatureType.count]] mock_subject_builder.get_all_subject_ids.return_value = subject_ids = [ "subjectA", "subjectB" ] mock_subject_builder.get_subject_dictionary.return_value = subject_dictionary = { "subjectA": [], "subjectB": [] } mock_train_test_splitter.leave_one_out.return_value = expected_data_splits = [ DataSplit(training_set="subjectA", testing_set="subjectB") ] mock_classifier_service.run_sw.side_effect = raw_performance_arrays = [ [ RawPerformance(true_labels=np.array([1, 2]), class_probabilities=np.array([3, 4])), RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([2, 3])) ], [ RawPerformance(true_labels=np.array([1, 1]), class_probabilities=np.array([4, 4])), RawPerformance(true_labels=np.array([0, 0]), class_probabilities=np.array([2, 2])) ] ] returned_summary = SleepWakeClassifierSummaryBuilder.build_leave_one_out( attributed_classifier, feature_sets) mock_subject_builder.get_all_subject_ids.assert_called_once_with() mock_subject_builder.get_subject_dictionary.assert_called_once_with() mock_train_test_splitter.leave_one_out.assert_called_once_with( subject_ids) mock_classifier_service.run_sw.assert_has_calls([ call(expected_data_splits, attributed_classifier, subject_dictionary, feature_sets[0]), call(expected_data_splits, attributed_classifier, subject_dictionary, feature_sets[1]) ]) self.assertEqual(returned_summary.attributed_classifier, attributed_classifier) self.assertEqual( returned_summary.performance_dictionary[tuple(feature_sets[0])], raw_performance_arrays[0]) self.assertEqual( returned_summary.performance_dictionary[tuple(feature_sets[1])], raw_performance_arrays[1])
def convert_three_class_to_two(raw_performance: RawPerformance): raw_performance.true_labels = SleepLabeler.label_sleep_wake( raw_performance.true_labels) number_of_samples = np.shape(raw_performance.class_probabilities)[0] for index in range(number_of_samples): raw_performance.class_probabilities[index, 1] = raw_performance.class_probabilities[index, 1] + \ raw_performance.class_probabilities[index, 2] raw_performance.class_probabilities = raw_performance.class_probabilities[:, : -1] return raw_performance
def test_build_roc_from_raw_performances(self): raw_performances = [ RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0, 1], [1, 0]])), RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]])) ] first_false_positive_rates, first_true_positive_rates, first_thresholds = roc_curve( raw_performances[0].true_labels, raw_performances[0].class_probabilities[:, 1], pos_label=SleepWakeLabel.sleep.value, drop_intermediate=False) second_false_positive_rates, second_true_positive_rates, second_thresholds = roc_curve( raw_performances[1].true_labels, raw_performances[1].class_probabilities[:, 1], pos_label=SleepWakeLabel.sleep.value, drop_intermediate=False) horizontal_axis_bins, vertical_axis_bins = CurvePerformanceBuilder.get_axes_bins( ) first_interpolated_true_positive_rates = np.interp( horizontal_axis_bins, first_false_positive_rates, first_true_positive_rates) second_interpolated_true_positive_rates = np.interp( horizontal_axis_bins, second_false_positive_rates, second_true_positive_rates) expected_true_positive_rates = ( first_interpolated_true_positive_rates + second_interpolated_true_positive_rates) / 2 horizontal_axis_bins = np.insert(horizontal_axis_bins, 0, 0, axis=0) expected_true_positive_rates = np.insert(expected_true_positive_rates, 0, 0, axis=0) roc_performance = CurvePerformanceBuilder.build_roc_from_raw( raw_performances, 1) self.assertListEqual(horizontal_axis_bins.tolist(), roc_performance.false_positive_rates.tolist()) self.assertListEqual(expected_true_positive_rates.tolist(), roc_performance.true_positive_rates.tolist())
def run_three_class_with_loaded_model(data_splits, classifier, subject_dictionary, feature_set): raw_performances = [] for ind in range(len(data_splits)): data_split = data_splits[ind] if ind == 0: training_x, training_y = ClassifierInputBuilder.get_three_class_inputs( data_split.training_set, subject_dictionary=subject_dictionary, feature_set=feature_set) classifier = ClassifierService.train_classifier( training_x, training_y, classifier, 'neg_log_loss') testing_x, testing_y = ClassifierInputBuilder.get_three_class_inputs( data_split.testing_set, subject_dictionary=subject_dictionary, feature_set=feature_set) class_probabilities = classifier.predict_proba(testing_x) raw_performance = RawPerformance( true_labels=testing_y, class_probabilities=class_probabilities) raw_performances.append(raw_performance) return raw_performances
def test_build_from_raw(self): threshold = 0.2 raw_performance = RawPerformance(true_labels=np.array([1, 0]), class_probabilities=np.array( [[0.1, 0.9], [0.3, 0.7]])) predicted_labels = np.array([1, 1]) kappa = cohen_kappa_score(raw_performance.true_labels, predicted_labels) sleep_predictive_value = precision_score(raw_performance.true_labels, predicted_labels, pos_label=1) wake_predictive_value = precision_score(raw_performance.true_labels, predicted_labels, pos_label=0) false_positive_rates, true_positive_rates, thresholds = roc_curve( raw_performance.true_labels, raw_performance.class_probabilities[:, 1], pos_label=1, drop_intermediate=False) auc_value = auc(false_positive_rates, true_positive_rates) expected_performance = SleepWakePerformance( accuracy=np.float64(0.5), wake_correct=np.float64(0), sleep_correct=np.float64(1.0), kappa=kappa, auc=auc_value, sleep_predictive_value=sleep_predictive_value, wake_predictive_value=wake_predictive_value) performance = PerformanceBuilder.build_with_sleep_threshold( raw_performance, threshold) TestHelper.assert_models_equal(self, expected_performance, performance)
def test_build_pr_from_raw_performances(self): raw_performances = [ RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0, 1], [1, 0]])), RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]])) ] first_precisions, first_recalls, first_thresholds = precision_recall_curve( raw_performances[0].true_labels, raw_performances[0].class_probabilities[:, 0], pos_label=SleepWakeLabel.wake.value) second_precisions, second_recalls, second_thresholds = precision_recall_curve( raw_performances[1].true_labels, raw_performances[1].class_probabilities[:, 0], pos_label=SleepWakeLabel.wake.value) horizontal_axis_bins, vertical_axis_bins = CurvePerformanceBuilder.get_axes_bins( ) first_interpolated_precisions = np.interp(horizontal_axis_bins, np.flip(first_recalls), np.flip(first_precisions)) second_interpolated_precisions = np.interp(horizontal_axis_bins, np.flip(second_recalls), np.flip(second_precisions)) expected_precisions = (first_interpolated_precisions + second_interpolated_precisions) / 2 horizontal_axis_bins = np.insert(horizontal_axis_bins, 0, 0, axis=0) expected_precisions = np.insert(expected_precisions, 0, 1, axis=0) pr_performance = CurvePerformanceBuilder.build_precision_recall_from_raw( raw_performances) self.assertListEqual(horizontal_axis_bins.tolist(), pr_performance.recalls.tolist()) self.assertListEqual(expected_precisions.tolist(), pr_performance.precisions.tolist())
def test_properties(self): true_labels = np.array([0, 1, 2]) class_probabilities = np.array([[0.1, 0.9], [0, 1]]) raw_performance = RawPerformance( true_labels=true_labels, class_probabilities=class_probabilities) self.assertEqual(raw_performance.true_labels.tolist(), true_labels.tolist()) self.assertEqual(raw_performance.class_probabilities.tolist(), class_probabilities.tolist())
def test_summarize_thresholds(self, mock_performance_builder, mock_performance_summarizer_average): raw_performances = [RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0, 1], [1, 0]])), RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]]))] mock_performance_builder.build_with_true_positive_rate_threshold.side_effect = ['return1', 'return2', 'return3', 'return4', 'return5', 'return6', 'return7', 'return8' ] mock_performance_summarizer_average.side_effect = averages = ['average1', 'average2', 'average3', 'average4'] thresholds, returned_averages = PerformanceSummarizer.summarize_thresholds(raw_performances) mock_performance_builder.build_with_true_positive_rate_threshold.assert_has_calls( [call(raw_performances[0], 0.8), call(raw_performances[1], 0.8), call(raw_performances[0], 0.9), call(raw_performances[1], 0.9), call(raw_performances[0], 0.93), call(raw_performances[1], 0.93), call(raw_performances[0], 0.95), call(raw_performances[1], 0.95)]) mock_performance_summarizer_average.assert_has_calls([call(['return1', 'return2']), call(['return3', 'return4']), call(['return5', 'return6']), call(['return7', 'return8']) ]) self.assertListEqual([0.8, 0.9, 0.93, 0.95], thresholds) self.assertListEqual(averages, returned_averages)
def test_properties(self): attributed_classifier = AttributedClassifier( name="Logistic Regression", classifier=LogisticRegression()) performance_dictionary = { (FeatureType.count, FeatureType.cosine): [ RawPerformance(true_labels=np.array([1, 0, 1, 0]), class_probabilities=np.array([0.1, 0.9])) ], FeatureType.count: [ RawPerformance(true_labels=np.array([0, 0, 1, 0]), class_probabilities=np.array([0.9, 0.1])) ] } classifier_summary = ClassifierSummary( attributed_classifier=attributed_classifier, performance_dictionary=performance_dictionary) TestHelper.assert_models_equal( self, attributed_classifier, classifier_summary.attributed_classifier) self.assertDictEqual(performance_dictionary, classifier_summary.performance_dictionary)
def run_single_data_split(training_x, training_y, testing_x, testing_y, attributed_classifier, scoring='roc_auc'): start_time = time.time() classifier = ClassifierService.train_classifier( training_x, training_y, attributed_classifier, scoring) class_probabilities = classifier.predict_proba(testing_x) raw_performance = RawPerformance( true_labels=testing_y, class_probabilities=class_probabilities) if Constants.VERBOSE: print('Completed data split in ' + str(time.time() - start_time)) return raw_performance
def test_print_table_sw(self, mock_print, mock_summarize_thresholds): first_raw_performances = [ RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0, 1], [1, 0]])), RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]]))] second_raw_performances = [ RawPerformance(true_labels=np.array([1, 1]), class_probabilities=np.array([[0, 1], [1, 0]])), RawPerformance(true_labels=np.array([0, 0]), class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]]))] performance_dictionary = {tuple([FeatureType.count, FeatureType.heart_rate]): first_raw_performances, tuple([FeatureType.count]): second_raw_performances} attributed_classifier = AttributedClassifier(name="Logistic Regression", classifier=LogisticRegression()) classifier_summary = ClassifierSummary(attributed_classifier=attributed_classifier, performance_dictionary=performance_dictionary) first_performance = SleepWakePerformance(accuracy=0, wake_correct=0, sleep_correct=0, kappa=0, auc=0, sleep_predictive_value=0, wake_predictive_value=0) second_performance = SleepWakePerformance(accuracy=1, wake_correct=1, sleep_correct=1, kappa=1, auc=1, sleep_predictive_value=1, wake_predictive_value=1) third_performance = SleepWakePerformance(accuracy=0.5, wake_correct=0.5, sleep_correct=0.5, kappa=0.5, auc=0.5, sleep_predictive_value=0.5, wake_predictive_value=0.5) fourth_performance = SleepWakePerformance(accuracy=0.2, wake_correct=0.2, sleep_correct=0.2, kappa=0.2, auc=0.2, sleep_predictive_value=0.2, wake_predictive_value=0.2) mock_summarize_thresholds.side_effect = [([0.3, 0.7], [first_performance, second_performance]), ([0.2, 0.8], [third_performance, fourth_performance])] TableBuilder.print_table_sw(classifier_summary) frontmatter = '\\begin{table} \\caption{Sleep/wake differentiation performance by Logistic Regression ' \ + 'across different feature inputs in the Apple Watch (PPG, MEMS) dataset} ' \ '\\begin{tabular}{l*{5}{c}} & Accuracy & Wake correct (specificity) ' \ '& Sleep correct (sensitivity) & $\\kappa$ & AUC \\\\ ' header_line_1 = '\\hline Motion, HR &' header_line_2 = '\\hline Motion only &' results_line_1 = '& ' + str(first_performance.accuracy) + ' & ' + str( first_performance.wake_correct) + ' & ' + str( first_performance.sleep_correct) + ' & ' + str(first_performance.kappa) + ' & \\\\' results_line_2 = '& ' + str(second_performance.accuracy) + ' & ' + str( second_performance.wake_correct) + ' & ' + str( second_performance.sleep_correct) + ' & ' + str(second_performance.kappa) + ' & \\\\' results_line_3 = '& ' + str(third_performance.accuracy) + ' & ' + str( third_performance.wake_correct) + ' & ' + str( third_performance.sleep_correct) + ' & ' + str(third_performance.kappa) + ' & \\\\' results_line_4 = str(fourth_performance.accuracy) + ' & ' + str( fourth_performance.wake_correct) + ' & ' + str( fourth_performance.sleep_correct) + ' & ' + str(fourth_performance.kappa) + ' & ' + str( fourth_performance.auc) + ' \\\\' backmatter = '\\hline \\end{tabular} \\label{tab:' \ + attributed_classifier.name[0:4] \ + 'params} \\small \\vspace{.2cm} ' \ '\\caption*{Fraction of wake correct, fraction of sleep correct, accuracy, ' \ '$\\kappa$, and AUC for sleep-wake predictions of Logistic Regression' \ ' with use of motion, HR, clock proxy, or combination of features. PPG, ' \ 'photoplethysmography; MEMS, microelectromechanical systems; HR, heart rate; ' \ 'AUC, area under the curve.} \\end{table}' mock_print.assert_has_calls([call(frontmatter), call(header_line_1), call(results_line_1), call(results_line_2), call(header_line_2), call(results_line_3), call(results_line_4), call(backmatter)])
def test_plot_pr(self, mock_curve_performance_builder, mock_feature_set_service, mock_plt): first_raw_performances = [ RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0, 1], [1, 0]])), RawPerformance(true_labels=np.array([0, 1]), class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]])) ] second_raw_performances = [ RawPerformance(true_labels=np.array([1, 1]), class_probabilities=np.array([[0, 1], [1, 0]])), RawPerformance(true_labels=np.array([0, 0]), class_probabilities=np.array([[0.2, 0.8], [0.1, 0.9]])) ] performance_dictionary = { tuple([FeatureType.count, FeatureType.heart_rate]): first_raw_performances, tuple([FeatureType.count]): second_raw_performances } attributed_classifier = AttributedClassifier( name="Logistic Regression", classifier=LogisticRegression()) classifier_summary = ClassifierSummary( attributed_classifier=attributed_classifier, performance_dictionary=performance_dictionary) first_pr_performance = PrecisionRecallPerformance( recalls=np.array([1]), precisions=np.array([.2])) second_pr_performance = PrecisionRecallPerformance( recalls=np.array([.3]), precisions=np.array([1])) first_label = 'Label 1' second_label = 'Label 2' first_color = '#ffffff' second_color = '#123456' mock_curve_performance_builder.build_precision_recall_from_raw.side_effect = [ first_pr_performance, second_pr_performance ] mock_feature_set_service.get_label.side_effect = [ first_label, second_label ] mock_feature_set_service.get_color.side_effect = [ first_color, second_color ] CurvePlotBuilder.build_pr_plot(classifier_summary) mock_curve_performance_builder.build_precision_recall_from_raw.assert_has_calls( [call(first_raw_performances)]) mock_curve_performance_builder.build_precision_recall_from_raw.assert_has_calls( [call(second_raw_performances)]) mock_feature_set_service.get_label.assert_has_calls( [call([FeatureType.count, FeatureType.heart_rate])]) mock_feature_set_service.get_color.assert_has_calls( [call([FeatureType.count, FeatureType.heart_rate])]) mock_feature_set_service.get_label.assert_has_calls( [call([FeatureType.count])]) mock_feature_set_service.get_color.assert_has_calls( [call([FeatureType.count])]) mock_plt.plot.assert_has_calls([ call(first_pr_performance.recalls, first_pr_performance.precisions, label=first_label, color=first_color) ]) mock_plt.plot.assert_has_calls([ call(second_pr_performance.recalls, second_pr_performance.precisions, label=second_label, color=second_color) ])