def figures_mc_three_class():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
trial_count = 20
three_class_performance_summaries = []
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = ThreeClassClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_one_vs_rest(classifier_summary)
three_class_performance_dictionary = CurvePlotBuilder.make_three_class_roc(
classifier_summary)
classifier_summary.performance_dictionary = three_class_performance_dictionary
three_class_performance_summaries.append(classifier_summary)
TableBuilder.print_table_three_class(three_class_performance_summaries)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_three_class_roc')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_ovr_rem_roc')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_ovr_nrem_roc')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_ovr_wake_roc')
def test_get_classifiers(self):
all_classifiers = utils.get_classifiers()
TestHelper.assert_models_equal(
self, all_classifiers[0],
AttributedClassifier(name='Random Forest',
classifier=RandomForestClassifier(
n_estimators=500,
max_features=1.0,
max_depth=10,
min_samples_split=10,
min_samples_leaf=1)))
TestHelper.assert_models_equal(
self, all_classifiers[1],
AttributedClassifier(name='Logistic Regression',
classifier=LogisticRegression(
penalty='l1',
solver='liblinear',
verbose=0)))
TestHelper.assert_models_equal(
self, all_classifiers[2],
AttributedClassifier(name='k-Nearest Neighbors',
classifier=KNeighborsClassifier()))
TestHelper.assert_models_equal(
self, all_classifiers[3],
AttributedClassifier(name='Neural Net',
classifier=MLPClassifier(
activation='relu',
hidden_layer_sizes=(30, 30, 30),
max_iter=1000,
alpha=0.01)))
def figures_compare_time_based_features():
classifiers = utils.get_classifiers()
feature_sets = [
[FeatureType.count, FeatureType.heart_rate],
[FeatureType.count, FeatureType.heart_rate, FeatureType.time],
[FeatureType.count, FeatureType.heart_rate, FeatureType.cosine],
[
FeatureType.count, FeatureType.heart_rate,
FeatureType.circadian_model
]
]
trial_count = 50
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_sw(classifier_summary, '_time_only')
CurvePlotBuilder.make_pr_sw(classifier_summary, '_time_only')
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_time_only_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count,
'_time_only_sw_roc')
def figures_mesa_three_class():
classifiers = utils.get_classifiers()
# Uncomment to just use MLP:
# classifiers = [AttributedClassifier(name='Neural Net', classifier=MLPClassifier(activation='relu', hidden_layer_sizes=(15, 15, 15),
# max_iter=1000, alpha=0.01, solver='lbfgs'))]
feature_sets = utils.get_base_feature_sets()
three_class_performance_summaries = []
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = ThreeClassClassifierSummaryBuilder.build_mesa_leave_one_out(
attributed_classifier, feature_sets)
PerformancePlotBuilder.make_bland_altman(classifier_summary, '_mesa')
PerformancePlotBuilder.make_single_threshold_histograms(
classifier_summary, '_mesa')
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = ThreeClassClassifierSummaryBuilder.build_mesa_all_combined(
attributed_classifier, feature_sets)
three_class_performance_dictionary = CurvePlotBuilder.make_three_class_roc(
classifier_summary, '_mesa')
classifier_summary.performance_dictionary = three_class_performance_dictionary
three_class_performance_summaries.append(classifier_summary)
CurvePlotBuilder.combine_sw_and_three_class_plots(
attributed_classifier, 1, 'mesa')
TableBuilder.print_table_three_class(three_class_performance_summaries)
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1,
'_mesa_three_class_roc')
def figures_mc_sleep_wake():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
trial_count = 20
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_monte_carlo(
attributed_classifier, feature_sets, trial_count)
CurvePlotBuilder.make_roc_sw(classifier_summary)
CurvePlotBuilder.make_pr_sw(classifier_summary)
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, trial_count, '_sw_roc')
def figure_leave_one_out_roc_and_pr():
classifiers = utils.get_classifiers()
feature_sets = utils.get_base_feature_sets()
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_leave_one_out(
attributed_classifier, feature_sets)
CurvePlotBuilder.make_roc_sw(classifier_summary)
CurvePlotBuilder.make_pr_sw(classifier_summary)
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(
classifiers, len(SubjectBuilder.get_all_subject_ids()), '_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(
classifiers, len(SubjectBuilder.get_all_subject_ids()), '_sw_roc')
def figures_mesa_sleep_wake():
classifiers = utils.get_classifiers()
# Uncomment to just use MLP:
# classifiers = [AttributedClassifier(name='Neural Net',
# classifier=MLPClassifier(activation='relu', hidden_layer_sizes=(15, 15, 15),
# max_iter=1000, alpha=0.01, solver='lbfgs'))]
feature_sets = utils.get_base_feature_sets()
for attributed_classifier in classifiers:
if Constants.VERBOSE:
print('Running ' + attributed_classifier.name + '...')
classifier_summary = SleepWakeClassifierSummaryBuilder.build_mesa(
attributed_classifier, feature_sets)
CurvePlotBuilder.make_roc_sw(classifier_summary, '_mesa')
CurvePlotBuilder.make_pr_sw(classifier_summary, '_mesa')
TableBuilder.print_table_sw(classifier_summary)
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_sw_pr')
CurvePlotBuilder.combine_plots_as_grid(classifiers, 1, '_mesa_sw_roc')