def build_pr_plot(classifier_summary: ClassifierSummary):
for feature_set in classifier_summary.performance_dictionary:
raw_performances = classifier_summary.performance_dictionary[
feature_set]
roc_performance = CurvePerformanceBuilder.build_precision_recall_from_raw(
raw_performances)
legend_text = FeatureSetService.get_label(list(feature_set))
plot_color = FeatureSetService.get_color(list(feature_set))
plt.plot(roc_performance.recalls,
roc_performance.precisions,
label=legend_text,
color=plot_color)
def make_three_class_roc(classifier_summary: ClassifierSummary,
description=''):
performance_dictionary = {}
for feature_set in classifier_summary.performance_dictionary:
raw_performances = classifier_summary.performance_dictionary[
feature_set]
sleep_wake_roc_performance, rem_roc_performance, nrem_roc_performance, three_class_performances = \
CurvePerformanceBuilder.build_three_class_roc_with_binary_search(
raw_performances)
performance_dictionary[
feature_set] = PerformanceSummarizer.average_three_class(
three_class_performances)
legend_text = FeatureSetService.get_label(list(feature_set))
plot_color = FeatureSetService.get_color(list(feature_set))
plt.plot(sleep_wake_roc_performance.false_positive_rates,
sleep_wake_roc_performance.true_positive_rates,
label=legend_text,
color=plot_color)
plt.plot(nrem_roc_performance.false_positive_rates,
nrem_roc_performance.true_positive_rates,
color=plot_color,
linestyle=':')
plt.plot(rem_roc_performance.false_positive_rates,
rem_roc_performance.true_positive_rates,
color=plot_color,
linestyle='--')
CurvePlotBuilder.tidy_plot()
CurvePlotBuilder.set_labels(classifier_summary.attributed_classifier,
'Fraction of wake scored as REM or NREM',
'Fraction of REM, NREM scored correctly',
(1.0, 0.4))
number_of_trials = len(
next(iter(classifier_summary.performance_dictionary.values())))
plt.savefig(
str(
Constants.FIGURE_FILE_PATH.joinpath(
classifier_summary.attributed_classifier.name + '_' +
str(number_of_trials) + '_' + description +
'_three_class_roc.png')))
plt.close()
return performance_dictionary
def build_roc_plot(classifier_summary: ClassifierSummary,
positive_class=1):
for feature_set in classifier_summary.performance_dictionary:
raw_performances = classifier_summary.performance_dictionary[
feature_set]
roc_performance = CurvePerformanceBuilder.build_roc_from_raw(
raw_performances, positive_class)
legend_text = FeatureSetService.get_label(list(feature_set))
plot_color = FeatureSetService.get_color(list(feature_set))
plt.plot(roc_performance.false_positive_rates,
roc_performance.true_positive_rates,
label=legend_text,
color=plot_color)
def print_table_three_class(classifier_summaries):
frontmatter = '\\begin{tabular}{l | l | c | c | c | c | c } & & Wake Correct & NREM' \
' Correct & REM Correct & Best accuracy & $\\kappa$ \\'
print(frontmatter)
for classifier_summary in classifier_summaries:
performance_dictionary = classifier_summary.performance_dictionary
attributed_classifier = classifier_summary.attributed_classifier
for feature_set in performance_dictionary:
if list(feature_set) == [FeatureType.count]:
print('\\hline ' + attributed_classifier.name + ' & ' +
FeatureSetService.get_label(list(feature_set)) +
' & ')
else:
print(' & ' +
FeatureSetService.get_label(list(feature_set)) +
' & ')
three_class_performance_summary = performance_dictionary[
feature_set]
wake_correct = three_class_performance_summary.wake_correct
nrem_correct = three_class_performance_summary.nrem_correct
rem_correct = three_class_performance_summary.rem_correct
accuracy = three_class_performance_summary.accuracy
kappa = three_class_performance_summary.kappa
print(
str(round(wake_correct, 3)) + ' & ' +
str(round(nrem_correct, 3)) + ' & ' +
str(round(rem_correct, 3)) + ' & ' +
str(round(accuracy, 3)) + ' & ' + str(round(kappa, 3)) +
'\\\\')
backmatter = '\\end{tabular} \\label{tab:REM_params} \\small \\vspace{.2cm}'
print(backmatter)
def print_table_sw(classifier_summary):
performance_dictionary = classifier_summary.performance_dictionary
attributed_classifier = classifier_summary.attributed_classifier
frontmatter = '\\begin{table} \\caption{Sleep/wake differentiation performance by ' \
+ attributed_classifier.name \
+ ' across different feature inputs in the Apple Watch (PPG, MEMS) dataset} ' \
'\\begin{tabular}{l*{5}{c}} & Accuracy & Wake correct (specificity) ' \
'& Sleep correct (sensitivity) & $\\kappa$ & AUC \\\\ '
print(frontmatter)
for feature_set in performance_dictionary:
raw_performances: [RawPerformance
] = performance_dictionary[feature_set]
print('\\hline ' + FeatureSetService.get_label(list(feature_set)) +
' &')
true_positive_thresholds, performance_summaries = PerformanceSummarizer.summarize_thresholds(
raw_performances)
for true_positive_threshold, performance in zip(
true_positive_thresholds, performance_summaries):
if true_positive_threshold == 0.8:
print(
str(round(performance.accuracy, 3)) + ' & ' +
str(round(performance.wake_correct, 3)) + ' & ' +
str(round(performance.sleep_correct, 3)) + ' & ' +
str(round(performance.kappa, 3)) + ' & ' +
str(round(performance.auc, 3)) + ' \\\\')
else:
print('& ' + str(round(performance.accuracy, 3)) + ' & ' +
str(round(performance.wake_correct, 3)) + ' & ' +
str(round(performance.sleep_correct, 3)) + ' & ' +
str(round(performance.kappa, 3)) + ' & \\\\')
backmatter = '\\hline \\end{tabular} \\label{tab:' + \
attributed_classifier.name[0:4] + \
'params} \\small \\vspace{.2cm} ' \
'\\caption*{Fraction of wake correct, fraction of sleep correct, accuracy, ' \
'$\\kappa$, and AUC for sleep-wake predictions of ' + attributed_classifier.name + \
' with use of motion, HR, clock proxy, or combination of features. PPG, ' \
'photoplethysmography; MEMS, microelectromechanical systems; HR, heart rate; ' \
'AUC, area under the curve.} \\end{table}'
print(backmatter)
def test_get_feature_set_labels(self):
self.assertEqual("Motion only",
FeatureSetService.get_label([FeatureType.count]))
self.assertEqual("HR only",
FeatureSetService.get_label([FeatureType.heart_rate]))
self.assertEqual(
"Motion, HR",
FeatureSetService.get_label(
[FeatureType.count, FeatureType.heart_rate]))
self.assertEqual(
"Motion, HR, and Clock",
FeatureSetService.get_label([
FeatureType.count, FeatureType.heart_rate,
FeatureType.circadian_model
]))
self.assertEqual(
"Motion, HR, and Time",
FeatureSetService.get_label(
[FeatureType.count, FeatureType.heart_rate, FeatureType.time]))
self.assertEqual(
"Motion, HR, and Cosine",
FeatureSetService.get_label([
FeatureType.count, FeatureType.heart_rate, FeatureType.cosine
]))
def test_get_feature_set_colors(self):
self.assertEqual(sns.xkcd_rgb["denim blue"],
FeatureSetService.get_color([FeatureType.count]))
self.assertEqual(sns.xkcd_rgb["yellow orange"],
FeatureSetService.get_color([FeatureType.heart_rate]))
self.assertEqual(
sns.xkcd_rgb["medium green"],
FeatureSetService.get_color(
[FeatureType.count, FeatureType.heart_rate]))
self.assertEqual(
sns.xkcd_rgb["medium pink"],
FeatureSetService.get_color([
FeatureType.count, FeatureType.heart_rate,
FeatureType.circadian_model
]))
self.assertEqual(
sns.xkcd_rgb["greyish"],
FeatureSetService.get_color(
[FeatureType.count, FeatureType.heart_rate, FeatureType.time]))
self.assertEqual(
sns.xkcd_rgb["plum"],
FeatureSetService.get_color([
FeatureType.count, FeatureType.heart_rate, FeatureType.cosine
]))
def make_bland_altman(classifier_summary: ClassifierSummary,
description=''):
fig, ax = plt.subplots(nrows=3, ncols=2, figsize=(8, 8))
wake_threshold = Constants.WAKE_THRESHOLD
rem_threshold = Constants.REM_THRESHOLD
for feature_set in classifier_summary.performance_dictionary:
raw_performances = classifier_summary.performance_dictionary[
feature_set]
number_of_subjects = len(raw_performances)
plot_color = FeatureSetService.get_color(feature_set)
for subject_index in range(number_of_subjects):
raw_performance = raw_performances[subject_index]
true_labels = raw_performance.true_labels
predicted_labels = PerformanceBuilder.apply_threshold_three_class(
raw_performance, wake_threshold, rem_threshold)
actual_sol = SleepMetricsCalculator.get_sleep_onset_latency(
true_labels)
predicted_sol = SleepMetricsCalculator.get_sleep_onset_latency(
predicted_labels)
sol_diff = (actual_sol - predicted_sol)
ax[0, 0].scatter(actual_sol, sol_diff, c=plot_color)
ax[0, 0].set_xlabel("SOL")
ax[0, 0].set_ylabel("Difference in SOL")
actual_waso = SleepMetricsCalculator.get_wake_after_sleep_onset(
true_labels)
predicted_waso = SleepMetricsCalculator.get_wake_after_sleep_onset(
predicted_labels)
waso_diff = (actual_waso - predicted_waso)
ax[0, 1].scatter(actual_waso, waso_diff, c=plot_color)
ax[0, 1].set_xlabel("WASO")
ax[0, 1].set_ylabel("Difference in WASO")
actual_tst = SleepMetricsCalculator.get_tst(true_labels)
predicted_tst = SleepMetricsCalculator.get_tst(
predicted_labels)
tst_diff = (actual_tst - predicted_tst)
ax[1, 0].scatter(actual_tst, tst_diff, c=plot_color)
ax[1, 0].set_xlabel("TST")
ax[1, 0].set_ylabel("Difference in TST")
actual_sleep_efficiency = SleepMetricsCalculator.get_sleep_efficiency(
true_labels)
predicted_sleep_efficiency = SleepMetricsCalculator.get_sleep_efficiency(
predicted_labels)
sleep_efficiency_diff = (actual_sleep_efficiency -
predicted_sleep_efficiency)
ax[1, 1].scatter(actual_sleep_efficiency,
sleep_efficiency_diff,
c=plot_color)
ax[1, 1].set_xlabel("Sleep efficiency")
ax[1, 1].set_ylabel("Difference in sleep efficiency")
actual_time_in_rem = SleepMetricsCalculator.get_time_in_rem(
true_labels)
predicted_time_in_rem = SleepMetricsCalculator.get_time_in_rem(
predicted_labels)
time_in_rem_diff = (actual_time_in_rem - predicted_time_in_rem)
ax[2, 0].scatter(actual_time_in_rem,
time_in_rem_diff,
c=plot_color)
ax[2, 0].set_xlabel("Time in REM")
ax[2, 0].set_ylabel("Difference in time in REM")
actual_time_in_nrem = SleepMetricsCalculator.get_time_in_nrem(
true_labels)
predicted_time_in_nrem = SleepMetricsCalculator.get_time_in_nrem(
predicted_labels)
time_in_nrem_diff = (actual_time_in_nrem -
predicted_time_in_nrem)
ax[2, 1].set_xlabel("Time in NREM")
ax[2, 1].set_ylabel("Difference in time in NREM")
font = font_manager.FontProperties(family='Arial',
style='normal',
size=10)
if subject_index == 0:
ax[2, 1].scatter(
actual_time_in_nrem,
time_in_nrem_diff,
c=plot_color,
label=FeatureSetService.get_label(feature_set))
ax[2, 1].legend(bbox_to_anchor=(1.04, 1),
loc="upper left",
borderaxespad=0,
prop=font)
else:
ax[2, 1].scatter(actual_time_in_nrem,
time_in_nrem_diff,
c=plot_color)
plt.tight_layout()
file_save_name = str(
Constants.FIGURE_FILE_PATH) + '/figure_' + classifier_summary.attributed_classifier.name + '_' + \
description + '_bland_altman.png'
plt.savefig(file_save_name, dpi=300)
plt.close()
def make_histogram_with_thresholds(classifier_summary: ClassifierSummary):
for feature_set in classifier_summary.performance_dictionary:
raw_performances = classifier_summary.performance_dictionary[
feature_set]
number_of_thresholds = 4
number_of_subjects = len(raw_performances)
fig, ax = plt.subplots(nrows=number_of_thresholds,
ncols=2,
figsize=(8, 8),
sharex=True,
sharey=True)
all_accuracies = np.zeros(
(number_of_subjects, number_of_thresholds))
all_wake_correct_fractions = np.zeros(
(number_of_subjects, number_of_thresholds))
for subject_index in range(number_of_subjects):
true_positive_thresholds, performance_summary = PerformanceSummarizer.summarize_thresholds(
[raw_performances[subject_index]])
for threshold_index in range(number_of_thresholds):
all_accuracies[subject_index,
threshold_index] = performance_summary[
threshold_index].accuracy
all_wake_correct_fractions[
subject_index, threshold_index] = performance_summary[
threshold_index].wake_correct
dt = 0.02
row_count = 0
font_size = 16
font_name = 'Arial'
for row in ax:
row[0].hist(all_accuracies[:, row_count].tolist(),
bins=np.arange(0, 1 + dt, dt),
color="skyblue",
ec="skyblue")
row[1].hist(all_wake_correct_fractions[:, row_count].tolist(),
bins=np.arange(0, 1 + dt, dt),
color="lightsalmon",
ec="lightsalmon")
if row_count == number_of_thresholds - 1:
row[0].set_xlabel('Accuracy',
fontsize=font_size,
fontname=font_name)
row[1].set_xlabel('Wake correct',
fontsize=font_size,
fontname=font_name)
row[0].set_ylabel('Count',
fontsize=font_size,
fontname=font_name)
row[0].set_xlim((0, 1))
row[1].set_xlim((0, 1))
row_count = row_count + 1
file_save_name = str(Constants.FIGURE_FILE_PATH) + '/' + FeatureSetService.get_label(feature_set) + '_' \
+ classifier_summary.attributed_classifier.name + '_histograms_with_thresholds.png'
plt.tight_layout()
plt.savefig(file_save_name, dpi=300)
plt.close()
image = Image.open(file_save_name)
width, height = image.size
scale_factor = 0.3
new_image = Image.new('RGB', (int(
(1 + scale_factor) * width), height), "white")
new_image.paste(image, (int(scale_factor * width), 0))
draw = ImageDraw.Draw(new_image)
font = ImageFont.truetype('/Library/Fonts/Arial Unicode.ttf', 75)
draw.text(
(int(scale_factor * width / 3), int((height * 0.9) * 0.125)),
"TPR = 0.8", (0, 0, 0),
font=font)
draw.text(
(int(scale_factor * width / 3), int(height * 0.9 * 0.375)),
"TPR = 0.9", (0, 0, 0),
font=font)
draw.text(
(int(scale_factor * width / 3), int(height * 0.9 * 0.625)),
"TPR = 0.93", (0, 0, 0),
font=font)
draw.text(
(int(scale_factor * width / 3), int(height * 0.9 * 0.875)),
"TPR = 0.95", (0, 0, 0),
font=font)
new_image.save(
str(Constants.FIGURE_FILE_PATH) + '/' +
'figure_threshold_histogram.png')