def load_experiment(data_path, start_hour):
# Configure an Experiment
exp = Experiment()
# Iterates over a set of files in a directory.
# Unfortunately, we have to do it manually with RawProcessing because we are modifying the annotations
for file in glob(data_path):
pp = RawProcessing(file,
# HR information
col_for_hr="hr",
# Activity information
cols_for_activity=["x", "y", "z"],
is_act_count=True,
# Datetime information
col_for_datetime="faketime",
strftime="%Y-%m-%d %H:%M:%S",
# Participant information
col_for_pid="pid")
w = Wearable(pp) # Creates a wearable from a pp object
w.data["hyp_annotation"] = (w.data["label"] > 0)
exp.add_wearable(w)
# Set frequency for every wearable in the collection
exp.set_freq_in_secs(15)
# Changing the hour the experiment starts from midnight (0) to 3pm (15)
exp.change_start_hour_for_experiment_day(start_hour)
return exp
def setup_experiment(file_path, diary_path, start_hour):
# Configure an Experiment
exp = Experiment()
# Iterates over a set of files in a directory.
# Unfortunately, we have to do it manually with RawProcessing because we are modifying the annotations
for file in glob(file_path):
pp = RawProcessing(file,
device_location="dw",
# HR information
col_for_hr="mean_hr",
# Activity information
cols_for_activity=["activity"],
is_act_count=True,
# Datetime information
col_for_datetime="linetime",
strftime="%Y-%m-%d %H:%M:%S",
# Participant information
col_for_pid="mesaid")
w = Wearable(pp) # Creates a wearable from a pp object
# Invert the two_stages flag. Now True means sleeping and False means awake
w.data["hyp_annotation"] = (w.data["stages"] > 0)
exp.add_wearable(w)
exp.set_freq_in_secs(30)
w.change_start_hour_for_experiment_day(start_hour)
diary = Diary().from_file(diary_path)
exp.add_diary(diary)
return exp
def configure_experiment(self, datapath: str, device_location: str, cols_for_activity, col_for_mets: str = None,
is_emno: bool = False, is_act_count: bool = False, col_for_datetime: str = "time",
start_of_week: int = -1, strftime: str = None, col_for_pid: str = None, pid: int = -1,
additional_data: object = None, col_for_hr: str = None):
# TODO: Missing a check to see if datapath exists.
if os.path.isdir(datapath):
if not datapath.endswith("*"):
datapath = os.path.join(datapath, "*")
else:
if '*' not in datapath:
datapath = datapath + "*"
for file in glob(datapath):
pp = RawProcessing(file,
device_location=device_location,
# activitiy information
cols_for_activity=cols_for_activity,
is_act_count=is_act_count,
is_emno=is_emno,
# Datatime information
col_for_datetime=col_for_datetime,
col_for_mets=col_for_mets,
start_of_week=start_of_week,
strftime=strftime,
# Participant information
col_for_pid=col_for_pid,
pid=pid,
additional_data=additional_data,
# HR information
col_for_hr=col_for_hr, )
w = Wearable(pp)
self.wearables[w.get_pid()] = w
def setUp(self):
# Loads some file
pp1 = RawProcessing()
pp1.load_file("../data/examples_mesa/mesa-sample-day5-invalid5hours.csv",
# activitiy information
cols_for_activity=["activity"],
is_act_count=True,
# Datatime information
col_for_datatime="linetime",
device_location="dw",
start_of_week="dayofweek",
# Participant information
col_for_pid="mesaid")
self.w_5day_invalid5hours = Wearable(pp1)
def load_experiment(data_path, diary_path, start_hour):
# Configure an Experiment
exp = Experiment()
# Iterates over a set of files in a directory.
# Unfortunately, we have to do it manually with RawProcessing because we are modifying the annotations
for file in glob(data_path):
pp = RawProcessing(
file,
# HR information
col_for_hr="HR",
# Activity information
cols_for_activity=["Axis1", "Axis2", "Axis3"],
is_act_count=False,
# Datetime information
col_for_datetime="time",
strftime="%Y-%b-%d %H:%M:%S",
# Participant information
col_for_pid="pid")
w = Wearable(pp) # Creates a wearable from a pp object
exp.add_wearable(w)
# Set frequency for every wearable in the collection
# exp.set_freq_in_secs(5)
# Changing the hour the experiment starts from midnight (0) to 3pm (15)
exp.change_start_hour_for_experiment_day(start_hour)
diary = Diary().from_file(diary_path)
exp.add_diary(diary)
return exp
def load_experiment(data_path, diary_path, start_hour):
# Configure an Experiment
exp = Experiment()
# Iterates over a set of files in a directory.
# Unfortunately, we have to do it manually with RawProcessing because we are modifying the annotations
print("Running %s" % data_path)
for file in glob(data_path):
print("FILE", file)
pp = RawProcessing(file,
cols_for_activity=["stdMET_highIC_Branch"],
is_act_count=False,
col_for_datetime="REALTIME",
strftime="%d-%b-%Y %H:%M:%S",
col_for_pid="id",
col_for_hr="mean_hr",
device_location="dw")
pp.data["hyp_act_x"] = pp.data[
"hyp_act_x"] - 1.0 # adjust for the BBVA dataset
w = Wearable(pp) # Creates a wearable from a pp object
exp.add_wearable(w)
# Set frequency for every wearable in the collection
exp.set_freq_in_secs(60)
# Changing the hour the experiment starts from midnight (0) to 3pm (15)
exp.change_start_hour_for_experiment_day(start_hour)
diary = Diary().from_file(diary_path)
exp.add_diary(diary)
return exp
def setup_experiment(file_path, diary_path, start_hour):
# Configure an Experiment
exp = Experiment()
# Iterates over a set of files in a directory.
# Unfortunately, we have to do it manually with RawProcessing because we are modifying the annotations
for file in glob(file_path):
pp = MESAPreProcessing(file)
w = Wearable(pp) # Creates a wearable from a pp object
# Invert the two_stages flag. Now True means sleeping and False means awake
w.data["interval_sleep"] = w.data["interval"].isin(["REST-S", "REST"])
exp.add_wearable(w)
exp.set_freq_in_secs(30)
w.change_start_hour_for_experiment_day(start_hour)
diary = Diary().from_file(diary_path)
exp.add_diary(diary)
return exp
def configure_experiment(self,
datapath: str,
device_location: str,
cols_for_activity,
col_for_mets: str = None,
is_emno: bool = False,
is_act_count: bool = False,
col_for_datetime: str = "time",
start_of_week: int = -1,
strftime: str = None,
col_for_pid: str = None,
pid: int = -1,
additional_data: object = None,
col_for_hr: str = None):
"""
Parameters
----------
datapath : str
DESCRIPTION. Path to the dat folder
device_location : str, optional
DESCRIPTION. The default is None. Where this device was located (options are: "bw", "hip", "dw", "ndw", "chest", "hp_ch", "hp_bw", "all")
# Configuration for activity
cols_for_activity : list / str
DESCRIPTION. Which columns record activity
col_for_mets : object, optional
DESCRIPTION. Column that records METs.
is_emno : bool, optional
DESCRIPTION. True if the cols_for_activity are already computed as the ENMO (Euclidean Norm Minus One)
is_act_count : bool, optional
DESCRIPTION. The default is False. True us cols_for_activity are already computed as counts
# Datetime parameters
col_for_datetime : str, optional
DESCRIPTION. The default is "time". Name of timestamp column.
start_of_week : int, optional
DESCRIPTION. The default is -1. Integer that represents the day at the start of the week
strftime : str, optional
DESCRIPTION. The default is None. Format to parse col_for_datetime
# PID parameters
col_for_pid : str, optional
DESCRIPTION. The default is None. Participant ID columns
pid : int, optional
DESCRIPTION. The default is -1.
# HR parameters
col_for_hr : str, optional
DESCRIPTION. The default is None. Column with heart rate data
# Any additional data?
additional_data : object, optional
DESCRIPTION. The default is None.
Returns
-------
None.
"""
# TODO: Missing a check to see if datapath exists.
if os.path.isdir(datapath):
if not datapath.endswith("*"):
datapath = os.path.join(datapath, "*")
else:
if '*' not in datapath:
datapath = datapath + "*"
for file in glob(datapath):
pp = RawProcessing(
file,
device_location=device_location,
# activitiy information
cols_for_activity=cols_for_activity,
is_act_count=is_act_count,
is_emno=is_emno,
# Datatime information
col_for_datetime=col_for_datetime,
col_for_mets=col_for_mets,
start_of_week=start_of_week,
strftime=strftime,
# Participant information
col_for_pid=col_for_pid,
pid=pid,
additional_data=additional_data,
# HR information
col_for_hr=col_for_hr,
)
w = Wearable(pp)
self.wearables[w.get_pid()] = w
def __sleep_boundaries_with_hr(wearable: Wearable, output_col: str, quantile: float = 0.4,
volarity_threshold: int = 5, rolling_win_in_minutes: int = 5,
sleep_search_window: tuple = (20, 12), min_window_length_in_minutes: int = 40,
volatility_window_in_minutes: int = 10, merge_blocks_gap_time_in_min: int = 240,
sleep_only_in_sleep_search_window: bool = False,
only_largest_sleep_period: bool = False):
if wearable.hr_col is None:
raise AttributeError("HR is not available for PID %s." % (wearable.get_pid()))
rolling_win_in_minutes = int(rolling_win_in_minutes * wearable.get_epochs_in_min())
min_window_length_in_minutes = int(min_window_length_in_minutes * wearable.get_epochs_in_min())
volatility_window_in_minutes = int(volatility_window_in_minutes * wearable.get_epochs_in_min())
df = wearable.data.copy()
df["hyp_sleep"], df["hyp_sleep_bin"], df["hyp_seq_length"], df[
"hyp_seq_id"] = SleepBoudaryDetector.__create_threshold_col_based_on_time(wearable.data, wearable.time_col,
wearable.hr_col,
sleep_search_window[0],
sleep_search_window[1],
quantile,
rolling_win_in_minutes,
sleep_only_in_sleep_search_window)
df['hyp_sleep_candidate'] = ((df["hyp_sleep_bin"] == 1.0) & (
df['hyp_seq_length'] > min_window_length_in_minutes)).astype(int)
df["hyp_sleep_vard"] = df[wearable.hr_col].rolling(volatility_window_in_minutes,
center=True).std().fillna(0)
df["hyp_seq_length"], df["hyp_seq_id"] = misc.get_consecutive_series(df, "hyp_sleep_candidate")
# Merge two sleep segments if their gap is smaller than X min (interval per day):
wearable.data = df
saved_hour_start_day = wearable.hour_start_experiment
wearable.change_start_hour_for_experiment_day(sleep_search_window[0])
grps = wearable.data.groupby(wearable.experiment_day_col)
tmp_df = []
for grp_id, grp_df in grps:
gdf = grp_df.copy()
gdf["hyp_sleep_candidate"], gdf["hyp_seq_length"], gdf["hyp_seq_id"] = misc.merge_sequences_given_tolerance(
gdf, wearable.time_col, "hyp_sleep_candidate", tolerance_in_minutes=merge_blocks_gap_time_in_min)
tmp_df.append(gdf)
wearable.data = pd.concat(tmp_df)
wearable.change_start_hour_for_experiment_day(saved_hour_start_day)
df = wearable.data.set_index(wearable.time_col)
new_sleep_segments = df[df["hyp_sleep_candidate"] == 1]["hyp_seq_id"].unique()
# Check if we can modify the sleep onset/offset
for sleep_seg_id in new_sleep_segments:
actual_seg = df[df["hyp_seq_id"] == sleep_seg_id]
if actual_seg.shape[0] == 0:
continue
start_time = actual_seg.index[0]
end_time = actual_seg.index[-1]
look_sleep_onset = df[start_time - timedelta(hours=4): start_time + timedelta(minutes=60)]
look_sleep_offset = df[end_time - timedelta(minutes=1): end_time + timedelta(minutes=120)]
new_sleep_onset = look_sleep_onset[look_sleep_onset["hyp_sleep_vard"] > volarity_threshold]
new_sleep_offset = look_sleep_offset[look_sleep_offset["hyp_sleep_vard"] > volarity_threshold]
new_start = new_sleep_onset.index[-1] if not new_sleep_onset.empty else start_time
new_end = new_sleep_offset.index[0] if not new_sleep_offset.empty else end_time
df.loc[new_start:new_end, "hyp_seq_id"] = sleep_seg_id
# df.loc[new_start:new_end, "hyp_seq_length"] = df.loc[new_start:new_end].shape[0]
df.loc[new_start:new_end, "hyp_sleep_candidate"] = 1
# Need to reorganize the sequences.
df["hyp_seq_length"], df["hyp_seq_id"] = misc.get_consecutive_series(df, "hyp_sleep_candidate")
# new_sleep_segments = df[df[col_win_night + '_sleep_candidate'] == 1][col_win_night + '_grpid'].unique()
wearable.data = df.reset_index()
if only_largest_sleep_period: # If true, we keep only one sleep period per night.
saved_hour_start_day = wearable.hour_start_experiment
wearable.change_start_hour_for_experiment_day(sleep_search_window[0])
grps = wearable.data.groupby(wearable.experiment_day_col)
tmp_df = []
for grp_id, grp_df in grps:
gdf = grp_df.copy()
gdf["hyp_seq_length"], gdf["hyp_seq_id"] = misc.get_consecutive_series(gdf, "hyp_sleep_candidate")
df_out = misc.find_largest_sequence(gdf, "hyp_sleep_candidate", output_col).replace(-1, False)
tmp_df.append(df_out)
wearable.data[output_col] = pd.concat(tmp_df)
wearable.change_start_hour_for_experiment_day(saved_hour_start_day)
else:
# Save final output
wearable.data[output_col] = False
wearable.data.loc[wearable.data[(wearable.data["hyp_sleep_candidate"] == 1)].index, output_col] = True
# Clean up!
wearable.data.drop(
columns=["hyp_sleep", "hyp_sleep_candidate", "hyp_seq_id",
"hyp_sleep_bin",
"hyp_sleep_vard", "hyp_seq_length"], inplace=True)
def __sleep_boundaries_with_angle_change_algorithm(wearable: Wearable, output_col: str,
start_hour: int = 15,
cols: list = [],
use_triaxial_activity=False,
q_sleep: float = 0.1,
minimum_len_in_minutes: int = 30,
merge_tolerance_in_minutes: int = 180,
factor: int = 15,
operator: str = "or", # Either 'or' or 'and'
only_largest_sleep_period: bool = False
):
df_time = wearable.data.copy()
df_time = df_time.set_index(wearable.time_col)
five_min = int(5 * wearable.get_epochs_in_min())
minimum_len_in_minutes = int(minimum_len_in_minutes * wearable.get_epochs_in_min())
if use_triaxial_activity:
# Step 1:
df_time["hyp_rolling_x"] = df_time["hyp_act_x"].rolling("5s").median().fillna(0.0)
df_time["hyp_rolling_y"] = df_time["hyp_act_y"].rolling("5s").median().fillna(0.0)
df_time["hyp_rolling_z"] = df_time["hyp_act_z"].rolling("5s").median().fillna(0.0)
df_time["hyp_act_z"].rolling(five_min).median().fillna(0.0)
df_time["hyp_angle_z"] = (np.arctan(
df_time["hyp_rolling_z"] / ((df_time['hyp_rolling_y'] ** 2 + df_time['hyp_rolling_x'] ** 2) ** (
1 / 2)))) * 180 / np.pi
# Step 2:
df_time["hyp_angle_z"] = df_time["hyp_angle_z"].fillna(0.0)
# Step 3:
df_time["hyp_angle_z"] = df_time["hyp_angle_z"].rolling("5s").mean().fillna(0.0)
cols += ["hyp_angle_z"]
if operator == "or":
df_time["hyp_sleep_candidate"] = False
else:
df_time["hyp_sleep_candidate"] = True
for col in cols:
# Paper's Step 4
df_time["hyp_" + col + '_diff'] = df_time[col].diff().abs()
# Paper's Step 5
df_time["hyp_" + col + '_5mm'] = df_time["hyp_" + col + '_diff'].rolling(five_min).median().fillna(0.0)
# Paper's Step 6
quantiles_per_day = df_time["hyp_" + col + '_5mm'].resample('24H', offset="%dh" % start_hour).quantile(
q_sleep).dropna()
# print(quantiles_per_day)
df_time["hyp_" + col + '_10pct'] = quantiles_per_day
if quantiles_per_day.index[0] < df_time.index[0]:
df_time.loc[df_time.index[0], "hyp_" + col + '_10pct'] = quantiles_per_day.iloc[0]
df_time["hyp_" + col + '_10pct'] = df_time["hyp_" + col + '_10pct'].fillna(method='ffill').fillna(
method='bfill')
df_time["hyp_" + col + '_bin'] = np.where(
(df_time["hyp_" + col + '_5mm'] - (df_time["hyp_" + col + '_10pct'] * factor)) > 0, 0, 1)
df_time["hyp_" + col + '_len'], _ = misc.get_consecutive_series(df_time, "hyp_" + col + '_bin')
# Paper's Step 7
if operator == "or":
df_time["hyp_sleep_candidate"] = df_time["hyp_sleep_candidate"] | (
(df_time["hyp_" + col + '_bin'] == 1.0) & (
df_time["hyp_" + col + '_len'] > minimum_len_in_minutes))
else:
df_time["hyp_sleep_candidate"] = df_time["hyp_sleep_candidate"] & \
((df_time["hyp_" + col + '_bin'] == 1.0)
& (df_time["hyp_" + col + '_len'] > minimum_len_in_minutes))
# Gets the largest sleep_candidate per night
wearable.data = df_time.reset_index()
# wearable.data[output_col] = wearable.data["hyp_sleep_candidate"]
wearable.data["hyp_seq_length"], wearable.data["hyp_seq_id"] = misc.get_consecutive_series(wearable.data,
"hyp_sleep_candidate")
# Paper's Step 8
wearable.data["hyp_sleep_candidate"], wearable.data["hyp_seq_length"], wearable.data[
"hyp_seq_id"] = misc.merge_sequences_given_tolerance(wearable.data, wearable.time_col,
"hyp_sleep_candidate", merge_tolerance_in_minutes)
# Paper's Step 9
if only_largest_sleep_period: # If true, we keep only one sleep period per night.
saved_hour_start_day = wearable.hour_start_experiment
wearable.change_start_hour_for_experiment_day(start_hour)
grps = wearable.data.groupby(wearable.experiment_day_col)
tmp_df = []
for grp_id, grp_df in grps:
gdf = grp_df.copy()
gdf["hyp_seq_length"], gdf["hyp_seq_id"] = misc.get_consecutive_series(gdf, "hyp_sleep_candidate")
df_out = misc.find_largest_sequence(gdf, "hyp_sleep_candidate", output_col).replace(-1, False)
tmp_df.append(df_out)
wearable.data[output_col] = pd.concat(tmp_df)
wearable.change_start_hour_for_experiment_day(saved_hour_start_day)
else:
# Save final output
wearable.data[output_col] = False
wearable.data.loc[wearable.data[(wearable.data["hyp_sleep_candidate"] == 1)].index, output_col] = True
# Cleaning up...
cols_to_drop = ["hyp_sleep_candidate", "hyp_seq_length", "hyp_seq_id"]
for col in cols:
cols_to_drop.append("hyp_" + col + '_diff')
cols_to_drop.append("hyp_" + col + '_5mm')
cols_to_drop.append("hyp_" + col + '_10pct')
cols_to_drop.append("hyp_" + col + '_len')
wearable.data.drop(columns=cols_to_drop, inplace=True)
def view_ml_format_in_one_row(wearable: Wearable,
signal_categories: list,
sleep_cols: list,
alphas: dict = None,
colors: dict = None,
edgecolors: dict = None,
labels: dict = None):
# Convert zoom to datatime object:
textstr = 'day: validation id \n'
cols = []
for signal in signal_categories:
if signal == "activity":
cols.append(wearable.get_activity_col())
elif signal == "sleep":
for sleep_col in sleep_cols:
if sleep_col not in wearable.data.keys():
raise ValueError("Could not find sleep_col (%s). Aborting." % sleep_col)
cols.append(sleep_col)
else:
cols.append(signal)
if len(cols) == 0:
raise ValueError("Aborting: Empty list of signals to show.")
if wearable.data.empty:
warnings.warn("Aborting: Dataframe for PID %s is empty." % wearable.get_pid())
return
cols.append(wearable.time_col)
df_plot = wearable.data[cols].set_index(wearable.time_col)
### Add column for experiment day. It will be resampled using the the mean
cols.append(wearable.experiment_day_col)
changed_experiment_hour = False
df_plot[wearable.experiment_day_col] = wearable.data[
[wearable.time_col, wearable.experiment_day_col]].set_index(wearable.time_col)[wearable.experiment_day_col]
### Init fig plot
fig, ax1 = plt.subplots(1, 1, figsize=(21, 3))
maxy = 2
### Plot Activity
if "activity" in signal_categories:
y = df_plot[wearable.get_activity_col()]
alpha, color, edgecolor, label = Viewer.__get_details(alphas, colors, edgecolors, labels, "activity",
None, default_label="Activity")
maxy = max(maxy, df_plot[wearable.get_activity_col()].max())
ax1.plot(df_plot.index, y, label=label, linewidth=2,
color=color, alpha=alpha)
### Plot Sleep
if "sleep" in signal_categories:
facecolors = ['royalblue', 'green', 'orange']
endy = 0
alpha = 1
addition = (maxy / len(sleep_cols)) if len(sleep_cols) > 0 else maxy
for i, sleep_col in enumerate(sleep_cols):
starty = endy
endy = endy + addition
sleeping = df_plot[sleep_col] # TODO: get a method instead of an attribute
ax1.fill_between(df_plot.index, starty, endy, where=~sleeping, facecolor='red',
alpha=0.3, label=sleep_col, edgecolor='red')
ax1.fill_between(df_plot.index, starty, endy, where=sleeping, facecolor=facecolors[i],
alpha=0.3, label=sleep_col, edgecolor='purple')
# X-tick label
labels = []
for day in np.unique(df_plot[wearable.experiment_day_col]):
labels.append('Active ' + str(day + 1))
labels.append('Sleep ' + str(day + 1))
# remove last sleep
labels = labels[:-1]
# get indices at the middle of awake and sleep sequences
mean_indices = Viewer.get_rolling_mean(df_plot)
for label, awake_sleep_index in zip(labels, mean_indices):
ax1.text(awake_sleep_index, -0.1, label, fontsize=14,
verticalalignment='center',
horizontalalignment='center',
transform=ax1.transAxes)
### X-tick params
ax1.tick_params(axis='x', which='both', bottom=True, top=False, labelbottom=True, rotation=0,
labelsize='medium', pad=20)
ax1.tick_params(axis='x', which='major', bottom=False, labelbottom=False)
ax1.tick_params(axis='y', which='major')
ax1.set_facecolor('snow')
new_start_datetime = df_plot.index[0]
new_end_datetime = df_plot.index[-1]
ax1.set_xlim(new_start_datetime, new_end_datetime)
ax1.set_ylim(df_plot[wearable.get_activity_col()].min() - 5, df_plot[wearable.get_activity_col()].max() + 5)
y_label = 'Activity'
ax1.set_ylabel("%s" % y_label, rotation=0, horizontalalignment="right", verticalalignment="center")
ax1.xaxis.set_minor_locator(dates.HourLocator(byhour=[15])) # every 4 hours
ax1.xaxis.set_minor_formatter(dates.DateFormatter('%H:%M')) # hours and minutes
ax1.set_title("PID = %s" % wearable.get_pid(), fontsize=16)
ax1.set_xlabel('Time')
print(ax1.get_xticks())
plt.subplots_adjust(hspace=1.0)
plt.show()
return ax1, plt
class TestWearable(TestCase):
def setUp(self):
# Loads some file
pp1 = RawProcessing()
pp1.load_file("../data/examples_mesa/mesa-sample-day5-invalid5hours.csv",
# activitiy information
cols_for_activity=["activity"],
is_act_count=True,
# Datatime information
col_for_datatime="linetime",
device_location="dw",
start_of_week="dayofweek",
# Participant information
col_for_pid="mesaid")
self.w_5day_invalid5hours = Wearable(pp1)
def test_get_activity_col(self):
col_name = self.w_5day_invalid5hours.get_activity_col()
self.assertEqual(col_name, "hyp_act_x")
self.assertIsInstance(col_name, str)
def test_get_pid(self):
pid = self.w_5day_invalid5hours.get_pid()
self.assertEqual(pid, "1")
self.assertIsInstance(pid, str)
def test_get_experiment_day_col(self):
exp_day_col = self.w_5day_invalid5hours.get_experiment_day_col()
self.assertEqual(exp_day_col, "hyp_exp_day")
self.assertIsInstance(exp_day_col, str)
def test_get_time_col(self):
time_col = self.w_5day_invalid5hours.get_time_col()
self.assertEqual(time_col, "hyp_time")
self.assertIsInstance(time_col, str)
def test_get_frequency_in_secs(self):
freq = self.w_5day_invalid5hours.get_frequency_in_secs()
self.assertEqual(freq, 30)
self.assertIsInstance(freq, int)
def test_get_epochs_in_min(self):
nepochs = self.w_5day_invalid5hours.get_epochs_in_min()
self.assertEqual(nepochs, 2.0)
self.assertIsInstance(nepochs, float)
def test_get_epochs_in_hour(self):
nepochs = self.w_5day_invalid5hours.get_epochs_in_hour()
self.assertEqual(nepochs, 120.0)
self.assertIsInstance(nepochs, float)
def test_change_start_hour_for_experiment_day(self):
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(0)
# We are expecting to have only one experiment day and this will be day 5
self.assertEqual(self.w_5day_invalid5hours.data["hyp_exp_day"].unique()[0], 5)
# We now start our day at hour 18
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(18)
# print(tsp.wearable.data[2155:2165])
# Check if transition from artificial day 4 to day 5 is done correctly
self.assertEqual(self.w_5day_invalid5hours.data.iloc[2159]["hyp_exp_day"], 4)
self.assertEqual(self.w_5day_invalid5hours.data.iloc[2160]["hyp_exp_day"], 5)
# Randomly change the start hour and return it back to 18
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(18)
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(0)
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(15)
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(18)
self.assertEqual(self.w_5day_invalid5hours.data.iloc[2159]["hyp_exp_day"], 4)
self.assertEqual(self.w_5day_invalid5hours.data.iloc[2160]["hyp_exp_day"], 5)
def test_has_nan_activity(self):
self.assertTrue(self.w_5day_invalid5hours.has_no_activity())
self.w_5day_invalid5hours.fill_no_activity(1)
self.assertFalse(self.w_5day_invalid5hours.has_no_activity())
def test_valid_invalid_days(self):
# Should not return anything yet, as we never marked any row as invalid
invalid_days = self.w_5day_invalid5hours.get_invalid_days()
self.assertSetEqual(invalid_days, set())
valid_days = self.w_5day_invalid5hours.get_valid_days()
self.assertSetEqual(valid_days, set([5]))
# We now force some an invalid day
tsp = TimeSeriesProcessing(self.w_5day_invalid5hours)
tsp.detect_non_wear(strategy="choi2011")
tsp.check_valid_days(min_activity_threshold=0, max_non_wear_minutes_per_day=60)
invalid_days = self.w_5day_invalid5hours.get_invalid_days()
self.assertSetEqual(invalid_days, set({5}))
def add_to_wearable(self, wearable: Wearable) -> None:
pid = wearable.get_pid()
if pid not in self.data.index:
raise KeyError("PID %s not found in the demographic data." % pid)
wearable.demographics = self.data.loc[pid]
from hypnospy import Experiment
if __name__ == "__main__":
# Configure an Experiment
exp = Experiment()
file_path = "./data/small_collection_hchs/*"
# Iterates over a set of files in a directory.
# Unfortunately, we have to do it manually with RawProcessing because we are modifying the annotations
for file in glob(file_path):
pp = ActiwatchSleepData(file,
col_for_datetime="time",
col_for_pid="pid")
w = Wearable(pp) # Creates a wearable from a pp object
exp.add_wearable(w)
tsp = TimeSeriesProcessing(exp)
tsp.fill_no_activity(-0.0001)
tsp.detect_non_wear(strategy="choi")
tsp.check_consecutive_days(5)
print("Valid days:", tsp.get_valid_days())
print("Invalid days:", tsp.get_invalid_days())
tsp.detect_sleep_boundaries(strategy="annotation",
annotation_hour_to_start_search=18)
tsp.invalidate_day_if_no_sleep()
print("Valid days:", tsp.get_valid_days())
import matplotlib.pyplot as plt
pp = RawProcessing(
"./data/shiftworker/Shiftworker.csv",
# HR information
col_for_hr="mean_hr",
# Activity information
cols_for_activity=["ACC"],
is_act_count=True,
# Datetime information
col_for_datetime="real_time",
strftime="%d/%m/%Y %H:%M",
# Participant information
col_for_pid="id")
w = Wearable(pp)
#Check out data columns
#print(w.data.head(10))
#Define parameters fo HR-based sleep algorithm
hr_quantile = 0.4
hr_min_window_length = 60
hr_merge_blocks = 180
hr_volarity = 5
#Time to consider as start and end of each experiment day - if equal the sleep labelling occurs
#over the entire 24 hours
start_hour = 18
end_hour = 18
def view_signals_wearable_ml_format(wearable: Wearable,
signal_categories: list,
other_signals: list,
signal_as_area: list,
sleep_cols: list,
select_days: list,
zoom: list,
alphas: dict = None,
colors: dict = None,
edgecolors: dict = None,
labels: dict = None,
text: list = []):
# Convert zoom to datatime object:
assert len(zoom) == 2
zoom_start = datetime.strptime(zoom[0], '%H:%M:%S')
zoom_end = datetime.strptime(zoom[1], '%H:%M:%S')
textstr = 'day: validation id \n'
cols = []
for signal in signal_categories:
if signal == "activity":
cols.append(wearable.get_activity_col())
elif signal == "sleep":
for sleep_col in sleep_cols:
if sleep_col not in wearable.data.keys():
raise ValueError(
"Could not find sleep_col (%s). Aborting." %
sleep_col)
cols.append(sleep_col)
else:
cols.append(signal)
if len(cols) == 0:
raise ValueError("Aborting: Empty list of signals to show.")
if wearable.data.empty:
warnings.warn("Aborting: Dataframe for PID %s is empty." %
wearable.get_pid())
return
cols.append(wearable.time_col)
for col in set(other_signals + signal_as_area):
cols.append(col)
if "validation" in text:
df_plot = wearable.data[cols + ['hyp_invalid']].set_index(
wearable.time_col)
else:
df_plot = wearable.data[cols].set_index(wearable.time_col)
# Add column for experiment day. It will be resampled using the the mean
cols.append(wearable.experiment_day_col)
changed_experiment_hour = False
if not Viewer.__is_default_zoom(
zoom_start, zoom_end
) and zoom_start.hour != wearable.hour_start_experiment:
changed_experiment_hour = True
saved_start_hour = wearable.hour_start_experiment
wearable.change_start_hour_for_experiment_day(zoom_start.hour)
df_plot[wearable.experiment_day_col] = wearable.data[[
wearable.time_col, wearable.experiment_day_col
]].set_index(wearable.time_col)[wearable.experiment_day_col]
if select_days is not None:
df_plot = df_plot[df_plot[wearable.experiment_day_col].isin(
select_days)]
if df_plot.empty:
raise ValueError(
"Invalid day selection: no remaining data to show.")
dfs_per_group = [
pd.DataFrame(group[1])
for group in df_plot.groupby(wearable.experiment_day_col)
]
max_sequence_length = [len(g) for g in dfs_per_group]
max_sequence_length = max(max_sequence_length)
fig, ax1 = plt.subplots(len(dfs_per_group), 1, figsize=(14, 8))
if len(dfs_per_group) == 1:
ax1 = [ax1]
for idx in range(len(dfs_per_group)):
maxy = 2
df_panel = dfs_per_group[idx]
padding_values = np.zeros(max_sequence_length - len(df_panel))
if "activity" in signal_categories:
y = df_panel[wearable.get_activity_col()]
alpha, color, edgecolor, label = Viewer.__get_details(
alphas,
colors,
edgecolors,
labels,
"activity",
None,
default_label="Activity")
maxy = max(maxy, df_panel[wearable.get_activity_col()].max())
ax1[idx].plot(df_panel.index,
y,
label=label,
linewidth=2,
color=color,
alpha=alpha)
if "sleep" in signal_categories:
facecolors = ['royalblue', 'green', 'orange']
endy = 0
alpha = 1
addition = (maxy /
len(sleep_cols)) if len(sleep_cols) > 0 else maxy
for i, sleep_col in enumerate(sleep_cols):
starty = endy
endy = endy + addition
sleeping = df_panel[
sleep_col] # TODO: get a method instead of an attribute
ax1[idx].fill_between(df_panel.index,
starty,
endy,
where=sleeping,
facecolor=facecolors[i],
alpha=0.7,
label=sleep_col)
if "validation" in text and "hyp_invalid" in df_panel.keys():
textstr = textstr + str(idx) + ": " + str(
df_panel['hyp_invalid'].unique()[0]) + '\n'
for i, col in enumerate(other_signals):
# colors = ["orange", "violet", "pink", "gray"] # Change to paramters
ax1[idx].plot(df_panel.index,
df_panel[col],
label=col,
linewidth=1,
color=colors[i],
alpha=alpha)
endy = 0
addition = 0 if len(signal_as_area) == 0 else (maxy /
len(signal_as_area))
for i, col in enumerate(signal_as_area):
alpha, color, edgecolor, label = Viewer.__get_details(
alphas,
colors,
edgecolors,
labels,
"area",
i,
default_label=col,
default_color="blue")
starty = endy
endy = endy + addition
ax1[idx].fill_between(df_panel.index,
starty,
endy,
where=df_panel[col],
facecolor=color,
alpha=alpha,
label=label)
ax1[idx].tick_params(axis='x',
which='both',
bottom=False,
top=False,
labelbottom=True,
rotation=0)
ax1[idx].tick_params(axis='x', which='major', labelsize='small')
ax1[idx].set_facecolor('snow')
new_start_datetime = df_panel.index[0]
freq = wearable.get_frequency_in_secs()
new_end_datetime = new_start_datetime + pd.DateOffset(
seconds=freq * max_sequence_length)
ax1[idx].set_xlim(new_start_datetime, new_end_datetime)
y_label = idx
ax1[idx].set_ylabel("%s" % y_label,
rotation=0,
horizontalalignment="right",
verticalalignment="center")
ax1[idx].xaxis.set_major_locator(dates.DayLocator(interval=1))
ax1[idx].xaxis.set_major_formatter(dates.DateFormatter('%m-%d'))
ax1[idx].xaxis.set_minor_locator(
dates.HourLocator(interval=4)) # every 4 hours
ax1[idx].xaxis.set_minor_formatter(
dates.DateFormatter('%H:%M')) # hours and minutes
ax1[idx].set_yticks([])
ax1[0].set_title("PID = %s" % wearable.get_pid(), fontsize=16)
ax1[-1].set_xlabel('Epochs')
handles, labels = ax1[-1].get_legend_handles_labels()
fig.legend(handles,
labels,
loc='lower center',
ncol=len(cols),
fontsize=14,
shadow=True)
# place a text box in upper left in axes coords
if "validation" in text and "hyp_invalid" in wearable.data.columns:
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
fig.text(0.93,
0.87,
textstr,
fontsize=14,
verticalalignment='top',
bbox=props)
fig.savefig('%s_signals_ml_format.pdf' % (wearable.get_pid()),
dpi=300,
transparent=True,
bbox_inches='tight')
plt.subplots_adjust(hspace=1.0)
plt.show()
plt.close()
def view_signals_wearable(wearable: Wearable,
signal_categories: list,
other_signals: list,
signal_as_area: list,
resample_to: str,
sleep_cols: list,
select_days: list,
zoom: list,
alphas: dict = None,
colors: dict = None,
edgecolors: dict = None,
labels: dict = None,
text: list = []):
# Convert zoom to datatime object:
assert len(zoom) == 2
zoom_start = datetime.strptime(zoom[0], '%H:%M:%S')
zoom_end = datetime.strptime(zoom[1], '%H:%M:%S')
textstr = 'day: validation id \n'
cols = []
for signal in signal_categories:
if signal == "activity":
cols.append(wearable.get_activity_col())
elif signal == "hr":
if wearable.get_hr_col():
cols.append(wearable.get_hr_col())
else:
raise KeyError("HR is not available for PID %s" %
wearable.get_pid())
elif signal == "pa_intensity":
if hasattr(wearable, 'pa_cutoffs') and hasattr(
wearable, 'pa_names'):
for pa in wearable.pa_names:
if pa in wearable.data.keys():
cols.append(pa)
else:
raise ValueError(
"PA Intensity levels not available for PID %s" %
(wearable.get_pid()))
elif signal == "sleep":
for sleep_col in sleep_cols:
if sleep_col not in wearable.data.keys():
raise ValueError(
"Could not find sleep_col (%s). Aborting." %
sleep_col)
cols.append(sleep_col)
elif signal == "diary" and wearable.diary_onset in wearable.data.keys() and \
wearable.diary_offset in wearable.data.keys():
cols.append(wearable.diary_onset)
cols.append(wearable.diary_offset)
else:
cols.append(signal)
if len(cols) == 0:
raise ValueError("Aborting: Empty list of signals to show.")
if wearable.data.empty:
warnings.warn("Aborting: Dataframe for PID %s is empty." %
wearable.get_pid())
return
cols.append(wearable.time_col)
for col in set(other_signals + signal_as_area):
cols.append(col)
if "validation" in text:
df_plot = wearable.data[cols + ['hyp_invalid']].set_index(
wearable.time_col)
else:
df_plot = wearable.data[cols].set_index(wearable.time_col)
if resample_to is not None:
df_plot = df_plot.resample(resample_to).mean()
# Add column for experiment day. It will be resampled using the the mean
cols.append(wearable.experiment_day_col)
changed_experiment_hour = False
if not Viewer.__is_default_zoom(
zoom_start, zoom_end
) and zoom_start.hour != wearable.hour_start_experiment:
changed_experiment_hour = True
saved_start_hour = wearable.hour_start_experiment
wearable.change_start_hour_for_experiment_day(zoom_start.hour)
if resample_to is not None:
df_plot[wearable.experiment_day_col] = wearable.data[[
wearable.time_col, wearable.experiment_day_col
]].set_index(wearable.time_col).resample(resample_to).median()
else:
df_plot[wearable.experiment_day_col] = wearable.data[[
wearable.time_col, wearable.experiment_day_col
]].set_index(wearable.time_col)[wearable.experiment_day_col]
if changed_experiment_hour:
wearable.change_start_hour_for_experiment_day(saved_start_hour)
# Daily version
# dfs_per_day = [pd.DataFrame(group[1]) for group in df_plot.groupby(df_plot.index.day)]
# Based on the experiment day gives us the correct chronological order of the days
if select_days is not None:
df_plot = df_plot[df_plot[wearable.experiment_day_col].isin(
select_days)]
if df_plot.empty:
raise ValueError(
"Invalid day selection: no remaining data to show.")
dfs_per_group = [
pd.DataFrame(group[1])
for group in df_plot.groupby(wearable.experiment_day_col)
]
fig, ax1 = plt.subplots(len(dfs_per_group), 1, figsize=(14, 8))
if len(dfs_per_group) == 1:
ax1 = [ax1]
for idx in range(len(dfs_per_group)):
maxy = 2
df_panel = dfs_per_group[idx]
if "activity" in signal_categories:
alpha, color, edgecolor, label = Viewer.__get_details(
alphas,
colors,
edgecolors,
labels,
"activity",
None,
default_label="Activity")
maxy = max(maxy, df_panel[wearable.get_activity_col()].max())
ax1[idx].plot(df_panel.index,
df_panel[wearable.get_activity_col()],
label=label,
linewidth=2,
color=color,
alpha=alpha)
if "pa_intensity" in signal_categories:
#TODO: colors should not be limited to only these four
pa_predefined_colors = [
"palegoldenrod", "honeydew", "palegreen", "forestgreen"
]
for i in range(len(wearable.pa_names)):
pa_filter = df_panel[wearable.pa_names[i]]
for j in range(len(wearable.pa_names)):
if i != j:
pa_filter &= (~df_panel[wearable.pa_names[j]])
ax1[idx].fill_between(df_panel.index,
0,
maxy,
where=pa_filter,
label=wearable.pa_names[i],
alpha=alpha,
facecolor=pa_predefined_colors[i],
edgecolor=pa_predefined_colors[i])
if "sleep" in signal_categories:
facecolors = ['royalblue', 'green', 'orange']
endy = 0
alpha = 1
addition = (maxy /
len(sleep_cols)) if len(sleep_cols) > 0 else maxy
for i, sleep_col in enumerate(sleep_cols):
starty = endy
endy = endy + addition
sleeping = df_panel[
sleep_col] # TODO: get a method instead of an attribute
ax1[idx].fill_between(df_panel.index,
starty,
endy,
where=sleeping,
facecolor=facecolors[i],
alpha=0.7,
label=sleep_col)
if "diary" in signal_categories and wearable.diary_onset in df_panel.keys(
) and wearable.diary_offset in df_panel.keys():
diary_event = df_panel[
(df_panel[wearable.diary_onset] == True) |
(df_panel[wearable.diary_offset] == True)].index
ax1[idx].vlines(x=diary_event,
ymin=0,
ymax=maxy,
facecolor='black',
alpha=alpha,
label='Diary',
linestyles="dashed")
if "validation" in text and "hyp_invalid" in df_panel.keys():
textstr = textstr + str(idx) + ": " + str(
df_panel['hyp_invalid'].unique()[0]) + '\n'
for i, col in enumerate(other_signals):
# colors = ["orange", "violet", "pink", "gray"] # Change to paramters
ax1[idx].plot(df_panel.index,
df_panel[col],
label=col,
linewidth=1,
color=colors[i],
alpha=alpha)
endy = 0
addition = 0 if len(signal_as_area) == 0 else (maxy /
len(signal_as_area))
for i, col in enumerate(signal_as_area):
alpha, color, edgecolor, label = Viewer.__get_details(
alphas,
colors,
edgecolors,
labels,
"area",
i,
default_label=col,
default_color="blue")
starty = endy
endy = endy + addition
ax1[idx].fill_between(df_panel.index,
starty,
endy,
where=df_panel[col],
facecolor=color,
alpha=alpha,
label=label)
# configure time limits (y-axis) for plot.
ax1[idx].tick_params(axis='x',
which='both',
bottom=False,
top=False,
labelbottom=True,
rotation=0)
ax1[idx].set_facecolor('snow')
# If the user has not specified a zoom...
if Viewer.__is_default_zoom(zoom_start, zoom_end):
new_start_datetime = df_panel.index[0] - timedelta(
hours=(df_panel.index[0].hour -
wearable.hour_start_experiment) % 24,
minutes=df_panel.index[0].minute,
seconds=df_panel.index[0].second),
new_end_datetime = df_panel.index[0] - timedelta(
hours=(df_panel.index[0].hour -
wearable.hour_start_experiment) % 24,
minutes=df_panel.index[0].minute,
seconds=df_panel.index[0].second) + timedelta(minutes=1439)
else:
new_start_date = df_panel.index[0].date()
new_start_datetime = datetime(new_start_date.year,
new_start_date.month,
new_start_date.day,
zoom_start.hour,
zoom_start.minute,
zoom_start.second)
new_end_date = df_panel.index[-1].date()
new_end_datetime = datetime(new_end_date.year,
new_end_date.month,
new_end_date.day, zoom_end.hour,
zoom_end.minute, zoom_end.second)
if new_end_datetime < new_start_datetime:
print("Changing it here")
new_end_datetime = datetime(new_end_date.year,
new_end_date.month,
new_end_date.day + 1,
int(zoom_end.hour),
int(zoom_end.minute),
int(zoom_end.second))
new_start_datetime = pd.to_datetime(new_start_datetime)
new_end_datetime = pd.to_datetime(new_end_datetime)
ax1[idx].set_xlim(new_start_datetime, new_end_datetime)
y_label = Viewer.get_day_label(df_panel)
ax1[idx].set_ylabel("%s" % y_label,
rotation=0,
horizontalalignment="right",
verticalalignment="center")
ax1[idx].set_xticks([])
ax1[idx].set_yticks([])
# create a twin of the axis that shares the x-axis
if "hr" in signal_categories:
alpha, color, edgecolor, label = Viewer.__get_details(
alphas,
colors,
edgecolors,
labels,
"hr",
None,
default_label="HR",
default_color="red")
ax2 = ax1[idx].twinx()
ax2.plot(df_panel.index,
df_panel[wearable.get_hr_col()],
label=label,
color=color)
ax2.set_ylim(df_panel[wearable.get_hr_col()].min() - 5,
df_panel[wearable.get_hr_col()].max() + 5)
ax2.set_xticks([])
ax2.set_yticks([])
ax1[0].set_title("PID = %s" % wearable.get_pid(), fontsize=16)
ax1[-1].set_xlabel('Time')
ax1[-1].xaxis.set_minor_locator(
dates.HourLocator(interval=4)) # every 4 hours
ax1[-1].xaxis.set_minor_formatter(
dates.DateFormatter('%H:%M')) # hours and minutes
handles, labels = ax1[-1].get_legend_handles_labels()
# handles2, labels2 = ax2.get_legend_handles_labels()
# fig.legend(handles + handles2, labels + labels2, loc='lower center', ncol=4)
# return fig
# ax.figure.savefig('%s_signals.pdf' % (self.get_pid()))
# fig.suptitle("%s" % self.get_pid(), fontsize=16)
fig.legend(handles,
labels,
loc='lower center',
ncol=len(cols),
fontsize=14,
shadow=True)
# place a text box in upper left in axes coords
if "validation" in text and "hyp_invalid" in wearable.data.columns:
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)
fig.text(0.93,
0.87,
textstr,
fontsize=14,
verticalalignment='top',
bbox=props)
fig.savefig('%s_signals.pdf' % (wearable.get_pid()),
dpi=300,
transparent=True,
bbox_inches='tight')
plt.show()
plt.close()
# Title: Dont forget to add a title!
from hypnospy import Wearable
from hypnospy.data import MESAPreProcessing
from hypnospy.analysis import Viewer, NonWearingDetector, SleepWakeAnalysis
#from tf.keras.preprocessing import timeseries_dataset_from_array
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
# MESAPreProcessing is a specialized class to preprocess csv files from Philips Actiwatch Spectrum devices used in the MESA Sleep experiment
# MESA Sleep dataset can be found here: https://sleepdata.org/datasets/mesa/
preprocessed = MESAPreProcessing("../data/examples_mesa/mesa-sample.csv")
# Wearable is the main object in HypnosPy.
w = Wearable(preprocessed)
# In HypnosPy, we have the concept of ``experiment day'' which by default starts at midnight (00 hours).
# We can easily change it to any other time we wish. For example, lets run this script with experiment days
# that start at 3pm (15h)
w.change_start_hour_for_experiment_day(15)
# Sleep Wake Analysis module
sw = SleepWakeAnalysis(w)
sw.run_sleep_algorithm(
"ScrippsClinic",
inplace=True) # runs alg and creates new col named 'ScrippsClinic'
sw.run_sleep_algorithm(
"Cole-Kripke",
inplace=True) # runs alg and creates new col named 'Cole-Kripke'
sw.run_sleep_algorithm(
