def __sleep_boundaries_with_angle_change_algorithm(wearable: Wearable, output_col: str,
start_hour: int = 15,
cols: list = [],
use_triaxial_activity=False,
q_sleep: float = 0.1,
minimum_len_in_minutes: int = 30,
merge_tolerance_in_minutes: int = 180,
factor: int = 15,
operator: str = "or", # Either 'or' or 'and'
only_largest_sleep_period: bool = False
):
df_time = wearable.data.copy()
df_time = df_time.set_index(wearable.time_col)
five_min = int(5 * wearable.get_epochs_in_min())
minimum_len_in_minutes = int(minimum_len_in_minutes * wearable.get_epochs_in_min())
if use_triaxial_activity:
# Step 1:
df_time["hyp_rolling_x"] = df_time["hyp_act_x"].rolling("5s").median().fillna(0.0)
df_time["hyp_rolling_y"] = df_time["hyp_act_y"].rolling("5s").median().fillna(0.0)
df_time["hyp_rolling_z"] = df_time["hyp_act_z"].rolling("5s").median().fillna(0.0)
df_time["hyp_act_z"].rolling(five_min).median().fillna(0.0)
df_time["hyp_angle_z"] = (np.arctan(
df_time["hyp_rolling_z"] / ((df_time['hyp_rolling_y'] ** 2 + df_time['hyp_rolling_x'] ** 2) ** (
1 / 2)))) * 180 / np.pi
# Step 2:
df_time["hyp_angle_z"] = df_time["hyp_angle_z"].fillna(0.0)
# Step 3:
df_time["hyp_angle_z"] = df_time["hyp_angle_z"].rolling("5s").mean().fillna(0.0)
cols += ["hyp_angle_z"]
if operator == "or":
df_time["hyp_sleep_candidate"] = False
else:
df_time["hyp_sleep_candidate"] = True
for col in cols:
# Paper's Step 4
df_time["hyp_" + col + '_diff'] = df_time[col].diff().abs()
# Paper's Step 5
df_time["hyp_" + col + '_5mm'] = df_time["hyp_" + col + '_diff'].rolling(five_min).median().fillna(0.0)
# Paper's Step 6
quantiles_per_day = df_time["hyp_" + col + '_5mm'].resample('24H', offset="%dh" % start_hour).quantile(
q_sleep).dropna()
# print(quantiles_per_day)
df_time["hyp_" + col + '_10pct'] = quantiles_per_day
if quantiles_per_day.index[0] < df_time.index[0]:
df_time.loc[df_time.index[0], "hyp_" + col + '_10pct'] = quantiles_per_day.iloc[0]
df_time["hyp_" + col + '_10pct'] = df_time["hyp_" + col + '_10pct'].fillna(method='ffill').fillna(
method='bfill')
df_time["hyp_" + col + '_bin'] = np.where(
(df_time["hyp_" + col + '_5mm'] - (df_time["hyp_" + col + '_10pct'] * factor)) > 0, 0, 1)
df_time["hyp_" + col + '_len'], _ = misc.get_consecutive_series(df_time, "hyp_" + col + '_bin')
# Paper's Step 7
if operator == "or":
df_time["hyp_sleep_candidate"] = df_time["hyp_sleep_candidate"] | (
(df_time["hyp_" + col + '_bin'] == 1.0) & (
df_time["hyp_" + col + '_len'] > minimum_len_in_minutes))
else:
df_time["hyp_sleep_candidate"] = df_time["hyp_sleep_candidate"] & \
((df_time["hyp_" + col + '_bin'] == 1.0)
& (df_time["hyp_" + col + '_len'] > minimum_len_in_minutes))
# Gets the largest sleep_candidate per night
wearable.data = df_time.reset_index()
# wearable.data[output_col] = wearable.data["hyp_sleep_candidate"]
wearable.data["hyp_seq_length"], wearable.data["hyp_seq_id"] = misc.get_consecutive_series(wearable.data,
"hyp_sleep_candidate")
# Paper's Step 8
wearable.data["hyp_sleep_candidate"], wearable.data["hyp_seq_length"], wearable.data[
"hyp_seq_id"] = misc.merge_sequences_given_tolerance(wearable.data, wearable.time_col,
"hyp_sleep_candidate", merge_tolerance_in_minutes)
# Paper's Step 9
if only_largest_sleep_period: # If true, we keep only one sleep period per night.
saved_hour_start_day = wearable.hour_start_experiment
wearable.change_start_hour_for_experiment_day(start_hour)
grps = wearable.data.groupby(wearable.experiment_day_col)
tmp_df = []
for grp_id, grp_df in grps:
gdf = grp_df.copy()
gdf["hyp_seq_length"], gdf["hyp_seq_id"] = misc.get_consecutive_series(gdf, "hyp_sleep_candidate")
df_out = misc.find_largest_sequence(gdf, "hyp_sleep_candidate", output_col).replace(-1, False)
tmp_df.append(df_out)
wearable.data[output_col] = pd.concat(tmp_df)
wearable.change_start_hour_for_experiment_day(saved_hour_start_day)
else:
# Save final output
wearable.data[output_col] = False
wearable.data.loc[wearable.data[(wearable.data["hyp_sleep_candidate"] == 1)].index, output_col] = True
# Cleaning up...
cols_to_drop = ["hyp_sleep_candidate", "hyp_seq_length", "hyp_seq_id"]
for col in cols:
cols_to_drop.append("hyp_" + col + '_diff')
cols_to_drop.append("hyp_" + col + '_5mm')
cols_to_drop.append("hyp_" + col + '_10pct')
cols_to_drop.append("hyp_" + col + '_len')
wearable.data.drop(columns=cols_to_drop, inplace=True)
def __sleep_boundaries_with_hr(wearable: Wearable, output_col: str, quantile: float = 0.4,
volarity_threshold: int = 5, rolling_win_in_minutes: int = 5,
sleep_search_window: tuple = (20, 12), min_window_length_in_minutes: int = 40,
volatility_window_in_minutes: int = 10, merge_blocks_gap_time_in_min: int = 240,
sleep_only_in_sleep_search_window: bool = False,
only_largest_sleep_period: bool = False):
if wearable.hr_col is None:
raise AttributeError("HR is not available for PID %s." % (wearable.get_pid()))
rolling_win_in_minutes = int(rolling_win_in_minutes * wearable.get_epochs_in_min())
min_window_length_in_minutes = int(min_window_length_in_minutes * wearable.get_epochs_in_min())
volatility_window_in_minutes = int(volatility_window_in_minutes * wearable.get_epochs_in_min())
df = wearable.data.copy()
df["hyp_sleep"], df["hyp_sleep_bin"], df["hyp_seq_length"], df[
"hyp_seq_id"] = SleepBoudaryDetector.__create_threshold_col_based_on_time(wearable.data, wearable.time_col,
wearable.hr_col,
sleep_search_window[0],
sleep_search_window[1],
quantile,
rolling_win_in_minutes,
sleep_only_in_sleep_search_window)
df['hyp_sleep_candidate'] = ((df["hyp_sleep_bin"] == 1.0) & (
df['hyp_seq_length'] > min_window_length_in_minutes)).astype(int)
df["hyp_sleep_vard"] = df[wearable.hr_col].rolling(volatility_window_in_minutes,
center=True).std().fillna(0)
df["hyp_seq_length"], df["hyp_seq_id"] = misc.get_consecutive_series(df, "hyp_sleep_candidate")
# Merge two sleep segments if their gap is smaller than X min (interval per day):
wearable.data = df
saved_hour_start_day = wearable.hour_start_experiment
wearable.change_start_hour_for_experiment_day(sleep_search_window[0])
grps = wearable.data.groupby(wearable.experiment_day_col)
tmp_df = []
for grp_id, grp_df in grps:
gdf = grp_df.copy()
gdf["hyp_sleep_candidate"], gdf["hyp_seq_length"], gdf["hyp_seq_id"] = misc.merge_sequences_given_tolerance(
gdf, wearable.time_col, "hyp_sleep_candidate", tolerance_in_minutes=merge_blocks_gap_time_in_min)
tmp_df.append(gdf)
wearable.data = pd.concat(tmp_df)
wearable.change_start_hour_for_experiment_day(saved_hour_start_day)
df = wearable.data.set_index(wearable.time_col)
new_sleep_segments = df[df["hyp_sleep_candidate"] == 1]["hyp_seq_id"].unique()
# Check if we can modify the sleep onset/offset
for sleep_seg_id in new_sleep_segments:
actual_seg = df[df["hyp_seq_id"] == sleep_seg_id]
if actual_seg.shape[0] == 0:
continue
start_time = actual_seg.index[0]
end_time = actual_seg.index[-1]
look_sleep_onset = df[start_time - timedelta(hours=4): start_time + timedelta(minutes=60)]
look_sleep_offset = df[end_time - timedelta(minutes=1): end_time + timedelta(minutes=120)]
new_sleep_onset = look_sleep_onset[look_sleep_onset["hyp_sleep_vard"] > volarity_threshold]
new_sleep_offset = look_sleep_offset[look_sleep_offset["hyp_sleep_vard"] > volarity_threshold]
new_start = new_sleep_onset.index[-1] if not new_sleep_onset.empty else start_time
new_end = new_sleep_offset.index[0] if not new_sleep_offset.empty else end_time
df.loc[new_start:new_end, "hyp_seq_id"] = sleep_seg_id
# df.loc[new_start:new_end, "hyp_seq_length"] = df.loc[new_start:new_end].shape[0]
df.loc[new_start:new_end, "hyp_sleep_candidate"] = 1
# Need to reorganize the sequences.
df["hyp_seq_length"], df["hyp_seq_id"] = misc.get_consecutive_series(df, "hyp_sleep_candidate")
# new_sleep_segments = df[df[col_win_night + '_sleep_candidate'] == 1][col_win_night + '_grpid'].unique()
wearable.data = df.reset_index()
if only_largest_sleep_period: # If true, we keep only one sleep period per night.
saved_hour_start_day = wearable.hour_start_experiment
wearable.change_start_hour_for_experiment_day(sleep_search_window[0])
grps = wearable.data.groupby(wearable.experiment_day_col)
tmp_df = []
for grp_id, grp_df in grps:
gdf = grp_df.copy()
gdf["hyp_seq_length"], gdf["hyp_seq_id"] = misc.get_consecutive_series(gdf, "hyp_sleep_candidate")
df_out = misc.find_largest_sequence(gdf, "hyp_sleep_candidate", output_col).replace(-1, False)
tmp_df.append(df_out)
wearable.data[output_col] = pd.concat(tmp_df)
wearable.change_start_hour_for_experiment_day(saved_hour_start_day)
else:
# Save final output
wearable.data[output_col] = False
wearable.data.loc[wearable.data[(wearable.data["hyp_sleep_candidate"] == 1)].index, output_col] = True
# Clean up!
wearable.data.drop(
columns=["hyp_sleep", "hyp_sleep_candidate", "hyp_seq_id",
"hyp_sleep_bin",
"hyp_sleep_vard", "hyp_seq_length"], inplace=True)
class TestWearable(TestCase):
def setUp(self):
# Loads some file
pp1 = RawProcessing()
pp1.load_file("../data/examples_mesa/mesa-sample-day5-invalid5hours.csv",
# activitiy information
cols_for_activity=["activity"],
is_act_count=True,
# Datatime information
col_for_datatime="linetime",
device_location="dw",
start_of_week="dayofweek",
# Participant information
col_for_pid="mesaid")
self.w_5day_invalid5hours = Wearable(pp1)
def test_get_activity_col(self):
col_name = self.w_5day_invalid5hours.get_activity_col()
self.assertEqual(col_name, "hyp_act_x")
self.assertIsInstance(col_name, str)
def test_get_pid(self):
pid = self.w_5day_invalid5hours.get_pid()
self.assertEqual(pid, "1")
self.assertIsInstance(pid, str)
def test_get_experiment_day_col(self):
exp_day_col = self.w_5day_invalid5hours.get_experiment_day_col()
self.assertEqual(exp_day_col, "hyp_exp_day")
self.assertIsInstance(exp_day_col, str)
def test_get_time_col(self):
time_col = self.w_5day_invalid5hours.get_time_col()
self.assertEqual(time_col, "hyp_time")
self.assertIsInstance(time_col, str)
def test_get_frequency_in_secs(self):
freq = self.w_5day_invalid5hours.get_frequency_in_secs()
self.assertEqual(freq, 30)
self.assertIsInstance(freq, int)
def test_get_epochs_in_min(self):
nepochs = self.w_5day_invalid5hours.get_epochs_in_min()
self.assertEqual(nepochs, 2.0)
self.assertIsInstance(nepochs, float)
def test_get_epochs_in_hour(self):
nepochs = self.w_5day_invalid5hours.get_epochs_in_hour()
self.assertEqual(nepochs, 120.0)
self.assertIsInstance(nepochs, float)
def test_change_start_hour_for_experiment_day(self):
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(0)
# We are expecting to have only one experiment day and this will be day 5
self.assertEqual(self.w_5day_invalid5hours.data["hyp_exp_day"].unique()[0], 5)
# We now start our day at hour 18
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(18)
# print(tsp.wearable.data[2155:2165])
# Check if transition from artificial day 4 to day 5 is done correctly
self.assertEqual(self.w_5day_invalid5hours.data.iloc[2159]["hyp_exp_day"], 4)
self.assertEqual(self.w_5day_invalid5hours.data.iloc[2160]["hyp_exp_day"], 5)
# Randomly change the start hour and return it back to 18
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(18)
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(0)
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(15)
self.w_5day_invalid5hours.change_start_hour_for_experiment_day(18)
self.assertEqual(self.w_5day_invalid5hours.data.iloc[2159]["hyp_exp_day"], 4)
self.assertEqual(self.w_5day_invalid5hours.data.iloc[2160]["hyp_exp_day"], 5)
def test_has_nan_activity(self):
self.assertTrue(self.w_5day_invalid5hours.has_no_activity())
self.w_5day_invalid5hours.fill_no_activity(1)
self.assertFalse(self.w_5day_invalid5hours.has_no_activity())
def test_valid_invalid_days(self):
# Should not return anything yet, as we never marked any row as invalid
invalid_days = self.w_5day_invalid5hours.get_invalid_days()
self.assertSetEqual(invalid_days, set())
valid_days = self.w_5day_invalid5hours.get_valid_days()
self.assertSetEqual(valid_days, set([5]))
# We now force some an invalid day
tsp = TimeSeriesProcessing(self.w_5day_invalid5hours)
tsp.detect_non_wear(strategy="choi2011")
tsp.check_valid_days(min_activity_threshold=0, max_non_wear_minutes_per_day=60)
invalid_days = self.w_5day_invalid5hours.get_invalid_days()
self.assertSetEqual(invalid_days, set({5}))