class TestWearable(TestCase): def setUp(self): # Loads some file pp1 = RawProcessing() pp1.load_file("../data/examples_mesa/mesa-sample-day5-invalid5hours.csv", # activitiy information cols_for_activity=["activity"], is_act_count=True, # Datatime information col_for_datatime="linetime", device_location="dw", start_of_week="dayofweek", # Participant information col_for_pid="mesaid") self.w_5day_invalid5hours = Wearable(pp1) def test_get_activity_col(self): col_name = self.w_5day_invalid5hours.get_activity_col() self.assertEqual(col_name, "hyp_act_x") self.assertIsInstance(col_name, str) def test_get_pid(self): pid = self.w_5day_invalid5hours.get_pid() self.assertEqual(pid, "1") self.assertIsInstance(pid, str) def test_get_experiment_day_col(self): exp_day_col = self.w_5day_invalid5hours.get_experiment_day_col() self.assertEqual(exp_day_col, "hyp_exp_day") self.assertIsInstance(exp_day_col, str) def test_get_time_col(self): time_col = self.w_5day_invalid5hours.get_time_col() self.assertEqual(time_col, "hyp_time") self.assertIsInstance(time_col, str) def test_get_frequency_in_secs(self): freq = self.w_5day_invalid5hours.get_frequency_in_secs() self.assertEqual(freq, 30) self.assertIsInstance(freq, int) def test_get_epochs_in_min(self): nepochs = self.w_5day_invalid5hours.get_epochs_in_min() self.assertEqual(nepochs, 2.0) self.assertIsInstance(nepochs, float) def test_get_epochs_in_hour(self): nepochs = self.w_5day_invalid5hours.get_epochs_in_hour() self.assertEqual(nepochs, 120.0) self.assertIsInstance(nepochs, float) def test_change_start_hour_for_experiment_day(self): self.w_5day_invalid5hours.change_start_hour_for_experiment_day(0) # We are expecting to have only one experiment day and this will be day 5 self.assertEqual(self.w_5day_invalid5hours.data["hyp_exp_day"].unique()[0], 5) # We now start our day at hour 18 self.w_5day_invalid5hours.change_start_hour_for_experiment_day(18) # print(tsp.wearable.data[2155:2165]) # Check if transition from artificial day 4 to day 5 is done correctly self.assertEqual(self.w_5day_invalid5hours.data.iloc[2159]["hyp_exp_day"], 4) self.assertEqual(self.w_5day_invalid5hours.data.iloc[2160]["hyp_exp_day"], 5) # Randomly change the start hour and return it back to 18 self.w_5day_invalid5hours.change_start_hour_for_experiment_day(18) self.w_5day_invalid5hours.change_start_hour_for_experiment_day(0) self.w_5day_invalid5hours.change_start_hour_for_experiment_day(15) self.w_5day_invalid5hours.change_start_hour_for_experiment_day(18) self.assertEqual(self.w_5day_invalid5hours.data.iloc[2159]["hyp_exp_day"], 4) self.assertEqual(self.w_5day_invalid5hours.data.iloc[2160]["hyp_exp_day"], 5) def test_has_nan_activity(self): self.assertTrue(self.w_5day_invalid5hours.has_no_activity()) self.w_5day_invalid5hours.fill_no_activity(1) self.assertFalse(self.w_5day_invalid5hours.has_no_activity()) def test_valid_invalid_days(self): # Should not return anything yet, as we never marked any row as invalid invalid_days = self.w_5day_invalid5hours.get_invalid_days() self.assertSetEqual(invalid_days, set()) valid_days = self.w_5day_invalid5hours.get_valid_days() self.assertSetEqual(valid_days, set([5])) # We now force some an invalid day tsp = TimeSeriesProcessing(self.w_5day_invalid5hours) tsp.detect_non_wear(strategy="choi2011") tsp.check_valid_days(min_activity_threshold=0, max_non_wear_minutes_per_day=60) invalid_days = self.w_5day_invalid5hours.get_invalid_days() self.assertSetEqual(invalid_days, set({5}))
def view_signals_wearable(wearable: Wearable, signal_categories: list, other_signals: list, signal_as_area: list, resample_to: str, sleep_cols: list, select_days: list, zoom: list, alphas: dict = None, colors: dict = None, edgecolors: dict = None, labels: dict = None, text: list = []): # Convert zoom to datatime object: assert len(zoom) == 2 zoom_start = datetime.strptime(zoom[0], '%H:%M:%S') zoom_end = datetime.strptime(zoom[1], '%H:%M:%S') textstr = 'day: validation id \n' cols = [] for signal in signal_categories: if signal == "activity": cols.append(wearable.get_activity_col()) elif signal == "hr": if wearable.get_hr_col(): cols.append(wearable.get_hr_col()) else: raise KeyError("HR is not available for PID %s" % wearable.get_pid()) elif signal == "pa_intensity": if hasattr(wearable, 'pa_cutoffs') and hasattr( wearable, 'pa_names'): for pa in wearable.pa_names: if pa in wearable.data.keys(): cols.append(pa) else: raise ValueError( "PA Intensity levels not available for PID %s" % (wearable.get_pid())) elif signal == "sleep": for sleep_col in sleep_cols: if sleep_col not in wearable.data.keys(): raise ValueError( "Could not find sleep_col (%s). Aborting." % sleep_col) cols.append(sleep_col) elif signal == "diary" and wearable.diary_onset in wearable.data.keys() and \ wearable.diary_offset in wearable.data.keys(): cols.append(wearable.diary_onset) cols.append(wearable.diary_offset) else: cols.append(signal) if len(cols) == 0: raise ValueError("Aborting: Empty list of signals to show.") if wearable.data.empty: warnings.warn("Aborting: Dataframe for PID %s is empty." % wearable.get_pid()) return cols.append(wearable.time_col) for col in set(other_signals + signal_as_area): cols.append(col) if "validation" in text: df_plot = wearable.data[cols + ['hyp_invalid']].set_index( wearable.time_col) else: df_plot = wearable.data[cols].set_index(wearable.time_col) if resample_to is not None: df_plot = df_plot.resample(resample_to).mean() # Add column for experiment day. It will be resampled using the the mean cols.append(wearable.experiment_day_col) changed_experiment_hour = False if not Viewer.__is_default_zoom( zoom_start, zoom_end ) and zoom_start.hour != wearable.hour_start_experiment: changed_experiment_hour = True saved_start_hour = wearable.hour_start_experiment wearable.change_start_hour_for_experiment_day(zoom_start.hour) if resample_to is not None: df_plot[wearable.experiment_day_col] = wearable.data[[ wearable.time_col, wearable.experiment_day_col ]].set_index(wearable.time_col).resample(resample_to).median() else: df_plot[wearable.experiment_day_col] = wearable.data[[ wearable.time_col, wearable.experiment_day_col ]].set_index(wearable.time_col)[wearable.experiment_day_col] if changed_experiment_hour: wearable.change_start_hour_for_experiment_day(saved_start_hour) # Daily version # dfs_per_day = [pd.DataFrame(group[1]) for group in df_plot.groupby(df_plot.index.day)] # Based on the experiment day gives us the correct chronological order of the days if select_days is not None: df_plot = df_plot[df_plot[wearable.experiment_day_col].isin( select_days)] if df_plot.empty: raise ValueError( "Invalid day selection: no remaining data to show.") dfs_per_group = [ pd.DataFrame(group[1]) for group in df_plot.groupby(wearable.experiment_day_col) ] fig, ax1 = plt.subplots(len(dfs_per_group), 1, figsize=(14, 8)) if len(dfs_per_group) == 1: ax1 = [ax1] for idx in range(len(dfs_per_group)): maxy = 2 df_panel = dfs_per_group[idx] if "activity" in signal_categories: alpha, color, edgecolor, label = Viewer.__get_details( alphas, colors, edgecolors, labels, "activity", None, default_label="Activity") maxy = max(maxy, df_panel[wearable.get_activity_col()].max()) ax1[idx].plot(df_panel.index, df_panel[wearable.get_activity_col()], label=label, linewidth=2, color=color, alpha=alpha) if "pa_intensity" in signal_categories: #TODO: colors should not be limited to only these four pa_predefined_colors = [ "palegoldenrod", "honeydew", "palegreen", "forestgreen" ] for i in range(len(wearable.pa_names)): pa_filter = df_panel[wearable.pa_names[i]] for j in range(len(wearable.pa_names)): if i != j: pa_filter &= (~df_panel[wearable.pa_names[j]]) ax1[idx].fill_between(df_panel.index, 0, maxy, where=pa_filter, label=wearable.pa_names[i], alpha=alpha, facecolor=pa_predefined_colors[i], edgecolor=pa_predefined_colors[i]) if "sleep" in signal_categories: facecolors = ['royalblue', 'green', 'orange'] endy = 0 alpha = 1 addition = (maxy / len(sleep_cols)) if len(sleep_cols) > 0 else maxy for i, sleep_col in enumerate(sleep_cols): starty = endy endy = endy + addition sleeping = df_panel[ sleep_col] # TODO: get a method instead of an attribute ax1[idx].fill_between(df_panel.index, starty, endy, where=sleeping, facecolor=facecolors[i], alpha=0.7, label=sleep_col) if "diary" in signal_categories and wearable.diary_onset in df_panel.keys( ) and wearable.diary_offset in df_panel.keys(): diary_event = df_panel[ (df_panel[wearable.diary_onset] == True) | (df_panel[wearable.diary_offset] == True)].index ax1[idx].vlines(x=diary_event, ymin=0, ymax=maxy, facecolor='black', alpha=alpha, label='Diary', linestyles="dashed") if "validation" in text and "hyp_invalid" in df_panel.keys(): textstr = textstr + str(idx) + ": " + str( df_panel['hyp_invalid'].unique()[0]) + '\n' for i, col in enumerate(other_signals): # colors = ["orange", "violet", "pink", "gray"] # Change to paramters ax1[idx].plot(df_panel.index, df_panel[col], label=col, linewidth=1, color=colors[i], alpha=alpha) endy = 0 addition = 0 if len(signal_as_area) == 0 else (maxy / len(signal_as_area)) for i, col in enumerate(signal_as_area): alpha, color, edgecolor, label = Viewer.__get_details( alphas, colors, edgecolors, labels, "area", i, default_label=col, default_color="blue") starty = endy endy = endy + addition ax1[idx].fill_between(df_panel.index, starty, endy, where=df_panel[col], facecolor=color, alpha=alpha, label=label) # configure time limits (y-axis) for plot. ax1[idx].tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=True, rotation=0) ax1[idx].set_facecolor('snow') # If the user has not specified a zoom... if Viewer.__is_default_zoom(zoom_start, zoom_end): new_start_datetime = df_panel.index[0] - timedelta( hours=(df_panel.index[0].hour - wearable.hour_start_experiment) % 24, minutes=df_panel.index[0].minute, seconds=df_panel.index[0].second), new_end_datetime = df_panel.index[0] - timedelta( hours=(df_panel.index[0].hour - wearable.hour_start_experiment) % 24, minutes=df_panel.index[0].minute, seconds=df_panel.index[0].second) + timedelta(minutes=1439) else: new_start_date = df_panel.index[0].date() new_start_datetime = datetime(new_start_date.year, new_start_date.month, new_start_date.day, zoom_start.hour, zoom_start.minute, zoom_start.second) new_end_date = df_panel.index[-1].date() new_end_datetime = datetime(new_end_date.year, new_end_date.month, new_end_date.day, zoom_end.hour, zoom_end.minute, zoom_end.second) if new_end_datetime < new_start_datetime: print("Changing it here") new_end_datetime = datetime(new_end_date.year, new_end_date.month, new_end_date.day + 1, int(zoom_end.hour), int(zoom_end.minute), int(zoom_end.second)) new_start_datetime = pd.to_datetime(new_start_datetime) new_end_datetime = pd.to_datetime(new_end_datetime) ax1[idx].set_xlim(new_start_datetime, new_end_datetime) y_label = Viewer.get_day_label(df_panel) ax1[idx].set_ylabel("%s" % y_label, rotation=0, horizontalalignment="right", verticalalignment="center") ax1[idx].set_xticks([]) ax1[idx].set_yticks([]) # create a twin of the axis that shares the x-axis if "hr" in signal_categories: alpha, color, edgecolor, label = Viewer.__get_details( alphas, colors, edgecolors, labels, "hr", None, default_label="HR", default_color="red") ax2 = ax1[idx].twinx() ax2.plot(df_panel.index, df_panel[wearable.get_hr_col()], label=label, color=color) ax2.set_ylim(df_panel[wearable.get_hr_col()].min() - 5, df_panel[wearable.get_hr_col()].max() + 5) ax2.set_xticks([]) ax2.set_yticks([]) ax1[0].set_title("PID = %s" % wearable.get_pid(), fontsize=16) ax1[-1].set_xlabel('Time') ax1[-1].xaxis.set_minor_locator( dates.HourLocator(interval=4)) # every 4 hours ax1[-1].xaxis.set_minor_formatter( dates.DateFormatter('%H:%M')) # hours and minutes handles, labels = ax1[-1].get_legend_handles_labels() # handles2, labels2 = ax2.get_legend_handles_labels() # fig.legend(handles + handles2, labels + labels2, loc='lower center', ncol=4) # return fig # ax.figure.savefig('%s_signals.pdf' % (self.get_pid())) # fig.suptitle("%s" % self.get_pid(), fontsize=16) fig.legend(handles, labels, loc='lower center', ncol=len(cols), fontsize=14, shadow=True) # place a text box in upper left in axes coords if "validation" in text and "hyp_invalid" in wearable.data.columns: props = dict(boxstyle='round', facecolor='wheat', alpha=0.5) fig.text(0.93, 0.87, textstr, fontsize=14, verticalalignment='top', bbox=props) fig.savefig('%s_signals.pdf' % (wearable.get_pid()), dpi=300, transparent=True, bbox_inches='tight') plt.show() plt.close()
def view_signals_wearable_ml_format(wearable: Wearable, signal_categories: list, other_signals: list, signal_as_area: list, sleep_cols: list, select_days: list, zoom: list, alphas: dict = None, colors: dict = None, edgecolors: dict = None, labels: dict = None, text: list = []): # Convert zoom to datatime object: assert len(zoom) == 2 zoom_start = datetime.strptime(zoom[0], '%H:%M:%S') zoom_end = datetime.strptime(zoom[1], '%H:%M:%S') textstr = 'day: validation id \n' cols = [] for signal in signal_categories: if signal == "activity": cols.append(wearable.get_activity_col()) elif signal == "sleep": for sleep_col in sleep_cols: if sleep_col not in wearable.data.keys(): raise ValueError( "Could not find sleep_col (%s). Aborting." % sleep_col) cols.append(sleep_col) else: cols.append(signal) if len(cols) == 0: raise ValueError("Aborting: Empty list of signals to show.") if wearable.data.empty: warnings.warn("Aborting: Dataframe for PID %s is empty." % wearable.get_pid()) return cols.append(wearable.time_col) for col in set(other_signals + signal_as_area): cols.append(col) if "validation" in text: df_plot = wearable.data[cols + ['hyp_invalid']].set_index( wearable.time_col) else: df_plot = wearable.data[cols].set_index(wearable.time_col) # Add column for experiment day. It will be resampled using the the mean cols.append(wearable.experiment_day_col) changed_experiment_hour = False if not Viewer.__is_default_zoom( zoom_start, zoom_end ) and zoom_start.hour != wearable.hour_start_experiment: changed_experiment_hour = True saved_start_hour = wearable.hour_start_experiment wearable.change_start_hour_for_experiment_day(zoom_start.hour) df_plot[wearable.experiment_day_col] = wearable.data[[ wearable.time_col, wearable.experiment_day_col ]].set_index(wearable.time_col)[wearable.experiment_day_col] if select_days is not None: df_plot = df_plot[df_plot[wearable.experiment_day_col].isin( select_days)] if df_plot.empty: raise ValueError( "Invalid day selection: no remaining data to show.") dfs_per_group = [ pd.DataFrame(group[1]) for group in df_plot.groupby(wearable.experiment_day_col) ] max_sequence_length = [len(g) for g in dfs_per_group] max_sequence_length = max(max_sequence_length) fig, ax1 = plt.subplots(len(dfs_per_group), 1, figsize=(14, 8)) if len(dfs_per_group) == 1: ax1 = [ax1] for idx in range(len(dfs_per_group)): maxy = 2 df_panel = dfs_per_group[idx] padding_values = np.zeros(max_sequence_length - len(df_panel)) if "activity" in signal_categories: y = df_panel[wearable.get_activity_col()] alpha, color, edgecolor, label = Viewer.__get_details( alphas, colors, edgecolors, labels, "activity", None, default_label="Activity") maxy = max(maxy, df_panel[wearable.get_activity_col()].max()) ax1[idx].plot(df_panel.index, y, label=label, linewidth=2, color=color, alpha=alpha) if "sleep" in signal_categories: facecolors = ['royalblue', 'green', 'orange'] endy = 0 alpha = 1 addition = (maxy / len(sleep_cols)) if len(sleep_cols) > 0 else maxy for i, sleep_col in enumerate(sleep_cols): starty = endy endy = endy + addition sleeping = df_panel[ sleep_col] # TODO: get a method instead of an attribute ax1[idx].fill_between(df_panel.index, starty, endy, where=sleeping, facecolor=facecolors[i], alpha=0.7, label=sleep_col) if "validation" in text and "hyp_invalid" in df_panel.keys(): textstr = textstr + str(idx) + ": " + str( df_panel['hyp_invalid'].unique()[0]) + '\n' for i, col in enumerate(other_signals): # colors = ["orange", "violet", "pink", "gray"] # Change to paramters ax1[idx].plot(df_panel.index, df_panel[col], label=col, linewidth=1, color=colors[i], alpha=alpha) endy = 0 addition = 0 if len(signal_as_area) == 0 else (maxy / len(signal_as_area)) for i, col in enumerate(signal_as_area): alpha, color, edgecolor, label = Viewer.__get_details( alphas, colors, edgecolors, labels, "area", i, default_label=col, default_color="blue") starty = endy endy = endy + addition ax1[idx].fill_between(df_panel.index, starty, endy, where=df_panel[col], facecolor=color, alpha=alpha, label=label) ax1[idx].tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=True, rotation=0) ax1[idx].tick_params(axis='x', which='major', labelsize='small') ax1[idx].set_facecolor('snow') new_start_datetime = df_panel.index[0] freq = wearable.get_frequency_in_secs() new_end_datetime = new_start_datetime + pd.DateOffset( seconds=freq * max_sequence_length) ax1[idx].set_xlim(new_start_datetime, new_end_datetime) y_label = idx ax1[idx].set_ylabel("%s" % y_label, rotation=0, horizontalalignment="right", verticalalignment="center") ax1[idx].xaxis.set_major_locator(dates.DayLocator(interval=1)) ax1[idx].xaxis.set_major_formatter(dates.DateFormatter('%m-%d')) ax1[idx].xaxis.set_minor_locator( dates.HourLocator(interval=4)) # every 4 hours ax1[idx].xaxis.set_minor_formatter( dates.DateFormatter('%H:%M')) # hours and minutes ax1[idx].set_yticks([]) ax1[0].set_title("PID = %s" % wearable.get_pid(), fontsize=16) ax1[-1].set_xlabel('Epochs') handles, labels = ax1[-1].get_legend_handles_labels() fig.legend(handles, labels, loc='lower center', ncol=len(cols), fontsize=14, shadow=True) # place a text box in upper left in axes coords if "validation" in text and "hyp_invalid" in wearable.data.columns: props = dict(boxstyle='round', facecolor='wheat', alpha=0.5) fig.text(0.93, 0.87, textstr, fontsize=14, verticalalignment='top', bbox=props) fig.savefig('%s_signals_ml_format.pdf' % (wearable.get_pid()), dpi=300, transparent=True, bbox_inches='tight') plt.subplots_adjust(hspace=1.0) plt.show() plt.close()
def view_ml_format_in_one_row(wearable: Wearable, signal_categories: list, sleep_cols: list, alphas: dict = None, colors: dict = None, edgecolors: dict = None, labels: dict = None): # Convert zoom to datatime object: textstr = 'day: validation id \n' cols = [] for signal in signal_categories: if signal == "activity": cols.append(wearable.get_activity_col()) elif signal == "sleep": for sleep_col in sleep_cols: if sleep_col not in wearable.data.keys(): raise ValueError("Could not find sleep_col (%s). Aborting." % sleep_col) cols.append(sleep_col) else: cols.append(signal) if len(cols) == 0: raise ValueError("Aborting: Empty list of signals to show.") if wearable.data.empty: warnings.warn("Aborting: Dataframe for PID %s is empty." % wearable.get_pid()) return cols.append(wearable.time_col) df_plot = wearable.data[cols].set_index(wearable.time_col) ### Add column for experiment day. It will be resampled using the the mean cols.append(wearable.experiment_day_col) changed_experiment_hour = False df_plot[wearable.experiment_day_col] = wearable.data[ [wearable.time_col, wearable.experiment_day_col]].set_index(wearable.time_col)[wearable.experiment_day_col] ### Init fig plot fig, ax1 = plt.subplots(1, 1, figsize=(21, 3)) maxy = 2 ### Plot Activity if "activity" in signal_categories: y = df_plot[wearable.get_activity_col()] alpha, color, edgecolor, label = Viewer.__get_details(alphas, colors, edgecolors, labels, "activity", None, default_label="Activity") maxy = max(maxy, df_plot[wearable.get_activity_col()].max()) ax1.plot(df_plot.index, y, label=label, linewidth=2, color=color, alpha=alpha) ### Plot Sleep if "sleep" in signal_categories: facecolors = ['royalblue', 'green', 'orange'] endy = 0 alpha = 1 addition = (maxy / len(sleep_cols)) if len(sleep_cols) > 0 else maxy for i, sleep_col in enumerate(sleep_cols): starty = endy endy = endy + addition sleeping = df_plot[sleep_col] # TODO: get a method instead of an attribute ax1.fill_between(df_plot.index, starty, endy, where=~sleeping, facecolor='red', alpha=0.3, label=sleep_col, edgecolor='red') ax1.fill_between(df_plot.index, starty, endy, where=sleeping, facecolor=facecolors[i], alpha=0.3, label=sleep_col, edgecolor='purple') # X-tick label labels = [] for day in np.unique(df_plot[wearable.experiment_day_col]): labels.append('Active ' + str(day + 1)) labels.append('Sleep ' + str(day + 1)) # remove last sleep labels = labels[:-1] # get indices at the middle of awake and sleep sequences mean_indices = Viewer.get_rolling_mean(df_plot) for label, awake_sleep_index in zip(labels, mean_indices): ax1.text(awake_sleep_index, -0.1, label, fontsize=14, verticalalignment='center', horizontalalignment='center', transform=ax1.transAxes) ### X-tick params ax1.tick_params(axis='x', which='both', bottom=True, top=False, labelbottom=True, rotation=0, labelsize='medium', pad=20) ax1.tick_params(axis='x', which='major', bottom=False, labelbottom=False) ax1.tick_params(axis='y', which='major') ax1.set_facecolor('snow') new_start_datetime = df_plot.index[0] new_end_datetime = df_plot.index[-1] ax1.set_xlim(new_start_datetime, new_end_datetime) ax1.set_ylim(df_plot[wearable.get_activity_col()].min() - 5, df_plot[wearable.get_activity_col()].max() + 5) y_label = 'Activity' ax1.set_ylabel("%s" % y_label, rotation=0, horizontalalignment="right", verticalalignment="center") ax1.xaxis.set_minor_locator(dates.HourLocator(byhour=[15])) # every 4 hours ax1.xaxis.set_minor_formatter(dates.DateFormatter('%H:%M')) # hours and minutes ax1.set_title("PID = %s" % wearable.get_pid(), fontsize=16) ax1.set_xlabel('Time') print(ax1.get_xticks()) plt.subplots_adjust(hspace=1.0) plt.show() return ax1, plt