def __init__(self, accel_filepath, temperature_filepath, log_filepath=None):
"""Class that stores accelerometer and temperature data from a GENEActiv, epochs the data,
reads in timestamps that mark device removal/reattachment from .csv, and plots results.
:arguments
-accel_filepath: full pathway to accelerometer file
-temperature_pathway: full pathway to temperature file
-log_filepath: full pathway to removal/reattachment file.
-File should contain 2 columns for off and on timestamps, respectively."""
self.accel_filepath = accel_filepath
self.temperature_filepath = temperature_filepath
self.log_filepath = log_filepath
# Reads in accelerometer data, epochs
self.accel_raw = ImportEDF.GENEActiv(filepath=self.accel_filepath, from_processed=False)
self.accel_epoch = EpochData.EpochAccel(raw_data=self.accel_raw, from_processed=False, processed_folder="")
# Reads in temperature data
self.temperature = ImportEDF.GENEActivTemperature(filepath=self.temperature_filepath, from_processed=False)
# Reads in removal/reattachment log
self.off_stamps, self.on_stamps = self.read_log()
self.plot_nonwear()
def import_data(self):
"""Imports wrist and ECG data"""
self.lw = ImportEDF.GENEActiv(filepath=self.lw_file,
load_raw=True,
start_offset=self.crop_dict["LW"])
self.rw = ImportEDF.GENEActiv(filepath=self.rw_file,
load_raw=True,
start_offset=self.crop_dict["RW"])
self.ecg = ImportEDF.Bittium(filepath=self.ecg_file,
load_accel=False,
start_offset=self.crop_dict["ECG"],
epoch_len=self.epoch_len)
def import_data(bittium_output, raw_edf):
# Data generated from Cardiscope
bf = pd.read_excel(bittium_output)
cols = [i for i in bf.columns]
cols[0] = 'Timestamp'
cols[bf.columns.get_loc("Ø(HR)")] = "HR"
bf.columns = cols
bf = bf.iloc[2:]
bf["Timestamp"] = pd.to_datetime(bf["Timestamp"])
# Imports raw EDF data
data = ImportEDF.Bittium(filepath=raw_edf,
start_offset=0,
end_offset=0,
epoch_len=15,
load_accel=True,
low_f=1,
high_f=25,
f_type="bandpass")
# Crops Cardioscope data of epochs before Bittium file started...
bf = bf.loc[bf["Timestamp"] >= data.timestamps[0]]
bf = bf.reset_index()
bf = bf.drop("index", axis=1)
return bf, data
def import_devs(epoch_len=1):
"""Imports chest, left ankle, and left wrist EDF files (axivity). Manually input start_offset to ensure sync.
Epochs using average vector magnitude (gravity-subtracted).
:argument
-epoch_len: in seconds
:returns
-data objects for each device
"""
# EDF import and sync
chest = ImportEDF.GENEActiv(filepath=root_folder + "Converted/" +
dev_fnames["Chest"],
load_raw=True)
la = ImportEDF.GENEActiv(filepath=root_folder + "Converted/" +
dev_fnames["LA"],
load_raw=True,
start_offset=91)
lw = ImportEDF.GENEActiv(filepath=root_folder + "Converted/" +
dev_fnames["LW"],
load_raw=True,
start_offset=100)
# Average vector magnitude epoching
chest.epoch = [
np.mean(chest.vm[i:i + int(chest.sample_rate)])
for i in range(0,
len(chest.vm) - int(chest.sample_rate),
int(epoch_len * chest.sample_rate))
]
lw.epoch = [
np.mean(lw.vm[i:i + int(lw.sample_rate)])
for i in range(0,
len(lw.vm) -
int(lw.sample_rate), int(epoch_len * lw.sample_rate))
]
la.epoch = [
np.mean(la.vm[i:i + int(la.sample_rate)])
for i in range(0,
len(la.vm) -
int(la.sample_rate), int(epoch_len * la.sample_rate))
]
return chest, la, lw
def load_raw_data(self):
accel = ImportEDF.GENEActiv(filepath=self.filepath,
load_raw=self.load_raw,
start_offset=self.start_offset,
end_offset=0)
return accel.timestamps, accel.x, accel.y, accel.z, accel.vm, accel.sample_rate, accel.starttime
def __init__(self,
subjectID=None,
filepath=None,
output_dir=None,
load_raw=False,
accel_only=False,
epoch_len=15,
start_offset=0,
end_offset=0,
ecg_object=None,
from_processed=True,
processed_folder=None,
write_results=False):
print()
print(
"======================================== WRIST ACCELEROMETER ========================================"
)
self.subjectID = subjectID
self.filepath = filepath
self.filename = self.filepath.split("/")[-1].split(".")[0]
self.output_dir = output_dir
self.load_raw = load_raw
self.accel_only = accel_only
self.epoch_len = epoch_len
self.start_offset = start_offset
self.end_offset = end_offset
self.ecg_obejct = ecg_object
self.from_processed = from_processed
self.processed_folder = processed_folder
self.write_results = write_results
# Loads raw accelerometer data and generates timestamps
self.raw = ImportEDF.GENEActiv(filepath=self.filepath,
start_offset=self.start_offset,
end_offset=self.end_offset,
load_raw=self.load_raw)
self.epoch = EpochData.EpochAccel(raw_data=self.raw,
accel_only=self.accel_only,
from_processed=self.from_processed,
processed_folder=processed_folder)
# Model
self.model = WristModel(accel_object=self, ecg_object=self.ecg_obejct)
# Write results
if self.write_results:
self.write_model()
def load_raw_data(self):
ecg = ImportEDF.Bittium(filepath=self.filepath,
load_accel=self.load_accel,
start_offset=self.start_offset,
end_offset=self.end_offset,
low_f=self.low_f,
high_f=self.high_f,
f_type=self.f_type)
self.sample_rate = ecg.sample_rate
self.accel_sample_rate = ecg.accel_sample_rate
raw = ecg.raw
filtered = ecg.filtered
timestamps = ecg.timestamps
if self.ecg_downsample != 1:
print("\n-ECG data will be downsampled by a factor of "
"{} to {}Hz...".format(
self.ecg_downsample,
round(self.sample_rate / self.ecg_downsample, 1)))
self.sample_rate = int(self.sample_rate / self.ecg_downsample)
timestamps = timestamps[::self.ecg_downsample]
raw = raw[::self.ecg_downsample]
filtered = filtered[::self.ecg_downsample]
self.df_raw = pd.DataFrame(list(zip(timestamps, raw, filtered)),
columns=["Timestamp", "Raw", "Filtered"])
if not self.from_processed:
t = [
i for i in self.df_raw["Timestamp"].iloc[::self.sample_rate *
self.epoch_len]
]
self.df_epoch = pd.DataFrame(list(zip(t)), columns=["Timestamp"])
if self.load_accel:
self.df_accel = pd.DataFrame(
list(
zip(
self.df_raw["Timestamp"].
iloc[::int(self.sample_rate / self.accel_sample_rate)],
ecg.x, ecg.y, ecg.z, ecg.vm)),
columns=["Timestamp", "X", "Y", "Z", "VM"])
# Calculates accel activity counts
self.epoch_accel(vm_data=ecg.vm)
f = pyedflib.EdfReader(self.filepath)
if f.getSignalHeader(1)["transducer"] == "X-axis":
self.valid_accel = True
f.close()
def check_sync(self):
"""Checks start times for all given files. Makes sure all devices start at same time."""
lw_crop_index = 0
rw_crop_index = 0
ecg_crop_index = 0
lw_starttime, lw_endtime, lw_fs, lw_duration = ImportEDF.check_file(
filepath=self.lw_file, print_summary=False)
rw_starttime, rw_endtime, rw_fs, rw_duration = ImportEDF.check_file(
filepath=self.rw_file, print_summary=False)
ecg_starttime, ecg_endtime, ecg_fs, ecg_duration = ImportEDF.check_file(
filepath=self.ecg_file, print_summary=False)
crop_time = max([lw_starttime, rw_starttime, ecg_starttime])
if lw_starttime < crop_time:
lw_crop_index = int(
(crop_time - lw_starttime).total_seconds() * lw_fs)
if rw_starttime < crop_time:
rw_crop_index = int(
(crop_time - rw_starttime).total_seconds() * rw_fs)
if ecg_starttime < crop_time:
ecg_crop_index = int(
(crop_time - ecg_starttime).total_seconds() * ecg_fs)
self.crop_dict = {
"LW": lw_crop_index,
"RW": rw_crop_index,
"ECG": ecg_crop_index
}
if lw_fs != rw_fs:
print(
"\n-Accelerometer sampling rates do not match. Errors will ensue."
)
if lw_fs == rw_fs:
self.accel_fs = lw_fs
def import_edf(filepath):
"""Loads in raw data using ImportEDF script. Returns GENEActiv class object.
:argument
-filepath: pathway to EDF file
"""
data = ImportEDF.GENEActiv(filepath=filepath,
load_raw=True,
start_offset=0,
end_offset=0)
return data
def import_data(self):
print("\nImporting data...")
if self.wrist_obj is None:
if "csv" in self.wrist_file:
self.lw = pd.read_csv(self.wrist_file, skiprows=100)
if "edf" in self.wrist_file:
d = ImportEDF.GENEActiv(filepath=self.wrist_file, load_raw=True)
self.lw = pd.DataFrame(list(zip(d.timestamps, d.x, d.y, d.z, [None for i in range(len(d.timestamps))],
[None for i in range(len(d.timestamps))],
[None for i in range(len(d.timestamps))])))
self.lw.columns = ["Timestamp", "x", "y", "z", "light", 'button', 'temperature']
self.lw["Timestamp"] = pd.to_datetime(self.lw["Timestamp"], format="%Y-%m-%d %H:%M:%S:%f")
self.lw["Timestamp"] = pd.date_range(start=self.lw['Timestamp'].iloc[0], periods=self.lw.shape[0],
freq='{}ms'.format(1000/self.sample_rate))
if self.ankle_obj is None:
if "csv" in self.ankle_file:
self.la = pd.read_csv(self.ankle_file, skiprows=100)
if "edf" in self.ankle_file:
d = ImportEDF.GENEActiv(filepath=self.ankle_file, load_raw=True)
# d.sample_rate = 50
self.la = pd.DataFrame(list(zip(d.timestamps, d.x, d.y, d.z, [None for i in range(len(d.timestamps))],
[None for i in range(len(d.timestamps))],
[None for i in range(len(d.timestamps))])))
self.la.columns = ["Timestamp", "x", "y", "z", "light", 'button', 'temperature']
self.la["Timestamp"] = pd.to_datetime(self.la["Timestamp"], format="%Y-%m-%d %H:%M:%S:%f")
self.la["Timestamp"] = pd.date_range(start=self.la['Timestamp'].iloc[0], periods=self.la.shape[0],
freq='{}ms'.format(1000/self.sample_rate))
def import_data(self):
if self.la_filepath is not None:
self.la = ImportEDF.GENEActiv(filepath=self.la_filepath,
start_offset=0,
end_offset=0,
load_raw=True)
self.la.timestamps = [
datetime.strptime(str(i)[:-3], "%Y-%m-%dT%H:%M:%S.%f")
for i in self.la.timestamps
]
self.la.epoch_stamps, self.la.svm = self.epoch_data(self.la)
self.la.name = "LAnkle"
if self.ra_filepath is not None:
self.ra = ImportEDF.GENEActiv(filepath=self.ra_filepath,
start_offset=0,
end_offset=0,
load_raw=True)
self.ra.timestamps = [
datetime.strptime(str(i)[:-3], "%Y-%m-%dT%H:%M:%S.%f")
for i in self.ra.timestamps
]
self.ra.epoch_stamps, self.ra.svm = self.epoch_data(self.ra)
self.ra.name = "RAnkle"
if self.lw_filepath is not None:
self.lw = ImportEDF.GENEActiv(filepath=self.lw_filepath,
start_offset=0,
end_offset=0,
load_raw=True)
self.lw.timestamps = [
datetime.strptime(str(i)[:-3], "%Y-%m-%dT%H:%M:%S.%f")
for i in self.lw.timestamps
]
self.lw.epoch_stamps, self.lw.svm = self.epoch_data(self.lw)
self.lw.name = "LWrist"
def import_ecg(self):
f = ImportEDF.Bittium(filepath=self.filename, load_accel=True)
self.x = f.x
self.y = f.y
self.z = f.z
self.fs = f.accel_sample_rate
self.vm = (np.sqrt(
np.square(np.array([self.x, self.y, self.z])).sum(axis=0)) -
1000) / 1000
self.vm[self.vm < 0] = 0
starttime = f.starttime
self.timestamps = np.asarray(
pd.date_range(start=starttime,
periods=len(self.x),
freq="{}ms".format(1000 / self.fs)))
def __init__(self,
subjectID=None,
filepath=None,
load_raw=False,
accel_only=False,
output_dir=None,
rvo2=None,
age=None,
epoch_len=15,
start_offset=0,
end_offset=0,
remove_baseline=False,
ecg_object=None,
from_processed=True,
treadmill_log_file=None,
processed_folder=None,
write_results=False):
print()
print(
"======================================== ANKLE ACCELEROMETER ========================================"
)
self.subjectID = subjectID
self.filepath = filepath
self.filename = self.filepath.split("/")[-1].split(".")[0]
self.load_raw = load_raw
self.accel_only = accel_only
self.output_dir = output_dir
self.rvo2 = rvo2
self.age = age
self.epoch_len = epoch_len
self.start_offset = start_offset
self.end_offset = end_offset
self.remove_baseline = remove_baseline
self.ecg_object = ecg_object
self.from_processed = from_processed
self.processed_folder = processed_folder
self.treadmill_log_file = treadmill_log_file
self.treadmill_complete = True
self.write_results = write_results
# Loads raw accelerometer data and generates timestamps
self.raw = ImportEDF.GENEActiv(filepath=self.filepath,
start_offset=self.start_offset,
end_offset=self.end_offset,
load_raw=self.load_raw)
self.epoch = EpochData.EpochAccel(raw_data=self.raw,
epoch_len=self.epoch_len,
remove_baseline=self.remove_baseline,
accel_only=self.accel_only,
from_processed=self.from_processed,
processed_folder=processed_folder)
if self.treadmill_log_file is None:
print("\n" +
"Need treadmill protocol data to continue. Try again.")
plt.title("Set treadmill protocol walk indexes on datasheet")
plt.plot(np.arange(0, len(self.epoch.svm)),
self.epoch.svm,
color='black')
plt.ylabel("Counts")
plt.xlabel("Epoch Index")
# Create Treadmill object
self.treadmill = Treadmill(ankle_object=self)
# Create AnkleModel object
self.model = AnkleModel(ankle_object=self,
write_results=self.write_results,
ecg_object=self.ecg_object)
if self.write_results:
self.write_model()
def create_plot_gif(wrist_file=None, ankle_file=None, start_time=None, stop_time=None,
sample_rate=75, plot_period_ms=100, wrist_obj=None, ankle_obj=None,
output_dir=None,
slide_window=False, remove_gravity=False, remove_high_f=False, remove_dc=True):
print("\nImporting data...")
if wrist_obj is None:
if "csv" in wrist_file:
lw = pd.read_csv(wrist_file, skiprows=100)
if "edf" in wrist_file:
d = ImportEDF.GENEActiv(filepath=wrist_file, load_raw=True)
d.sample_rate = 50
lw = pd.DataFrame(list(zip(d.timestamps, d.x, d.y, d.z, [None for i in range(len(d.timestamps))],
[None for i in range(len(d.timestamps))], [None for i in range(len(d.timestamps))])))
lw.columns = ["Timestamp", "x", "y", "z", "light", 'button', 'temperature']
lw["Timestamp"] = pd.to_datetime(lw["Timestamp"], format="%Y-%m-%d %H:%M:%S:%f")
if start_time is not None and stop_time is not None:
lw = lw.loc[(lw["Timestamp"] >= pd.to_datetime(start_time)) &
(lw["Timestamp"] < pd.to_datetime(stop_time))]
if ankle_obj is None:
if "csv" in ankle_file:
la = pd.read_csv(ankle_file, skiprows=100)
if "edf" in ankle_file:
d = ImportEDF.GENEActiv(filepath=ankle_file, load_raw=True)
d.sample_rate = 50
la = pd.DataFrame(list(zip(d.timestamps, d.x, d.y, d.z, [None for i in range(len(d.timestamps))],
[None for i in range(len(d.timestamps))],
[None for i in range(len(d.timestamps))])))
la.columns = ["Timestamp", "x", "y", "z", "light", 'button', 'temperature']
la["Timestamp"] = pd.to_datetime(la["Timestamp"], format="%Y-%m-%d %H:%M:%S:%f")
if start_time is not None and stop_time is not None:
la = la.loc[(la["Timestamp"] >= pd.to_datetime(start_time)) &
(la["Timestamp"] < pd.to_datetime(stop_time))]
filenames = []
# Converts cropped data to list
time = [i / sample_rate for i in range(lw.shape[0])]
lw_x = [i for i in lw["x"]]
lw_y = [i for i in lw["y"]]
lw_z = [i for i in lw["z"]]
la_x = [i for i in la["x"]]
la_y = [i for i in la["y"]]
la_z = [i for i in la["z"]]
if remove_gravity:
print("-Filtering data to remove gravity...")
lw_x = filter_signal(data=lw_x, filter_type="highpass", high_f=0.1, filter_order=2, sample_f=sample_rate)
lw_y = filter_signal(data=lw_y, filter_type="highpass", high_f=0.1, filter_order=2, sample_f=sample_rate)
lw_z = filter_signal(data=lw_z, filter_type="highpass", high_f=0.1, filter_order=2, sample_f=sample_rate)
la_x = filter_signal(data=la_x, filter_type="highpass", high_f=0.1, filter_order=2, sample_f=sample_rate)
la_y = filter_signal(data=la_y, filter_type="highpass", high_f=0.1, filter_order=2, sample_f=sample_rate)
la_z = filter_signal(data=la_z, filter_type="highpass", high_f=0.1, filter_order=2, sample_f=sample_rate)
if remove_high_f:
print("-Filtering data to remove high frequency...")
lw_x = filter_signal(data=lw_x, filter_type="lowpass", low_f=5, filter_order=2, sample_f=sample_rate)
lw_y = filter_signal(data=lw_y, filter_type="lowpass", low_f=5, filter_order=2, sample_f=sample_rate)
lw_z = filter_signal(data=lw_z, filter_type="lowpass", low_f=5, filter_order=2, sample_f=sample_rate)
la_x = filter_signal(data=la_x, filter_type="lowpass", low_f=5, filter_order=2, sample_f=sample_rate)
la_y = filter_signal(data=la_y, filter_type="lowpass", low_f=5, filter_order=2, sample_f=sample_rate)
la_z = filter_signal(data=la_z, filter_type="lowpass", low_f=5, filter_order=2, sample_f=sample_rate)
if remove_dc:
print("\n-Removing DC component from signal...")
lw_x = [i - np.mean(lw_x) for i in lw_x]
lw_y = [i - np.mean(lw_y) for i in lw_y]
lw_z = [i - np.mean(lw_z) for i in lw_z]
la_x = [i - np.mean(la_x) for i in la_x]
la_y = [i - np.mean(la_y) for i in la_y]
la_z = [i - np.mean(la_z) for i in la_z]
min_x = min([min(lw_x), min(la_x)])
min_y = min([min(lw_y), min(la_y)])
min_z = min([min(lw_z), min(la_z)])
max_x = max([max(lw_x), max(la_x)])
max_y = max([max(lw_y), max(la_y)])
max_z = max([max(lw_z), max(la_z)])
min_all = min([min_x, min_y, min_z])
max_all = max([max_x, max_y, max_z])
plot_rate = int(np.ceil(plot_period_ms / (1000 / sample_rate)))
if plot_rate == 0:
plot_rate = 1
print("\n-Data will be plotted in {}ms increments...\n".format(plot_period_ms))
for i in range(0, lw.shape[0], plot_rate):
print("-Generating plot {} of {}...".format(int((i/plot_rate))+1, int(len(range(0, lw.shape[0], plot_rate)))))
fig, (ax1, ax2) = plt.subplots(2, figsize=(10, 6))
plt.subplots_adjust(right=.75, left=.07, hspace=.3)
ax1.plot(time[:i], lw_x[:i], color='black')
ax1.plot(time[:i], lw_y[:i], color='red')
ax1.plot(time[:i], lw_z[:i], color='dodgerblue')
ax1.axvline(time[i], color='limegreen')
ax2.plot(time[:i], la_x[:i], color='black')
ax2.plot(time[:i], la_y[:i], color='red')
ax2.plot(time[:i], la_z[:i], color='dodgerblue')
ax2.axvline(time[i], color='limegreen')
ax1.set_ylim(min_all - .5, max_all + .5)
ax2.set_ylim(min_all - .5, max_all + .5)
if not slide_window:
ax1.set_xlim(0, len(lw_x)/sample_rate)
ax2.set_xlim(0, len(la_x)/sample_rate)
if slide_window:
if time[i] <= 12.5:
ax1.set_xlim(0, 15)
ax2.set_xlim(0, 15)
if time[i] > 12.5:
ax1.set_xlim(time[i]-7.5, time[i]+7.5)
ax2.set_xlim(time[i]-7.5, time[i]+7.5)
ax2.set_xlabel("Time (seconds)")
ax1.set_ylabel("Acceleration")
ax2.set_ylabel("Acceleration")
ax1.set_title("Left Wrist")
ax2.set_title("Left Ankle")
ax1.set_ylabel("Acceleration")
# create file name and append it to a list
filename = f'{i}.png'
filenames.append(filename)
plt.savefig(output_dir + filename)
plt.close()
# build gif
print("\nCombining images into gif...")
with imageio.get_writer(output_dir + "Output.gif", mode='I') as writer:
for filename in filenames:
image = imageio.imread(output_dir + filename)
writer.append_data(image)
# Remove files
for filename in set(filenames):
os.remove(output_dir + filename)
print("\nComplete.")
return lw, la
def __init__(self, filepath=None, run_qc_check=True,
start_offset=0, end_offset=0,
epoch_len=15, load_accel=False,
filter_data=False, low_f=1, high_f=30, f_type="bandpass"):
"""Class that contains raw and processed ECG data.
:argument
DATA IMPORT
-filepath: full pathway to EDF file
-start_offset, end_offset: indexes used to crop data to match other devices
FILTERING: used for visualization (not peak detection)
-filter: whether or not to filter the data
-low_f, high_1: cut-off frequencies for the filter. Set to None if irrelevant. In Hz.
-f_type: type of filter; "lowpass", "highpass", "bandpass"
"""
print()
print("============================================= ECG DATA ==============================================")
self.filepath = filepath
self.filename = filepath.split("/")[-1]
self.subject_id = self.filename.split("_")[2]
self.epoch_len = epoch_len
self.start_offset = start_offset
self.end_offset = end_offset
self.load_accel = load_accel
self.filter_data = filter_data
self.low_f = low_f
self.high_f = high_f
self.f_type = f_type
self.accel_sample_rate = 1
self.accel_x = None
self.accel_y = None
self.accel_z = None
self.accel_vm = None
self.svm = []
# Raw data
self.ecg = ImportEDF.Bittium(filepath=self.filepath, load_accel=self.load_accel,
start_offset=self.start_offset, end_offset=self.end_offset,
low_f=self.low_f, high_f=self.high_f, f_type=self.f_type)
self.sample_rate = self.ecg.sample_rate
self.accel_sample_rate = self.ecg.accel_sample_rate
self.raw = self.ecg.raw
self.filtered = self.ecg.filtered
self.timestamps = self.ecg.timestamps
self.epoch_timestamps = self.ecg.epoch_timestamps
self.accel_x, self.accel_y, self.accel_z, self.accel_vm = self.ecg.x, self.ecg.y, self.ecg.z, self.ecg.vm
del self.ecg
self.wavelet = self.wavelet_transform()[:len(self.timestamps)]
# Performs quality control check on raw data and epochs data
if run_qc_check:
self.epoch_validity, self.epoch_hr, self.avg_voltage, self.rr_sd, self.r_peaks = self.check_quality()
# List of epoched heart rates but any invalid epoch is marked as None instead of 0 (as is self.epoch_hr)
self.valid_hr = [self.epoch_hr[i] if self.epoch_validity[i] == 0 else None for i in range(len(self.epoch_hr))]
self.quality_report = self.generate_quality_report()
self.rolling_avg_hr = None
def __init__(self,
subject_id=None,
raw_filepath=None,
proc_filepath=None,
filename=None,
temperature_filepath=None,
output_dir=None,
load_raw=False,
accel_only=False,
epoch_len=15,
start_offset=0,
end_offset=0,
ecg_object=None,
from_processed=True,
processed_folder=None):
print()
print(
"======================================== WRIST ACCELEROMETER ========================================"
)
self.subject_id = subject_id
self.filepath = raw_filepath
self.proc_filepath = proc_filepath
self.temperature_filepath = temperature_filepath
self.filename = filename
self.output_dir = output_dir
self.load_raw = load_raw
self.accel_only = accel_only
self.epoch_len = epoch_len
self.start_offset = start_offset
self.end_offset = end_offset
self.ecg_object = ecg_object
self.from_processed = from_processed
self.processed_folder = processed_folder
# Loads raw accelerometer data and generates timestamps
self.raw = ImportEDF.GENEActiv(filepath=self.filepath,
start_offset=self.start_offset,
end_offset=self.end_offset,
load_raw=self.load_raw)
self.epoch = EpochData.EpochAccel(
raw_data=self.raw,
accel_type="wrist",
raw_filename=self.filename,
proc_filepath=self.proc_filepath,
accel_only=self.accel_only,
epoch_len=self.epoch_len,
from_processed=self.from_processed,
processed_folder=self.processed_folder)
# Model
self.model = WristModel(accel_object=self, ecg_object=self.ecg_object)
# Temperature data
self.temperature = ImportEDF.GENEActivTemperature(
filepath=self.temperature_filepath)
if self.load_raw:
self.temperature.sample_rate = 1 / (300 / self.raw.sample_rate)
if not self.load_raw:
self.temperature.sample_rate = 0.25
def crop_files(self):
if self.from_processed:
print("Data is being imported from processed. Skipping file crop.")
return None
if not self.from_processed:
# File summaries
print("Raw EDF file summaries...")
ImportEDF.check_file(self.ankle_filepath)
ImportEDF.check_file(self.wrist_filepath)
ImportEDF.check_file(self.ecg_filepath)
# If only one device available
if self.load_ecg + self.load_wrist + self.load_ankle == 1:
self.start_offset_dict = {"Ankle": 0, "Wrist": 0, "ECG": 0}
self.end_offset_dict = {"Ankle": 0, "Wrist": 0, "ECG": 0}
# If ECG and at least one accelerometer are available
if self.ecg_filepath is not None and (
self.wrist_filepath is not None
or self.ankle_filepath is not None):
# Reads in data from file if available
if self.crop_index_file is not None:
self.start_offset_dict, self.end_offset_dict, self.crop_indexes_found = \
ImportCropIndexes.import_crop_indexes(subject=self.subjectID, crop_file=self.crop_index_file)
# Reads data from raw if crop file not available or no data found for participant
if self.crop_index_file is None or not self.crop_indexes_found:
self.start_offset_dict = DeviceSync.crop_start(
subject_object=self)
self.end_offset_dict = DeviceSync.crop_end(
subject_object=self)
# If ECG not available but wrist and ankle accelerometers are
if self.ecg_filepath is None and self.wrist_filepath is not None and self.ankle_filepath is not None:
# Reads from csv if available
if self.crop_index_file is not None:
self.start_offset_dict, self.end_offset_dict, self.crop_indexes_found = \
ImportCropIndexes.import_crop_indexes(subject=self.subjectID, crop_file=self.crop_index_file)
# Reads from raw if participant not found in csv or csv does not exist
if not self.crop_indexes_found:
print(
"Crop file not entered/found. Ankle treadmill protocol indexes may be incorrect."
)
self.start_offset_dict = DeviceSync.crop_start(
subject_object=self)
# Overwrites end indexes with values from raw accel files (excludes ECG)
self.end_offset_dict = DeviceSync.crop_end(subject_object=self)
# Sets to default values if reading from processed (values not used) if not raw data is read in
if self.from_processed and not self.load_raw_ecg and not self.load_raw_ankle and not self.load_raw_wrist:
self.start_offset_dict = {"Ankle": 0, "Wrist": 0, "ECG": 0}
self.end_offset_dict = {"Ankle": 0, "Wrist": 0, "ECG": 0}
print("Start indexes: ankle = {}, wrist = {}, ECG = {}".format(
self.start_offset_dict["Ankle"],
self.start_offset_dict["Wrist"],
self.start_offset_dict["ECG"]))
print("Data points to be read: ankle = {}, wrist = {}, ECG = {}".
format(self.end_offset_dict["Ankle"],
self.end_offset_dict["Wrist"],
self.end_offset_dict["ECG"]))
import Filtering
from ecgdetectors import Detectors
import pandas as pd
import ECG_Quality_Check
"""================================================== LIU ET AL. 2018 =============================================="""
"""
data = ImportEDF.Bittium(filepath="/Users/kyleweber/Desktop/Student Supervision/Kin 472 - Megan/Data/Converted/"
"Collection 1/3LeadHIIT1.EDF",
start_offset=0, end_offset=0, epoch_len=10, load_accel=False,
low_f=.67, high_f=30, f_type="bandpass")
"""
data = ImportEDF.Bittium(
filepath="/Users/kyleweber/Desktop/Data/ECG Files/OND07_WTL_3023_01_BF.EDF",
start_offset=0,
end_offset=0,
epoch_len=10,
load_accel=False,
low_f=.67,
high_f=30,
f_type="bandpass")
detectors = Detectors(data.sample_rate)
# Number of datapoints corresponding to 125ms window
err_size = int(125 / (1000 / data.sample_rate))
def run_algorithm(threshold=.9, win_len=10, show_plot=True):
# Data lists
bsqi_list = []
def __init__(self,
subject_id=None,
raw_filepath=None,
proc_filepath=None,
filename=None,
load_raw=False,
accel_only=False,
output_dir=None,
rvo2=None,
age=None,
bmi=1,
epoch_len=15,
start_offset=0,
end_offset=0,
remove_baseline=False,
ecg_object=None,
from_processed=True,
treadmill_log_file=None,
treadmill_regression_file=None,
processed_folder=None,
write_results=False):
print()
print(
"======================================== ANKLE ACCELEROMETER ========================================"
)
self.subject_id = subject_id
self.filepath = raw_filepath
self.filename = filename
self.proc_filepath = proc_filepath
self.load_raw = load_raw
self.accel_only = accel_only
self.output_dir = output_dir
self.rvo2 = rvo2
self.age = age
self.bmi = bmi
self.epoch_len = epoch_len
self.start_offset = start_offset
self.end_offset = end_offset
self.remove_baseline = remove_baseline
self.ecg_object = ecg_object
self.from_processed = from_processed
self.processed_folder = processed_folder
self.treadmill_log_file = treadmill_log_file
self.treadmill_regression_file = treadmill_regression_file
self.write_results = write_results
# Loads raw accelerometer data and generates timestamps
self.raw = ImportEDF.GENEActiv(filepath=self.filepath,
start_offset=self.start_offset,
end_offset=self.end_offset,
load_raw=self.load_raw)
self.epoch = EpochData.EpochAccel(
raw_data=self.raw,
accel_type="ankle",
raw_filename=self.filename,
proc_filepath=self.proc_filepath,
epoch_len=self.epoch_len,
remove_baseline=self.remove_baseline,
accel_only=self.accel_only,
from_processed=self.from_processed,
processed_folder=self.processed_folder)
# Create Treadmill object
self.treadmill = Treadmill(ankle_object=self)
# Create AnkleModel object
self.model = AnkleModel(ankle_object=self,
bmi=self.bmi,
write_results=self.write_results,
ecg_object=self.ecg_object)
edf_folder = "/Volumes/nimbal$/Data/ReMiNDD/Raw data/Bittium/"
# csv file to write to (full path)
data_file = "/Users/kyleweber/Desktop/ECG_Datafile.csv"
# Seconds
epoch_length = 15
# ===================================================== PROCESSING ====================================================
file_list = os.listdir(edf_folder)
file_list = [i for i in file_list if "BF" in i]
rand_sub = random.randint(0, len(file_list) - 1)
start_time, end_time, fs, duration = \
ImportEDF.check_file(edf_folder + file_list[rand_sub],
print_summary=False)
rand_start = randint(0, duration * fs - 45 * fs)
print("\nImporting file {}".format(file_list[rand_sub]))
ecg_object = ECG.ECG(filepath=edf_folder+file_list[rand_sub], age=0,
start_offset=rand_start, end_offset=3 * epoch_length * fs,
epoch_len=epoch_length, load_raw=True, load_accel=True, from_processed=False)
qc_data = ECG.CheckQuality(ecg_object=ecg_object, start_index=epoch_length*fs,
template_data='wavelet', voltage_thresh=250, epoch_len=epoch_length)
parameters_dict = {"Initials": initials,
"ID": file_list[rand_sub].split(".")[0], "StartInd": rand_start,
"VisualInspection": None,
"OrphanidouAlgorithm": "Valid" if qc_data.rule_check_dict["Valid Period"] else "Invalid",
# ax2.plot(np.arange(len(s2))/60, high_var_flag, color='purple')
def remove_low_var_epochs(data, low_var_flag):
print("\nZeroing regions of non-wear...")
d = data.copy()
for i, val in enumerate(low_var_flag):
if val > 0:
d[int(i * 125):int((i + 1) * 125)] = np.zeros(125)
print("Complete.")
return d
ecg = ImportEDF.Bittium(filepath="/Users/kyleweber/Desktop/BG7_FastFix.edf")
data = downsample_data(raw_voltage=ecg.raw, old_fs=ecg.sample_rate, new_fs=125)
# data = convert_to_mv(raw_data=data)
# data = filter_data(raw_data=data)
# s2, low_var_flag, high_var_flag = check_variance(data=data, fs=125, nw_thresh=.00015, noise_thresh=1, fill_gaps=5)
#plot_variance()
# d = remove_low_var_epochs(data, low_var_flag)
"""
# BG7_Lead
d = [11400000, int(1.22e7)] # clean
# d = [int(1.46e7), int(1.48e7)] # nw
f, Pxx = scipy.signal.welch(x=data[d[0]:d[1]], fs=125, nperseg=int(125*60), scaling="density")
psd = pd.DataFrame(list(zip(f.transpose(), Pxx.transpose())), columns=["Freq", "Power"])
del f, Pxx
def qc_check(
raw_edf_folder='/Users/kyleweber/Desktop/Data/OND07/EDF/',
# output_file="/Users/kyleweber/Desktop/Data/OND07/Tabular Data/QualityControl_Output.csv"
output_file="/Users/kyleweber/Desktop/ECGNonwear.csv",
subject_num=None,
start=None,
epoch_len=15,
sample_rate=250,
write_results=True,
show_fft=False):
"""Imports a random segment of a random ECG file, runs the quality check algorithm on it, plots the raw and
filtered data, and appends the quality check results to a .csv file.
"""
plt.close()
print("\n" + "Plotting random section of data from random participant...")
sub_list = np.arange(3002, 3045)
if subject_num is None and start is None:
subjectID = random.choice(
np.delete(sub_list, np.argwhere(sub_list == 3038)))
if subject_num is not None:
subjectID = subject_num
ecg_filepath = raw_edf_folder + "OND07_WTL_{}_01_BF.EDF".format(subjectID)
print("Using file {}".format(ecg_filepath))
# if subject_num is None and start is None:
if start is None:
file_start, file_end, fs = ImportEDF.check_file(ecg_filepath,
print_summary=False)
file_duration = ((file_end - file_start).days * 86400 +
(file_end - file_start).seconds) * fs
start_index = randint(0, file_duration - epoch_len * sample_rate)
start_index -= start_index % (sample_rate * epoch_len)
if start is not None:
start_index = start
print("Testing index {}-{} ({}-second window).".format(
start_index, start_index + epoch_len * sample_rate, epoch_len))
ecg_object = ECG.ECG(filepath=ecg_filepath,
age=0,
start_offset=start_index,
end_offset=epoch_len * sample_rate,
epoch_len=15,
load_raw=True,
load_accel=True,
from_processed=False,
write_results=False)
ecg_object.subjectID = subjectID
plt.ion()
validity_data = ECG.ECG.plot_random_qc(self=ecg_object,
input_index=0).rule_check_dict
plt.show(block=True)
plt.ioff()
plt.close()
user_entry = input()
if user_entry == "1":
user_entry = "Non-wear"
else:
user_entry = "Wear"
output_data = [
subjectID, start_index, validity_data["Valid Period"],
validity_data["HR Valid"], validity_data["HR"],
validity_data["Max RR Interval Valid"],
validity_data["Max RR Interval"], validity_data["RR Ratio Valid"],
validity_data["RR Ratio"], validity_data["Voltage Range Valid"],
validity_data["Voltage Range"], validity_data["Correlation Valid"],
validity_data["Correlation"], validity_data["Accel Counts"],
validity_data["Accel Flatline"], validity_data["Accel SD"], user_entry
]
if write_results:
with open(output_file, "a") as outfile:
writer = csv.writer(outfile, lineterminator="\n", delimiter=",")
writer.writerow(output_data)
print("Result saved.")
if not write_results:
print("Result not saved.")
df_fft = None
if show_fft:
y = np.fft.fft(ecg_object.raw[:ecg_object.sample_rate * epoch_len])
f = np.fft.fftfreq(len(y), 1 / ecg_object.sample_rate)
df_fft = pd.DataFrame(list(zip(f, np.abs(y))),
columns=["Freq", "Power"])
df_data = pd.DataFrame(list(
zip(np.arange(0, epoch_len, 1 / ecg_object.sample_rate),
ecg_object.raw[:ecg_object.sample_rate * epoch_len])),
columns=["Time (s)", "Voltage"])
fig, (ax1, ax2) = plt.subplots(2, figsize=(9, 6))
fig.subplots_adjust(hspace=.25)
plt.suptitle("ECG FFT ({}-second window)".format(epoch_len))
ax1.plot(df_data["Time (s)"], df_data["Voltage"], color='red')
ax2.plot(df_fft["Freq"].loc[df_fft["Freq"] >= 0],
df_fft["Power"].loc[df_fft["Freq"] >= 0],
color='black')
ax2.axvline(60,
linestyle='dashed',
color='orange',
label="60Hz",
linewidth=1)
if ecg_object.epoch_validity[0] == 0:
ax2.axvline(ecg_object.epoch_hr[0] / 60,
color='red',
linewidth=1,
label="HR ({}bpm = {}Hz)".format(
ecg_object.epoch_hr[0],
round(ecg_object.epoch_hr[0] / 60, 1)))
for i in range(2, 6):
ax2.axvline(ecg_object.epoch_hr[0] / 60 * i,
color='red',
linewidth=1,
linestyle='dashed',
label='HR Harmonic')
ax2.set_xlim(-1, 65)
ax2.set_xticks(np.arange(0, 65, 5))
ax1.set_ylabel("Voltage")
ax1.set_xlabel("Time (s)")
ax2.set_ylabel("Power")
ax2.set_xlabel("Hz")
ax2.legend()
return ecg_object, output_data, user_entry, df_fft
def plot_segment(segment=None):
plt.close("all")
if segment is None:
rando = random.choice(num_list)
num_list.remove(rando)
if segment is not None:
rando = segment
data = ImportEDF.Bittium(
filepath="/Users/kyleweber/Desktop/Data/OND07/EDF/OND07_WTL_{}_01_BF.EDF"
.format(df["ID"].loc[rando]),
start_offset=df["Index"].loc[rando],
end_offset=int(15 * 250),
epoch_len=15,
load_accel=False,
low_f=1,
high_f=30,
f_type="bandpass")
filt = Filtering.filter_signal(data=data.raw,
sample_f=data.sample_rate,
low_f=.6,
high_f=30,
filter_type='bandpass')
r = ECG_Quality_Check.RedmondQC(ecg_signal=data.raw,
sample_rate=data.sample_rate,
start_index=0,
stop_index=None,
epoch_len=data.epoch_len)
fig, (ax1, ax2, ax3, ax4) = plt.subplots(4, sharex='col', figsize=(10, 6))
ax1.set_title("Row {}, {} % valid".format(
rando, round(r.final_mask.count("Valid") * 100 / len(r.final_mask),
2)))
ax1.plot(np.arange(0, len(data.raw)) / data.sample_rate,
data.raw,
color='red',
label="Raw")
ax1.plot(np.arange(0, len(data.raw)) / data.sample_rate,
filt,
color='black',
label="Filt")
ax1.legend()
ax2.plot(np.arange(0, len(data.raw)) / data.sample_rate,
r.lowf_data,
color='dodgerblue',
label="Low F Data")
ax2.legend()
ax3.plot(np.arange(0, len(data.raw)) / data.sample_rate,
r.highf_data,
color='green',
label='High F Data')
ax3.axhline(y=30, color='black')
ax3.legend()
ax4.plot(np.arange(0, len(r.final_mask)) / data.sample_rate,
r.final_mask,
color='black')
ax4.plot(np.arange(0, len(r.lowf_mask)) / data.sample_rate,
r.lowf_mask,
color='dodgerblue')
ax4.plot(np.arange(0, len(r.highf_mask)) / data.sample_rate,
r.highf_mask,
color='green')
return r
def __init__(self,
filepath=None,
output_dir=None,
age=0,
start_offset=0,
end_offset=0,
rest_hr_window=60,
n_epochs_rest=10,
epoch_len=15,
load_accel=False,
filter=False,
low_f=1,
high_f=30,
f_type="bandpass",
load_raw=False,
from_processed=True,
write_results=True):
"""Class that contains raw and processed ECG data.
:argument
DATA IMPORT
-filepath: full pathway to EDF file
-load_raw: boolean of whether to load raw ECG data. Can be used in addition to from_processed = True
-from_processed: boolean of whether to read in already processed data
(epoch timestamps, epoch HR, quality control check)
-output_dir: where files are written to OR where processed data files are read in from
-start_offset, end_offset: indexes used to crop data to match other devices
DATA EPOCHING
-rest_hr_window: number of seconds over which HR is averaged when calculating resting HR
-Creates a rolling average of HR over this many seconds (rounded to match epoch length)
-n_epochs_rest: number of epochs used in the resting HR calculation
(averages HR over the n_epochs_rest lower HRs)
-epoch_len: time period over which data is processed, seconds
FILTERING
-filter: whether or not to filter the data
-low_f, high_1: cut-off frequencies for the filter. Set to None if irrelevant. In Hz.
-f_type: type of filter; "lowpass", "highpass", "bandpass"
OTHER
-write_results: boolean of whether to write output file
-age: participant age in years. Needed for HRmax calculation.
"""
print()
print(
"============================================= ECG DATA =============================================="
)
self.filepath = filepath
self.filename = self.filepath.split("/")[-1].split(".")[0]
self.subjectID = self.filename.split("_")[2]
self.output_dir = output_dir
self.age = age
self.epoch_len = epoch_len
self.rest_hr_window = rest_hr_window
self.n_epochs_rest = n_epochs_rest
self.start_offset = start_offset
self.end_offset = end_offset
self.filter = filter
self.low_f = low_f
self.high_f = high_f
self.f_type = f_type
self.load_raw = load_raw
self.load_accel = load_accel
self.from_processed = from_processed
self.write_results = write_results
self.accel_sample_rate = 1
self.accel_x = None
self.accel_y = None
self.accel_z = None
self.accel_vm = None
self.svm = []
# Raw data
if self.load_raw:
self.ecg = ImportEDF.Bittium(filepath=self.filepath,
load_accel=self.load_accel,
start_offset=self.start_offset,
end_offset=self.end_offset,
filter=self.filter,
low_f=self.low_f,
high_f=self.high_f,
f_type=self.f_type)
self.sample_rate = self.ecg.sample_rate
self.accel_sample_rate = self.ecg.accel_sample_rate
self.raw = self.ecg.raw
self.filtered = self.ecg.filtered
self.timestamps = self.ecg.timestamps
self.epoch_timestamps = self.ecg.epoch_timestamps
self.accel_x, self.accel_y, self.accel_z, self.accel_vm = self.ecg.x, self.ecg.y, self.ecg.z, self.ecg.vm
del self.ecg
if self.load_accel:
self.epoch_accel()
# Performs quality control check on raw data and epochs data
if self.from_processed:
self.epoch_validity, self.epoch_hr = None, None
if not self.from_processed:
self.epoch_validity, self.epoch_hr, self.avg_voltage, self.rr_sd = self.check_quality(
)
# Loads epoched data from existing file
if self.from_processed:
self.epoch_timestamps, self.epoch_validity, self.epoch_hr = self.load_processed(
)
# List of epoched heart rates but any invalid epoch is marked as None instead of 0 (as is self.epoch_hr)
self.valid_hr = [
self.epoch_hr[i] if self.epoch_validity[i] == 0 else None
for i in range(len(self.epoch_hr))
]
self.quality_report = self.generate_quality_report()
self.rolling_avg_hr = None
self.rest_hr = None
self.perc_hrr = None
self.epoch_intensity = None
self.epoch_intensity_totals = None
# This block is called later from the Subject class after sleep data is processed
# self.rolling_avg_hr, self.rest_hr, self.awake_hr = self.find_resting_hr(window_size=self.rest_hr_window)
# self.perc_hrr = self.calculate_percent_hrr()
# self.epoch_intensity, self.intensity_totals = self.calculate_intensity()
if self.write_results:
self.write_model()
"/Users/kyleweber/Desktop/Data/OND07/Tabular Data/ECG_QualityControl_Testing_KW.xlsx"
)
percent_valid = []
for row in df.itertuples():
subj_id = row.ID
start_index = row.Index
print("-Importing subject {}, index {}...".format(subj_id, start_index))
data = ImportEDF.Bittium(
filepath="/Users/kyleweber/Desktop/Data/OND07/EDF/OND07_WTL_{}_01_BF.EDF"
.format(subj_id),
start_offset=start_index,
end_offset=int(start_index + 15 * 250),
epoch_len=15,
load_accel=False,
low_f=1,
high_f=30,
f_type="bandpass")
r = ECG_Quality_Check.RedmondQC(ecg_signal=data.raw,
sample_rate=data.sample_rate,
start_index=0,
stop_index=None,
epoch_len=data.epoch_len)
percent_val = round(
r.final_mask.count("Valid") * 100 / len(r.final_mask), 2)
percent_valid.append(percent_val)
def import_all_accels(self, timestamp=None, duration=120):
"""Imports all available accelerometer and temperature data from participants in specified time period.
Crops data to start 20 minutes before 'timestamp' and reads in 'duration' number of minutes."""
# Timestamp formatting ----------------------------------------------------------------------------------------
if timestamp is None:
timestamp = self.nw_log.iloc[0]["start_time"]
if timestamp is not None:
try:
timestamp = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S")
except (TypeError, ValueError):
try:
timestamp = timestamp + timedelta(minutes=-10)
except TypeError:
timestamp = datetime.strptime(
timestamp,
"%Y-%m-%dT%H:%M:%S") + timedelta(minutes=-20)
start_time, end_time, fs, dur = ImportEDF.check_file(
self.filename, print_summary=False)
self.timestamp = timestamp
self.duration = duration
start_index = int((timestamp - start_time).total_seconds() * fs)
stop_index = int(duration * fs * 60)
# Left ankle -------------------------------------------------------------------------------------------------
if os.path.exists(self.filename_blank.format("LAnkle")):
la = GENEActiv(filepath=self.filename_blank.format("LAnkle"),
load_raw=True,
start_offset=start_index,
end_offset=stop_index)
la_time = la.timestamps
lax = la.x
lay = la.y
laz = la.z
if not os.path.exists(self.filename_blank.format("LAnkle")):
print("-LAnkle file not found.")
la_time = [
timestamp + timedelta(seconds=i / fs)
for i in range(stop_index)
]
lax = [0 for i in range(stop_index)]
lay = [0 for i in range(stop_index)]
laz = [0 for i in range(stop_index)]
if os.path.exists(self.temp_filename_blank.format("LAnkle")):
la_temp = GENEActivTemperature(
filepath=self.temp_filename_blank.format("LAnkle"),
start_offset=0,
end_offset=0)
df = pd.DataFrame(list(zip(la_temp.timestamps,
la_temp.temperature)),
columns=["Time", "Temp"])
df = df.loc[(df["Time"] >= timestamp) & (
df["Time"] <= timestamp + timedelta(minutes=duration))]
lat = df["Temp"]
la_temp_time = df["Time"]
if not os.path.exists(self.temp_filename_blank.format("LAnkle")):
print("-LAnkle temperature file not found.")
lat = [None for i in range(int(stop_index / 300))]
la_temp_time = [
timestamp + timedelta(seconds=i / 300)
for i in range(int(stop_index / 300))
]
# Right ankle -------------------------------------------------------------------------------------------------
if os.path.exists(self.filename_blank.format("RAnkle")):
ra = GENEActiv(filepath=self.filename_blank.format("RAnkle"),
load_raw=True,
start_offset=start_index,
end_offset=stop_index)
ra_time = ra.timestamps
rax = ra.x
ray = ra.y
raz = ra.z
if not os.path.exists(self.filename_blank.format("RAnkle")):
print("-RAnkle file not found.")
ra_time = [
timestamp + timedelta(seconds=i / fs)
for i in range(stop_index)
]
rax = [0 for i in range(stop_index)]
ray = [0 for i in range(stop_index)]
raz = [0 for i in range(stop_index)]
if os.path.exists(self.temp_filename_blank.format("RAnkle")):
ra_temp = GENEActivTemperature(
filepath=self.temp_filename_blank.format("RAnkle"),
start_offset=0,
end_offset=0)
df = pd.DataFrame(list(zip(ra_temp.timestamps,
ra_temp.temperature)),
columns=["Time", "Temp"])
df = df.loc[(df["Time"] >= timestamp) & (
df["Time"] <= timestamp + timedelta(minutes=duration))]
rat = df["Temp"]
ra_temp_time = df["Time"]
if not os.path.exists(self.temp_filename_blank.format("RAnkle")):
print("-RAnkle temperature file not found.")
rat = [0 for i in range(int(stop_index / 300))]
ra_temp_time = [None for i in range(int(stop_index / 300))]
# Left wrist --------------------------------------------------------------------------------------------------
if os.path.exists(self.filename_blank.format("LWrist")):
lw = GENEActiv(filepath=self.filename_blank.format("LWrist"),
load_raw=True,
start_offset=start_index,
end_offset=stop_index)
lw_time = lw.timestamps
lwx = lw.x
lwy = lw.y
lwz = lw.z
if not os.path.exists(self.filename_blank.format("LWrist")):
print("-LWrist file not found.")
lw_time = [
timestamp + timedelta(seconds=i / fs)
for i in range(stop_index)
]
lwx = [0 for i in range(stop_index)]
lwy = [0 for i in range(stop_index)]
lwz = [0 for i in range(stop_index)]
if os.path.exists(self.temp_filename_blank.format("LWrist")):
lw_temp = GENEActivTemperature(
filepath=self.temp_filename_blank.format("LWrist"),
start_offset=0,
end_offset=0)
df = pd.DataFrame(list(zip(lw_temp.timestamps,
lw_temp.temperature)),
columns=["Time", "Temp"])
df = df.loc[(df["Time"] >= timestamp) & (
df["Time"] <= timestamp + timedelta(minutes=duration))]
lwt = df["Temp"]
lw_temp_time = df["Time"]
if not os.path.exists(self.temp_filename_blank.format("LWrist")):
print("-LWrist temperature file not found.")
lwt = [0 for i in range(int(stop_index / 300))]
lw_temp_time = [None for i in range(int(stop_index / 300))]
# Right wrist -------------------------------------------------------------------------------------------------
if os.path.exists(self.filename_blank.format("RWrist")):
rw = GENEActiv(filepath=self.filename_blank.format("RWrist"),
load_raw=True,
start_offset=start_index,
end_offset=stop_index)
rw_time = rw.timestamps
rwx = rw.x
rwy = rw.y
rwz = rw.z
if not os.path.exists(self.filename_blank.format("RWrist")):
print("-RWrist file not found.")
rw_time = [
timestamp + timedelta(seconds=i / fs)
for i in range(stop_index)
]
rwx = [0 for i in range(stop_index)]
rwy = [0 for i in range(stop_index)]
rwz = [0 for i in range(stop_index)]
if os.path.exists(self.temp_filename_blank.format("RWrist")):
rw_temp = GENEActivTemperature(
filepath=self.temp_filename_blank.format("RWrist"),
start_offset=0,
end_offset=0)
df = pd.DataFrame(list(zip(rw_temp.timestamps,
rw_temp.temperature)),
columns=["Time", "Temp"])
df = df.loc[(df["Time"] >= timestamp) & (
df["Time"] <= timestamp + timedelta(minutes=duration))]
rwt = df["Temp"]
rw_temp_time = df["Time"]
if not os.path.exists(self.temp_filename_blank.format("RWrist")):
print("-RWrist temperature file not found.")
rwt = [0 for i in range(int(stop_index / 300))]
rw_temp_time = [None for i in range(int(stop_index / 300))]
# Combining data ----------------------------------------------------------------------------------------------
df = pd.DataFrame(list(
zip(la_time, lax, lay, laz, ra_time, rax, ray, raz, lw_time, lwx,
lwy, lwz, rw_time, rwx, rwy, rwz)),
columns=[
"LA_time", "LA_x", "LA_y", "LA_z", "RA_time",
"RA_x", "RA_y", "RA_z", "LW_time", "LW_x",
"LW_y", "LW_z", "RW_time", "RW_x", "RW_y", "RW_z"
])
df_temp = pd.DataFrame(list(
zip(la_temp_time, lat, ra_temp_time, rat, lw_temp_time, lwt,
rw_temp_time, rwt)),
columns=[
"LA_time", "LA_temp", "RA_time", "RA_temp",
"LW_time", "LW_temp", "RW_time", "RW_temp"
])
print("\nIMPORTED DATA FROM {} to {}.".format(
timestamp, timestamp + timedelta(minutes=duration)))
return df, df_temp
def run_all_subjs():
# df_sleep = pd.read_excel("/Users/kyleweber/Desktop/Test.xlsx")
df_sleep = pd.read_excel(
"/Users/kyleweber/Desktop/Data/OND07/Tabular Data/OND07_SleepLogs_Reformatted.xlsx"
)
epoch_len = 15
file_dir = "/Volumes/Kyle's External HD/OND07 Bittium/"
files = [i for i in os.listdir(file_dir) if "edf" in i or "EDF" in i]
subjs = df_sleep['ID'].unique()
for subj in subjs[-4:]:
if os.path.exists(file_dir + f"{subj}_01_BF.EDF"):
data = ImportEDF.Bittium(filepath=file_dir + f"{subj}_01_BF.EDF",
start_offset=0,
end_offset=0,
epoch_len=epoch_len,
load_accel=False,
low_f=1,
high_f=30,
f_type="bandpass")
orphanidou = []
for i in range(0, len(data.raw[::2]),
int(data.sample_rate / 2 * epoch_len)):
if i % 1000 == 0:
print(f"{round(100*i/(len(data.raw)/2), 1)}%")
d = CheckQuality(raw_data=data.raw[::2],
sample_rate=data.sample_rate / 2,
start_index=i,
template_data='raw',
voltage_thresh=250,
epoch_len=epoch_len)
orphanidou.append("Valid" if d.valid_period else "Invalid")
epoch_stamps = data.timestamps[::int(data.sample_rate * epoch_len)]
sleep_mask = np.zeros(len(epoch_stamps))
for row in df_sleep.loc[df_sleep["ID"] == subj].itertuples():
start_ind = int(
(row.sleep - epoch_stamps[0]).total_seconds() / epoch_len)
end_ind = int(
(row.wake - epoch_stamps[0]).total_seconds() / epoch_len)
sleep_mask[start_ind:end_ind] = 1
df_qc = pd.DataFrame(np.array(
[epoch_stamps, sleep_mask, orphanidou]).transpose(),
columns=["Timestamp", "SleepMask", "QC"])
df_qc["Timestamp"] = pd.to_datetime(df_qc["Timestamp"])
df_qc.to_excel(
f"/Users/kyleweber/Desktop/ECG Output/{subj}_ECG_Output.xlsx",
index=False)
plt.close("all")
fig, axes = plt.subplots(3, sharex='col', figsize=(10, 6))
axes[0].plot(data.timestamps[::5], data.filtered[::5])
axes[1].plot(df_qc["Timestamp"], df_qc["SleepMask"])
axes[2].plot(df_qc["Timestamp"], df_qc["QC"])
plt.savefig(
f"/Users/kyleweber/Desktop/ECG Output/Plots/{subj}.png")
del data, df_qc
def import_daily_data(self, epoch_len, avg_n_beats=10, day_num=None):
print("\nProcessing continuous ECG data...")
# Empty DF
df_all = pd.DataFrame(columns=["Timestamp", "HR", "AvgHR"])
if day_num is None:
tally = 0
for start_ind, stop_ind in zip(self.start_inds, self.stop_inds):
print("======================== Day {} ======================== ".format(tally + 1))
data = ImportEDF.Bittium(filepath=self.filename,
start_offset=start_ind, end_offset=stop_ind, epoch_len=epoch_len,
load_accel=False,
low_f=.67, high_f=30, f_type="bandpass")
all_peaks, swt, filt = find_all_peaks(raw_data=data.raw, fs=data.sample_rate,
use_epochs=False, plot_data=False, epoch_len=epoch_len)
if tally >= 0:
all_peaks2 = [i + start_ind for i in all_peaks]
if tally == 0:
all_peaks2 = all_peaks
inst_hr = []
for b1, b2 in zip(all_peaks[:], all_peaks[1:]):
inst_hr.append(60 / ((b2 - b1) / data.sample_rate))
keep_index = []
for row in range(0, len(inst_hr) - 1):
if abs(inst_hr[row + 1] - inst_hr[row]) < 5:
keep_index.append(row)
peak_ind = [all_peaks[i] for i in keep_index]
df = pd.DataFrame(list(zip([data.timestamps[i] for i in peak_ind],
[all_peaks2[i] for i in keep_index],
[inst_hr[i] for i in keep_index])),
columns=["Timestamp", "Peak", "HR"])
avg_hr = [sum(df["HR"].iloc[i:i + avg_n_beats]) / avg_n_beats for i in range(df.shape[0])]
for i in range(len(avg_hr) - df.shape[0]):
avg_hr.append(None)
df["AvgHR"] = avg_hr
df_all = df_all.append(df)
tally += 1
if type(day_num) is int:
print("======================== Day {} ======================== ".format(day_num))
data = ImportEDF.Bittium(filepath=self.filename,
start_offset=self.start_inds[day_num-1],
end_offset=self.stop_inds[day_num-1], epoch_len=epoch_len,
load_accel=False,
low_f=.67, high_f=30, f_type="bandpass")
all_peaks, swt, filt = find_all_peaks(raw_data=data.raw, fs=data.sample_rate,
use_epochs=False, plot_data=False, epoch_len=epoch_len)
if day_num >= 0:
all_peaks2 = [i + self.start_inds[day_num - 1] for i in all_peaks]
if day_num == 0:
all_peaks2 = all_peaks
inst_hr = []
for b1, b2 in zip(all_peaks[:], all_peaks[1:]):
inst_hr.append(60 / ((b2 - b1) / data.sample_rate))
keep_index = []
for row in range(0, len(inst_hr) - 1):
if abs(inst_hr[row + 1] - inst_hr[row]) < 5:
keep_index.append(row)
peak_ind = [all_peaks[i] for i in keep_index]
df = pd.DataFrame(list(zip([data.timestamps[i] for i in peak_ind],
[all_peaks2[i] for i in keep_index],
[inst_hr[i] for i in keep_index])),
columns=["Timestamp", "Peak", "HR"])
avg_hr = [sum(df["HR"].iloc[i:i + avg_n_beats]) / avg_n_beats for i in range(df.shape[0])]
for i in range(len(avg_hr) - df.shape[0]):
avg_hr.append(None)
df["AvgHR"] = avg_hr
df_all = df_all.append(df)
return df_all
def replace_unusable_data(ecg_signal, annotations_df, value=None):
df_bad = annotations_df.loc[annotations_df["quality"] == "Q3"]
ecg = np.array(ecg_signal)
for row in df_bad.itertuples():
ecg[int(row.start_idx):int(row.end_idx)] = value
return ecg
data = ImportEDF.Bittium(filepath="/Users/kyleweber/Desktop/007_OmegaSnap.EDF",
start_offset=0,
end_offset=0,
epoch_len=15,
load_accel=False,
low_f=1,
high_f=25,
f_type="bandpass")
f = Filtering.filter_signal(data=data.raw,
filter_type='bandpass',
low_f=.5,
high_f=30,
filter_order=3,
sample_f=data.sample_rate)
# No bandpass/notch filtering
w = wiener_filter.awwf(f, data.sample_rate)
filt, wiener_filt, snr, annots, thresh = qc.annotate_ecg_quality(
sample_rate=data.sample_rate, signal=data.raw)
