def test_ecgprocessing_timestamp_correction(self):
ecg_corrected = timestamp_correct(self.ecg_datastream, self._fs)
rr_datastream_from_raw = compute_rr_intervals(self.ecg_datastream,
self._fs)
rr_datastream_from_corrected = compute_rr_intervals(
ecg_corrected, self._fs)
test_result = (len(rr_datastream_from_corrected.data) * 100) / len(
rr_datastream_from_raw.data)
self.assertGreaterEqual(test_result, 99)
def cStress(rdd: RDD) -> RDD:
# TODO: TWH Temporary
ecg_sampling_frequency = 64.0
rip_sampling_frequency = 64.0
accel_sampling_frequency = 64.0 / 6.0
# Timestamp correct datastreams
ecg_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['ecg'],
sampling_frequency=ecg_sampling_frequency)))
rip_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['rip'],
sampling_frequency=rip_sampling_frequency)))
accelx_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['accelx'],
sampling_frequency=accel_sampling_frequency)))
accely_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['accely'],
sampling_frequency=accel_sampling_frequency)))
accelz_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['accelz'],
sampling_frequency=accel_sampling_frequency)))
accel_group = accelx_corrected.join(accely_corrected).join(
accelz_corrected).map(fix_two_joins)
accel = accel_group.map(lambda ds: (
ds[0],
autosense_sequence_align(datastreams=[ds[1][0], ds[1][1], ds[1][2]],
sampling_frequency=accel_sampling_frequency)))
# Accelerometer Feature Computation
accel_features = accel.map(
lambda ds: (ds[0], accelerometer_features(ds[1], window_length=10.0)))
# rip features
peak_valley = rip_corrected.map(
lambda ds: (ds[0], rip.compute_peak_valley(rip=ds[1])))
rip_features = peak_valley.map(
lambda ds: (ds[0], rip_feature_computation(ds[1][0], ds[1][1])))
# r-peak datastream computation
ecg_rr_rdd = ecg_corrected.map(lambda ds: (ds[
0], compute_rr_intervals(ds[1], ecg_sampling_frequency)))
ecg_features = ecg_rr_rdd.map(lambda ds: (ds[
0], ecg_feature_computation(ds[1], window_size=60, window_offset=60)))
# return rip_features.join(ecg_features).join(accel_features).map(fix_two_joins)
return ecg_features
def test_compare_running_time(self):
start_time_whole = time.time()
rr_intervals = compute_rr_intervals(self.ecg_ds,
self.ecg_sampling_frequency)
elapse_time_whole = time.time() - start_time_whole
print('elapse_time_whole =', elapse_time_whole)
window_size = 60
window_offset = 1
start_time_win = time.time()
for key, data in window_iter(self.ecg_ds.data, window_size,
window_offset):
elapse_time_win = time.time() - start_time_win
print(len(data),
'time =',
'{:.2f}'.format(elapse_time_win),
end='\r',
flush=True)
rr_intervals = compute_rr_intervals(
DataStream(None, None, data=data), self.ecg_sampling_frequency)
def test_compute_rr_intervals(self): ecg_rrInterval, ecg_rpeak = compute_rr_intervals(self.ecg_datastream,fs=self._fs) #plot fig, ax = plt.subplots(figsize=(400, 10)) ecg_time = [i.start_time for i in self.ecg_datastream.data] ecg_sample = [i.sample for i in self.ecg_datastream.data] rpeak_time = [i.start_time for i in ecg_rpeak.data] rpeak_sample = [i.sample for i in ecg_rpeak.data] ax.plot(ecg_time, ecg_sample, 'ro--', markersize=3, linewidth=1) ax.plot(rpeak_time, rpeak_sample, 'bo', markersize=5) path_save = '/home/hyeok/research/md2k/data/minnesota-analysis/test/rpeaks.png' fig.savefig(path_save)
def setUpClass(cls):
super(TestECGFeatures, cls).setUpClass()
tz = pytz.timezone('US/Eastern')
ecg = []
ecg_sampling_frequency = 64.0
with gzip.open(
os.path.join(os.path.dirname(__file__), 'res/ecg.csv.gz'),
'rt') as f:
for l in f:
values = list(map(int, l.split(',')))
ecg.append(
DataPoint.from_tuple(
datetime.datetime.fromtimestamp(values[0] / 1000000.0,
tz=tz), values[1]))
ecg_ds = DataStream(None, None)
ecg_ds.datapoints = ecg
cls.rr_intervals = compute_rr_intervals(ecg_ds, ecg_sampling_frequency)
def cStress(rdd: RDD) -> RDD:
# TODO: TWH Temporary
ecg_sampling_frequency = 64.0
rip_sampling_frequency = 64.0
accel_sampling_frequency = 64.0 / 6.0
# Timestamp correct datastreams
ecg_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['ecg'],
sampling_frequency=ecg_sampling_frequency)))
rip_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['rip'],
sampling_frequency=rip_sampling_frequency)))
accelx_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['accelx'],
sampling_frequency=accel_sampling_frequency)))
accely_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['accely'],
sampling_frequency=accel_sampling_frequency)))
accelz_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['accelz'],
sampling_frequency=accel_sampling_frequency)))
ecg_quality = ecg_corrected.map(lambda ds:
(ds[0], ecg_data_quality(ds[1])))
rip_quality = rip_corrected.map(lambda ds:
(ds[0], rip_data_quality(ds[1])))
accel_group = accelx_corrected.join(accely_corrected).join(
accelz_corrected).map(fix_two_joins)
accel = accel_group.map(lambda ds: (
ds[0],
autosense_sequence_align(datastreams=[ds[1][0], ds[1][1], ds[1][2]],
sampling_frequency=accel_sampling_frequency)))
# Accelerometer Feature Computation
accel_features = accel.map(
lambda ds: (ds[0], accelerometer_features(ds[1], window_length=10.0)))
windowed_accel_features = accel_features.map(
lambda ds: (ds[0], window_accel(ds[1], window_size=60)))
rip_corrected_and_quality = rip_corrected.join(rip_quality)
# rip features
peak_valley = rip_corrected_and_quality.map(lambda ds: (ds[
0], rip.compute_peak_valley(rip=ds[1][0], rip_quality=ds[1][1])))
rip_cycle_features = peak_valley.map(
lambda ds: (ds[0], rip_cycle_feature_computation(ds[1][0])))
windowed_rip_features = rip_cycle_features.map(
lambda ds: (ds[0],
window_rip(peak_datastream=ds[1][0],
valley_datastream=ds[1][1],
inspiration_duration=ds[1][2],
expiration_duration=ds[1][3],
respiration_duration=ds[1][4],
inspiration_expiration_ratio=ds[1][5],
stretch=ds[1][6],
window_size=60,
window_offset=60)))
ecg_corrected_and_quality = ecg_corrected.join(ecg_quality)
# r-peak datastream computation
ecg_rr_rdd = ecg_corrected_and_quality.map(lambda ds: (
ds[0],
compute_rr_intervals(
ecg=ds[1][0], ecg_quality=ds[1][1], fs=ecg_sampling_frequency)))
ecg_rr_quality = ecg_rr_rdd.map(lambda ds:
(ds[0], compute_outlier_ecg(ds[1])))
ecg_rr_and_quality = ecg_rr_rdd.join(ecg_rr_quality)
windowed_ecg_features = ecg_rr_and_quality.map(
lambda ds: (ds[0],
ecg_feature_computation(datastream=ds[1][0],
quality_datastream=ds[1][1],
window_size=60,
window_offset=60)))
peak_valley_rr_int = peak_valley.join(
ecg_rr_rdd) # TODO: Add RR_Quality here?
# rsa_cycle_features = peak_valley_rr_int.map(
# lambda ds: (ds[0], compute_rsa_cycle_feature(valleys=ds[1][1], rr_int=ds[1][2])))
# windowed_rsa_features = rsa_cycle_features.map(lambda ds: (ds[0], window_rsa(ds[0][0], window_size=60)))
#
# combined_features = windowed_accel_features.join(windowed_ecg_features).join(windowed_rip_features).join(
# windowed_rsa_features)
# Fix joins here
feature_vector_ecg_rip = windowed_ecg_features.map(
lambda ds: (ds[0], generate_cStress_feature_vector(ds[1])))
stress_ground_truth = rdd.map(lambda ds:
(ds['participant'], ds['stress_marks']))
feature_vector_with_ground_truth = feature_vector_ecg_rip.join(
stress_ground_truth)
train_data_with_ground_truth_and_subjects = feature_vector_with_ground_truth.map(
lambda ds: analyze_events_with_features(participant=ds[0],
stress_mark_stream=ds[1][1],
feature_stream=ds[1][0]))
return train_data_with_ground_truth_and_subjects # Data stream with data points (ST, ET, [...37 values...])
def cStress(raw_ecg: DataStream, raw_rip: DataStream, raw_accel_x: DataStream,
raw_accel_y: DataStream, raw_accel_z: DataStream) -> DataStream:
"""
:return:
:param raw_ecg:
:param raw_rip:
:param raw_accel_x:
:param raw_accel_y:
:param raw_accel_z:
"""
# Algorithm Constants
# TODO: Once metadata is implemented
# ecg_sampling_frequency = rawecg.get_metadata('samplingFrequency') # 64.0
# rip_sampling_frequency = rawrip.get_metadata('samplingFrequency') # 64.0 / 3.0
# accel_sampling_frequency = rawaccelx.get_metadata('samplingFrequency') # 64.0 / 6.0
# TODO: TWH Temporary
ecg_sampling_frequency = 64.0
rip_sampling_frequency = 64.0 / 3.0
accel_sampling_frequency = 64.0 / 6.0
# Timestamp correct datastreams
ecg_corrected = timestamp_correct(
datastream=raw_ecg, sampling_frequency=ecg_sampling_frequency)
rip_corrected = timestamp_correct(
datastream=raw_rip, sampling_frequency=rip_sampling_frequency)
accel = timestamp_correct_and_sequence_align(
datastream_array=[raw_accel_x, raw_accel_y, raw_accel_z],
sampling_frequency=accel_sampling_frequency)
# ECG and RIP signal morphology data quality
ecg_data_quality = ECGDataQuality(
ecg_corrected,
windowsize=5.0, # What does windowsize mean here?
bufferLength=3,
acceptableOutlierPercent=50,
outlierThresholdHigh=4500,
outlierThresholdLow=20,
badSegmentThreshod=2,
ecgBandLooseThreshold=47)
# ecg_corrected.add_span_stream(ecg_data_quality)
rip_data_quality = RIPDataQuality(
rip_corrected,
windowsize=5.0, # What does windowsize mean here?
bufferLength=5,
acceptableOutlierPercent=50,
outlierThresholdHigh=4500,
outlierThresholdLow=20,
badSegmentThreshod=2,
ripBandOffThreshold=20,
ripBandLooseThreshold=150)
# rip_corrected.add_span_stream(rip_data_quality)
# Accelerometer Feature Computation
accelerometer_magnitude, accelerometer_win_mag_deviations, accel_activity = accelerometer_features(
accel)
print('mag-length:', len(accelerometer_magnitude.datapoints))
print('mag-deviations-length:',
len(accelerometer_win_mag_deviations.datapoints))
print('accel_activity-length:', len(accel_activity.datapoints))
# r-peak datastream computation
ecg_rr_datastream = compute_rr_intervals(raw_ecg, ecg_sampling_frequency)
ecg_features = ecg_feature_computation(ecg_rr_datastream,
window_size=60,
window_offset=60)
print(len(ecg_features))
# TODO: TWH Fix when feature vector result is available
return raw_ecg
def cStress(rdd: RDD) -> RDD:
# TODO: TWH Temporary
ecg_sampling_frequency = 64.0
rip_sampling_frequency = 64.0
accel_sampling_frequency = 64.0 / 6.0
# Timestamp correct datastreams
ecg_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['ecg'],
sampling_frequency=ecg_sampling_frequency)))
rip_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['rip'],
sampling_frequency=rip_sampling_frequency)))
accelx_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['accelx'],
sampling_frequency=accel_sampling_frequency)))
accely_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['accely'],
sampling_frequency=accel_sampling_frequency)))
accelz_corrected = rdd.map(lambda ds: (
ds['participant'],
timestamp_correct(datastream=ds['accelz'],
sampling_frequency=accel_sampling_frequency)))
ecg_quality = ecg_corrected.map(lambda ds:
(ds[0], ecg_data_quality(ds[1])))
rip_quality = rip_corrected.map(lambda ds:
(ds[0], rip_data_quality(ds[1])))
accel_group = accelx_corrected.join(accely_corrected).join(
accelz_corrected).map(fix_two_joins)
accel = accel_group.map(lambda ds: (
ds[0],
autosense_sequence_align(datastreams=[ds[1][0], ds[1][1], ds[1][2]],
sampling_frequency=accel_sampling_frequency)))
# Accelerometer Feature Computation
accel_features = accel.map(
lambda ds: (ds[0], accelerometer_features(ds[1], window_length=10.0)))
windowed_accel_features = accel_features.map(
lambda ds: (ds[0], window_accel(ds[1], window_size=60)))
rip_corrected_and_quality = rip_corrected.join(rip_quality)
# rip features
peak_valley = rip_corrected_and_quality.map(lambda ds: (ds[
0], rip.compute_peak_valley(rip=ds[1][0], rip_quality=ds[1][1])))
rip_cycle_features = peak_valley.map(
lambda ds: (ds[0], rip_feature_computation(ds[1][0])))
windowed_rip_features = rip_cycle_features.map(
lambda ds: (ds[0],
window_rip(inspiration_duration=ds[1][0],
expiration_duration=ds[1][1], ...,
window_size=60)))
ecg_corrected_and_quality = ecg_corrected.join(ecg_quality)
# r-peak datastream computation
ecg_rr_rdd = ecg_corrected_and_quality.map(lambda ds: (
ds[0],
compute_rr_intervals(
ecg=ds[1][0], ecg_quality=ds[1][1], fs=ecg_sampling_frequency)))
ecg_rr_quality = ecg_rr_rdd.map(lambda ds:
(ds[0], compute_outlier_ecg(ds[1])))
ecg_rr_and_quality = ecg_rr_rdd.join(ecg_rr_quality)
windowed_ecg_features = ecg_rr_and_quality.map(lambda ds: (
ds[0],
ecg_feature_computation(
rr_intervals=ds[1][0], data_quality=ds[1][1], window_size=60)))
peak_valley_rr_int = peak_valley.join(
ecg_rr_rdd) # TODO: Add RR_Quality here?
rsa_cycle_features = peak_valley_rr_int.map(lambda ds: (ds[
0], compute_rsa_cycle_feature(valleys=ds[1][1], rr_int=ds[1][2])))
windowed_rsa_features = rsa_cycle_features.map(
lambda ds: (ds[0], window_rsa(ds[0][0], window_size=60)))
combined_features = windowed_accel_features.join(
windowed_ecg_features).join(windowed_rip_features).join(
windowed_rsa_features)
# Fix joins here
feature_vector_ecg_rip = combined_features.map(lambda ds: (
ds[0],
generate_cStress_feature_vector(
accel=ds[1][0], ecg=ds[1][1], rip=ds[1][2], rsa=ds[1][3])))
return feature_vector_ecg_rip # Data stream with data points (ST, ET, [...37 values...])
