def test_accelerometer_features(self): ds = autosense_sequence_align([self.accelx_ds, self.accely_ds, self.accelz_ds], self.sampling_frequency) accelerometer_magnitude, accelerometer_win_mag_deviations, accel_activity = accelerometer_features(ds) self.assertEqual(len(accelerometer_magnitude.data), 62870) self.assertEqual(len(accelerometer_win_mag_deviations.data), 687) self.assertEqual(len(accel_activity.data), 687) self.assertEqual(len([dp for dp in accel_activity.data if dp.sample]), 0) # TODO: Is this correct
def cStress(rdd: RDD) -> RDD: # TODO: TWH Temporary ecg_sampling_frequency = 64.0 rip_sampling_frequency = 64.0 accel_sampling_frequency = 64.0 / 6.0 # Timestamp correct datastreams ecg_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['ecg'], sampling_frequency=ecg_sampling_frequency))) rip_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['rip'], sampling_frequency=rip_sampling_frequency))) accelx_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['accelx'], sampling_frequency=accel_sampling_frequency))) accely_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['accely'], sampling_frequency=accel_sampling_frequency))) accelz_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['accelz'], sampling_frequency=accel_sampling_frequency))) accel_group = accelx_corrected.join(accely_corrected).join( accelz_corrected).map(fix_two_joins) accel = accel_group.map(lambda ds: ( ds[0], autosense_sequence_align(datastreams=[ds[1][0], ds[1][1], ds[1][2]], sampling_frequency=accel_sampling_frequency))) # Accelerometer Feature Computation accel_features = accel.map( lambda ds: (ds[0], accelerometer_features(ds[1], window_length=10.0))) # rip features peak_valley = rip_corrected.map( lambda ds: (ds[0], rip.compute_peak_valley(rip=ds[1]))) rip_features = peak_valley.map( lambda ds: (ds[0], rip_feature_computation(ds[1][0], ds[1][1]))) # r-peak datastream computation ecg_rr_rdd = ecg_corrected.map(lambda ds: (ds[ 0], compute_rr_intervals(ds[1], ecg_sampling_frequency))) ecg_features = ecg_rr_rdd.map(lambda ds: (ds[ 0], ecg_feature_computation(ds[1], window_size=60, window_offset=60))) # return rip_features.join(ecg_features).join(accel_features).map(fix_two_joins) return ecg_features
def setUpClass(cls): super(TestVector, cls).setUpClass() tz = pytz.timezone('US/Eastern') cls.ecg = [] cls.sample_rate = 64.0 with gzip.open(os.path.join(os.path.dirname(__file__), 'res/ecg.csv.gz'), 'rt') as f: for l in f: values = list(map(int, l.split(','))) cls.ecg.append( DataPoint.from_tuple(datetime.datetime.fromtimestamp(values[0] / 1000000.0, tz=tz), values[1])) cls.ds = DataStream(None, None, data=cls.ecg) accelx = [] accel_sample_rate = 64.0 / 6 with gzip.open(os.path.join(os.path.dirname(__file__), 'res/accelx.csv.gz'), 'rt') as f: for l in f: values = list(map(int, l.split(','))) accelx.append( DataPoint.from_tuple(datetime.datetime.fromtimestamp(values[0] / 1000000.0, tz=tz), values[1])) accelx = DataStream(None, None, data=accelx) accely = [] with gzip.open(os.path.join(os.path.dirname(__file__), 'res/accely.csv.gz'), 'rt') as f: for l in f: values = list(map(int, l.split(','))) accely.append( DataPoint.from_tuple(datetime.datetime.fromtimestamp(values[0] / 1000000.0, tz=tz), values[1])) accely = DataStream(None, None, data=accely) accelz = [] with gzip.open(os.path.join(os.path.dirname(__file__), 'res/accelz.csv.gz'), 'rt') as f: for l in f: values = list(map(int, l.split(','))) accelz.append( DataPoint.from_tuple(datetime.datetime.fromtimestamp(values[0] / 1000000.0, tz=tz), values[1])) accelz = DataStream(None, None, data=accelz) cls.accel = autosense_sequence_align([accelx, accely, accelz], accel_sample_rate)
def cStress(rdd: RDD) -> RDD: # TODO: TWH Temporary ecg_sampling_frequency = 64.0 rip_sampling_frequency = 64.0 accel_sampling_frequency = 64.0 / 6.0 # Timestamp correct datastreams ecg_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['ecg'], sampling_frequency=ecg_sampling_frequency))) rip_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['rip'], sampling_frequency=rip_sampling_frequency))) accelx_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['accelx'], sampling_frequency=accel_sampling_frequency))) accely_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['accely'], sampling_frequency=accel_sampling_frequency))) accelz_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['accelz'], sampling_frequency=accel_sampling_frequency))) ecg_quality = ecg_corrected.map(lambda ds: (ds[0], ecg_data_quality(ds[1]))) rip_quality = rip_corrected.map(lambda ds: (ds[0], rip_data_quality(ds[1]))) accel_group = accelx_corrected.join(accely_corrected).join( accelz_corrected).map(fix_two_joins) accel = accel_group.map(lambda ds: ( ds[0], autosense_sequence_align(datastreams=[ds[1][0], ds[1][1], ds[1][2]], sampling_frequency=accel_sampling_frequency))) # Accelerometer Feature Computation accel_features = accel.map( lambda ds: (ds[0], accelerometer_features(ds[1], window_length=10.0))) windowed_accel_features = accel_features.map( lambda ds: (ds[0], window_accel(ds[1], window_size=60))) rip_corrected_and_quality = rip_corrected.join(rip_quality) # rip features peak_valley = rip_corrected_and_quality.map(lambda ds: (ds[ 0], rip.compute_peak_valley(rip=ds[1][0], rip_quality=ds[1][1]))) rip_cycle_features = peak_valley.map( lambda ds: (ds[0], rip_cycle_feature_computation(ds[1][0]))) windowed_rip_features = rip_cycle_features.map( lambda ds: (ds[0], window_rip(peak_datastream=ds[1][0], valley_datastream=ds[1][1], inspiration_duration=ds[1][2], expiration_duration=ds[1][3], respiration_duration=ds[1][4], inspiration_expiration_ratio=ds[1][5], stretch=ds[1][6], window_size=60, window_offset=60))) ecg_corrected_and_quality = ecg_corrected.join(ecg_quality) # r-peak datastream computation ecg_rr_rdd = ecg_corrected_and_quality.map(lambda ds: ( ds[0], compute_rr_intervals( ecg=ds[1][0], ecg_quality=ds[1][1], fs=ecg_sampling_frequency))) ecg_rr_quality = ecg_rr_rdd.map(lambda ds: (ds[0], compute_outlier_ecg(ds[1]))) ecg_rr_and_quality = ecg_rr_rdd.join(ecg_rr_quality) windowed_ecg_features = ecg_rr_and_quality.map( lambda ds: (ds[0], ecg_feature_computation(datastream=ds[1][0], quality_datastream=ds[1][1], window_size=60, window_offset=60))) peak_valley_rr_int = peak_valley.join( ecg_rr_rdd) # TODO: Add RR_Quality here? # rsa_cycle_features = peak_valley_rr_int.map( # lambda ds: (ds[0], compute_rsa_cycle_feature(valleys=ds[1][1], rr_int=ds[1][2]))) # windowed_rsa_features = rsa_cycle_features.map(lambda ds: (ds[0], window_rsa(ds[0][0], window_size=60))) # # combined_features = windowed_accel_features.join(windowed_ecg_features).join(windowed_rip_features).join( # windowed_rsa_features) # Fix joins here feature_vector_ecg_rip = windowed_ecg_features.map( lambda ds: (ds[0], generate_cStress_feature_vector(ds[1]))) stress_ground_truth = rdd.map(lambda ds: (ds['participant'], ds['stress_marks'])) feature_vector_with_ground_truth = feature_vector_ecg_rip.join( stress_ground_truth) train_data_with_ground_truth_and_subjects = feature_vector_with_ground_truth.map( lambda ds: analyze_events_with_features(participant=ds[0], stress_mark_stream=ds[1][1], feature_stream=ds[1][0])) return train_data_with_ground_truth_and_subjects # Data stream with data points (ST, ET, [...37 values...])
def cStress(rdd: RDD) -> RDD: # TODO: TWH Temporary ecg_sampling_frequency = 64.0 rip_sampling_frequency = 64.0 accel_sampling_frequency = 64.0 / 6.0 # Timestamp correct datastreams ecg_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['ecg'], sampling_frequency=ecg_sampling_frequency))) rip_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['rip'], sampling_frequency=rip_sampling_frequency))) accelx_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['accelx'], sampling_frequency=accel_sampling_frequency))) accely_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['accely'], sampling_frequency=accel_sampling_frequency))) accelz_corrected = rdd.map(lambda ds: ( ds['participant'], timestamp_correct(datastream=ds['accelz'], sampling_frequency=accel_sampling_frequency))) ecg_quality = ecg_corrected.map(lambda ds: (ds[0], ecg_data_quality(ds[1]))) rip_quality = rip_corrected.map(lambda ds: (ds[0], rip_data_quality(ds[1]))) accel_group = accelx_corrected.join(accely_corrected).join( accelz_corrected).map(fix_two_joins) accel = accel_group.map(lambda ds: ( ds[0], autosense_sequence_align(datastreams=[ds[1][0], ds[1][1], ds[1][2]], sampling_frequency=accel_sampling_frequency))) # Accelerometer Feature Computation accel_features = accel.map( lambda ds: (ds[0], accelerometer_features(ds[1], window_length=10.0))) windowed_accel_features = accel_features.map( lambda ds: (ds[0], window_accel(ds[1], window_size=60))) rip_corrected_and_quality = rip_corrected.join(rip_quality) # rip features peak_valley = rip_corrected_and_quality.map(lambda ds: (ds[ 0], rip.compute_peak_valley(rip=ds[1][0], rip_quality=ds[1][1]))) rip_cycle_features = peak_valley.map( lambda ds: (ds[0], rip_feature_computation(ds[1][0]))) windowed_rip_features = rip_cycle_features.map( lambda ds: (ds[0], window_rip(inspiration_duration=ds[1][0], expiration_duration=ds[1][1], ..., window_size=60))) ecg_corrected_and_quality = ecg_corrected.join(ecg_quality) # r-peak datastream computation ecg_rr_rdd = ecg_corrected_and_quality.map(lambda ds: ( ds[0], compute_rr_intervals( ecg=ds[1][0], ecg_quality=ds[1][1], fs=ecg_sampling_frequency))) ecg_rr_quality = ecg_rr_rdd.map(lambda ds: (ds[0], compute_outlier_ecg(ds[1]))) ecg_rr_and_quality = ecg_rr_rdd.join(ecg_rr_quality) windowed_ecg_features = ecg_rr_and_quality.map(lambda ds: ( ds[0], ecg_feature_computation( rr_intervals=ds[1][0], data_quality=ds[1][1], window_size=60))) peak_valley_rr_int = peak_valley.join( ecg_rr_rdd) # TODO: Add RR_Quality here? rsa_cycle_features = peak_valley_rr_int.map(lambda ds: (ds[ 0], compute_rsa_cycle_feature(valleys=ds[1][1], rr_int=ds[1][2]))) windowed_rsa_features = rsa_cycle_features.map( lambda ds: (ds[0], window_rsa(ds[0][0], window_size=60))) combined_features = windowed_accel_features.join( windowed_ecg_features).join(windowed_rip_features).join( windowed_rsa_features) # Fix joins here feature_vector_ecg_rip = combined_features.map(lambda ds: ( ds[0], generate_cStress_feature_vector( accel=ds[1][0], ecg=ds[1][1], rip=ds[1][2], rsa=ds[1][3]))) return feature_vector_ecg_rip # Data stream with data points (ST, ET, [...37 values...])