def persist_data(execution_context, data_descriptor, annotations, stream_id, name, owner_id, data, CC_obj): """ Parse old schema and map it into new CerebralCortex data provenance sechema :param filename: :param data: :param CC_obj: """ stream_type = "datastream" ds = DataStream(identifier=stream_id, owner=owner_id, name=name, data_descriptor=data_descriptor, execution_context=execution_context, annotations=annotations, stream_type=stream_type, data=data) CC_obj.save_datastream(ds)
def test_compare_running_time(self): start_time_whole = time.time() rr_intervals = compute_rr_intervals(self.ecg_ds, self.ecg_sampling_frequency) elapse_time_whole = time.time() - start_time_whole print('elapse_time_whole =', elapse_time_whole) window_size = 60 window_offset = 1 start_time_win = time.time() for key, data in window_iter(self.ecg_ds.data, window_size, window_offset): elapse_time_win = time.time() - start_time_win print(len(data), 'time =', '{:.2f}'.format(elapse_time_win), end='\r', flush=True) rr_intervals = compute_rr_intervals( DataStream(None, None, data=data), self.ecg_sampling_frequency)
def test_06_store_stream(self): identifier = "6db98dfb-d6e8-4b27-8d55-95b20fa0f754" owner = "06634264-56bc-4c92-abd7-377dbbad79dd" name = "data-store-test" data_descriptor = {} execution_context = json.loads( '{"execution_context": {"algorithm": {"method": "cerebralcortex.data_processor.data_diagnostic.BatteryDataMarker"}}}' ) annotations = {} datapoints = [] stream_type = "datastream" start_time = datetime.datetime(2017, 4, 24, 0, 0, 1) end_time = datetime.datetime(2017, 4, 24, 0, 0, 2) localtz = timezone('US/Central') start_time = localtz.localize(start_time) end_time = localtz.localize(end_time) sample = {'Foo3': 123} dp1 = DataPoint(start_time=start_time, end_time=end_time, sample=sample) datapoints.append(dp1) ds = DataStream(identifier, owner, name, data_descriptor, execution_context, annotations, stream_type, start_time, end_time, datapoints) self.CC.save_datastream(ds) stream = self.CC.get_datastream(identifier, data_type=DataSet.COMPLETE) self.assertEqual(stream._identifier, identifier) self.assertEqual(stream._owner, owner) self.assertEqual(stream._name, name) self.assertEqual(stream._data_descriptor, data_descriptor) self.assertEqual(stream._execution_context, execution_context) self.assertEqual(stream._annotations, annotations) self.assertEqual(stream._datastream_type, stream_type) self.assertEqual(stream.data[0].start_time, start_time) self.assertEqual(stream.data[0].end_time, end_time) self.assertEqual(stream.data[0].sample, sample)
def normalize(datastream: DataStream) -> DataStream: """ :param datastream: :return: """ input_data = np.array([i.sample for i in datastream.datapoints]) data = preprocessing.normalize(input_data, axis=0) result_data = [ DataPoint.from_tuple(start_time=i.start_time, sample=None) for i in datastream.datapoints ] for i, dp in enumerate(result_data): dp.sample = data[i] result = DataStream.from_datastream(input_streams=[datastream]) result.datapoints = result_data return result
def magnitude(datastream: DataStream) -> DataStream: """ :param datastream: :return: """ result = DataStream.from_datastream(input_streams=[datastream]) if datastream.data is None or len(datastream.data) == 0: result.data = [] return result input_data = np.array([i.sample for i in datastream.data]) data = norm(input_data, axis=1).tolist() result.data = [ DataPoint.from_tuple(start_time=v.start_time, sample=data[i]) for i, v in enumerate(datastream.data) ] return result
def test_compare_running_time(self): start_time_whole = time.time() result = timestamp_correct(self.accelx, sampling_frequency=self.sample_rate) elapse_time_whole = time.time() - start_time_whole print('elapse_time_whole =', elapse_time_whole) window_size = 60 window_offset = 1 start_time_win = time.time() for key, data in window_iter(self.accelx.data, window_size, window_offset): elapse_time_win = time.time() - start_time_win print(len(data), data[-1].start_time, 'time =', '{:.2f}'.format(elapse_time_win), end='\r', flush=True) result = timestamp_correct(DataStream(None, None, data=data), sampling_frequency=self.sample_rate)
def RIPDataQuality(datastream: DataStream, windowsize: float = 5.0, bufferLength=5, acceptableOutlierPercent=50, outlierThresholdHigh=4500, outlierThresholdLow=20, badSegmentThreshod=2, ripBandOffThreshold=20, ripBandLooseThreshold=150) -> SpanStream: """ :param datastream: :param windowsize: :param bufferLength: :param acceptableOutlierPercent: :param outlierThresholdHigh: :param outlierThresholdLow: :param badSegmentThreshod: :param ripBandOffThreshold: :param ripBandLooseThreshold: :return: """ # windows = window(datastream.datapoints, window_size=windowsize) # TODO: Do something with windows here result = DataStream.from_datastream(input_streams=[datastream]) # # Do something here for data quality # ripQuality = [] # for i in range(1, 10): # ripQuality.append(Span(result.getID(), # starttime=datetime.now(), # endtime=datetime.now(), # label=DataQuality.GOOD)) # # result.set_spans(ripQuality) return result
def autosense_sequence_align(datastreams: List[DataStream], sampling_frequency: float) -> DataStream: result = DataStream.from_datastream(input_streams=datastreams) result.data = [] if len(datastreams) == 0: return result start_time = None #print (len(datastreams)) for ds in datastreams: ts = ds.data[0].start_time #print (ts) if not start_time: start_time = ts elif start_time < ts: start_time = ts print(start_time) start_time -= datetime.timedelta(seconds=1.0 / sampling_frequency) print(start_time) data_block = [] max_index = np.Inf for ds in datastreams: d = [i for i in ds.data if i.start_time > start_time] if len(d) < max_index: max_index = len(d) data_block.append(d) print(max_index) data_array = np.array(data_block) dimensions = data_array.shape[0] for i in range(0, max_index): sample = [data_array[d][i].sample for d in range(0, dimensions)] result.data.append( DataPoint.from_tuple(data_array[0][i].start_time, sample)) return result
def generate_cStress_feature_vector(ecg_features: List[DataStream]): """ :param ecg_features: DataStream :return feature vector DataStream """ final_feature_vector = [] for i in range(len(ecg_features[0].data)): feature_vector = [] for ef in ecg_features: if ef.data[i].sample is None: continue feature_vector.append(ef.data[i].sample) final_feature_vector.append( DataPoint.from_tuple(start_time=ef.data[i].start_time, end_time=ef.data[i].end_time, sample=feature_vector)) feature_vector_ds = DataStream.from_datastream(ecg_features) feature_vector_ds.data = final_feature_vector return feature_vector_ds
def magnitude(datastream: DataStream) -> DataStream: """ :param datastream: :return: """ input_data = np.array([i.sample for i in datastream.datapoints]) data = norm(input_data, axis=1).tolist( ) # TODO: Fix function to not compute normalized magnitudes result_data = [ DataPoint.from_tuple(start_time=i.start_time, sample=None) for i in datastream.datapoints ] for i, dp in enumerate(result_data): dp.sample = data[i] result = DataStream.from_datastream(input_streams=[datastream]) result.datapoints = result_data return result
def map_datapoint_and_metadata_to_datastream(self, stream_id: int, data: list) -> DataStream: """ This method will map the datapoint and metadata to datastream object :param stream_id: :param data: list :return: datastream object """ # query datastream(mysql) for metadata datastream_info = Metadata(self.CC_obj).get_stream_info(stream_id) ownerID = datastream_info[0]["owner"] name = datastream_info[0]["name"] data_descriptor = json.loads(datastream_info[0]["data_descriptor"]) execution_context = json.loads(datastream_info[0]["execution_context"]) annotations = json.loads(datastream_info[0]["annotations"]) stream_type = datastream_info[0]["type"] start_time = datastream_info[0]["start_time"] end_time = datastream_info[0]["end_time"] return DataStream(stream_id, ownerID, name, data_descriptor, execution_context, annotations, stream_type, start_time, end_time, data)
def getBreathRate(datastream: DataStream, window_size: float, window_offset: float): """ Computes breath rate :param datastream: DataStream :return: breathing rate per minute datastream """ window_data = window_sliding(datastream.data, window_size, window_offset) breath_rates = [] for key, value in window_data.items(): starttime, endtime = key totalCycle = len(value) breath_rates.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=totalCycle)) datastream_breath_rates = DataStream.from_datastream([datastream]) datastream_breath_rates.data = breath_rates return datastream_breath_rates
def compute_datastream_rsa(valleys_datastream: DataStream, rr_datastream: DataStream): """ :param valleys_datastream: :param rr_datastream: :return: rsa_datastream """ rsa_ds = DataStream.from_datastream([valleys_datastream]) rsa_datapoints = [] for index, value in enumerate(valleys_datastream.data[:-2]): rsa_sample = rsa_calculate(valleys_datastream.data[index].start_time, valleys_datastream.data[index + 2].start_time, rr_datastream) if rsa_sample != -1.0: rsa_datapoints.append(DataPoint.from_tuple(start_time=valleys_datastream.data[index].start_time, end_time=valleys_datastream.data[index + 2].start_time, sample=rsa_sample)) if not rsa_datapoints: print('Error computing RSA!') rsa_ds.data = rsa_datapoints return rsa_ds
def rip_feature_computation( peaks_datastream: DataStream, valleys_datastream: DataStream) -> Tuple[DataStream]: """ Respiration Feature Implementation. The respiration feature values are derived from the following paper: 'puffMarker: a multi-sensor approach for pinpointing the timing of first lapse in smoking cessation' Removed due to lack of current use in the implementation roc_max = [] # 8. ROC_MAX = max(sample[j]-sample[j-1]) roc_min = [] # 9. ROC_MIN = min(sample[j]-sample[j-1]) :param peaks_datastream: DataStream :param valleys_datastream: DataStream :return: RIP Feature DataStreams """ # TODO: This needs fixed to prevent crashing the execution pipeline if peaks_datastream is None or valleys_datastream is None: return None # TODO: This needs fixed to prevent crashing the execution pipeline if len(peaks_datastream.data) == 0 or len(valleys_datastream.data) == 0: return None inspiration_duration = [] # 1 Inhalation duration expiration_duration = [] # 2 Exhalation duration respiration_duration = [] # 3 Respiration duration inspiration_expiration_ratio = [] # 4 Inhalation and Exhalation ratio stretch = [] # 5 Stretch upper_stretch = [] # 6. Upper portion of the stretch calculation lower_stretch = [] # 7. Lower portion of the stretch calculation delta_previous_inspiration_duration = [] # 10. BD_INSP = INSP(i)-INSP(i-1) delta_previous_expiration_duration = [] # 11. BD_EXPR = EXPR(i)-EXPR(i-1) delta_previous_respiration_duration = [] # 12. BD_RESP = RESP(i)-RESP(i-1) delta_previous_stretch_duration = [ ] # 14. BD_Stretch= Stretch(i)-Stretch(i-1) delta_next_inspiration_duration = [] # 19. FD_INSP = INSP(i)-INSP(i+1) delta_next_expiration_duration = [] # 20. FD_EXPR = EXPR(i)-EXPR(i+1) delta_next_respiration_duration = [] # 21. FD_RESP = RESP(i)-RESP(i+1) delta_next_stretch_duration = [] # 23. FD_Stretch= Stretch(i)-Stretch(i+1) neighbor_ratio_expiration_duration = [ ] # 29. D5_EXPR(i) = EXPR(i) / avg(EXPR(i-2)...EXPR(i+2)) neighbor_ratio_stretch_duration = [ ] # 32. D5_Stretch = Stretch(i) / avg(Stretch(i-2)...Stretch(i+2)) valleys = valleys_datastream.data peaks = peaks_datastream.data[:-1] for i, peak in enumerate(peaks): valley_start_time = valleys[i].start_time delta = peak.start_time - valleys[i].start_time inspiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta.total_seconds())) delta = valleys[i + 1].start_time - peak.start_time expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta.total_seconds())) delta = valleys[i + 1].start_time - valley_start_time respiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta.total_seconds())) ratio = (peak.start_time - valley_start_time) / ( valleys[i + 1].start_time - peak.start_time) inspiration_expiration_ratio.append( DataPoint.from_tuple(start_time=valley_start_time, sample=ratio)) value = peak.sample - valleys[i + 1].sample stretch.append( DataPoint.from_tuple(start_time=valley_start_time, sample=value)) # upper_stretch.append(DataPoint.from_tuple(start_time=valley_start_time, sample=(peak.sample - valleys[i + 1][1]) / 2)) # TODO: Fix this by adding a tracking moving average and compute upper stretch from this to the peak and lower from this to the valley # lower_stretch.append(DataPoint.from_tuple(start_time=valley_start_time, sample=(-peak.sample + valleys[i + 1][1]) / 2)) # TODO: Fix this by adding a tracking moving average and compute upper stretch from this to the peak and lower from this to the valley for i in range(len(inspiration_duration)): valley_start_time = valleys[i].start_time if i == 0: # Edge case delta_previous_inspiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_previous_expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_previous_respiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_previous_stretch_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) else: delta = inspiration_duration[i].sample - inspiration_duration[ i - 1].sample delta_previous_inspiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = expiration_duration[i].sample - expiration_duration[ i - 1].sample delta_previous_expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = respiration_duration[i].sample - respiration_duration[ i - 1].sample delta_previous_respiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = stretch[i].sample - stretch[i - 1].sample delta_previous_stretch_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) if i == len(inspiration_duration) - 1: delta_next_inspiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_next_expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_next_respiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_next_stretch_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) else: delta = inspiration_duration[i].sample - inspiration_duration[ i + 1].sample delta_next_inspiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = expiration_duration[i].sample - expiration_duration[ i + 1].sample delta_next_expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = respiration_duration[i].sample - respiration_duration[ i + 1].sample delta_next_respiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = stretch[i].sample - stretch[i + 1].sample delta_next_stretch_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) stretch_average = 0 expiration_average = 0 count = 0.0 for j in [-2, -1, 1, 2]: if i + j < 0 or i + j >= len(inspiration_duration): continue stretch_average += stretch[i + j].sample expiration_average += expiration_duration[i + j].sample count += 1 stretch_average /= count expiration_average /= count ratio = stretch[i].sample / stretch_average neighbor_ratio_stretch_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=ratio)) ratio = expiration_duration[i].sample / expiration_average neighbor_ratio_expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=ratio)) # Begin assembling datastream for output inspiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) inspiration_duration_datastream.data = inspiration_duration expiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) expiration_duration_datastream.data = expiration_duration respiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) respiration_duration_datastream.data = respiration_duration inspiration_expiration_ratio_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) inspiration_expiration_ratio_datastream.data = inspiration_expiration_ratio stretch_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) stretch_datastream.data = stretch upper_stretch_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) upper_stretch_datastream.data = upper_stretch lower_stretch_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) lower_stretch_datastream.data = lower_stretch delta_previous_inspiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_previous_inspiration_duration_datastream.data = delta_previous_inspiration_duration delta_previous_expiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_previous_expiration_duration_datastream.data = delta_previous_expiration_duration delta_previous_respiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_previous_respiration_duration_datastream.data = delta_previous_respiration_duration delta_previous_stretch_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_previous_stretch_duration_datastream.data = delta_previous_stretch_duration delta_next_inspiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_next_inspiration_duration_datastream.data = delta_next_inspiration_duration delta_next_expiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_next_expiration_duration_datastream.data = delta_next_expiration_duration delta_next_respiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_next_respiration_duration_datastream.data = delta_next_respiration_duration delta_next_stretch_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_next_stretch_duration_datastream.data = delta_next_stretch_duration neighbor_ratio_expiration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) neighbor_ratio_expiration_datastream.data = neighbor_ratio_expiration_duration neighbor_ratio_stretch_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) neighbor_ratio_stretch_datastream.data = neighbor_ratio_stretch_duration return inspiration_duration_datastream, \ expiration_duration_datastream, \ respiration_duration_datastream, \ inspiration_expiration_ratio_datastream, \ stretch_datastream, \ upper_stretch_datastream, \ lower_stretch_datastream, \ delta_previous_inspiration_duration_datastream, \ delta_previous_expiration_duration_datastream, \ delta_previous_respiration_duration_datastream, \ delta_previous_stretch_duration_datastream, \ delta_next_inspiration_duration_datastream, \ delta_next_expiration_duration_datastream, \ delta_next_respiration_duration_datastream, \ delta_next_stretch_duration_datastream, \ neighbor_ratio_expiration_datastream, \ neighbor_ratio_stretch_datastream
def timestamp_correct( datastream: DataStream, sampling_frequency: float, min_available_gaps: int = 3600, # TODO: Does this matter anymore? min_split_gap: datetime.timedelta = datetime.timedelta(seconds=30), max_data_points_per_segment: int = 100000000) -> DataStream: try: result = DataStream.from_datastream([datastream]) result.data = [] if len(datastream.data) == 0: return result data = datastream.data time_deltas = np.diff([dp.start_time for dp in data]) gap_points = [data[0]] for index, value in enumerate(time_deltas): if value > min_split_gap: gap_points.append(data[index]) gap_points.append(data[-1]) segments = [] segment_data = [] gap_index = 0 low_time = gap_points[gap_index].start_time high_time = gap_points[gap_index + 1].start_time for dp in data: if len(segment_data) >= max_data_points_per_segment: segments.append( interpolate_gaps(segment_data, sampling_frequency)) segment_data = [] if low_time <= dp.start_time <= high_time: segment_data.append(dp) else: segments.append( interpolate_gaps(segment_data, sampling_frequency)) gap_index += 1 low_time = gap_points[gap_index].start_time high_time = gap_points[gap_index + 1].start_time segment_data = [] segments.append(interpolate_gaps(segment_data, sampling_frequency)) for s in segments: begin_time = s[0].start_time.timestamp() end_time = s[-1].start_time.timestamp() x = np.array([ i for i in frange(begin_time, end_time, 1.0 / sampling_frequency) ], dtype='float') y = np.array([dp.start_time.timestamp() for dp in s], dtype='float') distance, path = fastdtw(x, y, radius=1) xx = [0 for i in y] for si, ei in path: xx[ei] = x[si] dtw_corrected_data = [] for index, dp in enumerate(s): ts = datetime.datetime.fromtimestamp(xx[index], tz=dp.start_time.tzinfo) dtw_corrected_data.append(DataPoint.from_tuple(ts, dp.sample)) result.data.extend(dtw_corrected_data) return result except: return []
def timestamp_correct(datastream: DataStream, sampling_frequency: float = None) -> DataStream: result = DataStream.from_datastream(input_streams=[datastream]) result.datapoints = datastream.datapoints return result
def detect_rpeak(ecg: DataStream, fs: float = 64, threshold: float = 0.5) -> DataStream: """ This program implements the Pan Tomkins algorithm on ECG signal to detect the R peaks Since the ecg array can have discontinuity in the timestamp arrays the rr-interval calculated in the algorithm is calculated in terms of the index in the sample array The algorithm consists of some major steps 1. computation of the moving window integration of the signal in terms of blackman window of a prescribed length 2. compute all the peaks of the moving window integration signal 3. adaptive thresholding with dynamic signal and noise thresholds applied to filter out the R peak locations 4. confirm the R peaks through differentiation from the nearby peaks and remove the false peaks :param ecg: ecg array of tuples (timestamp,value) :param fs: sampling frequency :param threshold: initial threshold to detect the R peak in a signal normalized by the 90th percentile. .5 is default. :return: R peak array of tuples (timestamp, Rpeak interval) """ data = ecg.datapoints sample = np.array([i.sample for i in data]) timestamp = np.array([i.start_time for i in data]) # computes the moving window integration of the signal blackman_win_len = np.ceil(fs / 5) # TODO: CODE_REVIEW: Hard coded value of 5 y = compute_moving_window_int(sample, fs, blackman_win_len) # TODO: CODE_REVIEW: Hard coded +/- 2. Is this a function of the blackman_win_len? peak_location = [ i for i in range(2, len(y) - 1) if check_peak(y[i - 2:i + 2]) ] peak_location_values = y[peak_location] # initial RR interval average running_rr_avg = sum(np.diff(peak_location)) / (len(peak_location) - 1) rpeak_temp1 = compute_r_peaks(threshold, running_rr_avg, y, peak_location_values, peak_location) rpeak_temp1 = remove_close_peaks(rpeak_temp1, sample, fs) index = confirm_peaks(rpeak_temp1, sample, fs) rpeak_timestamp = timestamp[index] rpeak_value = np.diff(rpeak_timestamp) rpeak_timestamp = rpeak_timestamp[1:] result_data = [] for k in range(len(rpeak_value)): result_data.append( DataPoint.from_tuple( rpeak_timestamp[k], rpeak_value[k].seconds + rpeak_value[k].microseconds / 1e6)) # Create resulting datastream to be returned result = DataStream.from_datastream([ecg]) result.datapoints = result_data return result
def store(data: OrderedDict, input_streams: dict, output_streams: dict, CC_obj: CerebralCortex): """ Store diagnostic results with its metadata in the data-store :param input_streams: :param data: :param CC_obj: :param config: :param algo_type: """ if data: #basic output stream info owner = input_streams[0]["owner_id"] dd_stream_id = output_streams["id"] dd_stream_name = output_streams["name"] stream_type = "ds" data_descriptor = [{ "NAME": "Data Quality (LED)", "DATA_TYPE": "int", "FREQUENCY": "0.33", "MAX_VALUE": "4", "MIN_VALUE": "0", "DESCRIPTION": "measures the Data Quality of LED. Values= GOOD(0), BAND_OFF(1), NOT_WORN(2), BAND_LOOSE(3), NOISE(4)" }] execution_context = { "platform_metadata": { "NAME": "MotionSense HRV", "DEVICE_ID": "" }, "processing_module": { "name": "", "environment": "cerebralcortex", "algorithm": [{ "method": "", "authors": ["Nasir Ali", " Md Azim Ullah"], "version": "0.0.1", "reference": { "url": "http://md2k.org/" }, "description": "" }], "description": "", "input_streams": input_streams, "output_streams": output_streams, "input_parameters": {} }, "datasource_metadata": { "NAME": "Data Quality (LED)", "DATA_TYPE": "org.md2k.datakitapi.datatype.DataTypeInt", "FREQUENCY": "0.33", "DESCRIPTION": "measures the Data Quality of LED. Values= GOOD(0), BAND_OFF(1), NOT_WORN(2), BAND_LOOSE(3), NOISE(4)" }, "application_metadata": { "NAME": "MotionSense", "DESCRIPTION": "Collects data from the motion sense. Sensors supported: [Accelerometer, Gyroscope, Battery, LED, DataQuality]", "VERSION_NAME": "0.0.1", "VERSION_NUMBER": "2000500" } } annotations = [] ds = DataStream(identifier=dd_stream_id, owner=owner, name=dd_stream_name, data_descriptor=data_descriptor, execution_context=execution_context, annotations=annotations, stream_type=stream_type, data=data) CC_obj.save_datastream(ds, "datastream")
def test_interpolate_gaps(self): ds = DataStream(None, None) ds.datapoints = self.ecg result = interpolate_gaps(ds.datapoints, self.sample_rate)
def detect_rpeak(ecg: DataStream, fs: float = 64, threshold: float = 0.5, blackman_win_len_range: float = 1 / 5): """ This program implements the Pan Tomkins algorithm on ECG signal to detect the R peaks Since the ecg array can have discontinuity in the timestamp arrays the rr-interval calculated in the algorithm is calculated in terms of the index in the sample array The algorithm consists of some major steps 1. computation of the moving window integration of the signal in terms of blackman window of a prescribed length 2. compute all the peaks of the moving window integration signal 3. adaptive thresholding with dynamic signal and noise thresholds applied to filter out the R peak locations 4. confirm the R peaks through differentiation from the nearby peaks and remove the false peaks :param ecg: ecg array of tuples (timestamp,value) :param fs: sampling frequency :param threshold: initial threshold to detect the R peak in a signal normalized by the 90th percentile. .5 is default. :param blackman_win_len_range : the range to calculate blackman window length :return: R peak array of tuples (timestamp, Rpeak interval) """ try: data = ecg.data sample = np.array([i.sample for i in data]) timestamp = np.array([i.start_time for i in data]) # computes the moving window integration of the signal blackman_win_len = np.ceil(fs * blackman_win_len_range) y = compute_moving_window_int(sample, fs, blackman_win_len) peak_location_values = [(i, y[i]) for i in range(2, len(y) - 1) if check_peak(y[i - 2:i + 3])] # initial RR interval average peak_location = [i[0] for i in peak_location_values] running_rr_avg = sum(np.diff(peak_location)) / (len(peak_location) - 1) rpeak_temp1 = compute_r_peaks(threshold, running_rr_avg, y, peak_location_values) rpeak_temp2 = remove_close_peaks(rpeak_temp1, sample, fs) index = confirm_peaks(rpeak_temp2, sample, fs) rpeak_timestamp = timestamp[index] rpeak_samples = sample[index] #--------------------------- # rpeak values: result_rpeaks = [] for k in range(len(rpeak_samples)): result_rpeaks.append( DataPoint.from_tuple(rpeak_timestamp[k], rpeak_samples[k])) #--------------------------- # rr_interval values rpeak_value = np.diff(rpeak_timestamp) rpeak_timestamp = rpeak_timestamp[1:] result_interval = [] for k in range(len(rpeak_value)): result_interval.append( DataPoint.from_tuple(rpeak_timestamp[k], rpeak_value[k].seconds + rpeak_value[k].microseconds / 1e6)) # Create resulting datastream to be returned # rpeaks result_rpeaks_ds = DataStream.from_datastream([ecg]) result_rpeaks_ds.data = result_rpeaks # rr_interval result_interval_ds = DataStream.from_datastream([ecg]) result_interval_ds.data = result_interval return result_interval_ds, result_rpeaks_ds except: # Create resulting datastream to be returned # rpeaks result_rpeaks_ds = DataStream.from_datastream([ecg]) result_rpeaks_ds.data = [] # rr_interval result_interval_ds = DataStream.from_datastream([ecg]) result_interval_ds.data = [] return result_interval_ds, result_rpeaks_ds
master="local[*]", name="Memphis cStress Development App") for i in range(1, 2): participant = "SI%02d" % i participant_UUID = uuid.uuid4() print(participant, participant_UUID) try: ecgRDD = CC.readfile( find(basedir, { "participant": participant, "datasource": "ecg" })).map(parser.data_processor).filter( lambda x: isinstance(x, DataPoint)) ecg = DataStream(CC, participant_UUID, data=ecgRDD.collect()) ripRDD = CC.readfile( find(basedir, { "participant": participant, "datasource": "rip" })).map(parser.data_processor).filter( lambda x: isinstance(x, DataPoint)) rip = DataStream(CC, participant_UUID, data=ripRDD.collect()) accelxRDD = CC.readfile( find(basedir, { "participant": participant, "datasource": "accelx" })).map(parser.data_processor).filter( lambda x: isinstance(x, DataPoint))
def ecg_feature_computation(datastream: DataStream, window_size: float, window_offset: float, low_frequency: float = 0.01, high_frequency: float = 0.7, low_rate_vlf: float = 0.0009, high_rate_vlf: float = 0.04, low_rate_hf: float = 0.15, high_rate_hf: float = 0.4, low_rate_lf: float = 0.04, high_rate_lf: float = 0.15): """ ECG Feature Implementation. The frequency ranges for High, Low and Very low heart rate variability values are derived from the following paper: 'Heart rate variability: standards of measurement, physiological interpretation and clinical use' :param high_rate_lf: float :param low_rate_lf: float :param high_rate_hf: float :param low_rate_hf: float :param high_rate_vlf: float :param low_rate_vlf: float :param high_frequency: float :param low_frequency: float :param datastream: DataStream :param window_size: float :param window_offset: float :return: ECG Feature DataStreams """ if datastream is None: return None if len(datastream.data) == 0: return None # perform windowing of datastream window_data = window_sliding(datastream.data, window_size, window_offset) # initialize each ecg feature array rr_variance_data = [] rr_mean_data = [] rr_median_data = [] rr_80percentile_data = [] rr_20percentile_data = [] rr_quartile_deviation_data = [] rr_HF_data = [] rr_LF_data = [] rr_VLF_data = [] rr_LF_HF_data = [] rr_heart_rate_data = [] # iterate over each window and calculate features for key, value in window_data.items(): starttime, endtime = key reference_data = np.array([i.sample for i in value]) rr_variance_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=np.var(reference_data))) power, frequency = lomb(data=value, low_frequency=low_frequency, high_frequency=high_frequency) rr_VLF_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=heart_rate_power( power, frequency, low_rate_vlf, high_rate_vlf))) rr_HF_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=heart_rate_power( power, frequency, low_rate_hf, high_rate_hf))) rr_LF_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=heart_rate_power( power, frequency, low_rate_lf, high_rate_lf))) if heart_rate_power(power, frequency, low_rate_hf, high_rate_hf) != 0: lf_hf = float( heart_rate_power(power, frequency, low_rate_lf, high_rate_lf) / heart_rate_power(power, frequency, low_rate_hf, high_rate_hf)) rr_LF_HF_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=lf_hf)) else: rr_LF_HF_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=0)) rr_mean_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=np.mean(reference_data))) rr_median_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=np.median(reference_data))) rr_quartile_deviation_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=(0.5 * (np.percentile(reference_data, 75) - np.percentile(reference_data, 25))))) rr_80percentile_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=np.percentile(reference_data, 80))) rr_20percentile_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=np.percentile(reference_data, 20))) rr_heart_rate_data.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=np.median(60 / reference_data))) rr_variance = DataStream.from_datastream([datastream]) rr_variance.data = rr_variance_data rr_vlf = DataStream.from_datastream([datastream]) rr_vlf.data = rr_VLF_data rr_hf = DataStream.from_datastream([datastream]) rr_hf.data = rr_HF_data rr_lf = DataStream.from_datastream([datastream]) rr_lf.data = rr_LF_data rr_lf_hf = DataStream.from_datastream([datastream]) rr_lf_hf.data = rr_LF_HF_data rr_mean = DataStream.from_datastream([datastream]) rr_mean.data = rr_mean_data rr_median = DataStream.from_datastream([datastream]) rr_median.data = rr_median_data rr_quartile = DataStream.from_datastream([datastream]) rr_quartile.data = rr_quartile_deviation_data rr_80 = DataStream.from_datastream([datastream]) rr_80.data = rr_80percentile_data rr_20 = DataStream.from_datastream([datastream]) rr_20.data = rr_20percentile_data rr_heart_rate = DataStream.from_datastream([datastream]) rr_heart_rate.data = rr_heart_rate_data return rr_variance, rr_vlf, rr_hf, rr_lf, rr_lf_hf, rr_mean, rr_median, rr_quartile, rr_80, rr_20, rr_heart_rate
def loader(identifier: int): participant = "SI%02d" % identifier participant_uuid = uuid.uuid4() try: ecg = DataStream(None, participant_uuid) ecg.data = readfile( find(basedir, { "participant": participant, "datasource": "ecg" })) rip = DataStream(None, participant_uuid) rip.data = readfile( find(basedir, { "participant": participant, "datasource": "rip" })) accelx = DataStream(None, participant_uuid) accelx.data = readfile( find(basedir, { "participant": participant, "datasource": "accelx" })) accely = DataStream(None, participant_uuid) accely.data = readfile( find(basedir, { "participant": participant, "datasource": "accely" })) accelz = DataStream(None, participant_uuid) accelz.data = readfile( find(basedir, { "participant": participant, "datasource": "accelz" })) stress_marks = DataStream(None, participant_uuid) stress_marks.data = readfile_ground_truth( find(basedir, { "participant": participant, "datasource": "stress_marks" })) return { "participant": participant, "ecg": ecg, "rip": rip, "accelx": accelx, "accely": accely, "accelz": accelz, "stress_marks": stress_marks } except Exception as e: print("File missing for %s" % participant) return {"ERROR": 'missing data file'}
def test_split_by_gaps(self): ds = DataStream(None, None) # ds.datapoints = self.ecg ds.datapoints = self.ecg[:500] result = split_by_gaps(ds)
def setUp(self): self.size = 100000 self.ds = DataStream(None, None) data = [DataPoint.from_tuple(datetime.datetime.now(), [random()]) for i in range(0, self.size)] self.ds.data = data
def compute_peak_valley( rip: DataStream, fs: float = 21.33, smoothing_factor: int = 5, time_window: int = 8, expiration_amplitude_threshold_perc: float = 0.10, threshold_expiration_duration: float = 0.312, inspiration_amplitude_threshold_perc: float = 0.10, max_amplitude_change_peak_correction: float = 30, min_neg_slope_count_peak_correction: int = 4, minimum_peak_to_valley_time_diff=0.31) -> [DataStream, DataStream]: """ Compute peak and valley from rip data and filter peak and valley. :param minimum_peak_to_valley_time_diff: :param inspiration_amplitude_threshold_perc: :param smoothing_factor: :return peak_datastream, valley_datastream: :param rip: :param fs: :param time_window: :param expiration_amplitude_threshold_perc: :param threshold_expiration_duration: :param max_amplitude_change_peak_correction: :param min_neg_slope_count_peak_correction: """ data_smooth = smooth(data=rip.data, span=smoothing_factor) window_length = int(round(time_window * fs)) data_mac = moving_average_curve(data_smooth, window_length=window_length) data_smooth_start_time_to_index = {} for index, data in enumerate(data_smooth): data_smooth_start_time_to_index[data.start_time] = index up_intercepts, down_intercepts = up_down_intercepts( data=data_smooth, mac=data_mac, data_start_time_to_index=data_smooth_start_time_to_index) up_intercepts_filtered, down_intercepts_filtered = filter_intercept_outlier( up_intercepts=up_intercepts, down_intercepts=down_intercepts) peaks, valleys = generate_peak_valley( up_intercepts=up_intercepts_filtered, down_intercepts=down_intercepts_filtered, data=data_smooth) valleys_corrected = correct_valley_position( peaks=peaks, valleys=valleys, up_intercepts=up_intercepts_filtered, data=data_smooth, data_start_time_to_index=data_smooth_start_time_to_index) peaks_corrected = correct_peak_position( peaks=peaks, valleys=valleys_corrected, up_intercepts=up_intercepts_filtered, data=data_smooth, max_amplitude_change_peak_correction= max_amplitude_change_peak_correction, min_neg_slope_count_peak_correction=min_neg_slope_count_peak_correction, data_start_time_to_index=data_smooth_start_time_to_index) # remove too close valley peak pair. peaks_filtered_close, valleys_filtered_close = remove_close_valley_peak_pair( peaks=peaks_corrected, valleys=valleys_corrected, minimum_peak_to_valley_time_diff=minimum_peak_to_valley_time_diff) # Remove small Expiration duration < 0.31 peaks_filtered_exp_dur, valleys_filtered_exp_dur = filter_expiration_duration_outlier( peaks=peaks_filtered_close, valleys=valleys_filtered_close, threshold_expiration_duration=threshold_expiration_duration) # filter out peak valley pair of inspiration of small amplitude. peaks_filtered_insp_amp, valleys_filtered_insp_amp = filter_small_amp_inspiration_peak_valley( peaks=peaks_filtered_exp_dur, valleys=valleys_filtered_exp_dur, inspiration_amplitude_threshold_perc= inspiration_amplitude_threshold_perc) # filter out peak valley pair of expiration of small amplitude. peaks_filtered_exp_amp, valleys_filtered_exp_amp = filter_small_amp_expiration_peak_valley( peaks=peaks_filtered_insp_amp, valleys=valleys_filtered_insp_amp, expiration_amplitude_threshold_perc=expiration_amplitude_threshold_perc ) peak_datastream = DataStream.from_datastream([rip]) peak_datastream.data = peaks_filtered_exp_amp valley_datastream = DataStream.from_datastream([rip]) valley_datastream.data = valleys_filtered_exp_amp return peak_datastream, valley_datastream
def get_stream(self, identifier: uuid) -> DataStream: return DataStream(identifier, data=[])
def test_07_stream_filter(self): identifier_anno = "6db98dfb-d6e8-4b27-8d55-95b20fa0f750" identifier_data = "6db98dfb-d6e8-4b27-8d55-95b20fa0f751" owner_id = "06634264-56bc-4c92-abd7-377dbbad79dd" name_anno = "data-store-test-annotation" name_data = "data-store-test-data" data_descriptor = {} execution_context_anno = json.loads( '{"execution_context": {"algorithm": {"method": "test.data_store.annotation.filter"}}}' ) execution_context_data = json.loads( '{"execution_context": {"algorithm": {"method": "test.data_store.data.filter"}}}' ) annotations_data = json.loads( '[{"name": "test-case","identifier": "6db98dfb-d6e8-4b27-8d55-95b20fa0f750"}]' ) annotations_anno = {} datapoints_anno = [] datapoints_data = [] result_data = Metadata(self.CC).is_id_created(owner_id, name_data, execution_context_data) if result_data["status"] != "new": identifier_data = result_data["id"] Metadata(self.CC).store_stream_info( identifier_anno, owner_id, name_anno, data_descriptor, execution_context_anno, annotations_anno, "annotations", datetime.datetime(2017, 4, 24, 0, 0, 1), datetime.datetime(2017, 4, 24, 0, 0, 5), result_data["status"]) result_anno = Metadata(self.CC).is_id_created(owner_id, name_data, execution_context_data) if result_anno["status"] != "new": identifier_anno = result_anno["id"] Metadata(self.CC).store_stream_info( identifier_data, owner_id, name_data, data_descriptor, execution_context_data, annotations_data, "datastream", datetime.datetime(2017, 4, 24, 0, 0, 1), datetime.datetime(2017, 4, 24, 0, 0, 5), result_anno["status"]) for i in range(0, 5): if (i % 2 == 0): sample_anno = 'good' else: sample_anno = 'bad' sample_data = i, i + 2, i + 3 start_time_anno = datetime.datetime(2017, 4, 24, 0, 0, i) end_time_anno = datetime.datetime(2017, 4, 24, 0, 0, (5 + i)) start_time_data = datetime.datetime(2017, 4, 24, 0, 0, i) end_time_data = datetime.datetime(2017, 4, 24, 0, 0, (3 + i)) localtz = timezone('US/Central') start_time_anno = localtz.localize(start_time_anno) end_time_anno = localtz.localize(end_time_anno) start_time_data = localtz.localize(start_time_data) end_time_data = localtz.localize(end_time_data) datapoints_anno.append( DataPoint(start_time=start_time_anno, end_time=end_time_anno, sample=sample_anno)) datapoints_data.append( DataPoint(start_time=start_time_data, end_time=end_time_data, sample=sample_data)) ds_anno = DataStream(uuid.UUID(identifier_anno), owner_id, name_anno, data_descriptor, execution_context_anno, annotations_data, "annotations", start_time_anno, end_time_anno, datapoints_anno) ds_data = DataStream(uuid.UUID(identifier_data), owner_id, name_data, data_descriptor, execution_context_data, annotations_anno, "datastream", start_time_anno, end_time_anno, datapoints_data) self.CC.save_datastream(ds_anno) self.CC.save_datastream(ds_data) filted_stream = self.CC.filter_stream(identifier_data, "test-case", "good") self.assertEqual(len(filted_stream), 5) for i in range(0, 5): sample_data = [i, i + 2, i + 3] start_time_data = datetime.datetime(2017, 4, 24, 0, 0, i) end_time_data = datetime.datetime(2017, 4, 24, 0, 0, (3 + i)) start_time_data = localtz.localize(start_time_data) end_time_data = localtz.localize(end_time_data) self.assertEqual(filted_stream[i].start_time, start_time_data) self.assertEqual(filted_stream[i].end_time, end_time_data) self.assertEqual(filted_stream[i].sample, sample_data)
def rip_feature_computation(peaks_datastream: DataStream, valleys_datastream: DataStream, rr_intervals_datastream: DataStream, window_size: float, window_offset: float) -> Tuple[DataStream]: """ Respiration Feature Implementation. The respiration feature values are derived from the following paper: 'puffMarker: a multi-sensor approach for pinpointing the timing of first lapse in smoking cessation' Removed due to lack of current use in the implementation roc_max = [] # 8. ROC_MAX = max(sample[j]-sample[j-1]) roc_min = [] # 9. ROC_MIN = min(sample[j]-sample[j-1]) :param peaks_datastream: DataStream :param valleys_datastream: DataStream :return: RIP Feature DataStreams """ # TODO: This needs fixed to prevent crashing the execution pipeline if peaks_datastream is None or valleys_datastream is None: return None # TODO: This needs fixed to prevent crashing the execution pipeline if len(peaks_datastream.data) == 0 or len(valleys_datastream.data) == 0: return None inspiration_duration = [] # 1 Inhalation duration expiration_duration = [] # 2 Exhalation duration respiration_duration = [] # 3 Respiration duration inspiration_expiration_ratio = [] # 4 Inhalation and Exhalation ratio stretch = [] # 5 Stretch upper_stretch = [] # 6. Upper portion of the stretch calculation lower_stretch = [] # 7. Lower portion of the stretch calculation delta_previous_inspiration_duration = [] # 10. BD_INSP = INSP(i)-INSP(i-1) delta_previous_expiration_duration = [] # 11. BD_EXPR = EXPR(i)-EXPR(i-1) delta_previous_respiration_duration = [] # 12. BD_RESP = RESP(i)-RESP(i-1) delta_previous_stretch_duration = [ ] # 14. BD_Stretch= Stretch(i)-Stretch(i-1) delta_next_inspiration_duration = [] # 19. FD_INSP = INSP(i)-INSP(i+1) delta_next_expiration_duration = [] # 20. FD_EXPR = EXPR(i)-EXPR(i+1) delta_next_respiration_duration = [] # 21. FD_RESP = RESP(i)-RESP(i+1) delta_next_stretch_duration = [] # 23. FD_Stretch= Stretch(i)-Stretch(i+1) neighbor_ratio_expiration_duration = [ ] # 29. D5_EXPR(i) = EXPR(i) / avg(EXPR(i-2)...EXPR(i+2)) neighbor_ratio_stretch_duration = [ ] # 32. D5_Stretch = Stretch(i) / avg(Stretch(i-2)...Stretch(i+2)) #---------------------------------------------------------------------- rsa = [] # RSA rrIntervals = rr_intervals_datastream.data #---------------------------------------------------------------------- valleys = valleys_datastream.data peaks = peaks_datastream.data[:-1] for i, peak in enumerate(peaks): valley_start_time = valleys[i].start_time delta = peak.start_time - valleys[i].start_time inspiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta.total_seconds())) delta = valleys[i + 1].start_time - peak.start_time expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta.total_seconds())) delta = valleys[i + 1].start_time - valley_start_time respiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta.total_seconds())) ratio = (peak.start_time - valley_start_time) / ( valleys[i + 1].start_time - peak.start_time) inspiration_expiration_ratio.append( DataPoint.from_tuple(start_time=valley_start_time, sample=ratio)) value = peak.sample - valleys[i + 1].sample stretch.append( DataPoint.from_tuple(start_time=valley_start_time, sample=value)) #---------------------------------------------------------------------- # RSA value = rsaCalculateCycle(valley_start_time, valleys[i + 1].start_time, rrIntervals) if value != -1: rsa.append( Data.Point.from_tuple(start_time=valley_start_time, sample=value)) #---------------------------------------------------------------------- # upper_stretch.append(DataPoint.from_tuple(start_time=valley_start_time, sample=(peak.sample - valleys[i + 1][1]) / 2)) # TODO: Fix this by adding a tracking moving average and compute upper stretch from this to the peak and lower from this to the valley # lower_stretch.append(DataPoint.from_tuple(start_time=valley_start_time, sample=(-peak.sample + valleys[i + 1][1]) / 2)) # TODO: Fix this by adding a tracking moving average and compute upper stretch from this to the peak and lower from this to the valley for i in range(len(inspiration_duration)): valley_start_time = valleys[i].start_time if i == 0: # Edge case delta_previous_inspiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_previous_expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_previous_respiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_previous_stretch_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) else: delta = inspiration_duration[i].sample - inspiration_duration[ i - 1].sample delta_previous_inspiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = expiration_duration[i].sample - expiration_duration[ i - 1].sample delta_previous_expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = respiration_duration[i].sample - respiration_duration[ i - 1].sample delta_previous_respiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = stretch[i].sample - stretch[i - 1].sample delta_previous_stretch_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) if i == len(inspiration_duration) - 1: delta_next_inspiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_next_expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_next_respiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) delta_next_stretch_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=0.0)) else: delta = inspiration_duration[i].sample - inspiration_duration[ i + 1].sample delta_next_inspiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = expiration_duration[i].sample - expiration_duration[ i + 1].sample delta_next_expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = respiration_duration[i].sample - respiration_duration[ i + 1].sample delta_next_respiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) delta = stretch[i].sample - stretch[i + 1].sample delta_next_stretch_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=delta)) stretch_average = 0 expiration_average = 0 count = 0.0 for j in [-2, -1, 1, 2]: if i + j < 0 or i + j >= len(inspiration_duration): continue stretch_average += stretch[i + j].sample expiration_average += expiration_duration[i + j].sample count += 1 stretch_average /= count expiration_average /= count ratio = stretch[i].sample / stretch_average neighbor_ratio_stretch_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=ratio)) ratio = expiration_duration[i].sample / expiration_average neighbor_ratio_expiration_duration.append( DataPoint.from_tuple(start_time=valley_start_time, sample=ratio)) # Begin assembling datastream for output inspiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) inspiration_duration_datastream.data = inspiration_duration expiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) expiration_duration_datastream.data = expiration_duration respiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) respiration_duration_datastream.data = respiration_duration inspiration_expiration_ratio_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) inspiration_expiration_ratio_datastream.data = inspiration_expiration_ratio stretch_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) stretch_datastream.data = stretch #---------------------------------------------------------------------- rsa_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) rsa_datastream.data = rsa #---------------------------------------------------------------------- upper_stretch_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) upper_stretch_datastream.data = upper_stretch lower_stretch_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) lower_stretch_datastream.data = lower_stretch delta_previous_inspiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_previous_inspiration_duration_datastream.data = delta_previous_inspiration_duration delta_previous_expiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_previous_expiration_duration_datastream.data = delta_previous_expiration_duration delta_previous_respiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_previous_respiration_duration_datastream.data = delta_previous_respiration_duration delta_previous_stretch_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_previous_stretch_duration_datastream.data = delta_previous_stretch_duration delta_next_inspiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_next_inspiration_duration_datastream.data = delta_next_inspiration_duration delta_next_expiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_next_expiration_duration_datastream.data = delta_next_expiration_duration delta_next_respiration_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_next_respiration_duration_datastream.data = delta_next_respiration_duration delta_next_stretch_duration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) delta_next_stretch_duration_datastream.data = delta_next_stretch_duration neighbor_ratio_expiration_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) neighbor_ratio_expiration_datastream.data = neighbor_ratio_expiration_duration neighbor_ratio_stretch_datastream = DataStream.from_datastream( [peaks_datastream, valleys_datastream]) neighbor_ratio_stretch_datastream.data = neighbor_ratio_stretch_duration #---------------------------------------------------------------------- # Aggregate minute level feature # Breath-rate valleys_window = window_sliding(valleys, window_size, window_offset) peaks_window = window_sliding(valleys, window_size, window_offset) insp_window = window_sliding(inspiration_duration, window_size, window_offset) exp_window = window_sliding(expiration_duration, window_size, window_offset) resp_window = window_sliding(respiration_duration, window_size, window_offset) inspExp_window = window_sliding(inspiration_expiration_ratio, window_size, window_offset) stretch_window = window_sliding(stretch, window_size, window_offset) rsa_window = window_sliding(rsa, window_size, window_offset) breath_rate = [] insp_min_vol = [] insp_qDev = [] insp_mean = [] insp_median = [] insp_80 = [] exp_qDev = [] exp_mean = [] exp_median = [] exp_80 = [] resp_qDev = [] resp_mean = [] resp_median = [] resp_80 = [] inspExp_qDev = [] inspExp_mean = [] inspExp_median = [] inspExp_80 = [] stretch_qDev = [] stretch_mean = [] stretch_median = [] stretch_80 = [] rsa_qDev = [] rsa_mean = [] rsa_median = [] rsa_80 = [] for key, value in valleys_window.items(): starttime, endtime = key # breath_rate breath_rate.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=len(valleys_window[key]))) # inspiration minute volume value = 0. for i in range(len(valleys_window[key])): peak_value = peaks_window[key][i].sample peak_time = peaks_window[key][i].start_time.timestamp() valley_value = valleys_window[key][i].sample valley_time = valleys_window[key][i].start_time.timestamp() if peak_time > valley_time: value += (peak_time - valley_time) * (peak_value - valley_value) / 2 insp_min_vol.append( DataPoint.from_tuple(start_time=starttime, end_time=endtime, sample=value)) # inspiration duration insp_qDev, insp_mean, insp_median, insp_80 = getStats( insp_window[key], insp_qDev, insp_mean, insp_median, insp_80) # Expiration duration exp_qDev, exp_mean, exp_median, exp_80 = getStats( exp_window[key], exp_qDev, exp_mean, exp_median, exp_80) # Respiration duration resp_qDev, resp_mean, resp_median, resp_80 = getStats( resp_window[key], resp_qDev, resp_mean, resp_median, resp_80) # Inspiration Expiration duration ratio inspExp_qDev, inspExp_mean, inspExp_median, inspExp_80 = getStats( inspExp_window[key], inspExp_qDev, inspExp_mean, inspExp_median, inspExp_80) # Stretch stretch_qDev, stretch_mean, stretch_median, stretch_80 = getStats( stretch_window[key], stretch_qDev, stretch_mean, stretch_median, stretch_80) # RSA rsa_qDev, rsa_mean, rsa_median, rsa_80 = getStats( rsa_window[key], rsa_qDev, rsa_mean, rsa_median, rsa_80) # To datastream struct # breath_rate breath_rate_datastream = DataStream.from_datastream( [rr_intervals_datastream]) breath_rate_datastream.data = breath_rate # inspiration minute volume insp_min_vol_datastream = DataStream.from_datastream( [rr_intervals_datastream]) insp_min_vol_datastream.data = insp_min_vol # Inspiration duration insp_qDev_datastream, insp_mean_datastream, insp_median_datastream, insp_80_datastream = \ list2datastream(insp_qDev, insp_mean, insp_median, insp_80, rr_intervals_datastream) # Expiration duration exp_qDev_datastream, exp_mean_datastream, exp_median_datastream, exp_80_datastream = \ list2datastream(exp_qDev, exp_mean, exp_median, exp_80, rr_intervals_datastream) # Respiration Duration resp_qDev_datastream, resp_mean_datastream, resp_median_datastream, resp_80_datastream = \ list2datastream(resp_qDev, resp_mean, resp_median, resp_80, rr_intervals_datastream) # Inspiration Expiration duration ratio inspExp_qDev_datastream, inspExp_mean_datastream, inspExp_median_datastream, inspExp_80_datastream = \ list2datastream(inspExp_qDev, inspExp_mean, inspExp_median, inspExp_80, rr_intervals_datastream) # Stretch stretch_qDev_datastream, stretch_mean_datastream, stretch_median_datastream, stretch_80_datastream = \ list2datastream(stretch_qDev, stretch_mean, stretch_median, stretch_80, rr_intervals_datastream) # RSA rsa_qDev_datastream, rsa_mean_datastream, rsa_median_datastream, rsa_80_datastream = \ list2datastream(rsa_qDev, rsa_mean, rsa_median, rsa_80, rr_intervals_datastream) #---------------------------------------------------------------------- return breath_rate_datastream, insp_min_vol_datastream, \ insp_qDev_datastream, insp_mean_datastream, insp_median_datastream, insp_80_datastream, \ exp_qDev_datastream, exp_mean_datastream, exp_median_datastream, exp_80_datastream, \ resp_qDev_datastream, resp_mean_datastream, resp_median_datastream, resp_80_datastream, \ inspExp_qDev_datastream, inspExp_mean_datastream, inspExp_median_datastream, inspExp_80_datastream, \ stretch_qDev_datastream, stretch_mean_datastream, stretch_median_datastream, stretch_80_datastream, \ rsa_qDev_datastream, rsa_mean_datastream, rsa_median_datastream, rsa_80_datastream '''