def persist_data(execution_context, data_descriptor, annotations, stream_id,
name, owner_id, data, CC_obj):
"""
Parse old schema and map it into new CerebralCortex data provenance sechema
:param filename:
:param data:
:param CC_obj:
"""
stream_type = "datastream"
ds = DataStream(identifier=stream_id,
owner=owner_id,
name=name,
data_descriptor=data_descriptor,
execution_context=execution_context,
annotations=annotations,
stream_type=stream_type,
data=data)
CC_obj.save_datastream(ds)
def test_compare_running_time(self):
start_time_whole = time.time()
rr_intervals = compute_rr_intervals(self.ecg_ds,
self.ecg_sampling_frequency)
elapse_time_whole = time.time() - start_time_whole
print('elapse_time_whole =', elapse_time_whole)
window_size = 60
window_offset = 1
start_time_win = time.time()
for key, data in window_iter(self.ecg_ds.data, window_size,
window_offset):
elapse_time_win = time.time() - start_time_win
print(len(data),
'time =',
'{:.2f}'.format(elapse_time_win),
end='\r',
flush=True)
rr_intervals = compute_rr_intervals(
DataStream(None, None, data=data), self.ecg_sampling_frequency)
def test_06_store_stream(self):
identifier = "6db98dfb-d6e8-4b27-8d55-95b20fa0f754"
owner = "06634264-56bc-4c92-abd7-377dbbad79dd"
name = "data-store-test"
data_descriptor = {}
execution_context = json.loads(
'{"execution_context": {"algorithm": {"method": "cerebralcortex.data_processor.data_diagnostic.BatteryDataMarker"}}}'
)
annotations = {}
datapoints = []
stream_type = "datastream"
start_time = datetime.datetime(2017, 4, 24, 0, 0, 1)
end_time = datetime.datetime(2017, 4, 24, 0, 0, 2)
localtz = timezone('US/Central')
start_time = localtz.localize(start_time)
end_time = localtz.localize(end_time)
sample = {'Foo3': 123}
dp1 = DataPoint(start_time=start_time,
end_time=end_time,
sample=sample)
datapoints.append(dp1)
ds = DataStream(identifier, owner, name, data_descriptor,
execution_context, annotations, stream_type,
start_time, end_time, datapoints)
self.CC.save_datastream(ds)
stream = self.CC.get_datastream(identifier, data_type=DataSet.COMPLETE)
self.assertEqual(stream._identifier, identifier)
self.assertEqual(stream._owner, owner)
self.assertEqual(stream._name, name)
self.assertEqual(stream._data_descriptor, data_descriptor)
self.assertEqual(stream._execution_context, execution_context)
self.assertEqual(stream._annotations, annotations)
self.assertEqual(stream._datastream_type, stream_type)
self.assertEqual(stream.data[0].start_time, start_time)
self.assertEqual(stream.data[0].end_time, end_time)
self.assertEqual(stream.data[0].sample, sample)
def normalize(datastream: DataStream) -> DataStream:
"""
:param datastream:
:return:
"""
input_data = np.array([i.sample for i in datastream.datapoints])
data = preprocessing.normalize(input_data, axis=0)
result_data = [
DataPoint.from_tuple(start_time=i.start_time, sample=None)
for i in datastream.datapoints
]
for i, dp in enumerate(result_data):
dp.sample = data[i]
result = DataStream.from_datastream(input_streams=[datastream])
result.datapoints = result_data
return result
def magnitude(datastream: DataStream) -> DataStream:
"""
:param datastream:
:return:
"""
result = DataStream.from_datastream(input_streams=[datastream])
if datastream.data is None or len(datastream.data) == 0:
result.data = []
return result
input_data = np.array([i.sample for i in datastream.data])
data = norm(input_data, axis=1).tolist()
result.data = [
DataPoint.from_tuple(start_time=v.start_time, sample=data[i])
for i, v in enumerate(datastream.data)
]
return result
def test_compare_running_time(self):
start_time_whole = time.time()
result = timestamp_correct(self.accelx,
sampling_frequency=self.sample_rate)
elapse_time_whole = time.time() - start_time_whole
print('elapse_time_whole =', elapse_time_whole)
window_size = 60
window_offset = 1
start_time_win = time.time()
for key, data in window_iter(self.accelx.data, window_size,
window_offset):
elapse_time_win = time.time() - start_time_win
print(len(data),
data[-1].start_time,
'time =',
'{:.2f}'.format(elapse_time_win),
end='\r',
flush=True)
result = timestamp_correct(DataStream(None, None, data=data),
sampling_frequency=self.sample_rate)
def RIPDataQuality(datastream: DataStream,
windowsize: float = 5.0,
bufferLength=5,
acceptableOutlierPercent=50,
outlierThresholdHigh=4500,
outlierThresholdLow=20,
badSegmentThreshod=2,
ripBandOffThreshold=20,
ripBandLooseThreshold=150) -> SpanStream:
"""
:param datastream:
:param windowsize:
:param bufferLength:
:param acceptableOutlierPercent:
:param outlierThresholdHigh:
:param outlierThresholdLow:
:param badSegmentThreshod:
:param ripBandOffThreshold:
:param ripBandLooseThreshold:
:return:
"""
# windows = window(datastream.datapoints, window_size=windowsize)
# TODO: Do something with windows here
result = DataStream.from_datastream(input_streams=[datastream])
# # Do something here for data quality
# ripQuality = []
# for i in range(1, 10):
# ripQuality.append(Span(result.getID(),
# starttime=datetime.now(),
# endtime=datetime.now(),
# label=DataQuality.GOOD))
#
# result.set_spans(ripQuality)
return result
def autosense_sequence_align(datastreams: List[DataStream],
sampling_frequency: float) -> DataStream:
result = DataStream.from_datastream(input_streams=datastreams)
result.data = []
if len(datastreams) == 0:
return result
start_time = None
#print (len(datastreams))
for ds in datastreams:
ts = ds.data[0].start_time
#print (ts)
if not start_time:
start_time = ts
elif start_time < ts:
start_time = ts
print(start_time)
start_time -= datetime.timedelta(seconds=1.0 / sampling_frequency)
print(start_time)
data_block = []
max_index = np.Inf
for ds in datastreams:
d = [i for i in ds.data if i.start_time > start_time]
if len(d) < max_index:
max_index = len(d)
data_block.append(d)
print(max_index)
data_array = np.array(data_block)
dimensions = data_array.shape[0]
for i in range(0, max_index):
sample = [data_array[d][i].sample for d in range(0, dimensions)]
result.data.append(
DataPoint.from_tuple(data_array[0][i].start_time, sample))
return result
def generate_cStress_feature_vector(ecg_features: List[DataStream]):
"""
:param ecg_features: DataStream
:return feature vector DataStream
"""
final_feature_vector = []
for i in range(len(ecg_features[0].data)):
feature_vector = []
for ef in ecg_features:
if ef.data[i].sample is None:
continue
feature_vector.append(ef.data[i].sample)
final_feature_vector.append(
DataPoint.from_tuple(start_time=ef.data[i].start_time,
end_time=ef.data[i].end_time,
sample=feature_vector))
feature_vector_ds = DataStream.from_datastream(ecg_features)
feature_vector_ds.data = final_feature_vector
return feature_vector_ds
def magnitude(datastream: DataStream) -> DataStream:
"""
:param datastream:
:return:
"""
input_data = np.array([i.sample for i in datastream.datapoints])
data = norm(input_data, axis=1).tolist(
) # TODO: Fix function to not compute normalized magnitudes
result_data = [
DataPoint.from_tuple(start_time=i.start_time, sample=None)
for i in datastream.datapoints
]
for i, dp in enumerate(result_data):
dp.sample = data[i]
result = DataStream.from_datastream(input_streams=[datastream])
result.datapoints = result_data
return result
def map_datapoint_and_metadata_to_datastream(self, stream_id: int,
data: list) -> DataStream:
"""
This method will map the datapoint and metadata to datastream object
:param stream_id:
:param data: list
:return: datastream object
"""
# query datastream(mysql) for metadata
datastream_info = Metadata(self.CC_obj).get_stream_info(stream_id)
ownerID = datastream_info[0]["owner"]
name = datastream_info[0]["name"]
data_descriptor = json.loads(datastream_info[0]["data_descriptor"])
execution_context = json.loads(datastream_info[0]["execution_context"])
annotations = json.loads(datastream_info[0]["annotations"])
stream_type = datastream_info[0]["type"]
start_time = datastream_info[0]["start_time"]
end_time = datastream_info[0]["end_time"]
return DataStream(stream_id, ownerID, name, data_descriptor,
execution_context, annotations, stream_type,
start_time, end_time, data)
def getBreathRate(datastream: DataStream, window_size: float,
window_offset: float):
"""
Computes breath rate
:param datastream: DataStream
:return: breathing rate per minute datastream
"""
window_data = window_sliding(datastream.data, window_size, window_offset)
breath_rates = []
for key, value in window_data.items():
starttime, endtime = key
totalCycle = len(value)
breath_rates.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=totalCycle))
datastream_breath_rates = DataStream.from_datastream([datastream])
datastream_breath_rates.data = breath_rates
return datastream_breath_rates
def compute_datastream_rsa(valleys_datastream: DataStream,
rr_datastream: DataStream):
"""
:param valleys_datastream:
:param rr_datastream:
:return: rsa_datastream
"""
rsa_ds = DataStream.from_datastream([valleys_datastream])
rsa_datapoints = []
for index, value in enumerate(valleys_datastream.data[:-2]):
rsa_sample = rsa_calculate(valleys_datastream.data[index].start_time,
valleys_datastream.data[index + 2].start_time,
rr_datastream)
if rsa_sample != -1.0:
rsa_datapoints.append(DataPoint.from_tuple(start_time=valleys_datastream.data[index].start_time,
end_time=valleys_datastream.data[index + 2].start_time,
sample=rsa_sample))
if not rsa_datapoints:
print('Error computing RSA!')
rsa_ds.data = rsa_datapoints
return rsa_ds
def rip_feature_computation(
peaks_datastream: DataStream,
valleys_datastream: DataStream) -> Tuple[DataStream]:
"""
Respiration Feature Implementation. The respiration feature values are
derived from the following paper:
'puffMarker: a multi-sensor approach for pinpointing the timing of first lapse in smoking cessation'
Removed due to lack of current use in the implementation
roc_max = [] # 8. ROC_MAX = max(sample[j]-sample[j-1])
roc_min = [] # 9. ROC_MIN = min(sample[j]-sample[j-1])
:param peaks_datastream: DataStream
:param valleys_datastream: DataStream
:return: RIP Feature DataStreams
"""
# TODO: This needs fixed to prevent crashing the execution pipeline
if peaks_datastream is None or valleys_datastream is None:
return None
# TODO: This needs fixed to prevent crashing the execution pipeline
if len(peaks_datastream.data) == 0 or len(valleys_datastream.data) == 0:
return None
inspiration_duration = [] # 1 Inhalation duration
expiration_duration = [] # 2 Exhalation duration
respiration_duration = [] # 3 Respiration duration
inspiration_expiration_ratio = [] # 4 Inhalation and Exhalation ratio
stretch = [] # 5 Stretch
upper_stretch = [] # 6. Upper portion of the stretch calculation
lower_stretch = [] # 7. Lower portion of the stretch calculation
delta_previous_inspiration_duration = [] # 10. BD_INSP = INSP(i)-INSP(i-1)
delta_previous_expiration_duration = [] # 11. BD_EXPR = EXPR(i)-EXPR(i-1)
delta_previous_respiration_duration = [] # 12. BD_RESP = RESP(i)-RESP(i-1)
delta_previous_stretch_duration = [
] # 14. BD_Stretch= Stretch(i)-Stretch(i-1)
delta_next_inspiration_duration = [] # 19. FD_INSP = INSP(i)-INSP(i+1)
delta_next_expiration_duration = [] # 20. FD_EXPR = EXPR(i)-EXPR(i+1)
delta_next_respiration_duration = [] # 21. FD_RESP = RESP(i)-RESP(i+1)
delta_next_stretch_duration = [] # 23. FD_Stretch= Stretch(i)-Stretch(i+1)
neighbor_ratio_expiration_duration = [
] # 29. D5_EXPR(i) = EXPR(i) / avg(EXPR(i-2)...EXPR(i+2))
neighbor_ratio_stretch_duration = [
] # 32. D5_Stretch = Stretch(i) / avg(Stretch(i-2)...Stretch(i+2))
valleys = valleys_datastream.data
peaks = peaks_datastream.data[:-1]
for i, peak in enumerate(peaks):
valley_start_time = valleys[i].start_time
delta = peak.start_time - valleys[i].start_time
inspiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta.total_seconds()))
delta = valleys[i + 1].start_time - peak.start_time
expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta.total_seconds()))
delta = valleys[i + 1].start_time - valley_start_time
respiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta.total_seconds()))
ratio = (peak.start_time - valley_start_time) / (
valleys[i + 1].start_time - peak.start_time)
inspiration_expiration_ratio.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=ratio))
value = peak.sample - valleys[i + 1].sample
stretch.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=value))
# upper_stretch.append(DataPoint.from_tuple(start_time=valley_start_time, sample=(peak.sample - valleys[i + 1][1]) / 2)) # TODO: Fix this by adding a tracking moving average and compute upper stretch from this to the peak and lower from this to the valley
# lower_stretch.append(DataPoint.from_tuple(start_time=valley_start_time, sample=(-peak.sample + valleys[i + 1][1]) / 2)) # TODO: Fix this by adding a tracking moving average and compute upper stretch from this to the peak and lower from this to the valley
for i in range(len(inspiration_duration)):
valley_start_time = valleys[i].start_time
if i == 0: # Edge case
delta_previous_inspiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_previous_expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_previous_respiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_previous_stretch_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
else:
delta = inspiration_duration[i].sample - inspiration_duration[
i - 1].sample
delta_previous_inspiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = expiration_duration[i].sample - expiration_duration[
i - 1].sample
delta_previous_expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = respiration_duration[i].sample - respiration_duration[
i - 1].sample
delta_previous_respiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = stretch[i].sample - stretch[i - 1].sample
delta_previous_stretch_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
if i == len(inspiration_duration) - 1:
delta_next_inspiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_next_expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_next_respiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_next_stretch_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
else:
delta = inspiration_duration[i].sample - inspiration_duration[
i + 1].sample
delta_next_inspiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = expiration_duration[i].sample - expiration_duration[
i + 1].sample
delta_next_expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = respiration_duration[i].sample - respiration_duration[
i + 1].sample
delta_next_respiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = stretch[i].sample - stretch[i + 1].sample
delta_next_stretch_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
stretch_average = 0
expiration_average = 0
count = 0.0
for j in [-2, -1, 1, 2]:
if i + j < 0 or i + j >= len(inspiration_duration):
continue
stretch_average += stretch[i + j].sample
expiration_average += expiration_duration[i + j].sample
count += 1
stretch_average /= count
expiration_average /= count
ratio = stretch[i].sample / stretch_average
neighbor_ratio_stretch_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=ratio))
ratio = expiration_duration[i].sample / expiration_average
neighbor_ratio_expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=ratio))
# Begin assembling datastream for output
inspiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
inspiration_duration_datastream.data = inspiration_duration
expiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
expiration_duration_datastream.data = expiration_duration
respiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
respiration_duration_datastream.data = respiration_duration
inspiration_expiration_ratio_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
inspiration_expiration_ratio_datastream.data = inspiration_expiration_ratio
stretch_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
stretch_datastream.data = stretch
upper_stretch_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
upper_stretch_datastream.data = upper_stretch
lower_stretch_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
lower_stretch_datastream.data = lower_stretch
delta_previous_inspiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_previous_inspiration_duration_datastream.data = delta_previous_inspiration_duration
delta_previous_expiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_previous_expiration_duration_datastream.data = delta_previous_expiration_duration
delta_previous_respiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_previous_respiration_duration_datastream.data = delta_previous_respiration_duration
delta_previous_stretch_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_previous_stretch_duration_datastream.data = delta_previous_stretch_duration
delta_next_inspiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_next_inspiration_duration_datastream.data = delta_next_inspiration_duration
delta_next_expiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_next_expiration_duration_datastream.data = delta_next_expiration_duration
delta_next_respiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_next_respiration_duration_datastream.data = delta_next_respiration_duration
delta_next_stretch_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_next_stretch_duration_datastream.data = delta_next_stretch_duration
neighbor_ratio_expiration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
neighbor_ratio_expiration_datastream.data = neighbor_ratio_expiration_duration
neighbor_ratio_stretch_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
neighbor_ratio_stretch_datastream.data = neighbor_ratio_stretch_duration
return inspiration_duration_datastream, \
expiration_duration_datastream, \
respiration_duration_datastream, \
inspiration_expiration_ratio_datastream, \
stretch_datastream, \
upper_stretch_datastream, \
lower_stretch_datastream, \
delta_previous_inspiration_duration_datastream, \
delta_previous_expiration_duration_datastream, \
delta_previous_respiration_duration_datastream, \
delta_previous_stretch_duration_datastream, \
delta_next_inspiration_duration_datastream, \
delta_next_expiration_duration_datastream, \
delta_next_respiration_duration_datastream, \
delta_next_stretch_duration_datastream, \
neighbor_ratio_expiration_datastream, \
neighbor_ratio_stretch_datastream
def timestamp_correct(
datastream: DataStream,
sampling_frequency: float,
min_available_gaps: int = 3600, # TODO: Does this matter anymore?
min_split_gap: datetime.timedelta = datetime.timedelta(seconds=30),
max_data_points_per_segment: int = 100000000) -> DataStream:
try:
result = DataStream.from_datastream([datastream])
result.data = []
if len(datastream.data) == 0:
return result
data = datastream.data
time_deltas = np.diff([dp.start_time for dp in data])
gap_points = [data[0]]
for index, value in enumerate(time_deltas):
if value > min_split_gap:
gap_points.append(data[index])
gap_points.append(data[-1])
segments = []
segment_data = []
gap_index = 0
low_time = gap_points[gap_index].start_time
high_time = gap_points[gap_index + 1].start_time
for dp in data:
if len(segment_data) >= max_data_points_per_segment:
segments.append(
interpolate_gaps(segment_data, sampling_frequency))
segment_data = []
if low_time <= dp.start_time <= high_time:
segment_data.append(dp)
else:
segments.append(
interpolate_gaps(segment_data, sampling_frequency))
gap_index += 1
low_time = gap_points[gap_index].start_time
high_time = gap_points[gap_index + 1].start_time
segment_data = []
segments.append(interpolate_gaps(segment_data, sampling_frequency))
for s in segments:
begin_time = s[0].start_time.timestamp()
end_time = s[-1].start_time.timestamp()
x = np.array([
i
for i in frange(begin_time, end_time, 1.0 / sampling_frequency)
],
dtype='float')
y = np.array([dp.start_time.timestamp() for dp in s],
dtype='float')
distance, path = fastdtw(x, y, radius=1)
xx = [0 for i in y]
for si, ei in path:
xx[ei] = x[si]
dtw_corrected_data = []
for index, dp in enumerate(s):
ts = datetime.datetime.fromtimestamp(xx[index],
tz=dp.start_time.tzinfo)
dtw_corrected_data.append(DataPoint.from_tuple(ts, dp.sample))
result.data.extend(dtw_corrected_data)
return result
except:
return []
def timestamp_correct(datastream: DataStream,
sampling_frequency: float = None) -> DataStream:
result = DataStream.from_datastream(input_streams=[datastream])
result.datapoints = datastream.datapoints
return result
def detect_rpeak(ecg: DataStream,
fs: float = 64,
threshold: float = 0.5) -> DataStream:
"""
This program implements the Pan Tomkins algorithm on ECG signal to detect the R peaks
Since the ecg array can have discontinuity in the timestamp arrays the rr-interval calculated
in the algorithm is calculated in terms of the index in the sample array
The algorithm consists of some major steps
1. computation of the moving window integration of the signal in terms of blackman window of a prescribed length
2. compute all the peaks of the moving window integration signal
3. adaptive thresholding with dynamic signal and noise thresholds applied to filter out the R peak locations
4. confirm the R peaks through differentiation from the nearby peaks and remove the false peaks
:param ecg: ecg array of tuples (timestamp,value)
:param fs: sampling frequency
:param threshold: initial threshold to detect the R peak in a signal normalized by the 90th percentile. .5 is default.
:return: R peak array of tuples (timestamp, Rpeak interval)
"""
data = ecg.datapoints
sample = np.array([i.sample for i in data])
timestamp = np.array([i.start_time for i in data])
# computes the moving window integration of the signal
blackman_win_len = np.ceil(fs /
5) # TODO: CODE_REVIEW: Hard coded value of 5
y = compute_moving_window_int(sample, fs, blackman_win_len)
# TODO: CODE_REVIEW: Hard coded +/- 2. Is this a function of the blackman_win_len?
peak_location = [
i for i in range(2,
len(y) - 1) if check_peak(y[i - 2:i + 2])
]
peak_location_values = y[peak_location]
# initial RR interval average
running_rr_avg = sum(np.diff(peak_location)) / (len(peak_location) - 1)
rpeak_temp1 = compute_r_peaks(threshold, running_rr_avg, y,
peak_location_values, peak_location)
rpeak_temp1 = remove_close_peaks(rpeak_temp1, sample, fs)
index = confirm_peaks(rpeak_temp1, sample, fs)
rpeak_timestamp = timestamp[index]
rpeak_value = np.diff(rpeak_timestamp)
rpeak_timestamp = rpeak_timestamp[1:]
result_data = []
for k in range(len(rpeak_value)):
result_data.append(
DataPoint.from_tuple(
rpeak_timestamp[k],
rpeak_value[k].seconds + rpeak_value[k].microseconds / 1e6))
# Create resulting datastream to be returned
result = DataStream.from_datastream([ecg])
result.datapoints = result_data
return result
def store(data: OrderedDict, input_streams: dict, output_streams: dict,
CC_obj: CerebralCortex):
"""
Store diagnostic results with its metadata in the data-store
:param input_streams:
:param data:
:param CC_obj:
:param config:
:param algo_type:
"""
if data:
#basic output stream info
owner = input_streams[0]["owner_id"]
dd_stream_id = output_streams["id"]
dd_stream_name = output_streams["name"]
stream_type = "ds"
data_descriptor = [{
"NAME":
"Data Quality (LED)",
"DATA_TYPE":
"int",
"FREQUENCY":
"0.33",
"MAX_VALUE":
"4",
"MIN_VALUE":
"0",
"DESCRIPTION":
"measures the Data Quality of LED. Values= GOOD(0), BAND_OFF(1), NOT_WORN(2), BAND_LOOSE(3), NOISE(4)"
}]
execution_context = {
"platform_metadata": {
"NAME": "MotionSense HRV",
"DEVICE_ID": ""
},
"processing_module": {
"name":
"",
"environment":
"cerebralcortex",
"algorithm": [{
"method": "",
"authors": ["Nasir Ali", " Md Azim Ullah"],
"version": "0.0.1",
"reference": {
"url": "http://md2k.org/"
},
"description": ""
}],
"description":
"",
"input_streams":
input_streams,
"output_streams":
output_streams,
"input_parameters": {}
},
"datasource_metadata": {
"NAME":
"Data Quality (LED)",
"DATA_TYPE":
"org.md2k.datakitapi.datatype.DataTypeInt",
"FREQUENCY":
"0.33",
"DESCRIPTION":
"measures the Data Quality of LED. Values= GOOD(0), BAND_OFF(1), NOT_WORN(2), BAND_LOOSE(3), NOISE(4)"
},
"application_metadata": {
"NAME": "MotionSense",
"DESCRIPTION":
"Collects data from the motion sense. Sensors supported: [Accelerometer, Gyroscope, Battery, LED, DataQuality]",
"VERSION_NAME": "0.0.1",
"VERSION_NUMBER": "2000500"
}
}
annotations = []
ds = DataStream(identifier=dd_stream_id,
owner=owner,
name=dd_stream_name,
data_descriptor=data_descriptor,
execution_context=execution_context,
annotations=annotations,
stream_type=stream_type,
data=data)
CC_obj.save_datastream(ds, "datastream")
def test_interpolate_gaps(self):
ds = DataStream(None, None)
ds.datapoints = self.ecg
result = interpolate_gaps(ds.datapoints, self.sample_rate)
def detect_rpeak(ecg: DataStream, fs: float = 64, threshold: float = 0.5, blackman_win_len_range: float = 1 / 5): """ This program implements the Pan Tomkins algorithm on ECG signal to detect the R peaks Since the ecg array can have discontinuity in the timestamp arrays the rr-interval calculated in the algorithm is calculated in terms of the index in the sample array The algorithm consists of some major steps 1. computation of the moving window integration of the signal in terms of blackman window of a prescribed length 2. compute all the peaks of the moving window integration signal 3. adaptive thresholding with dynamic signal and noise thresholds applied to filter out the R peak locations 4. confirm the R peaks through differentiation from the nearby peaks and remove the false peaks :param ecg: ecg array of tuples (timestamp,value) :param fs: sampling frequency :param threshold: initial threshold to detect the R peak in a signal normalized by the 90th percentile. .5 is default. :param blackman_win_len_range : the range to calculate blackman window length :return: R peak array of tuples (timestamp, Rpeak interval) """ try: data = ecg.data sample = np.array([i.sample for i in data]) timestamp = np.array([i.start_time for i in data]) # computes the moving window integration of the signal blackman_win_len = np.ceil(fs * blackman_win_len_range) y = compute_moving_window_int(sample, fs, blackman_win_len) peak_location_values = [(i, y[i]) for i in range(2, len(y) - 1) if check_peak(y[i - 2:i + 3])] # initial RR interval average peak_location = [i[0] for i in peak_location_values] running_rr_avg = sum(np.diff(peak_location)) / (len(peak_location) - 1) rpeak_temp1 = compute_r_peaks(threshold, running_rr_avg, y, peak_location_values) rpeak_temp2 = remove_close_peaks(rpeak_temp1, sample, fs) index = confirm_peaks(rpeak_temp2, sample, fs) rpeak_timestamp = timestamp[index] rpeak_samples = sample[index] #--------------------------- # rpeak values: result_rpeaks = [] for k in range(len(rpeak_samples)): result_rpeaks.append( DataPoint.from_tuple(rpeak_timestamp[k], rpeak_samples[k])) #--------------------------- # rr_interval values rpeak_value = np.diff(rpeak_timestamp) rpeak_timestamp = rpeak_timestamp[1:] result_interval = [] for k in range(len(rpeak_value)): result_interval.append( DataPoint.from_tuple(rpeak_timestamp[k], rpeak_value[k].seconds + rpeak_value[k].microseconds / 1e6)) # Create resulting datastream to be returned # rpeaks result_rpeaks_ds = DataStream.from_datastream([ecg]) result_rpeaks_ds.data = result_rpeaks # rr_interval result_interval_ds = DataStream.from_datastream([ecg]) result_interval_ds.data = result_interval return result_interval_ds, result_rpeaks_ds except: # Create resulting datastream to be returned # rpeaks result_rpeaks_ds = DataStream.from_datastream([ecg]) result_rpeaks_ds.data = [] # rr_interval result_interval_ds = DataStream.from_datastream([ecg]) result_interval_ds.data = [] return result_interval_ds, result_rpeaks_ds
master="local[*]",
name="Memphis cStress Development App")
for i in range(1, 2):
participant = "SI%02d" % i
participant_UUID = uuid.uuid4()
print(participant, participant_UUID)
try:
ecgRDD = CC.readfile(
find(basedir, {
"participant": participant,
"datasource": "ecg"
})).map(parser.data_processor).filter(
lambda x: isinstance(x, DataPoint))
ecg = DataStream(CC, participant_UUID, data=ecgRDD.collect())
ripRDD = CC.readfile(
find(basedir, {
"participant": participant,
"datasource": "rip"
})).map(parser.data_processor).filter(
lambda x: isinstance(x, DataPoint))
rip = DataStream(CC, participant_UUID, data=ripRDD.collect())
accelxRDD = CC.readfile(
find(basedir, {
"participant": participant,
"datasource": "accelx"
})).map(parser.data_processor).filter(
lambda x: isinstance(x, DataPoint))
def ecg_feature_computation(datastream: DataStream,
window_size: float,
window_offset: float,
low_frequency: float = 0.01,
high_frequency: float = 0.7,
low_rate_vlf: float = 0.0009,
high_rate_vlf: float = 0.04,
low_rate_hf: float = 0.15,
high_rate_hf: float = 0.4,
low_rate_lf: float = 0.04,
high_rate_lf: float = 0.15):
"""
ECG Feature Implementation. The frequency ranges for High, Low and Very low heart rate variability values are
derived from the following paper:
'Heart rate variability: standards of measurement, physiological interpretation and clinical use'
:param high_rate_lf: float
:param low_rate_lf: float
:param high_rate_hf: float
:param low_rate_hf: float
:param high_rate_vlf: float
:param low_rate_vlf: float
:param high_frequency: float
:param low_frequency: float
:param datastream: DataStream
:param window_size: float
:param window_offset: float
:return: ECG Feature DataStreams
"""
if datastream is None:
return None
if len(datastream.data) == 0:
return None
# perform windowing of datastream
window_data = window_sliding(datastream.data, window_size, window_offset)
# initialize each ecg feature array
rr_variance_data = []
rr_mean_data = []
rr_median_data = []
rr_80percentile_data = []
rr_20percentile_data = []
rr_quartile_deviation_data = []
rr_HF_data = []
rr_LF_data = []
rr_VLF_data = []
rr_LF_HF_data = []
rr_heart_rate_data = []
# iterate over each window and calculate features
for key, value in window_data.items():
starttime, endtime = key
reference_data = np.array([i.sample for i in value])
rr_variance_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=np.var(reference_data)))
power, frequency = lomb(data=value,
low_frequency=low_frequency,
high_frequency=high_frequency)
rr_VLF_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=heart_rate_power(
power, frequency, low_rate_vlf,
high_rate_vlf)))
rr_HF_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=heart_rate_power(
power, frequency, low_rate_hf,
high_rate_hf)))
rr_LF_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=heart_rate_power(
power, frequency, low_rate_lf,
high_rate_lf)))
if heart_rate_power(power, frequency, low_rate_hf, high_rate_hf) != 0:
lf_hf = float(
heart_rate_power(power, frequency, low_rate_lf, high_rate_lf) /
heart_rate_power(power, frequency, low_rate_hf, high_rate_hf))
rr_LF_HF_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=lf_hf))
else:
rr_LF_HF_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=0))
rr_mean_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=np.mean(reference_data)))
rr_median_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=np.median(reference_data)))
rr_quartile_deviation_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=(0.5 *
(np.percentile(reference_data, 75) -
np.percentile(reference_data, 25)))))
rr_80percentile_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=np.percentile(reference_data, 80)))
rr_20percentile_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=np.percentile(reference_data, 20)))
rr_heart_rate_data.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=np.median(60 / reference_data)))
rr_variance = DataStream.from_datastream([datastream])
rr_variance.data = rr_variance_data
rr_vlf = DataStream.from_datastream([datastream])
rr_vlf.data = rr_VLF_data
rr_hf = DataStream.from_datastream([datastream])
rr_hf.data = rr_HF_data
rr_lf = DataStream.from_datastream([datastream])
rr_lf.data = rr_LF_data
rr_lf_hf = DataStream.from_datastream([datastream])
rr_lf_hf.data = rr_LF_HF_data
rr_mean = DataStream.from_datastream([datastream])
rr_mean.data = rr_mean_data
rr_median = DataStream.from_datastream([datastream])
rr_median.data = rr_median_data
rr_quartile = DataStream.from_datastream([datastream])
rr_quartile.data = rr_quartile_deviation_data
rr_80 = DataStream.from_datastream([datastream])
rr_80.data = rr_80percentile_data
rr_20 = DataStream.from_datastream([datastream])
rr_20.data = rr_20percentile_data
rr_heart_rate = DataStream.from_datastream([datastream])
rr_heart_rate.data = rr_heart_rate_data
return rr_variance, rr_vlf, rr_hf, rr_lf, rr_lf_hf, rr_mean, rr_median, rr_quartile, rr_80, rr_20, rr_heart_rate
def loader(identifier: int):
participant = "SI%02d" % identifier
participant_uuid = uuid.uuid4()
try:
ecg = DataStream(None, participant_uuid)
ecg.data = readfile(
find(basedir, {
"participant": participant,
"datasource": "ecg"
}))
rip = DataStream(None, participant_uuid)
rip.data = readfile(
find(basedir, {
"participant": participant,
"datasource": "rip"
}))
accelx = DataStream(None, participant_uuid)
accelx.data = readfile(
find(basedir, {
"participant": participant,
"datasource": "accelx"
}))
accely = DataStream(None, participant_uuid)
accely.data = readfile(
find(basedir, {
"participant": participant,
"datasource": "accely"
}))
accelz = DataStream(None, participant_uuid)
accelz.data = readfile(
find(basedir, {
"participant": participant,
"datasource": "accelz"
}))
stress_marks = DataStream(None, participant_uuid)
stress_marks.data = readfile_ground_truth(
find(basedir, {
"participant": participant,
"datasource": "stress_marks"
}))
return {
"participant": participant,
"ecg": ecg,
"rip": rip,
"accelx": accelx,
"accely": accely,
"accelz": accelz,
"stress_marks": stress_marks
}
except Exception as e:
print("File missing for %s" % participant)
return {"ERROR": 'missing data file'}
def test_split_by_gaps(self):
ds = DataStream(None, None)
# ds.datapoints = self.ecg
ds.datapoints = self.ecg[:500]
result = split_by_gaps(ds)
def setUp(self):
self.size = 100000
self.ds = DataStream(None, None)
data = [DataPoint.from_tuple(datetime.datetime.now(), [random()]) for i in range(0, self.size)]
self.ds.data = data
def compute_peak_valley(
rip: DataStream,
fs: float = 21.33,
smoothing_factor: int = 5,
time_window: int = 8,
expiration_amplitude_threshold_perc: float = 0.10,
threshold_expiration_duration: float = 0.312,
inspiration_amplitude_threshold_perc: float = 0.10,
max_amplitude_change_peak_correction: float = 30,
min_neg_slope_count_peak_correction: int = 4,
minimum_peak_to_valley_time_diff=0.31) -> [DataStream, DataStream]:
"""
Compute peak and valley from rip data and filter peak and valley.
:param minimum_peak_to_valley_time_diff:
:param inspiration_amplitude_threshold_perc:
:param smoothing_factor:
:return peak_datastream, valley_datastream:
:param rip:
:param fs:
:param time_window:
:param expiration_amplitude_threshold_perc:
:param threshold_expiration_duration:
:param max_amplitude_change_peak_correction:
:param min_neg_slope_count_peak_correction:
"""
data_smooth = smooth(data=rip.data, span=smoothing_factor)
window_length = int(round(time_window * fs))
data_mac = moving_average_curve(data_smooth, window_length=window_length)
data_smooth_start_time_to_index = {}
for index, data in enumerate(data_smooth):
data_smooth_start_time_to_index[data.start_time] = index
up_intercepts, down_intercepts = up_down_intercepts(
data=data_smooth,
mac=data_mac,
data_start_time_to_index=data_smooth_start_time_to_index)
up_intercepts_filtered, down_intercepts_filtered = filter_intercept_outlier(
up_intercepts=up_intercepts, down_intercepts=down_intercepts)
peaks, valleys = generate_peak_valley(
up_intercepts=up_intercepts_filtered,
down_intercepts=down_intercepts_filtered,
data=data_smooth)
valleys_corrected = correct_valley_position(
peaks=peaks,
valleys=valleys,
up_intercepts=up_intercepts_filtered,
data=data_smooth,
data_start_time_to_index=data_smooth_start_time_to_index)
peaks_corrected = correct_peak_position(
peaks=peaks,
valleys=valleys_corrected,
up_intercepts=up_intercepts_filtered,
data=data_smooth,
max_amplitude_change_peak_correction=
max_amplitude_change_peak_correction,
min_neg_slope_count_peak_correction=min_neg_slope_count_peak_correction,
data_start_time_to_index=data_smooth_start_time_to_index)
# remove too close valley peak pair.
peaks_filtered_close, valleys_filtered_close = remove_close_valley_peak_pair(
peaks=peaks_corrected,
valleys=valleys_corrected,
minimum_peak_to_valley_time_diff=minimum_peak_to_valley_time_diff)
# Remove small Expiration duration < 0.31
peaks_filtered_exp_dur, valleys_filtered_exp_dur = filter_expiration_duration_outlier(
peaks=peaks_filtered_close,
valleys=valleys_filtered_close,
threshold_expiration_duration=threshold_expiration_duration)
# filter out peak valley pair of inspiration of small amplitude.
peaks_filtered_insp_amp, valleys_filtered_insp_amp = filter_small_amp_inspiration_peak_valley(
peaks=peaks_filtered_exp_dur,
valleys=valleys_filtered_exp_dur,
inspiration_amplitude_threshold_perc=
inspiration_amplitude_threshold_perc)
# filter out peak valley pair of expiration of small amplitude.
peaks_filtered_exp_amp, valleys_filtered_exp_amp = filter_small_amp_expiration_peak_valley(
peaks=peaks_filtered_insp_amp,
valleys=valleys_filtered_insp_amp,
expiration_amplitude_threshold_perc=expiration_amplitude_threshold_perc
)
peak_datastream = DataStream.from_datastream([rip])
peak_datastream.data = peaks_filtered_exp_amp
valley_datastream = DataStream.from_datastream([rip])
valley_datastream.data = valleys_filtered_exp_amp
return peak_datastream, valley_datastream
def get_stream(self, identifier: uuid) -> DataStream:
return DataStream(identifier, data=[])
def test_07_stream_filter(self):
identifier_anno = "6db98dfb-d6e8-4b27-8d55-95b20fa0f750"
identifier_data = "6db98dfb-d6e8-4b27-8d55-95b20fa0f751"
owner_id = "06634264-56bc-4c92-abd7-377dbbad79dd"
name_anno = "data-store-test-annotation"
name_data = "data-store-test-data"
data_descriptor = {}
execution_context_anno = json.loads(
'{"execution_context": {"algorithm": {"method": "test.data_store.annotation.filter"}}}'
)
execution_context_data = json.loads(
'{"execution_context": {"algorithm": {"method": "test.data_store.data.filter"}}}'
)
annotations_data = json.loads(
'[{"name": "test-case","identifier": "6db98dfb-d6e8-4b27-8d55-95b20fa0f750"}]'
)
annotations_anno = {}
datapoints_anno = []
datapoints_data = []
result_data = Metadata(self.CC).is_id_created(owner_id, name_data,
execution_context_data)
if result_data["status"] != "new":
identifier_data = result_data["id"]
Metadata(self.CC).store_stream_info(
identifier_anno, owner_id, name_anno, data_descriptor,
execution_context_anno, annotations_anno, "annotations",
datetime.datetime(2017, 4, 24, 0, 0, 1),
datetime.datetime(2017, 4, 24, 0, 0, 5), result_data["status"])
result_anno = Metadata(self.CC).is_id_created(owner_id, name_data,
execution_context_data)
if result_anno["status"] != "new":
identifier_anno = result_anno["id"]
Metadata(self.CC).store_stream_info(
identifier_data, owner_id, name_data, data_descriptor,
execution_context_data, annotations_data, "datastream",
datetime.datetime(2017, 4, 24, 0, 0, 1),
datetime.datetime(2017, 4, 24, 0, 0, 5), result_anno["status"])
for i in range(0, 5):
if (i % 2 == 0):
sample_anno = 'good'
else:
sample_anno = 'bad'
sample_data = i, i + 2, i + 3
start_time_anno = datetime.datetime(2017, 4, 24, 0, 0, i)
end_time_anno = datetime.datetime(2017, 4, 24, 0, 0, (5 + i))
start_time_data = datetime.datetime(2017, 4, 24, 0, 0, i)
end_time_data = datetime.datetime(2017, 4, 24, 0, 0, (3 + i))
localtz = timezone('US/Central')
start_time_anno = localtz.localize(start_time_anno)
end_time_anno = localtz.localize(end_time_anno)
start_time_data = localtz.localize(start_time_data)
end_time_data = localtz.localize(end_time_data)
datapoints_anno.append(
DataPoint(start_time=start_time_anno,
end_time=end_time_anno,
sample=sample_anno))
datapoints_data.append(
DataPoint(start_time=start_time_data,
end_time=end_time_data,
sample=sample_data))
ds_anno = DataStream(uuid.UUID(identifier_anno), owner_id, name_anno,
data_descriptor, execution_context_anno,
annotations_data, "annotations", start_time_anno,
end_time_anno, datapoints_anno)
ds_data = DataStream(uuid.UUID(identifier_data), owner_id, name_data,
data_descriptor, execution_context_data,
annotations_anno, "datastream", start_time_anno,
end_time_anno, datapoints_data)
self.CC.save_datastream(ds_anno)
self.CC.save_datastream(ds_data)
filted_stream = self.CC.filter_stream(identifier_data, "test-case",
"good")
self.assertEqual(len(filted_stream), 5)
for i in range(0, 5):
sample_data = [i, i + 2, i + 3]
start_time_data = datetime.datetime(2017, 4, 24, 0, 0, i)
end_time_data = datetime.datetime(2017, 4, 24, 0, 0, (3 + i))
start_time_data = localtz.localize(start_time_data)
end_time_data = localtz.localize(end_time_data)
self.assertEqual(filted_stream[i].start_time, start_time_data)
self.assertEqual(filted_stream[i].end_time, end_time_data)
self.assertEqual(filted_stream[i].sample, sample_data)
def rip_feature_computation(peaks_datastream: DataStream,
valleys_datastream: DataStream,
rr_intervals_datastream: DataStream,
window_size: float,
window_offset: float) -> Tuple[DataStream]:
"""
Respiration Feature Implementation. The respiration feature values are
derived from the following paper:
'puffMarker: a multi-sensor approach for pinpointing the timing of first lapse in smoking cessation'
Removed due to lack of current use in the implementation
roc_max = [] # 8. ROC_MAX = max(sample[j]-sample[j-1])
roc_min = [] # 9. ROC_MIN = min(sample[j]-sample[j-1])
:param peaks_datastream: DataStream
:param valleys_datastream: DataStream
:return: RIP Feature DataStreams
"""
# TODO: This needs fixed to prevent crashing the execution pipeline
if peaks_datastream is None or valleys_datastream is None:
return None
# TODO: This needs fixed to prevent crashing the execution pipeline
if len(peaks_datastream.data) == 0 or len(valleys_datastream.data) == 0:
return None
inspiration_duration = [] # 1 Inhalation duration
expiration_duration = [] # 2 Exhalation duration
respiration_duration = [] # 3 Respiration duration
inspiration_expiration_ratio = [] # 4 Inhalation and Exhalation ratio
stretch = [] # 5 Stretch
upper_stretch = [] # 6. Upper portion of the stretch calculation
lower_stretch = [] # 7. Lower portion of the stretch calculation
delta_previous_inspiration_duration = [] # 10. BD_INSP = INSP(i)-INSP(i-1)
delta_previous_expiration_duration = [] # 11. BD_EXPR = EXPR(i)-EXPR(i-1)
delta_previous_respiration_duration = [] # 12. BD_RESP = RESP(i)-RESP(i-1)
delta_previous_stretch_duration = [
] # 14. BD_Stretch= Stretch(i)-Stretch(i-1)
delta_next_inspiration_duration = [] # 19. FD_INSP = INSP(i)-INSP(i+1)
delta_next_expiration_duration = [] # 20. FD_EXPR = EXPR(i)-EXPR(i+1)
delta_next_respiration_duration = [] # 21. FD_RESP = RESP(i)-RESP(i+1)
delta_next_stretch_duration = [] # 23. FD_Stretch= Stretch(i)-Stretch(i+1)
neighbor_ratio_expiration_duration = [
] # 29. D5_EXPR(i) = EXPR(i) / avg(EXPR(i-2)...EXPR(i+2))
neighbor_ratio_stretch_duration = [
] # 32. D5_Stretch = Stretch(i) / avg(Stretch(i-2)...Stretch(i+2))
#----------------------------------------------------------------------
rsa = [] # RSA
rrIntervals = rr_intervals_datastream.data
#----------------------------------------------------------------------
valleys = valleys_datastream.data
peaks = peaks_datastream.data[:-1]
for i, peak in enumerate(peaks):
valley_start_time = valleys[i].start_time
delta = peak.start_time - valleys[i].start_time
inspiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta.total_seconds()))
delta = valleys[i + 1].start_time - peak.start_time
expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta.total_seconds()))
delta = valleys[i + 1].start_time - valley_start_time
respiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta.total_seconds()))
ratio = (peak.start_time - valley_start_time) / (
valleys[i + 1].start_time - peak.start_time)
inspiration_expiration_ratio.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=ratio))
value = peak.sample - valleys[i + 1].sample
stretch.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=value))
#----------------------------------------------------------------------
# RSA
value = rsaCalculateCycle(valley_start_time, valleys[i + 1].start_time,
rrIntervals)
if value != -1:
rsa.append(
Data.Point.from_tuple(start_time=valley_start_time,
sample=value))
#----------------------------------------------------------------------
# upper_stretch.append(DataPoint.from_tuple(start_time=valley_start_time, sample=(peak.sample - valleys[i + 1][1]) / 2)) # TODO: Fix this by adding a tracking moving average and compute upper stretch from this to the peak and lower from this to the valley
# lower_stretch.append(DataPoint.from_tuple(start_time=valley_start_time, sample=(-peak.sample + valleys[i + 1][1]) / 2)) # TODO: Fix this by adding a tracking moving average and compute upper stretch from this to the peak and lower from this to the valley
for i in range(len(inspiration_duration)):
valley_start_time = valleys[i].start_time
if i == 0: # Edge case
delta_previous_inspiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_previous_expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_previous_respiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_previous_stretch_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
else:
delta = inspiration_duration[i].sample - inspiration_duration[
i - 1].sample
delta_previous_inspiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = expiration_duration[i].sample - expiration_duration[
i - 1].sample
delta_previous_expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = respiration_duration[i].sample - respiration_duration[
i - 1].sample
delta_previous_respiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = stretch[i].sample - stretch[i - 1].sample
delta_previous_stretch_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
if i == len(inspiration_duration) - 1:
delta_next_inspiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_next_expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_next_respiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
delta_next_stretch_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=0.0))
else:
delta = inspiration_duration[i].sample - inspiration_duration[
i + 1].sample
delta_next_inspiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = expiration_duration[i].sample - expiration_duration[
i + 1].sample
delta_next_expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = respiration_duration[i].sample - respiration_duration[
i + 1].sample
delta_next_respiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
delta = stretch[i].sample - stretch[i + 1].sample
delta_next_stretch_duration.append(
DataPoint.from_tuple(start_time=valley_start_time,
sample=delta))
stretch_average = 0
expiration_average = 0
count = 0.0
for j in [-2, -1, 1, 2]:
if i + j < 0 or i + j >= len(inspiration_duration):
continue
stretch_average += stretch[i + j].sample
expiration_average += expiration_duration[i + j].sample
count += 1
stretch_average /= count
expiration_average /= count
ratio = stretch[i].sample / stretch_average
neighbor_ratio_stretch_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=ratio))
ratio = expiration_duration[i].sample / expiration_average
neighbor_ratio_expiration_duration.append(
DataPoint.from_tuple(start_time=valley_start_time, sample=ratio))
# Begin assembling datastream for output
inspiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
inspiration_duration_datastream.data = inspiration_duration
expiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
expiration_duration_datastream.data = expiration_duration
respiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
respiration_duration_datastream.data = respiration_duration
inspiration_expiration_ratio_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
inspiration_expiration_ratio_datastream.data = inspiration_expiration_ratio
stretch_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
stretch_datastream.data = stretch
#----------------------------------------------------------------------
rsa_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
rsa_datastream.data = rsa
#----------------------------------------------------------------------
upper_stretch_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
upper_stretch_datastream.data = upper_stretch
lower_stretch_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
lower_stretch_datastream.data = lower_stretch
delta_previous_inspiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_previous_inspiration_duration_datastream.data = delta_previous_inspiration_duration
delta_previous_expiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_previous_expiration_duration_datastream.data = delta_previous_expiration_duration
delta_previous_respiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_previous_respiration_duration_datastream.data = delta_previous_respiration_duration
delta_previous_stretch_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_previous_stretch_duration_datastream.data = delta_previous_stretch_duration
delta_next_inspiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_next_inspiration_duration_datastream.data = delta_next_inspiration_duration
delta_next_expiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_next_expiration_duration_datastream.data = delta_next_expiration_duration
delta_next_respiration_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_next_respiration_duration_datastream.data = delta_next_respiration_duration
delta_next_stretch_duration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
delta_next_stretch_duration_datastream.data = delta_next_stretch_duration
neighbor_ratio_expiration_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
neighbor_ratio_expiration_datastream.data = neighbor_ratio_expiration_duration
neighbor_ratio_stretch_datastream = DataStream.from_datastream(
[peaks_datastream, valleys_datastream])
neighbor_ratio_stretch_datastream.data = neighbor_ratio_stretch_duration
#----------------------------------------------------------------------
# Aggregate minute level feature
# Breath-rate
valleys_window = window_sliding(valleys, window_size, window_offset)
peaks_window = window_sliding(valleys, window_size, window_offset)
insp_window = window_sliding(inspiration_duration, window_size,
window_offset)
exp_window = window_sliding(expiration_duration, window_size,
window_offset)
resp_window = window_sliding(respiration_duration, window_size,
window_offset)
inspExp_window = window_sliding(inspiration_expiration_ratio, window_size,
window_offset)
stretch_window = window_sliding(stretch, window_size, window_offset)
rsa_window = window_sliding(rsa, window_size, window_offset)
breath_rate = []
insp_min_vol = []
insp_qDev = []
insp_mean = []
insp_median = []
insp_80 = []
exp_qDev = []
exp_mean = []
exp_median = []
exp_80 = []
resp_qDev = []
resp_mean = []
resp_median = []
resp_80 = []
inspExp_qDev = []
inspExp_mean = []
inspExp_median = []
inspExp_80 = []
stretch_qDev = []
stretch_mean = []
stretch_median = []
stretch_80 = []
rsa_qDev = []
rsa_mean = []
rsa_median = []
rsa_80 = []
for key, value in valleys_window.items():
starttime, endtime = key
# breath_rate
breath_rate.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=len(valleys_window[key])))
# inspiration minute volume
value = 0.
for i in range(len(valleys_window[key])):
peak_value = peaks_window[key][i].sample
peak_time = peaks_window[key][i].start_time.timestamp()
valley_value = valleys_window[key][i].sample
valley_time = valleys_window[key][i].start_time.timestamp()
if peak_time > valley_time:
value += (peak_time - valley_time) * (peak_value -
valley_value) / 2
insp_min_vol.append(
DataPoint.from_tuple(start_time=starttime,
end_time=endtime,
sample=value))
# inspiration duration
insp_qDev, insp_mean, insp_median, insp_80 = getStats(
insp_window[key], insp_qDev, insp_mean, insp_median, insp_80)
# Expiration duration
exp_qDev, exp_mean, exp_median, exp_80 = getStats(
exp_window[key], exp_qDev, exp_mean, exp_median, exp_80)
# Respiration duration
resp_qDev, resp_mean, resp_median, resp_80 = getStats(
resp_window[key], resp_qDev, resp_mean, resp_median, resp_80)
# Inspiration Expiration duration ratio
inspExp_qDev, inspExp_mean, inspExp_median, inspExp_80 = getStats(
inspExp_window[key], inspExp_qDev, inspExp_mean, inspExp_median,
inspExp_80)
# Stretch
stretch_qDev, stretch_mean, stretch_median, stretch_80 = getStats(
stretch_window[key], stretch_qDev, stretch_mean, stretch_median,
stretch_80)
# RSA
rsa_qDev, rsa_mean, rsa_median, rsa_80 = getStats(
rsa_window[key], rsa_qDev, rsa_mean, rsa_median, rsa_80)
# To datastream struct
# breath_rate
breath_rate_datastream = DataStream.from_datastream(
[rr_intervals_datastream])
breath_rate_datastream.data = breath_rate
# inspiration minute volume
insp_min_vol_datastream = DataStream.from_datastream(
[rr_intervals_datastream])
insp_min_vol_datastream.data = insp_min_vol
# Inspiration duration
insp_qDev_datastream, insp_mean_datastream, insp_median_datastream, insp_80_datastream = \
list2datastream(insp_qDev, insp_mean, insp_median, insp_80, rr_intervals_datastream)
# Expiration duration
exp_qDev_datastream, exp_mean_datastream, exp_median_datastream, exp_80_datastream = \
list2datastream(exp_qDev, exp_mean, exp_median, exp_80, rr_intervals_datastream)
# Respiration Duration
resp_qDev_datastream, resp_mean_datastream, resp_median_datastream, resp_80_datastream = \
list2datastream(resp_qDev, resp_mean, resp_median, resp_80, rr_intervals_datastream)
# Inspiration Expiration duration ratio
inspExp_qDev_datastream, inspExp_mean_datastream, inspExp_median_datastream, inspExp_80_datastream = \
list2datastream(inspExp_qDev, inspExp_mean, inspExp_median, inspExp_80, rr_intervals_datastream)
# Stretch
stretch_qDev_datastream, stretch_mean_datastream, stretch_median_datastream, stretch_80_datastream = \
list2datastream(stretch_qDev, stretch_mean, stretch_median, stretch_80, rr_intervals_datastream)
# RSA
rsa_qDev_datastream, rsa_mean_datastream, rsa_median_datastream, rsa_80_datastream = \
list2datastream(rsa_qDev, rsa_mean, rsa_median, rsa_80, rr_intervals_datastream)
#----------------------------------------------------------------------
return breath_rate_datastream, insp_min_vol_datastream, \
insp_qDev_datastream, insp_mean_datastream, insp_median_datastream, insp_80_datastream, \
exp_qDev_datastream, exp_mean_datastream, exp_median_datastream, exp_80_datastream, \
resp_qDev_datastream, resp_mean_datastream, resp_median_datastream, resp_80_datastream, \
inspExp_qDev_datastream, inspExp_mean_datastream, inspExp_median_datastream, inspExp_80_datastream, \
stretch_qDev_datastream, stretch_mean_datastream, stretch_median_datastream, stretch_80_datastream, \
rsa_qDev_datastream, rsa_mean_datastream, rsa_median_datastream, rsa_80_datastream
'''
