def complementary_filter(accel: List[DataPoint],
gyro: List[DataPoint],
fq=16.0):
ts = [v.start_time for v in accel]
acc_x = [v.sample[0] for v in accel]
acc_y = [v.sample[1] for v in accel]
acc_z = [v.sample[2] for v in accel]
gyr_x = [v.sample[0] for v in gyro]
gyr_y = [v.sample[1] for v in gyro]
gyr_z = [v.sample[2] for v in gyro]
dt = 1.0 / fq # 1/16.0;
M_PI = math.pi
hpf = 0.85
lpf = 0.15
thetaX_acc = [0] * len(acc_x) # math.atan2(-acc_z,acc_y)*180/M_PI;
thetaY_acc = [0] * len(acc_x) # math.atan2(acc_x,acc_z)*180/M_PI;
thetaZ_acc = [0] * len(acc_x) # math.atan2(acc_y,acc_x)*180/M_PI;
thetaX = [0] * len(gyr_x)
thetaY = [0] * len(gyr_y)
thetaZ = [0] * len(gyr_z)
for index in range(len(gyr_x)):
thetaX_acc[index] = math.atan2(-acc_z[index],
acc_y[index]) * 180 / M_PI
thetaY_acc[index] = math.atan2(acc_x[index], acc_z[index]) * 180 / M_PI
thetaZ_acc[index] = math.atan2(acc_y[index], acc_x[index]) * 180 / M_PI
if index == 0:
thetaX[index] = hpf * thetaX[index] * dt + lpf * thetaX_acc[index]
thetaY[index] = hpf * thetaY[index] * dt + lpf * thetaY_acc[index]
thetaZ[index] = hpf * thetaZ[index] * dt + lpf * thetaZ_acc[index]
else:
thetaX[index] = hpf * (thetaX[index - 1] +
gyr_x[index] * dt) + lpf * thetaX_acc[index]
thetaY[index] = hpf * (thetaY[index - 1] +
gyr_y[index] * dt) + lpf * thetaY_acc[index]
thetaZ[index] = hpf * (thetaZ[index - 1] +
gyr_z[index] * dt) + lpf * thetaZ_acc[index]
rolls = [
DataPoint(start_time=v.start_time, sample=thetaX[i])
for i, v in enumerate(accel)
]
pitches = [
DataPoint(start_time=v.start_time, sample=thetaY[i])
for i, v in enumerate(accel)
]
yaws = [
DataPoint(start_time=v.start_time, sample=thetaZ[i])
for i, v in enumerate(accel)
]
return rolls, pitches, yaws
def get_random_EMA(cur_dir, filename) -> List[DataPoint]:
emas = get_EMA_data(cur_dir, filename)
data = []
for ema in emas:
d = ema[2]
jsn_file = json.loads(d)
status = jsn_file['status']
# print(d)
if status == 'COMPLETED':
is_smoked = jsn_file['question_answers'][39]['response'][0]
# print(is_smoked, status)
if is_smoked.lower() == 'yes':
nSmoked = jsn_file['question_answers'][40]['response'][0]
if int(nSmoked) == 1:
nQI = 41
else:
nQI = 42
# options: ["0 - 2 hrs", "2 hrs - 4 hrs", "4 hrs - 6 hrs", "6 hrs - 8 hrs", "8 hrs - 10 hrs", "10 hrs - 12 hrs", "More than 12 hrs"]
howlong_ago = jsn_file['question_answers'][nQI]['response']
if howlong_ago is None:
howlong_ago = jsn_file['question_answers'][nQI +
1]['response']
sample = [int(nSmoked)]
# print(howlong_ago, nSmoked)
for hla in howlong_ago:
hla = str(hla)
if hla in ["More than 12 hrs"]:
sample.extend(
[12 * 60 * 60 * 1000, 24 * 60 * 60 * 1000])
continue
st = hla.split('-')[0]
et = hla.split('-')[1]
st = st.split(' ')[0]
st = int(st.strip()) * 60 * 60 * 1000
et = et.strip().split(' ')[0]
et = int(et.strip()) * 60 * 60 * 1000
sample.extend([st, et])
# print([ema[0], ema[1], nSmoked, howlong_ago, sample])
# data.append([ema[0], ema[1], int(nSmoked)])
if len(sample) > 3:
print('great than 1 timeslots', len(sample) / 2)
data.append(
DataPoint(start_time=ema[0], offset=ema[1], sample=sample))
else:
data.append(
DataPoint(start_time=ema[0], offset=ema[1], sample=[0]))
return data
def line_parser_offset(input):
ts, offset, sample = input.split(',', 2)
start_time = int(float(ts)) / 1000.0
sample = convert_sample(sample)
if len(sample) == 1:
sample = sample[0]
return DataPoint(start_time=datetime.fromtimestamp(start_time, tz), offset=offset, sample=sample)
def get_smoking_EMA(cur_dir, filename) -> List[DataPoint]:
emas = get_EMA_data(cur_dir, filename)
data = []
for ema in emas:
d = ema[2]
jsn_file = json.loads(d)
status = jsn_file['status']
if status == 'COMPLETED':
is_smoked = jsn_file['question_answers'][0]['question_answer'][0:3]
# print(is_smoked)
if is_smoked.lower() == 'yes':
data.append(DataPoint(start_time=ema[0], offset=ema[1], sample=1))
# data.append([ema[0], ema[1], 1])
else:
data.append(DataPoint(start_time=ema[0], offset=ema[1], sample=0))
# data.append([ema[0], ema[1], 0])
return data
def line_parser(input, sd_indx, fd_indx, dd_indx, cd_indx):
ts, sample = input.split(',', 1)
sample = convert_sample(sample)
ts = sample[6]
# gt = sample[8]
# gt = sample[8+6] #smoking conf
# gt = sample[8+9] #eating conf
start_time = int(float(ts)) / 1000.0
accel_dp = DataPoint(start_time=datetime.fromtimestamp(start_time, tz),
offset='0', sample=[sample[0], -sample[1], sample[2]])
gyro_dp = DataPoint(start_time=datetime.fromtimestamp(start_time, tz), offset='0',
sample=[sample[3], -sample[4], sample[5]])
gt_sd = None
gt_fd = None
gt_dd = None
gt_cd = None
if sd_indx != -1:
gt_sd = DataPoint(start_time=datetime.fromtimestamp(start_time, tz), offset='0', sample=sample[sd_indx])
if fd_indx != -1:
gt_fd = DataPoint(start_time=datetime.fromtimestamp(start_time, tz), offset='0', sample=sample[fd_indx])
if dd_indx != -1:
gt_dd = DataPoint(start_time=datetime.fromtimestamp(start_time, tz), offset='0', sample=sample[dd_indx])
if cd_indx != -1:
gt_cd = DataPoint(start_time=datetime.fromtimestamp(start_time, tz), offset='0', sample=sample[cd_indx])
return accel_dp, gyro_dp, gt_sd, gt_fd, gt_dd, gt_cd
def magnitude(data: List[DataPoint]):
input_sample = np.array([i.sample for i in data])
mag_sample = norm(input_sample, axis=1).tolist()
result = [
DataPoint(start_time=v.start_time,
offset=v.start_time,
sample=mag_sample[i]) for i, v in enumerate(data)
]
return result
def calculate_pitch(accel_data: List[DataPoint]):
pitch_list = []
for dp in accel_data:
ay = dp.sample[1]
az = dp.sample[2]
ptch = 180 * math.atan2(-ay, -az) / math.pi
pitch_list.append(
DataPoint(start_time=dp.start_time,
end_time=dp.end_time,
offset=dp.offset,
sample=ptch))
return pitch_list
def calculate_yaw(accel_data: List[DataPoint]):
yaw_list = []
for dp in accel_data:
ax = dp.sample[0]
ay = dp.sample[1]
yw = 180 * math.atan2(ay, ax) / math.pi
yaw_list.append(
DataPoint(start_time=dp.start_time,
end_time=dp.end_time,
offset=dp.offset,
sample=yw))
return yaw_list
def classify_puffs(features):
clf = get_posture_model()
labels = []
for dp in features:
predicted_label = clf.predict([dp.sample])
predicted_label = int(str(predicted_label))
labels.append(
DataPoint(start_time=dp.start_time,
offset=dp.offset,
end_time=dp.end_time,
sample=predicted_label))
return labels
def get_smoking_self_report(cur_dir, filename) -> List[DataPoint]:
emas = get_EMA_data(cur_dir, filename)
data = []
for ema in emas:
d = ema[2]
jsn_file = json.loads(d)
status = jsn_file['message']
if 'YES' in status:
# print(status)
data.append(DataPoint(start_time=ema[0], offset=ema[1], sample=1))
# print(ema)
# data.append([ema[0], ema[1], status])
return data
def calculate_roll(accel_data: List[DataPoint]):
roll_list = []
for dp in accel_data:
ax = dp.sample[0]
ay = dp.sample[1]
az = dp.sample[2]
rll = 180 * math.atan2(ax, math.sqrt(ay * ay + az * az)) / math.pi
roll_list.append(
DataPoint(start_time=dp.start_time,
end_time=dp.end_time,
offset=dp.offset,
sample=rll))
return roll_list
def moving_average_convergence_divergence(
slow_moving_average_data: List[DataPoint],
fast_moving_average_data: List[DataPoint], THRESHOLD: float,
near: int):
'''
Generates intersection points of two moving average signals
:param slow_moving_average_data:
:param fast_moving_average_data:
:param THRESHOLD: Cut-off value
:param near: # of nearest point to ignore
:return:
'''
slow_moving_average = np.array(
[data.sample for data in slow_moving_average_data])
fast_moving_average = np.array(
[data.sample for data in fast_moving_average_data])
index_list = [0] * len(slow_moving_average)
cur_index = 0
for index in range(len(slow_moving_average)):
diff = slow_moving_average[index] - fast_moving_average[index]
if diff > THRESHOLD:
if cur_index == 0:
index_list[cur_index] = index
cur_index = cur_index + 1
index_list[cur_index] = index
else:
if index <= index_list[cur_index] + near:
index_list[cur_index] = index
else:
cur_index = cur_index + 1
index_list[cur_index] = index
cur_index = cur_index + 1
index_list[cur_index] = index
intersection_points = []
if cur_index > 0:
for index in range(0, cur_index, 2):
start_index = index_list[index]
end_index = index_list[index + 1]
start_time = slow_moving_average_data[start_index].start_time
end_time = slow_moving_average_data[end_index].start_time
intersection_points.append(
DataPoint(start_time=start_time,
end_time=end_time,
sample=[index_list[index], index_list[index + 1]]))
return intersection_points
def get_accelerometer(data_dir, wrist) -> List[DataPoint]:
if wrist in [LEFT_WRIST]:
accel_x = load_data(data_dir + ax_left_filename)
accel_y = load_data(data_dir + ay_left_filename)
accel_z = load_data(data_dir + az_left_filename)
else:
accel_x = load_data(data_dir + ax_right_filename)
accel_y = load_data(data_dir + ay_right_filename)
accel_z = load_data(data_dir + az_right_filename)
accel = []
for index, val in enumerate(accel_x):
sample = [accel_x[index].sample, accel_y[index].sample, accel_z[index].sample]
accel.append(DataPoint(start_time=val.start_time, offset=val.offset, sample=sample))
return accel
def get_gyroscope(data_dir, wrist) -> List[DataPoint]:
if wrist in [LEFT_WRIST]:
gyro_x = load_data(data_dir + gx_left_filename)
gyro_y = load_data(data_dir + gy_left_filename)
gyro_z = load_data(data_dir + gz_left_filename)
else:
gyro_x = load_data(data_dir + gx_right_filename)
gyro_y = load_data(data_dir + gy_right_filename)
gyro_z = load_data(data_dir + gz_right_filename)
gyro = []
for index, val in enumerate(gyro_x):
sample = [gyro_x[index].sample, gyro_y[index].sample, gyro_z[index].sample]
gyro.append(DataPoint(start_time=val.start_time, offset=val.offset, sample=sample))
return gyro
def merge_two_datastream(accel: List[DataPoint], gyro: List[DataPoint]):
# usually accel is 16Hz and gyro is 32 Hz
# make gyro 16 Hz
A = np.array(
[[dp.start_time.timestamp(), dp.sample[0], dp.sample[1], dp.sample[2]]
for dp in accel])
G = np.array(
[[dp.start_time.timestamp(), dp.sample[0], dp.sample[1], dp.sample[2]]
for dp in gyro])
At = A[:, 0]
Gt = G[:, 0]
Gx = G[:, 1]
Gy = G[:, 2]
Gz = G[:, 3]
i = 0
j = 0
_Gx = [0] * len(At)
_Gy = [0] * len(At)
_Gz = [0] * len(At)
while (i < len(At)) and (j < len(Gt)):
while Gt[j] < At[i]:
j = j + 1
if j >= len(Gt):
break
if j < len(Gt):
if (At[i] == Gt[j]) | (j == 0):
_Gx[i] = Gx[j]
_Gy[i] = Gy[j]
_Gz[i] = Gz[j]
else:
_Gx[i] = getInterpoletedValue(Gx[j - 1], Gx[j], Gt[j - 1],
Gt[j], At[i])
_Gy[i] = getInterpoletedValue(Gy[j - 1], Gy[j], Gt[j - 1],
Gt[j], At[i])
_Gz[i] = getInterpoletedValue(Gz[j - 1], Gz[j], Gt[j - 1],
Gt[j], At[i])
i = i + 1
gyro = [
DataPoint(start_time=dp.start_time,
end_time=dp.end_time,
offset=dp.offset,
sample=[_Gx[i], _Gy[i], _Gz[i]])
for i, dp in enumerate(accel)
]
return gyro
def smooth(data: List[DataPoint], span: int = 5) -> List[DataPoint]:
"""
Smooths data using moving average filter over a span.
The first few elements of data_smooth are given by
data_smooth(1) = data(1)
data_smooth(2) = (data(1) + data(2) + data(3))/3
data_smooth(3) = (data(1) + data(2) + data(3) + data(4) + data(5))/5
data_smooth(4) = (data(2) + data(3) + data(4) + data(5) + data(6))/5
for more details follow the below links:
https://www.mathworks.com/help/curvefit/smooth.html
http://stackoverflow.com/a/40443565
:return: data_smooth
:param data:
:param span:
"""
if data is None or len(data) == 0:
return []
sample = [i.sample for i in data]
sample_middle = np.convolve(sample, np.ones(span, dtype=int),
'valid') / span
divisor = np.arange(1, span - 1, 2)
sample_start = np.cumsum(sample[:span - 1])[::2] / divisor
sample_end = (np.cumsum(sample[:-span:-1])[::2] / divisor)[::-1]
sample_smooth = np.concatenate((sample_start, sample_middle, sample_end))
data_smooth = []
if len(sample_smooth) == len(data):
for i, item in enumerate(data):
dp = DataPoint.from_tuple(sample=sample_smooth[i],
start_time=item.start_time,
end_time=item.end_time)
data_smooth.append(dp)
else:
raise Exception(
"Smoothed data length does not match with original data length.")
return data_smooth
def get_marked_smoking_puffs_filtered(data_dir, wrist) -> List[DataPoint]:
if wrist in [LEFT_WRIST]:
filename = data_dir + 'puff_timestamp_leftwrist.csv'
label = 1
else:
filename = data_dir + 'puff_timestamp_rightwrist.csv'
label = 2
fp = open(filename)
file_content = fp.read()
fp.close()
lines = file_content.splitlines()
puff_timings = []
for line in lines:
start_time = int(line) / 1000.0
tz = pytz.timezone('US/Central')
puff_timings.append(DataPoint(start_time=datetime.fromtimestamp(start_time, tz), sample=label))
return puff_timings
def moving_average_curve(data: List[DataPoint],
window_length: int) -> List[DataPoint]:
"""
Moving average curve from filtered (using moving average) samples.
:return: mac
:param data:
:param window_length:
"""
if data is None or len(data) == 0:
return []
sample = [i.sample for i in data]
mac = []
for i in range(window_length, len(sample) - (window_length + 1)):
sample_avg = np.mean(sample[i - window_length:i + window_length + 1])
mac.append(
DataPoint.from_tuple(sample=sample_avg,
start_time=data[i].start_time,
end_time=data[i].end_time))
return mac
def get_all_marked_smoking_puffs(data_dir, wrist, pid, sid) -> List[DataPoint]:
if wrist in [LEFT_WRIST]:
filename = ground_truth_file + pid + '_' + sid + '_smoking_puff_left.csv'
label = 1
else:
filename = ground_truth_file + pid + '_' + sid + '_smoking_puff_right.csv'
label = 2
try:
fp = open(filename)
file_content = fp.read()
fp.close()
except:
return []
lines = file_content.splitlines()
puff_timings = []
for line in lines:
start_time = int(line.split(',')[0]) / 1000.0
puff_timings.append(DataPoint(start_time=datetime.fromtimestamp(start_time), sample=label))
return puff_timings
def generate_smoking_episode(puff_labels) -> List[DataPoint]:
'''
Generates smoking episodes from classified puffs
:param puff_labels:
:return: list of smoking episodes
'''
only_puffs = [dp for dp in puff_labels if dp.sample > 0]
smoking_episode_data = []
cur_index = 0
while cur_index < len(only_puffs):
temp_index = cur_index
dp = only_puffs[temp_index]
prev = dp
temp_index = temp_index + 1
if temp_index >= len(only_puffs):
break
while ((
(only_puffs[temp_index].start_time - dp.start_time <=
timedelta(seconds=MINIMUM_TIME_DIFFERENCE_FIRST_AND_LAST_PUFFS))
| (only_puffs[temp_index].start_time - prev.start_time <
timedelta(seconds=MINIMUM_INTER_PUFF_DURATION)))):
prev = only_puffs[temp_index]
temp_index = temp_index + 1
if temp_index >= len(only_puffs):
break
temp_index = temp_index - 1
if (temp_index - cur_index + 1) >= MINIMUM_PUFFS_IN_EPISODE:
wrist = get_smoking_wrist(only_puffs, cur_index, temp_index)
smoking_episode_data.append(
DataPoint(start_time=only_puffs[cur_index].start_time,
end_time=only_puffs[temp_index].start_time,
sample=wrist))
cur_index = temp_index + 1
else:
cur_index = cur_index + 1
return smoking_episode_data
def line_parser(input):
ts, sample = input.split(',')
start_time = int(float(ts)) / 1000.0
sample = float(sample)
return DataPoint(start_time=datetime.fromtimestamp(start_time, tz), sample=sample)
def moving_average_convergence_divergence_new(
slow_moving_average_data: List[DataPoint],
fast_moving_average_data: List[DataPoint], accel: List[DataPoint]):
'''
Generates intersection points of two moving average signals
:param slow_moving_average_data:
:param fast_moving_average_data:
:param THRESHOLD: Cut-off value
:param near: # of nearest point to ignore
:return:
'''
s = np.array([data.sample for data in slow_moving_average_data])
f = np.array([data.sample for data in fast_moving_average_data])
bit_map = [0] * len(s)
for i in range(len(s)):
if f[i] > s[i]:
bit_map[i] = 0
else:
bit_map[i] = 1
for i in range(len(s)):
if bit_map[i] == 0 and bit_map[max(0, i - 4)] == 1 and bit_map[min(
len(s), i + 4)] == 1:
bit_map[i] = 1
cur_index = 0
intersection_points = []
while cur_index < len(s):
if bit_map[cur_index] == 1:
start_index = cur_index
while cur_index < len(s) and bit_map[cur_index] == 1:
cur_index = cur_index + 1
end_index = cur_index - 1
diff = []
i = start_index - 1
while i >= 0 and bit_map[i] == 0:
diff.append(f[i] - s[i])
i = i - 1
i = i + 1
if len(diff) > 0:
prev_peak_diff = np.mean(diff)
else:
prev_peak_diff = 0
# prev_peak_dur = (slow_moving_average_data[start_index].start_time - slow_moving_average_data[
# i].start_time).total_seconds()
prev_peak_start_index = i
ay = [accel[i].sample[1] for i in range(i, start_index)]
prev_ay_mean = np.mean(ay)
prev_ay_sd = np.std(ay)
diff = []
i = end_index + 1
while i < len(s) and bit_map[i] == 0:
diff.append(f[i] - s[i])
i = i + 1
i = i - 1
if len(diff) > 0:
next_peak_diff = np.mean(diff)
else:
next_peak_diff = 0
# next_peak_dur = (slow_moving_average_data[i].start_time - slow_moving_average_data[
# end_index].start_time).total_seconds()
next_peak_start_index = i
ay = [accel[i].sample[1] for i in range(end_index, i)]
nxt_ay_mean = np.mean(ay)
nxt_ay_sd = np.std(ay)
# if prev_peak_dur > 0.45 and next_peak_dur > 0.45 and prev_peak_dur < 4 and next_peak_dur < 4:
intersection_points.append(
DataPoint(
start_time=slow_moving_average_data[start_index].
start_time,
end_time=slow_moving_average_data[end_index].start_time,
sample=[
start_index, end_index, prev_peak_start_index,
next_peak_start_index, prev_peak_diff, next_peak_diff,
prev_ay_mean, nxt_ay_mean, prev_ay_sd, nxt_ay_sd
]))
else:
cur_index = cur_index + 1
return intersection_points