def plot_MIT_baseline(df_resample, fps, out_path):
fps=29.969664
df_resample = df_resample.reset_index()
df_resample['time'] -= df_resample['time'][0]
df_resample['time'] /= 1000
df_resample = df_resample.set_index('time')
fig, axes = plt.subplots(4, 1, figsize=(20, 10))
grid = plt.GridSpec(2, 3)
axes[0].plot(df_resample['accx'], label='accx')
axes[0].plot(df_resample['accy'], label='accy')
axes[0].plot(df_resample['accz'], label='accz')
axes[0].legend(loc='upper right')
axes[0].set_title('Accelerometer data')
# plt.set_color_cycle(['red', 'black', 'blue', 'yellow', 'grey'])
axes[1].plot(df_resample['diff_flowx'], 'b', label='diff_flowx')
axes[2].plot(df_resample['diff_flowy'], 'orange', label='diff_flowy')
#ax2.legend(loc='upper right')
axes[1].set_title('diff flow x')
axes[2].set_title('diff flow y')
fftshift = cross_correlation_using_fft(df_resample['diff_flowx'].values, df_resample['accx'].values)
shift = compute_shift(fftshift)
axes[3].plot(fftshift)
axes[3].set_title('diff flowx/accx delta={:.1f} ms'.format(shift * 1000/fps))
plt.subplots_adjust(hspace=0.5)
plt.savefig(out_path)
plt.close()
def baseline(mode):
# load video data
# session_list = open('../../CMU/session_list').readlines()
video_dir = '../../CMU/video'
opt_dir = '../../CMU/opt_flow'
IMU_dir = '../../CMU/sensor/'
data_dir = '../../CMU/data'
FPS = 30
baseline_dir = os.path.join(data_dir, 'baseline')
# offsets = np.zeros(len(session_list))
sensors = ['2794', '2795', '2796', '3261', '3337']
video = '7150991'
load_df = True
# verbose = True
draw = True
sensor_dict = {'2794': 'Left Arm', '2795': 'Back', '2796': 'Left Leg', '3261': 'Right Leg', '3337': 'Right Arm'}
#########################################################
## Baseline #
#########################################################
valid_session_file = '../../CMU/valid_sessions_win30_max60.pkl'
offsets, session_list = pickle.load(open(valid_session_file, 'rb'))
session_list = session_list['2794']
if mode == 'x':
mode_video = 'x'
mode_imu = 'X'
if mode == 'PCA':
mode_video = 'PCA'
mode_imu = 'PCA'
shifts_baseline = {}
skipped_sessions = {}
valid_sessions = {}
for sensor in sensors:
shifts_baseline[sensor] = np.zeros(len(session_list))
skipped_sessions[sensor] = []
valid_sessions[sensor] = []
os.makedirs('figures/baseline_{}_{}/{}'.format(mode_video, mode_imu, sensor), exist_ok=True)
# load ground truth
offsets = np.zeros(len(session_list))
for i, session in enumerate(session_list):
session = session.strip()
opt_file = glob.glob(os.path.join(opt_dir, session+'_Video', session+'_7150991-*.pkl'))
if len(opt_file) > 0:
opt_file = opt_file[0]
offsets[i] = int(opt_file[opt_file.find('-')+1:-4])
offsets = offsets * 1000/FPS
for i, session in enumerate(session_list):
session = session.strip()
print('======== Procession session {} ========'.format(session))
file = glob.glob(os.path.join(video_dir, session+'_Video', 'STime{}-time-*synch.txt'.format(video)))
out_dir = os.path.join(baseline_dir, session)
os.makedirs(out_dir, exist_ok=True)
df_file = os.path.join(out_dir, 'df_video_{}.pkl'.format(video))
print('Processing video {}'.format(video))
if load_df and os.path.exists(df_file):
print(' loading video df')
df_video = pd.read_pickle(df_file)
else:
print(' creating video df')
if file:
df_video = read_video(file[0])
check_df(df_video, delta=0.033332)
else:
print('skipping session{}, video sync doesn\'t exist'.format(session))
for sensor in sensors:
shifts_baseline[sensor][i] = np.nan
skipped_sessions[sensor].append(session)
continue
opt_file = glob.glob(os.path.join(opt_dir, session+'_Video', session+'_7150991-*.pkl'))
if opt_file:
opt_file = opt_file[0]
#offsets[i] = int(opt_file[opt_file.find('-')+1:-4])
else:
print('skipping session{}, optical flow doesn\'t exist'.format(session))
for sensor in sensors:
shifts_baseline[sensor][i] = np.nan
skipped_sessions[sensor].append(session)
continue
motion = pickle.load(open(opt_file, 'rb'))
if len(df_video) < len(motion):
print('skipping session{}, sync stamps less than frames'.format(session))
for sensor in sensors:
shifts_baseline[sensor][i] = np.nan
skipped_sessions[sensor].append(session)
continue
df_video = df_video[:len(motion)]
df_video['flowx'] = motion[:, 0]
df_video['flowy'] = motion[:, 1]
df_video['diff_flowx'] = df_video['flowx'].diff()
df_video['diff_flowy'] = df_video['flowy'].diff()
df_video = df_video[1:]
df_video = pca_flow(df_video)
# df_video.to_pickle(df_file)
# load sensor data
for sensor in sensors:
print('Processing sensor {}'.format(sensor))
df_file = os.path.join(out_dir, 'df_sensor_{}.pkl'.format(sensor))
if os.path.exists(df_file):
print(' loading sensor df')
df_imu = pd.read_pickle(df_file)
else:
print(' creating sensor df')
sensor_file = glob.glob(os.path.join(IMU_dir, session+'_3DMGX1', '{}_*-time*.txt'.format(sensor)))
if sensor_file:
sensor_file = sensor_file[0]
else:
print('skipping session{}, session time doesn\'t exist'.format(session))
skipped_sessions[sensor].append(session)
shifts_baseline[sensor][i] = np.nan
continue
df_imu = read_sensor(sensor_file)
df_imu = check_df(df_imu, delta=0.008)
df_imu = pca_sensor(df_imu)
# df_imu.to_pickle(df_file)
st_time = max([df_imu.iloc[0]['SysTime'], df_video.iloc[0]['SysTime']])-0.01
en_time = min([df_imu.iloc[len(df_imu) - 1]['SysTime'], df_video.iloc[len(df_video) - 1]['SysTime']])+0.01
df_video_tmp = df_video[(df_video['SysTime'] >= st_time) & (df_video['SysTime'] < en_time)]
df_imu = df_imu[(df_imu['SysTime'] >= st_time) & (df_imu['SysTime'] < en_time)]
vid_time_stamps = df_video_tmp['SysTime'].values
df_imu = resample(df_imu, 'SysTime', samplingRate=0,
gapTolerance=200, fixedTimeColumn=vid_time_stamps)
if df_imu is None:
print(' no intersection between video and imu, skip session {}'.format(session))
skipped_sessions[sensor].append(session)
shifts_baseline[sensor][i] = np.nan
continue
if len(df_video_tmp) != len(df_imu):
print(' lengths of video and imu not equal, skip session {}'.format(session))
print('len_vid = {}, len_sen = {}'.format(len(df_video_tmp), len(df_imu)))
skipped_sessions[sensor].append(session)
shifts_baseline[sensor][i] = np.nan
#df_imu.to_pickle(df_file)
continue
fftshift = cross_correlation_using_fft(df_video_tmp['diff_flow{}'.format(mode_video)].values, \
df_imu['Accel_{}'.format(mode_imu)].values)
shifts_baseline[sensor][i] = compute_shift(fftshift)
if draw:
path ='figures/baseline_{}_{}/{}/{}.jpg'.format(mode_video, mode_imu, sensor, session)
plt.figure()
plt.plot(fftshift[::-1])
plt.title('Video / {} Sensor, gt = {:.3f}s, predition = {:.3f}s'.format(sensor_dict[sensor], offsets[i]/1000, shifts_baseline[sensor][i] /FPS))
plt.savefig(path)
plt.close()
valid_sessions[sensor].append(session)
#plt.figure()
#plt.plot(fftshift)
#plt.savefig('{}_{}_{}.png'.format(mode, session, sensor))
for sensor in sensors:
shifts_baseline[sensor][67:91] = np.nan
error = compute_error(offsets, shifts_baseline)
error.to_csv(os.path.join(baseline_dir, 'df_error_baseline_{}_{}.csv'.format(mode_video, mode_imu)))
print(mode_video, mode_imu)
print(error)
#print(offsets, shifts_baseline)
# pickle.dump([offsets, shifts_baseline], open(os.path.join(baseline_dir, 'results_baseline_{}_{}.pkl'.format(mode_video, mode_imu)), 'wb'))
# pickle.dump(valid_sessions, open(os.path.join(baseline_dir, 'valid_sessions.pkl'), 'wb'))
#
# Analysis
_, shifts_baseline = pickle.load(open(os.path.join(baseline_dir, 'results_baseline_{}_{}.pkl'.format(mode_video, mode_imu)), 'rb'))
summ_mat = offsets.reshape(-1, 1)
for sensor in sensors:
summ_mat = np.concatenate([summ_mat, shifts_baseline[sensor].reshape(-1, 1) - offsets.reshape(-1, 1)], axis=1)
df_summ = pd.DataFrame(data=summ_mat, columns=['Ground truth', 'imu2794', 'imu2795', 'imu2796', 'imu3261', 'imu3337'], index=session_list)
df_summ.to_csv(os.path.join(baseline_dir, 'df_summ_baseline_{}_{}.csv'.format(mode_video, mode_imu)))
print(offsets)
def drift_confidence(ts1, ts2):
fftshift = cross_correlation_using_fft(ts1, ts2)
dist = max(abs(fftshift-median(fftshift)))
shift = compute_shift(fftshift)
conf = dist/stdev(fftshift)
return conf, shift
def drift_confidence(df_resample, out_path, fps, pca=1, save_fig=0):
"""
Args:
df_resample:
out_path:
fps:
pca:
save_fig:
Returns:
"""
if pca:
flow_key = 'diffflow_pca'
acc_key = 'acc_pca'
else:
flow_key = 'diff_flowx'
acc_key = 'accx'
fftshift = cross_correlation_using_fft(df_resample[flow_key].values,
df_resample[acc_key].values)
dist = max(abs(fftshift - median(fftshift)))
shift = compute_shift(fftshift)
fx_ay_drift = shift * 1000 / fps
fx_ay_conf = dist / stdev(fftshift)
if save_fig:
fig, ax = plt.subplots(2, 4, figsize=(20, 10))
plt.subplot(2, 4, 1)
plt.plot(df_resample['accx'])
plt.plot(df_resample['accy'])
plt.plot(df_resample['accz'])
plt.title('acc x, y, z')
plt.subplot(2, 4, 5)
plt.plot(df_resample['diff_flowx'])
plt.plot(df_resample['diff_flowy'])
plt.title('diff_flow x & y')
plt.subplot(2, 4, 2)
fftshift = cross_correlation_using_fft(
df_resample['diff_flowx'].values, df_resample['diff_flowy'].values)
dist = max(abs(fftshift - median(fftshift)))
shift = compute_shift(fftshift)
plt.plot(fftshift)
plt.title("\n".join(wrap('fx fy {:.1f} ms, k{:.1f}, std{:.1f}, dm{:.1f}, ndm{:.1f}'.format(\
shift * 1000/fps, kurtosis(fftshift), stdev(fftshift), dist, dist/stdev(fftshift)), 40)))
plt.subplot(2, 4, 3)
fftshift = cross_correlation_using_fft(
df_resample['diff_flowsquare'].values,
df_resample['accsquare'].values)
dist = max(abs(fftshift - median(fftshift)))
shift = compute_shift(fftshift)
plt.plot(fftshift)
plt.title("\n".join(wrap('fsq asq {:.1f} ms, k{:.1f}, std{:.1f}, dm{:.1f}, ndm{:.1f}'.format(\
shift * 1000/fps, kurtosis(fftshift), stdev(fftshift), dist, dist/stdev(fftshift)), 40)))
plt.subplot(2, 4, 4)
fftshift = cross_correlation_using_fft(
df_resample['diff_flowx'].values, df_resample['accx'].values)
dist = max(abs(fftshift - median(fftshift)))
shift = compute_shift(fftshift)
plt.plot(fftshift)
plt.title("\n".join(wrap('fx ax {:.1f} ms, k{:.1f}, std{:.1f}, dm{:.1f}, ndm{:.1f}'.format(\
shift * 1000/fps, kurtosis(fftshift), stdev(fftshift), dist, dist/stdev(fftshift)), 40)))
plt.subplot(2, 4, 6)
fftshift = cross_correlation_using_fft(
df_resample['diff_flowy'].values, df_resample['accz'].values)
dist = max(abs(fftshift - median(fftshift)))
shift = compute_shift(fftshift)
plt.plot(fftshift)
plt.title("\n".join(wrap('fy az {:.1f} ms, k{:.1f}, std{:.1f}, dm{:.1f}, ndm{:.1f}'.format(\
shift * 1000/fps, kurtosis(fftshift), stdev(fftshift), dist, dist/stdev(fftshift)), 40)))
plt.subplot(2, 4, 7)
fftshift = cross_correlation_using_fft(
df_resample['diff_flowx'].values, df_resample['accy'].values)
dist = max(abs(fftshift - median(fftshift)))
shift = compute_shift(fftshift)
plt.plot(fftshift)
plt.title("\n".join(wrap(r'fx ay $\bf{{{:.1f}}}$ ms, k{:.1f}, std{:.1f}, dm{:.1f}, ndm$\bf{{{:.1f}}}$'.format(\
shift * 1000/fps, kurtosis(fftshift), stdev(fftshift), dist, dist/stdev(fftshift)), 40)))
fig.tight_layout()
fig.subplots_adjust(top=0.8)
plt.savefig(out_path)
plt.close()
return fx_ay_drift, fx_ay_conf
def baseline_MIT_video_MD2K(window_size_sec, stride_sec, num_offsets, max_offset, window_criterion, offset_sec = 0, plot=0, use_PCA=0):
# #subjects = ['202', '205', '211', '235', '236', '238', '240', '243']
# dir_path = os.path.dirname(os.path.realpath(__file__))
# # df_start_time = pd.read_csv(settings['STARTTIME_TEST_FILE'])
# # df_start_time = csv_read(os.path.join(dir_path, '../../Sense2StopSync/start_time.csv')).set_index('video_name')
# df_start_time = csv_read(settings['STARTTIME_TEST_FILE']).set_index('video_name')
# video_names = df_start_time.index.tolist()
# subjects = list(set([vid.split(' ')[0] for vid in video_names]))
# fps = 29.969664
# data_dir = '/media/yun/08790233DP/sync_data_2nd'
# with open(data_dir+'/all_video' + title_suffix + '_info_dataset.pkl', 'rb') as handle:
# info_dataset = pickle.load(handle)
# video_all = []
# for info in info_dataset:
# video_all.append(info[0])
# counter = Counter(video_all)
# print(counter)
# # select the qualified videos with more than 20 windows
# qualify_videos = []
# # set the parameter number of qualified windows
# qualified_window_num = 200
# for vid in counter:
# if counter[vid] > qualified_window_num:
# qualify_videos.append(vid)
# print(len(qualify_videos), 'videos have more than ', qualified_window_num, ' qualified windows.\n')
# print(qualify_videos)
df_subjects = []
suffix = 'num_offsets{}'.format(num_offsets) if num_offsets else ''
title_suffix = '_win{}_str{}_offset{}_rdoffset{}_maxoffset{}_wincrt{}'.\
format(window_size_sec, stride_sec, offset_sec, num_offsets, max_offset, window_criterion)
data_dir = './'
df_start_time = pd.read_csv(settings['STARTTIME_TEST_FILE'])
qualify_videos = df_start_time["video_name"].tolist()
print(qualify_videos)
print(len(qualify_videos))
subjects = list(set([vid[:3] for vid in qualify_videos]))
print(subjects)
print(len(subjects))
fps = settings['FPS']
dir_path = './'
DEVICE = 'CHEST'
SENSOR = 'ACCELEROMETER'
SENSORS = ['ACCELEROMETER_X', 'ACCELEROMETER_Y', 'ACCELEROMETER_Z']
sensor_col_header = ['accx', 'accy', 'accz']
start_time_file = settings['STARTTIME_FILE']
RAW_PATH = settings['raw_path']
RESAMPLE_PATH = settings['reliability_resample_path']
create_folder("result/baseline_pca")
for sub in subjects:
DEVICE = 'CHEST'
SENSOR = 'ACCELEROMETER'
SENSORS = ['ACCELEROMETER_X', 'ACCELEROMETER_Y', 'ACCELEROMETER_Z']
sensor_col_header = ['accx', 'accy', 'accz']
# start_time_file = os.path.join(dir_path, '../../Sense2StopSync/start_time.csv')
start_time_file = settings["STARTTIME_TEST_FILE"]
flow_dir = os.path.join(dir_path, '../../data/flow_pwc/sub{}'.format(sub))
RAW_PATH = os.path.join(dir_path, '../../data/RAW/wild/')
RESAMPLE_PATH = os.path.join(dir_path, '../../data/RESAMPLE200/wild/')
flow_files = [f for f in os.listdir(flow_dir) if os.path.isfile(os.path.join(flow_dir, f))]
flow_files = [f for f in flow_files if f.endswith('.pkl')]
print('subject', sub, ': ', len(flow_files), 'total videos (including unqualified ones)')
video_list = []
offset_list = []
for f in flow_files:
vid_name = f[:-4]
if vid_name not in qualify_videos:
# print(vid_name, 'flow file not exist')
continue
vid_path = os.path.join(flow_dir, vid_name+'.pkl')
out_path = os.path.join(data_dir, 'figures/figures_MIT_pca/corr_flow_averaged_acc_{}.png'.format(vid_name))
# load start end time
offset = 0
df_start_time = pd.read_csv(start_time_file, index_col='video_name')
if vid_name not in df_start_time.index:
# print(vid_name, 'not exist in starttime csv')
continue
print(vid_name)
start_time = df_start_time.loc[vid_name]['start_time']+offset
# load optical flow data and assign unixtime to each frame
motion = pickle.load(open(vid_path, 'rb'))
# step = 1000.0/30.0
step = 1000.0/fps
length = motion.shape[0]
timestamps_int = np.arange(start_time, start_time + length * step, step).astype(int)
# # load sensor data
# interval = [int(start_time), int(start_time) + length * step]
# df = read_data_datefolder_hourfile(RESAMPLE_PATH, sub, DEVICE, SENSOR, *interval)
# len_raw_sensor = len(df)
# # load sensor reliability data
# df_rel = read_data_datefolder_hourfile(RESAMPLE_PATH, sub, DEVICE, SENSOR + '_reliability', *interval)
# # use the threshold ">=8Hz" to select 'good' seconds
# rel_seconds = df_rel[df_rel['SampleCounts'] > 7].sort_values(by='Time')['Time'].values
timestamps_int = timestamps_int[:min(len(timestamps_int), motion.shape[0])]
motion = motion[:min(len(timestamps_int), motion.shape[0]), :]
assert len(timestamps_int) == motion.shape[0]
df_flow = pd.DataFrame({'time': timestamps_int, 'flowx': motion[:, 0], 'flowy': motion[:, 1]})
df_flow['second'] = (df_flow['time']/1000).astype(int)
# # extract the optical flow frames of the good seconds according to sensor data
# df_flow_rel = pd.concat([df_flow[df_flow['second']==i] for i in rel_seconds]).reset_index()
## remove/keep video based on data quality
# print(len(df_flow_rel)/len(df_flow))
# if len(df_flow_rel)/len(df_flow) < 0.7:
# continue
fixedTimeCol = df_flow['time'].values
if CUBIC:
df_flow['time'] = pd.to_datetime(df_flow['time'], unit='ms')
df_flow = df_flow[['flowx', 'flowy', 'time']].set_index('time')
# extract the data of consecutive chunk and resample according to video frame timestamp
df_list = []
for S, col in zip(SENSORS, sensor_col_header):
df = read_data_datefolder_hourfile(RAW_PATH, sub, DEVICE, S, fixedTimeCol[0], fixedTimeCol[-1])
df = df[['time', col]]
if CUBIC == 0:
df_sensor_resample = resample(df, 'time', samplingRate=0,
gapTolerance=200, fixedTimeColumn=fixedTimeCol).set_index('time')
else:
df["time"] = pd.to_datetime(df["time"], unit="ms")
df = df.set_index("time")
df_sensor_resample = df.resample(FRAME_INTERVAL).mean()
# FRAME_INTERVAL as 0.03336707S is the most closest value to 1/29.969664 pandas accepts
df_sensor_resample = df_sensor_resample.interpolate(method="spline", order=3) # cubic spline interpolation
df_list.append(df_sensor_resample)
if CUBIC == 0:
df_list.append(df_flow)
df_resample = pd.concat(df_list, axis=1)
else:
df_sensors = pd.concat(df_list, axis=1)
df_list = [df_sensors, df_flow]
df_resample = pd.merge_asof(df_list[1], df_list[0], on='time', tolerance=pd.Timedelta("30ms"), \
direction='nearest').set_index('time')
# two method to fill na values
# 1 fill with 0
#df_resample = df_resample.fillna(0)
# 2 ffill
df_resample = df_resample.fillna(method='ffill')
df_resample = df_resample.dropna(how='any')
df_resample['diff_flowx'] = df_resample['flowx'].diff()
df_resample['diff_flowy'] = df_resample['flowy'].diff()
df_resample = df_resample.dropna(how='any')
pca_sensor = PCA(n_components=1)
df_resample[['accx', 'accy', 'accz']] -= df_resample[['accx', 'accy', 'accz']].mean()
df_resample['acc_pca'] = pca_sensor.fit_transform(df_resample[['accx', 'accy', 'accz']].to_numpy())
diffflow_mat = df_resample[['diff_flowx', 'diff_flowy']].to_numpy()
diffflow_mat -= np.mean(diffflow_mat, axis=0)
pca_diffflow = PCA(n_components=1)
df_resample['diffflow_pca'] = pca_diffflow.fit_transform(diffflow_mat)
if use_PCA == 1:
fftshift = cross_correlation_using_fft(df_resample['diffflow_pca'].values, df_resample['acc_pca'].values)
else:
fftshift = cross_correlation_using_fft(df_resample['diff_flowx'].values, df_resample['accx'].values)
shift = compute_shift(fftshift)
shift_ms = shift * 1000 / fps
# print('diff_flowx accx delta={:.1f} ms'.format(shift_ms))
video_list.append(vid_name)
offset_list.append(shift_ms)
print(vid_name, shift_ms)
if plot:
plot_MIT_baseline(df_resample, fps, out_path)
df_subj = pd.DataFrame({'video': video_list, 'offset': offset_list})
# print((df_subj))
df_subjects.append(df_subj)
result_df = pd.concat(df_subjects)
result_df = result_df.reset_index()
ave_error = np.mean(np.abs(result_df['offset'].values))
PV300 = np.sum(np.abs(result_df['offset'].values) < 300) / len(result_df) * 100
PV700 = np.sum(np.abs(result_df['offset'].values) < 700) / len(result_df) * 100
result_df.to_csv(os.path.join(data_dir, 'result/baseline_pca/baseline_MIT_entirevideo_MD2K_offset_pad' + title_suffix + '.csv'), index=None)
print("ave_error", "PV300", "PV700")
print(ave_error, PV300, PV700)
print("# videos: ", len(result_df))
return result_df