def test_1x1_nearest(self):
"""Ensure the do-nothing resample is working"""
img = np.ones((1, 1), dtype=np.float64)
result = resample(img, w=1, h=1, method='nearest')
self.assertEqual(img.shape, result.shape)
self.assertEqual(result.dtype, np.float32)
self.assertEqual(img[0, 0], result[0, 0])
def resample(self, weight, noise=(0.0, 0.0, 0.0), eps=1e-3, inv=False):
# cost -> probability
if inv:
weight = 1.0 / (np.add(weight, eps))
# weight rectification
weight[np.isnan(weight)] = 0.0
# naive choice
#idx = np.random.choice(self.size_, size=self.size_, p=prob)
#particles = self.particles_[idx]
# "perfect" resampler
#print 'weight statistics', weight.min(), weight.max(), weight.std() #np.min(weight), np.max(weight), np.std(weight)
weight = (self.gamma_ * self.weights_) + (1.0 - self.gamma_) * weight
self.particles_, self.weights_ = resample(self.particles_, weight)
# apply noise (simple method to sample more variance in candidates)
self.particles_ = np.random.normal(loc=self.particles_, scale=noise)
self.recalc_ = True # need to recompute best particle
# "naive" best
#best_idx = np.argmax(self.weights_)
#return self.particles_[best_idx].copy()
# "weighted mean" best
return self.best
def normalize(clip):
# Convert to 32-bit floating point first of all.
clip = floatscale(clip)
# If there are multiple channels, mix down to mono.
if clip.num_channels > 1:
clip = clip.mean(axis=0)
assert 1 == clip.ndim
# Resample down to no more than 22050 Hz.
if clip.sample_rate > 22050:
clip = resample(clip, 22050)
return clip
def forward(self, text_embed, return_loss = True):
width, num_cutouts = self.image_size, self.num_cutouts
out = self.model()
if not return_loss:
return out
pieces = []
for ch in range(num_cutouts):
size = int(width * torch.zeros(1,).normal_(mean=.8, std=.3).clip(.5, .95))
offsetx = torch.randint(0, width - size, ())
offsety = torch.randint(0, width - size, ())
apper = out[:, :, offsetx:offsetx + size, offsety:offsety + size]
if (self.experimental_resample):
apper = resample(apper, (224, 224))
else:
apper = F.interpolate(apper, (224, 224), **self.interpolation_settings)
pieces.append(apper)
into = torch.cat(pieces)
into = normalize_image(into)
image_embed = perceptor.encode_image(into)
latents, soft_one_hot_classes = self.model.latents()
num_latents = latents.shape[0]
latent_thres = self.model.latents.thresh_lat
lat_loss = torch.abs(1 - torch.std(latents, dim=1)).mean() + \
torch.abs(torch.mean(latents, dim = 1)).mean() + \
4 * torch.max(torch.square(latents).mean(), latent_thres)
for array in latents:
mean = torch.mean(array)
diffs = array - mean
var = torch.mean(torch.pow(diffs, 2.0))
std = torch.pow(var, 0.5)
zscores = diffs / std
skews = torch.mean(torch.pow(zscores, 3.0))
kurtoses = torch.mean(torch.pow(zscores, 4.0)) - 3.0
lat_loss = lat_loss + torch.abs(kurtoses) / num_latents + torch.abs(skews) / num_latents
cls_loss = ((50 * torch.topk(soft_one_hot_classes, largest = False, dim = 1, k = 999)[0]) ** 2).mean()
sim_loss = -self.loss_coef * torch.cosine_similarity(text_embed, image_embed, dim = -1).mean()
return (lat_loss, cls_loss, sim_loss)
def cv_main():
bad_acc = [6.0, 7.0, 8.0, 12.0]
pca = PCA(n_components=3)
with sqlite3.connect(SQLITE_DATABASE_FILE) as conn:
if os.path.exists('mhealth_features.pkl'):
features_mhealth = pd.read_pickle('mhealth_features.pkl')
else:
data = pd.read_sql_query(uci_mhealth.raw_table_query_shared_data, conn)
# data = drop_activities(bad_acc, data)
data = resample(data, 100)
data = deg2rad(data)
# data = data.drop(['timestamp'], axis = 1)
sliding_windows_mhealth = preprocess.full_df_to_sliding_windows(data)
# sliding_windows_mhealth = uci_mhealth.to_sliding_windows_shared_data(conn)
features_mhealth = extract_features(sliding_windows_mhealth, all_feature)
features_mhealth.to_pickle('mhealth_features.pkl')
if os.path.exists('pamap_features.pkl'):
features_pamap = pd.read_pickle('pamap_features.pkl')
else:
data = pd.read_sql_query(uci_pamap2.raw_table_query_shared_data, conn)
# data = resample(data, 50)
data = data.drop(['timestamp'], axis = 1)
sliding_windows_pamap = preprocess.full_df_to_sliding_windows(data)
# sliding_windows_pamap = uci_pamap2.to_sliding_windows_shared_data(conn)
features_pamap = extract_features(sliding_windows_pamap, all_feature)
features_pamap.to_pickle('pamap_features.pkl')
# features_mhealth = extract_features(sliding_windows_mhealth, all_feature)
# features_mhealth.to_pickle('mhealth_features.pkl')
x = StandardScaler().fit_transform(features_mhealth)
pc_mhealth = (pca.fit_transform(x))
x = StandardScaler().fit_transform(features_pamap)
pc_pamap = (pca.fit_transform(x))
[c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12] = centroid(pc_mhealth, features_mhealth)
pamap_labels = []
for i in range(len(features_pamap)):
label = euclidean([c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12], pc_pamap[i, :])
pamap_labels.append(label)
# print(pc_pamap[i, :])
# break
features_pamap['mhealth labels'] = pamap_labels
features_pamap.to_pickle('pamap_features.pkl')
df = activity_similarity(features_pamap, pamap_labels)
print(df)
df.to_pickle('pamap_mhealth_mapping.pkl')
def fetch_meter(meter, connection):
query = ("SELECT date_time, value FROM " + meter + " " +
"WHERE strftime('%s', date_time) " +
"BETWEEN strftime('%s', ?) AND strftime('%s', ?)")
cursor = connection.execute(query, (start_string, end_string))
readings = [[parse_date(row['date_time']), row['value']]
for row in cursor]
if RESAMPLING:
readings = resample(readings, start, end, RESAMPLING_FREQUENCY)
readings = [[calendar.timegm(d.timetuple()) * 1000, v]
for d, v in readings]
metadata = get_meter_metadata(connection, meter)
return {'metadata': metadata, 'readings': readings}
def _resample(self):
"""リサンプリング
"""
wcum = xp.cumsum(self._w)
num = self._num
# idxs = [ i for n in xp.sort(xp.random.rand(num))
# for i in range(num) if n <= wcum[i] ]
# start = 0
# idxs = [ 0 for i in xrange(num) ]
# for i, n in enumerate( xp.sort(xp.random.rand(num)) ):
# for j in range(start, num):
# if n <= wcum[j]:
# idxs[i] = start = j
# break
idxs = resample.resample(num, wcum)
self._pars = self._pars[:, idxs]
self._w = _normalize(self._w[idxs])
def resample(readings, start, end, frequency):
import pandas as pd
from resample import resample
def write_reading(i, v):
d = i.to_pydatetime()
value = v.item()
return [d, value]
if len(readings) > 0:
datetimes, values = zip(*readings)
else:
datetimes, values = [], []
series = pd.Series(values, index=pd.to_datetime(datetimes))
resampled = resample(series, start.replace(second=00), end, frequency,
'nan').dropna()
return [write_reading(i, resampled[i]) for i in resampled.index]
def parse_rx_buff():
global rx_buff, windDAQfile
while True:
idx = rx_buff.find('\r')
if idx == -1:
break
line = rx_buff[:idx]
try:
data = line.split(' ')
# DATA
windSpeed = float(data[1])
windDir = float(data[2])
status = float(data[3].split('*')[0])
time = datetime.datetime.now().timestamp()
except:
windDAQfile.log('Error in converting str to float: line = [' +
line)
try:
# print(time, windSpeed, windDir, status)
time_k, data_k = resample.resample(Fs, time,
[windSpeed, windDir, status])
for j in range(len(time_k)):
timestamp = datetime.datetime.fromtimestamp(
time_k[j]).strftime('%m-%d %H:%M:%S.%f')
print('{}, Ws={:5.1f} (m/s), Wd={:3.0f} (deg)'.format(
timestamp[0:-4], data_k[j][0], data_k[j][1]))
try:
windDAQfile.save(time_k, data_k)
except:
windDAQfile.log('Error in data saving')
except:
windDAQfile.log('Error in resampling')
# try:
if state[1] == 2:
pitft.plotyy(time_k, data_k)
#except:
# windDAQfile.log('Error in plotyy')
rx_buff = rx_buff[(idx + 2):]
return
# --------------------- 数据载入和整理 -------------------------------
target_class = ['W', 'N1', 'N2', 'N3', 'REM']
npz = np.load('channel0-feature.npz')
trainX = npz['X_train']
trainY = npz['y_train']
testX = npz['X_test']
testY = npz['y_test']
# print(trainX.shape)
# assert False
if opt.rebalanced:
trainX, trainY = resample(trainX, trainY, mode='max')
net_arch_list = []
for gap in [False]:
for ap in [2, 3, 4, 6, 8, 16]:
for conv in [[]]:
# for conv in [[[48, 3]], [[48, 3], [12, 3]], [[96, 12], [48, 12], [12, 12]]]:
for dropout in [0.3, 0.5]:
for fc in [[50, 30], [80, 50, 30]]:
# for ap in [2,3,4,6,8]:
# for conv in [[]]:
# # for conv in [[[48, 3]], [[48, 3], [12, 3]], [[96, 12], [48, 12], [12, 12]]]:
# for dropout in [0.5]:
# for fc in [[192,96]]:
net_arch_list.append({
def realtimePitchshift(pitch_ratio):
import wave
import pyaudio
import struct
import time
import numpy as np
from peakdetct import locate_peaks
from resample import resample
num_channels = 1 # Number of channels
Fs = 16000 # Sampling rate (frames/second)
duration = 8
signal_length = duration * Fs
width = 2 # Number of bytes per sample
# define the window size, synthesis hop size, and analysis hop size
window_size = 2048
synthesis_hopsize = window_size / 4
analysis_hopsize = int(synthesis_hopsize / pitch_ratio)
# signal blocks for procesynthesis_hopsizeing and output
delta_phase = np.zeros(window_size / 2 + 1) # delta phase
syn_phase = np.zeros(window_size / 2 + 1,
dtype=complex) # synthesis phase angle
k = np.linspace(0, window_size / 2, window_size / 2 + 1) # ramp
last_phase = np.zeros(window_size / 2 + 1) # last frame phase
accum_phase = np.zeros(window_size / 2 + 1) # accumulated phase
current_frame = np.zeros(window_size / 2 + 1)
expected_phase = k * 2 * np.pi * analysis_hopsize / window_size # expected phase
p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paInt16,
channels=1,
rate=Fs,
input=True,
output=True,
frames_per_buffer=Fs / 4)
# window function
win = np.hanning(window_size)
# human voice frequency range 80 - 250 Hz
k_min = int(20 * window_size / Fs) - 1
# get first chunk of input data
indata = np.zeros(4 * analysis_hopsize + window_size)
indata[0:window_size] = np.array(
struct.unpack('h' * window_size, stream.read(window_size)))
indata[window_size:] = np.array(
struct.unpack('h' * 4 * analysis_hopsize,
stream.read(4 * analysis_hopsize)))
blocksize = window_size + synthesis_hopsize
dout = np.zeros(4 * synthesis_hopsize + window_size)
zpad = np.zeros(blocksize) # zero pad when reading new frames
dataout = ''
Ampmax = 2**15 - 1
pk_indices = range(1025)
print "start."
# while True:
for i in range(int(
(signal_length - window_size) / (analysis_hopsize * 4))):
# initialize the pointers
read_pt = 0
write_pt = 0
while read_pt <= 4 * analysis_hopsize:
# analysis
# take the spectra of the current window
current_frame = np.fft.rfft(win *
indata[read_pt:read_pt + window_size])
# current_frame[:k_min] = 0
# current_frame[k_max:] = 0
# take the phase difference of two consecutive window
current_phase = np.angle(current_frame)
current_magn = abs(current_frame)
delta_phase = current_phase - last_phase
last_phase = np.copy(current_phase)
# subtract expected phase to get delta phase
delta_phase -= expected_phase
delta_phase = np.unwrap(delta_phase)
# accumulate delta phase
accum_phase[pk_indices] = accum_phase[pk_indices] + (
delta_phase[pk_indices] + expected_phase[pk_indices]
) * synthesis_hopsize / analysis_hopsize
# define the region of influence
rotation_angle = accum_phase[pk_indices] - current_phase[pk_indices]
start_point = 0
for k in range(len(pk_indices) - 1):
peak = pk_indices[k]
next_peak = pk_indices[k + 1]
end_point = int((peak + next_peak) / 2)
ri_indices = range(start_point, peak) + range(
peak + 1, end_point)
accum_phase[
ri_indices] = rotation_angle[k] + current_phase[ri_indices]
start_point = end_point
# last peak
ri_indices = range(start_point, next_peak)
accum_phase[ri_indices] = rotation_angle[
len(pk_indices) - 1] + current_phase[ri_indices]
# peak detect
pk_indices = locate_peaks(current_magn)
if len(pk_indices) == 0:
pk_indices = [1]
# synthesis
syn_phase.real, syn_phase.imag = np.cos(accum_phase), np.sin(
accum_phase)
dout[write_pt:write_pt +
window_size] += win * np.fft.irfft(current_magn * syn_phase)
read_pt += analysis_hopsize
write_pt += synthesis_hopsize
output_frame = dout[0:blocksize]
output_frame[output_frame > Ampmax] = Ampmax
output_frame[output_frame < -Ampmax] = -Ampmax
dataout = dataout + struct.pack('h' * len(output_frame),
*list(output_frame))
dout = np.concatenate((dout[blocksize:], zpad))
indata[0:window_size - analysis_hopsize] = np.copy(indata[read_pt:])
next_frame = stream.read(5 * analysis_hopsize)
# zero pad the last frame if necessary
if len(next_frame) < 10 * analysis_hopsize:
zpad2 = np.zeros(5 * analysis_hopsize - len(next_frame) / 2)
zpad2 = struct.pack('h' * len(zpad2), *list(zpad2))
next_frame = next_frame + zpad2
indata[window_size - analysis_hopsize:] = np.array(
struct.unpack('h' * 5 * analysis_hopsize, next_frame))
if len(dataout) / 2 > Fs / pitch_ratio / 4:
samplen = len(dataout) / 2
resamp = resample(np.array(struct.unpack('h' * samplen, dataout)),
pitch_ratio)
output_data = struct.pack('h' * len(resamp), *list(resamp))
stream.write(output_data)
dataout = ''
if len(dataout) / 2 > 0:
samplen = len(dataout) / 2
resamp = resample(np.array(struct.unpack('h' * samplen, dataout)),
pitch_ratio)
output_data = struct.pack('h' * len(resamp), *list(resamp))
stream.write(output_data)
print "done."
stream.stop_stream()
stream.close()
p.terminate()
def analysis(t, data_original, samplerate):
print("analysis(): Starting analysis..")
t_orig = list(range(len(data_original)))
new_sample_rate = 12000
# Resample signal if over 12 kHz
if (samplerate > new_sample_rate):
print("Resampled to " + str(new_sample_rate) + " Hz")
data_resample = resample.resample(samplerate, new_sample_rate,
data_original)
samplerate = new_sample_rate
# Apply Butterworth filter with center Frequency 2400 Hz
filter_config = scipy.signal.butter(1,
2400,
btype='lowpass',
fs=samplerate,
output='sos')
data_resample = signal.sosfilt(filter_config,
np.asarray(data_resample))
filter_config = scipy.signal.butter(1,
2400,
btype='highpass',
fs=samplerate,
output='sos')
data_resample = signal.sosfilt(filter_config,
np.asarray(data_resample))
else:
data_resample = data_original
t = list(range(len(data_resample)))
# Apply Hilbert transform to demodulate signal
data_hilbert = hilbert(data_resample)
# Derivative
data_hilbert_deriv = pd.Series(data_hilbert).diff()
peaks, _ = scipy.signal.find_peaks(data_hilbert_deriv,
distance=int(math.floor(samplerate *
.2)),
prominence=.05)
# Special exception for some sample rates in order to guarantee proper display (namely test images like argentina.wav)
if (samplerate == 11025):
width = int(samplerate / 2) + 1
else:
width = int(samplerate / 4)
new_peaks = []
check_start = 0
check_count = 0
# Loop through peaks and determine if there are 20 evenly spaced peaks indicating a sync header
for i in range(len(peaks) - 1):
if ((peaks[i + 1] - peaks[i]) >
(width * .2)) and ((peaks[i + 1] - peaks[i]) < (width * 1.8)):
check_start = i
check_count = check_count + 1
else:
check_start = 0
check_count = 0
if check_count > 20:
new_peaks.append(peaks[check_start])
if (check_count > 20):
print("Found image headers..")
if len(new_peaks) == 0:
new_peaks.append(0)
plt.figure()
data_hilbert_copy = data_hilbert[:math.floor(len(data_hilbert) / width) *
width]
# Rotate the array so that the image is aligned to the left so that it is not split down the middle
data_hilbert_copy = np.roll(np.reshape(data_hilbert_copy, (-1, (width))),
(0, -new_peaks[0]))
reshaped = np.asarray(data_hilbert_copy)
plt.cla()
plt.imshow(reshaped,
cmap='gray',
vmin=np.mean(data_hilbert_copy) * .7,
vmax=np.mean(data_hilbert_copy) * 1.4)
plt.ylabel("Pixel Number (row)")
plt.xlabel("Pixel Number (column)")
plt.draw()
#plt.show()
plt.show(block=False)
plt.pause(.05)
detailed_graphs = input(
"Enter 1 for detailed analysis graphs, otherwise press any key to quit:"
)
if (detailed_graphs == '1'):
############################################
# Analysis Plots
############################################
#Un-Demodulated Raw Data
plt.figure()
plt.title("Original Signal")
plt.ylabel("Amplitude (unitless)")
plt.xlabel("Sample Number (n)")
plt.plot(t_orig, data_original)
plt.draw()
plt.show(block=False)
#Demodulated (Hilbert)
plt.figure()
plt.title("Demodulated (Hilbert) Signal")
plt.ylabel("Amplitude (unitless)")
plt.xlabel("Sample Number (n)")
plt.plot(t[:len(data_hilbert)], data_hilbert)
plt.draw()
plt.show(block=False)
# Plot the FFT
time_step = 1 / samplerate
sig_fft = fftpack.fft(data_original)
power = np.abs(sig_fft)**2
sample_freq = fftpack.fftfreq(len(data_original), d=time_step)
plt.figure()
plt.title("FFT of Original Signal")
plt.plot(sample_freq, power)
plt.xlabel('Frequency [Hz]')
plt.ylabel('Amplitude (unitless)')
plt.draw()
plt.show(block=False)
#Derivative
data_hilbert_deriv = pd.Series(data_hilbert).diff()
peaks, _ = scipy.signal.find_peaks(data_hilbert_deriv,
distance=int(
math.floor(samplerate * .2)),
prominence=.05)
plt.figure()
plt.title("First Derivative of Demodulated Signal")
plt.ylabel("Amplitude (unitless)")
plt.xlabel("Sample Number (n)")
plt.plot(data_hilbert_deriv)
plt.plot(peaks, data_hilbert_deriv[peaks], "x")
plt.draw()
plt.show(block=False)
plt.figure()
plt.title("Orig Signal With Peak Detection")
plt.ylabel("Amplitude (unitless)")
plt.xlabel("Sample Number (n)")
plt.plot(data_hilbert)
plt.plot(new_peaks, data_hilbert[new_peaks], "bo")
plt.draw()
plt.show()
else:
return 0
frame = cv2.flip(frame, 1)
init_t = init_target(frame)
if init_t is not None:
target, target_model, target_hist, weights, particles = init_t
#TODO MAKE EVENT FOR MOUSE RELEASE INSTEAD OF THIS
else:
ret, frame = capture.read()
while target is None:
init_t = init_target(frame)
if init_t is not None:
target, target_model, target_hist, weights, particles = init_t
print(target_model[4] * target_model[5])
while True:
start = time.time()
particles = resample(particles, weights)
particles = propagate(particles, noise, (width, height),
Events[EVENT_RANDOM_GENERATOR])
# get frame
ret, frame = capture.read()
if not ret:
break
if CAPTURE_FROM == WEBCAM:
frame = cv2.flip(frame, 1)
# draw all particles
frame_copy = copy.deepcopy(frame)
if Events[EVENT_SHOW_PARTICLES]:
for particle in particles:
top_left, bottom_right = particle_center_to_particle_corners(
particle)
continue
for sensor in sensors:
# load df
df_file = os.path.join(baseline_dir, session, 'df_sensor_{}.pkl'.format(sensor))
df_imu = pd.read_pickle(df_file)
st_time = max([df_imu.iloc[0]['SysTime'], df_video.iloc[0]['SysTime']])
en_time = min([df_imu.iloc[len(df_imu) - 1]['SysTime'], df_video.iloc[len(df_video) - 1]['SysTime']])
df_video_tmp = df_video[(df_video['SysTime'] >= st_time) & (df_video['SysTime'] < en_time)]
vid_time_stamps = df_video_tmp['SysTime'].values
if len(vid_time_stamps) == 0:
print(' no intersection between video and imu, skip session {}'.format(session))
skipped_sessions[sensor].append(session)
shifts_syncwise[sensor][i] = np.nan
continue
df_imu = resample(df_imu, 'SysTime', samplingRate=0,
gapTolerance=200, fixedTimeColumn=vid_time_stamps)
df_imu = df_imu[(df_imu['SysTime'] >= st_time) & (df_imu['SysTime'] < en_time)]
if len(df_video_tmp) != len(df_imu):
print(' lengths of video and imu not equal, skip session {}'.format(session))
print('len_vid = {}, len_sen = {}'.format(len(df_video_tmp), len(df_imu)))
skipped_sessions[sensor].append(session)
shifts_syncwise[sensor][i] = np.nan
continue
num_windows = (len(df_video_tmp) - wind_length * FPS) // (FPS * wind_stride) + 1
if num_windows <= 1:
print('not enough windows')
skipped_sessions[sensor].append(session)
shifts_syncwise[sensor][i] = np.nan
continue
valid_sessions[sensor].append(session)
def syncwise():
# load video data
# session_list = open('../../CMU/session_list').readlines()
# video_dir = '../../CMU/video'
opt_dir = '../../CMU/opt_flow'
# IMU_dir = '../../CMU/sensor/'
data_dir = '../../CMU/data'
# FPS = 30
baseline_dir = os.path.join(data_dir, 'baseline')
# offsets = np.zeros(len(session_list))
sensors = ['2794', '2795', '2796', '3261', '3337']
video = '7150991'
# load_df = True
# verbose = True
# draw = True
# sensor_dict = {'2794': 'Left Arm', '2795': 'Back', '2796': 'Left Leg', '3261': 'Right Leg', '3337': 'Right Arm'}
##############################################
## SyncWISE #
##############################################
FPS = 30
wind_stride = 1
wind_length = 60
num_rand = 10
max_shift = 60
mode_video = 'PCA'
mode_imu = 'PCA'
load_window = True
load_score = False
load_results = False
shifts_syncwise = {}
skipped_sessions = {}
# valid_sessions = {}
summ_mats = {}
systematic_window = False
print('systemic windows', systematic_window)
suffix = '_win{}_max{}'.format(wind_length, max_shift)
#sync_data_dir = '/media/yun/08790233DP/SyncWise_subset/SyncWise'+suffix
sync_data_dir = 'tmp_data/SyncWise' + suffix
#sync_data_dir = 'SyncWise'+suffix
print(' mode:', mode_video, mode_imu, wind_length, max_shift)
os.makedirs('figures/win{}_max{}'.format(wind_length, max_shift), exist_ok=True)
offsets, valid_sessions = pickle.load(open('../../CMU/valid_sessions_win30_max60.pkl', 'rb'))
session_list = valid_sessions['2794']
for sensor in sensors:
shifts_syncwise[sensor] = np.zeros(len(session_list))
skipped_sessions[sensor] = []
valid_sessions[sensor] = []
summ_mats[sensor] = []
os.makedirs('figures/win{}_max{}/{}'.format(wind_length, max_shift, sensor), exist_ok=True)
# load ground truth
offsets = np.zeros(len(session_list))
for i, session in enumerate(session_list):
session = session.strip()
opt_file = glob.glob(os.path.join(opt_dir, session+'_Video', session+'_7150991-*.pkl'))
if len(opt_file) > 0:
opt_file = opt_file[0]
offsets[i] = int(opt_file[opt_file.find('-')+1:-4])
offsets = offsets * 1000/FPS
for i, session in enumerate(session_list):
#if i < 87:
#continue
st = time.time()
session = session.strip()
out_dir = os.path.join(sync_data_dir, session)
os.makedirs(out_dir, exist_ok=True)
print('Processing video {}'.format(session))
# load labels
#pkl_file = open(os.path.join(baseline_dir, 'results_baseline_x_X.pkl'), 'rb')
#offsets, shifts_baseline = pickle.load(pkl_file)
df_file = os.path.join(baseline_dir, session, 'df_video_{}.pkl'.format(video))
if os.path.exists(df_file):
df_video = pd.read_pickle(df_file)
else:
print('skipping video {}'.format(session))
for sensor in sensors:
shifts_syncwise[sensor][i] = np.nan
skipped_sessions[sensor].append(session)
continue
for sensor in sensors:
# load df
df_file = os.path.join(baseline_dir, session, 'df_sensor_{}.pkl'.format(sensor))
df_imu = pd.read_pickle(df_file)
st_time = max([df_imu.iloc[0]['SysTime'], df_video.iloc[0]['SysTime']])
en_time = min([df_imu.iloc[len(df_imu) - 1]['SysTime'], df_video.iloc[len(df_video) - 1]['SysTime']])
df_video_tmp = df_video[(df_video['SysTime'] >= st_time) & (df_video['SysTime'] < en_time)]
vid_time_stamps = df_video_tmp['SysTime'].values
if len(vid_time_stamps) == 0:
print(' no intersection between video and imu, skip session {}'.format(session))
skipped_sessions[sensor].append(session)
shifts_syncwise[sensor][i] = np.nan
continue
df_imu = resample(df_imu, 'SysTime', samplingRate=0,
gapTolerance=200, fixedTimeColumn=vid_time_stamps)
df_imu = df_imu[(df_imu['SysTime'] >= st_time) & (df_imu['SysTime'] < en_time)]
if len(df_video_tmp) != len(df_imu):
print(' lengths of video and imu not equal, skip session {}'.format(session))
print('len_vid = {}, len_sen = {}'.format(len(df_video_tmp), len(df_imu)))
skipped_sessions[sensor].append(session)
shifts_syncwise[sensor][i] = np.nan
continue
num_windows = (len(df_video_tmp) - wind_length * FPS) // (FPS * wind_stride) + 1
if num_windows <= 1:
print('not enough windows')
skipped_sessions[sensor].append(session)
shifts_syncwise[sensor][i] = np.nan
continue
valid_sessions[sensor].append(session)
# one window every win_stride seconds
st_win = time.time()
win_path = os.path.join(out_dir, 'win_{}_{}.pkl'.format(sensor, session))
if load_window and os.path.exists(win_path):
window_list = pickle.load(open(win_path, 'rb'))
elif not os.path.exists(win_path):
window_list = [None] * num_windows * num_rand
for j in range(num_windows):
vid_start_frame = j*FPS*wind_stride
df_win_vid = df_video_tmp[vid_start_frame: vid_start_frame + wind_length * FPS]
for k in range(num_rand):
if systematic_window:
rand_offset = int((-max_shift + k * 2 * max_shift / (num_rand - 1)) * FPS)
else:
rand_offset = random.randint(-max_shift * FPS, max_shift * FPS)
rand_offset = np.min([np.max([-vid_start_frame, rand_offset]), len(df_video_tmp) - wind_length * FPS - vid_start_frame])
sen_start_frame = vid_start_frame + rand_offset
df_win_sen = df_imu[sen_start_frame: sen_start_frame + wind_length * FPS]
window_list[j * num_rand + k] = (df_win_vid, df_win_sen, rand_offset)
# pickle.dump(window_list, open(win_path, 'wb'))
en_win = time.time()
print(' Time to get window: ', en_win - st_win)
st_score = time.time()
score_path = os.path.join(out_dir, 'score_{}_{}_xx.pkl'.format(sensor, session))
if load_score and os.path.exists(score_path):
scores_mat = pickle.load(open(score_path, 'rb'))
elif not os.path.exists(score_path):
scores_mat = np.zeros((num_windows * num_rand, 2))
# calculate conf, drift
for j in range(len(window_list)):
df_win_vid, df_win_sen, rand_offset = window_list[j]
scores_mat[j] = drift_confidence(df_win_vid['diff_flow{}'.format(mode_video)].values, \
df_win_sen['Accel_{}'.format(mode_imu)].values)
scores_mat[j, 1] += rand_offset
#scores_mat[:, 1] = scores_mat[:, 1] * 1000/FPS
# pickle.dump(scores_mat, open(score_path, 'wb'))
else:
scores_mat = np.zeros((num_windows * num_rand, 2))
# calculate conf, drift
for j in range(len(window_list)):
df_win_vid, df_win_sen, rand_offset = window_list[j]
scores_mat[j] = drift_confidence(df_win_vid['diff_flow{}'.format(mode_video)].values, \
df_win_sen['Accel_{}'.format(mode_imu)].values)
scores_mat[j, 1] += rand_offset
# scores_mat[:, 1] = scores_mat[:, 1] * 1000/FPS
# pickle.dump(scores_mat, open(score_path, 'wb'))
en_score = time.time()
print(scores_mat[0, :])
print(' time to get scores: ', en_score - st_score)
st_res = time.time()
#path = '/home/yun/Dropbox (GaTech)/code/figures/{}_{}.jpg'.format(session, sensor)
res_path = os.path.join(out_dir, 'res_{}_{}_xx.pkl'.format(sensor, session))
path = 'figures/win{}_max{}/{}/{}.jpg'.format(wind_length, max_shift, sensor, session)
if load_results and os.path.exists(res_path):
res = pickle.load(open(res_path, 'rb'))
else:
res = gaussianVoting(scores_mat, kernel_var=90, draw=True, path=path)
# pickle.dump(res, open(res_path, 'wb'))
en_res = time.time()
print(' time to get res:', en_res - st_res)
# 'session', 'offset', 'abs_offset', 'num_segs', 'conf', 'mu', 'sigma', 'mu_var', 'sigma_var', 'abs_mu'
summ = [session, res[0], abs(res[0]), num_windows * num_rand, res[1], res[2][0][0], res[2][0][1], res[2][1][0, 0], \
res[2][1][1, 1], abs(res[2][0][0])]
summ_mats[sensor].append(summ)
shifts_syncwise[sensor][i] = summ[1]
en = time.time()
print(' processing time {} seconds'.format(en-st))
# result_file = os.path.join(sync_data_dir, 'results_SyncWISE_win{}_max{}_xx.pkl'.format(wind_length, max_shift))
#offsets, shifts_syncwise = pickle.load(open(result_file, 'rb'))
# pickle.dump([offsets, shifts_syncwise], open(result_file, 'wb'))
# valid_session_file = os.path.join(sync_data_dir, 'valid_sessions_win{}_max{}.pkl'.format(wind_length, max_shift))
# pickle.dump([offsets, valid_sessions], open(valid_session_file, 'wb'))
for sensor in sensors:
shifts_syncwise[sensor][67:91] = np.nan
error = compute_error(offsets, shifts_syncwise)
print(suffix)
print(error)
error.to_csv(os.path.join(sync_data_dir, 'df_error{}.csv'.format(sensor, suffix)))
cols = ['session', 'offset', 'abs_offset', 'num_segs', 'conf', 'mu', 'sigma', 'mu_var', 'sigma_var', 'abs_mu']
for sensor in sensors:
df_summ = pd.DataFrame(data=summ_mats[sensor], columns=cols)
df_summ.to_csv(os.path.join(sync_data_dir, 'df_summary_sensor{}{}.csv'.format(sensor, suffix)))
def cv_main():
with sqlite3.connect(SQLITE_DATABASE_FILE) as conn:
if os.path.exists('mhealth_features.pkl'):
features_mhealth = pd.read_pickle('mhealth_features.pkl')
features_mhealth = axes_normaliser_1(
features_mhealth, ['chest', 'left_ankle', 'right_lower_arm'],
['acc', 'gyro'])
# print(features_mhealth)
else:
data_mhealth = pd.read_sql_query(
uci_mhealth.raw_table_query_shared_data, conn)
data_mhealth = resample(data_mhealth, 100)
data_mhealth = deg2rad(data_mhealth)
data_mhealth = axes_normaliser_1(
data_mhealth, ['chest', 'left_ankle', 'right_lower_arm'],
['chest', 'ankle', 'hand'], ['acc', 'gyro'], 1, 4, [])
data_mhealth.index = np.linspace(0,
len(data_mhealth) - 1,
len(data_mhealth))
data_mhealth.to_pickle('mhealth_resampled.pkl')
# data = axes_normaliser_2(data, ['chest', 'ankle', 'hand'], ['acc', 'gyro'])
# print(data)
# data_mhealth.to_pickle('mhealth_reformatted.pickle')
# print(data)
# data = drop_activities(bad_acc, data)
# sliding_windows_mhealth = preprocess.full_df_to_sliding_windows(data)
# sliding_windows_mhealth = uci_mhealth.to_sliding_windows_shared_data(conn)
# features_mhealth = extract_features(sliding_windows_mhealth, all_feature)
# features_mhealth.to_pickle('mhealth_features.pkl')
if os.path.exists('pamap_features.pkl'):
features_pamap = pd.read_pickle('pamap_features.pkl')
else:
data_pamap = pd.read_sql_query(
uci_pamap2.raw_table_query_shared_data, conn)
maps = [[1, 3], [3, 1], [5, 11], [6, 9]]
subj_maps = np.unique(data_mhealth['subject_id'].values)
data_pamap = axes_normaliser_1(data_pamap,
['chest', 'ankle', 'hand'],
['chest', 'ankle', 'hand'],
['acc', 'gyro'], 3, 4, maps)
data_pamap.to_pickle('pamap.pkl')
data_pamap.index = [
x + len(data_mhealth) for x in data_pamap.index
]
# print(data)
# data.to_pickle('pamap_reformatted.pickle')
#
# # # data = resample(data, 50)
# # data = data.drop(['timestamp'], axis = 1)
# # sliding_windows_pamap = preprocess.full_df_to_sliding_windows(data)
# # # sliding_windows_pamap = uci_pamap2.to_sliding_windows_shared_data(conn)
# # features_pamap = extract_features(sliding_windows_pamap, all_feature)
# # features_pamap.to_pickle('pamap_features.pkl')
# if os.path.exists('smartphone_features.pkl'):
# features_pamap = pd.read_pickle('smartphone_features.pkl')
# else:
# data_smartphone = pd.read_sql_query(uci_smartphone.raw_table_query_shared_data, conn)
# maps = [[1, 4], [2, 50], [3, 51], [4, 2], [5, 1], [6, 3]]
# subj_maps = np.unique(data_smartphone['subject_id'].values)
# data_smartphone = axes_normaliser_1(data_smartphone, ['body'], ['chest'], ['acc', 'gyro'], 5, 1, maps, subj_maps)
# data_smartphone.index = [x + len(data_pamap) +len(data_mhealth) for x in data_smartphone.index]
# # print(data)
# data.to_pickle('pamap_reformatted.pickle')
#
# # # data = resample(data, 50)
# # data = data.drop(['timestamp'], axis = 1)
# # sliding_windows_pamap = preprocess.full_df_to_sliding_windows(data)
# # # sliding_windows_pamap = uci_pamap2.to_sliding_windows_shared_data(conn)
# # features_pamap = extract_features(sliding_windows_pamap, all_feature)
# # features_pamap.to_pickle('pamap_features.pkl')
data = pd.concat([data_mhealth, data_pamap])
data.to_pickle('data.pkl')
data = axes_normaliser_2(data, ['chest', 'ankle', 'hand'], ['acc', 'gyro'])
data.to_pickle('fully_reformatted_10000_resampled.pkl')
p.measure(s)
# Normalize likelihoods.
if not particle.normalize(particles):
print("lost")
particles = gen_particles()
continue
# Report centroid.
c = particle.centroid(particles)
print("actual:", vehicle.x, " imputed:", c)
states.append((
vehicle.clone(),
[p.clone() for p in particles],
c,
))
# Resample as needed.
if t >= rst:
print("resampling")
particles = resample.resample(particles, nparticles)
t = 0
# Advance the state.
vehicle.advance(dt)
for p in particles:
p.advance(dt)
t += dt
render.render(sensors, states, dt)
settings = open("settings.csv", "r")
filename = settings.read()
settings.close()
print("Reading file: " + str(filename))
#t,data_original,samplerate = read_wav.read_wav('argentina.wav')
#samplerate = 11025
t, data_original, samplerate = read_raw.read_raw(str(filename))
print("analysis(): Starting analysis..")
new_sample_rate = 12000
print(new_sample_rate)
data_original = rs.resample(samplerate, new_sample_rate, data_original)
samplerate = new_sample_rate
t = list(range(len(data_original)))
# Apply Butterworth filter with center Frequency 2400 Hz
filter_config = scipy.signal.butter(1,
2400,
btype='lowpass',
fs=samplerate,
output='sos')
data_original = signal.sosfilt(filter_config, np.asarray(data_original))
filter_config = scipy.signal.butter(1,
2400,
btype='highpass',
fs=samplerate,
#import pyximport; pyximport.install()
#import resample_cython
from resample import resample
import pylab as pl
l = misc.ascent()[::2, ::2] / 1.
l.shape = l.shape + (1, )
l = l.astype('float32')
#a = np.random.randn(5, 3, 4).astype('float32')
l = np.tile(l, 96)
print l.shape
#r = resample_cython.upsample_cython(a, (6, 4, 4))
#print r
import time
N = 100
start = time.time()
for i in xrange(N):
#out = np.empty((1024, 1024, l.shape[-1]), dtype='float32')
out = resample(l, (1024, 1024, l.shape[-1]))
print out.shape
end = time.time()
fps = N / (end - start)
print fps
print 1. / fps
#pl.matshow(out[:,:,-1])
#pl.matshow(out[:,:,20])
#pl.show()
def interactive(self):
"""
Start interactive mode.
- See help in the class for key mapping
"""
global ncuarf, nxcuar
extv = self.extv
peaks = self.pixs
pl.clf()
# Plot the reference lists. (in a Fits file).
# Arc file WCS
cdelt = self.wcs['cdelt']
crval = self.wcs['crval']
crpix = self.wcs['crpix']
# Reffile min, max wavelength
wmin = self.wmin
wmax = self.wmax
# 1st subplot --------------------- REFFILE
# Resample cuarf to the ARC size
xsize = self.lpix.size
resmp = False
cuarf = pf.getdata(self.reffile)
if resmp:
newf = resample(cuarf,(xsize,))
else:
newf = cuarf
# Scale cuarf to the same as lpix
spec_max = np.max(self.lpix)
ymax = newf.max()
ncuarf = newf*(spec_max/ymax)
self.uu=ncuarf
print 'WCS:',self.wcs, 'size of reff: ',ncuarf.size
halfw = max(crval-wmin,wmax-crval)
#print 'Reff:',crval,halfw,(wmin,wmax),crval-halfw,crval+halfw
# x-axis values in wavelength units
nxcuar = np.linspace(wmin+1,wmax+1,len(newf))
pref = pl.subplot(211)
pref.plot(nxcuar,ncuarf)
basename = os.path.basename(self.reffile)
pref.set_title('Reference: '+basename+' ('+self.ad.instrument()+')')
self.subu = pref
# 2nd subplot ---------------------- ARC
bname = os.path.basename(self.filename)
if extv>1:
bname+='[SCI,%d]'%extv
xlabel = bname+' middle row'
halfp = max(crpix-0,len(self.lpix)-crpix)
#print 'Half crpix:',halfp,(crpix+halfp,max(0,crpix-halfp))
pspec = pl.subplot(212)
self.subm = pspec
pspec.plot(self.lpix)
pspec.set_xlim(len(self.lpix)-1,0)
#pspec.set_xlim(crpix+halfp,max(0,crpix-halfp))
pspec.set_xlabel(xlabel)
# Align the Reference plot to the Arc plot
nwmin = (0-crpix)*cdelt + crval
nwmax = (len(self.lpix)-crpix)*cdelt + crval
#nwmin = halfp*cdelt + crval
#nwmax = -halfp*cdelt + crval
pref.set_xlim(nwmax,nwmin)
indx = np.searchsorted(nxcuar,(nwmin,nwmax))
if cdelt > 0:
indx = indx -1
p1 = min(indx)
p2 = min(max(indx),len(nxcuar)-1)
#print '.....Align 1st plot:',halfp,(nxcuar[p1],nxcuar[p2])
#pref.set_xlim(nxcuar[p1],nxcuar[p2])
#if cdelt < 0:
# pref.set_xlim(nwmax,nwmin)
#else:
#pref.set_xlim(nwmin,nwmax)
ymax = ncuarf[p1:p2].max()
pref.set_ylim(-ymax*.1,ymax+ymax*.2)
# Redefine default keymapping 'l'
pl.rcParams['keymap.yscale'] = 'i'
self.cid = pl.connect('key_press_event',self.key_events)
pl.draw()
def test_bilinear_downsample(self):
img = np.uint8([[[1, 2, 3], [5, 6, 9]]])
result = resample(img, w=1, h=1, method='bilinear')
self.assertEqual(flatten(result[0, 0, :]),
flatten(np.uint8([3, 4, 6])))
def test_2_to_4_nearest(self):
img = np.float32([[1, 2], [3, 4]])
result = resample(img, w=4, h=4, method='nearest')
self.assertEqual(result.shape, (4, 4))
self.assertEqual(result[1, 1], 1)
self.assertEqual(result[2, 2], 4)
def pitchshift(wavfile,pitch_ratio,mode):
import wave
import pyaudio
import struct
import time
import numpy as np
from peakdetct import locate_peaks
from resample import resample
# Read the wave file properties
wf = wave.open( wavfile, 'rb')
num_channels = wf.getnchannels() # Number of channels
Fs = wf.getframerate() # Sampling rate (frames/second)
signal_length = wf.getnframes() # Signal length
width = wf.getsampwidth() # Number of bytes per sample
if num_channels > 1:
print "Multichannel wavefiles are not supported."
return
# define the window size, synthesis hop size, and analysis hop size
window_size = 2048
synthesis_hopsize = window_size/4
analysis_hopsize = int(synthesis_hopsize/pitch_ratio)
# signal blocks for procesynthesis_hopsizeing and output
delta_phase = np.zeros(window_size/2+1) # delta phase
syn_phase = np.zeros(window_size/2+1, dtype=complex) # synthesis phase angle
k = np.linspace(0, window_size/2, window_size/2+1) # ramp
last_phase = np.zeros(window_size/2+1) # last frame phase
accum_phase = np.zeros(window_size/2+1) # accumulated phase
current_frame = np.zeros(window_size/2+1)
expected_phase = k*2*np.pi*analysis_hopsize/window_size # expected phase
p = pyaudio.PyAudio()
stream = p.open(format = pyaudio.paInt16,
channels = 1,
rate = Fs,
input = False,
output = True,
frames_per_buffer = 5*analysis_hopsize)
# window function
win = np.hanning(window_size)
# get first chunk of input data
indata = np.zeros(4*analysis_hopsize+window_size)
indata[0:window_size] = np.array(struct.unpack('h'*window_size,wf.readframes(window_size)))
indata[window_size:] = np.array(struct.unpack('h'*4*analysis_hopsize,wf.readframes(4*analysis_hopsize)))
blocksize = window_size+synthesis_hopsize
dout = np.zeros(4*synthesis_hopsize+window_size)
zpad = np.zeros(blocksize) # zero pad when reading new frames
dataout = ''
Ampmax = 2**15-1
pk_indices = range(1025)
for i in range(int((signal_length-window_size)/(analysis_hopsize*4))):
# initialize the pointers
read_pt = 0
write_pt = 0
while read_pt <= 4*analysis_hopsize:
# analysis
# take the spectra of the current window
current_frame = np.fft.rfft(win*indata[read_pt:read_pt+window_size])
# take the phase difference of two consecutive window
current_phase = np.angle(current_frame)
current_magn = abs(current_frame)
delta_phase = current_phase - last_phase
last_phase = np.copy(current_phase)
# subtract expected phase to get delta phase
delta_phase -= expected_phase
delta_phase = np.unwrap(delta_phase)
# accumulate delta phase
accum_phase[pk_indices] = accum_phase[pk_indices] + (delta_phase[pk_indices] + expected_phase[pk_indices])*synthesis_hopsize/analysis_hopsize
# define the region of influence
rotation_angle = accum_phase[pk_indices] - current_phase[pk_indices]
start_point = 0
for k2 in range(len(pk_indices)-1):
peak = pk_indices[k2]
next_peak = pk_indices[k2+1]
end_point = int((peak + next_peak)/2)+1
ri_indices = range(start_point,peak)+ range(peak+1,end_point)
accum_phase[ri_indices] = rotation_angle[k2] + current_phase[ri_indices]
start_point = end_point
# last peak
ri_indices = range(start_point,next_peak)
accum_phase[ri_indices] = rotation_angle[len(pk_indices)-1] + current_phase[ri_indices]
# peak detect
pk_indices = locate_peaks(current_magn)
if len(pk_indices) == 0:
pk_indices = [1]
# synthesis
syn_phase.real, syn_phase.imag = np.cos(accum_phase), np.sin(accum_phase)
dout[write_pt:write_pt+window_size] += win*np.fft.irfft(current_magn*syn_phase)
read_pt += analysis_hopsize
write_pt += synthesis_hopsize
output_frame = dout[0:blocksize]
# clip
output_frame[output_frame>Ampmax] = Ampmax
dataout = dataout+struct.pack('h'*len(output_frame), *list(output_frame))
dout = np.concatenate((dout[blocksize:],zpad))
indata[0:window_size-analysis_hopsize] = np.copy(indata[read_pt:])
next_frame = wf.readframes(5*analysis_hopsize)
# zero pad the last frame if necessary
if len(next_frame) < 10*analysis_hopsize:
zpad2 = np.zeros(5*analysis_hopsize-len(next_frame)/2)
zpad2 = struct.pack('h'*len(zpad2), *list(zpad2))
next_frame = next_frame+zpad2
indata[window_size-analysis_hopsize:] = np.array(struct.unpack('h'*5*analysis_hopsize,next_frame))
stream.stop_stream()
stream.close()
p.terminate()
if mode == "tune":
samplen = len(dataout)/2
resamp = resample(np.array(struct.unpack('h'*samplen,dataout)),pitch_ratio)
output_data = struct.pack('h'*len(resamp), *list(resamp))
elif mode == "stretch":
output_data = dataout
if mode == "tune":
output_wavefile = wavfile[:-4]+'_tuned.wav'
elif mode == "stretch":
output_wavefile = wavfile[:-4]+'_stretched.wav'
print 'Writing to wave file', output_wavefile
wf = wave.open(output_wavefile, 'w') # wave file
wf.setnchannels(1) # two channel (stereo)
wf.setsampwidth(width) # two bytes per sample
wf.setframerate(Fs) # samples per second
wf.writeframes(output_data)
wf.close()
time.sleep(0.1)
# Now play from the wave file
wf = wave.open(output_wavefile, 'rb')
# Define callback function for playing audio
def callback_play(input_string, block_size, time_info, status):
output_string = wf.readframes(block_size)
return (output_string, pyaudio.paContinue)
p2 = pyaudio.PyAudio()
stream2 = p2.open(format = p2.get_format_from_width(width),
channels = 1,
rate = Fs,
input = False,
output = True,
stream_callback = callback_play)
print '* Playing wave file', output_wavefile, '...'
stream2.start_stream()
while stream2.is_active():
time.sleep(0.1)
stream2.stop_stream()
print 'Done.'
stream2.close()
p2.terminate()
def baseline_MIT_video_MD2K(window_size_sec, stride_sec, num_offsets, max_offset, window_criterion, offset_sec = 0, plot=0, use_PCA=0):
# #subjects = ['202', '205', '211', '235', '236', '238', '240', '243']
# dir_path = os.path.dirname(os.path.realpath(__file__))
# # df_start_time = pd.read_csv(settings['STARTTIME_TEST_FILE'])
# # df_start_time = csv_read(os.path.join(dir_path, '../../Sense2StopSync/start_time.csv')).set_index('video_name')
# df_start_time = csv_read(settings['STARTTIME_TEST_FILE']).set_index('video_name')
# video_names = df_start_time.index.tolist()
# subjects = list(set([vid.split(' ')[0] for vid in video_names]))
# fps = 29.969664
# data_dir = '/media/yun/08790233DP/sync_data_2nd'
# with open(data_dir+'/all_video' + title_suffix + '_info_dataset.pkl', 'rb') as handle:
# info_dataset = pickle.load(handle)
# video_all = []
# for info in info_dataset:
# video_all.append(info[0])
# counter = Counter(video_all)
# print(counter)
# # select the qualified videos with more than 20 windows
# qualify_videos = []
# # set the parameter number of qualified windows
# qualified_window_num = 200
# for vid in counter:
# if counter[vid] > qualified_window_num:
# qualify_videos.append(vid)
# print(len(qualify_videos), 'videos have more than ', qualified_window_num, ' qualified windows.\n')
# print(qualify_videos)
df_subjects = []
suffix = 'num_offsets{}'.format(num_offsets) if num_offsets else ''
title_suffix = '_win{}_str{}_offset{}_rdoffset{}_maxoffset{}_wincrt{}'.\
format(window_size_sec, stride_sec, offset_sec, num_offsets, max_offset, window_criterion)
data_dir = './'
df_start_time = pd.read_csv(settings['STARTTIME_TEST_FILE'])
qualify_videos = df_start_time["video_name"].tolist()
print(qualify_videos)
print(len(qualify_videos))
subjects = list(set([vid[:3] for vid in qualify_videos]))
print(subjects)
print(len(subjects))
fps = settings['FPS']
dir_path = './'
DEVICE = 'CHEST'
SENSOR = 'ACCELEROMETER'
SENSORS = ['ACCELEROMETER_X', 'ACCELEROMETER_Y', 'ACCELEROMETER_Z']
sensor_col_header = ['accx', 'accy', 'accz']
start_time_file = settings['STARTTIME_FILE']
RAW_PATH = settings['raw_path']
RESAMPLE_PATH = settings['reliability_resample_path']
create_folder("result/baseline_pca")
for sub in subjects:
DEVICE = 'CHEST'
SENSOR = 'ACCELEROMETER'
SENSORS = ['ACCELEROMETER_X', 'ACCELEROMETER_Y', 'ACCELEROMETER_Z']
sensor_col_header = ['accx', 'accy', 'accz']
# start_time_file = os.path.join(dir_path, '../../Sense2StopSync/start_time.csv')
start_time_file = settings["STARTTIME_TEST_FILE"]
flow_dir = os.path.join(dir_path, '../../data/flow_pwc/sub{}'.format(sub))
RAW_PATH = os.path.join(dir_path, '../../data/RAW/wild/')
RESAMPLE_PATH = os.path.join(dir_path, '../../data/RESAMPLE200/wild/')
flow_files = [f for f in os.listdir(flow_dir) if os.path.isfile(os.path.join(flow_dir, f))]
flow_files = [f for f in flow_files if f.endswith('.pkl')]
print('subject', sub, ': ', len(flow_files), 'total videos (including unqualified ones)')
video_list = []
offset_list = []
for f in flow_files:
vid_name = f[:-4]
if vid_name not in qualify_videos:
# print(vid_name, 'flow file not exist')
continue
vid_path = os.path.join(flow_dir, vid_name+'.pkl')
out_path = os.path.join(data_dir, 'figures/figures_MIT_pca/corr_flow_averaged_acc_{}.png'.format(vid_name))
# load start end time
offset = 0
df_start_time = pd.read_csv(start_time_file, index_col='video_name')
if vid_name not in df_start_time.index:
# print(vid_name, 'not exist in starttime csv')
continue
print(vid_name)
start_time = df_start_time.loc[vid_name]['start_time']+offset
# load optical flow data and assign unixtime to each frame
motion = pickle.load(open(vid_path, 'rb'))
# step = 1000.0/30.0
step = 1000.0/fps
length = motion.shape[0]
timestamps_int = np.arange(start_time, start_time + length * step, step).astype(int)
# # load sensor data
# interval = [int(start_time), int(start_time) + length * step]
# df = read_data_datefolder_hourfile(RESAMPLE_PATH, sub, DEVICE, SENSOR, *interval)
# len_raw_sensor = len(df)
# # load sensor reliability data
# df_rel = read_data_datefolder_hourfile(RESAMPLE_PATH, sub, DEVICE, SENSOR + '_reliability', *interval)
# # use the threshold ">=8Hz" to select 'good' seconds
# rel_seconds = df_rel[df_rel['SampleCounts'] > 7].sort_values(by='Time')['Time'].values
timestamps_int = timestamps_int[:min(len(timestamps_int), motion.shape[0])]
motion = motion[:min(len(timestamps_int), motion.shape[0]), :]
assert len(timestamps_int) == motion.shape[0]
df_flow = pd.DataFrame({'time': timestamps_int, 'flowx': motion[:, 0], 'flowy': motion[:, 1]})
df_flow['second'] = (df_flow['time']/1000).astype(int)
# # extract the optical flow frames of the good seconds according to sensor data
# df_flow_rel = pd.concat([df_flow[df_flow['second']==i] for i in rel_seconds]).reset_index()
## remove/keep video based on data quality
# print(len(df_flow_rel)/len(df_flow))
# if len(df_flow_rel)/len(df_flow) < 0.7:
# continue
fixedTimeCol = df_flow['time'].values
if CUBIC:
df_flow['time'] = pd.to_datetime(df_flow['time'], unit='ms')
df_flow = df_flow[['flowx', 'flowy', 'time']].set_index('time')
# extract the data of consecutive chunk and resample according to video frame timestamp
df_list = []
for S, col in zip(SENSORS, sensor_col_header):
df = read_data_datefolder_hourfile(RAW_PATH, sub, DEVICE, S, fixedTimeCol[0], fixedTimeCol[-1])
df = df[['time', col]]
if CUBIC == 0:
df_sensor_resample = resample(df, 'time', samplingRate=0,
gapTolerance=200, fixedTimeColumn=fixedTimeCol).set_index('time')
else:
df["time"] = pd.to_datetime(df["time"], unit="ms")
df = df.set_index("time")
df_sensor_resample = df.resample(FRAME_INTERVAL).mean()
# FRAME_INTERVAL as 0.03336707S is the most closest value to 1/29.969664 pandas accepts
df_sensor_resample = df_sensor_resample.interpolate(method="spline", order=3) # cubic spline interpolation
df_list.append(df_sensor_resample)
if CUBIC == 0:
df_list.append(df_flow)
df_resample = pd.concat(df_list, axis=1)
else:
df_sensors = pd.concat(df_list, axis=1)
df_list = [df_sensors, df_flow]
df_resample = pd.merge_asof(df_list[1], df_list[0], on='time', tolerance=pd.Timedelta("30ms"), \
direction='nearest').set_index('time')
# two method to fill na values
# 1 fill with 0
#df_resample = df_resample.fillna(0)
# 2 ffill
df_resample = df_resample.fillna(method='ffill')
df_resample = df_resample.dropna(how='any')
df_resample['diff_flowx'] = df_resample['flowx'].diff()
df_resample['diff_flowy'] = df_resample['flowy'].diff()
df_resample = df_resample.dropna(how='any')
pca_sensor = PCA(n_components=1)
df_resample[['accx', 'accy', 'accz']] -= df_resample[['accx', 'accy', 'accz']].mean()
df_resample['acc_pca'] = pca_sensor.fit_transform(df_resample[['accx', 'accy', 'accz']].to_numpy())
diffflow_mat = df_resample[['diff_flowx', 'diff_flowy']].to_numpy()
diffflow_mat -= np.mean(diffflow_mat, axis=0)
pca_diffflow = PCA(n_components=1)
df_resample['diffflow_pca'] = pca_diffflow.fit_transform(diffflow_mat)
if use_PCA == 1:
fftshift = cross_correlation_using_fft(df_resample['diffflow_pca'].values, df_resample['acc_pca'].values)
else:
fftshift = cross_correlation_using_fft(df_resample['diff_flowx'].values, df_resample['accx'].values)
shift = compute_shift(fftshift)
shift_ms = shift * 1000 / fps
# print('diff_flowx accx delta={:.1f} ms'.format(shift_ms))
video_list.append(vid_name)
offset_list.append(shift_ms)
print(vid_name, shift_ms)
if plot:
plot_MIT_baseline(df_resample, fps, out_path)
df_subj = pd.DataFrame({'video': video_list, 'offset': offset_list})
# print((df_subj))
df_subjects.append(df_subj)
result_df = pd.concat(df_subjects)
result_df = result_df.reset_index()
ave_error = np.mean(np.abs(result_df['offset'].values))
PV300 = np.sum(np.abs(result_df['offset'].values) < 300) / len(result_df) * 100
PV700 = np.sum(np.abs(result_df['offset'].values) < 700) / len(result_df) * 100
result_df.to_csv(os.path.join(data_dir, 'result/baseline_pca/baseline_MIT_entirevideo_MD2K_offset_pad' + title_suffix + '.csv'), index=None)
print("ave_error", "PV300", "PV700")
print(ave_error, PV300, PV700)
print("# videos: ", len(result_df))
return result_df
cx = np.append(x, x[0])
cy = np.append(y, y[0])
plt.axis([0, len(im[1]), 0, len(im)])
plt.plot(cx, cy, '+-')
#plt.gca().invert_xaxis()
#plt.gca().invert_yaxis()
plt.show
if __name__ == "__main__":
from resample import resample
im = np.zeros((300,300))
im[120:180,120:180] = 1
x,y,points = getInitialCurve(im, 30)
drawCurve(x, y, im)
plt.plot(points[:,0], points[:,1], 'r+')
f = np.vstack((x,y)).T
f = np.matrix(f)
rf = np.array(resample(f))
rfx = rf[:,0]
rfy = rf[:,1]
drawCurve(rfx, rfy)
#import pyximport; pyximport.install()
#import resample_cython
from resample import resample
import pylab as pl
l = misc.lena()[::2, ::2]/1.
l.shape = l.shape + (1,)
l = l.astype('float32')
#a = np.random.randn(5, 3, 4).astype('float32')
l = np.tile(l, 96)
print l.shape
#r = resample_cython.upsample_cython(a, (6, 4, 4))
#print r
import time
N = 100
start = time.time()
for i in xrange(N):
#out = np.empty((1024, 1024, l.shape[-1]), dtype='float32')
out =resample(l, (1024, 1024, l.shape[-1]))
print out.shape
end = time.time()
fps = N / (end - start)
print fps
print 1. / fps
#pl.matshow(out[:,:,-1])
#pl.matshow(out[:,:,20])
#pl.show()
ax_stim_clark = f.add_subplot(gs[0, 0])
ax_stim_clark.plot(clark_reader.t_hard_edges,
clark_reader.stim_hard_edges,
color='black', lw=2.4)
ax_stim_clark.patch.set_visible(False)
ax_stim_clark.set_frame_on(False)
ax_photor = f.add_subplot(gs[1: 3, 0], sharex=ax_stim_clark)
ax_photor.plot(t_clark - pre_stimulus_t, data_array_clark['s0'], lw=2.)
ax_L1_clark = f.add_subplot(gs[3: 5, 0], sharex=ax_stim_clark)
ax_L2_clark = f.add_subplot(gs[5: 7, 0], sharex=ax_stim_clark)
t_indices = np.where(t_clark > 2.0)
new_t = t_clark[t_indices] - 2.0
resampled_x_L1 = resample(clark_reader.response_hard_edges_L1[:, 1],
clark_reader.response_hard_edges_L1[:, 0], new_t)
resampled_x_L2 = resample(clark_reader.response_hard_edges_L2[:, 1],
clark_reader.response_hard_edges_L2[:, 0], new_t)
sim, model_z, clark_z = similarity.similarity_z(data_array_clark['L1'][t_indices],
resampled_x_L1)
ax_L1_clark.plot(new_t, model_z, label='model', lw=2.)
ax_L1_clark.plot(new_t, clark_z, label='Clark', lw=2.)
ax_L1_clark.legend(loc=1, fontsize=17)
ax_L1_clark.legend(bbox_to_anchor=(1.05, 0.999), loc=1, fontsize=17)
sim, model_z, clark_z = similarity.similarity_z(data_array_clark['L2'][t_indices],
resampled_x_L2)
ax_L2_clark.plot(new_t, model_z, label='model', lw=2.)
def baseline(mode):
# load video data
# session_list = open('../../CMU/session_list').readlines()
video_dir = '../../CMU/video'
opt_dir = '../../CMU/opt_flow'
IMU_dir = '../../CMU/sensor/'
data_dir = '../../CMU/data'
FPS = 30
baseline_dir = os.path.join(data_dir, 'baseline')
# offsets = np.zeros(len(session_list))
sensors = ['2794', '2795', '2796', '3261', '3337']
video = '7150991'
load_df = True
# verbose = True
draw = True
sensor_dict = {'2794': 'Left Arm', '2795': 'Back', '2796': 'Left Leg', '3261': 'Right Leg', '3337': 'Right Arm'}
#########################################################
## Baseline #
#########################################################
valid_session_file = '../../CMU/valid_sessions_win30_max60.pkl'
offsets, session_list = pickle.load(open(valid_session_file, 'rb'))
session_list = session_list['2794']
if mode == 'x':
mode_video = 'x'
mode_imu = 'X'
if mode == 'PCA':
mode_video = 'PCA'
mode_imu = 'PCA'
shifts_baseline = {}
skipped_sessions = {}
valid_sessions = {}
for sensor in sensors:
shifts_baseline[sensor] = np.zeros(len(session_list))
skipped_sessions[sensor] = []
valid_sessions[sensor] = []
os.makedirs('figures/baseline_{}_{}/{}'.format(mode_video, mode_imu, sensor), exist_ok=True)
# load ground truth
offsets = np.zeros(len(session_list))
for i, session in enumerate(session_list):
session = session.strip()
opt_file = glob.glob(os.path.join(opt_dir, session+'_Video', session+'_7150991-*.pkl'))
if len(opt_file) > 0:
opt_file = opt_file[0]
offsets[i] = int(opt_file[opt_file.find('-')+1:-4])
offsets = offsets * 1000/FPS
for i, session in enumerate(session_list):
session = session.strip()
print('======== Procession session {} ========'.format(session))
file = glob.glob(os.path.join(video_dir, session+'_Video', 'STime{}-time-*synch.txt'.format(video)))
out_dir = os.path.join(baseline_dir, session)
os.makedirs(out_dir, exist_ok=True)
df_file = os.path.join(out_dir, 'df_video_{}.pkl'.format(video))
print('Processing video {}'.format(video))
if load_df and os.path.exists(df_file):
print(' loading video df')
df_video = pd.read_pickle(df_file)
else:
print(' creating video df')
if file:
df_video = read_video(file[0])
check_df(df_video, delta=0.033332)
else:
print('skipping session{}, video sync doesn\'t exist'.format(session))
for sensor in sensors:
shifts_baseline[sensor][i] = np.nan
skipped_sessions[sensor].append(session)
continue
opt_file = glob.glob(os.path.join(opt_dir, session+'_Video', session+'_7150991-*.pkl'))
if opt_file:
opt_file = opt_file[0]
#offsets[i] = int(opt_file[opt_file.find('-')+1:-4])
else:
print('skipping session{}, optical flow doesn\'t exist'.format(session))
for sensor in sensors:
shifts_baseline[sensor][i] = np.nan
skipped_sessions[sensor].append(session)
continue
motion = pickle.load(open(opt_file, 'rb'))
if len(df_video) < len(motion):
print('skipping session{}, sync stamps less than frames'.format(session))
for sensor in sensors:
shifts_baseline[sensor][i] = np.nan
skipped_sessions[sensor].append(session)
continue
df_video = df_video[:len(motion)]
df_video['flowx'] = motion[:, 0]
df_video['flowy'] = motion[:, 1]
df_video['diff_flowx'] = df_video['flowx'].diff()
df_video['diff_flowy'] = df_video['flowy'].diff()
df_video = df_video[1:]
df_video = pca_flow(df_video)
# df_video.to_pickle(df_file)
# load sensor data
for sensor in sensors:
print('Processing sensor {}'.format(sensor))
df_file = os.path.join(out_dir, 'df_sensor_{}.pkl'.format(sensor))
if os.path.exists(df_file):
print(' loading sensor df')
df_imu = pd.read_pickle(df_file)
else:
print(' creating sensor df')
sensor_file = glob.glob(os.path.join(IMU_dir, session+'_3DMGX1', '{}_*-time*.txt'.format(sensor)))
if sensor_file:
sensor_file = sensor_file[0]
else:
print('skipping session{}, session time doesn\'t exist'.format(session))
skipped_sessions[sensor].append(session)
shifts_baseline[sensor][i] = np.nan
continue
df_imu = read_sensor(sensor_file)
df_imu = check_df(df_imu, delta=0.008)
df_imu = pca_sensor(df_imu)
# df_imu.to_pickle(df_file)
st_time = max([df_imu.iloc[0]['SysTime'], df_video.iloc[0]['SysTime']])-0.01
en_time = min([df_imu.iloc[len(df_imu) - 1]['SysTime'], df_video.iloc[len(df_video) - 1]['SysTime']])+0.01
df_video_tmp = df_video[(df_video['SysTime'] >= st_time) & (df_video['SysTime'] < en_time)]
df_imu = df_imu[(df_imu['SysTime'] >= st_time) & (df_imu['SysTime'] < en_time)]
vid_time_stamps = df_video_tmp['SysTime'].values
df_imu = resample(df_imu, 'SysTime', samplingRate=0,
gapTolerance=200, fixedTimeColumn=vid_time_stamps)
if df_imu is None:
print(' no intersection between video and imu, skip session {}'.format(session))
skipped_sessions[sensor].append(session)
shifts_baseline[sensor][i] = np.nan
continue
if len(df_video_tmp) != len(df_imu):
print(' lengths of video and imu not equal, skip session {}'.format(session))
print('len_vid = {}, len_sen = {}'.format(len(df_video_tmp), len(df_imu)))
skipped_sessions[sensor].append(session)
shifts_baseline[sensor][i] = np.nan
#df_imu.to_pickle(df_file)
continue
fftshift = cross_correlation_using_fft(df_video_tmp['diff_flow{}'.format(mode_video)].values, \
df_imu['Accel_{}'.format(mode_imu)].values)
shifts_baseline[sensor][i] = compute_shift(fftshift)
if draw:
path ='figures/baseline_{}_{}/{}/{}.jpg'.format(mode_video, mode_imu, sensor, session)
plt.figure()
plt.plot(fftshift[::-1])
plt.title('Video / {} Sensor, gt = {:.3f}s, predition = {:.3f}s'.format(sensor_dict[sensor], offsets[i]/1000, shifts_baseline[sensor][i] /FPS))
plt.savefig(path)
plt.close()
valid_sessions[sensor].append(session)
#plt.figure()
#plt.plot(fftshift)
#plt.savefig('{}_{}_{}.png'.format(mode, session, sensor))
for sensor in sensors:
shifts_baseline[sensor][67:91] = np.nan
error = compute_error(offsets, shifts_baseline)
error.to_csv(os.path.join(baseline_dir, 'df_error_baseline_{}_{}.csv'.format(mode_video, mode_imu)))
print(mode_video, mode_imu)
print(error)
#print(offsets, shifts_baseline)
# pickle.dump([offsets, shifts_baseline], open(os.path.join(baseline_dir, 'results_baseline_{}_{}.pkl'.format(mode_video, mode_imu)), 'wb'))
# pickle.dump(valid_sessions, open(os.path.join(baseline_dir, 'valid_sessions.pkl'), 'wb'))
#
# Analysis
_, shifts_baseline = pickle.load(open(os.path.join(baseline_dir, 'results_baseline_{}_{}.pkl'.format(mode_video, mode_imu)), 'rb'))
summ_mat = offsets.reshape(-1, 1)
for sensor in sensors:
summ_mat = np.concatenate([summ_mat, shifts_baseline[sensor].reshape(-1, 1) - offsets.reshape(-1, 1)], axis=1)
df_summ = pd.DataFrame(data=summ_mat, columns=['Ground truth', 'imu2794', 'imu2795', 'imu2796', 'imu3261', 'imu3337'], index=session_list)
df_summ.to_csv(os.path.join(baseline_dir, 'df_summ_baseline_{}_{}.csv'.format(mode_video, mode_imu)))
print(offsets)
voltageResults, strainResults = splitCycles(combined,
byPeak=True,
strainProminence=.001,
voltageProminence=.1)
strainCycles = strainResults['cycles']
strainTimes = strainResults['times']
voltageCycles = voltageResults['cycles']
voltageTimes = voltageResults['times']
newVoltageCycles = []
newStrainCycles = []
for i in range(len(voltageCycles)):
newVoltageCycle, newStrainCycle, _, _ = (resample(voltageCycles.loc[i, :],
strainCycles.loc[i, :],
voltageTimes.loc[i, :],
strainTimes.loc[i, :]))
newVoltageCycles.append(newVoltageCycle)
newStrainCycles.append(newStrainCycle)\
newStrainCycles = pd.DataFrame(newStrainCycles)
newVoltageCycles = pd.DataFrame(newVoltageCycles)
plt.figure(0)
plt.plot(newStrainCycles.loc[0], 'ro')
plt.plot(strainCycles.loc[0], 'bo')
newStrainCycles.to_csv('strainCycles.csv', index=False, na_rep="NA")
newVoltageCycles.to_csv('voltageCycles.csv', index=False, na_rep="NA")
import resample
import argparse
import heapq
parser = argparse.ArgumentParser()
parser.add_argument("-src", help="Source file")
parser.add_argument("-dst", help="Destination file")
args = parser.parse_args()
print "Resampling", args.src, "to", args.dst
resample.resample(args.src, args.dst, resampling_fn=lambda x: min(x), cutoff_less=True)
if save_result:
loc_results = np.empty((len(poses), numParticles, 4))
for frame_idx in range(start_idx, len(poses)):
if visualize:
visualizer.update(frame_idx, particles)
visualizer.fig.canvas.draw()
visualizer.fig.canvas.flush_events()
# motion model
particles = motion_model(particles, commands[frame_idx])
# only update the weight when the car moves
if commands[frame_idx, 1] > 0.2 / grid_res or is_initial:
is_initial = False
# grid-based method
particles = sensor_model.update_weights(particles, frame_idx)
# resampling
particles = resample(particles)
if save_result:
loc_results[frame_idx, :len(particles)] = particles
print('finished frame:', frame_idx)
if save_result:
print('Saving localization results...')
np.savez_compressed('localization_results_'+str(start_idx), loc_results)
plot_traj_result(loc_results, poses, numParticles=numParticles, start_idx=start_idx)
