def __init__(self, img, flowfield, dm_out, gridsize = 20, threshold = 8, plot = False): self.img = img self.nx = img.shape[0] self.ny = img.shape[1] self.threshold = threshold mask, ctrs, fd = self.backsub() self.distmesh = DistMesh(img, h0 = gridsize) self.distmesh.createMesh(ctrs, fd, img, plot = plot) self.distmesh.save(dm_out) self.t = 0 self.flowfield = flowfield self.writer = None flowzeros = np.zeros((self.nx, self.ny, 2)) self.kf = KalmanFilter(self.distmesh, img, flowzeros, cuda = False) self.N = self.kf.state.N
flow_in = './synthetictests/' + name + '/' + ff + '/' + ff + '_flow'
#Output
img_out = './synthetictests/' + name + '/' + ff + '_' + notes + '_pred/'
if not os.path.isdir(img_out):
os.makedirs(img_out)
gridsize = 18
threshold = 8
#Create KF
print 'Loading synthetic data streams'
capture = VideoStream(v_in, threshold)
frame = capture.current_frame()
mask, ctrs, fd = capture.backsub()
distmesh = DistMesh(frame, h0 = gridsize)
distmesh.load(dm_in)
#Load true data
f_mesh = open(m_in, 'r')
lines = f_mesh.readlines()
nF = len(lines)-1
nX = int(lines[0].split(',')[1])
truestates = np.zeros((nF, nX*4), dtype = np.float32)
predstates = np.zeros((nF, nX*4), dtype = np.float32)
for i in range(1,nF+1):
line = lines[i]
truestates[i-1,:] = [float(x) for x in line.split(',')[1:]]
predstates[0,:] = truestates[0,:]
def main():
usage = """run_kalmanfilter.py [input_avi_file] [optic_flow_path] [output_avi_file] [threshold]
HydraGL. State space model using an extended Kalman filter to track Hydra in video
Dependencies:
-Vispy*
-Numpy
-PyCuda**
-DistMesh
-HDF
-OpenCV2
-matplotlib
Notes:
* Uses OpenGL rendering. If using remotely, you'll need to set up a VirtualGL server
** If have a CUDA compatible graphics card
Example:
./run_kalmanfilter.py ./video/johntest_brightcontrast_short.avi ...
./video/johntest_brightcontrast_short/flow ./video/output.avi -s 15
For help:
./run_kalmanfilter.py -h
Ben Lansdell
02/16/2016
"""
parser = argparse.ArgumentParser()
parser.add_argument('fn_in', default='./video/johntest_brightcontrast_short.avi',
help='input video file, any format readable by OpenCV', nargs = '?')
parser.add_argument('flow_in', default='./video/johntest_brightcontrast_short/flow',
help='input optic flow path', nargs = '?')
parser.add_argument('fn_out', default='./video/johntest_brightcontrast_short_output.avi',
help='avi output video file', nargs='?')
parser.add_argument('-n', '--name', default='johntest_brightcontrast_short',
help='name for saving run images', nargs='?')
parser.add_argument('-t', '--threshold', default=9,
help='threshold intensity below which is background', type = int)
parser.add_argument('-s', '--gridsize', default=22,
help='edge length for mesh (smaller is finer; unstable much further below 18)', type = int)
parser.add_argument('-c', '--cuda', default=True,
help='whether or not to do analysis on CUDA', type = bool)
args = parser.parse_args()
if len(sys.argv) == 1:
print("No command line arguments provided, using defaults")
capture = VideoStream(args.fn_in, args.threshold)
frame = capture.current_frame()
mask, ctrs, fd = capture.backsub()
distmesh = DistMesh(frame, h0 = args.gridsize)
distmesh.createMesh(ctrs, fd, frame, plot = True)
#Load flow data from directory
flowstream = FlowStream(args.flow_in)
ret_flow, flowframe = flowstream.peek()
if ret_flow:
kf = IteratedMSKalmanFilter(distmesh, frame, flowframe, cuda = args.cuda, sparse = True, multi = True)
else:
print 'Cannot read flow stream'
return
#kf.compute(capture.gray_frame(), flowframe)
nI = 3
count = 0
while(capture.isOpened()):
count += 1
print 'Frame %d' % count
ret, frame, grayframe, mask = capture.read()
ret_flow, flowframe = flowstream.read()
if ret is False or ret_flow is False:
break
#for i in range(nI):
# print 'Iteration %d' % i
# raw_input("Finished. Press Enter to continue")
# kf.compute(grayframe, flowframe)
kf.compute(grayframe, flowframe, mask, imageoutput = 'screenshots/' + args.name + '_frame_' + str(count))
capture.release()
output.release()
cv2.destroyAllWindows()
raw_input("Finished. Press ENTER to exit")
import matplotlib.pyplot as plot cuda = True gridsize = 30 threshold = 9 name = 'test_data' #Grab test data video, flow = test_data(680, 680) #video, flow = test_data_texture(680, 680) #video, flow = test_data_image() flowframe = flow[:,:,:,0] frame = video[:,:,0] #Make mesh distmesh = DistMesh(frame, h0 = gridsize) mask, ctrs, h = findObjectThreshold(frame, threshold = threshold) distmesh.createMesh(ctrs, h, frame, plot=False) nx = 680 start = nx//3 end = 2*nx//3 kf = IteratedMSKalmanFilter(distmesh, frame, flowframe, cuda) #Now perturb the positions a bit... kf.state.X = kf.state.X*1.2 kf.state.refresh() rend = kf.state.renderer cuda = kf.state.renderer.cudagl
print 'Loading frame', idx ret, frame, mask = capture.read() if idx == nref: refframe = frame.copy() vid[:,:,idx] = frame masks[:,:,idx] = mask #Generate ctrs and fd (mask, ctrs, fd) = findObjectThreshold(tracking_mask, threshold = threshold) #distmesh = DistMesh(refframe, h0 = gridsize) #distmesh.createMesh(ctrs, fd, refframe, plot = True) #Save this distmesh and reload it for quicker testing #distmesh.save(dm_out) distmesh = DistMesh(refframe, h0 = gridsize) distmesh.load(dm_out) refpositions = distmesh.p #Create dummy input for flow frame flowframe = np.zeros((nx, nx, 2)) #Create Kalman Filter object to store mesh and make use of plotting functions kf = IteratedMSKalmanFilter(distmesh, refframe, flowframe, cuda = cuda) N = kf.size()/4 totalframes = 0 for vidx in range(nV): #Load MFSF data a = loadmat(mfsf_matfiles[vidx])
flow_in = './synthetictests/' + name + '/' + ff + '/' + ff + '_flow'
#Output
img_out = './synthetictests/' + name + '/' + ff + '_' + notes + '_pred/'
if not os.path.isdir(img_out):
os.makedirs(img_out)
gridsize = 18
threshold = 8
#Create KF
print 'Loading synthetic data streams'
capture = VideoStream(v_in, threshold)
frame = capture.current_frame()
mask, ctrs, fd = capture.backsub()
distmesh = DistMesh(frame, h0 = gridsize)
distmesh.load(dm_in)
#Load true data
f_mesh = open(m_in, 'r')
lines = f_mesh.readlines()
nF = len(lines)-1
nX = int(lines[0].split(',')[1])
truestates = np.zeros((nF, nX*4), dtype = np.float32)
predstates = np.zeros((nF, nX*4), dtype = np.float32)
for i in range(1,nF+1):
line = lines[i]
truestates[i-1,:] = [float(x) for x in line.split(',')[1:]]
predstates[0,:] = truestates[0,:]
def runIEKF(eps_Z, eps_J, eps_M, eps_F):
print 'Running with:'
print '- eps_Z:', eps_Z
print '- eps_J:', eps_J
print '- eps_M:', eps_M
print '- eps_F:', eps_F
if mask_flow:
notes = 'masked_iekf'
else:
notes = 'iekf'
notes += '_eps_Z_%f'%eps_Z
notes += '_eps_J_%f'%eps_J
notes += '_eps_M_%f'%eps_M
notes += '_eps_F_%f'%eps_F
img_out = './synthetictests/' + name + '/' + ff + '_' + notes + '_pred/'
if not os.path.isdir(img_out):
os.makedirs(img_out)
print 'Loading synthetic data streams'
capture = VideoStream(v_in, threshold)
frame = capture.current_frame()
mask, ctrs, fd = capture.backsub()
distmesh = DistMesh(frame, h0 = gridsize)
distmesh.load(dm_in)
#Load true data
f_mesh = open(m_in, 'r')
lines = f_mesh.readlines()
nF = len(lines)-1
nX = int(lines[0].split(',')[1])
truestates = np.zeros((nF, nX*4), dtype = np.float32)
predstates = np.zeros((nF, nX*4), dtype = np.float32)
for i in range(1,nF+1):
line = lines[i]
truestates[i-1,:] = [float(x) for x in line.split(',')[1:]]
predstates[0,:] = truestates[0,:]
rms_vel = np.zeros(nF)
rms_pos = np.zeros(nF)
flowstream = FlowStream(flow_in)
ret_flow, flowframe = flowstream.read()
kf = IteratedKalmanFilter(distmesh, frame, flowframe, cuda = cuda, sparse = sparse,\
eps_F = eps_F, eps_Z = eps_Z, eps_J = eps_J, eps_M = eps_M, nI = nI)
count = 0
print 'Tracking with Kalman filter'
while(capture.isOpened() and count < max_frames):
#for idx in range(1):
count += 1
ret, frame, grayframe, mask = capture.read()
ret_flow, flowframe = flowstream.read()
if ret is False or ret_flow is False:
break
print 'Frame %d' % count
kf.compute(grayframe, flowframe, mask, maskflow = mask_flow, imageoutput = img_out+'solution_frame_%03d'%count)
#kf.compute(grayframe, flowframe, mask, maskflow = mask_flow)
predstates[count,:] = np.squeeze(kf.state.X)
r_pos = truestates[count,0:(2*nX)]-predstates[count,0:(2*nX)]
r_vel = truestates[count,(2*nX):]-predstates[count,(2*nX):]
rms_pos[count] = np.sqrt(np.mean(np.multiply(r_pos, r_pos)))
rms_vel[count] = np.sqrt(np.mean(np.multiply(r_vel, r_vel)))
print 'RMS_pos:', rms_pos[count], 'RMS_vel:', rms_vel[count]
print 'Saving'
np.savez('./synthetictests/' + name + '/' + ff + '_' + notes + '_pred.npz', predstates, truestates, rms_pos, rms_vel)
print 'Done... how\'d we do?'
#Make plot of tracking error
plt.plot(range(nF), rms_pos, label='RMS position')
plt.plot(range(nF), rms_vel, label='RMS velocity')
plt.legend(loc='upper left')
plt.ylabel('RMS')
plt.xlabel('Frame')
plt.savefig('./synthetictests/' + name + '/' + ff + '_' + notes + '_pred_rms.eps')
#!/usr/bin/env python import sys, argparse from kalman import KalmanFilter, KFState from renderer import * from cuda import * from distmesh_dyn import DistMesh import numpy as np fn_in ='./video/johntest_brightcontrast_short.tif' threshold = 9 capture = VideoStream(fn_in, threshold) frame = capture.current_frame(backsub = True) mask, ctrs, fd = capture.backsub() distmesh = DistMesh(frame) distmesh.createMesh(ctrs, fd, frame, True) flowframe = None #capture.backsub(hdf.read()) #Create KalmanFilter object one step at a time (kf = KalmanFilter(distmesh, frame)) #Create KFState object (KFState.__init__) im = frame nx = im.shape[0] eps_Q = 1 eps_R = 1e-3 _ver = np.array(distmesh.p, np.float32) _vel = np.ones(_ver.shape, np.float32) #Create renderer (renderer = Renderer(distmesh, _vel, nx, im)) ##############################################################
class TestMesh:
"""Takes an initial frame (object), creates a mesh and morphs the mesh points
over time according to a provided flow field. Saves the results and the true
set of mesh points"""
def __init__(self, img, flowfield, dm_out, gridsize = 20, threshold = 8, plot = False):
self.img = img
self.nx = img.shape[0]
self.ny = img.shape[1]
self.threshold = threshold
mask, ctrs, fd = self.backsub()
self.distmesh = DistMesh(img, h0 = gridsize)
self.distmesh.createMesh(ctrs, fd, img, plot = plot)
self.distmesh.save(dm_out)
self.t = 0
self.flowfield = flowfield
self.writer = None
flowzeros = np.zeros((self.nx, self.ny, 2))
self.kf = KalmanFilter(self.distmesh, img, flowzeros, cuda = False)
self.N = self.kf.state.N
def forward(self):
#Update mesh points according to the velocity flow field
for i in range(self.N):
x = self.kf.state.X[2*i]
y = self.kf.state.X[2*i+1]
(vx, vy) = self.flowfield([x, y], self.t)
self.kf.state.X[2*i] += vx
self.kf.state.X[2*i+1] += vy
self.kf.state.X[2*self.N + 2*i] = vx
self.kf.state.X[2*self.N + 2*i+1] = vy
def render(self):
self.kf.state.renderer.update_vertex_buffer(self.kf.state.vertices(), self.kf.state.velocities())
#self.kf.state.renderer.on_draw(None)
pred_img = self.kf.state.renderer.getpredimg()
return pred_img
def _createwriter(self, video_out):
fourcc = cv2.VideoWriter_fourcc(*'XVID')
framesize = (self.nx, self.ny)
self.writer = cv2.VideoWriter(video_out, fourcc, 2.0, framesize[::-1])
def _strState(self):
return "X," + ','.join([str(x[0]) for x in self.kf.state.X]) + '\n'
def run(self, video_out, mesh_out, steps = 100):
#Number of time steps
f_out = open(mesh_out, 'w')
#Write: mesh size
f_out.write("size,%d\n"%self.N)
#self._createwriter(video_out)
#assert self.writer is not None, "Cannot create VideoWriter object"
print "Simulating", steps, "steps of mesh warping"
for i in range(steps):
self.forward()
pred_img = self.render()
#self.writer.write(pred_img)
#Or just save the images
fn_out = video_out + "_frame_%03d"%i + ".png"
cv2.imwrite(fn_out,pred_img)
f_out.write(self._strState())
f_out.close()
#self.writer.release()
cv2.destroyAllWindows()
print "Done"
def backsub(self):
(mask, ctrs, fd) = findObjectThreshold(self.img, threshold = self.threshold)
return mask, ctrs, fd
