def flow_err_deepflow(fn_in1, fn_in2):
flow1 = readFlo(fn_in1)
flow2 = readFlo(fn_in2)
nx = flow1.shape[0]
ny = flow1.shape[1]
#Flip x and y flow
flow1 = np.transpose(flow1, [1, 0, 2])
flow2 = np.transpose(flow2, [1, 0, 2])
flow1 = flow1[:, :, ::-1]
flow2 = flow2[:, :, ::-1]
#Perform mapping and then reverse mapping, then perform reverse mapping then mapping
#Make mesh grid
fwdmeshy, fwdmeshx = [
a.astype(np.float32) for a in np.meshgrid(np.arange(nx), np.arange(ny))
]
#Perturb mesh grid by forward flow
#Round to integers
fwdx = fwdmeshx + np.ceil(flow1[:, :, 0])
fwdy = fwdmeshy + np.ceil(flow1[:, :, 1])
fwdx = np.maximum(0, np.minimum(nx - 1, fwdx))
fwdy = np.maximum(0, np.minimum(nx - 1, fwdy))
#Look up flow field using this perturbed map
fwdremapx = fwdx + flow2[fwdx.astype(int), fwdy.astype(int), 0]
fwdremapy = fwdy + flow2[fwdx.astype(int), fwdy.astype(int), 1]
fwdremapx -= fwdmeshx
fwdremapy -= fwdmeshy
fwderr = np.sqrt(fwdremapx**2 + fwdremapy**2)
return fwderr
def main():
usage = """deepflow_simmatrix.py [input_dir]
Expects refframes directory in [input_dir] to be already filled with i-frames
Example:
./deepflow_simmatrix_viz.py ./simmatrix/20160412/
For help:
./deepflow_simmatrix_viz.py -h
Ben Lansdell
01/04/2016
"""
parser = argparse.ArgumentParser()
parser.add_argument(
'path_in', help='input directory with frames already placed in it')
args = parser.parse_args()
#Test code
class Args:
pass
args = Args()
args.path_in = './simmatrix/20160412/'
#Get all files in
iframes = sorted(glob(args.path_in + 'refframes/*.tif'))
nF = len(iframes)
nx_tile = 64
#Load images
images = []
for fn_in in iframes: # do stuff with image
# to open a tiff file for reading:
tif = TIFF.open(fn_in, mode='r')
image = tif.read_image()
image = cv2.resize(image, (nx_tile, nx_tile))
images.append(image)
tif.close()
D = np.zeros((nF, nF))
#Run DeepFlow
for i in range(nF):
im1 = iframes[i]
fn1 = int(os.path.splitext(os.path.basename(im1))[0].split('_')[1])
for j in range(i + 1, nF):
im2 = iframes[j]
fn2 = int(os.path.splitext(os.path.basename(im2))[0].split('_')[1])
print("DeepFlow between frame %d and %d" % (fn1, fn2))
flow_in1 = args.path_in + 'corrmatrix/%04d_%04d.flo' % (fn1, fn2)
flow_in2 = args.path_in + 'corrmatrix/%04d_%04d.flo' % (fn2, fn1)
#Read in flow
flow1 = readFlo(flow_in1)
flow2 = readFlo(flow_in2)
ny, nx = flow1.shape[0:2]
#For each run we compute the average reconstruction error
fwdmeshy, fwdmeshx = [
a.astype(np.float32)
for a in np.meshgrid(np.arange(nx), np.arange(ny))
]
#Perturb mesh grid by forward flow
#Round to integers
fwdx = fwdmeshx + np.ceil(flow1[:, :, 0])
fwdy = fwdmeshy + np.ceil(flow1[:, :, 1])
fwdx = np.maximum(0, np.minimum(nx - 1, fwdx))
fwdy = np.maximum(0, np.minimum(nx - 1, fwdy))
#Look up flow field using this perturbed map
fwdremapx = fwdx + flow2[fwdx.astype(int), fwdy.astype(int), 0]
fwdremapy = fwdy + flow2[fwdx.astype(int), fwdy.astype(int), 1]
fwdremapx -= fwdmeshx
fwdremapy -= fwdmeshy
fwderr = np.sqrt(fwdremapx**2 + fwdremapy**2)
#fwdtracked = fwderr < threshold
D[i, j] = np.mean(fwderr)
D[j, i] = D[i, j]
# Plot distance matrix.
fig1 = pylab.figure(figsize=(8, 8))
axmatrix1 = fig1.add_axes([0.3, 0.1, 0.6, 0.6])
im = axmatrix1.matshow(D,
aspect='auto',
origin='lower',
cmap=pylab.cm.YlGnBu)
fn_out = args.path_in + 'similarity.png'
fig1.savefig(fn_out)
#Once we've loaded this data we view the similarity matrix
fig = pylab.figure(figsize=(8, 8))
ax1 = fig.add_axes([0.09, 0.1, 0.2, 0.6])
Y = sch.linkage(D, method='centroid')
Z1 = sch.dendrogram(Y, orientation='right')
ax1.set_xticks([])
ax1.set_yticks([])
# Compute and plot second dendrogram.
ax2 = fig.add_axes([0.3, 0.71, 0.6, 0.2])
Y = sch.linkage(D, method='single')
Z2 = sch.dendrogram(Y)
ax2.set_xticks([])
ax2.set_yticks([])
# Plot distance matrix.
axmatrix = fig.add_axes([0.3, 0.1, 0.6, 0.6])
idx1 = Z1['leaves']
idx2 = Z2['leaves']
D = D[idx1, :]
D = D[:, idx2]
im = axmatrix.matshow(D,
aspect='auto',
origin='lower',
cmap=pylab.cm.YlGnBu)
axmatrix.set_xticks([])
axmatrix.set_yticks([])
# Plot colorbar.
axcolor = fig.add_axes([0.91, 0.1, 0.02, 0.6])
pylab.colorbar(im, cax=axcolor)
#fig.show()
fn_out = args.path_in + 'dendrogram.png'
fig.savefig(fn_out)
#Make another version of this plot but bigger and with frame snippets
#Load all the iframe images and resize
fn_out_d1 = args.path_in + 'dend_d1_tile.png'
fn_out_d2 = args.path_in + 'dend_d2_tile.png'
im_d1 = images[idx1[0]]
im_d2 = images[idx2[0]]
for idx in range(1, nF):
im_d1 = np.hstack((im_d1, images[idx1[idx]]))
im_d2 = np.hstack((im_d2, images[idx2[idx]]))
cv2.imwrite(fn_out_d1, im_d1)
cv2.imwrite(fn_out_d2, im_d2)
def continuation(name, segmentation, res_dir, mfsf_in, iframes, rframes):
usage = """Continue MFSF optic flow fields from separate videos into the one flow field
whose coordinates are relative to a set of reference frames specified.
Example:
./continue_mfsf.py --rframes 1,501 --iframes 1,251,501,751 ./simmatrix/20160412/ ./mfsf_output/
For help:
./continue_mfsf.py -h
Ben Lansdell
1/5/2017
"""
#Test code
#name = '20160412'
#segmentation = './simmatrix/20160412/seg_admm/gpu_MS_lambda_1.00e-04_rho_1.00e-03_niter_3000.npy'
#res_dir = './simmatrix/'
#mfsf_in = './mfsf_output/'
#iframes = [1, 251, 501, 751, 1001, 1251, 1501, 1751, 2001, 2251, 2501, 2751, 3001, 3251, 3501, 3751, 4001,\
# 4251, 4501, 4751]
#rframes = [1, 501]
#name = 'moon'
#segmentation = './simmatrix/moon/seg_admm/gpu_MS_lambda_1.00e-04_rho_1.00e-03_niter_1000_1,101,201,301,401,501,601,701,801,901,1001,1101,1201,1301,1401.npz'
#res_dir = './simmatrix/'
#mfsf_in = './mfsf_output/'
#iframes = [1,101,201,301,401,501,601,701,801,901,1001,1101,1201,1301,1401]
#rframes = [1,201,401,601]
name = 'earth'
segmentation = './simmatrix/earth/seg_admm/gpu_MS_lambda_1.00e-04_rho_1.00e-03_niter_3000_1,51,101,151,201,251,301,351,401,451,501,551,601,651,701,751,801.npz'
res_dir = './simmatrix/'
mfsf_in = './mfsf_output/'
iframes = [
1, 51, 101, 151, 201, 251, 301, 351, 401, 451, 501, 551, 601, 651, 701,
751, 801
]
rframes = [1, 101, 201, 301, 401]
#Prepare output directory
dr = res_dir + '/' + name + '/continuation/'
if not os.path.exists(dr):
os.makedirs(dr)
#Load MS segmenting results for each reference frame
u_s = np.load(segmentation)
u_s = u_s['u_s']
print u_s.shape
nR = len(rframes)
nF = len(iframes)
nframes = iframes[1] - iframes[0]
#For each MFSF file
for vidx in range(nF):
#Load MFSF data
fn_in = mfsf_in + '/' + name + '/stack%04d_nref1_nframe%d/result.mat' % (
vidx + 1, nframes)
fn2 = iframes[vidx]
print "Continuing video (frame %d)" % fn2
try:
a = loadmat(fn_in)
params = a['parmsOF']
u1 = a['u']
v1 = a['v']
except NotImplementedError:
print "Failed to read using loadmat, using hdf5 library to read %s" % fn_in
f = h5py.File(fn_in, 'r')
#Note the x and y axes may need to be switched here...
u1 = np.transpose(np.array(f.get('u')), (1, 2, 0))
v1 = np.transpose(np.array(f.get('v')), (1, 2, 0))
#Continue paths for both reference frames
#and save a mask of which paths should actually be continued based on
#segmentation
seg = np.argmax(cv2.resize(u_s[:, :, :, vidx], u1.shape[0:2]), axis=2)
for k in range(nR):
fn1 = rframes[k]
#Load in flow results for each reference frame to the iframe
if fn1 != fn2:
#Make a Stitcher. Takes the second set's flow data as input
thestitch = Stitcher(u1, v1)
fn_in = res_dir + '/' + name + '/corrmatrix/%04d_%04d.flo' % (
fn1, fn2)
flow = readFlo(fn_in)
u0 = flow[:, :, 0]
v0 = flow[:, :, 1]
(u, v) = thestitch.run(u0, v0)
del thestitch
unreachable = gc.collect()
#Threshold and label paths that are to be continued with this rframe
mask = (seg == k)
else:
u = u1
v = v1
mask = np.ones(u1.shape[0:2])
#Save output matrix
mdict = {'u': u, 'v': v, 'mask': mask}
savemat(dr + '/mfsf_r_%04d_l_%04d.mat' % (fn1, fn2), mdict)
def continuation(path_in, mfsf_in, iframes, rframes):
usage = """Continue MFSF optic flow fields from separate videos into the one flow field
whose coordinates are relative to a set of reference frames specified.
Example:
./continue_dm_mfsf.py --rframes 1,501 --iframes 1,251,501,751 ./simmatrix/20160412/ ./mfsf_output/
For help:
./continue_dm_mfsf.py -h
Ben Lansdell
1/14/2017
"""
#Test code
path_in = './simmatrix/20160412/seg_admm/gpu_MS_lambda_1.00e-04_rho_1.00e-03_niter_3000.npy'
name = './simmatrix/20160412/'
mfsf_in = './mfsf_output/'
iframes = [1, 251, 501, 751, 1001, 1251, 1501]
rframes = [1, 501]
dm_frames = [1, 501, 1251]
forward_mfsf = [[501, 751], [1251, 1501]]
reverse_mfsf = [[501, 251], [1251, 1001]]
#Prepare output directory
dr = name + './continuation_mfsf/'
if not os.path.exists(dr):
os.makedirs(dr)
#Load MS segmenting results for each reference frame
u_s = np.load(path_in)
nR = len(rframes)
nF = len(iframes)
nD = len(dm_frames)
#For each DM file
for vidx2 in range(nD):
vidx = np.nonzero(np.array(iframes) == dm_frames[vidx2])[0][0]
#Load MFSF data
fn_in = mfsf_in + 'stack%04d_nref1_nframe250/result.mat'%(vidx+1)
fn2 = iframes[vidx]
print "Continuing video with DeepMatching (frame %d)"%fn2
(u1, v1) = read_MFSF(fn_in)
#Continue paths for both reference frames
#and save a mask of which paths should actually be continued based on
#segmentation
seg = np.argmax(cv2.resize(u_s[:,:,:,vidx], u1.shape[0:2]), axis = 2)
for k in range(nR):
fn1 = rframes[k]
#Load in flow results for each reference frame to the iframe
if fn1 != fn2:
#Make a Stitcher. Takes the second set's flow data as input
thestitch = Stitcher(u1, v1)
fn_in = name + '/corrmatrix/%04d_%04d.flo'%(fn1,fn2)
flow = readFlo(fn_in)
u0 = flow[:,:,0]
v0 = flow[:,:,1]
(u, v) = thestitch.run(u0, v0)
del thestitch
unreachable = gc.collect()
#Threshold and label paths that are to be continued with this rframe
mask = (seg == k)
else:
u = u1
v = v1
mask = np.ones(u1.shape[0:2])
#Save output matrix
mdict = {'u':u, 'v':v, 'mask':mask}
savemat(dr + '/mfsf_r_%04d_l_%04d.mat'%(fn1, fn2), mdict)
#Then do the MFSF frames
#forward_mfsf = [[501, 751], [1251, 1501]]
for vidx in range(len(forward_mfsf)):
#This is the already shifted one
vn1 = forward_mfsf[vidx][0]
vidx1 = np.nonzero(np.array(iframes) == vn1)[0][0]
#This is the to be shifted one
vn2 = forward_mfsf[vidx][1]
vidx2 = np.nonzero(np.array(iframes) == vn2)[0][0]
print "Continuing video with MFSF (frame %d)"%vn2
fn_in2 = mfsf_in + 'stack%04d_nref1_nframe250/result.mat'%(vidx2+1)
(u2,v2) = read_MFSF(fn_in2)
#Load in v1's MFSF data for both r frames
for k in range(nR):
#Create a Stitcher
thestitch = Stitcher(u2, v2)
fn1 = rframes[k]
#Load MFSF data
#fn_in1 = mfsf_in + 'stack%04d_nref1_nframe250/result.mat'%(vidx1+1)
fn_in1 = dr + '/mfsf_r_%04d_l_%04d.mat'%(fn1, vn1)
#(u1, v1) = load_MFSF(fn_in1)
a = loadmat(fn_in1)
u1 = a['u']
v1 = a['v']
mask = a['mask']
#Load in flow results for each reference frame to the iframe
#Make a Stitcher. Takes the second set's flow data as input
(u, v) = thestitch.run(u1, v1)
del thestitch
unreachable = gc.collect()
#Threshold and label paths that are to be continued with this rframe
#Save output matrix (inherit the mask from the parent continuation)
mdict = {'u':u, 'v':v, 'mask':mask}
savemat(dr + '/mfsf_r_%04d_l_%04d_m_%04d.mat'%(fn1, vn1, vn2), mdict)
#reverse_mfsf = [[501, 251], [1251, 1001]]
for vidx in range(len(reverse_mfsf)):
#This is the already shifted one
vn1 = reverse_mfsf[vidx][0]
vidx1 = np.nonzero(np.array(iframes) == vn1)[0][0]
#This is the to be shifted one
vn2 = reverse_mfsf[vidx][1]
vidx2 = np.nonzero(np.array(iframes) == vn2)[0][0]
print "Continuing video with MFSF (frame %d)"%vn2
fn_in2 = mfsf_in + 'stack%04d_nref1_nframe250/result.mat'%(vidx2+1)
(u2,v2) = read_MFSF(fn_in2)
#Load in v1's MFSF data for both r frames
for k in range(nR):
#Create a Stitcher
thestitch = StitcherReverse(u2, v2)
fn1 = rframes[k]
#Load MFSF data
fn_in1 = dr + '/mfsf_r_%04d_l_%04d.mat'%(fn1, vn1)
#fn_in1 = mfsf_in + 'stack%04d_nref1_nframe250/result.mat'%(vidx1+1)
a = loadmat(fn_in1)
u1 = a['u']
v1 = a['v']
mask = a['mask']
#Load in flow results for each reference frame to the iframe
#Make a Stitcher. Takes the second set's flow data as input
(u, v) = thestitch.run(u1, v1)
del thestitch
unreachable = gc.collect()
#Threshold and label paths that are to be continued with this rframe
#Save output matrix (inherit the mask from the parent continuation)
mdict = {'u':u, 'v':v, 'mask':mask}
savemat(dr + '/mfsf_r_%04d_l_%04d_m_%04d.mat'%(fn1, vn1, vn2), mdict)
def main():
usage = """deepflow_corrmatrix.py [register_dir]...
Generate confidence maps based on optical flow estimation
Example:
./deepflow_corrmatrix.py ./register/20160412stk0001/
For help:
./deepflow_corrmatrix.py -h
Ben Lansdell
10/30/2016
"""
parser = argparse.ArgumentParser()
parser.add_argument('dir_in', help='output directory')
args = parser.parse_args()
#Test code for interactive dev
class Args:
pass
args = Args()
args.dir_in = './register/20160412stk0001/'
#args.dir_in = './register/test/'
#Get the set of reference frames...
refframes = [int(i[-8:-4]) for i in glob(args.dir_in + 'refframes/*.png')]
nF = len(refframes)
threshold = 4
radius = 6
#Load DeepFlow results
for r1 in refframes:
for r2 in refframes:
if r1 != r2:
print("Load DeepFlow between frame %d and %d" %(r1, r2))
fn_in1 = args.dir_in + 'corrmatrix/%04d_%04d.flo'%(r1, r2)
fn_in2 = args.dir_in + 'corrmatrix/%04d_%04d.flo'%(r2, r1)
flow1 = readFlo(fn_in1)
flow2 = readFlo(fn_in2)
nx = flow1.shape[0]
ny = flow1.shape[1]
#Flip x and y flow
flow1 = np.transpose(flow1, [1,0,2])
flow2 = np.transpose(flow2, [1,0,2])
flow1 = flow1[:,:,::-1]
flow2 = flow2[:,:,::-1]
#Perform mapping and then reverse mapping, then perform reverse mapping then mapping
#Make mesh grid
fwdmeshy, fwdmeshx = [a.astype(np.float32) for a in np.meshgrid(np.arange(nx), np.arange(ny))]
revmeshy, revmeshx = [a.astype(np.float32) for a in np.meshgrid(np.arange(nx), np.arange(ny))]
#Perturb mesh grid by forward flow
#Round to integers
fwdx = fwdmeshx + np.ceil(flow1[:,:,0])
fwdy = fwdmeshy + np.ceil(flow1[:,:,1])
fwdx = np.maximum(0, np.minimum(nx-1, fwdx))
fwdy = np.maximum(0, np.minimum(nx-1, fwdy))
#Look up flow field using this perturbed map
fwdremapx = fwdx + flow2[fwdx.astype(int),fwdy.astype(int),0]
fwdremapy = fwdy + flow2[fwdx.astype(int),fwdy.astype(int),1]
fwdremapx -= fwdmeshx
fwdremapy -= fwdmeshy
fwderr = np.sqrt(fwdremapx**2 + fwdremapy**2)
#Determine based on absolute value of displacement whether 'mapped' or not...
#perhaps model as GMM/some sort of smoothing....
#plt.pcolor(fwderr)
#plt.colorbar()
#plt.show()
#Plot flow data
#absflow1 = np.sqrt(flow1[:,:,0]**2 + flow1[:,:,1]**2)
#absflow2 = np.sqrt(flow2[:,:,0]**2 + flow2[:,:,1]**2)
#Load reference frame
fn_in1 = args.dir_in + 'refframes/frame_%04d.png'%r1
rf1 = cv2.imread(fn_in1)
fn_in2 = args.dir_in + 'refframes/frame_%04d.png'%r2
rf2 = cv2.imread(fn_in2)
hsv = np.zeros_like(rf1)
hsv[...,1] = 255
mag, ang = cv2.cartToPolar(flow1[...,0], flow1[...,1])
hsv[...,0] = ang*180/np.pi/2
hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)
bgr1 = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
dst1 = cv2.addWeighted(rf1,0.7,bgr1,0.3,0)
#cv2.imshow('f',dst1)
cv2.imwrite('./dst1.png', dst1)
hsv = np.zeros_like(rf2)
hsv[...,1] = 255
mag, ang = cv2.cartToPolar(flow2[...,0], flow2[...,1])
hsv[...,0] = ang*180/np.pi/2
hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)
bgr2 = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
dst2 = cv2.addWeighted(rf2,0.7,bgr2,0.3,0)
#cv2.imshow('f',dst2)
cv2.imwrite('./dst2.png', dst2)
#Overlay with reference frame...
im_fwderr = cv2.normalize(fwderr, fwderr, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)
im_fwderr = cv2.cvtColor(im_fwderr, cv2.COLOR_GRAY2BGR)
im_fwderr = cv2.applyColorMap(im_fwderr, cv2.COLORMAP_JET)
dst = cv2.addWeighted(rf1,0.7,im_fwderr,0.3,0)
#cv2.imshow('Error', dst)
cv2.imwrite('./error.png', dst)
#Concat into one big image...
vis1 = np.concatenate((rf1, rf2, bgr1), axis=1)
vis2 = np.concatenate((rf2, rf1, bgr2), axis=1)
vis3 = np.concatenate((bgr1, bgr2, im_fwderr), axis=1)
vis = np.concatenate((vis1, vis2, vis3), axis = 0)
#cv2.imshow('Error', vis)
cv2.imwrite('./error.png', dst)
#Other measures of error include just comparing the flow mapped R1 to R2
#and vice versa
#Can use CV's remap function
rf1_recon = cv2.remap(rf2, fwdmeshx - flow1[:,:,0], fwdmeshy - flow1[:,:,1], cv2.INTER_LINEAR)
rf2_recon = cv2.remap(rf1, revmeshx - flow2[:,:,0], revmeshy - flow2[:,:,1], cv2.INTER_LINEAR)
#Compare these reconstructions to the original frames....
vis1 = np.concatenate((rf1, rf1_recon), axis = 1)
vis2 = np.concatenate((rf2, rf2_recon), axis = 1)
vis = np.concatenate((vis1, vis2), axis = 0)
#cv2.imshow('Recon', vis)
cv2.imwrite('./test.png', vis)
#Run local comparisons within a window...
sim1cc = np.zeros((nx,ny))
sim2cc = np.zeros((nx,ny))
sim1l2 = np.zeros((nx,ny))
sim2l2 = np.zeros((nx,ny))
gr_rf1 = cv2.cvtColor(rf1, cv2.COLOR_BGR2GRAY)
gr_rf2 = cv2.cvtColor(rf2, cv2.COLOR_BGR2GRAY)
gr_rf1_rc = cv2.cvtColor(rf1_recon, cv2.COLOR_BGR2GRAY)
gr_rf2_rc = cv2.cvtColor(rf2_recon, cv2.COLOR_BGR2GRAY)
r2 = radius**2
for i in range(radius, nx-radius):
for j in range(radius, ny-radius):
#Generate window around current point
pts1 = gr_rf1[i-radius:i+radius, j-radius:j+radius]
pts2 = gr_rf1_rc[i-radius:i+radius, j-radius:j+radius]
pts1 = np.reshape(pts1, (-1,1))
pts2 = np.reshape(pts2, (-1,1))
#Measure similiarty here
l2 = np.sqrt(np.sum((pts1 - pts2)**2))/r2
cc = np.corrcoef(pts1.T, pts2.T)[0,1]
sim1cc[i,j] = cc if not np.isnan(cc) else 0
sim1l2[i,j] = l2
#Generate window around current point
pts1 = gr_rf2[i-radius:i+radius, j-radius:j+radius]
pts2 = gr_rf2_rc[i-radius:i+radius, j-radius:j+radius]
pts1 = np.reshape(pts1, (-1,1))
pts2 = np.reshape(pts2, (-1,1))
#Measure similiarty here
l2 = np.sqrt(np.sum((pts1 - pts2)**2))/r2
cc = np.corrcoef(pts1.T, pts2.T)[0,1]
sim2cc[i,j] = cc if not np.isnan(cc) else 0
sim2l2[i,j] = l2
#Save these results
cv2.imwrite('./test_l2_err1.png', sim1l2)
cv2.imwrite('./test_l2_err2.png', sim2l2)
cv2.imwrite('./test_corr_err1.png', sim1cc*255)
cv2.imwrite('./test_corr_err2.png', sim2cc*255)