def label_util1():
scan = dset.Scan('../data/util1-real')
scan.im_files[3] = os.path.join(scan.path, 'raw/2014-05-24 17.34.10.jpg')
scan.im_files[4] = os.path.join(scan.path, 'raw/2014-05-24 17.35.07.jpg')
label_box(scan, 3)
mesh = box.load_from_mat('../data/util1-real/cube.mat')
ig.show([box.draw_faces(mesh, scan, 3, hires=0), scan.im(3)])
def label_util2():
scan = dset.Scan('../data/util2-real')
#ig.show([[frame, ig.resize(scan.im(frame), 0.5), f, ig.resize(ig.load(f), scan.scale*0.5)] for frame, f in zip(scan.frames, glob.glob(ut.pjoin(scan.path, 'raw', '*.jpg')))])
frame = 0
scan.im_files[
frame] = '/data/vision/billf/camo/camo/nondetect/data/util2-real/raw/2014-05-25 17.17.02.jpg'
label_box(scan, frame)
mesh = box.load_from_mat('../data/util2-real/cube.mat')
ig.show([box.draw_faces(mesh, scan, frame), scan.im(frame)])
def tour(scan,
mesh,
texel_colors,
frames,
n=40,
im_wait=2.2,
plane_idx=2,
par=0,
outline_start=np.inf,
outline_end=np.inf,
bad_pairs=[],
other_planes=[],
mesh_occ=None,
start_scale=1.25,
start_wo_outline=False):
#def tour(scan, mesh, texel_colors, frames, n = 40, im_wait = 2.2, plane_idx = 2, par = 0, bad_pairs = [], other_planes = [], mesh_occ = None):
if 0:
idx = mesh.face_idx
idx = idx[plane_idx:plane_idx + 1]
only_plane = box.Mesh(idx, mesh.mesh_pts)
ig.show(box.draw_faces(only_plane, scan, 0))
return
if mesh_occ is None:
mesh_occ = figures.MeshOcc(scan, None)
def f(frame1,
frame2,
n=n,
im_wait=im_wait,
plane_idx=plane_idx,
texel_colors=[texel_colors],
scan=scan,
mesh=mesh,
other_planes=other_planes,
bad_pairs=bad_pairs,
frames=frames,
mesh_occ=mesh_occ,
outline_start=outline_start,
outline_end=outline_end,
start_scale=start_scale,
start_wo_outline=start_wo_outline):
texel_colors = texel_colors[0]
ims = []
ps = np.linspace(0, 1, n)
frame1_idx = frames.index(frame1)
frame2_idx = frames.index(frame2)
for pi, p in enumerate(ps):
print p
p1 = 1 - p
p2 = p
R1, c1 = scan.R(frame1), scan.center(frame1)
R2, c2 = scan.R(frame2), scan.center(frame2)
Rp = slerp_R(R1, R2, p)
cp = p1 * c1 + p2 * c2
Kp = p1 * scan.K(frame1) + p2 * scan.K(frame2)
tp = -np.dot(Rp, cp)
Pp = mvg.compose_P(Kp, Rp, tp)
tc = texel_colors.copy()
if outline_start <= frame1_idx < outline_end: # or outline_start <= frame2_idx < outline_end:
if frame2_idx == outline_end:
interp_occ = p1
else:
interp_occ = 1.
print 'interp_occ =', interp_occ
scan_with_interp, frame_interp = scan.add_virtual_camera(
Kp, Rp, dset.t_from_Rc(Rp, cp), scan.im(frame1))
frame_round = (frame1 if p <= 0.5 else frame1)
tc = texel_colors.copy()
tc = figures.mark_occlusion_texels(
tc,
scan_with_interp,
mesh,
frame_interp,
thresh=2,
p=interp_occ,
mesh_occ_mask=np.array(
np.round(mesh_occ.mask(frame_round)), 'bool'))
if pi == 0 or pi == len(ps) - 1:
frame = (frame1 if pi == 0 else frame2)
#tc = figures.mark_occlusion_texels(tc, scan, mesh, frame, mesh_occ_mask = np.array(np.round(mesh_occ.mask(frame)), 'bool'), thresh = 2)
if frame == frames[0]:
# 2x brings it on par with a normal frame (which are counted twice); add another factor to
# give people a chance to scan
wait_time = int(start_scale * 2 * im_wait * n)
elif frame == frames[-1]:
wait_time = int(2 * im_wait * n)
else:
wait_time = int(im_wait * n)
if start_wo_outline:
ims += ([mesh.render(scan, frame, texel_colors)] * 3)
im = mesh.render(scan, frame, tc)
im0 = scan.im(frame)
ims += [mesh_occ.apply_mask(im, im0, mesh_occ.mask(frame))
] * wait_time
else:
im1 = scan.im(frame1)
im2 = scan.im(frame2)
planes = [mesh.face_planes[plane_idx]] + other_planes
#print other_planes
best_dists = None
for plane in planes:
# backproject pixel into both frames, using the plane as the geometry
# average the colors
im = np.zeros_like(im1)
#print plane
ray_dirs = ut.col_from_im(
ut.mult_im(Rp.T, mvg.ray_directions(Kp, im.shape)))
dists = (-plane[3] - np.dot(cp, plane[:3])) / np.dot(
ray_dirs, plane[:3])
if best_dists is None:
best_dists = dists
else:
# asdf
#best_dists[dists < best_dists] = 0
if 1:
ok = ((best_dists < 0) &
(dists >= 0)) | ((dists >= 0) &
(dists < best_dists))
best_dists[ok] = dists[ok]
pts = cp + ray_dirs * best_dists[:, np.newaxis]
proj1 = scan.project(frame1, pts)
proj2 = scan.project(frame2, pts)
color1 = ig.lookup_bilinear(im1, proj1[:, 0], proj1[:, 1])
color2 = ig.lookup_bilinear(im2, proj2[:, 0], proj2[:, 1])
in_bounds1 = ig.lookup_bilinear(np.ones(im1.shape[:2]),
proj1[:, 0], proj1[:, 1]) > 0
in_bounds2 = ig.lookup_bilinear(np.ones(im2.shape[:2]),
proj2[:, 0], proj2[:, 1]) > 0
p1s = np.array([p1] * len(proj1))
p1s[-in_bounds1] = 0
p2s = np.array([p2] * len(proj2))
p2s[-in_bounds2] = 0
s = p1s + p2s
p1s = p1s / np.maximum(s, 0.00001)
p2s = p2s / np.maximum(s, 0.00001)
#mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2)
#mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2)
mask1 = ig.lookup_bilinear(mesh_occ.mask(frame1), proj1[:, 0],
proj1[:, 1])
mask2 = ig.lookup_bilinear(mesh_occ.mask(frame2), proj2[:, 0],
proj2[:, 1])
mask = ut.im_from_col(im.shape[:2], mask1 * p1s + mask2 * p2s)
#mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2)
#mask[mask >= 0.5] = 1
if (frame1, frame2) in bad_pairs:
assert mesh_occ.path is None
ims.append(p1 * mesh.render(scan, frame1, tc, mask=mask) +
p2 * mesh.render(scan, frame2, tc, mask=mask))
else:
im = ut.im_from_col(
im.shape, color1 * p1s[:, np.newaxis] +
color2 * p2s[:, np.newaxis])
ims.append(
mesh.render(scan, (Rp, Pp, Kp, cp),
tc,
im=im,
mask=mask))
return ims
ims = ut.flatten(ip.map(par, f, (frames[:-1], frames[1:])))
#ut.toplevel_locals()
#ig.show([('animation', ims, 100*10./n)])
#ig.show([('animation', ims, 8*10./n)])
#ig.show([imtable.Video(ims, fps = 0.25*float(im_wait*n))])
#ig.show([imtable.Video(ims, fps = 0.75*float(im_wait*n))])
ut.toplevel_locals()
url = ig.show([imtable.Video(ims, fps=0.75 * float(im_wait * n))])
del ims
return url
def tour(scan, mesh, texel_colors, frames, n = 40, im_wait = 2.2,
plane_idx = 2, par = 0, outline_start = np.inf, outline_end = np.inf, bad_pairs = [], other_planes = [],
mesh_occ = None, start_scale = 1.25, start_wo_outline = False):
#def tour(scan, mesh, texel_colors, frames, n = 40, im_wait = 2.2, plane_idx = 2, par = 0, bad_pairs = [], other_planes = [], mesh_occ = None):
if 0:
idx = mesh.face_idx
idx = idx[plane_idx:plane_idx+1]
only_plane = box.Mesh(idx, mesh.mesh_pts)
ig.show(box.draw_faces(only_plane, scan, 0))
return
if mesh_occ is None:
mesh_occ = figures.MeshOcc(scan, None)
def f(frame1, frame2, n = n, im_wait = im_wait, plane_idx = plane_idx, texel_colors = [texel_colors], scan = scan, mesh = mesh,
other_planes = other_planes, bad_pairs = bad_pairs, frames = frames, mesh_occ = mesh_occ, outline_start = outline_start,
outline_end = outline_end, start_scale = start_scale, start_wo_outline = start_wo_outline):
texel_colors = texel_colors[0]
ims = []
ps = np.linspace(0, 1, n)
frame1_idx = frames.index(frame1)
frame2_idx = frames.index(frame2)
for pi, p in enumerate(ps):
print p
p1 = 1-p
p2 = p
R1, c1 = scan.R(frame1), scan.center(frame1)
R2, c2 = scan.R(frame2), scan.center(frame2)
Rp = slerp_R(R1, R2, p)
cp = p1*c1 + p2*c2
Kp = p1*scan.K(frame1) + p2*scan.K(frame2)
tp = -np.dot(Rp, cp)
Pp = mvg.compose_P(Kp, Rp, tp)
tc = texel_colors.copy()
if outline_start <= frame1_idx < outline_end:# or outline_start <= frame2_idx < outline_end:
if frame2_idx == outline_end:
interp_occ = p1
else:
interp_occ = 1.
print 'interp_occ =', interp_occ
scan_with_interp, frame_interp = scan.add_virtual_camera(Kp, Rp, dset.t_from_Rc(Rp, cp), scan.im(frame1))
frame_round = (frame1 if p <= 0.5 else frame1)
tc = texel_colors.copy()
tc = figures.mark_occlusion_texels(tc, scan_with_interp, mesh, frame_interp, thresh = 2,
p = interp_occ, mesh_occ_mask = np.array(np.round(mesh_occ.mask(frame_round)), 'bool'))
if pi == 0 or pi == len(ps)-1:
frame = (frame1 if pi == 0 else frame2)
#tc = figures.mark_occlusion_texels(tc, scan, mesh, frame, mesh_occ_mask = np.array(np.round(mesh_occ.mask(frame)), 'bool'), thresh = 2)
if frame == frames[0]:
# 2x brings it on par with a normal frame (which are counted twice); add another factor to
# give people a chance to scan
wait_time = int(start_scale*2*im_wait*n)
elif frame == frames[-1]:
wait_time = int(2*im_wait*n)
else:
wait_time = int(im_wait*n)
if start_wo_outline:
ims += ([mesh.render(scan, frame, texel_colors)]*3)
im = mesh.render(scan, frame, tc)
im0 = scan.im(frame)
ims += [mesh_occ.apply_mask(im, im0, mesh_occ.mask(frame))]*wait_time
else:
im1 = scan.im(frame1)
im2 = scan.im(frame2)
planes = [mesh.face_planes[plane_idx]] + other_planes
#print other_planes
best_dists = None
for plane in planes:
# backproject pixel into both frames, using the plane as the geometry
# average the colors
im = np.zeros_like(im1)
#print plane
ray_dirs = ut.col_from_im(ut.mult_im(Rp.T, mvg.ray_directions(Kp, im.shape)))
dists = (-plane[3] - np.dot(cp, plane[:3]))/np.dot(ray_dirs, plane[:3])
if best_dists is None:
best_dists = dists
else:
# asdf
#best_dists[dists < best_dists] = 0
if 1:
ok = ((best_dists < 0) & (dists >= 0)) | ((dists >= 0) & (dists < best_dists))
best_dists[ok] = dists[ok]
pts = cp + ray_dirs*best_dists[:, np.newaxis]
proj1 = scan.project(frame1, pts)
proj2 = scan.project(frame2, pts)
color1 = ig.lookup_bilinear(im1, proj1[:, 0], proj1[:, 1])
color2 = ig.lookup_bilinear(im2, proj2[:, 0], proj2[:, 1])
in_bounds1 = ig.lookup_bilinear(np.ones(im1.shape[:2]), proj1[:, 0], proj1[:, 1]) > 0
in_bounds2 = ig.lookup_bilinear(np.ones(im2.shape[:2]), proj2[:, 0], proj2[:, 1]) > 0
p1s = np.array([p1]*len(proj1))
p1s[-in_bounds1] = 0
p2s = np.array([p2]*len(proj2))
p2s[-in_bounds2] = 0
s = p1s + p2s
p1s = p1s / np.maximum(s, 0.00001)
p2s = p2s / np.maximum(s, 0.00001)
#mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2)
#mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2)
mask1 = ig.lookup_bilinear(mesh_occ.mask(frame1), proj1[:, 0], proj1[:, 1])
mask2 = ig.lookup_bilinear(mesh_occ.mask(frame2), proj2[:, 0], proj2[:, 1])
mask = ut.im_from_col(im.shape[:2], mask1*p1s + mask2*p2s)
#mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2)
#mask[mask >= 0.5] = 1
if (frame1, frame2) in bad_pairs:
assert mesh_occ.path is None
ims.append(p1*mesh.render(scan, frame1, tc, mask = mask) + p2*mesh.render(scan, frame2, tc, mask = mask))
else:
im = ut.im_from_col(im.shape, color1*p1s[:, np.newaxis] + color2*p2s[:, np.newaxis])
ims.append(mesh.render(scan, (Rp, Pp, Kp, cp), tc, im = im, mask = mask))
return ims
ims = ut.flatten(ip.map(par, f, (frames[:-1], frames[1:])))
#ut.toplevel_locals()
#ig.show([('animation', ims, 100*10./n)])
#ig.show([('animation', ims, 8*10./n)])
#ig.show([imtable.Video(ims, fps = 0.25*float(im_wait*n))])
#ig.show([imtable.Video(ims, fps = 0.75*float(im_wait*n))])
ut.toplevel_locals()
url = ig.show([imtable.Video(ims, fps = 0.75*float(im_wait*n))])
del ims
return url
def label_box(seq,
root=0,
side_len1=None,
side_len2=None,
side_len3=None,
y_flip=True,
mode='normal'):
print seq
if type(seq) == type(''):
scan = dset.Scan(seq, None)
else:
scan = seq
seq = scan.path
if mode == 'normal':
_, _, tracks = dset.read_bundler(scan.bundle_file, scan.full_shape)
pts = np.array([t[0] for t in tracks])
proj = scan.project(root, pts)
w = 1
pylab.clf()
im_with_pts = ig.draw_pts(scan.im(root), proj, width=w)
pylab.imshow(im_with_pts)
rect = ut.bbox2d(pylab.ginput(2, timeout=-1))
#rect = (1782.005828476269, 1431.7364696086595, 529.75936719400488, 354.40549542048279)
print rect
ok = ut.land(rect[0] <= proj[:, 0], proj[:, 0] <= rect[0] + rect[2],
rect[1] <= proj[:, 1], proj[:, 1] <= rect[1] + rect[3])
pts_in_box = pts[ok]
thresh = pylab.dist(scan.center(root), scan.center(root + 1)) / 50.
plane, _ = planefit.fit_plane_ransac(pts_in_box, thresh)
if plane[1] < 0 and y_flip:
plane *= -1
ins = planefit.plane_inliers(plane, pts, thresh)
pylab.clf()
colors = np.zeros_like(pts)
colors[:, 0] = 255
colors[ins] = (0, 255, 0)
im_ins = ig.draw_pts(scan.im(root),
map(ut.itup, proj),
map(ut.itup, colors),
width=w)
pylab.clf()
pylab.imshow(im_ins)
if not input('ok? '):
return
print 'click 2 points (used to recalibrate the plane)'
rect = ut.bbox2d(pylab.ginput(2, timeout=-1))
ok = ut.land(rect[0] <= proj[:, 0], proj[:, 0] <= rect[0] + rect[2],
rect[1] <= proj[:, 1], proj[:, 1] <= rect[1] + rect[3])
pts_in_box = pts[ok]
print 'plane before', plane
plane[3] = -np.median(np.dot(pts_in_box, plane[:3]))
print 'plane after', plane[3]
if 1:
print 'hack'
im_ins = scan.im(root)
pylab.clf()
pylab.imshow(im_ins)
print 'click 3 base points'
px = pylab.ginput(3, timeout=-1)
#px = [(2270.2989175686921, 1482.9937552039967), (2297.2764363030801, 1555.8330557868442), (2405.1865112406322, 1550.4375520399667)]
def backproj(p):
ray = ut.normalized(
np.dot(mvg.pixel_ray_matrix(scan.R(root), scan.K(root)),
ut.homog(p)))
c = scan.center(root)
dist = (-plane[3] - np.dot(c, plane[:3])) / np.dot(ray, plane[:3])
assert dist >= 0
pt = c + ray * dist
print planefit.dist_to_plane(plane, pt[np.newaxis, :])
return pt
sc = 1.
while 1:
cb = np.array(map(backproj, px))
v1 = cb[0] - cb[1]
v2 = cb[2] - cb[1]
if side_len1 is None:
side_len1 = 0.5 * (np.linalg.norm(v1) + np.linalg.norm(v2))
if side_len2 is None:
side_len2 = side_len1
if side_len3 is None:
side_len3 = side_len1
a1 = sc * side_len1
a2 = sc * side_len2
a3 = sc * side_len3
print 'side1', a1, 'side2', a2, 'side3', a3, 'thresh =', thresh, \
'v1 =', np.linalg.norm(v1), 'v2 = ', np.linalg.norm(v2)
R = np.zeros((3, 3))
cr = ut.normalized(np.cross(v1, plane[:3]))
cr *= np.sign(np.dot(cr, v2))
R[0] = a1 * ut.normalized(v1)
R[1] = a2 * ut.normalized(cr)
R[2] = a3 * ut.normalized(plane[:3])
print ut.normax(R, 1)
mesh_pts = []
for zi in xrange(2):
for yi in xrange(2):
for xi in xrange(2):
mesh_pts.append(cb[1] + R[0] * xi + R[1] * yi +
R[2] * zi)
face_idx = -1 + np.array(
[[1, 2, 4, 3],
np.array([1, 2, 4, 3]) + 4, [1, 2, 2 + 4, 1 + 4],
[2, 4, 4 + 4, 2 + 4], [4, 3, 3 + 4, 4 + 4],
[3, 1, 1 + 4, 3 + 4]])
mesh = box.Mesh(face_idx, mesh_pts, texsize=128)
# show a preview
scan_ = dset.Scan(seq)
ig.show(
[[1 + i,
box.draw_faces(mesh, scan_, i, hires=0),
scan_.im(i)] for i in [root, root + 1]])
if input('ok? '):
box.save_mesh(ut.pjoin(seq, 'cube.mat'), mesh)
break
else:
sc = float(input('scale? '))
time.sleep(2)
else:
mesh = box.load_from_mat(ut.pjoin(seq, 'cube.mat'))
scan = dset.Scan(seq, use_cams_file=False)
print 'Already marked as bad:'
good_cams_file = os.path.join(scan.path, 'good_cams.txt')
if os.path.exists(good_cams_file):
inds = map(int, open(good_cams_file, 'r').read().split())
file_ids = map(scan.file_index, scan.im_files)
bad = sorted(set(file_ids) - set(inds))
print '\n'.join(map(str, bad))
if 1:
ig.show([[
scan.file_index(scan.im_files[frame]),
box.draw_faces(mesh, scan, frame, hires=0)
] for frame in xrange(scan.length)])
inp = input('Bad cameras (as string): ')
if inp != 'skip':
bad_cams = map(int, inp.split())
all_idx = map(scan.file_index, scan.im_files)
good_cams = sorted(set(all_idx) - set(bad_cams))
ut.write_lines(ut.pjoin(seq, 'good_cams.txt'), map(str, good_cams))