def analyze_costs(scan,
mesh,
results,
costs,
texel_colors,
label_info,
cost_names=None,
smooth_faces=True,
frames=None):
""" Visualize which costs contributed to the solution (warning: slow) """
if frames is None:
frames = range(
scan.length
) #sorted(set(map(int, np.linspace(0, scan.length-1, 10))))
if cost_names is None:
cost_names = range(len(costs))
subcosts = np.array(
[cost[range(len(results)), results].copy() for cost in costs])
#total = float(np.max(subcosts))
print 'costs:'
table = [cost_names, map(ut.pr, [np.sum(x) for x in subcosts])]
if smooth_faces:
for subcost in subcosts:
print 'subcost', np.sum(subcost)
as_juv = np.squeeze(mesh.index_as_juv(subcost))
for j in xrange(as_juv.shape[0]):
D, I = scipy.ndimage.distance_transform_edt(
as_juv[j] == 0, return_indices=True)
res = np.where(D < 20, as_juv[j][I[0], I[1]], as_juv[j]).copy()
as_juv[j, :, :] = res
print np.max(as_juv)
subcost[:] = mesh.index_as_flat(as_juv).flatten()
labeling_colors = np.array(ut.distinct_colors(len(label_info)))
labeling_colors[ut.land(label_info[:, 1] == 0, label_info[:,
2] == 0)] = 255
for frame in frames:
row = [mesh.render(scan, frame, texel_colors)]
frame_lc = labeling_colors.copy()
frame_lc[label_info[:, 0] != frame] = 0
row.append(mesh.render(scan, frame, frame_lc[results]))
for subcost in subcosts:
total = float(np.max(subcost))
colors = np.clip(
255 * np.tile(subcost[:, np.newaxis], (1, 3)) / total, 0, 255)
row.append(mesh.render(scan, frame, colors))
table.append(row)
ig.show(table)
def backproj(face_visible):
# (alpha*r + t)*n + d = 0
# alpha*r'n + t'*n + d = 0
# (-d - t'*n)/(r'*n)
ray_juv = -1 + np.zeros(rays.shape, 'd')
best_dist = np.inf + np.zeros(rays.shape[0])
if len(rays) == 0:
return ray_juv, best_dist
for j in np.nonzero(face_visible)[0]:
plane = self.face_planes[j]
dist = (-plane[3] - np.dot(center, plane[:3])) / np.dot(
rays, plane[:3])
#dist2 = (-plane[3] - np.dot(center, plane[:3]))/np.dot(rays, plane[:3])
pts = dist[:, na] * rays + center
p1, edges = self.face_edges[j]
# R = edges / ut.normax(edges, 1)[:, na]
# uv = np.array((self.texsize-1)*np.dot(R/ut.normax(edges, 1)[:, na], (pts - p1).T), 'd').T
uv = np.linalg.lstsq(edges.T, (pts - p1).T)[0].T
in_bounds = ut.land(0 <= uv[:, 0], uv[:, 0] <= 1,
0 <= uv[:, 1], uv[:, 1] <= 1)
uv = np.array((self.texsize - 1) * uv)
visible = ut.land(in_bounds, 0 <= dist, dist < best_dist)
best_dist[visible] = dist[visible]
ray_juv[visible, 0] = j
ray_juv[visible, 1] = uv[visible, 0]
ray_juv[visible, 2] = uv[visible, 1]
#print np.bincount(np.array(ray_juv[ray_juv[:, 0] >= 0, 0], 'l'), minlength = 6)
# in first call to backproj this assertion might fail (need to double check the rest of the code if that happens)
assert np.sum(
np.bincount(np.array(ray_juv[ray_juv[:, 0] >= 0, 0], 'l'),
minlength=6) > 0) <= 3
return ray_juv, best_dist
def backproj(face_visible):
# (alpha*r + t)*n + d = 0
# alpha*r'n + t'*n + d = 0
# (-d - t'*n)/(r'*n)
ray_juv = -1 + np.zeros(rays.shape, 'd')
best_dist = np.inf + np.zeros(rays.shape[0])
if len(rays) == 0:
return ray_juv, best_dist
for j in np.nonzero(face_visible)[0]:
plane = self.face_planes[j]
dist = (-plane[3] - np.dot(center, plane[:3]))/np.dot(rays, plane[:3])
#dist2 = (-plane[3] - np.dot(center, plane[:3]))/np.dot(rays, plane[:3])
pts = dist[:, na]*rays + center
p1, edges = self.face_edges[j]
# R = edges / ut.normax(edges, 1)[:, na]
# uv = np.array((self.texsize-1)*np.dot(R/ut.normax(edges, 1)[:, na], (pts - p1).T), 'd').T
uv = np.linalg.lstsq(edges.T, (pts - p1).T)[0].T
in_bounds = ut.land(0 <= uv[:, 0], uv[:, 0] <= 1,
0 <= uv[:, 1], uv[:, 1] <= 1)
uv = np.array((self.texsize-1)*uv)
visible = ut.land(in_bounds, 0 <= dist, dist < best_dist)
best_dist[visible] = dist[visible]
ray_juv[visible, 0] = j
ray_juv[visible, 1] = uv[visible, 0]
ray_juv[visible, 2] = uv[visible, 1]
#print np.bincount(np.array(ray_juv[ray_juv[:, 0] >= 0, 0], 'l'), minlength = 6)
# in first call to backproj this assertion might fail (need to double check the rest of the code if that happens)
assert np.sum(np.bincount(np.array(ray_juv[ray_juv[:, 0] >= 0, 0], 'l'), minlength = 6) > 0) <= 3
return ray_juv, best_dist
def occlusion_texels(scan, mesh, frame, thresh = 1., only_border = True):
occ_mask = np.array(occlusion_mask(scan, mesh, frame, thresh = thresh), 'd')
vis = mesh.texel_visible(scan, frame)
proj = np.array(scan.project(frame, mesh.texel_pts), 'l')
occ = np.zeros(mesh.ntexels, 'bool')
occ[vis] = occ_mask[proj[vis, 1], proj[vis, 0]]
if only_border:
occ = ut.land(occ, mesh.on_border)
assert np.all(vis[occ])
return occ
def occlusion_texels(scan, mesh, frame, thresh=1., only_border=True):
occ_mask = np.array(occlusion_mask(scan, mesh, frame, thresh=thresh), 'd')
vis = mesh.texel_visible(scan, frame)
proj = np.array(scan.project(frame, mesh.texel_pts), 'l')
occ = np.zeros(mesh.ntexels, 'bool')
occ[vis] = occ_mask[proj[vis, 1], proj[vis, 0]]
if only_border:
occ = ut.land(occ, mesh.on_border)
assert np.all(vis[occ])
return occ
def analyze_costs(scan, mesh, results, costs, texel_colors, label_info, cost_names = None, smooth_faces = True, frames = None):
""" Visualize which costs contributed to the solution (warning: slow) """
if frames is None:
frames = range(scan.length)#sorted(set(map(int, np.linspace(0, scan.length-1, 10))))
if cost_names is None:
cost_names = range(len(costs))
subcosts = np.array([cost[range(len(results)), results].copy() for cost in costs])
#total = float(np.max(subcosts))
print 'costs:'
table = [cost_names, map(ut.pr, [np.sum(x) for x in subcosts])]
if smooth_faces:
for subcost in subcosts:
print 'subcost', np.sum(subcost)
as_juv = np.squeeze(mesh.index_as_juv(subcost))
for j in xrange(as_juv.shape[0]):
D, I = scipy.ndimage.distance_transform_edt(as_juv[j] == 0, return_indices = True)
res = np.where(D < 20, as_juv[j][I[0], I[1]], as_juv[j]).copy()
as_juv[j, :, :] = res
print np.max(as_juv)
subcost[:] = mesh.index_as_flat(as_juv).flatten()
labeling_colors = np.array(ut.distinct_colors(len(label_info)))
labeling_colors[ut.land(label_info[:, 1] == 0, label_info[:, 2] == 0)] = 255
for frame in frames:
row = [mesh.render(scan, frame, texel_colors)]
frame_lc = labeling_colors.copy()
frame_lc[label_info[:, 0] != frame] = 0
row.append(mesh.render(scan, frame, frame_lc[results]))
for subcost in subcosts:
total = float(np.max(subcost))
colors = np.clip(255*np.tile(subcost[:, np.newaxis], (1, 3))/total, 0, 255)
row.append(mesh.render(scan, frame, colors))
table.append(row)
ig.show(table)
def pixels_in_bounds(im_shape, xs, ys):
return ut.land(0 <= xs, xs < im_shape[1],
0 <= ys, ys < im_shape[0])
def pixels_in_bounds(im_shape, xs, ys):
return ut.land(0 <= xs, xs < im_shape[1], 0 <= ys, ys < im_shape[0])
def label_box(seq,
root=0,
side_len1=None,
side_len2=None,
side_len3=None,
y_flip=True,
mode='normal'):
print seq
if type(seq) == type(''):
scan = dset.Scan(seq, None)
else:
scan = seq
seq = scan.path
if mode == 'normal':
_, _, tracks = dset.read_bundler(scan.bundle_file, scan.full_shape)
pts = np.array([t[0] for t in tracks])
proj = scan.project(root, pts)
w = 1
pylab.clf()
im_with_pts = ig.draw_pts(scan.im(root), proj, width=w)
pylab.imshow(im_with_pts)
rect = ut.bbox2d(pylab.ginput(2, timeout=-1))
#rect = (1782.005828476269, 1431.7364696086595, 529.75936719400488, 354.40549542048279)
print rect
ok = ut.land(rect[0] <= proj[:, 0], proj[:, 0] <= rect[0] + rect[2],
rect[1] <= proj[:, 1], proj[:, 1] <= rect[1] + rect[3])
pts_in_box = pts[ok]
thresh = pylab.dist(scan.center(root), scan.center(root + 1)) / 50.
plane, _ = planefit.fit_plane_ransac(pts_in_box, thresh)
if plane[1] < 0 and y_flip:
plane *= -1
ins = planefit.plane_inliers(plane, pts, thresh)
pylab.clf()
colors = np.zeros_like(pts)
colors[:, 0] = 255
colors[ins] = (0, 255, 0)
im_ins = ig.draw_pts(scan.im(root),
map(ut.itup, proj),
map(ut.itup, colors),
width=w)
pylab.clf()
pylab.imshow(im_ins)
if not input('ok? '):
return
print 'click 2 points (used to recalibrate the plane)'
rect = ut.bbox2d(pylab.ginput(2, timeout=-1))
ok = ut.land(rect[0] <= proj[:, 0], proj[:, 0] <= rect[0] + rect[2],
rect[1] <= proj[:, 1], proj[:, 1] <= rect[1] + rect[3])
pts_in_box = pts[ok]
print 'plane before', plane
plane[3] = -np.median(np.dot(pts_in_box, plane[:3]))
print 'plane after', plane[3]
if 1:
print 'hack'
im_ins = scan.im(root)
pylab.clf()
pylab.imshow(im_ins)
print 'click 3 base points'
px = pylab.ginput(3, timeout=-1)
#px = [(2270.2989175686921, 1482.9937552039967), (2297.2764363030801, 1555.8330557868442), (2405.1865112406322, 1550.4375520399667)]
def backproj(p):
ray = ut.normalized(
np.dot(mvg.pixel_ray_matrix(scan.R(root), scan.K(root)),
ut.homog(p)))
c = scan.center(root)
dist = (-plane[3] - np.dot(c, plane[:3])) / np.dot(ray, plane[:3])
assert dist >= 0
pt = c + ray * dist
print planefit.dist_to_plane(plane, pt[np.newaxis, :])
return pt
sc = 1.
while 1:
cb = np.array(map(backproj, px))
v1 = cb[0] - cb[1]
v2 = cb[2] - cb[1]
if side_len1 is None:
side_len1 = 0.5 * (np.linalg.norm(v1) + np.linalg.norm(v2))
if side_len2 is None:
side_len2 = side_len1
if side_len3 is None:
side_len3 = side_len1
a1 = sc * side_len1
a2 = sc * side_len2
a3 = sc * side_len3
print 'side1', a1, 'side2', a2, 'side3', a3, 'thresh =', thresh, \
'v1 =', np.linalg.norm(v1), 'v2 = ', np.linalg.norm(v2)
R = np.zeros((3, 3))
cr = ut.normalized(np.cross(v1, plane[:3]))
cr *= np.sign(np.dot(cr, v2))
R[0] = a1 * ut.normalized(v1)
R[1] = a2 * ut.normalized(cr)
R[2] = a3 * ut.normalized(plane[:3])
print ut.normax(R, 1)
mesh_pts = []
for zi in xrange(2):
for yi in xrange(2):
for xi in xrange(2):
mesh_pts.append(cb[1] + R[0] * xi + R[1] * yi +
R[2] * zi)
face_idx = -1 + np.array(
[[1, 2, 4, 3],
np.array([1, 2, 4, 3]) + 4, [1, 2, 2 + 4, 1 + 4],
[2, 4, 4 + 4, 2 + 4], [4, 3, 3 + 4, 4 + 4],
[3, 1, 1 + 4, 3 + 4]])
mesh = box.Mesh(face_idx, mesh_pts, texsize=128)
# show a preview
scan_ = dset.Scan(seq)
ig.show(
[[1 + i,
box.draw_faces(mesh, scan_, i, hires=0),
scan_.im(i)] for i in [root, root + 1]])
if input('ok? '):
box.save_mesh(ut.pjoin(seq, 'cube.mat'), mesh)
break
else:
sc = float(input('scale? '))
time.sleep(2)
else:
mesh = box.load_from_mat(ut.pjoin(seq, 'cube.mat'))
scan = dset.Scan(seq, use_cams_file=False)
print 'Already marked as bad:'
good_cams_file = os.path.join(scan.path, 'good_cams.txt')
if os.path.exists(good_cams_file):
inds = map(int, open(good_cams_file, 'r').read().split())
file_ids = map(scan.file_index, scan.im_files)
bad = sorted(set(file_ids) - set(inds))
print '\n'.join(map(str, bad))
if 1:
ig.show([[
scan.file_index(scan.im_files[frame]),
box.draw_faces(mesh, scan, frame, hires=0)
] for frame in xrange(scan.length)])
inp = input('Bad cameras (as string): ')
if inp != 'skip':
bad_cams = map(int, inp.split())
all_idx = map(scan.file_index, scan.im_files)
good_cams = sorted(set(all_idx) - set(bad_cams))
ut.write_lines(ut.pjoin(seq, 'good_cams.txt'), map(str, good_cams))