def normals_label(tans, qs, keep, labels, s, local_normals, labels_frame):
label = 1
for i in range(len(keep) - 1):
j = keep[i]
k = s + i
if labels[j] != 0:
labels_frame[k] = label
else:
labels_frame[k] = 0
normal = Geom.cross3(tans[j], qs[j])
mag = Geom.norm3(normal)
if mag == 0:
local_normals[k] = 1, 0, 0
labels_frame[k] = 0
else:
normal /= mag
local_normals[k] = normal
if labels[j + 1] != labels[j]:
label += 1
j = keep[-1]
normal = Geom.cross3(tans[j], qs[j])
normal /= Geom.norm3(normal)
local_normals[k] = normal
local_normals[s + len(keep) - 1] = normal
labels_frame[s + len(keep) - 1] = 0
def find_contours(rgb, d, max_depth=2**16 - 1, snakes=False):
holes = Image.find_blobs(d == 0)
N = len(holes)
hole_lookup = np.full(d.shape, -1, int)
for i, hole in enumerate(holes):
I, J = hole.T
hole_lookup[I, J] = i
max_val = np.iinfo(d.dtype).max
blob_mins = min_touching(hole_lookup, d, N, max_val)
thresh = 30
outlines, flags = find_outlines(d, hole_lookup, blob_mins, thresh=thresh)
contours, values, _ = Image.find_contours(d,
thresh=100 * thresh,
min_len=10,
values=outlines,
flags=flags,
dtype=float)
if snakes:
grey = rgb2grey(rgb)
edges = 255 * canny(grey, sigma=2).astype(np.uint8)
edges = gaussian(edges, sigma=2)
new_contours = []
normals = []
for i, cont in enumerate(contours):
vals = values[i]
if (vals > max_depth).any():
continue
pts = Geom.smooth_points(cont, sigma=3, support=5)
if snakes:
# pts += 2*Geom.smooth_contour_normals(pts, radius=4)
pts = active_contour(edges,
pts,
alpha=0.1,
beta=20,
gamma=0.005,
w_line=5,
bc='fixed')
# outlines has been conveniently emptied by find_contours
coords = Image.draw_poly(outlines, np.round(pts).astype(int), val=vals)
new_contours.append(coords)
normals.append(Geom.smooth_contour_normals(coords, radius=4))
# show_d(outlines)
return outlines, new_contours, normals
def render_lsd(seq):
plypath = os.path.join(seq, 'render_me.ply')
verts, edges = IO.read_ply(plypath)
lsd_mesh = Mesh()
lsd_mesh.verts = verts.astype(np.float32)
campath = os.path.join(seq, 'render.py')
Rs, ts = IO.import_blender_cam(campath)
W, H = 1920, 1080
im = empty((H, W, 3), np.uint8)
cam = Geom.cam_params(29.97, W, H, 35) # Blender defaults
cam = cam.astype(np.float32)
render_out = os.path.join(seq, 'render')
Util.try_mkdir(render_out)
im[:] = 255
pts = lsd_mesh.project_verts(Rs[0].astype(np.float32), ts[0].astype(np.float32), cam)
pt_buf = buffer_points(pts, np.r_[255, 82, 82], alpha=0.2, size=2)
render_buffer(im, pt_buf, bg=255)
out = os.path.join(render_out, 'frame_0.png')
IO.imwrite(out, im)
def normals_from_ray_pairs(Qs):
'''
Each sequential ray is treated as bases for normals
'''
N = Qs.shape[0]
normals = np.empty_like(Qs)
for i in range(0, N - 1):
n = Geom.cross3(Qs[i + 1], Qs[i])
mag = Geom.norm3(n)
for j in range(3):
normals[i, j] = n[j] / mag
normals[-1] = normals[-2]
return normals
def set_poses(self, R, t):
self.F = R.shape[0]
self.Rs = R.astype(np.float32)
self.ts = t.astype(np.float32)
self.Cs = Geom.global_to_local(R, t)[1].astype(np.float32)
if len(self.frame_range) == 0:
self.frame_range = zeros((self.F, 2), np.uint32)
def apply_bounding_boxes(self, frames, bboxes):
tw = Geom.global_to_local(self.Rs, self.ts)[1]
center = tw.mean(axis=0)
size = 5.0 * np.abs(tw - center).max(axis=0)
# center = np.r_[0.0, 0.0, 1.0]
# size = np.r_[1.0, 1.0, 1.0]
bbox_min, bbox_max = Geom.frustrum_bounding_box(self.Rs,
self.ts,
self.cam,
frames,
bboxes,
center=center,
size=size,
res=100)
print("BBOX:", bbox_min, bbox_max)
self.bbox_min = bbox_min
self.bbox_max = bbox_max
def render_frame(im, object_mesh, occlusion_mesh, poly_mesh, R, t, cam, light=None, color=None, poly_color=None):
if light is None:
# lazi, lalt = np.deg2rad(30), np.deg2rad(60)
lazi, lalt = np.deg2rad(30), np.deg2rad(45)
light = np.r_[cos(lazi)*cos(lalt), sin(lazi)*cos(lalt), sin(lalt)].astype(np.float32)
light /= Geom.norm3(light)
light = dot(R.T, light).astype(np.float64)
# light = light.astype(np.float64)
# raise Exception("TODO: Light")
view = -dot(R.T, t)
pts_obj = object_mesh.project_verts(R, t, cam)
obj_map = np.full(im.shape[:2], np.inf, np.float32)
render_depth(obj_map, pts_obj, object_mesh)
# --- Buffer occlusion lines ----
if color is None:
color = np.r_[0, 155, 255]
pts_occl = occlusion_mesh.project_verts(R, t, cam)
norm_proj = dot(occlusion_mesh.normals, R.T)
if len(occlusion_mesh.edges) == 0:
line_buf = buffer_points(pts_occl, color, alpha=0.1, size=3)
else:
line_buf = buffer_lines(pts_occl, norm_proj, occlusion_mesh, color, view, light, alpha=0.1, shade_method=NORMAL)
if poly_mesh is not None:
if poly_color is None:
poly_color = np.r_[255, 166, 0]
pts_poly = poly_mesh.project_verts(R, t, cam)
norm_poly = dot(poly_mesh.normals, R.T)
poly_buf = buffer_lines(pts_poly, norm_poly, poly_mesh, poly_color, view, light, linewidth=4, shade_method=FLAT)
poly_buf[:, 2] = 0.001
else:
poly_buf = np.empty((0, line_buf.shape[1]), line_buf.dtype)
# Make buffer out of object depth map, and append to line buffer
I, J = np.where(obj_map != np.inf)
depths = 1.02*obj_map[I, J]
object_buf = empty((len(I), line_buf.shape[1]), line_buf.dtype)
object_buf[:, 0] = I
object_buf[:, 1] = J
object_buf[:, 2] = depths
object_buf[:, 3] = 0.4
object_buf[:, 4:] = 255, 255, 255
final_buf = np.vstack((line_buf, poly_buf, object_buf))
render_buffer(im, final_buf, bg=255)
def find_intersections(Cs, Qs, plucker, ray_ind, plane, sel):
'''
Cs, Qs, plucker: Canonical and Plucker representations of set of rays
ray_ind: Specific inds of the rays/ray triangle that's being intersected
plane: Plane describing the ray triangle
sel: Subset of rays to check
'''
N = Cs.shape[0]
pl1 = plucker[ray_ind:ray_ind + 2]
intersects = empty(N, np.int64)
ps = empty((N, 3), Cs.dtype)
us = empty((N, 3), Cs.dtype)
j = 0
for k, i in enumerate(sel):
pl2 = plucker[i:i + 2]
skew = empty(4, np.int16)
for a in range(2):
for b in range(2):
skew[2 * a + b] = np.sign(Geom.check_plucker(pl1[a], pl2[b]))
if abs(skew.sum()) == 4:
# Both rays skew on the same side of the other 2 rays
continue
# Find the intersection of the rays with the plane
c1, c2 = Cs[i], Cs[i + 1]
q1, q2 = Qs[i], Qs[i + 1]
t1, int1 = Geom.intersect_line_plane(c1, q1, plane)
t2, int2 = Geom.intersect_line_plane(c2, q2, plane)
if not int1 or not int2:
continue
intersects[j] = k
ps[j] = c1 + t1 * q1
us[j] = (c2 + t2 * q2) - ps[j]
j += 1
return ps[:j], us[:j], intersects[:j]
def find_in_cone(cone_center, cone_ray, cone_radius, Cs, Qs):
N = Cs.shape[0]
sel = empty(N, np.int64)
j = 0
for i in range(N):
in_cone = Geom.check_intersect_cone_ray(cone_center, cone_ray,
cone_radius, Cs[i], Qs[i])
if in_cone:
sel[j] = i
j += 1
return sel[:j]
def check_ray_bounds(qs, R, t, labels, b_min, b_max):
C = -dot(R.T, t)
Qs = dot(qs, R)
N = Qs.shape[0]
good = empty(N, np.int64)
j = 0
for i in range(N):
if Geom.intersect_ray_aabb(C, Qs[i], b_min, b_max):
good[j] = i
j += 1
else:
labels[i - 1:i + 2] = 0
return good[:j]
def find_close_rays(Cs,
Qs,
frames,
ind,
eps=1e-3,
min_angle=np.pi / 6,
min_depth=0.1,
max_depth=np.inf,
cone=False):
c = Cs[ind]
q = Qs[ind]
frame = frames[ind]
N = Cs.shape[0]
sel = empty(N, np.int64)
j = 0
for i in range(N):
if frames[i] == frame:
continue
c2 = Cs[i]
q2 = Qs[i]
dist, depth = Geom.line_distance_3d(c, q, c2, q2)
if (not cone and dist > eps) or (
cone and dist * depth > eps) or not (0.1 < depth < max_depth):
continue
# # Want to check if it's close enough to plane defined as
# # tangent to q and n
# dc = (c2 - c)
# dc -= dc * dot(dc, q) # Remove q component
# mag = norm3(dc)
# if mag == 0:
# # Other view lies on the same ray
# continue
# dc /= mag
# if abs(dot(dc, n)) < min_cos:
# continue
sel[j] = i
j += 1
return sel[:j]
def test_render_cube():
import pynutmeg
import time
fig = pynutmeg.figure('cube', 'figs/imshow.qml')
im = empty((720, 1280, 3), np.uint8)
azi = 0
alt = np.deg2rad(40)
dist = 10
cam = Geom.cam_params(29.97, 1280, 720, 35) # Blender defaults
cam = cam.astype(np.float32)
mesh = get_cube_mesh(1)
obj_mesh = Mesh()
n = 0
while True:
n += 1
azi += np.deg2rad(2)
tw = dist * np.array([cos(alt)*cos(azi), cos(alt)*sin(azi), sin(alt)], np.float32)
alpha = -np.pi/2 - alt
beta = np.pi/2 + azi
Rx = np.array([
[1, 0, 0],
[0, cos(alpha), -sin(alpha)],
[0, sin(alpha), cos(alpha)]
], np.float32)
Rz = np.array([
[cos(beta), -sin(beta), 0],
[sin(beta), cos(beta), 0],
[0, 0, 1]
], np.float32)
Rw = dot(Rz, Rx)
t = -dot(Rw.T, tw)
R = Rw.T
im[:] = 255
render_frame(im, obj_mesh, mesh, R, t, cam)
time.sleep(0.005)
fig.set('ax.im', binary=im)
def intersect_pairs(c, q):
'''
Find intersections of each sequential pair of rays
c: ray origin
q: ray direction
'''
N = c.shape[1]
lines = empty((N, 3))
lines[:, 0] = -q[1]
lines[:, 1] = q[0]
lines[:, 2] = -sum_ax(c.T * lines[:, :2], axis=1)
points = empty((N - 1, 2))
for i in range(N - 1):
p = Geom.cross3(lines[i], lines[i + 1])
points[i] = p[:2] / p[2]
return points.T
def smooth_polys(verts, starts, sizes, support=10):
poly_dir = np.empty_like(verts)
S = len(starts)
for p in range(S):
start = starts[p]
size = sizes[p]
poly = verts[start:start + size]
N = len(poly)
for i in range(N):
s = max(0, i - support)
e = min(N - 1, i + support)
v = poly[e] - poly[s]
v /= Geom.norm3(v)
poly_dir[start + i] = v
return poly_dir
def estimate_normals(pts, labels, support=3):
N = pts.shape[0]
ns = empty((N, 2), np.float32)
curvs = empty(N, np.float32)
for i in range(N):
label = labels[i]
s = max(0, i - support)
e = min(N, i + support + 1)
while labels[s] != label:
s += 1
while labels[e - 1] != label:
e -= 1
P = pts[s:e]
normal, curv = Geom.fit_normal_2d(P)
ns[i] = normal
curvs[i] = curv
return ns, curvs
def find_intersections(Cs, Qs, plucker, ray_ind, planes, sel):
'''
Plucker coords make the collision check quite fast, requiring few ops.
Cs, Qs, plucker: Canonical and Plucker representations of set of rays
ray_ind: Specific inds of the rays/ray triangle that's being intersected
plane: Plane describing the ray triangle
sel: Subset of rays to check
'''
N = Cs.shape[0]
pl1a, pl1b = plucker[ray_ind], plucker[ray_ind + 1]
plane = planes[ray_ind]
intersects = empty(N, np.int64)
ps1 = empty((N, 3), Cs.dtype)
us = empty((N, 3), Cs.dtype)
j = 0
pl2a = plucker.T[:, sel]
pl2b = plucker.T[:, sel + 1]
skews = np.sign( Geom.check_plucker(pl1a, pl2a) ) \
+ np.sign( Geom.check_plucker(pl1a, pl2b) ) \
+ np.sign( Geom.check_plucker(pl1b, pl2a) ) \
+ np.sign( Geom.check_plucker(pl1b, pl2b) )
for s, i in enumerate(sel):
if abs(skews[s]) == 4:
# Both rays skew on the same side of the other 2 rays
continue
# Find the intersection of the rays with the plane
c1, c2 = Cs[i], Cs[i + 1]
q1, q2 = Qs[i], Qs[i + 1]
t1, int1 = Geom.intersect_line_plane(c1, q1, plane)
t2, int2 = Geom.intersect_line_plane(c2, q2, plane)
if not int1 or not int2:
continue
intersects[j] = i
for k in range(3):
ps1[j, k] = c1[k] + t1 * q1[k]
us[j, k] = (c2[k] + t2 * q2[k]) - ps1[j, k]
# ps2[j, k] = c2[k] + t2*q2[k]
j += 1
return ps1[:j], us[:j], intersects[:j]
def test_contours_rgb():
import Edge_Tracker
seq = 'data/synth/plant_out'
out_d = os.path.join(seq, 'outlines_out')
IO.imshow(zeros((3, 3, 3), np.uint8))
try:
os.makedirs(out_d)
except FileExistsError:
pass
h, w = 360, 640
cam = Geom.cam_params(f=29.9, sw=35.0, w=w, h=h)
fx, fy, cx, cy = cam
def to_homogenious(pts):
N = pts.shape[0]
out = empty((N, 3))
out[:, 2] = 1
out[:, 0] = (pts[:, 0] - cx) * (1 / fx)
out[:, 1] = (pts[:, 1] - cy) * (1 / fy)
return out
def from_homogenious(pts):
x = (fx * pts[:, 0] + cx).astype(int)
y = (fy * pts[:, 1] + cy).astype(int)
return x, y
i = 0
contour_set = []
normal_set = []
P_last = None
pose = zeros(6)
try:
for im, D in load_datas(seq, imext='png'):
print("Frame {}".format(i))
P, normals, edge_map = find_contours_rgb(im, D)
P_hom = to_homogenious(P)
H, W = im.shape[:2]
if P_last is not None:
P_tr, pose, tree = Edge_Tracker.align(P_last, P_hom, pose=pose)
P_tr = from_homogenious(P_tr)
np.clip(P_tr[0], 0, W - 1, out=P_tr[0])
np.clip(P_tr[1], 0, H - 1, out=P_tr[1])
align = np.zeros_like(im)
last_u, last_v = from_homogenious(P_last)
align[P[:, 1], P[:, 0], 2] = 255
align[last_v, last_u, 1] = 255
align[P_tr[1], P_tr[0], 0] = 255
IO.imshow(align, "align")
# outlines, contours, normals = find_contours_rgb(im, D, snakes=False)
# imio.imsave(os.path.join(out_d, "{:04d}.png".format(i)), outlines.astype(np.uint16)*2**3)
# contour_set.append(contours)
# normal_set.append(normals)
# time.sleep(1)
P_last = P_hom
i += 1
except Exception as e:
print(e)
raise
np.savez(os.path.join(seq, 'contours.npz'),
contours=contour_set,
normals=normal_set)
gt_lms = np.loadtxt(gt_lms_path)
nL = gt_lms.shape[0]
mesh, meshfaces = read_obj('mean.obj')
tls = meshfaces - 1
print("template mean mesh norm :",
np.linalg.norm(mesh[2087, :] - mesh[14471, :]))
print("ground truth mesh norm: ", np.linalg.norm(gt_lms[0] - gt_lms[3]))
##scale up template mesh##
scale = np.linalg.norm(mesh[2087, :] -
mesh[14471, :]) / np.linalg.norm(gt_lms[0] - gt_lms[3])
mesh = mesh / scale
R, t = Geom.rigid_transform_3D(mesh[bfm_landmark_indices, :], gt_lms)
mesh = dot(R, mesh.T) + t.reshape(3, 1)
mesh = mesh.T
nV = mesh.shape[0]
centre = np.mean(target, axis=0)
scale_factor = (np.linalg.norm(mesh[2087, :] - mesh[14471, :])) * 2
mesh_norms = get_vertex_normals(mesh, tls)
t_orig = target.copy()
target = target - centre
target = target / scale_factor
mesh = mesh - centre
mesh = mesh / scale_factor
orig_mesh = mesh.copy()
def normalize(A):
return A / Geom.norm1d(A)
def cache_segment(Cs, Qs, planes, pluckers, frames, labels, raygrid, inds,
eps):
starts = []
deltas = []
depths = []
sideps = []
sideds = []
ray_frames = []
planar_angles = []
sel_inds = []
j = 0
M = len(inds)
for ind in range(M):
ray_ind = inds[ind]
j += 1
print(j)
print("Checking ray_ind:", ray_ind)
# TODO: Check that this is still working correctly for the triangles
in_cone = raygrid.rays_near_ind(ray_ind)
print("Near: {}/{}".format(len(in_cone), raygrid.n_rays))
if len(in_cone) == 0:
print("Total grid for ray:", raygrid.ray2vox_count[ray_ind])
center = Cs[ray_ind]
ray = Qs[ray_ind:ray_ind + 2].mean(axis=0)
ray /= Geom.norm3(ray)
plane = planes[ray_ind]
normal = plane[:3]
tangent = cross3(ray, normal)
print("Intersect planes")
# TODO: Use Plucker coords to quickly calculate intersection dual ray segments (infinite triangles?)
# Will need labels. Precalc Plucker before (do same as the planes)
# ps -> us, should describe how the other segment intersects this plane
# in terms of it's width
ps, us, intersecting = find_intersections(Cs, Qs, pluckers, ray_ind,
plane, in_cone)
print("Project")
R = empty((3, 3), ps.dtype)
R[0] = ray
R[1] = tangent
R[2] = normal
ps2d = dot(R[:2], (ps - center).T)
us2d = dot(R[:2], us.T)
# Solve for the intersection with the center of the segment
# [x, 0] = p + t*u
us2d[1, us2d[1] == 0] = 1e-10
ts = -ps2d[1] / us2d[1]
ds = ps2d[0] + ts * us2d[0]
# Keep only lines that are more vertical
# vert = np.abs(us2d[0] / (np.abs(us2d[1]) + 1e-12)) < 0.4
crossing = ps2d[1] * (ps2d[1] + us2d[1]) < 0
vert = np.arctan(us2d[0] / us2d[1]) < 0.85
# near = np.abs(ps2d[1]) < eps*ps2d[0]
forward = (ds > 0.2) & (ds < 1e6)
intersecting = in_cone[intersecting]
centers = dot(R, (Cs[intersecting] - center).T)
sidep = empty((2, len(intersecting)), Cs.dtype)
sidep[0] = centers[0]
sidep[1] = centers[2]
rays = empty((3, len(intersecting)), Cs.dtype)
rays[0] = ds - centers[0]
rays[1] = -centers[2]
rays[2] = -centers[1]
# rays = dot(R, Qs[intersecting].T)
sided = np.empty_like(sidep)
sided[0] = rays[0]
sided[1] = rays[1]
# hori = np.abs(sided[1] / sided[0]) < 1.5*eps
sel = np.where(forward & crossing)[0]
ps2d = ps2d[:, sel]
us2d = us2d[:, sel]
starts.append(ps2d)
deltas.append(us2d)
depths.append(ds[sel])
ray_frames.append(frames[intersecting[sel]])
sideps.append(sidep[:, sel])
sideds.append(1.5 * sided[:, sel])
planar_angles.append(np.arctan(centers[1, sel] / centers[2, sel]))
# ray_dot = rays[1, sel] / norm(rays[:,sel], axis=0)
# planar_angles.append( np.arctan(ray_dot) )
sel_inds.append(intersecting[sel])
return starts, deltas, depths, ray_frames, sideps, sideds, sel_inds, planar_angles
def render_animation(seq, sub, dopoly):
import pynutmeg
import time
import os
import RayCloud
fig = pynutmeg.figure('cube', 'figs/imshow.qml')
W, H = 1920, 1080
im = empty((H, W, 3), np.uint8)
cam = Geom.cam_params(29.97, W, H, 35) # Blender defaults
cam = cam.astype(np.float32)
print("Loading data")
cloud = RayCloud.load(os.path.join(seq, sub))
datapath = os.path.join(seq, 'occlusion_data.npz')
data = np.load(datapath)
mesh_path = os.path.join(seq, 'occlusion_mesh.npz')
occl_mesh = load_mesh(mesh_path)
# occl_mesh = Mesh()
# occl_mesh.verts = data['verts'].astype(np.float32)
edges = data['edges'].astype(np.uint32)
ply_path = os.path.join(seq, 'model.ply')
if os.path.exists(ply_path):
obj_mesh = from_ply(ply_path)
else:
obj_mesh = Mesh()
if dopoly:
poly_data = os.path.join(seq, sub, 'poly_data.npz')
polynpz = np.load(poly_data)
poly_mesh = Mesh()
poly_mesh.verts = polynpz['verts'].astype(np.float32)
poly_mesh.edges = polynpz['edges'].astype(np.uint32)
else:
poly_mesh = None
# pers_mesh_path = os.path.join(seq, 'persistent_mesh.npz')
# pers_mesh = load_mesh(pers_mesh_path)
campath = os.path.join(seq, 'animation.py')
Rs, ts = IO.import_blender_cam(campath)
# Grab frame info
inds = data['inds']
frames = cloud.frames[inds]
render_out = os.path.join(seq, 'animate')
Util.try_mkdir(render_out)
F = len(Rs)
N = len(frames)
print("Loaded", frames.max())
end_frame = 0
for f in range(0, F, 10):
print("Frame:", f)
# R = cloud.Rs[f]
# t = cloud.ts[f]
R = Rs[f].astype(np.float32)
t = ts[f].astype(np.float32)
while end_frame < N and frames[end_frame] < f:
end_frame += 1
occl_mesh.edges = edges[:end_frame]
im[:] = 255
if len(occl_mesh.edges) > 0:
t0 = time.time()
render_frame(im, obj_mesh, occl_mesh, None, R, t, cam)
print("Render time: {} ms".format( int((time.time() - t0)*1000) ))
# time.sleep(0.005)
fig.set('ax.im', binary=im)
out = os.path.join(render_out, 'frame_{:05d}.png'.format(f))
IO.imwrite(out, im)
def test_render_seq(seq, sub, dopoly):
import pynutmeg
import time
import os
import RayCloud
fig = pynutmeg.figure('cube', 'figs/imshow.qml')
W, H = 1920, 1080
im = empty((H, W, 3), np.uint8)
cam = Geom.cam_params(29.97, W, H, 35) # Blender defaults
cam = cam.astype(np.float32)
print("Loading data")
cloud = RayCloud.load(os.path.join(seq, sub))
mesh_path = os.path.join(seq, 'occlusion_mesh.npz')
occl_mesh = load_mesh(mesh_path)
ply_path = os.path.join(seq, 'model.ply')
if os.path.exists(ply_path):
obj_mesh = from_ply(ply_path)
else:
obj_mesh = Mesh()
if dopoly:
poly_data = os.path.join(seq, sub, 'poly_data.npz')
polynpz = np.load(poly_data)
poly_mesh = Mesh()
poly_mesh.verts = polynpz['verts'].astype(np.float32)
poly_mesh.edges = polynpz['edges'].astype(np.uint32)
else:
poly_mesh = None
pers_mesh_path = os.path.join(seq, 'persistent_mesh.npz')
pers_mesh = load_mesh(pers_mesh_path)
campath = os.path.join(seq, 'render.py')
Rs, ts = IO.import_blender_cam(campath)
render_out = os.path.join(seq, 'render')
Util.try_mkdir(render_out)
F = min(3, len(Rs))
print("Loaded")
for f in range(F):
print("Frame:", f)
# R = cloud.Rs[f]
# t = cloud.ts[f]
R = Rs[f].astype(np.float32)
t = ts[f].astype(np.float32)
im[:] = 255
t0 = time.time()
render_frame(im, obj_mesh, occl_mesh, None, R, t, cam)
print("dt:", (time.time() - t0)*1000)
# time.sleep(0.005)
fig.set('ax.im', binary=im)
out = os.path.join(render_out, 'frame_{}.png'.format(f))
IO.imwrite(out, im)
im[:] = 255
t0 = time.time()
render_frame(im, obj_mesh, pers_mesh, poly_mesh, R, t, cam, color=np.r_[255, 82, 82], poly_color=np.r_[0,0,0])
print("dt:", (time.time() - t0)*1000)
fig.set('ax.im', binary=im)
out = os.path.join(render_out, 'frame_pers_{}.png'.format(f))
IO.imwrite(out, im)
def visualize_intersections(seq, sub, skip=20, imtype='png'):
edge_dir = os.path.join(seq, 'edges', '*.' + imtype)
paths = glob.glob(edge_dir)
voxel_dir = os.path.join(seq, sub + '_voxel')
# ---------- Set up the figure -----------
fig = pynutmeg.figure('segments', 'figs/intersection.qml')
fig.set_gui('figs/intersection_gui.qml')
# Parameters
sld_frame = fig.parameter('frame')
sld_frameoffset = fig.parameter('frameoffset')
sld_segment = fig.parameter('segment')
sld_index = fig.parameter('index')
sld_anglesupport = fig.parameter('anglesupport')
sld_planarsupport = fig.parameter('planarsupport')
sld_support = fig.parameter('framesupport')
btn_cache = fig.parameter('cachebtn')
btn_export = fig.parameter('exportbtn')
btn_cache.wait_changed(5)
btn_export.wait_changed(1)
# ------------- Load in data -------------
print("Loading rays")
cloud = RayCloud.load(os.path.join(seq, sub))
F = int(cloud.frames.max())
sld_frame.set(maximumValue=F - 1, stepSize=skip)
sld_support.set(maximumValue=1000, stepSize=skip)
N = cloud.N
Cs, Qs, Ns = cloud.global_rays()
planes = empty((N, 4), Qs.dtype)
planes[:, :3] = Ns
planes[:, 3] = -(Cs * Ns).sum(axis=1)
plucker = Geom.to_plucker(Cs, Qs)
# Get a rough scale for closeness threshold based on
# size of camera center bounding box
print("Loading voxels")
raygrid = RayVoxel.load(voxel_dir)
# longest = (bbox_max - bbox_min).max()
# eps = longest * 1e-2
eps = 1 / cloud.cam[0]
print("Eps:", eps)
# Load the cam so we can offset the image properly
fx, fy, cx, cy = cloud.cam
# Make image show in homogenious coords
fig.set('ax.im',
xOffset=-(cx + 0.5) / fx,
yOffset=-(cy + 0.5) / fy,
xScale=1 / fx,
yScale=1 / fy)
fig.set('fit', minX=0.4, maxX=1, minY=-1, maxY=1)
# Make sure the figure's online
pynutmeg.wait_for_nutmeg()
pynutmeg.check_errors()
# Init state vars
frame = 0
label = 1
index = 0
frame_offset = 0
labels = empty(0, int)
max_label = 1
max_ind = 0
s, e = 0, 0
ray_ind = 0
frame_changed = True
cache = [[], [], []]
validcache = False
cachechanged = False
cluster_sel = empty(0, int)
hough = zeros((500, 1000), np.uint32)
while True:
# Check parameter update
if sld_frame.changed:
frame = max(0, sld_frame.read())
frame_changed = True
if sld_segment.changed:
label = sld_segment.read()
segment_changed = True
if sld_index.changed:
index = sld_index.read()
index_changed = True
# Apply updated values
if frame_changed:
E = IO.imread(paths[frame])
fig.set('ax.im', binary=255 - E)
s, e = cloud.frame_range[frame]
labels = cloud.labels_frame[s:e]
if len(labels) > 0:
max_label = labels.max()
sld_segment.set(maximumValue=int(max_label))
else:
sld_segment.set(maximumValue=0)
label = 0
segment_changed = True
frame_changed = False
if segment_changed:
segment_inds = s + np.where(labels == label)[0]
max_ind = max(0, len(segment_inds))
sld_index.set(maximumValue=max_ind)
if len(segment_inds) > 0:
P_seg = cloud.local_rays[np.r_[segment_inds,
segment_inds[-1] + 1]].T
fig.set('ax.P0', x=P_seg[0], y=P_seg[1])
else:
fig.set('ax.P0', x=[], y=[])
fig.set('ax.P1', x=[], y=[])
fig.set('ax.rays', x=[], y=[])
index = min(max_ind, index)
index_changed = True
segment_changed = False
validcache = False
# if sld_frameoffset.changed:
# print("Recalculation frame offset...")
# frame_offset = sld_frameoffset.read()
# # Slow, but don't care, atm...
# Cs, Qs, Ns = cloud.global_rays(frame_offset)
# planes = empty((N,4), Qs.dtype)
# planes[:,:3] = Ns
# planes[:,3] = -(Cs*Ns).sum(axis=1)
# plucker = Geom.to_plucker(Cs, Qs)
# print("Done")
# validcache = False
if index_changed and index >= 0 and index < len(segment_inds):
ray_ind = segment_inds[index]
P_seg = cloud.local_rays[ray_ind:ray_ind + 2]
P_ind = P_seg.mean(axis=0).reshape(-1, 1)
fig.set('ax.P1', x=P_ind[0], y=P_ind[1])
tx, ty, _ = P_seg[1] - P_seg[0]
mag = sqrt(tx * tx + ty * ty)
# nx, ny = Geom.project_normal(q, cloud.local_normals[ray_ind])
nx, ny = -ty / mag * 3e-2, tx / mag * 3e-2
L = empty((2, 2))
L[:, 0] = P_ind[:2, 0]
L[:, 1] = L[:, 0] + (nx, ny)
fig.set('ax.rays', x=L[0], y=L[1])
if (index_changed or sld_support.changed or sld_anglesupport.changed
or sld_planarsupport.changed or cachechanged) and validcache:
frame_support = max(sld_support.read(),
2 * sld_support.read() - frame)
angle_support = sld_anglesupport.read() / 10000
planarsupport = sld_planarsupport.read() / 1000
cachechanged = False
# print("Cache: {}, Index: {}".format(len(cache[0]), index))
if len(cache[0]) > 0 and 0 <= index < len(cache[0]):
frames = cache[3][index]
df = frames - frame
planar_angles = cache[7][index]
keep = np.where((np.abs(df) <= frame_support)
| (np.abs(planar_angles) <= planarsupport))[0]
P = cache[0][index][:, keep]
deltas = cache[1][index][:, keep]
depths = cache[2][index][keep]
# angles = np.arctan(deltas[0]/deltas[1])
angleres = np.deg2rad(5)
centers = cache[4][index][:, keep]
rays2d = cache[5][index][:, keep]
rays2d /= norm(rays2d, axis=0)
print("Sel:", len(cache[6][index]))
print("Clustering")
# cluster_inds, radius, depth, ray_angles, inlier_frac = ClassifyEdges.find_cluster_line3(
# deltas, rays2d, depths,
# angle_support, res=(angleres, 1e-2),
# thresh=(np.deg2rad(10), 4e-3))
result = ClassifyEdges.find_cluster_line4(
centers,
rays2d,
depth_thresh=angle_support,
percentile=0.15)
cluster_inds, radius, depth, dual_centers, dual_rays, inlier_frac = result
print(".. Done")
# np.savez('tmp/frame_cluster/ray_{}.npz'.format(ray_ind), frames=frames-frame, depths=depths, angles=angles, cluster_inds=cluster_inds)
# print("Saved cluster", ray_ind)
cluster_sel = cache[6][index][keep][cluster_inds]
# fig.set('fit.P0', x=depths, y=ray_angles)
# fig.set('fit.P1', x=depths[cluster_inds], y=ray_angles[cluster_inds])
# fig.set('fit.P2', x=depths[line_cluster], y=ray_angles[line_cluster])
x1, y1 = dual_centers - 10 * dual_rays
x2, y2 = dual_centers + 10 * dual_rays
fig.set('fit.rays', x=x1, y=y1, endX=x2, endY=y2)
fig.set('fit.rays2',
x=x1[cluster_inds],
y=y1[cluster_inds],
endX=x2[cluster_inds],
endY=y2[cluster_inds])
# fig.set('fit.rays3', x=x1[line_cluster], y=y1[line_cluster], endX=x2[line_cluster], endY=y2[line_cluster])
print(cluster_sel.shape)
# c_out = Cs[cluster_sel]
# q_out = (Cs[ray_ind] + Qs[ray_ind] * depths[cluster_inds].reshape(-1,1)) - c_out
# verts = np.vstack((c_out, c_out + 1.2*q_out))
# edges = empty((len(verts)//2, 2), np.uint32)
# edges[:] = np.r_[0:len(verts)].reshape(2,-1).T
# IO.save_point_cloud("tmp/cluster_rays.ply", verts, edges)
# fig.set('fit.P2', x=depths[init_inds], y=ray_space[init_inds])
if len(cluster_inds) >= 10:
# hist *= (0.04/hist.max())
# fig.set('tangent.l1', x=histx, y=hist)
# fig.set('fit.P0', x=angles, y=depths)
# fig.set('fit.P1', x=angles[cluster_inds], y=depths[cluster_inds])
P = P[:, cluster_inds]
deltas = deltas[:, cluster_inds]
x1, y1 = P
x2, y2 = P + deltas
fig.set('tangent.rays', x=x1, y=y1, endX=x2, endY=y2)
fig.set('tangent.l0', x=[0.0, 1.5], y=[0.0, 0.0])
fig.set('tangent.P0', x=depths, y=zeros(len(depths)))
# Determine tangent angle
intersect_angles = np.arctan(-deltas[0] / deltas[1])
# Zero is verticle
alpha = np.median(intersect_angles)
qa = np.r_[-np.sin(alpha), np.cos(alpha)]
a1 = np.r_[depth, 0] + 0.05 * qa
a2 = np.r_[depth, 0] - 0.05 * qa
fig.set('tangent.l1', x=[a1[0], a2[0]], y=[a1[1], a2[1]])
# Draw the other axis
Q2 = 2 * rays2d[:, cluster_inds]
P2a = centers[:, cluster_inds]
P2b = P2a + Q2
fig.set('normal.rays',
x=P2a[0],
y=P2a[1],
endX=P2b[0],
endY=P2b[1])
fig.set('normal.l0', x=[0.0, 1.5], y=[0.0, 0.0])
# fig.set('fit.P0', x=angles2, y=depths)
# fig.set('fit.P1', x=angles2[cluster_inds], y=depths[cluster_inds])
# np.savez('tmp/circle3.npz', P=P2a, Q=Q2, eps=eps)
# depth_std = np.std(depths[cluster_inds])
# nearby = np.where( np.abs(ray_angles[cluster_inds]) < 1.5*angle_support )[0]
# maxangle = np.percentile(np.abs(ray_angles[cluster_inds]), 95)
print("Radius:", radius, depth)
# frac = len(cluster_inds)/len(depths)
print("Frac:", len(cluster_inds), inlier_frac)
# print("Nearby:", len(nearby))
# if angle_range > np.deg2rad(20):
# print("Hard edge", len(cluster_inds))
if inlier_frac > 0.05 and len(cluster_inds) > 100:
if abs(radius) < 1e-2:
print("Hard edge", len(cluster_inds))
else:
print("Occlusion", len(cluster_inds))
else:
print("Cluster too small:", len(cluster_inds))
index_changed = False
if btn_cache.read_changed() or not validcache:
if len(segment_inds) > 0 and label != 0:
print("Caching segment... Total indices: {}".format(
len(segment_inds)),
flush=True)
cache = cache_segment(Cs,
Qs,
planes,
plucker,
cloud.frames,
cloud.labels_frame,
raygrid,
segment_inds,
eps=eps)
validcache = True
cachechanged = True
print("Done")
# TODO: Output cluster segments to .ply for blender visualization.......
if btn_export.read_changed() and validcache and len(cluster_sel) > 0:
export_triangles('tmp/cluster_tris.ply', Cs, Qs * 1.5, cluster_sel,
ray_ind)
c_out = Cs[cluster_sel]
q_out = (Cs[ray_ind] +
Qs[ray_ind] * depths[cluster_inds].reshape(-1, 1)) - c_out
verts = np.vstack((c_out, c_out + 1.5 * q_out))
edges = empty((len(verts) // 2, 2), np.uint32)
edges[:] = np.r_[0:len(verts)].reshape(2, -1).T
IO.save_point_cloud("tmp/cluster_rays.ply", verts, edges)
print("Saved .ply")
time.sleep(0.005)
def buffer_lines(pts, norms_proj, mesh, color, view, light, alpha=1.0, shade_method=PHONG, linewidth=1):
'''
Use the depth buffer, dbuf, to figure out whether the object needs to be "washed out"
when drawing.
Use the normal information to make clear the normal direction of the line.
'''
edges = mesh.edges
normals = mesh.normals
E = edges.shape[0]
# Output: [i, j, d, alpha, r, g, b]
size = 1024
P = 7
out = empty((size, P), np.float32)
N = 0
for e in range(E):
edge = edges[e]
x1, y1, z1 = pts[edge[0]]
x2, y2, z2 = pts[edge[1]]
view_local = (view - mesh.verts[edge[0]]).astype(np.float64)
# n1, n2 = normals[edge[0]], normals[edge[1]]
normal = normals[e].astype(np.float64)
normal_l = norms_proj[e].astype(np.float64)
normal_l /= Geom.norm3(normal_l)
# light_angle = abs(dot(normal, light))*1.5 + 0.3
# print(light_angle)
I, J, a = Drawing.line_aa(x1, y1, x2, y2, linewidth)
M = len(I)
depths = np.linspace(z1, z2, M)
# Resize if necessary
resize = False
while size < N + M:
size *= 2
resize = True
if resize:
newout = empty((size, P), np.float32)
newout[:N] = out[:N]
out = newout
if shade_method == PHONG:
out_color = phong_shade(color, light, normal, view_local)
elif shade_method == NORMAL:
out_color = normal_shade(light, normal, normal_l)
else:
out_color = color.astype(np.float64)
# Copy it in
for i, n in enumerate(range(N, N + M)):
out[n, 0] = I[i]
out[n, 1] = J[i]
out[n, 2] = depths[i]
out[n, 3] = a[i] * alpha
out[n, 4:] = out_color
N += M
return out[:N]
def detect(seq, sub, cloud, config, imtype='png'):
# ------------- Load in data for sequence ---------------
voxel_dir = os.path.join(seq, sub + '_voxel')
# Get a rough scale for closeness threshold based on
# size of camera center bounding box
print("Loading voxels")
raygrid = RayVoxel.load(voxel_dir)
# ------------ Precalc values from data -----------------
N = cloud.N
Cs, Qs, Ns = cloud.global_rays()
planes = empty((N, 4), Qs.dtype)
planes[:, :3] = Ns
planes[:, 3] = -(Cs * Ns).sum(axis=1)
pluckers = Geom.to_plucker(Cs, Qs)
# Load the cam so we can offset the image properly
fx, fy, cx, cy = cloud.cam
# Initialize all the output data
score = zeros(N, int)
total = zeros(N, int)
curv_mean = zeros(N)
m2 = zeros(N)
depths = zeros((N, 3), np.float32)
radii = np.full(N, np.inf, np.float32)
inlier_frac = zeros(N, np.float32)
edge_angles = np.full(N, 0, np.float32)
neighbors = np.full((N, 300), -1, np.int32)
status = zeros(N, np.int64)
# --------------------- The meat ------------------------
ray_inds = np.arange(0, N, dtype=np.int64)
args = (Cs, Qs, pluckers, planes, cloud.labels_frame, cloud.frames,
raygrid, config, score, total, curv_mean, m2, depths, radii,
inlier_frac, edge_angles, neighbors, status)
distribute(detect_loop,
ray_inds,
args,
watcher=loop_watcher,
w_args=(status, ),
n_workers=16)
# Estimate final variance of curvature
invalid = np.where(total == 0)[0]
total[invalid] = 1
curv_var = m2 / total
curv_var[invalid] = np.inf
# Count the neighbors
neighbor_count, reciprocated = count_neighbors(neighbors)
# --- Save ---
segout = os.path.join(seq, sub, 'segment_stats.npz')
np.savez(segout,
score=score,
total=total,
curv_mean=curv_mean,
curv_var=curv_var,
depths=depths,
radii=radii,
inlier_frac=inlier_frac,
edge_angles=edge_angles,
neighbors=neighbors,
neighbor_count=neighbor_count,
reciprocated=reciprocated)
def detect_loop(ray_inds, Cs, Qs, pluckers, planes, labels, frames, raygrid,
config, score, total, curv_mean, m2, depths_out, radii_out,
inlier_out, edge_angles_out, neighbors, status):
'''
score: Output, integer array classifying the type of each segment
'''
# Each valid segment is labelled with a non-zero label
# tim = Util.LoopTimer()
# loop_i = 0
for ray_ind in ray_inds:
# tim.loop_start(loop_i)
# loop_i += 1
if labels[ray_ind] == 0:
status[ray_ind] = -1
score[ray_ind] = 0
continue
# First find other segments that are nearby, and might overlap
# The RayVoxel.Grid class offers very fast lookups at the cost of memory
near_grid = raygrid.rays_near_ind(ray_ind)
# tim.add("RayGrid Search")
if len(near_grid) < 10:
status[ray_ind] = -1
continue
# Filter frame support
near = filter_frames(near_grid, frames, ray_ind, config.frame_support)
# close_frames = np.abs(frames[near_grid] - frames[ray_ind]) <= config.frame_support
# near = near_grid[close_frames]
# tim.add("Filter frames")
# Intersect the shortlisted segments with this segment
# ps, us describe the intersection locations of the other segments'
# start- and end-points.
ps1, us, intersecting = find_intersections(Cs, Qs, pluckers, ray_ind,
planes, near)
# print("Near:", len(near))
# tim.add("Find intersections")
if len(intersecting) < 10:
status[ray_ind] = -1
continue
ray = 0.5 * (Qs[ray_ind] + Qs[ray_ind + 1])
ray /= Geom.norm3(ray)
plane = planes[ray_ind]
normal = plane[:3]
tangent = Geom.cross3(ray, normal)
# Create rotation matrix to orthoganally project intersections
# into plane of this segment. This plane is tangent to the edge
R_tang = empty((2, 3), ps1.dtype)
R_tang[0] = ray
R_tang[1] = tangent
center = Cs[ray_ind]
ps2d = dot(R_tang, (ps1 - center).T)
us2d = dot(R_tang, us.T)
px, py = ps2d[0], ps2d[1]
ux, uy = us2d[0], us2d[1]
# Solve for the intersection with the center of the segment
# [x, 0] = p + t*u
uy[uy == 0] = 1e-10
ts = -py / uy
depths = px + ts * ux
# Project the intersecting segments into the normal plane.
# This plane's normal is roughly aligned with the edge direction.
R_norm = empty((3, 3), ps1.dtype)
R_norm[0] = ray
R_norm[1] = normal
R_norm[2] = tangent
centers = dot(R_norm, (Cs[intersecting] - center).T)
# plane_angles = np.arctan(centers[1]/centers[2])
# Check where start and end points straddles center ray of this segment
crossing = py * (py + uy) < 0
good_depths = (depths > 0.2) & (depths < 1e6)
# close_frames = np.abs(frames[intersecting] - frames[ray_ind]) <= config.frame_support
# in_normal_plane = np.abs(plane_angles) <= config.planar_support
sel = np.where(good_depths & crossing)[0]
# sel = np.where(good_depths & crossing & (close_frames | in_normal_plane))[0]
# tim.add("Intersect filter")
# print("Sel:", len(sel))
# We're looking for clusters of line segments in the tangent plane
# These indicate a high likelihood of a persistent edge
# frames = cloud.frames[ intersecting[sel] ]
# p = ps2d[:,sel]
u = us2d[:, sel]
d = depths[sel]
c_sel = centers[:, sel]
rays = empty((2, len(sel)), centers.dtype)
rays[0] = d - c_sel[0]
rays[1] = -c_sel[1]
rays /= CircleRays.norm_ax(rays, axis=0)
# tim.add("Prepare clustering")
dthresh = config.depth_thresh
# Cluster based on circle space
cluster_inds, radius, depth, dual_centers, dual_rays, inlier_frac = find_cluster_line4(
c_sel, rays, depth_thresh=dthresh, percentile=config.inlier_thresh)
# tim.add("Clustering")
cluster_full = intersecting[sel[cluster_inds]]
M = min(neighbors.shape[1], len(cluster_full))
if inlier_frac > 0.02:
if abs(radius) < config.min_radius:
# Hard edge
status[ray_ind] = 1
score[ray_ind] += len(cluster_full)
else:
status[ray_ind] = 2
score[ray_ind] += len(cluster_full)
# Want to estimate the depths of the rays on either side of the segment
u_cluster = u[:, cluster_inds]
intersect_angles = np.arctan(-u_cluster[0] / u_cluster[1])
# Zero is verticle
alpha = np.median(intersect_angles)
# Find angle between rays
theta = 0.5 * arccos(dot(Qs[ray_ind], Qs[ray_ind + 1]))
d1 = depth * sin(PI / 2 + alpha) / sin(PI / 2 - alpha - theta)
d2 = depth * sin(PI / 2 - alpha) / sin(PI / 2 + alpha - theta)
depths_out[ray_ind, 0] = d1
depths_out[ray_ind, 1] = d2
depths_out[ray_ind, 2] = depth
# depths_out[ray_ind] = depth
radii_out[ray_ind] = radius
inlier_out[ray_ind] = inlier_frac
edge_angles_out[ray_ind] = alpha
ray_c = rays[:, cluster_inds]
cluster_angles = np.abs(np.arctan(ray_c[1] / ray_c[0]))
closest = Util.argsort(cluster_angles)
neighbors[ray_ind, :M] = cluster_full[closest[:M]]
else:
status[ray_ind] = -1
score[ray_ind] = 0
labels[ray_ind] = 0
continue
