def render_lsd(seq):
plypath = os.path.join(seq, 'render_me.ply')
verts, edges = IO.read_ply(plypath)
lsd_mesh = Mesh()
lsd_mesh.verts = verts.astype(np.float32)
campath = os.path.join(seq, 'render.py')
Rs, ts = IO.import_blender_cam(campath)
W, H = 1920, 1080
im = empty((H, W, 3), np.uint8)
cam = Geom.cam_params(29.97, W, H, 35) # Blender defaults
cam = cam.astype(np.float32)
render_out = os.path.join(seq, 'render')
Util.try_mkdir(render_out)
im[:] = 255
pts = lsd_mesh.project_verts(Rs[0].astype(np.float32), ts[0].astype(np.float32), cam)
pt_buf = buffer_points(pts, np.r_[255, 82, 82], alpha=0.2, size=2)
render_buffer(im, pt_buf, bg=255)
out = os.path.join(render_out, 'frame_0.png')
IO.imwrite(out, im)
def show_edges(seq, cloud, thresh, min_score, imtype='png'):
edge_dir = os.path.join(seq, 'edges', '*.' + imtype)
paths = glob.glob(edge_dir)
segout = os.path.join(seq, 'segment_stats.npz')
# --- Load data ---
npz = np.load(segout)
score = npz['score']
# total = npz['total']
# curv_mean = npz['curv_mean']
# curv_var = npz['curv_var']
# --- Prepare data ---
F = len(paths)
fx, fy, cx, cy = cloud.cam
labels = cloud.labels_frame
focal = cloud.cam[:2]
center = cloud.cam[2:]
outfolder = os.path.join(seq, 'hard_edges')
Util.try_mkdir(outfolder)
# --- Do it ---
print("Writing out frames...")
j = 0
for f in range(F):
if f % 5:
continue
print("Frame {}".format(f))
path = paths[f]
im = IO.imread(path)
col = color.gray2rgb(255 - im)
s, e = cloud.frame_range[f]
for i in range(s, e):
if labels[i] == 0 or score[i] < min_score:
continue
line = empty(4)
line[:2] = cloud.local_rays[i, :2] * focal + center
line[2:] = cloud.local_rays[i + 1, :2] * focal + center
np.round(line, out=line)
I, J = Drawing.line(*(line.astype(int)))
Drawing.draw_points(col, I, J, color=(255, 0, 0))
Drawing.draw_points(col, I + 1, J, color=(255, 0, 0))
Drawing.draw_points(col, I, J + 1, color=(255, 0, 0))
# col[I, J] = 255, 0, 0
out = os.path.join(outfolder, '{:04d}.png'.format(f))
j += 1
IO.imwrite(out, col)
def render_buffer(im, buf, bg=1):
'''
Order by depth and mix alpha values at the same pixel
buf: (Nx7) array with each element as [i, j, d, alpha, r, g, b]
'''
inds = Util.lex_order(buf[:, :3].T, maxs=(720, 1280, 20))[::-1]
_render_buffer_jit(im, buf, inds, bg)
def write_clusters(outdir, lookup, P1, P2, indmap):
Util.try_mkdir(outdir)
verts_all = []
edges_all = []
E = 0
for key, cluster in lookup.items():
# path = os.path.join(outdir, 'set_{}.ply'.format(key))
inds = indmap[cluster]
verts = np.vstack((P1[inds], P2[inds]))
V = len(verts)
edges1 = np.r_[0:V].reshape(2, -1).T
edges2 = np.c_[np.r_[0:V - 1], np.r_[1:V]]
edges = np.vstack((edges1, edges2))
verts_all.append(verts)
edges_all.append(edges + E)
E += len(edges)
path = os.path.join(outdir, '_full.ply')
IO.save_point_cloud(path, np.vstack(verts_all), np.vstack(edges_all))
def find_cluster_line3(deltas, rays, depths, init_support, res, thresh):
'''
Given a set of line segments, defined by (x1, y1) = p and (x2, y2) = p + u,
locate clusters
The segments are intersection points of ray segments.
ray_angles are projected ray angles of camera ray angles.
Depths are distance the segments intersect the center of the ray of interest
'''
N = rays.shape[1]
if N == 0:
return empty(0, np.int64), 100.0, 0.0, empty(0, depths.dtype), 0.0
# Note: (ps, us, depths, frames) should be ordered by frames
ray_angles = empty(N, depths.dtype)
for i in range(N):
iqy = 1 / rays[1, i]
pos = (1 - rays[0, i]) * iqy
neg = -(1 + rays[0, i]) * iqy
if abs(pos) < abs(neg):
ray_angles[i] = pos
else:
ray_angles[i] = neg
dc = np.abs(ray_angles)
for k in range(2):
sel = np.where(dc <= init_support)[0]
if len(sel) == 0:
init_support += 0.02
continue
# Search for a "big enough" cluster of depths according to nearby cameras
hist, histx = Util.histogram(depths[sel],
bins=int(2 / res[1]),
rng=(0, 2))
i = hist.argmax()
if hist[i] > 10:
break
init_support += 0.02
else:
return empty(0, np.int64), 100.0, 0.0, ray_angles, 0.0
peak_depth = 0.5 * (histx[i] + histx[i + 1])
sel = np.where(dc < init_support * 5)[0]
depth, radius, inside, good = CircleRays.sequential_circle_space(
depths[sel], ray_angles[sel], peak_depth, init_support, thresh[1])
return sel[inside], radius, depth, ray_angles, len(inside) / len(sel)
def find_contours_rgb(rgb, D, d_sigma=2, d_scale=1):
from skimage.morphology import skeletonize
# from skimage.filters import gaussian
tim = Util.Timer()
# grey = rgb2grey(rgb)
grey = Image.rgb2grey(rgb)
tim.add("Gray")
edges_b = Image.find_edges(grey, sigma=2)
tim.add("Find edges")
edge_map = skeletonize(edges_b)
tim.add("Skel")
I, J = np.where(edge_map)
N = len(I)
P = empty((2, N), int)
P[0] = J # x
P[1] = I # y
# Find normals based on depth
# D_blur = gaussian(D, sigma=d_sigma)
D_blur = D
dgy, dgx = Image.gradient(D_blur)
grey_blur = Image.gaussian(grey, sigma=2)
gy, gx = Image.gradient(grey_blur)
tim.add("Grad")
normals = empty((2, N))
normals[0] = gx[I, J]
normals[1] = gy[I, J]
normals /= norm(normals, axis=0)
# Use depth map to make sure pointing correct way
dnormals = zeros((2, N))
dnormals[0] = dgx[I // d_scale, J // d_scale]
dnormals[1] = dgy[I // d_scale, J // d_scale]
mag = norm(dnormals, axis=0)
sel = mag > 0
dnormals[:, sel] /= mag[sel]
flip = (normals * dnormals).sum(axis=0) > 0
normals[:, flip] *= -1
tim.add("The rest")
# print(tim)
return P.T, normals.T, edge_map
def break_segments(pts, curvs, labels, max_breaks=10, max_curv=15e-3):
'''
Break segments (given by ordered labels) at points of high curvature.
Note: A bit of magic in the choice of max_curv. It will depend on the
support used when estimating the curvature. The discretization of pixesl
means a smaller support will have more noise.
In future, could use rate of change of normals along the line... Later.
'''
acurvs = np.abs(curvs)
curv_smooth = Image.gaussian1d(acurvs, 2.0)
peaks = Util.find_peaks(curv_smooth)
corners = peaks[np.where(acurvs[peaks] > max_curv)[0]]
newlabels = labels * max_breaks
# Relabel
N = len(newlabels)
for j in corners:
label = newlabels[j]
newlabel = label + 1
i = j
while i < N and newlabels[i] == label:
newlabels[i] = newlabel
i += 1
# Compress labels
newlabel = 1
i = 0
while i < N:
currentlabel = newlabels[i]
j = i
while j < N and newlabels[j] == currentlabel:
newlabels[j] = newlabel
j += 1
i = j
newlabel += 1
return newlabels
def rays_near_point(self, p):
# Convert to voxel coords:
coords = (self.scale * (p - self.offset)).astype(np.int64)
inds = empty(3 * 3 * 3, np.int64)
n = 0
# for di, dj, dk in ((0,0,0), (-1,0,0), (1,0,0), (0,-1,0), (0,1,0), (0,0,-1), (0,0,1)):
for di in range(-1, 2):
for dj in range(-1, 2):
for dk in range(-1, 2):
c = coords.copy()
c[0] += di
c[1] += dj
c[2] += dk
ind = self.to_linear_ind(c)
if ind >= 0 and ind < self.N:
inds[n] = ind
n += 1
# ind = self.to_linear_ind(coords)
# if ind < 0 or ind >= self.N:
# return empty(0, np.int64)
N = 0
count = empty(n, np.int64)
cumsum = empty(n + 1, np.int64)
for i in range(n):
ind = inds[i]
count[i] = self.vox2ray_count[ind]
cumsum[i] = N
N += count[i]
cumsum[-1] = N
# N = self.vox2ray_count[ind]
sel = empty(N, np.int64)
for i in range(n):
ind = inds[i]
sel[cumsum[i]:cumsum[i + 1]] = self.vox2ray_inds[ind, :count[i]]
# sel[:] = self.vox2ray_inds[ind, :N]
return Util.unique(sel)
def load(self, data):
total = 0
jump = 0
down = 0
fail = 0
for i in data:
total += 1
if self.found(i):
jump += 1
if self._cfgs["vmode"] == "full":
self.output(
str(total) + ".跳过：" + urllib.parse.urlparse(i).path)
#continue
else:
if self._cfgs["vmode"] == "full":
self.output(
str(total) + ".[正在下载]:" +
urllib.parse.urlparse(i).path)
try:
ua = Util.randUA()
req = urllib.request.Request(i, headers={"User-Agent": ua})
response = urllib.request.urlopen(req)
except urllib.error.URLError as reason:
fail += 1
self.output("URLError:" + str(reason))
except urllib.error.HTTPError as reason:
fail += 1
self.output("HTTPError:" + str(reason))
else:
down += 1
img = response.read()
with open(self.fullpath(i), "wb") as f:
f.write(img)
self.output("实时总数%s，跳过%s，下载%s，失败%s" % (total, jump, down, fail))
return True
#!/usr/bin/python3
#-*-coding:utf-8-*-
from tools import Util
'''
入口功能test
'''
#u = Util.Util()
#ua = u.getUA()
#print(ua)
print(Util.randUA())
if __name__ == "__main__":
print("welcome")
def render_animation(seq, sub, dopoly):
import pynutmeg
import time
import os
import RayCloud
fig = pynutmeg.figure('cube', 'figs/imshow.qml')
W, H = 1920, 1080
im = empty((H, W, 3), np.uint8)
cam = Geom.cam_params(29.97, W, H, 35) # Blender defaults
cam = cam.astype(np.float32)
print("Loading data")
cloud = RayCloud.load(os.path.join(seq, sub))
datapath = os.path.join(seq, 'occlusion_data.npz')
data = np.load(datapath)
mesh_path = os.path.join(seq, 'occlusion_mesh.npz')
occl_mesh = load_mesh(mesh_path)
# occl_mesh = Mesh()
# occl_mesh.verts = data['verts'].astype(np.float32)
edges = data['edges'].astype(np.uint32)
ply_path = os.path.join(seq, 'model.ply')
if os.path.exists(ply_path):
obj_mesh = from_ply(ply_path)
else:
obj_mesh = Mesh()
if dopoly:
poly_data = os.path.join(seq, sub, 'poly_data.npz')
polynpz = np.load(poly_data)
poly_mesh = Mesh()
poly_mesh.verts = polynpz['verts'].astype(np.float32)
poly_mesh.edges = polynpz['edges'].astype(np.uint32)
else:
poly_mesh = None
# pers_mesh_path = os.path.join(seq, 'persistent_mesh.npz')
# pers_mesh = load_mesh(pers_mesh_path)
campath = os.path.join(seq, 'animation.py')
Rs, ts = IO.import_blender_cam(campath)
# Grab frame info
inds = data['inds']
frames = cloud.frames[inds]
render_out = os.path.join(seq, 'animate')
Util.try_mkdir(render_out)
F = len(Rs)
N = len(frames)
print("Loaded", frames.max())
end_frame = 0
for f in range(0, F, 10):
print("Frame:", f)
# R = cloud.Rs[f]
# t = cloud.ts[f]
R = Rs[f].astype(np.float32)
t = ts[f].astype(np.float32)
while end_frame < N and frames[end_frame] < f:
end_frame += 1
occl_mesh.edges = edges[:end_frame]
im[:] = 255
if len(occl_mesh.edges) > 0:
t0 = time.time()
render_frame(im, obj_mesh, occl_mesh, None, R, t, cam)
print("Render time: {} ms".format( int((time.time() - t0)*1000) ))
# time.sleep(0.005)
fig.set('ax.im', binary=im)
out = os.path.join(render_out, 'frame_{:05d}.png'.format(f))
IO.imwrite(out, im)
def test_render_seq(seq, sub, dopoly):
import pynutmeg
import time
import os
import RayCloud
fig = pynutmeg.figure('cube', 'figs/imshow.qml')
W, H = 1920, 1080
im = empty((H, W, 3), np.uint8)
cam = Geom.cam_params(29.97, W, H, 35) # Blender defaults
cam = cam.astype(np.float32)
print("Loading data")
cloud = RayCloud.load(os.path.join(seq, sub))
mesh_path = os.path.join(seq, 'occlusion_mesh.npz')
occl_mesh = load_mesh(mesh_path)
ply_path = os.path.join(seq, 'model.ply')
if os.path.exists(ply_path):
obj_mesh = from_ply(ply_path)
else:
obj_mesh = Mesh()
if dopoly:
poly_data = os.path.join(seq, sub, 'poly_data.npz')
polynpz = np.load(poly_data)
poly_mesh = Mesh()
poly_mesh.verts = polynpz['verts'].astype(np.float32)
poly_mesh.edges = polynpz['edges'].astype(np.uint32)
else:
poly_mesh = None
pers_mesh_path = os.path.join(seq, 'persistent_mesh.npz')
pers_mesh = load_mesh(pers_mesh_path)
campath = os.path.join(seq, 'render.py')
Rs, ts = IO.import_blender_cam(campath)
render_out = os.path.join(seq, 'render')
Util.try_mkdir(render_out)
F = min(3, len(Rs))
print("Loaded")
for f in range(F):
print("Frame:", f)
# R = cloud.Rs[f]
# t = cloud.ts[f]
R = Rs[f].astype(np.float32)
t = ts[f].astype(np.float32)
im[:] = 255
t0 = time.time()
render_frame(im, obj_mesh, occl_mesh, None, R, t, cam)
print("dt:", (time.time() - t0)*1000)
# time.sleep(0.005)
fig.set('ax.im', binary=im)
out = os.path.join(render_out, 'frame_{}.png'.format(f))
IO.imwrite(out, im)
im[:] = 255
t0 = time.time()
render_frame(im, obj_mesh, pers_mesh, poly_mesh, R, t, cam, color=np.r_[255, 82, 82], poly_color=np.r_[0,0,0])
print("dt:", (time.time() - t0)*1000)
fig.set('ax.im', binary=im)
out = os.path.join(render_out, 'frame_pers_{}.png'.format(f))
IO.imwrite(out, im)
def detect_loop(ray_inds, Cs, Qs, pluckers, planes, labels, frames, raygrid,
config, score, total, curv_mean, m2, depths_out, radii_out,
inlier_out, edge_angles_out, neighbors, status):
'''
score: Output, integer array classifying the type of each segment
'''
# Each valid segment is labelled with a non-zero label
# tim = Util.LoopTimer()
# loop_i = 0
for ray_ind in ray_inds:
# tim.loop_start(loop_i)
# loop_i += 1
if labels[ray_ind] == 0:
status[ray_ind] = -1
score[ray_ind] = 0
continue
# First find other segments that are nearby, and might overlap
# The RayVoxel.Grid class offers very fast lookups at the cost of memory
near_grid = raygrid.rays_near_ind(ray_ind)
# tim.add("RayGrid Search")
if len(near_grid) < 10:
status[ray_ind] = -1
continue
# Filter frame support
near = filter_frames(near_grid, frames, ray_ind, config.frame_support)
# close_frames = np.abs(frames[near_grid] - frames[ray_ind]) <= config.frame_support
# near = near_grid[close_frames]
# tim.add("Filter frames")
# Intersect the shortlisted segments with this segment
# ps, us describe the intersection locations of the other segments'
# start- and end-points.
ps1, us, intersecting = find_intersections(Cs, Qs, pluckers, ray_ind,
planes, near)
# print("Near:", len(near))
# tim.add("Find intersections")
if len(intersecting) < 10:
status[ray_ind] = -1
continue
ray = 0.5 * (Qs[ray_ind] + Qs[ray_ind + 1])
ray /= Geom.norm3(ray)
plane = planes[ray_ind]
normal = plane[:3]
tangent = Geom.cross3(ray, normal)
# Create rotation matrix to orthoganally project intersections
# into plane of this segment. This plane is tangent to the edge
R_tang = empty((2, 3), ps1.dtype)
R_tang[0] = ray
R_tang[1] = tangent
center = Cs[ray_ind]
ps2d = dot(R_tang, (ps1 - center).T)
us2d = dot(R_tang, us.T)
px, py = ps2d[0], ps2d[1]
ux, uy = us2d[0], us2d[1]
# Solve for the intersection with the center of the segment
# [x, 0] = p + t*u
uy[uy == 0] = 1e-10
ts = -py / uy
depths = px + ts * ux
# Project the intersecting segments into the normal plane.
# This plane's normal is roughly aligned with the edge direction.
R_norm = empty((3, 3), ps1.dtype)
R_norm[0] = ray
R_norm[1] = normal
R_norm[2] = tangent
centers = dot(R_norm, (Cs[intersecting] - center).T)
# plane_angles = np.arctan(centers[1]/centers[2])
# Check where start and end points straddles center ray of this segment
crossing = py * (py + uy) < 0
good_depths = (depths > 0.2) & (depths < 1e6)
# close_frames = np.abs(frames[intersecting] - frames[ray_ind]) <= config.frame_support
# in_normal_plane = np.abs(plane_angles) <= config.planar_support
sel = np.where(good_depths & crossing)[0]
# sel = np.where(good_depths & crossing & (close_frames | in_normal_plane))[0]
# tim.add("Intersect filter")
# print("Sel:", len(sel))
# We're looking for clusters of line segments in the tangent plane
# These indicate a high likelihood of a persistent edge
# frames = cloud.frames[ intersecting[sel] ]
# p = ps2d[:,sel]
u = us2d[:, sel]
d = depths[sel]
c_sel = centers[:, sel]
rays = empty((2, len(sel)), centers.dtype)
rays[0] = d - c_sel[0]
rays[1] = -c_sel[1]
rays /= CircleRays.norm_ax(rays, axis=0)
# tim.add("Prepare clustering")
dthresh = config.depth_thresh
# Cluster based on circle space
cluster_inds, radius, depth, dual_centers, dual_rays, inlier_frac = find_cluster_line4(
c_sel, rays, depth_thresh=dthresh, percentile=config.inlier_thresh)
# tim.add("Clustering")
cluster_full = intersecting[sel[cluster_inds]]
M = min(neighbors.shape[1], len(cluster_full))
if inlier_frac > 0.02:
if abs(radius) < config.min_radius:
# Hard edge
status[ray_ind] = 1
score[ray_ind] += len(cluster_full)
else:
status[ray_ind] = 2
score[ray_ind] += len(cluster_full)
# Want to estimate the depths of the rays on either side of the segment
u_cluster = u[:, cluster_inds]
intersect_angles = np.arctan(-u_cluster[0] / u_cluster[1])
# Zero is verticle
alpha = np.median(intersect_angles)
# Find angle between rays
theta = 0.5 * arccos(dot(Qs[ray_ind], Qs[ray_ind + 1]))
d1 = depth * sin(PI / 2 + alpha) / sin(PI / 2 - alpha - theta)
d2 = depth * sin(PI / 2 - alpha) / sin(PI / 2 + alpha - theta)
depths_out[ray_ind, 0] = d1
depths_out[ray_ind, 1] = d2
depths_out[ray_ind, 2] = depth
# depths_out[ray_ind] = depth
radii_out[ray_ind] = radius
inlier_out[ray_ind] = inlier_frac
edge_angles_out[ray_ind] = alpha
ray_c = rays[:, cluster_inds]
cluster_angles = np.abs(np.arctan(ray_c[1] / ray_c[0]))
closest = Util.argsort(cluster_angles)
neighbors[ray_ind, :M] = cluster_full[closest[:M]]
else:
status[ray_ind] = -1
score[ray_ind] = 0
labels[ray_ind] = 0
continue