def SecondFundamentalForm(v, f):
from chumpy import hstack, vstack
from chumpy.linalg import Pinv
nbrs = MatVecMult(FirstEdgesMtx(v, f, want_big=True), v.ravel()).reshape(
(-1, 3))
b0 = VertNormals(f=f, v=v)
b1 = NormalizedNx3(CrossProduct(b0, nbrs - v)).reshape((-1, 3))
b2 = NormalizedNx3(CrossProduct(b0, b1)).reshape((-1, 3))
cnct = get_vert_connectivity(np.asarray(v), f)
ffs = []
for i in range(v.size / 3):
nbrs = v[np.nonzero(np.asarray(cnct[i].todense()).ravel())[0]] - row(
v[i])
us = nbrs.dot(b2[i])
vs = nbrs.dot(b1[i])
hs = nbrs.dot(b0[i])
coeffs = Pinv(
hstack((col((us * .5)**2), col(us * vs), col(
(vs * .5)**2)))).dot(hs)
ffs.append(row(coeffs))
# if i == 3586:
# import pdb; pdb.set_trace()
ffs = vstack(ffs)
return ffs
def get_earthmesh(trans, rotation):
from .serialization import load_mesh
from copy import deepcopy
if not hasattr(get_earthmesh, 'm'):
def wg(url):
dest = join('/tmp', split(url)[1])
if not exists(dest):
wget(url, dest)
wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.obj')
wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.mtl')
wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.jpg')
fname = join('/tmp', 'nasa_earth.obj')
mesh = load_mesh(fname)
mesh.v = np.asarray(mesh.v, order='C')
mesh.vc = mesh.v*0 + 1
mesh.v -= row(np.mean(mesh.v, axis=0))
mesh.v /= np.max(mesh.v)
mesh.v *= 2.0
get_earthmesh.mesh = mesh
mesh = deepcopy(get_earthmesh.mesh)
mesh.v = mesh.v.dot(cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0])
mesh.v = mesh.v + row(np.asarray(trans))
return mesh
def get_earthmesh(trans, rotation):
from opendr.serialization import load_mesh
from copy import deepcopy
if not hasattr(get_earthmesh, 'm'):
def wg(url):
dest = join('/tmp', split(url)[1])
if not exists(dest):
wget(url, dest)
wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.obj')
wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.mtl')
wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.jpg')
fname = join('/tmp', 'nasa_earth.obj')
mesh = load_mesh(fname)
mesh.v = np.asarray(mesh.v, order='C')
mesh.vc = mesh.v * 0 + 1
mesh.v -= row(np.mean(mesh.v, axis=0))
mesh.v /= np.max(mesh.v)
mesh.v *= 2.0
get_earthmesh.mesh = mesh
mesh = deepcopy(get_earthmesh.mesh)
mesh.v = mesh.v.dot(
cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0])
mesh.v = mesh.v + row(np.asarray(trans))
return mesh
def compute_dr_wrt(self, obj):
if obj is self.a:
if not hasattr(self, '_dr_cached'):
IS = np.arange(len(self.idxs))
JS = self.idxs.ravel()
ij = np.vstack((row(IS), row(JS)))
data = np.ones(len(self.idxs))
self._dr_cached = sp.csc_matrix(
(data, ij), shape=(len(self.idxs), np.prod(self.a.shape)))
return self._dr_cached
def read_and_process_mesh(fname, trans, rotation):
mesh = load_mesh(fname)
mesh.v = np.asarray(mesh.v, order='C')
mesh.vc = np.ones_like(mesh.v)
mesh.v -= row(np.mean(mesh.v, axis=0))
mesh.v /= np.max(mesh.v)
mesh.v *= 2.0
mesh.v = mesh.v.dot(
cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0])
if hasattr(mesh, "vn"):
mesh.vn = mesh.vn.dot(
cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0])
mesh.v = mesh.v + row(np.asarray(trans))
return mesh
def compute_dr_wrt(self, wrt):
if wrt is not self.v:
return None
cplus = self.cplus
cminus = self.cminus
vplus = self.f[:,cplus]
vminus = self.f[:,cminus]
vplus3 = row(np.hstack([col(vplus*3), col(vplus*3+1), col(vplus*3+2)]))
vminus3 = row(np.hstack([col(vminus*3), col(vminus*3+1), col(vminus*3+2)]))
IS = row(np.arange(0,vplus3.size))
ones = np.ones(vplus3.size)
shape = (self.f.size, self.v.r.size)
return sp.csc_matrix((ones, np.vstack([IS, vplus3])), shape=shape) - sp.csc_matrix((ones, np.vstack([IS, vminus3])), shape=shape)
def compute_dr_wrt(self, wrt):
result = super(TexturedRenderer, self).compute_dr_wrt(wrt)
if wrt is self.vc:
cim = self.draw_color_image(with_vertex_colors=False).ravel()
cim = sp.spdiags(row(cim), [0], cim.size, cim.size)
result = cim.dot(result)
elif wrt is self.texture_image:
IS = np.nonzero(self.visibility_image.ravel() != 4294967295)[0]
JS = self.texcoord_image_quantized.ravel()[IS]
clr_im = self.draw_color_image(with_vertex_colors=True,
with_texture_on=False)
if False:
cv2.imshow('clr_im', clr_im)
cv2.imshow('texmap', self.texture_image.r)
cv2.waitKey(1)
r = clr_im[:, :, 0].ravel()[IS]
g = clr_im[:, :, 1].ravel()[IS]
b = clr_im[:, :, 2].ravel()[IS]
data = np.concatenate((r, g, b))
IS = np.concatenate((IS * 3, IS * 3 + 1, IS * 3 + 2))
JS = np.concatenate((JS * 3, JS * 3 + 1, JS * 3 + 2))
return sp.csc_matrix((data, (IS, JS)),
shape=(self.r.size, wrt.r.size))
return result
def compute_dr_wrt(self, wrt):
result = super(TexturedRenderer, self).compute_dr_wrt(wrt)
if wrt is self.vc:
cim = self.draw_color_image(with_vertex_colors=False).ravel()
cim = sp.spdiags(row(cim), [0], cim.size, cim.size)
result = cim.dot(result)
elif wrt is self.texture_image:
IS = np.nonzero(self.visibility_image.ravel() != 4294967295)[0]
JS = self.texcoord_image_quantized.ravel()[IS]
clr_im = self.draw_color_image(with_vertex_colors=True, with_texture_on=False)
if False:
cv2.imshow('clr_im', clr_im)
cv2.imshow('texmap', self.texture_image.r)
cv2.waitKey(1)
r = clr_im[:,:,0].ravel()[IS]
g = clr_im[:,:,1].ravel()[IS]
b = clr_im[:,:,2].ravel()[IS]
data = np.concatenate((r,g,b))
IS = np.concatenate((IS*3, IS*3+1, IS*3+2))
JS = np.concatenate((JS*3, JS*3+1, JS*3+2))
return sp.csc_matrix((data, (IS, JS)), shape=(self.r.size, wrt.r.size))
return result
def draw_boundary_images(glf, glb, v, f, vpe, fpe, camera):
"""Assumes camera is set up correctly, and that glf has any texmapping on necessary."""
glf.Clear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glb.Clear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
# Figure out which edges are on pairs of differently visible triangles
from opendr.geometry import TriNormals
tn = TriNormals(v, f).r.reshape((-1,3))
campos = -cv2.Rodrigues(camera.rt.r)[0].T.dot(camera.t.r)
rays_to_verts = v.reshape((-1,3)) - row(campos)
rays_to_faces = rays_to_verts[f[:,0]] + rays_to_verts[f[:,1]] + rays_to_verts[f[:,2]]
dps = np.sum(rays_to_faces * tn, axis=1)
dps = dps[fpe[:,0]] * dps[fpe[:,1]]
silhouette_edges = np.asarray(np.nonzero(dps<=0)[0], np.uint32)
non_silhouette_edges = np.nonzero(dps>0)[0]
lines_e = vpe[silhouette_edges]
lines_v = v
visibility = draw_edge_visibility(glb, lines_v, lines_e, f, hidden_wireframe=True)
shape = visibility.shape
visibility = visibility.ravel()
visible = np.nonzero(visibility.ravel() != 4294967295)[0]
visibility[visible] = silhouette_edges[visibility[visible]]
result = visibility.reshape(shape)
return result
def unproject_points(self, uvd, camera_space=False):
cam = ProjectPoints3D(
**{k: getattr(self, k)
for k in self.dterms if hasattr(self, k)})
try:
xy_undistorted_camspace = cv2.undistortPoints(
np.asarray(uvd[:, :2].reshape((1, -1, 2)).copy()),
np.asarray(cam.camera_mtx), cam.k.r)
xyz_camera_space = np.hstack(
(xy_undistorted_camspace.squeeze(), col(uvd[:, 2])))
xyz_camera_space[:, :2] *= col(
xyz_camera_space[:, 2]) # scale x,y by z
if camera_space:
return xyz_camera_space
other_answer = xyz_camera_space - row(cam.view_mtx[:,
3]) # translate
result = other_answer.dot(cam.view_mtx[:, :3]) # rotate
except: # slow way, probably not so good. But doesn't require cv2.undistortPoints.
cam.v = np.ones_like(uvd)
ch.minimize(cam - uvd,
x0=[cam.v],
method='dogleg',
options={'disp': 0})
result = cam.v.r
return result
def compute_dr_wrt(self, wrt):
if wrt is not self.camera and wrt is not self.v:
return None
visibility = self.visibility_image
visible = np.nonzero(visibility.ravel() != 4294967295)[0]
barycentric = self.barycentric_image
if wrt is self.camera:
shape = visibility.shape
depth = self.depth_image
if self.overdraw:
result1 = common.dImage_wrt_2dVerts_bnd(
depth, visible, visibility, barycentric,
self.frustum['width'], self.frustum['height'],
self.v.r.size / 3, self.f,
self.boundaryid_image != 4294967295)
else:
result1 = common.dImage_wrt_2dVerts(depth, visible, visibility,
barycentric,
self.frustum['width'],
self.frustum['height'],
self.v.r.size / 3, self.f)
# result1 = common.dImage_wrt_2dVerts(depth, visible, visibility, barycentric, self.frustum['width'], self.frustum['height'], self.v.r.size/3, self.f)
return result1
elif wrt is self.v:
IS = np.tile(col(visible), (1, 9)).ravel()
JS = col(self.f[visibility.ravel()[visible]].ravel())
JS = np.hstack((JS * 3, JS * 3 + 1, JS * 3 + 2)).ravel()
# FIXME: there should be a faster way to get the camera axis.
# But it should be carefully tested with distortion present!
pts = np.array([[self.camera.c.r[0], self.camera.c.r[1], 2],
[self.camera.c.r[0], self.camera.c.r[1], 1]])
pts = self.camera.unproject_points(pts)
cam_axis = pts[0, :] - pts[1, :]
if True: # use barycentric coordinates (correct way)
w = visibility.shape[1]
pxs = np.asarray(visible % w, np.int32)
pys = np.asarray(np.floor(np.floor(visible) / w), np.int32)
bc0 = col(barycentric[pys, pxs, 0])
bc1 = col(barycentric[pys, pxs, 1])
bc2 = col(barycentric[pys, pxs, 2])
bc = np.hstack(
(bc0, bc0, bc0, bc1, bc1, bc1, bc2, bc2, bc2)).ravel()
else: # each vert contributes equally (an approximation)
bc = 1. / 3.
data = np.tile(row(cam_axis), (IS.size / 3, 1)).ravel() * bc
result2 = sp.csc_matrix(
(data, (IS, JS)),
shape=(self.frustum['height'] * self.frustum['width'],
self.v.r.size))
return result2
def draw_boundary_images(glf, glb, v, f, vpe, fpe, camera):
"""Assumes camera is set up correctly, and that glf has any texmapping on necessary."""
glf.Clear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glb.Clear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
# Figure out which edges are on pairs of differently visible triangles
from opendr.geometry import TriNormals
tn = TriNormals(v, f).r.reshape((-1, 3))
campos = -cv2.Rodrigues(camera.rt.r)[0].T.dot(camera.t.r)
rays_to_verts = v.reshape((-1, 3)) - row(campos)
rays_to_faces = rays_to_verts[f[:, 0]] + rays_to_verts[
f[:, 1]] + rays_to_verts[f[:, 2]]
dps = np.sum(rays_to_faces * tn, axis=1)
dps = dps[fpe[:, 0]] * dps[fpe[:, 1]]
silhouette_edges = np.asarray(np.nonzero(dps <= 0)[0], np.uint32)
non_silhouette_edges = np.nonzero(dps > 0)[0]
lines_e = vpe[silhouette_edges]
lines_v = v
visibility = draw_edge_visibility(glb,
lines_v,
lines_e,
f,
hidden_wireframe=True)
shape = visibility.shape
visibility = visibility.ravel()
visible = np.nonzero(visibility.ravel() != 4294967295)[0]
visibility[visible] = silhouette_edges[visibility[visible]]
result = visibility.reshape(shape)
return result
def get_earthmesh(trans, rotation):
fname = "example_data/nasa_earth.obj"
mesh = load_mesh(fname)
if not hasattr(get_earthmesh, "mesh"):
mesh.v = np.asarray(mesh.v, order='C')
mesh.vc = mesh.v * 0 + 1
mesh.v -= row(np.mean(mesh.v, axis=0))
mesh.v /= np.max(mesh.v)
mesh.v *= 2.0
get_earthmesh.mesh = mesh
mesh = deepcopy(get_earthmesh.mesh)
mesh.v = mesh.v.dot(
cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0])
mesh.v = mesh.v + row(np.asarray(trans))
return mesh
def compute_d1(self):
# To stay consistent with numpy, we must upgrade 1D arrays to 2D
ar = row(self.a.r) if len(self.a.r.shape)<2 else self.a.r.reshape((-1, self.a.r.shape[-1]))
br = col(self.b.r) if len(self.b.r.shape)<2 else self.b.r.reshape((self.b.r.shape[0], -1))
if ar.ndim <= 2:
return sp.kron(sp.eye(ar.shape[0], ar.shape[0]),br.T)
else:
raise NotImplementedError
def get_vert_connectivity(mesh_v, mesh_f):
"""Returns a sparse matrix (of size #verts x #verts) where each nonzero
element indicates a neighborhood relation. For example, if there is a
nonzero element in position (15,12), that means vertex 15 is connected
by an edge to vertex 12."""
vpv = sp.csc_matrix((len(mesh_v),len(mesh_v)))
# for each column in the faces...
for i in range(3):
IS = mesh_f[:,i]
JS = mesh_f[:,(i+1)%3]
data = np.ones(len(IS))
ij = np.vstack((row(IS.flatten()), row(JS.flatten())))
mtx = sp.csc_matrix((data, ij), shape=vpv.shape)
vpv = vpv + mtx + mtx.T
return vpv
def get_vert_connectivity(mesh_v, mesh_f):
"""Returns a sparse matrix (of size #verts x #verts) where each nonzero
element indicates a neighborhood relation. For example, if there is a
nonzero element in position (15,12), that means vertex 15 is connected
by an edge to vertex 12."""
vpv = sp.csc_matrix((len(mesh_v),len(mesh_v)))
# for each column in the faces...
for i in range(3):
IS = mesh_f[:,i]
JS = mesh_f[:,(i+1)%3]
data = np.ones(len(IS))
ij = np.vstack((row(IS.flatten()), row(JS.flatten())))
mtx = sp.csc_matrix((data, ij), shape=vpv.shape)
vpv = vpv + mtx + mtx.T
return vpv
def flow_to(self, v_next, cam_next):
from chumpy.ch import MatVecMult
color_image = self.r
visibility = self.visibility_image
pxpos = np.zeros_like(self.color_image)
pxpos[:, :, 0] = np.tile(row(np.arange(self.color_image.shape[1])),
(self.color_image.shape[0], 1))
pxpos[:, :, 2] = np.tile(col(np.arange(self.color_image.shape[0])),
(1, self.color_image.shape[1]))
visible = np.nonzero(visibility.ravel() != 4294967295)[0]
num_visible = len(visible)
barycentric = self.barycentric_image
# map 3d to 3d
JS = col(self.f[visibility.ravel()[visible]]).ravel()
IS = np.tile(col(np.arange(JS.size / 3)), (1, 3)).ravel()
data = barycentric.reshape((-1, 3))[visible].ravel()
# replicate to xyz
IS = np.concatenate((IS * 3, IS * 3 + 1, IS * 3 + 2))
JS = np.concatenate((JS * 3, JS * 3 + 1, JS * 3 + 2))
data = np.concatenate((data, data, data))
verts_to_visible = sp.csc_matrix((data, (IS, JS)),
shape=(np.max(IS) + 1, self.v.r.size))
v_old = self.camera.v
cam_old = self.camera
if cam_next is None:
cam_next = self.camera
self.camera.v = MatVecMult(verts_to_visible, self.v.r)
r1 = self.camera.r.copy()
self.camera = cam_next
self.camera.v = MatVecMult(verts_to_visible, v_next)
r2 = self.camera.r.copy()
n_channels = self.camera.shape[1]
flow = r2 - r1
flow_im = np.zeros(
(self.frustum['height'], self.frustum['width'], n_channels)).reshape(
(-1, n_channels))
flow_im[visible] = flow
flow_im = flow_im.reshape(
(self.frustum['height'], self.frustum['width'], n_channels))
self.camera = cam_old
self.camera.v = v_old
return flow_im
def get_body_mesh(obj_path, trans, rotation):
from opendr.serialization import load_mesh
from copy import deepcopy
fname = obj_path
mesh = load_mesh(fname)
mesh.v = np.asarray(mesh.v, order='C')
mesh.vc = mesh.v * 0 + 1
mesh.v -= row(np.mean(mesh.v, axis=0))
mesh.v /= np.max(mesh.v)
mesh.v *= 2.0
get_body_mesh.mesh = mesh
mesh = deepcopy(get_body_mesh.mesh)
mesh.v = mesh.v.dot(
cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0])
mesh.v = mesh.v + row(np.asarray(trans))
return mesh
def compute_d1(self):
# To stay consistent with numpy, we must upgrade 1D arrays to 2D
mtx1r = row(self.mtx1.r) if len(self.mtx1.r.shape)<2 else self.mtx1.r
mtx2r = col(self.mtx2.r) if len(self.mtx2.r.shape)<2 else self.mtx2.r
if mtx1r.ndim <= 2:
return sp.kron(sp.eye(mtx1r.shape[0], mtx1r.shape[0]),mtx2r.T)
else:
mtx2f = mtx2r.reshape((-1, mtx2r.shape[-2], mtx2r.shape[-1]))
mtx2f = np.rollaxis(mtx2f, -1, -2) #transpose basically
result = sp.block_diag([np.kron(np.eye(mtx1r.shape[-2], mtx1r.shape[-2]),m2) for m2 in mtx2f])
assert(result.shape[0] == self.r.size)
return result
def flow_to(self, v_next, cam_next):
from chumpy.ch import MatVecMult
color_image = self.r
visibility = self.visibility_image
pxpos = np.zeros_like(self.color_image)
pxpos[:,:,0] = np.tile(row(np.arange(self.color_image.shape[1])), (self.color_image.shape[0], 1))
pxpos[:,:,2] = np.tile(col(np.arange(self.color_image.shape[0])), (1, self.color_image.shape[1]))
visible = np.nonzero(visibility.ravel() != 4294967295)[0]
num_visible = len(visible)
barycentric = self.barycentric_image
# map 3d to 3d
JS = col(self.f[visibility.ravel()[visible]]).ravel()
IS = np.tile(col(np.arange(JS.size/3)), (1, 3)).ravel()
data = barycentric.reshape((-1,3))[visible].ravel()
# replicate to xyz
IS = np.concatenate((IS*3, IS*3+1, IS*3+2))
JS = np.concatenate((JS*3, JS*3+1, JS*3+2))
data = np.concatenate((data, data, data))
verts_to_visible = sp.csc_matrix((data, (IS, JS)), shape=(np.max(IS)+1, self.v.r.size))
v_old = self.camera.v
cam_old = self.camera
if cam_next is None:
cam_next = self.camera
self.camera.v = MatVecMult(verts_to_visible, self.v.r)
r1 = self.camera.r.copy()
self.camera = cam_next
self.camera.v = MatVecMult(verts_to_visible, v_next)
r2 = self.camera.r.copy()
n_channels = self.camera.shape[1]
flow = r2 - r1
flow_im = np.zeros((self.frustum['height'], self.frustum['width'], n_channels)).reshape((-1,n_channels))
flow_im[visible] = flow
flow_im = flow_im.reshape((self.frustum['height'], self.frustum['width'], n_channels))
self.camera = cam_old
self.camera.v = v_old
return flow_im
def compute_vpe_boundary_idxs(v, f, camera, fpe):
# Figure out which edges are on pairs of differently visible triangles
from geometry import TriNormals
tn = TriNormals(v, f).r.reshape((-1,3))
#ray = cv2.Rodrigues(camera.rt.r)[0].T[:,2]
campos = -cv2.Rodrigues(camera.rt.r)[0].T.dot(camera.t.r)
rays_to_verts = v.reshape((-1,3)) - row(campos)
rays_to_faces = rays_to_verts[f[:,0]] + rays_to_verts[f[:,1]] + rays_to_verts[f[:,2]]
faces_invisible = np.sum(rays_to_faces * tn, axis=1)
dps = faces_invisible[fpe[:,0]] * faces_invisible[fpe[:,1]]
silhouette_edges = np.asarray(np.nonzero(dps<=0)[0], np.uint32)
return silhouette_edges, faces_invisible < 0
def dImage_wrt_2dVerts(observed, visible, visibility, barycentric, image_width, image_height, num_verts, f):
"""Construct a sparse jacobian that relates 2D projected vertex positions
(in the columns) to pixel values (in the rows). This can be done
in two steps."""
num_verts = np.int32(num_verts)
n_channels = np.atleast_3d(observed).shape[2]
shape = visibility.shape
# Step 1: get the structure ready, ie the IS and the JS
IS = np.tile(col(visible), (1, 2*f.shape[1])).ravel()
JS = f[visibility.ravel()[visible]].reshape((-1,1))
JS = np.hstack((JS*2, JS*2+1)).ravel()
pxs = np.asarray(visible % shape[1], np.int32)
pys = np.asarray(np.floor(np.floor(visible) / shape[1]), np.int32)
if n_channels > 1:
IS = np.concatenate([IS*n_channels+i for i in range(n_channels)])
JS = np.concatenate([JS for i in range(n_channels)])
# Step 2: get the data ready, ie the actual values of the derivatives
ksize=1
sobel_normalizer = cv2.Sobel(np.asarray(np.tile(row(np.arange(10)), (10, 1)), np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize)[5,5]
xdiff = -cv2.Sobel(observed, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / sobel_normalizer
ydiff = cv2.Sobel(observed, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / sobel_normalizer
xdiff = np.atleast_3d(xdiff)
ydiff = np.atleast_3d(ydiff)
datas = []
# The data is weighted according to barycentric coordinates
bc0 = barycentric[pys, pxs, 0].reshape((-1,1))
bc1 = barycentric[pys, pxs, 1].reshape((-1,1))
bc2 = barycentric[pys, pxs, 2].reshape((-1,1))
for k in range(n_channels):
dxs = xdiff[pys, pxs, k]
dys = ydiff[pys, pxs, k]
if f.shape[1] == 3:
datas.append(np.hstack((dxs.reshape((-1,1))*bc0,dys.reshape((-1,1))*bc0,dxs.reshape((-1,1))*bc1,dys.reshape((-1,1))*bc1,dxs.reshape((-1,1))*bc2,dys.reshape((-1,1))*bc2)).ravel())
else:
datas.append(np.hstack((dxs.reshape((-1,1))*bc0,dys.reshape((-1,1))*bc0,dxs.reshape((-1,1))*bc1,dys.reshape((-1,1))*bc1)).ravel())
data = np.concatenate(datas)
ij = np.vstack((IS.ravel(), JS.ravel())).astype(np.int32)
result = sp.csc_matrix((data, ij), shape=(image_width*image_height*n_channels, num_verts*2))
return result
def test_derivatives(self):
import chumpy as ch
from chumpy.utils import row
import numpy as np
from .renderer import DepthRenderer
rn = DepthRenderer()
# Assign attributes to renderer
from .util_tests import get_earthmesh
m = get_earthmesh(trans=ch.array([0, 0, 4]), rotation=ch.zeros(3))
w, h = (320, 240)
from .camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v,
rt=ch.zeros(3),
t=ch.zeros(3),
f=ch.array([w, w]) / 2.,
c=ch.array([w, h]) / 2.,
k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3))
if visualize:
import matplotlib.pyplot as plt
plt.figure()
for which in range(3):
r1 = rn.r
adder = np.zeros(3)
adder[which] = .01
change = rn.v.r * 0 + row(adder)
dr_pred = rn.dr_wrt(rn.v).dot(change.ravel()).reshape(rn.shape)
rn.v = rn.v.r + change
r2 = rn.r
dr_emp = r2 - r1
# print np.mean(np.abs(dr_pred-dr_emp))
self.assertLess(np.mean(np.abs(dr_pred - dr_emp)), .031)
if visualize:
plt.subplot(2, 3, which + 1)
plt.imshow(dr_pred)
plt.clim(-.01, .01)
plt.title('emp')
plt.subplot(2, 3, which + 4)
plt.imshow(dr_emp)
plt.clim(-.01, .01)
plt.title('pred')
def compute_vpe_boundary_idxs(v, f, camera, fpe):
# Figure out which edges are on pairs of differently visible triangles
from geometry import TriNormals
tn = TriNormals(v, f).r.reshape((-1,3))
#ray = cv2.Rodrigues(camera.rt.r)[0].T[:,2]
campos = -cv2.Rodrigues(camera.rt.r)[0].T.dot(camera.t.r)
rays_to_verts = v.reshape((-1,3)) - row(campos)
rays_to_faces = rays_to_verts[f[:,0]] + rays_to_verts[f[:,1]] + rays_to_verts[f[:,2]]
faces_invisible = np.sum(rays_to_faces * tn, axis=1)
dps = faces_invisible[fpe[:,0]] * faces_invisible[fpe[:,1]]
silhouette_edges = np.asarray(np.nonzero(dps<=0)[0], np.uint32)
return silhouette_edges, faces_invisible < 0
def SecondFundamentalForm(v, f):
from chumpy import hstack, vstack
from chumpy.linalg import Pinv
nbrs = MatVecMult(FirstEdgesMtx(v, f, want_big=True), v.ravel()).reshape((-1,3))
b0 = NormalizedNx3(VertNormalsScaled(f=f, v=v)).reshape((-1,3))
b1 = NormalizedNx3(CrossProduct(b0, nbrs-v)).reshape((-1,3))
b2 = NormalizedNx3(CrossProduct(b0, b1)).reshape((-1,3))
cnct = get_vert_connectivity(v.r, f)
ffs = []
for i in range(v.size/3):
nbrs = v[np.nonzero(np.asarray(cnct[i].todense()).ravel())[0]] - row(v[i])
us = nbrs.dot(b2[i])
vs = nbrs.dot(b1[i])
hs = nbrs.dot(b0[i])
coeffs = Pinv(hstack((col((us*.5)**2), col(us*vs), col((vs*.5)**2)))).dot(hs)
ffs.append(row(coeffs))
# if i == 3586:
# import pdb; pdb.set_trace()
ffs = vstack(ffs)
return ffs
def compute_dr_wrt(self, wrt):
if wrt is not self.camera and wrt is not self.v:
return None
visibility = self.visibility_image
visible = np.nonzero(visibility.ravel() != 4294967295)[0]
barycentric = self.barycentric_image
if wrt is self.camera:
shape = visibility.shape
depth = self.depth_image
if self.overdraw:
result1 = common.dImage_wrt_2dVerts_bnd(depth, visible, visibility, barycentric, self.frustum['width'], self.frustum['height'], self.v.r.size/3, self.f, self.boundaryid_image != 4294967295)
else:
result1 = common.dImage_wrt_2dVerts(depth, visible, visibility, barycentric, self.frustum['width'], self.frustum['height'], self.v.r.size/3, self.f)
# result1 = common.dImage_wrt_2dVerts(depth, visible, visibility, barycentric, self.frustum['width'], self.frustum['height'], self.v.r.size/3, self.f)
return result1
elif wrt is self.v:
IS = np.tile(col(visible), (1, 9)).ravel()
JS = col(self.f[visibility.ravel()[visible]].ravel())
JS = np.hstack((JS*3, JS*3+1, JS*3+2)).ravel()
# FIXME: there should be a faster way to get the camera axis.
# But it should be carefully tested with distortion present!
pts = np.array([
[self.camera.c.r[0], self.camera.c.r[1], 2],
[self.camera.c.r[0], self.camera.c.r[1], 1]
])
pts = self.camera.unproject_points(pts)
cam_axis = pts[0,:] - pts[1,:]
if True: # use barycentric coordinates (correct way)
w = visibility.shape[1]
pxs = np.asarray(visible % w, np.int32)
pys = np.asarray(np.floor(np.floor(visible) / w), np.int32)
bc0 = col(barycentric[pys, pxs, 0])
bc1 = col(barycentric[pys, pxs, 1])
bc2 = col(barycentric[pys, pxs, 2])
bc = np.hstack((bc0,bc0,bc0,bc1,bc1,bc1,bc2,bc2,bc2)).ravel()
else: # each vert contributes equally (an approximation)
bc = 1. / 3.
data = np.tile(row(cam_axis), (IS.size/3,1)).ravel() * bc
result2 = sp.csc_matrix((data, (IS, JS)), shape=(self.frustum['height']*self.frustum['width'], self.v.r.size))
return result2
def dImage_wrt_2dVerts(observed, visible, visibility, barycentric, image_width, image_height, num_verts, f):
"""Construct a sparse jacobian that relates 2D projected vertex positions
(in the columns) to pixel values (in the rows). This can be done
in two steps."""
n_channels = np.atleast_3d(observed).shape[2]
shape = visibility.shape
# Step 1: get the structure ready, ie the IS and the JS
IS = np.tile(col(visible), (1, 2*f.shape[1])).ravel()
JS = col(f[visibility.ravel()[visible]].ravel())
JS = np.hstack((JS*2, JS*2+1)).ravel()
pxs = np.asarray(visible % shape[1], np.int32)
pys = np.asarray(np.floor(np.floor(visible) / shape[1]), np.int32)
if n_channels > 1:
IS = np.concatenate([IS*n_channels+i for i in range(n_channels)])
JS = np.concatenate([JS for i in range(n_channels)])
# Step 2: get the data ready, ie the actual values of the derivatives
ksize=1
sobel_normalizer = cv2.Sobel(np.asarray(np.tile(row(np.arange(10)), (10, 1)), np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize)[5,5]
xdiff = -cv2.Sobel(observed, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / sobel_normalizer
ydiff = -cv2.Sobel(observed, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / sobel_normalizer
xdiff = np.atleast_3d(xdiff)
ydiff = np.atleast_3d(ydiff)
datas = []
# The data is weighted according to barycentric coordinates
bc0 = col(barycentric[pys, pxs, 0])
bc1 = col(barycentric[pys, pxs, 1])
bc2 = col(barycentric[pys, pxs, 2])
for k in range(n_channels):
dxs = xdiff[pys, pxs, k]
dys = ydiff[pys, pxs, k]
if f.shape[1] == 3:
datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1,col(dxs)*bc2,col(dys)*bc2)).ravel())
else:
datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1)).ravel())
data = np.concatenate(datas)
ij = np.vstack((IS.ravel(), JS.ravel()))
result = sp.csc_matrix((data, ij), shape=(image_width*image_height*n_channels, num_verts*2))
return result
def compute_d2(self):
# To stay consistent with numpy, we must upgrade 1D arrays to 2D
mtx1r = row(self.mtx1.r) if len(self.mtx1.r.shape)<2 else self.mtx1.r
mtx2r = col(self.mtx2.r) if len(self.mtx2.r.shape)<2 else self.mtx2.r
if mtx2r.ndim <= 1:
return self.mtx1r
elif mtx2r.ndim <= 2:
return sp.kron(mtx1r, sp.eye(mtx2r.shape[1],mtx2r.shape[1]))
else:
mtx1f = mtx1r.reshape((-1, mtx1r.shape[-2], mtx1r.shape[-1]))
result = sp.block_diag([np.kron(m1, np.eye(mtx2r.shape[-1],mtx2r.shape[-1])) for m1 in mtx1f])
assert(result.shape[0] == self.r.size)
return result
def unproject_points(self, uvd, camera_space=False):
cam = ProjectPoints3D(**{k: getattr(self, k) for k in self.dterms if hasattr(self, k)})
try:
xy_undistorted_camspace = cv2.undistortPoints(np.asarray(uvd[:,:2].reshape((1,-1,2)).copy()), np.asarray(cam.camera_mtx), cam.k.r)
xyz_camera_space = np.hstack((xy_undistorted_camspace.squeeze(), col(uvd[:,2])))
xyz_camera_space[:,:2] *= col(xyz_camera_space[:,2]) # scale x,y by z
if camera_space:
return xyz_camera_space
other_answer = xyz_camera_space - row(cam.view_mtx[:,3]) # translate
result = other_answer.dot(cam.view_mtx[:,:3]) # rotate
except: # slow way, probably not so good. But doesn't require cv2.undistortPoints.
cam.v = np.ones_like(uvd)
ch.minimize(cam - uvd, x0=[cam.v], method='dogleg', options={'disp': 0})
result = cam.v.r
return result
def compute_dr_wrt(self, wrt):
if wrt is not self.a:
return None
if self.axis is not None:
raise NotImplementedError
IS = np.tile(row(np.arange(self.a.size)), (self.a.size, 1))
JS = IS.T
IS = IS.ravel()
JS = JS.ravel()
which = IS >= JS
IS = IS[which]
JS = JS[which]
data = np.ones_like(IS)
result = sp.csc_matrix((data, (IS, JS)), shape=(self.a.size, self.a.size))
return result
def test_derivatives(self):
import chumpy as ch
from chumpy.utils import row
import numpy as np
from renderer import DepthRenderer
rn = DepthRenderer()
# Assign attributes to renderer
from util_tests import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)
from camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3))
if visualize:
import matplotlib.pyplot as plt
plt.figure()
for which in range(3):
r1 = rn.r
adder = np.zeros(3)
adder[which] = .01
change = rn.v.r * 0 + row(adder)
dr_pred = rn.dr_wrt(rn.v).dot(change.ravel()).reshape(rn.shape)
rn.v = rn.v.r + change
r2 = rn.r
dr_emp = r2 - r1
# print np.mean(np.abs(dr_pred-dr_emp))
self.assertLess(np.mean(np.abs(dr_pred-dr_emp)), .031)
if visualize:
plt.subplot(2,3,which+1)
plt.imshow(dr_pred)
plt.clim(-.01,.01)
plt.title('emp')
plt.subplot(2,3,which+4)
plt.imshow(dr_emp)
plt.clim(-.01,.01)
plt.title('pred')
def on_changed(self, which):
if 'vn' in which:
vn = self.vn.r.reshape((-1,3))
# Conversion from normals to spherical harmonics found in...
# http://en.wikipedia.org/wiki/Spherical_coordinate_system#Cartesian_coordinates
self.theta = np.arccos(vn[:,2])
self.phi = np.arctan2(vn[:,1], vn[:,0])
# vnswapped = np.swapaxes(vn, 0,1)
self.sh_coeffs = real_sh_coeff(vn)
self.num_verts = self.sh_coeffs.shape[0]
if 'light_color' in which or self.mtx.shape[1] != self.num_verts:
nc = self.num_channels
IS = np.arange(self.num_verts*nc)
JS = np.repeat(np.arange(self.num_verts), nc)
data = (row(self.light_color)*np.ones((self.num_verts, nc))).ravel()
self.mtx = sp.csc_matrix((data, (IS,JS)), shape=(self.num_verts*nc, self.num_verts))
def on_changed(self, which):
if 'vn' in which:
vn = self.vn.r.reshape((-1,3))
# Conversion from normals to spherical harmonics found in...
# http://en.wikipedia.org/wiki/Spherical_coordinate_system#Cartesian_coordinates
self.theta = np.arccos(vn[:,2])
self.phi = np.arctan2(vn[:,1], vn[:,0])
self.sh_coeffs = real_sh_coeff(vn)
self.num_verts = self.sh_coeffs.shape[0]
if 'light_color' in which or self.mtx.shape[1] != self.num_verts:
nc = self.num_channels
IS = np.arange(self.num_verts*nc)
JS = np.repeat(np.arange(self.num_verts), nc)
data = (row(self.light_color)*np.ones((self.num_verts, nc))).ravel()
self.mtx = sp.csc_matrix((data, (IS,JS)), shape=(self.num_verts*nc, self.num_verts))
def compute_dr_wrt(self, wrt):
if wrt is not self.x:
return
if self.axis == None:
return row(np.ones((1, len(self.x.r))))/len(self.x.r)
else:
uid = tuple(list(self.x.shape) + [self.axis])
if uid not in self.dr_cache:
idxs_presum = np.arange(self.x.size).reshape(self.x.shape)
idxs_presum = np.rollaxis(idxs_presum, self.axis, 0)
idxs_postsum = np.arange(self.r.size).reshape(self.r.shape)
tp = np.ones(idxs_presum.ndim, dtype=np.uint32)
tp[0] = idxs_presum.shape[0]
idxs_postsum = np.tile(idxs_postsum, tp)
data = np.ones(idxs_postsum.size) / self.x.shape[self.axis]
result = sp.csc_matrix((data, (idxs_postsum.ravel(), idxs_presum.ravel())), (self.r.size, wrt.size))
self.dr_cache[uid] = result
return self.dr_cache[uid]
def compute_dr_wrt(self, wrt):
if wrt is self.x:
if visualize:
import matplotlib.pyplot as plt
residuals = np.sum(self.r**2)
print '------> RESIDUALS %.2e' % (residuals, )
print '------> CURRENT GUESS %s' % (str(self.x.r), )
plt.figure(123)
if not hasattr(self, 'vs'):
self.vs = []
self.xs = []
self.ys = []
self.vs.append(residuals)
self.xs.append(self.x.r[0])
self.ys.append(self.x.r[1])
plt.clf()
plt.subplot(1, 2, 1)
plt.plot(self.vs)
plt.subplot(1, 2, 2)
plt.plot(self.xs, self.ys)
plt.draw()
return row(rosen_der(self.x.r))
def test_spherical_harmonics(self):
global visualize
if visualize:
plt.ion()
# Get mesh
v, f = get_sphere_mesh()
from geometry import VertNormals
vn = VertNormals(v=v, f=f)
#vn = Ch(mesh.estimate_vertex_normals())
# Get camera
cam, frustum = getcam()
# Get renderer
from renderer import ColoredRenderer
cam.v = v
cr = ColoredRenderer(f=f, camera=cam, frustum=frustum, v=v)
sh_red = SphericalHarmonics(vn=vn, light_color=np.array([1,0,0]))
sh_green = SphericalHarmonics(vn=vn, light_color=np.array([0,1,0]))
cr.vc = sh_red + sh_green
ims_baseline = []
for comp_idx, subplot_idx in enumerate([3,7,8,9,11,12,13,14,15]):
sh_comps = np.zeros(9)
sh_comps[comp_idx] = 1
sh_red.components = Ch(sh_comps)
sh_green.components = Ch(-sh_comps)
newim = cr.r.reshape((frustum['height'], frustum['width'], 3))
ims_baseline.append(newim)
if visualize:
plt.subplot(3,5,subplot_idx)
plt.imshow(newim)
plt.axis('off')
offset = row(.4 * (np.random.rand(3)-.5))
#offset = row(np.array([1.,1.,1.]))*.05
vn_shifted = (vn.r + offset)
vn_shifted = vn_shifted / col(np.sqrt(np.sum(vn_shifted**2, axis=1)))
vn_shifted = vn_shifted.ravel()
vn_shifted[vn_shifted>1.] = 1
vn_shifted[vn_shifted<-1.] = -1
vn_shifted = Ch(vn_shifted)
cr.replace(sh_red.vn, vn_shifted)
if True:
for comp_idx in range(9):
if visualize:
plt.figure(comp_idx+2)
sh_comps = np.zeros(9)
sh_comps[comp_idx] = 1
sh_red.components = Ch(sh_comps)
sh_green.components = Ch(-sh_comps)
pred = cr.dr_wrt(vn_shifted).dot(col(vn_shifted.r.reshape(vn.r.shape) - vn.r)).reshape((frustum['height'], frustum['width'], 3))
if visualize:
plt.subplot(1,2,1)
plt.imshow(pred)
plt.title('pred (comp %d)' % (comp_idx,))
plt.subplot(1,2,2)
newim = cr.r.reshape((frustum['height'], frustum['width'], 3))
emp = newim - ims_baseline[comp_idx]
if visualize:
plt.imshow(emp)
plt.title('empirical (comp %d)' % (comp_idx,))
pred_flat = pred.ravel()
emp_flat = emp.ravel()
nnz = np.unique(np.concatenate((np.nonzero(pred_flat)[0], np.nonzero(emp_flat)[0])))
if comp_idx != 0:
med_diff = np.median(np.abs(pred_flat[nnz]-emp_flat[nnz]))
med_obs = np.median(np.abs(emp_flat[nnz]))
if comp_idx == 4 or comp_idx == 8:
self.assertTrue(med_diff / med_obs < .6)
else:
self.assertTrue(med_diff / med_obs < .3)
if visualize:
plt.axis('off')
def loop_subdivider(mesh_v, mesh_f):
IS = []
JS = []
data = []
vc = get_vert_connectivity(mesh_v, mesh_f)
ve = get_vertices_per_edge(mesh_v, mesh_f)
vo = get_vert_opposites_per_edge(mesh_v, mesh_f)
if True:
# New values for each vertex
for idx in xrange(len(mesh_v)):
# find neighboring vertices
nbrs = np.nonzero(vc[:,idx])[0]
nn = len(nbrs)
if nn < 3:
wt = 0.
elif nn == 3:
wt = 3./16.
elif nn > 3:
wt = 3. / (8. * nn)
else:
raise Exception('nn should be 3 or more')
if wt > 0.:
for nbr in nbrs:
IS.append(idx)
JS.append(nbr)
data.append(wt)
JS.append(idx)
IS.append(idx)
data.append(1. - (wt * nn))
start = len(mesh_v)
edge_to_midpoint = {}
if True:
# New values for each edge:
# new edge verts depend on the verts they span
for idx, vs in enumerate(ve):
vsl = list(vs)
vsl.sort()
IS.append(start + idx)
IS.append(start + idx)
JS.append(vsl[0])
JS.append(vsl[1])
data.append(3./8)
data.append(3./8)
opposites = vo[(vsl[0], vsl[1])]
for opp in opposites:
IS.append(start + idx)
JS.append(opp)
data.append(2./8./len(opposites))
edge_to_midpoint[(vsl[0], vsl[1])] = start + idx
edge_to_midpoint[(vsl[1], vsl[0])] = start + idx
f = []
for f_i, old_f in enumerate(mesh_f):
ff = np.concatenate((old_f, old_f))
for i in range(3):
v0 = edge_to_midpoint[(ff[i], ff[i+1])]
v1 = ff[i+1]
v2 = edge_to_midpoint[(ff[i+1], ff[i+2])]
f.append(row(np.array([v0,v1,v2])))
v0 = edge_to_midpoint[(ff[0], ff[1])]
v1 = edge_to_midpoint[(ff[1], ff[2])]
v2 = edge_to_midpoint[(ff[2], ff[3])]
f.append(row(np.array([v0,v1,v2])))
f = np.vstack(f)
IS = np.array(IS, dtype=np.uint32)
JS = np.array(JS, dtype=np.uint32)
if True: # for x,y,z coords
IS = np.concatenate((IS*3, IS*3+1, IS*3+2))
JS = np.concatenate((JS*3, JS*3+1, JS*3+2))
data = np.concatenate ((data,data,data))
ij = np.vstack((IS.flatten(), JS.flatten()))
mtx = sp.csc_matrix((data, ij))
return mtx, f
def loop_subdivider(mesh_v, mesh_f):
IS = []
JS = []
data = []
vc = get_vert_connectivity(mesh_v, mesh_f)
ve = get_vertices_per_edge(mesh_v, mesh_f)
vo = get_vert_opposites_per_edge(mesh_v, mesh_f)
if True:
# New values for each vertex
for idx in xrange(len(mesh_v)):
# find neighboring vertices
nbrs = np.nonzero(vc[:,idx])[0]
nn = len(nbrs)
if nn < 3:
wt = 0.
elif nn == 3:
wt = 3./16.
elif nn > 3:
wt = 3. / (8. * nn)
else:
raise Exception('nn should be 3 or more')
if wt > 0.:
for nbr in nbrs:
IS.append(idx)
JS.append(nbr)
data.append(wt)
JS.append(idx)
IS.append(idx)
data.append(1. - (wt * nn))
start = len(mesh_v)
edge_to_midpoint = {}
if True:
# New values for each edge:
# new edge verts depend on the verts they span
for idx, vs in enumerate(ve):
vsl = list(vs)
vsl.sort()
IS.append(start + idx)
IS.append(start + idx)
JS.append(vsl[0])
JS.append(vsl[1])
data.append(3./8)
data.append(3./8)
opposites = vo[(vsl[0], vsl[1])]
for opp in opposites:
IS.append(start + idx)
JS.append(opp)
data.append(2./8./len(opposites))
edge_to_midpoint[(vsl[0], vsl[1])] = start + idx
edge_to_midpoint[(vsl[1], vsl[0])] = start + idx
f = []
for f_i, old_f in enumerate(mesh_f):
ff = np.concatenate((old_f, old_f))
for i in range(3):
v0 = edge_to_midpoint[(ff[i], ff[i+1])]
v1 = ff[i+1]
v2 = edge_to_midpoint[(ff[i+1], ff[i+2])]
f.append(row(np.array([v0,v1,v2])))
v0 = edge_to_midpoint[(ff[0], ff[1])]
v1 = edge_to_midpoint[(ff[1], ff[2])]
v2 = edge_to_midpoint[(ff[2], ff[3])]
f.append(row(np.array([v0,v1,v2])))
f = np.vstack(f)
IS = np.array(IS, dtype=np.uint32)
JS = np.array(JS, dtype=np.uint32)
if True: # for x,y,z coords
IS = np.concatenate((IS*3, IS*3+1, IS*3+2))
JS = np.concatenate((JS*3, JS*3+1, JS*3+2))
data = np.concatenate ((data,data,data))
ij = np.vstack((IS.flatten(), JS.flatten()))
mtx = sp.csc_matrix((data, ij))
return mtx, f
def dImage_wrt_2dVerts_bnd(observed, visible, visibility, barycentric, image_width, image_height, num_verts, f, bnd_bool):
"""Construct a sparse jacobian that relates 2D projected vertex positions
(in the columns) to pixel values (in the rows). This can be done
in two steps."""
n_channels = np.atleast_3d(observed).shape[2]
shape = visibility.shape
# Step 1: get the structure ready, ie the IS and the JS
IS = np.tile(col(visible), (1, 2*f.shape[1])).ravel()
JS = col(f[visibility.ravel()[visible]].ravel())
JS = np.hstack((JS*2, JS*2+1)).ravel()
pxs = np.asarray(visible % shape[1], np.int32)
pys = np.asarray(np.floor(np.floor(visible) / shape[1]), np.int32)
if n_channels > 1:
IS = np.concatenate([IS*n_channels+i for i in range(n_channels)])
JS = np.concatenate([JS for i in range(n_channels)])
# Step 2: get the data ready, ie the actual values of the derivatives
ksize = 1
bndf = bnd_bool.astype(np.float64)
nbndf = np.logical_not(bnd_bool).astype(np.float64)
sobel_normalizer = cv2.Sobel(np.asarray(np.tile(row(np.arange(10)), (10, 1)), np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize)[5,5]
bnd_nan = bndf.reshape((observed.shape[0], observed.shape[1], -1)).copy()
bnd_nan.ravel()[bnd_nan.ravel()>0] = np.nan
bnd_nan += 1
obs_nonbnd = np.atleast_3d(observed) * bnd_nan
ydiffnb, xdiffnb = nangradients(obs_nonbnd)
observed = np.atleast_3d(observed)
if observed.shape[2] > 1:
ydiffbnd, xdiffbnd, _ = np.gradient(observed)
else:
ydiffbnd, xdiffbnd = np.gradient(observed.squeeze())
ydiffbnd = np.atleast_3d(ydiffbnd)
xdiffbnd = np.atleast_3d(xdiffbnd)
# This corrects for a bias imposed boundary differences begin spread over two pixels
# (by np.gradients or similar) but only counted once (since OpenGL's line
# drawing spans 1 pixel)
xdiffbnd *= 2.0
ydiffbnd *= 2.0
xdiffnb = -xdiffnb
ydiffnb = -ydiffnb
xdiffbnd = -xdiffbnd
ydiffbnd = -ydiffbnd
# ydiffnb *= 0
# xdiffnb *= 0
if False:
import matplotlib.pyplot as plt
plt.figure()
plt.subplot(121)
plt.imshow(xdiffnb)
plt.title('xdiffnb')
plt.subplot(122)
plt.imshow(xdiffbnd)
plt.title('xdiffbnd')
import pdb; pdb.set_trace()
idxs = np.isnan(xdiffnb.ravel())
xdiffnb.ravel()[idxs] = xdiffbnd.ravel()[idxs]
idxs = np.isnan(ydiffnb.ravel())
ydiffnb.ravel()[idxs] = ydiffbnd.ravel()[idxs]
if True: # should be right thing
xdiff = xdiffnb
ydiff = ydiffnb
else: #should be old way
xdiff = xdiffbnd
ydiff = ydiffbnd
# TODO: NORMALIZER IS WRONG HERE
# xdiffnb = -cv2.Sobel(obs_nonbnd, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / np.atleast_3d(cv2.Sobel(row(np.arange(obs_nonbnd.shape[1])).astype(np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize))
# ydiffnb = -cv2.Sobel(obs_nonbnd, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / np.atleast_3d(cv2.Sobel(col(np.arange(obs_nonbnd.shape[0])).astype(np.float64), cv2.CV_64F, dx=0, dy=1, ksize=ksize))
#
# xdiffnb.ravel()[np.isnan(xdiffnb.ravel())] = 0.
# ydiffnb.ravel()[np.isnan(ydiffnb.ravel())] = 0.
# xdiffnb.ravel()[np.isinf(xdiffnb.ravel())] = 0.
# ydiffnb.ravel()[np.isinf(ydiffnb.ravel())] = 0.
# xdiffnb = np.atleast_3d(xdiffnb)
# ydiffnb = np.atleast_3d(ydiffnb)
#
# xdiffbnd = -cv2.Sobel(observed, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / sobel_normalizer
# ydiffbnd = -cv2.Sobel(observed, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / sobel_normalizer
#
# xdiff = xdiffnb * np.atleast_3d(nbndf)
# xdiff.ravel()[np.isnan(xdiff.ravel())] = 0
# xdiff += xdiffbnd*np.atleast_3d(bndf)
#
# ydiff = ydiffnb * np.atleast_3d(nbndf)
# ydiff.ravel()[np.isnan(ydiff.ravel())] = 0
# ydiff += ydiffbnd*np.atleast_3d(bndf)
#import pdb; pdb.set_trace()
#xdiff = xdiffnb
#ydiff = ydiffnb
#import pdb; pdb.set_trace()
datas = []
# The data is weighted according to barycentric coordinates
bc0 = col(barycentric[pys, pxs, 0])
bc1 = col(barycentric[pys, pxs, 1])
bc2 = col(barycentric[pys, pxs, 2])
for k in range(n_channels):
dxs = xdiff[pys, pxs, k]
dys = ydiff[pys, pxs, k]
if f.shape[1] == 3:
datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1,col(dxs)*bc2,col(dys)*bc2)).ravel())
else:
datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1)).ravel())
data = np.concatenate(datas)
ij = np.vstack((IS.ravel(), JS.ravel()))
result = sp.csc_matrix((data, ij), shape=(image_width*image_height*n_channels, num_verts*2))
return result
def test_spherical_harmonics(self):
global visualize
if visualize:
plt.ion()
# Get mesh
v, f = get_sphere_mesh()
from geometry import VertNormals
vn = VertNormals(v=v, f=f)
#vn = Ch(mesh.estimate_vertex_normals())
# Get camera
cam, frustum = getcam()
# Get renderer
from renderer import ColoredRenderer
cam.v = v
cr = ColoredRenderer(f=f, camera=cam, frustum=frustum, v=v)
sh_red = SphericalHarmonics(vn=vn, light_color=np.array([1,0,0]))
sh_green = SphericalHarmonics(vn=vn, light_color=np.array([0,1,0]))
cr.vc = sh_red + sh_green
ims_baseline = []
for comp_idx, subplot_idx in enumerate([3,7,8,9,11,12,13,14,15]):
sh_comps = np.zeros(9)
sh_comps[comp_idx] = 1
sh_red.components = Ch(sh_comps)
sh_green.components = Ch(-sh_comps)
newim = cr.r.reshape((frustum['height'], frustum['width'], 3))
ims_baseline.append(newim)
if visualize:
plt.subplot(3,5,subplot_idx)
plt.imshow(newim)
plt.axis('off')
offset = row(.4 * (np.random.rand(3)-.5))
#offset = row(np.array([1.,1.,1.]))*.05
vn_shifted = (vn.r + offset)
vn_shifted = vn_shifted / col(np.sqrt(np.sum(vn_shifted**2, axis=1)))
vn_shifted = vn_shifted.ravel()
vn_shifted[vn_shifted>1.] = 1
vn_shifted[vn_shifted<-1.] = -1
vn_shifted = Ch(vn_shifted)
cr.replace(sh_red.vn, vn_shifted)
if True:
for comp_idx in range(9):
if visualize:
plt.figure(comp_idx+2)
sh_comps = np.zeros(9)
sh_comps[comp_idx] = 1
sh_red.components = Ch(sh_comps)
sh_green.components = Ch(-sh_comps)
pred = cr.dr_wrt(vn_shifted).dot(col(vn_shifted.r.reshape(vn.r.shape) - vn.r)).reshape((frustum['height'], frustum['width'], 3))
if visualize:
plt.subplot(1,2,1)
plt.imshow(pred)
plt.title('pred (comp %d)' % (comp_idx,))
plt.subplot(1,2,2)
newim = cr.r.reshape((frustum['height'], frustum['width'], 3))
emp = newim - ims_baseline[comp_idx]
if visualize:
plt.imshow(emp)
plt.title('empirical (comp %d)' % (comp_idx,))
pred_flat = pred.ravel()
emp_flat = emp.ravel()
nnz = np.unique(np.concatenate((np.nonzero(pred_flat)[0], np.nonzero(emp_flat)[0])))
if comp_idx != 0:
med_diff = np.median(np.abs(pred_flat[nnz]-emp_flat[nnz]))
med_obs = np.median(np.abs(emp_flat[nnz]))
if comp_idx == 4 or comp_idx == 8:
self.assertTrue(med_diff / med_obs < .6)
else:
self.assertTrue(med_diff / med_obs < .3)
if visualize:
plt.axis('off')
def compute_dr_wrt(self, wrt):
if wrt is self.x:
return row(self.r * np.linalg.inv(self.x.r).T)
def dImage_wrt_2dVerts_bnd(observed, visible, visibility, barycentric, image_width, image_height, num_verts, f, bnd_bool):
"""Construct a sparse jacobian that relates 2D projected vertex positions
(in the columns) to pixel values (in the rows). This can be done
in two steps."""
n_channels = np.atleast_3d(observed).shape[2]
shape = visibility.shape
# Step 1: get the structure ready, ie the IS and the JS
IS = np.tile(col(visible), (1, 2*f.shape[1])).ravel()
JS = col(f[visibility.ravel()[visible]].ravel())
JS = np.hstack((JS*2, JS*2+1)).ravel()
pxs = np.asarray(visible % shape[1], np.int32)
pys = np.asarray(np.floor(np.floor(visible) / shape[1]), np.int32)
if n_channels > 1:
IS = np.concatenate([IS*n_channels+i for i in range(n_channels)])
JS = np.concatenate([JS for i in range(n_channels)])
# Step 2: get the data ready, ie the actual values of the derivatives
ksize = 1
bndf = bnd_bool.astype(np.float64)
nbndf = np.logical_not(bnd_bool).astype(np.float64)
sobel_normalizer = cv2.Sobel(np.asarray(np.tile(row(np.arange(10)), (10, 1)), np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize)[5,5]
bnd_nan = bndf.reshape((observed.shape[0], observed.shape[1], -1)).copy()
bnd_nan.ravel()[bnd_nan.ravel()>0] = np.nan
bnd_nan += 1
obs_nonbnd = np.atleast_3d(observed) * bnd_nan
ydiffnb, xdiffnb = nangradients(obs_nonbnd)
observed = np.atleast_3d(observed)
if observed.shape[2] > 1:
ydiffbnd, xdiffbnd, _ = np.gradient(observed)
else:
ydiffbnd, xdiffbnd = np.gradient(observed.squeeze())
ydiffbnd = np.atleast_3d(ydiffbnd)
xdiffbnd = np.atleast_3d(xdiffbnd)
# This corrects for a bias imposed boundary differences begin spread over two pixels
# (by np.gradients or similar) but only counted once (since OpenGL's line
# drawing spans 1 pixel)
xdiffbnd *= 2.0
ydiffbnd *= 2.0
xdiffnb = -xdiffnb
ydiffnb = -ydiffnb
xdiffbnd = -xdiffbnd
ydiffbnd = -ydiffbnd
# ydiffnb *= 0
# xdiffnb *= 0
if False:
import matplotlib.pyplot as plt
plt.figure()
plt.subplot(121)
plt.imshow(xdiffnb)
plt.title('xdiffnb')
plt.subplot(122)
plt.imshow(xdiffbnd)
plt.title('xdiffbnd')
# import pdb; pdb.set_trace()
idxs = np.isnan(xdiffnb.ravel())
xdiffnb.ravel()[idxs] = xdiffbnd.ravel()[idxs]
idxs = np.isnan(ydiffnb.ravel())
ydiffnb.ravel()[idxs] = ydiffbnd.ravel()[idxs]
if True: # should be right thing
xdiff = xdiffnb
ydiff = ydiffnb
else: #should be old way
xdiff = xdiffbnd
ydiff = ydiffbnd
# TODO: NORMALIZER IS WRONG HERE
# xdiffnb = -cv2.Sobel(obs_nonbnd, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / np.atleast_3d(cv2.Sobel(row(np.arange(obs_nonbnd.shape[1])).astype(np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize))
# ydiffnb = -cv2.Sobel(obs_nonbnd, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / np.atleast_3d(cv2.Sobel(col(np.arange(obs_nonbnd.shape[0])).astype(np.float64), cv2.CV_64F, dx=0, dy=1, ksize=ksize))
#
# xdiffnb.ravel()[np.isnan(xdiffnb.ravel())] = 0.
# ydiffnb.ravel()[np.isnan(ydiffnb.ravel())] = 0.
# xdiffnb.ravel()[np.isinf(xdiffnb.ravel())] = 0.
# ydiffnb.ravel()[np.isinf(ydiffnb.ravel())] = 0.
# xdiffnb = np.atleast_3d(xdiffnb)
# ydiffnb = np.atleast_3d(ydiffnb)
#
# xdiffbnd = -cv2.Sobel(observed, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / sobel_normalizer
# ydiffbnd = -cv2.Sobel(observed, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / sobel_normalizer
#
# xdiff = xdiffnb * np.atleast_3d(nbndf)
# xdiff.ravel()[np.isnan(xdiff.ravel())] = 0
# xdiff += xdiffbnd*np.atleast_3d(bndf)
#
# ydiff = ydiffnb * np.atleast_3d(nbndf)
# ydiff.ravel()[np.isnan(ydiff.ravel())] = 0
# ydiff += ydiffbnd*np.atleast_3d(bndf)
#import pdb; pdb.set_trace()
#xdiff = xdiffnb
#ydiff = ydiffnb
#import pdb; pdb.set_trace()
datas = []
# The data is weighted according to barycentric coordinates
bc0 = col(barycentric[pys, pxs, 0])
bc1 = col(barycentric[pys, pxs, 1])
bc2 = col(barycentric[pys, pxs, 2])
for k in range(n_channels):
dxs = xdiff[pys, pxs, k]
dys = ydiff[pys, pxs, k]
if f.shape[1] == 3:
datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1,col(dxs)*bc2,col(dys)*bc2)).ravel())
else:
datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1)).ravel())
data = np.concatenate(datas)
ij = np.vstack((IS.ravel(), JS.ravel()))
result = sp.csc_matrix((data, ij), shape=(image_width*image_height*n_channels, num_verts*2))
return result
def compute_dr_wrt(self, wrt):
if wrt is self.x:
return row(self.x.r.ravel() * 2.)
