def SecondFundamentalForm(v, f): from chumpy import hstack, vstack from chumpy.linalg import Pinv nbrs = MatVecMult(FirstEdgesMtx(v, f, want_big=True), v.ravel()).reshape( (-1, 3)) b0 = VertNormals(f=f, v=v) b1 = NormalizedNx3(CrossProduct(b0, nbrs - v)).reshape((-1, 3)) b2 = NormalizedNx3(CrossProduct(b0, b1)).reshape((-1, 3)) cnct = get_vert_connectivity(np.asarray(v), f) ffs = [] for i in range(v.size / 3): nbrs = v[np.nonzero(np.asarray(cnct[i].todense()).ravel())[0]] - row( v[i]) us = nbrs.dot(b2[i]) vs = nbrs.dot(b1[i]) hs = nbrs.dot(b0[i]) coeffs = Pinv( hstack((col((us * .5)**2), col(us * vs), col( (vs * .5)**2)))).dot(hs) ffs.append(row(coeffs)) # if i == 3586: # import pdb; pdb.set_trace() ffs = vstack(ffs) return ffs
def get_earthmesh(trans, rotation): from .serialization import load_mesh from copy import deepcopy if not hasattr(get_earthmesh, 'm'): def wg(url): dest = join('/tmp', split(url)[1]) if not exists(dest): wget(url, dest) wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.obj') wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.mtl') wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.jpg') fname = join('/tmp', 'nasa_earth.obj') mesh = load_mesh(fname) mesh.v = np.asarray(mesh.v, order='C') mesh.vc = mesh.v*0 + 1 mesh.v -= row(np.mean(mesh.v, axis=0)) mesh.v /= np.max(mesh.v) mesh.v *= 2.0 get_earthmesh.mesh = mesh mesh = deepcopy(get_earthmesh.mesh) mesh.v = mesh.v.dot(cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0]) mesh.v = mesh.v + row(np.asarray(trans)) return mesh
def get_earthmesh(trans, rotation): from opendr.serialization import load_mesh from copy import deepcopy if not hasattr(get_earthmesh, 'm'): def wg(url): dest = join('/tmp', split(url)[1]) if not exists(dest): wget(url, dest) wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.obj') wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.mtl') wg('http://files.is.tue.mpg.de/mloper/opendr/images/nasa_earth.jpg') fname = join('/tmp', 'nasa_earth.obj') mesh = load_mesh(fname) mesh.v = np.asarray(mesh.v, order='C') mesh.vc = mesh.v * 0 + 1 mesh.v -= row(np.mean(mesh.v, axis=0)) mesh.v /= np.max(mesh.v) mesh.v *= 2.0 get_earthmesh.mesh = mesh mesh = deepcopy(get_earthmesh.mesh) mesh.v = mesh.v.dot( cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0]) mesh.v = mesh.v + row(np.asarray(trans)) return mesh
def compute_dr_wrt(self, obj): if obj is self.a: if not hasattr(self, '_dr_cached'): IS = np.arange(len(self.idxs)) JS = self.idxs.ravel() ij = np.vstack((row(IS), row(JS))) data = np.ones(len(self.idxs)) self._dr_cached = sp.csc_matrix( (data, ij), shape=(len(self.idxs), np.prod(self.a.shape))) return self._dr_cached
def read_and_process_mesh(fname, trans, rotation): mesh = load_mesh(fname) mesh.v = np.asarray(mesh.v, order='C') mesh.vc = np.ones_like(mesh.v) mesh.v -= row(np.mean(mesh.v, axis=0)) mesh.v /= np.max(mesh.v) mesh.v *= 2.0 mesh.v = mesh.v.dot( cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0]) if hasattr(mesh, "vn"): mesh.vn = mesh.vn.dot( cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0]) mesh.v = mesh.v + row(np.asarray(trans)) return mesh
def compute_dr_wrt(self, wrt): if wrt is not self.v: return None cplus = self.cplus cminus = self.cminus vplus = self.f[:,cplus] vminus = self.f[:,cminus] vplus3 = row(np.hstack([col(vplus*3), col(vplus*3+1), col(vplus*3+2)])) vminus3 = row(np.hstack([col(vminus*3), col(vminus*3+1), col(vminus*3+2)])) IS = row(np.arange(0,vplus3.size)) ones = np.ones(vplus3.size) shape = (self.f.size, self.v.r.size) return sp.csc_matrix((ones, np.vstack([IS, vplus3])), shape=shape) - sp.csc_matrix((ones, np.vstack([IS, vminus3])), shape=shape)
def compute_dr_wrt(self, wrt): result = super(TexturedRenderer, self).compute_dr_wrt(wrt) if wrt is self.vc: cim = self.draw_color_image(with_vertex_colors=False).ravel() cim = sp.spdiags(row(cim), [0], cim.size, cim.size) result = cim.dot(result) elif wrt is self.texture_image: IS = np.nonzero(self.visibility_image.ravel() != 4294967295)[0] JS = self.texcoord_image_quantized.ravel()[IS] clr_im = self.draw_color_image(with_vertex_colors=True, with_texture_on=False) if False: cv2.imshow('clr_im', clr_im) cv2.imshow('texmap', self.texture_image.r) cv2.waitKey(1) r = clr_im[:, :, 0].ravel()[IS] g = clr_im[:, :, 1].ravel()[IS] b = clr_im[:, :, 2].ravel()[IS] data = np.concatenate((r, g, b)) IS = np.concatenate((IS * 3, IS * 3 + 1, IS * 3 + 2)) JS = np.concatenate((JS * 3, JS * 3 + 1, JS * 3 + 2)) return sp.csc_matrix((data, (IS, JS)), shape=(self.r.size, wrt.r.size)) return result
def compute_dr_wrt(self, wrt): result = super(TexturedRenderer, self).compute_dr_wrt(wrt) if wrt is self.vc: cim = self.draw_color_image(with_vertex_colors=False).ravel() cim = sp.spdiags(row(cim), [0], cim.size, cim.size) result = cim.dot(result) elif wrt is self.texture_image: IS = np.nonzero(self.visibility_image.ravel() != 4294967295)[0] JS = self.texcoord_image_quantized.ravel()[IS] clr_im = self.draw_color_image(with_vertex_colors=True, with_texture_on=False) if False: cv2.imshow('clr_im', clr_im) cv2.imshow('texmap', self.texture_image.r) cv2.waitKey(1) r = clr_im[:,:,0].ravel()[IS] g = clr_im[:,:,1].ravel()[IS] b = clr_im[:,:,2].ravel()[IS] data = np.concatenate((r,g,b)) IS = np.concatenate((IS*3, IS*3+1, IS*3+2)) JS = np.concatenate((JS*3, JS*3+1, JS*3+2)) return sp.csc_matrix((data, (IS, JS)), shape=(self.r.size, wrt.r.size)) return result
def draw_boundary_images(glf, glb, v, f, vpe, fpe, camera): """Assumes camera is set up correctly, and that glf has any texmapping on necessary.""" glf.Clear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); glb.Clear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); # Figure out which edges are on pairs of differently visible triangles from opendr.geometry import TriNormals tn = TriNormals(v, f).r.reshape((-1,3)) campos = -cv2.Rodrigues(camera.rt.r)[0].T.dot(camera.t.r) rays_to_verts = v.reshape((-1,3)) - row(campos) rays_to_faces = rays_to_verts[f[:,0]] + rays_to_verts[f[:,1]] + rays_to_verts[f[:,2]] dps = np.sum(rays_to_faces * tn, axis=1) dps = dps[fpe[:,0]] * dps[fpe[:,1]] silhouette_edges = np.asarray(np.nonzero(dps<=0)[0], np.uint32) non_silhouette_edges = np.nonzero(dps>0)[0] lines_e = vpe[silhouette_edges] lines_v = v visibility = draw_edge_visibility(glb, lines_v, lines_e, f, hidden_wireframe=True) shape = visibility.shape visibility = visibility.ravel() visible = np.nonzero(visibility.ravel() != 4294967295)[0] visibility[visible] = silhouette_edges[visibility[visible]] result = visibility.reshape(shape) return result
def unproject_points(self, uvd, camera_space=False): cam = ProjectPoints3D( **{k: getattr(self, k) for k in self.dterms if hasattr(self, k)}) try: xy_undistorted_camspace = cv2.undistortPoints( np.asarray(uvd[:, :2].reshape((1, -1, 2)).copy()), np.asarray(cam.camera_mtx), cam.k.r) xyz_camera_space = np.hstack( (xy_undistorted_camspace.squeeze(), col(uvd[:, 2]))) xyz_camera_space[:, :2] *= col( xyz_camera_space[:, 2]) # scale x,y by z if camera_space: return xyz_camera_space other_answer = xyz_camera_space - row(cam.view_mtx[:, 3]) # translate result = other_answer.dot(cam.view_mtx[:, :3]) # rotate except: # slow way, probably not so good. But doesn't require cv2.undistortPoints. cam.v = np.ones_like(uvd) ch.minimize(cam - uvd, x0=[cam.v], method='dogleg', options={'disp': 0}) result = cam.v.r return result
def compute_dr_wrt(self, wrt): if wrt is not self.camera and wrt is not self.v: return None visibility = self.visibility_image visible = np.nonzero(visibility.ravel() != 4294967295)[0] barycentric = self.barycentric_image if wrt is self.camera: shape = visibility.shape depth = self.depth_image if self.overdraw: result1 = common.dImage_wrt_2dVerts_bnd( depth, visible, visibility, barycentric, self.frustum['width'], self.frustum['height'], self.v.r.size / 3, self.f, self.boundaryid_image != 4294967295) else: result1 = common.dImage_wrt_2dVerts(depth, visible, visibility, barycentric, self.frustum['width'], self.frustum['height'], self.v.r.size / 3, self.f) # result1 = common.dImage_wrt_2dVerts(depth, visible, visibility, barycentric, self.frustum['width'], self.frustum['height'], self.v.r.size/3, self.f) return result1 elif wrt is self.v: IS = np.tile(col(visible), (1, 9)).ravel() JS = col(self.f[visibility.ravel()[visible]].ravel()) JS = np.hstack((JS * 3, JS * 3 + 1, JS * 3 + 2)).ravel() # FIXME: there should be a faster way to get the camera axis. # But it should be carefully tested with distortion present! pts = np.array([[self.camera.c.r[0], self.camera.c.r[1], 2], [self.camera.c.r[0], self.camera.c.r[1], 1]]) pts = self.camera.unproject_points(pts) cam_axis = pts[0, :] - pts[1, :] if True: # use barycentric coordinates (correct way) w = visibility.shape[1] pxs = np.asarray(visible % w, np.int32) pys = np.asarray(np.floor(np.floor(visible) / w), np.int32) bc0 = col(barycentric[pys, pxs, 0]) bc1 = col(barycentric[pys, pxs, 1]) bc2 = col(barycentric[pys, pxs, 2]) bc = np.hstack( (bc0, bc0, bc0, bc1, bc1, bc1, bc2, bc2, bc2)).ravel() else: # each vert contributes equally (an approximation) bc = 1. / 3. data = np.tile(row(cam_axis), (IS.size / 3, 1)).ravel() * bc result2 = sp.csc_matrix( (data, (IS, JS)), shape=(self.frustum['height'] * self.frustum['width'], self.v.r.size)) return result2
def draw_boundary_images(glf, glb, v, f, vpe, fpe, camera): """Assumes camera is set up correctly, and that glf has any texmapping on necessary.""" glf.Clear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) glb.Clear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) # Figure out which edges are on pairs of differently visible triangles from opendr.geometry import TriNormals tn = TriNormals(v, f).r.reshape((-1, 3)) campos = -cv2.Rodrigues(camera.rt.r)[0].T.dot(camera.t.r) rays_to_verts = v.reshape((-1, 3)) - row(campos) rays_to_faces = rays_to_verts[f[:, 0]] + rays_to_verts[ f[:, 1]] + rays_to_verts[f[:, 2]] dps = np.sum(rays_to_faces * tn, axis=1) dps = dps[fpe[:, 0]] * dps[fpe[:, 1]] silhouette_edges = np.asarray(np.nonzero(dps <= 0)[0], np.uint32) non_silhouette_edges = np.nonzero(dps > 0)[0] lines_e = vpe[silhouette_edges] lines_v = v visibility = draw_edge_visibility(glb, lines_v, lines_e, f, hidden_wireframe=True) shape = visibility.shape visibility = visibility.ravel() visible = np.nonzero(visibility.ravel() != 4294967295)[0] visibility[visible] = silhouette_edges[visibility[visible]] result = visibility.reshape(shape) return result
def get_earthmesh(trans, rotation): fname = "example_data/nasa_earth.obj" mesh = load_mesh(fname) if not hasattr(get_earthmesh, "mesh"): mesh.v = np.asarray(mesh.v, order='C') mesh.vc = mesh.v * 0 + 1 mesh.v -= row(np.mean(mesh.v, axis=0)) mesh.v /= np.max(mesh.v) mesh.v *= 2.0 get_earthmesh.mesh = mesh mesh = deepcopy(get_earthmesh.mesh) mesh.v = mesh.v.dot( cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0]) mesh.v = mesh.v + row(np.asarray(trans)) return mesh
def compute_d1(self): # To stay consistent with numpy, we must upgrade 1D arrays to 2D ar = row(self.a.r) if len(self.a.r.shape)<2 else self.a.r.reshape((-1, self.a.r.shape[-1])) br = col(self.b.r) if len(self.b.r.shape)<2 else self.b.r.reshape((self.b.r.shape[0], -1)) if ar.ndim <= 2: return sp.kron(sp.eye(ar.shape[0], ar.shape[0]),br.T) else: raise NotImplementedError
def get_vert_connectivity(mesh_v, mesh_f): """Returns a sparse matrix (of size #verts x #verts) where each nonzero element indicates a neighborhood relation. For example, if there is a nonzero element in position (15,12), that means vertex 15 is connected by an edge to vertex 12.""" vpv = sp.csc_matrix((len(mesh_v),len(mesh_v))) # for each column in the faces... for i in range(3): IS = mesh_f[:,i] JS = mesh_f[:,(i+1)%3] data = np.ones(len(IS)) ij = np.vstack((row(IS.flatten()), row(JS.flatten()))) mtx = sp.csc_matrix((data, ij), shape=vpv.shape) vpv = vpv + mtx + mtx.T return vpv
def flow_to(self, v_next, cam_next): from chumpy.ch import MatVecMult color_image = self.r visibility = self.visibility_image pxpos = np.zeros_like(self.color_image) pxpos[:, :, 0] = np.tile(row(np.arange(self.color_image.shape[1])), (self.color_image.shape[0], 1)) pxpos[:, :, 2] = np.tile(col(np.arange(self.color_image.shape[0])), (1, self.color_image.shape[1])) visible = np.nonzero(visibility.ravel() != 4294967295)[0] num_visible = len(visible) barycentric = self.barycentric_image # map 3d to 3d JS = col(self.f[visibility.ravel()[visible]]).ravel() IS = np.tile(col(np.arange(JS.size / 3)), (1, 3)).ravel() data = barycentric.reshape((-1, 3))[visible].ravel() # replicate to xyz IS = np.concatenate((IS * 3, IS * 3 + 1, IS * 3 + 2)) JS = np.concatenate((JS * 3, JS * 3 + 1, JS * 3 + 2)) data = np.concatenate((data, data, data)) verts_to_visible = sp.csc_matrix((data, (IS, JS)), shape=(np.max(IS) + 1, self.v.r.size)) v_old = self.camera.v cam_old = self.camera if cam_next is None: cam_next = self.camera self.camera.v = MatVecMult(verts_to_visible, self.v.r) r1 = self.camera.r.copy() self.camera = cam_next self.camera.v = MatVecMult(verts_to_visible, v_next) r2 = self.camera.r.copy() n_channels = self.camera.shape[1] flow = r2 - r1 flow_im = np.zeros( (self.frustum['height'], self.frustum['width'], n_channels)).reshape( (-1, n_channels)) flow_im[visible] = flow flow_im = flow_im.reshape( (self.frustum['height'], self.frustum['width'], n_channels)) self.camera = cam_old self.camera.v = v_old return flow_im
def get_body_mesh(obj_path, trans, rotation): from opendr.serialization import load_mesh from copy import deepcopy fname = obj_path mesh = load_mesh(fname) mesh.v = np.asarray(mesh.v, order='C') mesh.vc = mesh.v * 0 + 1 mesh.v -= row(np.mean(mesh.v, axis=0)) mesh.v /= np.max(mesh.v) mesh.v *= 2.0 get_body_mesh.mesh = mesh mesh = deepcopy(get_body_mesh.mesh) mesh.v = mesh.v.dot( cv2.Rodrigues(np.asarray(np.array(rotation), np.float64))[0]) mesh.v = mesh.v + row(np.asarray(trans)) return mesh
def compute_d1(self): # To stay consistent with numpy, we must upgrade 1D arrays to 2D mtx1r = row(self.mtx1.r) if len(self.mtx1.r.shape)<2 else self.mtx1.r mtx2r = col(self.mtx2.r) if len(self.mtx2.r.shape)<2 else self.mtx2.r if mtx1r.ndim <= 2: return sp.kron(sp.eye(mtx1r.shape[0], mtx1r.shape[0]),mtx2r.T) else: mtx2f = mtx2r.reshape((-1, mtx2r.shape[-2], mtx2r.shape[-1])) mtx2f = np.rollaxis(mtx2f, -1, -2) #transpose basically result = sp.block_diag([np.kron(np.eye(mtx1r.shape[-2], mtx1r.shape[-2]),m2) for m2 in mtx2f]) assert(result.shape[0] == self.r.size) return result
def flow_to(self, v_next, cam_next): from chumpy.ch import MatVecMult color_image = self.r visibility = self.visibility_image pxpos = np.zeros_like(self.color_image) pxpos[:,:,0] = np.tile(row(np.arange(self.color_image.shape[1])), (self.color_image.shape[0], 1)) pxpos[:,:,2] = np.tile(col(np.arange(self.color_image.shape[0])), (1, self.color_image.shape[1])) visible = np.nonzero(visibility.ravel() != 4294967295)[0] num_visible = len(visible) barycentric = self.barycentric_image # map 3d to 3d JS = col(self.f[visibility.ravel()[visible]]).ravel() IS = np.tile(col(np.arange(JS.size/3)), (1, 3)).ravel() data = barycentric.reshape((-1,3))[visible].ravel() # replicate to xyz IS = np.concatenate((IS*3, IS*3+1, IS*3+2)) JS = np.concatenate((JS*3, JS*3+1, JS*3+2)) data = np.concatenate((data, data, data)) verts_to_visible = sp.csc_matrix((data, (IS, JS)), shape=(np.max(IS)+1, self.v.r.size)) v_old = self.camera.v cam_old = self.camera if cam_next is None: cam_next = self.camera self.camera.v = MatVecMult(verts_to_visible, self.v.r) r1 = self.camera.r.copy() self.camera = cam_next self.camera.v = MatVecMult(verts_to_visible, v_next) r2 = self.camera.r.copy() n_channels = self.camera.shape[1] flow = r2 - r1 flow_im = np.zeros((self.frustum['height'], self.frustum['width'], n_channels)).reshape((-1,n_channels)) flow_im[visible] = flow flow_im = flow_im.reshape((self.frustum['height'], self.frustum['width'], n_channels)) self.camera = cam_old self.camera.v = v_old return flow_im
def compute_vpe_boundary_idxs(v, f, camera, fpe): # Figure out which edges are on pairs of differently visible triangles from geometry import TriNormals tn = TriNormals(v, f).r.reshape((-1,3)) #ray = cv2.Rodrigues(camera.rt.r)[0].T[:,2] campos = -cv2.Rodrigues(camera.rt.r)[0].T.dot(camera.t.r) rays_to_verts = v.reshape((-1,3)) - row(campos) rays_to_faces = rays_to_verts[f[:,0]] + rays_to_verts[f[:,1]] + rays_to_verts[f[:,2]] faces_invisible = np.sum(rays_to_faces * tn, axis=1) dps = faces_invisible[fpe[:,0]] * faces_invisible[fpe[:,1]] silhouette_edges = np.asarray(np.nonzero(dps<=0)[0], np.uint32) return silhouette_edges, faces_invisible < 0
def dImage_wrt_2dVerts(observed, visible, visibility, barycentric, image_width, image_height, num_verts, f): """Construct a sparse jacobian that relates 2D projected vertex positions (in the columns) to pixel values (in the rows). This can be done in two steps.""" num_verts = np.int32(num_verts) n_channels = np.atleast_3d(observed).shape[2] shape = visibility.shape # Step 1: get the structure ready, ie the IS and the JS IS = np.tile(col(visible), (1, 2*f.shape[1])).ravel() JS = f[visibility.ravel()[visible]].reshape((-1,1)) JS = np.hstack((JS*2, JS*2+1)).ravel() pxs = np.asarray(visible % shape[1], np.int32) pys = np.asarray(np.floor(np.floor(visible) / shape[1]), np.int32) if n_channels > 1: IS = np.concatenate([IS*n_channels+i for i in range(n_channels)]) JS = np.concatenate([JS for i in range(n_channels)]) # Step 2: get the data ready, ie the actual values of the derivatives ksize=1 sobel_normalizer = cv2.Sobel(np.asarray(np.tile(row(np.arange(10)), (10, 1)), np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize)[5,5] xdiff = -cv2.Sobel(observed, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / sobel_normalizer ydiff = cv2.Sobel(observed, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / sobel_normalizer xdiff = np.atleast_3d(xdiff) ydiff = np.atleast_3d(ydiff) datas = [] # The data is weighted according to barycentric coordinates bc0 = barycentric[pys, pxs, 0].reshape((-1,1)) bc1 = barycentric[pys, pxs, 1].reshape((-1,1)) bc2 = barycentric[pys, pxs, 2].reshape((-1,1)) for k in range(n_channels): dxs = xdiff[pys, pxs, k] dys = ydiff[pys, pxs, k] if f.shape[1] == 3: datas.append(np.hstack((dxs.reshape((-1,1))*bc0,dys.reshape((-1,1))*bc0,dxs.reshape((-1,1))*bc1,dys.reshape((-1,1))*bc1,dxs.reshape((-1,1))*bc2,dys.reshape((-1,1))*bc2)).ravel()) else: datas.append(np.hstack((dxs.reshape((-1,1))*bc0,dys.reshape((-1,1))*bc0,dxs.reshape((-1,1))*bc1,dys.reshape((-1,1))*bc1)).ravel()) data = np.concatenate(datas) ij = np.vstack((IS.ravel(), JS.ravel())).astype(np.int32) result = sp.csc_matrix((data, ij), shape=(image_width*image_height*n_channels, num_verts*2)) return result
def test_derivatives(self): import chumpy as ch from chumpy.utils import row import numpy as np from .renderer import DepthRenderer rn = DepthRenderer() # Assign attributes to renderer from .util_tests import get_earthmesh m = get_earthmesh(trans=ch.array([0, 0, 4]), rotation=ch.zeros(3)) w, h = (320, 240) from .camera import ProjectPoints rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w, w]) / 2., c=ch.array([w, h]) / 2., k=ch.zeros(5)) rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h} rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3)) if visualize: import matplotlib.pyplot as plt plt.figure() for which in range(3): r1 = rn.r adder = np.zeros(3) adder[which] = .01 change = rn.v.r * 0 + row(adder) dr_pred = rn.dr_wrt(rn.v).dot(change.ravel()).reshape(rn.shape) rn.v = rn.v.r + change r2 = rn.r dr_emp = r2 - r1 # print np.mean(np.abs(dr_pred-dr_emp)) self.assertLess(np.mean(np.abs(dr_pred - dr_emp)), .031) if visualize: plt.subplot(2, 3, which + 1) plt.imshow(dr_pred) plt.clim(-.01, .01) plt.title('emp') plt.subplot(2, 3, which + 4) plt.imshow(dr_emp) plt.clim(-.01, .01) plt.title('pred')
def SecondFundamentalForm(v, f): from chumpy import hstack, vstack from chumpy.linalg import Pinv nbrs = MatVecMult(FirstEdgesMtx(v, f, want_big=True), v.ravel()).reshape((-1,3)) b0 = NormalizedNx3(VertNormalsScaled(f=f, v=v)).reshape((-1,3)) b1 = NormalizedNx3(CrossProduct(b0, nbrs-v)).reshape((-1,3)) b2 = NormalizedNx3(CrossProduct(b0, b1)).reshape((-1,3)) cnct = get_vert_connectivity(v.r, f) ffs = [] for i in range(v.size/3): nbrs = v[np.nonzero(np.asarray(cnct[i].todense()).ravel())[0]] - row(v[i]) us = nbrs.dot(b2[i]) vs = nbrs.dot(b1[i]) hs = nbrs.dot(b0[i]) coeffs = Pinv(hstack((col((us*.5)**2), col(us*vs), col((vs*.5)**2)))).dot(hs) ffs.append(row(coeffs)) # if i == 3586: # import pdb; pdb.set_trace() ffs = vstack(ffs) return ffs
def compute_dr_wrt(self, wrt): if wrt is not self.camera and wrt is not self.v: return None visibility = self.visibility_image visible = np.nonzero(visibility.ravel() != 4294967295)[0] barycentric = self.barycentric_image if wrt is self.camera: shape = visibility.shape depth = self.depth_image if self.overdraw: result1 = common.dImage_wrt_2dVerts_bnd(depth, visible, visibility, barycentric, self.frustum['width'], self.frustum['height'], self.v.r.size/3, self.f, self.boundaryid_image != 4294967295) else: result1 = common.dImage_wrt_2dVerts(depth, visible, visibility, barycentric, self.frustum['width'], self.frustum['height'], self.v.r.size/3, self.f) # result1 = common.dImage_wrt_2dVerts(depth, visible, visibility, barycentric, self.frustum['width'], self.frustum['height'], self.v.r.size/3, self.f) return result1 elif wrt is self.v: IS = np.tile(col(visible), (1, 9)).ravel() JS = col(self.f[visibility.ravel()[visible]].ravel()) JS = np.hstack((JS*3, JS*3+1, JS*3+2)).ravel() # FIXME: there should be a faster way to get the camera axis. # But it should be carefully tested with distortion present! pts = np.array([ [self.camera.c.r[0], self.camera.c.r[1], 2], [self.camera.c.r[0], self.camera.c.r[1], 1] ]) pts = self.camera.unproject_points(pts) cam_axis = pts[0,:] - pts[1,:] if True: # use barycentric coordinates (correct way) w = visibility.shape[1] pxs = np.asarray(visible % w, np.int32) pys = np.asarray(np.floor(np.floor(visible) / w), np.int32) bc0 = col(barycentric[pys, pxs, 0]) bc1 = col(barycentric[pys, pxs, 1]) bc2 = col(barycentric[pys, pxs, 2]) bc = np.hstack((bc0,bc0,bc0,bc1,bc1,bc1,bc2,bc2,bc2)).ravel() else: # each vert contributes equally (an approximation) bc = 1. / 3. data = np.tile(row(cam_axis), (IS.size/3,1)).ravel() * bc result2 = sp.csc_matrix((data, (IS, JS)), shape=(self.frustum['height']*self.frustum['width'], self.v.r.size)) return result2
def dImage_wrt_2dVerts(observed, visible, visibility, barycentric, image_width, image_height, num_verts, f): """Construct a sparse jacobian that relates 2D projected vertex positions (in the columns) to pixel values (in the rows). This can be done in two steps.""" n_channels = np.atleast_3d(observed).shape[2] shape = visibility.shape # Step 1: get the structure ready, ie the IS and the JS IS = np.tile(col(visible), (1, 2*f.shape[1])).ravel() JS = col(f[visibility.ravel()[visible]].ravel()) JS = np.hstack((JS*2, JS*2+1)).ravel() pxs = np.asarray(visible % shape[1], np.int32) pys = np.asarray(np.floor(np.floor(visible) / shape[1]), np.int32) if n_channels > 1: IS = np.concatenate([IS*n_channels+i for i in range(n_channels)]) JS = np.concatenate([JS for i in range(n_channels)]) # Step 2: get the data ready, ie the actual values of the derivatives ksize=1 sobel_normalizer = cv2.Sobel(np.asarray(np.tile(row(np.arange(10)), (10, 1)), np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize)[5,5] xdiff = -cv2.Sobel(observed, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / sobel_normalizer ydiff = -cv2.Sobel(observed, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / sobel_normalizer xdiff = np.atleast_3d(xdiff) ydiff = np.atleast_3d(ydiff) datas = [] # The data is weighted according to barycentric coordinates bc0 = col(barycentric[pys, pxs, 0]) bc1 = col(barycentric[pys, pxs, 1]) bc2 = col(barycentric[pys, pxs, 2]) for k in range(n_channels): dxs = xdiff[pys, pxs, k] dys = ydiff[pys, pxs, k] if f.shape[1] == 3: datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1,col(dxs)*bc2,col(dys)*bc2)).ravel()) else: datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1)).ravel()) data = np.concatenate(datas) ij = np.vstack((IS.ravel(), JS.ravel())) result = sp.csc_matrix((data, ij), shape=(image_width*image_height*n_channels, num_verts*2)) return result
def compute_d2(self): # To stay consistent with numpy, we must upgrade 1D arrays to 2D mtx1r = row(self.mtx1.r) if len(self.mtx1.r.shape)<2 else self.mtx1.r mtx2r = col(self.mtx2.r) if len(self.mtx2.r.shape)<2 else self.mtx2.r if mtx2r.ndim <= 1: return self.mtx1r elif mtx2r.ndim <= 2: return sp.kron(mtx1r, sp.eye(mtx2r.shape[1],mtx2r.shape[1])) else: mtx1f = mtx1r.reshape((-1, mtx1r.shape[-2], mtx1r.shape[-1])) result = sp.block_diag([np.kron(m1, np.eye(mtx2r.shape[-1],mtx2r.shape[-1])) for m1 in mtx1f]) assert(result.shape[0] == self.r.size) return result
def unproject_points(self, uvd, camera_space=False): cam = ProjectPoints3D(**{k: getattr(self, k) for k in self.dterms if hasattr(self, k)}) try: xy_undistorted_camspace = cv2.undistortPoints(np.asarray(uvd[:,:2].reshape((1,-1,2)).copy()), np.asarray(cam.camera_mtx), cam.k.r) xyz_camera_space = np.hstack((xy_undistorted_camspace.squeeze(), col(uvd[:,2]))) xyz_camera_space[:,:2] *= col(xyz_camera_space[:,2]) # scale x,y by z if camera_space: return xyz_camera_space other_answer = xyz_camera_space - row(cam.view_mtx[:,3]) # translate result = other_answer.dot(cam.view_mtx[:,:3]) # rotate except: # slow way, probably not so good. But doesn't require cv2.undistortPoints. cam.v = np.ones_like(uvd) ch.minimize(cam - uvd, x0=[cam.v], method='dogleg', options={'disp': 0}) result = cam.v.r return result
def compute_dr_wrt(self, wrt): if wrt is not self.a: return None if self.axis is not None: raise NotImplementedError IS = np.tile(row(np.arange(self.a.size)), (self.a.size, 1)) JS = IS.T IS = IS.ravel() JS = JS.ravel() which = IS >= JS IS = IS[which] JS = JS[which] data = np.ones_like(IS) result = sp.csc_matrix((data, (IS, JS)), shape=(self.a.size, self.a.size)) return result
def test_derivatives(self): import chumpy as ch from chumpy.utils import row import numpy as np from renderer import DepthRenderer rn = DepthRenderer() # Assign attributes to renderer from util_tests import get_earthmesh m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3)) w, h = (320, 240) from camera import ProjectPoints rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5)) rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h} rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3)) if visualize: import matplotlib.pyplot as plt plt.figure() for which in range(3): r1 = rn.r adder = np.zeros(3) adder[which] = .01 change = rn.v.r * 0 + row(adder) dr_pred = rn.dr_wrt(rn.v).dot(change.ravel()).reshape(rn.shape) rn.v = rn.v.r + change r2 = rn.r dr_emp = r2 - r1 # print np.mean(np.abs(dr_pred-dr_emp)) self.assertLess(np.mean(np.abs(dr_pred-dr_emp)), .031) if visualize: plt.subplot(2,3,which+1) plt.imshow(dr_pred) plt.clim(-.01,.01) plt.title('emp') plt.subplot(2,3,which+4) plt.imshow(dr_emp) plt.clim(-.01,.01) plt.title('pred')
def on_changed(self, which): if 'vn' in which: vn = self.vn.r.reshape((-1,3)) # Conversion from normals to spherical harmonics found in... # http://en.wikipedia.org/wiki/Spherical_coordinate_system#Cartesian_coordinates self.theta = np.arccos(vn[:,2]) self.phi = np.arctan2(vn[:,1], vn[:,0]) # vnswapped = np.swapaxes(vn, 0,1) self.sh_coeffs = real_sh_coeff(vn) self.num_verts = self.sh_coeffs.shape[0] if 'light_color' in which or self.mtx.shape[1] != self.num_verts: nc = self.num_channels IS = np.arange(self.num_verts*nc) JS = np.repeat(np.arange(self.num_verts), nc) data = (row(self.light_color)*np.ones((self.num_verts, nc))).ravel() self.mtx = sp.csc_matrix((data, (IS,JS)), shape=(self.num_verts*nc, self.num_verts))
def on_changed(self, which): if 'vn' in which: vn = self.vn.r.reshape((-1,3)) # Conversion from normals to spherical harmonics found in... # http://en.wikipedia.org/wiki/Spherical_coordinate_system#Cartesian_coordinates self.theta = np.arccos(vn[:,2]) self.phi = np.arctan2(vn[:,1], vn[:,0]) self.sh_coeffs = real_sh_coeff(vn) self.num_verts = self.sh_coeffs.shape[0] if 'light_color' in which or self.mtx.shape[1] != self.num_verts: nc = self.num_channels IS = np.arange(self.num_verts*nc) JS = np.repeat(np.arange(self.num_verts), nc) data = (row(self.light_color)*np.ones((self.num_verts, nc))).ravel() self.mtx = sp.csc_matrix((data, (IS,JS)), shape=(self.num_verts*nc, self.num_verts))
def compute_dr_wrt(self, wrt): if wrt is not self.x: return if self.axis == None: return row(np.ones((1, len(self.x.r))))/len(self.x.r) else: uid = tuple(list(self.x.shape) + [self.axis]) if uid not in self.dr_cache: idxs_presum = np.arange(self.x.size).reshape(self.x.shape) idxs_presum = np.rollaxis(idxs_presum, self.axis, 0) idxs_postsum = np.arange(self.r.size).reshape(self.r.shape) tp = np.ones(idxs_presum.ndim, dtype=np.uint32) tp[0] = idxs_presum.shape[0] idxs_postsum = np.tile(idxs_postsum, tp) data = np.ones(idxs_postsum.size) / self.x.shape[self.axis] result = sp.csc_matrix((data, (idxs_postsum.ravel(), idxs_presum.ravel())), (self.r.size, wrt.size)) self.dr_cache[uid] = result return self.dr_cache[uid]
def compute_dr_wrt(self, wrt): if wrt is self.x: if visualize: import matplotlib.pyplot as plt residuals = np.sum(self.r**2) print '------> RESIDUALS %.2e' % (residuals, ) print '------> CURRENT GUESS %s' % (str(self.x.r), ) plt.figure(123) if not hasattr(self, 'vs'): self.vs = [] self.xs = [] self.ys = [] self.vs.append(residuals) self.xs.append(self.x.r[0]) self.ys.append(self.x.r[1]) plt.clf() plt.subplot(1, 2, 1) plt.plot(self.vs) plt.subplot(1, 2, 2) plt.plot(self.xs, self.ys) plt.draw() return row(rosen_der(self.x.r))
def test_spherical_harmonics(self): global visualize if visualize: plt.ion() # Get mesh v, f = get_sphere_mesh() from geometry import VertNormals vn = VertNormals(v=v, f=f) #vn = Ch(mesh.estimate_vertex_normals()) # Get camera cam, frustum = getcam() # Get renderer from renderer import ColoredRenderer cam.v = v cr = ColoredRenderer(f=f, camera=cam, frustum=frustum, v=v) sh_red = SphericalHarmonics(vn=vn, light_color=np.array([1,0,0])) sh_green = SphericalHarmonics(vn=vn, light_color=np.array([0,1,0])) cr.vc = sh_red + sh_green ims_baseline = [] for comp_idx, subplot_idx in enumerate([3,7,8,9,11,12,13,14,15]): sh_comps = np.zeros(9) sh_comps[comp_idx] = 1 sh_red.components = Ch(sh_comps) sh_green.components = Ch(-sh_comps) newim = cr.r.reshape((frustum['height'], frustum['width'], 3)) ims_baseline.append(newim) if visualize: plt.subplot(3,5,subplot_idx) plt.imshow(newim) plt.axis('off') offset = row(.4 * (np.random.rand(3)-.5)) #offset = row(np.array([1.,1.,1.]))*.05 vn_shifted = (vn.r + offset) vn_shifted = vn_shifted / col(np.sqrt(np.sum(vn_shifted**2, axis=1))) vn_shifted = vn_shifted.ravel() vn_shifted[vn_shifted>1.] = 1 vn_shifted[vn_shifted<-1.] = -1 vn_shifted = Ch(vn_shifted) cr.replace(sh_red.vn, vn_shifted) if True: for comp_idx in range(9): if visualize: plt.figure(comp_idx+2) sh_comps = np.zeros(9) sh_comps[comp_idx] = 1 sh_red.components = Ch(sh_comps) sh_green.components = Ch(-sh_comps) pred = cr.dr_wrt(vn_shifted).dot(col(vn_shifted.r.reshape(vn.r.shape) - vn.r)).reshape((frustum['height'], frustum['width'], 3)) if visualize: plt.subplot(1,2,1) plt.imshow(pred) plt.title('pred (comp %d)' % (comp_idx,)) plt.subplot(1,2,2) newim = cr.r.reshape((frustum['height'], frustum['width'], 3)) emp = newim - ims_baseline[comp_idx] if visualize: plt.imshow(emp) plt.title('empirical (comp %d)' % (comp_idx,)) pred_flat = pred.ravel() emp_flat = emp.ravel() nnz = np.unique(np.concatenate((np.nonzero(pred_flat)[0], np.nonzero(emp_flat)[0]))) if comp_idx != 0: med_diff = np.median(np.abs(pred_flat[nnz]-emp_flat[nnz])) med_obs = np.median(np.abs(emp_flat[nnz])) if comp_idx == 4 or comp_idx == 8: self.assertTrue(med_diff / med_obs < .6) else: self.assertTrue(med_diff / med_obs < .3) if visualize: plt.axis('off')
def loop_subdivider(mesh_v, mesh_f): IS = [] JS = [] data = [] vc = get_vert_connectivity(mesh_v, mesh_f) ve = get_vertices_per_edge(mesh_v, mesh_f) vo = get_vert_opposites_per_edge(mesh_v, mesh_f) if True: # New values for each vertex for idx in xrange(len(mesh_v)): # find neighboring vertices nbrs = np.nonzero(vc[:,idx])[0] nn = len(nbrs) if nn < 3: wt = 0. elif nn == 3: wt = 3./16. elif nn > 3: wt = 3. / (8. * nn) else: raise Exception('nn should be 3 or more') if wt > 0.: for nbr in nbrs: IS.append(idx) JS.append(nbr) data.append(wt) JS.append(idx) IS.append(idx) data.append(1. - (wt * nn)) start = len(mesh_v) edge_to_midpoint = {} if True: # New values for each edge: # new edge verts depend on the verts they span for idx, vs in enumerate(ve): vsl = list(vs) vsl.sort() IS.append(start + idx) IS.append(start + idx) JS.append(vsl[0]) JS.append(vsl[1]) data.append(3./8) data.append(3./8) opposites = vo[(vsl[0], vsl[1])] for opp in opposites: IS.append(start + idx) JS.append(opp) data.append(2./8./len(opposites)) edge_to_midpoint[(vsl[0], vsl[1])] = start + idx edge_to_midpoint[(vsl[1], vsl[0])] = start + idx f = [] for f_i, old_f in enumerate(mesh_f): ff = np.concatenate((old_f, old_f)) for i in range(3): v0 = edge_to_midpoint[(ff[i], ff[i+1])] v1 = ff[i+1] v2 = edge_to_midpoint[(ff[i+1], ff[i+2])] f.append(row(np.array([v0,v1,v2]))) v0 = edge_to_midpoint[(ff[0], ff[1])] v1 = edge_to_midpoint[(ff[1], ff[2])] v2 = edge_to_midpoint[(ff[2], ff[3])] f.append(row(np.array([v0,v1,v2]))) f = np.vstack(f) IS = np.array(IS, dtype=np.uint32) JS = np.array(JS, dtype=np.uint32) if True: # for x,y,z coords IS = np.concatenate((IS*3, IS*3+1, IS*3+2)) JS = np.concatenate((JS*3, JS*3+1, JS*3+2)) data = np.concatenate ((data,data,data)) ij = np.vstack((IS.flatten(), JS.flatten())) mtx = sp.csc_matrix((data, ij)) return mtx, f
def dImage_wrt_2dVerts_bnd(observed, visible, visibility, barycentric, image_width, image_height, num_verts, f, bnd_bool): """Construct a sparse jacobian that relates 2D projected vertex positions (in the columns) to pixel values (in the rows). This can be done in two steps.""" n_channels = np.atleast_3d(observed).shape[2] shape = visibility.shape # Step 1: get the structure ready, ie the IS and the JS IS = np.tile(col(visible), (1, 2*f.shape[1])).ravel() JS = col(f[visibility.ravel()[visible]].ravel()) JS = np.hstack((JS*2, JS*2+1)).ravel() pxs = np.asarray(visible % shape[1], np.int32) pys = np.asarray(np.floor(np.floor(visible) / shape[1]), np.int32) if n_channels > 1: IS = np.concatenate([IS*n_channels+i for i in range(n_channels)]) JS = np.concatenate([JS for i in range(n_channels)]) # Step 2: get the data ready, ie the actual values of the derivatives ksize = 1 bndf = bnd_bool.astype(np.float64) nbndf = np.logical_not(bnd_bool).astype(np.float64) sobel_normalizer = cv2.Sobel(np.asarray(np.tile(row(np.arange(10)), (10, 1)), np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize)[5,5] bnd_nan = bndf.reshape((observed.shape[0], observed.shape[1], -1)).copy() bnd_nan.ravel()[bnd_nan.ravel()>0] = np.nan bnd_nan += 1 obs_nonbnd = np.atleast_3d(observed) * bnd_nan ydiffnb, xdiffnb = nangradients(obs_nonbnd) observed = np.atleast_3d(observed) if observed.shape[2] > 1: ydiffbnd, xdiffbnd, _ = np.gradient(observed) else: ydiffbnd, xdiffbnd = np.gradient(observed.squeeze()) ydiffbnd = np.atleast_3d(ydiffbnd) xdiffbnd = np.atleast_3d(xdiffbnd) # This corrects for a bias imposed boundary differences begin spread over two pixels # (by np.gradients or similar) but only counted once (since OpenGL's line # drawing spans 1 pixel) xdiffbnd *= 2.0 ydiffbnd *= 2.0 xdiffnb = -xdiffnb ydiffnb = -ydiffnb xdiffbnd = -xdiffbnd ydiffbnd = -ydiffbnd # ydiffnb *= 0 # xdiffnb *= 0 if False: import matplotlib.pyplot as plt plt.figure() plt.subplot(121) plt.imshow(xdiffnb) plt.title('xdiffnb') plt.subplot(122) plt.imshow(xdiffbnd) plt.title('xdiffbnd') import pdb; pdb.set_trace() idxs = np.isnan(xdiffnb.ravel()) xdiffnb.ravel()[idxs] = xdiffbnd.ravel()[idxs] idxs = np.isnan(ydiffnb.ravel()) ydiffnb.ravel()[idxs] = ydiffbnd.ravel()[idxs] if True: # should be right thing xdiff = xdiffnb ydiff = ydiffnb else: #should be old way xdiff = xdiffbnd ydiff = ydiffbnd # TODO: NORMALIZER IS WRONG HERE # xdiffnb = -cv2.Sobel(obs_nonbnd, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / np.atleast_3d(cv2.Sobel(row(np.arange(obs_nonbnd.shape[1])).astype(np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize)) # ydiffnb = -cv2.Sobel(obs_nonbnd, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / np.atleast_3d(cv2.Sobel(col(np.arange(obs_nonbnd.shape[0])).astype(np.float64), cv2.CV_64F, dx=0, dy=1, ksize=ksize)) # # xdiffnb.ravel()[np.isnan(xdiffnb.ravel())] = 0. # ydiffnb.ravel()[np.isnan(ydiffnb.ravel())] = 0. # xdiffnb.ravel()[np.isinf(xdiffnb.ravel())] = 0. # ydiffnb.ravel()[np.isinf(ydiffnb.ravel())] = 0. # xdiffnb = np.atleast_3d(xdiffnb) # ydiffnb = np.atleast_3d(ydiffnb) # # xdiffbnd = -cv2.Sobel(observed, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / sobel_normalizer # ydiffbnd = -cv2.Sobel(observed, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / sobel_normalizer # # xdiff = xdiffnb * np.atleast_3d(nbndf) # xdiff.ravel()[np.isnan(xdiff.ravel())] = 0 # xdiff += xdiffbnd*np.atleast_3d(bndf) # # ydiff = ydiffnb * np.atleast_3d(nbndf) # ydiff.ravel()[np.isnan(ydiff.ravel())] = 0 # ydiff += ydiffbnd*np.atleast_3d(bndf) #import pdb; pdb.set_trace() #xdiff = xdiffnb #ydiff = ydiffnb #import pdb; pdb.set_trace() datas = [] # The data is weighted according to barycentric coordinates bc0 = col(barycentric[pys, pxs, 0]) bc1 = col(barycentric[pys, pxs, 1]) bc2 = col(barycentric[pys, pxs, 2]) for k in range(n_channels): dxs = xdiff[pys, pxs, k] dys = ydiff[pys, pxs, k] if f.shape[1] == 3: datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1,col(dxs)*bc2,col(dys)*bc2)).ravel()) else: datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1)).ravel()) data = np.concatenate(datas) ij = np.vstack((IS.ravel(), JS.ravel())) result = sp.csc_matrix((data, ij), shape=(image_width*image_height*n_channels, num_verts*2)) return result
def compute_dr_wrt(self, wrt): if wrt is self.x: return row(self.r * np.linalg.inv(self.x.r).T)
def dImage_wrt_2dVerts_bnd(observed, visible, visibility, barycentric, image_width, image_height, num_verts, f, bnd_bool): """Construct a sparse jacobian that relates 2D projected vertex positions (in the columns) to pixel values (in the rows). This can be done in two steps.""" n_channels = np.atleast_3d(observed).shape[2] shape = visibility.shape # Step 1: get the structure ready, ie the IS and the JS IS = np.tile(col(visible), (1, 2*f.shape[1])).ravel() JS = col(f[visibility.ravel()[visible]].ravel()) JS = np.hstack((JS*2, JS*2+1)).ravel() pxs = np.asarray(visible % shape[1], np.int32) pys = np.asarray(np.floor(np.floor(visible) / shape[1]), np.int32) if n_channels > 1: IS = np.concatenate([IS*n_channels+i for i in range(n_channels)]) JS = np.concatenate([JS for i in range(n_channels)]) # Step 2: get the data ready, ie the actual values of the derivatives ksize = 1 bndf = bnd_bool.astype(np.float64) nbndf = np.logical_not(bnd_bool).astype(np.float64) sobel_normalizer = cv2.Sobel(np.asarray(np.tile(row(np.arange(10)), (10, 1)), np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize)[5,5] bnd_nan = bndf.reshape((observed.shape[0], observed.shape[1], -1)).copy() bnd_nan.ravel()[bnd_nan.ravel()>0] = np.nan bnd_nan += 1 obs_nonbnd = np.atleast_3d(observed) * bnd_nan ydiffnb, xdiffnb = nangradients(obs_nonbnd) observed = np.atleast_3d(observed) if observed.shape[2] > 1: ydiffbnd, xdiffbnd, _ = np.gradient(observed) else: ydiffbnd, xdiffbnd = np.gradient(observed.squeeze()) ydiffbnd = np.atleast_3d(ydiffbnd) xdiffbnd = np.atleast_3d(xdiffbnd) # This corrects for a bias imposed boundary differences begin spread over two pixels # (by np.gradients or similar) but only counted once (since OpenGL's line # drawing spans 1 pixel) xdiffbnd *= 2.0 ydiffbnd *= 2.0 xdiffnb = -xdiffnb ydiffnb = -ydiffnb xdiffbnd = -xdiffbnd ydiffbnd = -ydiffbnd # ydiffnb *= 0 # xdiffnb *= 0 if False: import matplotlib.pyplot as plt plt.figure() plt.subplot(121) plt.imshow(xdiffnb) plt.title('xdiffnb') plt.subplot(122) plt.imshow(xdiffbnd) plt.title('xdiffbnd') # import pdb; pdb.set_trace() idxs = np.isnan(xdiffnb.ravel()) xdiffnb.ravel()[idxs] = xdiffbnd.ravel()[idxs] idxs = np.isnan(ydiffnb.ravel()) ydiffnb.ravel()[idxs] = ydiffbnd.ravel()[idxs] if True: # should be right thing xdiff = xdiffnb ydiff = ydiffnb else: #should be old way xdiff = xdiffbnd ydiff = ydiffbnd # TODO: NORMALIZER IS WRONG HERE # xdiffnb = -cv2.Sobel(obs_nonbnd, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / np.atleast_3d(cv2.Sobel(row(np.arange(obs_nonbnd.shape[1])).astype(np.float64), cv2.CV_64F, dx=1, dy=0, ksize=ksize)) # ydiffnb = -cv2.Sobel(obs_nonbnd, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / np.atleast_3d(cv2.Sobel(col(np.arange(obs_nonbnd.shape[0])).astype(np.float64), cv2.CV_64F, dx=0, dy=1, ksize=ksize)) # # xdiffnb.ravel()[np.isnan(xdiffnb.ravel())] = 0. # ydiffnb.ravel()[np.isnan(ydiffnb.ravel())] = 0. # xdiffnb.ravel()[np.isinf(xdiffnb.ravel())] = 0. # ydiffnb.ravel()[np.isinf(ydiffnb.ravel())] = 0. # xdiffnb = np.atleast_3d(xdiffnb) # ydiffnb = np.atleast_3d(ydiffnb) # # xdiffbnd = -cv2.Sobel(observed, cv2.CV_64F, dx=1, dy=0, ksize=ksize) / sobel_normalizer # ydiffbnd = -cv2.Sobel(observed, cv2.CV_64F, dx=0, dy=1, ksize=ksize) / sobel_normalizer # # xdiff = xdiffnb * np.atleast_3d(nbndf) # xdiff.ravel()[np.isnan(xdiff.ravel())] = 0 # xdiff += xdiffbnd*np.atleast_3d(bndf) # # ydiff = ydiffnb * np.atleast_3d(nbndf) # ydiff.ravel()[np.isnan(ydiff.ravel())] = 0 # ydiff += ydiffbnd*np.atleast_3d(bndf) #import pdb; pdb.set_trace() #xdiff = xdiffnb #ydiff = ydiffnb #import pdb; pdb.set_trace() datas = [] # The data is weighted according to barycentric coordinates bc0 = col(barycentric[pys, pxs, 0]) bc1 = col(barycentric[pys, pxs, 1]) bc2 = col(barycentric[pys, pxs, 2]) for k in range(n_channels): dxs = xdiff[pys, pxs, k] dys = ydiff[pys, pxs, k] if f.shape[1] == 3: datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1,col(dxs)*bc2,col(dys)*bc2)).ravel()) else: datas.append(np.hstack((col(dxs)*bc0,col(dys)*bc0,col(dxs)*bc1,col(dys)*bc1)).ravel()) data = np.concatenate(datas) ij = np.vstack((IS.ravel(), JS.ravel())) result = sp.csc_matrix((data, ij), shape=(image_width*image_height*n_channels, num_verts*2)) return result
def compute_dr_wrt(self, wrt): if wrt is self.x: return row(self.x.r.ravel() * 2.)