def test_spectrogram(): # http://matplotlib.org/examples/pylab_examples/specgram_demo.html dt = 1./0.0005 t = np.arange(0., 20., dt) #t = np.arange(0., 3., dt) s1 = np.sin((2*np.pi)*100*t) s2 = 2 * np.sin((2*np.pi)*400*t) s2[-((10 < t) & (t < 12))] = 0 nse = 0.01 * np.random.randn(len(t)) if 0: x = s1 else: x = s1 + s2 + nse freqs, spec, spec_times = make_specgram(x, dt) pl.clf() ax1 = pl.subplot(211) ax1.plot(t, x) if 1: lsp = spec.copy() lsp[spec > 0] = np.log(spec[spec > 0]) lsp = ut.clip_rescale(lsp, -10, np.percentile(lsp, 99)) else: lsp = spec.copy() lsp = ut.clip_rescale(lsp, 0, np.percentile(lsp, 99)) ax2 = pl.subplot(212, sharex = ax1) ax2.imshow(lsp.T, cmap = pl.cm.jet, extent = (0., t[-1], np.min(freqs), np.max(freqs)), aspect = 'auto') ig.show(vis_specgram(freqs, spec, spec_times)) ut.toplevel_locals()
def interior_mrf(scan, mesh, shift_dim=1, shift_dist=30., data_weight=1., full_occ=0, occ_border=None, occ_weight=0.5): """ Runs the Interior MRF camouflage method """ # controls the importance of smoothness vs. the local evidence data_weight = 1. / 4 * (64. / mesh.texsize) geom = Geom(scan, mesh) labels = make_labels(scan, shift_dim, shift_dist) label_color, label_valid = label_colors(scan, mesh, geom, labels, invisible_colors=True) stable_cost = stability_costs(scan, mesh, labels) occ_cost = occ_weight * occlusion_costs(scan, mesh, labels, geom) face_weight = np.ones(mesh.nfaces) node_visible = np.any(label_valid, axis=1) fw = face_weight[mesh.tex2juv[:, 0]][:, na] data_cost = np.array(fw * (occ_cost + stable_cost), 'float32') data_cost[-label_valid] = 1e5 # no invalid label? any label will do data_cost[-node_visible] = 0 data_cost *= data_weight m = MRF() m.edges = make_graph(mesh).edges() m.data_cost = data_cost m.labels = labels m.label_color = label_color m.node_visible = node_visible m.sameview_prior = np.zeros(mesh.ntexels, 'float32') m.smooth_prior = 1. results = mrf.solve_mrf(m) assert np.all( np.logical_or(-node_visible, label_valid[range(len(results)), results])) colors = from_color_space_2d(label_color[range(len(results)), results]) ut.toplevel_locals() occ_total = np.sum((fw * occ_cost)[range(len(results)), results]) stable_total = np.sum((fw * stable_cost)[range(len(results)), results]) print 'Occlusion cost:', occ_total print 'Stability cost:', stable_total return colors, results, labels, (occ_total, stable_total)
def test_box(path, inds = None, hires = 1, as_cycle = False, label_faces = False, mesh = None): scan = dset.Scan(path) if mesh is None: mesh = load_from_mat(os.path.join(path, 'cube.mat')) ut.toplevel_locals() if inds is None: inds = range(scan.length) ims = [draw_faces(mesh, scan, i, hires = hires, label_faces = label_faces) for i in inds] if as_cycle: ig.show([('cycle', ims)]) else: ig.show(ims)
def test_plane_fitting(): noise = 0.0 plane1 = np.array([0., 1./2**0.5, 1/2**0.5, 1.]) plane2 = np.array([1., 0., 0., 5.]) pts1 = sample_plane(plane1, 100, 1, noise) pts2 = sample_plane(plane2, 20, 1, noise) pts = np.vstack([pts1, pts2]) plane, _ = fit_plane_ransac(pts, 0.05) #plane = fit_plane(pts1) print plane, 'should be', np.array([0., 1., 1., 1.]) true_ins = plane_inliers(plane, pts, 0.05) print 'ninliers', len(true_ins), 'should be', len(plane_inliers(plane1, pts, 0.05)), 'other hypothesis', len(plane_inliers(plane2, pts, 0.05)) ut.toplevel_locals()
def interior_mrf(scan, mesh, shift_dim = 1, shift_dist = 30., data_weight = 1., full_occ = 0, occ_border = None, occ_weight = 0.5): """ Runs the Interior MRF camouflage method """ # controls the importance of smoothness vs. the local evidence data_weight = 1./4*(64./mesh.texsize) geom = Geom(scan, mesh) labels = make_labels(scan, shift_dim, shift_dist) label_color, label_valid = label_colors(scan, mesh, geom, labels, invisible_colors = True) stable_cost = stability_costs(scan, mesh, labels) occ_cost = occ_weight*occlusion_costs(scan, mesh, labels, geom) face_weight = np.ones(mesh.nfaces) node_visible = np.any(label_valid, axis = 1) fw = face_weight[mesh.tex2juv[:, 0]][:, na] data_cost = np.array(fw*(occ_cost + stable_cost), 'float32') data_cost[-label_valid] = 1e5 # no invalid label? any label will do data_cost[-node_visible] = 0 data_cost *= data_weight m = MRF() m.edges = make_graph(mesh).edges() m.data_cost = data_cost m.labels = labels m.label_color = label_color m.node_visible = node_visible m.sameview_prior = np.zeros(mesh.ntexels, 'float32') m.smooth_prior = 1. results = mrf.solve_mrf(m) assert np.all(np.logical_or(-node_visible, label_valid[range(len(results)), results])) colors = from_color_space_2d(label_color[range(len(results)), results]) ut.toplevel_locals() occ_total = np.sum((fw*occ_cost)[range(len(results)), results]) stable_total = np.sum((fw*stable_cost)[range(len(results)), results]) print 'Occlusion cost:', occ_total print 'Stability cost:', stable_total return colors, results, labels, (occ_total, stable_total)
def test_box(path, inds=None, hires=1, as_cycle=False, label_faces=False, mesh=None): scan = dset.Scan(path) if mesh is None: mesh = load_from_mat(os.path.join(path, 'cube.mat')) ut.toplevel_locals() if inds is None: inds = range(scan.length) ims = [ draw_faces(mesh, scan, i, hires=hires, label_faces=label_faces) for i in inds ] if as_cycle: ig.show([('cycle', ims)]) else: ig.show(ims)
def tour(scan, mesh, texel_colors, frames, n=40, im_wait=2.2, plane_idx=2, par=0, outline_start=np.inf, outline_end=np.inf, bad_pairs=[], other_planes=[], mesh_occ=None, start_scale=1.25, start_wo_outline=False): #def tour(scan, mesh, texel_colors, frames, n = 40, im_wait = 2.2, plane_idx = 2, par = 0, bad_pairs = [], other_planes = [], mesh_occ = None): if 0: idx = mesh.face_idx idx = idx[plane_idx:plane_idx + 1] only_plane = box.Mesh(idx, mesh.mesh_pts) ig.show(box.draw_faces(only_plane, scan, 0)) return if mesh_occ is None: mesh_occ = figures.MeshOcc(scan, None) def f(frame1, frame2, n=n, im_wait=im_wait, plane_idx=plane_idx, texel_colors=[texel_colors], scan=scan, mesh=mesh, other_planes=other_planes, bad_pairs=bad_pairs, frames=frames, mesh_occ=mesh_occ, outline_start=outline_start, outline_end=outline_end, start_scale=start_scale, start_wo_outline=start_wo_outline): texel_colors = texel_colors[0] ims = [] ps = np.linspace(0, 1, n) frame1_idx = frames.index(frame1) frame2_idx = frames.index(frame2) for pi, p in enumerate(ps): print p p1 = 1 - p p2 = p R1, c1 = scan.R(frame1), scan.center(frame1) R2, c2 = scan.R(frame2), scan.center(frame2) Rp = slerp_R(R1, R2, p) cp = p1 * c1 + p2 * c2 Kp = p1 * scan.K(frame1) + p2 * scan.K(frame2) tp = -np.dot(Rp, cp) Pp = mvg.compose_P(Kp, Rp, tp) tc = texel_colors.copy() if outline_start <= frame1_idx < outline_end: # or outline_start <= frame2_idx < outline_end: if frame2_idx == outline_end: interp_occ = p1 else: interp_occ = 1. print 'interp_occ =', interp_occ scan_with_interp, frame_interp = scan.add_virtual_camera( Kp, Rp, dset.t_from_Rc(Rp, cp), scan.im(frame1)) frame_round = (frame1 if p <= 0.5 else frame1) tc = texel_colors.copy() tc = figures.mark_occlusion_texels( tc, scan_with_interp, mesh, frame_interp, thresh=2, p=interp_occ, mesh_occ_mask=np.array( np.round(mesh_occ.mask(frame_round)), 'bool')) if pi == 0 or pi == len(ps) - 1: frame = (frame1 if pi == 0 else frame2) #tc = figures.mark_occlusion_texels(tc, scan, mesh, frame, mesh_occ_mask = np.array(np.round(mesh_occ.mask(frame)), 'bool'), thresh = 2) if frame == frames[0]: # 2x brings it on par with a normal frame (which are counted twice); add another factor to # give people a chance to scan wait_time = int(start_scale * 2 * im_wait * n) elif frame == frames[-1]: wait_time = int(2 * im_wait * n) else: wait_time = int(im_wait * n) if start_wo_outline: ims += ([mesh.render(scan, frame, texel_colors)] * 3) im = mesh.render(scan, frame, tc) im0 = scan.im(frame) ims += [mesh_occ.apply_mask(im, im0, mesh_occ.mask(frame)) ] * wait_time else: im1 = scan.im(frame1) im2 = scan.im(frame2) planes = [mesh.face_planes[plane_idx]] + other_planes #print other_planes best_dists = None for plane in planes: # backproject pixel into both frames, using the plane as the geometry # average the colors im = np.zeros_like(im1) #print plane ray_dirs = ut.col_from_im( ut.mult_im(Rp.T, mvg.ray_directions(Kp, im.shape))) dists = (-plane[3] - np.dot(cp, plane[:3])) / np.dot( ray_dirs, plane[:3]) if best_dists is None: best_dists = dists else: # asdf #best_dists[dists < best_dists] = 0 if 1: ok = ((best_dists < 0) & (dists >= 0)) | ((dists >= 0) & (dists < best_dists)) best_dists[ok] = dists[ok] pts = cp + ray_dirs * best_dists[:, np.newaxis] proj1 = scan.project(frame1, pts) proj2 = scan.project(frame2, pts) color1 = ig.lookup_bilinear(im1, proj1[:, 0], proj1[:, 1]) color2 = ig.lookup_bilinear(im2, proj2[:, 0], proj2[:, 1]) in_bounds1 = ig.lookup_bilinear(np.ones(im1.shape[:2]), proj1[:, 0], proj1[:, 1]) > 0 in_bounds2 = ig.lookup_bilinear(np.ones(im2.shape[:2]), proj2[:, 0], proj2[:, 1]) > 0 p1s = np.array([p1] * len(proj1)) p1s[-in_bounds1] = 0 p2s = np.array([p2] * len(proj2)) p2s[-in_bounds2] = 0 s = p1s + p2s p1s = p1s / np.maximum(s, 0.00001) p2s = p2s / np.maximum(s, 0.00001) #mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2) #mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2) mask1 = ig.lookup_bilinear(mesh_occ.mask(frame1), proj1[:, 0], proj1[:, 1]) mask2 = ig.lookup_bilinear(mesh_occ.mask(frame2), proj2[:, 0], proj2[:, 1]) mask = ut.im_from_col(im.shape[:2], mask1 * p1s + mask2 * p2s) #mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2) #mask[mask >= 0.5] = 1 if (frame1, frame2) in bad_pairs: assert mesh_occ.path is None ims.append(p1 * mesh.render(scan, frame1, tc, mask=mask) + p2 * mesh.render(scan, frame2, tc, mask=mask)) else: im = ut.im_from_col( im.shape, color1 * p1s[:, np.newaxis] + color2 * p2s[:, np.newaxis]) ims.append( mesh.render(scan, (Rp, Pp, Kp, cp), tc, im=im, mask=mask)) return ims ims = ut.flatten(ip.map(par, f, (frames[:-1], frames[1:]))) #ut.toplevel_locals() #ig.show([('animation', ims, 100*10./n)]) #ig.show([('animation', ims, 8*10./n)]) #ig.show([imtable.Video(ims, fps = 0.25*float(im_wait*n))]) #ig.show([imtable.Video(ims, fps = 0.75*float(im_wait*n))]) ut.toplevel_locals() url = ig.show([imtable.Video(ims, fps=0.75 * float(im_wait * n))]) del ims return url
def boundary_mrf(scan, mesh, shift_dim = 0, shift_dist = 1, max_stable = MAX_STABLE, occ_weight = 0.5, stable_weight = 1., per_term_stable = np.inf, use_face_mrf = True): """ Run the Boundary MRF model """ geom = Geom(scan, mesh) labels = make_labels(scan, shift_dim, shift_dist) label_color, label_valid = label_colors(scan, mesh, geom, labels) stable_cost = stable_weight*stability_costs(scan, mesh, labels) if occ_weight == 0: print 'no occ cost!' occ_cost = np.zeros((mesh.ntexels, len(labels))) else: occ_cost = occ_weight*occlusion_costs(scan, mesh, labels, geom) node_visible = np.any(label_valid, axis = 1) data_cost = np.array(occ_cost + stable_cost, 'float32') data_cost[-label_valid] = 1e5 # no valid label? any label will do data_cost[-node_visible] = 0 assert np.max(data_cost[label_valid]) <= 1e3 mismatch_cost = 1e5 sameview_prior = 1e7 + np.zeros(mesh.ntexels, 'float32') sameview_prior[mesh.on_border] = mismatch_cost m = MRF() m.edges = make_graph(mesh).edges() m.data_cost = data_cost m.labels = labels m.label_color = label_color m.node_visible = node_visible m.sameview_prior = sameview_prior m.smooth_prior = 0. def en(r): u = np.sum(np.array(data_cost, 'd')[range(len(r)), r]) s = np.sum(mismatched(m, r)*mismatch_cost) # 2* is to deal w/ bug in gc energy estimate print u + 2*s, (u, 2*s) if use_face_mrf: # Use a brute-force solver (written in Cython) results = mrf.solve_face_mrf(mesh, m) print 'Energy for brute-force solver', en(results) assert len(np.unique(results[m.node_visible])) <= 2 if 0: # verify that the brute-force solver gets a better result than alpha-expansion results2 = mrf.solve_mrf(m) print 'Energy for alpha-expansion solver:', en(results2) assert en(results) <= en(results2) else: # Solve using alpha-expansion (requires gco) results = mrf.solve_mrf(m) colors = from_color_space_2d(label_color[range(len(results)), results]) ut.toplevel_locals() occ_total = np.sum(occ_cost[range(len(results)), results]) stable_total = np.sum(stable_cost[range(len(results)), results]) print 'Occlusion cost:', occ_total print 'Stability cost:', stable_total return colors, results, labels, (occ_total, stable_total)
def boundary_mrf(scan, mesh, shift_dim=0, shift_dist=1, max_stable=MAX_STABLE, occ_weight=0.5, stable_weight=1., per_term_stable=np.inf, use_face_mrf=True): """ Run the Boundary MRF model """ geom = Geom(scan, mesh) labels = make_labels(scan, shift_dim, shift_dist) label_color, label_valid = label_colors(scan, mesh, geom, labels) stable_cost = stable_weight * stability_costs(scan, mesh, labels) if occ_weight == 0: print 'no occ cost!' occ_cost = np.zeros((mesh.ntexels, len(labels))) else: occ_cost = occ_weight * occlusion_costs(scan, mesh, labels, geom) node_visible = np.any(label_valid, axis=1) data_cost = np.array(occ_cost + stable_cost, 'float32') data_cost[-label_valid] = 1e5 # no valid label? any label will do data_cost[-node_visible] = 0 assert np.max(data_cost[label_valid]) <= 1e3 mismatch_cost = 1e5 sameview_prior = 1e7 + np.zeros(mesh.ntexels, 'float32') sameview_prior[mesh.on_border] = mismatch_cost m = MRF() m.edges = make_graph(mesh).edges() m.data_cost = data_cost m.labels = labels m.label_color = label_color m.node_visible = node_visible m.sameview_prior = sameview_prior m.smooth_prior = 0. def en(r): u = np.sum(np.array(data_cost, 'd')[range(len(r)), r]) s = np.sum(mismatched(m, r) * mismatch_cost) # 2* is to deal w/ bug in gc energy estimate print u + 2 * s, (u, 2 * s) if use_face_mrf: # Use a brute-force solver (written in Cython) results = mrf.solve_face_mrf(mesh, m) print 'Energy for brute-force solver', en(results) assert len(np.unique(results[m.node_visible])) <= 2 if 0: # verify that the brute-force solver gets a better result than alpha-expansion results2 = mrf.solve_mrf(m) print 'Energy for alpha-expansion solver:', en(results2) assert en(results) <= en(results2) else: # Solve using alpha-expansion (requires gco) results = mrf.solve_mrf(m) colors = from_color_space_2d(label_color[range(len(results)), results]) ut.toplevel_locals() occ_total = np.sum(occ_cost[range(len(results)), results]) stable_total = np.sum(stable_cost[range(len(results)), results]) print 'Occlusion cost:', occ_total print 'Stability cost:', stable_total return colors, results, labels, (occ_total, stable_total)
def tour(scan, mesh, texel_colors, frames, n = 40, im_wait = 2.2, plane_idx = 2, par = 0, outline_start = np.inf, outline_end = np.inf, bad_pairs = [], other_planes = [], mesh_occ = None, start_scale = 1.25, start_wo_outline = False): #def tour(scan, mesh, texel_colors, frames, n = 40, im_wait = 2.2, plane_idx = 2, par = 0, bad_pairs = [], other_planes = [], mesh_occ = None): if 0: idx = mesh.face_idx idx = idx[plane_idx:plane_idx+1] only_plane = box.Mesh(idx, mesh.mesh_pts) ig.show(box.draw_faces(only_plane, scan, 0)) return if mesh_occ is None: mesh_occ = figures.MeshOcc(scan, None) def f(frame1, frame2, n = n, im_wait = im_wait, plane_idx = plane_idx, texel_colors = [texel_colors], scan = scan, mesh = mesh, other_planes = other_planes, bad_pairs = bad_pairs, frames = frames, mesh_occ = mesh_occ, outline_start = outline_start, outline_end = outline_end, start_scale = start_scale, start_wo_outline = start_wo_outline): texel_colors = texel_colors[0] ims = [] ps = np.linspace(0, 1, n) frame1_idx = frames.index(frame1) frame2_idx = frames.index(frame2) for pi, p in enumerate(ps): print p p1 = 1-p p2 = p R1, c1 = scan.R(frame1), scan.center(frame1) R2, c2 = scan.R(frame2), scan.center(frame2) Rp = slerp_R(R1, R2, p) cp = p1*c1 + p2*c2 Kp = p1*scan.K(frame1) + p2*scan.K(frame2) tp = -np.dot(Rp, cp) Pp = mvg.compose_P(Kp, Rp, tp) tc = texel_colors.copy() if outline_start <= frame1_idx < outline_end:# or outline_start <= frame2_idx < outline_end: if frame2_idx == outline_end: interp_occ = p1 else: interp_occ = 1. print 'interp_occ =', interp_occ scan_with_interp, frame_interp = scan.add_virtual_camera(Kp, Rp, dset.t_from_Rc(Rp, cp), scan.im(frame1)) frame_round = (frame1 if p <= 0.5 else frame1) tc = texel_colors.copy() tc = figures.mark_occlusion_texels(tc, scan_with_interp, mesh, frame_interp, thresh = 2, p = interp_occ, mesh_occ_mask = np.array(np.round(mesh_occ.mask(frame_round)), 'bool')) if pi == 0 or pi == len(ps)-1: frame = (frame1 if pi == 0 else frame2) #tc = figures.mark_occlusion_texels(tc, scan, mesh, frame, mesh_occ_mask = np.array(np.round(mesh_occ.mask(frame)), 'bool'), thresh = 2) if frame == frames[0]: # 2x brings it on par with a normal frame (which are counted twice); add another factor to # give people a chance to scan wait_time = int(start_scale*2*im_wait*n) elif frame == frames[-1]: wait_time = int(2*im_wait*n) else: wait_time = int(im_wait*n) if start_wo_outline: ims += ([mesh.render(scan, frame, texel_colors)]*3) im = mesh.render(scan, frame, tc) im0 = scan.im(frame) ims += [mesh_occ.apply_mask(im, im0, mesh_occ.mask(frame))]*wait_time else: im1 = scan.im(frame1) im2 = scan.im(frame2) planes = [mesh.face_planes[plane_idx]] + other_planes #print other_planes best_dists = None for plane in planes: # backproject pixel into both frames, using the plane as the geometry # average the colors im = np.zeros_like(im1) #print plane ray_dirs = ut.col_from_im(ut.mult_im(Rp.T, mvg.ray_directions(Kp, im.shape))) dists = (-plane[3] - np.dot(cp, plane[:3]))/np.dot(ray_dirs, plane[:3]) if best_dists is None: best_dists = dists else: # asdf #best_dists[dists < best_dists] = 0 if 1: ok = ((best_dists < 0) & (dists >= 0)) | ((dists >= 0) & (dists < best_dists)) best_dists[ok] = dists[ok] pts = cp + ray_dirs*best_dists[:, np.newaxis] proj1 = scan.project(frame1, pts) proj2 = scan.project(frame2, pts) color1 = ig.lookup_bilinear(im1, proj1[:, 0], proj1[:, 1]) color2 = ig.lookup_bilinear(im2, proj2[:, 0], proj2[:, 1]) in_bounds1 = ig.lookup_bilinear(np.ones(im1.shape[:2]), proj1[:, 0], proj1[:, 1]) > 0 in_bounds2 = ig.lookup_bilinear(np.ones(im2.shape[:2]), proj2[:, 0], proj2[:, 1]) > 0 p1s = np.array([p1]*len(proj1)) p1s[-in_bounds1] = 0 p2s = np.array([p2]*len(proj2)) p2s[-in_bounds2] = 0 s = p1s + p2s p1s = p1s / np.maximum(s, 0.00001) p2s = p2s / np.maximum(s, 0.00001) #mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2) #mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2) mask1 = ig.lookup_bilinear(mesh_occ.mask(frame1), proj1[:, 0], proj1[:, 1]) mask2 = ig.lookup_bilinear(mesh_occ.mask(frame2), proj2[:, 0], proj2[:, 1]) mask = ut.im_from_col(im.shape[:2], mask1*p1s + mask2*p2s) #mask = p1*mesh_occ.mask(frame1) + p2*mesh_occ.mask(frame2) #mask[mask >= 0.5] = 1 if (frame1, frame2) in bad_pairs: assert mesh_occ.path is None ims.append(p1*mesh.render(scan, frame1, tc, mask = mask) + p2*mesh.render(scan, frame2, tc, mask = mask)) else: im = ut.im_from_col(im.shape, color1*p1s[:, np.newaxis] + color2*p2s[:, np.newaxis]) ims.append(mesh.render(scan, (Rp, Pp, Kp, cp), tc, im = im, mask = mask)) return ims ims = ut.flatten(ip.map(par, f, (frames[:-1], frames[1:]))) #ut.toplevel_locals() #ig.show([('animation', ims, 100*10./n)]) #ig.show([('animation', ims, 8*10./n)]) #ig.show([imtable.Video(ims, fps = 0.25*float(im_wait*n))]) #ig.show([imtable.Video(ims, fps = 0.75*float(im_wait*n))]) ut.toplevel_locals() url = ig.show([imtable.Video(ims, fps = 0.75*float(im_wait*n))]) del ims return url