def main(): sm = SceneManager() sm.instantiate(sm) snd_mngr = SoundManager() snd_mngr.instantiate(snd_mngr) pl = Player('player.png', 400) pl.event.register_listener(snd_mngr.event_listener) pl.instantiate(pl, pygame.math.Vector2(0, 0), 0) item = Item() item.event.register_listener(pl.event_listener) item.instantiate(item) while (True): display.screen.fill((0, 50, 0)) gb.updateAll() event = pygame.event.poll() if event.type == pygame.QUIT: break #text = text_field.create_text("Hello, World", ["comicsansms"], 72, (0, 128, 0)) #display.screen.blit(text, #(320 - text.get_width() // 2, 240 - text.get_height() // 2)) pygame.display.flip() utils.clock.tick(60)
class App: def __init__(self): pyxel.init(256, 224, palette=[ 0x000000, 0x1D2B53, 0x7E2553, 0x008751, 0xAB5236, 0x5F574F, 0xC2C3C7, 0xFFF1E8, 0xFF004D, 0xFFA300, 0xFFEC27, 0x00E436, 0x29ADFF, 0x83769C, 0xFF77A8, 0xFFCCAA ]) pyxel.image(0).load(0, 0, os.path.join(os.getcwd(), 'asset/system_neko.bmp')) im = Image.open(os.path.join(os.getcwd(), 'asset/font.png')) rgb_im = im.convert('RGB') self.font = Font(rgb_im) self.scene_manager = SceneManager(self) pyxel.run(self.update, self.draw) def update(self): if pyxel.btnp(pyxel.KEY_Q): pyxel.quit() self.scene_manager.update() def draw(self): self.scene_manager.draw()
def __init__(self): pygame.init() resolution = (800, 400) self.display = pygame.display.set_mode(resolution) self.scene_manager = SceneManager(self.display, resolution) pygame.display.set_caption('Pong')
def load_scene_manager(scenes, initial=None): scene_manager = SceneManager(scenes) for scene in scenes.values(): scene.scene_manager = scene_manager if initial: scene_manager.switch_scene(initial) return scene_manager
def __onClickExcute(self): if not self.firstClick: p.changeDynamics(self.colId, -1, 10) self.base_rotate = [0, 0, 1, 0] self.col_local_pos = [0, 0, 0] self.__ChangeStatus(STATUS_FLY) currentScene = SceneManager.GetCurrentScene() multi = currentScene.speedButton.GetCurrentTickZone() currentScene.SetState("STATE_PLAYING") currentScene.pauseButton.Display() self.firstClick = True data = self.GetData() if data: farMul, highMul = data['baseFarMultiply'], data[ 'baseHighMultiply'] else: farMul, highMul = 1.0, 1.0 farMul = farMul * multi highMul = highMul * 0.5 * multi pos, orn = p.getBasePositionAndOrientation(self.colId) force = [0, -self.camDis[1] * 10000 * farMul, 20000 * highMul] p.applyExternalForce(self.colId, -1, force, pos, flags=p.WORLD_FRAME)
def OnClickExcute(self): currentScene = SceneManager.GetCurrentScene() if currentScene.state == "STATE_PAUSE": currentScene.SetState("STATE_PLAYING") elif currentScene.state == "STATE_PLAYING": currentScene.SetState("STATE_PAUSE") else: return
def main(): Logger.instance().log_debug("main"); Config.instance().init(); Camera.instance().init(); SceneManager.instance().init(); SceneManager.instance().run(); SceneManager.instance().quit(); Camera.instance().quit();
def DisplayStageResult(self): percentDestroy = self.finalScene.GetDestroyPercent() if percentDestroy < 0.5: rate = "UNCLEAR" elif percentDestroy < 0.6: rate = "ONE_STAR_RATE" elif percentDestroy < 0.8: rate = "TWO_STAR_RATE" else: rate = "THREE_STAR_RATE" self.endStageCanvas.Display(rate) self.percentDestroy = 0 currentScene = SceneManager.GetCurrentScene() currentScene.HideUI() if os.path.exists("TestVoxel/currentStageResult.pickle"): os.remove("TestVoxel/currentStageResult.pickle")
def __init__(self): Solstice._platform_specific_init() self._config = Configuration() self._translations_init() self._pygame_init() self._screen = Screen(self._config, _('Solstice')) self._control = Control(self) # TODO: Change data file name (don't use .zip extension) self._resource_manager = ResourceManager(self, 'data.zip') self._sound_player = SoundPlayer(self) self._scenes = { 'logo': LogoScene(self), 'intro': IntroScene(self), 'game': GameScene(self), 'elevator': ElevatorScene(self) } # The first parameter is the game 'context' self._scene_manager = SceneManager(self, 'logo')
def OnClickExcute(self): SceneManager.GetCurrentScene().SetState("STATE_RESET")
import game_globals as glb from scene_manager import SceneManager from menu_scene import * from game_scene import * from splash import * if __name__ == '__main__': pygame.init() pygame.font.init() pygame.mouse.set_visible(0) scenes = [ main_menu, options_menu, difficulty_menu, game_scene, you_loose_splash_screen, you_win_splash_screen, final_splash_screen ] # create scene manager scene_manager = SceneManager(glb.WIDTH, glb.HEIGHT, glb.FPS) # build and register scenes for scene in scenes: if scene != game_scene: # we build main game scene later, only when difficulty is chosen scene.build() scene.bind_scene_manager(scene_manager) # add starting scene and launch game loop scene_manager.push_scene(main_menu) scene_manager.main_loop()
import pybullet as p import time import pybullet_data from pyxie.apputil import graphicsHelper import pyxie import pyvmath as vmath import math import random from game_scene import GameScene from scene_manager import SceneManager import imgui import sys SCREEN_WIDTH = 520 SCREEN_HEIGHT = 900 pyxie.window(True, SCREEN_WIDTH , SCREEN_HEIGHT) imgui.create_context() gameScene = GameScene() SceneManager.SetCurrentScene(gameScene) while(1): SceneManager.GetCurrentScene().Update() SceneManager.GetCurrentScene().Render() pyxie.swap() # game loop
def main(): scenemanager = SceneManager() scenemanager.loop()
def run(image_name, image_path, colmap_folder, out_folder, min_track_len, min_tri_angle, max_tri_angle, ref_surf_name, max_num_points=None, estimate_falloff=True, use_image_weighted_derivatives=True, get_initial_warp=True): min_track_len = int(min_track_len) min_tri_angle = int(min_tri_angle) max_tri_angle = int(max_tri_angle) #est_global_ref.estimate_overall_ref_model(colmap_folder,min_track_len,min_tri_angle,max_tri_angle,image_path) try: max_num_points = int(max_num_points) except: max_num_points = None get_initial_warp = (get_initial_warp not in ('False', 'false')) estimate_falloff = (estimate_falloff not in ('False', 'false')) use_image_weighted_derivatives = (use_image_weighted_derivatives not in ('False', 'false')) #if not image_path.endswith('/'): # image_path += '/' print 'Loading COLMAP data' scene_manager = SceneManager(colmap_folder) scene_manager.load_cameras() scene_manager.load_images() image_id = scene_manager.get_image_id_from_name(image_name) image = scene_manager.images[image_id] camera = scene_manager.get_camera(image.camera_id) # image pose R = util.quaternion_to_rotation_matrix(image.qvec) print 'Loading 3D points' scene_manager.load_points3D() scene_manager.filter_points3D(min_track_len, min_tri_angle=min_tri_angle, max_tri_angle=max_tri_angle, image_list=set([image_id])) points3D, points2D = scene_manager.get_points3D(image_id) points3D = points3D.dot(R.T) + image.tvec[np.newaxis, :] # need to remove redundant points # http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array points2D_view = np.ascontiguousarray(points2D).view( np.dtype((np.void, points2D.dtype.itemsize * points2D.shape[1]))) _, idx = np.unique(points2D_view, return_index=True) points2D, points3D = points2D[idx], points3D[idx] # further rule out any points too close to the image border (for bilinear # interpolation) mask = ((points2D[:, 0] < camera.width - 1) & (points2D[:, 1] < camera.height - 1)) points2D, points3D = points2D[mask], points3D[mask] points2D_image = points2D.copy() # coordinates in image space points2D = np.hstack((points2D, np.ones((points2D.shape[0], 1)))) points2D = points2D.dot(np.linalg.inv(camera.get_camera_matrix()).T)[:, :2] if (max_num_points is not None and max_num_points > 0 and max_num_points < len(points2D)): np.random.seed(0) # fix the "random" points selected selected_points = np.random.choice(len(points2D), max_num_points, False) points2D = points2D[selected_points] points2D_image = points2D_image[selected_points] points3D = points3D[selected_points] print len(points3D), 'total points' #perturb points #import pdb;pdb.set_trace() #perturb_points=np.random.choice(len(points2D),max_num_points*0.3,False) #randomperturb=np.random.rand(perturb_points.size)*2 #points3D[perturb_points,2]=points3D[perturb_points,2]*randomperturb #points3D[2][2]=points3D[2][2]*0.5 #points3D[4][2]=points3D[4][2]*1.5 #points3D[8][2]=points3D[8][2]*0.3 #points3D[9][2]=points3D[9][2]*2. # load image #image_file = scene_manager.image_path + image.name image_file = image_path + image.name im_rgb = skimage.img_as_float(skimage.io.imread(image_file)) # color image L = skimage.color.rgb2lab(im_rgb)[:, :, 0] * 0.01 #import pdb;pdb.set_trace() #skimage.io.imsave('test.png',L) L = np.maximum(L, 1e-6) # unfortunately, can't have black pixels, since we # divide by L # initial values on unit sphere x, y = camera.get_image_grid() print 'Computing nearest neighbors' #import pdb;pdb.set_trace() nn_idxs, weights = compute_nn(points2D.astype(np.float32), camera.width, camera.height, camera.fx, camera.fy, camera.cx, camera.cy) nn_idxs = np.swapaxes(nn_idxs, 0, 2) nn_idxs = np.swapaxes(nn_idxs, 0, 1) weights = np.swapaxes(weights, 0, 2) weights = np.swapaxes(weights, 0, 1) #kdtree = KDTree(points2D) #weights, nn_idxs = kdtree.query(np.c_[x.ravel(),y.ravel()], KD_TREE_KNN) #weights = weights.reshape(camera.height, camera.width, KD_TREE_KNN) #nn_idxs = nn_idxs.reshape(camera.height, camera.width, KD_TREE_KNN) # figure out pixel neighborhoods for each point #neighborhoods = [] neighborhood_mask = np.zeros((camera.height, camera.width), dtype=np.bool) for v, u in points2D_image: rr, cc = circle(int(u), int(v), MAX_POINT_DISTANCE, (camera.height, camera.width)) #neighborhoods.append((rr, cc)) neighborhood_mask[rr, cc] = True # turn distances into weights for the nearest neighbors np.exp(-weights, weights) # in-place # create initial surface on unit sphere S0 = np.dstack((x, y, np.ones_like(x))) S0 /= np.linalg.norm(S0, axis=-1)[:, :, np.newaxis] r_fixed = np.linalg.norm(points3D, axis=-1) # fixed 3D depths specular_mask = (L < 1.) vmask = np.zeros_like(S0[:, :, 2]) for k, (u, v) in enumerate(points2D_image): j0, i0 = int(u), int(v) # upper left pixel for the (u,v) coordinate vmask[i0, j0] = points3D[k, 2] # iterative SFS algorithm # model_type: reflectance model type # fit_falloff: True to fit the 1/r^m falloff parameter; False to fix it at 2 # extra_params: extra parameters to the reflectance model fit function def run_iterative_sfs(out_file, model_type, fit_falloff, *extra_params): #if os.path.exists(out_file + '_z.bin'): # don't re-run existing results # return if model_type == sfs.LAMBERTIAN_MODEL: model_name = 'LAMBERTIAN_MODEL' elif model_type == sfs.OREN_NAYAR_MODEL: model_name = 'OREN_NAYAR_MODEL' elif model_type == sfs.PHONG_MODEL: model_name = 'PHONG_MODEL' elif model_type == sfs.COOK_TORRANCE_MODEL: model_name = 'COOK_TORRANCE_MODEL' elif model_type == sfs.POWER_MODEL: model_name = 'POWER_MODEL' if model_type == sfs.POWER_MODEL: print '%s_%i' % (model_name, extra_params[0]) else: print model_name S = S0.copy() z = S0[:, :, 2] for iteration in xrange(MAX_NUM_ITER): print 'Iteration', iteration z_old = z if get_initial_warp: #z_gt=util.load_point_ply('C:\\Users\\user\\Documents\\UNC\\Research\\ColonProject\\code\\Rui\\SFS_CPU\\frame0859.jpg_gt.ply') # warp to 3D points if iteration > -1: S, r_ratios = nearest_neighbor_warp( weights, nn_idxs, points2D_image, r_fixed, util.generate_surface(camera, z)) z_est = np.maximum(S[:, :, 2], 1e-6) S = util.generate_surface(camera, z_est) else: z_est = z S = util.generate_surface(camera, z_est) #util.save_sfs_ply('warp' + '.ply', S, im_rgb) #util.save_xyz('test.xyz',points3D); #z=z_est #break #z_est=z else: #import pdb;pdb.set_trace() #z_est = extract_depth_map(camera,ref_surf_name,R,image) z_est = np.fromfile( 'C:\Users\user\Documents\UNC\Research\ColonProject\code\SFS_Program_from_True\endo_evaluation\gt_surfaces\\frame0859.jpg.bin', dtype=np.float32).reshape(camera.height, camera.width) / WORLD_SCALE z_est = z_est.astype(float) #S, r_ratios = nearest_neighbor_warp(weights, nn_idxs, # points2D_image, r_fixed, util.generate_surface(camera, z_est)) z_est = np.maximum(z_est[:, :], 1e-6) #Sworld = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R) S = util.generate_surface(camera, z_est) #util.save_sfs_ply('test' + '.ply', S, im_rgb) #util.save_sfs_ply(out_file + '_warp_%i.ply' % iteration, Sworld, im_rgb) #import pdb;pdb.set_trace() # if we need to, make corrections for non-positive depths #S = util.generate_surface(camera, z_est) mask = (z_est < INIT_Z) specular_mask = (L < 0.8) dark_mask = (L > 0.1) _mask = np.logical_and(specular_mask, mask) _mask = np.logical_and(_mask, dark_mask) # fit reflectance model r = np.linalg.norm(S, axis=-1) ndotl = util.calculate_ndotl(camera, S) falloff, model_params, residual = fit_reflectance_model( model_type, L[_mask], r.ravel(), r[_mask], ndotl.ravel(), ndotl[_mask], fit_falloff, camera.width, camera.height, *extra_params) #r = np.linalg.norm(S[specular_mask], axis=-1) #import pdb;pdb.set_trace() #model_params=np.array([26.15969874,-27.674055,-12.52426,7.579855,21.9768004,24.3911142,-21.7282996,-19.850894,-11.62229,-4.837014]) #model_params=np.array([-19.4837,-490.4796,812.4527,-426.09107,139.2602,351.8061,-388.1591,875.5013,-302.4748,-414.4384]) #falloff = 1.2 #ndotl = util.calculate_ndotl(camera, S)[specular_mask] #falloff, model_params, residual = fit_reflectance_model(model_type, # L[specular_mask], r, ndotl, fit_falloff, *extra_params) #r = np.linalg.norm(S[neighborhood_mask], axis=-1) #ndotl = util.calculate_ndotl(camera, S)[neighborhood_mask] #falloff, model_params, residual = fit_reflectance_model(model_type, # L[neighborhood_mask], r, ndotl, fit_falloff, *extra_params) # lambda values reflect our confidence in the current surface: 0 # corresponds to only using SFS at a pixel, 1 corresponds to equally # weighting SFS and the current estimate, and larger values increasingly # favor using only the current estimate rdiff = np.abs(r_fixed - get_estimated_r(S, points2D_image)) w = np.log10(r_fixed) - np.log10(rdiff) - np.log10(2.) lambdas = (np.sum(weights * w[nn_idxs], axis=-1) / np.sum(weights, axis=-1)) lambdas = np.maximum(lambdas, 0.) # just in case # lambdas[~mask] = 0 #if iteration == 0: # don't use current estimated surface on first pass #lambdas = np.zeros_like(z) #else: # r_ratios_postwarp = r_fixed / get_estimated_r(S, points2D_image) # ratio_diff = np.abs(r_ratios_prewarp - r_ratios_postwarp) # ratio_diff[ratio_diff == 0] = 1e-10 # arbitrarily high lambda # feature_lambdas = 1. / ratio_diff # lambdas = (np.sum(weights * feature_lambdas[nn_idxs], axis=-1) / # np.sum(weights, axis=-1)) # run SFS H_lut, dH_lut = compute_H_LUT(model_type, model_params, NUM_H_LUT_BINS) #import pdb;pdb.set_trace() # run SFS #H_lut = np.ascontiguousarray(H_lut.astype(np.float32)) #dH_lut = np.ascontiguousarray(dH_lut.astype(np.float32)) z = run_sfs(H_lut, dH_lut, camera, L, lambdas, z_est, model_type, model_params, falloff, vmask, use_image_weighted_derivatives) # check for convergence #diff = np.sum(np.abs(z_old[specular_mask] - z[specular_mask])) #if diff < CONVERGENCE_THRESHOLD * camera.height * camera.width: # break # save the surface #S = util.generate_surface(camera, z) #S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R) #util.save_sfs_ply(out_file + '_%i.ply' % iteration, S, im_rgb) else: print 'DID NOT CONVERGE' #import pdb;pdb.set_trace() S = util.generate_surface(camera, z) #S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R) util.save_sfs_ply(out_file + '.ply', S, im_rgb) z.astype(np.float32).tofile(out_file + '_z.bin') # save the surface #S = util.generate_surface(camera, z) #S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R) #S, r_ratios = nearest_neighbor_warp(weights, nn_idxs, # points2D_image, r_fixed, util.generate_surface(camera, z)) #util.save_sfs_ply(out_file + '_warped.ply', S, im_rgb) #z = np.maximum(S[:,:,2], 1e-6) #z.astype(np.float32).tofile(out_file + '_warped_z.bin') #reflectance_models.save_reflectance_model(out_file + '_reflectance.txt', # model_name, residual, model_params, falloff, *extra_params) print # now, actually run SFS if not out_folder.endswith('/'): out_folder += '/' if not os.path.exists(out_folder): os.makedirs(out_folder) # if not os.path.exists(out_folder + 'lambertian/'): # os.mkdir(out_folder + 'lambertian/') # run_iterative_sfs(out_folder + 'lambertian/' + image.name, # sfs.LAMBERTIAN_MODEL, estimate_falloff) # if not os.path.exists(out_folder + 'oren_nayar/'): # os.mkdir(out_folder + 'oren_nayar/') # run_iterative_sfs(out_folder + 'oren_nayar/' + image.name, # sfs.OREN_NAYAR_MODEL, estimate_falloff) # if not os.path.exists(out_folder + 'phong/'): # os.mkdir(out_folder + 'phong/') # run_iterative_sfs(out_folder + 'phong/' + image.name, sfs.PHONG_MODEL, # estimate_falloff) # if not os.path.exists(out_folder + 'cook_torrance/'): # os.mkdir(out_folder + 'cook_torrance/') # run_iterative_sfs(out_folder + 'cook_torrance/' + image.name, # sfs.COOK_TORRANCE_MODEL, estimate_falloff) #for K in [1, 2, 3, 4, 5, 10, 20, 50, 100]: #for K in [10, 20, 50]: #for K in [1, 2, 3, 4, 5]: for K in [100]: #if not os.path.exists(out_folder + 'power_model_%i/' % K): # os.mkdir(out_folder + 'power_model_%i/' % K) run_iterative_sfs(out_folder + image.name, sfs.POWER_MODEL, estimate_falloff, K)
def run(colmap_folder, sfs_results_path, gt_path, out_folder): #import pdb;pdb.set_trace(); if not sfs_results_path.endswith('/'): sfs_results_path += '/' if not gt_path.endswith('/'): gt_path += '/' if not out_folder.endswith('/'): out_folder += '/' scene_manager = SceneManager(colmap_folder) scene_manager.load_cameras() camera = scene_manager.cameras[1] # assume single camera # initial values on unit sphere x, y = camera.get_image_grid() r_scale = np.sqrt(x * x + y * y + 1.) image = np.empty((camera.height, camera.width, 3)) #image[:,:,0] = 1. if not os.path.exists(out_folder): os.mkdir(out_folder) # iterate through rendered BRDFs #for brdf_name in os.listdir(sfs_results_path): for brdf_name in ['phantom']: brdf_folder = sfs_results_path + brdf_name + '/' if not os.path.isdir(brdf_folder): continue with open(out_folder + brdf_name + '.csv', 'w') as f: print>>f, HEADER1 % brdf_name print>>f, HEADER2 # iterate through reflectance models #for rm_name in os.listdir(brdf_folder): #for model_name in MODELS: model_folder = brdf_folder + '/'#model_name + '/' # percent of pixels within 1, 2, 5mm of the GT surface data = [list() for _ in THRESHOLDS] ### datasum = [] ### # iterate through SFS surfaces for filename in os.listdir(model_folder): if not filename.endswith('_z.bin'): continue name = filename[:-6] #import pdb;pdb.set_trace(); util.save_sfs_ply('%s_%s.ply' % (name, 'est'), np.dstack((x, y, np.ones_like(x))) * np.fromfile(model_folder + filename, dtype=np.float32).reshape( camera.height, camera.width, 1)) # util.save_sfs_ply('%s_%s.ply' % (name, 'gt'), np.dstack((x, y, np.ones_like(x)))/WORLD_SCALE * np.fromfile(gt_path + name+'.bin', dtype=np.float32).reshape( camera.height, camera.width, 1)) #break r_est = WORLD_SCALE * r_scale * np.fromfile( model_folder + filename, dtype=np.float32).reshape( camera.height, camera.width) r_gt = r_scale * np.fromfile( gt_path + name + '.bin', dtype=np.float32).reshape( camera.height, camera.width) rdiff = np.abs(r_est - r_gt) #rdiff = np.concatenate(( # rdiff[:,:20].ravel(), rdiff[:,-20:].ravel(), # rdiff[20:-20,:20].ravel(), rdiff[20:-20,-20:].ravel())) #rdiff = rdiff[(r_gt > 5.0) & (r_gt < 20.0)] inv_size = 1. / float(rdiff.size) for i, t in enumerate(THRESHOLDS): data[i].append(np.count_nonzero(rdiff < t) * inv_size) #data[i].append(np.mean((rdiff / r_gt)[rdiff < t])) #data[i].append(np.mean((r_est / r_gt))) ### datasum.append(1. - (r_est / r_gt).ravel()) datasum = np.concatenate(datasum) print np.mean(datasum), np.std(datasum) ### data = np.array(data) data = np.mean(data, axis=1), np.std(data, axis=1) model_name='power' print>>f, model_name + ',' + ','.join('%f,%f' % d for d in izip(*data))
def estimate_overall_ref_model(colmap_folder,min_track_len,min_tri_angle,max_tri_angle,image_path): print 'Loading COLMAP data' scene_manager = SceneManager(colmap_folder) scene_manager.load_cameras() scene_manager.load_images() images=['frame0859.jpg', 'frame0867.jpg', 'frame0875.jpg', 'frame0883.jpg', 'frame0891.jpg'] _L=np.array([]) _r=np.array([]) _ndotl=np.array([]) for image_name in images: image_id = scene_manager.get_image_id_from_name(image_name) image = scene_manager.images[image_id] camera = scene_manager.get_camera(image.camera_id) # image pose R = util.quaternion_to_rotation_matrix(image.qvec) print 'Loading 3D points' scene_manager.load_points3D() scene_manager.filter_points3D(min_track_len, min_tri_angle=min_tri_angle, max_tri_angle=max_tri_angle, image_list=set([image_id])) points3D, points2D = scene_manager.get_points3D(image_id) points3D = points3D.dot(R.T) + image.tvec[np.newaxis,:] # need to remove redundant points # http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array points2D_view = np.ascontiguousarray(points2D).view( np.dtype((np.void, points2D.dtype.itemsize * points2D.shape[1]))) _, idx = np.unique(points2D_view, return_index=True) points2D, points3D = points2D[idx], points3D[idx] # further rule out any points too close to the image border (for bilinear # interpolation) mask = ( (points2D[:,0] < camera.width - 1) & (points2D[:,1] < camera.height - 1)) points2D, points3D = points2D[mask], points3D[mask] points2D_image = points2D.copy() # coordinates in image space points2D = np.hstack((points2D, np.ones((points2D.shape[0], 1)))) points2D = points2D.dot(np.linalg.inv(camera.get_camera_matrix()).T)[:,:2] print len(points3D), 'total points' # load image #image_file = scene_manager.image_path + image.name image_file = image_path + image.name im_rgb = skimage.img_as_float(skimage.io.imread(image_file)) # color image L = skimage.color.rgb2lab(im_rgb)[:,:,0] * 0.01 L = np.maximum(L, 1e-6) # unfortunately, can't have black pixels, since we # divide by L # initial values on unit sphere x, y = camera.get_image_grid() print 'Computing nearest neighbors' kdtree = KDTree(points2D) weights, nn_idxs = kdtree.query(np.c_[x.ravel(),y.ravel()], KD_TREE_KNN) weights = weights.reshape(camera.height, camera.width, KD_TREE_KNN) nn_idxs = nn_idxs.reshape(camera.height, camera.width, KD_TREE_KNN) # turn distances into weights for the nearest neighbors np.exp(-weights, weights) # in-place # create initial surface on unit sphere S0 = np.dstack((x, y, np.ones_like(x))) S0 /= np.linalg.norm(S0, axis=-1)[:,:,np.newaxis] r_fixed = np.linalg.norm(points3D, axis=-1) # fixed 3D depths S = S0.copy() z = S0[:,:,2] S, r_ratios = nearest_neighbor_warp(weights, nn_idxs, points2D_image, r_fixed, util.generate_surface(camera, z)) z_est = np.maximum(S[:,:,2], 1e-6) S = util.generate_surface(camera, z_est) r = np.linalg.norm(S, axis=-1) ndotl = util.calculate_ndotl(camera, S) if _L.size == 0: _L=L.ravel() _r=r.ravel() _ndotl=ndotl.ravel() else: _L=np.append(_L,L.ravel()) _r=np.append(_r,r.ravel()) _ndotl=np.append(_ndotl,ndotl.ravel()) # falloff, model_params, residual = fit_reflectance_model(sfs.POWER_MODEL, _L, _r, _ndotl, False, 5) import pdb;pdb.set_trace();
def run(colmap_folder, sfs_results_path, gt_path, out_folder): #import pdb;pdb.set_trace(); if not sfs_results_path.endswith('/'): sfs_results_path += '/' if not gt_path.endswith('/'): gt_path += '/' if not out_folder.endswith('/'): out_folder += '/' scene_manager = SceneManager(colmap_folder) scene_manager.load_cameras() camera = scene_manager.cameras[1] # assume single camera # initial values on unit sphere x, y = camera.get_image_grid() r_scale = np.sqrt(x * x + y * y + 1.) image = np.empty((camera.height, camera.width, 3)) #image[:,:,0] = 1. if not os.path.exists(out_folder): os.mkdir(out_folder) # iterate through rendered BRDFs #for brdf_name in os.listdir(sfs_results_path): for brdf_name in ['phantom']: brdf_folder = sfs_results_path + brdf_name + '/' if not os.path.isdir(brdf_folder): continue with open(out_folder + brdf_name + '.csv', 'w') as f: print >> f, HEADER1 % brdf_name print >> f, HEADER2 # iterate through reflectance models #for rm_name in os.listdir(brdf_folder): #for model_name in MODELS: model_folder = brdf_folder + '/' #model_name + '/' # percent of pixels within 1, 2, 5mm of the GT surface data = [list() for _ in THRESHOLDS] ### datasum = [] ### # iterate through SFS surfaces for filename in os.listdir(model_folder): if not filename.endswith('_z.bin'): continue name = filename[:-6] #import pdb;pdb.set_trace(); util.save_sfs_ply( '%s_%s.ply' % (name, 'est'), np.dstack((x, y, np.ones_like(x))) * np.fromfile( model_folder + filename, dtype=np.float32).reshape( camera.height, camera.width, 1)) # util.save_sfs_ply( '%s_%s.ply' % (name, 'gt'), np.dstack( (x, y, np.ones_like(x))) / WORLD_SCALE * np.fromfile( gt_path + name + '.bin', dtype=np.float32).reshape( camera.height, camera.width, 1)) #break r_est = WORLD_SCALE * r_scale * np.fromfile( model_folder + filename, dtype=np.float32).reshape( camera.height, camera.width) r_gt = r_scale * np.fromfile(gt_path + name + '.bin', dtype=np.float32).reshape( camera.height, camera.width) rdiff = np.abs(r_est - r_gt) #rdiff = np.concatenate(( # rdiff[:,:20].ravel(), rdiff[:,-20:].ravel(), # rdiff[20:-20,:20].ravel(), rdiff[20:-20,-20:].ravel())) #rdiff = rdiff[(r_gt > 5.0) & (r_gt < 20.0)] inv_size = 1. / float(rdiff.size) for i, t in enumerate(THRESHOLDS): data[i].append(np.count_nonzero(rdiff < t) * inv_size) #data[i].append(np.mean((rdiff / r_gt)[rdiff < t])) #data[i].append(np.mean((r_est / r_gt))) ### datasum.append(1. - (r_est / r_gt).ravel()) datasum = np.concatenate(datasum) print np.mean(datasum), np.std(datasum) ### data = np.array(data) data = np.mean(data, axis=1), np.std(data, axis=1) model_name = 'power' print >> f, model_name + ',' + ','.join('%f,%f' % d for d in izip(*data))
def main(_): with tf.Graph().as_default(): #Load image and label x1 = tf.placeholder(shape=[1, 224, 224, 3], dtype=tf.float32) x2 = tf.placeholder(shape=[1, 224, 224, 3], dtype=tf.float32) gt_x1_depth = tf.placeholder(shape=[1, 224, 224, 1], dtype=tf.float32) img_list = sorted(glob(FLAGS.dataset_dir + '/*.jpg')) pred_poses = tf.placeholder(shape=[1, 4, 4], dtype=tf.float32) intrinsics = tf.placeholder(shape=[1, 3, 3], dtype=tf.float32) tf_points3D = tf.placeholder(shape=[224, 224], dtype=tf.float32) tf_multiply = tf.placeholder(shape=[224, 224], dtype=tf.float32) tf_points2D = tf.placeholder(shape=[None, 2], dtype=tf.float32) #import pdb;pdb.set_trace() m_stack_intrinsics = get_multi_scale_intrinsics( intrinsics, FLAGS.num_scales, 224.0 / 720.0, 224.0 / 240.0) # # Define the model: with tf.name_scope("Prediction"): pred_disp, depth_net_endpoints = disp_net(x1, is_training=True) scale_factor = get_scale_factor(tf_points3D, 1.0 / pred_disp[0][0, :, :, 0], tf_points2D) with tf.name_scope("compute_loss"): pixel_loss = 0 smooth_loss = 0 curr_proj_image = [] for s in range(FLAGS.num_scales): smooth_loss += FLAGS.smooth_weight/(2**s) * \ compute_smooth_loss(pred_disp[s]) curr_src_image = tf.image.resize_area( x1, [int(224 / (2**s)), int(224 / (2**s))]) curr_tgt_image = tf.image.resize_area( x2, [int(224 / (2**s)), int(224 / (2**s))]) curr_gt_x1_depth = tf.image.resize_area( gt_x1_depth, [int(224 / (2**s)), int(224 / (2**s))]) #import pdb;pdb.set_trace() curr_proj_image.append( projective_inverse_warp( curr_tgt_image, #tf.squeeze(1.0/curr_gt_x1_depth,axis=3), tf.squeeze(1.0 / pred_disp[s], axis=3), tf.matmul(pred_poses, scale_factor), #pred_poses, m_stack_intrinsics[:, s, :, :])) curr_proj_error = tf.abs(curr_src_image - curr_proj_image[s]) curr_depth_error = tf.abs(curr_gt_x1_depth - scale_factor[0, 0, 0] * pred_disp[s]) pixel_loss += tf.reduce_mean(curr_proj_error) pixel_loss += tf.reduce_mean( curr_depth_error) * FLAGS.data_weight / (2**s) total_loss = pixel_loss + smooth_loss saver = tf.train.Saver([var for var in tf.model_variables()]) checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate, FLAGS.beta1) global_step = tf.Variable(0, name='global_step', trainable=False) train_op = optimizer.minimize(total_loss, global_step=global_step) with tf.Session() as sess: init = tf.global_variables_initializer() sess.run(init) saver.restore(sess, checkpoint) I1 = pil.open(img_list[0]) I1 = np.array(I1) I1 = cv2.resize(I1, (224, 224), interpolation=cv2.INTER_AREA) I2 = pil.open(img_list[1]) I2 = np.array(I2) I2 = cv2.resize(I2, (224, 224), interpolation=cv2.INTER_AREA) m_intric = np.array([[567.239, 0.0, 376.04], [0.0, 261.757, 100.961], [0.0, 0.0, 1.0]]) z = np.fromfile( '/home/wrlife/project/deeplearning/depth_prediction/backprojtest/oldcase1/frame1819.jpg_z.bin', dtype=np.float32).reshape(FLAGS.image_height, FLAGS.image_width, 1) #================================ # #================================ #Get camera translation matrix r1, t1, _, _ = util.get_camera_pose( FLAGS.dataset_dir + "/images.txt", 'frame1819.jpg') r2, t2, _, _ = util.get_camera_pose( FLAGS.dataset_dir + "/images.txt", 'frame1829.jpg') scene_manager = SceneManager(FLAGS.dataset_dir) scene_manager.load_cameras() scene_manager.load_images() scene_manager.load_points3D() image_name = 'frame1819.jpg' image_id = scene_manager.get_image_id_from_name(image_name) image = scene_manager.images[image_id] camera = scene_manager.get_camera(image.camera_id) points3D, points2D = scene_manager.get_points3D(image_id) R = util.quaternion_to_rotation_matrix(image.qvec) points3D = points3D.dot(R.T) + image.tvec[np.newaxis, :] #points2D = np.hstack((points2D, np.ones((points2D.shape[0], 1)))) #points2D = points2D.dot(np.linalg.inv(camera.get_camera_matrix()).T)[:,:2] vmask = np.zeros_like(z) mulmask = np.zeros_like(z) for k, (u, v) in enumerate(points2D): j0, i0 = int(u), int( v) # upper left pixel for the (u,v) coordinate vmask[i0, j0] = points3D[k, 2] mulmask[i0, j0] = 1.0 z = cv2.resize(z, (224, 224), interpolation=cv2.INTER_AREA) z = z[:, :, np.newaxis] #z = z/100 z = 1.0 / z vmask = cv2.resize(vmask, (224, 224), interpolation=cv2.INTER_NEAREST) mulmask = cv2.resize(mulmask, (224, 224), interpolation=cv2.INTER_NEAREST) resized_points2D = [] for i in range(224): for j in range(224): if mulmask[i, j] != 0: resized_points2D.append([i, j]) # import pdb;pdb.set_trace() # z_stack = [] # points3D_stack = [] # for i in range(len(resized_points2D)): # i0=int(resized_points2D[i][0]) # j0=int(resized_points2D[i][1]) # z_stack.append(z[i0,j0]) # points3D_stack.append(vmask[i0,j0]) # import pdb;pdb.set_trace() # m_s = np.median(points3D_stack)/np.median(z_stack) # scale_m = np.eye(4) # scale_m[0,0] = m_s # scale_m[1,1] = m_s # scale_m[2,2] = m_s # #z = m_s*z pad = np.array([[0, 0, 0, 1]]) homo1 = np.append(r1, t1.reshape(3, 1), 1) homo1 = np.append(homo1, pad, 0) homo2 = np.append(r2, t2.reshape(3, 1), 1) homo2 = np.append(homo2, pad, 0) src2tgt_proj = np.dot(inv(homo2), homo1) #pred_poses = np.array([src2tgt_proj[0,3],src2tgt_proj[1,3],src2tgt_proj[2,3],]) I1 = I1 / 255.0 I2 = I2 / 255.0 for i in range(1, 100001): _, pred, reproject_img, tmp_total_loss, tmp_proj, tmp_scale = sess.run( [ train_op, pred_disp, curr_proj_image, total_loss, pred_poses, scale_factor ], feed_dict={ x1: I1[None, :, :, :], x2: I2[None, :, :, :], pred_poses: src2tgt_proj[None, :, :], intrinsics: m_intric[None, :, :], gt_x1_depth: z[None, :, :, :], tf_points3D: vmask, tf_multiply: mulmask, tf_points2D: np.array(resized_points2D, dtype=np.float32) }) # z=cv2.resize(pred[0][0,:,:,0],(FLAGS.image_width,FLAGS.image_height)) # z.astype(np.float32).tofile(FLAGS.output_dir+img_list[i].split('/')[-1]+'_z.bin') if i >= 0 and i % 10 == 0: print('Step %s - Loss: %.2f ' % (i, tmp_total_loss)) print('Scale %f' % (tmp_scale[0, 0, 0])) if i == 1: test = reproject_img[0] test = test * 255 test = test.astype(np.uint8) plt.imshow(test[0]) plt.show() import pdb pdb.set_trace() if i == 200: test = reproject_img[0] test = test * 255 test = test.astype(np.uint8) plt.imshow(test[0]) plt.show() import pdb pdb.set_trace() #print("The %dth frame is processed"%(i)) if i == 500: z_out = cv2.resize(1.0 / pred[0][0, :, :, 0], (FLAGS.image_width, FLAGS.image_height)) z_out.astype( np.float32).tofile(FLAGS.output_dir + img_list[0].split('/')[-1] + '_z.bin') if i == 1000: test = reproject_img[0] test = test * 255 test = test.astype(np.uint8) plt.imshow(test[0]) plt.show() import pdb pdb.set_trace()
def onSceneManager(self): SceneManager(self)
def OnClickRetryButtonExcute(self): currentScene = SceneManager.GetCurrentScene() currentScene.resetDataButton.OnClickExcute() currentScene.SetState("STATE_RETRY")
def estimate_overall_ref_model(colmap_folder, min_track_len, min_tri_angle, max_tri_angle, image_path): print 'Loading COLMAP data' scene_manager = SceneManager(colmap_folder) scene_manager.load_cameras() scene_manager.load_images() images = [ 'frame0859.jpg', 'frame0867.jpg', 'frame0875.jpg', 'frame0883.jpg', 'frame0891.jpg' ] _L = np.array([]) _r = np.array([]) _ndotl = np.array([]) for image_name in images: image_id = scene_manager.get_image_id_from_name(image_name) image = scene_manager.images[image_id] camera = scene_manager.get_camera(image.camera_id) # image pose R = util.quaternion_to_rotation_matrix(image.qvec) print 'Loading 3D points' scene_manager.load_points3D() scene_manager.filter_points3D(min_track_len, min_tri_angle=min_tri_angle, max_tri_angle=max_tri_angle, image_list=set([image_id])) points3D, points2D = scene_manager.get_points3D(image_id) points3D = points3D.dot(R.T) + image.tvec[np.newaxis, :] # need to remove redundant points # http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array points2D_view = np.ascontiguousarray(points2D).view( np.dtype((np.void, points2D.dtype.itemsize * points2D.shape[1]))) _, idx = np.unique(points2D_view, return_index=True) points2D, points3D = points2D[idx], points3D[idx] # further rule out any points too close to the image border (for bilinear # interpolation) mask = ((points2D[:, 0] < camera.width - 1) & (points2D[:, 1] < camera.height - 1)) points2D, points3D = points2D[mask], points3D[mask] points2D_image = points2D.copy() # coordinates in image space points2D = np.hstack((points2D, np.ones((points2D.shape[0], 1)))) points2D = points2D.dot(np.linalg.inv( camera.get_camera_matrix()).T)[:, :2] print len(points3D), 'total points' # load image #image_file = scene_manager.image_path + image.name image_file = image_path + image.name im_rgb = skimage.img_as_float( skimage.io.imread(image_file)) # color image L = skimage.color.rgb2lab(im_rgb)[:, :, 0] * 0.01 L = np.maximum( L, 1e-6) # unfortunately, can't have black pixels, since we # divide by L # initial values on unit sphere x, y = camera.get_image_grid() print 'Computing nearest neighbors' kdtree = KDTree(points2D) weights, nn_idxs = kdtree.query(np.c_[x.ravel(), y.ravel()], KD_TREE_KNN) weights = weights.reshape(camera.height, camera.width, KD_TREE_KNN) nn_idxs = nn_idxs.reshape(camera.height, camera.width, KD_TREE_KNN) # turn distances into weights for the nearest neighbors np.exp(-weights, weights) # in-place # create initial surface on unit sphere S0 = np.dstack((x, y, np.ones_like(x))) S0 /= np.linalg.norm(S0, axis=-1)[:, :, np.newaxis] r_fixed = np.linalg.norm(points3D, axis=-1) # fixed 3D depths S = S0.copy() z = S0[:, :, 2] S, r_ratios = nearest_neighbor_warp(weights, nn_idxs, points2D_image, r_fixed, util.generate_surface(camera, z)) z_est = np.maximum(S[:, :, 2], 1e-6) S = util.generate_surface(camera, z_est) r = np.linalg.norm(S, axis=-1) ndotl = util.calculate_ndotl(camera, S) if _L.size == 0: _L = L.ravel() _r = r.ravel() _ndotl = ndotl.ravel() else: _L = np.append(_L, L.ravel()) _r = np.append(_r, r.ravel()) _ndotl = np.append(_ndotl, ndotl.ravel()) # falloff, model_params, residual = fit_reflectance_model( sfs.POWER_MODEL, _L, _r, _ndotl, False, 5) import pdb pdb.set_trace()
class Solstice(object): @staticmethod def _platform_specific_init(): if platform.system() == 'Windows': os.environ['SDL_AUDIODRIVER'] = 'dsound' def __init__(self): Solstice._platform_specific_init() self._config = Configuration() self._translations_init() self._pygame_init() self._screen = Screen(self._config, _('Solstice')) self._control = Control(self) # TODO: Change data file name (don't use .zip extension) self._resource_manager = ResourceManager(self, 'data.zip') self._sound_player = SoundPlayer(self) self._scenes = { 'logo': LogoScene(self), 'intro': IntroScene(self), 'game': GameScene(self), 'elevator': ElevatorScene(self) } # The first parameter is the game 'context' self._scene_manager = SceneManager(self, 'logo') def _translations_init(self): gettext.bindtextdomain('solstice', self._config.locale_path) gettext.textdomain('solstice') def _sound_preinit(self): if self._config.sound or self._config.music: pygame.mixer.pre_init(22050, -16, 2, 1024) def _pygame_init(self): self._sound_preinit() os.environ['SDL_VIDEO_CENTERED'] = '1' pygame.init() def run(self): self._scene_manager.run() def exit(self, exit_code): self._config.save() pygame.quit() sys.exit(exit_code) @property def config(self): return self._config @property def screen(self): return self._screen @property def resource_manager(self): return self._resource_manager @property def control(self): return self._control @property def sound_player(self): return self._sound_player @property def scenes(self): return self._scenes @property def scene_manager(self): return self._scene_manager
for event in pygame.event.get(): if event.type == QUIT: pygame.quit() exit(0) if event.type == pygame.KEYDOWN: if event.key == K_ESCAPE: esc = True if event.key == pygame.K_DOWN: down = True if event.key == pygame.K_UP: up = True if event.key == pygame.K_RETURN: enter = True if event.type == pygame.KEYUP: if event.key == K_ESCAPE: esc = False if event.key == pygame.K_DOWN: down = False if event.key == pygame.K_UP: up = False if event.key == pygame.K_RETURN: enter = False SceneManager.current_scene.tick(up, down, enter, esc) SceneManager.current_scene.draw() SceneManager.check_for_switch() pygame.display.update() SceneManager.display.fill([0, 0, 0])
def cvt_bin_txt(model_folder): converter = SceneManager( model_folder ) converter.load_cameras() converter.load_images() converter.load_points3D() converter.save_cameras(output_folder=model_folder, output_file=None, binary=False) converter.save_images(output_folder=model_folder, output_file=None, binary=False) converter.save_points3D(output_folder=model_folder, output_file=None, binary=False)
def run(image_name, image_path, colmap_folder, out_folder, min_track_len, min_tri_angle, max_tri_angle, ref_surf_name, max_num_points=None, estimate_falloff=True, use_image_weighted_derivatives=True,get_initial_warp=True): min_track_len = int(min_track_len) min_tri_angle = int(min_tri_angle) max_tri_angle = int(max_tri_angle) #est_global_ref.estimate_overall_ref_model(colmap_folder,min_track_len,min_tri_angle,max_tri_angle,image_path) try: max_num_points = int(max_num_points) except: max_num_points = None get_initial_warp = (get_initial_warp not in ('False', 'false')) estimate_falloff = (estimate_falloff not in ('False', 'false')) use_image_weighted_derivatives = (use_image_weighted_derivatives not in ('False', 'false')) #if not image_path.endswith('/'): # image_path += '/' print 'Loading COLMAP data' scene_manager = SceneManager(colmap_folder) scene_manager.load_cameras() scene_manager.load_images() image_id = scene_manager.get_image_id_from_name(image_name) image = scene_manager.images[image_id] camera = scene_manager.get_camera(image.camera_id) # image pose R = util.quaternion_to_rotation_matrix(image.qvec) print 'Loading 3D points' scene_manager.load_points3D() scene_manager.filter_points3D(min_track_len, min_tri_angle=min_tri_angle, max_tri_angle=max_tri_angle, image_list=set([image_id])) points3D, points2D = scene_manager.get_points3D(image_id) points3D = points3D.dot(R.T) + image.tvec[np.newaxis,:] # need to remove redundant points # http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array points2D_view = np.ascontiguousarray(points2D).view( np.dtype((np.void, points2D.dtype.itemsize * points2D.shape[1]))) _, idx = np.unique(points2D_view, return_index=True) points2D, points3D = points2D[idx], points3D[idx] # further rule out any points too close to the image border (for bilinear # interpolation) mask = ( (points2D[:,0] < camera.width - 1) & (points2D[:,1] < camera.height - 1)) points2D, points3D = points2D[mask], points3D[mask] points2D_image = points2D.copy() # coordinates in image space points2D = np.hstack((points2D, np.ones((points2D.shape[0], 1)))) points2D = points2D.dot(np.linalg.inv(camera.get_camera_matrix()).T)[:,:2] if (max_num_points is not None and max_num_points > 0 and max_num_points < len(points2D)): np.random.seed(0) # fix the "random" points selected selected_points = np.random.choice(len(points2D), max_num_points, False) points2D = points2D[selected_points] points2D_image = points2D_image[selected_points] points3D = points3D[selected_points] print len(points3D), 'total points' #perturb points #import pdb;pdb.set_trace() #perturb_points=np.random.choice(len(points2D),max_num_points*0.3,False) #randomperturb=np.random.rand(perturb_points.size)*2 #points3D[perturb_points,2]=points3D[perturb_points,2]*randomperturb #points3D[2][2]=points3D[2][2]*0.5 #points3D[4][2]=points3D[4][2]*1.5 #points3D[8][2]=points3D[8][2]*0.3 #points3D[9][2]=points3D[9][2]*2. # load image #image_file = scene_manager.image_path + image.name image_file = image_path + image.name im_rgb = skimage.img_as_float(skimage.io.imread(image_file)) # color image L = skimage.color.rgb2lab(im_rgb)[:,:,0] * 0.01 #import pdb;pdb.set_trace() #skimage.io.imsave('test.png',L) L = np.maximum(L, 1e-6) # unfortunately, can't have black pixels, since we # divide by L # initial values on unit sphere x, y = camera.get_image_grid() print 'Computing nearest neighbors' #import pdb;pdb.set_trace() nn_idxs, weights = compute_nn(points2D.astype(np.float32), camera.width, camera.height, camera.fx, camera.fy, camera.cx, camera.cy) nn_idxs=np.swapaxes(nn_idxs,0,2) nn_idxs=np.swapaxes(nn_idxs,0,1) weights=np.swapaxes(weights,0,2) weights=np.swapaxes(weights,0,1) #kdtree = KDTree(points2D) #weights, nn_idxs = kdtree.query(np.c_[x.ravel(),y.ravel()], KD_TREE_KNN) #weights = weights.reshape(camera.height, camera.width, KD_TREE_KNN) #nn_idxs = nn_idxs.reshape(camera.height, camera.width, KD_TREE_KNN) # figure out pixel neighborhoods for each point #neighborhoods = [] neighborhood_mask = np.zeros((camera.height, camera.width), dtype=np.bool) for v, u in points2D_image: rr, cc = circle(int(u), int(v), MAX_POINT_DISTANCE, (camera.height, camera.width)) #neighborhoods.append((rr, cc)) neighborhood_mask[rr,cc] = True # turn distances into weights for the nearest neighbors np.exp(-weights, weights) # in-place # create initial surface on unit sphere S0 = np.dstack((x, y, np.ones_like(x))) S0 /= np.linalg.norm(S0, axis=-1)[:,:,np.newaxis] r_fixed = np.linalg.norm(points3D, axis=-1) # fixed 3D depths specular_mask = (L < 1.) vmask=np.zeros_like(S0[:,:,2]) for k, (u, v) in enumerate(points2D_image): j0, i0 = int(u), int(v) # upper left pixel for the (u,v) coordinate vmask[i0,j0]=points3D[k,2] # iterative SFS algorithm # model_type: reflectance model type # fit_falloff: True to fit the 1/r^m falloff parameter; False to fix it at 2 # extra_params: extra parameters to the reflectance model fit function def run_iterative_sfs(out_file, model_type, fit_falloff, *extra_params): #if os.path.exists(out_file + '_z.bin'): # don't re-run existing results # return if model_type == sfs.LAMBERTIAN_MODEL: model_name = 'LAMBERTIAN_MODEL' elif model_type == sfs.OREN_NAYAR_MODEL: model_name = 'OREN_NAYAR_MODEL' elif model_type == sfs.PHONG_MODEL: model_name = 'PHONG_MODEL' elif model_type == sfs.COOK_TORRANCE_MODEL: model_name = 'COOK_TORRANCE_MODEL' elif model_type == sfs.POWER_MODEL: model_name = 'POWER_MODEL' if model_type == sfs.POWER_MODEL: print '%s_%i' % (model_name, extra_params[0]) else: print model_name S = S0.copy() z = S0[:,:,2] for iteration in xrange(MAX_NUM_ITER): print 'Iteration', iteration z_old = z if get_initial_warp: #z_gt=util.load_point_ply('C:\\Users\\user\\Documents\\UNC\\Research\\ColonProject\\code\\Rui\\SFS_CPU\\frame0859.jpg_gt.ply') # warp to 3D points if iteration>-1: S, r_ratios = nearest_neighbor_warp(weights, nn_idxs, points2D_image, r_fixed, util.generate_surface(camera, z)) z_est = np.maximum(S[:,:,2], 1e-6) S=util.generate_surface(camera, z_est) else: z_est=z S=util.generate_surface(camera, z_est) #util.save_sfs_ply('warp' + '.ply', S, im_rgb) #util.save_xyz('test.xyz',points3D); #z=z_est #break #z_est=z else: #import pdb;pdb.set_trace() #z_est = extract_depth_map(camera,ref_surf_name,R,image) z_est=np.fromfile( 'C:\Users\user\Documents\UNC\Research\ColonProject\code\SFS_Program_from_True\endo_evaluation\gt_surfaces\\frame0859.jpg.bin', dtype=np.float32).reshape( camera.height, camera.width)/WORLD_SCALE z_est=z_est.astype(float) #S, r_ratios = nearest_neighbor_warp(weights, nn_idxs, # points2D_image, r_fixed, util.generate_surface(camera, z_est)) z_est = np.maximum(z_est[:,:], 1e-6) #Sworld = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R) S = util.generate_surface(camera, z_est) #util.save_sfs_ply('test' + '.ply', S, im_rgb) #util.save_sfs_ply(out_file + '_warp_%i.ply' % iteration, Sworld, im_rgb) #import pdb;pdb.set_trace() # if we need to, make corrections for non-positive depths #S = util.generate_surface(camera, z_est) mask = (z_est < INIT_Z) specular_mask=(L<0.8) dark_mask=(L>0.1) _mask=np.logical_and(specular_mask,mask) _mask=np.logical_and(_mask,dark_mask) # fit reflectance model r = np.linalg.norm(S, axis=-1) ndotl = util.calculate_ndotl(camera, S) falloff, model_params, residual = fit_reflectance_model(model_type, L[_mask],r.ravel(), r[_mask],ndotl.ravel(), ndotl[_mask], fit_falloff,camera.width,camera.height, *extra_params) #r = np.linalg.norm(S[specular_mask], axis=-1) #import pdb;pdb.set_trace() #model_params=np.array([26.15969874,-27.674055,-12.52426,7.579855,21.9768004,24.3911142,-21.7282996,-19.850894,-11.62229,-4.837014]) #model_params=np.array([-19.4837,-490.4796,812.4527,-426.09107,139.2602,351.8061,-388.1591,875.5013,-302.4748,-414.4384]) #falloff = 1.2 #ndotl = util.calculate_ndotl(camera, S)[specular_mask] #falloff, model_params, residual = fit_reflectance_model(model_type, # L[specular_mask], r, ndotl, fit_falloff, *extra_params) #r = np.linalg.norm(S[neighborhood_mask], axis=-1) #ndotl = util.calculate_ndotl(camera, S)[neighborhood_mask] #falloff, model_params, residual = fit_reflectance_model(model_type, # L[neighborhood_mask], r, ndotl, fit_falloff, *extra_params) # lambda values reflect our confidence in the current surface: 0 # corresponds to only using SFS at a pixel, 1 corresponds to equally # weighting SFS and the current estimate, and larger values increasingly # favor using only the current estimate rdiff = np.abs(r_fixed - get_estimated_r(S, points2D_image)) w = np.log10(r_fixed) - np.log10(rdiff) - np.log10(2.) lambdas = (np.sum(weights * w[nn_idxs], axis=-1) / np.sum(weights, axis=-1)) lambdas = np.maximum(lambdas, 0.) # just in case # lambdas[~mask] = 0 #if iteration == 0: # don't use current estimated surface on first pass #lambdas = np.zeros_like(z) #else: # r_ratios_postwarp = r_fixed / get_estimated_r(S, points2D_image) # ratio_diff = np.abs(r_ratios_prewarp - r_ratios_postwarp) # ratio_diff[ratio_diff == 0] = 1e-10 # arbitrarily high lambda # feature_lambdas = 1. / ratio_diff # lambdas = (np.sum(weights * feature_lambdas[nn_idxs], axis=-1) / # np.sum(weights, axis=-1)) # run SFS H_lut, dH_lut = compute_H_LUT(model_type, model_params, NUM_H_LUT_BINS) #import pdb;pdb.set_trace() # run SFS #H_lut = np.ascontiguousarray(H_lut.astype(np.float32)) #dH_lut = np.ascontiguousarray(dH_lut.astype(np.float32)) z = run_sfs(H_lut,dH_lut,camera, L, lambdas, z_est, model_type, model_params, falloff,vmask, use_image_weighted_derivatives) # check for convergence #diff = np.sum(np.abs(z_old[specular_mask] - z[specular_mask])) #if diff < CONVERGENCE_THRESHOLD * camera.height * camera.width: # break # save the surface #S = util.generate_surface(camera, z) #S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R) #util.save_sfs_ply(out_file + '_%i.ply' % iteration, S, im_rgb) else: print 'DID NOT CONVERGE' #import pdb;pdb.set_trace() S = util.generate_surface(camera, z) #S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R) util.save_sfs_ply(out_file + '.ply', S, im_rgb) z.astype(np.float32).tofile(out_file + '_z.bin') # save the surface #S = util.generate_surface(camera, z) #S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R) #S, r_ratios = nearest_neighbor_warp(weights, nn_idxs, # points2D_image, r_fixed, util.generate_surface(camera, z)) #util.save_sfs_ply(out_file + '_warped.ply', S, im_rgb) #z = np.maximum(S[:,:,2], 1e-6) #z.astype(np.float32).tofile(out_file + '_warped_z.bin') #reflectance_models.save_reflectance_model(out_file + '_reflectance.txt', # model_name, residual, model_params, falloff, *extra_params) print # now, actually run SFS if not out_folder.endswith('/'): out_folder += '/' if not os.path.exists(out_folder): os.makedirs(out_folder) # if not os.path.exists(out_folder + 'lambertian/'): # os.mkdir(out_folder + 'lambertian/') # run_iterative_sfs(out_folder + 'lambertian/' + image.name, # sfs.LAMBERTIAN_MODEL, estimate_falloff) # if not os.path.exists(out_folder + 'oren_nayar/'): # os.mkdir(out_folder + 'oren_nayar/') # run_iterative_sfs(out_folder + 'oren_nayar/' + image.name, # sfs.OREN_NAYAR_MODEL, estimate_falloff) # if not os.path.exists(out_folder + 'phong/'): # os.mkdir(out_folder + 'phong/') # run_iterative_sfs(out_folder + 'phong/' + image.name, sfs.PHONG_MODEL, # estimate_falloff) # if not os.path.exists(out_folder + 'cook_torrance/'): # os.mkdir(out_folder + 'cook_torrance/') # run_iterative_sfs(out_folder + 'cook_torrance/' + image.name, # sfs.COOK_TORRANCE_MODEL, estimate_falloff) #for K in [1, 2, 3, 4, 5, 10, 20, 50, 100]: #for K in [10, 20, 50]: #for K in [1, 2, 3, 4, 5]: for K in [100]: #if not os.path.exists(out_folder + 'power_model_%i/' % K): # os.mkdir(out_folder + 'power_model_%i/' % K) run_iterative_sfs(out_folder + image.name, sfs.POWER_MODEL, estimate_falloff, K)