def main():
sm = SceneManager()
sm.instantiate(sm)
snd_mngr = SoundManager()
snd_mngr.instantiate(snd_mngr)
pl = Player('player.png', 400)
pl.event.register_listener(snd_mngr.event_listener)
pl.instantiate(pl, pygame.math.Vector2(0, 0), 0)
item = Item()
item.event.register_listener(pl.event_listener)
item.instantiate(item)
while (True):
display.screen.fill((0, 50, 0))
gb.updateAll()
event = pygame.event.poll()
if event.type == pygame.QUIT:
break
#text = text_field.create_text("Hello, World", ["comicsansms"], 72, (0, 128, 0))
#display.screen.blit(text,
#(320 - text.get_width() // 2, 240 - text.get_height() // 2))
pygame.display.flip()
utils.clock.tick(60)
class App:
def __init__(self):
pyxel.init(256,
224,
palette=[
0x000000, 0x1D2B53, 0x7E2553, 0x008751, 0xAB5236,
0x5F574F, 0xC2C3C7, 0xFFF1E8, 0xFF004D, 0xFFA300,
0xFFEC27, 0x00E436, 0x29ADFF, 0x83769C, 0xFF77A8,
0xFFCCAA
])
pyxel.image(0).load(0, 0,
os.path.join(os.getcwd(), 'asset/system_neko.bmp'))
im = Image.open(os.path.join(os.getcwd(), 'asset/font.png'))
rgb_im = im.convert('RGB')
self.font = Font(rgb_im)
self.scene_manager = SceneManager(self)
pyxel.run(self.update, self.draw)
def update(self):
if pyxel.btnp(pyxel.KEY_Q):
pyxel.quit()
self.scene_manager.update()
def draw(self):
self.scene_manager.draw()
def __init__(self):
pygame.init()
resolution = (800, 400)
self.display = pygame.display.set_mode(resolution)
self.scene_manager = SceneManager(self.display, resolution)
pygame.display.set_caption('Pong')
def load_scene_manager(scenes, initial=None):
scene_manager = SceneManager(scenes)
for scene in scenes.values():
scene.scene_manager = scene_manager
if initial:
scene_manager.switch_scene(initial)
return scene_manager
def __onClickExcute(self):
if not self.firstClick:
p.changeDynamics(self.colId, -1, 10)
self.base_rotate = [0, 0, 1, 0]
self.col_local_pos = [0, 0, 0]
self.__ChangeStatus(STATUS_FLY)
currentScene = SceneManager.GetCurrentScene()
multi = currentScene.speedButton.GetCurrentTickZone()
currentScene.SetState("STATE_PLAYING")
currentScene.pauseButton.Display()
self.firstClick = True
data = self.GetData()
if data:
farMul, highMul = data['baseFarMultiply'], data[
'baseHighMultiply']
else:
farMul, highMul = 1.0, 1.0
farMul = farMul * multi
highMul = highMul * 0.5 * multi
pos, orn = p.getBasePositionAndOrientation(self.colId)
force = [0, -self.camDis[1] * 10000 * farMul, 20000 * highMul]
p.applyExternalForce(self.colId,
-1,
force,
pos,
flags=p.WORLD_FRAME)
def OnClickExcute(self):
currentScene = SceneManager.GetCurrentScene()
if currentScene.state == "STATE_PAUSE":
currentScene.SetState("STATE_PLAYING")
elif currentScene.state == "STATE_PLAYING":
currentScene.SetState("STATE_PAUSE")
else:
return
def main():
Logger.instance().log_debug("main");
Config.instance().init();
Camera.instance().init();
SceneManager.instance().init();
SceneManager.instance().run();
SceneManager.instance().quit();
Camera.instance().quit();
def DisplayStageResult(self):
percentDestroy = self.finalScene.GetDestroyPercent()
if percentDestroy < 0.5:
rate = "UNCLEAR"
elif percentDestroy < 0.6:
rate = "ONE_STAR_RATE"
elif percentDestroy < 0.8:
rate = "TWO_STAR_RATE"
else:
rate = "THREE_STAR_RATE"
self.endStageCanvas.Display(rate)
self.percentDestroy = 0
currentScene = SceneManager.GetCurrentScene()
currentScene.HideUI()
if os.path.exists("TestVoxel/currentStageResult.pickle"):
os.remove("TestVoxel/currentStageResult.pickle")
def __init__(self):
Solstice._platform_specific_init()
self._config = Configuration()
self._translations_init()
self._pygame_init()
self._screen = Screen(self._config, _('Solstice'))
self._control = Control(self)
# TODO: Change data file name (don't use .zip extension)
self._resource_manager = ResourceManager(self, 'data.zip')
self._sound_player = SoundPlayer(self)
self._scenes = {
'logo': LogoScene(self),
'intro': IntroScene(self),
'game': GameScene(self),
'elevator': ElevatorScene(self)
}
# The first parameter is the game 'context'
self._scene_manager = SceneManager(self, 'logo')
def OnClickExcute(self):
SceneManager.GetCurrentScene().SetState("STATE_RESET")
import game_globals as glb
from scene_manager import SceneManager
from menu_scene import *
from game_scene import *
from splash import *
if __name__ == '__main__':
pygame.init()
pygame.font.init()
pygame.mouse.set_visible(0)
scenes = [
main_menu, options_menu, difficulty_menu, game_scene,
you_loose_splash_screen, you_win_splash_screen, final_splash_screen
]
# create scene manager
scene_manager = SceneManager(glb.WIDTH, glb.HEIGHT, glb.FPS)
# build and register scenes
for scene in scenes:
if scene != game_scene:
# we build main game scene later, only when difficulty is chosen
scene.build()
scene.bind_scene_manager(scene_manager)
# add starting scene and launch game loop
scene_manager.push_scene(main_menu)
scene_manager.main_loop()
import pybullet as p import time import pybullet_data from pyxie.apputil import graphicsHelper import pyxie import pyvmath as vmath import math import random from game_scene import GameScene from scene_manager import SceneManager import imgui import sys SCREEN_WIDTH = 520 SCREEN_HEIGHT = 900 pyxie.window(True, SCREEN_WIDTH , SCREEN_HEIGHT) imgui.create_context() gameScene = GameScene() SceneManager.SetCurrentScene(gameScene) while(1): SceneManager.GetCurrentScene().Update() SceneManager.GetCurrentScene().Render() pyxie.swap() # game loop
def main():
scenemanager = SceneManager()
scenemanager.loop()
def run(image_name,
image_path,
colmap_folder,
out_folder,
min_track_len,
min_tri_angle,
max_tri_angle,
ref_surf_name,
max_num_points=None,
estimate_falloff=True,
use_image_weighted_derivatives=True,
get_initial_warp=True):
min_track_len = int(min_track_len)
min_tri_angle = int(min_tri_angle)
max_tri_angle = int(max_tri_angle)
#est_global_ref.estimate_overall_ref_model(colmap_folder,min_track_len,min_tri_angle,max_tri_angle,image_path)
try:
max_num_points = int(max_num_points)
except:
max_num_points = None
get_initial_warp = (get_initial_warp not in ('False', 'false'))
estimate_falloff = (estimate_falloff not in ('False', 'false'))
use_image_weighted_derivatives = (use_image_weighted_derivatives
not in ('False', 'false'))
#if not image_path.endswith('/'):
# image_path += '/'
print 'Loading COLMAP data'
scene_manager = SceneManager(colmap_folder)
scene_manager.load_cameras()
scene_manager.load_images()
image_id = scene_manager.get_image_id_from_name(image_name)
image = scene_manager.images[image_id]
camera = scene_manager.get_camera(image.camera_id)
# image pose
R = util.quaternion_to_rotation_matrix(image.qvec)
print 'Loading 3D points'
scene_manager.load_points3D()
scene_manager.filter_points3D(min_track_len,
min_tri_angle=min_tri_angle,
max_tri_angle=max_tri_angle,
image_list=set([image_id]))
points3D, points2D = scene_manager.get_points3D(image_id)
points3D = points3D.dot(R.T) + image.tvec[np.newaxis, :]
# need to remove redundant points
# http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array
points2D_view = np.ascontiguousarray(points2D).view(
np.dtype((np.void, points2D.dtype.itemsize * points2D.shape[1])))
_, idx = np.unique(points2D_view, return_index=True)
points2D, points3D = points2D[idx], points3D[idx]
# further rule out any points too close to the image border (for bilinear
# interpolation)
mask = ((points2D[:, 0] < camera.width - 1) &
(points2D[:, 1] < camera.height - 1))
points2D, points3D = points2D[mask], points3D[mask]
points2D_image = points2D.copy() # coordinates in image space
points2D = np.hstack((points2D, np.ones((points2D.shape[0], 1))))
points2D = points2D.dot(np.linalg.inv(camera.get_camera_matrix()).T)[:, :2]
if (max_num_points is not None and max_num_points > 0
and max_num_points < len(points2D)):
np.random.seed(0) # fix the "random" points selected
selected_points = np.random.choice(len(points2D), max_num_points,
False)
points2D = points2D[selected_points]
points2D_image = points2D_image[selected_points]
points3D = points3D[selected_points]
print len(points3D), 'total points'
#perturb points
#import pdb;pdb.set_trace()
#perturb_points=np.random.choice(len(points2D),max_num_points*0.3,False)
#randomperturb=np.random.rand(perturb_points.size)*2
#points3D[perturb_points,2]=points3D[perturb_points,2]*randomperturb
#points3D[2][2]=points3D[2][2]*0.5
#points3D[4][2]=points3D[4][2]*1.5
#points3D[8][2]=points3D[8][2]*0.3
#points3D[9][2]=points3D[9][2]*2.
# load image
#image_file = scene_manager.image_path + image.name
image_file = image_path + image.name
im_rgb = skimage.img_as_float(skimage.io.imread(image_file)) # color image
L = skimage.color.rgb2lab(im_rgb)[:, :, 0] * 0.01
#import pdb;pdb.set_trace()
#skimage.io.imsave('test.png',L)
L = np.maximum(L, 1e-6) # unfortunately, can't have black pixels, since we
# divide by L
# initial values on unit sphere
x, y = camera.get_image_grid()
print 'Computing nearest neighbors'
#import pdb;pdb.set_trace()
nn_idxs, weights = compute_nn(points2D.astype(np.float32), camera.width,
camera.height, camera.fx, camera.fy,
camera.cx, camera.cy)
nn_idxs = np.swapaxes(nn_idxs, 0, 2)
nn_idxs = np.swapaxes(nn_idxs, 0, 1)
weights = np.swapaxes(weights, 0, 2)
weights = np.swapaxes(weights, 0, 1)
#kdtree = KDTree(points2D)
#weights, nn_idxs = kdtree.query(np.c_[x.ravel(),y.ravel()], KD_TREE_KNN)
#weights = weights.reshape(camera.height, camera.width, KD_TREE_KNN)
#nn_idxs = nn_idxs.reshape(camera.height, camera.width, KD_TREE_KNN)
# figure out pixel neighborhoods for each point
#neighborhoods = []
neighborhood_mask = np.zeros((camera.height, camera.width), dtype=np.bool)
for v, u in points2D_image:
rr, cc = circle(int(u), int(v), MAX_POINT_DISTANCE,
(camera.height, camera.width))
#neighborhoods.append((rr, cc))
neighborhood_mask[rr, cc] = True
# turn distances into weights for the nearest neighbors
np.exp(-weights, weights) # in-place
# create initial surface on unit sphere
S0 = np.dstack((x, y, np.ones_like(x)))
S0 /= np.linalg.norm(S0, axis=-1)[:, :, np.newaxis]
r_fixed = np.linalg.norm(points3D, axis=-1) # fixed 3D depths
specular_mask = (L < 1.)
vmask = np.zeros_like(S0[:, :, 2])
for k, (u, v) in enumerate(points2D_image):
j0, i0 = int(u), int(v) # upper left pixel for the (u,v) coordinate
vmask[i0, j0] = points3D[k, 2]
# iterative SFS algorithm
# model_type: reflectance model type
# fit_falloff: True to fit the 1/r^m falloff parameter; False to fix it at 2
# extra_params: extra parameters to the reflectance model fit function
def run_iterative_sfs(out_file, model_type, fit_falloff, *extra_params):
#if os.path.exists(out_file + '_z.bin'): # don't re-run existing results
# return
if model_type == sfs.LAMBERTIAN_MODEL:
model_name = 'LAMBERTIAN_MODEL'
elif model_type == sfs.OREN_NAYAR_MODEL:
model_name = 'OREN_NAYAR_MODEL'
elif model_type == sfs.PHONG_MODEL:
model_name = 'PHONG_MODEL'
elif model_type == sfs.COOK_TORRANCE_MODEL:
model_name = 'COOK_TORRANCE_MODEL'
elif model_type == sfs.POWER_MODEL:
model_name = 'POWER_MODEL'
if model_type == sfs.POWER_MODEL:
print '%s_%i' % (model_name, extra_params[0])
else:
print model_name
S = S0.copy()
z = S0[:, :, 2]
for iteration in xrange(MAX_NUM_ITER):
print 'Iteration', iteration
z_old = z
if get_initial_warp:
#z_gt=util.load_point_ply('C:\\Users\\user\\Documents\\UNC\\Research\\ColonProject\\code\\Rui\\SFS_CPU\\frame0859.jpg_gt.ply')
# warp to 3D points
if iteration > -1:
S, r_ratios = nearest_neighbor_warp(
weights, nn_idxs, points2D_image, r_fixed,
util.generate_surface(camera, z))
z_est = np.maximum(S[:, :, 2], 1e-6)
S = util.generate_surface(camera, z_est)
else:
z_est = z
S = util.generate_surface(camera, z_est)
#util.save_sfs_ply('warp' + '.ply', S, im_rgb)
#util.save_xyz('test.xyz',points3D);
#z=z_est
#break
#z_est=z
else:
#import pdb;pdb.set_trace()
#z_est = extract_depth_map(camera,ref_surf_name,R,image)
z_est = np.fromfile(
'C:\Users\user\Documents\UNC\Research\ColonProject\code\SFS_Program_from_True\endo_evaluation\gt_surfaces\\frame0859.jpg.bin',
dtype=np.float32).reshape(camera.height,
camera.width) / WORLD_SCALE
z_est = z_est.astype(float)
#S, r_ratios = nearest_neighbor_warp(weights, nn_idxs,
# points2D_image, r_fixed, util.generate_surface(camera, z_est))
z_est = np.maximum(z_est[:, :], 1e-6)
#Sworld = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R)
S = util.generate_surface(camera, z_est)
#util.save_sfs_ply('test' + '.ply', S, im_rgb)
#util.save_sfs_ply(out_file + '_warp_%i.ply' % iteration, Sworld, im_rgb)
#import pdb;pdb.set_trace()
# if we need to, make corrections for non-positive depths
#S = util.generate_surface(camera, z_est)
mask = (z_est < INIT_Z)
specular_mask = (L < 0.8)
dark_mask = (L > 0.1)
_mask = np.logical_and(specular_mask, mask)
_mask = np.logical_and(_mask, dark_mask)
# fit reflectance model
r = np.linalg.norm(S, axis=-1)
ndotl = util.calculate_ndotl(camera, S)
falloff, model_params, residual = fit_reflectance_model(
model_type, L[_mask], r.ravel(), r[_mask], ndotl.ravel(),
ndotl[_mask], fit_falloff, camera.width, camera.height,
*extra_params)
#r = np.linalg.norm(S[specular_mask], axis=-1)
#import pdb;pdb.set_trace()
#model_params=np.array([26.15969874,-27.674055,-12.52426,7.579855,21.9768004,24.3911142,-21.7282996,-19.850894,-11.62229,-4.837014])
#model_params=np.array([-19.4837,-490.4796,812.4527,-426.09107,139.2602,351.8061,-388.1591,875.5013,-302.4748,-414.4384])
#falloff = 1.2
#ndotl = util.calculate_ndotl(camera, S)[specular_mask]
#falloff, model_params, residual = fit_reflectance_model(model_type,
# L[specular_mask], r, ndotl, fit_falloff, *extra_params)
#r = np.linalg.norm(S[neighborhood_mask], axis=-1)
#ndotl = util.calculate_ndotl(camera, S)[neighborhood_mask]
#falloff, model_params, residual = fit_reflectance_model(model_type,
# L[neighborhood_mask], r, ndotl, fit_falloff, *extra_params)
# lambda values reflect our confidence in the current surface: 0
# corresponds to only using SFS at a pixel, 1 corresponds to equally
# weighting SFS and the current estimate, and larger values increasingly
# favor using only the current estimate
rdiff = np.abs(r_fixed - get_estimated_r(S, points2D_image))
w = np.log10(r_fixed) - np.log10(rdiff) - np.log10(2.)
lambdas = (np.sum(weights * w[nn_idxs], axis=-1) /
np.sum(weights, axis=-1))
lambdas = np.maximum(lambdas, 0.) # just in case
#
lambdas[~mask] = 0
#if iteration == 0: # don't use current estimated surface on first pass
#lambdas = np.zeros_like(z)
#else:
# r_ratios_postwarp = r_fixed / get_estimated_r(S, points2D_image)
# ratio_diff = np.abs(r_ratios_prewarp - r_ratios_postwarp)
# ratio_diff[ratio_diff == 0] = 1e-10 # arbitrarily high lambda
# feature_lambdas = 1. / ratio_diff
# lambdas = (np.sum(weights * feature_lambdas[nn_idxs], axis=-1) /
# np.sum(weights, axis=-1))
# run SFS
H_lut, dH_lut = compute_H_LUT(model_type, model_params,
NUM_H_LUT_BINS)
#import pdb;pdb.set_trace()
# run SFS
#H_lut = np.ascontiguousarray(H_lut.astype(np.float32))
#dH_lut = np.ascontiguousarray(dH_lut.astype(np.float32))
z = run_sfs(H_lut, dH_lut, camera, L, lambdas, z_est, model_type,
model_params, falloff, vmask,
use_image_weighted_derivatives)
# check for convergence
#diff = np.sum(np.abs(z_old[specular_mask] - z[specular_mask]))
#if diff < CONVERGENCE_THRESHOLD * camera.height * camera.width:
# break
# save the surface
#S = util.generate_surface(camera, z)
#S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R)
#util.save_sfs_ply(out_file + '_%i.ply' % iteration, S, im_rgb)
else:
print 'DID NOT CONVERGE'
#import pdb;pdb.set_trace()
S = util.generate_surface(camera, z)
#S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R)
util.save_sfs_ply(out_file + '.ply', S, im_rgb)
z.astype(np.float32).tofile(out_file + '_z.bin')
# save the surface
#S = util.generate_surface(camera, z)
#S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R)
#S, r_ratios = nearest_neighbor_warp(weights, nn_idxs,
# points2D_image, r_fixed, util.generate_surface(camera, z))
#util.save_sfs_ply(out_file + '_warped.ply', S, im_rgb)
#z = np.maximum(S[:,:,2], 1e-6)
#z.astype(np.float32).tofile(out_file + '_warped_z.bin')
#reflectance_models.save_reflectance_model(out_file + '_reflectance.txt',
# model_name, residual, model_params, falloff, *extra_params)
print
# now, actually run SFS
if not out_folder.endswith('/'):
out_folder += '/'
if not os.path.exists(out_folder):
os.makedirs(out_folder)
# if not os.path.exists(out_folder + 'lambertian/'):
# os.mkdir(out_folder + 'lambertian/')
# run_iterative_sfs(out_folder + 'lambertian/' + image.name,
# sfs.LAMBERTIAN_MODEL, estimate_falloff)
# if not os.path.exists(out_folder + 'oren_nayar/'):
# os.mkdir(out_folder + 'oren_nayar/')
# run_iterative_sfs(out_folder + 'oren_nayar/' + image.name,
# sfs.OREN_NAYAR_MODEL, estimate_falloff)
# if not os.path.exists(out_folder + 'phong/'):
# os.mkdir(out_folder + 'phong/')
# run_iterative_sfs(out_folder + 'phong/' + image.name, sfs.PHONG_MODEL,
# estimate_falloff)
# if not os.path.exists(out_folder + 'cook_torrance/'):
# os.mkdir(out_folder + 'cook_torrance/')
# run_iterative_sfs(out_folder + 'cook_torrance/' + image.name,
# sfs.COOK_TORRANCE_MODEL, estimate_falloff)
#for K in [1, 2, 3, 4, 5, 10, 20, 50, 100]:
#for K in [10, 20, 50]:
#for K in [1, 2, 3, 4, 5]:
for K in [100]:
#if not os.path.exists(out_folder + 'power_model_%i/' % K):
# os.mkdir(out_folder + 'power_model_%i/' % K)
run_iterative_sfs(out_folder + image.name, sfs.POWER_MODEL,
estimate_falloff, K)
def run(colmap_folder, sfs_results_path, gt_path, out_folder):
#import pdb;pdb.set_trace();
if not sfs_results_path.endswith('/'):
sfs_results_path += '/'
if not gt_path.endswith('/'):
gt_path += '/'
if not out_folder.endswith('/'):
out_folder += '/'
scene_manager = SceneManager(colmap_folder)
scene_manager.load_cameras()
camera = scene_manager.cameras[1] # assume single camera
# initial values on unit sphere
x, y = camera.get_image_grid()
r_scale = np.sqrt(x * x + y * y + 1.)
image = np.empty((camera.height, camera.width, 3))
#image[:,:,0] = 1.
if not os.path.exists(out_folder):
os.mkdir(out_folder)
# iterate through rendered BRDFs
#for brdf_name in os.listdir(sfs_results_path):
for brdf_name in ['phantom']:
brdf_folder = sfs_results_path + brdf_name + '/'
if not os.path.isdir(brdf_folder): continue
with open(out_folder + brdf_name + '.csv', 'w') as f:
print>>f, HEADER1 % brdf_name
print>>f, HEADER2
# iterate through reflectance models
#for rm_name in os.listdir(brdf_folder):
#for model_name in MODELS:
model_folder = brdf_folder + '/'#model_name + '/'
# percent of pixels within 1, 2, 5mm of the GT surface
data = [list() for _ in THRESHOLDS]
###
datasum = []
###
# iterate through SFS surfaces
for filename in os.listdir(model_folder):
if not filename.endswith('_z.bin'): continue
name = filename[:-6]
#import pdb;pdb.set_trace();
util.save_sfs_ply('%s_%s.ply' % (name, 'est'),
np.dstack((x, y, np.ones_like(x))) *
np.fromfile(model_folder + filename, dtype=np.float32).reshape(
camera.height, camera.width, 1))
#
util.save_sfs_ply('%s_%s.ply' % (name, 'gt'),
np.dstack((x, y, np.ones_like(x)))/WORLD_SCALE *
np.fromfile(gt_path + name+'.bin', dtype=np.float32).reshape(
camera.height, camera.width, 1))
#break
r_est = WORLD_SCALE * r_scale * np.fromfile(
model_folder + filename, dtype=np.float32).reshape(
camera.height, camera.width)
r_gt = r_scale * np.fromfile(
gt_path + name + '.bin', dtype=np.float32).reshape(
camera.height, camera.width)
rdiff = np.abs(r_est - r_gt)
#rdiff = np.concatenate((
# rdiff[:,:20].ravel(), rdiff[:,-20:].ravel(),
# rdiff[20:-20,:20].ravel(), rdiff[20:-20,-20:].ravel()))
#rdiff = rdiff[(r_gt > 5.0) & (r_gt < 20.0)]
inv_size = 1. / float(rdiff.size)
for i, t in enumerate(THRESHOLDS):
data[i].append(np.count_nonzero(rdiff < t) * inv_size)
#data[i].append(np.mean((rdiff / r_gt)[rdiff < t]))
#data[i].append(np.mean((r_est / r_gt)))
###
datasum.append(1. - (r_est / r_gt).ravel())
datasum = np.concatenate(datasum)
print np.mean(datasum), np.std(datasum)
###
data = np.array(data)
data = np.mean(data, axis=1), np.std(data, axis=1)
model_name='power'
print>>f, model_name + ',' + ','.join('%f,%f' % d for d in izip(*data))
def estimate_overall_ref_model(colmap_folder,min_track_len,min_tri_angle,max_tri_angle,image_path):
print 'Loading COLMAP data'
scene_manager = SceneManager(colmap_folder)
scene_manager.load_cameras()
scene_manager.load_images()
images=['frame0859.jpg', 'frame0867.jpg', 'frame0875.jpg', 'frame0883.jpg', 'frame0891.jpg']
_L=np.array([])
_r=np.array([])
_ndotl=np.array([])
for image_name in images:
image_id = scene_manager.get_image_id_from_name(image_name)
image = scene_manager.images[image_id]
camera = scene_manager.get_camera(image.camera_id)
# image pose
R = util.quaternion_to_rotation_matrix(image.qvec)
print 'Loading 3D points'
scene_manager.load_points3D()
scene_manager.filter_points3D(min_track_len,
min_tri_angle=min_tri_angle, max_tri_angle=max_tri_angle,
image_list=set([image_id]))
points3D, points2D = scene_manager.get_points3D(image_id)
points3D = points3D.dot(R.T) + image.tvec[np.newaxis,:]
# need to remove redundant points
# http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array
points2D_view = np.ascontiguousarray(points2D).view(
np.dtype((np.void, points2D.dtype.itemsize * points2D.shape[1])))
_, idx = np.unique(points2D_view, return_index=True)
points2D, points3D = points2D[idx], points3D[idx]
# further rule out any points too close to the image border (for bilinear
# interpolation)
mask = (
(points2D[:,0] < camera.width - 1) & (points2D[:,1] < camera.height - 1))
points2D, points3D = points2D[mask], points3D[mask]
points2D_image = points2D.copy() # coordinates in image space
points2D = np.hstack((points2D, np.ones((points2D.shape[0], 1))))
points2D = points2D.dot(np.linalg.inv(camera.get_camera_matrix()).T)[:,:2]
print len(points3D), 'total points'
# load image
#image_file = scene_manager.image_path + image.name
image_file = image_path + image.name
im_rgb = skimage.img_as_float(skimage.io.imread(image_file)) # color image
L = skimage.color.rgb2lab(im_rgb)[:,:,0] * 0.01
L = np.maximum(L, 1e-6) # unfortunately, can't have black pixels, since we
# divide by L
# initial values on unit sphere
x, y = camera.get_image_grid()
print 'Computing nearest neighbors'
kdtree = KDTree(points2D)
weights, nn_idxs = kdtree.query(np.c_[x.ravel(),y.ravel()], KD_TREE_KNN)
weights = weights.reshape(camera.height, camera.width, KD_TREE_KNN)
nn_idxs = nn_idxs.reshape(camera.height, camera.width, KD_TREE_KNN)
# turn distances into weights for the nearest neighbors
np.exp(-weights, weights) # in-place
# create initial surface on unit sphere
S0 = np.dstack((x, y, np.ones_like(x)))
S0 /= np.linalg.norm(S0, axis=-1)[:,:,np.newaxis]
r_fixed = np.linalg.norm(points3D, axis=-1) # fixed 3D depths
S = S0.copy()
z = S0[:,:,2]
S, r_ratios = nearest_neighbor_warp(weights, nn_idxs,
points2D_image, r_fixed, util.generate_surface(camera, z))
z_est = np.maximum(S[:,:,2], 1e-6)
S = util.generate_surface(camera, z_est)
r = np.linalg.norm(S, axis=-1)
ndotl = util.calculate_ndotl(camera, S)
if _L.size == 0:
_L=L.ravel()
_r=r.ravel()
_ndotl=ndotl.ravel()
else:
_L=np.append(_L,L.ravel())
_r=np.append(_r,r.ravel())
_ndotl=np.append(_ndotl,ndotl.ravel())
#
falloff, model_params, residual = fit_reflectance_model(sfs.POWER_MODEL,
_L, _r, _ndotl, False, 5)
import pdb;pdb.set_trace();
def run(colmap_folder, sfs_results_path, gt_path, out_folder):
#import pdb;pdb.set_trace();
if not sfs_results_path.endswith('/'):
sfs_results_path += '/'
if not gt_path.endswith('/'):
gt_path += '/'
if not out_folder.endswith('/'):
out_folder += '/'
scene_manager = SceneManager(colmap_folder)
scene_manager.load_cameras()
camera = scene_manager.cameras[1] # assume single camera
# initial values on unit sphere
x, y = camera.get_image_grid()
r_scale = np.sqrt(x * x + y * y + 1.)
image = np.empty((camera.height, camera.width, 3))
#image[:,:,0] = 1.
if not os.path.exists(out_folder):
os.mkdir(out_folder)
# iterate through rendered BRDFs
#for brdf_name in os.listdir(sfs_results_path):
for brdf_name in ['phantom']:
brdf_folder = sfs_results_path + brdf_name + '/'
if not os.path.isdir(brdf_folder): continue
with open(out_folder + brdf_name + '.csv', 'w') as f:
print >> f, HEADER1 % brdf_name
print >> f, HEADER2
# iterate through reflectance models
#for rm_name in os.listdir(brdf_folder):
#for model_name in MODELS:
model_folder = brdf_folder + '/' #model_name + '/'
# percent of pixels within 1, 2, 5mm of the GT surface
data = [list() for _ in THRESHOLDS]
###
datasum = []
###
# iterate through SFS surfaces
for filename in os.listdir(model_folder):
if not filename.endswith('_z.bin'): continue
name = filename[:-6]
#import pdb;pdb.set_trace();
util.save_sfs_ply(
'%s_%s.ply' % (name, 'est'),
np.dstack((x, y, np.ones_like(x))) * np.fromfile(
model_folder + filename, dtype=np.float32).reshape(
camera.height, camera.width, 1))
#
util.save_sfs_ply(
'%s_%s.ply' % (name, 'gt'),
np.dstack(
(x, y, np.ones_like(x))) / WORLD_SCALE * np.fromfile(
gt_path + name + '.bin', dtype=np.float32).reshape(
camera.height, camera.width, 1))
#break
r_est = WORLD_SCALE * r_scale * np.fromfile(
model_folder + filename, dtype=np.float32).reshape(
camera.height, camera.width)
r_gt = r_scale * np.fromfile(gt_path + name + '.bin',
dtype=np.float32).reshape(
camera.height, camera.width)
rdiff = np.abs(r_est - r_gt)
#rdiff = np.concatenate((
# rdiff[:,:20].ravel(), rdiff[:,-20:].ravel(),
# rdiff[20:-20,:20].ravel(), rdiff[20:-20,-20:].ravel()))
#rdiff = rdiff[(r_gt > 5.0) & (r_gt < 20.0)]
inv_size = 1. / float(rdiff.size)
for i, t in enumerate(THRESHOLDS):
data[i].append(np.count_nonzero(rdiff < t) * inv_size)
#data[i].append(np.mean((rdiff / r_gt)[rdiff < t]))
#data[i].append(np.mean((r_est / r_gt)))
###
datasum.append(1. - (r_est / r_gt).ravel())
datasum = np.concatenate(datasum)
print np.mean(datasum), np.std(datasum)
###
data = np.array(data)
data = np.mean(data, axis=1), np.std(data, axis=1)
model_name = 'power'
print >> f, model_name + ',' + ','.join('%f,%f' % d
for d in izip(*data))
def main(_):
with tf.Graph().as_default():
#Load image and label
x1 = tf.placeholder(shape=[1, 224, 224, 3], dtype=tf.float32)
x2 = tf.placeholder(shape=[1, 224, 224, 3], dtype=tf.float32)
gt_x1_depth = tf.placeholder(shape=[1, 224, 224, 1], dtype=tf.float32)
img_list = sorted(glob(FLAGS.dataset_dir + '/*.jpg'))
pred_poses = tf.placeholder(shape=[1, 4, 4], dtype=tf.float32)
intrinsics = tf.placeholder(shape=[1, 3, 3], dtype=tf.float32)
tf_points3D = tf.placeholder(shape=[224, 224], dtype=tf.float32)
tf_multiply = tf.placeholder(shape=[224, 224], dtype=tf.float32)
tf_points2D = tf.placeholder(shape=[None, 2], dtype=tf.float32)
#import pdb;pdb.set_trace()
m_stack_intrinsics = get_multi_scale_intrinsics(
intrinsics, FLAGS.num_scales, 224.0 / 720.0, 224.0 / 240.0)
# # Define the model:
with tf.name_scope("Prediction"):
pred_disp, depth_net_endpoints = disp_net(x1, is_training=True)
scale_factor = get_scale_factor(tf_points3D,
1.0 / pred_disp[0][0, :, :, 0],
tf_points2D)
with tf.name_scope("compute_loss"):
pixel_loss = 0
smooth_loss = 0
curr_proj_image = []
for s in range(FLAGS.num_scales):
smooth_loss += FLAGS.smooth_weight/(2**s) * \
compute_smooth_loss(pred_disp[s])
curr_src_image = tf.image.resize_area(
x1,
[int(224 / (2**s)), int(224 / (2**s))])
curr_tgt_image = tf.image.resize_area(
x2,
[int(224 / (2**s)), int(224 / (2**s))])
curr_gt_x1_depth = tf.image.resize_area(
gt_x1_depth,
[int(224 / (2**s)), int(224 / (2**s))])
#import pdb;pdb.set_trace()
curr_proj_image.append(
projective_inverse_warp(
curr_tgt_image,
#tf.squeeze(1.0/curr_gt_x1_depth,axis=3),
tf.squeeze(1.0 / pred_disp[s], axis=3),
tf.matmul(pred_poses, scale_factor),
#pred_poses,
m_stack_intrinsics[:, s, :, :]))
curr_proj_error = tf.abs(curr_src_image - curr_proj_image[s])
curr_depth_error = tf.abs(curr_gt_x1_depth -
scale_factor[0, 0, 0] * pred_disp[s])
pixel_loss += tf.reduce_mean(curr_proj_error)
pixel_loss += tf.reduce_mean(
curr_depth_error) * FLAGS.data_weight / (2**s)
total_loss = pixel_loss + smooth_loss
saver = tf.train.Saver([var for var in tf.model_variables()])
checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate,
FLAGS.beta1)
global_step = tf.Variable(0, name='global_step', trainable=False)
train_op = optimizer.minimize(total_loss, global_step=global_step)
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
saver.restore(sess, checkpoint)
I1 = pil.open(img_list[0])
I1 = np.array(I1)
I1 = cv2.resize(I1, (224, 224), interpolation=cv2.INTER_AREA)
I2 = pil.open(img_list[1])
I2 = np.array(I2)
I2 = cv2.resize(I2, (224, 224), interpolation=cv2.INTER_AREA)
m_intric = np.array([[567.239, 0.0, 376.04],
[0.0, 261.757, 100.961], [0.0, 0.0, 1.0]])
z = np.fromfile(
'/home/wrlife/project/deeplearning/depth_prediction/backprojtest/oldcase1/frame1819.jpg_z.bin',
dtype=np.float32).reshape(FLAGS.image_height,
FLAGS.image_width, 1)
#================================
#
#================================
#Get camera translation matrix
r1, t1, _, _ = util.get_camera_pose(
FLAGS.dataset_dir + "/images.txt", 'frame1819.jpg')
r2, t2, _, _ = util.get_camera_pose(
FLAGS.dataset_dir + "/images.txt", 'frame1829.jpg')
scene_manager = SceneManager(FLAGS.dataset_dir)
scene_manager.load_cameras()
scene_manager.load_images()
scene_manager.load_points3D()
image_name = 'frame1819.jpg'
image_id = scene_manager.get_image_id_from_name(image_name)
image = scene_manager.images[image_id]
camera = scene_manager.get_camera(image.camera_id)
points3D, points2D = scene_manager.get_points3D(image_id)
R = util.quaternion_to_rotation_matrix(image.qvec)
points3D = points3D.dot(R.T) + image.tvec[np.newaxis, :]
#points2D = np.hstack((points2D, np.ones((points2D.shape[0], 1))))
#points2D = points2D.dot(np.linalg.inv(camera.get_camera_matrix()).T)[:,:2]
vmask = np.zeros_like(z)
mulmask = np.zeros_like(z)
for k, (u, v) in enumerate(points2D):
j0, i0 = int(u), int(
v) # upper left pixel for the (u,v) coordinate
vmask[i0, j0] = points3D[k, 2]
mulmask[i0, j0] = 1.0
z = cv2.resize(z, (224, 224), interpolation=cv2.INTER_AREA)
z = z[:, :, np.newaxis]
#z = z/100
z = 1.0 / z
vmask = cv2.resize(vmask, (224, 224),
interpolation=cv2.INTER_NEAREST)
mulmask = cv2.resize(mulmask, (224, 224),
interpolation=cv2.INTER_NEAREST)
resized_points2D = []
for i in range(224):
for j in range(224):
if mulmask[i, j] != 0:
resized_points2D.append([i, j])
# import pdb;pdb.set_trace()
# z_stack = []
# points3D_stack = []
# for i in range(len(resized_points2D)):
# i0=int(resized_points2D[i][0])
# j0=int(resized_points2D[i][1])
# z_stack.append(z[i0,j0])
# points3D_stack.append(vmask[i0,j0])
# import pdb;pdb.set_trace()
# m_s = np.median(points3D_stack)/np.median(z_stack)
# scale_m = np.eye(4)
# scale_m[0,0] = m_s
# scale_m[1,1] = m_s
# scale_m[2,2] = m_s
# #z = m_s*z
pad = np.array([[0, 0, 0, 1]])
homo1 = np.append(r1, t1.reshape(3, 1), 1)
homo1 = np.append(homo1, pad, 0)
homo2 = np.append(r2, t2.reshape(3, 1), 1)
homo2 = np.append(homo2, pad, 0)
src2tgt_proj = np.dot(inv(homo2), homo1)
#pred_poses = np.array([src2tgt_proj[0,3],src2tgt_proj[1,3],src2tgt_proj[2,3],])
I1 = I1 / 255.0
I2 = I2 / 255.0
for i in range(1, 100001):
_, pred, reproject_img, tmp_total_loss, tmp_proj, tmp_scale = sess.run(
[
train_op, pred_disp, curr_proj_image, total_loss,
pred_poses, scale_factor
],
feed_dict={
x1: I1[None, :, :, :],
x2: I2[None, :, :, :],
pred_poses: src2tgt_proj[None, :, :],
intrinsics: m_intric[None, :, :],
gt_x1_depth: z[None, :, :, :],
tf_points3D: vmask,
tf_multiply: mulmask,
tf_points2D: np.array(resized_points2D,
dtype=np.float32)
})
# z=cv2.resize(pred[0][0,:,:,0],(FLAGS.image_width,FLAGS.image_height))
# z.astype(np.float32).tofile(FLAGS.output_dir+img_list[i].split('/')[-1]+'_z.bin')
if i >= 0 and i % 10 == 0:
print('Step %s - Loss: %.2f ' % (i, tmp_total_loss))
print('Scale %f' % (tmp_scale[0, 0, 0]))
if i == 1:
test = reproject_img[0]
test = test * 255
test = test.astype(np.uint8)
plt.imshow(test[0])
plt.show()
import pdb
pdb.set_trace()
if i == 200:
test = reproject_img[0]
test = test * 255
test = test.astype(np.uint8)
plt.imshow(test[0])
plt.show()
import pdb
pdb.set_trace()
#print("The %dth frame is processed"%(i))
if i == 500:
z_out = cv2.resize(1.0 / pred[0][0, :, :, 0],
(FLAGS.image_width, FLAGS.image_height))
z_out.astype(
np.float32).tofile(FLAGS.output_dir +
img_list[0].split('/')[-1] +
'_z.bin')
if i == 1000:
test = reproject_img[0]
test = test * 255
test = test.astype(np.uint8)
plt.imshow(test[0])
plt.show()
import pdb
pdb.set_trace()
def onSceneManager(self):
SceneManager(self)
def OnClickRetryButtonExcute(self):
currentScene = SceneManager.GetCurrentScene()
currentScene.resetDataButton.OnClickExcute()
currentScene.SetState("STATE_RETRY")
def estimate_overall_ref_model(colmap_folder, min_track_len, min_tri_angle,
max_tri_angle, image_path):
print 'Loading COLMAP data'
scene_manager = SceneManager(colmap_folder)
scene_manager.load_cameras()
scene_manager.load_images()
images = [
'frame0859.jpg', 'frame0867.jpg', 'frame0875.jpg', 'frame0883.jpg',
'frame0891.jpg'
]
_L = np.array([])
_r = np.array([])
_ndotl = np.array([])
for image_name in images:
image_id = scene_manager.get_image_id_from_name(image_name)
image = scene_manager.images[image_id]
camera = scene_manager.get_camera(image.camera_id)
# image pose
R = util.quaternion_to_rotation_matrix(image.qvec)
print 'Loading 3D points'
scene_manager.load_points3D()
scene_manager.filter_points3D(min_track_len,
min_tri_angle=min_tri_angle,
max_tri_angle=max_tri_angle,
image_list=set([image_id]))
points3D, points2D = scene_manager.get_points3D(image_id)
points3D = points3D.dot(R.T) + image.tvec[np.newaxis, :]
# need to remove redundant points
# http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array
points2D_view = np.ascontiguousarray(points2D).view(
np.dtype((np.void, points2D.dtype.itemsize * points2D.shape[1])))
_, idx = np.unique(points2D_view, return_index=True)
points2D, points3D = points2D[idx], points3D[idx]
# further rule out any points too close to the image border (for bilinear
# interpolation)
mask = ((points2D[:, 0] < camera.width - 1) &
(points2D[:, 1] < camera.height - 1))
points2D, points3D = points2D[mask], points3D[mask]
points2D_image = points2D.copy() # coordinates in image space
points2D = np.hstack((points2D, np.ones((points2D.shape[0], 1))))
points2D = points2D.dot(np.linalg.inv(
camera.get_camera_matrix()).T)[:, :2]
print len(points3D), 'total points'
# load image
#image_file = scene_manager.image_path + image.name
image_file = image_path + image.name
im_rgb = skimage.img_as_float(
skimage.io.imread(image_file)) # color image
L = skimage.color.rgb2lab(im_rgb)[:, :, 0] * 0.01
L = np.maximum(
L, 1e-6) # unfortunately, can't have black pixels, since we
# divide by L
# initial values on unit sphere
x, y = camera.get_image_grid()
print 'Computing nearest neighbors'
kdtree = KDTree(points2D)
weights, nn_idxs = kdtree.query(np.c_[x.ravel(), y.ravel()],
KD_TREE_KNN)
weights = weights.reshape(camera.height, camera.width, KD_TREE_KNN)
nn_idxs = nn_idxs.reshape(camera.height, camera.width, KD_TREE_KNN)
# turn distances into weights for the nearest neighbors
np.exp(-weights, weights) # in-place
# create initial surface on unit sphere
S0 = np.dstack((x, y, np.ones_like(x)))
S0 /= np.linalg.norm(S0, axis=-1)[:, :, np.newaxis]
r_fixed = np.linalg.norm(points3D, axis=-1) # fixed 3D depths
S = S0.copy()
z = S0[:, :, 2]
S, r_ratios = nearest_neighbor_warp(weights, nn_idxs, points2D_image,
r_fixed,
util.generate_surface(camera, z))
z_est = np.maximum(S[:, :, 2], 1e-6)
S = util.generate_surface(camera, z_est)
r = np.linalg.norm(S, axis=-1)
ndotl = util.calculate_ndotl(camera, S)
if _L.size == 0:
_L = L.ravel()
_r = r.ravel()
_ndotl = ndotl.ravel()
else:
_L = np.append(_L, L.ravel())
_r = np.append(_r, r.ravel())
_ndotl = np.append(_ndotl, ndotl.ravel())
#
falloff, model_params, residual = fit_reflectance_model(
sfs.POWER_MODEL, _L, _r, _ndotl, False, 5)
import pdb
pdb.set_trace()
class Solstice(object):
@staticmethod
def _platform_specific_init():
if platform.system() == 'Windows':
os.environ['SDL_AUDIODRIVER'] = 'dsound'
def __init__(self):
Solstice._platform_specific_init()
self._config = Configuration()
self._translations_init()
self._pygame_init()
self._screen = Screen(self._config, _('Solstice'))
self._control = Control(self)
# TODO: Change data file name (don't use .zip extension)
self._resource_manager = ResourceManager(self, 'data.zip')
self._sound_player = SoundPlayer(self)
self._scenes = {
'logo': LogoScene(self),
'intro': IntroScene(self),
'game': GameScene(self),
'elevator': ElevatorScene(self)
}
# The first parameter is the game 'context'
self._scene_manager = SceneManager(self, 'logo')
def _translations_init(self):
gettext.bindtextdomain('solstice', self._config.locale_path)
gettext.textdomain('solstice')
def _sound_preinit(self):
if self._config.sound or self._config.music:
pygame.mixer.pre_init(22050, -16, 2, 1024)
def _pygame_init(self):
self._sound_preinit()
os.environ['SDL_VIDEO_CENTERED'] = '1'
pygame.init()
def run(self):
self._scene_manager.run()
def exit(self, exit_code):
self._config.save()
pygame.quit()
sys.exit(exit_code)
@property
def config(self):
return self._config
@property
def screen(self):
return self._screen
@property
def resource_manager(self):
return self._resource_manager
@property
def control(self):
return self._control
@property
def sound_player(self):
return self._sound_player
@property
def scenes(self):
return self._scenes
@property
def scene_manager(self):
return self._scene_manager
for event in pygame.event.get():
if event.type == QUIT:
pygame.quit()
exit(0)
if event.type == pygame.KEYDOWN:
if event.key == K_ESCAPE:
esc = True
if event.key == pygame.K_DOWN:
down = True
if event.key == pygame.K_UP:
up = True
if event.key == pygame.K_RETURN:
enter = True
if event.type == pygame.KEYUP:
if event.key == K_ESCAPE:
esc = False
if event.key == pygame.K_DOWN:
down = False
if event.key == pygame.K_UP:
up = False
if event.key == pygame.K_RETURN:
enter = False
SceneManager.current_scene.tick(up, down, enter, esc)
SceneManager.current_scene.draw()
SceneManager.check_for_switch()
pygame.display.update()
SceneManager.display.fill([0, 0, 0])
def cvt_bin_txt(model_folder):
converter = SceneManager( model_folder )
converter.load_cameras()
converter.load_images()
converter.load_points3D()
converter.save_cameras(output_folder=model_folder, output_file=None, binary=False)
converter.save_images(output_folder=model_folder, output_file=None, binary=False)
converter.save_points3D(output_folder=model_folder, output_file=None, binary=False)
def run(image_name, image_path, colmap_folder, out_folder, min_track_len,
min_tri_angle, max_tri_angle, ref_surf_name, max_num_points=None,
estimate_falloff=True, use_image_weighted_derivatives=True,get_initial_warp=True):
min_track_len = int(min_track_len)
min_tri_angle = int(min_tri_angle)
max_tri_angle = int(max_tri_angle)
#est_global_ref.estimate_overall_ref_model(colmap_folder,min_track_len,min_tri_angle,max_tri_angle,image_path)
try:
max_num_points = int(max_num_points)
except:
max_num_points = None
get_initial_warp = (get_initial_warp not in ('False', 'false'))
estimate_falloff = (estimate_falloff not in ('False', 'false'))
use_image_weighted_derivatives = (use_image_weighted_derivatives not in ('False', 'false'))
#if not image_path.endswith('/'):
# image_path += '/'
print 'Loading COLMAP data'
scene_manager = SceneManager(colmap_folder)
scene_manager.load_cameras()
scene_manager.load_images()
image_id = scene_manager.get_image_id_from_name(image_name)
image = scene_manager.images[image_id]
camera = scene_manager.get_camera(image.camera_id)
# image pose
R = util.quaternion_to_rotation_matrix(image.qvec)
print 'Loading 3D points'
scene_manager.load_points3D()
scene_manager.filter_points3D(min_track_len,
min_tri_angle=min_tri_angle, max_tri_angle=max_tri_angle,
image_list=set([image_id]))
points3D, points2D = scene_manager.get_points3D(image_id)
points3D = points3D.dot(R.T) + image.tvec[np.newaxis,:]
# need to remove redundant points
# http://stackoverflow.com/questions/16970982/find-unique-rows-in-numpy-array
points2D_view = np.ascontiguousarray(points2D).view(
np.dtype((np.void, points2D.dtype.itemsize * points2D.shape[1])))
_, idx = np.unique(points2D_view, return_index=True)
points2D, points3D = points2D[idx], points3D[idx]
# further rule out any points too close to the image border (for bilinear
# interpolation)
mask = (
(points2D[:,0] < camera.width - 1) & (points2D[:,1] < camera.height - 1))
points2D, points3D = points2D[mask], points3D[mask]
points2D_image = points2D.copy() # coordinates in image space
points2D = np.hstack((points2D, np.ones((points2D.shape[0], 1))))
points2D = points2D.dot(np.linalg.inv(camera.get_camera_matrix()).T)[:,:2]
if (max_num_points is not None and max_num_points > 0 and
max_num_points < len(points2D)):
np.random.seed(0) # fix the "random" points selected
selected_points = np.random.choice(len(points2D), max_num_points, False)
points2D = points2D[selected_points]
points2D_image = points2D_image[selected_points]
points3D = points3D[selected_points]
print len(points3D), 'total points'
#perturb points
#import pdb;pdb.set_trace()
#perturb_points=np.random.choice(len(points2D),max_num_points*0.3,False)
#randomperturb=np.random.rand(perturb_points.size)*2
#points3D[perturb_points,2]=points3D[perturb_points,2]*randomperturb
#points3D[2][2]=points3D[2][2]*0.5
#points3D[4][2]=points3D[4][2]*1.5
#points3D[8][2]=points3D[8][2]*0.3
#points3D[9][2]=points3D[9][2]*2.
# load image
#image_file = scene_manager.image_path + image.name
image_file = image_path + image.name
im_rgb = skimage.img_as_float(skimage.io.imread(image_file)) # color image
L = skimage.color.rgb2lab(im_rgb)[:,:,0] * 0.01
#import pdb;pdb.set_trace()
#skimage.io.imsave('test.png',L)
L = np.maximum(L, 1e-6) # unfortunately, can't have black pixels, since we
# divide by L
# initial values on unit sphere
x, y = camera.get_image_grid()
print 'Computing nearest neighbors'
#import pdb;pdb.set_trace()
nn_idxs, weights = compute_nn(points2D.astype(np.float32), camera.width,
camera.height, camera.fx, camera.fy, camera.cx, camera.cy)
nn_idxs=np.swapaxes(nn_idxs,0,2)
nn_idxs=np.swapaxes(nn_idxs,0,1)
weights=np.swapaxes(weights,0,2)
weights=np.swapaxes(weights,0,1)
#kdtree = KDTree(points2D)
#weights, nn_idxs = kdtree.query(np.c_[x.ravel(),y.ravel()], KD_TREE_KNN)
#weights = weights.reshape(camera.height, camera.width, KD_TREE_KNN)
#nn_idxs = nn_idxs.reshape(camera.height, camera.width, KD_TREE_KNN)
# figure out pixel neighborhoods for each point
#neighborhoods = []
neighborhood_mask = np.zeros((camera.height, camera.width), dtype=np.bool)
for v, u in points2D_image:
rr, cc = circle(int(u), int(v), MAX_POINT_DISTANCE,
(camera.height, camera.width))
#neighborhoods.append((rr, cc))
neighborhood_mask[rr,cc] = True
# turn distances into weights for the nearest neighbors
np.exp(-weights, weights) # in-place
# create initial surface on unit sphere
S0 = np.dstack((x, y, np.ones_like(x)))
S0 /= np.linalg.norm(S0, axis=-1)[:,:,np.newaxis]
r_fixed = np.linalg.norm(points3D, axis=-1) # fixed 3D depths
specular_mask = (L < 1.)
vmask=np.zeros_like(S0[:,:,2])
for k, (u, v) in enumerate(points2D_image):
j0, i0 = int(u), int(v) # upper left pixel for the (u,v) coordinate
vmask[i0,j0]=points3D[k,2]
# iterative SFS algorithm
# model_type: reflectance model type
# fit_falloff: True to fit the 1/r^m falloff parameter; False to fix it at 2
# extra_params: extra parameters to the reflectance model fit function
def run_iterative_sfs(out_file, model_type, fit_falloff, *extra_params):
#if os.path.exists(out_file + '_z.bin'): # don't re-run existing results
# return
if model_type == sfs.LAMBERTIAN_MODEL:
model_name = 'LAMBERTIAN_MODEL'
elif model_type == sfs.OREN_NAYAR_MODEL:
model_name = 'OREN_NAYAR_MODEL'
elif model_type == sfs.PHONG_MODEL:
model_name = 'PHONG_MODEL'
elif model_type == sfs.COOK_TORRANCE_MODEL:
model_name = 'COOK_TORRANCE_MODEL'
elif model_type == sfs.POWER_MODEL:
model_name = 'POWER_MODEL'
if model_type == sfs.POWER_MODEL:
print '%s_%i' % (model_name, extra_params[0])
else:
print model_name
S = S0.copy()
z = S0[:,:,2]
for iteration in xrange(MAX_NUM_ITER):
print 'Iteration', iteration
z_old = z
if get_initial_warp:
#z_gt=util.load_point_ply('C:\\Users\\user\\Documents\\UNC\\Research\\ColonProject\\code\\Rui\\SFS_CPU\\frame0859.jpg_gt.ply')
# warp to 3D points
if iteration>-1:
S, r_ratios = nearest_neighbor_warp(weights, nn_idxs,
points2D_image, r_fixed, util.generate_surface(camera, z))
z_est = np.maximum(S[:,:,2], 1e-6)
S=util.generate_surface(camera, z_est)
else:
z_est=z
S=util.generate_surface(camera, z_est)
#util.save_sfs_ply('warp' + '.ply', S, im_rgb)
#util.save_xyz('test.xyz',points3D);
#z=z_est
#break
#z_est=z
else:
#import pdb;pdb.set_trace()
#z_est = extract_depth_map(camera,ref_surf_name,R,image)
z_est=np.fromfile(
'C:\Users\user\Documents\UNC\Research\ColonProject\code\SFS_Program_from_True\endo_evaluation\gt_surfaces\\frame0859.jpg.bin', dtype=np.float32).reshape(
camera.height, camera.width)/WORLD_SCALE
z_est=z_est.astype(float)
#S, r_ratios = nearest_neighbor_warp(weights, nn_idxs,
# points2D_image, r_fixed, util.generate_surface(camera, z_est))
z_est = np.maximum(z_est[:,:], 1e-6)
#Sworld = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R)
S = util.generate_surface(camera, z_est)
#util.save_sfs_ply('test' + '.ply', S, im_rgb)
#util.save_sfs_ply(out_file + '_warp_%i.ply' % iteration, Sworld, im_rgb)
#import pdb;pdb.set_trace()
# if we need to, make corrections for non-positive depths
#S = util.generate_surface(camera, z_est)
mask = (z_est < INIT_Z)
specular_mask=(L<0.8)
dark_mask=(L>0.1)
_mask=np.logical_and(specular_mask,mask)
_mask=np.logical_and(_mask,dark_mask)
# fit reflectance model
r = np.linalg.norm(S, axis=-1)
ndotl = util.calculate_ndotl(camera, S)
falloff, model_params, residual = fit_reflectance_model(model_type,
L[_mask],r.ravel(), r[_mask],ndotl.ravel(), ndotl[_mask], fit_falloff,camera.width,camera.height, *extra_params)
#r = np.linalg.norm(S[specular_mask], axis=-1)
#import pdb;pdb.set_trace()
#model_params=np.array([26.15969874,-27.674055,-12.52426,7.579855,21.9768004,24.3911142,-21.7282996,-19.850894,-11.62229,-4.837014])
#model_params=np.array([-19.4837,-490.4796,812.4527,-426.09107,139.2602,351.8061,-388.1591,875.5013,-302.4748,-414.4384])
#falloff = 1.2
#ndotl = util.calculate_ndotl(camera, S)[specular_mask]
#falloff, model_params, residual = fit_reflectance_model(model_type,
# L[specular_mask], r, ndotl, fit_falloff, *extra_params)
#r = np.linalg.norm(S[neighborhood_mask], axis=-1)
#ndotl = util.calculate_ndotl(camera, S)[neighborhood_mask]
#falloff, model_params, residual = fit_reflectance_model(model_type,
# L[neighborhood_mask], r, ndotl, fit_falloff, *extra_params)
# lambda values reflect our confidence in the current surface: 0
# corresponds to only using SFS at a pixel, 1 corresponds to equally
# weighting SFS and the current estimate, and larger values increasingly
# favor using only the current estimate
rdiff = np.abs(r_fixed - get_estimated_r(S, points2D_image))
w = np.log10(r_fixed) - np.log10(rdiff) - np.log10(2.)
lambdas = (np.sum(weights * w[nn_idxs], axis=-1) /
np.sum(weights, axis=-1))
lambdas = np.maximum(lambdas, 0.) # just in case
#
lambdas[~mask] = 0
#if iteration == 0: # don't use current estimated surface on first pass
#lambdas = np.zeros_like(z)
#else:
# r_ratios_postwarp = r_fixed / get_estimated_r(S, points2D_image)
# ratio_diff = np.abs(r_ratios_prewarp - r_ratios_postwarp)
# ratio_diff[ratio_diff == 0] = 1e-10 # arbitrarily high lambda
# feature_lambdas = 1. / ratio_diff
# lambdas = (np.sum(weights * feature_lambdas[nn_idxs], axis=-1) /
# np.sum(weights, axis=-1))
# run SFS
H_lut, dH_lut = compute_H_LUT(model_type, model_params, NUM_H_LUT_BINS)
#import pdb;pdb.set_trace()
# run SFS
#H_lut = np.ascontiguousarray(H_lut.astype(np.float32))
#dH_lut = np.ascontiguousarray(dH_lut.astype(np.float32))
z = run_sfs(H_lut,dH_lut,camera, L, lambdas, z_est, model_type, model_params, falloff,vmask,
use_image_weighted_derivatives)
# check for convergence
#diff = np.sum(np.abs(z_old[specular_mask] - z[specular_mask]))
#if diff < CONVERGENCE_THRESHOLD * camera.height * camera.width:
# break
# save the surface
#S = util.generate_surface(camera, z)
#S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R)
#util.save_sfs_ply(out_file + '_%i.ply' % iteration, S, im_rgb)
else:
print 'DID NOT CONVERGE'
#import pdb;pdb.set_trace()
S = util.generate_surface(camera, z)
#S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R)
util.save_sfs_ply(out_file + '.ply', S, im_rgb)
z.astype(np.float32).tofile(out_file + '_z.bin')
# save the surface
#S = util.generate_surface(camera, z)
#S = (S - image.tvec[np.newaxis,np.newaxis,:]).dot(R)
#S, r_ratios = nearest_neighbor_warp(weights, nn_idxs,
# points2D_image, r_fixed, util.generate_surface(camera, z))
#util.save_sfs_ply(out_file + '_warped.ply', S, im_rgb)
#z = np.maximum(S[:,:,2], 1e-6)
#z.astype(np.float32).tofile(out_file + '_warped_z.bin')
#reflectance_models.save_reflectance_model(out_file + '_reflectance.txt',
# model_name, residual, model_params, falloff, *extra_params)
print
# now, actually run SFS
if not out_folder.endswith('/'):
out_folder += '/'
if not os.path.exists(out_folder):
os.makedirs(out_folder)
# if not os.path.exists(out_folder + 'lambertian/'):
# os.mkdir(out_folder + 'lambertian/')
# run_iterative_sfs(out_folder + 'lambertian/' + image.name,
# sfs.LAMBERTIAN_MODEL, estimate_falloff)
# if not os.path.exists(out_folder + 'oren_nayar/'):
# os.mkdir(out_folder + 'oren_nayar/')
# run_iterative_sfs(out_folder + 'oren_nayar/' + image.name,
# sfs.OREN_NAYAR_MODEL, estimate_falloff)
# if not os.path.exists(out_folder + 'phong/'):
# os.mkdir(out_folder + 'phong/')
# run_iterative_sfs(out_folder + 'phong/' + image.name, sfs.PHONG_MODEL,
# estimate_falloff)
# if not os.path.exists(out_folder + 'cook_torrance/'):
# os.mkdir(out_folder + 'cook_torrance/')
# run_iterative_sfs(out_folder + 'cook_torrance/' + image.name,
# sfs.COOK_TORRANCE_MODEL, estimate_falloff)
#for K in [1, 2, 3, 4, 5, 10, 20, 50, 100]:
#for K in [10, 20, 50]:
#for K in [1, 2, 3, 4, 5]:
for K in [100]:
#if not os.path.exists(out_folder + 'power_model_%i/' % K):
# os.mkdir(out_folder + 'power_model_%i/' % K)
run_iterative_sfs(out_folder + image.name,
sfs.POWER_MODEL, estimate_falloff, K)