def main_loop():
global DESKTOP_MODE, SERVER_MODE, VIEW_MODE
window_server = np.ones((256, 256, 3), dtype=np.uint8) * 255
window_desktop = np.ones((512, 1024, 3), dtype=np.uint8) * 255
create_opengl_context(256, 256)
renderer = AlbedoRender(width=256, height=256, multi_sample_rate=1)
renderer.set_attrib(0, scene.vert_data)
renderer.set_attrib(1, scene.uv_data)
renderer.set_texture("TargetTexture", scene.texture_image)
def render(extrinsic, intrinsic):
uniform_dict = {"ModelMat": extrinsic, "PerspMat": intrinsic}
renderer.draw(uniform_dict)
color = (renderer.get_color() * 255).astype(np.uint8)
background = cv2.cvtColor(color, cv2.COLOR_RGB2BGR)
background = cv2.resize(background, (256, 256))
return background
for data_dict in tqdm.tqdm(loader):
render_norm = data_dict["render_norm"] # [256, 256, 3] RGB
render_tex = data_dict["render_tex"] # [256, 256, 3] RGB
mask = data_dict["mask"]
extrinsic = data_dict["extrinsic"]
intrinsic = data_dict["intrinsic"]
if SERVER_MODE == "NORM":
background = render(extrinsic, intrinsic)
if mask is None:
window_server = background
else:
window_server = np.uint8(mask * render_norm +
(1 - mask) * background)
elif SERVER_MODE == "TEXTURE":
background = render(extrinsic, intrinsic)
if mask is None:
window_server = background
else:
window_server = np.uint8(mask * render_tex +
(1 - mask) * background)
else:
if render_norm is not None:
window_server = np.uint8(render_norm)
yield window_server
def __init__(self, size=(512, 512), debug=False):
self.floors = ["carpet", "drum", "grass", "mousemat", "table"]
self.vert_data, self.uv_data, self.texture_image = _load_floor()
self.intrinsic = _load_intrinsic()
create_opengl_context(size[0], size[1])
self.renderer = AlbedoRender(width=size[0], height=size[1], multi_sample_rate=1)
self.renderer.set_attrib(0, self.vert_data)
self.renderer.set_attrib(1, self.uv_data)
self.renderer.set_texture('TargetTexture', self.texture_image)
self.extrinsic = np.array([
[1.0, 0.0, 0.0, 0.0],
[0.0, 1.0, 0.0, 0.0],
[0.0, 0.0, 1.0, -2.0],
[0.0, 0.0, 0.0, 1.0],
], dtype=np.float32)
self.step = 0
def main_loop():
global DESKTOP_MODE, SERVER_MODE, VIEW_MODE
window_server = np.ones((256, 256, 3), dtype=np.uint8) * 255
window_desktop = np.ones((512, 1024, 3), dtype=np.uint8) * 255
create_opengl_context(256, 256)
renderer = AlbedoRender(width=256, height=256, multi_sample_rate=1)
renderer.set_attrib(0, scene.vert_data)
renderer.set_attrib(1, scene.uv_data)
renderer.set_texture('TargetTexture', scene.texture_image)
def render(extrinsic, intrinsic):
uniform_dict = {'ModelMat': extrinsic, 'PerspMat': intrinsic}
renderer.draw(uniform_dict)
color = (renderer.get_color() * 255).astype(np.uint8)
background = cv2.cvtColor(color, cv2.COLOR_RGB2BGR)
background = cv2.resize(background, (256, 256))
return background
for data_dict in tqdm.tqdm(loader):
render_norm = data_dict["render_norm"] # [256, 256, 3] RGB
render_tex = data_dict["render_tex"] # [256, 256, 3] RGB
mask = data_dict["mask"]
extrinsic = data_dict["extrinsic"]
intrinsic = data_dict["intrinsic"]
if DESKTOP_MODE is not None:
input4c = data_dict["segm"].cpu().numpy()[0].transpose(
1, 2, 0) # [512, 512, 4]
input = (input4c[:, :, 0:3] * 0.5) + 0.5
if DESKTOP_MODE == 'SEGM':
segmentation = (input4c[:, :, 0:3] * input4c[:, :, 3:4] *
0.5) + 0.5
window_desktop = np.uint8(
np.hstack([input * 255, segmentation * 255])) # RGB
elif DESKTOP_MODE == 'NORM':
if render_norm is None:
render_norm = np.ones((256, 256, 3), dtype=np.float32) * 255
window_desktop = np.uint8(
np.hstack([input * 255,
cv2.resize(render_norm, (512, 512))])) # RGB
elif DESKTOP_MODE == 'TEXTURE':
if render_tex is None:
render_tex = np.ones((256, 256, 3), dtype=np.float32) * 255
window_desktop = np.uint8(
np.hstack([input * 255,
cv2.resize(render_tex, (512, 512))])) # RGB
elif DESKTOP_MODE == 'TEXTURE_NORM':
if render_tex is None:
render_tex = np.ones((256, 256, 3), dtype=np.float32) * 255
if render_norm is None:
render_norm = np.ones((256, 256, 3), dtype=np.float32) * 255
window_desktop = np.uint8(np.vstack([render_tex,
render_norm])) # RGB
else:
window_desktop = None
if DESKTOP_MODE is not None:
# window_desktop = cv2.resize(window_desktop, (2400, 1200))
cv2.namedWindow('window_desktop', cv2.WINDOW_NORMAL)
# cv2.setWindowProperty('window_desktop', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
cv2.imshow('window_desktop', window_desktop[:, :, ::-1])
if args.use_server:
if SERVER_MODE == 'NORM':
background = render(extrinsic, intrinsic)
if mask is None:
window_server = background
else:
window_server = np.uint8(mask * render_norm +
(1 - mask) * background)
elif SERVER_MODE == 'TEXTURE':
background = render(extrinsic, intrinsic)
if mask is None:
window_server = background
else:
window_server = np.uint8(mask * render_tex +
(1 - mask) * background)
else:
if render_norm is not None:
window_server = np.uint8(render_norm)
# yield window_desktop, window_server
(flag, encodedImage) = cv2.imencode(".jpg",
window_server[:, :, ::-1])
if not flag:
continue
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' +
bytearray(encodedImage) + b'\r\n')
key = cv2.waitKey(1)
if key == ord('q'):
DESKTOP_MODE = 'SEGM'
elif key == ord('w'):
DESKTOP_MODE = 'NORM'
elif key == ord('e'):
DESKTOP_MODE = 'TEXTURE'
elif key == ord('r'):
DESKTOP_MODE = 'TEXTURE_NORM'
# elif key == ord('a'):
# SERVER_MODE = 'SEGM'
elif key == ord('s'):
SERVER_MODE = 'NORM'
elif key == ord('d'):
SERVER_MODE = 'TEXTURE'
elif key == ord('f'):
SERVER_MODE = None
elif key == ord('z'):
VIEW_MODE = 'FRONT'
elif key == ord('x'):
VIEW_MODE = 'BACK'
elif key == ord('c'):
VIEW_MODE = 'LEFT'
elif key == ord('v'):
VIEW_MODE = 'RIGHT'
elif key == ord('b'):
VIEW_MODE = 'AUTO'
elif key == ord('n'):
VIEW_MODE = 'LOAD'
elif key == ord('g'):
scene.shift_floor()