def createRendererGT(glMode, chAz, chEl, chDist, center, v, vc, f_list, vn,
light_color, chComponent, chVColors, targetPosition,
chDisplacement, width, height, uv, haveTextures_list,
textures_list, frustum, win):
renderer = TexturedRenderer()
renderer.set(glMode=glMode)
vflat = [item for sublist in v for item in sublist]
rangeMeshes = range(len(vflat))
# vch = [ch.array(vflat[mesh]) for mesh in rangeMeshes]
vch = vflat
if len(vch) == 1:
vstack = vch[0]
else:
vstack = ch.vstack(vch)
camera, modelRotation, _ = setupCamera(
vstack, chAz, chEl, chDist, center + targetPosition + chDisplacement,
width, height)
vnflat = [item for sublist in vn for item in sublist]
# vnch = [ch.array(vnflat[mesh]) for mesh in rangeMeshes]
# vnchnorm = [vnch[mesh]/ch.sqrt(vnch[mesh][:,0]**2 + vnch[mesh][:,1]**2 + vnch[mesh][:,2]**2).reshape([-1,1]) for mesh in rangeMeshes]
vcflat = [item for sublist in vc for item in sublist]
vcch = [vcflat[mesh] for mesh in rangeMeshes]
# vc_list = computeGlobalAndPointLighting(vch, vnch, vcch, lightPosGT, chGlobalConstantGT, light_colorGT)
setupTexturedRenderer(renderer, vstack, vch, f_list, vcch, vnflat, uv,
haveTextures_list, textures_list, camera, frustum,
win)
return renderer
def __init__(self,
width=800,
height=600,
near=0.5,
far=1000,
faces=None,
tex=None,
vt=None,
ft=None):
self.colors = {
'pink': [.9, .7, .7],
'light_blue': [0.65098039, 0.74117647, 0.85882353],
'blue': [0.65098039, 0.74117647, 0.85882353]
}
self.width = width
self.height = height
self.faces = faces
self.tex = tex
self.vt = vt
self.ft = ft
self.renderer = TexturedRenderer(f=faces,
vt=self.vt,
ft=self.ft,
bgcolor=np.ones(3),
texture_image=self.tex)
def createRendererTarget(glMode, chAz, chEl, chDist, center, v, vc, f_list, vn,
light_color, chComponent, chVColors, targetPosition,
chDisplacement, width, height, uv, haveTextures_list,
textures_list, frustum, win):
renderer = TexturedRenderer()
renderer.set(glMode=glMode)
vflat = [item for sublist in v for item in sublist]
rangeMeshes = range(len(vflat))
vnflat = [item for sublist in vn for item in sublist]
vcflat = [item for sublist in vc for item in sublist]
# vcch = [np.ones_like(vcflat[mesh])*chVColors.reshape([1,3]) for mesh in rangeMeshes]
vc_list = computeSphericalHarmonics(vnflat, vcflat, light_color,
chComponent)
if len(vflat) == 1:
vstack = vflat[0]
else:
vstack = ch.vstack(vflat)
camera, modelRotation, _ = setupCamera(
vstack, chAz, chEl, chDist, center + targetPosition + chDisplacement,
width, height)
setupTexturedRenderer(renderer, vstack, vflat, f_list, vc_list, vnflat, uv,
haveTextures_list, textures_list, camera, frustum,
win)
return renderer
def create_renderer(w=640,
h=480,
rt=np.zeros(3),
t=np.zeros(3),
f=None,
c=None,
k=None,
near=.5,
far=10.,
mesh=None):
f = np.array([w, w]) / 2. if f is None else f
c = np.array([w, h]) / 2. if c is None else c
k = np.zeros(5) if k is None else k
if mesh is not None and hasattr(mesh, 'texture_image'):
from opendr.renderer import TexturedRenderer
rn = TexturedRenderer()
rn.texture_image = mesh.texture_image
if rn.texture_image.max() > 1:
rn.texture_image[:] = rn.texture_image[:].r / 255.
rn.ft = mesh.ft
rn.vt = mesh.vt
else:
from opendr.renderer import ColoredRenderer
rn = ColoredRenderer()
rn.camera = ProjectPoints(rt=rt, t=t, f=f, c=c, k=k)
rn.frustum = {'near': near, 'far': far, 'height': h, 'width': w}
return rn
def test_earth():
m = get_earthmesh(trans=ch.array([0, 0, 0]), rotation=ch.zeros(3))
# Create V, A, U, f: geometry, brightness, camera, renderer
V = ch.array(m.v)
A = SphericalHarmonics(vn=VertNormals(v=V, f=m.f),
components=[3., 2., 0., 0., 0., 0., 0., 0., 0.],
light_color=ch.ones(3))
# camera
U = ProjectPoints(v=V,
f=[w, w],
c=[w / 2., h / 2.],
k=ch.zeros(5),
t=ch.zeros(3),
rt=ch.zeros(3))
f = TexturedRenderer(vc=A,
camera=U,
f=m.f,
bgcolor=[0., 0., 0.],
texture_image=m.texture_image,
vt=m.vt,
ft=m.ft,
frustum={
'width': w,
'height': h,
'near': 1,
'far': 20
})
# Parameterize the vertices
translation, rotation = ch.array([0, 0, 8]), ch.zeros(3)
f.v = translation + V.dot(Rodrigues(rotation))
observed = f.r
np.random.seed(1)
# this is reactive
# in the sense that changes to values will affect function which depend on them.
translation[:] = translation.r + np.random.rand(3)
rotation[:] = rotation.r + np.random.rand(3) * .2
# Create the energy
E_raw = f - observed
E_pyr = gaussian_pyramid(E_raw, n_levels=6, normalization='size')
Image.fromarray((observed * 255).astype(np.uint8)).save(
os.path.join(save_dir, "reference.png"))
step = 0
Image.fromarray((f.r * 255).astype(np.uint8)).save(
os.path.join(save_dir, "step_{:05d}.png".format(step)))
print('OPTIMIZING TRANSLATION, ROTATION, AND LIGHT PARMS')
free_variables = [translation, rotation]
ch.minimize({'pyr': E_pyr}, x0=free_variables, callback=create_callback(f))
ch.minimize({'raw': E_raw}, x0=free_variables, callback=create_callback(f))
def _create_renderer_mesh( # pylint: disable=too-many-arguments
w=640,
h=480,
rt=np.zeros(3),
t=np.zeros(3),
f=None,
c=None,
k=None,
near=1.,
far=10.,
texture=None):
"""Create a renderer for the specified parameters."""
f = np.array([w, w]) / 2. if f is None else f
c = np.array([w, h]) / 2. if c is None else c
k = np.zeros(5) if k is None else k
if texture is not None:
rn = TexturedRenderer()
else:
rn = ColoredRenderer()
rn.camera = ProjectPoints(rt=rt, t=t, f=f, c=c, k=k)
rn.frustum = {'near': near, 'far': far, 'height': h, 'width': w}
if texture is not None:
rn.texture_image = np.asarray(texture, np.float64) / 255.
rn.texture_image = rn.texture_image[:, :, ::-1]
#print rn.texture_image.shape
return rn
def createRenderer(glMode, cameraParams, v, vc, f_list, vn, uv,
haveTextures_list, textures_list, frustum, win):
renderer = TexturedRenderer()
renderer.set(glMode=glMode)
vflat = [item for sublist in v for item in sublist]
if len(vflat) == 1:
vstack = vflat[0]
else:
vstack = ch.vstack(vflat)
camera, modelRotation, _ = setupCamera(vstack, cameraParams)
vnflat = [item for sublist in vn for item in sublist]
vcflat = [item for sublist in vc for item in sublist]
setupTexturedRenderer(renderer, vstack, vflat, f_list, vcflat, vnflat, uv,
haveTextures_list, textures_list, camera, frustum,
win)
return renderer
def generate(self, img_bgr, texture_bgr):
img = img_bgr
self.set_texture(texture_bgr)
vert_shifted, theta, cam_for_render = self.hmr.predict(img)
pose = theta[self.num_cam:(self.num_cam + self.num_theta)]
beta = theta[(self.num_cam + self.num_theta):]
self.body.pose[:] = pose
self.body.betas[:] = beta
rn_vis = TexturedRenderer()
rn_vis.camera = ProjectPoints(t=np.zeros(3),
rt=np.zeros(3),
c=cam_for_render[1:],
f=np.ones(2) * cam_for_render[0],
k=np.zeros(5),
v=vert_shifted)
rn_vis.frustum = {
'near': 0.1,
'far': 1000.,
'width': self.width,
'height': self.height
}
rn_vis.set(v=vert_shifted,
f=self.m.f,
vc=self.m.vc,
texture_image=self.m.texture_image,
ft=self.m.ft,
vt=self.m.vt,
bgcolor=np.zeros(3))
# rn_vis.background_image = img_bgr / 255. if img_bgr.max() > 1 else img_bgr
out_img = rn_vis.r
out_img = (out_img * 255).astype(np.uint8)
out_img = cv2.cvtColor(out_img, cv2.COLOR_RGB2BGR)
silhouette_rn = ColoredRenderer()
silhouette_rn.camera = ProjectPoints(v=self.body,
rt=np.zeros(3),
t=np.zeros(3),
f=np.ones(2) * cam_for_render[0],
c=cam_for_render[1:],
k=np.zeros(5))
silhouette_rn.frustum = {
'near': 0.1,
'far': 1000.,
'width': self.width,
'height': self.height
}
silhouette_rn.set(v=vert_shifted,
f=self.m.f,
vc=self.m.vc,
bgcolor=np.zeros(3))
return out_img, texture_dr_wrt(rn_vis,
silhouette_rn.r), silhouette_rn.r
def render(self, thetas, texture_bgr, rotate=np.array([0, 0, 0]), background_img=None):
"""
get the rendered image and rendered silhouette
:param thetas: model parameters, 3 * camera parameter + 72 * body pose + 10 * body shape
:param texture_bgr: texture image in bgr format
:return: the rendered image and deviation of rendered image to texture image
(rendered image, deviation of rendered image, silhouette)
"""
self.set_texture(texture_bgr)
thetas = thetas.reshape(-1)
cams = thetas[:self.num_cam]
theta = thetas[self.num_cam: (self.num_cam + self.num_theta)]
beta = thetas[(self.num_cam + self.num_theta):]
self.body.pose[:] = theta
self.body.betas[:] = beta
#
# size = cams[0] * min(self.w, self.h)
# position = cams[1:3] * min(self.w, self.h) / 2 + min(self.w, self.h) / 2
"""
####################################################################
ATTENTION!
I do not know why the flength is 500.
But it worked
####################################################################
"""
texture_rn = TexturedRenderer()
texture_rn.camera = ProjectPoints(v=self.body, rt=rotate, t=ch.array([0, 0, 2]),
f=np.ones(2) * self.img_size * 0.62,
c=np.array([self.w / 2, self.h / 2]),
k=ch.zeros(5))
texture_rn.frustum = {'near': 1., 'far': 10., 'width': self.w, 'height': self.h}
texture_rn.set(v=self.body, f=self.m.f, vc=self.m.vc, texture_image=self.m.texture_image, ft=self.m.ft,
vt=self.m.vt)
if background_img is not None:
texture_rn.background_image = background_img / 255. if background_img.max() > 1 else background_img
silhouette_rn = ColoredRenderer()
silhouette_rn.camera = ProjectPoints(v=self.body, rt=rotate, t=ch.array([0, 0, 2]),
f=np.ones(2) * self.img_size * 0.62,
c=np.array([self.w / 2, self.h / 2]),
k=ch.zeros(5))
silhouette_rn.frustum = {'near': 1., 'far': 10., 'width': self.w, 'height': self.h}
silhouette_rn.set(v=self.body, f=self.m.f, vc=np.ones_like(self.body), bgcolor=np.zeros(3))
return texture_rn.r, texture_dr_wrt(texture_rn, silhouette_rn.r), silhouette_rn.r
class TexRenderer(object):
"""
Render mesh using OpenDR for visualization.
"""
def __init__(self,
width=800,
height=600,
near=0.5,
far=1000,
faces=None,
tex=None,
vt=None,
ft=None):
self.colors = {
'pink': [.9, .7, .7],
'light_blue': [0.65098039, 0.74117647, 0.85882353],
'blue': [0.65098039, 0.74117647, 0.85882353]
}
self.width = width
self.height = height
self.faces = faces
self.tex = tex
self.vt = vt
self.ft = ft
self.renderer = TexturedRenderer(f=faces,
vt=self.vt,
ft=self.ft,
bgcolor=np.ones(3),
texture_image=self.tex)
def render(self,
vertices,
faces=None,
img=None,
camera_t=np.zeros([3], dtype=np.float32),
camera_rot=np.zeros([3], dtype=np.float32),
camera_center=None,
use_bg=False,
bg_color=(0.0, 0.0, 0.0),
body_color=None,
focal_length=5000,
disp_text=False,
gt_keyp=None,
pred_keyp=None,
**kwargs):
if img is not None:
height, width = img.shape[:2]
else:
height, width = self.height, self.width
if faces is None:
faces = self.faces
if camera_center is None:
camera_center = np.array([width * 0.5, height * 0.5])
self.renderer.camera = ProjectPoints(rt=camera_rot,
t=camera_t,
f=focal_length * np.ones(2),
c=camera_center,
k=np.zeros(5))
dist = np.abs(self.renderer.camera.t.r[2] -
np.mean(vertices, axis=0)[2])
far = dist + 20
self.renderer.frustum = {
'near': 1.0,
'far': far,
'width': width,
'height': height
}
if img is not None:
if use_bg:
self.renderer.background_image = img
else:
self.renderer.background_image = np.ones_like(img) * np.array(
bg_color)
if body_color is None:
color = self.colors['blue']
else:
color = self.colors[body_color]
if isinstance(self.renderer, TexturedRenderer):
color = [1., 1., 1.]
self.renderer.set(v=vertices,
f=faces,
vt=self.vt,
ft=self.ft,
vc=color,
bgcolor=np.ones(3),
texture_image=self.tex)
albedo = self.renderer.vc
# Construct Back Light (on back right corner)
yrot = np.radians(120)
self.renderer.vc = LambertianPointLight(
f=self.renderer.f,
v=self.renderer.v,
num_verts=self.renderer.v.shape[0],
light_pos=rotateY(np.array([-200, -100, -100]), yrot),
vc=albedo,
light_color=np.array([1, 1, 1]))
# Construct Left Light
self.renderer.vc += LambertianPointLight(
f=self.renderer.f,
v=self.renderer.v,
num_verts=self.renderer.v.shape[0],
light_pos=rotateY(np.array([800, 10, 300]), yrot),
vc=albedo,
light_color=np.array([1, 1, 1]))
# Construct Right Light
self.renderer.vc += LambertianPointLight(
f=self.renderer.f,
v=self.renderer.v,
num_verts=self.renderer.v.shape[0],
light_pos=rotateY(np.array([-500, 500, 1000]), yrot),
vc=albedo,
light_color=np.array([1, 1, 1]))
tmp = self.renderer.r
return tmp
def generateSceneImages(width, height, envMapFilename, envMapMean, phiOffset,
chAzGT, chElGT, chDistGT, light_colorGT, chComponentGT,
glMode):
replaceableScenesFile = '../databaseFull/fields/scene_replaceables_backup.txt'
sceneLines = [line.strip() for line in open(replaceableScenesFile)]
for sceneIdx in np.arange(len(sceneLines)):
sceneNumber, sceneFileName, instances, roomName, roomInstanceNum, targetIndices, targetPositions = scene_io_utils.getSceneInformation(
sceneIdx, replaceableScenesFile)
sceneDicFile = 'data/scene' + str(sceneNumber) + '.pickle'
bpy.ops.wm.read_factory_settings()
scene_io_utils.loadSceneBlendData(sceneIdx, replaceableScenesFile)
scene = bpy.data.scenes['Main Scene']
bpy.context.screen.scene = scene
scene_io_utils.setupScene(scene, roomInstanceNum, scene.world,
scene.camera, width, height, 16, True, False)
scene.update()
scene.render.resolution_x = width #perhaps set resolution in code
scene.render.resolution_y = height
scene.render.tile_x = height / 2
scene.render.tile_y = width
scene.cycles.samples = 100
addEnvironmentMapWorld(envMapFilename, scene)
setEnviornmentMapStrength(0.3 / envMapMean, scene)
rotateEnviornmentMap(phiOffset, scene)
if not os.path.exists('scenes/' + str(sceneNumber)):
os.makedirs('scenes/' + str(sceneNumber))
for targetIdx, targetIndex in enumerate(targetIndices):
targetPosition = targetPositions[targetIdx]
rendererGT.clear()
del rendererGT
v, f_list, vc, vn, uv, haveTextures_list, textures_list = scene_io_utils.loadSavedScene(
sceneDicFile)
# removeObjectData(targetIndex, v, f_list, vc, vn, uv, haveTextures_list, textures_list)
# addObjectData(v, f_list, vc, vn, uv, haveTextures_list, textures_list, v_teapots[currentTeapotModel][0], f_list_teapots[currentTeapotModel][0], vc_teapots[currentTeapotModel][0], vn_teapots[currentTeapotModel][0], uv_teapots[currentTeapotModel][0], haveTextures_list_teapots[currentTeapotModel][0], textures_list_teapots[currentTeapotModel][0])
vflat = [item for sublist in v for item in sublist]
rangeMeshes = range(len(vflat))
vch = [ch.array(vflat[mesh]) for mesh in rangeMeshes]
# vch[0] = ch.dot(vch[0], scaleMatGT) + targetPosition
if len(vch) == 1:
vstack = vch[0]
else:
vstack = ch.vstack(vch)
cameraGT, modelRotationGT = setupCamera(vstack, chAzGT, chElGT,
chDistGT, targetPosition,
width, height)
# cameraGT, modelRotationGT = setupCamera(vstack, chAzGT, chElGT, chDistGT, center + targetPosition, width, height)
vnflat = [item for sublist in vn for item in sublist]
vnch = [ch.array(vnflat[mesh]) for mesh in rangeMeshes]
vnchnorm = [
vnch[mesh] /
ch.sqrt(vnch[mesh][:, 0]**2 + vnch[mesh][:, 1]**2 +
vnch[mesh][:, 2]**2).reshape([-1, 1])
for mesh in rangeMeshes
]
vcflat = [item for sublist in vc for item in sublist]
vcch = [ch.array(vcflat[mesh]) for mesh in rangeMeshes]
vc_list = computeSphericalHarmonics(vnchnorm, vcch, light_colorGT,
chComponentGT)
rendererGT = TexturedRenderer()
rendererGT.set(glMode=glMode)
setupTexturedRenderer(rendererGT, vstack, vch, f_list, vc_list,
vnchnorm, uv, haveTextures_list,
textures_list, cameraGT, frustum, win)
cv2.imwrite(
'scenes/' + str(sceneNumber) + '/opendr_' + str(targetIndex) +
'.png', 255 * rendererGT.r[:, :, [2, 1, 0]])
placeCamera(scene.camera, -chAzGT[0].r * 180 / np.pi,
chElGT[0].r * 180 / np.pi, chDistGT, targetPosition)
scene.update()
scene.render.filepath = 'scenes/' + str(
sceneNumber) + '/blender_' + str(targetIndex) + '.png'
bpy.ops.render.render(write_still=True)
bg = cv2.imread(bg_file)
x = 0
y = 0
bg_im = bg[y:y+h, x:x+w].astype(np.float64)/255
txt_idx = num % len(txt_list)
txt_file = txt_list[txt_idx]
txt = cv2.imread(txt_file)
txt = cv2.cvtColor(txt, cv2.COLOR_BGR2RGB)
txt_im = txt.astype(np.float64)/255
cam = ProjectPoints(v=m.r, rt=np.zeros(3), t = np.array([0, -.1, .5]), f=np.array([w,w])/2., c=np.array([w,h])/2, k=np.zeros(5))
rn = TexturedRenderer(v=m.r, f=m.f, vc=np.ones_like(m.r), vt=tmpl.vt, ft=tmpl.ft,
texture_image = txt_im, background_image = bg_im,
camera = cam,
frustum = {'near': .1, 'far': 10., 'width': w, 'height': h},
overdraw=False)
data = 255 * rn.r
cv2.imshow('render_SMIL', rn.r)
file_name = 'syn' + str(num) + '.jpg'
syn_file = os.path.join(syn_folder, file_name)
cv2.imwrite(syn_file, data)
