def __init__(self, camera_mode='projection', K = None, R = None, t = None,\
smpl_model_path = "/hdd/zen/dev/ActMix/actmix/DataGen/MotionRender/ActmixTorchGenerator/smpl_models/",\
texture_path = "/hdd/zen/data/SURREAL/smpl_data/", dtype = torch.float32, gender = "male", image_size = 256, \
device =(torch.device("cuda", index=0) if torch.cuda.is_available() else torch.device("cpu")), \
background_color=[0,0,0],
):
self.dtype = dtype
self.camera_mode = camera_mode
self.image_size = image_size
r = sciR.from_euler('zxy', [0, 0, 0], degrees = True)
vet_rot = r.as_matrix()
fx = 500.0
fy = 500.0
cx = 512.0
cy = 512.0
def_K = np.array( [ [fx, 0., cx],
[ 0. ,fx, cy],
[0.,0.,1.]])
def_K = def_K[None, :]
def_R = vet_rot[None, :]
def_t = np.array([0,0,1.5])
if K is None:
K = def_K
if R is None:
R = def_R
if t is None:
t = def_t
if camera_mode == "look_at":
self.renderer = nr.Renderer(camera_mode='look_at', viewing_angle = 30, image_size = image_size, background_color= background_color)
self.set_render_angles(2.5, 0, 0)
elif camera_mode == "projection":
self.renderer = nr.Renderer(K = K, R = R, t = t, camera_mode='projection', image_size = image_size, light_intensity_ambient=1, background_color= background_color)
self.renderer = self.renderer.to(device)
self.gender = gender
self.faces = np.load(os.path.join(smpl_model_path, "smpl_faces.npy"))[np.newaxis, :,:]
self.device = device
self.faces_uv = np.load(os.path.join(texture_path, 'final_faces_uv_mapping.npy'))
self.uv_sampler = torch.from_numpy(self.faces_uv.reshape(-1, 2, 2, 2)).to(device).type(self.dtype)
self.uv_sampler = self.uv_sampler.view(-1, 13776, 2*2, 2)
self.male_texture_dataset = TextureDataset(texture_path, "male")
self.male_smpl_p = SMPL_Parser(device = device, gender = "male")
self.female_texture_dataset = TextureDataset(texture_path, "female")
self.female_smpl_p = SMPL_Parser(device = device, gender = "female")
self.smpl_p = self.male_smpl_p if self.gender == 'male' else self.female_smpl_p
self.texture_dataset = self.male_texture_dataset if self.gender == 'male' else self.female_texture_dataset
def __init__(self, filename_obj, filename_ref=None):
super(Model, self).__init__()
# Load mesh vertices and faces
vertices, faces = load_off(filename_obj)
self.register_buffer('vertices', vertices[None, :, :])
self.register_buffer('faces', faces[None, :, :])
# create textures
texture_size = 2
textures = torch.ones(1,
self.faces.shape[1],
texture_size,
texture_size,
texture_size,
3,
dtype=torch.float32)
self.register_buffer('textures', textures)
# load reference image
image_ref = torch.from_numpy(
(imread(filename_ref).max(-1)).astype(np.float32))
# print(image_ref.unique())
# # TODO: try it without thresholding, just normalize
self.register_buffer('image_ref', image_ref)
# camera parameters
self.camera_position = nn.Parameter(
torch.from_numpy(np.array([6, 10, -14], dtype=np.float32)))
# setup renderer
renderer = nr.Renderer(camera_mode='look_at')
renderer.eye = self.camera_position
self.renderer = renderer
def __init__(self,
img_size=256,
face_path='models/smpl_faces.npy',
t_size=1):
self.renderer = nr.Renderer(img_size,
camera_mode='look_at',
perspective=False)
self.set_light_dir([1, .5, -1], int_dir=0.3, int_amb=0.7)
self.set_bgcolor([1, 1, 1.])
self.img_size = img_size
self.faces_np = np.load(face_path).astype(np.int)
self.faces = to_variable(torch.IntTensor(self.faces_np).cuda())
if self.faces.dim() == 2:
self.faces = torch.unsqueeze(self.faces, 0)
# Default color:
default_tex = np.ones(
(1, self.faces.shape[1], t_size, t_size, t_size, 3))
self.default_tex = to_variable(torch.FloatTensor(default_tex).cuda())
# Default camera:
cam = np.hstack([0.9, 0, 0])
default_cam = to_variable(torch.FloatTensor(cam).cuda())
self.default_cam = torch.unsqueeze(default_cam, 0)
# Setup proj fn:
self.proj_fn = orthographic_proj_withz_idrot
def __init__(self, filename_obj, filename_ref):
super(Model, self).__init__()
vertices, faces = nr.load_obj(filename_obj)
self.register_buffer('vertices', vertices[None, :, :])
self.register_buffer('faces', faces[None, :, :])
# create textures
texture_size = 4
textures = torch.zeros(1,
self.faces.shape[1],
texture_size,
texture_size,
texture_size,
3,
dtype=torch.float32)
self.textures = nn.Parameter(textures)
# load reference image
image_ref = torch.from_numpy(
imread(filename_ref).astype('float32') / 255.).permute(2, 0,
1)[None, ::]
self.register_buffer('image_ref', image_ref)
# setup renderer
renderer = nr.Renderer(camera_mode='look_at')
renderer.perspective = False
renderer.light_intensity_directional = 0.0
renderer.light_intensity_ambient = 1.0
self.renderer = renderer
def test_backward_case1(self):
"""Backward if non-zero gradient is out of a face."""
vertices = [
[0.8, 0.8, 1.],
[0.0, -0.5, 1.],
[0.2, -0.4, 1.]]
faces = [[0, 1, 2]]
pxi = 35
pyi = 25
grad_ref = [
[1.6725862, -0.26021874, 0.],
[1.41986704, -1.64284933, 0.],
[0., 0., 0.],
]
renderer = nr.Renderer(camera_mode='look_at')
renderer.image_size = 64
renderer.anti_aliasing = False
renderer.perspective = False
vertices = torch.from_numpy(np.array(vertices, np.float32)).cuda()
faces = torch.from_numpy(np.array(faces, np.int32)).cuda()
grad_ref = torch.from_numpy(np.array(grad_ref, np.float32)).cuda()
vertices, faces, grad_ref = utils.to_minibatch((vertices, faces, grad_ref))
vertices.requires_grad = True
images = renderer(vertices, faces, mode='silhouettes')
loss = torch.sum(torch.abs(images[:, pyi, pxi] - 1))
loss.backward()
assert(torch.allclose(vertices.grad, grad_ref, rtol=1e-2))
def __init__(self, filename_obj, filename_ref):
super(Model, self).__init__()
with self.init_scope():
# load .obj
vertices, faces = neural_renderer.load_obj(filename_obj)
self.vertices = chainer.Parameter(vertices[None, :, :])
self.faces = faces[None, :, :]
# create textures
texture_size = 2
textures = np.ones((1, self.faces.shape[1], texture_size,
texture_size, texture_size, 3), 'float32')
self.textures = textures
# load reference image
#self.image_ref = scipy.misc.imread(filename_ref).astype('float32').mean(-1) / 255.
image_ref = scipy.misc.imread(filename_ref).astype('float32').mean(
-1) / 255.
h = image_ref.shape[0]
w = image_ref.shape[1]
hcrop = wcrop = self.image_size = 256 #np.min(image_ref.shape[:1])
top = int(math.floor((h - hcrop) / 2.0))
bottom = int(math.ceil((h + hcrop) / 2.0))
left = int(math.floor((w - wcrop) / 2.0))
right = int(math.ceil((w + wcrop) / 2.0))
if len(image_ref.shape) > 2:
self.image_ref = image_ref[top:bottom, left:right, :]
else:
self.image_ref = image_ref[top:bottom, left:right]
# setup renderer
renderer = neural_renderer.Renderer()
self.renderer = renderer
def __init__(self, vertices, faces, filename_ref, camera_distance, camera_elevation, camera_azimuth,
base_textures=None):
super(ModelTextures, self).__init__()
self.register_buffer('vertices', vertices)
self.register_buffer('faces', faces)
# Create textures
# If no base textures are specified they are initialized to white
if base_textures is None:
texture_size = 2
textures = torch.ones(1, self.faces.shape[1], texture_size, texture_size, texture_size, 3,
dtype=torch.float32).cuda()
self.textures = nn.Parameter(textures)
else:
self.textures = nn.Parameter(base_textures)
# load reference image
image_ref = torch.from_numpy(imread(filename_ref).astype('float32') / 255.).permute(2, 0, 1)[None, ::]
image_ref = image_ref.cuda()
self.register_buffer('image_ref', image_ref)
# save camera parameters
self.camera_x = camera_distance
self.camera_y = camera_elevation
self.camera_z = camera_azimuth
# setup renderer
renderer = nr.Renderer(camera_mode='look_at', far=200)
renderer.light_intensity_directional = 0
renderer.light_intensity_ambient = 1
renderer.eye = (self.camera_x, self.camera_y, self.camera_z)
self.renderer = renderer
def __init__(self, filename_obj, filename_ref):
super(Model, self).__init__()
with self.init_scope():
# load .obj
vertices, faces = neural_renderer.load_obj(filename_obj)
self.vertices = vertices[None, :, :]
self.faces = faces[None, :, :]
# create textures
texture_size = 4
textures = np.zeros((1, self.faces.shape[1], texture_size,
texture_size, texture_size, 3), 'float32')
self.textures = chainer.Parameter(textures)
# load reference image
self.image_ref = scipy.misc.imread(filename_ref).astype(
'float32') / 255.
# setup renderer
renderer = neural_renderer.Renderer()
renderer.perspective = False
renderer.light_intensity_directional = 0.0
renderer.light_intensity_ambient = 1.0
self.renderer = renderer
def test_forward_case3(self):
"""Whether a silhouette by neural renderer matches that by Blender."""
# load teapot
vertices, faces, textures = utils.load_teapot_batch()
vertices = vertices.cuda()
faces = faces.cuda()
textures = textures.cuda()
# create renderer
renderer = nr.Renderer()
renderer.camera.position = self.position
renderer.camera.image_size = 256
renderer.anti_aliasing = False
renderer.light_intensity_ambient = 1.0
renderer.light_intensity_directional = 0.0
images = renderer(vertices, faces, textures)
images = images.detach().cpu().numpy()
image = images[2].mean(0)
# load reference image by blender
ref = imread(os.path.join(data_dir, 'teapot_blender.png'))
ref = (ref.min(axis=-1) != 255).astype(np.float32)
assert (np.allclose(ref, image))
def __init__(self, img_size, camera_data, vis3d):
super(SMPLModel, self).__init__()
# camera
self.camera_t = torch.load('runner_srcipts/camera.pt')['camera_t'][None, :]
self.camera_f = torch.FloatTensor(camera_data['camera_f']).cuda()[None, :]
self.camera_c = torch.FloatTensor(camera_data['camera_c']).cuda()[None, :]
self.camera_rt = torch.FloatTensor(camera_data['camera_rt']).cuda()[None, :]
self.dist_coeffs = torch.FloatTensor([[0., 0., 0., 0., 0.]]).repeat(self.camera_f.shape[0], 1).cuda()
self.img_size = img_size
self.orig_size = camera_data['height']
self.vis3d = vis3d
# SMPL
self.smpl = SMPL('hmr/models/neutral_smpl_with_cocoplus_reg.pkl', 'cuda', 'coco').cuda()
initial_thetas = torch.FloatTensor(np.concatenate([[np.pi, 0, 0], MEAN_POSE])).cuda()
initial_betas = torch.randn(10).cuda()
self.faces = self.smpl.faces[None, :, :]
self.theta = nn.Parameter(initial_thetas[None, :])
self.betas = nn.Parameter(initial_betas[None, :])
# renderer
self.renderer = neural_renderer.Renderer(
image_size=self.img_size,
camera_f=self.camera_f,
camera_c=self.camera_c,
camera_rt=self.camera_rt,
camera_t=self.camera_t,
orig_size=self.orig_size,
near=1.0,
far=100.0,
camera_mode='projection_by_params'
)
def render_img_from_mesh(mesh, e, a, d, file_name):
renderer = nr.Renderer(camera_mode='look_at',
image_size=512,
background_color=[1, 1, 1],
light_intensity_ambient=0.3)
renderer = renderer.to(device)
renderer.eye = nr.get_points_from_angles(d, e, a)
texture_size = 2
v = torch.Tensor(mesh.vertices)[None, :, :]
f = torch.Tensor(mesh.faces)[None, :, :]
v = v.to(device)
f = f.int().to(device)
textures = torch.ones(1,
f.shape[1],
texture_size,
texture_size,
texture_size,
3,
dtype=torch.float32).cuda()
textures[:] = 0.9
images, _, _ = renderer(v, f, textures)
image1 = images.detach().cpu().numpy()[0].transpose(1, 2, 0)
image2 = (image1 * 255).astype(np.uint8)
scipy.misc.imsave(file_name, image2)
return image1
def __init__(self, opt):
self.opt = opt
self.render_size = opt.crop_size
d = './3ddfa/train.configs'
w_shp = _load(osp.join(d, 'w_shp_sim.npy'))
w_exp = _load(osp.join(d, 'w_exp_sim.npy')) # simplified version
u_shp = _load(osp.join(d, 'u_shp.npy'))
u_exp = _load(osp.join(d, 'u_exp.npy'))
self.keypoints = _load(osp.join(d, 'keypoints_sim.npy'))
self.texMU = _load(osp.join(d, 'texMU.npy')) # 159645 * 1
self.texPC = _load(osp.join(d, 'texPC.npy')) # 159645 * 199
self.texEV = _load(osp.join(d, 'texEV.npy')) # 199 * 1
self.keypoints = _load(osp.join(d, 'keypoints_sim.npy'))
self.n_tex_para = len(self.texEV)
self.pose_noise = opt.pose_noise
self.large_pose = opt.large_pose
u = u_shp + u_exp
# tri = sio.loadmat('/home/sensetime/Documents/3DDFA/visualize/tri.mat')['tri'] # 3 * 53215
tri = sio.loadmat('./3ddfa/visualize/tri.mat')['tri'] # 3 * 53215
faces_np = np.expand_dims(tri.T, axis=0).astype(np.int32) - 1
self.std_size = 120
self.current_gpu = opt.gpu_ids
with torch.cuda.device(self.current_gpu):
self.faces = torch.from_numpy(faces_np).cuda()
self.renderer = nr.Renderer(camera_mode='look',
image_size=self.render_size,
perspective=False,
light_intensity_directional=0,
light_intensity_ambient=1)
self.u_cuda = torch.from_numpy(u.astype(np.float32)).cuda()
self.w_shp_cuda = torch.from_numpy(w_shp.astype(np.float32)).cuda()
self.w_exp_cuda = torch.from_numpy(w_exp.astype(np.float32)).cuda()
def __init__(self,
vertices,
faces,
img_size=64,
mode='silhouettes',
dataset='CVPR18'):
super(Mesh_Renderer, self).__init__()
if dataset == 'CVPR18':
self.elevation = 30.
self.distance = 2.732
vAngle = 15
elif dataset == 'NIPS17':
vAngle = 14.9314
self.vertices = vertices
self.register_buffer('faces', faces)
self.img_size = img_size
# create textures
texture_size = 2
textures = torch.ones(self.faces.shape[0],
self.faces.shape[1],
texture_size,
texture_size,
texture_size,
3,
dtype=torch.float32)
self.register_buffer('textures', textures)
self.mode = mode
# setup renderer
renderer = nr.Renderer(camera_mode='look_at',
image_size=self.img_size,
viewing_angle=vAngle)
self.renderer = renderer
def __init__(self, image_size, anti_aliasing, camera_mode, perspective):
renderer = neural_renderer.Renderer(image_size=image_size,
anti_aliasing=anti_aliasing,
camera_mode=camera_mode,
perspective=perspective,
background_color=[0, 0, 0])
self.renderer = renderer
def __init__(self, cfg):
super(DepthRenderer, self).__init__()
min_depth = cfg.MODEL.MVSNET.MIN_DEPTH
max_depth = min_depth + (cfg.MODEL.MVSNET.DEPTH_INTERVAL \
* cfg.MODEL.MVSNET.NUM_DEPTHS)
self.renderer = nr.Renderer(
camera_mode='projection',
near=min_depth, far=max_depth,
anti_aliasing=False
)
fx, fy = cfg.MODEL.MVSNET.FOCAL_LENGTH
cx, cy = cfg.MODEL.MVSNET.PRINCIPAL_POINT
self.camera_k = torch.tensor(
[[fx, 0, cx],
[0, fy, cy],
[0, 0, 1]],
dtype=torch.float32
)
self.dist_coeffs = torch.zeros(5, dtype=torch.float32)
# conversion from shapenet convention (East-Up_South)
# to renderer convention (East-Down-North)
# final rotation: R_renderer_shapenet * extrinsics
# inverse y and z, equivalent to inverse x, but gives positive z
rvec = np.array([np.pi, 0., 0.], dtype=np.float32)
R = cv2.Rodrigues(rvec)[0]
T = np.eye(4, dtype=np.float32)
T[:3, :3] = R
self.T_renderer_shapenet = torch.from_numpy(T)
self.T_shapenet_renderer = torch.inverse(self.T_renderer_shapenet)
def test_texture(self):
position, rotation = position_rotation_from_angles(2, 15, 30)
camera = nr.Camera(position=position, rotation=rotation)
renderer = nr.Renderer(camera=camera)
vertices, faces, textures = nr.load_obj(os.path.join(
data_dir, '1cde62b063e14777c9152a706245d48/model.obj'),
load_texture=True)
images = renderer(vertices[None, :, :], faces[None, :, :],
textures[None, :, :, :, :, :]).permute(
0, 2, 3, 1).detach().cpu().numpy()
image = (images[0] * 255).astype(np.uint8)
imsave(os.path.join(data_dir, 'car.png'), image)
vertices, faces, textures = nr.load_obj(os.path.join(
data_dir, '4e49873292196f02574b5684eaec43e9/model.obj'),
load_texture=True,
texture_size=16)
position, rotation = position_rotation_from_angles(2, 15, -90)
renderer.camera.position = position
renderer.camera.rotation = rotation
images = renderer(vertices[None, :, :], faces[None, :, :],
textures[None, :, :, :, :, :]).permute(
0, 2, 3, 1).detach().cpu().numpy()
image = (images[0] * 255).astype(np.uint8)
imsave(os.path.join(data_dir, 'display.png'), image)
def __init__(self, filename_obj, filename_ref=None):
super(Model, self).__init__()
with self.init_scope():
# load .obj
vertices, faces = neural_renderer.load_obj(filename_obj)
self.vertices = vertices[None, :, :]
self.faces = faces[None, :, :]
# create textures
texture_size = 2
textures = np.ones((1, self.faces.shape[1], texture_size,
texture_size, texture_size, 3), 'float32')
self.textures = textures
# load reference image
if filename_ref is not None:
self.image_ref = (scipy.misc.imread(filename_ref).max(-1) !=
0).astype('float32')
else:
self.image_ref = None
# camera parameters
self.camera_position = chainer.Parameter(
np.array([6, 10, -14], 'float32'))
# setup renderer
renderer = neural_renderer.Renderer()
renderer.eye = self.camera_position
self.renderer = renderer
def stest_forward_case2(self):
data = [[
'./tests/data/4e49873292196f02574b5684eaec43e9/model.obj',
neural_renderer.get_points_from_angles(2.5, 10, -90),
'./tests/data/4e49873292196f02574b5684eaec43e9.png',
],
[
'./tests/data/1cde62b063e14777c9152a706245d48/model.obj',
neural_renderer.get_points_from_angles(2.5, 10, 60),
'./tests/data/1cde62b063e14777c9152a706245d48.png',
]]
renderer = neural_renderer.Renderer()
renderer.draw_backside = False
for i, (filename, viewpoint, reference) in enumerate(data):
renderer.viewpoints = viewpoint
ref = neural_renderer.imread(reference)
vertices, faces, vertices_t, faces_t, textures = neural_renderer.load_obj(
filename, load_textures=True)
vertices, faces, vertices_t, faces_t, textures = neural_renderer.to_gpu(
(vertices[None, :, :], faces, vertices_t[None, :, :], faces_t,
textures[None, :, :, :]))
images = renderer.render(vertices, faces, vertices_t, faces_t,
textures).data
image = images[0].transpose((1, 2, 0))
# imageio.toimage(image.get(), cmin=0, cmax=1).save(reference)
chainer.testing.assert_allclose(ref, image, atol=1e-2)
def test_backward_case2(self):
vertices = [[0.8, 0.8, 1.], [-0.5, -0.8, 1.], [0.8, -0.8, 1.]]
faces = [[0, 1, 2]]
pyi = 40
pxi = 50
renderer = neural_renderer.Renderer()
renderer.image_size = 64
renderer.anti_aliasing = False
renderer.perspective = False
vertices = chainer.Variable(cp.array(vertices, 'float32'))
faces = cp.array(faces, 'int32')
images = renderer.render_silhouettes(vertices[None, :, :],
faces[None, :, :])
loss = cf.sum(cf.absolute(images[:, pyi, pxi]))
loss.backward()
for i in range(3):
for j in range(2):
axis = 'x' if j == 0 else 'y'
vertices2 = cp.copy(vertices.data)
vertices2[i, j] -= 1. / vertices.grad[i, j]
images = renderer.render_silhouettes(vertices2[None, :, :],
faces[None, :, :])
image = np.tile(images[0].data.get()[:, :, None], (1, 1, 3))
image[pyi, pxi] = [1, 0, 0]
ref = scipy.misc.imread(
'./tests/data/rasterize_silhouettes_case2_v%d_%s.png' %
(i, axis))
ref = ref.astype('float32') / 255
chainer.testing.assert_allclose(ref, image)
def __init__(self, opt):
self.opt = opt
self.render_size = opt.crop_size
self.d = './3ddfa/train.configs'
w_shp = _load(osp.join(self.d, 'w_shp_sim.npy'))
w_exp = _load(osp.join(self.d, 'w_exp_sim.npy')) # simplified version
u_shp = _load(osp.join(self.d, 'u_shp.npy'))
u_exp = _load(osp.join(self.d, 'u_exp.npy'))
self.keypoints = _load(osp.join(self.d, 'keypoints_sim.npy'))
self.pose_noise = getattr(opt, 'pose_noise', False)
self.large_pose = getattr(opt, 'large_pose', False)
u = u_shp + u_exp
tri = sio.loadmat('./3ddfa/visualize/tri.mat')['tri'] # 3 * 53215
faces_np = np.expand_dims(tri.T, axis=0).astype(np.int32) - 1
self.std_size = 120
self.current_gpu = opt.gpu_ids
with torch.cuda.device(self.current_gpu):
self.faces = torch.from_numpy(faces_np).cuda()
self.renderer = nr.Renderer(camera_mode='look', image_size=self.render_size, perspective=False,
light_intensity_directional=0, light_intensity_ambient=1)
self.u_cuda = torch.from_numpy(u.astype(np.float32)).cuda()
self.w_shp_cuda = torch.from_numpy(w_shp.astype(np.float32)).cuda()
self.w_exp_cuda = torch.from_numpy(w_exp.astype(np.float32)).cuda()
def test_backward_case1(self):
vertices = [
[-0.9, -0.9, 2.],
[-0.8, 0.8, 1.],
[0.8, 0.8, 0.5]]
faces = [[0, 1, 2]]
renderer = nr.Renderer(camera_mode='look_at')
renderer.image_size = 64
renderer.anti_aliasing = False
renderer.perspective = False
renderer.camera_mode = 'none'
vertices = torch.from_numpy(np.array(vertices, np.float32)).cuda()
faces = torch.from_numpy(np.array(faces, np.int32)).cuda()
vertices, faces = utils.to_minibatch((vertices, faces))
vertices.requires_grad = True
images = renderer(vertices, faces, mode='depth')
loss = torch.sum((images[0, 15, 20] - 1)**2)
loss.backward()
grad = vertices.grad.clone()
grad2 = np.zeros_like(grad)
for i in range(3):
for j in range(3):
eps = 1e-3
vertices2 = vertices.clone()
vertices2[i, j] += eps
images = renderer.render_depth(vertices2, faces)
loss2 = torch.sum((images[0, 15, 20] - 1)**2)
grad2[i, j] = ((loss2 - loss) / eps).item()
assert(np.allclose(grad, grad2, atol=1e-3))
def __init__(self, vertices, faces, filename_ref, camera_x, camera_y,
camera_z):
super(ModelMorphing, self).__init__()
self.vertices = nn.Parameter(vertices)
self.register_buffer('faces', faces)
# create textures
texture_size = 2
textures = torch.ones(1,
self.faces.shape[1],
texture_size,
texture_size,
texture_size,
3,
dtype=torch.float32)
self.register_buffer('textures', textures)
# load reference image
image_ref = torch.from_numpy(
imread(filename_ref).astype(np.float32).mean(-1) / 255.)[None, ::]
self.register_buffer('image_ref', image_ref)
# save camera parameters
self.camera_x = camera_x
self.camera_y = camera_y
self.camera_z = camera_z
# setup renderer
renderer = nr.Renderer(camera_mode='look_at', far=200)
renderer.eye = (self.camera_x, self.camera_y, self.camera_z)
self.renderer = renderer
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--filename_input', type=str, default=os.path.join(data_dir, 'teapot.obj'))
parser.add_argument('-o', '--filename_output', type=str, default=os.path.join(data_dir, 'example1.gif'))
parser.add_argument('-g', '--gpu', type=int, default=0)
args = parser.parse_args()
# other settings
camera_distance = 2.732
elevation = 30
texture_size = 2
# load .obj
vertices, faces = nr.load_obj(args.filename_input)
vertices = vertices[None, :, :] # [num_vertices, XYZ] -> [batch_size=1, num_vertices, XYZ]
faces = faces[None, :, :] # [num_faces, 3] -> [batch_size=1, num_faces, 3]
# create texture [batch_size=1, num_faces, texture_size, texture_size, texture_size, RGB]
textures = torch.ones(1, faces.shape[1], texture_size, texture_size, texture_size, 3, dtype=torch.float32).cuda()
# to gpu
# create renderer
renderer = nr.Renderer(camera_mode='look_at')
# draw object
loop = tqdm.tqdm(range(0, 360, 4))
writer = imageio.get_writer(args.filename_output, mode='I')
for num, azimuth in enumerate(loop):
loop.set_description('Drawing')
renderer.eye = nr.get_points_from_angles(camera_distance, elevation, azimuth)
images = renderer(vertices, faces, textures) # [batch_size, RGB, image_size, image_size]
image = images.detach().cpu().numpy()[0].transpose((1, 2, 0)) # [image_size, image_size, RGB]
writer.append_data((255*image).astype(np.uint8))
writer.close()
def test_texture(self):
renderer = neural_renderer.Renderer()
vertices, faces, textures = neural_renderer.load_obj(
'./tests/data/1cde62b063e14777c9152a706245d48/model.obj',
load_texture=True)
vertices = chainer.cuda.to_gpu(vertices)
faces = chainer.cuda.to_gpu(faces)
textures = chainer.cuda.to_gpu(textures)
renderer.eye = neural_renderer.get_points_from_angles(2, 15, 30)
images = renderer.render(vertices[None, :, :], faces[None, :, :],
textures[None, :, :, :, :, :]).data.get()
scipy.misc.imsave('./tests/data/car.png',
scipy.misc.toimage(images[0]))
vertices, faces, textures = neural_renderer.load_obj(
'./tests/data/4e49873292196f02574b5684eaec43e9/model.obj',
load_texture=True,
texture_size=16)
vertices = chainer.cuda.to_gpu(vertices)
faces = chainer.cuda.to_gpu(faces)
textures = chainer.cuda.to_gpu(textures)
renderer.eye = neural_renderer.get_points_from_angles(2, 15, -90)
images = renderer.render(vertices[None, :, :], faces[None, :, :],
textures[None, :, :, :, :, :]).data.get()
scipy.misc.imsave('./tests/data/display.png',
scipy.misc.toimage(images[0]))
def test_backward_case2(self):
"""Backward if non-zero gradient is on a face."""
vertices = [
[0.8, 0.8, 1.],
[-0.5, -0.8, 1.],
[0.8, -0.8, 1.]]
faces = [[0, 1, 2]]
pyi = 40
pxi = 50
grad_ref = [
[0.98646867, 1.04628897, 0.],
[-1.03415668, - 0.10403691, 0.],
[3.00094461, - 1.55173182, 0.],
]
renderer = nr.Renderer(camera_mode='look_at')
renderer.image_size = 64
renderer.anti_aliasing = False
renderer.perspective = False
vertices = torch.from_numpy(np.array(vertices, np.float32)).cuda()
faces = torch.from_numpy(np.array(faces, np.int32)).cuda()
grad_ref = torch.from_numpy(np.array(grad_ref, np.float32)).cuda()
vertices, faces, grad_ref = utils.to_minibatch((vertices, faces, grad_ref))
vertices.requires_grad = True
images = renderer(vertices, faces, mode='silhouettes')
loss = torch.sum(torch.abs(images[:, pyi, pxi]))
loss.backward()
assert(torch.allclose(vertices.grad, grad_ref, rtol=1e-2))
def __init__(self,
args):
super(CircleNet, self).__init__()
self.num_nudes = args.num_nodes
self.dec_dim = args.dec_dim
self.dec_size = args.dec_size
self.image_size = args.image_size
self.stages = args.stages
if args.arch == 'resnet':
kwargs = {'stages': self.stages}
res_dims = [256, 512, 1024, 2048]
self.backbone = resnet_encoder(pretrained=True, **kwargs)
dec_skip_dims = [res_dims[i] for i in self.stages][::-1]
self.disp = resnet_decoder(dec_skip_dims, 2, self.dec_dim, self.dec_size, drop=args.drop)
elif args.arch == 'unet':
self.backbone = unet_encoder(args.enc_dim, drop=args.drop)
self.disp = unet_decoder(self.backbone.dims, drop=args.drop)
self.texture_size = 2
self.camera_distance = 1
self.elevation = 0
self.azimuth = 0
self.renderer = nr.Renderer(camera_mode='look_at', image_size=self.image_size, light_intensity_ambient=1,
light_intensity_directional=1, perspective=False)
def __init__(self, filename_obj, filename_ref):
super(Model, self).__init__()
# load .obj
vertices, faces = nr.load_obj(filename_obj)
self.vertices = nn.Parameter(vertices[None, :, :])
self.register_buffer('faces', faces[None, :, :])
# create textures
texture_size = 2
textures = torch.ones(1,
self.faces.shape[1],
texture_size,
texture_size,
texture_size,
3,
dtype=torch.float32)
self.register_buffer('textures', textures)
# load reference image
image_ref = torch.from_numpy(
imread(filename_ref).astype(np.float32).mean(-1) / 255.)[None, ::]
self.register_buffer('image_ref', image_ref)
# setup renderer
renderer = nr.Renderer(camera_mode='look_at')
self.renderer = renderer
def test_texture(self):
renderer = nr.Renderer(camera_mode='look_at')
vertices, faces, textures = nr.load_obj(os.path.join(
data_dir, '1cde62b063e14777c9152a706245d48/model.obj'),
load_texture=True)
renderer.eye = nr.get_points_from_angles(2, 15, 30)
images, depth, silhouette = renderer.render(
vertices[None, :, :], faces[None, :, :],
textures[None, :, :, :, :, :])
images = images.permute(0, 2, 3, 1).detach().cpu().numpy()
silhouette = silhouette.detach().cpu().numpy()
depth = depth.detach().cpu().numpy()
imsave(os.path.join(data_dir, 'car.png'), images[0])
cv2.imshow("r", images[0, :, :, ::-1])
cv2.imshow("d", ColorizeDepth(depth[0], 1.5, 2.5))
cv2.imshow("s", silhouette[0])
cv2.waitKey()
vertices, faces, textures = nr.load_obj(os.path.join(
data_dir, '4e49873292196f02574b5684eaec43e9/model.obj'),
load_texture=True,
texture_size=16)
renderer.eye = nr.get_points_from_angles(2, 15, -90)
images, _, _ = renderer.render(vertices[None, :, :], faces[None, :, :],
textures[None, :, :, :, :, :])
images = images.permute(0, 2, 3, 1).detach().cpu().numpy()
imsave(os.path.join(data_dir, 'display.png'), images[0])
def __init__(self, filename_obj, filename_ref=None):
super(Model, self).__init__()
# load .obj
vertices, faces = nr.load_obj(filename_obj)
self.register_buffer('vertices', vertices[None, :, :])
self.register_buffer('faces', faces[None, :, :])
# create textures
texture_size = 2
textures = torch.ones(1,
self.faces.shape[1],
texture_size,
texture_size,
texture_size,
3,
dtype=torch.float32)
self.register_buffer('textures', textures)
# load reference image
image_ref = torch.from_numpy(
(imread(filename_ref).max(-1) != 0).astype(np.float32))
self.register_buffer('image_ref', image_ref)
# camera parameters
self.camera_position = nn.Parameter(
torch.from_numpy(np.array([6, 10, -14], dtype=np.float32)))
# setup renderer
renderer = nr.Renderer(camera_mode='look_at')
renderer.eye = self.camera_position
self.renderer = renderer
def __init__(self, filename_obj):
super(Model, self).__init__()
# Load mesh vertices and faces
vertices, faces = load_off(filename_obj)
self.register_buffer('vertices', vertices[None, :, :])
self.register_buffer('faces', faces[None, :, :])
# create textures
texture_size = 2
textures = torch.ones(1,
self.faces.shape[1],
texture_size,
texture_size,
texture_size,
3,
dtype=torch.float32)
self.register_buffer('textures', textures)
# camera parameters
self.camera_position = nn.Parameter(
torch.from_numpy(np.array([3, 5, -7], dtype=np.float32)))
# setup renderer
renderer = nr.Renderer(camera_mode='look_at')
renderer.eye = self.camera_position
self.renderer = renderer
self.entropy = 0
