def run_on_image(self, images, id_str):
deprocess = imagenet_deprocess(rescale_image=False)
voxel_scores, meshes_pred = self.predictor(images)
img = image_to_numpy(deprocess(images[0][0]))
vis_utils.visualize_prediction(id_str, img, meshes_pred[-1][0],
self.output_dir)
def evaluate_test(model, data_loader, vis_preds=False):
"""
This function evaluates the model on the dataset defined by data_loader.
The metrics reported are described in Table 2 of our paper.
"""
# Note that all eval runs on main process
assert comm.is_main_process()
deprocess = imagenet_deprocess(rescale_image=False)
device = torch.device("cuda:0")
# evaluation
class_names = {
"02828884": "bench",
"03001627": "chair",
"03636649": "lamp",
"03691459": "speaker",
"04090263": "firearm",
"04379243": "table",
"04530566": "watercraft",
"02691156": "plane",
"02933112": "cabinet",
"02958343": "car",
"03211117": "monitor",
"04256520": "couch",
"04401088": "cellphone",
}
num_instances = {i: 0 for i in class_names}
chamfer = {i: 0 for i in class_names}
normal = {i: 0 for i in class_names}
f1_01 = {i: 0 for i in class_names}
f1_03 = {i: 0 for i in class_names}
f1_05 = {i: 0 for i in class_names}
num_batch_evaluated = 0
for batch in data_loader:
batch = data_loader.postprocess(batch, device)
imgs, meshes_gt, _, _, _, id_strs, _imgs = batch
#NOTE: _imgs contains all of the other images in belonging to this model
#We have to select the next-best-view from that list of images
sids = [id_str.split("-")[0] for id_str in id_strs]
for sid in sids:
num_instances[sid] += 1
with inference_context(model):
voxel_scores, meshes_pred = model(imgs)
#TODO: Render masks from predicted mesh for each view
cur_metrics = compare_meshes(meshes_pred[-1],
meshes_gt,
reduce=False)
cur_metrics["verts_per_mesh"] = meshes_pred[-1].num_verts_per_mesh(
).cpu()
cur_metrics["faces_per_mesh"] = meshes_pred[-1].num_faces_per_mesh(
).cpu()
for i, sid in enumerate(sids):
chamfer[sid] += cur_metrics["Chamfer-L2"][i].item()
normal[sid] += cur_metrics["AbsNormalConsistency"][i].item()
f1_01[sid] += cur_metrics["F1@%f" % 0.1][i].item()
f1_03[sid] += cur_metrics["F1@%f" % 0.3][i].item()
f1_05[sid] += cur_metrics["F1@%f" % 0.5][i].item()
if vis_preds:
img = image_to_numpy(deprocess(imgs[i]))
vis_utils.visualize_prediction(id_strs[i], img,
meshes_pred[-1][i],
"/tmp/output")
num_batch_evaluated += 1
logger.info("Evaluated %d / %d batches" %
(num_batch_evaluated, len(data_loader)))
vis_utils.print_instances_class_histogram(
num_instances,
class_names,
{
"chamfer": chamfer,
"normal": normal,
"f1_01": f1_01,
"f1_03": f1_03,
"f1_05": f1_05
},
)
def run_on_image(self, image, id_str, gt_verts, gt_faces):
deprocess = imagenet_deprocess(rescale_image=False)
with torch.no_grad():
voxel_scores, meshes_pred = self.predictor(image)
sid, mid, iid = id_str.split('-')
iid = int(iid)
#Transform vertex space
metadata_path = os.path.join('./datasets/shapenet/ShapeNetV1processed',
sid, mid, "metadata.pt")
metadata = torch.load(metadata_path)
K = metadata["intrinsic"]
RTs = metadata["extrinsics"]
rot_y_90 = torch.tensor([[0, 0, 1, 0], [0, 1, 0, 0], [-1, 0, 0, 0],
[0, 0, 0, 1]]).to(RTs)
mesh = meshes_pred[-1][0]
#For some strange reason all classes (expect vehicle class) require a 90 degree rotation about the y-axis
#for the GT mesh
invRT = torch.inverse(RTs[iid].mm(rot_y_90))
invRT_no_rot = torch.inverse(RTs[iid])
mesh._verts_list[0] = project_verts(mesh._verts_list[0], invRT.cpu())
#Get look at view extrinsics
render_metadata_path = os.path.join(
'datasets/shapenet/ShapeNetRenderingExtrinsics', sid, mid,
'rendering_metadata.pt')
render_metadata = torch.load(render_metadata_path)
render_RTs = render_metadata['extrinsics']
plt.figure(figsize=(10, 10))
R = render_RTs[iid][:3, :3].unsqueeze(0)
T = render_RTs[iid][:3, 3].unsqueeze(0)
cameras = OpenGLPerspectiveCameras(R=R, T=T)
#Phong Renderer
lights = PointLights(location=[[0.0, 0.0, -3.0]])
raster_settings = RasterizationSettings(image_size=256,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
phong_renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(lights=lights))
#Silhouette Renderer
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=256,
blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma,
faces_per_pixel=50,
)
silhouette_renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftSilhouetteShader(blend_params=blend_params))
verts, faces = mesh.get_mesh_verts_faces(0)
verts_rgb = torch.ones_like(verts)[None]
textures = Textures(verts_rgb=verts_rgb)
mesh.textures = textures
verts_rgb = torch.ones_like(gt_verts)[None]
textures = Textures(verts_rgb=verts_rgb)
#Invert without the rotation for the vehicle class
if sid == '02958343':
gt_verts = project_verts(gt_verts, invRT_no_rot.cpu())
else:
gt_verts = project_verts(gt_verts, invRT.cpu())
gt_mesh = Meshes(verts=[gt_verts], faces=[gt_faces], textures=textures)
img = image_to_numpy(deprocess(image[0]))
mesh_image = phong_renderer(meshes_world=mesh, R=R, T=T)
gt_silh_image = (silhouette_renderer(meshes_world=gt_mesh, R=R, T=T) >
0).float()
silhouette_image = (silhouette_renderer(meshes_world=mesh, R=R, T=T) >
0).float()
plt.subplot(2, 2, 1)
plt.imshow(img)
plt.title('input image')
plt.subplot(2, 2, 2)
plt.imshow(mesh_image[0, ..., :3].cpu().numpy())
plt.title('rendered mesh')
plt.subplot(2, 2, 3)
plt.imshow(gt_silh_image[0, ..., 3].cpu().numpy())
plt.title('silhouette of gt mesh')
plt.subplot(2, 2, 4)
plt.imshow(silhouette_image[0, ..., 3].cpu().numpy())
plt.title('silhouette of rendered mesh')
plt.show()
#plt.savefig('./output_demo/figures/'+id_str+'.png')
vis_utils.visualize_prediction(id_str, img, mesh, self.output_dir)
def evaluate_test(model, data_loader, vis_preds=False):
"""
This function evaluates the model on the dataset defined by data_loader.
The metrics reported are described in Table 2 of our paper.
"""
# Note that all eval runs on main process
assert comm.is_main_process()
deprocess = imagenet_deprocess(rescale_image=False)
device = torch.device("cuda:0")
# evaluation
class_names = {
"02828884": "bench",
"03001627": "chair",
"03636649": "lamp",
"03691459": "speaker",
"04090263": "firearm",
"04379243": "table",
"04530566": "watercraft",
"02691156": "plane",
"02933112": "cabinet",
"02958343": "car",
"03211117": "monitor",
"04256520": "couch",
"04401088": "cellphone",
}
num_instances = {i: 0 for i in class_names}
chamfer = {i: 0 for i in class_names}
normal = {i: 0 for i in class_names}
f1_01 = {i: 0 for i in class_names}
f1_03 = {i: 0 for i in class_names}
f1_05 = {i: 0 for i in class_names}
num_batch_evaluated = 0
for batch in data_loader:
batch = data_loader.postprocess(batch, device)
sids = [id_str.split("-")[0] for id_str in batch["id_strs"]]
for sid in sids:
num_instances[sid] += 1
with inference_context(model):
model_kwargs = {}
module = model.module if hasattr(model, "module") else model
if isinstance(module, VoxMeshMultiViewHead):
model_kwargs["intrinsics"] = batch["intrinsics"]
model_kwargs["extrinsics"] = batch["extrinsics"]
if isinstance(module, VoxMeshDepthHead):
model_kwargs["masks"] = batch["masks"]
model_outputs = model(batch["imgs"], **model_kwargs)
voxel_scores = model_outputs["voxel_scores"]
meshes_pred = model_outputs["meshes_pred"]
cur_metrics = compare_meshes(meshes_pred[-1], batch["meshes"], reduce=False)
cur_metrics["verts_per_mesh"] = meshes_pred[-1].num_verts_per_mesh().cpu()
cur_metrics["faces_per_mesh"] = meshes_pred[-1].num_faces_per_mesh().cpu()
for i, sid in enumerate(sids):
chamfer[sid] += cur_metrics["Chamfer-L2"][i].item()
normal[sid] += cur_metrics["AbsNormalConsistency"][i].item()
f1_01[sid] += cur_metrics["F1@%f" % 0.1][i].item()
f1_03[sid] += cur_metrics["F1@%f" % 0.3][i].item()
f1_05[sid] += cur_metrics["F1@%f" % 0.5][i].item()
if vis_preds:
img = image_to_numpy(deprocess(batch["imgs"][i]))
vis_utils.visualize_prediction(
batch["id_strs"][i], img, meshes_pred[-1][i], "/tmp/output"
)
num_batch_evaluated += 1
logger.info("Evaluated %d / %d batches" % (num_batch_evaluated, len(data_loader)))
vis_utils.print_instances_class_histogram(
num_instances,
class_names,
{"chamfer": chamfer, "normal": normal, "f1_01": f1_01, "f1_03": f1_03, "f1_05": f1_05},
)