def main(opt): ply_start = '''ply format ascii 1.0 element vertex {} property float x property float y property float z property uchar red property uchar green property uchar blue end_header\n''' model = body_models.create(model_path='../3d_data/models', model_type='smpl', gender='male', ext='pkl') smpl = pickle.load(open('../3d_data/densepose_uv.pkl', 'rb')) faces = np.array(smpl['f_extended'], dtype=np.int64).reshape((-1, 3)) uv_faceid = io.loadmat( '../3d_data/DensePoseData/UV_data/UV_Processed.mat')['All_FaceIndices'] uv = smpl['uv'] # with open('../3d_data/nongrey_male_0110.jpg', 'rb') as file: texture = cv2.imread('../3d_data/nongrey_male_0110.jpg') output = model(return_verts=True) vertices = output.vertices.detach().cpu().numpy().squeeze() smpl_vertices = np.array([vertices[i] for i in smpl['v_extended']]) smpl_uv_visual = trimesh.visual.TextureVisuals(uv=uv, image=texture) # smpl_render = Render(width=opt.image_width, height=opt.image_height, # camera_distance=opt.camera_distance, pose_y=opt.global_y, # focal_length=opt.focal_length) smpl_render = Render(width=opt.image_width, height=opt.image_height, pose_y=opt.global_y) smpl_render.set_render(vertices=smpl_vertices, faces=faces, visual=smpl_uv_visual) smpl_norm_vertices = smpl_render.vertices smpl_render_uv = smpl_render.render_visual( flags=pyrender.RenderFlags.UV_RENDERING, face_id=uv_faceid) smpl_render.set_render(vertices=smpl_vertices, faces=faces) norm_vertices = smpl_render.vertices color = (norm_vertices * 255).astype('uint8') concatenated_smpl = np.concatenate((norm_vertices, color), axis=1) ply_start = ply_start.format(norm_vertices.shape[0]) with open(opt.save_loc, 'w') as write_file: write_file.write(ply_start) np.savetxt(write_file, concatenated_smpl, fmt=' '.join(['%0.8f'] * 3 + ['%d'] * 3))
def main(opt): model = body_models.create(model_path='../3d_data/models', model_type='smpl', gender='male', ext='pkl') smpl = pickle.load(open('../3d_data/densepose_uv.pkl', 'rb')) faces = np.array(smpl['f_extended'], dtype=np.int64).reshape((-1, 3)) uv_faceid = io.loadmat( '../3d_data/DensePoseData/UV_data/UV_Processed.mat')['All_FaceIndices'] uv = smpl['uv'] # with open('../3d_data/nongrey_male_0110.jpg', 'rb') as file: texture = cv2.imread('../3d_data/nongrey_male_0110.jpg') global_tr = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) # set up the rendering objects focal_length = opt.focal_length * opt.image_height # mesh_camera = pyrender.IntrinsicsCamera(focal_length, focal_length, opt.image_width / 2, opt.image_height / 2, # opt.znear, opt.zfar) mesh_camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0, znear=0.05) camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0, znear=0.05) camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.05], [0.0, 0.0, 0.0, 1.0]]) mesh_tr = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, opt.global_y + 0.11], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) mesh_camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, opt.camera_distance], [0.0, 0.0, 0.0, 1.0]]) render = pyrender.OffscreenRenderer(opt.image_width, opt.image_height) output = model(return_verts=True) vertices = output.vertices.detach().cpu().numpy().squeeze() mesh_verts = np.array([vertices[i] for i in smpl['v_extended']]) visual_check = trimesh.visual.TextureVisuals(uv=uv, image=texture) tri_mesh_scene = trimesh.Trimesh(vertices=mesh_verts, faces=faces, visual=visual_check) mesh_body = pyrender.Mesh.from_trimesh(tri_mesh_scene) mesh_scene = pyrender.Scene(ambient_light=[0.5, 0.5, 0.5], bg_color=[-1.0, -1.0, -1.0]) mesh_scene.add(mesh_body, pose=mesh_tr) mesh_scene.add(mesh_camera, pose=mesh_camera_pose) rendered_uv, depth = render.render(scene=mesh_scene, flags=pyrender.RenderFlags.UV_RENDERING) rendered_uv = rendered_uv.copy() mask = rendered_uv[:, :, 2] != -1. temp_2 = rendered_uv[:, :, 2] temp_2[mask] = np.take(uv_faceid, temp_2[mask].astype('int')) rendered_uv[:, :, 2] = temp_2 cv2.imshow('UV', rendered_uv) bounds = tri_mesh_scene.bounding_box_oriented.extents mesh_verts -= mesh_scene.centroid mesh_verts /= bounds # mesh_verts *= 2 mesh_verts = mesh_verts + 1 / 2 face_select = faces[uv_faceid[:, 0] == 1] # verts = np.concatenate((uv, np.ones(uv.shape[:2] + (1,))), axis=2) # uv[:, 2] = 1 verts = (uv * 2) - 1 visual = trimesh.visual.ColorVisuals(vertex_colors=uv) tri_mesh = trimesh.Trimesh(vertices=verts, faces=face_select, visual=visual) # tri_mesh mesh = pyrender.Mesh.from_trimesh(tri_mesh) # tri_mesh.show() scene = pyrender.Scene(ambient_light=[0.5, 0.5, 0.5], bg_color=[-1.0, -1.0, -1.0]) scene.add(mesh, pose=global_tr) scene.add(camera, pose=camera_pose) rendered_color_visual, depth = render.render( scene=scene, flags=pyrender.RenderFlags.SKIP_CULL_FACES) # pyrender.Viewer(scene, render_flags={'cull_faces': False}) cv2.imshow('Part UV', rendered_color_visual) # cv2.waitKey(0) rendered_interp, _ = render.render( scene=scene, flags=pyrender.RenderFlags.BARYCENTRIC_COORDINATES | pyrender.RenderFlags.SKIP_CULL_FACES) tri_id, _ = render.render(scene=scene, flags=pyrender.RenderFlags.TRIANGLE_ID_RENDERING | pyrender.RenderFlags.SKIP_CULL_FACES) vertex_stream = np.take(mesh_verts, face_select, axis=0) tri_id = tri_id[:, :, 0] rendered_interp = rendered_interp.reshape(rendered_interp.shape + (1, )).repeat([3], axis=-1) out_view = vertex_stream[tri_id.astype('int')] * rendered_interp out_view = out_view.sum(axis=-2) # rendered_uv[rendered_uv == -1] = 0 # rendered_uv[:, :, 2] /= 255 out_view[rendered_color_visual < 0] = 0 # cv2.imwrite('../saves/checks/mesh_normalized_uv.jpg', (rendered_uv * 255).astype('uint8')) cv2.imshow('Coords', out_view) cv2.imwrite('../saves/checks/mesh_uv_render.jpg', (out_view * 255).astype('uint8')) cv2.waitKey(0)
def main(opt): model = body_models.create(model_path='../3d_data/models', model_type='smpl', gender='male', ext='pkl') smpl = pickle.load(open('../3d_data/densepose_uv.pkl', 'rb')) faces = np.array(smpl['f_extended'], dtype=np.int64).reshape((-1, 3)) uv_faceid = io.loadmat( '../3d_data/DensePoseData/UV_data/UV_Processed.mat')['All_FaceIndices'] uv = smpl['uv'] uv_vertices = (uv * 2) - 1 # with open('../3d_data/nongrey_male_0110.jpg', 'rb') as file: texture = cv2.imread('../3d_data/nongrey_male_0110.jpg') output = model(return_verts=True) vertices = output.vertices.detach().cpu().numpy().squeeze() smpl_vertices = np.array([vertices[i] for i in smpl['v_extended']]) smpl_uv_visual = trimesh.visual.TextureVisuals(uv=uv, image=texture) # smpl_render = Render(width=opt.image_width, height=opt.image_height, pose_y=opt.global_y) smpl_render = Render(width=opt.image_width, height=opt.image_height, camera_distance=opt.camera_distance, pose_y=opt.global_y, focal_length=opt.focal_length) smpl_render.set_render(vertices=smpl_vertices, faces=faces) smpl_norm_vertices = smpl_render.vertices smpl_uv = smpl_render.render_interpolate(vertices=uv, skip_cull=False) smpl_noc = smpl_render.render_interpolate( vertices=smpl_norm_vertices).interpolated smpl_class_id = match_faces(smpl_uv.triangle, uv_faceid) smpl_class_id = smpl_class_id.reshape(smpl_class_id.shape + (1, )) smpl_class_id[smpl_class_id == -1] = 0 smpl_uv_stack = np.array([]).reshape( (0, opt.image_height, opt.image_width, 2)) aggregate_textures = np.array([], dtype='float64').reshape( (0, 3, opt.image_height, opt.image_width)) uv_render = Render(width=opt.image_width, height=opt.image_height) for idx in range(1, 25): # face_select = faces[uv_faceid[:, 0] == idx, :] id_select = np.unique( np.hstack( np.where(faces == vert)[0] for face in faces[uv_faceid[:, 0] == idx, :] for vert in face)) face_select = faces[id_select, :] uv_render.set_render(vertices=uv_vertices, faces=face_select, normalize=False) # out_view = uv_render.render_visual(flags=pyrender.RenderFlags.SKIP_CULL_FACES) out_view = np.flip(uv_render.render_interpolate( vertices=smpl_norm_vertices).interpolated.transpose([2, 0, 1]), axis=1) aggregate_textures = np.concatenate( [aggregate_textures, out_view.reshape((1, ) + out_view.shape)]) smpl_uv_stack = np.concatenate([ smpl_uv_stack, (smpl_uv.interpolated[:, :, :-1] * (smpl_class_id.repeat([2], axis=-1) == idx)).reshape((1, ) + smpl_uv.interpolated[:, :, :-1].shape) ]) # cv2.imshow("Part Texture", aggregate_textures[idx - 1].transpose([1, 2, 0])) # cv2.waitKey(0) texture_map = torch.from_numpy(aggregate_textures) smpl_uv_stack = torch.from_numpy((smpl_uv_stack * 2) - 1) output_textured_uv = 0 for idx in range(0, 24): output_textured_uv += torch.nn.functional.grid_sample( texture_map[idx:idx + 1], smpl_uv_stack[idx:idx + 1], mode='bilinear', padding_mode='border') output_textured_uv = output_textured_uv[0].cpu().numpy().transpose( [1, 2, 0]) cv2.imshow("Resampled UV", output_textured_uv) cv2.imwrite('../saves/checks/sampled_NOC_render.jpg', (output_textured_uv * 255).astype('uint8')) cv2.imshow("NOC", smpl_noc) cv2.imwrite('../saves/checks/NOC_render.jpg', (smpl_noc * 255).astype('uint8')) cv2.imshow("Rendered UV", smpl_uv.interpolated) cv2.imwrite('../saves/checks/UV_render.jpg', (smpl_uv.interpolated * 255).astype('uint8')) cv2.imshow("Rendered Class", smpl_class_id.astype('uint8')) print( "Image mean: ", np.mean( cv2.subtract(output_textured_uv.astype('float64'), smpl_noc.astype('float64')))) cv2.waitKey(0)
def main(opt): model = body_models.create(model_path='../3d_data/models', model_type='smpl', gender='male', ext='pkl') smpl = pickle.load(open('../3d_data/densepose_uv.pkl', 'rb')) faces = np.array(smpl['f_extended'], dtype=np.int64).reshape((-1, 3)) uv_faceid = io.loadmat( '../3d_data/DensePoseData/UV_data/UV_Processed.mat')['All_FaceIndices'] uv = smpl['uv'] # with open('../3d_data/nongrey_male_0110.jpg', 'rb') as file: texture = cv2.imread('../3d_data/nongrey_male_0110.jpg') output = model(return_verts=True) vertices = output.vertices.detach().cpu().numpy().squeeze() smpl_vertices = np.array([vertices[i] for i in smpl['v_extended']]) smpl_uv_visual = trimesh.visual.TextureVisuals(uv=uv, image=texture) # smpl_render = Render(width=opt.image_width, height=opt.image_height, # camera_distance=opt.camera_distance, pose_y=opt.global_y, # focal_length=opt.focal_length) smpl_render = Render(width=opt.image_width, height=opt.image_height, pose_y=opt.global_y) smpl_render.set_render(vertices=smpl_vertices, faces=faces, visual=smpl_uv_visual) smpl_norm_vertices = smpl_render.vertices smpl_render_uv = smpl_render.render_visual( flags=pyrender.RenderFlags.UV_RENDERING, face_id=uv_faceid) smpl_render.set_render(vertices=smpl_vertices, faces=faces) norm_vertices = smpl_render.vertices smpl_render_norm = smpl_render.render_interpolate(vertices=uv).interpolated smpl_body_uv = smpl_render_uv[:, :, :2] smpl_body_class = smpl_render_uv[:, :, 2:3] smpl_uv_stack = np.array([]).reshape( (0, opt.image_height, opt.image_width, 2)) uv_vertices = (uv * 2) - 1 uv_render = Render(width=opt.image_width, height=opt.image_height) aggregate_textures = np.array([], dtype='float64').reshape( (0, 3, opt.image_height, opt.image_width)) for idx in range(1, 4): face_select = faces[uv_faceid[:, 0] == idx] uv_visual = trimesh.visual.ColorVisuals(vertex_colors=uv) uv_render.set_render(vertices=uv_vertices, faces=face_select, visual=uv_visual, normalize=False) # out_view = uv_render.render_visual(flags=pyrender.RenderFlags.SKIP_CULL_FACES) out_view = uv_render.render_interpolate( vertices=smpl_norm_vertices).interpolated.transpose([2, 0, 1]) aggregate_textures = np.concatenate( [aggregate_textures, out_view.reshape((1, ) + out_view.shape)]) smpl_uv_stack = np.concatenate([ smpl_uv_stack, (smpl_body_uv * (smpl_body_class.repeat([2], axis=-1) == idx)).reshape((1, ) + smpl_body_uv.shape) ]) cv2.imshow( "Part body UV", np.concatenate( [smpl_uv_stack[idx - 1], np.zeros(smpl_body_class.shape)], axis=-1)) cv2.imshow("Part Texture", aggregate_textures[idx - 1].transpose([1, 2, 0])) cv2.waitKey(0) texture_map = torch.from_numpy(aggregate_textures) smpl_uv_stack = torch.from_numpy((smpl_uv_stack * 2) - 1) output_textured_uv = 0 for idx in range(0, 3): output_textured_uv += torch.nn.functional.grid_sample( texture_map[idx:idx + 1], smpl_uv_stack[idx:idx + 1], mode='bilinear', padding_mode='border') output_textured_uv = output_textured_uv[0].cpu().numpy().transpose( [1, 2, 0]) cv2.imshow("Resampled UV", output_textured_uv) cv2.imshow("Real Norm", smpl_render_norm) cv2.waitKey(0)
def main(opt): coco_folder = os.environ['COCO'] save_annotation_file = opt.output if not os.path.exists(save_annotation_file): os.mkdir(save_annotation_file) annotation_dict = { 'minival': [ COCO(coco_folder + '/annotations/densepose_coco_2014_minival.json'), COCO(coco_folder + '/annotations/person_keypoints_val2014.json'), 'val2014' ], 'valminusminival': [ COCO(coco_folder + '/annotations/densepose_coco_2014_valminusminival.json'), COCO(coco_folder + '/annotations/person_keypoints_val2014.json'), 'val2014' ], 'train': [ COCO(coco_folder + '/annotations/densepose_coco_2014_train.json'), COCO(coco_folder + '/annotations/person_keypoints_train2014.json'), 'train2014' ], 'test': [ COCO(coco_folder + '/annotations/densepose_coco_2014_test.json'), 'test2014' ] } ################################# # SMPL prep model = body_models.create(model_path='../3d_data/models', model_type='smpl', gender='male', ext='pkl') smpl = pickle.load(open('../3d_data/densepose_uv.pkl', 'rb')) faces = np.array(smpl['f_extended'], dtype=np.int64).reshape((-1, 3)) uv_faceid = io.loadmat( '../3d_data/DensePoseData/UV_data/UV_Processed.mat')['All_FaceIndices'] uv = smpl['uv'] uv_vertices = (uv * 2) - 1 # with open('../3d_data/nongrey_male_0110.jpg', 'rb') as file: # texture = cv2.imread('../3d_data/nongrey_male_0110.jpg') seg_path = os.path.join(coco_folder, 'background') if not os.path.exists(seg_path): os.mkdir(seg_path) output = model(return_verts=True) vertices = output.vertices.detach().cpu().numpy().squeeze() smpl_vertices = np.array([vertices[i] for i in smpl['v_extended']]) smpl_render = Render(width=opt.image_width, height=opt.image_height) smpl_render.set_render(vertices=smpl_vertices, faces=faces) smpl_norm_vertices = smpl_render.vertices # smpl_uv = smpl_render.render_interpolate(vertices=uv, skip_cull=False) # smpl_noc = smpl_render.render_interpolate(vertices=smpl_norm_vertices).interpolated aggregate_textures = np.array([], dtype='float64').reshape( (0, 3, opt.image_height, opt.image_width)) uv_render = Render(width=opt.image_width, height=opt.image_height) kernel = np.ones((3, 3), np.uint8) for idx in range(1, 25): # face_select = faces[uv_faceid[:, 0] == idx, :] id_select = np.unique( np.hstack( np.where(faces == vert)[0] for face in faces[uv_faceid[:, 0] == idx, :] for vert in face)) face_select = faces[id_select, :] uv_render.set_render(vertices=uv_vertices, faces=face_select, normalize=False) out_view = np.flip(uv_render.render_interpolate( vertices=smpl_norm_vertices).interpolated.transpose([2, 0, 1]), axis=1) aggregate_textures = np.concatenate( [aggregate_textures, out_view.reshape((1, ) + out_view.shape)]) texture_map = torch.from_numpy(aggregate_textures) print("SMPL textures loaded in memory.\n") ################################# for key in annotation_dict: dp_coco = annotation_dict[key][0] person_coco = annotation_dict[key][1] parent_dir = annotation_dict[key][2] seg_key_path = os.path.join(seg_path, parent_dir) if not os.path.exists(seg_key_path): os.mkdir(seg_key_path) im_ids = dp_coco.getImgIds() # len_ids = len(im_ids) key_list = [] for idx, im_id in enumerate( tqdm(im_ids, desc="Key [{}] Progress".format(key), ncols=100)): im_dict = {} im = dp_coco.loadImgs(im_id)[0] person_im = person_coco.loadImgs(im_id)[0] im_name = os.path.join(coco_folder, parent_dir, im['file_name']) image = cv2.imread(im_name) # im_dict['image'] = image im_dict['file_name'] = os.path.join(parent_dir, im['file_name']) ann_ids = dp_coco.getAnnIds(imgIds=im['id']) anns = dp_coco.loadAnns(ann_ids) person_anns = person_coco.loadAnns(ann_ids) im_dict['points'] = {} zero_im = np.zeros((image.shape[0], image.shape[1])) person_seg = np.zeros((image.shape[0], image.shape[1])) point_dict = im_dict['points'] point_dict['yx'] = np.array([], dtype='int').reshape((0, 2)) point_dict['iuv'] = np.array([]).reshape((0, 3)) xy_mask = np.zeros((image.shape[0], image.shape[1], 1)) zero_point_iuv = np.zeros_like(image) zero_point_uv = np.zeros((24, image.shape[0], image.shape[1], 2)) for person_ann in person_anns: person_seg += person_coco.annToMask(person_ann) for ann in anns: if 'dp_masks' in ann.keys(): bbr = (np.array(ann['bbox'])).astype('int') mask = GetDensePoseMask(ann['dp_masks']) x1, y1, x2, y2 = bbr[0], bbr[ 1], bbr[0] + bbr[2], bbr[1] + bbr[3] x2 = min([x2, image.shape[1]]) y2 = min([y2, image.shape[0]]) mask_im = cv2.resize(mask, (int(x2 - x1), int(y2 - y1)), interpolation=cv2.INTER_NEAREST) # mask_bool = np.tile((mask_im == 0)[:, :, np.newaxis], [1, 1, 3]) zero_im[y1:y2, x1:x2] += mask_im img_x = np.array(ann['dp_x']) / 255. * bbr[ 2] + x1 # Stretch the points to current box. img_y = np.array(ann['dp_y']) / 255. * bbr[ 3] + y1 # Stretch the points to current box. img_x = img_x.astype('int') - 1 * (img_x >= image.shape[1]) img_y = img_y.astype('int') - 1 * (img_y >= image.shape[0]) point_dict['yx'] = np.concatenate( [point_dict['yx'], np.array([img_y, img_x]).T]) point_i = np.array(ann['dp_I']).astype('int') point_u = np.array(ann['dp_U']) point_v = np.array(ann['dp_V']) point_dict['iuv'] = np.concatenate( (point_dict['iuv'], np.array([point_i, point_u, point_v]).T)) zero_point_iuv[img_y, img_x, :] = np.array( [point_i, point_u, point_v]).T xy_mask[img_y, img_x, 0] = 1 zero_point_uv[point_i - 1, img_y, img_x] = np.array([point_u, point_v]).T uv_stack = torch.from_numpy((zero_point_uv * 2) - 1) output_noc = 0 for idx in range(0, 24): output_noc += torch.nn.functional.grid_sample( texture_map[idx:idx + 1], uv_stack[idx:idx + 1], mode='bilinear', padding_mode='border') output_noc = output_noc[0].cpu().numpy().transpose([1, 2, 0]) zero_im = zero_im + person_seg zero_im = (zero_im > 0).astype('float32') zero_im = cv2.dilate(zero_im, kernel, iterations=1) cv2.imwrite(os.path.join(seg_key_path, im['file_name']), (zero_im == 0.0).astype('uint8')) point_dict['noc'] = output_noc[point_dict['yx'][:, 0], point_dict['yx'][:, 1], :] # print(np.min(point_dict['yx'])) key_list.append(im_dict) # cv2.imshow("Image", image) # cv2.imshow("Background Image", (zero_im == 0.0).astype('uint8') * 255) # cv2.imshow("IUV", (zero_point_iuv * 30).astype('uint8')) # cv2.imshow("NOC sampled", (output_noc * 255).astype('uint8')) # # cv2.waitKey(0) # progress_bar(idx + 1, len_ids, prefix="Progress for {}:".format(key), suffix="Complete") save_file = os.path.join(save_annotation_file, '{}.pkl'.format(key)) with open(save_file, 'wb') as write_file: pickle.dump(key_list, write_file, protocol=pickle.HIGHEST_PROTOCOL)