def new_eval_net(data_loader, model, opts): model.eval() dataset = data_loader.dataset scales = [1., 0.5, 0.75, 1.25, 1.5, 2.0] assert (scales[0] == 1) n_scales = len(scales) from data_process.process_utils import resize_hm import cv2 index = 0 with torch.no_grad(): img, heat_map, paf, ignore_mask, _ = dataset[index] height = img.shape[1] width = img.shape[2] img_torch = torch.from_numpy(img).float().cuda() img_torch = torch.unsqueeze(img_torch, 0) heatmaps, pafs = model(img_torch) heatmap = heatmaps.data.cpu().numpy()[0] paf = pafs.data.cpu().numpy()[0] print(np.sum(heatmap < 0)) print(np.sum(heatmap > 1)) heatmap = resize_hm(heatmap, height) paf = resize_hm(paf, height) print(np.sum(heatmap < 0)) print(np.sum(heatmap > 1)) raw_img, heat_map, paf, ignore_mask, keypoints = dataset.get_item_raw( index, False) cv2.imwrite("eval/img.jpg", raw_img * 255) for i in range(19): cv2.imwrite("eval/hm_" + str(i) + ".jpg", heatmap[i] * 255)
def eval_net(data_loader, model, opts): model.eval() dataset = data_loader.dataset scales = [1., 0.5, 0.75, 1.25, 1.5, 2.0] assert (scales[0]==1) n_scales = len(scales) outputs = [] dataset_len = 100 #len(dataset) keypoints_list = [] runtimes = [] with torch.no_grad(): for i in range(dataset_len): print(i) start = time.time() imgs, heatmap_t, paf_t, ignore_mask_t, keypoints = dataset.get_imgs_multiscale(i, scales,flip=False) n_imgs = len(imgs) assert(n_imgs == n_scales) heights = list(map(lambda x: x.shape[1], imgs)) widths = list(map(lambda x: x.shape[2], imgs)) max_h, max_w = max(heights), max(widths) imgs_np = np.zeros((n_imgs, 3, max_h, max_w)) for j in range(n_imgs): img = imgs[j] h, w = img.shape[1], img.shape[2] imgs_np[j,:,:h,:w] = img img_basic = imgs[0] heatmap_avg_lst = [] paf_avg_lst = [] print("first loop", time.time() - start) for j in range(0, n_imgs): imgs_torch = torch.from_numpy(imgs_np[j:j+1]).float().to(opts.device) heatmaps, pafs = model(imgs_torch) heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8] paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8] heatmap = resize_hm(heatmap, (widths[0], heights[0])) paf = resize_hm(paf, (widths[0], heights[0])) heatmap_avg_lst += [heatmap] paf_avg_lst += [paf] heatmap_avg = sum(heatmap_avg_lst)/n_imgs paf_avg = sum(paf_avg_lst)/n_imgs print("second loop", time.time() - start) #visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg) img_basic = denormalize(img_basic) param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5} canvas, to_plot, candidate, subset = decode_pose(img_basic, param, heatmap_avg, paf_avg) final = time.time()-start runtimes += [final] print("both loops took ", final) keypoints_list.append(keypoints) append_result(dataset.indices[i], subset, candidate, outputs) vis_path = os.path.join(opts.saveDir, 'viz') if not os.path.exists(vis_path): os.makedirs(vis_path) cv2.imwrite(vis_path+'/{}.png'.format(i), to_plot) print ("runtime statistics for all images") print(scipy.stats.describe(runtimes)) return outputs, dataset.indices[:dataset_len]
def visualize_output_single(img, heatmap_t, paf_t, ignore_mask_t, heatmap_o, paf_o): img = (denormalize(img) * 255).astype('uint8') heatmap_o = resize_hm(heatmap_o, (img.shape[1], img.shape[0])) paf_o = resize_hm(paf_o, (img.shape[1], img.shape[0])) heatmap_t = resize_hm(heatmap_t, (img.shape[1], img.shape[0])) paf_t = resize_hm(paf_t, (img.shape[1], img.shape[0])) ignore_mask = cv2.resize(ignore_mask_t, (img.shape[1], img.shape[0])) visualize_heatmap(img, heatmap_o, 'heatmap_out') visualize_heatmap(img, heatmap_t, 'heatmap_target') visualize_paf(img, reshape_paf(paf_o), 'paf_out') visualize_paf(img, reshape_paf(paf_t), 'paf_target') visualize_masks(img, ignore_mask)
def eval_oneshot(img_path, saveDir, model, opts): img = cv2.imread(img_path) img = img.astype('float32') / 255. imgs = [] scales = [1., 0.5, 0.75, 1.25, 1.5, 2.0] for scale in scales: width, height = img.shape[1], img.shape[0] new_width, new_height = int(scale* width), int(scale*height) scaled_img = cv2.resize(img.copy(), (new_width, new_height)) scaled_img = normalize(scaled_img) imgs.append(scaled_img) assert(len(imgs) == len(scales)) start = time.time() heights = list(map(lambda x: x.shape[1], imgs)) widths = list(map(lambda x: x.shape[2], imgs)) max_h, max_w = max(heights), max(widths) imgs_np = np.zeros((len(imgs), 3, max_h, max_w)) for j in range(len(imgs)): img = imgs[j] h, w = img.shape[1], img.shape[2] imgs_np[j,:,:h,:w] = img img_basic = imgs[0] heatmap_avg_lst = [] paf_avg_lst = [] print("first loop", time.time() - start) with torch.no_grad(): for j in range(len(imgs)): imgs_torch = torch.from_numpy(imgs_np[j:j+1]).float().cuda() heatmaps, pafs = model(imgs_torch) heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8] paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8] heatmap = resize_hm(heatmap, (widths[0], heights[0])) paf = resize_hm(paf, (widths[0], heights[0])) heatmap_avg_lst += [heatmap] paf_avg_lst += [paf] heatmap_avg = sum(heatmap_avg_lst)/len(imgs) paf_avg = sum(paf_avg_lst)/len(imgs) print("second loop", time.time() - start) #visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg) img_basic = denormalize(img_basic) param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5} canvas, to_plot, candidate, subset = decode_pose(img_basic, param, heatmap_avg, paf_avg) final = time.time()-start print("both loops took ", final) vis_path = os.path.join(saveDir, 'viz') if not os.path.exists(vis_path): os.makedirs(vis_path) cv2.imwrite(vis_path+'/out.png', to_plot)
def test_net(data_loader, model, opts): model.eval() dataset = data_loader.dataset outputs = [] dataset_len = 100 #len(dataset) runtimes = [] with torch.no_grad(): for i, imgs in enumerate(dataset): print(i) start = time.time() n_imgs = len(imgs) heights = list(map(lambda x: x.shape[1], imgs)) widths = list(map(lambda x: x.shape[2], imgs)) img_basic = imgs[0] heatmap_avg_lst = [] paf_avg_lst = [] print("first loop", time.time() - start) for j in range(0, n_imgs): imgs_torch = torch.from_numpy(imgs[j:j + 1]).float().to( opts.device) heatmaps, pafs = model(imgs_torch) heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j] // 8, :widths[j] // 8] paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j] // 8, :widths[j] // 8] heatmap = resize_hm(heatmap, (widths[0], heights[0])) paf = resize_hm(paf, (widths[0], heights[0])) heatmap_avg_lst += [heatmap] paf_avg_lst += [paf] heatmap_avg = sum(heatmap_avg_lst) / n_imgs paf_avg = sum(paf_avg_lst) / n_imgs print("second loop", time.time() - start) #visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg) img_basic = denormalize(img_basic) param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5} canvas, to_plot, candidate, subset = decode_pose( img_basic, param, heatmap_avg, paf_avg) final = time.time() - start runtimes += [final] print("both loops took ", final) append_result(dataset.idx, subset, candidate, outputs) vis_path = Path(opts.saveDir, 'viz') vis_path.mkdir(parents=True, exist_ok=True) cv2.imwrite(str(vis_path) + '/{}.png'.format(i), to_plot) print("runtime statistics for all images")
def run_pose_model(person_id, img, model): model.eval() scales = [1., 0.5, 0.75, 1.25, 1.5, 2.0] assert (scales[0]==1) n_scales = len(scales) outputs = [] keypoints_list = [] with torch.no_grad(): imgs = get_imgs_multiscale(img, scales) n_imgs = len(imgs) assert(n_imgs == n_scales) heights = list(map(lambda x: x.shape[1], imgs)) widths = list(map(lambda x: x.shape[2], imgs)) max_h, max_w = max(heights), max(widths) imgs_np = np.zeros((n_imgs, 3, max_h, max_w)) for j in range(n_imgs): img = imgs[j] h, w = img.shape[1], img.shape[2] imgs_np[j,:,:h,:w] = img img_basic = imgs[0] heatmap_avg_lst = [] paf_avg_lst = [] for j in range(0, n_imgs): imgs_torch = torch.from_numpy(imgs_np[j:j+1]).float().cuda() heatmaps, pafs = model(imgs_torch) heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8] paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8] heatmap = resize_hm(heatmap, (widths[0], heights[0])) paf = resize_hm(paf, (widths[0], heights[0])) heatmap_avg_lst += [heatmap] paf_avg_lst += [paf] heatmap_avg = sum(heatmap_avg_lst)/n_imgs paf_avg = sum(paf_avg_lst)/n_imgs img_basic = denormalize(img_basic) param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5} canvas, to_plot, candidate, subset = decode_pose(img_basic, param, heatmap_avg, paf_avg) append_result(person_id, subset, candidate, outputs) if not os.path.exists('./viz_result'): os.makedirs('./viz_result') cv2.imwrite('./viz_result/{}_pose.png'.format(person_id), to_plot) return outputs
def eval_net(data_loader, model, opts): model.eval() dataset = data_loader.dataset scales = [1., 0.75, 1.25] assert (scales[0] == 1) n_scales = len(scales) outputs = [] dataset_len = 100 #len(dataset) for i in range(dataset_len): imgs, heatmap_t, paf_t, ignore_mask_t = dataset.get_imgs_multiscale( i, scales, flip=False) n_imgs = len(imgs) assert (n_imgs == n_scales) heights = list(map(lambda x: x.shape[1], imgs)) widths = list(map(lambda x: x.shape[2], imgs)) max_h, max_w = max(heights), max(widths) imgs_np = np.zeros((n_imgs, 3, max_h, max_w)) for j in range(n_imgs): img = imgs[j] h, w = img.shape[1], img.shape[2] imgs_np[j, :, :h, :w] = img img_basic = imgs[0] heatmap_avg = np.zeros(heatmap_t.shape) paf_avg = np.zeros(paf_t.shape) for j in range(0, n_imgs): imgs_torch = torch.from_numpy(imgs_np[j:j + 1]).float().cuda() heatmaps, pafs = model(imgs_torch) heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j] // 8, :widths[j] // 8] paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j] // 8, :widths[j] // 8] heatmap = resize_hm(heatmap, (widths[0], heights[0])) paf = resize_hm(paf, (widths[0], heights[0])) heatmap_avg += heatmap / n_imgs paf_avg += paf / n_imgs #visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg) img_basic = denormalize(img_basic) param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5} canvas, to_plot, candidate, subset = decode_pose( img_basic, param, heatmap_avg, paf_avg) append_result(dataset.indices[i], subset, candidate, outputs) vis_path = os.path.join(opts.saveDir, 'viz') if not os.path.exists(vis_path): os.makedirs(vis_path) cv2.imwrite(vis_path + '/{}.png'.format(i), to_plot) return outputs, dataset.indices[:dataset_len]
def visualize_output_single(img, heatmap_t, paf_t, ignore_mask_t, heatmap_o, paf_o): img = (denormalize(img) * 255).astype('uint8') heatmap_o = resize_hm(heatmap_o, (img.shape[1], img.shape[0])) paf_o = resize_hm(paf_o, (img.shape[1], img.shape[0])) heatmap_t = resize_hm(heatmap_t, (img.shape[1], img.shape[0])) paf_t = resize_hm(paf_t, (img.shape[1], img.shape[0])) ignore_mask = cv2.resize(ignore_mask_t, (img.shape[1], img.shape[0])) cv2.namedWindow('heatmap_out') cv2.namedWindow('heatmap_target') cv2.namedWindow('paf_out') cv2.namedWindow('paf_target') cv2.namedWindow('masked_img') cv2.startWindowThread() visualize_heatmap(img, heatmap_o, 'heatmap_out') visualize_heatmap(img, heatmap_t, 'heatmap_target') visualize_heatmap(img, paf_o, 'paf_out') visualize_heatmap(img, paf_t, 'paf_target') visualize_masks(img, ignore_mask)
def eval_net(data_loader, model, opts, ids_in_ckpt=[], fraction=[0, 0.3]): model.eval() dataset = data_loader.dataset scales = [1., 0.5, 1.25, 1.5, 1.75, 2.0] # scales = [1.] assert (scales[0]==1) n_scales = len(scales) dataset_len = len(dataset) # keypoints_list = [] runtimes = [] with torch.no_grad(): for i in range(int(dataset_len * fraction[0]), int(dataset_len * fraction[1])): if dataset.indices[i] in ids_in_ckpt: print(f"skip {i}th image of image_id {dataset.indices[i]}.") continue print(i) start = time.time() # imgs, heatmap_t, paf_t, ignore_mask_t = dataset.get_imgs_multiscale(i, scales, flip=False, to_resize=False) if opts["to_test"]: imgs, orig_shape = dataset.get_imgs_multiscale_resize(i, scales, flip=False, resize_factor=1) heatmap_t = np.zeros((opts["model"]["nJoints"], opts["test"]["hmSize"], opts["test"]["hmSize"])) paf_t = np.zeros((opts["model"]["nLimbs"], opts["test"]["hmSize"], opts["test"]["hmSize"])) ignore_mask_t = np.zeros((opts["test"]["hmSize"], opts["test"]["hmSize"])) else: imgs, heatmap_t, paf_t, ignore_mask_t, orig_shape = dataset.get_imgs_multiscale_resize(i, scales, flip=False, resize_factor=1) resize_factors = tuple([orig_shape[i] / imgs[0].shape[i + 1] for i in [1, 0]]) # resize_factors = (1, 1) print(f"input_size:{imgs[0].shape}, origin_size:{orig_shape}") n_imgs = len(imgs) assert(n_imgs == n_scales) heights = list(map(lambda x: x.shape[1], imgs)) widths = list(map(lambda x: x.shape[2], imgs)) max_h, max_w = max(heights), max(widths) imgs_np = np.zeros((n_imgs, 3, max_h, max_w)) for j in range(n_imgs): img = imgs[j] h, w = img.shape[1], img.shape[2] imgs_np[j,:,:h,:w] = img img_basic = imgs[0] heatmap_avg_lst = [] paf_avg_lst = [] print("first loop", time.time() - start) for j in range(0, n_imgs): imgs_torch = torch.from_numpy(imgs_np[j:j+1]).float().cuda() heatmaps, pafs = model(imgs_torch) heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8] paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8] heatmap = resize_hm(heatmap, (widths[0], heights[0])) paf = resize_hm(paf, (widths[0], heights[0])) heatmap_avg_lst += [heatmap] paf_avg_lst += [paf] heatmap_avg = sum(heatmap_avg_lst)/n_imgs paf_avg = sum(paf_avg_lst)/n_imgs print("second loop", time.time() - start) if opts["viz"]["vizOut"]: visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg) img_basic = denormalize(img_basic) param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5} vis_path = os.path.join(opts["env"]["saveDir"], "val" if not opts["to_test"] else "test" + '_viz') def native(): candidates, subset = decode_pose(img_basic, param, heatmap_avg, paf_avg) plot_pose_pdf(img_basic, candidates, subset, os.path.join(vis_path, f"{i}.pdf")) outputs = append_result(dataset.indices[i], subset, candidates) return outputs def openpose_inf(): outputs = openpose_decode_pose(img_basic, param, heatmap_avg, paf_avg, opts, dataset.indices[i], os.path.join(vis_path, f"{i}.pdf"), resize_shape=None) return outputs if not os.path.exists(vis_path): os.makedirs(vis_path) outputs = native() # native() for out in outputs: out['keypoints'][::3] = [i * resize_factors[0] for i in out['keypoints'][::3]] out['keypoints'][1::3] = [i * resize_factors[1] for i in out['keypoints'][1::3]] final = time.time()-start print("both loops took ", final) yield outputs, dataset.indices[i]
def test_net(data_loader, model, opts): model.eval() dataset = data_loader.dataset scales = [1., 0.5, 0.75, 1.25, 1.5, 2.0] # scales = [1.] assert (scales[0] == 1) n_scales = len(scales) dataset_len = len(dataset) #len(dataset) runtimes = [] with torch.no_grad(): for i in range(dataset_len): print(i) start = time.time() imgs = dataset.get_imgs_multiscale(i, scales, flip=False) n_imgs = len(imgs) assert (n_imgs == n_scales) heights = list(map(lambda x: x.shape[1], imgs)) widths = list(map(lambda x: x.shape[2], imgs)) max_h, max_w = max(heights), max(widths) imgs_np = np.zeros((n_imgs, 3, max_h, max_w)) for j in range(n_imgs): img = imgs[j] h, w = img.shape[1], img.shape[2] imgs_np[j, :, :h, :w] = img img_basic = imgs[0] heatmap_avg_lst = [] paf_avg_lst = [] for j in range(0, n_imgs): if torch.cuda.is_available(): imgs_torch = torch.from_numpy(imgs_np[j:j + 1]).float().cuda() else: imgs_torch = torch.from_numpy(imgs_np[j:j + 1]).float().cpu() pafs, heatmap = model(imgs_torch) heatmap = heatmap.data.cpu().numpy()[0, :, :heights[j] // 8, :widths[j] // 8] paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j] // 8, :widths[j] // 8] heatmap = resize_hm(heatmap, (widths[0], heights[0])) paf = resize_hm(paf, (widths[0], heights[0])) heatmap_avg_lst += [heatmap] paf_avg_lst += [paf] heatmap_avg = sum(heatmap_avg_lst) / n_imgs paf_avg = sum(paf_avg_lst) / n_imgs #visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg) img_basic = denormalize(img_basic) param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5} canvas, to_plot, candidate, subset = decode_pose( img_basic, param, heatmap_avg, paf_avg) final = time.time() - start runtimes += [final] image_id = dataset.indices[i] print(image_id) if len(candidate) > 0: candidate = candidate[:, :3] vis_path = os.path.join(opts.saveDir, 'viz') if not os.path.exists(vis_path): os.makedirs(vis_path) cv2.imwrite(vis_path + '/{}'.format(image_id), to_plot) print("All images finished")