def new_eval_net(data_loader, model, opts):
model.eval()
dataset = data_loader.dataset
scales = [1., 0.5, 0.75, 1.25, 1.5, 2.0]
assert (scales[0] == 1)
n_scales = len(scales)
from data_process.process_utils import resize_hm
import cv2
index = 0
with torch.no_grad():
img, heat_map, paf, ignore_mask, _ = dataset[index]
height = img.shape[1]
width = img.shape[2]
img_torch = torch.from_numpy(img).float().cuda()
img_torch = torch.unsqueeze(img_torch, 0)
heatmaps, pafs = model(img_torch)
heatmap = heatmaps.data.cpu().numpy()[0]
paf = pafs.data.cpu().numpy()[0]
print(np.sum(heatmap < 0))
print(np.sum(heatmap > 1))
heatmap = resize_hm(heatmap, height)
paf = resize_hm(paf, height)
print(np.sum(heatmap < 0))
print(np.sum(heatmap > 1))
raw_img, heat_map, paf, ignore_mask, keypoints = dataset.get_item_raw(
index, False)
cv2.imwrite("eval/img.jpg", raw_img * 255)
for i in range(19):
cv2.imwrite("eval/hm_" + str(i) + ".jpg", heatmap[i] * 255)
def eval_net(data_loader, model, opts):
model.eval()
dataset = data_loader.dataset
scales = [1., 0.5, 0.75, 1.25, 1.5, 2.0]
assert (scales[0]==1)
n_scales = len(scales)
outputs = []
dataset_len = 100 #len(dataset)
keypoints_list = []
runtimes = []
with torch.no_grad():
for i in range(dataset_len):
print(i)
start = time.time()
imgs, heatmap_t, paf_t, ignore_mask_t, keypoints = dataset.get_imgs_multiscale(i, scales,flip=False)
n_imgs = len(imgs)
assert(n_imgs == n_scales)
heights = list(map(lambda x: x.shape[1], imgs))
widths = list(map(lambda x: x.shape[2], imgs))
max_h, max_w = max(heights), max(widths)
imgs_np = np.zeros((n_imgs, 3, max_h, max_w))
for j in range(n_imgs):
img = imgs[j]
h, w = img.shape[1], img.shape[2]
imgs_np[j,:,:h,:w] = img
img_basic = imgs[0]
heatmap_avg_lst = []
paf_avg_lst = []
print("first loop", time.time() - start)
for j in range(0, n_imgs):
imgs_torch = torch.from_numpy(imgs_np[j:j+1]).float().to(opts.device)
heatmaps, pafs = model(imgs_torch)
heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8]
paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8]
heatmap = resize_hm(heatmap, (widths[0], heights[0]))
paf = resize_hm(paf, (widths[0], heights[0]))
heatmap_avg_lst += [heatmap]
paf_avg_lst += [paf]
heatmap_avg = sum(heatmap_avg_lst)/n_imgs
paf_avg = sum(paf_avg_lst)/n_imgs
print("second loop", time.time() - start)
#visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg)
img_basic = denormalize(img_basic)
param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5}
canvas, to_plot, candidate, subset = decode_pose(img_basic, param, heatmap_avg, paf_avg)
final = time.time()-start
runtimes += [final]
print("both loops took ", final)
keypoints_list.append(keypoints)
append_result(dataset.indices[i], subset, candidate, outputs)
vis_path = os.path.join(opts.saveDir, 'viz')
if not os.path.exists(vis_path):
os.makedirs(vis_path)
cv2.imwrite(vis_path+'/{}.png'.format(i), to_plot)
print ("runtime statistics for all images")
print(scipy.stats.describe(runtimes))
return outputs, dataset.indices[:dataset_len]
def visualize_output_single(img, heatmap_t, paf_t, ignore_mask_t, heatmap_o, paf_o):
img = (denormalize(img) * 255).astype('uint8')
heatmap_o = resize_hm(heatmap_o, (img.shape[1], img.shape[0]))
paf_o = resize_hm(paf_o, (img.shape[1], img.shape[0]))
heatmap_t = resize_hm(heatmap_t, (img.shape[1], img.shape[0]))
paf_t = resize_hm(paf_t, (img.shape[1], img.shape[0]))
ignore_mask = cv2.resize(ignore_mask_t, (img.shape[1], img.shape[0]))
visualize_heatmap(img, heatmap_o, 'heatmap_out')
visualize_heatmap(img, heatmap_t, 'heatmap_target')
visualize_paf(img, reshape_paf(paf_o), 'paf_out')
visualize_paf(img, reshape_paf(paf_t), 'paf_target')
visualize_masks(img, ignore_mask)
def eval_oneshot(img_path, saveDir, model, opts):
img = cv2.imread(img_path)
img = img.astype('float32') / 255.
imgs = []
scales = [1., 0.5, 0.75, 1.25, 1.5, 2.0]
for scale in scales:
width, height = img.shape[1], img.shape[0]
new_width, new_height = int(scale* width), int(scale*height)
scaled_img = cv2.resize(img.copy(), (new_width, new_height))
scaled_img = normalize(scaled_img)
imgs.append(scaled_img)
assert(len(imgs) == len(scales))
start = time.time()
heights = list(map(lambda x: x.shape[1], imgs))
widths = list(map(lambda x: x.shape[2], imgs))
max_h, max_w = max(heights), max(widths)
imgs_np = np.zeros((len(imgs), 3, max_h, max_w))
for j in range(len(imgs)):
img = imgs[j]
h, w = img.shape[1], img.shape[2]
imgs_np[j,:,:h,:w] = img
img_basic = imgs[0]
heatmap_avg_lst = []
paf_avg_lst = []
print("first loop", time.time() - start)
with torch.no_grad():
for j in range(len(imgs)):
imgs_torch = torch.from_numpy(imgs_np[j:j+1]).float().cuda()
heatmaps, pafs = model(imgs_torch)
heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8]
paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8]
heatmap = resize_hm(heatmap, (widths[0], heights[0]))
paf = resize_hm(paf, (widths[0], heights[0]))
heatmap_avg_lst += [heatmap]
paf_avg_lst += [paf]
heatmap_avg = sum(heatmap_avg_lst)/len(imgs)
paf_avg = sum(paf_avg_lst)/len(imgs)
print("second loop", time.time() - start)
#visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg)
img_basic = denormalize(img_basic)
param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5}
canvas, to_plot, candidate, subset = decode_pose(img_basic, param, heatmap_avg, paf_avg)
final = time.time()-start
print("both loops took ", final)
vis_path = os.path.join(saveDir, 'viz')
if not os.path.exists(vis_path):
os.makedirs(vis_path)
cv2.imwrite(vis_path+'/out.png', to_plot)
def test_net(data_loader, model, opts):
model.eval()
dataset = data_loader.dataset
outputs = []
dataset_len = 100 #len(dataset)
runtimes = []
with torch.no_grad():
for i, imgs in enumerate(dataset):
print(i)
start = time.time()
n_imgs = len(imgs)
heights = list(map(lambda x: x.shape[1], imgs))
widths = list(map(lambda x: x.shape[2], imgs))
img_basic = imgs[0]
heatmap_avg_lst = []
paf_avg_lst = []
print("first loop", time.time() - start)
for j in range(0, n_imgs):
imgs_torch = torch.from_numpy(imgs[j:j + 1]).float().to(
opts.device)
heatmaps, pafs = model(imgs_torch)
heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j] //
8, :widths[j] // 8]
paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j] //
8, :widths[j] // 8]
heatmap = resize_hm(heatmap, (widths[0], heights[0]))
paf = resize_hm(paf, (widths[0], heights[0]))
heatmap_avg_lst += [heatmap]
paf_avg_lst += [paf]
heatmap_avg = sum(heatmap_avg_lst) / n_imgs
paf_avg = sum(paf_avg_lst) / n_imgs
print("second loop", time.time() - start)
#visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg)
img_basic = denormalize(img_basic)
param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5}
canvas, to_plot, candidate, subset = decode_pose(
img_basic, param, heatmap_avg, paf_avg)
final = time.time() - start
runtimes += [final]
print("both loops took ", final)
append_result(dataset.idx, subset, candidate, outputs)
vis_path = Path(opts.saveDir, 'viz')
vis_path.mkdir(parents=True, exist_ok=True)
cv2.imwrite(str(vis_path) + '/{}.png'.format(i), to_plot)
print("runtime statistics for all images")
def run_pose_model(person_id, img, model):
model.eval()
scales = [1., 0.5, 0.75, 1.25, 1.5, 2.0]
assert (scales[0]==1)
n_scales = len(scales)
outputs = []
keypoints_list = []
with torch.no_grad():
imgs = get_imgs_multiscale(img, scales)
n_imgs = len(imgs)
assert(n_imgs == n_scales)
heights = list(map(lambda x: x.shape[1], imgs))
widths = list(map(lambda x: x.shape[2], imgs))
max_h, max_w = max(heights), max(widths)
imgs_np = np.zeros((n_imgs, 3, max_h, max_w))
for j in range(n_imgs):
img = imgs[j]
h, w = img.shape[1], img.shape[2]
imgs_np[j,:,:h,:w] = img
img_basic = imgs[0]
heatmap_avg_lst = []
paf_avg_lst = []
for j in range(0, n_imgs):
imgs_torch = torch.from_numpy(imgs_np[j:j+1]).float().cuda()
heatmaps, pafs = model(imgs_torch)
heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8]
paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8]
heatmap = resize_hm(heatmap, (widths[0], heights[0]))
paf = resize_hm(paf, (widths[0], heights[0]))
heatmap_avg_lst += [heatmap]
paf_avg_lst += [paf]
heatmap_avg = sum(heatmap_avg_lst)/n_imgs
paf_avg = sum(paf_avg_lst)/n_imgs
img_basic = denormalize(img_basic)
param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5}
canvas, to_plot, candidate, subset = decode_pose(img_basic, param, heatmap_avg, paf_avg)
append_result(person_id, subset, candidate, outputs)
if not os.path.exists('./viz_result'):
os.makedirs('./viz_result')
cv2.imwrite('./viz_result/{}_pose.png'.format(person_id), to_plot)
return outputs
def eval_net(data_loader, model, opts):
model.eval()
dataset = data_loader.dataset
scales = [1., 0.75, 1.25]
assert (scales[0] == 1)
n_scales = len(scales)
outputs = []
dataset_len = 100 #len(dataset)
for i in range(dataset_len):
imgs, heatmap_t, paf_t, ignore_mask_t = dataset.get_imgs_multiscale(
i, scales, flip=False)
n_imgs = len(imgs)
assert (n_imgs == n_scales)
heights = list(map(lambda x: x.shape[1], imgs))
widths = list(map(lambda x: x.shape[2], imgs))
max_h, max_w = max(heights), max(widths)
imgs_np = np.zeros((n_imgs, 3, max_h, max_w))
for j in range(n_imgs):
img = imgs[j]
h, w = img.shape[1], img.shape[2]
imgs_np[j, :, :h, :w] = img
img_basic = imgs[0]
heatmap_avg = np.zeros(heatmap_t.shape)
paf_avg = np.zeros(paf_t.shape)
for j in range(0, n_imgs):
imgs_torch = torch.from_numpy(imgs_np[j:j + 1]).float().cuda()
heatmaps, pafs = model(imgs_torch)
heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j] //
8, :widths[j] // 8]
paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j] //
8, :widths[j] // 8]
heatmap = resize_hm(heatmap, (widths[0], heights[0]))
paf = resize_hm(paf, (widths[0], heights[0]))
heatmap_avg += heatmap / n_imgs
paf_avg += paf / n_imgs
#visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg)
img_basic = denormalize(img_basic)
param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5}
canvas, to_plot, candidate, subset = decode_pose(
img_basic, param, heatmap_avg, paf_avg)
append_result(dataset.indices[i], subset, candidate, outputs)
vis_path = os.path.join(opts.saveDir, 'viz')
if not os.path.exists(vis_path):
os.makedirs(vis_path)
cv2.imwrite(vis_path + '/{}.png'.format(i), to_plot)
return outputs, dataset.indices[:dataset_len]
def visualize_output_single(img, heatmap_t, paf_t, ignore_mask_t, heatmap_o,
paf_o):
img = (denormalize(img) * 255).astype('uint8')
heatmap_o = resize_hm(heatmap_o, (img.shape[1], img.shape[0]))
paf_o = resize_hm(paf_o, (img.shape[1], img.shape[0]))
heatmap_t = resize_hm(heatmap_t, (img.shape[1], img.shape[0]))
paf_t = resize_hm(paf_t, (img.shape[1], img.shape[0]))
ignore_mask = cv2.resize(ignore_mask_t, (img.shape[1], img.shape[0]))
cv2.namedWindow('heatmap_out')
cv2.namedWindow('heatmap_target')
cv2.namedWindow('paf_out')
cv2.namedWindow('paf_target')
cv2.namedWindow('masked_img')
cv2.startWindowThread()
visualize_heatmap(img, heatmap_o, 'heatmap_out')
visualize_heatmap(img, heatmap_t, 'heatmap_target')
visualize_heatmap(img, paf_o, 'paf_out')
visualize_heatmap(img, paf_t, 'paf_target')
visualize_masks(img, ignore_mask)
def eval_net(data_loader, model, opts, ids_in_ckpt=[], fraction=[0, 0.3]):
model.eval()
dataset = data_loader.dataset
scales = [1., 0.5, 1.25, 1.5, 1.75, 2.0]
# scales = [1.]
assert (scales[0]==1)
n_scales = len(scales)
dataset_len = len(dataset)
# keypoints_list = []
runtimes = []
with torch.no_grad():
for i in range(int(dataset_len * fraction[0]), int(dataset_len * fraction[1])):
if dataset.indices[i] in ids_in_ckpt:
print(f"skip {i}th image of image_id {dataset.indices[i]}.")
continue
print(i)
start = time.time()
# imgs, heatmap_t, paf_t, ignore_mask_t = dataset.get_imgs_multiscale(i, scales, flip=False, to_resize=False)
if opts["to_test"]:
imgs, orig_shape = dataset.get_imgs_multiscale_resize(i, scales, flip=False, resize_factor=1)
heatmap_t = np.zeros((opts["model"]["nJoints"], opts["test"]["hmSize"], opts["test"]["hmSize"]))
paf_t = np.zeros((opts["model"]["nLimbs"], opts["test"]["hmSize"], opts["test"]["hmSize"]))
ignore_mask_t = np.zeros((opts["test"]["hmSize"], opts["test"]["hmSize"]))
else:
imgs, heatmap_t, paf_t, ignore_mask_t, orig_shape = dataset.get_imgs_multiscale_resize(i, scales,
flip=False,
resize_factor=1)
resize_factors = tuple([orig_shape[i] / imgs[0].shape[i + 1] for i in [1, 0]])
# resize_factors = (1, 1)
print(f"input_size:{imgs[0].shape}, origin_size:{orig_shape}")
n_imgs = len(imgs)
assert(n_imgs == n_scales)
heights = list(map(lambda x: x.shape[1], imgs))
widths = list(map(lambda x: x.shape[2], imgs))
max_h, max_w = max(heights), max(widths)
imgs_np = np.zeros((n_imgs, 3, max_h, max_w))
for j in range(n_imgs):
img = imgs[j]
h, w = img.shape[1], img.shape[2]
imgs_np[j,:,:h,:w] = img
img_basic = imgs[0]
heatmap_avg_lst = []
paf_avg_lst = []
print("first loop", time.time() - start)
for j in range(0, n_imgs):
imgs_torch = torch.from_numpy(imgs_np[j:j+1]).float().cuda()
heatmaps, pafs = model(imgs_torch)
heatmap = heatmaps[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8]
paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j]//8, :widths[j]//8]
heatmap = resize_hm(heatmap, (widths[0], heights[0]))
paf = resize_hm(paf, (widths[0], heights[0]))
heatmap_avg_lst += [heatmap]
paf_avg_lst += [paf]
heatmap_avg = sum(heatmap_avg_lst)/n_imgs
paf_avg = sum(paf_avg_lst)/n_imgs
print("second loop", time.time() - start)
if opts["viz"]["vizOut"]:
visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg)
img_basic = denormalize(img_basic)
param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5}
vis_path = os.path.join(opts["env"]["saveDir"], "val" if not opts["to_test"] else "test" + '_viz')
def native():
candidates, subset = decode_pose(img_basic, param, heatmap_avg, paf_avg)
plot_pose_pdf(img_basic, candidates, subset, os.path.join(vis_path, f"{i}.pdf"))
outputs = append_result(dataset.indices[i], subset, candidates)
return outputs
def openpose_inf():
outputs = openpose_decode_pose(img_basic, param, heatmap_avg, paf_avg, opts,
dataset.indices[i], os.path.join(vis_path, f"{i}.pdf"),
resize_shape=None)
return outputs
if not os.path.exists(vis_path):
os.makedirs(vis_path)
outputs = native() # native()
for out in outputs:
out['keypoints'][::3] = [i * resize_factors[0] for i in out['keypoints'][::3]]
out['keypoints'][1::3] = [i * resize_factors[1] for i in out['keypoints'][1::3]]
final = time.time()-start
print("both loops took ", final)
yield outputs, dataset.indices[i]
def test_net(data_loader, model, opts):
model.eval()
dataset = data_loader.dataset
scales = [1., 0.5, 0.75, 1.25, 1.5, 2.0]
# scales = [1.]
assert (scales[0] == 1)
n_scales = len(scales)
dataset_len = len(dataset) #len(dataset)
runtimes = []
with torch.no_grad():
for i in range(dataset_len):
print(i)
start = time.time()
imgs = dataset.get_imgs_multiscale(i, scales, flip=False)
n_imgs = len(imgs)
assert (n_imgs == n_scales)
heights = list(map(lambda x: x.shape[1], imgs))
widths = list(map(lambda x: x.shape[2], imgs))
max_h, max_w = max(heights), max(widths)
imgs_np = np.zeros((n_imgs, 3, max_h, max_w))
for j in range(n_imgs):
img = imgs[j]
h, w = img.shape[1], img.shape[2]
imgs_np[j, :, :h, :w] = img
img_basic = imgs[0]
heatmap_avg_lst = []
paf_avg_lst = []
for j in range(0, n_imgs):
if torch.cuda.is_available():
imgs_torch = torch.from_numpy(imgs_np[j:j +
1]).float().cuda()
else:
imgs_torch = torch.from_numpy(imgs_np[j:j +
1]).float().cpu()
pafs, heatmap = model(imgs_torch)
heatmap = heatmap.data.cpu().numpy()[0, :, :heights[j] //
8, :widths[j] // 8]
paf = pafs[-1].data.cpu().numpy()[0, :, :heights[j] //
8, :widths[j] // 8]
heatmap = resize_hm(heatmap, (widths[0], heights[0]))
paf = resize_hm(paf, (widths[0], heights[0]))
heatmap_avg_lst += [heatmap]
paf_avg_lst += [paf]
heatmap_avg = sum(heatmap_avg_lst) / n_imgs
paf_avg = sum(paf_avg_lst) / n_imgs
#visualize_output_single(img_basic, heatmap_t, paf_t, ignore_mask_t, heatmap_avg, paf_avg)
img_basic = denormalize(img_basic)
param = {'thre1': 0.1, 'thre2': 0.05, 'thre3': 0.5}
canvas, to_plot, candidate, subset = decode_pose(
img_basic, param, heatmap_avg, paf_avg)
final = time.time() - start
runtimes += [final]
image_id = dataset.indices[i]
print(image_id)
if len(candidate) > 0:
candidate = candidate[:, :3]
vis_path = os.path.join(opts.saveDir, 'viz')
if not os.path.exists(vis_path):
os.makedirs(vis_path)
cv2.imwrite(vis_path + '/{}'.format(image_id), to_plot)
print("All images finished")