def predict(data_loader, net, encoder, config):
net.eval()
nes = []
for i, (imgs, heatmaps, vismaps, kpts) in tqdm(enumerate(data_loader)):
data = Variable(imgs.cuda(async=True), volatile=True)
hm_preds1, hm_preds2 = net(data)
hm_preds2 = F.relu(hm_preds2, False)
for j in range(len(imgs)):
img = imgs[j]
img = np.transpose(img.numpy(), (1, 2, 0))
img = ((img * config.sigma + config.mu) * 255).astype(np.uint8)
hm_pred = hm_preds2[j].data.cpu().numpy()
heatmap = heatmaps[j].numpy()
x, y = encoder.decode_np(hm_pred, scale=1, stride=config.hm_stride, method='exp')
keypoints = np.stack([x, y, np.ones(x.shape)], axis=1).astype(np.int16)
kpt = kpts[j].numpy()
ne = normalized_error(keypoints, kpt, img.shape[1])
nes.append(ne)
kp_img = draw_keypoints(img, keypoints, kpt)
hps = []
for k in range(len(hm_pred)):
hp = hm_pred[k]
hp2 = heatmap[k]
hp = print_heatmap(hp, img, config)
hp2 = print_heatmap(hp2, img, config)
hps.append(np.concatenate([hp, hp2], axis=0))
cv2.imwrite('/home/storage/lsy/fashion/tmp/%d-%d-hm.png' % (i, j), np.concatenate(hps, axis=1))
cv2.imwrite('/home/storage/lsy/fashion/tmp/%d-%d-kp.png' % (i, j), kp_img)
print(np.nanmean(np.array(nes)))
def main(**kwargs):
opt._parse(kwargs)
# n_gpu = utils.set_gpu(opt.gpu)
val_dataset = FashionAIKeypoints(opt, phase='val')
encoder = val_dataset.encoder
nes = []
print('Evaluating: {}'.format(opt.category))
print('Validation sample number: {}'.format(len(val_dataset)))
cudnn.benchmark = True
net = getattr(models, opt.model)(opt)
checkpoint = torch.load(opt.load_checkpoint_path) # Must be before cuda
net.load_state_dict(checkpoint['state_dict'])
net = net.cuda()
# net = DataParallel(net)
net.eval()
for idx in tqdm(range(len(val_dataset))):
img_path = val_dataset.get_image_path(idx)
kpts = val_dataset.get_keypoints(idx)
img0 = cv2.imread(img_path) #X
img0_flip = cv2.flip(img0, 1)
img_h, img_w, _ = img0.shape
scale = opt.img_max_size / max(img_w, img_h)
with torch.no_grad():
hm_pred = utils.compute_keypoints(opt, img0, net, encoder)
hm_pred2 = utils.compute_keypoints(opt,
img0_flip,
net,
encoder,
doflip=True)
x, y = encoder.decode_np(hm_pred + hm_pred2, scale, opt.hm_stride,
(img_w / 2, img_h / 2))
keypoints = np.stack([x, y, np.ones(x.shape)], axis=1).astype(np.int16)
if args.visual:
kpt_img = utils.draw_keypoints(img0, keypoints)
save_img_path = str(opt.db_path /
'tmp/one_{0}{1}.png'.format(opt.category, idx))
cv2.imwrite(save_img_path, kpt_img)
left, right = opt.datum
x1, y1, v1 = kpts[left]
x2, y2, v2 = kpts[right]
if v1 == -1 or v2 == -1:
continue
width = math.sqrt((x2 - x1)**2 + (y2 - y1)**2)
ne = utils.normalized_error(keypoints, kpts, width)
nes.append([ne])
nes = np.array(nes)
print(np.mean(nes, axis=0))
# Read left and right image for panoramic stitching
im_left = imread('.\sample image\DT_left.jpg')
im_right = imread('.\sample image\DT_right.jpg')
# Create the SIFT feature point detector object
sift = SIFT_create()
# Identify the keypoints and SIFT descriptors
skp_left = sift.detect(im_left)
skp_right = sift.detect(im_right)
skp_left, sd_left = sift.compute(im_left, skp_left)
skp_right, sd_right = sift.compute(im_right, skp_right)
# Plot the keypoints on the image
keypoints_left = draw_keypoints(im_left, skp_left)
plot_image(keypoints_left, 'Keypoints on Left Image')
keypoints_right = draw_keypoints(im_right, skp_right)
plot_image(keypoints_right, 'Keypoints on Right Image')
# Adjust the descriptors to be of equal sizes
if sd_left.size < sd_right.size:
sd_right = sd_right[0:sd_left.shape[0], 0:sd_left.shape[1]]
elif sd_left.size >= sd_right.size:
sd_left = sd_left[0:sd_right.shape[0], 0:sd_right.shape[1]]
# Identify the matched pairs of points
match_points = create_matched_pairs(sd_left, sd_right, 0.5)
# Plot the matched pairs on the image
large_image = match_image_points(im_left, im_right, skp_left, skp_right, match_points, True)
except Exception:
# No object found
continue
orig_img = cv2.imread(meta['path'][0])
orig_inp = im_to_torch(cv2.cvtColor(orig_img, cv2.COLOR_BGR2RGB))
cropped_inputs, pt1, pt2 = crop_from_dets(orig_inp, bboxes[0], 320,
256)
hms = pose_model(cropped_inputs.unsqueeze(0).cuda()).cpu()
try:
_, pred_kps, pred_kps_score = getPrediction(
hms, pt1.unsqueeze(0), pt2.unsqueeze(0), 320, 256, 80, 64)
except Exception:
continue
if args.name == 'linemod-single':
f = bench.frames[args.seq][int(idx)]
elif args.name == 'linemod-occ':
f = bench.frames['02'][int(idx)]
annot = f['annots'][f['obj_ids'].index(int(args.seq))]
gt_kps = annot['kps']
# save_dir = os.path.join('./results/hms/%s' % idx)
# draw_heatmap(hms[0], save_dir)
save_path = os.path.join('./results/kps/%s.png' % idx)
draw_keypoints(img_path, gt_kps, pred_kps[0].numpy(),
bboxes[0].numpy(), pred_kps_score[0].squeeze().numpy(),
save_path)
def corner_display(imgs, fe):
img_final = []
im1 = []
y = []
for img in imgs:
start1 = time.time()
pts, desc, heatmap = fe.run(img)
end1 = time.time()
points = utils.select_top_k(pts)
im1.append(utils.draw_keypoints(img, points, (0, 255, 0)))
# utils.plot_imgs(im1, ylabel='current solution', dpi=200, cmap='gray', titles=titles)
img_final.append(im1)
y.append('deep learning')
im2 = []
for img in imgs:
dst = cv2.cornerHarris(img, 2, 3, 0.04)
dst = cv2.dilate(dst, None)
dst = np.array(dst)
xs, ys = np.where(dst >= 0.00001)
pts = np.zeros((3, len(xs)))
pts[0, :] = ys
pts[1, :] = xs
pts[2, :] = dst[xs, ys]
pts, _ = fe.nms_fast(pts, img.shape[0], img.shape[1], dist_thresh=fe.nms_dist)
points = utils.select_top_k(pts)
im2.append(utils.draw_keypoints(img, points, (0, 255, 0)))
#utils.plot_imgs(im2, ylabel='harris', dpi=200, cmap='gray', titles=titles)
img_final.append(im2)
y.append('harris')
im3 = []
for img in imgs:
#dst = cv2.cornerSubPix(img, dst, 500, 0.01, 10, )
dst = cv2.goodFeaturesToTrack(img, 300, 0.01, 10, blockSize=3, k=0.04)
dst = dst.astype(int)
points = list(np.squeeze(dst).T)
im3.append(utils.draw_keypoints(img, points, (0, 255, 0)))
# utils.plot_imgs(im3, ylabel='Shi', dpi=200, cmap='gray', titles=titles)
img_final.append(im3)
y.append('subpix')
im4 = []
for img in imgs:
fast = cv2.FastFeatureDetector_create(threshold=70, nonmaxSuppression=True)
img = img * 255.
img = np.uint8(img)
kp = fast.detect(img, None)
# kp.sort()
# kp = kp[:300]
pts = np.zeros((2, len(kp)), np.int)
i = 0
for k in kp:
pts[0, i] = k.pt[0]
pts[1, i] = k.pt[1]
i += 1
points = pts.tolist()
im4.append(utils.draw_keypoints(img, points, (1, 255, 1)))
# orb = cv2.ORB_create()
# kp, des = orb.detectAndCompute(img, None)
# print(type(kp[0]))
img_final.append(im4)
y.append('fast')
title = '300 points'
utils.plot_imgs(img_final, ylabel=y, dpi=200, cmap='gray', title=title)
def main(**kwargs):
opt._parse(kwargs)
# n_gpu = utils.set_gpu(opt.gpu)
test_dataset = FashionAIKeypoints(opt, phase='test')
encoder = test_dataset.encoder
df = utils.data_frame_template()
print('Testing: {}'.format(opt.category))
print('Testing sample number: {}'.format(len(val_dataset)))
cudnn.benchmark = True
net1 = getattr(models, opt.model[0])(opt)
checkpoint = torch.load(opt.load_checkpoint_path) # Must be before cuda
net1.load_state_dict(checkpoint['state_dict'])
net1 = net1.cuda()
# net1 = DataParallel(net)
net1.eval()
net2 = getattr(models, opt.model[1])(opt)
checkpoint = torch.load(opt.load_checkpoint_path_2) # Must be before cuda
net2.load_state_dict(checkpoint['state_dict'])
net2 = net2.cuda()
# net2 = DataParallel(net2)
net2.eval()
for idx in tqdm(range(len(test_dataset))):
img_path = test_dataset.get_image_path(idx)
img0 = cv2.imread(img_path) # BGR
img0_flip = cv2.flip(img0, 1)
img_h, img_w, _ = img0.shape
scale = opt.img_max_size / max(img_w, img_h)
hm_pred = utils.compute_keypoints(opt, img0, net, encoder)
hm_pred_flip = utils.compute_keypoints(opt,
img0_flip,
net,
encoder,
doflip=True)
hm_pred2 = utils.compute_keypoints(opt, img0, net2, encoder)
hm_pred_flip2 = utils.compute_keypoints(opt,
img0_flip,
net2,
encoder,
doflip=True)
x, y = encoder.decode_np(
hm_pred + hm_pred_flip + hm_pred2 + hm_pred_flip2, scale,
opt.hm_stride, (img_w / 2, img_h / 2))
keypoints = np.stack([x, y, np.ones(x.shape)], axis=1).astype(np.int16)
row = test_dataset.anno_df.iloc[idx]
df.at[idx, 'image_id'] = row['image_id']
df.at[idx, 'image_category'] = row['image_category']
for k, kpt_name in enumerate(opt.keypoints[opt.category]):
df.at[idx, kpt_name] = str(keypoints[k, 0]) + '_' + str(
keypoints[k, 1]) + '_1'
if args.visual:
kp_img = utils.draw_keypoints(img0, keypoints)
save_img_path = str(
opt.db_path /
'tmp/ensemble_{0}{1}.png'.format(opt.category, idx))
cv2.imwrite(save_img_path, kpt_img)
df.fillna('-1_-1_-1', inplace=True)
print(df.head(5))
df.to_csv(opt.pred_path / 'ensemble_{}.csv'.format(opt.category),
index=False)
img_h, img_w, _ = img0.shape
scale = opt.img_max_size / max(img_w, img_h)
with torch.no_grad():
hm_pred = utils.compute_keypoints(opt, img0, net, encoder)
hm_pred_flip = utils.compute_keypoints(opt, img0_flip, net, encoder, doflip=True)
utils.
x, y = encoder.decode_np(hm_pred+hm_pred_flip, scale, opt.hm_stride, (img_w/2, img_h/2))
keypoints = np.stack([x, y, np.ones(x.shape)], axis=1).astype(np.int16)
row = test_dataset.anno_df.iloc[idx]
df.at[idx, 'image_id'] = row['image_id']
df.at[idx, 'image_category'] = row['image_category']
for k, kpt_name in enumerate(opt.keypoints[opt.category]):
df.at[idx, kpt_name] = str(keypoints[k,0])+'_'+str(keypoints[k,1])+'_1'
if args.visual:
kpt_img = utils.draw_keypoints(img0, keypoints)
save_img_path = str(opt.db_path / 'tmp/one_{0}{1}.png'.format(opt.category, idx))
cv2.imwrite(save_img_path, kpt_img)
df.fillna('-1_-1_-1', inplace=True)
print(df.head(5))
df.to_csv(opt.pred_path / 'one_{}.csv'.format(opt.category), index=False)
if __name__ == '__main__':
import fire
fire.Fire()
if imgs is not None and bboxs is not None:
for img, bbox in zip(imgs, bboxs):
bbox = np.array([int(box) for box in bbox])
x = bbox[0]
y = bbox[1]
w = bbox[2] - bbox[0]
h = bbox[3] - bbox[1]
cv2.rectangle(display_img, (x, y), (x + w, y + h), (0, 255, 0),
2)
if args.mode == "emo":
points = handler.get(frame, bbox, get_all=True)
display_img = draw_keypoints(display_img, points)
points = points[0]
preproc_img = preprocess(args, points, frame, [x, y, w, h])
emo = clf.predict(np.expand_dims(preproc_img, 0))
display_img = put_text(display_img,
LABELS_DICT_EMO[np.argmax(emo[0])],
(x, y - 20))
elif args.mode == "reco":
f2 = model.get_feature(img)
name = compare_emdbs(embds_dict, f2)
display_img = put_text(display_img, name, (x, y - 40))
# print(sim, ": ", sim >= 0.5 and sim < 1.01)