def preprocess(self, image):
image = image[np.newaxis]
image = image / 256.0 - 0.5
center_map = utils.make_gaussian(self.input_img_size,
self.input_img_size / 2,
self.input_img_size / 2,
self.center_map_radius)
center_map = center_map[np.newaxis, :, :, np.newaxis]
return image, center_map
def __getitem__(self, idx):
"""returns crop from original image (img),
path to original image, heatmaps (np.ndarray (17,94,70))"""
annos = self.coco.anns[self.idxs[idx]]
org_img_path = os.path.join(
self.root, self.coco.imgs[annos['image_id']]['file_name'])
box_crop, kpts_crop = self._crop_box(org_img_path, annos)
kpts_t = [(kpts_crop[i], kpts_crop[i + 1])
for i in range(0, len(kpts_crop), 3)
if kpts_crop[i + 2] != 0]
kpts_idxs = [
e for e, i in enumerate(range(0, len(kpts_crop), 3))
if kpts_crop[i + 2] != 0
]
if self.transform:
try:
transformed = self.transform(image=box_crop, keypoints=kpts_t)
except:
print("Something is gone wrong")
print(org_img_path)
print(kpts_t)
print(box_crop.shape)
img = transformed['image']
kpts = transformed['keypoints']
keypoints = []
j = 0
for i in range(self.num_keypoints):
if i not in kpts_idxs:
keypoints.extend([0, 0, 0])
else:
keypoints.extend([int(kpts[j][0]), int(kpts[j][1]), 2])
j += 1
_, height, width = img.shape
scale_w = width / self.hm_size[0]
scale_h = height / self.hm_size[1]
heatmaps = [
make_gaussian(self.hm_size[0], self.hm_size[1],
int(keypoints[p] / scale_w),
int(keypoints[p + 1] /
scale_h)) if keypoints[p + 2] != 0 else np.zeros(
(self.hm_size[1], self.hm_size[0]))
for p in range(0, len(keypoints), 3)
]
heatmaps = torch.tensor(heatmaps)
return img, org_img_path, heatmaps, keypoints
def main(perform_exp_n_times: int, iters_per_experiment: int,
show_plots: bool):
print('TensorFlow version %s' % str(tf.__version__))
print('PyTorch version %s' % str(torch.__version__))
# allowing soft growth of GPU in tensorflow
gpus = tf.config.experimental.list_physical_devices('GPU')
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
# create an image of circular masks
g1 = create_circular_mask(21, 21, radius=4)
g2 = create_circular_mask(21, 21, radius=5)
g3 = create_circular_mask(21, 21, [3, 3], radius=3)
image = np.stack([g1, g2, g3], axis=-1)[None]
# create kernel to convolve image with
gauss_kernel = np.stack([make_gaussian(5, 1)] * 3, axis=-1)[..., None]
# plot image and kernel
plot_images([image[0, ...], gauss_kernel[..., 0]], [
'Image shape %s' % str(image.shape),
'Kernel shape %s' % str(gauss_kernel.shape)
], 'Image to process', show_plots)
convolved_tf = cnn2d_tf(image, gauss_kernel)
convolved_torch = cnn2d_torch(image, gauss_kernel)
# plot tf and torch convolutions
plot_images([convolved_tf[0, ..., 0], convolved_torch[0, ..., 0]], [
'TF2 Conv shape %s' % str(convolved_tf.shape),
'PyTorch Conv shape %s' % str(convolved_torch.shape)
], 'Difference between convolutions %.2f' %
np.mean(convolved_tf - convolved_torch), show_plots)
delta_time_list = []
for i in range(perform_exp_n_times):
start_time = time.time()
for i in range(iters_per_experiment):
cnn2d_tf(image, gauss_kernel)
delta_time_list.append(time.time() - start_time)
print("Time usage conv2d TF2: %s +/- %s" %
(str(timer(np.mean(delta_time_list))),
str(timer(np.std(delta_time_list), True))))
delta_time_list = []
for i in range(perform_exp_n_times):
start_time = time.time()
for i in range(iters_per_experiment):
cnn2d_torch(image, gauss_kernel)
delta_time_list.append(time.time() - start_time)
print("Time usage conv2d PyTorch: %s +/- %s" %
(str(timer(np.mean(delta_time_list))),
str(timer(np.std(delta_time_list), True))))
def __generate_hmap(self, cvmat, kpAnnolst):
# kpnum + background
gthmp = np.zeros((cvmat.shape[0], cvmat.shape[1], getKpNum(self.category)), dtype=np.float)
for i, _kpAnn in enumerate(kpAnnolst):
if _kpAnn.visibility == -1:
continue
radius = 100
gaussMask = make_gaussian(radius, radius, 20, None)
# avoid out of boundary
top_x, top_y = max(0, _kpAnn.x - radius/2), max(0, _kpAnn.y - radius/2)
bottom_x, bottom_y = min(cvmat.shape[1], _kpAnn.x + radius/2), min(cvmat.shape[0], _kpAnn.y + radius/2)
top_x_offset = top_x - (_kpAnn.x - radius/2)
top_y_offset = top_y - (_kpAnn.y - radius/2)
gthmp[ top_y:bottom_y, top_x:bottom_x, i] = gaussMask[top_y_offset:top_y_offset + bottom_y-top_y,
top_x_offset:top_x_offset + bottom_x-top_x]
return gthmp
output_image[:, int(box_size / 2 - math.floor(image.shape[1] / 2)): int(
box_size / 2 + math.floor(image.shape[1] / 2) + offset), :] = image
cur_hand_joints_x = map(lambda x: x + (box_size / 2 - math.floor(image.shape[1] / 2)),
cur_hand_joints_x)
cur_hand_joints_x = np.asarray(cur_hand_joints_x)
cur_hand_joints_y = np.asarray(cur_hand_joints_y)
if SHOW_INFO:
hmap = np.zeros((box_size, box_size))
# Plot joints
for i in range(num_of_joints):
cv2.circle(output_image, (int(cur_hand_joints_x[i]), int(cur_hand_joints_y[i])), 3, (0, 255, 0), 2)
# Generate joint gaussian map
part_heatmap = utils.make_gaussian(output_image.shape[0], gaussian_radius,
[cur_hand_joints_x[i], cur_hand_joints_y[i]])
hmap += part_heatmap * 50
else:
for i in range(num_of_joints):
output_heatmaps[:, :, i] = utils.make_gaussian(box_size, gaussian_radius,
[cur_hand_joints_x[i], cur_hand_joints_y[i]])
else:
scale = box_size / (cur_img.shape[1] * 1.0)
# Relocalize points
cur_hand_joints_x = map(lambda x: x * scale, cur_hand_joints_x)
cur_hand_joints_y = map(lambda x: x * scale, cur_hand_joints_y)
# Resize image
image = cv2.resize(cur_img, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)
cur_hand_joints_x = np.asarray(cur_hand_joints_x)
cur_hand_joints_y = np.asarray(cur_hand_joints_y)
if SHOW_INFO:
hmap = np.zeros((box_size, box_size))
# Plot joints
for i in range(num_of_joints):
cv2.circle(
output_image,
(int(cur_hand_joints_x[i]), int(cur_hand_joints_y[i])),
3, (0, 255, 0), 2)
# Generate joint gaussian map
part_heatmap = utils.make_gaussian(
output_image.shape[0], gaussian_radius,
[cur_hand_joints_x[i], cur_hand_joints_y[i]])
hmap += part_heatmap * 50
else:
for i in range(num_of_joints):
output_heatmaps[:, :, i] = utils.make_gaussian(
box_size, gaussian_radius,
[cur_hand_joints_x[i], cur_hand_joints_y[i]])
else:
scale = box_size / (cur_img.shape[1] * 1.0)
# Relocalize points
cur_hand_joints_x = map(lambda x: x * scale, cur_hand_joints_x)
cur_hand_joints_y = map(lambda x: x * scale, cur_hand_joints_y)