def read_and_decode_cpm(tfr_queue, img_size, num_joints, center_radius):
tfr_reader = tf.TFRecordReader()
_, serialized_example = tfr_reader.read(tfr_queue)
queue_images = []
queue_center_maps = []
queue_labels = []
queue_orig_images = []
for i in range(2):
features = tf.parse_single_example(serialized_example,
features={
'image': tf.FixedLenFeature([], tf.string),
'heatmaps': tf.FixedLenFeature(
[int(img_size * img_size * (num_joints + 1))], tf.float32)
})
# img_size = 128
# center_radius = 11
img = tf.decode_raw(features['image'], tf.uint8)
img = tf.reshape(img, [img_size, img_size, 3])
img = tf.cast(img, tf.float32)
img = img[..., ::-1]
img = tf.image.random_contrast(img, 0.7, 1)
img = tf.image.random_brightness(img, max_delta=0.9)
img = tf.image.random_hue(img, 0.05)
img = tf.image.random_saturation(img, 0.7, 1.1)
img = img[..., ::-1]
# heatmap = tf.decode_raw(features['heatmaps'], tf.float32)
heatmap = tf.reshape(features['heatmaps'], [img_size, img_size, (num_joints + 1)])
# heatmap = tf.Print(heatmap, [heatmap])
# create centermap
center_map = tf.constant((cpm_utils.make_gaussian(img_size, center_radius,
[int(img_size / 2), int(img_size / 2)])).reshape(
(img_size, img_size, 1)), name='center_map')
center_map = tf.cast(center_map, tf.float32)
# merge img + centermap + heatmap
merged_img_heatmap = tf.concat([img, center_map, heatmap], axis=2)
# subtract mean before pad
mean_volume = tf.concat((128 * tf.ones(shape=(img_size, img_size, 3)),
tf.zeros(shape=(img_size, img_size, (num_joints + 1))),
tf.ones(shape=(img_size, img_size, 1))), axis=2)
merged_img_heatmap -= mean_volume
# preprocessing
preprocessed_merged_img_c_heatmap, _, _ = preprocess(merged_img_heatmap,
label=None,
crop_off_ratio=0.05,
rotation_angle=0.8,
has_bbox=False,
do_flip_lr=True,
do_flip_ud=False,
low_sat=None,
high_sat=None,
max_bright_delta=None,
max_hue_delta=None)
padded_img_size = img_size # * (1 + tf.random_uniform([], minval=0.0, maxval=0.3))
padded_img_size = tf.cast(padded_img_size, tf.int32)
# resize pad
preprocessed_merged_img_c_heatmap = tf.image.resize_image_with_crop_or_pad(preprocessed_merged_img_c_heatmap,
padded_img_size, padded_img_size)
preprocessed_merged_img_c_heatmap += tf.concat((128 * tf.ones(shape=(padded_img_size, padded_img_size, 3)),
tf.zeros(
shape=(padded_img_size, padded_img_size, (num_joints + 1))),
tf.ones(shape=(padded_img_size, padded_img_size, 1))), axis=2)
preprocessed_merged_img_c_heatmap = tf.image.resize_images(preprocessed_merged_img_c_heatmap,
size=[img_size, img_size])
with tf.control_dependencies([preprocessed_merged_img_c_heatmap]):
# preprocessed_img = tf.slice(preprocessed_merged_img_c_heatmap, [0,0,0], [368,368,3])
# preprocessed_center_maps = tf.slice(preprocessed_merged_img_c_heatmap, [0,0,3], [368,368,1])
# preprocessed_heatmaps = tf.slice(preprocessed_merged_img_c_heatmap, [0,0,4], [368,368,13])
preprocessed_img, preprocessed_center_maps, preprocessed_heatmaps = tf.split(
preprocessed_merged_img_c_heatmap, [3, 1, (num_joints + 1)], axis=2)
# Normalize image value
preprocessed_img /= 256
preprocessed_img -= 0.5
queue_images.append(preprocessed_img)
queue_center_maps.append(preprocessed_center_maps)
queue_labels.append(preprocessed_heatmaps)
queue_orig_images.append(img)
return queue_images, queue_center_maps, queue_labels, queue_orig_images
def read_and_decode_cpm(tfr_queue, img_size, num_joints, center_radius):
tfr_reader = tf.TFRecordReader()
_, serialized_example = tfr_reader.read(tfr_queue)
queue_images = []
queue_center_maps = []
queue_labels = []
queue_orig_images = []
for i in range(2):
features = tf.parse_single_example(serialized_example,
features={
'image': tf.FixedLenFeature([], tf.string),
'heatmaps': tf.FixedLenFeature(
[int(img_size * img_size * (num_joints + 1))], tf.float32)
})
# img_size = 128
# center_radius = 11
img = tf.decode_raw(features['image'], tf.uint8)
img = tf.reshape(img, [img_size, img_size, 3])
img = tf.cast(img, tf.float32)
img = img[..., ::-1]
img = tf.image.random_contrast(img, 0.7, 1)
img = tf.image.random_brightness(img, max_delta=0.9)
img = tf.image.random_hue(img, 0.05)
img = tf.image.random_saturation(img, 0.7, 1.1)
img = img[..., ::-1]
# heatmap = tf.decode_raw(features['heatmaps'], tf.float32)
heatmap = tf.reshape(features['heatmaps'], [img_size, img_size, (num_joints + 1)])
# create centermap
center_map = tf.constant((cpm_utils.make_gaussian(img_size, center_radius,
[int(img_size / 2), int(img_size / 2)])).reshape(
(img_size, img_size, 1)), name='center_map')
center_map = tf.cast(center_map, tf.float32)
# merge img + centermap + heatmap
merged_img_heatmap = tf.concat([img, center_map, heatmap], axis=2)
# subtract mean before pad
mean_volume = tf.concat((128 * tf.ones(shape=(img_size, img_size, 3)),
tf.zeros(shape=(img_size, img_size, (num_joints + 1))),
tf.ones(shape=(img_size, img_size, 1))), axis=2)
merged_img_heatmap -= mean_volume
# preprocessing
preprocessed_merged_img_c_heatmap, _, _ = preprocess(merged_img_heatmap,
label=None,
crop_off_ratio=0.05,
rotation_angle=0.8,
has_bbox=False,
do_flip_lr=True,
do_flip_ud=False,
low_sat=None,
high_sat=None,
max_bright_delta=None,
max_hue_delta=None)
padded_img_size = img_size # * (1 + tf.random_uniform([], minval=0.0, maxval=0.3))
padded_img_size = tf.cast(padded_img_size, tf.int32)
# resize pad
preprocessed_merged_img_c_heatmap = tf.image.resize_image_with_crop_or_pad(preprocessed_merged_img_c_heatmap,
padded_img_size, padded_img_size)
preprocessed_merged_img_c_heatmap += tf.concat((128 * tf.ones(shape=(padded_img_size, padded_img_size, 3)),
tf.zeros(
shape=(padded_img_size, padded_img_size, (num_joints + 1))),
tf.ones(shape=(padded_img_size, padded_img_size, 1))), axis=2)
preprocessed_merged_img_c_heatmap = tf.image.resize_images(preprocessed_merged_img_c_heatmap,
size=[img_size, img_size])
with tf.control_dependencies([preprocessed_merged_img_c_heatmap]):
# preprocessed_img = tf.slice(preprocessed_merged_img_c_heatmap, [0,0,0], [368,368,3])
# preprocessed_center_maps = tf.slice(preprocessed_merged_img_c_heatmap, [0,0,3], [368,368,1])
# preprocessed_heatmaps = tf.slice(preprocessed_merged_img_c_heatmap, [0,0,4], [368,368,13])
preprocessed_img, preprocessed_center_maps, preprocessed_heatmaps = tf.split(
preprocessed_merged_img_c_heatmap, [3, 1, (num_joints + 1)], axis=2)
# Normalize image value
preprocessed_img /= 256
preprocessed_img -= 0.5
queue_images.append(preprocessed_img)
queue_center_maps.append(preprocessed_center_maps)
queue_labels.append(preprocessed_heatmaps)
queue_orig_images.append(img)
return queue_images, queue_center_maps, queue_labels, queue_orig_images
cur_body_joints_x = np.asarray(list(cur_body_joints_x))
cur_body_joints_y = np.asarray(list(cur_body_joints_y))
if SHOW_INFO:
hmap = np.zeros((box_size, box_size))
# Plot joints
for i in range(num_of_joints):
cv2.circle(
output_image,
(int(cur_body_joints_x[i]), int(cur_body_joints_y[i])),
3, (0, 255, 0), 2)
# Generate joint gaussian map
part_heatmap = utils.make_gaussian(
output_image.shape[0], gaussian_radius,
[cur_body_joints_x[i], cur_body_joints_y[i]])
hmap += part_heatmap * 50
else:
for i in range(num_of_joints):
output_heatmaps[:, :, i] = utils.make_gaussian(
box_size, gaussian_radius,
[cur_body_joints_x[i], cur_body_joints_y[i]])
else:
scale = box_size / (cur_img.shape[1] * 1.0)
# Relocalize points
cur_body_joints_x = map(lambda x: x * scale, cur_body_joints_x)
cur_body_joints_y = map(lambda x: x * scale, cur_body_joints_y)
cur_hand_joints_x = np.asarray(cur_hand_joints_x)
cur_hand_joints_y = np.asarray(cur_hand_joints_y)
if SHOW_INFO:
hmap = np.zeros((heatmap_size, heatmap_size))
# Plot joints
for i in range(num_of_joints):
cv2.circle(
heatmap_output_image,
(int(cur_hand_joints_x[i]), int(cur_hand_joints_y[i])), 1,
(0, 255, 0), 2)
# Generate joint gaussian map
part_heatmap = cpm_utils.make_gaussian(
heatmap_output_image.shape[0], gaussian_radius,
[cur_hand_joints_x[i], cur_hand_joints_y[i]])
hmap += part_heatmap * 50
else:
for i in range(num_of_joints):
output_heatmaps[:, :, i] = cpm_utils.make_gaussian(
heatmap_size, gaussian_radius,
[cur_hand_joints_x[i], cur_hand_joints_y[i]])
else:
img_scale = img_size / (cur_img.shape[1] * 1.0)
heatmap_scale = heatmap_size / (cur_img.shape[1] * 1.0)
# Relocalize points
cur_hand_joints_x = map(lambda x: x * heatmap_scale, cur_hand_joints_x)
cur_hand_joints_y = map(lambda x: x * heatmap_scale, cur_hand_joints_y)
vP_temp.append(vP[set[iP]])
joint_temp[iP, :] = joint[iJoint][set[iP],:]
X,n = RANSAC_Triangulation(vP_temp, joint_temp,200)
Joint3D.append(X)
# for iP in range(len(vP)):
# vCamera_time[iP].heatmap1 = np.zeros((heatmap_size, heatmap_size, num_of_joints))
for iJoint in range(num_of_joints):
for iP in range(len(vP)):
x = Projection(vP[iP], Joint3D[iJoint])
y = Heatmap2Image(x, vCamera_time[iP].cropping_para[6], np.float(heatmap_size) / img_size,
vCamera_time[iP].cropping_para[:2], vCamera_time[iP].cropping_para[4:6])
ht = cpm_utils.make_gaussian(heatmap_size, float(gaussian_radius) * float(heatmap_size) / heatmap_size,
y)
vCamera_time[iP].heatmap1[:,:,iJoint] = ht
for iP in range(len(vCamera_time)):
vCamera_time[iP].heatmap1[:,:,-1] = np.ones((heatmap_size, heatmap_size)) - np.amax(vCamera_time[iP].heatmap1[:,:,:-1], axis=2)
conf = np.sum(confident_joint, axis=0)
print(conf)
max_camera = np.argmax(conf)
index_set = GetNearViewCamera(vCamera_time, max_camera, np.pi/5)
# print("ref %d" % vCamera_time[max_camera].frame)
# for i in range(len(index_set)):
# print(vCamera_time[index_set[i]].frame)
for iJoint in range(num_of_joints):
img_scale = img_size / (cur_img.shape[1] * 1.0)
image = cv2.resize(cur_img, (0, 0),
fx=img_scale,
fy=img_scale,
interpolation=cv2.INTER_LANCZOS4)
# heatmap_image = cv2.resize(cur_img, (0, 0), fx=img_scale, fy=img_scale, interpolation=cv2.INTER_LANCZOS4)
offset_x = 0
offset_y = int(img_size / 2 - math.floor(image.shape[0] / 2))
output_image[offset_y:offset_y + image.shape[0],
offset_x:offset_x + image.shape[1], :] = image
for i in range(num_of_joints):
ht = cpm_utils.make_gaussian(
img_size,
float(gaussian_radius) * float(img_size) / heatmap_size, [
offset_x + cur_hand_joints_x[i] * img_scale,
offset_y + cur_hand_joints_y[i] * img_scale
])
ht = cv2.resize(ht, (0, 0),
fx=float(heatmap_size) / img_size,
fy=float(heatmap_size) / img_size,
interpolation=cv2.INTER_LANCZOS4)
output_heatmaps[:, :, i] = ht
cropping_param = [
int(float(cur_hand_bbox[0])),
int(float(cur_hand_bbox[1])),
int(float(cur_hand_bbox[2])),
int(float(cur_hand_bbox[3])), offset_x, offset_y, img_scale
]