def test_create_model_input_with_forced_off_masks(self):
# Shape = [2, 13, 2].
keypoints_2d = tf.ones_like([2, 13, 2], dtype=tf.float32)
# Shape = [2, 13].
keypoint_masks_2d = tf.constant(
[[1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0],
[1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0,
1.0]])
# Shape = [2, 39].
_, side_outputs = input_generator.create_model_input(
keypoints_2d,
keypoint_masks_2d,
keypoints_3d=None,
model_input_keypoint_type=common.
MODEL_INPUT_KEYPOINT_TYPE_2D_INPUT,
model_input_keypoint_mask_type=(
common.
MODEL_INPUT_KEYPOINT_MASK_TYPE_MASK_KEYPOINTS_AND_AS_INPUT),
normalize_keypoints_2d=False,
keypoint_profile_2d=keypoint_profiles.Std13KeypointProfile2D(),
keypoint_dropout_probs=(0.5, 0.5),
forced_mask_off_part_names=['HEAD', 'LEFT_SHOULDER', 'LEFT_ANKLE'],
seed=0)
self.assertCountEqual(
side_outputs.keys(),
{'preprocessed_keypoints_2d', 'preprocessed_keypoint_masks_2d'})
expected_preprocessed_keypoint_masks_2d = np.array(
[[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0,
1.0]])
self.assertAllClose(side_outputs['preprocessed_keypoint_masks_2d'],
expected_preprocessed_keypoint_masks_2d)
def test_create_model_masked_keypoints_with_masks_2d_input(self):
keypoints_2d = tf.constant([[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]],
[[7.0, 8.0], [9.0, 10.0], [11.0, 12.0]]])
keypoint_masks_2d = tf.constant([[1.0, 0.0, 1.0], [0.0, 0.0, 1.0]])
features, side_outputs = input_generator.create_model_input(
keypoints_2d,
keypoint_masks_2d,
keypoints_3d=None,
model_input_keypoint_type=common.
MODEL_INPUT_KEYPOINT_TYPE_2D_INPUT,
model_input_keypoint_mask_type=(
common.
MODEL_INPUT_KEYPOINT_MASK_TYPE_MASK_KEYPOINTS_AND_AS_INPUT),
normalize_keypoints_2d=False,
rescale_features=True)
expected_features = np.array(
[[1.0, 2.0, 1.0, 0.0, 0.0, 0.0, 5.0, 6.0, 1.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 11.0, 12.0, 1.0]])
expected_features *= np.array([[3.0 / 2.0], [3.0 / 1.0]])
self.assertAllClose(features, expected_features)
self.assertCountEqual(
side_outputs.keys(),
{'preprocessed_keypoints_2d', 'preprocessed_keypoint_masks_2d'})
self.assertAllClose(side_outputs['preprocessed_keypoints_2d'],
[[[1.0, 2.0], [0.0, 0.0], [5.0, 6.0]],
[[0.0, 0.0], [0.0, 0.0], [11.0, 12.0]]])
self.assertAllClose(side_outputs['preprocessed_keypoint_masks_2d'],
[[1.0, 0.0, 1.0], [0.0, 0.0, 1.0]])
def test_create_model_input_with_instance_keypoint_dropout(self):
# Shape = [2, 3, 4, 2].
keypoints_2d = tf.constant([
[[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]],
[[11.0, 12.0], [13.0, 14.0], [15.0, 16.0], [17.0, 18.0]],
[[21.0, 22.0], [23.0, 24.0], [25.0, 26.0], [27.0, 28.0]]],
[[[31.0, 32.0], [33.0, 34.0], [35.0, 36.0], [37.0, 38.0]],
[[41.0, 42.0], [43.0, 44.0], [45.0, 46.0], [47.0, 48.0]],
[[51.0, 52.0], [53.0, 54.0], [55.0, 56.0], [57.0, 58.0]]],
])
# Shape = [2, 3, 4].
keypoint_masks_2d = tf.constant([[
[0.0, 0.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0],
], [
[1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 0.0],
[0.0, 0.0, 1.0, 1.0],
]])
# Shape = [2, 3, 12].
features, side_outputs = input_generator.create_model_input(
keypoints_2d,
keypoint_masks_2d,
keypoints_3d=None,
model_input_keypoint_type=common.MODEL_INPUT_KEYPOINT_TYPE_2D_INPUT,
model_input_keypoint_mask_type=(
common.MODEL_INPUT_KEYPOINT_MASK_TYPE_MASK_KEYPOINTS_AND_AS_INPUT),
normalize_keypoints_2d=False,
keypoint_dropout_probs=(0.1, 0.8),
rescale_features=True,
seed=0)
self.assertCountEqual(
side_outputs.keys(),
{'preprocessed_keypoints_2d', 'preprocessed_keypoint_masks_2d'})
expected_features = np.array([
[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 5.0, 6.0, 1.0, 7.0, 8.0, 1.0],
[11.0, 12.0, 1.0, 13.0, 14.0, 1.0, 15.0, 16.0, 1.0, 17.0, 18.0, 1.0],
[21.0, 22.0, 1.0, 23.0, 24.0, 1.0, 25.0, 26.0, 1.0, 27.0, 28.0, 1.0]],
[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 35.0, 36.0, 1.0, 0.0, 0.0, 0.0],
[41.0, 42.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 55.0, 56.0, 1.0, 57.0, 58.0, 1.0]],
])
expected_features *= np.array([[[4.0 / 2.0], [4.0 / 4.0], [4.0 / 4.0]],
[[4.0 / 1.0], [4.0 / 1.0], [4.0 / 2.0]]])
self.assertAllClose(features, expected_features)
self.assertAllClose(
side_outputs['preprocessed_keypoints_2d'],
[[[[0.0, 0.0], [0.0, 0.0], [5.0, 6.0], [7.0, 8.0]],
[[11.0, 12.0], [13.0, 14.0], [15.0, 16.0], [17.0, 18.0]],
[[21.0, 22.0], [23.0, 24.0], [25.0, 26.0], [27.0, 28.0]]],
[[[0.0, 0.0], [0.0, 0.0], [35.0, 36.0], [0.0, 0.0]],
[[41.0, 42.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]],
[[0.0, 0.0], [0.0, 0.0], [55.0, 56.0], [57.0, 58.0]]]])
self.assertAllClose(
side_outputs['preprocessed_keypoint_masks_2d'],
[[[0.0, 0.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]],
[[0.0, 0.0, 1.0, 0.0], [1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 1.0]]])
def test_create_model_keypoints_2d_input(self):
keypoint_profile_2d = keypoint_profiles.KeypointProfile2D(
name='Dummy',
keypoint_names=[('A', keypoint_profiles.LeftRightType.UNKNOWN),
('B', keypoint_profiles.LeftRightType.UNKNOWN),
('C', keypoint_profiles.LeftRightType.UNKNOWN)],
offset_keypoint_names=['A', 'B'],
scale_keypoint_name_pairs=[(['A', 'B'], ['B']), (['A'], ['B',
'C'])],
segment_name_pairs=[],
scale_distance_reduction_fn=tf.math.reduce_sum,
scale_unit=1.0)
# Shape = [2, 3, 2].
keypoints_2d = tf.constant([[[0.0, 1.0], [2.0, 3.0], [4.0, 5.0]],
[[10.0, 11.0], [12.0, 13.0], [14.0,
15.0]]])
keypoint_masks_2d = tf.ones([2, 3], dtype=tf.float32)
# Shape = [2, 6].
features, side_outputs = input_generator.create_model_input(
keypoints_2d,
keypoint_masks_2d,
keypoints_3d=None,
model_input_keypoint_type=common.
MODEL_INPUT_KEYPOINT_TYPE_2D_INPUT,
normalize_keypoints_2d=True,
keypoint_profile_2d=keypoint_profile_2d)
sqrt_2 = 1.414213562
self.assertAllClose(features,
[[
-0.25 / sqrt_2, -0.25 / sqrt_2, 0.25 / sqrt_2,
0.25 / sqrt_2, 0.75 / sqrt_2, 0.75 / sqrt_2
],
[
-0.25 / sqrt_2, -0.25 / sqrt_2, 0.25 / sqrt_2,
0.25 / sqrt_2, 0.75 / sqrt_2, 0.75 / sqrt_2
]])
self.assertCountEqual(side_outputs.keys(), [
'preprocessed_keypoints_2d', 'preprocessed_keypoint_masks_2d',
'offset_points_2d', 'scale_distances_2d'
])
self.assertAllClose(
side_outputs['preprocessed_keypoints_2d'],
[[[-0.25 / sqrt_2, -0.25 / sqrt_2], [0.25 / sqrt_2, 0.25 / sqrt_2],
[0.75 / sqrt_2, 0.75 / sqrt_2]],
[[-0.25 / sqrt_2, -0.25 / sqrt_2], [0.25 / sqrt_2, 0.25 / sqrt_2],
[0.75 / sqrt_2, 0.75 / sqrt_2]]])
self.assertAllClose(side_outputs['preprocessed_keypoint_masks_2d'],
[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]])
self.assertAllClose(side_outputs['offset_points_2d'],
[[[1.0, 2.0]], [[11.0, 12.0]]])
self.assertAllClose(side_outputs['scale_distances_2d'],
[[[4.0 * sqrt_2]], [[4.0 * sqrt_2]]])
def main(_):
"""Runs inference."""
keypoint_profile_2d = (keypoint_profiles.create_keypoint_profile_or_die(
FLAGS.input_keypoint_profile_name_2d))
g = tf.Graph()
with g.as_default():
keypoints_2d, keypoint_masks_2d = read_inputs(keypoint_profile_2d)
model_inputs, _ = input_generator.create_model_input(
keypoints_2d,
keypoint_masks_2d=keypoint_masks_2d,
keypoints_3d=None,
model_input_keypoint_type=common.
MODEL_INPUT_KEYPOINT_TYPE_2D_INPUT,
model_input_keypoint_mask_type=FLAGS.
model_input_keypoint_mask_type,
keypoint_profile_2d=keypoint_profile_2d,
# Fix seed for determinism.
seed=1)
embedder_fn = models.get_embedder(
base_model_type=FLAGS.base_model_type,
embedding_type=FLAGS.embedding_type,
num_embedding_components=FLAGS.num_embedding_components,
embedding_size=FLAGS.embedding_size,
num_embedding_samples=FLAGS.num_embedding_samples,
is_training=False,
num_fc_blocks=FLAGS.num_fc_blocks,
num_fcs_per_block=FLAGS.num_fcs_per_block,
num_hidden_nodes=FLAGS.num_hidden_nodes,
num_bottleneck_nodes=FLAGS.num_bottleneck_nodes,
weight_max_norm=FLAGS.weight_max_norm)
outputs, _ = embedder_fn(model_inputs)
if FLAGS.use_moving_average:
variables_to_restore = (
pipeline_utils.get_moving_average_variables_to_restore())
saver = tf.train.Saver(variables_to_restore)
else:
saver = tf.train.Saver()
scaffold = tf.train.Scaffold(init_op=tf.global_variables_initializer(),
saver=saver)
session_creator = tf.train.ChiefSessionCreator(
scaffold=scaffold,
master=FLAGS.master,
checkpoint_filename_with_path=FLAGS.checkpoint_path)
with tf.train.MonitoredSession(session_creator=session_creator,
hooks=None) as sess:
outputs_result = sess.run(outputs)
tf.gfile.MakeDirs(FLAGS.output_dir)
for key in [
common.KEY_EMBEDDING_MEANS, common.KEY_EMBEDDING_STDDEVS,
common.KEY_EMBEDDING_SAMPLES
]:
if key in outputs_result:
output = outputs_result[key]
np.savetxt(os.path.join(FLAGS.output_dir, key + '.csv'),
output.reshape([output.shape[0], -1]),
delimiter=',')
def create_model_input(inputs,
model_input_keypoint_type,
keypoint_profile_2d=None,
keypoint_profile_3d=None,
normalize_keypoints_2d=True,
min_keypoint_score_2d=-1.0,
azimuth_range=(-math.pi, math.pi),
elevation_range=(-math.pi / 6.0, math.pi / 6.0),
roll_range=(-math.pi / 6.0, math.pi / 6.0),
seed=None):
"""Creates model input features (2D keypoints) from input keypoints.
Note that this function mainly duplicates `create_model_input` in
`v1.input_generator.py` for compatible with tf2.
IMPORTANT: We assume that 2D keypoints from the inputs have been normalized by
image size. This function will reads image sizes from the input and
denormalize the 2D keypoints with them. No normalization is expected and no
denormalization will be performed for 3D keypoints.
Args:
inputs: A dictionary for tensor inputs.
model_input_keypoint_type: An enum string for model input keypoint type. See
`MODEL_INPUT_KEYPOINT_TYPE_*` for supported values.
keypoint_profile_2d: A KeypointProfile2D object for input 2D keypoints.
Required for normalizing 2D keypoints. Also required when 3D-to-2D
projection is involved.
keypoint_profile_3d: A KeypointProfile3D object for input 3D keypoints. Only
used when 3D-to-2D projection is involved.
normalize_keypoints_2d: A boolean for whether to normalize 2D keypoints at
the end.
min_keypoint_score_2d: A float for the minimum score to consider a 2D
keypoint as invalid.
azimuth_range: A tuple for minimum and maximum azimuth angles to randomly
rotate 3D keypoints with.
elevation_range: A tuple for minimum and maximum elevation angles to
randomly rotate 3D keypoints with.
roll_range: A tuple for minimum and maximum roll angles to randomly rotate
3D keypoints with.
seed: An integer for random seed.
Returns:
features: A tensor for input features. Shape = [..., feature_dim].
side_outputs: A dictionary for side outputs, which includes
`offset_points_2d` (shape = [..., 1, 2]) and `scale_distances_2d` (shape =
[..., 1, 1]) if `normalize_keypoints_2d` is True.
"""
keypoints_2d = keypoint_utils.denormalize_points_by_image_size(
inputs[common.KEY_KEYPOINTS_2D],
image_sizes=inputs[common.KEY_IMAGE_SIZES])
keypoint_scores_2d = inputs[common.KEY_KEYPOINT_SCORES_2D]
if min_keypoint_score_2d < 0.0:
keypoint_masks_2d = tf.ones_like(keypoint_scores_2d, dtype=tf.float32)
else:
keypoint_masks_2d = tf.cast(
tf.math.greater_equal(keypoint_scores_2d, min_keypoint_score_2d),
dtype=tf.float32)
keypoints_3d = inputs.get(common.KEY_KEYPOINTS_3D, None)
features, side_outputs = input_generator.create_model_input(
keypoints_2d,
keypoint_masks_2d,
keypoints_3d,
model_input_keypoint_type,
normalize_keypoints_2d=normalize_keypoints_2d,
keypoint_profile_2d=keypoint_profile_2d,
keypoint_profile_3d=keypoint_profile_3d,
azimuth_range=azimuth_range,
elevation_range=elevation_range,
roll_range=roll_range,
seed=seed)
# IMPORTANT: It is better not to modify `inputs` in TF2. Instead, we save
# results in the `side_outputs` for further computation.
side_outputs.update({
common.KEY_KEYPOINTS_2D: keypoints_2d,
common.KEY_KEYPOINT_MASKS_2D: keypoint_masks_2d
})
return features, side_outputs
