def evaluate_checkpoints(self):
print("Loading model... ")
# Load pipeline config and build a detection model
configs = config_util.get_configs_from_pipeline_file(
self._pipeline_config_path)
self._detection_model = model_builder.build(
model_config=configs["model"], is_training=False)
# Load evaluation inputs
strategy = tf.compat.v2.distribute.get_strategy()
eval_input = strategy.experimental_distribute_dataset(
inputs.eval_input(
eval_config=configs['eval_config'],
eval_input_config=configs['eval_input_configs'][0],
model_config=configs['model'],
model=self._detection_model))
self._ckpt_paths = tf.train.get_checkpoint_state(
self._training_loop_path).all_model_checkpoint_paths
global_step = tf.compat.v2.Variable(0,
trainable=False,
dtype=tf.compat.v2.dtypes.int64)
results = []
if os.path.exists(self._eval_results_filename):
with open(self._eval_results_filename, "rb") as f:
results = pickle.load(f)
calculated_steps = [result[0] for result in results]
for ckpt_path in self._ckpt_paths:
# Restore checkpoint
ckpt = tf.compat.v2.train.Checkpoint(model=self._detection_model,
step=global_step)
ckpt.restore(ckpt_path).expect_partial()
step_value = int(global_step.read_value())
if step_value in calculated_steps:
print(
"Evaluation for global step {:d} already done, skipping..."
.format(step_value))
continue
print("Running evaluation for checkpoint {}...".format(ckpt_path))
evaluation = model_lib_v2.eager_eval_loop(
detection_model=self._detection_model,
configs=configs,
eval_dataset=eval_input,
global_step=global_step)
results.append((step_value, evaluation))
calculated_steps.append(step_value)
with open(self._eval_results_filename, "wb") as f:
pickle.dump(obj=results, file=f)
def eval_continuously(
pipeline_config_path,
config_override=None,
train_steps=None,
sample_1_of_n_eval_examples=1,
sample_1_of_n_eval_on_train_examples=1,
use_tpu=False,
override_eval_num_epochs=True,
postprocess_on_cpu=False,
model_dir=None,
checkpoint_dir=None,
wait_interval=180,
timeout=3600,
eval_index=0,
save_final_config=False,
**kwargs):
"""Run continuous evaluation of a detection model eagerly.
This method builds the model, and continously restores it from the most
recent training checkpoint in the checkpoint directory & evaluates it
on the evaluation data.
Args:
pipeline_config_path: A path to a pipeline config file.
config_override: A pipeline_pb2.TrainEvalPipelineConfig text proto to
override the config from `pipeline_config_path`.
train_steps: Number of training steps. If None, the number of training steps
is set from the `TrainConfig` proto.
sample_1_of_n_eval_examples: Integer representing how often an eval example
should be sampled. If 1, will sample all examples.
sample_1_of_n_eval_on_train_examples: Similar to
`sample_1_of_n_eval_examples`, except controls the sampling of training
data for evaluation.
use_tpu: Boolean, whether training and evaluation should run on TPU.
override_eval_num_epochs: Whether to overwrite the number of epochs to 1 for
eval_input.
postprocess_on_cpu: When use_tpu and postprocess_on_cpu are true,
postprocess is scheduled on the host cpu.
model_dir: Directory to output resulting evaluation summaries to.
checkpoint_dir: Directory that contains the training checkpoints.
wait_interval: The mimmum number of seconds to wait before checking for a
new checkpoint.
timeout: The maximum number of seconds to wait for a checkpoint. Execution
will terminate if no new checkpoints are found after these many seconds.
eval_index: int, If given, only evaluate the dataset at the given
index. By default, evaluates dataset at 0'th index.
save_final_config: Whether to save the pipeline config file to the model
directory.
**kwargs: Additional keyword arguments for configuration override.
"""
get_configs_from_pipeline_file = MODEL_BUILD_UTIL_MAP[
'get_configs_from_pipeline_file']
create_pipeline_proto_from_configs = MODEL_BUILD_UTIL_MAP[
'create_pipeline_proto_from_configs']
merge_external_params_with_configs = MODEL_BUILD_UTIL_MAP[
'merge_external_params_with_configs']
configs = get_configs_from_pipeline_file(
pipeline_config_path, config_override=config_override)
kwargs.update({
'sample_1_of_n_eval_examples': sample_1_of_n_eval_examples,
'use_bfloat16': configs['train_config'].use_bfloat16 and use_tpu
})
if train_steps is not None:
kwargs['train_steps'] = train_steps
if override_eval_num_epochs:
kwargs.update({'eval_num_epochs': 1})
tf.logging.warning(
'Forced number of epochs for all eval validations to be 1.')
configs = merge_external_params_with_configs(
configs, None, kwargs_dict=kwargs)
if model_dir and save_final_config:
tf.logging.info('Saving pipeline config file to directory {}'.format(
model_dir))
pipeline_config_final = create_pipeline_proto_from_configs(configs)
config_util.save_pipeline_config(pipeline_config_final, model_dir)
model_config = configs['model']
train_input_config = configs['train_input_config']
eval_config = configs['eval_config']
eval_input_configs = configs['eval_input_configs']
eval_on_train_input_config = copy.deepcopy(train_input_config)
eval_on_train_input_config.sample_1_of_n_examples = (
sample_1_of_n_eval_on_train_examples)
if override_eval_num_epochs and eval_on_train_input_config.num_epochs != 1:
tf.logging.warning('Expected number of evaluation epochs is 1, but '
'instead encountered `eval_on_train_input_config'
'.num_epochs` = '
'{}. Overwriting `num_epochs` to 1.'.format(
eval_on_train_input_config.num_epochs))
eval_on_train_input_config.num_epochs = 1
if kwargs['use_bfloat16']:
tf.compat.v2.keras.mixed_precision.experimental.set_policy('mixed_bfloat16')
eval_input_config = eval_input_configs[eval_index]
strategy = tf.compat.v2.distribute.get_strategy()
with strategy.scope():
detection_model = MODEL_BUILD_UTIL_MAP['detection_model_fn_base'](
model_config=model_config, is_training=True)
eval_input = strategy.experimental_distribute_dataset(
inputs.eval_input(
eval_config=eval_config,
eval_input_config=eval_input_config,
model_config=model_config,
model=detection_model))
global_step = tf.compat.v2.Variable(
0, trainable=False, dtype=tf.compat.v2.dtypes.int64)
optimizer, _ = optimizer_builder.build(
configs['train_config'].optimizer, global_step=global_step)
for latest_checkpoint in tf.train.checkpoints_iterator(
checkpoint_dir, timeout=timeout, min_interval_secs=wait_interval):
ckpt = tf.compat.v2.train.Checkpoint(
step=global_step, model=detection_model, optimizer=optimizer)
# We run the detection_model on dummy inputs in order to ensure that the
# model and all its variables have been properly constructed. Specifically,
# this is currently necessary prior to (potentially) creating shadow copies
# of the model variables for the EMA optimizer.
if eval_config.use_moving_averages:
unpad_groundtruth_tensors = (eval_config.batch_size == 1 and not use_tpu)
_ensure_model_is_built(detection_model, eval_input,
unpad_groundtruth_tensors)
optimizer.shadow_copy(detection_model)
ckpt.restore(latest_checkpoint).expect_partial()
if eval_config.use_moving_averages:
optimizer.swap_weights()
summary_writer = tf.compat.v2.summary.create_file_writer(
os.path.join(model_dir, 'eval', eval_input_config.name))
with summary_writer.as_default():
eval_metrics = eager_eval_loop(
detection_model,
configs,
eval_input,
use_tpu=use_tpu,
postprocess_on_cpu=postprocess_on_cpu,
global_step=global_step,
)
return eval_metrics
def eval_continuously(hparams,
pipeline_config_path,
config_override=None,
train_steps=None,
sample_1_of_n_eval_examples=1,
sample_1_of_n_eval_on_train_examples=1,
use_tpu=False,
override_eval_num_epochs=True,
postprocess_on_cpu=False,
export_to_tpu=None,
model_dir=None,
checkpoint_dir=None,
wait_interval=180,
timeout=3600,
**kwargs):
"""Run continuous evaluation of a detection model eagerly.
This method builds the model, and continously restores it from the most
recent training checkpoint in the checkpoint directory & evaluates it
on the evaluation data.
Args:
hparams: A `HParams`.
pipeline_config_path: A path to a pipeline config file.
config_override: A pipeline_pb2.TrainEvalPipelineConfig text proto to
override the config from `pipeline_config_path`.
train_steps: Number of training steps. If None, the number of training steps
is set from the `TrainConfig` proto.
sample_1_of_n_eval_examples: Integer representing how often an eval example
should be sampled. If 1, will sample all examples.
sample_1_of_n_eval_on_train_examples: Similar to
`sample_1_of_n_eval_examples`, except controls the sampling of training
data for evaluation.
use_tpu: Boolean, whether training and evaluation should run on TPU.
override_eval_num_epochs: Whether to overwrite the number of epochs to 1 for
eval_input.
postprocess_on_cpu: When use_tpu and postprocess_on_cpu are true,
postprocess is scheduled on the host cpu.
export_to_tpu: When use_tpu and export_to_tpu are true,
`export_savedmodel()` exports a metagraph for serving on TPU besides the
one on CPU. If export_to_tpu is not provided, we will look for it in
hparams too.
model_dir: Directory to output resulting evaluation summaries to.
checkpoint_dir: Directory that contains the training checkpoints.
wait_interval: The mimmum number of seconds to wait before checking for a
new checkpoint.
timeout: The maximum number of seconds to wait for a checkpoint. Execution
will terminate if no new checkpoints are found after these many seconds.
**kwargs: Additional keyword arguments for configuration override.
"""
get_configs_from_pipeline_file = MODEL_BUILD_UTIL_MAP[
'get_configs_from_pipeline_file']
merge_external_params_with_configs = MODEL_BUILD_UTIL_MAP[
'merge_external_params_with_configs']
configs = get_configs_from_pipeline_file(pipeline_config_path,
config_override=config_override)
kwargs.update({
'sample_1_of_n_eval_examples':
sample_1_of_n_eval_examples,
'use_bfloat16':
configs['train_config'].use_bfloat16 and use_tpu
})
if train_steps is not None:
kwargs['train_steps'] = train_steps
if override_eval_num_epochs:
kwargs.update({'eval_num_epochs': 1})
tf.logging.warning(
'Forced number of epochs for all eval validations to be 1.')
configs = merge_external_params_with_configs(configs,
hparams,
kwargs_dict=kwargs)
model_config = configs['model']
train_input_config = configs['train_input_config']
eval_config = configs['eval_config']
eval_input_configs = configs['eval_input_configs']
eval_on_train_input_config = copy.deepcopy(train_input_config)
eval_on_train_input_config.sample_1_of_n_examples = (
sample_1_of_n_eval_on_train_examples)
if override_eval_num_epochs and eval_on_train_input_config.num_epochs != 1:
tf.logging.warning('Expected number of evaluation epochs is 1, but '
'instead encountered `eval_on_train_input_config'
'.num_epochs` = '
'{}. Overwriting `num_epochs` to 1.'.format(
eval_on_train_input_config.num_epochs))
eval_on_train_input_config.num_epochs = 1
if kwargs['use_bfloat16']:
tf.compat.v2.keras.mixed_precision.experimental.set_policy(
'mixed_bfloat16')
detection_model = model_builder.build(model_config=model_config,
is_training=True)
# Create the inputs.
eval_inputs = []
for eval_input_config in eval_input_configs:
next_eval_input = inputs.eval_input(
eval_config=eval_config,
eval_input_config=eval_input_config,
model_config=model_config,
model=detection_model)
eval_inputs.append((eval_input_config.name, next_eval_input))
# Read export_to_tpu from hparams if not passed.
if export_to_tpu is None:
export_to_tpu = hparams.get('export_to_tpu', False)
tf.logging.info('eval_continuously: use_tpu %s, export_to_tpu %s', use_tpu,
export_to_tpu)
global_step = tf.compat.v2.Variable(0,
trainable=False,
dtype=tf.compat.v2.dtypes.int64)
for latest_checkpoint in tf.train.checkpoints_iterator(
checkpoint_dir, timeout=timeout, min_interval_secs=wait_interval):
ckpt = tf.compat.v2.train.Checkpoint(step=global_step,
model=detection_model)
ckpt.restore(latest_checkpoint).expect_partial()
for eval_name, eval_input in eval_inputs:
summary_writer = tf.compat.v2.summary.create_file_writer(
model_dir + '/eval' + eval_name)
with summary_writer.as_default():
eager_eval_loop(detection_model,
configs,
eval_input,
use_tpu=use_tpu,
postprocess_on_cpu=postprocess_on_cpu,
global_step=global_step)
def main(unused_argv):
MODEL_BUILD_UTIL_MAP = model_lib.MODEL_BUILD_UTIL_MAP
get_configs_from_pipeline_file = MODEL_BUILD_UTIL_MAP[
'get_configs_from_pipeline_file']
configs = get_configs_from_pipeline_file(
FLAGS.pipeline_config_path, config_override=None)
model_config = configs['model']
eval_config = configs['eval_config']
eval_input_configs = configs['eval_input_configs']
detection_model = MODEL_BUILD_UTIL_MAP['detection_model_fn_base'](
model_config=model_config, is_training=True)
# Create the inputs.
eval_inputs = []
for eval_input_config in eval_input_configs:
next_eval_input = inputs.eval_input(
eval_config=eval_config,
eval_input_config=eval_input_config,
model_config=model_config,
model=detection_model)
eval_inputs.append((eval_input_config.name, next_eval_input))
if FLAGS.mode != '3':
# Scan for new model files
model_list = generate_model_list(FLAGS.model_dir)
timeout_count = 0
latest_model = 'ckpt-0' if len(model_list) == 0 else model_list[-1]
start_model_num = int(latest_model.split('-')[-1])
while(timeout_count<=FLAGS.eval_timeout):
print('Wait for {} seconds before starting a new evaluation.'.format(FLAGS.wait_interval))
time.sleep(FLAGS.wait_interval)
timeout_count += FLAGS.wait_interval
model_list = generate_model_list(FLAGS.model_dir)
if model_list[-1] != latest_model:
timeout_count = 0
print('\nEvaluating {}'.format(model_list[-1]))
tStart = time.time()
latest_model = model_list[-1]
eval_metric = generate_eval_metric(detection_model, FLAGS.model_dir, latest_model, eval_inputs, configs)
print('Takes {} to evaluate {}'.format(show_time_taken(tStart), latest_model))
# Automatic labeling restart after at least 10 models produced
model_num = int(latest_model.split('-')[-1])
if model_num >= start_model_num+10:
eval_index = eval_metric[FLAGS.eval_index]
if eval_index >= float(FLAGS.eval_threshold) and FLAGS.mode == '2':
break
else:
model_list = generate_model_list(FLAGS.model_dir)
for model_step in model_list:
print('\nEvaluating {}'.format(model_step))
tStart = time.time()
eval_metric = generate_eval_metric(detection_model, FLAGS.model_dir, model_step, eval_inputs, configs)
print('Takes {} to evaluate {}'.format(show_time_taken(tStart), model_step))
def eval_all_checkpoints(pipeline_config_path,
config_override=None,
train_steps=None,
sample_1_of_n_eval_examples=1,
sample_1_of_n_eval_on_train_examples=1,
use_tpu=False,
override_eval_num_epochs=True,
postprocess_on_cpu=False,
model_dir=None,
checkpoint_dir=None,
eval_index=None,
**kwargs):
get_configs_from_pipeline_file = MODEL_BUILD_UTIL_MAP[
'get_configs_from_pipeline_file']
merge_external_params_with_configs = MODEL_BUILD_UTIL_MAP[
'merge_external_params_with_configs']
configs = get_configs_from_pipeline_file(pipeline_config_path,
config_override=config_override)
kwargs.update({
'sample_1_of_n_eval_examples':
sample_1_of_n_eval_examples,
'use_bfloat16':
configs['train_config'].use_bfloat16 and use_tpu
})
if train_steps is not None:
kwargs['train_steps'] = train_steps
if override_eval_num_epochs:
kwargs.update({'eval_num_epochs': 1})
tf.logging.warning(
'Forced number of epochs for all eval validations to be 1.')
configs = merge_external_params_with_configs(configs,
None,
kwargs_dict=kwargs)
model_config = configs['model']
train_input_config = configs['train_input_config']
eval_config = configs['eval_config']
eval_input_configs = configs['eval_input_configs']
eval_on_train_input_config = copy.deepcopy(train_input_config)
eval_on_train_input_config.sample_1_of_n_examples = (
sample_1_of_n_eval_on_train_examples)
if override_eval_num_epochs and eval_on_train_input_config.num_epochs != 1:
tf.logging.warning('Expected number of evaluation epochs is 1, but '
'instead encountered `eval_on_train_input_config'
'.num_epochs` = '
'{}. Overwriting `num_epochs` to 1.'.format(
eval_on_train_input_config.num_epochs))
eval_on_train_input_config.num_epochs = 1
if kwargs['use_bfloat16']:
tf.compat.v2.keras.mixed_precision.experimental.set_policy(
'mixed_bfloat16')
detection_model = MODEL_BUILD_UTIL_MAP['detection_model_fn_base'](
model_config=model_config, is_training=True)
# Create the inputs.
eval_inputs = []
for eval_input_config in eval_input_configs:
next_eval_input = inputs.eval_input(
eval_config=eval_config,
eval_input_config=eval_input_config,
model_config=model_config,
model=detection_model)
eval_inputs.append((eval_input_config.name, next_eval_input))
if eval_index is not None:
eval_inputs = [eval_inputs[eval_index]]
global_step = tf.compat.v2.Variable(0,
trainable=False,
dtype=tf.compat.v2.dtypes.int64)
ckpt = tf.compat.v2.train.Checkpoint(step=global_step,
model=detection_model)
ckpt_list = []
import re
def atoi(text):
return int(text) if text.isdigit() else text
def natural_keys(text):
return [atoi(c) for c in re.split(r'(\d+)', text)]
for c in os.listdir(checkpoint_dir):
if c.endswith('.index'):
ckpt_list.append(os.path.splitext(c)[0])
ckpt_list.sort(key=natural_keys)
for eval_name, eval_input in eval_inputs:
summary_writer = tf.compat.v2.summary.create_file_writer(
os.path.join(model_dir, 'eval', eval_name))
for c in ckpt_list:
ckpt.restore(os.path.join(checkpoint_dir, c)).expect_partial()
print(c)
with summary_writer.as_default():
eager_eval_loop(detection_model,
configs,
eval_input,
use_tpu=use_tpu,
postprocess_on_cpu=postprocess_on_cpu,
global_step=global_step)
def eval_continuously(hparams,
pipeline_config_path,
config_override=None,
train_steps=None,
sample_1_of_n_eval_examples=1,
sample_1_of_n_eval_on_train_examples=1,
use_tpu=False,
override_eval_num_epochs=True,
postprocess_on_cpu=False,
export_to_tpu=None,
model_dir=None,
checkpoint_dir=None,
wait_interval=180,
**kwargs):
get_configs_from_pipeline_file = MODEL_BUILD_UTIL_MAP[
'get_configs_from_pipeline_file']
merge_external_params_with_configs = MODEL_BUILD_UTIL_MAP[
'merge_external_params_with_configs']
configs = get_configs_from_pipeline_file(pipeline_config_path,
config_override=config_override)
kwargs.update({
'sample_1_of_n_eval_examples':
sample_1_of_n_eval_examples,
'use_bfloat16':
configs['train_config'].use_bfloat16 and use_tpu
})
if train_steps is not None:
kwargs['train_steps'] = train_steps
if override_eval_num_epochs:
kwargs.update({'eval_num_epochs': 1})
tf.logging.warning(
'Forced number of epochs for all eval validations to be 1.')
configs = merge_external_params_with_configs(configs,
hparams,
kwargs_dict=kwargs)
model_config = configs['model']
train_input_config = configs['train_input_config']
eval_config = configs['eval_config']
eval_input_configs = configs['eval_input_configs']
eval_on_train_input_config = copy.deepcopy(train_input_config)
eval_on_train_input_config.sample_1_of_n_examples = (
sample_1_of_n_eval_on_train_examples)
if override_eval_num_epochs and eval_on_train_input_config.num_epochs != 1:
tf.logging.warning('Expected number of evaluation epochs is 1, but '
'instead encountered `eval_on_train_input_config'
'.num_epochs` = '
'{}. Overwriting `num_epochs` to 1.'.format(
eval_on_train_input_config.num_epochs))
eval_on_train_input_config.num_epochs = 1
detection_model = model_builder.build(model_config=model_config,
is_training=True)
# Create the inputs.
eval_inputs = []
for eval_input_config in eval_input_configs:
next_eval_input = inputs.eval_input(
eval_config=eval_config,
eval_input_config=eval_input_config,
model_config=model_config,
model=detection_model)
eval_inputs.append((eval_input_config.name, next_eval_input))
# Read export_to_tpu from hparams if not passed.
if export_to_tpu is None:
export_to_tpu = hparams.get('export_to_tpu', False)
tf.logging.info('eval_continuously: use_tpu %s, export_to_tpu %s', use_tpu,
export_to_tpu)
global_step = tf.compat.v2.Variable(0,
trainable=False,
dtype=tf.compat.v2.dtypes.int64)
prev_checkpoint = None
waiting = False
while True:
ckpt = tf.compat.v2.train.Checkpoint(step=global_step,
model=detection_model)
manager = tf.compat.v2.train.CheckpointManager(ckpt,
checkpoint_dir,
max_to_keep=3)
latest_checkpoint = manager.latest_checkpoint
if prev_checkpoint == latest_checkpoint:
if prev_checkpoint is None:
tf.logging.info(
'No checkpoints found yet. Trying again in %s seconds.' %
wait_interval)
time.sleep(wait_interval)
else:
if waiting:
tf.logging.info(
'Terminating eval after %s seconds of no new '
'checkpoints.' % wait_interval)
break
else:
tf.logging.info(
'No new checkpoint found. Will try again '
'in %s seconds and terminate if no checkpoint '
'appears.' % wait_interval)
waiting = True
time.sleep(wait_interval)
else:
tf.logging.info('New checkpoint found. Starting evaluation.')
waiting = False
prev_checkpoint = latest_checkpoint
ckpt.restore(latest_checkpoint)
for eval_name, eval_input in eval_inputs:
summary_writer = tf.compat.v2.summary.create_file_writer(
model_dir + '/eval' + eval_name)
with summary_writer.as_default():
eager_eval_loop(detection_model,
configs,
eval_input,
use_tpu=use_tpu,
postprocess_on_cpu=postprocess_on_cpu,
global_step=global_step)
def run(callbacks=None):
configs = config_util.get_configs_from_pipeline_file(config_file)
model_config = configs['model']
detection_model = model_builder.build(model_config=model_config,
is_training=False)
#tf.config.experimental_run_functions_eagerly(True)
checkpoint_dir = app.FLAGS.checkpoint_dir
checkpoint_path = tf.train.latest_checkpoint(
app.FLAGS.model_dir if not len(checkpoint_dir) > 0 else checkpoint_dir,
latest_filename=None)
# Restore checkpoint
ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)
ckpt.restore(checkpoint_path).expect_partial()
def get_model_detection_function(model):
"""Get a tf.function for detection."""
@tf.function
def detect_fn(image):
"""Detect objects in image."""
image, shapes = model.preprocess(image)
prediction_dict = model.predict(image, shapes)
detections = model.postprocess(prediction_dict, shapes)
return detections, prediction_dict, tf.reshape(shapes, [-1])
return detect_fn
detect_fn = get_model_detection_function(detection_model)
eval_config = configs['eval_config']
eval_input_configs = configs['eval_input_configs']
eval_inputs = []
for eval_input_config in eval_input_configs:
next_eval_input = inputs.eval_input(
eval_config=eval_config,
eval_input_config=eval_input_config,
model_config=model_config,
model=detection_model)
eval_inputs.append((eval_input_config.name, next_eval_input))
index = 0
annotation_min_score = 0.5
annotations_center = []
annotations_box = []
empty_json = "{}"
for eval_name, eval_input in eval_inputs:
for images_dict, labels in eval_input:
orig_image = images_dict['original_image']
images = tf.cast(orig_image, tf.float32)
detections, prediction_outputs, shapes = detect_fn(images)
images_shape = shapes.numpy()[0:2]
image_shape_multiplier = tf.cast(tf.tile(shapes[0:2], [2]),
tf.float32)
def shape_boxes(boxes):
boxes_px = boxes * image_shape_multiplier
return boxes_px
def get_boxes(numpy=True):
boxes = detections['detection_boxes'][i]
shaped = shape_boxes(boxes)
if numpy:
shaped = shaped.numpy()
return shaped
for i in range(images.shape[0]):
index += 1
boxes = get_boxes(numpy=False)
classes = detections['detection_classes'][i] + label_offset
scores = detections['detection_scores'][i]
y_angles = detections['detection_y_rotation_angles'][i]
source_id = images_dict['source_id'][i]
cv_camera_matrix = images_dict['camera_intrinsic'][i].numpy(
).reshape(3, 3)
cv_distortion_coefficients = images_dict['camera_distortion'][
i].numpy()
if get_poses:
from DicePoseFinding import get_dice_pose_results
dice_pose_results = get_dice_pose_results(
boxes, classes, scores, y_angles, cv_camera_matrix,
cv_distortion_coefficients)
else:
dice_pose_results = []
boxes = boxes.numpy()
classes = classes.numpy()
scores = scores.numpy()
source_id = source_id.numpy()
if show_images:
import matplotlib.patches as patches
image = images[i]
image_np = image.numpy() / 255.0
rotation_angles = detections[
'detection_y_rotation_angles'][i].numpy()
matplotlib.use('TkAgg')
plt.figure(figsize=(12, 16))
plt.imshow(image_np)
ax = plt.gca()
for dice_pose_result in dice_pose_results:
approx_pose_result = dice_pose_result.additional_data[
'approx_pose_result']
from TransformUtil import transform_points_3d
import DiceConfig, DiceProjection
import cv2
local_dots_visible_in_eye_space = DiceProjection.get_local_dots_facing_camera_from_eye_space_pose(
dice_pose_result.pose_pyrender)
#NB We should use cv-space pose when using cv2.projectPoints
if local_dots_visible_in_eye_space.size > 0:
pose_points, pose_points_jacobian = cv2.projectPoints(
local_dots_visible_in_eye_space,
dice_pose_result.pose_cv.rotation_rodrigues,
dice_pose_result.pose_cv.translation,
cv_camera_matrix, cv_distortion_coefficients)
pose_points = np.squeeze(pose_points)
ax.scatter(pose_points[:, 0],
pose_points[:, 1],
s=4)
comp_pts = dice_pose_result.comparison_points_cv[
dice_pose_result.comparison_indices]
proj_pts = dice_pose_result.projected_points[
dice_pose_result.projected_indices]
ax.plot(
np.vstack([comp_pts[:, 0], proj_pts[:, 0]]),
np.vstack([comp_pts[:, 1], proj_pts[:, 1]]),
'g-')
local_dots_visible_in_eye_space_approx = DiceProjection.get_local_dots_facing_camera_from_eye_space_pose(
approx_pose_result.pose_pyrender)
if show_approx and local_dots_visible_in_eye_space_approx.size > 0:
pose_points, pose_points_jacobian = cv2.projectPoints(
local_dots_visible_in_eye_space_approx,
approx_pose_result.pose_cv.rotation_rodrigues,
approx_pose_result.pose_cv.translation,
cv_camera_matrix, cv_distortion_coefficients)
pose_points = np.squeeze(pose_points)
ax.scatter(pose_points[:, 0],
pose_points[:, 1],
s=4)
comp_pts = approx_pose_result.comparison_points_cv[
approx_pose_result.comparison_indices]
proj_pts = approx_pose_result.projected_points[
approx_pose_result.projected_indices]
ax.plot(
np.vstack([comp_pts[:, 0], proj_pts[:, 0]]),
np.vstack([comp_pts[:, 1], proj_pts[:, 1]]),
'y-')
try:
local_dots_projected = dice_pose_result.additional_data[
'local_dots_projected']
dot_centers_transformed = dice_pose_result.additional_data[
'dot_centers_transformed']
#ax.scatter(local_dots_projected[:, 0], local_dots_projected[:, 1], marker='+')
#ax.scatter(dot_centers_transformed[:, 0], dot_centers_transformed[:, 1], marker='s')
except KeyError:
pass
for j in range(boxes.shape[0]):
box = tuple(boxes[j].tolist())
score = scores[j]
class_id = classes[j]
rot_angle = (rotation_angles[j] %
(2 * np.pi)) * 180. / np.pi
if score > min_score:
rect = patches.Rectangle((box[1], box[0]),
box[3] - box[1],
box[2] - box[0],
linewidth=1,
edgecolor='r',
facecolor='none')
ax.add_patch(rect)
if class_id > 0:
plt.text(
box[1], box[0],
'<{:.2f}>:{}:{:.2f}\n@ {:.0f} {:.0f}'.
format(score, class_id, rot_angle, box[1],
box[0]), {'color': 'r'})
else:
#plt.text(box[1], box[0], ' {:.2f}:D'.format(score), {'color': 'r'})
pass
label_rotation_angles = labels['y_rotation_angle'][
i].numpy()
label_boxes = labels['groundtruth_boxes'][i].numpy()
label_classes = labels['groundtruth_classes'][i].numpy()
label_classes_id = np.argmax(label_classes, axis=-1)
label_box_px = shape_boxes(label_boxes).numpy()
num_gt_boxes = labels['num_groundtruth_boxes'].numpy()[0]
for j in range(num_gt_boxes):
box = tuple(label_box_px[j].tolist())
rect = patches.Rectangle((box[1], box[0]),
box[3] - box[1],
box[2] - box[0],
linewidth=1,
linestyle=':',
edgecolor='g',
facecolor='none')
ax.add_patch(rect)
class_id = label_classes_id[j]
rot_angle = (label_rotation_angles[j] %
(2 * np.pi)) * 180. / np.pi
if class_id > 0:
plt.text(box[3], box[0],
'{}:{:.2f}'.format(class_id, rot_angle),
{'color': 'g'})
else:
#plt.text(box[3], box[0], 'D', {'color': 'g'})
pass
plt.show()
for j in range(boxes.shape[0]):
box = boxes[j].tolist()
score = scores[j]
class_id = classes[j]
if score > annotation_min_score:
x_min = box[1]
y_min = box[0]
width = box[3] - box[1]
height = box[2] - box[0]
x_mid = (box[1] + box[3]) / 2
y_mid = (box[0] + box[2]) / 2
region_attributes_string = "{\"type\":" + str(
class_id) + "}"
region_shape_string_box = "{\"name\":\"rect\",\"x\":" + str(
x_min) + ",\"y\":" + str(
y_min) + ",\"width\":" + str(
width) + ",\"height\":" + str(height) + "}"
annotations_box.append([
source_id, empty_json, empty_json, empty_json, j,
region_shape_string_box, region_attributes_string
])
region_shape_string_center = "{\"name\":\"point\",\"cx\":" + str(
x_mid) + ",\"cy\":" + str(y_mid) + "}"
annotations_center.append([
source_id, empty_json, empty_json, empty_json, j,
region_shape_string_center,
region_attributes_string
])
print(index)
if write_annotations:
import csv
with open('output_annotation_box.csv', 'w', newline='') as csvfile:
writer = csv.writer(csvfile)
for row in annotations_box:
writer.writerow(row)
with open('output_annotation_point.csv', 'w', newline='') as csvfile:
writer = csv.writer(csvfile)
for row in annotations_center:
writer.writerow(row)
print("DONE")
