def test_prefetch_tensors_with_fully_defined_shapes(self):
with self.test_session() as sess:
batch_size = 10
image_size = 32
num_batches = 5
examples = tf.Variable(tf.constant(0, dtype=tf.int64))
counter = examples.count_up_to(num_batches)
image = tf.random_normal([batch_size, image_size,
image_size, 3],
dtype=tf.float32,
name='images')
label = tf.random_uniform([batch_size, 1], 0, 10,
dtype=tf.int32, name='labels')
prefetch_queue = prefetcher.prefetch(tensor_dict={'counter': counter,
'image': image,
'label': label},
capacity=100)
tensor_dict = prefetch_queue.dequeue()
self.assertAllEqual(tensor_dict['image'].get_shape().as_list(),
[batch_size, image_size, image_size, 3])
self.assertAllEqual(tensor_dict['label'].get_shape().as_list(),
[batch_size, 1])
tf.initialize_all_variables().run()
with slim.queues.QueueRunners(sess):
for _ in range(num_batches):
results = sess.run(tensor_dict)
self.assertEquals(results['image'].shape,
(batch_size, image_size, image_size, 3))
self.assertEquals(results['label'].shape, (batch_size, 1))
with self.assertRaises(tf.errors.OutOfRangeError):
sess.run(tensor_dict)
def _extract_groundtruth_tensors(create_input_dict_fn):
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image],
0)
tensor_dict = {'image_id': input_dict[fields.InputDataFields.source_id]}
normalized_gt_boxlist = box_list.BoxList(
input_dict[fields.InputDataFields.groundtruth_boxes])
gt_boxlist = box_list_ops.scale(normalized_gt_boxlist,
tf.shape(original_image)[1],
tf.shape(original_image)[2])
groundtruth_boxes = gt_boxlist.get()
groundtruth_classes = input_dict[
fields.InputDataFields.groundtruth_classes]
tensor_dict['groundtruth_boxes'] = groundtruth_boxes
tensor_dict['groundtruth_classes'] = groundtruth_classes
tensor_dict['area'] = input_dict[fields.InputDataFields.groundtruth_area]
tensor_dict['difficult'] = input_dict[
fields.InputDataFields.groundtruth_difficult]
# subset annotations
if fields.InputDataFields.groundtruth_subset in input_dict:
tensor_dict['groundtruth_subset'] \
= input_dict[fields.InputDataFields.groundtruth_subset]
return tensor_dict
def __init__(self, tensor_dict, batch_size, batch_queue_capacity,
num_batch_queue_threads, prefetch_queue_capacity):
"""Constructs a batch queue holding tensor_dict.
Args:
tensor_dict: dictionary of tensors to batch.
batch_size: batch size.
batch_queue_capacity: max capacity of the queue from which the tensors are
batched.
num_batch_queue_threads: number of threads to use for batching.
prefetch_queue_capacity: max capacity of the queue used to prefetch
assembled batches.
"""
# Remember static shapes to set shapes of batched tensors.
static_shapes = collections.OrderedDict(
{key: tensor.get_shape() for key, tensor in tensor_dict.items()})
# Remember runtime shapes to unpad tensors after batching.
runtime_shapes = collections.OrderedDict(
{(key + rt_shape_str): tf.shape(tensor)
for key, tensor in tensor_dict.items()})
all_tensors = tensor_dict
all_tensors.update(runtime_shapes)
batched_tensors = tf.train.batch(
all_tensors,
capacity=batch_queue_capacity,
batch_size=batch_size,
dynamic_pad=True,
num_threads=num_batch_queue_threads)
self._queue = prefetcher.prefetch(batched_tensors,
prefetch_queue_capacity)
self._static_shapes = static_shapes
self._batch_size = batch_size
def __init__(self, tensor_dict, batch_size, batch_queue_capacity,
num_batch_queue_threads, prefetch_queue_capacity):
"""Constructs a batch queue holding tensor_dict.
Args:
tensor_dict: dictionary of tensors to batch.
batch_size: batch size.
batch_queue_capacity: max capacity of the queue from which the tensors are
batched.
num_batch_queue_threads: number of threads to use for batching.
prefetch_queue_capacity: max capacity of the queue used to prefetch
assembled batches.
"""
# Remember static shapes to set shapes of batched tensors.
static_shapes = collections.OrderedDict(
{key: tensor.get_shape()
for key, tensor in tensor_dict.items()})
# Remember runtime shapes to unpad tensors after batching.
runtime_shapes = collections.OrderedDict({
(key + rt_shape_str): tf.shape(tensor)
for key, tensor in tensor_dict.items()
})
all_tensors = tensor_dict
all_tensors.update(runtime_shapes)
batched_tensors = tf.train.batch(all_tensors,
capacity=batch_queue_capacity,
batch_size=batch_size,
dynamic_pad=True,
num_threads=num_batch_queue_threads)
self._queue = prefetcher.prefetch(batched_tensors,
prefetch_queue_capacity)
self._static_shapes = static_shapes
self._batch_size = batch_size
def _extract_prediction_tensors(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image],
0)
original_audio = tf.expand_dims(input_dict[fields.InputDataFields.audio],
0)
preprocessed_image = model.preprocess(tf.to_float(original_image), False)
preprocessed_audio = model.preprocess(tf.to_float(original_audio), True)
prediction_dict = model.predict(preprocessed_image, preprocessed_audio)
detections = model.postprocess(prediction_dict)
groundtruth = None
if not ignore_groundtruth:
groundtruth = {
fields.InputDataFields.groundtruth_boxes:
input_dict[fields.InputDataFields.groundtruth_boxes],
fields.InputDataFields.groundtruth_classes:
input_dict[fields.InputDataFields.groundtruth_classes],
fields.InputDataFields.groundtruth_area:
input_dict[fields.InputDataFields.groundtruth_area],
fields.InputDataFields.groundtruth_is_crowd:
input_dict[fields.InputDataFields.groundtruth_is_crowd],
fields.InputDataFields.groundtruth_difficult:
input_dict[fields.InputDataFields.groundtruth_difficult],
fields.InputDataFields.groundtruth_image_classes:
input_dict[fields.InputDataFields.groundtruth_image_classes]
}
if fields.InputDataFields.groundtruth_group_of in input_dict:
groundtruth[fields.InputDataFields.groundtruth_group_of] = (
input_dict[fields.InputDataFields.groundtruth_group_of])
if fields.DetectionResultFields.detection_masks in detections:
groundtruth[fields.InputDataFields.groundtruth_instance_masks] = (
input_dict[fields.InputDataFields.groundtruth_instance_masks])
return eval_util.result_dict_for_single_example(
original_image,
input_dict[fields.InputDataFields.source_id],
detections,
groundtruth,
class_agnostic=(fields.DetectionResultFields.detection_classes
not in detections),
scale_to_absolute=True)
def _extract_prediction_tensors(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0)
preprocessed_image, true_image_shapes = model.preprocess(
tf.to_float(original_image))
prediction_dict = model.predict(preprocessed_image, true_image_shapes)
detections = model.postprocess(prediction_dict, true_image_shapes)
groundtruth = None
if not ignore_groundtruth:
groundtruth = {
fields.InputDataFields.groundtruth_boxes:
input_dict[fields.InputDataFields.groundtruth_boxes],
fields.InputDataFields.groundtruth_classes:
input_dict[fields.InputDataFields.groundtruth_classes],
fields.InputDataFields.groundtruth_area:
input_dict[fields.InputDataFields.groundtruth_area],
fields.InputDataFields.groundtruth_is_crowd:
input_dict[fields.InputDataFields.groundtruth_is_crowd],
fields.InputDataFields.groundtruth_difficult:
input_dict[fields.InputDataFields.groundtruth_difficult]
}
if fields.InputDataFields.groundtruth_group_of in input_dict:
groundtruth[fields.InputDataFields.groundtruth_group_of] = (
input_dict[fields.InputDataFields.groundtruth_group_of])
if fields.DetectionResultFields.detection_masks in detections:
groundtruth[fields.InputDataFields.groundtruth_instance_masks] = (
input_dict[fields.InputDataFields.groundtruth_instance_masks])
return eval_util.result_dict_for_single_example(
original_image,
input_dict[fields.InputDataFields.source_id],
detections,
groundtruth,
class_agnostic=(
fields.DetectionResultFields.detection_classes not in detections),
scale_to_absolute=True)
def build_input(tfrecord_paths):
"""Builds the graph's input.
Args:
tfrecord_paths: List of paths to the input TFRecords
Returns:
serialized_example_tensor: The next serialized example. String scalar Tensor
image_tensor: The decoded image of the example. Uint8 tensor,
shape=[1, None, None,3]
"""
filename_queue = tf.train.string_input_producer(
tfrecord_paths, shuffle=False, num_epochs=1)
tf_record_reader = tf.TFRecordReader()
_, serialized_example_tensor = tf_record_reader.read(filename_queue)
# *** MODIFIED
prefetch_queue = prefetcher.prefetch({'serialized_example_tensor': serialized_example_tensor}, 100)
dequeue = prefetch_queue.dequeue()
serialized_example_tensor = dequeue['serialized_example_tensor']
# *** MODIFIED ENDS
features = tf.parse_single_example(
serialized_example_tensor,
features={
standard_fields.TfExampleFields.image_encoded:
tf.FixedLenFeature([], tf.string),
})
encoded_image = features[standard_fields.TfExampleFields.image_encoded]
image_tensor = tf.image.decode_image(encoded_image, channels=3)
image_tensor.set_shape([None, None, 3])
image_tensor = tf.expand_dims(image_tensor, 0)
# # *** MODIFIED
# batch = tf.train.batch(
# [serialized_example_tensor, image_tensor],
# batch_size=24,
# enqueue_many=False,
# num_threads=6,
# capacity=5 * 24)
# return batch[0], batch[1]
# # *** MODIFIED ENDS
return serialized_example_tensor, image_tensor
def _extract_prediction_tensors(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
input_dict = create_input_dict_fn()
batch = None
if 'batch' in input_dict:
batch = input_dict.pop('batch')
else:
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
# consistent format for images and videos
for key, value in input_dict.iteritems():
input_dict[key] = (value, )
detections = _create_detection_op(model, input_dict, batch)
# Print out anaylsis of the model.
tf.contrib.tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=tf.contrib.tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
tf.contrib.tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=tf.contrib.tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
num_frames = len(input_dict[fields.InputDataFields.image])
ret = []
for i in range(num_frames):
original_image = tf.expand_dims(
input_dict[fields.InputDataFields.image][i], 0)
groundtruth = None
if not ignore_groundtruth:
groundtruth = {
fields.InputDataFields.groundtruth_boxes:
input_dict[fields.InputDataFields.groundtruth_boxes][i],
fields.InputDataFields.groundtruth_classes:
input_dict[fields.InputDataFields.groundtruth_classes][i],
}
optional_keys = (
fields.InputDataFields.groundtruth_area,
fields.InputDataFields.groundtruth_is_crowd,
fields.InputDataFields.groundtruth_difficult,
fields.InputDataFields.groundtruth_group_of,
)
for opt_key in optional_keys:
if opt_key in input_dict:
groundtruth[opt_key] = input_dict[opt_key][i]
if fields.DetectionResultFields.detection_masks in detections:
groundtruth[
fields.InputDataFields.groundtruth_instance_masks] = (
input_dict[fields.InputDataFields.
groundtruth_instance_masks][i])
detections_frame = {
key: tf.expand_dims(value[i], 0)
for key, value in detections.iteritems()
}
source_id = (batch.key[0] if batch is not None else
input_dict[fields.InputDataFields.source_id][i])
ret.append(
eval_util.result_dict_for_single_example(
original_image,
source_id,
detections_frame,
groundtruth,
class_agnostic=(fields.DetectionResultFields.detection_classes
not in detections),
scale_to_absolute=True))
return ret
def _extract_prediction_tensors(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0)
preprocessed_image = model.preprocess(tf.to_float(original_image))
prediction_dict = model.predict(preprocessed_image)
detections = model.postprocess(prediction_dict)
original_image_shape = tf.shape(original_image)
absolute_detection_boxlist = box_list_ops.to_absolute_coordinates(
box_list.BoxList(tf.squeeze(detections['detection_boxes'], axis=0)),
original_image_shape[1], original_image_shape[2])
label_id_offset = 1
tensor_dict = {
'original_image': original_image,
'image_id': input_dict[fields.InputDataFields.source_id],
'detection_boxes': absolute_detection_boxlist.get(),
'detection_scores': tf.squeeze(detections['detection_scores'], axis=0),
'detection_classes': (
tf.squeeze(detections['detection_classes'], axis=0) +
label_id_offset),
}
if 'detection_masks' in detections:
detection_masks = tf.squeeze(detections['detection_masks'],
axis=0)
detection_boxes = tf.squeeze(detections['detection_boxes'],
axis=0)
# TODO: This should be done in model's postprocess function ideally.
detection_masks_reframed = ops.reframe_box_masks_to_image_masks(
detection_masks,
detection_boxes,
original_image_shape[1], original_image_shape[2])
detection_masks_reframed = tf.to_float(tf.greater(detection_masks_reframed,
0.5))
tensor_dict['detection_masks'] = detection_masks_reframed
# load groundtruth fields into tensor_dict
if not ignore_groundtruth:
normalized_gt_boxlist = box_list.BoxList(
input_dict[fields.InputDataFields.groundtruth_boxes])
gt_boxlist = box_list_ops.scale(normalized_gt_boxlist,
tf.shape(original_image)[1],
tf.shape(original_image)[2])
groundtruth_boxes = gt_boxlist.get()
groundtruth_classes = input_dict[fields.InputDataFields.groundtruth_classes]
tensor_dict['groundtruth_boxes'] = groundtruth_boxes
tensor_dict['groundtruth_classes'] = groundtruth_classes
tensor_dict['area'] = input_dict[fields.InputDataFields.groundtruth_area]
tensor_dict['is_crowd'] = input_dict[
fields.InputDataFields.groundtruth_is_crowd]
tensor_dict['difficult'] = input_dict[
fields.InputDataFields.groundtruth_difficult]
if 'detection_masks' in tensor_dict:
tensor_dict['groundtruth_instance_masks'] = input_dict[
fields.InputDataFields.groundtruth_instance_masks]
return tensor_dict
def _extract_prediction_tensors(model,
input_k_shot,
create_input_dict_fn,
ignore_groundtruth=False,
use_target_class_in_predictions=False):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
tree_k_shot = model._k_shot
bag_size = model._bag_size
num_classes = model._num_classes
num_negative_bags = model._num_negative_bags
total_images = num_negative_bags + input_k_shot
input_dict, thread = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=3)
input_dict = prefetch_queue.dequeue()
def _r(t):
return tf.reshape(t, [-1,]+t.shape[2:].as_list())
def reorder_for_k2(t):
'''
when tree_k_shot is 2, and we have negative bags and input_k_shot is greater than 2,
this function reorders the list of inputs in a way that it puts the negative elements
after every 2*bag_size elements.
Notes:
- assumes positive elements are in first bag_size*input_k_shot elements of the input list
and the rest of the elements are negative bags i.e., num_negative_bags*bag_size last
elements of the list.
'''
if num_negative_bags > 0 and tree_k_shot == 2 and input_k_shot != 2:
#TDOO: 'implemented, but not tested yet! use embed to check'
pos_part = t[:bag_size*input_k_shot]
neg_part = t[bag_size*input_k_shot:]
assert input_k_shot % 2 == 0, 'input_k_shot should be multiple of 2'
l = [pos_part[bag_size*2*i:bag_size*2*(i+1)] + neg_part for i in range(input_k_shot/2)]
t = [i for x in l for i in x]
return t
def _s(t):
return reorder_for_k2(tf.split(_r(t), total_images*bag_size))
images_list = _s(input_dict[fields.supportset])
groundtruth_classes_list = _s(input_dict[fields.groundtruth_classes])
original_groundtruth_classes = input_dict[fields.original_groundtruth_classes]
original_groundtruth_classes_list = _s(original_groundtruth_classes)
one_hot_gt_classes_list = _s(tf.one_hot(tf.to_int32(original_groundtruth_classes), num_classes + 1))
boxes_list = _s(input_dict[fields.groundtruth_boxes])
images = tf.concat(images_list,0)
float_images = tf.to_float(images)
input_dict[fields.supportset] = float_images
preprocessed_images = [model.preprocess(float_image)
for float_image in
tf.split(float_images, len(images_list))]
model.provide_groundtruth(boxes_list, one_hot_gt_classes_list, None)
if fields.proposal_objectness in input_dict:
model._groundtruth_lists['objectness_score'] = input_dict[fields.proposal_objectness]
if use_target_class_in_predictions:
target_class = input_dict[fields.groundtruth_target_class]
model._groundtruth_lists['target_class'] = target_class[tf.newaxis]
preprocessed_image = tf.concat(preprocessed_images, 0)
prediction_dict = model.predict(preprocessed_image)
detections = model.postprocess(prediction_dict)
tensor_dict = dict()
tensor_dict.update(model._tree_debug_tensors(
include_k2_cross_similarty=False)) #FLAGS.add_mrf
tensor_dict['input_k_shot'] = tf.constant(input_k_shot)
tensor_dict['num_negative_bags'] = tf.constant(num_negative_bags)
label_id_offset = 1
# Convert to the absolute coordinates
for key,val in tensor_dict.items():
if 'multiclass' in key:
tensor_dict.pop(key)
if isinstance(val, dict):
for mkey in val:
if 'classes' in mkey:
val[mkey] = val[mkey] + label_id_offset
groundtruth_classes = tf.concat(groundtruth_classes_list[:input_k_shot*bag_size], 0)
groundtruth_classes = tf.reshape(groundtruth_classes, [input_k_shot, bag_size])
original_groundtruth_classes = tf.concat(original_groundtruth_classes_list[:input_k_shot*bag_size],0)
original_groundtruth_classes = tf.reshape(original_groundtruth_classes,
[input_k_shot, bag_size])
if not fields.original_images in input_dict:
raise Exception('Please add this tensors to mini-imagenet model, guzu.' \
' It is added but ridam passe kallat')
tensor_dict['original_image'] = input_dict[fields.original_images]
tensor_dict['filenames'] = input_dict[fields.filename]
ndict = dict(boxes=[], classes=[])
ndict['target_class'] = input_dict[fields.groundtruth_target_class]
for i in range(input_k_shot):
ndict['boxes'].append(
input_dict[fields.groundtruth_image_boxes+'_{}'.format(i)])
ndict['classes'].append(input_dict[
fields.groundtruth_image_classes+'_{}'.format(i)])
#ndict['fea_groundtruth_classes'] = groundtruth_classes
#ndict['fea_original_groundtruth_classes'] = original_groundtruth_classes
#ndict['fea_boxes'] = input_dict[fields.groundtruth_boxes]
tensor_dict['groundtruth'] = ndict
####
#tensor_dict['meta_info']['extra_tensors'] = util.get_extra_tensors_dict()
####
return tensor_dict, thread
def _extract_anchros_and_losses(model,
create_input_dict_fn,
quantize=False,
ignore_groundtruth=False):
"""Constructs tensorflow detection graph and returns output tensors.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
prediction_groundtruth_dict: A dictionary with postprocessed tensors (keyed
by standard_fields.DetectionResultsFields) and optional groundtruth
tensors (keyed by standard_fields.InputDataFields).
losses_dict: A dictionary containing detection losses. This is empty when
ignore_groundtruth is true.
"""
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image],
0)
preprocessed_image, true_image_shapes = model.preprocess(
tf.to_float(original_image))
prediction_dict = model.predict(preprocessed_image, true_image_shapes)
detections = model.postprocess(prediction_dict, true_image_shapes)
if quantize:
from tensorflow.contrib.quantize import experimental_create_eval_graph
experimental_create_eval_graph()
# g = tf.get_default_graph()
# print(g.get_operations())
groundtruth = None
losses_dict = {}
if not ignore_groundtruth:
groundtruth = {
fields.InputDataFields.groundtruth_boxes:
input_dict[fields.InputDataFields.groundtruth_boxes],
fields.InputDataFields.groundtruth_classes:
input_dict[fields.InputDataFields.groundtruth_classes],
fields.InputDataFields.groundtruth_area:
input_dict[fields.InputDataFields.groundtruth_area],
fields.InputDataFields.groundtruth_is_crowd:
input_dict[fields.InputDataFields.groundtruth_is_crowd],
fields.InputDataFields.groundtruth_difficult:
input_dict[fields.InputDataFields.groundtruth_difficult]
}
if fields.InputDataFields.groundtruth_group_of in input_dict:
groundtruth[fields.InputDataFields.groundtruth_group_of] = (
input_dict[fields.InputDataFields.groundtruth_group_of])
if fields.DetectionResultFields.detection_masks in detections:
groundtruth[fields.InputDataFields.groundtruth_instance_masks] = (
input_dict[fields.InputDataFields.groundtruth_instance_masks])
label_id_offset = 1
model.provide_groundtruth(
[input_dict[fields.InputDataFields.groundtruth_boxes]], [
tf.one_hot(
input_dict[fields.InputDataFields.groundtruth_classes] -
label_id_offset,
depth=model.num_classes)
])
losses_dict.update(model.loss(prediction_dict, true_image_shapes))
result_dict = eval_util.result_dict_for_single_example(
original_image,
input_dict[fields.InputDataFields.source_id],
detections,
groundtruth,
class_agnostic=(fields.DetectionResultFields.detection_classes
not in detections),
scale_to_absolute=True)
# model.preprocess
result_dict['preprocessed_image'] = preprocessed_image
result_dict['true_image_shapes'] = true_image_shapes
# model.predict
result_dict['class_predictions_with_background'] = (
prediction_dict['class_predictions_with_background'])
result_dict['feature_maps'] = prediction_dict['feature_maps']
result_dict['preprocessed_inputs'] = prediction_dict['preprocessed_inputs']
result_dict['box_encodings'] = prediction_dict['box_encodings']
result_dict['anchors'] = prediction_dict['anchors']
# model.detections DEBUG ONLY
# result_dict['detection_boxes_all'] = detections['detection_boxes_all']
# print('YMK in _extract_anchros_and_losses')
# import ipdb
# ipdb.set_trace()
return result_dict, losses_dict
def _extract_prediction_tensors(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0)
preprocessed_image = model.preprocess(tf.to_float(original_image))
prediction_dict = model.predict(preprocessed_image)
detections = model.postprocess(prediction_dict)
original_image_shape = tf.shape(original_image)
absolute_detection_boxlist = box_list_ops.to_absolute_coordinates(
box_list.BoxList(tf.squeeze(detections['detection_boxes'], axis=0)),
original_image_shape[1], original_image_shape[2])
label_id_offset = 1
tensor_dict = {
'original_image': original_image,
'image_id': input_dict[fields.InputDataFields.source_id],
'detection_boxes': absolute_detection_boxlist.get(),
'detection_scores': tf.squeeze(detections['detection_scores'], axis=0),
'detection_classes': (
tf.squeeze(detections['detection_classes'], axis=0) +
label_id_offset),
}
if 'detection_masks' in detections:
detection_masks = tf.squeeze(detections['detection_masks'],
axis=0)
detection_boxes = tf.squeeze(detections['detection_boxes'],
axis=0)
# TODO: This should be done in model's postprocess function ideally.
detection_masks_reframed = ops.reframe_box_masks_to_image_masks(
detection_masks,
detection_boxes,
original_image_shape[1], original_image_shape[2])
detection_masks_reframed = tf.to_float(tf.greater(detection_masks_reframed,
0.5))
tensor_dict['detection_masks'] = detection_masks_reframed
# load groundtruth fields into tensor_dict
if not ignore_groundtruth:
normalized_gt_boxlist = box_list.BoxList(
input_dict[fields.InputDataFields.groundtruth_boxes])
gt_boxlist = box_list_ops.scale(normalized_gt_boxlist,
tf.shape(original_image)[1],
tf.shape(original_image)[2])
groundtruth_boxes = gt_boxlist.get()
groundtruth_classes = input_dict[fields.InputDataFields.groundtruth_classes]
tensor_dict['groundtruth_boxes'] = groundtruth_boxes
tensor_dict['groundtruth_classes'] = groundtruth_classes
tensor_dict['area'] = input_dict[fields.InputDataFields.groundtruth_area]
tensor_dict['is_crowd'] = input_dict[
fields.InputDataFields.groundtruth_is_crowd]
tensor_dict['difficult'] = input_dict[
fields.InputDataFields.groundtruth_difficult]
if 'detection_masks' in tensor_dict:
tensor_dict['groundtruth_instance_masks'] = input_dict[
fields.InputDataFields.groundtruth_instance_masks]
return tensor_dict
for key, tensor in tensor_dict.iteritems()})
=======
runtime_shapes = collections.OrderedDict(
{(key + rt_shape_str): tf.shape(tensor)
for key, tensor in tensor_dict.iteritems()})
>>>>>>> LOCAL
all_tensors = tensor_dict
all_tensors.update(runtime_shapes)
batched_tensors = tf.train.batch(
all_tensors,
capacity=batch_queue_capacity,
batch_size=batch_size,
dynamic_pad=True,
num_threads=num_batch_queue_threads)
self._queue = prefetcher.prefetch(batched_tensors,
prefetch_queue_capacity)
self._static_shapes = static_shapes
self._batch_size = batch_size
def dequeue(self):
"""Dequeues a batch of tensor_dict from the BatchQueue.
TODO: use allow_smaller_final_batch to allow running over the whole eval set
Returns:
A list of tensor_dicts of the requested batch_size.
"""
batched_tensors = self._queue.dequeue()
# Separate input tensors from tensors containing their runtime shapes.
tensors = {}
shapes = {}
def _extract_prediction_tensors(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
input_dict = create_input_dict_fn()
batch = None
if 'batch' in input_dict:
batch = input_dict.pop('batch')
else:
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
# consistent format for images and videos
for key, value in input_dict.iteritems():
input_dict[key] = (value,)
detections = _create_detection_op(model, input_dict, batch)
# Print out anaylsis of the model.
tf.contrib.tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=tf.contrib.tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
tf.contrib.tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=tf.contrib.tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
num_frames = len(input_dict[fields.InputDataFields.image])
ret = []
for i in range(num_frames):
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image][i],
0)
groundtruth = None
if not ignore_groundtruth:
groundtruth = {
fields.InputDataFields.groundtruth_boxes:
input_dict[fields.InputDataFields.groundtruth_boxes][i],
fields.InputDataFields.groundtruth_classes:
input_dict[fields.InputDataFields.groundtruth_classes][i],
}
optional_keys = (
fields.InputDataFields.groundtruth_area,
fields.InputDataFields.groundtruth_is_crowd,
fields.InputDataFields.groundtruth_difficult,
fields.InputDataFields.groundtruth_group_of,
)
for opt_key in optional_keys:
if opt_key in input_dict:
groundtruth[opt_key] = input_dict[opt_key][i]
if fields.DetectionResultFields.detection_masks in detections:
groundtruth[fields.InputDataFields.groundtruth_instance_masks] = (
input_dict[fields.InputDataFields.groundtruth_instance_masks][i])
detections_frame = {
key: tf.expand_dims(value[i], 0)
for key, value in detections.iteritems()
}
source_id = (
batch.key[0] if batch is not None else
input_dict[fields.InputDataFields.source_id][i])
ret.append(
eval_util.result_dict_for_single_example(
original_image,
source_id,
detections_frame,
groundtruth,
class_agnostic=(fields.DetectionResultFields.detection_classes
not in detections),
scale_to_absolute=True))
return ret
def _extract_predictions_and_losses(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Constructs tensorflow detection graph and returns output tensors.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
prediction_groundtruth_dict: A dictionary with postprocessed tensors (keyed
by standard_fields.DetectionResultsFields) and optional groundtruth
tensors (keyed by standard_fields.InputDataFields).
losses_dict: A dictionary containing detection losses. This is empty when
ignore_groundtruth is true.
"""
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0)
preprocessed_image, true_image_shapes = model.preprocess(
tf.to_float(original_image))
prediction_dict = model.predict(preprocessed_image, true_image_shapes)
detections = model.postprocess(prediction_dict, true_image_shapes)
groundtruth = None
losses_dict = {}
if not ignore_groundtruth:
groundtruth = {
fields.InputDataFields.groundtruth_boxes:
input_dict[fields.InputDataFields.groundtruth_boxes],
fields.InputDataFields.groundtruth_classes:
input_dict[fields.InputDataFields.groundtruth_classes],
fields.InputDataFields.groundtruth_area:
input_dict[fields.InputDataFields.groundtruth_area],
fields.InputDataFields.groundtruth_is_crowd:
input_dict[fields.InputDataFields.groundtruth_is_crowd],
fields.InputDataFields.groundtruth_difficult:
input_dict[fields.InputDataFields.groundtruth_difficult]
}
if fields.InputDataFields.groundtruth_group_of in input_dict:
groundtruth[fields.InputDataFields.groundtruth_group_of] = (
input_dict[fields.InputDataFields.groundtruth_group_of])
groundtruth_masks_list = None
if fields.DetectionResultFields.detection_masks in detections:
groundtruth[fields.InputDataFields.groundtruth_instance_masks] = (
input_dict[fields.InputDataFields.groundtruth_instance_masks])
groundtruth_masks_list = [
input_dict[fields.InputDataFields.groundtruth_instance_masks]]
groundtruth_keypoints_list = None
if fields.DetectionResultFields.detection_keypoints in detections:
groundtruth[fields.InputDataFields.groundtruth_keypoints] = (
input_dict[fields.InputDataFields.groundtruth_keypoints])
groundtruth_keypoints_list = [
input_dict[fields.InputDataFields.groundtruth_keypoints]]
label_id_offset = 1
model.provide_groundtruth(
[input_dict[fields.InputDataFields.groundtruth_boxes]],
[tf.one_hot(input_dict[fields.InputDataFields.groundtruth_classes]
- label_id_offset, depth=model.num_classes)],
groundtruth_masks_list, groundtruth_keypoints_list)
losses_dict.update(model.loss(prediction_dict, true_image_shapes))
result_dict = eval_util.result_dict_for_single_example(
original_image,
input_dict[fields.InputDataFields.source_id],
detections,
groundtruth,
class_agnostic=(
fields.DetectionResultFields.detection_classes not in detections),
scale_to_absolute=True)
return result_dict, losses_dict
def _extract_prediction_tensors(model,
create_input_dict_fn,
ignore_groundtruth=False,
provide_groundtruth_to_model=False,
calc_loss=False):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
provide_groundtruth_to_model: whether to use model.provide_groundtruth()
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
mtl = model._mtl
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
if calc_loss or mtl.window or mtl.edgemask:
provide_groundtruth_to_model = True
# Get groundtruth information
if provide_groundtruth_to_model:
(_, groundtruth_boxes_list, groundtruth_ignore_list,
groundtruth_classes_list, groundtruth_masks_list, _,
window_boxes_list, window_classes_list, groundtruth_closeness_list,
groundtruth_edgemask_list) = _get_inputs([input_dict],
model.num_classes,
with_filename=False)
if any(mask is None for mask in groundtruth_masks_list):
groundtruth_masks_list = None
model.provide_groundtruth(groundtruth_boxes_list,
groundtruth_classes_list,
groundtruth_closeness_list,
groundtruth_ignore_list,
groundtruth_masks_list)
model.provide_window(window_boxes_list, window_classes_list)
model.provide_edgemask(groundtruth_edgemask_list)
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image],
0)
preprocessed_image = model.preprocess(tf.to_float(original_image))
prediction_dict = model.predict(preprocessed_image)
if mtl.window:
prediction_dict = model.predict_with_window(prediction_dict)
if mtl.edgemask:
prediction_dict = model.predict_edgemask(prediction_dict)
if mtl.refine:
prediction_dict = model.predict_with_mtl_results(prediction_dict)
detections = model.postprocess(prediction_dict)
original_image_shape = tf.shape(original_image)
absolute_detection_boxlist = box_list_ops.to_absolute_coordinates(
box_list.BoxList(tf.squeeze(detections['detection_boxes'], axis=0)),
original_image_shape[1], original_image_shape[2])
label_id_offset = 1
tensor_dict = {
'original_image':
original_image,
'image_id':
input_dict[fields.InputDataFields.source_id],
'detection_boxes':
absolute_detection_boxlist.get(),
'detection_scores':
tf.squeeze(detections['detection_scores'], axis=0),
'detection_classes':
(tf.squeeze(detections['detection_classes'], axis=0) +
label_id_offset),
}
if 'detection_thresholds' in detections:
tensor_dict['detection_thresholds'] = \
tf.squeeze(detections['detection_thresholds'], axis=0)
if 'detection_masks' in detections:
detection_masks = tf.squeeze(detections['detection_masks'], axis=0)
detection_boxes = tf.squeeze(detections['detection_boxes'], axis=0)
# TODO: This should be done in model's postprocess function ideally.
detection_masks_reframed = ops.reframe_box_masks_to_image_masks(
detection_masks, detection_boxes, original_image_shape[1],
original_image_shape[2])
detection_masks_reframed = tf.to_float(
tf.greater(detection_masks_reframed, 0.5))
tensor_dict['detection_masks'] = detection_masks_reframed
# load groundtruth fields into tensor_dict
if not ignore_groundtruth:
normalized_gt_boxlist = box_list.BoxList(
input_dict[fields.InputDataFields.groundtruth_boxes])
gt_boxlist = box_list_ops.scale(normalized_gt_boxlist,
tf.shape(original_image)[1],
tf.shape(original_image)[2])
groundtruth_boxes = gt_boxlist.get()
groundtruth_classes = input_dict[
fields.InputDataFields.groundtruth_classes]
tensor_dict['groundtruth_boxes'] = groundtruth_boxes
tensor_dict['groundtruth_classes'] = groundtruth_classes
tensor_dict['area'] = input_dict[
fields.InputDataFields.groundtruth_area]
tensor_dict['difficult'] = input_dict[
fields.InputDataFields.groundtruth_difficult]
if 'detection_masks' in tensor_dict:
tensor_dict['groundtruth_instance_masks'] = input_dict[
fields.InputDataFields.groundtruth_instance_masks]
# Subset annotations
if fields.InputDataFields.groundtruth_subset in input_dict:
tensor_dict['groundtruth_subset'] \
= input_dict[fields.InputDataFields.groundtruth_subset]
if calc_loss:
losses_dict = model.loss(prediction_dict)
for loss_name, loss_tensor in losses_dict.iteritems():
loss_tensor = tf.check_numerics(loss_tensor,
'%s is inf or nan.' % loss_name,
name='Loss/' + loss_name)
tensor_dict['Loss/' + loss_name] = loss_tensor
# mtl groundtruth
if mtl.window:
tensor_dict['window_classes_gt'] = input_dict[
fields.InputDataFields.window_classes]
tensor_dict['window_classes_dt'] = prediction_dict[
'window_class_predictions']
if mtl.closeness:
tensor_dict['closeness_gt'] = input_dict[
fields.InputDataFields.groundtruth_closeness]
tensor_dict['closeness_dt'] = prediction_dict['closeness_predictions']
if mtl.edgemask:
tensor_dict['edgemask_gt'] = input_dict[
fields.InputDataFields.groundtruth_edgemask_masks]
tensor_dict['edgemask_dt'] = prediction_dict['edgemask_predictions']
return tensor_dict
def _extract_prediction_tensors(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
k_shot = model._k_shot
input_dict, thread = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=100)
input_dict = prefetch_queue.dequeue()
images = input_dict[fields.CoLocInputDataFields.supportset]
images_list = [image for image in tf.split(images, k_shot)]
float_images = tf.to_float(images)
input_dict[fields.CoLocInputDataFields.supportset] = float_images
preprocessed_images = [
model.preprocess(float_image)
for float_image in tf.split(float_images, k_shot)
]
preprocessed_image = tf.concat(preprocessed_images, 0)
prediction_dict = model.predict(preprocessed_image)
detections = model.postprocess(prediction_dict)
original_image_shape = tf.shape(images)
def _absolute_boxes(normalized_boxes):
absolute_detection_boxlist_list = [
box_list_ops.to_absolute_coordinates(
box_list.BoxList(tf.squeeze(k, axis=0)),
original_image_shape[1], original_image_shape[2])
for k in tf.split(normalized_boxes, k_shot)
]
return tf.stack([db.get() for db in absolute_detection_boxlist_list])
tensor_dict = {'original_image': images}
if detections.has_key('rpn_detection_boxes'):
tensor_dict['rpn'] = {
'boxes': detections['rpn_detection_boxes'],
'scores': detections['rpn_detection_scores'],
'classes': detections['rpn_detection_classes'],
'class_agnostic': tf.constant(True)
}
if detections.has_key('detection_boxes'):
tensor_dict['detection'] = {
'boxes': detections['detection_boxes'],
'scores': detections['detection_scores'],
'classes': detections['detection_classes'],
'class_agnostic': tf.constant(False)
}
label_id_offset = 1
if hasattr(model, '_tree_debug_tensors'):
tensor_dict.update(model._tree_debug_tensors())
# Convert to the absolute coordinates
for key, val in tensor_dict.items():
if isinstance(val, dict):
for mkey in val:
if 'boxes' in mkey:
val[mkey] = _absolute_boxes(val[mkey])
if 'classes' in mkey:
val[mkey] = val[mkey] + label_id_offset
if not ignore_groundtruth:
groundtruth_boxes_list = []
groundtruth_classes_list = []
groundtruth_target_class = input_dict[
fields.CoLocInputDataFields.groundtruth_target_class]
for k in xrange(k_shot):
normalized_gt_boxlist = box_list.BoxList(
input_dict[fields.CoLocInputDataFields.groundtruth_boxes +
'_{}'.format(k)])
gt_boxlist = box_list_ops.scale(normalized_gt_boxlist,
original_image_shape[1],
original_image_shape[2])
groundtruth_boxes = gt_boxlist.get()
groundtruth_classes = input_dict[
fields.CoLocInputDataFields.groundtruth_classes +
'_{}'.format(k)]
groundtruth_boxes_list.append(groundtruth_boxes)
groundtruth_classes_list.append(groundtruth_classes)
ndict = dict()
ndict['boxes'] = groundtruth_boxes_list
ndict['classes'] = groundtruth_classes_list
ndict['target_class'] = groundtruth_target_class
tensor_dict['groundtruth'] = ndict
return tensor_dict, thread
def build_input(tfrecord_paths):
"""Builds the graph's input.
Args:
tfrecord_paths: List of paths to the input TFRecords
Returns:
serialized_example_tensor: The next serialized example. String scalar Tensor
image_tensor: The decoded image of the example. Uint8 tensor,
shape=[1, None, None,3]
"""
filename_queue = tf.train.string_input_producer(
tfrecord_paths, shuffle=False, num_epochs=1)
tf_record_reader = tf.TFRecordReader()
_, serialized_example_tensor = tf_record_reader.read(filename_queue)
# *** MODIFIED
prefetch_queue = prefetcher.prefetch({'serialized_example_tensor': serialized_example_tensor}, 100)
dequeue = prefetch_queue.dequeue()
serialized_example_tensor = dequeue['serialized_example_tensor']
# *** MODIFIED ENDS
features = tf.parse_single_example(
serialized_example_tensor,
features={
standard_fields.TfExampleFields.image_encoded:
tf.FixedLenFeature([], tf.string),
})
encoded_image = features[standard_fields.TfExampleFields.image_encoded]
image_tensor = tf.image.decode_image(encoded_image, channels=3)
image_tensor.set_shape([None, None, 3])
# image_tensor = tf.expand_dims(image_tensor, 0)
# # *** MODIFIED
# batch = tf.train.batch(
# [serialized_example_tensor, image_tensor],
# batch_size=24,
# enqueue_many=False,
# num_threads=6,
# capacity=5 * 24)
# return batch[0], batch[1]
image_resizer_text_proto = """
keep_aspect_ratio_resizer {
min_dimension: 800
max_dimension: 1365
}
"""
image_resizer_config = image_resizer_pb2.ImageResizer()
text_format.Merge(image_resizer_text_proto, image_resizer_config)
image_resizer_fn = image_resizer_builder.build(image_resizer_config)
resized_image_tensor, _ = image_resizer_fn(image_tensor)
# resized_image_tensor = tf.image.convert_image_dtype(resized_image_tensor, dtype=tf.uint8)
resized_image_tensor = tf.cast(resized_image_tensor, dtype=tf.uint8)
resized_image_tensor = tf.expand_dims(resized_image_tensor, 0)
# # *** MODIFIED ENDS
return serialized_example_tensor, resized_image_tensor#image_tensor
def _extract_prediction_tensors(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) # TODO
input_dict = prefetch_queue.dequeue()
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image],
0)
next_image = input_dict.get(fields.InputDataFields.next_image)
image_input = tf.to_float(original_image)
if next_image is not None:
next_image = tf.to_float(next_image)
image_input = tf.concat(
[image_input, tf.expand_dims(next_image, 0)], 3)
depth = input_dict.get(fields.InputDataFields.groundtruth_depth)
next_depth = input_dict.get(
fields.InputDataFields.groundtruth_next_depth)
image_input.set_shape([1, None, None, 6])
if depth is not None and next_depth is not None:
camera_intrinsics = input_dict[
fields.InputDataFields.camera_intrinsics]
coords = motion_util.get_3D_coords(tf.expand_dims(depth, 0),
camera_intrinsics)
next_coords = motion_util.get_3D_coords(
tf.expand_dims(next_depth, 0), camera_intrinsics)
image_input = tf.concat([image_input, coords, next_coords], 3)
image_input.set_shape([1, None, None, 12])
preprocessed_image = model.preprocess(image_input)
prediction_dict = model.predict(preprocessed_image)
detections = model.postprocess(prediction_dict)
original_image_shape = tf.shape(original_image)
absolute_detection_boxlist = box_list_ops.to_absolute_coordinates(
box_list.BoxList(tf.squeeze(detections['detection_boxes'], axis=0)),
original_image_shape[1], original_image_shape[2])
label_id_offset = 1
tensor_dict = {
'original_image':
original_image,
'image_id':
input_dict[fields.InputDataFields.source_id],
'detection_boxes':
absolute_detection_boxlist.get(),
'detection_scores':
tf.squeeze(detections['detection_scores'], axis=0),
'detection_classes':
(tf.squeeze(detections['detection_classes'], axis=0) +
label_id_offset),
}
if 'detection_masks' in detections:
detection_masks = tf.squeeze(detections['detection_masks'], axis=0)
detection_boxes = tf.squeeze(detections['detection_boxes'], axis=0)
# TODO: This should be done in model's postprocess function ideally.
detection_masks_reframed = ops.reframe_box_masks_to_image_masks(
detection_masks, detection_boxes, original_image_shape[1],
original_image_shape[2])
detection_masks_reframed = tf.to_float(
tf.greater(detection_masks_reframed, 0.5))
tensor_dict['detection_masks'] = detection_masks_reframed
if 'detection_motions' in detections:
detection_motions = tf.squeeze(detections['detection_motions'], axis=0)
detection_motions_with_matrices = (
motion_util.postprocess_detection_motions(detection_motions,
keep_logits=False))
tensor_dict['detection_motions'] = detection_motions_with_matrices
if 'camera_motion' in detections:
camera_motion_with_matrices = tf.squeeze(
motion_util.postprocess_camera_motion(detections['camera_motion']),
axis=0)
tensor_dict['camera_motion'] = camera_motion_with_matrices
tensor_dict['groundtruth_camera_motion'] = input_dict[
fields.InputDataFields.groundtruth_camera_motion]
# load groundtruth fields into tensor_dict
if not ignore_groundtruth:
normalized_gt_boxlist = box_list.BoxList(
input_dict[fields.InputDataFields.groundtruth_boxes])
gt_boxlist = box_list_ops.scale(normalized_gt_boxlist,
tf.shape(original_image)[1],
tf.shape(original_image)[2])
groundtruth_boxes = gt_boxlist.get()
groundtruth_classes = input_dict[
fields.InputDataFields.groundtruth_classes]
tensor_dict['groundtruth_boxes'] = groundtruth_boxes
tensor_dict['groundtruth_classes'] = groundtruth_classes
tensor_dict['area'] = input_dict[
fields.InputDataFields.groundtruth_area]
tensor_dict['is_crowd'] = input_dict[
fields.InputDataFields.groundtruth_is_crowd]
tensor_dict['difficult'] = input_dict[
fields.InputDataFields.groundtruth_difficult]
if 'detection_masks' in tensor_dict:
tensor_dict['groundtruth_instance_masks'] = input_dict[
fields.InputDataFields.groundtruth_instance_masks]
if 'detection_motions' in tensor_dict:
tensor_dict['groundtruth_camera_motion'] = input_dict[
fields.InputDataFields.groundtruth_camera_motion]
tensor_dict['groundtruth_instance_motions'] = input_dict[
fields.InputDataFields.groundtruth_instance_motions]
tensor_dict['camera_intrinsics'] = input_dict[
fields.InputDataFields.camera_intrinsics]
if fields.InputDataFields.groundtruth_flow in input_dict:
tensor_dict['groundtruth_flow'] = input_dict[
fields.InputDataFields.groundtruth_flow]
if not 'depth' in tensor_dict:
tensor_dict['depth'] = input_dict[
fields.InputDataFields.groundtruth_depth]
else:
tensor_dict['groundtruth_depth'] = input_dict[
fields.InputDataFields.groundtruth_depth]
return tensor_dict
def _extract_predictions_and_losses(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Constructs tensorflow detection graph and returns output tensors.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
prediction_groundtruth_dict: A dictionary with postprocessed tensors (keyed
by standard_fields.DetectionResultsFields) and optional groundtruth
tensors (keyed by standard_fields.InputDataFields).
losses_dict: A dictionary containing detection losses. This is empty when
ignore_groundtruth is true.
"""
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image],
0)
preprocessed_image, true_image_shapes = model.preprocess(
tf.cast(original_image, dtype=tf.float32))
prediction_dict = model.predict(preprocessed_image, true_image_shapes)
detections = model.postprocess(prediction_dict, true_image_shapes)
groundtruth = None
losses_dict = {}
if not ignore_groundtruth:
groundtruth = {
fields.InputDataFields.groundtruth_boxes:
input_dict[fields.InputDataFields.groundtruth_boxes],
fields.InputDataFields.groundtruth_classes:
input_dict[fields.InputDataFields.groundtruth_classes],
fields.InputDataFields.groundtruth_area:
input_dict[fields.InputDataFields.groundtruth_area],
fields.InputDataFields.groundtruth_is_crowd:
input_dict[fields.InputDataFields.groundtruth_is_crowd],
fields.InputDataFields.groundtruth_difficult:
input_dict[fields.InputDataFields.groundtruth_difficult]
}
if fields.InputDataFields.groundtruth_group_of in input_dict:
groundtruth[fields.InputDataFields.groundtruth_group_of] = (
input_dict[fields.InputDataFields.groundtruth_group_of])
groundtruth_masks_list = None
if fields.DetectionResultFields.detection_masks in detections:
groundtruth[fields.InputDataFields.groundtruth_instance_masks] = (
input_dict[fields.InputDataFields.groundtruth_instance_masks])
groundtruth_masks_list = [
input_dict[fields.InputDataFields.groundtruth_instance_masks]
]
groundtruth_keypoints_list = None
if fields.DetectionResultFields.detection_keypoints in detections:
groundtruth[fields.InputDataFields.groundtruth_keypoints] = (
input_dict[fields.InputDataFields.groundtruth_keypoints])
groundtruth_keypoints_list = [
input_dict[fields.InputDataFields.groundtruth_keypoints]
]
label_id_offset = 1
model.provide_groundtruth(
[input_dict[fields.InputDataFields.groundtruth_boxes]], [
tf.one_hot(
input_dict[fields.InputDataFields.groundtruth_classes] -
label_id_offset,
depth=model.num_classes)
],
groundtruth_masks_list=groundtruth_masks_list,
groundtruth_keypoints_list=groundtruth_keypoints_list)
losses_dict.update(model.loss(prediction_dict, true_image_shapes))
result_dict = eval_util.result_dict_for_single_example(
original_image,
input_dict[fields.InputDataFields.source_id],
detections,
groundtruth,
class_agnostic=(fields.DetectionResultFields.detection_classes
not in detections),
scale_to_absolute=True)
return result_dict, losses_dict
def _extract_prediction_tensors(model,
input_k_shot,
create_input_dict_fn,
ignore_groundtruth=False,
use_target_class_in_predictions=False,
bcd_outside_images=0):
"""Restores the model in a tensorflow session.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
bcd_outside_images: maximum number of images outside the subproblem.
We expect to get one feature from each bag outside
the subproblem.
Returns:
tensor_dict: A tensor dictionary with evaluations.
"""
tree_k_shot = model._k_shot
bag_size = model._bag_size
num_classes = model._num_classes
num_negative_bags = model._num_negative_bags
total_images = num_negative_bags + input_k_shot
input_dict, thread = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=10)
input_dict = prefetch_queue.dequeue()
extra_dict = {}
reduced_dict = {}
## Append additional variables for the outside bag features
num_bcd_bags = int(np.ceil(bcd_outside_images / float(bag_size)))
bcd_bags = None
bcd_reload_op = None
field_bcd_supportset = 'bcd_supportset'
if bcd_outside_images > 0:
assert (num_negative_bags == 0)
assert ('dense' in FLAGS.mrf_type)
shape = [num_bcd_bags
] + input_dict[fields.supportset].shape.as_list()[1:]
#bcd_supportset = tf.placeholder(tf.float32, shape=shape,
#name=field_bcd_supportset)
bcd_supportset = tf.constant(0.0,
dtype=tf.float32,
shape=shape,
name=field_bcd_supportset)
bcd_supportset_var, bcd_reload_op = wrap_with_variable(
bcd_supportset, 0)
input_dict[field_bcd_supportset] = bcd_supportset_var
##
for k, v in input_dict.iteritems():
#if 'groundtruth_image' in k or 'original_images' in
#k or 'target_class' in k or 'filename' in k:
if ('groundtruth_image' in k or 'original_images' in k
or 'filename' in k or field_bcd_supportset in k):
extra_dict[k] = v
else:
#######
#if k == fields.supportset:
#v = tf.zeros_like(v)
#embed()
#####
reduced_dict[k] = v
#reduced_dict.pop('filename')
input_dict, ops = wrap_dict(reduced_dict)
input_dict.update(extra_dict)
reload_op = tf.group(*ops)
def _r(t):
return tf.reshape(t, [
-1,
] + t.shape[2:].as_list())
def reorder_for_k2(t):
'''
when tree_k_shot is 2, and we have negative bags and input_k_shot is greater than 2,
this function reorders the list of inputs in a way that it puts the negative elements
after every 2*bag_size elements.
Notes:
- assumes positive elements are in first bag_size*input_k_shot elements of the input list
and the rest of the elements are negative bags i.e., num_negative_bags*bag_size last
elements of the list.
'''
if num_negative_bags > 0 and tree_k_shot == 2 and input_k_shot != 2:
pos_part = t[:bag_size * input_k_shot]
neg_part = t[bag_size * input_k_shot:]
assert input_k_shot % 2 == 0, 'input_k_shot should be multiple of 2'
l = [
pos_part[bag_size * 2 * i:bag_size * 2 * (i + 1)] + neg_part
for i in range(input_k_shot / 2)
]
t = [i for x in l for i in x]
return t
def _s(t):
return reorder_for_k2(tf.split(_r(t), total_images * bag_size))
images_list = _s(input_dict[fields.supportset])
groundtruth_classes_list = _s(input_dict[fields.groundtruth_classes])
original_groundtruth_classes = input_dict[
fields.original_groundtruth_classes]
original_groundtruth_classes_list = _s(original_groundtruth_classes)
one_hot_gt_classes_list = _s(
tf.one_hot(tf.to_int32(original_groundtruth_classes), num_classes + 1))
boxes_list = _s(input_dict[fields.groundtruth_boxes])
proposal_objectness_list = input_dict.get(fields.proposal_objectness, None)
if proposal_objectness_list is not None:
proposal_objectness_list = _s(proposal_objectness_list)
if use_target_class_in_predictions:
target_class = input_dict[fields.groundtruth_target_class]
model._groundtruth_lists['target_class'] = target_class[tf.newaxis]
bcd_bags = None
if field_bcd_supportset in input_dict:
bcd_bags = tf.split(_r(input_dict[field_bcd_supportset]),
num_bcd_bags * bag_size)
if FLAGS.add_mrf:
lists = [
images_list, groundtruth_classes_list,
original_groundtruth_classes_list, one_hot_gt_classes_list,
boxes_list
]
if proposal_objectness_list is not None:
lists.append(proposal_objectness_list)
extend_for_mrf(lists, bcd_bags, input_k_shot, bag_size,
num_negative_bags)
if is_multipart(input_k_shot + num_bcd_bags, bag_size):
problem_size = (
2 + num_negative_bags
) * bag_size ###TODO for K1 and BCD: add min(2, input_k_shot)
total_problems = len(images_list) / problem_size
max_batch = get_max_batch(total_problems, bag_size)
assert total_problems % max_batch == 0, 'problems should be divisible by max_batch'
batch_indices = range(total_problems // max_batch)
l = max_batch * problem_size
#start and end indices
se = [(l * i, l * (i + 1)) for i in batch_indices]
#if total_problems % max_batch != 0:
# se += [(se[-1][1], len(images_list))]
np_ranges = np.array(se)
tf_ranges = tf.constant(np_ranges)
tensor_dict = get_tensor_dict(model, images_list, boxes_list,
one_hot_gt_classes_list,
proposal_objectness_list, tf_ranges,
l, bag_size)
gt_dict = {}
gt_dict['num_sel'] = len(se)
add_groundtruth(gt_dict, input_dict, input_k_shot, bag_size,
num_negative_bags, groundtruth_classes_list,
original_groundtruth_classes_list)
tensor_dict['reload_op'] = reload_op
if bcd_reload_op is not None:
tensor_dict['bcd_reload_op'] = bcd_reload_op
tensor_dict['bcd_supportset_placeholder'] = bcd_supportset
tensor_dict[fields.supportset] = input_dict[fields.supportset]
tensor_dict['target_class'] = input_dict[
fields.groundtruth_target_class]
return [tensor_dict, gt_dict], thread
tf_ranges = tf.constant([(0, len(images_list))])
tensor_dict = get_tensor_dict(model, images_list, boxes_list,
one_hot_gt_classes_list,
proposal_objectness_list, tf_ranges,
len(images_list), bag_size)
gt_dict = {}
gt_dict['num_sel'] = 1
add_groundtruth(gt_dict, input_dict, input_k_shot, bag_size,
num_negative_bags, groundtruth_classes_list,
original_groundtruth_classes_list)
tensor_dict['reload_op'] = reload_op
if bcd_reload_op is not None:
tensor_dict['bcd_reload_op'] = bcd_reload_op
tensor_dict['bcd_supportset_placeholder'] = bcd_supportset
tensor_dict[fields.supportset] = input_dict[fields.supportset]
tensor_dict['target_class'] = input_dict[fields.groundtruth_target_class]
return [tensor_dict, gt_dict], thread
