def test_prefetch_tensors_with_fully_defined_shapes(self): with self.test_session() as sess: batch_size = 10 image_size = 32 num_batches = 5 examples = tf.Variable(tf.constant(0, dtype=tf.int64)) counter = examples.count_up_to(num_batches) image = tf.random_normal([batch_size, image_size, image_size, 3], dtype=tf.float32, name='images') label = tf.random_uniform([batch_size, 1], 0, 10, dtype=tf.int32, name='labels') prefetch_queue = prefetcher.prefetch(tensor_dict={'counter': counter, 'image': image, 'label': label}, capacity=100) tensor_dict = prefetch_queue.dequeue() self.assertAllEqual(tensor_dict['image'].get_shape().as_list(), [batch_size, image_size, image_size, 3]) self.assertAllEqual(tensor_dict['label'].get_shape().as_list(), [batch_size, 1]) tf.initialize_all_variables().run() with slim.queues.QueueRunners(sess): for _ in range(num_batches): results = sess.run(tensor_dict) self.assertEquals(results['image'].shape, (batch_size, image_size, image_size, 3)) self.assertEquals(results['label'].shape, (batch_size, 1)) with self.assertRaises(tf.errors.OutOfRangeError): sess.run(tensor_dict)
def _extract_groundtruth_tensors(create_input_dict_fn): input_dict = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0) tensor_dict = {'image_id': input_dict[fields.InputDataFields.source_id]} normalized_gt_boxlist = box_list.BoxList( input_dict[fields.InputDataFields.groundtruth_boxes]) gt_boxlist = box_list_ops.scale(normalized_gt_boxlist, tf.shape(original_image)[1], tf.shape(original_image)[2]) groundtruth_boxes = gt_boxlist.get() groundtruth_classes = input_dict[ fields.InputDataFields.groundtruth_classes] tensor_dict['groundtruth_boxes'] = groundtruth_boxes tensor_dict['groundtruth_classes'] = groundtruth_classes tensor_dict['area'] = input_dict[fields.InputDataFields.groundtruth_area] tensor_dict['difficult'] = input_dict[ fields.InputDataFields.groundtruth_difficult] # subset annotations if fields.InputDataFields.groundtruth_subset in input_dict: tensor_dict['groundtruth_subset'] \ = input_dict[fields.InputDataFields.groundtruth_subset] return tensor_dict
def __init__(self, tensor_dict, batch_size, batch_queue_capacity, num_batch_queue_threads, prefetch_queue_capacity): """Constructs a batch queue holding tensor_dict. Args: tensor_dict: dictionary of tensors to batch. batch_size: batch size. batch_queue_capacity: max capacity of the queue from which the tensors are batched. num_batch_queue_threads: number of threads to use for batching. prefetch_queue_capacity: max capacity of the queue used to prefetch assembled batches. """ # Remember static shapes to set shapes of batched tensors. static_shapes = collections.OrderedDict( {key: tensor.get_shape() for key, tensor in tensor_dict.items()}) # Remember runtime shapes to unpad tensors after batching. runtime_shapes = collections.OrderedDict( {(key + rt_shape_str): tf.shape(tensor) for key, tensor in tensor_dict.items()}) all_tensors = tensor_dict all_tensors.update(runtime_shapes) batched_tensors = tf.train.batch( all_tensors, capacity=batch_queue_capacity, batch_size=batch_size, dynamic_pad=True, num_threads=num_batch_queue_threads) self._queue = prefetcher.prefetch(batched_tensors, prefetch_queue_capacity) self._static_shapes = static_shapes self._batch_size = batch_size
def __init__(self, tensor_dict, batch_size, batch_queue_capacity, num_batch_queue_threads, prefetch_queue_capacity): """Constructs a batch queue holding tensor_dict. Args: tensor_dict: dictionary of tensors to batch. batch_size: batch size. batch_queue_capacity: max capacity of the queue from which the tensors are batched. num_batch_queue_threads: number of threads to use for batching. prefetch_queue_capacity: max capacity of the queue used to prefetch assembled batches. """ # Remember static shapes to set shapes of batched tensors. static_shapes = collections.OrderedDict( {key: tensor.get_shape() for key, tensor in tensor_dict.items()}) # Remember runtime shapes to unpad tensors after batching. runtime_shapes = collections.OrderedDict({ (key + rt_shape_str): tf.shape(tensor) for key, tensor in tensor_dict.items() }) all_tensors = tensor_dict all_tensors.update(runtime_shapes) batched_tensors = tf.train.batch(all_tensors, capacity=batch_queue_capacity, batch_size=batch_size, dynamic_pad=True, num_threads=num_batch_queue_threads) self._queue = prefetcher.prefetch(batched_tensors, prefetch_queue_capacity) self._static_shapes = static_shapes self._batch_size = batch_size
def _extract_prediction_tensors(model, create_input_dict_fn, ignore_groundtruth=False): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: tensor_dict: A tensor dictionary with evaluations. """ input_dict = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0) original_audio = tf.expand_dims(input_dict[fields.InputDataFields.audio], 0) preprocessed_image = model.preprocess(tf.to_float(original_image), False) preprocessed_audio = model.preprocess(tf.to_float(original_audio), True) prediction_dict = model.predict(preprocessed_image, preprocessed_audio) detections = model.postprocess(prediction_dict) groundtruth = None if not ignore_groundtruth: groundtruth = { fields.InputDataFields.groundtruth_boxes: input_dict[fields.InputDataFields.groundtruth_boxes], fields.InputDataFields.groundtruth_classes: input_dict[fields.InputDataFields.groundtruth_classes], fields.InputDataFields.groundtruth_area: input_dict[fields.InputDataFields.groundtruth_area], fields.InputDataFields.groundtruth_is_crowd: input_dict[fields.InputDataFields.groundtruth_is_crowd], fields.InputDataFields.groundtruth_difficult: input_dict[fields.InputDataFields.groundtruth_difficult], fields.InputDataFields.groundtruth_image_classes: input_dict[fields.InputDataFields.groundtruth_image_classes] } if fields.InputDataFields.groundtruth_group_of in input_dict: groundtruth[fields.InputDataFields.groundtruth_group_of] = ( input_dict[fields.InputDataFields.groundtruth_group_of]) if fields.DetectionResultFields.detection_masks in detections: groundtruth[fields.InputDataFields.groundtruth_instance_masks] = ( input_dict[fields.InputDataFields.groundtruth_instance_masks]) return eval_util.result_dict_for_single_example( original_image, input_dict[fields.InputDataFields.source_id], detections, groundtruth, class_agnostic=(fields.DetectionResultFields.detection_classes not in detections), scale_to_absolute=True)
def _extract_prediction_tensors(model, create_input_dict_fn, ignore_groundtruth=False): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: tensor_dict: A tensor dictionary with evaluations. """ input_dict = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0) preprocessed_image, true_image_shapes = model.preprocess( tf.to_float(original_image)) prediction_dict = model.predict(preprocessed_image, true_image_shapes) detections = model.postprocess(prediction_dict, true_image_shapes) groundtruth = None if not ignore_groundtruth: groundtruth = { fields.InputDataFields.groundtruth_boxes: input_dict[fields.InputDataFields.groundtruth_boxes], fields.InputDataFields.groundtruth_classes: input_dict[fields.InputDataFields.groundtruth_classes], fields.InputDataFields.groundtruth_area: input_dict[fields.InputDataFields.groundtruth_area], fields.InputDataFields.groundtruth_is_crowd: input_dict[fields.InputDataFields.groundtruth_is_crowd], fields.InputDataFields.groundtruth_difficult: input_dict[fields.InputDataFields.groundtruth_difficult] } if fields.InputDataFields.groundtruth_group_of in input_dict: groundtruth[fields.InputDataFields.groundtruth_group_of] = ( input_dict[fields.InputDataFields.groundtruth_group_of]) if fields.DetectionResultFields.detection_masks in detections: groundtruth[fields.InputDataFields.groundtruth_instance_masks] = ( input_dict[fields.InputDataFields.groundtruth_instance_masks]) return eval_util.result_dict_for_single_example( original_image, input_dict[fields.InputDataFields.source_id], detections, groundtruth, class_agnostic=( fields.DetectionResultFields.detection_classes not in detections), scale_to_absolute=True)
def build_input(tfrecord_paths): """Builds the graph's input. Args: tfrecord_paths: List of paths to the input TFRecords Returns: serialized_example_tensor: The next serialized example. String scalar Tensor image_tensor: The decoded image of the example. Uint8 tensor, shape=[1, None, None,3] """ filename_queue = tf.train.string_input_producer( tfrecord_paths, shuffle=False, num_epochs=1) tf_record_reader = tf.TFRecordReader() _, serialized_example_tensor = tf_record_reader.read(filename_queue) # *** MODIFIED prefetch_queue = prefetcher.prefetch({'serialized_example_tensor': serialized_example_tensor}, 100) dequeue = prefetch_queue.dequeue() serialized_example_tensor = dequeue['serialized_example_tensor'] # *** MODIFIED ENDS features = tf.parse_single_example( serialized_example_tensor, features={ standard_fields.TfExampleFields.image_encoded: tf.FixedLenFeature([], tf.string), }) encoded_image = features[standard_fields.TfExampleFields.image_encoded] image_tensor = tf.image.decode_image(encoded_image, channels=3) image_tensor.set_shape([None, None, 3]) image_tensor = tf.expand_dims(image_tensor, 0) # # *** MODIFIED # batch = tf.train.batch( # [serialized_example_tensor, image_tensor], # batch_size=24, # enqueue_many=False, # num_threads=6, # capacity=5 * 24) # return batch[0], batch[1] # # *** MODIFIED ENDS return serialized_example_tensor, image_tensor
def _extract_prediction_tensors(model, create_input_dict_fn, ignore_groundtruth=False): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: tensor_dict: A tensor dictionary with evaluations. """ input_dict = create_input_dict_fn() batch = None if 'batch' in input_dict: batch = input_dict.pop('batch') else: prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() # consistent format for images and videos for key, value in input_dict.iteritems(): input_dict[key] = (value, ) detections = _create_detection_op(model, input_dict, batch) # Print out anaylsis of the model. tf.contrib.tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=tf.contrib.tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) tf.contrib.tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=tf.contrib.tfprof.model_analyzer.FLOAT_OPS_OPTIONS) num_frames = len(input_dict[fields.InputDataFields.image]) ret = [] for i in range(num_frames): original_image = tf.expand_dims( input_dict[fields.InputDataFields.image][i], 0) groundtruth = None if not ignore_groundtruth: groundtruth = { fields.InputDataFields.groundtruth_boxes: input_dict[fields.InputDataFields.groundtruth_boxes][i], fields.InputDataFields.groundtruth_classes: input_dict[fields.InputDataFields.groundtruth_classes][i], } optional_keys = ( fields.InputDataFields.groundtruth_area, fields.InputDataFields.groundtruth_is_crowd, fields.InputDataFields.groundtruth_difficult, fields.InputDataFields.groundtruth_group_of, ) for opt_key in optional_keys: if opt_key in input_dict: groundtruth[opt_key] = input_dict[opt_key][i] if fields.DetectionResultFields.detection_masks in detections: groundtruth[ fields.InputDataFields.groundtruth_instance_masks] = ( input_dict[fields.InputDataFields. groundtruth_instance_masks][i]) detections_frame = { key: tf.expand_dims(value[i], 0) for key, value in detections.iteritems() } source_id = (batch.key[0] if batch is not None else input_dict[fields.InputDataFields.source_id][i]) ret.append( eval_util.result_dict_for_single_example( original_image, source_id, detections_frame, groundtruth, class_agnostic=(fields.DetectionResultFields.detection_classes not in detections), scale_to_absolute=True)) return ret
def _extract_prediction_tensors(model, create_input_dict_fn, ignore_groundtruth=False): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: tensor_dict: A tensor dictionary with evaluations. """ input_dict = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0) preprocessed_image = model.preprocess(tf.to_float(original_image)) prediction_dict = model.predict(preprocessed_image) detections = model.postprocess(prediction_dict) original_image_shape = tf.shape(original_image) absolute_detection_boxlist = box_list_ops.to_absolute_coordinates( box_list.BoxList(tf.squeeze(detections['detection_boxes'], axis=0)), original_image_shape[1], original_image_shape[2]) label_id_offset = 1 tensor_dict = { 'original_image': original_image, 'image_id': input_dict[fields.InputDataFields.source_id], 'detection_boxes': absolute_detection_boxlist.get(), 'detection_scores': tf.squeeze(detections['detection_scores'], axis=0), 'detection_classes': ( tf.squeeze(detections['detection_classes'], axis=0) + label_id_offset), } if 'detection_masks' in detections: detection_masks = tf.squeeze(detections['detection_masks'], axis=0) detection_boxes = tf.squeeze(detections['detection_boxes'], axis=0) # TODO: This should be done in model's postprocess function ideally. detection_masks_reframed = ops.reframe_box_masks_to_image_masks( detection_masks, detection_boxes, original_image_shape[1], original_image_shape[2]) detection_masks_reframed = tf.to_float(tf.greater(detection_masks_reframed, 0.5)) tensor_dict['detection_masks'] = detection_masks_reframed # load groundtruth fields into tensor_dict if not ignore_groundtruth: normalized_gt_boxlist = box_list.BoxList( input_dict[fields.InputDataFields.groundtruth_boxes]) gt_boxlist = box_list_ops.scale(normalized_gt_boxlist, tf.shape(original_image)[1], tf.shape(original_image)[2]) groundtruth_boxes = gt_boxlist.get() groundtruth_classes = input_dict[fields.InputDataFields.groundtruth_classes] tensor_dict['groundtruth_boxes'] = groundtruth_boxes tensor_dict['groundtruth_classes'] = groundtruth_classes tensor_dict['area'] = input_dict[fields.InputDataFields.groundtruth_area] tensor_dict['is_crowd'] = input_dict[ fields.InputDataFields.groundtruth_is_crowd] tensor_dict['difficult'] = input_dict[ fields.InputDataFields.groundtruth_difficult] if 'detection_masks' in tensor_dict: tensor_dict['groundtruth_instance_masks'] = input_dict[ fields.InputDataFields.groundtruth_instance_masks] return tensor_dict
def _extract_prediction_tensors(model, input_k_shot, create_input_dict_fn, ignore_groundtruth=False, use_target_class_in_predictions=False): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: tensor_dict: A tensor dictionary with evaluations. """ tree_k_shot = model._k_shot bag_size = model._bag_size num_classes = model._num_classes num_negative_bags = model._num_negative_bags total_images = num_negative_bags + input_k_shot input_dict, thread = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=3) input_dict = prefetch_queue.dequeue() def _r(t): return tf.reshape(t, [-1,]+t.shape[2:].as_list()) def reorder_for_k2(t): ''' when tree_k_shot is 2, and we have negative bags and input_k_shot is greater than 2, this function reorders the list of inputs in a way that it puts the negative elements after every 2*bag_size elements. Notes: - assumes positive elements are in first bag_size*input_k_shot elements of the input list and the rest of the elements are negative bags i.e., num_negative_bags*bag_size last elements of the list. ''' if num_negative_bags > 0 and tree_k_shot == 2 and input_k_shot != 2: #TDOO: 'implemented, but not tested yet! use embed to check' pos_part = t[:bag_size*input_k_shot] neg_part = t[bag_size*input_k_shot:] assert input_k_shot % 2 == 0, 'input_k_shot should be multiple of 2' l = [pos_part[bag_size*2*i:bag_size*2*(i+1)] + neg_part for i in range(input_k_shot/2)] t = [i for x in l for i in x] return t def _s(t): return reorder_for_k2(tf.split(_r(t), total_images*bag_size)) images_list = _s(input_dict[fields.supportset]) groundtruth_classes_list = _s(input_dict[fields.groundtruth_classes]) original_groundtruth_classes = input_dict[fields.original_groundtruth_classes] original_groundtruth_classes_list = _s(original_groundtruth_classes) one_hot_gt_classes_list = _s(tf.one_hot(tf.to_int32(original_groundtruth_classes), num_classes + 1)) boxes_list = _s(input_dict[fields.groundtruth_boxes]) images = tf.concat(images_list,0) float_images = tf.to_float(images) input_dict[fields.supportset] = float_images preprocessed_images = [model.preprocess(float_image) for float_image in tf.split(float_images, len(images_list))] model.provide_groundtruth(boxes_list, one_hot_gt_classes_list, None) if fields.proposal_objectness in input_dict: model._groundtruth_lists['objectness_score'] = input_dict[fields.proposal_objectness] if use_target_class_in_predictions: target_class = input_dict[fields.groundtruth_target_class] model._groundtruth_lists['target_class'] = target_class[tf.newaxis] preprocessed_image = tf.concat(preprocessed_images, 0) prediction_dict = model.predict(preprocessed_image) detections = model.postprocess(prediction_dict) tensor_dict = dict() tensor_dict.update(model._tree_debug_tensors( include_k2_cross_similarty=False)) #FLAGS.add_mrf tensor_dict['input_k_shot'] = tf.constant(input_k_shot) tensor_dict['num_negative_bags'] = tf.constant(num_negative_bags) label_id_offset = 1 # Convert to the absolute coordinates for key,val in tensor_dict.items(): if 'multiclass' in key: tensor_dict.pop(key) if isinstance(val, dict): for mkey in val: if 'classes' in mkey: val[mkey] = val[mkey] + label_id_offset groundtruth_classes = tf.concat(groundtruth_classes_list[:input_k_shot*bag_size], 0) groundtruth_classes = tf.reshape(groundtruth_classes, [input_k_shot, bag_size]) original_groundtruth_classes = tf.concat(original_groundtruth_classes_list[:input_k_shot*bag_size],0) original_groundtruth_classes = tf.reshape(original_groundtruth_classes, [input_k_shot, bag_size]) if not fields.original_images in input_dict: raise Exception('Please add this tensors to mini-imagenet model, guzu.' \ ' It is added but ridam passe kallat') tensor_dict['original_image'] = input_dict[fields.original_images] tensor_dict['filenames'] = input_dict[fields.filename] ndict = dict(boxes=[], classes=[]) ndict['target_class'] = input_dict[fields.groundtruth_target_class] for i in range(input_k_shot): ndict['boxes'].append( input_dict[fields.groundtruth_image_boxes+'_{}'.format(i)]) ndict['classes'].append(input_dict[ fields.groundtruth_image_classes+'_{}'.format(i)]) #ndict['fea_groundtruth_classes'] = groundtruth_classes #ndict['fea_original_groundtruth_classes'] = original_groundtruth_classes #ndict['fea_boxes'] = input_dict[fields.groundtruth_boxes] tensor_dict['groundtruth'] = ndict #### #tensor_dict['meta_info']['extra_tensors'] = util.get_extra_tensors_dict() #### return tensor_dict, thread
def _extract_anchros_and_losses(model, create_input_dict_fn, quantize=False, ignore_groundtruth=False): """Constructs tensorflow detection graph and returns output tensors. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: prediction_groundtruth_dict: A dictionary with postprocessed tensors (keyed by standard_fields.DetectionResultsFields) and optional groundtruth tensors (keyed by standard_fields.InputDataFields). losses_dict: A dictionary containing detection losses. This is empty when ignore_groundtruth is true. """ input_dict = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0) preprocessed_image, true_image_shapes = model.preprocess( tf.to_float(original_image)) prediction_dict = model.predict(preprocessed_image, true_image_shapes) detections = model.postprocess(prediction_dict, true_image_shapes) if quantize: from tensorflow.contrib.quantize import experimental_create_eval_graph experimental_create_eval_graph() # g = tf.get_default_graph() # print(g.get_operations()) groundtruth = None losses_dict = {} if not ignore_groundtruth: groundtruth = { fields.InputDataFields.groundtruth_boxes: input_dict[fields.InputDataFields.groundtruth_boxes], fields.InputDataFields.groundtruth_classes: input_dict[fields.InputDataFields.groundtruth_classes], fields.InputDataFields.groundtruth_area: input_dict[fields.InputDataFields.groundtruth_area], fields.InputDataFields.groundtruth_is_crowd: input_dict[fields.InputDataFields.groundtruth_is_crowd], fields.InputDataFields.groundtruth_difficult: input_dict[fields.InputDataFields.groundtruth_difficult] } if fields.InputDataFields.groundtruth_group_of in input_dict: groundtruth[fields.InputDataFields.groundtruth_group_of] = ( input_dict[fields.InputDataFields.groundtruth_group_of]) if fields.DetectionResultFields.detection_masks in detections: groundtruth[fields.InputDataFields.groundtruth_instance_masks] = ( input_dict[fields.InputDataFields.groundtruth_instance_masks]) label_id_offset = 1 model.provide_groundtruth( [input_dict[fields.InputDataFields.groundtruth_boxes]], [ tf.one_hot( input_dict[fields.InputDataFields.groundtruth_classes] - label_id_offset, depth=model.num_classes) ]) losses_dict.update(model.loss(prediction_dict, true_image_shapes)) result_dict = eval_util.result_dict_for_single_example( original_image, input_dict[fields.InputDataFields.source_id], detections, groundtruth, class_agnostic=(fields.DetectionResultFields.detection_classes not in detections), scale_to_absolute=True) # model.preprocess result_dict['preprocessed_image'] = preprocessed_image result_dict['true_image_shapes'] = true_image_shapes # model.predict result_dict['class_predictions_with_background'] = ( prediction_dict['class_predictions_with_background']) result_dict['feature_maps'] = prediction_dict['feature_maps'] result_dict['preprocessed_inputs'] = prediction_dict['preprocessed_inputs'] result_dict['box_encodings'] = prediction_dict['box_encodings'] result_dict['anchors'] = prediction_dict['anchors'] # model.detections DEBUG ONLY # result_dict['detection_boxes_all'] = detections['detection_boxes_all'] # print('YMK in _extract_anchros_and_losses') # import ipdb # ipdb.set_trace() return result_dict, losses_dict
def _extract_prediction_tensors(model, create_input_dict_fn, ignore_groundtruth=False): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: tensor_dict: A tensor dictionary with evaluations. """ input_dict = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0) preprocessed_image = model.preprocess(tf.to_float(original_image)) prediction_dict = model.predict(preprocessed_image) detections = model.postprocess(prediction_dict) original_image_shape = tf.shape(original_image) absolute_detection_boxlist = box_list_ops.to_absolute_coordinates( box_list.BoxList(tf.squeeze(detections['detection_boxes'], axis=0)), original_image_shape[1], original_image_shape[2]) label_id_offset = 1 tensor_dict = { 'original_image': original_image, 'image_id': input_dict[fields.InputDataFields.source_id], 'detection_boxes': absolute_detection_boxlist.get(), 'detection_scores': tf.squeeze(detections['detection_scores'], axis=0), 'detection_classes': ( tf.squeeze(detections['detection_classes'], axis=0) + label_id_offset), } if 'detection_masks' in detections: detection_masks = tf.squeeze(detections['detection_masks'], axis=0) detection_boxes = tf.squeeze(detections['detection_boxes'], axis=0) # TODO: This should be done in model's postprocess function ideally. detection_masks_reframed = ops.reframe_box_masks_to_image_masks( detection_masks, detection_boxes, original_image_shape[1], original_image_shape[2]) detection_masks_reframed = tf.to_float(tf.greater(detection_masks_reframed, 0.5)) tensor_dict['detection_masks'] = detection_masks_reframed # load groundtruth fields into tensor_dict if not ignore_groundtruth: normalized_gt_boxlist = box_list.BoxList( input_dict[fields.InputDataFields.groundtruth_boxes]) gt_boxlist = box_list_ops.scale(normalized_gt_boxlist, tf.shape(original_image)[1], tf.shape(original_image)[2]) groundtruth_boxes = gt_boxlist.get() groundtruth_classes = input_dict[fields.InputDataFields.groundtruth_classes] tensor_dict['groundtruth_boxes'] = groundtruth_boxes tensor_dict['groundtruth_classes'] = groundtruth_classes tensor_dict['area'] = input_dict[fields.InputDataFields.groundtruth_area] tensor_dict['is_crowd'] = input_dict[ fields.InputDataFields.groundtruth_is_crowd] tensor_dict['difficult'] = input_dict[ fields.InputDataFields.groundtruth_difficult] if 'detection_masks' in tensor_dict: tensor_dict['groundtruth_instance_masks'] = input_dict[ fields.InputDataFields.groundtruth_instance_masks] return tensor_dict
for key, tensor in tensor_dict.iteritems()}) ======= runtime_shapes = collections.OrderedDict( {(key + rt_shape_str): tf.shape(tensor) for key, tensor in tensor_dict.iteritems()}) >>>>>>> LOCAL all_tensors = tensor_dict all_tensors.update(runtime_shapes) batched_tensors = tf.train.batch( all_tensors, capacity=batch_queue_capacity, batch_size=batch_size, dynamic_pad=True, num_threads=num_batch_queue_threads) self._queue = prefetcher.prefetch(batched_tensors, prefetch_queue_capacity) self._static_shapes = static_shapes self._batch_size = batch_size def dequeue(self): """Dequeues a batch of tensor_dict from the BatchQueue. TODO: use allow_smaller_final_batch to allow running over the whole eval set Returns: A list of tensor_dicts of the requested batch_size. """ batched_tensors = self._queue.dequeue() # Separate input tensors from tensors containing their runtime shapes. tensors = {} shapes = {}
def _extract_prediction_tensors(model, create_input_dict_fn, ignore_groundtruth=False): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: tensor_dict: A tensor dictionary with evaluations. """ input_dict = create_input_dict_fn() batch = None if 'batch' in input_dict: batch = input_dict.pop('batch') else: prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() # consistent format for images and videos for key, value in input_dict.iteritems(): input_dict[key] = (value,) detections = _create_detection_op(model, input_dict, batch) # Print out anaylsis of the model. tf.contrib.tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=tf.contrib.tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) tf.contrib.tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=tf.contrib.tfprof.model_analyzer.FLOAT_OPS_OPTIONS) num_frames = len(input_dict[fields.InputDataFields.image]) ret = [] for i in range(num_frames): original_image = tf.expand_dims(input_dict[fields.InputDataFields.image][i], 0) groundtruth = None if not ignore_groundtruth: groundtruth = { fields.InputDataFields.groundtruth_boxes: input_dict[fields.InputDataFields.groundtruth_boxes][i], fields.InputDataFields.groundtruth_classes: input_dict[fields.InputDataFields.groundtruth_classes][i], } optional_keys = ( fields.InputDataFields.groundtruth_area, fields.InputDataFields.groundtruth_is_crowd, fields.InputDataFields.groundtruth_difficult, fields.InputDataFields.groundtruth_group_of, ) for opt_key in optional_keys: if opt_key in input_dict: groundtruth[opt_key] = input_dict[opt_key][i] if fields.DetectionResultFields.detection_masks in detections: groundtruth[fields.InputDataFields.groundtruth_instance_masks] = ( input_dict[fields.InputDataFields.groundtruth_instance_masks][i]) detections_frame = { key: tf.expand_dims(value[i], 0) for key, value in detections.iteritems() } source_id = ( batch.key[0] if batch is not None else input_dict[fields.InputDataFields.source_id][i]) ret.append( eval_util.result_dict_for_single_example( original_image, source_id, detections_frame, groundtruth, class_agnostic=(fields.DetectionResultFields.detection_classes not in detections), scale_to_absolute=True)) return ret
def _extract_predictions_and_losses(model, create_input_dict_fn, ignore_groundtruth=False): """Constructs tensorflow detection graph and returns output tensors. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: prediction_groundtruth_dict: A dictionary with postprocessed tensors (keyed by standard_fields.DetectionResultsFields) and optional groundtruth tensors (keyed by standard_fields.InputDataFields). losses_dict: A dictionary containing detection losses. This is empty when ignore_groundtruth is true. """ input_dict = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0) preprocessed_image, true_image_shapes = model.preprocess( tf.to_float(original_image)) prediction_dict = model.predict(preprocessed_image, true_image_shapes) detections = model.postprocess(prediction_dict, true_image_shapes) groundtruth = None losses_dict = {} if not ignore_groundtruth: groundtruth = { fields.InputDataFields.groundtruth_boxes: input_dict[fields.InputDataFields.groundtruth_boxes], fields.InputDataFields.groundtruth_classes: input_dict[fields.InputDataFields.groundtruth_classes], fields.InputDataFields.groundtruth_area: input_dict[fields.InputDataFields.groundtruth_area], fields.InputDataFields.groundtruth_is_crowd: input_dict[fields.InputDataFields.groundtruth_is_crowd], fields.InputDataFields.groundtruth_difficult: input_dict[fields.InputDataFields.groundtruth_difficult] } if fields.InputDataFields.groundtruth_group_of in input_dict: groundtruth[fields.InputDataFields.groundtruth_group_of] = ( input_dict[fields.InputDataFields.groundtruth_group_of]) groundtruth_masks_list = None if fields.DetectionResultFields.detection_masks in detections: groundtruth[fields.InputDataFields.groundtruth_instance_masks] = ( input_dict[fields.InputDataFields.groundtruth_instance_masks]) groundtruth_masks_list = [ input_dict[fields.InputDataFields.groundtruth_instance_masks]] groundtruth_keypoints_list = None if fields.DetectionResultFields.detection_keypoints in detections: groundtruth[fields.InputDataFields.groundtruth_keypoints] = ( input_dict[fields.InputDataFields.groundtruth_keypoints]) groundtruth_keypoints_list = [ input_dict[fields.InputDataFields.groundtruth_keypoints]] label_id_offset = 1 model.provide_groundtruth( [input_dict[fields.InputDataFields.groundtruth_boxes]], [tf.one_hot(input_dict[fields.InputDataFields.groundtruth_classes] - label_id_offset, depth=model.num_classes)], groundtruth_masks_list, groundtruth_keypoints_list) losses_dict.update(model.loss(prediction_dict, true_image_shapes)) result_dict = eval_util.result_dict_for_single_example( original_image, input_dict[fields.InputDataFields.source_id], detections, groundtruth, class_agnostic=( fields.DetectionResultFields.detection_classes not in detections), scale_to_absolute=True) return result_dict, losses_dict
def _extract_prediction_tensors(model, create_input_dict_fn, ignore_groundtruth=False, provide_groundtruth_to_model=False, calc_loss=False): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. provide_groundtruth_to_model: whether to use model.provide_groundtruth() Returns: tensor_dict: A tensor dictionary with evaluations. """ mtl = model._mtl input_dict = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() if calc_loss or mtl.window or mtl.edgemask: provide_groundtruth_to_model = True # Get groundtruth information if provide_groundtruth_to_model: (_, groundtruth_boxes_list, groundtruth_ignore_list, groundtruth_classes_list, groundtruth_masks_list, _, window_boxes_list, window_classes_list, groundtruth_closeness_list, groundtruth_edgemask_list) = _get_inputs([input_dict], model.num_classes, with_filename=False) if any(mask is None for mask in groundtruth_masks_list): groundtruth_masks_list = None model.provide_groundtruth(groundtruth_boxes_list, groundtruth_classes_list, groundtruth_closeness_list, groundtruth_ignore_list, groundtruth_masks_list) model.provide_window(window_boxes_list, window_classes_list) model.provide_edgemask(groundtruth_edgemask_list) original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0) preprocessed_image = model.preprocess(tf.to_float(original_image)) prediction_dict = model.predict(preprocessed_image) if mtl.window: prediction_dict = model.predict_with_window(prediction_dict) if mtl.edgemask: prediction_dict = model.predict_edgemask(prediction_dict) if mtl.refine: prediction_dict = model.predict_with_mtl_results(prediction_dict) detections = model.postprocess(prediction_dict) original_image_shape = tf.shape(original_image) absolute_detection_boxlist = box_list_ops.to_absolute_coordinates( box_list.BoxList(tf.squeeze(detections['detection_boxes'], axis=0)), original_image_shape[1], original_image_shape[2]) label_id_offset = 1 tensor_dict = { 'original_image': original_image, 'image_id': input_dict[fields.InputDataFields.source_id], 'detection_boxes': absolute_detection_boxlist.get(), 'detection_scores': tf.squeeze(detections['detection_scores'], axis=0), 'detection_classes': (tf.squeeze(detections['detection_classes'], axis=0) + label_id_offset), } if 'detection_thresholds' in detections: tensor_dict['detection_thresholds'] = \ tf.squeeze(detections['detection_thresholds'], axis=0) if 'detection_masks' in detections: detection_masks = tf.squeeze(detections['detection_masks'], axis=0) detection_boxes = tf.squeeze(detections['detection_boxes'], axis=0) # TODO: This should be done in model's postprocess function ideally. detection_masks_reframed = ops.reframe_box_masks_to_image_masks( detection_masks, detection_boxes, original_image_shape[1], original_image_shape[2]) detection_masks_reframed = tf.to_float( tf.greater(detection_masks_reframed, 0.5)) tensor_dict['detection_masks'] = detection_masks_reframed # load groundtruth fields into tensor_dict if not ignore_groundtruth: normalized_gt_boxlist = box_list.BoxList( input_dict[fields.InputDataFields.groundtruth_boxes]) gt_boxlist = box_list_ops.scale(normalized_gt_boxlist, tf.shape(original_image)[1], tf.shape(original_image)[2]) groundtruth_boxes = gt_boxlist.get() groundtruth_classes = input_dict[ fields.InputDataFields.groundtruth_classes] tensor_dict['groundtruth_boxes'] = groundtruth_boxes tensor_dict['groundtruth_classes'] = groundtruth_classes tensor_dict['area'] = input_dict[ fields.InputDataFields.groundtruth_area] tensor_dict['difficult'] = input_dict[ fields.InputDataFields.groundtruth_difficult] if 'detection_masks' in tensor_dict: tensor_dict['groundtruth_instance_masks'] = input_dict[ fields.InputDataFields.groundtruth_instance_masks] # Subset annotations if fields.InputDataFields.groundtruth_subset in input_dict: tensor_dict['groundtruth_subset'] \ = input_dict[fields.InputDataFields.groundtruth_subset] if calc_loss: losses_dict = model.loss(prediction_dict) for loss_name, loss_tensor in losses_dict.iteritems(): loss_tensor = tf.check_numerics(loss_tensor, '%s is inf or nan.' % loss_name, name='Loss/' + loss_name) tensor_dict['Loss/' + loss_name] = loss_tensor # mtl groundtruth if mtl.window: tensor_dict['window_classes_gt'] = input_dict[ fields.InputDataFields.window_classes] tensor_dict['window_classes_dt'] = prediction_dict[ 'window_class_predictions'] if mtl.closeness: tensor_dict['closeness_gt'] = input_dict[ fields.InputDataFields.groundtruth_closeness] tensor_dict['closeness_dt'] = prediction_dict['closeness_predictions'] if mtl.edgemask: tensor_dict['edgemask_gt'] = input_dict[ fields.InputDataFields.groundtruth_edgemask_masks] tensor_dict['edgemask_dt'] = prediction_dict['edgemask_predictions'] return tensor_dict
def _extract_prediction_tensors(model, create_input_dict_fn, ignore_groundtruth=False): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: tensor_dict: A tensor dictionary with evaluations. """ k_shot = model._k_shot input_dict, thread = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=100) input_dict = prefetch_queue.dequeue() images = input_dict[fields.CoLocInputDataFields.supportset] images_list = [image for image in tf.split(images, k_shot)] float_images = tf.to_float(images) input_dict[fields.CoLocInputDataFields.supportset] = float_images preprocessed_images = [ model.preprocess(float_image) for float_image in tf.split(float_images, k_shot) ] preprocessed_image = tf.concat(preprocessed_images, 0) prediction_dict = model.predict(preprocessed_image) detections = model.postprocess(prediction_dict) original_image_shape = tf.shape(images) def _absolute_boxes(normalized_boxes): absolute_detection_boxlist_list = [ box_list_ops.to_absolute_coordinates( box_list.BoxList(tf.squeeze(k, axis=0)), original_image_shape[1], original_image_shape[2]) for k in tf.split(normalized_boxes, k_shot) ] return tf.stack([db.get() for db in absolute_detection_boxlist_list]) tensor_dict = {'original_image': images} if detections.has_key('rpn_detection_boxes'): tensor_dict['rpn'] = { 'boxes': detections['rpn_detection_boxes'], 'scores': detections['rpn_detection_scores'], 'classes': detections['rpn_detection_classes'], 'class_agnostic': tf.constant(True) } if detections.has_key('detection_boxes'): tensor_dict['detection'] = { 'boxes': detections['detection_boxes'], 'scores': detections['detection_scores'], 'classes': detections['detection_classes'], 'class_agnostic': tf.constant(False) } label_id_offset = 1 if hasattr(model, '_tree_debug_tensors'): tensor_dict.update(model._tree_debug_tensors()) # Convert to the absolute coordinates for key, val in tensor_dict.items(): if isinstance(val, dict): for mkey in val: if 'boxes' in mkey: val[mkey] = _absolute_boxes(val[mkey]) if 'classes' in mkey: val[mkey] = val[mkey] + label_id_offset if not ignore_groundtruth: groundtruth_boxes_list = [] groundtruth_classes_list = [] groundtruth_target_class = input_dict[ fields.CoLocInputDataFields.groundtruth_target_class] for k in xrange(k_shot): normalized_gt_boxlist = box_list.BoxList( input_dict[fields.CoLocInputDataFields.groundtruth_boxes + '_{}'.format(k)]) gt_boxlist = box_list_ops.scale(normalized_gt_boxlist, original_image_shape[1], original_image_shape[2]) groundtruth_boxes = gt_boxlist.get() groundtruth_classes = input_dict[ fields.CoLocInputDataFields.groundtruth_classes + '_{}'.format(k)] groundtruth_boxes_list.append(groundtruth_boxes) groundtruth_classes_list.append(groundtruth_classes) ndict = dict() ndict['boxes'] = groundtruth_boxes_list ndict['classes'] = groundtruth_classes_list ndict['target_class'] = groundtruth_target_class tensor_dict['groundtruth'] = ndict return tensor_dict, thread
def build_input(tfrecord_paths): """Builds the graph's input. Args: tfrecord_paths: List of paths to the input TFRecords Returns: serialized_example_tensor: The next serialized example. String scalar Tensor image_tensor: The decoded image of the example. Uint8 tensor, shape=[1, None, None,3] """ filename_queue = tf.train.string_input_producer( tfrecord_paths, shuffle=False, num_epochs=1) tf_record_reader = tf.TFRecordReader() _, serialized_example_tensor = tf_record_reader.read(filename_queue) # *** MODIFIED prefetch_queue = prefetcher.prefetch({'serialized_example_tensor': serialized_example_tensor}, 100) dequeue = prefetch_queue.dequeue() serialized_example_tensor = dequeue['serialized_example_tensor'] # *** MODIFIED ENDS features = tf.parse_single_example( serialized_example_tensor, features={ standard_fields.TfExampleFields.image_encoded: tf.FixedLenFeature([], tf.string), }) encoded_image = features[standard_fields.TfExampleFields.image_encoded] image_tensor = tf.image.decode_image(encoded_image, channels=3) image_tensor.set_shape([None, None, 3]) # image_tensor = tf.expand_dims(image_tensor, 0) # # *** MODIFIED # batch = tf.train.batch( # [serialized_example_tensor, image_tensor], # batch_size=24, # enqueue_many=False, # num_threads=6, # capacity=5 * 24) # return batch[0], batch[1] image_resizer_text_proto = """ keep_aspect_ratio_resizer { min_dimension: 800 max_dimension: 1365 } """ image_resizer_config = image_resizer_pb2.ImageResizer() text_format.Merge(image_resizer_text_proto, image_resizer_config) image_resizer_fn = image_resizer_builder.build(image_resizer_config) resized_image_tensor, _ = image_resizer_fn(image_tensor) # resized_image_tensor = tf.image.convert_image_dtype(resized_image_tensor, dtype=tf.uint8) resized_image_tensor = tf.cast(resized_image_tensor, dtype=tf.uint8) resized_image_tensor = tf.expand_dims(resized_image_tensor, 0) # # *** MODIFIED ENDS return serialized_example_tensor, resized_image_tensor#image_tensor
def _extract_prediction_tensors(model, create_input_dict_fn, ignore_groundtruth=False): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: tensor_dict: A tensor dictionary with evaluations. """ input_dict = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) # TODO input_dict = prefetch_queue.dequeue() original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0) next_image = input_dict.get(fields.InputDataFields.next_image) image_input = tf.to_float(original_image) if next_image is not None: next_image = tf.to_float(next_image) image_input = tf.concat( [image_input, tf.expand_dims(next_image, 0)], 3) depth = input_dict.get(fields.InputDataFields.groundtruth_depth) next_depth = input_dict.get( fields.InputDataFields.groundtruth_next_depth) image_input.set_shape([1, None, None, 6]) if depth is not None and next_depth is not None: camera_intrinsics = input_dict[ fields.InputDataFields.camera_intrinsics] coords = motion_util.get_3D_coords(tf.expand_dims(depth, 0), camera_intrinsics) next_coords = motion_util.get_3D_coords( tf.expand_dims(next_depth, 0), camera_intrinsics) image_input = tf.concat([image_input, coords, next_coords], 3) image_input.set_shape([1, None, None, 12]) preprocessed_image = model.preprocess(image_input) prediction_dict = model.predict(preprocessed_image) detections = model.postprocess(prediction_dict) original_image_shape = tf.shape(original_image) absolute_detection_boxlist = box_list_ops.to_absolute_coordinates( box_list.BoxList(tf.squeeze(detections['detection_boxes'], axis=0)), original_image_shape[1], original_image_shape[2]) label_id_offset = 1 tensor_dict = { 'original_image': original_image, 'image_id': input_dict[fields.InputDataFields.source_id], 'detection_boxes': absolute_detection_boxlist.get(), 'detection_scores': tf.squeeze(detections['detection_scores'], axis=0), 'detection_classes': (tf.squeeze(detections['detection_classes'], axis=0) + label_id_offset), } if 'detection_masks' in detections: detection_masks = tf.squeeze(detections['detection_masks'], axis=0) detection_boxes = tf.squeeze(detections['detection_boxes'], axis=0) # TODO: This should be done in model's postprocess function ideally. detection_masks_reframed = ops.reframe_box_masks_to_image_masks( detection_masks, detection_boxes, original_image_shape[1], original_image_shape[2]) detection_masks_reframed = tf.to_float( tf.greater(detection_masks_reframed, 0.5)) tensor_dict['detection_masks'] = detection_masks_reframed if 'detection_motions' in detections: detection_motions = tf.squeeze(detections['detection_motions'], axis=0) detection_motions_with_matrices = ( motion_util.postprocess_detection_motions(detection_motions, keep_logits=False)) tensor_dict['detection_motions'] = detection_motions_with_matrices if 'camera_motion' in detections: camera_motion_with_matrices = tf.squeeze( motion_util.postprocess_camera_motion(detections['camera_motion']), axis=0) tensor_dict['camera_motion'] = camera_motion_with_matrices tensor_dict['groundtruth_camera_motion'] = input_dict[ fields.InputDataFields.groundtruth_camera_motion] # load groundtruth fields into tensor_dict if not ignore_groundtruth: normalized_gt_boxlist = box_list.BoxList( input_dict[fields.InputDataFields.groundtruth_boxes]) gt_boxlist = box_list_ops.scale(normalized_gt_boxlist, tf.shape(original_image)[1], tf.shape(original_image)[2]) groundtruth_boxes = gt_boxlist.get() groundtruth_classes = input_dict[ fields.InputDataFields.groundtruth_classes] tensor_dict['groundtruth_boxes'] = groundtruth_boxes tensor_dict['groundtruth_classes'] = groundtruth_classes tensor_dict['area'] = input_dict[ fields.InputDataFields.groundtruth_area] tensor_dict['is_crowd'] = input_dict[ fields.InputDataFields.groundtruth_is_crowd] tensor_dict['difficult'] = input_dict[ fields.InputDataFields.groundtruth_difficult] if 'detection_masks' in tensor_dict: tensor_dict['groundtruth_instance_masks'] = input_dict[ fields.InputDataFields.groundtruth_instance_masks] if 'detection_motions' in tensor_dict: tensor_dict['groundtruth_camera_motion'] = input_dict[ fields.InputDataFields.groundtruth_camera_motion] tensor_dict['groundtruth_instance_motions'] = input_dict[ fields.InputDataFields.groundtruth_instance_motions] tensor_dict['camera_intrinsics'] = input_dict[ fields.InputDataFields.camera_intrinsics] if fields.InputDataFields.groundtruth_flow in input_dict: tensor_dict['groundtruth_flow'] = input_dict[ fields.InputDataFields.groundtruth_flow] if not 'depth' in tensor_dict: tensor_dict['depth'] = input_dict[ fields.InputDataFields.groundtruth_depth] else: tensor_dict['groundtruth_depth'] = input_dict[ fields.InputDataFields.groundtruth_depth] return tensor_dict
def _extract_predictions_and_losses(model, create_input_dict_fn, ignore_groundtruth=False): """Constructs tensorflow detection graph and returns output tensors. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. Returns: prediction_groundtruth_dict: A dictionary with postprocessed tensors (keyed by standard_fields.DetectionResultsFields) and optional groundtruth tensors (keyed by standard_fields.InputDataFields). losses_dict: A dictionary containing detection losses. This is empty when ignore_groundtruth is true. """ input_dict = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=500) input_dict = prefetch_queue.dequeue() original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0) preprocessed_image, true_image_shapes = model.preprocess( tf.cast(original_image, dtype=tf.float32)) prediction_dict = model.predict(preprocessed_image, true_image_shapes) detections = model.postprocess(prediction_dict, true_image_shapes) groundtruth = None losses_dict = {} if not ignore_groundtruth: groundtruth = { fields.InputDataFields.groundtruth_boxes: input_dict[fields.InputDataFields.groundtruth_boxes], fields.InputDataFields.groundtruth_classes: input_dict[fields.InputDataFields.groundtruth_classes], fields.InputDataFields.groundtruth_area: input_dict[fields.InputDataFields.groundtruth_area], fields.InputDataFields.groundtruth_is_crowd: input_dict[fields.InputDataFields.groundtruth_is_crowd], fields.InputDataFields.groundtruth_difficult: input_dict[fields.InputDataFields.groundtruth_difficult] } if fields.InputDataFields.groundtruth_group_of in input_dict: groundtruth[fields.InputDataFields.groundtruth_group_of] = ( input_dict[fields.InputDataFields.groundtruth_group_of]) groundtruth_masks_list = None if fields.DetectionResultFields.detection_masks in detections: groundtruth[fields.InputDataFields.groundtruth_instance_masks] = ( input_dict[fields.InputDataFields.groundtruth_instance_masks]) groundtruth_masks_list = [ input_dict[fields.InputDataFields.groundtruth_instance_masks] ] groundtruth_keypoints_list = None if fields.DetectionResultFields.detection_keypoints in detections: groundtruth[fields.InputDataFields.groundtruth_keypoints] = ( input_dict[fields.InputDataFields.groundtruth_keypoints]) groundtruth_keypoints_list = [ input_dict[fields.InputDataFields.groundtruth_keypoints] ] label_id_offset = 1 model.provide_groundtruth( [input_dict[fields.InputDataFields.groundtruth_boxes]], [ tf.one_hot( input_dict[fields.InputDataFields.groundtruth_classes] - label_id_offset, depth=model.num_classes) ], groundtruth_masks_list=groundtruth_masks_list, groundtruth_keypoints_list=groundtruth_keypoints_list) losses_dict.update(model.loss(prediction_dict, true_image_shapes)) result_dict = eval_util.result_dict_for_single_example( original_image, input_dict[fields.InputDataFields.source_id], detections, groundtruth, class_agnostic=(fields.DetectionResultFields.detection_classes not in detections), scale_to_absolute=True) return result_dict, losses_dict
def _extract_prediction_tensors(model, input_k_shot, create_input_dict_fn, ignore_groundtruth=False, use_target_class_in_predictions=False, bcd_outside_images=0): """Restores the model in a tensorflow session. Args: model: model to perform predictions with. create_input_dict_fn: function to create input tensor dictionaries. ignore_groundtruth: whether groundtruth should be ignored. bcd_outside_images: maximum number of images outside the subproblem. We expect to get one feature from each bag outside the subproblem. Returns: tensor_dict: A tensor dictionary with evaluations. """ tree_k_shot = model._k_shot bag_size = model._bag_size num_classes = model._num_classes num_negative_bags = model._num_negative_bags total_images = num_negative_bags + input_k_shot input_dict, thread = create_input_dict_fn() prefetch_queue = prefetcher.prefetch(input_dict, capacity=10) input_dict = prefetch_queue.dequeue() extra_dict = {} reduced_dict = {} ## Append additional variables for the outside bag features num_bcd_bags = int(np.ceil(bcd_outside_images / float(bag_size))) bcd_bags = None bcd_reload_op = None field_bcd_supportset = 'bcd_supportset' if bcd_outside_images > 0: assert (num_negative_bags == 0) assert ('dense' in FLAGS.mrf_type) shape = [num_bcd_bags ] + input_dict[fields.supportset].shape.as_list()[1:] #bcd_supportset = tf.placeholder(tf.float32, shape=shape, #name=field_bcd_supportset) bcd_supportset = tf.constant(0.0, dtype=tf.float32, shape=shape, name=field_bcd_supportset) bcd_supportset_var, bcd_reload_op = wrap_with_variable( bcd_supportset, 0) input_dict[field_bcd_supportset] = bcd_supportset_var ## for k, v in input_dict.iteritems(): #if 'groundtruth_image' in k or 'original_images' in #k or 'target_class' in k or 'filename' in k: if ('groundtruth_image' in k or 'original_images' in k or 'filename' in k or field_bcd_supportset in k): extra_dict[k] = v else: ####### #if k == fields.supportset: #v = tf.zeros_like(v) #embed() ##### reduced_dict[k] = v #reduced_dict.pop('filename') input_dict, ops = wrap_dict(reduced_dict) input_dict.update(extra_dict) reload_op = tf.group(*ops) def _r(t): return tf.reshape(t, [ -1, ] + t.shape[2:].as_list()) def reorder_for_k2(t): ''' when tree_k_shot is 2, and we have negative bags and input_k_shot is greater than 2, this function reorders the list of inputs in a way that it puts the negative elements after every 2*bag_size elements. Notes: - assumes positive elements are in first bag_size*input_k_shot elements of the input list and the rest of the elements are negative bags i.e., num_negative_bags*bag_size last elements of the list. ''' if num_negative_bags > 0 and tree_k_shot == 2 and input_k_shot != 2: pos_part = t[:bag_size * input_k_shot] neg_part = t[bag_size * input_k_shot:] assert input_k_shot % 2 == 0, 'input_k_shot should be multiple of 2' l = [ pos_part[bag_size * 2 * i:bag_size * 2 * (i + 1)] + neg_part for i in range(input_k_shot / 2) ] t = [i for x in l for i in x] return t def _s(t): return reorder_for_k2(tf.split(_r(t), total_images * bag_size)) images_list = _s(input_dict[fields.supportset]) groundtruth_classes_list = _s(input_dict[fields.groundtruth_classes]) original_groundtruth_classes = input_dict[ fields.original_groundtruth_classes] original_groundtruth_classes_list = _s(original_groundtruth_classes) one_hot_gt_classes_list = _s( tf.one_hot(tf.to_int32(original_groundtruth_classes), num_classes + 1)) boxes_list = _s(input_dict[fields.groundtruth_boxes]) proposal_objectness_list = input_dict.get(fields.proposal_objectness, None) if proposal_objectness_list is not None: proposal_objectness_list = _s(proposal_objectness_list) if use_target_class_in_predictions: target_class = input_dict[fields.groundtruth_target_class] model._groundtruth_lists['target_class'] = target_class[tf.newaxis] bcd_bags = None if field_bcd_supportset in input_dict: bcd_bags = tf.split(_r(input_dict[field_bcd_supportset]), num_bcd_bags * bag_size) if FLAGS.add_mrf: lists = [ images_list, groundtruth_classes_list, original_groundtruth_classes_list, one_hot_gt_classes_list, boxes_list ] if proposal_objectness_list is not None: lists.append(proposal_objectness_list) extend_for_mrf(lists, bcd_bags, input_k_shot, bag_size, num_negative_bags) if is_multipart(input_k_shot + num_bcd_bags, bag_size): problem_size = ( 2 + num_negative_bags ) * bag_size ###TODO for K1 and BCD: add min(2, input_k_shot) total_problems = len(images_list) / problem_size max_batch = get_max_batch(total_problems, bag_size) assert total_problems % max_batch == 0, 'problems should be divisible by max_batch' batch_indices = range(total_problems // max_batch) l = max_batch * problem_size #start and end indices se = [(l * i, l * (i + 1)) for i in batch_indices] #if total_problems % max_batch != 0: # se += [(se[-1][1], len(images_list))] np_ranges = np.array(se) tf_ranges = tf.constant(np_ranges) tensor_dict = get_tensor_dict(model, images_list, boxes_list, one_hot_gt_classes_list, proposal_objectness_list, tf_ranges, l, bag_size) gt_dict = {} gt_dict['num_sel'] = len(se) add_groundtruth(gt_dict, input_dict, input_k_shot, bag_size, num_negative_bags, groundtruth_classes_list, original_groundtruth_classes_list) tensor_dict['reload_op'] = reload_op if bcd_reload_op is not None: tensor_dict['bcd_reload_op'] = bcd_reload_op tensor_dict['bcd_supportset_placeholder'] = bcd_supportset tensor_dict[fields.supportset] = input_dict[fields.supportset] tensor_dict['target_class'] = input_dict[ fields.groundtruth_target_class] return [tensor_dict, gt_dict], thread tf_ranges = tf.constant([(0, len(images_list))]) tensor_dict = get_tensor_dict(model, images_list, boxes_list, one_hot_gt_classes_list, proposal_objectness_list, tf_ranges, len(images_list), bag_size) gt_dict = {} gt_dict['num_sel'] = 1 add_groundtruth(gt_dict, input_dict, input_k_shot, bag_size, num_negative_bags, groundtruth_classes_list, original_groundtruth_classes_list) tensor_dict['reload_op'] = reload_op if bcd_reload_op is not None: tensor_dict['bcd_reload_op'] = bcd_reload_op tensor_dict['bcd_supportset_placeholder'] = bcd_supportset tensor_dict[fields.supportset] = input_dict[fields.supportset] tensor_dict['target_class'] = input_dict[fields.groundtruth_target_class] return [tensor_dict, gt_dict], thread