def post_processing_mc_dropout_ensembles(self, input_im):
"""
This function produces results using multiple runs of MC dropout, through fusion before or after
the non-maximum suppression step.
Args:
input_im (list): an input im list generated from dataset handler.
Returns:
result (instances): object instances
"""
if self.cfg.PROBABILISTIC_INFERENCE.ENSEMBLES_DROPOUT.BOX_MERGE_MODE == 'pre_nms':
return self.post_processing_standard_nms(input_im)
else:
outputs_dict = self.model(
input_im,
return_anchorwise_output=False,
num_mc_dropout_runs=self.num_mc_dropout_runs)
n_fms = len(self.model.in_features)
outputs_list = [{
key: value[i * n_fms:(i + 1) *
n_fms] if value is not None else value
for key, value in outputs_dict.items()
} for i in range(self.num_mc_dropout_runs)]
# Merge results:
results = [
inference_utils.general_standard_nms_postprocessing(
input_im,
self.retinanet_probabilistic_inference(input_im,
outputs=outputs),
self.model.test_nms_thresh,
self.model.max_detections_per_image)
for outputs in outputs_list
]
# Append per-ensemble outputs after NMS has been performed.
ensemble_pred_box_list = [
result.pred_boxes.tensor for result in results
]
ensemble_pred_prob_vectors_list = [
result.pred_cls_probs for result in results
]
ensembles_class_idxs_list = [
result.pred_classes for result in results
]
ensembles_pred_box_covariance_list = [
result.pred_boxes_covariance for result in results
]
return inference_utils.general_black_box_ensembles_post_processing(
input_im, ensemble_pred_box_list, ensembles_class_idxs_list,
ensemble_pred_prob_vectors_list,
ensembles_pred_box_covariance_list, self.model.test_nms_thresh,
self.model.max_detections_per_image,
self.cfg.PROBABILISTIC_INFERENCE.AFFINITY_THRESHOLD)
def post_processing_mc_dropout_ensembles(self, input_im):
"""
This function produces results using monte-carlo dropout ensembles.
Args:
input_im (list): an input im list generated from dataset handler.
Returns:
result (instances): object instances
"""
if self.cfg.PROBABILISTIC_INFERENCE.ENSEMBLES.BOX_MERGE_MODE == 'pre_nms':
# In generalized rcnn models, association cannot be achieved on an anchor level when using
# dropout as anchor order might shift. To overcome this problem, the anchor statistics function
# is used to perform the association and to fuse covariance
# results.
return self.post_processing_output_statistics(input_im)
else:
outputs_list = self.model(
input_im,
return_anchorwise_output=False,
num_mc_dropout_runs=self.num_mc_dropout_runs)
# Merge results:
results = [
inference_utils.general_standard_nms_postprocessing(
input_im,
self.generalized_rcnn_probabilistic_inference(
input_im, outputs=outputs), self.test_nms_thresh,
self.test_topk_per_image) for outputs in outputs_list
]
# Append per-ensemble outputs after NMS has been performed.
ensemble_pred_box_list = [
result.pred_boxes.tensor for result in results
]
ensemble_pred_prob_vectors_list = [
result.pred_cls_probs for result in results
]
ensembles_class_idxs_list = [
result.pred_classes for result in results
]
ensembles_pred_box_covariance_list = [
result.pred_boxes_covariance for result in results
]
return inference_utils.general_black_box_ensembles_post_processing(
input_im,
ensemble_pred_box_list,
ensembles_class_idxs_list,
ensemble_pred_prob_vectors_list,
ensembles_pred_box_covariance_list,
self.test_nms_thresh,
self.test_topk_per_image,
self.cfg.PROBABILISTIC_INFERENCE.AFFINITY_THRESHOLD,
is_generalized_rcnn=True,
merging_method=self.cfg.PROBABILISTIC_INFERENCE.ENSEMBLES.
BOX_FUSION_MODE)
def post_processing_ensembles(self, input_im, model_dict):
"""
This function produces results using multiple runs of independently trained models, through fusion before or after
the non-maximum suppression step.
Args:
input_im (list): an input im list generated from dataset handler.
model_dict (dict): dictionary containing list of models comprising the ensemble.
Returns:
result (instances): object instances
"""
if self.cfg.PROBABILISTIC_INFERENCE.ENSEMBLES.BOX_MERGE_MODE == 'pre_nms':
outputs_list = []
for model in model_dict:
outputs = model(input_im, return_anchorwise_output=True)
outputs_list.append(outputs)
outputs = self.retinanet_probabilistic_inference(
input_im, ensemble_inference=True, outputs_list=outputs_list)
return inference_utils.general_standard_nms_postprocessing(
input_im, outputs, self.model.test_nms_thresh, self.model.max_detections_per_image)
else:
outputs_list = []
for model in model_dict:
self.model = model
outputs_list.append(
self.post_processing_standard_nms(input_im))
# Merge results:
ensemble_pred_box_list = []
ensemble_pred_prob_vectors_list = []
ensembles_class_idxs_list = []
ensembles_pred_box_covariance_list = []
for results in outputs_list:
# Append per-ensemble outputs after NMS has been performed.
ensemble_pred_box_list.append(results.pred_boxes.tensor)
ensemble_pred_prob_vectors_list.append(results.pred_cls_probs)
ensembles_class_idxs_list.append(results.pred_classes)
ensembles_pred_box_covariance_list.append(
results.pred_boxes_covariance)
return inference_utils.general_black_box_ensembles_post_processing(
input_im,
ensemble_pred_box_list,
ensembles_class_idxs_list,
ensemble_pred_prob_vectors_list,
ensembles_pred_box_covariance_list,
self.model.test_nms_thresh,
self.model.max_detections_per_image,
self.cfg.PROBABILISTIC_INFERENCE.AFFINITY_THRESHOLD,
merging_method=self.cfg.PROBABILISTIC_INFERENCE.ENSEMBLES.BOX_FUSION_MODE)
def post_processing_standard_nms(self, input_im):
"""
This function produces results using standard non-maximum suppression. The function takes into
account any probabilistic modeling method when computing the results.
Args:
input_im (list): an input im list generated from dataset handler.
Returns:
result (instances): object instances
"""
outputs = self.generalized_rcnn_probabilistic_inference(input_im)
return inference_utils.general_standard_nms_postprocessing(
input_im, outputs, self.test_nms_thresh, self.test_topk_per_image)
def post_processing_standard_nms(self, input_im):
"""
This function produces results using standard non-maximum suppression. The function takes into
account any probabilistic modeling method when computing the results. It can combine aleatoric uncertainty
from heteroscedastic regression and epistemic uncertainty from monte-carlo dropout for both classification and
regression results.
Args:
input_im (list): an input im list generated from dataset handler.
Returns:
result (instances): object instances
"""
outputs = self.retinanet_probabilistic_inference(input_im)
return inference_utils.general_standard_nms_postprocessing(
input_im, outputs, self.model.test_nms_thresh, self.model.max_detections_per_image)
def post_processing_mc_dropout_ensembles(self, input_im):
if self.cfg.PROBABILISTIC_INFERENCE.ENSEMBLES.BOX_MERGE_MODE == 'pre_nms':
raise NotImplementedError
else:
# Merge results:
results = [
inference_utils.general_standard_nms_postprocessing(
input_im, self.detr_probabilistic_inference(input_im),
self.test_nms_thresh, self.test_topk_per_image)
for _ in range(self.num_mc_dropout_runs)
]
# Append per-ensemble outputs after NMS has been performed.
ensemble_pred_box_list = [
result.pred_boxes.tensor for result in results
]
ensemble_pred_prob_vectors_list = [
result.pred_cls_probs for result in results
]
ensembles_class_idxs_list = [
result.pred_classes for result in results
]
ensembles_pred_box_covariance_list = [
result.pred_boxes_covariance for result in results
]
return inference_utils.general_black_box_ensembles_post_processing(
input_im,
ensemble_pred_box_list,
ensembles_class_idxs_list,
ensemble_pred_prob_vectors_list,
ensembles_pred_box_covariance_list,
self.test_nms_thresh,
self.test_topk_per_image,
self.cfg.PROBABILISTIC_INFERENCE.AFFINITY_THRESHOLD,
is_generalized_rcnn=True,
merging_method=self.cfg.PROBABILISTIC_INFERENCE.ENSEMBLES.
BOX_FUSION_MODE)
