def associate_detections_to_trackers(detections, trackers, iou_threshold=0.3):
"""
Assigns detections to tracked object (both represented as bounding boxes)
Returns 3 lists of matches, unmatched_detections and unmatched_trackers
"""
if (len(trackers) == 0):
return np.empty(
(0, 2), dtype=int), np.arange(len(detections)), np.empty((0, 5),
dtype=int)
iou_matrix = np.zeros((len(detections), len(trackers)), dtype=np.float32)
for d, det in enumerate(detections):
for t, trk in enumerate(trackers):
iou_matrix[d, t] = iou(det, trk)
# matched_indices = linear_assignment(-iou_matrix)
row_ind, col_ind = linear_sum_assignment(-iou_matrix)
matched_indices = np.concatenate(
(np.expand_dims(row_ind, axis=1), np.expand_dims(col_ind, axis=1)),
axis=1)
# matched_indices = linear_sum_assignment(-iou_matrix)
unmatched_detections = []
for d, det in enumerate(detections):
if (d not in matched_indices[:, 0]):
unmatched_detections.append(d)
unmatched_trackers = []
for t, trk in enumerate(trackers):
if (t not in matched_indices[:, 1]):
unmatched_trackers.append(t)
# filter out matched with low IOU
matches = []
for m in matched_indices:
if (iou_matrix[m[0], m[1]] < iou_threshold):
unmatched_detections.append(m[0])
unmatched_trackers.append(m[1])
else:
matches.append(m.reshape(1, 2))
if (len(matches) == 0):
matches = np.empty((0, 2), dtype=int)
else:
matches = np.concatenate(matches, axis=0)
return matches, unmatched_detections, unmatched_trackers
def evaluation(eval_loader,
model,
criterion,
num_classes,
batch_size,
task,
ep_idx,
progress_log,
vis_params,
batch_metrics=None,
dataset='val',
device=None,
debug=False):
"""
Evaluate the model and return the updated metrics
:param eval_loader: data loader
:param model: model to evaluate
:param criterion: loss criterion
:param num_classes: number of classes
:param batch_size: number of samples to process simultaneously
:param task: segmentation or classification
:param ep_idx: epoch index (for hypertrainer log)
:param progress_log: progress log file (for hypertrainer log)
:param batch_metrics: (int) Metrics computed every (int) batches. If left blank, will not perform metrics.
:param dataset: (str) 'val or 'tst'
:param device: device used by pytorch (cpu ou cuda)
:return: (dict) eval_metrics
"""
eval_metrics = create_metrics_dict(num_classes)
model.eval()
for m in model.modules():
if isinstance(m, nn.BatchNorm2d):
m.track_running_stats = False
vis_at_eval = get_key_def('vis_at_evaluation', vis_params['visualization'],
False)
vis_batch_range = get_key_def('vis_batch_range',
vis_params['visualization'], None)
min_vis_batch, max_vis_batch, increment = vis_batch_range
with tqdm(eval_loader,
dynamic_ncols=True,
desc=f'Iterating {dataset} batches with {device.type}') as _tqdm:
for batch_index, data in enumerate(_tqdm):
progress_log.open('a', buffering=1).write(
tsv_line(ep_idx, dataset, batch_index, len(eval_loader),
time.time()))
with torch.no_grad():
inputs = data['sat_img'].to(device)
labels = data['map_img'].to(device)
labels_flatten = flatten_labels(labels)
outputs = model(inputs)
if isinstance(outputs, OrderedDict):
outputs = outputs['out']
if vis_batch_range is not None and vis_at_eval and batch_index in range(
min_vis_batch, max_vis_batch, increment):
vis_path = progress_log.parent.joinpath('visualization')
if ep_idx == 0 and batch_index == min_vis_batch:
tqdm.write(
f'Visualizing on {dataset} outputs for batches in range {vis_batch_range}. All '
f'images will be saved to {vis_path}\n')
vis_from_batch(params,
inputs,
outputs,
batch_index=batch_index,
vis_path=vis_path,
labels=labels,
dataset=dataset,
ep_num=ep_idx + 1)
outputs_flatten = flatten_outputs(outputs, num_classes)
loss = criterion(outputs, labels)
eval_metrics['loss'].update(loss.item(), batch_size)
if (dataset == 'val') and (batch_metrics is not None):
# Compute metrics every n batches. Time consuming.
assert batch_metrics <= len(_tqdm), f"Batch_metrics ({batch_metrics} is smaller than batch size " \
f"{len(_tqdm)}. Metrics in validation loop won't be computed"
if (
batch_index + 1
) % batch_metrics == 0: # +1 to skip val loop at very beginning
a, segmentation = torch.max(outputs_flatten, dim=1)
eval_metrics = iou(segmentation, labels_flatten,
batch_size, num_classes,
eval_metrics)
eval_metrics = report_classification(
segmentation,
labels_flatten,
batch_size,
eval_metrics,
ignore_index=get_key_def("ignore_index",
params["training"], None))
elif dataset == 'tst':
a, segmentation = torch.max(outputs_flatten, dim=1)
eval_metrics = iou(segmentation, labels_flatten,
batch_size, num_classes, eval_metrics)
eval_metrics = report_classification(
segmentation,
labels_flatten,
batch_size,
eval_metrics,
ignore_index=get_key_def("ignore_index",
params["training"], None))
_tqdm.set_postfix(
OrderedDict(dataset=dataset,
loss=f'{eval_metrics["loss"].avg:.4f}'))
if debug and device.type == 'cuda':
res, mem = gpu_stats(device=device.index)
_tqdm.set_postfix(
OrderedDict(
device=device,
gpu_perc=f'{res.gpu} %',
gpu_RAM=
f'{mem.used/(1024**2):.0f}/{mem.total/(1024**2):.0f} MiB'
))
print(f"{dataset} Loss: {eval_metrics['loss'].avg}")
if batch_metrics is not None:
print(f"{dataset} precision: {eval_metrics['precision'].avg}")
print(f"{dataset} recall: {eval_metrics['recall'].avg}")
print(f"{dataset} fscore: {eval_metrics['fscore'].avg}")
print(f"{dataset} iou: {eval_metrics['iou'].avg}")
return eval_metrics
def evaluation(eval_loader,
model,
criterion,
num_classes,
batch_size,
ep_idx,
progress_log,
scale,
vis_params,
batch_metrics=None,
dataset='val',
device=None,
debug=False):
"""
Evaluate the model and return the updated metrics
:param eval_loader: data loader
:param model: model to evaluate
:param criterion: loss criterion
:param num_classes: number of classes
:param batch_size: number of samples to process simultaneously
:param ep_idx: epoch index (for hypertrainer log)
:param progress_log: progress log file (for hypertrainer log)
:param scale: Scale to which values in sat img have been redefined. Useful during visualization
:param vis_params: (Dict) Parameters useful during visualization
:param batch_metrics: (int) Metrics computed every (int) batches. If left blank, will not perform metrics.
:param dataset: (str) 'val or 'tst'
:param device: device used by pytorch (cpu ou cuda)
:param debug: if True, debug functions will be performed
:return: (dict) eval_metrics
"""
eval_metrics = create_metrics_dict(num_classes)
model.eval()
for batch_index, data in enumerate(tqdm(eval_loader, dynamic_ncols=True, desc=f'Iterating {dataset} '
f'batches with {device.type}')):
progress_log.open('a', buffering=1).write(tsv_line(ep_idx, dataset, batch_index, len(eval_loader), time.time()))
with torch.no_grad():
try: # For HPC when device 0 not available. Error: RuntimeError: CUDA error: invalid device ordinal
inputs = data['sat_img'].to(device)
labels = data['map_img'].to(device)
except RuntimeError:
logging.exception(f'Unable to use device {device}. Trying "cuda:0"')
device = torch.device('cuda')
inputs = data['sat_img'].to(device)
labels = data['map_img'].to(device)
labels_flatten = flatten_labels(labels)
outputs = model(inputs)
if isinstance(outputs, OrderedDict):
outputs = outputs['out']
# vis_batch_range: range of batches to perform visualization on. see README.md for more info.
# vis_at_eval: (bool) if True, will perform visualization at eval time, as long as vis_batch_range is valid
if vis_params['vis_batch_range'] and vis_params['vis_at_eval']:
min_vis_batch, max_vis_batch, increment = vis_params['vis_batch_range']
if batch_index in range(min_vis_batch, max_vis_batch, increment):
vis_path = progress_log.parent.joinpath('visualization')
if ep_idx == 0 and batch_index == min_vis_batch:
logging.info(
f'Visualizing on {dataset} outputs for batches in range {vis_params["vis_batch_range"]} '
f'images will be saved to {vis_path}\n')
vis_from_batch(vis_params, inputs, outputs,
batch_index=batch_index,
vis_path=vis_path,
labels=labels,
dataset=dataset,
ep_num=ep_idx + 1,
scale=scale)
outputs_flatten = flatten_outputs(outputs, num_classes)
loss = criterion(outputs, labels)
eval_metrics['loss'].update(loss.item(), batch_size)
if (dataset == 'val') and (batch_metrics is not None):
# Compute metrics every n batches. Time consuming.
if not batch_metrics <= len(eval_loader):
logging.error(f"Batch_metrics ({batch_metrics}) is smaller than batch size "
f"{len(eval_loader)}. Metrics in validation loop won't be computed")
if (batch_index + 1) % batch_metrics == 0: # +1 to skip val loop at very beginning
a, segmentation = torch.max(outputs_flatten, dim=1)
eval_metrics = iou(segmentation, labels_flatten, batch_size, num_classes, eval_metrics)
eval_metrics = report_classification(segmentation, labels_flatten, batch_size, eval_metrics,
ignore_index=eval_loader.dataset.dontcare)
elif dataset == 'tst':
a, segmentation = torch.max(outputs_flatten, dim=1)
eval_metrics = iou(segmentation, labels_flatten, batch_size, num_classes, eval_metrics)
eval_metrics = report_classification(segmentation, labels_flatten, batch_size, eval_metrics,
ignore_index=eval_loader.dataset.dontcare)
logging.debug(OrderedDict(dataset=dataset, loss=f'{eval_metrics["loss"].avg:.4f}'))
if debug and device.type == 'cuda':
res, mem = gpu_stats(device=device.index)
logging.debug(OrderedDict(device=device, gpu_perc=f'{res.gpu} %',
gpu_RAM=f'{mem.used / (1024 ** 2):.0f}/{mem.total / (1024 ** 2):.0f} MiB'))
logging.info(f"{dataset} Loss: {eval_metrics['loss'].avg}")
if batch_metrics is not None:
logging.info(f"{dataset} precision: {eval_metrics['precision'].avg}")
logging.info(f"{dataset} recall: {eval_metrics['recall'].avg}")
logging.info(f"{dataset} fscore: {eval_metrics['fscore'].avg}")
logging.info(f"{dataset} iou: {eval_metrics['iou'].avg}")
return eval_metrics
def generate_targets(gt_labels, gt_boxes, regions, image_shape,
foreground_iou_interval, background_iou_interval):
"""
Generate target labels and target boxes associated to each input region for a single image.
Args:
- gt_labels: A tensor of shape [max_num_objects, num_classes + 1] representing the
ground-truth class labels possibly passed with zeros.
- gt_boxes: A tensor of shape [max_num_objects, 4] representing the
ground-truth bounding boxes possibly passed with zeros.
- regions: A tensor of shape [num_regions, 4] representing the reference regions.
This corresponds to the anchors for the RPN and to the RoIs for Fast-RCNN.
- image_shape: Shape of the input image.
- foreground_iou_interval: Regions that have an IoU overlap with a ground-truth
bounding box in this interval are labeled as foreground. Note that if there are no regions
within this interval, the region with the highest IoU with any ground truth box will be labeled
as foreground to ensure that there is always at least one positive example per image.
- background_iou_interval: Regions that have an IoU overlap with a ground-truth
bounding box in this interval are labeled as background. Regions that are neither labeled as
foreground, nor background are ignored.
Returns:
- target_labels: Tensor of shape [num_regions, num_classes + 1] representing the one-hot encoded
target labels for each region. The target label for background regions is [1, 0, ..., 0], whereas the
target label for ignored regions is [0, ..., 0].
- target_boxes_encoded: Tensor of shape [num_regions, num_classes, 4] representing the
encoded target bounding boxes for each region. The values of this tensor are all zeros,
except at positions [i, j] such as:
(1) the i'th region is labeled as a foreground region.
(2) the object associated with the ground-truth box with the highest IoU with the i'th region
(let's call it object k) is of the j'th class (without background).
in this case, target_boxes_encoded[i, j] = gt_boxes[k]
"""
num_classes = tf.shape(gt_labels)[1] - 1
num_regions = tf.shape(regions)[0]
# Remove padding gt_boxes & gt_labels
non_padding_inds = tf.where(tf.reduce_sum(gt_labels, -1) != 0.0)
gt_boxes = tf.gather_nd(gt_boxes, non_padding_inds)
gt_labels = tf.gather_nd(gt_labels, non_padding_inds)
# Rescale gt_boxes to be in absolute coordnates
abs_gt_boxes = to_absolute(gt_boxes, image_shape)
# Compute pairwise IoU overlap between regions and ground-truth boxes
ious = iou(regions, abs_gt_boxes, pairwise=True)
max_iou_indices = tf.math.argmax(ious, axis=-1)
max_iou_per_region = tf.reduce_max(ious, axis=-1)
max_iou = tf.reduce_max(max_iou_per_region)
# Create target class labels
background_labels_mask, foreground_labels_mask = _get_labels_masks(
max_iou_per_region, max_iou, num_classes, foreground_iou_interval,
background_iou_interval)
background_labels = tf.one_hot(tf.zeros(num_regions, dtype=tf.int32),
num_classes + 1)
foreground_labels = tf.gather(gt_labels, max_iou_indices)
target_labels = tf.zeros([num_regions, num_classes + 1])
target_labels = tf.where(background_labels_mask, background_labels,
target_labels)
target_labels = tf.where(foreground_labels_mask, foreground_labels,
target_labels)
# Create target boxes
foreground_boxes = tf.gather(abs_gt_boxes, max_iou_indices)
foreground_boxes_encoded = encode(foreground_boxes, regions)
foreground_boxes_encoded = tf.tile(
tf.expand_dims(foreground_boxes_encoded, 1), [1, num_classes, 1])
foreground_boxes_mask = tf.cast(target_labels[:, 1:], dtype=tf.bool)
foreground_boxes_mask = tf.tile(tf.expand_dims(foreground_boxes_mask, -1),
[1, 1, 4])
target_boxes_encoded = tf.zeros([num_regions, num_classes, 4])
target_boxes_encoded = tf.where(foreground_boxes_mask,
foreground_boxes_encoded,
target_boxes_encoded)
return target_labels, target_boxes_encoded