def _sample_to_sseg_heatmap(self, imdata, sample):
annots = sample['annots']
aids = annots['aids']
cids = annots['cids']
boxes = annots['rel_boxes']
# Clip boxes to the image boundary
input_dims = imdata.shape[0:2]
boxes = boxes.clip(0, 0, input_dims[1], input_dims[0])
class_idxs = np.array([self.cid_to_cidx[cid] for cid in cids])
segmentations = annots['rel_ssegs']
raw_dets = kwimage.Detections(
aids=aids,
boxes=boxes,
class_idxs=class_idxs,
segmentations=segmentations,
classes=self.classes,
datakeys=['aids'],
)
keep = []
for i, s in enumerate(raw_dets.data['segmentations']):
# TODO: clip polygons
m = s.to_mask(input_dims)
if m.area > 0:
keep.append(i)
dets = raw_dets.take(keep)
heatmap = dets.rasterize(bg_size=(1, 1), input_dims=input_dims,
soften=0, exclude=['diameter', 'class_probs', 'offset'])
return heatmap
def _labels_to_true_dets(harn, inp_size, labels, _aidbase=1, undo_lb=True):
""" Convert batch groundtruth to coco-style annotations for scoring """
indices = labels['indices']
orig_sizes = labels['orig_sizes']
targets = labels['cxywh']
gt_weights = labels['gt_weights']
letterbox = harn.datasets[harn.current_tag].letterbox
# On the training set, we need to add truth due to augmentation
bsize = len(indices)
for ix in range(bsize):
target = targets[ix].view(-1, 5)
import kwimage
true_det = kwimage.Detections(
boxes=kwimage.Boxes(target[:, 1:5].float(), 'cxywh'),
class_idxs=target[:, 0].long(),
weights=gt_weights[ix],
)
true_det = true_det.numpy()
flags = true_det.class_idxs != -1
true_det = true_det.compress(flags)
if undo_lb:
orig_size = orig_sizes[ix].cpu().numpy()
true_det.data['boxes'] = letterbox._boxes_letterbox_invert(
true_det.boxes, orig_size, inp_size)
true_det.data['aids'] = np.arange(_aidbase, _aidbase + len(true_det))
gx = int(indices[ix].data.cpu().numpy())
# if util.IS_PROFILING:
# torch.cuda.synchronize()
yield gx, true_det
def _devcheck_sample_full_image():
"""
"""
import kwimage
import numpy as np
sampler = grab_camvid_sampler()
cid_to_cidx = sampler.catgraph.id_to_idx
classes = sampler.catgraph
# Try loading an entire image
img, annots = sampler.load_image_with_annots(1)
file = img['imdata']
imdata = file[:]
aids = [ann['id'] for ann in annots]
_annots = sampler.dset.annots(aids)
sseg_list = []
for s in _annots.lookup('segmentation'):
m = kwimage.MultiPolygon.coerce(s)
sseg_list.append(m)
aids = _annots.aids
cids = _annots.cids
boxes = _annots.boxes
segmentations = kwimage.PolygonList(sseg_list)
class_idxs = np.array([cid_to_cidx[cid] for cid in cids])
dets = kwimage.Detections(
aids=aids,
boxes=boxes,
class_idxs=class_idxs,
segmentations=segmentations,
classes=classes,
datakeys=['aids'],
)
if 1:
print('dets = {!r}'.format(dets))
print('dets.data = {!r}'.format(dets.data))
print('dets.meta = {!r}'.format(dets.meta))
if ub.argflag('--show'):
import kwplot
with ub.Timer('dets.draw_on'):
canvas = imdata.copy()
canvas = dets.draw_on(canvas)
kwplot.imshow(canvas, pnum=(1, 2, 1), title='dets.draw_on')
with ub.Timer('dets.draw'):
kwplot.imshow(imdata,
pnum=(1, 2, 2),
docla=True,
title='dets.draw')
dets.draw()
def _sample_to_sseg_heatmap(self, sample):
imdata = sample['im']
annots = sample['annots']
aids = annots['aids']
cids = annots['cids']
boxes = annots['rel_boxes']
# Clip boxes to the image boundary
input_dims = imdata.shape[0:2]
boxes = boxes.clip(0, 0, input_dims[1], input_dims[0])
class_idxs = np.array([self.cid_to_cidx[cid] for cid in cids])
segmentations = annots['rel_ssegs']
raw_dets = kwimage.Detections(
aids=aids,
boxes=boxes,
class_idxs=class_idxs,
segmentations=segmentations,
classes=self.classes,
datakeys=['aids'],
)
keep = []
for i, s in enumerate(raw_dets.data['segmentations']):
# TODO: clip polygons
m = s.to_mask(input_dims)
if m.area > 0:
keep.append(i)
dets = raw_dets.take(keep)
heatmap = dets.rasterize(bg_size=(1, 1),
input_dims=input_dims,
soften=0,
exclude=['diameter', 'class_probs', 'offset'])
try:
# TODO: THIS MAY NOT BE THE CORRECT TRANSFORM
input_shape = (
1,
3,
) + input_dims
output_dims = getattr(self, '_output_dims', None)
if output_dims is None:
output_shape = self.raw_model.output_shape_for(input_shape)
output_dims = self.output_dims = output_shape[2:]
sf = np.array(output_dims) / np.array(input_dims)
heatmap = heatmap.scale(sf,
output_dims=output_dims,
interpolation='nearest')
except Exception:
pass
return heatmap
def select_positive_regions(targets, window_dims=(300, 300), thresh=0.0,
rng=None, verbose=0):
"""
Reduce positive example redundency by selecting disparate positive samples
Example:
>>> from ndsampler.coco_regions import *
>>> import kwcoco
>>> dset = kwcoco.CocoDataset.demo('shapes8')
>>> targets = tabular_coco_targets(dset)
>>> window_dims = (300, 300)
>>> selected = select_positive_regions(targets, window_dims)
>>> print(len(selected))
>>> print(len(dset.anns))
"""
unique_gids, groupxs = kwarray.group_indices(targets['gid'])
gid_to_groupx = dict(zip(unique_gids, groupxs))
wh, ww = window_dims
rng = kwarray.ensure_rng(rng)
selection = []
# Get all the bounding boxes
cxs, cys = ub.take(targets, ['cx', 'cy'])
n = len(targets)
cxs = cxs.astype(np.float32)
cys = cys.astype(np.float32)
wws = np.full(n, ww, dtype=np.float32)
whs = np.full(n, wh, dtype=np.float32)
cxywh = np.hstack([a[:, None] for a in [cxs, cys, wws, whs]])
boxes = kwimage.Boxes(cxywh, 'cxywh').to_tlbr()
iter_ = ub.ProgIter(gid_to_groupx.items(),
enabled=verbose,
label='select positive regions',
total=len(gid_to_groupx), adjust=0, freq=32)
for gid, groupx in iter_:
# Select all candiate windows in this image
cand_windows = boxes.take(groupx, axis=0)
# Randomize which candidate windows have the highest scores so the
# selection can vary each epoch.
cand_scores = rng.rand(len(cand_windows))
cand_dets = kwimage.Detections(boxes=cand_windows, scores=cand_scores)
# Non-max supresssion is really similar to set-cover
keep = cand_dets.non_max_supression(thresh=thresh)
selection.extend(groupx[keep])
selection = np.array(sorted(selection))
return selection
def _kwiver_to_kwimage_detections(detected_objects):
"""
Convert vital detected object sets to kwimage.Detections
Args:
detected_objects (kwiver.vital.types.DetectedObjectSet)
Returns:
kwimage.Detections
"""
import ubelt as ub
import kwimage
boxes = []
scores = []
class_idxs = []
classes = []
if len(detected_objects) > 0:
obj = ub.peek(detected_objects)
classes = obj.type().all_class_names()
for obj in detected_objects:
box = obj.bounding_box()
tlbr = [box.min_x(), box.min_y(), box.max_x(), box.max_y()]
score = obj.confidence()
cname = obj.type().get_most_likely_class()
cidx = classes.index(cname)
boxes.append(tlbr)
scores.append(score)
class_idxs.append(cidx)
dets = kwimage.Detections(
boxes=kwimage.Boxes(np.array(boxes), 'tlbr'),
scores=np.array(scores),
class_idxs=np.array(class_idxs),
classes=classes,
)
return dets
def draw_batch(harn,
batch,
outputs,
batch_dets,
idx=None,
thresh=None,
orig_img=None,
num_extra=3):
"""
Returns:
np.ndarray: numpy image
Example:
>>> # DISABLE_DOCTSET
>>> harn = setup_harn(bsize=1, datasets='special:voc', pretrained='lightnet')
>>> harn.initialize()
>>> batch = harn._demo_batch(0, 'train')
>>> outputs, loss = harn.run_batch(batch)
>>> batch_dets = harn.raw_model.coder.decode_batch(outputs)
>>> stacked = harn.draw_batch(batch, outputs, batch_dets)
>>> # xdoc: +REQUIRES(--show)
>>> kwplot.autompl() # xdoc: +SKIP
>>> kwplot.imshow(stacked)
>>> kwplot.show_if_requested()
"""
import cv2
inputs = batch['im']
labels = batch['label']
orig_sizes = labels['orig_sizes']
classes = harn.datasets['train'].sampler.classes
if idx is None:
idxs = range(len(inputs))
else:
idxs = [idx]
imgs = []
for idx in idxs:
chw01 = inputs[idx]
pred_dets = batch_dets[idx]
# pred_dets.meta['classes'] = classes
import kwimage
true_dets = kwimage.Detections(
boxes=kwimage.Boxes(labels['cxywh'][idx], 'cxywh'),
class_idxs=labels['class_idxs'][idx].view(-1),
weights=labels['weight'][idx],
classes=classes,
)
pred_dets = pred_dets.numpy()
true_dets = true_dets.numpy()
true_dets = true_dets.compress(true_dets.class_idxs != -1)
if thresh is not None:
pred_dets = pred_dets.compress(pred_dets.scores > thresh)
# only show so many predictions
num_max = len(true_dets) + num_extra
sortx = pred_dets.argsort(reverse=True)
pred_dets = pred_dets.take(sortx[0:num_max])
hwc01 = chw01.cpu().numpy().transpose(1, 2, 0)
inp_size = np.array(hwc01.shape[0:2][::-1])
true_dets.boxes.scale(inp_size, inplace=True)
pred_dets.boxes.scale(inp_size, inplace=True)
letterbox = harn.datasets[harn.current_tag].letterbox
orig_size = orig_sizes[idx].cpu().numpy()
target_size = inp_size
img = letterbox._img_letterbox_invert(hwc01, orig_size,
target_size)
img = np.clip(img, 0, 1)
# we are given the original image, to avoid artifacts from
# inverting a downscale
assert orig_img is None or orig_img.shape == img.shape
true_dets.data['boxes'] = letterbox._boxes_letterbox_invert(
true_dets.boxes, orig_size, target_size)
pred_dets.data['boxes'] = letterbox._boxes_letterbox_invert(
pred_dets.boxes, orig_size, target_size)
# shift, scale, embed_size = letterbox._letterbox_transform(orig_size, target_size)
# fig = kwplot.figure(doclf=True, fnum=1)
# kwplot.imshow(img, colorspace='rgb')
canvas = (img * 255).astype(np.uint8)
canvas = true_dets.draw_on(canvas, color='green')
canvas = pred_dets.draw_on(canvas, color='blue')
canvas = cv2.resize(canvas, (300, 300))
imgs.append(canvas)
stacked = imgs[0] if len(imgs) == 1 else kwimage.stack_images_grid(
imgs)
return stacked
def __getitem__(self, index):
"""
Example:
>>> # DISABLE_DOCTSET
>>> self = DetectDataset.demo(backend='npy')
>>> index = 0
>>> item = self[index]
>>> hwc01 = item['im'].numpy().transpose(1, 2, 0)
>>> print(hwc01.shape)
>>> norm_boxes = item['label']['targets'].numpy().reshape(-1, 5)[:, 1:5]
>>> inp_size = hwc01.shape[-2::-1]
>>> # xdoc: +REQUIRES(--show)
>>> import kwplot
>>> kwplot.figure(doclf=True, fnum=1)
>>> kwplot.autompl() # xdoc: +SKIP
>>> kwplot.imshow(hwc01)
>>> inp_boxes = kwimage.Boxes(norm_boxes, 'cxywh').scale(inp_size)
>>> inp_boxes.draw()
>>> kwplot.show_if_requested()
"""
import kwimage
if isinstance(index, tuple):
# Get size index from the batch loader
index, size_index = index
if size_index is None:
inp_size = self.input_dims
else:
inp_size = self.multi_scale_inp_size[size_index]
else:
inp_size = self.input_dims
classes = self.sampler.classes
coco_dset = self.sampler.dset
gid = self.sampler.image_ids[index]
img = self.sampler.dset.imgs[gid]
tr = {
'gid': gid,
'cx': img['width'] / 2.0,
'cy': img['height'] / 2.0,
'width': img['width'],
'height': img['height'],
}
sample = self.sampler.load_sample(tr)
# sample = self.sampler.load_item(index, window_dims=self.input_dims)
annots = sample['annots']
aids = annots['aids']
cids = annots['cids']
image = sample['im']
boxes = annots['rel_boxes'].to_tlbr()
dets = kwimage.Detections(
boxes=boxes,
class_idxs=np.array([classes.id_to_idx[cid] for cid in cids]),
weights=np.array(
[coco_dset.anns[aid].get('weight', 1.0) for aid in aids]),
)
inp_size = np.array(inp_size)
orig_size = np.array(image.shape[0:2][::-1])
if self.augmenter:
if len(dets):
# Ensure the same augmentor is used for bboxes and iamges
seq_det = self.augmenter.to_deterministic()
input_dims = image.shape[0:2]
image = seq_det.augment_image(image)
output_dims = image.shape[0:2]
dets = dets.warp(seq_det,
input_dims=input_dims,
output_dims=output_dims)
# Clip any bounding boxes that went out of bounds
h, w = image.shape[0:2]
tlbr = dets.boxes.to_tlbr()
old_area = tlbr.area
tlbr = tlbr.clip(0, 0, w - 1, h - 1, inplace=True)
new_area = tlbr.area
dets.data['boxes'] = tlbr
# Remove any boxes that have gone significantly out of bounds.
remove_thresh = 0.1
flags = (new_area / old_area).ravel() > remove_thresh
dets = dets.compress(flags)
# Apply letterbox resize transform to train and test
self.letterbox.target_size = inp_size
input_dims = image.shape[0:2]
image = self.letterbox.augment_image(image)
output_dims = image.shape[0:2]
if len(dets):
dets = dets.warp(self.letterbox,
input_dims=input_dims,
output_dims=output_dims)
# Remove any boxes that are no longer visible or out of bounds
flags = (dets.boxes.area > 0).ravel()
dets = dets.compress(flags)
chw01 = torch.FloatTensor(image.transpose(2, 0, 1) / 255.0)
# Lightnet YOLO accepts truth tensors in the format:
# [class_id, center_x, center_y, w, h]
# where coordinates are noramlized between 0 and 1
cxywh_norm = dets.boxes.toformat('cxywh').scale(1 / inp_size)
# Return index information in the label as well
orig_size = torch.LongTensor(orig_size)
index = torch.LongTensor([index])
bg_weight = torch.FloatTensor([1.0])
label = {
'cxywh': torch.FloatTensor(cxywh_norm.data),
'class_idxs': torch.LongTensor(dets.class_idxs[:, None]),
'weight': torch.FloatTensor(dets.weights),
'indices': index,
'orig_sizes': orig_size,
'bg_weights': bg_weight
}
item = {
'im': chw01,
'label': label,
}
return item
def demo(cls, **kwargs):
"""
Creates random true boxes and predicted boxes that have some noisy
offset from the truth.
Kwargs:
nclasses (int, default=1): number of foreground classes.
nimgs (int, default=1): number of images in the coco datasts.
nboxes (int, default=1): boxes per image.
n_fp (int, default=0): number of false positives.
n_fn (int, default=0): number of false negatives.
box_noise (float, default=0): std of a normal distribution used to
perterb both box location and box size.
cls_noise (float, default=0): probability that a class label will
change. Must be within 0 and 1.
anchors (ndarray, default=None): used to create random boxes
null_pred (bool, default=0):
if True, predicted classes are returned as null, which means
only localization scoring is suitable.
with_probs (bool, default=1):
if True, includes per-class probabilities with predictions
Example:
>>> # xdoctest: +REQUIRES(module:ndsampler)
>>> kwargs = {}
>>> # Seed the RNG
>>> kwargs['rng'] = 0
>>> # Size parameters determine how big the data is
>>> kwargs['nimgs'] = 5
>>> kwargs['nboxes'] = 7
>>> kwargs['nclasses'] = 11
>>> # Noise parameters perterb predictions further from the truth
>>> kwargs['n_fp'] = 3
>>> kwargs['box_noise'] = 0.1
>>> kwargs['cls_noise'] = 0.5
>>> dmet = DetectionMetrics.demo(**kwargs)
>>> print('dmet.classes = {}'.format(dmet.classes))
dmet.classes = <CategoryTree(nNodes=12, maxDepth=3, maxBreadth=4...)>
>>> # Can grab kwimage.Detection object for any image
>>> print(dmet.true_detections(gid=0))
<Detections(4)>
>>> print(dmet.pred_detections(gid=0))
<Detections(7)>
Example:
>>> # xdoctest: +REQUIRES(module:ndsampler)
>>> # Test case with null predicted categories
>>> dmet = DetectionMetrics.demo(nimgs=30, null_pred=1, nclasses=3,
>>> nboxes=10, n_fp=10, box_noise=0.3,
>>> with_probs=False)
>>> dmet.gid_to_pred_dets[0].data
>>> dmet.gid_to_true_dets[0].data
>>> cfsn_vecs = dmet.confusion_vectors()
>>> binvecs_ovr = cfsn_vecs.binarize_ovr()
>>> binvecs_per = cfsn_vecs.binarize_peritem()
>>> pr_per = binvecs_per.precision_recall()
>>> pr_ovr = binvecs_ovr.precision_recall()
>>> print('pr_per = {!r}'.format(pr_per))
>>> print('pr_ovr = {!r}'.format(pr_ovr))
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> kwplot.autompl()
>>> pr_per.draw(fnum=1)
>>> pr_ovr['perclass'].draw(fnum=2)
"""
import kwimage
import kwarray
import ndsampler
# Parse kwargs
rng = kwarray.ensure_rng(kwargs.get('rng', 0))
nclasses = kwargs.get('nclasses', 1)
nimgs = kwargs.get('nimgs', 1)
box_noise = kwargs.get('box_noise', 0)
cls_noise = kwargs.get('cls_noise', 0)
null_pred = kwargs.get('null_pred', False)
with_probs = kwargs.get('with_probs', True)
# specify an amount of overlap between true and false scores
score_noise = kwargs.get('score_noise', 0.2)
anchors = kwargs.get('anchors', None)
scale = 100.0
# Build random variables
from kwarray import distributions
DiscreteUniform = distributions.DiscreteUniform.seeded(rng=rng)
def _parse_arg(key, default):
value = kwargs.get(key, default)
try:
low, high = value
return (low, high + 1)
except Exception:
return (0, value + 1)
nboxes_RV = DiscreteUniform(*_parse_arg('nboxes', 1))
n_fp_RV = DiscreteUniform(*_parse_arg('n_fp', 0))
n_fn_RV = DiscreteUniform(*_parse_arg('n_fn', 0))
box_noise_RV = distributions.Normal(0, box_noise, rng=rng)
cls_noise_RV = distributions.Bernoulli(cls_noise, rng=rng)
# the values of true and false scores starts off with no overlap and
# the overlap increases as the score noise increases.
def _interp(v1, v2, alpha):
return v1 * alpha + (1 - alpha) * v2
mid = 0.5
# true_high = 2.0
true_high = 1.0
true_low = _interp(0, mid, score_noise)
false_high = _interp(true_high, mid - 1e-3, score_noise)
true_mean = _interp(0.5, .8, score_noise)
false_mean = _interp(0.5, .2, score_noise)
true_score_RV = distributions.TruncNormal(mean=true_mean,
std=.5,
low=true_low,
high=true_high,
rng=rng)
false_score_RV = distributions.TruncNormal(mean=false_mean,
std=.5,
low=0,
high=false_high,
rng=rng)
frgnd_cx_RV = distributions.DiscreteUniform(1, nclasses + 1, rng=rng)
# Create the category hierarcy
graph = nx.DiGraph()
graph.add_node('background', id=0)
for cid in range(1, nclasses + 1):
# binary heap encoding of a tree
cx = cid - 1
parent_cx = (cx - 1) // 2
node = 'cat_{}'.format(cid)
graph.add_node(node, id=cid)
if parent_cx > 0:
supercategory = 'cat_{}'.format(parent_cx + 1)
graph.add_edge(supercategory, node)
classes = ndsampler.CategoryTree(graph)
dmet = cls()
dmet.classes = classes
for gid in range(nimgs):
# Sample random variables
nboxes_ = nboxes_RV()
n_fp_ = n_fp_RV()
n_fn_ = n_fn_RV()
imgname = 'img_{}'.format(gid)
dmet._register_imagename(imgname, gid)
# Generate random ground truth detections
true_boxes = kwimage.Boxes.random(num=nboxes_,
scale=scale,
anchors=anchors,
rng=rng,
format='cxywh')
# Prevent 0 sized boxes: increase w/h by 1
true_boxes.data[..., 2:4] += 1
true_cxs = frgnd_cx_RV(len(true_boxes))
true_weights = np.ones(len(true_boxes), dtype=np.int32)
# Initialize predicted detections as a copy of truth
pred_boxes = true_boxes.copy()
pred_cxs = true_cxs.copy()
# Perterb box coordinates
pred_boxes.data = np.abs(
pred_boxes.data.astype(np.float) + box_noise_RV())
# Perterb class predictions
change = cls_noise_RV(len(pred_cxs))
pred_cxs_swap = frgnd_cx_RV(len(pred_cxs))
pred_cxs[change] = pred_cxs_swap[change]
# Drop true positive boxes
if n_fn_:
pred_boxes.data = pred_boxes.data[n_fn_:]
pred_cxs = pred_cxs[n_fn_:]
# pred_scores = np.linspace(true_min, true_max, len(pred_boxes))[::-1]
n_tp_ = len(pred_boxes)
pred_scores = true_score_RV(n_tp_)
# Add false positive boxes
if n_fp_:
false_boxes = kwimage.Boxes.random(num=n_fp_,
scale=scale,
rng=rng,
format='cxywh')
false_cxs = frgnd_cx_RV(n_fp_)
false_scores = false_score_RV(n_fp_)
pred_boxes.data = np.vstack(
[pred_boxes.data, false_boxes.data])
pred_cxs = np.hstack([pred_cxs, false_cxs])
pred_scores = np.hstack([pred_scores, false_scores])
# Transform the scores for the assigned class into a predicted
# probability for each class. (Currently a bit hacky).
class_probs = _demo_construct_probs(pred_cxs,
pred_scores,
classes,
rng,
hacked=kwargs.get('hacked', 0))
true_dets = kwimage.Detections(boxes=true_boxes,
class_idxs=true_cxs,
weights=true_weights)
pred_dets = kwimage.Detections(boxes=pred_boxes,
class_idxs=pred_cxs,
scores=pred_scores)
# Hack in the probs
if with_probs:
pred_dets.data['probs'] = class_probs
if null_pred:
pred_dets.data['class_idxs'] = np.array([None] *
len(pred_dets),
dtype=object)
dmet.add_truth(true_dets, imgname=imgname)
dmet.add_predictions(pred_dets, imgname=imgname)
return dmet
def _devcheck_load_sub_image():
import kwimage
import numpy as np
sampler = grab_camvid_sampler()
cid_to_cidx = sampler.catgraph.id_to_idx
classes = sampler.catgraph
# Try loading a subregion of an image
sample = sampler.load_positive(2)
imdata = sample['im']
annots = sample['annots']
aids = annots['aids']
cids = annots['cids']
boxes = annots['rel_boxes']
class_idxs = np.array([cid_to_cidx[cid] for cid in cids])
segmentations = annots['rel_ssegs']
raw_dets = kwimage.Detections(
aids=aids,
boxes=boxes,
class_idxs=class_idxs,
segmentations=segmentations,
classes=classes,
datakeys=['aids'],
)
# Clip boxes to the image boundary
input_dims = imdata.shape[0:2]
raw_dets.data['boxes'] = raw_dets.boxes.clip(0, 0, input_dims[1],
input_dims[0])
keep = []
for i, s in enumerate(raw_dets.data['segmentations']):
# TODO: clip polygons
m = s.to_mask(input_dims)
if m.area > 0:
keep.append(i)
dets = raw_dets.take(keep)
heatmap = dets.rasterize(bg_size=(1, 1), input_dims=input_dims)
if 1:
print('dets = {!r}'.format(dets))
print('dets.data = {!r}'.format(dets.data))
print('dets.meta = {!r}'.format(dets.meta))
if ub.argflag('--show'):
import kwplot
kwplot.autompl()
heatmap.draw()
draw_boxes = 1
kwplot.figure(doclf=True)
with ub.Timer('dets.draw_on'):
canvas = imdata.copy()
# TODO: add logic to color by class
canvas = dets.draw_on(canvas, boxes=draw_boxes, color='random')
kwplot.imshow(canvas, pnum=(1, 2, 1), title='dets.draw_on')
with ub.Timer('dets.draw'):
kwplot.imshow(imdata,
pnum=(1, 2, 2),
docla=True,
title='dets.draw')
dets.draw(boxes=draw_boxes, color='random')
def _decode(self, output):
"""
Returns array of detections for every image in batch
CommandLine:
python ~/code/netharn/netharn/box_models/yolo2/light_postproc.py GetBoundingBoxes._decode
Examples:
>>> # xdoctest: +REQUIRES(module:kwimage)
>>> import torch
>>> torch.random.manual_seed(0)
>>> anchors = np.array([(1.3221, 1.73145), (3.19275, 4.00944), (5.05587, 8.09892), (9.47112, 4.84053), (11.2364, 10.0071)])
>>> self = GetBoundingBoxes(anchors=anchors, num_classes=20, conf_thresh=.14, nms_thresh=0.5)
>>> output = torch.randn(16, 5, 5 + 20, 9, 9)
>>> from netharn import XPU
>>> output = XPU.coerce('auto').move(output)
>>> batch_dets = self._decode(output.data)
>>> assert len(batch_dets) == 16
Ignore:
>>> from netharn.models.yolo2.yolo2 import * # NOQA
>>> info = dev_demodata()
>>> outputs = info['outputs']
>>> cxywh_energy = output['cxywh_energy']
>>> raw = info['raw']
>>> raw_ = raw.clone()
>>> self = GetBoundingBoxes(anchors=info['model'].anchors, num_classes=20, conf_thresh=.14, nms_thresh=0.5)
>>> dets = self._decode(raw)[0]
>>> dets.scores
>>> self, output = ub.take(info, ['coder', 'outputs'])
>>> batch_dets = self.decode_batch(output)
>>> dets = batch_dets[0]
>>> dets.scores
"""
import kwimage
# dont modify inplace
raw_ = output.clone()
# Variables
bsize = raw_.shape[0]
h, w = raw_.shape[-2:]
device = raw_.device
if self.anchors.device != device:
self.anchors = self.anchors.to(device)
# Compute xc,yc, w,h, box_score on Tensor
lin_x = torch.linspace(0, w - 1, w,
device=device).repeat(h, 1).view(h * w)
lin_y = torch.linspace(0, h - 1, h, device=device).repeat(
w, 1).t().contiguous().view(h * w)
anchor_w = self.anchors[:, 0].contiguous().view(1, self.num_anchors, 1)
anchor_h = self.anchors[:, 1].contiguous().view(1, self.num_anchors, 1)
# -1 == 5+num_classes (we can drop feature maps if 1 class)
output_ = raw_.view(bsize, self.num_anchors, -1, h * w)
output_[:, :, 0, :].sigmoid_().add_(lin_x).div_(w) # X center
output_[:, :, 1, :].sigmoid_().add_(lin_y).div_(h) # Y center
output_[:, :, 2, :].exp_().mul_(anchor_w).div_(w) # Width
output_[:, :, 3, :].exp_().mul_(anchor_h).div_(h) # Height
output_[:, :, 4, :].sigmoid_() # Box score
# output_[:, :, 0:4].sum()
# torch.all(cxywh.view(-1) == output_[:, :, 0:4].contiguous().view(-1))
# Compute class_score
if self.num_classes > 1:
cls_scores = torch.nn.functional.softmax(output_[:, :, 5:, :], 2)
cls_max, cls_max_idx = torch.max(cls_scores, 2)
cls_max.mul_(output_[:, :, 4, :])
else:
cls_max = output_[:, :, 4, :]
cls_max_idx = torch.zeros_like(cls_max)
# Save detection if conf*class_conf is higher than threshold
# Newst lightnet code, which is based on my mode1 code
score_thresh = cls_max > self.conf_thresh
score_thresh_flat = score_thresh.view(-1)
if score_thresh.sum() == 0:
batch_dets = []
for i in range(bsize):
batch_dets.append(
kwimage.Detections(
boxes=kwimage.Boxes(
torch.empty((0, 4),
dtype=torch.float32,
device=device), 'cxywh'),
scores=torch.empty(0,
dtype=torch.float32,
device=device),
class_idxs=torch.empty(0,
dtype=torch.int64,
device=device),
))
else:
# Mask select boxes > conf_thresh
coords = output_.transpose(2, 3)[..., 0:4]
coords = coords[score_thresh[...,
None].expand_as(coords)].view(-1, 4)
scores = cls_max[score_thresh]
class_idxs = cls_max_idx[score_thresh]
stacked_dets = kwimage.Detections(
boxes=kwimage.Boxes(coords, 'cxywh'),
scores=scores,
class_idxs=class_idxs,
)
# Get indexes of splits between images of batch
max_det_per_batch = len(self.anchors) * h * w
slices = [
slice(max_det_per_batch * i, max_det_per_batch * (i + 1))
for i in range(bsize)
]
det_per_batch = torch.IntTensor(
[score_thresh_flat[s].int().sum() for s in slices])
split_idx = torch.cumsum(det_per_batch, dim=0)
batch_dets = []
start = 0
for end in split_idx:
dets = stacked_dets[start:end]
dets = dets.non_max_supress(thresh=self.nms_thresh)
batch_dets.append(dets)
start = end
return batch_dets
def decode_batch(self, output, forloss=False):
"""
Returns array of detections for every image in batch
Example:
>>> # xdoc: +REQUIRES(--download, module:ndsampler)
>>> from netharn.models.yolo2.yolo2 import * # NOQA
>>> self = YoloCoder.demo()
>>> output = self.demo_output()
>>> batch_dets = self.decode_batch(output)
>>> batch_dets = self.decode_batch(output, forloss=True)
Example:
>>> # xdoc: +REQUIRES(--download, module:ndsampler)
>>> info = dev_demodata()
>>> self, output = ub.take(info, ['coder', 'outputs'])
>>> batch_dets = self.decode_batch(output)
>>> dets = batch_dets[0].sort().scale(info['orig_sizes'][0])
>>> print('dets.boxes = {!r}'.format(dets.boxes))
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> kwplot.figure(fnum=1, doclf=True)
>>> kwplot.imshow(info['rgb255'], colorspace='rgb')
>>> dets.draw()
>>> kwplot.show_if_requested()
"""
import kwimage
# dont modify inplace
# output = output.clone()
class_energy = output['class_energy']
score_energy = output['score_energy']
cxywh_energy = output['cxywh_energy']
# Variables
nB = class_energy.shape[0]
nH, nW = class_energy.shape[-2:]
nA = self.num_anchors
device = class_energy.device
if self.anchors.device != device:
self.anchors = self.anchors.to(device)
# Compute xc,yc, nW,nH, box_score on Tensor
lin_x = torch.linspace(0, nW - 1, nW, device=device).repeat(nH, 1)
lin_y = torch.linspace(0, nH - 1, nH, device=device).repeat(nW, 1).t().contiguous()
anchor_w = self.anchors[:, 0].contiguous().view(1, nA, 1, 1)
anchor_h = self.anchors[:, 1].contiguous().view(1, nA, 1, 1)
if forloss:
# TODO : rectify
coord = torch.empty_like(cxywh_energy)
coord[:, :, 0:2, :, :] = cxywh_energy[:, :, 0:2, :, :].sigmoid() # cx,cy
coord[:, :, 2:4, :, :] = cxywh_energy[:, :, 2:4, :, :] # w,h
with torch.no_grad():
pred_boxes = torch.empty_like(cxywh_energy, device=device).view(-1, 4)
pred_boxes[:, 0] = (coord[:, :, 0, :, :] + lin_x).view(-1)
pred_boxes[:, 1] = (coord[:, :, 1, :, :] + lin_y).view(-1)
pred_boxes[:, 2] = (coord[:, :, 2, :, :].exp() * anchor_w).view(-1)
pred_boxes[:, 3] = (coord[:, :, 3, :, :].exp() * anchor_h).view(-1)
info = {
'coord': coord,
'pred_boxes': pred_boxes,
}
return info
else:
cxywh = cxywh_energy.clone()
# cxywh_ = cxywh.view(nB, self.num_anchors, -1, nH, nW)
cxywh[:, :, 0, :].sigmoid_().add_(lin_x).div_(nW) # X center
cxywh[:, :, 1, :].sigmoid_().add_(lin_y).div_(nH) # Y center
cxywh[:, :, 2, :].exp_().mul_(anchor_w).div_(nW) # Width
cxywh[:, :, 3, :].exp_().mul_(anchor_h).div_(nH) # Height
score = score_energy.sigmoid() # Box score
# Compute class_score
if len(self.classes) > 1:
cls_scores = torch.nn.functional.softmax(class_energy, dim=2)
cls_max, cls_max_idx = torch.max(cls_scores, 2, keepdim=True)
cls_max.mul_(score)
else:
cls_max = score
cls_max_idx = torch.zeros_like(cls_max)
# Save detection if conf*class_conf is higher than threshold
flags = cls_max >= self.conf_thresh
flags_flat = flags.view(-1)
if flags.sum() == 0:
batch_dets = []
for i in range(nB):
batch_dets.append(kwimage.Detections(
boxes=kwimage.Boxes(torch.empty((0, 4), dtype=torch.float32, device=device), 'cxywh'),
scores=torch.empty(0, dtype=torch.float32, device=device),
class_idxs=torch.empty(0, dtype=torch.int64, device=device),
classes=self.classes
))
else:
# Permute so the bbox dim (i.e. xywh) is trailing
coords = cxywh.permute(0, 1, 3, 4, 2).contiguous().view(-1, 4)
coords = coords[flags.view(-1)]
scores = cls_max[flags]
class_idxs = cls_max_idx[flags]
stacked_dets = kwimage.Detections(
boxes=kwimage.Boxes(coords, 'cxywh'),
scores=scores,
class_idxs=class_idxs,
classes=self.classes
)
# Get indexes of splits between images of batch
max_det_per_batch = len(self.anchors) * nH * nW
m = max_det_per_batch
flags_flat.int = flags_flat.int()
slices = [slice(m * i, m * (i + 1)) for i in range(nB)]
det_per_batch = torch.IntTensor([flags_flat[s].sum()
for s in slices])
split_idx = torch.cumsum(det_per_batch, dim=0)
batch_dets = []
start = 0
for end in split_idx:
dets = stacked_dets[start: end]
dets = dets.non_max_supress(thresh=self.nms_thresh)
batch_dets.append(dets)
start = end
return batch_dets
def perterb_coco(coco_dset, **kwargs):
"""
Perterbs a coco dataset
Args:
rng (int, default=0):
box_noise (int, default=0):
cls_noise (int, default=0):
null_pred (bool, default=False):
with_probs (bool, default=False):
score_noise (float, default=0.2):
hacked (int, default=1):
Example:
>>> from kwcoco.demo.perterb import * # NOQA
>>> from kwcoco.demo.perterb import _demo_construct_probs
>>> import kwcoco
>>> coco_dset = true_dset = kwcoco.CocoDataset.demo('shapes8')
>>> kwargs = {
>>> 'box_noise': 0.5,
>>> 'n_fp': 3,
>>> 'with_probs': 1,
>>> }
>>> pred_dset = perterb_coco(true_dset, **kwargs)
>>> pred_dset._check_json_serializable()
Ignore:
import xdev
from kwcoco.demo.perterb import perterb_coco # NOQA
defaultkw = xdev.get_func_kwargs(perterb_coco)
for k, v in defaultkw.items():
desc = ''
print('{} ({}, default={}): {}'.format(k, type(v).__name__, v, desc))
"""
import kwimage
import kwarray
# Parse kwargs
rng = kwarray.ensure_rng(kwargs.get('rng', 0))
box_noise = kwargs.get('box_noise', 0)
cls_noise = kwargs.get('cls_noise', 0)
null_pred = kwargs.get('null_pred', False)
with_probs = kwargs.get('with_probs', False)
# specify an amount of overlap between true and false scores
score_noise = kwargs.get('score_noise', 0.2)
# Build random variables
from kwarray import distributions
DiscreteUniform = distributions.DiscreteUniform.seeded(rng=rng)
def _parse_arg(key, default):
value = kwargs.get(key, default)
try:
low, high = value
return (low, high + 1)
except Exception:
return (value, value + 1)
n_fp_RV = DiscreteUniform(*_parse_arg('n_fp', 0))
n_fn_RV = DiscreteUniform(*_parse_arg('n_fn', 0))
box_noise_RV = distributions.Normal(0, box_noise, rng=rng)
cls_noise_RV = distributions.Bernoulli(cls_noise, rng=rng)
# the values of true and false scores starts off with no overlap and
# the overlap increases as the score noise increases.
def _interp(v1, v2, alpha):
return v1 * alpha + (1 - alpha) * v2
mid = 0.5
# true_high = 2.0
true_high = 1.0
false_low = 0.0
true_low = _interp(0, mid, score_noise)
false_high = _interp(true_high, mid - 1e-3, score_noise)
true_mean = _interp(0.5, .8, score_noise)
false_mean = _interp(0.5, .2, score_noise)
true_score_RV = distributions.TruncNormal(mean=true_mean,
std=.5,
low=true_low,
high=true_high,
rng=rng)
false_score_RV = distributions.TruncNormal(mean=false_mean,
std=.5,
low=false_low,
high=false_high,
rng=rng)
# Create the category hierarcy
classes = coco_dset.object_categories()
cids = coco_dset.cats.keys()
cidxs = [classes.id_to_idx[c] for c in cids]
frgnd_cx_RV = distributions.CategoryUniform(cidxs, rng=rng)
new_dset = coco_dset.copy()
remove_aids = []
false_anns = []
index_invalidated = False
for gid in coco_dset.imgs.keys():
# Sample random variables
n_fp_ = n_fp_RV()
n_fn_ = n_fn_RV()
true_annots = coco_dset.annots(gid=gid)
aids = true_annots.aids
for aid in aids:
# Perterb box coordinates
ann = new_dset.anns[aid]
new_bbox = (np.array(ann['bbox']) + box_noise_RV(4)).tolist()
new_x, new_y, new_w, new_h = new_bbox
allow_neg_boxes = 0
if not allow_neg_boxes:
new_w = max(new_w, 0)
new_h = max(new_h, 0)
ann['bbox'] = [new_x, new_y, new_w, new_h]
ann['score'] = float(true_score_RV(1)[0])
if cls_noise_RV():
# Perterb class predictions
ann['category_id'] = classes.idx_to_id[frgnd_cx_RV()]
index_invalidated = True
# Drop true positive boxes
if n_fn_:
import kwarray
drop_idxs = kwarray.shuffle(np.arange(len(aids)), rng=rng)[0:n_fn_]
remove_aids.extend(list(ub.take(aids, drop_idxs)))
# Add false positive boxes
if n_fp_:
try:
img = coco_dset.imgs[gid]
scale = (img['width'], img['height'])
except KeyError:
scale = 100
false_boxes = kwimage.Boxes.random(num=n_fp_,
scale=scale,
rng=rng,
format='cxywh')
false_cxs = frgnd_cx_RV(n_fp_)
false_scores = false_score_RV(n_fp_)
false_dets = kwimage.Detections(
boxes=false_boxes,
class_idxs=false_cxs,
scores=false_scores,
classes=classes,
)
for ann in list(false_dets.to_coco('new')):
ann['category_id'] = classes.node_to_id[ann.pop(
'category_name')]
ann['image_id'] = gid
false_anns.append(ann)
if null_pred:
raise NotImplementedError
if index_invalidated:
new_dset.index.clear()
new_dset._build_index()
new_dset.remove_annotations(remove_aids)
for ann in false_anns:
new_dset.add_annotation(**ann)
# Hack in the probs
if with_probs:
annots = new_dset.annots()
pred_cids = annots.lookup('category_id')
pred_cxs = np.array([classes.id_to_idx[cid] for cid in pred_cids])
pred_scores = np.array(annots.lookup('score'))
# Transform the scores for the assigned class into a predicted
# probability for each class. (Currently a bit hacky).
pred_probs = _demo_construct_probs(pred_cxs,
pred_scores,
classes,
rng,
hacked=kwargs.get('hacked', 1))
for aid, prob in zip(annots.aids, pred_probs):
new_dset.anns[aid]['prob'] = prob.tolist()
return new_dset
def draw_batch(harn,
batch,
outputs,
batch_dets,
idx=None,
thresh=None,
orig_img=None):
"""
Returns:
np.ndarray: numpy image
Example:
>>> # DISABLE_DOCTSET
>>> harn = setup_yolo_harness(bsize=1)
>>> harn.initialize()
>>> batch = harn._demo_batch(0, 'train')
>>> weights_fpath = light_yolo.demo_voc_weights()
>>> state_dict = harn.xpu.load(weights_fpath)['weights']
>>> harn.model.module.load_state_dict(state_dict)
>>> outputs, loss = harn.run_batch(batch)
>>> harn.on_batch(batch, outputs, loss)
>>> # xdoc: +REQUIRES(--show)
>>> import kwplot
>>> batch_dets = harn.model.module.postprocess(outputs)
>>> kwplot.autompl() # xdoc: +SKIP
>>> stacked = harn.draw_batch(batch, outputs, batch_dets, thresh=0.01)
>>> kwplot.imshow(stacked)
>>> kwplot.show_if_requested()
"""
import cv2
import kwimage
inputs, labels = batch
targets = labels['targets']
orig_sizes = labels['orig_sizes']
if idx is None:
idxs = range(len(inputs))
else:
idxs = [idx]
imgs = []
for idx in idxs:
chw01 = inputs[idx]
target = targets[idx].view(-1, 5)
pred_dets = batch_dets[idx]
label_names = harn.datasets[harn.current_tag].label_names
pred_dets.meta['classes'] = label_names
true_dets = kwimage.Detections(boxes=kwimage.Boxes(
target[:, 1:5], 'cxywh'),
class_idxs=target[:, 0].int(),
classes=label_names)
pred_dets = pred_dets.numpy()
true_dets = true_dets.numpy()
true_dets = true_dets.compress(true_dets.class_idxs != -1)
if thresh is not None:
pred_dets = pred_dets.compress(pred_dets.scores > thresh)
hwc01 = chw01.cpu().numpy().transpose(1, 2, 0)
inp_size = np.array(hwc01.shape[0:2][::-1])
true_dets.boxes.scale(inp_size, inplace=True)
pred_dets.boxes.scale(inp_size, inplace=True)
letterbox = harn.datasets[harn.current_tag].letterbox
orig_size = orig_sizes[idx].cpu().numpy()
target_size = inp_size
img = letterbox._img_letterbox_invert(hwc01, orig_size,
target_size)
img = np.clip(img, 0, 1)
# we are given the original image, to avoid artifacts from
# inverting a downscale
assert orig_img is None or orig_img.shape == img.shape
true_dets.data['boxes'] = letterbox._boxes_letterbox_invert(
true_dets.boxes, orig_size, target_size)
pred_dets.data['boxes'] = letterbox._boxes_letterbox_invert(
pred_dets.boxes, orig_size, target_size)
# shift, scale, embed_size = letterbox._letterbox_transform(orig_size, target_size)
# fig = kwplot.figure(doclf=True, fnum=1)
# kwplot.imshow(img, colorspace='rgb')
canvas = (img * 255).astype(np.uint8)
canvas = true_dets.draw_on(canvas, color='green')
canvas = pred_dets.draw_on(canvas, color='blue')
canvas = cv2.resize(canvas, (300, 300))
imgs.append(canvas)
# if IS_PROFILING:
# torch.cuda.synchronize()
stacked = imgs[0] if len(imgs) == 1 else kwimage.stack_images_grid(
imgs)
return stacked
