def _sample_to_sseg_heatmap(self, imdata, sample): annots = sample['annots'] aids = annots['aids'] cids = annots['cids'] boxes = annots['rel_boxes'] # Clip boxes to the image boundary input_dims = imdata.shape[0:2] boxes = boxes.clip(0, 0, input_dims[1], input_dims[0]) class_idxs = np.array([self.cid_to_cidx[cid] for cid in cids]) segmentations = annots['rel_ssegs'] raw_dets = kwimage.Detections( aids=aids, boxes=boxes, class_idxs=class_idxs, segmentations=segmentations, classes=self.classes, datakeys=['aids'], ) keep = [] for i, s in enumerate(raw_dets.data['segmentations']): # TODO: clip polygons m = s.to_mask(input_dims) if m.area > 0: keep.append(i) dets = raw_dets.take(keep) heatmap = dets.rasterize(bg_size=(1, 1), input_dims=input_dims, soften=0, exclude=['diameter', 'class_probs', 'offset']) return heatmap
def _labels_to_true_dets(harn, inp_size, labels, _aidbase=1, undo_lb=True): """ Convert batch groundtruth to coco-style annotations for scoring """ indices = labels['indices'] orig_sizes = labels['orig_sizes'] targets = labels['cxywh'] gt_weights = labels['gt_weights'] letterbox = harn.datasets[harn.current_tag].letterbox # On the training set, we need to add truth due to augmentation bsize = len(indices) for ix in range(bsize): target = targets[ix].view(-1, 5) import kwimage true_det = kwimage.Detections( boxes=kwimage.Boxes(target[:, 1:5].float(), 'cxywh'), class_idxs=target[:, 0].long(), weights=gt_weights[ix], ) true_det = true_det.numpy() flags = true_det.class_idxs != -1 true_det = true_det.compress(flags) if undo_lb: orig_size = orig_sizes[ix].cpu().numpy() true_det.data['boxes'] = letterbox._boxes_letterbox_invert( true_det.boxes, orig_size, inp_size) true_det.data['aids'] = np.arange(_aidbase, _aidbase + len(true_det)) gx = int(indices[ix].data.cpu().numpy()) # if util.IS_PROFILING: # torch.cuda.synchronize() yield gx, true_det
def _devcheck_sample_full_image(): """ """ import kwimage import numpy as np sampler = grab_camvid_sampler() cid_to_cidx = sampler.catgraph.id_to_idx classes = sampler.catgraph # Try loading an entire image img, annots = sampler.load_image_with_annots(1) file = img['imdata'] imdata = file[:] aids = [ann['id'] for ann in annots] _annots = sampler.dset.annots(aids) sseg_list = [] for s in _annots.lookup('segmentation'): m = kwimage.MultiPolygon.coerce(s) sseg_list.append(m) aids = _annots.aids cids = _annots.cids boxes = _annots.boxes segmentations = kwimage.PolygonList(sseg_list) class_idxs = np.array([cid_to_cidx[cid] for cid in cids]) dets = kwimage.Detections( aids=aids, boxes=boxes, class_idxs=class_idxs, segmentations=segmentations, classes=classes, datakeys=['aids'], ) if 1: print('dets = {!r}'.format(dets)) print('dets.data = {!r}'.format(dets.data)) print('dets.meta = {!r}'.format(dets.meta)) if ub.argflag('--show'): import kwplot with ub.Timer('dets.draw_on'): canvas = imdata.copy() canvas = dets.draw_on(canvas) kwplot.imshow(canvas, pnum=(1, 2, 1), title='dets.draw_on') with ub.Timer('dets.draw'): kwplot.imshow(imdata, pnum=(1, 2, 2), docla=True, title='dets.draw') dets.draw()
def _sample_to_sseg_heatmap(self, sample): imdata = sample['im'] annots = sample['annots'] aids = annots['aids'] cids = annots['cids'] boxes = annots['rel_boxes'] # Clip boxes to the image boundary input_dims = imdata.shape[0:2] boxes = boxes.clip(0, 0, input_dims[1], input_dims[0]) class_idxs = np.array([self.cid_to_cidx[cid] for cid in cids]) segmentations = annots['rel_ssegs'] raw_dets = kwimage.Detections( aids=aids, boxes=boxes, class_idxs=class_idxs, segmentations=segmentations, classes=self.classes, datakeys=['aids'], ) keep = [] for i, s in enumerate(raw_dets.data['segmentations']): # TODO: clip polygons m = s.to_mask(input_dims) if m.area > 0: keep.append(i) dets = raw_dets.take(keep) heatmap = dets.rasterize(bg_size=(1, 1), input_dims=input_dims, soften=0, exclude=['diameter', 'class_probs', 'offset']) try: # TODO: THIS MAY NOT BE THE CORRECT TRANSFORM input_shape = ( 1, 3, ) + input_dims output_dims = getattr(self, '_output_dims', None) if output_dims is None: output_shape = self.raw_model.output_shape_for(input_shape) output_dims = self.output_dims = output_shape[2:] sf = np.array(output_dims) / np.array(input_dims) heatmap = heatmap.scale(sf, output_dims=output_dims, interpolation='nearest') except Exception: pass return heatmap
def select_positive_regions(targets, window_dims=(300, 300), thresh=0.0, rng=None, verbose=0): """ Reduce positive example redundency by selecting disparate positive samples Example: >>> from ndsampler.coco_regions import * >>> import kwcoco >>> dset = kwcoco.CocoDataset.demo('shapes8') >>> targets = tabular_coco_targets(dset) >>> window_dims = (300, 300) >>> selected = select_positive_regions(targets, window_dims) >>> print(len(selected)) >>> print(len(dset.anns)) """ unique_gids, groupxs = kwarray.group_indices(targets['gid']) gid_to_groupx = dict(zip(unique_gids, groupxs)) wh, ww = window_dims rng = kwarray.ensure_rng(rng) selection = [] # Get all the bounding boxes cxs, cys = ub.take(targets, ['cx', 'cy']) n = len(targets) cxs = cxs.astype(np.float32) cys = cys.astype(np.float32) wws = np.full(n, ww, dtype=np.float32) whs = np.full(n, wh, dtype=np.float32) cxywh = np.hstack([a[:, None] for a in [cxs, cys, wws, whs]]) boxes = kwimage.Boxes(cxywh, 'cxywh').to_tlbr() iter_ = ub.ProgIter(gid_to_groupx.items(), enabled=verbose, label='select positive regions', total=len(gid_to_groupx), adjust=0, freq=32) for gid, groupx in iter_: # Select all candiate windows in this image cand_windows = boxes.take(groupx, axis=0) # Randomize which candidate windows have the highest scores so the # selection can vary each epoch. cand_scores = rng.rand(len(cand_windows)) cand_dets = kwimage.Detections(boxes=cand_windows, scores=cand_scores) # Non-max supresssion is really similar to set-cover keep = cand_dets.non_max_supression(thresh=thresh) selection.extend(groupx[keep]) selection = np.array(sorted(selection)) return selection
def _kwiver_to_kwimage_detections(detected_objects): """ Convert vital detected object sets to kwimage.Detections Args: detected_objects (kwiver.vital.types.DetectedObjectSet) Returns: kwimage.Detections """ import ubelt as ub import kwimage boxes = [] scores = [] class_idxs = [] classes = [] if len(detected_objects) > 0: obj = ub.peek(detected_objects) classes = obj.type().all_class_names() for obj in detected_objects: box = obj.bounding_box() tlbr = [box.min_x(), box.min_y(), box.max_x(), box.max_y()] score = obj.confidence() cname = obj.type().get_most_likely_class() cidx = classes.index(cname) boxes.append(tlbr) scores.append(score) class_idxs.append(cidx) dets = kwimage.Detections( boxes=kwimage.Boxes(np.array(boxes), 'tlbr'), scores=np.array(scores), class_idxs=np.array(class_idxs), classes=classes, ) return dets
def draw_batch(harn, batch, outputs, batch_dets, idx=None, thresh=None, orig_img=None, num_extra=3): """ Returns: np.ndarray: numpy image Example: >>> # DISABLE_DOCTSET >>> harn = setup_harn(bsize=1, datasets='special:voc', pretrained='lightnet') >>> harn.initialize() >>> batch = harn._demo_batch(0, 'train') >>> outputs, loss = harn.run_batch(batch) >>> batch_dets = harn.raw_model.coder.decode_batch(outputs) >>> stacked = harn.draw_batch(batch, outputs, batch_dets) >>> # xdoc: +REQUIRES(--show) >>> kwplot.autompl() # xdoc: +SKIP >>> kwplot.imshow(stacked) >>> kwplot.show_if_requested() """ import cv2 inputs = batch['im'] labels = batch['label'] orig_sizes = labels['orig_sizes'] classes = harn.datasets['train'].sampler.classes if idx is None: idxs = range(len(inputs)) else: idxs = [idx] imgs = [] for idx in idxs: chw01 = inputs[idx] pred_dets = batch_dets[idx] # pred_dets.meta['classes'] = classes import kwimage true_dets = kwimage.Detections( boxes=kwimage.Boxes(labels['cxywh'][idx], 'cxywh'), class_idxs=labels['class_idxs'][idx].view(-1), weights=labels['weight'][idx], classes=classes, ) pred_dets = pred_dets.numpy() true_dets = true_dets.numpy() true_dets = true_dets.compress(true_dets.class_idxs != -1) if thresh is not None: pred_dets = pred_dets.compress(pred_dets.scores > thresh) # only show so many predictions num_max = len(true_dets) + num_extra sortx = pred_dets.argsort(reverse=True) pred_dets = pred_dets.take(sortx[0:num_max]) hwc01 = chw01.cpu().numpy().transpose(1, 2, 0) inp_size = np.array(hwc01.shape[0:2][::-1]) true_dets.boxes.scale(inp_size, inplace=True) pred_dets.boxes.scale(inp_size, inplace=True) letterbox = harn.datasets[harn.current_tag].letterbox orig_size = orig_sizes[idx].cpu().numpy() target_size = inp_size img = letterbox._img_letterbox_invert(hwc01, orig_size, target_size) img = np.clip(img, 0, 1) # we are given the original image, to avoid artifacts from # inverting a downscale assert orig_img is None or orig_img.shape == img.shape true_dets.data['boxes'] = letterbox._boxes_letterbox_invert( true_dets.boxes, orig_size, target_size) pred_dets.data['boxes'] = letterbox._boxes_letterbox_invert( pred_dets.boxes, orig_size, target_size) # shift, scale, embed_size = letterbox._letterbox_transform(orig_size, target_size) # fig = kwplot.figure(doclf=True, fnum=1) # kwplot.imshow(img, colorspace='rgb') canvas = (img * 255).astype(np.uint8) canvas = true_dets.draw_on(canvas, color='green') canvas = pred_dets.draw_on(canvas, color='blue') canvas = cv2.resize(canvas, (300, 300)) imgs.append(canvas) stacked = imgs[0] if len(imgs) == 1 else kwimage.stack_images_grid( imgs) return stacked
def __getitem__(self, index): """ Example: >>> # DISABLE_DOCTSET >>> self = DetectDataset.demo(backend='npy') >>> index = 0 >>> item = self[index] >>> hwc01 = item['im'].numpy().transpose(1, 2, 0) >>> print(hwc01.shape) >>> norm_boxes = item['label']['targets'].numpy().reshape(-1, 5)[:, 1:5] >>> inp_size = hwc01.shape[-2::-1] >>> # xdoc: +REQUIRES(--show) >>> import kwplot >>> kwplot.figure(doclf=True, fnum=1) >>> kwplot.autompl() # xdoc: +SKIP >>> kwplot.imshow(hwc01) >>> inp_boxes = kwimage.Boxes(norm_boxes, 'cxywh').scale(inp_size) >>> inp_boxes.draw() >>> kwplot.show_if_requested() """ import kwimage if isinstance(index, tuple): # Get size index from the batch loader index, size_index = index if size_index is None: inp_size = self.input_dims else: inp_size = self.multi_scale_inp_size[size_index] else: inp_size = self.input_dims classes = self.sampler.classes coco_dset = self.sampler.dset gid = self.sampler.image_ids[index] img = self.sampler.dset.imgs[gid] tr = { 'gid': gid, 'cx': img['width'] / 2.0, 'cy': img['height'] / 2.0, 'width': img['width'], 'height': img['height'], } sample = self.sampler.load_sample(tr) # sample = self.sampler.load_item(index, window_dims=self.input_dims) annots = sample['annots'] aids = annots['aids'] cids = annots['cids'] image = sample['im'] boxes = annots['rel_boxes'].to_tlbr() dets = kwimage.Detections( boxes=boxes, class_idxs=np.array([classes.id_to_idx[cid] for cid in cids]), weights=np.array( [coco_dset.anns[aid].get('weight', 1.0) for aid in aids]), ) inp_size = np.array(inp_size) orig_size = np.array(image.shape[0:2][::-1]) if self.augmenter: if len(dets): # Ensure the same augmentor is used for bboxes and iamges seq_det = self.augmenter.to_deterministic() input_dims = image.shape[0:2] image = seq_det.augment_image(image) output_dims = image.shape[0:2] dets = dets.warp(seq_det, input_dims=input_dims, output_dims=output_dims) # Clip any bounding boxes that went out of bounds h, w = image.shape[0:2] tlbr = dets.boxes.to_tlbr() old_area = tlbr.area tlbr = tlbr.clip(0, 0, w - 1, h - 1, inplace=True) new_area = tlbr.area dets.data['boxes'] = tlbr # Remove any boxes that have gone significantly out of bounds. remove_thresh = 0.1 flags = (new_area / old_area).ravel() > remove_thresh dets = dets.compress(flags) # Apply letterbox resize transform to train and test self.letterbox.target_size = inp_size input_dims = image.shape[0:2] image = self.letterbox.augment_image(image) output_dims = image.shape[0:2] if len(dets): dets = dets.warp(self.letterbox, input_dims=input_dims, output_dims=output_dims) # Remove any boxes that are no longer visible or out of bounds flags = (dets.boxes.area > 0).ravel() dets = dets.compress(flags) chw01 = torch.FloatTensor(image.transpose(2, 0, 1) / 255.0) # Lightnet YOLO accepts truth tensors in the format: # [class_id, center_x, center_y, w, h] # where coordinates are noramlized between 0 and 1 cxywh_norm = dets.boxes.toformat('cxywh').scale(1 / inp_size) # Return index information in the label as well orig_size = torch.LongTensor(orig_size) index = torch.LongTensor([index]) bg_weight = torch.FloatTensor([1.0]) label = { 'cxywh': torch.FloatTensor(cxywh_norm.data), 'class_idxs': torch.LongTensor(dets.class_idxs[:, None]), 'weight': torch.FloatTensor(dets.weights), 'indices': index, 'orig_sizes': orig_size, 'bg_weights': bg_weight } item = { 'im': chw01, 'label': label, } return item
def demo(cls, **kwargs): """ Creates random true boxes and predicted boxes that have some noisy offset from the truth. Kwargs: nclasses (int, default=1): number of foreground classes. nimgs (int, default=1): number of images in the coco datasts. nboxes (int, default=1): boxes per image. n_fp (int, default=0): number of false positives. n_fn (int, default=0): number of false negatives. box_noise (float, default=0): std of a normal distribution used to perterb both box location and box size. cls_noise (float, default=0): probability that a class label will change. Must be within 0 and 1. anchors (ndarray, default=None): used to create random boxes null_pred (bool, default=0): if True, predicted classes are returned as null, which means only localization scoring is suitable. with_probs (bool, default=1): if True, includes per-class probabilities with predictions Example: >>> # xdoctest: +REQUIRES(module:ndsampler) >>> kwargs = {} >>> # Seed the RNG >>> kwargs['rng'] = 0 >>> # Size parameters determine how big the data is >>> kwargs['nimgs'] = 5 >>> kwargs['nboxes'] = 7 >>> kwargs['nclasses'] = 11 >>> # Noise parameters perterb predictions further from the truth >>> kwargs['n_fp'] = 3 >>> kwargs['box_noise'] = 0.1 >>> kwargs['cls_noise'] = 0.5 >>> dmet = DetectionMetrics.demo(**kwargs) >>> print('dmet.classes = {}'.format(dmet.classes)) dmet.classes = <CategoryTree(nNodes=12, maxDepth=3, maxBreadth=4...)> >>> # Can grab kwimage.Detection object for any image >>> print(dmet.true_detections(gid=0)) <Detections(4)> >>> print(dmet.pred_detections(gid=0)) <Detections(7)> Example: >>> # xdoctest: +REQUIRES(module:ndsampler) >>> # Test case with null predicted categories >>> dmet = DetectionMetrics.demo(nimgs=30, null_pred=1, nclasses=3, >>> nboxes=10, n_fp=10, box_noise=0.3, >>> with_probs=False) >>> dmet.gid_to_pred_dets[0].data >>> dmet.gid_to_true_dets[0].data >>> cfsn_vecs = dmet.confusion_vectors() >>> binvecs_ovr = cfsn_vecs.binarize_ovr() >>> binvecs_per = cfsn_vecs.binarize_peritem() >>> pr_per = binvecs_per.precision_recall() >>> pr_ovr = binvecs_ovr.precision_recall() >>> print('pr_per = {!r}'.format(pr_per)) >>> print('pr_ovr = {!r}'.format(pr_ovr)) >>> # xdoctest: +REQUIRES(--show) >>> import kwplot >>> kwplot.autompl() >>> pr_per.draw(fnum=1) >>> pr_ovr['perclass'].draw(fnum=2) """ import kwimage import kwarray import ndsampler # Parse kwargs rng = kwarray.ensure_rng(kwargs.get('rng', 0)) nclasses = kwargs.get('nclasses', 1) nimgs = kwargs.get('nimgs', 1) box_noise = kwargs.get('box_noise', 0) cls_noise = kwargs.get('cls_noise', 0) null_pred = kwargs.get('null_pred', False) with_probs = kwargs.get('with_probs', True) # specify an amount of overlap between true and false scores score_noise = kwargs.get('score_noise', 0.2) anchors = kwargs.get('anchors', None) scale = 100.0 # Build random variables from kwarray import distributions DiscreteUniform = distributions.DiscreteUniform.seeded(rng=rng) def _parse_arg(key, default): value = kwargs.get(key, default) try: low, high = value return (low, high + 1) except Exception: return (0, value + 1) nboxes_RV = DiscreteUniform(*_parse_arg('nboxes', 1)) n_fp_RV = DiscreteUniform(*_parse_arg('n_fp', 0)) n_fn_RV = DiscreteUniform(*_parse_arg('n_fn', 0)) box_noise_RV = distributions.Normal(0, box_noise, rng=rng) cls_noise_RV = distributions.Bernoulli(cls_noise, rng=rng) # the values of true and false scores starts off with no overlap and # the overlap increases as the score noise increases. def _interp(v1, v2, alpha): return v1 * alpha + (1 - alpha) * v2 mid = 0.5 # true_high = 2.0 true_high = 1.0 true_low = _interp(0, mid, score_noise) false_high = _interp(true_high, mid - 1e-3, score_noise) true_mean = _interp(0.5, .8, score_noise) false_mean = _interp(0.5, .2, score_noise) true_score_RV = distributions.TruncNormal(mean=true_mean, std=.5, low=true_low, high=true_high, rng=rng) false_score_RV = distributions.TruncNormal(mean=false_mean, std=.5, low=0, high=false_high, rng=rng) frgnd_cx_RV = distributions.DiscreteUniform(1, nclasses + 1, rng=rng) # Create the category hierarcy graph = nx.DiGraph() graph.add_node('background', id=0) for cid in range(1, nclasses + 1): # binary heap encoding of a tree cx = cid - 1 parent_cx = (cx - 1) // 2 node = 'cat_{}'.format(cid) graph.add_node(node, id=cid) if parent_cx > 0: supercategory = 'cat_{}'.format(parent_cx + 1) graph.add_edge(supercategory, node) classes = ndsampler.CategoryTree(graph) dmet = cls() dmet.classes = classes for gid in range(nimgs): # Sample random variables nboxes_ = nboxes_RV() n_fp_ = n_fp_RV() n_fn_ = n_fn_RV() imgname = 'img_{}'.format(gid) dmet._register_imagename(imgname, gid) # Generate random ground truth detections true_boxes = kwimage.Boxes.random(num=nboxes_, scale=scale, anchors=anchors, rng=rng, format='cxywh') # Prevent 0 sized boxes: increase w/h by 1 true_boxes.data[..., 2:4] += 1 true_cxs = frgnd_cx_RV(len(true_boxes)) true_weights = np.ones(len(true_boxes), dtype=np.int32) # Initialize predicted detections as a copy of truth pred_boxes = true_boxes.copy() pred_cxs = true_cxs.copy() # Perterb box coordinates pred_boxes.data = np.abs( pred_boxes.data.astype(np.float) + box_noise_RV()) # Perterb class predictions change = cls_noise_RV(len(pred_cxs)) pred_cxs_swap = frgnd_cx_RV(len(pred_cxs)) pred_cxs[change] = pred_cxs_swap[change] # Drop true positive boxes if n_fn_: pred_boxes.data = pred_boxes.data[n_fn_:] pred_cxs = pred_cxs[n_fn_:] # pred_scores = np.linspace(true_min, true_max, len(pred_boxes))[::-1] n_tp_ = len(pred_boxes) pred_scores = true_score_RV(n_tp_) # Add false positive boxes if n_fp_: false_boxes = kwimage.Boxes.random(num=n_fp_, scale=scale, rng=rng, format='cxywh') false_cxs = frgnd_cx_RV(n_fp_) false_scores = false_score_RV(n_fp_) pred_boxes.data = np.vstack( [pred_boxes.data, false_boxes.data]) pred_cxs = np.hstack([pred_cxs, false_cxs]) pred_scores = np.hstack([pred_scores, false_scores]) # Transform the scores for the assigned class into a predicted # probability for each class. (Currently a bit hacky). class_probs = _demo_construct_probs(pred_cxs, pred_scores, classes, rng, hacked=kwargs.get('hacked', 0)) true_dets = kwimage.Detections(boxes=true_boxes, class_idxs=true_cxs, weights=true_weights) pred_dets = kwimage.Detections(boxes=pred_boxes, class_idxs=pred_cxs, scores=pred_scores) # Hack in the probs if with_probs: pred_dets.data['probs'] = class_probs if null_pred: pred_dets.data['class_idxs'] = np.array([None] * len(pred_dets), dtype=object) dmet.add_truth(true_dets, imgname=imgname) dmet.add_predictions(pred_dets, imgname=imgname) return dmet
def _devcheck_load_sub_image(): import kwimage import numpy as np sampler = grab_camvid_sampler() cid_to_cidx = sampler.catgraph.id_to_idx classes = sampler.catgraph # Try loading a subregion of an image sample = sampler.load_positive(2) imdata = sample['im'] annots = sample['annots'] aids = annots['aids'] cids = annots['cids'] boxes = annots['rel_boxes'] class_idxs = np.array([cid_to_cidx[cid] for cid in cids]) segmentations = annots['rel_ssegs'] raw_dets = kwimage.Detections( aids=aids, boxes=boxes, class_idxs=class_idxs, segmentations=segmentations, classes=classes, datakeys=['aids'], ) # Clip boxes to the image boundary input_dims = imdata.shape[0:2] raw_dets.data['boxes'] = raw_dets.boxes.clip(0, 0, input_dims[1], input_dims[0]) keep = [] for i, s in enumerate(raw_dets.data['segmentations']): # TODO: clip polygons m = s.to_mask(input_dims) if m.area > 0: keep.append(i) dets = raw_dets.take(keep) heatmap = dets.rasterize(bg_size=(1, 1), input_dims=input_dims) if 1: print('dets = {!r}'.format(dets)) print('dets.data = {!r}'.format(dets.data)) print('dets.meta = {!r}'.format(dets.meta)) if ub.argflag('--show'): import kwplot kwplot.autompl() heatmap.draw() draw_boxes = 1 kwplot.figure(doclf=True) with ub.Timer('dets.draw_on'): canvas = imdata.copy() # TODO: add logic to color by class canvas = dets.draw_on(canvas, boxes=draw_boxes, color='random') kwplot.imshow(canvas, pnum=(1, 2, 1), title='dets.draw_on') with ub.Timer('dets.draw'): kwplot.imshow(imdata, pnum=(1, 2, 2), docla=True, title='dets.draw') dets.draw(boxes=draw_boxes, color='random')
def _decode(self, output): """ Returns array of detections for every image in batch CommandLine: python ~/code/netharn/netharn/box_models/yolo2/light_postproc.py GetBoundingBoxes._decode Examples: >>> # xdoctest: +REQUIRES(module:kwimage) >>> import torch >>> torch.random.manual_seed(0) >>> anchors = np.array([(1.3221, 1.73145), (3.19275, 4.00944), (5.05587, 8.09892), (9.47112, 4.84053), (11.2364, 10.0071)]) >>> self = GetBoundingBoxes(anchors=anchors, num_classes=20, conf_thresh=.14, nms_thresh=0.5) >>> output = torch.randn(16, 5, 5 + 20, 9, 9) >>> from netharn import XPU >>> output = XPU.coerce('auto').move(output) >>> batch_dets = self._decode(output.data) >>> assert len(batch_dets) == 16 Ignore: >>> from netharn.models.yolo2.yolo2 import * # NOQA >>> info = dev_demodata() >>> outputs = info['outputs'] >>> cxywh_energy = output['cxywh_energy'] >>> raw = info['raw'] >>> raw_ = raw.clone() >>> self = GetBoundingBoxes(anchors=info['model'].anchors, num_classes=20, conf_thresh=.14, nms_thresh=0.5) >>> dets = self._decode(raw)[0] >>> dets.scores >>> self, output = ub.take(info, ['coder', 'outputs']) >>> batch_dets = self.decode_batch(output) >>> dets = batch_dets[0] >>> dets.scores """ import kwimage # dont modify inplace raw_ = output.clone() # Variables bsize = raw_.shape[0] h, w = raw_.shape[-2:] device = raw_.device if self.anchors.device != device: self.anchors = self.anchors.to(device) # Compute xc,yc, w,h, box_score on Tensor lin_x = torch.linspace(0, w - 1, w, device=device).repeat(h, 1).view(h * w) lin_y = torch.linspace(0, h - 1, h, device=device).repeat( w, 1).t().contiguous().view(h * w) anchor_w = self.anchors[:, 0].contiguous().view(1, self.num_anchors, 1) anchor_h = self.anchors[:, 1].contiguous().view(1, self.num_anchors, 1) # -1 == 5+num_classes (we can drop feature maps if 1 class) output_ = raw_.view(bsize, self.num_anchors, -1, h * w) output_[:, :, 0, :].sigmoid_().add_(lin_x).div_(w) # X center output_[:, :, 1, :].sigmoid_().add_(lin_y).div_(h) # Y center output_[:, :, 2, :].exp_().mul_(anchor_w).div_(w) # Width output_[:, :, 3, :].exp_().mul_(anchor_h).div_(h) # Height output_[:, :, 4, :].sigmoid_() # Box score # output_[:, :, 0:4].sum() # torch.all(cxywh.view(-1) == output_[:, :, 0:4].contiguous().view(-1)) # Compute class_score if self.num_classes > 1: cls_scores = torch.nn.functional.softmax(output_[:, :, 5:, :], 2) cls_max, cls_max_idx = torch.max(cls_scores, 2) cls_max.mul_(output_[:, :, 4, :]) else: cls_max = output_[:, :, 4, :] cls_max_idx = torch.zeros_like(cls_max) # Save detection if conf*class_conf is higher than threshold # Newst lightnet code, which is based on my mode1 code score_thresh = cls_max > self.conf_thresh score_thresh_flat = score_thresh.view(-1) if score_thresh.sum() == 0: batch_dets = [] for i in range(bsize): batch_dets.append( kwimage.Detections( boxes=kwimage.Boxes( torch.empty((0, 4), dtype=torch.float32, device=device), 'cxywh'), scores=torch.empty(0, dtype=torch.float32, device=device), class_idxs=torch.empty(0, dtype=torch.int64, device=device), )) else: # Mask select boxes > conf_thresh coords = output_.transpose(2, 3)[..., 0:4] coords = coords[score_thresh[..., None].expand_as(coords)].view(-1, 4) scores = cls_max[score_thresh] class_idxs = cls_max_idx[score_thresh] stacked_dets = kwimage.Detections( boxes=kwimage.Boxes(coords, 'cxywh'), scores=scores, class_idxs=class_idxs, ) # Get indexes of splits between images of batch max_det_per_batch = len(self.anchors) * h * w slices = [ slice(max_det_per_batch * i, max_det_per_batch * (i + 1)) for i in range(bsize) ] det_per_batch = torch.IntTensor( [score_thresh_flat[s].int().sum() for s in slices]) split_idx = torch.cumsum(det_per_batch, dim=0) batch_dets = [] start = 0 for end in split_idx: dets = stacked_dets[start:end] dets = dets.non_max_supress(thresh=self.nms_thresh) batch_dets.append(dets) start = end return batch_dets
def decode_batch(self, output, forloss=False): """ Returns array of detections for every image in batch Example: >>> # xdoc: +REQUIRES(--download, module:ndsampler) >>> from netharn.models.yolo2.yolo2 import * # NOQA >>> self = YoloCoder.demo() >>> output = self.demo_output() >>> batch_dets = self.decode_batch(output) >>> batch_dets = self.decode_batch(output, forloss=True) Example: >>> # xdoc: +REQUIRES(--download, module:ndsampler) >>> info = dev_demodata() >>> self, output = ub.take(info, ['coder', 'outputs']) >>> batch_dets = self.decode_batch(output) >>> dets = batch_dets[0].sort().scale(info['orig_sizes'][0]) >>> print('dets.boxes = {!r}'.format(dets.boxes)) >>> # xdoctest: +REQUIRES(--show) >>> import kwplot >>> kwplot.figure(fnum=1, doclf=True) >>> kwplot.imshow(info['rgb255'], colorspace='rgb') >>> dets.draw() >>> kwplot.show_if_requested() """ import kwimage # dont modify inplace # output = output.clone() class_energy = output['class_energy'] score_energy = output['score_energy'] cxywh_energy = output['cxywh_energy'] # Variables nB = class_energy.shape[0] nH, nW = class_energy.shape[-2:] nA = self.num_anchors device = class_energy.device if self.anchors.device != device: self.anchors = self.anchors.to(device) # Compute xc,yc, nW,nH, box_score on Tensor lin_x = torch.linspace(0, nW - 1, nW, device=device).repeat(nH, 1) lin_y = torch.linspace(0, nH - 1, nH, device=device).repeat(nW, 1).t().contiguous() anchor_w = self.anchors[:, 0].contiguous().view(1, nA, 1, 1) anchor_h = self.anchors[:, 1].contiguous().view(1, nA, 1, 1) if forloss: # TODO : rectify coord = torch.empty_like(cxywh_energy) coord[:, :, 0:2, :, :] = cxywh_energy[:, :, 0:2, :, :].sigmoid() # cx,cy coord[:, :, 2:4, :, :] = cxywh_energy[:, :, 2:4, :, :] # w,h with torch.no_grad(): pred_boxes = torch.empty_like(cxywh_energy, device=device).view(-1, 4) pred_boxes[:, 0] = (coord[:, :, 0, :, :] + lin_x).view(-1) pred_boxes[:, 1] = (coord[:, :, 1, :, :] + lin_y).view(-1) pred_boxes[:, 2] = (coord[:, :, 2, :, :].exp() * anchor_w).view(-1) pred_boxes[:, 3] = (coord[:, :, 3, :, :].exp() * anchor_h).view(-1) info = { 'coord': coord, 'pred_boxes': pred_boxes, } return info else: cxywh = cxywh_energy.clone() # cxywh_ = cxywh.view(nB, self.num_anchors, -1, nH, nW) cxywh[:, :, 0, :].sigmoid_().add_(lin_x).div_(nW) # X center cxywh[:, :, 1, :].sigmoid_().add_(lin_y).div_(nH) # Y center cxywh[:, :, 2, :].exp_().mul_(anchor_w).div_(nW) # Width cxywh[:, :, 3, :].exp_().mul_(anchor_h).div_(nH) # Height score = score_energy.sigmoid() # Box score # Compute class_score if len(self.classes) > 1: cls_scores = torch.nn.functional.softmax(class_energy, dim=2) cls_max, cls_max_idx = torch.max(cls_scores, 2, keepdim=True) cls_max.mul_(score) else: cls_max = score cls_max_idx = torch.zeros_like(cls_max) # Save detection if conf*class_conf is higher than threshold flags = cls_max >= self.conf_thresh flags_flat = flags.view(-1) if flags.sum() == 0: batch_dets = [] for i in range(nB): batch_dets.append(kwimage.Detections( boxes=kwimage.Boxes(torch.empty((0, 4), dtype=torch.float32, device=device), 'cxywh'), scores=torch.empty(0, dtype=torch.float32, device=device), class_idxs=torch.empty(0, dtype=torch.int64, device=device), classes=self.classes )) else: # Permute so the bbox dim (i.e. xywh) is trailing coords = cxywh.permute(0, 1, 3, 4, 2).contiguous().view(-1, 4) coords = coords[flags.view(-1)] scores = cls_max[flags] class_idxs = cls_max_idx[flags] stacked_dets = kwimage.Detections( boxes=kwimage.Boxes(coords, 'cxywh'), scores=scores, class_idxs=class_idxs, classes=self.classes ) # Get indexes of splits between images of batch max_det_per_batch = len(self.anchors) * nH * nW m = max_det_per_batch flags_flat.int = flags_flat.int() slices = [slice(m * i, m * (i + 1)) for i in range(nB)] det_per_batch = torch.IntTensor([flags_flat[s].sum() for s in slices]) split_idx = torch.cumsum(det_per_batch, dim=0) batch_dets = [] start = 0 for end in split_idx: dets = stacked_dets[start: end] dets = dets.non_max_supress(thresh=self.nms_thresh) batch_dets.append(dets) start = end return batch_dets
def perterb_coco(coco_dset, **kwargs): """ Perterbs a coco dataset Args: rng (int, default=0): box_noise (int, default=0): cls_noise (int, default=0): null_pred (bool, default=False): with_probs (bool, default=False): score_noise (float, default=0.2): hacked (int, default=1): Example: >>> from kwcoco.demo.perterb import * # NOQA >>> from kwcoco.demo.perterb import _demo_construct_probs >>> import kwcoco >>> coco_dset = true_dset = kwcoco.CocoDataset.demo('shapes8') >>> kwargs = { >>> 'box_noise': 0.5, >>> 'n_fp': 3, >>> 'with_probs': 1, >>> } >>> pred_dset = perterb_coco(true_dset, **kwargs) >>> pred_dset._check_json_serializable() Ignore: import xdev from kwcoco.demo.perterb import perterb_coco # NOQA defaultkw = xdev.get_func_kwargs(perterb_coco) for k, v in defaultkw.items(): desc = '' print('{} ({}, default={}): {}'.format(k, type(v).__name__, v, desc)) """ import kwimage import kwarray # Parse kwargs rng = kwarray.ensure_rng(kwargs.get('rng', 0)) box_noise = kwargs.get('box_noise', 0) cls_noise = kwargs.get('cls_noise', 0) null_pred = kwargs.get('null_pred', False) with_probs = kwargs.get('with_probs', False) # specify an amount of overlap between true and false scores score_noise = kwargs.get('score_noise', 0.2) # Build random variables from kwarray import distributions DiscreteUniform = distributions.DiscreteUniform.seeded(rng=rng) def _parse_arg(key, default): value = kwargs.get(key, default) try: low, high = value return (low, high + 1) except Exception: return (value, value + 1) n_fp_RV = DiscreteUniform(*_parse_arg('n_fp', 0)) n_fn_RV = DiscreteUniform(*_parse_arg('n_fn', 0)) box_noise_RV = distributions.Normal(0, box_noise, rng=rng) cls_noise_RV = distributions.Bernoulli(cls_noise, rng=rng) # the values of true and false scores starts off with no overlap and # the overlap increases as the score noise increases. def _interp(v1, v2, alpha): return v1 * alpha + (1 - alpha) * v2 mid = 0.5 # true_high = 2.0 true_high = 1.0 false_low = 0.0 true_low = _interp(0, mid, score_noise) false_high = _interp(true_high, mid - 1e-3, score_noise) true_mean = _interp(0.5, .8, score_noise) false_mean = _interp(0.5, .2, score_noise) true_score_RV = distributions.TruncNormal(mean=true_mean, std=.5, low=true_low, high=true_high, rng=rng) false_score_RV = distributions.TruncNormal(mean=false_mean, std=.5, low=false_low, high=false_high, rng=rng) # Create the category hierarcy classes = coco_dset.object_categories() cids = coco_dset.cats.keys() cidxs = [classes.id_to_idx[c] for c in cids] frgnd_cx_RV = distributions.CategoryUniform(cidxs, rng=rng) new_dset = coco_dset.copy() remove_aids = [] false_anns = [] index_invalidated = False for gid in coco_dset.imgs.keys(): # Sample random variables n_fp_ = n_fp_RV() n_fn_ = n_fn_RV() true_annots = coco_dset.annots(gid=gid) aids = true_annots.aids for aid in aids: # Perterb box coordinates ann = new_dset.anns[aid] new_bbox = (np.array(ann['bbox']) + box_noise_RV(4)).tolist() new_x, new_y, new_w, new_h = new_bbox allow_neg_boxes = 0 if not allow_neg_boxes: new_w = max(new_w, 0) new_h = max(new_h, 0) ann['bbox'] = [new_x, new_y, new_w, new_h] ann['score'] = float(true_score_RV(1)[0]) if cls_noise_RV(): # Perterb class predictions ann['category_id'] = classes.idx_to_id[frgnd_cx_RV()] index_invalidated = True # Drop true positive boxes if n_fn_: import kwarray drop_idxs = kwarray.shuffle(np.arange(len(aids)), rng=rng)[0:n_fn_] remove_aids.extend(list(ub.take(aids, drop_idxs))) # Add false positive boxes if n_fp_: try: img = coco_dset.imgs[gid] scale = (img['width'], img['height']) except KeyError: scale = 100 false_boxes = kwimage.Boxes.random(num=n_fp_, scale=scale, rng=rng, format='cxywh') false_cxs = frgnd_cx_RV(n_fp_) false_scores = false_score_RV(n_fp_) false_dets = kwimage.Detections( boxes=false_boxes, class_idxs=false_cxs, scores=false_scores, classes=classes, ) for ann in list(false_dets.to_coco('new')): ann['category_id'] = classes.node_to_id[ann.pop( 'category_name')] ann['image_id'] = gid false_anns.append(ann) if null_pred: raise NotImplementedError if index_invalidated: new_dset.index.clear() new_dset._build_index() new_dset.remove_annotations(remove_aids) for ann in false_anns: new_dset.add_annotation(**ann) # Hack in the probs if with_probs: annots = new_dset.annots() pred_cids = annots.lookup('category_id') pred_cxs = np.array([classes.id_to_idx[cid] for cid in pred_cids]) pred_scores = np.array(annots.lookup('score')) # Transform the scores for the assigned class into a predicted # probability for each class. (Currently a bit hacky). pred_probs = _demo_construct_probs(pred_cxs, pred_scores, classes, rng, hacked=kwargs.get('hacked', 1)) for aid, prob in zip(annots.aids, pred_probs): new_dset.anns[aid]['prob'] = prob.tolist() return new_dset
def draw_batch(harn, batch, outputs, batch_dets, idx=None, thresh=None, orig_img=None): """ Returns: np.ndarray: numpy image Example: >>> # DISABLE_DOCTSET >>> harn = setup_yolo_harness(bsize=1) >>> harn.initialize() >>> batch = harn._demo_batch(0, 'train') >>> weights_fpath = light_yolo.demo_voc_weights() >>> state_dict = harn.xpu.load(weights_fpath)['weights'] >>> harn.model.module.load_state_dict(state_dict) >>> outputs, loss = harn.run_batch(batch) >>> harn.on_batch(batch, outputs, loss) >>> # xdoc: +REQUIRES(--show) >>> import kwplot >>> batch_dets = harn.model.module.postprocess(outputs) >>> kwplot.autompl() # xdoc: +SKIP >>> stacked = harn.draw_batch(batch, outputs, batch_dets, thresh=0.01) >>> kwplot.imshow(stacked) >>> kwplot.show_if_requested() """ import cv2 import kwimage inputs, labels = batch targets = labels['targets'] orig_sizes = labels['orig_sizes'] if idx is None: idxs = range(len(inputs)) else: idxs = [idx] imgs = [] for idx in idxs: chw01 = inputs[idx] target = targets[idx].view(-1, 5) pred_dets = batch_dets[idx] label_names = harn.datasets[harn.current_tag].label_names pred_dets.meta['classes'] = label_names true_dets = kwimage.Detections(boxes=kwimage.Boxes( target[:, 1:5], 'cxywh'), class_idxs=target[:, 0].int(), classes=label_names) pred_dets = pred_dets.numpy() true_dets = true_dets.numpy() true_dets = true_dets.compress(true_dets.class_idxs != -1) if thresh is not None: pred_dets = pred_dets.compress(pred_dets.scores > thresh) hwc01 = chw01.cpu().numpy().transpose(1, 2, 0) inp_size = np.array(hwc01.shape[0:2][::-1]) true_dets.boxes.scale(inp_size, inplace=True) pred_dets.boxes.scale(inp_size, inplace=True) letterbox = harn.datasets[harn.current_tag].letterbox orig_size = orig_sizes[idx].cpu().numpy() target_size = inp_size img = letterbox._img_letterbox_invert(hwc01, orig_size, target_size) img = np.clip(img, 0, 1) # we are given the original image, to avoid artifacts from # inverting a downscale assert orig_img is None or orig_img.shape == img.shape true_dets.data['boxes'] = letterbox._boxes_letterbox_invert( true_dets.boxes, orig_size, target_size) pred_dets.data['boxes'] = letterbox._boxes_letterbox_invert( pred_dets.boxes, orig_size, target_size) # shift, scale, embed_size = letterbox._letterbox_transform(orig_size, target_size) # fig = kwplot.figure(doclf=True, fnum=1) # kwplot.imshow(img, colorspace='rgb') canvas = (img * 255).astype(np.uint8) canvas = true_dets.draw_on(canvas, color='green') canvas = pred_dets.draw_on(canvas, color='blue') canvas = cv2.resize(canvas, (300, 300)) imgs.append(canvas) # if IS_PROFILING: # torch.cuda.synchronize() stacked = imgs[0] if len(imgs) == 1 else kwimage.stack_images_grid( imgs) return stacked