def draw_perclass_prcurve(cx_to_peritem, classes=None, prefix='', fnum=1, **kw): """ Example: >>> # xdoctest: +REQUIRES(module:ndsampler) >>> # xdoctest: +REQUIRES(module:kwplot) >>> from netharn.metrics import DetectionMetrics >>> dmet = DetectionMetrics.demo( >>> nimgs=10, nboxes=(0, 10), n_fp=(0, 1), nclasses=3) >>> cfsn_vecs = dmet.confusion_vectors() >>> classes = cfsn_vecs.classes >>> cx_to_peritem = cfsn_vecs.binarize_ovr().precision_recall()['perclass'] >>> # xdoctest: +REQUIRES(--show) >>> import kwplot >>> kwplot.autompl() >>> draw_perclass_prcurve(cx_to_peritem, classes) >>> kwplot.show_if_requested() """ import kwplot # Sort by descending AP cxs = list(cx_to_peritem.keys()) priority = np.array([item['ap'] for item in cx_to_peritem.values()]) priority[np.isnan(priority)] = -np.inf cxs = list(ub.take(cxs, np.argsort(priority)))[::-1] aps = [] xydata = ub.odict() for cx in cxs: peritem = cx_to_peritem[cx] catname = classes[cx] if isinstance(cx, int) else cx ap = peritem['ap'] if 'pr' in peritem: pr = peritem['pr'] elif 'ppv' in peritem: pr = (peritem['ppv'], peritem['tpr']) elif 'prec' in peritem: pr = (peritem['prec'], peritem['rec']) else: raise KeyError('pr, prec, or ppv not in peritem') if np.isfinite(ap): aps.append(ap) (precision, recall) = pr else: aps.append(np.nan) precision, recall = [0], [0] if precision is None and recall is None: # I thought AP=nan in this case, but I missed something precision, recall = [0], [0] nsupport = int(peritem['nsupport']) if 'realpos_total' in peritem: z = peritem['realpos_total'] if abs(z - int(z)) < 1e-8: label = 'ap={:0.2f}: {} ({:d}/{:d})'.format(ap, catname, int(peritem['realpos_total']), round(nsupport, 2)) else: label = 'ap={:0.2f}: {} ({:.2f}/{:d})'.format(ap, catname, round(peritem['realpos_total'], 2), round(nsupport, 2)) else: label = 'ap={:0.2f}: {} ({:d})'.format(ap, catname, round(nsupport, 2)) xydata[label] = (recall, precision) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Mean of empty slice', RuntimeWarning) mAP = np.nanmean(aps) ax = kwplot.multi_plot( xydata=xydata, fnum=fnum, xlim=(0, 1), ylim=(0, 1), xpad=0.01, ypad=0.01, xlabel='recall', ylabel='precision', title=prefix + 'perclass mAP={:.4f}'.format(mAP), legend_loc='lower right', color='distinct', linestyle='cycle', marker='cycle', **kw ) return ax
def _devcheck_voc_consistency2(): """ # CHECK FOR ISSUES WITH MY MAP COMPUTATION TODO: Check how cocoeval works https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py """ import pandas as pd from kwcoco.metrics.detections import DetectionMetrics xdata = [] ydatas = ub.ddict(list) dmets = [] for box_noise in np.linspace(0, 8, 20): dmet = DetectionMetrics.demo( nimgs=20, nboxes=(0, 20), classes=3, rng=0, # anchors=np.array([[.5, .5], [.3, .3], [.1, .3], [.2, .1]]), box_noise=box_noise, # n_fp=0 if box_noise == 0 else (0, 3), # n_fn=0 if box_noise == 0 else (0, 3), # cls_noise=0 if box_noise == 0 else .3, ) dmets.append(dmet) nh_scores = dmet.score_kwcoco(bias=0) voc_scores = dmet.score_voc(bias=0) coco_scores = dmet.score_coco() nh_map = nh_scores['mAP'] voc_map = voc_scores['mAP'] coco_map = coco_scores['mAP'] print('nh_map = {!r}'.format(nh_map)) print('voc_map = {!r}'.format(voc_map)) print('coco_map = {!r}'.format(coco_map)) xdata.append(box_noise) ydatas['voc'].append(voc_map) ydatas['kwcoco'].append(nh_map) ydatas['coco'].append(coco_map) ydf = pd.DataFrame(ydatas) print(ydf) import kwplot kwplot.autompl() kwplot.multi_plot(xdata=xdata, ydata=ydatas, fnum=1, doclf=True) if False: dmet_ = dmets[-1] dmet_ = dmets[0] print('true = ' + ub.repr2(dmet_.true.dataset, nl=2, precision=2)) print('pred = ' + ub.repr2(dmet_.pred.dataset, nl=2, precision=2)) dmet = DetectionMetrics() for gid in range(0, 5): print('----') print('gid = {!r}'.format(gid)) dmet.true = dmet_.true.subset([gid]) dmet.pred = dmet_.pred.subset([gid]) nh_scores = dmet.score_kwcoco(bias=0) voc_scores = dmet.score_voc(bias=0) coco_scores = dmet.score_coco() nh_map = nh_scores['mAP'] voc_map = voc_scores['mAP'] coco_map = coco_scores['mAP'] print('nh_map = {!r}'.format(nh_map)) print('voc_map = {!r}'.format(voc_map)) print('coco_map = {!r}'.format(coco_map)) print('true = ' + ub.repr2(dmet.true.dataset, nl=2)) print('pred = ' + ub.repr2(dmet.pred.dataset, nl=2))
def _devcheck_voc_consistency(): """ # CHECK FOR ISSUES WITH MY MAP COMPUTATION TODO: Check how cocoeval works https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py """ import pandas as pd import kwcoco as nh # method = 'voc2012' method = 'voc2007' bias = 0 bias = 0 # classes = [0, 1, 2] classes = [0] classname = 0 # nimgs = 5 # nboxes = 2 nimgs = 5 nboxes = 5 nbad = 1 bg_weight = 1.0 iou_thresh = 0.5 bg_cls = -1 xdata = [] ydatas = ub.ddict(list) for noise in np.linspace(0, 5, 10): recs = {} lines = [] confusions = [] rng = np.random.RandomState(0) detmetrics = DetectionMetrics() true_coco = nh.data.coco_api.CocoDataset() pred_coco = nh.data.coco_api.CocoDataset() cid = true_coco.add_category('cat1') cid = pred_coco.add_category('cat1') for imgname in range(nimgs): # Create voc style data imgname = str(imgname) import kwimage true_boxes = kwimage.Boxes.random(num=nboxes, scale=100., rng=rng, format='cxywh') pred_boxes = true_boxes.copy() pred_boxes.data = pred_boxes.data.astype( np.float) + (rng.rand() * noise) if nbad: pred_boxes.data = np.vstack([ pred_boxes.data, kwimage.Boxes.random(num=nbad, scale=100., rng=rng, format='cxywh').data ]) true_cxs = rng.choice(classes, size=len(true_boxes)) pred_cxs = true_cxs.copy() change = rng.rand(len(true_cxs)) < (noise / 5) pred_cxs_swap = rng.choice(classes, size=len(pred_cxs)) pred_cxs[change] = pred_cxs_swap[change] if nbad: pred_cxs = np.hstack( [pred_cxs, rng.choice(classes, size=nbad)]) np.array([0] * len(true_boxes)) pred_cxs = np.array([0] * len(pred_boxes)) recs[imgname] = [] for bbox in true_boxes.to_tlbr().data: recs[imgname].append({ 'bbox': bbox, 'difficult': False, 'name': classname }) for bbox, score in zip(pred_boxes.to_tlbr().data, np.arange(len(pred_boxes))): lines.append([imgname, score] + list(bbox)) # lines.append('{} {} {} {} {} {}'.format(imgname, score, *bbox)) # Create MS-COCO style data gid = true_coco.add_image(imgname) gid = pred_coco.add_image(imgname) for bbox in true_boxes.to_xywh(): true_coco.add_annotation(gid, cid, bbox=bbox, iscrowd=False, ignore=0, area=bbox.area[0]) for bbox, score in zip(pred_boxes.to_xywh(), np.arange(len(pred_boxes))): pred_coco.add_annotation(gid, cid, bbox=bbox, iscrowd=False, ignore=0, score=score, area=bbox.area[0]) # Create kwcoco style confusion data true_weights = np.array([1] * len(true_boxes)) pred_scores = np.arange(len(pred_boxes)) y = pd.DataFrame( detection_confusions(true_boxes, true_cxs, true_weights, pred_boxes, pred_scores, pred_cxs, bg_weight=1.0, iou_thresh=0.5, bg_cls=-1, bias=bias)) y['gx'] = int(imgname) y = (y) confusions.append(y) from pycocotools import cocoeval as coco_score cocoGt = true_coco._aspycoco() cocoDt = pred_coco._aspycoco() evaler = coco_score.COCOeval(cocoGt, cocoDt, iouType='bbox') evaler.evaluate() evaler.accumulate() evaler.summarize() coco_ap = evaler.stats[1] y = pd.concat(confusions) mine_ap = score_detection_assignment(y, method=method)['ap'] voc_rec, voc_prec, voc_ap = voc_eval(lines, recs, classname, iou_thresh=0.5, method=method, bias=bias) eav_prec, eav_rec, eav_ap1 = _multiclass_ap(y) eav_ap2 = _ave_precision(eav_rec, eav_prec, method=method) voc_ap2 = _ave_precision(voc_rec, voc_prec, method=method) eav_ap = eav_ap2 print('noise = {!r}'.format(noise)) print('mine_ap = {!r}'.format(mine_ap.values.mean())) print('voc_ap = {!r}'.format(voc_ap)) print('eav_ap = {!r}'.format(eav_ap)) print('---') xdata.append(noise) ydatas['voc'].append(voc_ap) ydatas['eav'].append(eav_ap) ydatas['kwcoco'].append(mine_ap.values.mean()) ydatas['coco'].append(coco_ap) ydf = pd.DataFrame(ydatas) print(ydf) import kwplot kwplot.autompl() kwplot.multi_plot(xdata=xdata, ydata=ydatas, fnum=1, doclf=True)
def draw_threshold_curves(info, keys=None, prefix='', fnum=1, **kw): """ Args: info (Measures | Dict) Example: >>> # xdoctest: +REQUIRES(module:kwplot) >>> import sys, ubelt >>> sys.path.append(ubelt.expandpath('~/code/kwcoco')) >>> from kwcoco.metrics.drawing import * # NOQA >>> from kwcoco.metrics import DetectionMetrics >>> dmet = DetectionMetrics.demo( >>> nimgs=10, nboxes=(0, 10), n_fp=(0, 1), classes=3) >>> cfsn_vecs = dmet.confusion_vectors() >>> info = cfsn_vecs.binarize_classless().measures() >>> keys = None >>> import kwplot >>> kwplot.autompl() >>> draw_threshold_curves(info, keys) >>> # xdoctest: +REQUIRES(--show) >>> kwplot.show_if_requested() """ import kwplot import kwimage thresh = info['thresholds'] if keys is None: keys = {'g1', 'f1', 'acc', 'mcc'} idx_to_colors = kwimage.Color.distinct(len(keys), space='rgba') idx_to_best_pt = {} xydata = {} colors = {} finite_flags = np.isfinite(thresh) for idx, key in enumerate(keys): color = idx_to_colors[idx] measure = info[key][finite_flags] if len(measure): try: max_idx = np.nanargmax(measure) offset = (~finite_flags[:max_idx]).sum() max_idx += offset best_thresh = thresh[max_idx] best_measure = measure[max_idx] best_label = '{}={:0.2f}@{:0.2f}'.format(key, best_measure, best_thresh) except ValueError: best_thresh = np.nan best_measure = np.nan else: best_thresh = np.nan best_measure = np.nan best_label = '{}={:0.2f}@{:0.2f}'.format(key, best_measure, best_thresh) label_suffix = _realpos_label_suffix(info) label = '{}: ({})'.format(best_label, label_suffix) xydata[label] = (thresh, measure) colors[label] = color idx_to_best_pt[idx] = (best_thresh, best_measure) ax = kwplot.multi_plot( xydata=xydata, fnum=fnum, xlim=(0, 1), ylim=(0, 1), xpad=0.01, ypad=0.01, xlabel='threshold', ylabel=key, title=prefix + 'threshold curves', legend_loc='lower right', color=colors, linestyle='cycle', marker='cycle', **kw ) for idx, best_pt in idx_to_best_pt.items(): best_thresh, best_measure = best_pt color = idx_to_colors[idx] ax.plot(best_thresh, best_measure, '*', color=color) return ax
def bench_bbox_iou_method(): """ On my system the torch impl was fastest (when the data was on the GPU). """ from kwimage.structs.boxes import _box_ious_torch, _box_ious_py, _bbox_ious_c ydata = ub.ddict(list) xdata = [ 10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 2000, 4000 ] bias = 0 if _bbox_ious_c is None: print('CYTHON IMPLEMENATION IS NOT AVAILABLE') for num in xdata: results = {} # Setup Timer N = max(20, int(1000 / num)) ti = ub.Timerit(N, bestof=10) # Setup input dat boxes1 = kwimage.Boxes.random(num, scale=10.0, rng=0, format='ltrb') boxes2 = kwimage.Boxes.random(num + 1, scale=10.0, rng=1, format='ltrb') ltrb1 = boxes1.tensor().data ltrb2 = boxes2.tensor().data for timer in ti.reset('iou-torch-cpu'): with timer: out = _box_ious_torch(ltrb1, ltrb2, bias) results[ti.label] = out.data.cpu().numpy() ydata[ti.label].append(ti.mean()) gpu = torch.device(0) ltrb1 = boxes1.tensor().data.to(gpu) ltrb2 = boxes2.tensor().data.to(gpu) for timer in ti.reset('iou-torch-gpu'): with timer: out = _box_ious_torch(ltrb1, ltrb2, bias) torch.cuda.synchronize() results[ti.label] = out.data.cpu().numpy() ydata[ti.label].append(ti.mean()) ltrb1 = boxes1.numpy().data ltrb2 = boxes2.numpy().data for timer in ti.reset('iou-numpy'): with timer: out = _box_ious_py(ltrb1, ltrb2, bias) results[ti.label] = out ydata[ti.label].append(ti.mean()) if _bbox_ious_c: ltrb1 = boxes1.numpy().data.astype(np.float32) ltrb2 = boxes2.numpy().data.astype(np.float32) for timer in ti.reset('iou-cython'): with timer: out = _bbox_ious_c(ltrb1, ltrb2, bias) results[ti.label] = out ydata[ti.label].append(ti.mean()) eq = partial(np.allclose, atol=1e-07) passed = ub.allsame(results.values(), eq) if passed: print( 'All methods produced the same answer for num={}'.format(num)) else: for k1, k2 in it.combinations(results.keys(), 2): v1 = results[k1] v2 = results[k2] if eq(v1, v2): print('pass: {} == {}'.format(k1, k2)) else: diff = np.abs(v1 - v2) print( 'FAIL: {} != {}: diff(max={}, mean={}, sum={})'.format( k1, k2, diff.max(), diff.mean(), diff.sum())) raise AssertionError('different methods report different results') print('num = {!r}'.format(num)) print('ti.measures = {}'.format( ub.repr2(ub.map_vals(ub.sorted_vals, ti.measures), align=':', nl=2, precision=6))) import kwplot kwplot.autoplt() kwplot.multi_plot(xdata, ydata, xlabel='num boxes', ylabel='seconds') kwplot.show_if_requested()
def draw_roc(info, prefix='', fnum=1, **kw): """ Args: info (Measures | Dict) NOTE: There needs to be enough negative examples for using ROC to make any sense! Example: >>> # xdoctest: +REQUIRES(module:kwplot, module:seaborn) >>> from kwcoco.metrics.drawing import * # NOQA >>> from kwcoco.metrics import DetectionMetrics >>> dmet = DetectionMetrics.demo(nimgs=30, null_pred=1, classes=3, >>> nboxes=10, n_fp=10, box_noise=0.3, >>> with_probs=False) >>> dmet.true_detections(0).data >>> cfsn_vecs = dmet.confusion_vectors(compat='mutex', prioritize='iou', bias=0) >>> print(cfsn_vecs.data._pandas().sort_values('score')) >>> classes = cfsn_vecs.classes >>> info = ub.peek(cfsn_vecs.binarize_ovr().measures()['perclass'].values()) >>> # xdoctest: +REQUIRES(--show) >>> import kwplot >>> kwplot.autompl() >>> draw_roc(info) >>> kwplot.show_if_requested() """ import kwplot try: fp_count = info['trunc_fp_count'] fp_rate = info['trunc_fpr'] tp_rate = info['trunc_tpr'] auc = info['trunc_auc'] except KeyError: fp_count = info['fp_count'] fp_rate = info['fpr'] tp_rate = info['tpr'] auc = info['auc'] realpos_total = info['realpos_total'] title = prefix + 'AUC*: {:.4f}'.format(auc) falsepos_total = fp_count[-1] if 0: # TODO: deprecate multi_plot for seaborn? fig = kwplot.figure(fnum=fnum) ax = fig.gca() import seaborn as sns xlabel = 'fpr (count={})'.format(falsepos_total) ylabel = 'tpr (count={})'.format(int(realpos_total)) data = { xlabel: list(fp_rate), ylabel: list(tp_rate), } sns.lineplot(data=data, x=xlabel, y=ylabel, markers='', ax=ax) ax.set_title(title) else: realpos_total_disp = inty_display(realpos_total) ax = kwplot.multi_plot( list(fp_rate), list(tp_rate), marker='', # xlabel='FA count (false positive count)', xlabel='fpr (count={})'.format(falsepos_total), ylabel='tpr (count={})'.format(realpos_total_disp), title=title, ylim=(0, 1), ypad=1e-2, xlim=(0, 1), xpad=1e-2, fnum=fnum, **kw) return ax
def draw_prcurve(info, prefix='', fnum=1, **kw): """ Draws a single pr curve. Args: info (Measures | Dict) Example: >>> # xdoctest: +REQUIRES(module:kwplot) >>> from kwcoco.metrics import DetectionMetrics >>> dmet = DetectionMetrics.demo( >>> nimgs=10, nboxes=(0, 10), n_fp=(0, 1), classes=3) >>> cfsn_vecs = dmet.confusion_vectors() >>> classes = cfsn_vecs.classes >>> info = cfsn_vecs.binarize_classless().measures() >>> import kwplot >>> kwplot.autompl() >>> draw_prcurve(info) >>> # xdoctest: +REQUIRES(--show) >>> kwplot.show_if_requested() """ import kwplot aps = [] ap = info['ap'] if 'pr' in info: pr = info['pr'] elif 'ppv' in info: pr = (info['ppv'], info['tpr']) elif 'prec' in info: pr = (info['prec'], info['rec']) else: raise KeyError('pr, prec, or ppv not in info') if np.isfinite(ap): aps.append(ap) (precision, recall) = pr else: precision, recall = [0], [0] if precision is None and recall is None: # I thought AP=nan in this case, but I missed something precision, recall = [0], [0] label_suffix = _realpos_label_suffix(info) label = 'ap={:0.2f}: ({})'.format(ap, label_suffix) ax = kwplot.multi_plot( xdata=recall, ydata=precision, fnum=fnum, label=label, xlim=(0, 1), ylim=(0, 1), xpad=0.01, ypad=0.01, xlabel='recall', ylabel='precision', title=prefix + 'classless AP={:.4f}'.format(ap), legend_loc='lower right', color='distinct', linestyle='cycle', marker='cycle', **kw ) # if 0: # # TODO: should show contour lines with F1 scores # x = np.arange(0.0, 1.0, 1e-3) # X, Y = np.meshgrid(x, x) # Z = np.round(2.XY/(X+Y),3) # Z[np.isnan(Z)] = 0 # levels = np.round(np.arange(0.1, 1.0, .1),1) # CS = ax.contour(X, Y, Z, # levels=levels, # linewidths=0.75, # cmap='copper') # location = zip(levels, levels) # ax.clabel(CS, inline=1, fontsize=9, manual=location, fmt='%.1f') # for c in CS.collections: # c.set_linestyle('dashed') return ax
def draw_perclass_prcurve(cx_to_info, classes=None, prefix='', fnum=1, **kw): """ Args: cx_to_info (PerClass_Measures | Dict): Example: >>> # xdoctest: +REQUIRES(module:kwplot) >>> from kwcoco.metrics.drawing import * # NOQA >>> from kwcoco.metrics import DetectionMetrics >>> dmet = DetectionMetrics.demo( >>> nimgs=3, nboxes=(0, 10), n_fp=(0, 3), n_fn=(0, 2), classes=3, score_noise=0.1, box_noise=0.1, with_probs=False) >>> cfsn_vecs = dmet.confusion_vectors() >>> print(cfsn_vecs.data.pandas()) >>> classes = cfsn_vecs.classes >>> cx_to_info = cfsn_vecs.binarize_ovr().measures()['perclass'] >>> print('cx_to_info = {}'.format(ub.repr2(cx_to_info, nl=1))) >>> import kwplot >>> kwplot.autompl() >>> draw_perclass_prcurve(cx_to_info, classes) >>> # xdoctest: +REQUIRES(--show) >>> kwplot.show_if_requested() Ignore: from kwcoco.metrics.drawing import * # NOQA import xdev globals().update(xdev.get_func_kwargs(draw_perclass_prcurve)) """ import kwplot # Sort by descending AP cxs = list(cx_to_info.keys()) priority = np.array([item['ap'] for item in cx_to_info.values()]) priority[np.isnan(priority)] = -np.inf cxs = list(ub.take(cxs, np.argsort(priority)))[::-1] aps = [] xydata = ub.odict() for cx in cxs: info = cx_to_info[cx] catname = classes[cx] if isinstance(cx, int) else cx ap = info['ap'] if 'pr' in info: pr = info['pr'] elif 'ppv' in info: pr = (info['ppv'], info['tpr']) elif 'prec' in info: pr = (info['prec'], info['rec']) else: raise KeyError('pr, prec, or ppv not in info') if np.isfinite(ap): aps.append(ap) (precision, recall) = pr else: aps.append(np.nan) precision, recall = [0], [0] if precision is None and recall is None: # I thought AP=nan in this case, but I missed something precision, recall = [0], [0] label_suffix = _realpos_label_suffix(info) label = 'ap={:0.2f}: {} ({})'.format(ap, catname, label_suffix) xydata[label] = (recall, precision) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Mean of empty slice', RuntimeWarning) mAP = np.nanmean(aps) if 0: import seaborn as sns import pandas as pd # sns.set() # TODO: deprecate multi_plot for seaborn? data_groups = { key: {'recall': r, 'precision': p} for key, (r, p) in xydata.items() } print('data_groups = {}'.format(ub.repr2(data_groups, nl=3))) longform = [] for key, subdata in data_groups.items(): subdata = pd.DataFrame.from_dict(subdata) subdata['label'] = key longform.append(subdata) data = pd.concat(longform) fig = kwplot.figure(fnum=fnum) ax = fig.gca() longform = [] for key, (r, p) in xydata.items(): subdata = pd.DataFrame.from_dict({'recall': r, 'precision': p, 'label': key}) longform.append(subdata) data = pd.concat(longform) palette = ub.dzip(xydata.keys(), kwplot.distinct_colors(len(xydata))) # markers = ub.dzip(xydata.keys(), kwplot.distinct_markers(len(xydata))) sns.lineplot( data=data, x='recall', y='precision', hue='label', style='label', ax=ax, # markers=markers, estimator=None, ci=0, hue_order=list(xydata.keys()), palette=palette, ) ax.set_xlim(0, 1) ax.set_ylim(0, 1) else: ax = kwplot.multi_plot( xydata=xydata, fnum=fnum, xlim=(0, 1), ylim=(0, 1), xpad=0.01, ypad=0.01, xlabel='recall', ylabel='precision', err_style='bars', title=prefix + 'OVR mAP={:.4f}'.format(mAP), legend_loc='lower right', color='distinct', linestyle='cycle', marker='cycle', **kw ) return ax
def draw_perclass_thresholds(cx_to_info, key='mcc', classes=None, prefix='', fnum=1, **kw): """ Args: cx_to_info (PerClass_Measures | Dict): Notes: Each category is inspected independently of one another, there is no notion of confusion. Example: >>> # xdoctest: +REQUIRES(module:kwplot) >>> from kwcoco.metrics.drawing import * # NOQA >>> from kwcoco.metrics import ConfusionVectors >>> cfsn_vecs = ConfusionVectors.demo() >>> classes = cfsn_vecs.classes >>> ovr_cfsn = cfsn_vecs.binarize_ovr(keyby='name') >>> cx_to_info = ovr_cfsn.measures()['perclass'] >>> import kwplot >>> kwplot.autompl() >>> key = 'mcc' >>> draw_perclass_thresholds(cx_to_info, key, classes) >>> # xdoctest: +REQUIRES(--show) >>> kwplot.show_if_requested() """ import kwplot # Sort by descending "best value" cxs = list(cx_to_info.keys()) try: priority = np.array([item['_max_' + key][0] for item in cx_to_info.values()]) priority[np.isnan(priority)] = -np.inf cxs = list(ub.take(cxs, np.argsort(priority)))[::-1] except KeyError: pass xydata = ub.odict() for cx in cxs: info = cx_to_info[cx] catname = classes[cx] if isinstance(cx, int) else cx thresholds = info['thresholds'] measure = info[key] try: best_label = info['max_{}'.format(key)] except KeyError: max_idx = measure.argmax() best_thresh = thresholds[max_idx] best_measure = measure[max_idx] best_label = '{}={:0.2f}@{:0.2f}'.format(key, best_measure, best_thresh) label_suffix = _realpos_label_suffix(info) label = '{}: {} ({})'.format(best_label, catname, label_suffix) xydata[label] = (thresholds, measure) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Mean of empty slice', RuntimeWarning) ax = kwplot.multi_plot( xydata=xydata, fnum=fnum, xlim=(0, 1), ylim=(0, 1), xpad=0.01, ypad=0.01, xlabel='threshold', ylabel=key, title=prefix + 'OVR {}'.format(key), legend_loc='lower right', color='distinct', linestyle='cycle', marker='cycle', **kw ) return ax
def _precompute_class_weights(dset, mode='median-idf'): """ Example: >>> # xdoctest: +REQUIRES(--download) >>> import sys, ubelt >>> sys.path.append(ubelt.expandpath('~/code/netharn/examples')) >>> from sseg_camvid import * # NOQA >>> harn = setup_harn(0, workers=0, xpu='cpu').initialize() >>> dset = harn.datasets['train'] """ assert mode in ['median-idf', 'log-median-idf'] total_freq = _cached_class_frequency(dset) def logb(arr, base): if base == 'e': return np.log(arr) elif base == 2: return np.log2(arr) elif base == 10: return np.log10(arr) else: out = np.log(arr) out /= np.log(base) return out _min, _max = np.percentile(total_freq, [5, 95]) is_valid = (_min <= total_freq) & (total_freq <= _max) if np.any(is_valid): middle_value = np.median(total_freq[is_valid]) else: middle_value = np.median(total_freq) # variant of median-inverse-frequency nonzero_freq = total_freq[total_freq != 0] if len(nonzero_freq): total_freq[total_freq == 0] = nonzero_freq.min() / 2 if mode == 'median-idf': weights = (middle_value / total_freq) weights[~np.isfinite(weights)] = 1.0 elif mode == 'log-median-idf': weights = (middle_value / total_freq) weights[~np.isfinite(weights)] = 1.0 base = 2 base = np.exp(1) weights = logb(weights + (base - 1), base) weights = np.maximum(weights, .1) weights = np.minimum(weights, 10) else: raise KeyError('mode = {!r}'.format(mode)) weights = np.round(weights, 2) cname_to_weight = ub.dzip(dset.classes, weights) print('weights: ' + ub.repr2(cname_to_weight)) if False: # Inspect the weights import kwplot kwplot.autoplt() cname_to_weight = ub.dzip(dset.classes, weights) cname_to_weight = ub.dict_subset(cname_to_weight, ub.argsort(cname_to_weight)) kwplot.multi_plot( ydata=list(cname_to_weight.values()), kind='bar', xticklabels=list(cname_to_weight.keys()), xtick_rotation=90, fnum=2, doclf=True) return weights
def eval_detections_cli(**kw): """ CommandLine: xdoctest -m ~/code/netharn/netharn/metrics/detect_metrics.py eval_detections_cli """ import scriptconfig as scfg import kwcoco class EvalDetectionCLI(scfg.Config): default = { 'true': scfg.Path(None, help='true coco dataset'), 'pred': scfg.Path(None, help='predicted coco dataset'), 'out_dpath': scfg.Path('./out', help='output directory') } pass config = EvalDetectionCLI() cmdline = kw.pop('cmdline', True) config.load(kw, cmdline=cmdline) true_coco = kwcoco.CocoDataset(config['true']) pred_coco = kwcoco.CocoDataset(config['pred']) from netharn.metrics.detect_metrics import DetectionMetrics dmet = DetectionMetrics.from_coco(true_coco, pred_coco) voc_info = dmet.score_voc() cls_info = voc_info['perclass'][0] tp = cls_info['tp'] fp = cls_info['fp'] fn = cls_info['fn'] tpr = cls_info['tpr'] ppv = cls_info['ppv'] fp = cls_info['fp'] # Compute the MCC as TN->inf thresh = cls_info['thresholds'] # https://erotemic.wordpress.com/2019/10/23/closed-form-of-the-mcc-when-tn-inf/ mcc_lim = tp / (np.sqrt(fn + tp) * np.sqrt(fp + tp)) f1 = 2 * (ppv * tpr) / (ppv + tpr) draw = False if draw: mcc_idx = mcc_lim.argmax() f1_idx = f1.argmax() import kwplot plt = kwplot.autoplt() kwplot.multi_plot( xdata=thresh, ydata=mcc_lim, xlabel='threshold', ylabel='mcc*', fnum=1, pnum=(1, 4, 1), title='MCC*', color=['blue'], ) plt.plot(thresh[mcc_idx], mcc_lim[mcc_idx], 'r*', markersize=20) plt.plot(thresh[f1_idx], mcc_lim[f1_idx], 'k*', markersize=20) kwplot.multi_plot( xdata=fp, ydata=tpr, xlabel='fp (fa)', ylabel='tpr (pd)', fnum=1, pnum=(1, 4, 2), title='ROC', color=['blue'], ) plt.plot(fp[mcc_idx], tpr[mcc_idx], 'r*', markersize=20) plt.plot(fp[f1_idx], tpr[f1_idx], 'k*', markersize=20) kwplot.multi_plot( xdata=tpr, ydata=ppv, xlabel='tpr (recall)', ylabel='ppv (precision)', fnum=1, pnum=(1, 4, 3), title='PR', color=['blue'], ) plt.plot(tpr[mcc_idx], ppv[mcc_idx], 'r*', markersize=20) plt.plot(tpr[f1_idx], ppv[f1_idx], 'k*', markersize=20) kwplot.multi_plot( xdata=thresh, ydata=f1, xlabel='threshold', ylabel='f1', fnum=1, pnum=(1, 4, 4), title='F1', color=['blue'], ) plt.plot(thresh[mcc_idx], f1[mcc_idx], 'r*', markersize=20) plt.plot(thresh[f1_idx], f1[f1_idx], 'k*', markersize=20)
def test_yolo_lr(): if 0: datasets = { 'train': nh.data.ToyData2d(size=3, border=1, n=18, rng=0), # 'vali': nh.data.ToyData2d(size=3, border=1, n=16, rng=1), } burn_in = 2.5 lr = 0.1 bstep = 2 bsize = 2 decay = 0.0005 simulated_bsize = bstep * bsize max_epoch = 4 points = { 0: lr * 1.0, 3: lr * 1.0, 4: lr * 0.1, } else: datasets = { 'train': nh.data.ToyData2d(size=3, border=1, n=16551 // 100, rng=0), 'vali': nh.data.ToyData2d(size=3, border=1, n=4952 // 100, rng=1), } # number of epochs to burn_in for. approx 1000 batches? burn_in = 3.86683584 lr = 0.001 bstep = 2 bsize = 32 decay = 0.0005 simulated_bsize = bstep * bsize max_epoch = 311 points = { 0: lr * 1.0 / simulated_bsize, 154: lr * 1.0 / simulated_bsize, # 1.5625e-05 155: lr * 0.1 / simulated_bsize, # 1.5625e-06 232: lr * 0.1 / simulated_bsize, 233: lr * 0.01 / simulated_bsize, # 1.5625e-07 } hyper = { # --- data first 'datasets' : datasets, 'nice' : 'restart_lr', 'workdir' : ub.ensure_app_cache_dir('netharn/test/restart_lr'), 'loaders' : {'batch_size': bsize}, 'xpu' : nh.XPU.coerce('cpu'), # --- algorithm second 'model' : (nh.models.ToyNet2d, {}), 'optimizer' : (nh.optimizers.SGD, { 'lr': points[0], 'weight_decay': decay * simulated_bsize }), 'criterion' : (nh.criterions.FocalLoss, {}), 'initializer' : (nh.initializers.NoOp, {}), 'scheduler': (nh.schedulers.YOLOScheduler, { 'points': points, 'burn_in': burn_in, 'dset_size': len(datasets['train']), 'batch_size': bsize, 'interpolate': False, }), 'dynamics' : {'batch_step': bstep}, 'monitor' : (nh.Monitor, {'max_epoch': max_epoch}), } harn = MyHarn(hyper=hyper) harn.preferences['prog_backend'] = 'progiter' harn.preferences['use_tensorboard'] = False # Delete previous data harn.initialize(reset='delete') # Cause the harness to fail try: harn.failpoint = 100 harn.run() except Failpoint: pass print('\nFAILPOINT REACHED\n') failpoint_lrs = set(harn._current_lrs()) old_harn = harn # Restarting the harness should begin at the same point harn = MyHarn(hyper=hyper) harn.preferences['prog_backend'] = 'progiter' harn.preferences['use_tensorboard'] = False harn.initialize() harn.xdata = old_harn.xdata harn.ydata = old_harn.ydata restart_lrs = set(harn._current_lrs()) print('failpoint_lrs = {!r}'.format(failpoint_lrs)) print('restart_lrs = {!r}'.format(restart_lrs)) harn.failpoint = None harn.run() if ub.argflag('--show'): import kwplot kwplot.autompl() kwplot.multi_plot(harn.xdata, harn.ydata) from matplotlib import pyplot as plt plt.show() assert restart_lrs == failpoint_lrs
def benchmark_hash_file(): """ CommandLine: python ~/code/ubelt/dev/bench_hash.py --show python ~/code/ubelt/dev/bench_hash.py --show """ import ubelt as ub import random # dpath = ub.ensuredir(ub.expandpath('$HOME/raid/data/tmp')) dpath = ub.ensuredir(ub.expandpath('$HOME/tmp')) rng = random.Random(0) # Create a pool of random chunks of data chunksize = int(2**20) pool_size = 8 part_pool = [_random_data(rng, chunksize) for _ in range(pool_size)] #ITEM = 'JUST A STRING' * 100 HASHERS = ['sha1', 'sha512', 'xxh32', 'xxh64', 'blake3'] scales = list(range(5, 10)) import os results = ub.AutoDict() # Use json is faster or at least as fast it most cases # xxhash is also significantly faster than sha512 ti = ub.Timerit(9, bestof=3, verbose=1, unit='ms') for s in ub.ProgIter(scales, desc='benchmark', verbose=3): N = 2**s print(' --- s={s}, N={N} --- '.format(s=s, N=N)) # Write a big file size_pool = [N] fpath = _write_random_file(dpath, part_pool, size_pool, rng) megabytes = os.stat(fpath).st_size / (2**20) print('megabytes = {!r}'.format(megabytes)) for hasher in HASHERS: for timer in ti.reset(hasher): ub.hash_file(fpath, hasher=hasher) results[hasher].update({N: ti.mean()}) col = {h: results[h][N] for h in HASHERS} sortx = ub.argsort(col) ranking = ub.dict_subset(col, sortx) print('walltime: ' + ub.repr2(ranking, precision=9, nl=0)) best = next(iter(ranking)) #pairs = list(ub.iter_window( 2)) pairs = [(k, best) for k in ranking] ratios = [ranking[k1] / ranking[k2] for k1, k2 in pairs] nicekeys = ['{}/{}'.format(k1, k2) for k1, k2 in pairs] relratios = ub.odict(zip(nicekeys, ratios)) print('speedup: ' + ub.repr2(relratios, precision=4, nl=0)) # xdoc +REQUIRES(--show) # import pytest # pytest.skip() import pandas as pd df = pd.DataFrame.from_dict(results) df.columns.name = 'hasher' df.index.name = 'N' ratios = df.copy().drop(columns=df.columns) for k1, k2 in [('sha512', 'xxh64'), ('sha1', 'xxh64'), ('xxh32', 'xxh64'), ('blake3', 'xxh64')]: ratios['{}/{}'.format(k1, k2)] = df[k1] / df[k2] print() print('Seconds per iteration') print(df.to_string(float_format='%.9f')) print() print('Ratios of seconds') print(ratios.to_string(float_format='%.2f')) print() print('Average Ratio (over all N)') print(ratios.mean().sort_values()) if ub.argflag('--show'): import kwplot kwplot.autompl() xdata = sorted(ub.peek(results.values()).keys()) ydata = ub.map_vals(lambda d: [d[x] for x in xdata], results) kwplot.multi_plot(xdata, ydata, xlabel='N', ylabel='seconds') kwplot.show_if_requested()
def benchamrk_det_nms(): """ Benchmarks different implementations of non-max-supression on the CPU, GPU, and using cython / numpy / torch. CommandLine: xdoctest -m ~/code/kwimage/dev/bench_nms.py benchamrk_det_nms --show SeeAlso: PJR Darknet NonMax supression https://github.com/pjreddie/darknet/blob/master/src/box.c Lightnet NMS https://gitlab.com/EAVISE/lightnet/blob/master/lightnet/data/transform/_postprocess.py#L116 """ # N = 200 # bestof = 50 N = 1 bestof = 1 # xdata = [10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 1500, 2000] # max number of boxes yolo will spit out at a time max_boxes = 19 * 19 * 5 xdata = [ 10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 1500, max_boxes ] # xdata = [10, 20, 40, 80, 100, 200, 300, 400, 500] # Demo values xdata = [0, 1, 2, 3, 10, 100, 200, 300, 500] if ub.argflag('--small'): xdata = [10, 100, 500, 1000, 1500, 2000, 5000, 10000] if ub.argflag('--medium'): xdata = [ 1000, 5000, 10000, 20000, 50000, ] if ub.argflag('--large'): xdata = [ 1000, 5000, 10000, 20000, 50000, 100000, ] if ub.argflag('--extra-large'): xdata = [ 1000, 2000, 10000, 20000, 40000, 100000, 200000, ] title_parts = [] SMALL_BOXES = ub.argflag('--small-boxes') if SMALL_BOXES: title_parts.append('small boxes') else: title_parts.append('large boxes') # NOTE: for large images we may have up to 21,850,753 detections! thresh = float(ub.argval('--thresh', default=0.4)) title_parts.append('thresh={:.2f}'.format(thresh)) from kwimage.algo.algo_nms import available_nms_impls valid_impls = available_nms_impls() print('valid_impls = {!r}'.format(valid_impls)) basis = { 'type': ['ndarray', 'tensor', 'tensor0'], # 'daq': [True, False], # 'daq': [False], # 'device': [None], # 'impl': valid_impls, 'impl': valid_impls + ['auto'], } if ub.argflag('--daq'): basis['daq'] = [True, False] # if torch.cuda.is_available(): # basis['device'].append(0) combos = [ ub.dzip(basis.keys(), vals) for vals in it.product(*basis.values()) ] def is_valid_combo(combo): # if combo['impl'] in {'py', 'cython_cpu'} and combo['device'] is not None: # return False # if combo['type'] == 'ndarray' and combo['impl'] == 'cython_gpu': # if combo['device'] is None: # return False # if combo['type'] == 'ndarray' and combo['impl'] != 'cython_gpu': # if combo['device'] is not None: # return False # if combo['type'].endswith('0'): # if combo['impl'] in {'numpy', 'cython_gpu', 'cython_cpu'}: # return False # if combo['type'] == 'ndarray': # if combo['impl'] in {'torch'}: # return False REMOVE_SLOW = True if REMOVE_SLOW: known_bad = [ { 'impl': 'torch', 'type': 'tensor' }, { 'impl': 'numpy', 'type': 'tensor' }, # {'impl': 'cython_gpu', 'type': 'tensor'}, { 'impl': 'cython_cpu', 'type': 'tensor' }, # {'impl': 'torch', 'type': 'tensor0'}, { 'impl': 'numpy', 'type': 'tensor0' }, # {'impl': 'cython_gpu', 'type': 'tensor0'}, # {'impl': 'cython_cpu', 'type': 'tensor0'}, { 'impl': 'torchvision', 'type': 'ndarray' }, ] for known in known_bad: if all(combo[key] == val for key, val in known.items()): return False return True combos = list(filter(is_valid_combo, combos)) times = ub.ddict(list) for num in xdata: if num > 10000: N = 1 bestof = 1 if num > 1000: N = 3 bestof = 1 if num > 100: N = 10 bestof = 3 elif num > 10: N = 100 bestof = 10 else: N = 1000 bestof = 10 print('\n\n---- number of boxes = {} ----\n'.format(num)) outputs = {} ti = ub.Timerit(N, bestof=bestof, verbose=1) # Build random test boxes and scores np_dets1 = kwimage.Detections.random(num // 2, scale=1000.0, rng=0) np_dets1.data['boxes'] = np_dets1.boxes.to_xywh() if SMALL_BOXES: max_dim = 100 np_dets1.boxes.data[..., 2] = np.minimum(np_dets1.boxes.width, max_dim).ravel() np_dets1.boxes.data[..., 3] = np.minimum(np_dets1.boxes.height, max_dim).ravel() np_dets2 = copy.deepcopy(np_dets1) np_dets2.boxes.translate(10, inplace=True) # add boxes that will definately be removed np_dets = kwimage.Detections.concatenate([np_dets1, np_dets2]) # make all scores unique to ensure comparability np_dets.scores[:] = np.linspace(0, 1, np_dets.num_boxes()) np_dets.data['scores'] = np_dets.scores.astype(np.float32) np_dets.boxes.data = np_dets.boxes.data.astype(np.float32) typed_data = {} # ---------------------------------- import netharn as nh for combo in combos: print('combo = {}'.format(ub.repr2(combo, nl=0))) label = nh.util.make_idstr(combo) mode = combo.copy() # if mode['impl'] == 'cython_gpu': # mode['device_id'] = mode['device'] mode_type = mode.pop('type') if mode_type in typed_data: dets = typed_data[mode_type] else: if mode_type == 'ndarray': dets = np_dets.numpy() elif mode_type == 'tensor': dets = np_dets.tensor(None) elif mode_type == 'tensor0': dets = np_dets.tensor(0) else: raise KeyError typed_data[mode_type] = dets for timer in ti.reset(label): with timer: keep = dets.non_max_supression(thresh=thresh, **mode) torch.cuda.synchronize() times[ti.label].append(ti.min()) outputs[ti.label] = ensure_numpy_indices(keep) # ---------------------------------- # Check that all kept boxes do not have more than `threshold` ious if 0: for key, keep_idxs in outputs.items(): kept = np_dets.take(keep_idxs).boxes ious = kept.ious(kept) max_iou = (np.tril(ious) - np.eye(len(ious))).max() if max_iou > thresh: print('{} produced a bad result with max_iou={}'.format( key, max_iou)) # Check result consistency: print('\nResult stats:') for key in sorted(outputs.keys()): print(' * {:<20}: num={}'.format(key, len(outputs[key]))) print('\nResult overlaps (method1, method2: jaccard):') datas = [] for k1, k2 in it.combinations(sorted(outputs.keys()), 2): idxs1 = set(outputs[k1]) idxs2 = set(outputs[k2]) jaccard = len(idxs1 & idxs2) / max(len(idxs1 | idxs2), 1) datas.append((k1, k2, jaccard)) datas = sorted(datas, key=lambda x: -x[2]) for k1, k2, jaccard in datas: print(' * {:<20}, {:<20}: {:0.4f}'.format(k1, k2, jaccard)) if True: ydata = {key: 1.0 / np.array(vals) for key, vals in times.items()} ylabel = 'Hz' reverse = True yscale = 'symlog' else: ydata = {key: np.array(vals) for key, vals in times.items()} ylabel = 'seconds' reverse = False yscale = 'linear' scores = {key: vals[-1] for key, vals in ydata.items()} ydata = ub.dict_subset(ydata, ub.argsort(scores, reverse=reverse)) ### times_of_interest = [0, 10, 100, 200, 1000] times_of_interest = xdata lines = [] record = lines.append record('### times_of_interest = {!r}'.format(times_of_interest)) for x in times_of_interest: if times_of_interest[-1] == x: record('else:') elif times_of_interest[0] == x: record('if num <= {}:'.format(x)) else: record('elif num <= {}:'.format(x)) if x in xdata: pos = xdata.index(x) score_wrt_x = {} for key, vals in ydata.items(): score_wrt_x[key] = vals[pos] typekeys = ['tensor0', 'tensor', 'ndarray'] type_groups = dict([(b, ub.group_items(score_wrt_x, lambda y: y.endswith(b))[True]) for b in typekeys]) # print('\n=========') # print('x = {!r}'.format(x)) record(' if code not in {!r}:'.format(set(typekeys))) record(' raise KeyError(code)') for typekey, group in type_groups.items(): # print('-------') record(' if code == {!r}:'.format(typekey)) # print('typekey = {!r}'.format(typekey)) # print('group = {!r}'.format(group)) group_x = ub.dict_isect(score_wrt_x, group) valid_keys = ub.argsort(group_x, reverse=True) valid_x = ub.dict_subset(group_x, valid_keys) # parts = [','.split(k) for k in valid_keys] ordered_impls = [] ordered_impls2 = ub.odict() for k in valid_keys: vals = valid_x[k] p = k.split(',') d = dict(i.split('=') for i in p) ordered_impls2[d['impl']] = vals ordered_impls.append(d['impl']) ordered_impls = list(ub.oset(ordered_impls) - {'auto'}) ordered_impls2.pop('auto') record(' # {}'.format( ub.repr2(ordered_impls2, precision=1, nl=0, explicit=True))) record(' preference = {}'.format( ub.repr2(ordered_impls, nl=0))) record('### end times of interest ') print(ub.indent('\n'.join(lines), ' ' * 8)) ### markers = { key: 'o' if 'auto' in key else '' for key, score in scores.items() } if ub.argflag('--daq'): markers = { key: '+' if 'daq=True' in key else '' for key, score in scores.items() } labels = { key: '{:.2f} {} - {}'.format(score, ylabel[0:3], key) for key, score in scores.items() } title = 'NSM-impl speed: ' + ', '.join(title_parts) import kwplot kwplot.autompl() kwplot.multi_plot( xdata, ydata, xlabel='num boxes', ylabel=ylabel, label=labels, yscale=yscale, title=title, marker=markers, # xscale='symlog', ) kwplot.show_if_requested()