def draw_perclass_prcurve(cx_to_peritem, classes=None, prefix='', fnum=1, **kw):
"""
Example:
>>> # xdoctest: +REQUIRES(module:ndsampler)
>>> # xdoctest: +REQUIRES(module:kwplot)
>>> from netharn.metrics import DetectionMetrics
>>> dmet = DetectionMetrics.demo(
>>> nimgs=10, nboxes=(0, 10), n_fp=(0, 1), nclasses=3)
>>> cfsn_vecs = dmet.confusion_vectors()
>>> classes = cfsn_vecs.classes
>>> cx_to_peritem = cfsn_vecs.binarize_ovr().precision_recall()['perclass']
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> kwplot.autompl()
>>> draw_perclass_prcurve(cx_to_peritem, classes)
>>> kwplot.show_if_requested()
"""
import kwplot
# Sort by descending AP
cxs = list(cx_to_peritem.keys())
priority = np.array([item['ap'] for item in cx_to_peritem.values()])
priority[np.isnan(priority)] = -np.inf
cxs = list(ub.take(cxs, np.argsort(priority)))[::-1]
aps = []
xydata = ub.odict()
for cx in cxs:
peritem = cx_to_peritem[cx]
catname = classes[cx] if isinstance(cx, int) else cx
ap = peritem['ap']
if 'pr' in peritem:
pr = peritem['pr']
elif 'ppv' in peritem:
pr = (peritem['ppv'], peritem['tpr'])
elif 'prec' in peritem:
pr = (peritem['prec'], peritem['rec'])
else:
raise KeyError('pr, prec, or ppv not in peritem')
if np.isfinite(ap):
aps.append(ap)
(precision, recall) = pr
else:
aps.append(np.nan)
precision, recall = [0], [0]
if precision is None and recall is None:
# I thought AP=nan in this case, but I missed something
precision, recall = [0], [0]
nsupport = int(peritem['nsupport'])
if 'realpos_total' in peritem:
z = peritem['realpos_total']
if abs(z - int(z)) < 1e-8:
label = 'ap={:0.2f}: {} ({:d}/{:d})'.format(ap, catname, int(peritem['realpos_total']), round(nsupport, 2))
else:
label = 'ap={:0.2f}: {} ({:.2f}/{:d})'.format(ap, catname, round(peritem['realpos_total'], 2), round(nsupport, 2))
else:
label = 'ap={:0.2f}: {} ({:d})'.format(ap, catname, round(nsupport, 2))
xydata[label] = (recall, precision)
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Mean of empty slice', RuntimeWarning)
mAP = np.nanmean(aps)
ax = kwplot.multi_plot(
xydata=xydata, fnum=fnum,
xlim=(0, 1), ylim=(0, 1), xpad=0.01, ypad=0.01,
xlabel='recall', ylabel='precision',
title=prefix + 'perclass mAP={:.4f}'.format(mAP),
legend_loc='lower right',
color='distinct', linestyle='cycle', marker='cycle', **kw
)
return ax
def _devcheck_voc_consistency2():
"""
# CHECK FOR ISSUES WITH MY MAP COMPUTATION
TODO:
Check how cocoeval works
https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py
"""
import pandas as pd
from kwcoco.metrics.detections import DetectionMetrics
xdata = []
ydatas = ub.ddict(list)
dmets = []
for box_noise in np.linspace(0, 8, 20):
dmet = DetectionMetrics.demo(
nimgs=20,
nboxes=(0, 20),
classes=3,
rng=0,
# anchors=np.array([[.5, .5], [.3, .3], [.1, .3], [.2, .1]]),
box_noise=box_noise,
# n_fp=0 if box_noise == 0 else (0, 3),
# n_fn=0 if box_noise == 0 else (0, 3),
# cls_noise=0 if box_noise == 0 else .3,
)
dmets.append(dmet)
nh_scores = dmet.score_kwcoco(bias=0)
voc_scores = dmet.score_voc(bias=0)
coco_scores = dmet.score_coco()
nh_map = nh_scores['mAP']
voc_map = voc_scores['mAP']
coco_map = coco_scores['mAP']
print('nh_map = {!r}'.format(nh_map))
print('voc_map = {!r}'.format(voc_map))
print('coco_map = {!r}'.format(coco_map))
xdata.append(box_noise)
ydatas['voc'].append(voc_map)
ydatas['kwcoco'].append(nh_map)
ydatas['coco'].append(coco_map)
ydf = pd.DataFrame(ydatas)
print(ydf)
import kwplot
kwplot.autompl()
kwplot.multi_plot(xdata=xdata, ydata=ydatas, fnum=1, doclf=True)
if False:
dmet_ = dmets[-1]
dmet_ = dmets[0]
print('true = ' + ub.repr2(dmet_.true.dataset, nl=2, precision=2))
print('pred = ' + ub.repr2(dmet_.pred.dataset, nl=2, precision=2))
dmet = DetectionMetrics()
for gid in range(0, 5):
print('----')
print('gid = {!r}'.format(gid))
dmet.true = dmet_.true.subset([gid])
dmet.pred = dmet_.pred.subset([gid])
nh_scores = dmet.score_kwcoco(bias=0)
voc_scores = dmet.score_voc(bias=0)
coco_scores = dmet.score_coco()
nh_map = nh_scores['mAP']
voc_map = voc_scores['mAP']
coco_map = coco_scores['mAP']
print('nh_map = {!r}'.format(nh_map))
print('voc_map = {!r}'.format(voc_map))
print('coco_map = {!r}'.format(coco_map))
print('true = ' + ub.repr2(dmet.true.dataset, nl=2))
print('pred = ' + ub.repr2(dmet.pred.dataset, nl=2))
def _devcheck_voc_consistency():
"""
# CHECK FOR ISSUES WITH MY MAP COMPUTATION
TODO:
Check how cocoeval works
https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py
"""
import pandas as pd
import kwcoco as nh
# method = 'voc2012'
method = 'voc2007'
bias = 0
bias = 0
# classes = [0, 1, 2]
classes = [0]
classname = 0
# nimgs = 5
# nboxes = 2
nimgs = 5
nboxes = 5
nbad = 1
bg_weight = 1.0
iou_thresh = 0.5
bg_cls = -1
xdata = []
ydatas = ub.ddict(list)
for noise in np.linspace(0, 5, 10):
recs = {}
lines = []
confusions = []
rng = np.random.RandomState(0)
detmetrics = DetectionMetrics()
true_coco = nh.data.coco_api.CocoDataset()
pred_coco = nh.data.coco_api.CocoDataset()
cid = true_coco.add_category('cat1')
cid = pred_coco.add_category('cat1')
for imgname in range(nimgs):
# Create voc style data
imgname = str(imgname)
import kwimage
true_boxes = kwimage.Boxes.random(num=nboxes,
scale=100.,
rng=rng,
format='cxywh')
pred_boxes = true_boxes.copy()
pred_boxes.data = pred_boxes.data.astype(
np.float) + (rng.rand() * noise)
if nbad:
pred_boxes.data = np.vstack([
pred_boxes.data,
kwimage.Boxes.random(num=nbad,
scale=100.,
rng=rng,
format='cxywh').data
])
true_cxs = rng.choice(classes, size=len(true_boxes))
pred_cxs = true_cxs.copy()
change = rng.rand(len(true_cxs)) < (noise / 5)
pred_cxs_swap = rng.choice(classes, size=len(pred_cxs))
pred_cxs[change] = pred_cxs_swap[change]
if nbad:
pred_cxs = np.hstack(
[pred_cxs, rng.choice(classes, size=nbad)])
np.array([0] * len(true_boxes))
pred_cxs = np.array([0] * len(pred_boxes))
recs[imgname] = []
for bbox in true_boxes.to_tlbr().data:
recs[imgname].append({
'bbox': bbox,
'difficult': False,
'name': classname
})
for bbox, score in zip(pred_boxes.to_tlbr().data,
np.arange(len(pred_boxes))):
lines.append([imgname, score] + list(bbox))
# lines.append('{} {} {} {} {} {}'.format(imgname, score, *bbox))
# Create MS-COCO style data
gid = true_coco.add_image(imgname)
gid = pred_coco.add_image(imgname)
for bbox in true_boxes.to_xywh():
true_coco.add_annotation(gid,
cid,
bbox=bbox,
iscrowd=False,
ignore=0,
area=bbox.area[0])
for bbox, score in zip(pred_boxes.to_xywh(),
np.arange(len(pred_boxes))):
pred_coco.add_annotation(gid,
cid,
bbox=bbox,
iscrowd=False,
ignore=0,
score=score,
area=bbox.area[0])
# Create kwcoco style confusion data
true_weights = np.array([1] * len(true_boxes))
pred_scores = np.arange(len(pred_boxes))
y = pd.DataFrame(
detection_confusions(true_boxes,
true_cxs,
true_weights,
pred_boxes,
pred_scores,
pred_cxs,
bg_weight=1.0,
iou_thresh=0.5,
bg_cls=-1,
bias=bias))
y['gx'] = int(imgname)
y = (y)
confusions.append(y)
from pycocotools import cocoeval as coco_score
cocoGt = true_coco._aspycoco()
cocoDt = pred_coco._aspycoco()
evaler = coco_score.COCOeval(cocoGt, cocoDt, iouType='bbox')
evaler.evaluate()
evaler.accumulate()
evaler.summarize()
coco_ap = evaler.stats[1]
y = pd.concat(confusions)
mine_ap = score_detection_assignment(y, method=method)['ap']
voc_rec, voc_prec, voc_ap = voc_eval(lines,
recs,
classname,
iou_thresh=0.5,
method=method,
bias=bias)
eav_prec, eav_rec, eav_ap1 = _multiclass_ap(y)
eav_ap2 = _ave_precision(eav_rec, eav_prec, method=method)
voc_ap2 = _ave_precision(voc_rec, voc_prec, method=method)
eav_ap = eav_ap2
print('noise = {!r}'.format(noise))
print('mine_ap = {!r}'.format(mine_ap.values.mean()))
print('voc_ap = {!r}'.format(voc_ap))
print('eav_ap = {!r}'.format(eav_ap))
print('---')
xdata.append(noise)
ydatas['voc'].append(voc_ap)
ydatas['eav'].append(eav_ap)
ydatas['kwcoco'].append(mine_ap.values.mean())
ydatas['coco'].append(coco_ap)
ydf = pd.DataFrame(ydatas)
print(ydf)
import kwplot
kwplot.autompl()
kwplot.multi_plot(xdata=xdata, ydata=ydatas, fnum=1, doclf=True)
def draw_threshold_curves(info, keys=None, prefix='', fnum=1, **kw):
"""
Args:
info (Measures | Dict)
Example:
>>> # xdoctest: +REQUIRES(module:kwplot)
>>> import sys, ubelt
>>> sys.path.append(ubelt.expandpath('~/code/kwcoco'))
>>> from kwcoco.metrics.drawing import * # NOQA
>>> from kwcoco.metrics import DetectionMetrics
>>> dmet = DetectionMetrics.demo(
>>> nimgs=10, nboxes=(0, 10), n_fp=(0, 1), classes=3)
>>> cfsn_vecs = dmet.confusion_vectors()
>>> info = cfsn_vecs.binarize_classless().measures()
>>> keys = None
>>> import kwplot
>>> kwplot.autompl()
>>> draw_threshold_curves(info, keys)
>>> # xdoctest: +REQUIRES(--show)
>>> kwplot.show_if_requested()
"""
import kwplot
import kwimage
thresh = info['thresholds']
if keys is None:
keys = {'g1', 'f1', 'acc', 'mcc'}
idx_to_colors = kwimage.Color.distinct(len(keys), space='rgba')
idx_to_best_pt = {}
xydata = {}
colors = {}
finite_flags = np.isfinite(thresh)
for idx, key in enumerate(keys):
color = idx_to_colors[idx]
measure = info[key][finite_flags]
if len(measure):
try:
max_idx = np.nanargmax(measure)
offset = (~finite_flags[:max_idx]).sum()
max_idx += offset
best_thresh = thresh[max_idx]
best_measure = measure[max_idx]
best_label = '{}={:0.2f}@{:0.2f}'.format(key, best_measure, best_thresh)
except ValueError:
best_thresh = np.nan
best_measure = np.nan
else:
best_thresh = np.nan
best_measure = np.nan
best_label = '{}={:0.2f}@{:0.2f}'.format(key, best_measure, best_thresh)
label_suffix = _realpos_label_suffix(info)
label = '{}: ({})'.format(best_label, label_suffix)
xydata[label] = (thresh, measure)
colors[label] = color
idx_to_best_pt[idx] = (best_thresh, best_measure)
ax = kwplot.multi_plot(
xydata=xydata, fnum=fnum,
xlim=(0, 1), ylim=(0, 1), xpad=0.01, ypad=0.01,
xlabel='threshold', ylabel=key,
title=prefix + 'threshold curves',
legend_loc='lower right',
color=colors,
linestyle='cycle', marker='cycle', **kw
)
for idx, best_pt in idx_to_best_pt.items():
best_thresh, best_measure = best_pt
color = idx_to_colors[idx]
ax.plot(best_thresh, best_measure, '*', color=color)
return ax
def bench_bbox_iou_method():
"""
On my system the torch impl was fastest (when the data was on the GPU).
"""
from kwimage.structs.boxes import _box_ious_torch, _box_ious_py, _bbox_ious_c
ydata = ub.ddict(list)
xdata = [
10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 2000, 4000
]
bias = 0
if _bbox_ious_c is None:
print('CYTHON IMPLEMENATION IS NOT AVAILABLE')
for num in xdata:
results = {}
# Setup Timer
N = max(20, int(1000 / num))
ti = ub.Timerit(N, bestof=10)
# Setup input dat
boxes1 = kwimage.Boxes.random(num, scale=10.0, rng=0, format='ltrb')
boxes2 = kwimage.Boxes.random(num + 1,
scale=10.0,
rng=1,
format='ltrb')
ltrb1 = boxes1.tensor().data
ltrb2 = boxes2.tensor().data
for timer in ti.reset('iou-torch-cpu'):
with timer:
out = _box_ious_torch(ltrb1, ltrb2, bias)
results[ti.label] = out.data.cpu().numpy()
ydata[ti.label].append(ti.mean())
gpu = torch.device(0)
ltrb1 = boxes1.tensor().data.to(gpu)
ltrb2 = boxes2.tensor().data.to(gpu)
for timer in ti.reset('iou-torch-gpu'):
with timer:
out = _box_ious_torch(ltrb1, ltrb2, bias)
torch.cuda.synchronize()
results[ti.label] = out.data.cpu().numpy()
ydata[ti.label].append(ti.mean())
ltrb1 = boxes1.numpy().data
ltrb2 = boxes2.numpy().data
for timer in ti.reset('iou-numpy'):
with timer:
out = _box_ious_py(ltrb1, ltrb2, bias)
results[ti.label] = out
ydata[ti.label].append(ti.mean())
if _bbox_ious_c:
ltrb1 = boxes1.numpy().data.astype(np.float32)
ltrb2 = boxes2.numpy().data.astype(np.float32)
for timer in ti.reset('iou-cython'):
with timer:
out = _bbox_ious_c(ltrb1, ltrb2, bias)
results[ti.label] = out
ydata[ti.label].append(ti.mean())
eq = partial(np.allclose, atol=1e-07)
passed = ub.allsame(results.values(), eq)
if passed:
print(
'All methods produced the same answer for num={}'.format(num))
else:
for k1, k2 in it.combinations(results.keys(), 2):
v1 = results[k1]
v2 = results[k2]
if eq(v1, v2):
print('pass: {} == {}'.format(k1, k2))
else:
diff = np.abs(v1 - v2)
print(
'FAIL: {} != {}: diff(max={}, mean={}, sum={})'.format(
k1, k2, diff.max(), diff.mean(), diff.sum()))
raise AssertionError('different methods report different results')
print('num = {!r}'.format(num))
print('ti.measures = {}'.format(
ub.repr2(ub.map_vals(ub.sorted_vals, ti.measures),
align=':',
nl=2,
precision=6)))
import kwplot
kwplot.autoplt()
kwplot.multi_plot(xdata, ydata, xlabel='num boxes', ylabel='seconds')
kwplot.show_if_requested()
def draw_roc(info, prefix='', fnum=1, **kw):
"""
Args:
info (Measures | Dict)
NOTE:
There needs to be enough negative examples for using ROC to make any
sense!
Example:
>>> # xdoctest: +REQUIRES(module:kwplot, module:seaborn)
>>> from kwcoco.metrics.drawing import * # NOQA
>>> from kwcoco.metrics import DetectionMetrics
>>> dmet = DetectionMetrics.demo(nimgs=30, null_pred=1, classes=3,
>>> nboxes=10, n_fp=10, box_noise=0.3,
>>> with_probs=False)
>>> dmet.true_detections(0).data
>>> cfsn_vecs = dmet.confusion_vectors(compat='mutex', prioritize='iou', bias=0)
>>> print(cfsn_vecs.data._pandas().sort_values('score'))
>>> classes = cfsn_vecs.classes
>>> info = ub.peek(cfsn_vecs.binarize_ovr().measures()['perclass'].values())
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> kwplot.autompl()
>>> draw_roc(info)
>>> kwplot.show_if_requested()
"""
import kwplot
try:
fp_count = info['trunc_fp_count']
fp_rate = info['trunc_fpr']
tp_rate = info['trunc_tpr']
auc = info['trunc_auc']
except KeyError:
fp_count = info['fp_count']
fp_rate = info['fpr']
tp_rate = info['tpr']
auc = info['auc']
realpos_total = info['realpos_total']
title = prefix + 'AUC*: {:.4f}'.format(auc)
falsepos_total = fp_count[-1]
if 0:
# TODO: deprecate multi_plot for seaborn?
fig = kwplot.figure(fnum=fnum)
ax = fig.gca()
import seaborn as sns
xlabel = 'fpr (count={})'.format(falsepos_total)
ylabel = 'tpr (count={})'.format(int(realpos_total))
data = {
xlabel: list(fp_rate),
ylabel: list(tp_rate),
}
sns.lineplot(data=data, x=xlabel, y=ylabel, markers='', ax=ax)
ax.set_title(title)
else:
realpos_total_disp = inty_display(realpos_total)
ax = kwplot.multi_plot(
list(fp_rate), list(tp_rate), marker='',
# xlabel='FA count (false positive count)',
xlabel='fpr (count={})'.format(falsepos_total),
ylabel='tpr (count={})'.format(realpos_total_disp),
title=title,
ylim=(0, 1), ypad=1e-2,
xlim=(0, 1), xpad=1e-2,
fnum=fnum, **kw)
return ax
def draw_prcurve(info, prefix='', fnum=1, **kw):
"""
Draws a single pr curve.
Args:
info (Measures | Dict)
Example:
>>> # xdoctest: +REQUIRES(module:kwplot)
>>> from kwcoco.metrics import DetectionMetrics
>>> dmet = DetectionMetrics.demo(
>>> nimgs=10, nboxes=(0, 10), n_fp=(0, 1), classes=3)
>>> cfsn_vecs = dmet.confusion_vectors()
>>> classes = cfsn_vecs.classes
>>> info = cfsn_vecs.binarize_classless().measures()
>>> import kwplot
>>> kwplot.autompl()
>>> draw_prcurve(info)
>>> # xdoctest: +REQUIRES(--show)
>>> kwplot.show_if_requested()
"""
import kwplot
aps = []
ap = info['ap']
if 'pr' in info:
pr = info['pr']
elif 'ppv' in info:
pr = (info['ppv'], info['tpr'])
elif 'prec' in info:
pr = (info['prec'], info['rec'])
else:
raise KeyError('pr, prec, or ppv not in info')
if np.isfinite(ap):
aps.append(ap)
(precision, recall) = pr
else:
precision, recall = [0], [0]
if precision is None and recall is None:
# I thought AP=nan in this case, but I missed something
precision, recall = [0], [0]
label_suffix = _realpos_label_suffix(info)
label = 'ap={:0.2f}: ({})'.format(ap, label_suffix)
ax = kwplot.multi_plot(
xdata=recall, ydata=precision, fnum=fnum, label=label,
xlim=(0, 1), ylim=(0, 1), xpad=0.01, ypad=0.01,
xlabel='recall', ylabel='precision',
title=prefix + 'classless AP={:.4f}'.format(ap),
legend_loc='lower right',
color='distinct', linestyle='cycle', marker='cycle', **kw
)
# if 0:
# # TODO: should show contour lines with F1 scores
# x = np.arange(0.0, 1.0, 1e-3)
# X, Y = np.meshgrid(x, x)
# Z = np.round(2.XY/(X+Y),3)
# Z[np.isnan(Z)] = 0
# levels = np.round(np.arange(0.1, 1.0, .1),1)
# CS = ax.contour(X, Y, Z,
# levels=levels,
# linewidths=0.75,
# cmap='copper')
# location = zip(levels, levels)
# ax.clabel(CS, inline=1, fontsize=9, manual=location, fmt='%.1f')
# for c in CS.collections:
# c.set_linestyle('dashed')
return ax
def draw_perclass_prcurve(cx_to_info, classes=None, prefix='', fnum=1, **kw):
"""
Args:
cx_to_info (PerClass_Measures | Dict):
Example:
>>> # xdoctest: +REQUIRES(module:kwplot)
>>> from kwcoco.metrics.drawing import * # NOQA
>>> from kwcoco.metrics import DetectionMetrics
>>> dmet = DetectionMetrics.demo(
>>> nimgs=3, nboxes=(0, 10), n_fp=(0, 3), n_fn=(0, 2), classes=3, score_noise=0.1, box_noise=0.1, with_probs=False)
>>> cfsn_vecs = dmet.confusion_vectors()
>>> print(cfsn_vecs.data.pandas())
>>> classes = cfsn_vecs.classes
>>> cx_to_info = cfsn_vecs.binarize_ovr().measures()['perclass']
>>> print('cx_to_info = {}'.format(ub.repr2(cx_to_info, nl=1)))
>>> import kwplot
>>> kwplot.autompl()
>>> draw_perclass_prcurve(cx_to_info, classes)
>>> # xdoctest: +REQUIRES(--show)
>>> kwplot.show_if_requested()
Ignore:
from kwcoco.metrics.drawing import * # NOQA
import xdev
globals().update(xdev.get_func_kwargs(draw_perclass_prcurve))
"""
import kwplot
# Sort by descending AP
cxs = list(cx_to_info.keys())
priority = np.array([item['ap'] for item in cx_to_info.values()])
priority[np.isnan(priority)] = -np.inf
cxs = list(ub.take(cxs, np.argsort(priority)))[::-1]
aps = []
xydata = ub.odict()
for cx in cxs:
info = cx_to_info[cx]
catname = classes[cx] if isinstance(cx, int) else cx
ap = info['ap']
if 'pr' in info:
pr = info['pr']
elif 'ppv' in info:
pr = (info['ppv'], info['tpr'])
elif 'prec' in info:
pr = (info['prec'], info['rec'])
else:
raise KeyError('pr, prec, or ppv not in info')
if np.isfinite(ap):
aps.append(ap)
(precision, recall) = pr
else:
aps.append(np.nan)
precision, recall = [0], [0]
if precision is None and recall is None:
# I thought AP=nan in this case, but I missed something
precision, recall = [0], [0]
label_suffix = _realpos_label_suffix(info)
label = 'ap={:0.2f}: {} ({})'.format(ap, catname, label_suffix)
xydata[label] = (recall, precision)
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Mean of empty slice', RuntimeWarning)
mAP = np.nanmean(aps)
if 0:
import seaborn as sns
import pandas as pd
# sns.set()
# TODO: deprecate multi_plot for seaborn?
data_groups = {
key: {'recall': r, 'precision': p}
for key, (r, p) in xydata.items()
}
print('data_groups = {}'.format(ub.repr2(data_groups, nl=3)))
longform = []
for key, subdata in data_groups.items():
subdata = pd.DataFrame.from_dict(subdata)
subdata['label'] = key
longform.append(subdata)
data = pd.concat(longform)
fig = kwplot.figure(fnum=fnum)
ax = fig.gca()
longform = []
for key, (r, p) in xydata.items():
subdata = pd.DataFrame.from_dict({'recall': r, 'precision': p, 'label': key})
longform.append(subdata)
data = pd.concat(longform)
palette = ub.dzip(xydata.keys(), kwplot.distinct_colors(len(xydata)))
# markers = ub.dzip(xydata.keys(), kwplot.distinct_markers(len(xydata)))
sns.lineplot(
data=data, x='recall', y='precision',
hue='label', style='label', ax=ax,
# markers=markers,
estimator=None,
ci=0,
hue_order=list(xydata.keys()),
palette=palette,
)
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
else:
ax = kwplot.multi_plot(
xydata=xydata, fnum=fnum,
xlim=(0, 1), ylim=(0, 1), xpad=0.01, ypad=0.01,
xlabel='recall', ylabel='precision',
err_style='bars',
title=prefix + 'OVR mAP={:.4f}'.format(mAP),
legend_loc='lower right',
color='distinct', linestyle='cycle', marker='cycle', **kw
)
return ax
def draw_perclass_thresholds(cx_to_info, key='mcc', classes=None, prefix='', fnum=1, **kw):
"""
Args:
cx_to_info (PerClass_Measures | Dict):
Notes:
Each category is inspected independently of one another, there is no
notion of confusion.
Example:
>>> # xdoctest: +REQUIRES(module:kwplot)
>>> from kwcoco.metrics.drawing import * # NOQA
>>> from kwcoco.metrics import ConfusionVectors
>>> cfsn_vecs = ConfusionVectors.demo()
>>> classes = cfsn_vecs.classes
>>> ovr_cfsn = cfsn_vecs.binarize_ovr(keyby='name')
>>> cx_to_info = ovr_cfsn.measures()['perclass']
>>> import kwplot
>>> kwplot.autompl()
>>> key = 'mcc'
>>> draw_perclass_thresholds(cx_to_info, key, classes)
>>> # xdoctest: +REQUIRES(--show)
>>> kwplot.show_if_requested()
"""
import kwplot
# Sort by descending "best value"
cxs = list(cx_to_info.keys())
try:
priority = np.array([item['_max_' + key][0] for item in cx_to_info.values()])
priority[np.isnan(priority)] = -np.inf
cxs = list(ub.take(cxs, np.argsort(priority)))[::-1]
except KeyError:
pass
xydata = ub.odict()
for cx in cxs:
info = cx_to_info[cx]
catname = classes[cx] if isinstance(cx, int) else cx
thresholds = info['thresholds']
measure = info[key]
try:
best_label = info['max_{}'.format(key)]
except KeyError:
max_idx = measure.argmax()
best_thresh = thresholds[max_idx]
best_measure = measure[max_idx]
best_label = '{}={:0.2f}@{:0.2f}'.format(key, best_measure, best_thresh)
label_suffix = _realpos_label_suffix(info)
label = '{}: {} ({})'.format(best_label, catname, label_suffix)
xydata[label] = (thresholds, measure)
with warnings.catch_warnings():
warnings.filterwarnings('ignore', 'Mean of empty slice', RuntimeWarning)
ax = kwplot.multi_plot(
xydata=xydata, fnum=fnum,
xlim=(0, 1), ylim=(0, 1), xpad=0.01, ypad=0.01,
xlabel='threshold', ylabel=key,
title=prefix + 'OVR {}'.format(key),
legend_loc='lower right',
color='distinct', linestyle='cycle', marker='cycle', **kw
)
return ax
def _precompute_class_weights(dset, mode='median-idf'):
"""
Example:
>>> # xdoctest: +REQUIRES(--download)
>>> import sys, ubelt
>>> sys.path.append(ubelt.expandpath('~/code/netharn/examples'))
>>> from sseg_camvid import * # NOQA
>>> harn = setup_harn(0, workers=0, xpu='cpu').initialize()
>>> dset = harn.datasets['train']
"""
assert mode in ['median-idf', 'log-median-idf']
total_freq = _cached_class_frequency(dset)
def logb(arr, base):
if base == 'e':
return np.log(arr)
elif base == 2:
return np.log2(arr)
elif base == 10:
return np.log10(arr)
else:
out = np.log(arr)
out /= np.log(base)
return out
_min, _max = np.percentile(total_freq, [5, 95])
is_valid = (_min <= total_freq) & (total_freq <= _max)
if np.any(is_valid):
middle_value = np.median(total_freq[is_valid])
else:
middle_value = np.median(total_freq)
# variant of median-inverse-frequency
nonzero_freq = total_freq[total_freq != 0]
if len(nonzero_freq):
total_freq[total_freq == 0] = nonzero_freq.min() / 2
if mode == 'median-idf':
weights = (middle_value / total_freq)
weights[~np.isfinite(weights)] = 1.0
elif mode == 'log-median-idf':
weights = (middle_value / total_freq)
weights[~np.isfinite(weights)] = 1.0
base = 2
base = np.exp(1)
weights = logb(weights + (base - 1), base)
weights = np.maximum(weights, .1)
weights = np.minimum(weights, 10)
else:
raise KeyError('mode = {!r}'.format(mode))
weights = np.round(weights, 2)
cname_to_weight = ub.dzip(dset.classes, weights)
print('weights: ' + ub.repr2(cname_to_weight))
if False:
# Inspect the weights
import kwplot
kwplot.autoplt()
cname_to_weight = ub.dzip(dset.classes, weights)
cname_to_weight = ub.dict_subset(cname_to_weight, ub.argsort(cname_to_weight))
kwplot.multi_plot(
ydata=list(cname_to_weight.values()),
kind='bar',
xticklabels=list(cname_to_weight.keys()),
xtick_rotation=90,
fnum=2, doclf=True)
return weights
def eval_detections_cli(**kw):
"""
CommandLine:
xdoctest -m ~/code/netharn/netharn/metrics/detect_metrics.py eval_detections_cli
"""
import scriptconfig as scfg
import kwcoco
class EvalDetectionCLI(scfg.Config):
default = {
'true': scfg.Path(None, help='true coco dataset'),
'pred': scfg.Path(None, help='predicted coco dataset'),
'out_dpath': scfg.Path('./out', help='output directory')
}
pass
config = EvalDetectionCLI()
cmdline = kw.pop('cmdline', True)
config.load(kw, cmdline=cmdline)
true_coco = kwcoco.CocoDataset(config['true'])
pred_coco = kwcoco.CocoDataset(config['pred'])
from netharn.metrics.detect_metrics import DetectionMetrics
dmet = DetectionMetrics.from_coco(true_coco, pred_coco)
voc_info = dmet.score_voc()
cls_info = voc_info['perclass'][0]
tp = cls_info['tp']
fp = cls_info['fp']
fn = cls_info['fn']
tpr = cls_info['tpr']
ppv = cls_info['ppv']
fp = cls_info['fp']
# Compute the MCC as TN->inf
thresh = cls_info['thresholds']
# https://erotemic.wordpress.com/2019/10/23/closed-form-of-the-mcc-when-tn-inf/
mcc_lim = tp / (np.sqrt(fn + tp) * np.sqrt(fp + tp))
f1 = 2 * (ppv * tpr) / (ppv + tpr)
draw = False
if draw:
mcc_idx = mcc_lim.argmax()
f1_idx = f1.argmax()
import kwplot
plt = kwplot.autoplt()
kwplot.multi_plot(
xdata=thresh,
ydata=mcc_lim,
xlabel='threshold',
ylabel='mcc*',
fnum=1,
pnum=(1, 4, 1),
title='MCC*',
color=['blue'],
)
plt.plot(thresh[mcc_idx], mcc_lim[mcc_idx], 'r*', markersize=20)
plt.plot(thresh[f1_idx], mcc_lim[f1_idx], 'k*', markersize=20)
kwplot.multi_plot(
xdata=fp,
ydata=tpr,
xlabel='fp (fa)',
ylabel='tpr (pd)',
fnum=1,
pnum=(1, 4, 2),
title='ROC',
color=['blue'],
)
plt.plot(fp[mcc_idx], tpr[mcc_idx], 'r*', markersize=20)
plt.plot(fp[f1_idx], tpr[f1_idx], 'k*', markersize=20)
kwplot.multi_plot(
xdata=tpr,
ydata=ppv,
xlabel='tpr (recall)',
ylabel='ppv (precision)',
fnum=1,
pnum=(1, 4, 3),
title='PR',
color=['blue'],
)
plt.plot(tpr[mcc_idx], ppv[mcc_idx], 'r*', markersize=20)
plt.plot(tpr[f1_idx], ppv[f1_idx], 'k*', markersize=20)
kwplot.multi_plot(
xdata=thresh,
ydata=f1,
xlabel='threshold',
ylabel='f1',
fnum=1,
pnum=(1, 4, 4),
title='F1',
color=['blue'],
)
plt.plot(thresh[mcc_idx], f1[mcc_idx], 'r*', markersize=20)
plt.plot(thresh[f1_idx], f1[f1_idx], 'k*', markersize=20)
def test_yolo_lr():
if 0:
datasets = {
'train': nh.data.ToyData2d(size=3, border=1, n=18, rng=0),
# 'vali': nh.data.ToyData2d(size=3, border=1, n=16, rng=1),
}
burn_in = 2.5
lr = 0.1
bstep = 2
bsize = 2
decay = 0.0005
simulated_bsize = bstep * bsize
max_epoch = 4
points = {
0: lr * 1.0,
3: lr * 1.0,
4: lr * 0.1,
}
else:
datasets = {
'train': nh.data.ToyData2d(size=3, border=1, n=16551 // 100, rng=0),
'vali': nh.data.ToyData2d(size=3, border=1, n=4952 // 100, rng=1),
}
# number of epochs to burn_in for. approx 1000 batches?
burn_in = 3.86683584
lr = 0.001
bstep = 2
bsize = 32
decay = 0.0005
simulated_bsize = bstep * bsize
max_epoch = 311
points = {
0: lr * 1.0 / simulated_bsize,
154: lr * 1.0 / simulated_bsize, # 1.5625e-05
155: lr * 0.1 / simulated_bsize, # 1.5625e-06
232: lr * 0.1 / simulated_bsize,
233: lr * 0.01 / simulated_bsize, # 1.5625e-07
}
hyper = {
# --- data first
'datasets' : datasets,
'nice' : 'restart_lr',
'workdir' : ub.ensure_app_cache_dir('netharn/test/restart_lr'),
'loaders' : {'batch_size': bsize},
'xpu' : nh.XPU.coerce('cpu'),
# --- algorithm second
'model' : (nh.models.ToyNet2d, {}),
'optimizer' : (nh.optimizers.SGD, {
'lr': points[0],
'weight_decay': decay * simulated_bsize
}),
'criterion' : (nh.criterions.FocalLoss, {}),
'initializer' : (nh.initializers.NoOp, {}),
'scheduler': (nh.schedulers.YOLOScheduler, {
'points': points,
'burn_in': burn_in,
'dset_size': len(datasets['train']),
'batch_size': bsize,
'interpolate': False,
}),
'dynamics' : {'batch_step': bstep},
'monitor' : (nh.Monitor, {'max_epoch': max_epoch}),
}
harn = MyHarn(hyper=hyper)
harn.preferences['prog_backend'] = 'progiter'
harn.preferences['use_tensorboard'] = False
# Delete previous data
harn.initialize(reset='delete')
# Cause the harness to fail
try:
harn.failpoint = 100
harn.run()
except Failpoint:
pass
print('\nFAILPOINT REACHED\n')
failpoint_lrs = set(harn._current_lrs())
old_harn = harn
# Restarting the harness should begin at the same point
harn = MyHarn(hyper=hyper)
harn.preferences['prog_backend'] = 'progiter'
harn.preferences['use_tensorboard'] = False
harn.initialize()
harn.xdata = old_harn.xdata
harn.ydata = old_harn.ydata
restart_lrs = set(harn._current_lrs())
print('failpoint_lrs = {!r}'.format(failpoint_lrs))
print('restart_lrs = {!r}'.format(restart_lrs))
harn.failpoint = None
harn.run()
if ub.argflag('--show'):
import kwplot
kwplot.autompl()
kwplot.multi_plot(harn.xdata, harn.ydata)
from matplotlib import pyplot as plt
plt.show()
assert restart_lrs == failpoint_lrs
def benchmark_hash_file():
"""
CommandLine:
python ~/code/ubelt/dev/bench_hash.py --show
python ~/code/ubelt/dev/bench_hash.py --show
"""
import ubelt as ub
import random
# dpath = ub.ensuredir(ub.expandpath('$HOME/raid/data/tmp'))
dpath = ub.ensuredir(ub.expandpath('$HOME/tmp'))
rng = random.Random(0)
# Create a pool of random chunks of data
chunksize = int(2**20)
pool_size = 8
part_pool = [_random_data(rng, chunksize) for _ in range(pool_size)]
#ITEM = 'JUST A STRING' * 100
HASHERS = ['sha1', 'sha512', 'xxh32', 'xxh64', 'blake3']
scales = list(range(5, 10))
import os
results = ub.AutoDict()
# Use json is faster or at least as fast it most cases
# xxhash is also significantly faster than sha512
ti = ub.Timerit(9, bestof=3, verbose=1, unit='ms')
for s in ub.ProgIter(scales, desc='benchmark', verbose=3):
N = 2**s
print(' --- s={s}, N={N} --- '.format(s=s, N=N))
# Write a big file
size_pool = [N]
fpath = _write_random_file(dpath, part_pool, size_pool, rng)
megabytes = os.stat(fpath).st_size / (2**20)
print('megabytes = {!r}'.format(megabytes))
for hasher in HASHERS:
for timer in ti.reset(hasher):
ub.hash_file(fpath, hasher=hasher)
results[hasher].update({N: ti.mean()})
col = {h: results[h][N] for h in HASHERS}
sortx = ub.argsort(col)
ranking = ub.dict_subset(col, sortx)
print('walltime: ' + ub.repr2(ranking, precision=9, nl=0))
best = next(iter(ranking))
#pairs = list(ub.iter_window( 2))
pairs = [(k, best) for k in ranking]
ratios = [ranking[k1] / ranking[k2] for k1, k2 in pairs]
nicekeys = ['{}/{}'.format(k1, k2) for k1, k2 in pairs]
relratios = ub.odict(zip(nicekeys, ratios))
print('speedup: ' + ub.repr2(relratios, precision=4, nl=0))
# xdoc +REQUIRES(--show)
# import pytest
# pytest.skip()
import pandas as pd
df = pd.DataFrame.from_dict(results)
df.columns.name = 'hasher'
df.index.name = 'N'
ratios = df.copy().drop(columns=df.columns)
for k1, k2 in [('sha512', 'xxh64'), ('sha1', 'xxh64'), ('xxh32', 'xxh64'),
('blake3', 'xxh64')]:
ratios['{}/{}'.format(k1, k2)] = df[k1] / df[k2]
print()
print('Seconds per iteration')
print(df.to_string(float_format='%.9f'))
print()
print('Ratios of seconds')
print(ratios.to_string(float_format='%.2f'))
print()
print('Average Ratio (over all N)')
print(ratios.mean().sort_values())
if ub.argflag('--show'):
import kwplot
kwplot.autompl()
xdata = sorted(ub.peek(results.values()).keys())
ydata = ub.map_vals(lambda d: [d[x] for x in xdata], results)
kwplot.multi_plot(xdata, ydata, xlabel='N', ylabel='seconds')
kwplot.show_if_requested()
def benchamrk_det_nms():
"""
Benchmarks different implementations of non-max-supression on the CPU, GPU,
and using cython / numpy / torch.
CommandLine:
xdoctest -m ~/code/kwimage/dev/bench_nms.py benchamrk_det_nms --show
SeeAlso:
PJR Darknet NonMax supression
https://github.com/pjreddie/darknet/blob/master/src/box.c
Lightnet NMS
https://gitlab.com/EAVISE/lightnet/blob/master/lightnet/data/transform/_postprocess.py#L116
"""
# N = 200
# bestof = 50
N = 1
bestof = 1
# xdata = [10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 1500, 2000]
# max number of boxes yolo will spit out at a time
max_boxes = 19 * 19 * 5
xdata = [
10, 20, 40, 80, 100, 200, 300, 400, 500, 600, 700, 1000, 1500,
max_boxes
]
# xdata = [10, 20, 40, 80, 100, 200, 300, 400, 500]
# Demo values
xdata = [0, 1, 2, 3, 10, 100, 200, 300, 500]
if ub.argflag('--small'):
xdata = [10, 100, 500, 1000, 1500, 2000, 5000, 10000]
if ub.argflag('--medium'):
xdata = [
1000,
5000,
10000,
20000,
50000,
]
if ub.argflag('--large'):
xdata = [
1000,
5000,
10000,
20000,
50000,
100000,
]
if ub.argflag('--extra-large'):
xdata = [
1000,
2000,
10000,
20000,
40000,
100000,
200000,
]
title_parts = []
SMALL_BOXES = ub.argflag('--small-boxes')
if SMALL_BOXES:
title_parts.append('small boxes')
else:
title_parts.append('large boxes')
# NOTE: for large images we may have up to 21,850,753 detections!
thresh = float(ub.argval('--thresh', default=0.4))
title_parts.append('thresh={:.2f}'.format(thresh))
from kwimage.algo.algo_nms import available_nms_impls
valid_impls = available_nms_impls()
print('valid_impls = {!r}'.format(valid_impls))
basis = {
'type': ['ndarray', 'tensor', 'tensor0'],
# 'daq': [True, False],
# 'daq': [False],
# 'device': [None],
# 'impl': valid_impls,
'impl': valid_impls + ['auto'],
}
if ub.argflag('--daq'):
basis['daq'] = [True, False]
# if torch.cuda.is_available():
# basis['device'].append(0)
combos = [
ub.dzip(basis.keys(), vals) for vals in it.product(*basis.values())
]
def is_valid_combo(combo):
# if combo['impl'] in {'py', 'cython_cpu'} and combo['device'] is not None:
# return False
# if combo['type'] == 'ndarray' and combo['impl'] == 'cython_gpu':
# if combo['device'] is None:
# return False
# if combo['type'] == 'ndarray' and combo['impl'] != 'cython_gpu':
# if combo['device'] is not None:
# return False
# if combo['type'].endswith('0'):
# if combo['impl'] in {'numpy', 'cython_gpu', 'cython_cpu'}:
# return False
# if combo['type'] == 'ndarray':
# if combo['impl'] in {'torch'}:
# return False
REMOVE_SLOW = True
if REMOVE_SLOW:
known_bad = [
{
'impl': 'torch',
'type': 'tensor'
},
{
'impl': 'numpy',
'type': 'tensor'
},
# {'impl': 'cython_gpu', 'type': 'tensor'},
{
'impl': 'cython_cpu',
'type': 'tensor'
},
# {'impl': 'torch', 'type': 'tensor0'},
{
'impl': 'numpy',
'type': 'tensor0'
},
# {'impl': 'cython_gpu', 'type': 'tensor0'},
# {'impl': 'cython_cpu', 'type': 'tensor0'},
{
'impl': 'torchvision',
'type': 'ndarray'
},
]
for known in known_bad:
if all(combo[key] == val for key, val in known.items()):
return False
return True
combos = list(filter(is_valid_combo, combos))
times = ub.ddict(list)
for num in xdata:
if num > 10000:
N = 1
bestof = 1
if num > 1000:
N = 3
bestof = 1
if num > 100:
N = 10
bestof = 3
elif num > 10:
N = 100
bestof = 10
else:
N = 1000
bestof = 10
print('\n\n---- number of boxes = {} ----\n'.format(num))
outputs = {}
ti = ub.Timerit(N, bestof=bestof, verbose=1)
# Build random test boxes and scores
np_dets1 = kwimage.Detections.random(num // 2, scale=1000.0, rng=0)
np_dets1.data['boxes'] = np_dets1.boxes.to_xywh()
if SMALL_BOXES:
max_dim = 100
np_dets1.boxes.data[..., 2] = np.minimum(np_dets1.boxes.width,
max_dim).ravel()
np_dets1.boxes.data[..., 3] = np.minimum(np_dets1.boxes.height,
max_dim).ravel()
np_dets2 = copy.deepcopy(np_dets1)
np_dets2.boxes.translate(10, inplace=True)
# add boxes that will definately be removed
np_dets = kwimage.Detections.concatenate([np_dets1, np_dets2])
# make all scores unique to ensure comparability
np_dets.scores[:] = np.linspace(0, 1, np_dets.num_boxes())
np_dets.data['scores'] = np_dets.scores.astype(np.float32)
np_dets.boxes.data = np_dets.boxes.data.astype(np.float32)
typed_data = {}
# ----------------------------------
import netharn as nh
for combo in combos:
print('combo = {}'.format(ub.repr2(combo, nl=0)))
label = nh.util.make_idstr(combo)
mode = combo.copy()
# if mode['impl'] == 'cython_gpu':
# mode['device_id'] = mode['device']
mode_type = mode.pop('type')
if mode_type in typed_data:
dets = typed_data[mode_type]
else:
if mode_type == 'ndarray':
dets = np_dets.numpy()
elif mode_type == 'tensor':
dets = np_dets.tensor(None)
elif mode_type == 'tensor0':
dets = np_dets.tensor(0)
else:
raise KeyError
typed_data[mode_type] = dets
for timer in ti.reset(label):
with timer:
keep = dets.non_max_supression(thresh=thresh, **mode)
torch.cuda.synchronize()
times[ti.label].append(ti.min())
outputs[ti.label] = ensure_numpy_indices(keep)
# ----------------------------------
# Check that all kept boxes do not have more than `threshold` ious
if 0:
for key, keep_idxs in outputs.items():
kept = np_dets.take(keep_idxs).boxes
ious = kept.ious(kept)
max_iou = (np.tril(ious) - np.eye(len(ious))).max()
if max_iou > thresh:
print('{} produced a bad result with max_iou={}'.format(
key, max_iou))
# Check result consistency:
print('\nResult stats:')
for key in sorted(outputs.keys()):
print(' * {:<20}: num={}'.format(key, len(outputs[key])))
print('\nResult overlaps (method1, method2: jaccard):')
datas = []
for k1, k2 in it.combinations(sorted(outputs.keys()), 2):
idxs1 = set(outputs[k1])
idxs2 = set(outputs[k2])
jaccard = len(idxs1 & idxs2) / max(len(idxs1 | idxs2), 1)
datas.append((k1, k2, jaccard))
datas = sorted(datas, key=lambda x: -x[2])
for k1, k2, jaccard in datas:
print(' * {:<20}, {:<20}: {:0.4f}'.format(k1, k2, jaccard))
if True:
ydata = {key: 1.0 / np.array(vals) for key, vals in times.items()}
ylabel = 'Hz'
reverse = True
yscale = 'symlog'
else:
ydata = {key: np.array(vals) for key, vals in times.items()}
ylabel = 'seconds'
reverse = False
yscale = 'linear'
scores = {key: vals[-1] for key, vals in ydata.items()}
ydata = ub.dict_subset(ydata, ub.argsort(scores, reverse=reverse))
###
times_of_interest = [0, 10, 100, 200, 1000]
times_of_interest = xdata
lines = []
record = lines.append
record('### times_of_interest = {!r}'.format(times_of_interest))
for x in times_of_interest:
if times_of_interest[-1] == x:
record('else:')
elif times_of_interest[0] == x:
record('if num <= {}:'.format(x))
else:
record('elif num <= {}:'.format(x))
if x in xdata:
pos = xdata.index(x)
score_wrt_x = {}
for key, vals in ydata.items():
score_wrt_x[key] = vals[pos]
typekeys = ['tensor0', 'tensor', 'ndarray']
type_groups = dict([(b,
ub.group_items(score_wrt_x,
lambda y: y.endswith(b))[True])
for b in typekeys])
# print('\n=========')
# print('x = {!r}'.format(x))
record(' if code not in {!r}:'.format(set(typekeys)))
record(' raise KeyError(code)')
for typekey, group in type_groups.items():
# print('-------')
record(' if code == {!r}:'.format(typekey))
# print('typekey = {!r}'.format(typekey))
# print('group = {!r}'.format(group))
group_x = ub.dict_isect(score_wrt_x, group)
valid_keys = ub.argsort(group_x, reverse=True)
valid_x = ub.dict_subset(group_x, valid_keys)
# parts = [','.split(k) for k in valid_keys]
ordered_impls = []
ordered_impls2 = ub.odict()
for k in valid_keys:
vals = valid_x[k]
p = k.split(',')
d = dict(i.split('=') for i in p)
ordered_impls2[d['impl']] = vals
ordered_impls.append(d['impl'])
ordered_impls = list(ub.oset(ordered_impls) - {'auto'})
ordered_impls2.pop('auto')
record(' # {}'.format(
ub.repr2(ordered_impls2, precision=1, nl=0,
explicit=True)))
record(' preference = {}'.format(
ub.repr2(ordered_impls, nl=0)))
record('### end times of interest ')
print(ub.indent('\n'.join(lines), ' ' * 8))
###
markers = {
key: 'o' if 'auto' in key else ''
for key, score in scores.items()
}
if ub.argflag('--daq'):
markers = {
key: '+' if 'daq=True' in key else ''
for key, score in scores.items()
}
labels = {
key: '{:.2f} {} - {}'.format(score, ylabel[0:3], key)
for key, score in scores.items()
}
title = 'NSM-impl speed: ' + ', '.join(title_parts)
import kwplot
kwplot.autompl()
kwplot.multi_plot(
xdata,
ydata,
xlabel='num boxes',
ylabel=ylabel,
label=labels,
yscale=yscale,
title=title,
marker=markers,
# xscale='symlog',
)
kwplot.show_if_requested()
