def ap(self, det, anno, iou=.5, csv=None):
if csv is not None:
base_path = Path(csv)
if self.fast_pr:
pr = bbb.pr(det, anno, iou)
if csv is not None:
np.savetxt(str(base_path), np.array(pr), delimiter=',')
return round(100 * bbb.ap(*pr), 2)
else:
aps = []
for c in tqdm(self.params.class_label_map):
anno_c = bbb.filter_discard(copy.deepcopy(anno),
[lambda a: a.class_label == c])
det_c = bbb.filter_discard(copy.deepcopy(det),
[lambda d: d.class_label == c])
pr = bbb.pr(det_c, anno_c)
if csv is not None:
np.savetxt(str(
base_path.with_name(base_path.stem + f'_{c}' +
base_path.suffix)),
np.array(pr),
delimiter=',')
aps.append(bbb.ap(*pr))
return round(100 * mean(aps), 2)
def bb_map(truth, pred):
import lightnet as ln
import torch
detection_to_brambox = ln.data.transform.TensorToBrambox(NETWORK_SIZE, LABELS)
def as_annos(truth, Win=1, Hin=1):
"""
Construct an BramBox annotation using the basic YOLO box format
"""
from brambox.boxes.annotations import Annotation
for true in truth:
anno = Annotation()
anno.class_id = true[0]
anno.class_label = LABELS[int(true[0])]
x_center, y_center, w, h = true[1:5]
anno.x_top_left = (x_center - w / 2) * Win
anno.y_top_left = (y_center - h / 2) * Hin
anno.width, anno.height = w * Win, h * Hin
anno.ignore = False
yield anno
true_bram = list(as_annos(truth, *NETWORK_SIZE))
pred_bram = detection_to_brambox([torch.Tensor(pred)])[0]
ln_det = {'a': pred_bram}
anno = {'a': true_bram}
import brambox.boxes as bbb
detections = ln_det
ground_truth = anno
overlap_threshold = 0.5
bb_precision, bb_recall = bbb.pr(detections, ground_truth, overlap_threshold)
bb_map = round(bbb.ap(bb_precision, bb_recall) * 100, 2)
return bb_map
def test(self):
log.info('Start testing')
self.network.eval()
tot_loss = 0
anno, det = {}, {}
with torch.no_grad():
for idx, (data, target) in enumerate(self.valid_loader):
data = data.to(self.device)
output, loss = self.network(data, target)
tot_loss += loss.item()*len(target)
key_val = len(anno)
anno.update({key_val+k: v for k,v in enumerate(target)})
det.update({key_val+k: v for k,v in enumerate(output)})
if self.sigint:
self.network.train()
return
pr = bbb.pr(det, anno)
m_ap = round(bbb.ap(*pr)*100, 2)
loss = tot_loss/len(anno)
self.log(f'Loss:{loss:.5f} mAP:{m_ap}%')
self.plot_valid_loss(np.array([loss]), np.array([self.batch]))
self.plot_valid_pr(np.array(pr[0]), np.array(pr[1]), update='replace')
if m_ap > self.best_map:
if self.best_map > 0:
self.plot_valid_pr(None, name=f'best - {self.best_map}%', update='remove', opts=None)
self.best_map = m_ap
self.network.save(os.path.join(self.backup_folder, 'best_map.pt'))
self.plot_valid_pr(np.array(pr[0]), np.array(pr[1]), name=f'best - {self.best_map}%', update='new', opts=dict(legend=[f'best - {self.best_map}%']))
self.network.train()
def test(self):
tot_loss = []
anno, det = {}, {}
for idx, (data, target) in enumerate(
tqdm(self.testloader, total=len(self.testloader))):
if self.cuda:
data = data.cuda(async=PIN_MEM)
data = torch.autograd.Variable(data, volatile=True)
output, loss = self.network(data, target)
tot_loss.append(loss.data[0] * len(target))
key_val = len(anno)
anno.update({key_val + k: v for k, v in enumerate(target)})
det.update({key_val + k: v for k, v in enumerate(output)})
if self.sigint:
return
pr = bbb.pr(det, anno)
m_ap = bbb.ap(*pr)
loss = round(sum(tot_loss) / len(anno), 5)
self.log('\nLoss:{loss} mAP:{m_ap:0.02f}%'.format(loss=loss,
m_ap=m_ap * 100.0))
self.visdom_plot_test_loss(np.array([loss]), np.array([self.batch]))
self.hyperdash_plot_train_loss('Loss Total (Test)', loss, log=False)
self.hyperdash_plot_train_loss('mAP (Test)', m_ap, log=False)
def test(weight, device, save_det):
log.debug('Creating network')
net = ln.models.TinyYolo(len(CLASS_LABELS), weight, CONF_THRESH, NMS_THRESH)
net.postprocess.append(ln.data.transform.TensorToBrambox(NETWORK_SIZE, CLASS_LABELS))
net = net.to(device)
net.eval()
log.debug('Creating dataset')
loader = torch.utils.data.DataLoader(
ln.models.DarknetDataset(TESTFILE, augment=False, input_dimension=NETWORK_SIZE, class_label_map=CLASS_LABELS),
batch_size = MINI_BATCH,
shuffle = False,
drop_last = False,
num_workers = WORKERS,
pin_memory = PIN_MEM,
collate_fn = ln.data.list_collate,
)
log.debug('Running network')
tot_loss = []
coord_loss = []
conf_loss = []
cls_loss = []
anno, det = {}, {}
num_det = 0
with torch.no_grad():
for idx, (data, box) in enumerate(tqdm(loader, total=len(loader))):
data = data.to(device)
output, loss = net(data, box)
tot_loss.append(net.loss.loss_tot.item()*len(box))
coord_loss.append(net.loss.loss_coord.item()*len(box))
conf_loss.append(net.loss.loss_conf.item()*len(box))
if net.loss.loss_cls is not None:
cls_loss.append(net.loss.loss_cls.item()*len(box))
key_val = len(anno)
anno.update({loader.dataset.keys[key_val+k]: v for k,v in enumerate(box)})
det.update({loader.dataset.keys[key_val+k]: v for k,v in enumerate(output)})
log.debug('Computing statistics')
pr = bbb.pr(det, anno)
m_ap = bbb.ap(*pr)*100
tot = sum(tot_loss)/len(anno)
coord = sum(coord_loss)/len(anno)
conf = sum(conf_loss)/len(anno)
if len(cls_loss) > 0:
cls = sum(cls_loss)/len(anno)
log.test(f'{net.seen//BATCH} mAP:{m_ap:.2f}% Loss:{tot:.5f} (Coord:{coord:.2f} Conf:{conf:.2f} Cls:{cls:.2f})')
else:
log.test(f'{net.seen//BATCH} mAP:{m_ap:.2f}% Loss:{tot:.5f} (Coord:{coord:.2f} Conf:{conf:.2f})')
if save_det is not None:
# Note: These detection boxes are the coordinates for the letterboxed images,
# you need ln.data.transform.ReverseLetterbox to have the right ones.
# Alternatively, you can save the letterboxed annotations, and use those for statistics later on!
bbb.generate('det_pickle', det, Path(arguments.save_det).with_suffix('.pkl'))
def test(self):
log.info('Start testing')
self.network.eval()
tot_loss = []
anno, det = {}, {}
with torch.no_grad():
for idx, (data, target) in enumerate(self.valid_loader):
data = data.to(self.device)
output, loss = self.network(data, target)
tot_loss.append(loss.item()*len(target))
key_val = len(anno)
anno.update({key_val+k: v for k,v in enumerate(target)})
det.update({key_val+k: v for k,v in enumerate(output)})
if self.sigint:
self.network.train()
return
m_ap = bbb.ap(*bbb.pr(det, anno)) * 100
loss = sum(tot_loss) / len(self.valid_loader.dataset)
self.log(f'Loss:{loss:.5f} mAP:{m_ap}%')
def test(self):
tot_loss = []
anno, det = {}, {}
for idx, (data, target) in enumerate(self.testloader):
if self.cuda:
data = data.cuda()
if torch.__version__.startswith('0.3'):
data = torch.autograd.Variable(data, volatile=True)
output, loss = self.network(data, target)
tot_loss.append(loss.data[0] * len(target))
else:
with torch.no_grad():
output, loss = self.network(data, target)
tot_loss.append(loss.item() * len(target))
key_val = len(anno)
anno.update({key_val + k: v for k, v in enumerate(target)})
det.update({key_val + k: v for k, v in enumerate(output)})
if self.sigint:
return
pr = bbb.pr(det, anno)
m_ap = bbb.ap(*pr)
loss = round(sum(tot_loss) / len(anno), 5)
self.log('Loss:{loss} mAP:{m_ap:0.02f}%'.format(loss,
m_ap=m_ap * 100.0))
self.plot_test_loss(np.array([loss]), np.array([self.batch]))
self.plot_test_pr.clear()
self.plot_test_pr(np.array(pr[0]),
np.array(pr[1]),
update='replace',
name='{self.batch} - {m_ap:0.02f}%'.format(
self=self, m_ap=m_ap * 100.0))
def generate_curves(ground_truth,
results,
pr=True,
title="",
filename="",
overlap=0.5,
only_plot=None,
linewidth=2,
figsize=(8, 6),
legendloc=3):
curves = []
scores = {}
# colors = plt.rcParams['axes.prop_cycle'].by_key()['color']
# colors = ['#1919ff', '#ff7f0e', '#ff1919', '#ff19ff', '#19ff19', '#19ff19']
colors = ['#1919ff', '#ff7f0e', '#ff1919', '#ff19ff', '#19ff19']
i = 0
linestyles = ['-', '-', '-', '--', '--', '--', '-.', ':']
for label, detections in results.items():
if pr:
ys, xs = bbb.pr(detections, ground_truth, overlap)
score = round(bbb.ap(ys, xs) * 100, 2)
else:
ys, xs = bbb.mr_fppi(detections, ground_truth, overlap)
score = round(lamr(ys, xs) * 100, 2)
color = colors[i % len(colors)]
linestyle = linestyles[i % len(linestyles)]
if only_plot is None or label in only_plot:
i += 1
curves += [(label, ys, xs, score, color, linestyle)]
# print(score)
# score = round(score,2)
# print(score)
scores[label] = score
if pr:
# sort from highest ap to lowest
sorted_curves = sorted(curves,
key=lambda curve: curve[3],
reverse=True)
else:
# sort from lowest to highest
sorted_curves = sorted(curves, key=lambda curve: curve[3])
fig, ax = plt.subplots(figsize=figsize)
for label, ys, xs, score, color, linestyle in sorted_curves:
# skip curves not mensioned in only_plot
if only_plot is not None and label not in only_plot:
continue
if pr:
plt.plot(xs,
ys,
color=color,
linestyle=linestyle,
label=f"{score:.2f}% {label}",
linewidth=linewidth)
else:
plt.loglog(xs,
ys,
color=color,
linestyle=linestyle,
label=f"{score:.2f}% {label}",
linewidth=linewidth)
plt.legend(loc=legendloc)
plt.gcf().suptitle(title, weight='bold', x=0.5, y=0.95)
if pr:
plt.grid(which='major')
plt.gca().set_ylabel('Precision')
plt.gca().set_xlabel('Recall')
plt.gca().set_xlim([0, 1])
plt.gca().set_ylim([0, 1])
plt.tight_layout(pad=0)
else:
# modify the y axis a bit
from matplotlib.ticker import FormatStrFormatter, LogLocator
subs = [1.0, 2.0, 3.0, 4.0, 5.0, 6.4, 8.0] # ticks to show per decade
ax.yaxis.set_minor_locator(LogLocator(subs=subs))
ax.yaxis.set_minor_formatter(FormatStrFormatter("%.2f"))
ax.yaxis.grid(which='minor')
ax.xaxis.grid(which='major')
plt.setp(ax.get_ymajorticklabels(),
visible=False) # disable major labels
plt.gca().set_ylabel('Miss rate')
plt.gca().set_xlabel('FPPI')
plt.gca().set_ylim([0.1, 1])
plt.gca().set_ylabel('Miss rate')
plt.gca().set_xlabel('FPPI')
plt.tight_layout(pad=0)
# plt.gca().set_xlim([0, 10])
if filename:
# plt.savefig(filename+'.pdf', format='pdf', bbox_inches = 'tight',pad_inches=0, transparent=True)
plt.savefig(filename + '.png', format='png')
return scores, plt.get_current_fig_manager()
def _ln_loop(ln_model, xpu, harn):
"""
Uses ln data, but nh map computation
"""
import lightnet as ln
ln_test = ub.import_module_from_path(ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
harn.current_tag = tag = 'test'
# Keep track of NH metrics
dmet = harn.dmets[tag]
dmet.pred.remove_all_annotations()
dmet.true.remove_all_annotations()
dmet.true._build_index()
dmet.pred._build_index()
# Keep track of LN metrics
anno = {}
ln_det = {}
resize_anno = {}
resize_ln_det = {}
ln_keys_to_gid = {}
for gid, img in dmet.true.imgs.items():
key = os.path.splitext(img['file_name'].split('VOCdevkit/')[1])[0]
ln_keys_to_gid[key] = gid
def brambox_to_labels(ln_loader, ln_bramboxes, inp_size, LABELS, offset=None):
""" convert brambox to netharn style labels """
import lightnet as ln
max_anno = max(map(len, ln_bramboxes))
ln_targets = [
ln.data.transform.BramboxToTensor.apply(
annos, inp_size, max_anno=max_anno, class_label_map=LABELS)
for annos in ln_bramboxes]
ln_targets = torch.stack(ln_targets)
gt_weights = -np.ones((len(ln_bramboxes), max_anno), dtype=np.float32)
for i, annos in enumerate(ln_bramboxes):
weights = 1.0 - np.array([anno.ignore for anno in annos], dtype=np.float32)
gt_weights[i, 0:len(annos)] = weights
gt_weights = torch.Tensor(gt_weights)
bg_weights = torch.FloatTensor(np.ones(len(ln_targets)))
if offset is not None:
# Hack to find image size, assume ordered iteration, which
# might be true for the test set.
orig_sizes = []
indices = []
for k in range(len(ln_bramboxes)):
key = ln_loader.dataset.keys[offset + k]
gid = ln_keys_to_gid[key]
orig_sizes += [(dmet.true.imgs[gid]['width'], dmet.true.imgs[gid]['height'])]
indices += [gid]
indices = torch.FloatTensor(indices)
orig_sizes = torch.FloatTensor(orig_sizes)
else:
indices = None
orig_sizes = None
ln_labels = {
'targets': ln_targets,
'gt_weights': gt_weights,
'orig_sizes': orig_sizes,
'indices': indices,
'bg_weights': bg_weights,
}
return ln_labels
def img_to_box(ln_loader, boxes, offset):
gname_lut = ln_loader.dataset.keys
return {gname_lut[offset + k]: v for k, v in enumerate(boxes)}
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, ln_model)
ln_loader = torch.utils.data.DataLoader(
ln_dset, batch_size=8, shuffle=False, drop_last=False,
num_workers=4, pin_memory=True, collate_fn=ln.data.list_collate,
)
detection_to_brambox = ln.data.transform.TensorToBrambox(ln_test.NETWORK_SIZE, ln_test.LABELS)
ln.data.transform.ReverseLetterbox
# ----------------------
# Postprocessing to transform yolo outputs into detections
# Basic difference here is the implementation of NMS
ln_postprocess = ln_model.postprocess
# ----------------------
# ln_results = []
moving_ave = nh.util.util_averages.CumMovingAve()
prog = ub.ProgIter(ln_loader, desc='')
with torch.no_grad():
ln_loader.dataset.keys
for bx, ln_batch in enumerate(prog):
ln_inputs, ln_bramboxes = ln_batch
# Convert brambox into components understood by netharn
ln_inputs = xpu.variable(ln_inputs)
inp_size = tuple(ln_inputs.shape[-2:][::-1])
# hack image index (assume they are sequential)
offset = len(anno)
ln_labels = brambox_to_labels(ln_loader, ln_bramboxes,
inp_size, ln_test.LABELS,
offset=offset)
ln_model.loss.seen = 1000000
ln_outputs = ln_model._forward(ln_inputs)
ln_loss_bram = ln_model.loss(ln_outputs, ln_bramboxes)
moving_ave.update(ub.odict([
('loss_bram', float(ln_loss_bram.sum())),
]))
# Display progress information
average_losses = moving_ave.average()
description = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(description, refresh=False)
# nh_outputs and ln_outputs should be the same, so no need to
# differentiate between them here.
ln_postout = ln_postprocess(ln_outputs.clone())
# ln_results.append((ln_postout, ln_labels, inp_size))
# Track NH stats
pred_anns = list(harn._postout_to_pred_ann(
inp_size, ln_labels, ln_postout, undo_lb=False,
_aidbase=len(dmet.pred.dataset['annotations']) + 1))
dmet.pred.add_annotations(pred_anns)
true_anns = list(harn._labels_to_true_ann(
inp_size, ln_labels, undo_lb=False,
_aidbase=len(dmet.true.dataset['annotations']) + 1))
dmet.true.add_annotations(true_anns)
# Track LN stats
ln_brambox_postout = detection_to_brambox([x.clone() for x in ln_postout])
anno.update(img_to_box(ln_loader, ln_bramboxes, offset))
ln_det.update(img_to_box(ln_loader, ln_brambox_postout, offset))
# Also track bb-stats in original sizes
ln_resize_annos = []
for bb_anns, orig_size in zip(ln_bramboxes, ln_labels['orig_sizes']):
old_bb_anns = copy.deepcopy(bb_anns)
new_bb_anns = ln.data.transform.ReverseLetterbox.apply(
[old_bb_anns], ln_test.NETWORK_SIZE, orig_size)[0]
ln_resize_annos.append(new_bb_anns)
ln_resize_dets = []
for bb_dets, orig_size in zip(ln_brambox_postout, ln_labels['orig_sizes']):
old_bb_dets = copy.deepcopy(bb_dets)
new_bb_dets = ln.data.transform.ReverseLetterbox.apply(
[old_bb_dets], ln_test.NETWORK_SIZE, orig_size)[0]
ln_resize_dets.append(new_bb_dets)
resize_anno.update(img_to_box(ln_loader, ln_resize_annos, offset))
resize_ln_det.update(img_to_box(ln_loader, ln_resize_dets, offset))
print('lightnet voc_mAP = {}'.format(dmet.score_voc()['mAP']))
print('lightnet nh_mAP = {}'.format(dmet.score_netharn()['mAP']))
# Compute mAP using brambox / lightnet
import brambox.boxes as bbb
ln_mAP = round(bbb.ap(*bbb.pr(ln_det, anno)) * 100, 2)
print('ln_bb_mAP = {!r}'.format(ln_mAP))
ln_resize_mAP = round(bbb.ap(*bbb.pr(resize_ln_det, resize_anno)) * 100, 2)
print('ln_resize_bb_mAP = {!r}'.format(ln_resize_mAP))
if False:
# Check sizes
for bb in resize_anno[ln_loader.dataset.keys[0]]:
print(list(map(float, [bb.x_top_left, bb.y_top_left, bb.width, bb.height])))
print(dmet.true.annots(gid=0).boxes)
# Check sizes
for bb in resize_ln_det[ln_loader.dataset.keys[0]][0:3]:
print(list(map(float, [bb.x_top_left, bb.y_top_left, bb.width, bb.height])))
print(dmet.pred.annots(gid=0).boxes[0:3])
def _test_with_lnstyle_data():
"""
Uses pretrained lightnet weights, and the lightnet data loader.
Uses my critrion and net implementations.
(already verified to produce the same outputs)
Checks to see if my loss and map calcluations are the same as lightnet
CommandLine:
python ~/code/netharn/netharn/examples/yolo_voc.py _test_with_lnstyle_data
Using LighNet Trained Weights:
LightNet Results:
TEST 30000 mAP:74.18% Loss:3.16839 (Coord:0.38 Conf:1.61 Cls:1.17)
My Results:
# Worse losses (due to different image loading)
loss: 5.00 {coord: 0.69, conf: 2.05, cls: 2.26}
mAP = 0.6227
The MAP is quite a bit worse... Why is that?
USING THE SAME WEIGHTS! I must be doing something wrong.
Results using the same Data Loader:
{cls: 2.22, conf: 2.05, coord: 0.65, loss: 4.91}
# Checking with extra info
{loss_bram: 3.17, loss_ten1: 4.91, loss_ten2: 4.91}
OH!, Is is just that BramBox has an ignore function?
- [X] Add ignore / weight to tensor version to see if its the same
YUP! {loss_bram: 3.17, loss_ten1: 4.91, loss_ten2: 4.91, nh_unweighted: 4.92, nh_weighted: 3.16}
TO CHECK:
- [ ] why is the loss different?
- [X] network input size is 416 in both
- [x] networks output the same data given the same input
- [x] loss outputs the same data given the same input (they do if seen is the same)
- [x] CONCLUSION: The loss is not actually different
- [ ] why is the mAP different?
- [x] does brambox compute AP differently?
... YES
CONCLUSION: several factors are at play
* brambox has a different AP computation
* netharn and lightnet non-max supressions are different
* The NMS seems to have the most significant impact
# """
import brambox.boxes as bbb
CHECK_SANITY = False
# Path to weights that we will be testing
# (These were trained using the lightnet harness and achived a good mAP)
weights_fpath = ub.truepath('~/code/lightnet/examples/yolo-voc/backup/weights_30000.pt')
# Load weights into a netharn model
harn = setup_harness(bsize=2)
harn.hyper.xpu = nh.XPU(0)
harn.initialize()
state_dict = harn.xpu.load(weights_fpath)['weights']
harn.model.module.load_state_dict(state_dict)
harn.model.eval()
nh_net = harn.model.module
# Load weights into a lightnet model
import os
import lightnet as ln
ln_test = ub.import_module_from_path(ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
ln_net = ln.models.Yolo(ln_test.CLASSES, weights_fpath,
ln_test.CONF_THRESH, ln_test.NMS_THRESH)
ln_net = harn.xpu.move(ln_net)
ln_net.eval()
# Sanity check, the weights should be the same
if CHECK_SANITY:
state1 = nh_net.state_dict()
state2 = ln_net.state_dict()
assert state1.keys() == state2.keys()
for k in state1.keys():
v1 = state1[k]
v2 = state2[k]
assert np.all(v1 == v2)
# Create a lightnet dataset loader
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, ln_net)
ln_loader = torch.utils.data.DataLoader(
ln_dset, batch_size=2, shuffle=False, drop_last=False, num_workers=0,
pin_memory=True, collate_fn=ln.data.list_collate,
)
# Create a netharn dataset loader
nh_loader = harn.loaders['test']
# ----------------------
# Postprocessing to transform yolo outputs into detections
# Basic difference here is the implementation of NMS
ln_postprocess = ln_net.postprocess
nh_postprocess = harn.model.module.postprocess
# ----------------------
# Define helper functions to deal with bramboxes
LABELS = ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car',
'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse',
'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train',
'tvmonitor']
NETWORK_SIZE = (416, 416)
detection_to_brambox = ln.data.TensorToBrambox(NETWORK_SIZE, LABELS)
# ----------------------
CHECK_LOSS = False
with torch.no_grad():
# Track netharn and lightnet results that will be scored
ln_batch_confusions = []
nh_batch_confusions = []
nh_batch_confusions0 = []
ln_results = []
nh_results = []
nh_results0 = []
anno = {}
ln_det = {}
nh_det = {}
nh_det0 = {}
moving_ave = nh.util.util_averages.CumMovingAve()
coco_truth = []
# ln_coco_detections = []
nh_coco_detections0 = []
prog = ub.ProgIter(zip(ln_loader, nh_loader), desc='')
for bx, (ln_batch, nh_batch) in enumerate(prog):
ln_inputs, ln_bramboxes = ln_batch
inp_size = tuple(ln_inputs.shape[-2:][::-1])
nh_inputs, nh_labels = nh_batch
nh_targets = nh_labels['targets']
nh_gt_weights = nh_labels['gt_weights']
# Convert brambox into components understood by netharn
ln_labels = brambox_to_labels(ln_bramboxes, inp_size, ln_test.LABELS)
ln_inputs = harn.xpu.variable(ln_inputs)
ln_targets = harn.xpu.variable(ln_labels['targets'])
ln_gt_weights = harn.xpu.variable(ln_labels['gt_weights']) # NOQA
ln_net.loss.seen = 1000000
ln_outputs = ln_net._forward(ln_inputs)
if CHECK_SANITY:
nh_outputs = harn.model(ln_inputs)
assert np.all(nh_outputs == ln_outputs)
ln_loss_bram = ln_net.loss(ln_outputs, ln_bramboxes)
moving_ave.update(ub.odict([
('loss_bram', float(ln_loss_bram.sum())),
]))
if CHECK_LOSS:
seen = ln_net.loss.seen
ln_loss_ten1 = harn.criterion(ln_outputs, ln_targets,
seen=seen)
ln_loss_ten2 = ln_net.loss(ln_outputs, ln_targets)
nh_weighted = harn.criterion(ln_outputs, nh_targets,
gt_weights=nh_gt_weights,
seen=seen)
nh_unweighted = harn.criterion(ln_outputs, nh_targets,
seen=seen)
moving_ave.update(ub.odict([
('loss_ten1', float(ln_loss_ten1.sum())),
('loss_ten2', float(ln_loss_ten2.sum())),
('nh_weighted', float(nh_weighted.sum())),
('nh_unweighted', float(nh_unweighted.sum())),
# ('coord', harn.criterion.loss_coord),
# ('conf', harn.criterion.loss_conf),
# ('cls', harn.criterion.loss_cls),
]))
# Display progress information
average_losses = moving_ave.average()
description = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(description, refresh=False)
# nh_outputs and ln_outputs should be the same, so no need to
# differentiate between them here.
ln_postout = ln_postprocess(ln_outputs.clone())
nh_postout = nh_postprocess(ln_outputs.clone())
# Should use the original NMS strategy
nh_postout0 = nh_postprocess(ln_outputs.clone(), mode=0)
ln_brambox_postout = detection_to_brambox([x.clone() for x in ln_postout])
nh_brambox_postout = detection_to_brambox([x.clone() for x in nh_postout])
nh_brambox_postout0 = detection_to_brambox([x.clone() for x in nh_postout0])
# Record data scored by brambox
offset = len(anno)
def img_to_box(boxes, offset):
gname_lut = ln_loader.dataset.keys
return {gname_lut[offset + k]: v for k, v in enumerate(boxes)}
anno.update(img_to_box(ln_bramboxes, offset))
ln_det.update(img_to_box(ln_brambox_postout, offset))
nh_det.update(img_to_box(nh_brambox_postout, offset))
nh_det0.update(img_to_box(nh_brambox_postout0, offset))
# Record data scored by netharn
ln_results.append((ln_postout, ln_labels, inp_size))
nh_results.append((nh_postout, nh_labels, inp_size))
nh_results0.append((nh_postout0, nh_labels, inp_size))
# preds, truths = harn._postout_to_coco(ln_postout, ln_labels, inp_size)
# ln_coco_detections.append(preds)
preds, truths = harn._postout_to_coco(nh_postout0, nh_labels, inp_size)
nh_coco_detections0.append(preds)
coco_truth.append(truths)
# kw = dict(bias=0)
# for y in harn._measure_confusion(ln_postout, ln_labels, inp_size, **kw):
# ln_batch_confusions.append(y)
# for y in harn._measure_confusion(nh_postout, nh_labels, inp_size, **kw):
# nh_batch_confusions.append(y)
# for y in harn._measure_confusion(nh_postout0, nh_labels, inp_size, **kw):
# nh_batch_confusions0.append(y)
if bx > 50:
break
# Compute mAP using brambox / lightnet
ln_mAP = round(bbb.ap(*bbb.pr(ln_det, anno)) * 100, 2)
nh_mAP = round(bbb.ap(*bbb.pr(nh_det, anno)) * 100, 2)
nh_mAP0 = round(bbb.ap(*bbb.pr(nh_det0, anno)) * 100, 2)
print('\n----')
print('ln_mAP = {!r}'.format(ln_mAP))
print('nh_mAP = {!r}'.format(nh_mAP))
print('nh_mAP0 = {!r}'.format(nh_mAP0))
# num_classes = len(LABELS)
# cls_labels = list(range(num_classes))
# # Compute mAP using netharn
# if False:
# is_tp = (y.true == y.pred) & (y.pred >= 0)
# is_fp = (y.true != y.pred) & (y.pred >= 0)
# rest = ~(is_fp | is_tp)
# y.true[is_tp] = 1
# y.pred[is_tp] = 1
# y.true[is_fp] = 0
# y.pred[is_fp] = 1
# y.true[rest] = 0
# y.pred[rest] = 0
# import sklearn
# import sklearn.metrics
# sklearn.metrics.average_precision_score(y.true, y.score, 'weighted',
# y.weight)
for bias in [0, 1]:
ln_batch_confusions = []
nh_batch_confusions = []
nh_batch_confusions0 = []
print('\n\n======\n\nbias = {!r}'.format(bias))
kw = dict(bias=bias, PREFER_WEIGHTED_TRUTH=False)
for ln_postout, ln_labels, inp_size in ln_results:
for y in harn._measure_confusion(ln_postout, ln_labels, inp_size, **kw):
ln_batch_confusions.append(y)
for nh_postout, nh_labels, inp_size in nh_results:
for y in harn._measure_confusion(nh_postout, nh_labels, inp_size, **kw):
nh_batch_confusions.append(y)
for nh_postout0, nh_labels, inp_size in nh_results0:
for y in harn._measure_confusion(nh_postout0, nh_labels, inp_size, **kw):
nh_batch_confusions0.append(y)
confusions = {
'lh': ln_batch_confusions,
# 'nh': nh_batch_confusions,
'nh0': nh_batch_confusions0,
}
for lbl, batch_confusions in confusions.items():
print('----')
print('\nlbl = {!r}'.format(lbl))
y = pd.concat([pd.DataFrame(c) for c in batch_confusions])
# aps = nh.metrics.ave_precisions(y, cls_labels, use_07_metric=True)
# aps = aps.rename(dict(zip(cls_labels, LABELS)), axis=0)
# mean_ap = np.nanmean(aps['ap'])
# print('mean_ap_07 = {:.2f}'.format(mean_ap * 100))
# aps = nh.metrics.ave_precisions(y, cls_labels, use_07_metric=False)
# aps = aps.rename(dict(zip(cls_labels, LABELS)), axis=0)
# mean_ap = np.nanmean(aps['ap'])
# print('mean_ap_12 = {:.2f}'.format(mean_ap * 100))
# Try the other way
from netharn.metrics.detections import _multiclass_ap
prec, recall, ap2 = _multiclass_ap(y)
print('ap2 = {!r}'.format(round(ap2 * 100, 2)))
def _ln_data_ln_map(ln_model, xpu, num=None):
"""
Compute the results on the ln test set using a ln model.
Weights can either be lightnet or netharn
"""
import os
import lightnet as ln
ln_test = ub.import_module_from_path(ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, ln_model)
ln_loader = torch.utils.data.DataLoader(
ln_dset, batch_size=2, shuffle=False, drop_last=False, num_workers=0,
pin_memory=True, collate_fn=ln.data.list_collate,
)
# ----------------------
# Postprocessing to transform yolo outputs into detections
# Basic difference here is the implementation of NMS
ln_postprocess = ln_model.postprocess
# ----------------------
# Define helper functions to deal with bramboxes
detection_to_brambox = ln.data.transform.TensorToBrambox(ln_test.NETWORK_SIZE,
ln_test.LABELS)
# ----------------------
def img_to_box(boxes, offset):
gname_lut = ln_loader.dataset.keys
return {gname_lut[offset + k]: v for k, v in enumerate(boxes)}
with torch.no_grad():
anno = {}
ln_det = {}
moving_ave = nh.util.util_averages.CumMovingAve()
prog = ub.ProgIter(ln_loader, desc='')
for bx, ln_batch in enumerate(prog):
ln_inputs, ln_bramboxes = ln_batch
# Convert brambox into components understood by netharn
ln_inputs = xpu.variable(ln_inputs)
ln_model.loss.seen = 1000000
ln_outputs = ln_model._forward(ln_inputs)
ln_loss_bram = ln_model.loss(ln_outputs, ln_bramboxes)
moving_ave.update(ub.odict([
('loss_bram', float(ln_loss_bram.sum())),
]))
# Display progress information
average_losses = moving_ave.average()
description = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(description, refresh=False)
# nh_outputs and ln_outputs should be the same, so no need to
# differentiate between them here.
ln_postout = ln_postprocess(ln_outputs.clone())
ln_brambox_postout = detection_to_brambox([x.clone() for x in ln_postout])
# Record data scored by brambox
offset = len(anno)
anno.update(img_to_box(ln_bramboxes, offset))
ln_det.update(img_to_box(ln_brambox_postout, offset))
if num is not None and bx >= num:
break
import brambox.boxes as bbb
# Compute mAP using brambox / lightnet
ln_mAP = round(bbb.ap(*bbb.pr(ln_det, anno)) * 100, 2)
print('\nln_mAP = {!r}'.format(ln_mAP))
return ln_mAP
detection = bbb.Detection.create(bounding_box)
detection.confidence = det[1]
data.append(detection)
detections[image.split('.')[0]] = data
# Read annotations file
annotations = bbb.parse('anno_darknet', args.annotations_path, image_width=img_w, image_height=img_h, class_label_map=classes)
# Generate PR-curve and compute AP for every individual class.
plt.figure()
for c in classes:
anno_c = bbb.filter_discard(copy.deepcopy(annotations), [lambda anno: anno.class_label == c])
det_c = bbb.filter_discard(copy.deepcopy(detections), [lambda det: det.class_label == c])
p, r = bbb.pr(det_c, anno_c)
ap = bbb.ap(p, r)
plt.plot(r, p, label=f'{c}: {round(ap * 100, 2)}%')
plt.gcf().suptitle('PR-curve individual')
plt.gca().set_ylabel('Precision')
plt.gca().set_xlabel('Recall')
plt.gca().set_xlim([0, 1])
plt.gca().set_ylim([0, 1])
plt.legend()
plt.show()
# Generate PR-curve and compute mAP
plt.figure()
p, r = bbb.pr(detections, annotations)
def test(arguments):
log.debug('Creating network')
net = ln.models.Yolo(CLASSES, arguments.weight, CONF_THRESH, NMS_THRESH)
net.postprocess.append(
ln.data.transform.TensorToBrambox(NETWORK_SIZE, LABELS))
net.eval()
if arguments.cuda:
net.cuda()
log.debug('Creating dataset')
loader = torch.utils.data.DataLoader(
CustomDataset(TESTFILE, net),
batch_size=MINI_BATCH,
shuffle=False,
drop_last=False,
num_workers=WORKERS if arguments.cuda else 0,
pin_memory=PIN_MEM if arguments.cuda else False,
collate_fn=ln.data.list_collate,
)
if arguments.visdom:
log.debug('Creating visdom visualisation wrappers')
vis = visdom.Visdom(port=VISDOM_PORT)
visdom_plot_pr = ln.engine.VisdomLinePlotter(
vis,
'pr',
opts=dict(xlabel='Recall',
ylabel='Precision',
title='Precision Recall',
xtickmin=0,
xtickmax=1,
ytickmin=0,
ytickmax=1,
showlegend=True))
if arguments.hyperdash:
log.debug('Creating hyperdash visualisation wrappers')
hd = hyperdash.Experiment('YOLOv2 Pascal VOC Test')
hyperdash_plot_pr = ln.engine.HyperdashLinePlotter(hd)
log.debug('Running network')
tot_loss = []
coord_loss = []
conf_loss = []
cls_loss = []
anno, det = {}, {}
for idx, (data, box) in enumerate(tqdm(loader, total=len(loader))):
if arguments.cuda:
data = data.cuda()
if torch.__version__.startswith('0.3'):
data = torch.autograd.Variable(data, volatile=True)
output, loss = net(data, box)
else:
with torch.no_grad():
output, loss = net(data, box)
if torch.__version__.startswith('0.3'):
tot_loss.append(net.loss.loss_tot.data[0] * len(box))
coord_loss.append(net.loss.loss_coord.data[0] * len(box))
conf_loss.append(net.loss.loss_conf.data[0] * len(box))
if net.loss.loss_cls is not None:
cls_loss.append(net.loss.loss_cls.data[0] * len(box))
else:
tot_loss.append(net.loss.loss_tot.item() * len(box))
coord_loss.append(net.loss.loss_coord.item() * len(box))
conf_loss.append(net.loss.loss_conf.item() * len(box))
if net.loss.loss_cls is not None:
cls_loss.append(net.loss.loss_cls.item() * len(box))
key_val = len(anno)
anno.update(
{loader.dataset.keys[key_val + k]: v
for k, v in enumerate(box)})
det.update({
loader.dataset.keys[key_val + k]: v
for k, v in enumerate(output)
})
log.debug('Computing statistics')
pr = bbb.pr(det, anno)
m_ap = round(bbb.ap(*pr) * 100, 2)
tot = round(sum(tot_loss) / len(anno), 5)
coord = round(sum(coord_loss) / len(anno), 2)
conf = round(sum(conf_loss) / len(anno), 2)
if len(cls_loss) > 0:
cls = round(sum(cls_loss) / len(anno), 2)
log.test(
'{seen} mAP:{m_ap}% Loss:{tot} (Coord:{coord} Conf:{conf} Cls:{cls})'
.format(seen=net.seen // BATCH,
m_ap=m_ap,
tot=tot,
coord=coord,
conf=conf,
cls=cls))
else:
log.test(
'{seen} mAP:{m_ap}% Loss:{tot} (Coord:{coord} Conf:{conf})'.format(
seen=net.seen // BATCH,
m_ap=m_ap,
tot=tot,
coord=coord,
conf=conf))
name = 'mAP: {m_ap}%'.format(m_ap=m_ap)
if arguments.visdom:
visdom_plot_pr(np.array(pr[0]), np.array(pr[1]), name=name)
if arguments.hyperdash:
now = time.time()
re_seen = None
for index, (re_, pr_) in enumerate(sorted(zip(pr[1], pr[0]))):
re_ = round(re_, 2)
if re_ != re_seen:
re_seen = re_
re_ = int(re_ * 100.0)
hyperdash_plot_pr(name, pr_, now + re_, log=False)
if arguments.save_det is not None:
# Note: These detection boxes are the coordinates for the letterboxed images,
# you need ln.data.ReverseLetterbox to have the right ones.
# Alternatively, you can save the letterboxed annotations, and use those for statistics later on!
bbb.generate('det_pickle', det,
Path(arguments.save_det).with_suffix('.pkl'))
#bbb.generate('anno_pickle', det, Path('anno-letterboxed_'+arguments.save_det).with_suffix('.pkl'))
if arguments.hyperdash:
hyperdash_plot_pr.close()
def evaluate_lightnet_model():
"""
Try to evaulate the model using the exact same VOC scoring metric
import ubelt as ub
import sys
sys.path.append(ub.truepath('~/code/netharn/netharn/examples/tests'))
from test_yolo import *
"""
import os # NOQA
import netharn as nh
import ubelt as ub
import lightnet as ln
import torch
ln_test = ub.import_module_from_path(
ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
xpu = nh.XPU.cast('auto')
ln_weights_fpath = ub.truepath(
'~/code/lightnet/examples/yolo-voc/backup/final.pt')
ln_weights_fpath = ub.truepath(
'~/code/lightnet/examples/yolo-voc/backup/weights_30000.pt')
from netharn.models.yolo2 import light_yolo
ln_weights_fpath = nh.models.yolo2.light_yolo.demo_voc_weights('darknet')
# Lightnet model, postprocess, and lightnet weights
ln_model = ln.models.Yolo(ln_test.CLASSES, ln_weights_fpath,
ln_test.CONF_THRESH, ln_test.NMS_THRESH)
ln_model = xpu.move(ln_model)
ln_model.eval()
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, ln_model)
ln_loader = torch.utils.data.DataLoader(
ln_dset,
batch_size=8,
shuffle=False,
drop_last=False,
num_workers=0,
pin_memory=True,
collate_fn=ln.data.list_collate,
)
# ----------------------
# Postprocessing to transform yolo outputs into detections
# Basic difference here is the implementation of NMS
ln_postprocess = ln.data.transform.util.Compose(
ln_model.postprocess.copy())
# ----------------------
# Define helper functions to deal with bramboxes
detection_to_brambox = ln.data.transform.TensorToBrambox(
ln_test.NETWORK_SIZE, ln_test.LABELS)
# hack so forward call behaves like it does in test
ln_model.postprocess.append(detection_to_brambox)
# ----------------------
def img_to_box(ln_loader, boxes, offset):
gname_lut = ln_loader.dataset.keys
return {gname_lut[offset + k]: v for k, v in enumerate(boxes)}
with torch.no_grad():
anno = {}
ln_det = {}
moving_ave = nh.util.util_averages.CumMovingAve()
prog = ub.ProgIter(ln_loader, desc='')
for bx, ln_raw_batch in enumerate(prog):
ln_raw_inputs, ln_bramboxes = ln_raw_batch
# Convert brambox into components understood by netharn
ln_inputs = xpu.variable(ln_raw_inputs)
ln_model.loss.seen = 1000000
ln_outputs = ln_model._forward(ln_inputs)
ln_loss_bram = ln_model.loss(ln_outputs, ln_bramboxes)
moving_ave.update(
ub.odict([
('loss_bram', float(ln_loss_bram.sum())),
]))
# Display progress information
average_losses = moving_ave.average()
description = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(description, refresh=False)
# nh_outputs and ln_outputs should be the same, so no need to
# differentiate between them here.
ln_postout = ln_postprocess(ln_outputs.clone())
ln_brambox_postout = detection_to_brambox(
[x.clone() for x in ln_postout])
# out, loss = ln_model.forward(ln_inputs, ln_bramboxes)
# Record data scored by brambox
offset = len(anno)
anno.update(img_to_box(ln_loader, ln_bramboxes, offset))
ln_det.update(img_to_box(ln_loader, ln_brambox_postout, offset))
import brambox.boxes as bbb
# Compute mAP using brambox / lightnet
pr = bbb.pr(ln_det, anno)
ln_mAP = round(bbb.ap(*pr) * 100, 2)
print('\nBrambox multiclass AP = {!r}'.format(ln_mAP))
devkit_dpath = ub.truepath('~/data/VOC/VOCdevkit/')
# Convert bramboxes to VOC style data for eval
class_to_dets = ub.ddict(list)
for gpath, bb_dets in ub.ProgIter(list(ln_det.items())):
from PIL import Image
from os.path import join
pil_image = Image.open(join(devkit_dpath, gpath) + '.jpg')
img_size = pil_image.size
bb_dets = ln.data.transform.ReverseLetterbox.apply(
[bb_dets], ln_test.NETWORK_SIZE, img_size)[0]
for det in bb_dets:
from os.path import basename, splitext
imgid = splitext(basename(gpath))[0]
score = det.confidence
tlwh = [
max(det.x_top_left, 0),
max(det.y_top_left, 0),
min(det.x_top_left + det.width, img_size[0]),
min(det.y_top_left + det.height, img_size[1])
]
# See: /home/joncrall/data/VOC/VOCdevkit/devkit_doc.pdf
# Each line is formatted as:
# <image identifier> <confidence> <left> <top> <right> <bottom>
class_to_dets[det.class_label].append(
list(ub.flatten([[imgid], [score], tlwh])))
# Calculate Original VOC measure
from os.path import join
results_path = join(devkit_dpath, 'results', 'VOC2007', 'Main')
detpath = join(results_path, 'comp3_det_test_{}.txt')
for lbl, dets in class_to_dets.items():
fpath = detpath.format(lbl)
with open(fpath, 'w') as file:
text = '\n'.join([' '.join(list(map(str, det))) for det in dets])
file.write(text)
import sys
sys.path.append('/home/joncrall/code/netharn/netharn/examples/tests')
from voc_eval_orig import voc_eval
use_07_metric = False
bias = 1.0
# for bias in [0.0, 1.0]:
# for use_07_metric in [True, False]:
class_aps = {}
class_curve = {}
annopath = join(devkit_dpath, 'VOC2007', 'Annotations', '{}.xml')
for classname in ub.ProgIter(list(class_to_dets.keys())):
cachedir = None
imagesetfile = join(devkit_dpath, 'VOC2007', 'ImageSets', 'Main',
'{}_test.txt').format(classname)
rec, prec, ap = voc_eval(detpath,
annopath,
imagesetfile,
classname,
cachedir,
ovthresh=0.5,
use_07_metric=use_07_metric,
bias=bias)
class_aps[classname] = ap
class_curve[classname] = (rec, prec)
mAP = np.mean(list(class_aps.values()))
print('Official* bias={} {} VOC mAP = {!r}'.format(
bias, '2007' if use_07_metric else '2012', mAP))
# I get 0.71091 without 07 metric
# I get 0.73164 without 07 metric
"""
Lightnet:
'aeroplane' : 0.7738,
'bicycle' : 0.8326,
'bird' : 0.7505,
'boat' : 0.6202,
'bottle' : 0.4614,
'bus' : 0.8078,
'car' : 0.8052,
'cat' : 0.8857,
'chair' : 0.5385,
'cow' : 0.7768,
'diningtable' : 0.7556,
'dog' : 0.8545,
'horse' : 0.8401,
'motorbike' : 0.8144,
'person' : 0.7608,
'pottedplant' : 0.4640,
'sheep' : 0.7398,
'sofa' : 0.7334,
'train' : 0.8697,
'tvmonitor' : 0.7533,
"""
"""
Official* bias=1.0 2012 VOC mAP = 0.7670408107201542
Darknet:
'aeroplane' : 0.7597,
'bicycle' : 0.8412,
'bird' : 0.7705,
'boat' : 0.6360,
'bottle' : 0.4902,
'bus' : 0.8164,
'car' : 0.8444,
'cat' : 0.8926,
'chair' : 0.5902,
'cow' : 0.8184,
'diningtable' : 0.7728,
'dog' : 0.8612,
'horse' : 0.8759,
'motorbike' : 0.8467,
'person' : 0.7855,
'pottedplant' : 0.5117,
'sheep' : 0.7889,
'sofa' : 0.7584,
'train' : 0.8997,
'tvmonitor' : 0.7806,
"""
"""
def generate_curves(ground_truth, results, pr=True, title="", saveplot="", overlap=0.5, only_plot=None,
linewidth=2, figsize=(8, 6), legendloc=3):
curves = []
scores = {}
# colors = plt.rcParams['axes.prop_cycle'].by_key()['color']
# colors = ['#1919ff', '#ff7f0e', '#ff1919', '#ff19ff', '#19ff19', '#19ff19']
colors = ['#1919ff', '#ff7f0e', '#ff1919', '#ff19ff', '#19ff19']
i = 0
linestyles = ['-', '--', '-.', ':']
for label, detections in results.items():
### because YOLO has stuck in small object, so the paper of this code on CVPRW prefer chose overlap 0.4, but here we choose 0.5 for all.
# if label=='YOLO_TLV' or label=='Ours: TD(V,V)' or label=='Ours: TD(T,T)' or label=='Ours: TD(VT,T)' or label=='Ours: BU(VAT,T)' or label == 'Ours: BU(VLT,T)':
# print(label)
# overlap = 0.4
if pr:
ys, xs = bbb.pr(detections, ground_truth, overlap)
score = round(bbb.ap(ys, xs) * 100, 2)
else:
ys, xs = bbb.mr_fppi(detections, ground_truth, overlap)
score = round(lamr(ys, xs) * 100, 2)
color = colors[i % len(colors)]
linestyle = linestyles[i % len(linestyles)]
if only_plot is None or label in only_plot:
i += 1
curves += [(label, ys, xs, score, color, linestyle)]
scores[label] = score
# if pr:
# # sort from highest ap to lowest
# sorted_curves = sorted(curves, key=lambda curve: curve[3], reverse=True)
# else:
# # sort from lowest to highest
# sorted_curves = sorted(curves, key=lambda curve: curve[3])
# fig, ax = plt.subplots(figsize=figsize)
# for label, ys, xs, score, color, linestyle in sorted_curves:
# # skip curves not mensioned in only_plot
# if only_plot is not None and label not in only_plot:
# continue
# if pr:
# plt.plot(xs, ys, color=color, linestyle=linestyle, label=f"{score}% {label}", linewidth=linewidth)
# else:
# plt.loglog(xs, ys, color=color, linestyle=linestyle, label=f"{score}% {label}", linewidth=linewidth)
#
# plt.legend(loc=legendloc)
#
# plt.gcf().suptitle(title, weight='bold')
# if pr:
# plt.grid(which='major')
# plt.gca().set_ylabel('Precision')
# plt.gca().set_xlabel('Recall')
# plt.gca().set_xlim([0, 1])
# plt.gca().set_ylim([0, 1])
# else:
# # modify the y axis a bit
# from matplotlib.ticker import FormatStrFormatter, LogLocator
# subs = [1.0, 2.0, 3.0, 4.0, 5.0, 6.4, 8.0] # ticks to show per decade
# ax.yaxis.set_minor_locator(LogLocator(subs=subs))
# ax.yaxis.set_minor_formatter(FormatStrFormatter("%.2f"))
# ax.yaxis.grid(which='minor')
# ax.xaxis.grid(which='major')
# plt.setp(ax.get_ymajorticklabels(), visible=False) # disable major labels
#
# plt.gca().set_ylabel('Miss rate')
# plt.gca().set_xlabel('FPPI')
# plt.gca().set_ylim([0.1, 1])
#
# plt.gca().set_xlim([0, 10])
# if saveplot:
# plt.savefig(saveplot, format='eps', dpi=1200)
return scores