def test_import_modpath_package():
assert '_tmproot373.sub1.sub2.testmod' not in sys.modules
temp = ub.TempDir().start()
# with ub.TempDir() as temp:
if True:
dpath = temp.dpath
# Create a dummy package heirachy
root = ub.ensuredir((dpath, '_tmproot373'))
sub1 = ub.ensuredir((root, 'sub1'))
sub2 = ub.ensuredir((sub1, 'sub2'))
ub.touch(join(root, '__init__.py'))
ub.touch(join(sub1, '__init__.py'))
ub.touch(join(sub2, '__init__.py'))
modpath = join(sub2, 'testmod.py')
text = ub.codeblock(
'''
a = 'value'
''')
ub.writeto(modpath, text)
assert temp.dpath not in sys.path
module = ub.import_module_from_path(modpath)
assert temp.dpath not in sys.path, 'pythonpath should remain clean'
assert module.a == 'value'
assert module.__file__ == modpath
assert module.__name__ == '_tmproot373.sub1.sub2.testmod'
assert '_tmproot373.sub1.sub2.testmod' in sys.modules
assert '_tmproot373.sub1.sub2' in sys.modules
assert '_tmproot373' in sys.modules
def model_definition(self):
info = self.unpack_info()
model_fpath = info['model_fpath']
module = ub.import_module_from_path(model_fpath)
export_version = getattr(module, '__pt_export_version__', '0')
export_version = list(map(int, export_version.split('.')))
if export_version >= [0, 2, 0]:
model_cls = module.get_model_cls()
initkw = module.get_initkw()
else:
# Hack to get information from older versions of pytorch_export
import inspect
from xdoctest import static_analysis
print('Hacking to grab model_cls and initkw')
model = module.make()
model_cls = model.__class__
source = inspect.getsource(module.make)
print(source)
initkw = static_analysis.parse_static_value('initkw',
source=source)
# Try to reconstruct initkw
model_ = (model_cls, initkw)
return model_
def main():
import xdoctest
import ubelt as ub
import sys
modpath = ub.modname_to_modpath('netharn.examples')
name_to_path = {}
submods = list(xdoctest.static_analysis.package_modpaths(modpath))
for submod in submods:
modname = ub.augpath(submod, dpath='', ext='')
if not modname.startswith('_'):
name_to_path[modname] = submod
print('name_to_path = {}'.format(ub.repr2(name_to_path, nl=1)))
chosen = None
for arg in sys.argv[1:2]:
print('arg = {!r}'.format(arg))
if arg in name_to_path:
chosen = name_to_path[arg]
break
print('chosen = {!r}'.format(chosen))
assert chosen is not None
module = ub.import_module_from_path(chosen)
print('module = {!r}'.format(module))
module.main()
def compare_loss():
harn = setup_harness(bsize=2)
harn.hyper.xpu = nh.XPU(0)
harn.initialize()
weights_fpath = ub.truepath(
'~/code/lightnet/examples/yolo-voc/backup/weights_30000.pt')
state_dict = harn.xpu.load(weights_fpath)['weights']
harn.model.module.load_state_dict(state_dict)
ln_test = ub.import_module_from_path(
ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
import lightnet as ln
net = ln.models.Yolo(ln_test.CLASSES, weights_fpath, ln_test.CONF_THRESH,
ln_test.NMS_THRESH)
net = harn.xpu.move(net)
import os
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, net)
ln_img, ln_label = ln_dset[0]
my_img, my_label = harn.datasets['test'][0]
my_targets = my_label[0][None, :]
ln_targets = [ln_label]
# Test model forward is the same for my image
ln_outputs = net._forward(harn.xpu.move(my_img[None, :]))
my_outputs = harn.model(harn.xpu.move(my_img[None, :]))
seen = net.loss.seen = 99999999
ln_loss = net.loss(ln_outputs, my_targets)
my_loss = harn.criterion(ln_outputs, my_targets, seen=seen)
print('my_loss = {!r}'.format(my_loss))
print('ln_loss = {!r}'.format(ln_loss))
ln_brambox_loss = net.loss(ln_outputs, ln_targets)
print('ln_brambox_loss = {!r}'.format(ln_brambox_loss))
inp_size = tuple(my_img.shape[-2:][::-1])
ln_tf_target = []
for anno in ln_targets[0]:
anno.class_label = anno.class_id
tf = ln.data.preprocess.BramboxToTensor._tf_anno(anno, inp_size, None)
ln_tf_target.append(tf)
ln_boxes = nh.util.Boxes(np.array(ln_tf_target)[:, 1:],
'cxywh').scale(inp_size)
my_boxes = nh.util.Boxes(np.array(my_targets[0])[:, 1:],
'cxywh').scale(inp_size)
nh.util.imshow(ln_img.numpy(), colorspace='rgb', fnum=1)
nh.util.draw_boxes(ln_boxes, color='blue')
nh.util.draw_boxes(my_boxes, color='red')
def _nh_data_nh_map(harn, num=10):
with torch.no_grad():
postprocess = harn.model.module.postprocess
# postprocess.conf_thresh = 0.001
# postprocess.nms_thresh = 0.5
batch_confusions = []
moving_ave = nh.util.util_averages.CumMovingAve()
loader = harn.loaders['test']
prog = ub.ProgIter(iter(loader), desc='')
for bx, batch in enumerate(prog):
inputs, labels = harn.prepare_batch(batch)
inp_size = np.array(inputs.shape[-2:][::-1])
outputs = harn.model(inputs)
loss = harn.criterion(outputs, labels['targets'],
gt_weights=labels['gt_weights'],
seen=1000000000)
moving_ave.update(ub.odict([
('loss', float(loss.sum())),
('coord', harn.criterion.loss_coord),
('conf', harn.criterion.loss_conf),
('cls', harn.criterion.loss_cls),
]))
average_losses = moving_ave.average()
desc = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(desc, refresh=False)
postout = postprocess(outputs)
for y in harn._measure_confusion(postout, labels, inp_size):
batch_confusions.append(y)
# batch_output.append((outputs.cpu().data.numpy().copy(), inp_size))
# batch_labels.append([x.cpu().data.numpy().copy() for x in labels])
if num is not None and bx >= num:
break
average_losses = moving_ave.average()
print('average_losses {}'.format(ub.repr2(average_losses)))
if False:
from netharn.util import mplutil
mplutil.qtensure() # xdoc: +SKIP
harn.visualize_prediction(batch, outputs, postout, thresh=.1)
y = pd.concat([pd.DataFrame(c) for c in batch_confusions])
precision, recall, ap = nh.metrics.detections._multiclass_ap(y)
ln_test = ub.import_module_from_path(ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
num_classes = len(ln_test.LABELS)
cls_labels = list(range(num_classes))
aps = nh.metrics.ave_precisions(y, cls_labels, use_07_metric=True)
aps = aps.rename(dict(zip(cls_labels, ln_test.LABELS)), axis=0)
# return ap
return ap, aps
def test_dynamic_init():
"""
python ~/code/mkinit/tests/test_with_dummy.py test_dynamic_init
"""
import mkinit
cache_dpath = ub.ensure_app_cache_dir('mkinit/tests')
paths = make_dummy_package(cache_dpath, 'dynamic_dummy_mod1')
module = ub.import_module_from_path(paths['root'])
text = mkinit.dynamic_mkinit.dynamic_init(module.__name__)
print(text)
for i in range(1, 15):
want = 'good_attr_{:02d}'.format(i)
assert want in text, 'missing {}'.format(want)
def test_import_modpath_basic():
assert 'testmod' not in sys.modules
with ub.TempDir() as temp:
modpath = join(temp.dpath, 'testmod.py')
text = ub.codeblock('''
a = 'value'
''')
ub.writeto(modpath, text)
assert temp.dpath not in sys.path
module = ub.import_module_from_path(modpath)
assert temp.dpath not in sys.path, 'pythonpath should remain clean'
assert module.a == 'value'
assert module.__file__ == modpath
assert module.__name__ == 'testmod'
assert 'testmod' in sys.modules
import ubelt as ub
except ImportError:
print('running this test script requires ubelt')
raise
# Statically check if ``line_profiler`` is installed outside of the repo.
# To do this, we make a copy of PYTHONPATH, remove the repodir, and use
# ubelt to check to see if ``line_profiler`` can be resolved to a path.
temp_path = list(map(abspath, sys.path))
if repo_dir in temp_path:
temp_path.remove(repo_dir)
modpath = ub.modname_to_modpath('line_profiler', sys_path=temp_path)
if modpath is not None:
# If it does, then import it. This should cause the installed version
# to be used on further imports even if the repo_dir is in the path.
print('Using installed version of line_profiler')
module = ub.import_module_from_path(modpath, index=0)
print('Installed module = {!r}'.format(module))
else:
print('No installed version of line_profiler found')
try:
print('Changing dirs to test_dir={!r}'.format(test_dir))
os.chdir(test_dir)
package_name = 'line_profiler'
pytest_args = [
'--cov-config', '../pyproject.toml',
'--cov-report', 'html',
'--cov-report', 'term',
'--cov-report', 'xml',
'--cov=' + package_name,
def ignore():
# inp_size = (96, 96)
inp_size = (416, 416)
W = H = inp_size[0] // 32
n_classes = 3
import ubelt as ub
for i in ub.ProgIter(range(1000)):
data1, anchors = demo_npdata(5,
W,
H,
inp_size=inp_size,
C=n_classes,
n=1000)
_tup1 = build_target_item(data1, inp_size, n_classes, anchors, epoch=1)
(_boxes1, _ious1, _classes1, _box_mask1, _iou_mask1,
_class_mask1) = _tup1
orig_darknet = ub.import_module_from_path(
ub.truepath('~/code/yolo2-pytorch/darknet.py'))
orig_darknet.cfg.anchors = anchors
orig_darknet.cfg.multi_scale_inp_size[0][:] = inp_size
orig_darknet.cfg.multi_scale_out_size[0][:] = [W, H]
orig_darknet.cfg.num_classes = n_classes
pbox, piou, gbox, gcls, gw = data1
data2 = (pbox, gbox, gcls, gw, piou)
_tup2 = orig_darknet._process_batch(data2, size_index=0)
(_boxes2, _ious2, _classes2, _box_mask2, _iou_mask2,
_class_mask2) = _tup2
if np.any(_box_mask1 != _box_mask2):
raise Exception
flags = ~np.isclose(_iou_mask1, _iou_mask2)
if np.any(flags):
print(np.where(flags))
print(_iou_mask1[flags])
print(_iou_mask2[flags])
raise Exception
flags = ~np.isclose(_class_mask1, _class_mask2)
if np.any(flags):
raise Exception
flags = ~np.isclose(_classes2, _classes1)
if np.any(flags):
raise Exception
flags = ~np.isclose(_ious1, _ious2)
if np.any(flags):
raise Exception
_ious1[flags]
_ious2[flags]
ba = _boxes1.reshape(-1, 4)
bb = _boxes2.reshape(-1, 4)
flags = ~np.isclose(ba, bb)
if np.any(flags):
print(ba[(~np.isclose(ba, bb)).sum(axis=-1) > 0])
print(bb[(~np.isclose(ba, bb)).sum(axis=-1) > 0])
raise Exception
def import_module_from_pyx(fname,
dpath=None,
error="raise",
autojit=True,
verbose=1,
recompile=False,
annotate=False):
"""
Attempts to import a module corresponding to a pyx file.
If the corresponding compiled module is not found, this can attempt to
JIT-cythonize the pyx file.
Parameters
----------
fname : str
The basename of the cython pyx file
dpath : str
The directory containing the cython pyx file
error : str
Can be "raise" or "ignore"
autojit : bool
If True, we will cythonize and compile the pyx file if possible.
verbose : int
verbosity level (higher is more verbose)
recompile : bool
if True force recompile
Returns
-------
ModuleType | None : module
Returns the compiled and imported module if possible, otherwise None
"""
if dpath is None:
dpath = dirname(fname)
pyx_fpath = join(dpath, fname)
if verbose > 3:
print('[xdev.import] pyx_fpath = {!r}'.format(pyx_fpath))
if not exists(pyx_fpath):
raise AssertionError("pyx file {!r} does not exist".format(pyx_fpath))
try:
# This functionality depends on ubelt
# TODO: the required functionality could be moved to nx.utils
import ubelt as ub
except Exception:
if verbose:
print("Autojit requires ubelt, which failed to import")
if error == "ignore":
module = None
elif error == "raise":
raise
else:
raise KeyError(error)
else:
if autojit:
if verbose > 3:
print('autojit = {!r}'.format(autojit))
# Try to JIT the cython module if we ship the pyx without the compiled
# library.
NUM_AUTOJIT_TRIES[pyx_fpath] += 1
if verbose > 3:
print('NUM_AUTOJIT_TRIES = {!r}'.format(NUM_AUTOJIT_TRIES))
print('MAX_AUTOJIT_TRIES = {!r}'.format(MAX_AUTOJIT_TRIES))
if recompile or NUM_AUTOJIT_TRIES[pyx_fpath] <= MAX_AUTOJIT_TRIES:
if verbose > 3:
print('try it')
try:
_autojit_cython(pyx_fpath,
verbose=verbose,
recompile=recompile,
annotate=annotate)
if verbose > 3:
print('got it')
except Exception as ex:
warnings.warn("Cython autojit failed: ex={!r}".format(ex))
if error == "raise":
raise
try:
# if recompile:
# # Doesnt work. The module is not reloaded.
# modname = ub.modpath_to_modname(pyx_fpath)
# import sys
# sys.modules.pop(modname, None)
module = ub.import_module_from_path(pyx_fpath)
except Exception:
if error == "ignore":
module = None
elif error == "raise":
raise
else:
raise KeyError(error)
return module
def evaluate_lightnet_model():
"""
Try to evaulate the model using the exact same VOC scoring metric
import ubelt as ub
import sys
sys.path.append(ub.truepath('~/code/netharn/netharn/examples/tests'))
from test_yolo import *
"""
import os # NOQA
import netharn as nh
import ubelt as ub
import lightnet as ln
import torch
ln_test = ub.import_module_from_path(
ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
xpu = nh.XPU.cast('auto')
ln_weights_fpath = ub.truepath(
'~/code/lightnet/examples/yolo-voc/backup/final.pt')
ln_weights_fpath = ub.truepath(
'~/code/lightnet/examples/yolo-voc/backup/weights_30000.pt')
from netharn.models.yolo2 import light_yolo
ln_weights_fpath = nh.models.yolo2.light_yolo.demo_voc_weights('darknet')
# Lightnet model, postprocess, and lightnet weights
ln_model = ln.models.Yolo(ln_test.CLASSES, ln_weights_fpath,
ln_test.CONF_THRESH, ln_test.NMS_THRESH)
ln_model = xpu.move(ln_model)
ln_model.eval()
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, ln_model)
ln_loader = torch.utils.data.DataLoader(
ln_dset,
batch_size=8,
shuffle=False,
drop_last=False,
num_workers=0,
pin_memory=True,
collate_fn=ln.data.list_collate,
)
# ----------------------
# Postprocessing to transform yolo outputs into detections
# Basic difference here is the implementation of NMS
ln_postprocess = ln.data.transform.util.Compose(
ln_model.postprocess.copy())
# ----------------------
# Define helper functions to deal with bramboxes
detection_to_brambox = ln.data.transform.TensorToBrambox(
ln_test.NETWORK_SIZE, ln_test.LABELS)
# hack so forward call behaves like it does in test
ln_model.postprocess.append(detection_to_brambox)
# ----------------------
def img_to_box(ln_loader, boxes, offset):
gname_lut = ln_loader.dataset.keys
return {gname_lut[offset + k]: v for k, v in enumerate(boxes)}
with torch.no_grad():
anno = {}
ln_det = {}
moving_ave = nh.util.util_averages.CumMovingAve()
prog = ub.ProgIter(ln_loader, desc='')
for bx, ln_raw_batch in enumerate(prog):
ln_raw_inputs, ln_bramboxes = ln_raw_batch
# Convert brambox into components understood by netharn
ln_inputs = xpu.variable(ln_raw_inputs)
ln_model.loss.seen = 1000000
ln_outputs = ln_model._forward(ln_inputs)
ln_loss_bram = ln_model.loss(ln_outputs, ln_bramboxes)
moving_ave.update(
ub.odict([
('loss_bram', float(ln_loss_bram.sum())),
]))
# Display progress information
average_losses = moving_ave.average()
description = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(description, refresh=False)
# nh_outputs and ln_outputs should be the same, so no need to
# differentiate between them here.
ln_postout = ln_postprocess(ln_outputs.clone())
ln_brambox_postout = detection_to_brambox(
[x.clone() for x in ln_postout])
# out, loss = ln_model.forward(ln_inputs, ln_bramboxes)
# Record data scored by brambox
offset = len(anno)
anno.update(img_to_box(ln_loader, ln_bramboxes, offset))
ln_det.update(img_to_box(ln_loader, ln_brambox_postout, offset))
import brambox.boxes as bbb
# Compute mAP using brambox / lightnet
pr = bbb.pr(ln_det, anno)
ln_mAP = round(bbb.ap(*pr) * 100, 2)
print('\nBrambox multiclass AP = {!r}'.format(ln_mAP))
devkit_dpath = ub.truepath('~/data/VOC/VOCdevkit/')
# Convert bramboxes to VOC style data for eval
class_to_dets = ub.ddict(list)
for gpath, bb_dets in ub.ProgIter(list(ln_det.items())):
from PIL import Image
from os.path import join
pil_image = Image.open(join(devkit_dpath, gpath) + '.jpg')
img_size = pil_image.size
bb_dets = ln.data.transform.ReverseLetterbox.apply(
[bb_dets], ln_test.NETWORK_SIZE, img_size)[0]
for det in bb_dets:
from os.path import basename, splitext
imgid = splitext(basename(gpath))[0]
score = det.confidence
tlwh = [
max(det.x_top_left, 0),
max(det.y_top_left, 0),
min(det.x_top_left + det.width, img_size[0]),
min(det.y_top_left + det.height, img_size[1])
]
# See: /home/joncrall/data/VOC/VOCdevkit/devkit_doc.pdf
# Each line is formatted as:
# <image identifier> <confidence> <left> <top> <right> <bottom>
class_to_dets[det.class_label].append(
list(ub.flatten([[imgid], [score], tlwh])))
# Calculate Original VOC measure
from os.path import join
results_path = join(devkit_dpath, 'results', 'VOC2007', 'Main')
detpath = join(results_path, 'comp3_det_test_{}.txt')
for lbl, dets in class_to_dets.items():
fpath = detpath.format(lbl)
with open(fpath, 'w') as file:
text = '\n'.join([' '.join(list(map(str, det))) for det in dets])
file.write(text)
import sys
sys.path.append('/home/joncrall/code/netharn/netharn/examples/tests')
from voc_eval_orig import voc_eval
use_07_metric = False
bias = 1.0
# for bias in [0.0, 1.0]:
# for use_07_metric in [True, False]:
class_aps = {}
class_curve = {}
annopath = join(devkit_dpath, 'VOC2007', 'Annotations', '{}.xml')
for classname in ub.ProgIter(list(class_to_dets.keys())):
cachedir = None
imagesetfile = join(devkit_dpath, 'VOC2007', 'ImageSets', 'Main',
'{}_test.txt').format(classname)
rec, prec, ap = voc_eval(detpath,
annopath,
imagesetfile,
classname,
cachedir,
ovthresh=0.5,
use_07_metric=use_07_metric,
bias=bias)
class_aps[classname] = ap
class_curve[classname] = (rec, prec)
mAP = np.mean(list(class_aps.values()))
print('Official* bias={} {} VOC mAP = {!r}'.format(
bias, '2007' if use_07_metric else '2012', mAP))
# I get 0.71091 without 07 metric
# I get 0.73164 without 07 metric
"""
Lightnet:
'aeroplane' : 0.7738,
'bicycle' : 0.8326,
'bird' : 0.7505,
'boat' : 0.6202,
'bottle' : 0.4614,
'bus' : 0.8078,
'car' : 0.8052,
'cat' : 0.8857,
'chair' : 0.5385,
'cow' : 0.7768,
'diningtable' : 0.7556,
'dog' : 0.8545,
'horse' : 0.8401,
'motorbike' : 0.8144,
'person' : 0.7608,
'pottedplant' : 0.4640,
'sheep' : 0.7398,
'sofa' : 0.7334,
'train' : 0.8697,
'tvmonitor' : 0.7533,
"""
"""
Official* bias=1.0 2012 VOC mAP = 0.7670408107201542
Darknet:
'aeroplane' : 0.7597,
'bicycle' : 0.8412,
'bird' : 0.7705,
'boat' : 0.6360,
'bottle' : 0.4902,
'bus' : 0.8164,
'car' : 0.8444,
'cat' : 0.8926,
'chair' : 0.5902,
'cow' : 0.8184,
'diningtable' : 0.7728,
'dog' : 0.8612,
'horse' : 0.8759,
'motorbike' : 0.8467,
'person' : 0.7855,
'pottedplant' : 0.5117,
'sheep' : 0.7889,
'sofa' : 0.7584,
'train' : 0.8997,
'tvmonitor' : 0.7806,
"""
"""
def _ln_loop(ln_model, xpu, harn):
"""
Uses ln data, but nh map computation
"""
import lightnet as ln
ln_test = ub.import_module_from_path(ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
harn.current_tag = tag = 'test'
# Keep track of NH metrics
dmet = harn.dmets[tag]
dmet.pred.remove_all_annotations()
dmet.true.remove_all_annotations()
dmet.true._build_index()
dmet.pred._build_index()
# Keep track of LN metrics
anno = {}
ln_det = {}
resize_anno = {}
resize_ln_det = {}
ln_keys_to_gid = {}
for gid, img in dmet.true.imgs.items():
key = os.path.splitext(img['file_name'].split('VOCdevkit/')[1])[0]
ln_keys_to_gid[key] = gid
def brambox_to_labels(ln_loader, ln_bramboxes, inp_size, LABELS, offset=None):
""" convert brambox to netharn style labels """
import lightnet as ln
max_anno = max(map(len, ln_bramboxes))
ln_targets = [
ln.data.transform.BramboxToTensor.apply(
annos, inp_size, max_anno=max_anno, class_label_map=LABELS)
for annos in ln_bramboxes]
ln_targets = torch.stack(ln_targets)
gt_weights = -np.ones((len(ln_bramboxes), max_anno), dtype=np.float32)
for i, annos in enumerate(ln_bramboxes):
weights = 1.0 - np.array([anno.ignore for anno in annos], dtype=np.float32)
gt_weights[i, 0:len(annos)] = weights
gt_weights = torch.Tensor(gt_weights)
bg_weights = torch.FloatTensor(np.ones(len(ln_targets)))
if offset is not None:
# Hack to find image size, assume ordered iteration, which
# might be true for the test set.
orig_sizes = []
indices = []
for k in range(len(ln_bramboxes)):
key = ln_loader.dataset.keys[offset + k]
gid = ln_keys_to_gid[key]
orig_sizes += [(dmet.true.imgs[gid]['width'], dmet.true.imgs[gid]['height'])]
indices += [gid]
indices = torch.FloatTensor(indices)
orig_sizes = torch.FloatTensor(orig_sizes)
else:
indices = None
orig_sizes = None
ln_labels = {
'targets': ln_targets,
'gt_weights': gt_weights,
'orig_sizes': orig_sizes,
'indices': indices,
'bg_weights': bg_weights,
}
return ln_labels
def img_to_box(ln_loader, boxes, offset):
gname_lut = ln_loader.dataset.keys
return {gname_lut[offset + k]: v for k, v in enumerate(boxes)}
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, ln_model)
ln_loader = torch.utils.data.DataLoader(
ln_dset, batch_size=8, shuffle=False, drop_last=False,
num_workers=4, pin_memory=True, collate_fn=ln.data.list_collate,
)
detection_to_brambox = ln.data.transform.TensorToBrambox(ln_test.NETWORK_SIZE, ln_test.LABELS)
ln.data.transform.ReverseLetterbox
# ----------------------
# Postprocessing to transform yolo outputs into detections
# Basic difference here is the implementation of NMS
ln_postprocess = ln_model.postprocess
# ----------------------
# ln_results = []
moving_ave = nh.util.util_averages.CumMovingAve()
prog = ub.ProgIter(ln_loader, desc='')
with torch.no_grad():
ln_loader.dataset.keys
for bx, ln_batch in enumerate(prog):
ln_inputs, ln_bramboxes = ln_batch
# Convert brambox into components understood by netharn
ln_inputs = xpu.variable(ln_inputs)
inp_size = tuple(ln_inputs.shape[-2:][::-1])
# hack image index (assume they are sequential)
offset = len(anno)
ln_labels = brambox_to_labels(ln_loader, ln_bramboxes,
inp_size, ln_test.LABELS,
offset=offset)
ln_model.loss.seen = 1000000
ln_outputs = ln_model._forward(ln_inputs)
ln_loss_bram = ln_model.loss(ln_outputs, ln_bramboxes)
moving_ave.update(ub.odict([
('loss_bram', float(ln_loss_bram.sum())),
]))
# Display progress information
average_losses = moving_ave.average()
description = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(description, refresh=False)
# nh_outputs and ln_outputs should be the same, so no need to
# differentiate between them here.
ln_postout = ln_postprocess(ln_outputs.clone())
# ln_results.append((ln_postout, ln_labels, inp_size))
# Track NH stats
pred_anns = list(harn._postout_to_pred_ann(
inp_size, ln_labels, ln_postout, undo_lb=False,
_aidbase=len(dmet.pred.dataset['annotations']) + 1))
dmet.pred.add_annotations(pred_anns)
true_anns = list(harn._labels_to_true_ann(
inp_size, ln_labels, undo_lb=False,
_aidbase=len(dmet.true.dataset['annotations']) + 1))
dmet.true.add_annotations(true_anns)
# Track LN stats
ln_brambox_postout = detection_to_brambox([x.clone() for x in ln_postout])
anno.update(img_to_box(ln_loader, ln_bramboxes, offset))
ln_det.update(img_to_box(ln_loader, ln_brambox_postout, offset))
# Also track bb-stats in original sizes
ln_resize_annos = []
for bb_anns, orig_size in zip(ln_bramboxes, ln_labels['orig_sizes']):
old_bb_anns = copy.deepcopy(bb_anns)
new_bb_anns = ln.data.transform.ReverseLetterbox.apply(
[old_bb_anns], ln_test.NETWORK_SIZE, orig_size)[0]
ln_resize_annos.append(new_bb_anns)
ln_resize_dets = []
for bb_dets, orig_size in zip(ln_brambox_postout, ln_labels['orig_sizes']):
old_bb_dets = copy.deepcopy(bb_dets)
new_bb_dets = ln.data.transform.ReverseLetterbox.apply(
[old_bb_dets], ln_test.NETWORK_SIZE, orig_size)[0]
ln_resize_dets.append(new_bb_dets)
resize_anno.update(img_to_box(ln_loader, ln_resize_annos, offset))
resize_ln_det.update(img_to_box(ln_loader, ln_resize_dets, offset))
print('lightnet voc_mAP = {}'.format(dmet.score_voc()['mAP']))
print('lightnet nh_mAP = {}'.format(dmet.score_netharn()['mAP']))
# Compute mAP using brambox / lightnet
import brambox.boxes as bbb
ln_mAP = round(bbb.ap(*bbb.pr(ln_det, anno)) * 100, 2)
print('ln_bb_mAP = {!r}'.format(ln_mAP))
ln_resize_mAP = round(bbb.ap(*bbb.pr(resize_ln_det, resize_anno)) * 100, 2)
print('ln_resize_bb_mAP = {!r}'.format(ln_resize_mAP))
if False:
# Check sizes
for bb in resize_anno[ln_loader.dataset.keys[0]]:
print(list(map(float, [bb.x_top_left, bb.y_top_left, bb.width, bb.height])))
print(dmet.true.annots(gid=0).boxes)
# Check sizes
for bb in resize_ln_det[ln_loader.dataset.keys[0]][0:3]:
print(list(map(float, [bb.x_top_left, bb.y_top_left, bb.width, bb.height])))
print(dmet.pred.annots(gid=0).boxes[0:3])
def check_inconsistency():
import netharn as nh
import numpy as np
import torch
import ubelt as ub
from netharn.models.yolo2 import light_yolo
from netharn.models.yolo2 import light_region_loss
yolo_voc = ub.import_module_from_path(ub.truepath('~/code/netharn/examples/yolo_voc.py'))
xpu = nh.XPU.cast('argv')
nice = ub.argval('--nice', default='Yolo2Baseline')
batch_size = 8
bstep = 8
workers = 0
decay = 0.0005
lr = 0.001
ovthresh = 0.5
simulated_bsize = bstep * batch_size
# We will divide the learning rate by the simulated batch size
datasets = {
# 'train': yolo_voc.YoloVOCDataset(years=[2007, 2012], split='trainval'),
'test': yolo_voc.YoloVOCDataset(years=[2007], split='test'),
}
loaders = {
key: dset.make_loader(batch_size=batch_size, num_workers=workers,
shuffle=(key == 'train'), pin_memory=True,
resize_rate=10 * bstep, drop_last=True)
for key, dset in datasets.items()
}
if workers > 0:
import cv2
cv2.setNumThreads(0)
assert simulated_bsize == 64, 'must be 64'
lr_step_points = {
0: 0, # Hack to see performance before any learning
1: 0,
2: lr * 1.0 / simulated_bsize,
3: lr * 1.0 / simulated_bsize,
}
max_epoch = 3
# Anchors
anchors = np.array([(1.3221, 1.73145), (3.19275, 4.00944),
(5.05587, 8.09892), (9.47112, 4.84053),
(11.2364, 10.0071)])
hyper = nh.HyperParams(**{
'nice': nice,
'workdir': ub.truepath('~/work/devcheck_yolo'),
'datasets': datasets,
'xpu': xpu,
# a single dict is applied to all datset loaders
'loaders': loaders,
'model': (light_yolo.Yolo, {
# 'num_classes': datasets['train'].num_classes,
'num_classes': 20,
'anchors': anchors,
# 'conf_thresh': 0.001,
'conf_thresh': 0.1, # make training a bit faster
# nms_thresh=0.5 to reproduce original yolo
# nms_thresh=0.4 to reproduce lightnet
'nms_thresh': 0.5 if not ub.argflag('--eav') else 0.4
}),
'criterion': (light_region_loss.RegionLoss, {
# 'num_classes': datasets['train'].num_classes,
'num_classes': 20,
'anchors': anchors,
'object_scale': 5.0,
'noobject_scale': 1.0,
'class_scale': 1.0,
'coord_scale': 1.0,
'thresh': 0.6, # iou_thresh
}),
'initializer': (nh.initializers.Pretrained, {
# 'fpath': light_yolo.initial_imagenet_weights(),
'fpath': light_yolo.demo_voc_weights(),
}),
'optimizer': (torch.optim.SGD, {
'lr': lr_step_points[0],
'momentum': 0.9,
'dampening': 0,
# multiplying by batch size was one of those unpublished details
'weight_decay': decay * simulated_bsize,
}),
'scheduler': (nh.schedulers.core.YOLOScheduler, {
'points': lr_step_points,
'interpolate': True,
'burn_in': 1,
# 'dset_size': len(datasets['train']), # when drop_last=False
'dset_size': len(datasets['test']), # when drop_last=False
'batch_size': batch_size,
}),
'monitor': (nh.Monitor, {
'minimize': ['loss'],
'maximize': ['mAP'],
'patience': max_epoch,
'max_epoch': max_epoch,
}),
# 'augment': datasets['train'].augmenter,
'dynamics': {'batch_step': bstep},
'other': {
'nice': nice,
'ovthresh': ovthresh,
},
})
print('max_epoch = {!r}'.format(max_epoch))
harn = yolo_voc.YoloHarn(hyper=hyper)
harn.config['use_tqdm'] = False
harn.intervals['log_iter_train'] = None
harn.intervals['log_iter_test'] = None
harn.intervals['log_iter_vali'] = None
harn.initialize()
harn.run()
def _test_with_lnstyle_data():
"""
Uses pretrained lightnet weights, and the lightnet data loader.
Uses my critrion and net implementations.
(already verified to produce the same outputs)
Checks to see if my loss and map calcluations are the same as lightnet
CommandLine:
python ~/code/netharn/netharn/examples/yolo_voc.py _test_with_lnstyle_data
Using LighNet Trained Weights:
LightNet Results:
TEST 30000 mAP:74.18% Loss:3.16839 (Coord:0.38 Conf:1.61 Cls:1.17)
My Results:
# Worse losses (due to different image loading)
loss: 5.00 {coord: 0.69, conf: 2.05, cls: 2.26}
mAP = 0.6227
The MAP is quite a bit worse... Why is that?
USING THE SAME WEIGHTS! I must be doing something wrong.
Results using the same Data Loader:
{cls: 2.22, conf: 2.05, coord: 0.65, loss: 4.91}
# Checking with extra info
{loss_bram: 3.17, loss_ten1: 4.91, loss_ten2: 4.91}
OH!, Is is just that BramBox has an ignore function?
- [X] Add ignore / weight to tensor version to see if its the same
YUP! {loss_bram: 3.17, loss_ten1: 4.91, loss_ten2: 4.91, nh_unweighted: 4.92, nh_weighted: 3.16}
TO CHECK:
- [ ] why is the loss different?
- [X] network input size is 416 in both
- [x] networks output the same data given the same input
- [x] loss outputs the same data given the same input (they do if seen is the same)
- [x] CONCLUSION: The loss is not actually different
- [ ] why is the mAP different?
- [x] does brambox compute AP differently?
... YES
CONCLUSION: several factors are at play
* brambox has a different AP computation
* netharn and lightnet non-max supressions are different
* The NMS seems to have the most significant impact
# """
import brambox.boxes as bbb
CHECK_SANITY = False
# Path to weights that we will be testing
# (These were trained using the lightnet harness and achived a good mAP)
weights_fpath = ub.truepath('~/code/lightnet/examples/yolo-voc/backup/weights_30000.pt')
# Load weights into a netharn model
harn = setup_harness(bsize=2)
harn.hyper.xpu = nh.XPU(0)
harn.initialize()
state_dict = harn.xpu.load(weights_fpath)['weights']
harn.model.module.load_state_dict(state_dict)
harn.model.eval()
nh_net = harn.model.module
# Load weights into a lightnet model
import os
import lightnet as ln
ln_test = ub.import_module_from_path(ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
ln_net = ln.models.Yolo(ln_test.CLASSES, weights_fpath,
ln_test.CONF_THRESH, ln_test.NMS_THRESH)
ln_net = harn.xpu.move(ln_net)
ln_net.eval()
# Sanity check, the weights should be the same
if CHECK_SANITY:
state1 = nh_net.state_dict()
state2 = ln_net.state_dict()
assert state1.keys() == state2.keys()
for k in state1.keys():
v1 = state1[k]
v2 = state2[k]
assert np.all(v1 == v2)
# Create a lightnet dataset loader
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, ln_net)
ln_loader = torch.utils.data.DataLoader(
ln_dset, batch_size=2, shuffle=False, drop_last=False, num_workers=0,
pin_memory=True, collate_fn=ln.data.list_collate,
)
# Create a netharn dataset loader
nh_loader = harn.loaders['test']
# ----------------------
# Postprocessing to transform yolo outputs into detections
# Basic difference here is the implementation of NMS
ln_postprocess = ln_net.postprocess
nh_postprocess = harn.model.module.postprocess
# ----------------------
# Define helper functions to deal with bramboxes
LABELS = ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car',
'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse',
'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train',
'tvmonitor']
NETWORK_SIZE = (416, 416)
detection_to_brambox = ln.data.TensorToBrambox(NETWORK_SIZE, LABELS)
# ----------------------
CHECK_LOSS = False
with torch.no_grad():
# Track netharn and lightnet results that will be scored
ln_batch_confusions = []
nh_batch_confusions = []
nh_batch_confusions0 = []
ln_results = []
nh_results = []
nh_results0 = []
anno = {}
ln_det = {}
nh_det = {}
nh_det0 = {}
moving_ave = nh.util.util_averages.CumMovingAve()
coco_truth = []
# ln_coco_detections = []
nh_coco_detections0 = []
prog = ub.ProgIter(zip(ln_loader, nh_loader), desc='')
for bx, (ln_batch, nh_batch) in enumerate(prog):
ln_inputs, ln_bramboxes = ln_batch
inp_size = tuple(ln_inputs.shape[-2:][::-1])
nh_inputs, nh_labels = nh_batch
nh_targets = nh_labels['targets']
nh_gt_weights = nh_labels['gt_weights']
# Convert brambox into components understood by netharn
ln_labels = brambox_to_labels(ln_bramboxes, inp_size, ln_test.LABELS)
ln_inputs = harn.xpu.variable(ln_inputs)
ln_targets = harn.xpu.variable(ln_labels['targets'])
ln_gt_weights = harn.xpu.variable(ln_labels['gt_weights']) # NOQA
ln_net.loss.seen = 1000000
ln_outputs = ln_net._forward(ln_inputs)
if CHECK_SANITY:
nh_outputs = harn.model(ln_inputs)
assert np.all(nh_outputs == ln_outputs)
ln_loss_bram = ln_net.loss(ln_outputs, ln_bramboxes)
moving_ave.update(ub.odict([
('loss_bram', float(ln_loss_bram.sum())),
]))
if CHECK_LOSS:
seen = ln_net.loss.seen
ln_loss_ten1 = harn.criterion(ln_outputs, ln_targets,
seen=seen)
ln_loss_ten2 = ln_net.loss(ln_outputs, ln_targets)
nh_weighted = harn.criterion(ln_outputs, nh_targets,
gt_weights=nh_gt_weights,
seen=seen)
nh_unweighted = harn.criterion(ln_outputs, nh_targets,
seen=seen)
moving_ave.update(ub.odict([
('loss_ten1', float(ln_loss_ten1.sum())),
('loss_ten2', float(ln_loss_ten2.sum())),
('nh_weighted', float(nh_weighted.sum())),
('nh_unweighted', float(nh_unweighted.sum())),
# ('coord', harn.criterion.loss_coord),
# ('conf', harn.criterion.loss_conf),
# ('cls', harn.criterion.loss_cls),
]))
# Display progress information
average_losses = moving_ave.average()
description = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(description, refresh=False)
# nh_outputs and ln_outputs should be the same, so no need to
# differentiate between them here.
ln_postout = ln_postprocess(ln_outputs.clone())
nh_postout = nh_postprocess(ln_outputs.clone())
# Should use the original NMS strategy
nh_postout0 = nh_postprocess(ln_outputs.clone(), mode=0)
ln_brambox_postout = detection_to_brambox([x.clone() for x in ln_postout])
nh_brambox_postout = detection_to_brambox([x.clone() for x in nh_postout])
nh_brambox_postout0 = detection_to_brambox([x.clone() for x in nh_postout0])
# Record data scored by brambox
offset = len(anno)
def img_to_box(boxes, offset):
gname_lut = ln_loader.dataset.keys
return {gname_lut[offset + k]: v for k, v in enumerate(boxes)}
anno.update(img_to_box(ln_bramboxes, offset))
ln_det.update(img_to_box(ln_brambox_postout, offset))
nh_det.update(img_to_box(nh_brambox_postout, offset))
nh_det0.update(img_to_box(nh_brambox_postout0, offset))
# Record data scored by netharn
ln_results.append((ln_postout, ln_labels, inp_size))
nh_results.append((nh_postout, nh_labels, inp_size))
nh_results0.append((nh_postout0, nh_labels, inp_size))
# preds, truths = harn._postout_to_coco(ln_postout, ln_labels, inp_size)
# ln_coco_detections.append(preds)
preds, truths = harn._postout_to_coco(nh_postout0, nh_labels, inp_size)
nh_coco_detections0.append(preds)
coco_truth.append(truths)
# kw = dict(bias=0)
# for y in harn._measure_confusion(ln_postout, ln_labels, inp_size, **kw):
# ln_batch_confusions.append(y)
# for y in harn._measure_confusion(nh_postout, nh_labels, inp_size, **kw):
# nh_batch_confusions.append(y)
# for y in harn._measure_confusion(nh_postout0, nh_labels, inp_size, **kw):
# nh_batch_confusions0.append(y)
if bx > 50:
break
# Compute mAP using brambox / lightnet
ln_mAP = round(bbb.ap(*bbb.pr(ln_det, anno)) * 100, 2)
nh_mAP = round(bbb.ap(*bbb.pr(nh_det, anno)) * 100, 2)
nh_mAP0 = round(bbb.ap(*bbb.pr(nh_det0, anno)) * 100, 2)
print('\n----')
print('ln_mAP = {!r}'.format(ln_mAP))
print('nh_mAP = {!r}'.format(nh_mAP))
print('nh_mAP0 = {!r}'.format(nh_mAP0))
# num_classes = len(LABELS)
# cls_labels = list(range(num_classes))
# # Compute mAP using netharn
# if False:
# is_tp = (y.true == y.pred) & (y.pred >= 0)
# is_fp = (y.true != y.pred) & (y.pred >= 0)
# rest = ~(is_fp | is_tp)
# y.true[is_tp] = 1
# y.pred[is_tp] = 1
# y.true[is_fp] = 0
# y.pred[is_fp] = 1
# y.true[rest] = 0
# y.pred[rest] = 0
# import sklearn
# import sklearn.metrics
# sklearn.metrics.average_precision_score(y.true, y.score, 'weighted',
# y.weight)
for bias in [0, 1]:
ln_batch_confusions = []
nh_batch_confusions = []
nh_batch_confusions0 = []
print('\n\n======\n\nbias = {!r}'.format(bias))
kw = dict(bias=bias, PREFER_WEIGHTED_TRUTH=False)
for ln_postout, ln_labels, inp_size in ln_results:
for y in harn._measure_confusion(ln_postout, ln_labels, inp_size, **kw):
ln_batch_confusions.append(y)
for nh_postout, nh_labels, inp_size in nh_results:
for y in harn._measure_confusion(nh_postout, nh_labels, inp_size, **kw):
nh_batch_confusions.append(y)
for nh_postout0, nh_labels, inp_size in nh_results0:
for y in harn._measure_confusion(nh_postout0, nh_labels, inp_size, **kw):
nh_batch_confusions0.append(y)
confusions = {
'lh': ln_batch_confusions,
# 'nh': nh_batch_confusions,
'nh0': nh_batch_confusions0,
}
for lbl, batch_confusions in confusions.items():
print('----')
print('\nlbl = {!r}'.format(lbl))
y = pd.concat([pd.DataFrame(c) for c in batch_confusions])
# aps = nh.metrics.ave_precisions(y, cls_labels, use_07_metric=True)
# aps = aps.rename(dict(zip(cls_labels, LABELS)), axis=0)
# mean_ap = np.nanmean(aps['ap'])
# print('mean_ap_07 = {:.2f}'.format(mean_ap * 100))
# aps = nh.metrics.ave_precisions(y, cls_labels, use_07_metric=False)
# aps = aps.rename(dict(zip(cls_labels, LABELS)), axis=0)
# mean_ap = np.nanmean(aps['ap'])
# print('mean_ap_12 = {:.2f}'.format(mean_ap * 100))
# Try the other way
from netharn.metrics.detections import _multiclass_ap
prec, recall, ap2 = _multiclass_ap(y)
print('ap2 = {!r}'.format(round(ap2 * 100, 2)))
def _test_with_lnstyle_data():
"""
Uses pretrained lightnet weights, and the lightnet data loader.
Uses my critrion and net implementations.
(already verified to produce the same outputs)
Checks to see if my loss and map calcluations are the same as lightnet
CommandLine:
python ~/code/netharn/netharn/examples/yolo_voc.py _test_with_lnstyle_data
"""
"""
Using LighNet Trained Weights:
LightNet Results:
TEST 30000 mAP:74.18% Loss:3.16839 (Coord:0.38 Conf:1.61 Cls:1.17)
My Results:
# Worse losses (due to different image loading)
loss: 5.00 {coord: 0.69, conf: 2.05, cls: 2.26}
mAP = 0.6227
The MAP is quite a bit worse... Why is that?
USING THE SAME WEIGHTS! I must be doing something wrong.
Results using the same Data Loader:
{cls: 2.22, conf: 2.05, coord: 0.65, loss: 4.91}
# Checking with extra info
{loss_bram: 3.17, loss_ten1: 4.91, loss_ten2: 4.91}
OH!, Is is just that BramBox has an ignore function?
- [X] Add ignore / weight to tensor version to see if its the same
YUP! {loss_bram: 3.17, loss_ten1: 4.91, loss_ten2: 4.91, my_unweighted: 4.92, my_weighted: 3.16}
TO CHECK:
- [ ] why is the loss different?
- [X] network input size is 416 in both
- [x] networks output the same data given the same input
- [x] loss outputs the same data given the same input (they do if seen is the same)
- [ ] is the data read and formated properly / letterbox done correctly?
- [ ] does the brambox version of loss work differently?
- [ ] check that we each format the first item in the test set the same
- [ ] why is the mAP different?
- [ ] does brambox compute AP differently?
"""
harn = setup_harness(bsize=2)
harn.hyper.xpu = nh.XPU(0)
harn.initialize()
weights_fpath = ub.truepath(
'~/code/lightnet/examples/yolo-voc/backup/weights_30000.pt')
state_dict = harn.xpu.load(weights_fpath)['weights']
harn.model.module.load_state_dict(state_dict)
ln_test = ub.import_module_from_path(
ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
import lightnet as ln
net = ln.models.Yolo(ln_test.CLASSES, weights_fpath, ln_test.CONF_THRESH,
ln_test.NMS_THRESH)
net = harn.xpu.move(net)
import os
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, net)
ln_loader = torch.utils.data.DataLoader(
ln_dset,
batch_size=2,
shuffle=False,
drop_last=False,
num_workers=0,
pin_memory=True,
collate_fn=ln.data.list_collate,
)
my_loader = harn.loaders['test']
harn.model.eval()
# class_label_map = ub.invert_dict(dict(enumerate(harn.datasets['train'].label_names)))
class_label_map = harn.datasets['train'].label_names
def brambox_to_labels(ln_bramboxes, inp_size):
max_anno = max(map(len, ln_bramboxes))
ln_targets = [
ln.data.preprocess.BramboxToTensor.apply(
annos,
inp_size,
max_anno=max_anno,
class_label_map=class_label_map) for annos in ln_bramboxes
]
ln_targets = torch.stack(ln_targets)
gt_weights = -np.ones((len(ln_bramboxes), max_anno), dtype=np.float32)
for i, annos in enumerate(ln_bramboxes):
weights = 1.0 - np.array([anno.ignore for anno in annos],
dtype=np.float32)
gt_weights[i, 0:len(annos)] = weights
gt_weights = torch.Tensor(gt_weights)
ln_labels = ln_targets, gt_weights, orig_sizes, indices, bg_weights
return ln_labels
with torch.no_grad():
postprocess = harn.model.module.postprocess
# postprocess.conf_thresh = 0.001
# postprocess.nms_thresh = 0.5
batch_confusions = []
moving_ave = nh.util.util_averages.CumMovingAve()
prog = ub.ProgIter(zip(ln_loader, my_loader), desc='')
for ln_batch, my_batch in prog:
ln_inputs, ln_bramboxes = ln_batch
inp_size = tuple(ln_inputs.shape[-2:][::-1])
# my_inputs, my_labels = my_batch
ln_labels = brambox_to_labels(ln_bramboxes, inp_size)
# my_targets = my_labels[0]
# gt_weights = my_labels[1]
ln_inputs = harn.xpu.variable(ln_inputs)
ln_targets = harn.xpu.variable(ln_labels[0])
gt_weights = harn.xpu.variable(ln_labels[1])
net.loss.seen = 1000000
ln_outputs = harn.model(ln_inputs)
# ln_loss_ten1 = harn.criterion(ln_outputs, ln_targets, seen=net.loss.seen)
# ln_loss_ten2 = net.loss(ln_outputs, ln_targets)
ln_loss_bram = net.loss(ln_outputs, ln_bramboxes)
# my_loss_weighted = harn.criterion(ln_outputs, my_targets, gt_weights=gt_weights, seen=seen)
# my_loss_unweighted = harn.criterion(ln_outputs, my_targets, seen=seen)
moving_ave.update(
ub.odict([
# ('loss_ten1', float(ln_loss_ten1.sum())),
# ('loss_ten2', float(ln_loss_ten2.sum())),
# ('my_weighted', float(my_loss_weighted.sum())),
# ('my_unweighted', float(my_loss_unweighted.sum())),
('loss_bram', float(ln_loss_bram.sum())),
# ('coord', harn.criterion.loss_coord),
# ('conf', harn.criterion.loss_conf),
# ('cls', harn.criterion.loss_cls),
]))
average_losses = moving_ave.average()
desc = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(desc, refresh=False)
# TODO: check that my postprocess of ln outputs gives good map
ln_postout = postprocess(ln_outputs)
bg_weights = torch.FloatTensor(np.ones(len(ln_targets)))
indices = None
orig_sizes = None
ln_labels = ln_targets, gt_weights, orig_sizes, indices, bg_weights
batch_confusions.extend(
list(harn._measure_confusion(ln_postout, ln_labels, inp_size)))
y = pd.concat([pd.DataFrame(c) for c in batch_confusions])
# TODO: write out a few visualizations
loader = harn.loaders['test']
num_classes = len(loader.dataset.label_names)
cls_labels = list(range(num_classes))
aps = nh.metrics.ave_precisions(y, cls_labels, use_07_metric=True)
aps = aps.rename(dict(zip(cls_labels, loader.dataset.label_names)), axis=0)
mean_ap = np.nanmean(aps['ap'])
max_ap = np.nanmax(aps['ap'])
print(aps)
print('mean_ap = {!r}'.format(mean_ap))
print('max_ap = {!r}'.format(max_ap))
def _ln_data_nh_map(ln_model, xpu, harn, num=None):
"""
Uses ln data, but nh map computation
"""
import os
import lightnet as ln
ln_test = ub.import_module_from_path(ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, ln_model)
ln_loader = torch.utils.data.DataLoader(
ln_dset, batch_size=2, shuffle=False, drop_last=False, num_workers=0,
pin_memory=True, collate_fn=ln.data.list_collate,
)
# ----------------------
# Postprocessing to transform yolo outputs into detections
# Basic difference here is the implementation of NMS
ln_postprocess = ln_model.postprocess
# ----------------------
with torch.no_grad():
ln_results = []
moving_ave = nh.util.util_averages.CumMovingAve()
prog = ub.ProgIter(ln_loader, desc='')
for bx, ln_batch in enumerate(prog):
ln_inputs, ln_bramboxes = ln_batch
# Convert brambox into components understood by netharn
ln_inputs = xpu.variable(ln_inputs)
inp_size = tuple(ln_inputs.shape[-2:][::-1])
ln_labels = brambox_to_labels(ln_bramboxes, inp_size, ln_test.LABELS)
ln_model.loss.seen = 1000000
ln_outputs = ln_model._forward(ln_inputs)
ln_loss_bram = ln_model.loss(ln_outputs, ln_bramboxes)
moving_ave.update(ub.odict([
('loss_bram', float(ln_loss_bram.sum())),
]))
# Display progress information
average_losses = moving_ave.average()
description = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(description, refresh=False)
# nh_outputs and ln_outputs should be the same, so no need to
# differentiate between them here.
ln_postout = ln_postprocess(ln_outputs.clone())
ln_results.append((ln_postout, ln_labels, inp_size))
if num is not None and bx >= num:
break
batch_confusions = []
kw = dict(bias=0, PREFER_WEIGHTED_TRUTH=False)
for ln_postout, ln_labels, inp_size in ln_results:
for y in harn._measure_confusion(ln_postout, ln_labels, inp_size, **kw):
batch_confusions.append(y)
y = pd.concat([pd.DataFrame(c) for c in batch_confusions])
num_classes = len(ln_test.LABELS)
cls_labels = list(range(num_classes))
precision, recall, mean_ap = nh.metrics.detections._multiclass_ap(y)
aps = nh.metrics.ave_precisions(y, cls_labels, use_07_metric=True)
aps = aps.rename(dict(zip(cls_labels, ln_test.LABELS)), axis=0)
# mean_ap = np.nanmean(aps['ap'])
return mean_ap
def _ln_data_ln_map(ln_model, xpu, num=None):
"""
Compute the results on the ln test set using a ln model.
Weights can either be lightnet or netharn
"""
import os
import lightnet as ln
ln_test = ub.import_module_from_path(ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
TESTFILE = ub.truepath('~/code/lightnet/examples/yolo-voc/data/test.pkl')
os.chdir(ub.truepath('~/code/lightnet/examples/yolo-voc/'))
ln_dset = ln_test.CustomDataset(TESTFILE, ln_model)
ln_loader = torch.utils.data.DataLoader(
ln_dset, batch_size=2, shuffle=False, drop_last=False, num_workers=0,
pin_memory=True, collate_fn=ln.data.list_collate,
)
# ----------------------
# Postprocessing to transform yolo outputs into detections
# Basic difference here is the implementation of NMS
ln_postprocess = ln_model.postprocess
# ----------------------
# Define helper functions to deal with bramboxes
detection_to_brambox = ln.data.transform.TensorToBrambox(ln_test.NETWORK_SIZE,
ln_test.LABELS)
# ----------------------
def img_to_box(boxes, offset):
gname_lut = ln_loader.dataset.keys
return {gname_lut[offset + k]: v for k, v in enumerate(boxes)}
with torch.no_grad():
anno = {}
ln_det = {}
moving_ave = nh.util.util_averages.CumMovingAve()
prog = ub.ProgIter(ln_loader, desc='')
for bx, ln_batch in enumerate(prog):
ln_inputs, ln_bramboxes = ln_batch
# Convert brambox into components understood by netharn
ln_inputs = xpu.variable(ln_inputs)
ln_model.loss.seen = 1000000
ln_outputs = ln_model._forward(ln_inputs)
ln_loss_bram = ln_model.loss(ln_outputs, ln_bramboxes)
moving_ave.update(ub.odict([
('loss_bram', float(ln_loss_bram.sum())),
]))
# Display progress information
average_losses = moving_ave.average()
description = ub.repr2(average_losses, nl=0, precision=2, si=True)
prog.set_description(description, refresh=False)
# nh_outputs and ln_outputs should be the same, so no need to
# differentiate between them here.
ln_postout = ln_postprocess(ln_outputs.clone())
ln_brambox_postout = detection_to_brambox([x.clone() for x in ln_postout])
# Record data scored by brambox
offset = len(anno)
anno.update(img_to_box(ln_bramboxes, offset))
ln_det.update(img_to_box(ln_brambox_postout, offset))
if num is not None and bx >= num:
break
import brambox.boxes as bbb
# Compute mAP using brambox / lightnet
ln_mAP = round(bbb.ap(*bbb.pr(ln_det, anno)) * 100, 2)
print('\nln_mAP = {!r}'.format(ln_mAP))
return ln_mAP
def _compare_map():
"""
Pascal 2007 + 2012 trainval has 16551 images
Pascal 2007 test has 4952 images
In Lightnet:
One batch is 64 images, so one epoch is 16551 / 64 = 259 iterations.
The LR says step at iteration 250, so thats just about one batch. No
special handling needed.
Most recent training run gave:
2018-06-03 00:57:31,830 : log_value(test epoch L_bbox, 0.4200094618143574, 160
2018-06-03 00:57:31,830 : log_value(test epoch L_iou, 1.6416475874762382, 160
2018-06-03 00:57:31,830 : log_value(test epoch L_cls, 1.3163336199137472, 160
LightNet Results:
TEST 30000 mAP:74.18% Loss:3.16839 (Coord:0.38 Conf:1.61 Cls:1.17)
My Results:
# Worse losses (due to different image loading)
loss: 5.00 {coord: 0.69, conf: 2.05, cls: 2.26}
mAP = 0.6227
The MAP is quite a bit worse... Why is that?
"""
import netharn as nh
import ubelt as ub
import sys
from os.path import exists # NOQA
sys.path.append(ub.truepath('~/code/netharn/netharn/examples')) # NOQA
from yolo_voc import setup_harness, light_yolo # NOQA
import shutil
import lightnet as ln
ln_test = ub.import_module_from_path(ub.truepath('~/code/lightnet/examples/yolo-voc/test.py'))
my_weights_fpath = ub.truepath('~/remote/namek/work/voc_yolo2/fit/nice/dynamic/torch_snapshots/_epoch_00000080.pt')
my_weights_fpath = ub.truepath('~/remote/namek/work/voc_yolo2/fit/nice/dynamic/torch_snapshots/_epoch_00000160.pt')
my_weights_fpath = ub.truepath('~/remote/namek/work/voc_yolo2/fit/nice/dynamic/torch_snapshots/_epoch_00000040.pt')
ln_weights_fpath = ub.truepath('~/remote/namek/code/lightnet/examples/yolo-voc/backup/weights_30000.pt')
ln_weights_fpath = ub.truepath('~/remote/namek/code/lightnet/examples/yolo-voc/backup/weights_45000.pt')
ln_weights_fpath = ub.truepath('~/remote/namek/code/lightnet/examples/yolo-voc/backup/final.pt')
assert exists(my_weights_fpath)
assert exists(ln_weights_fpath)
ln_weights_fpath_ = ub.truepath('~/tmp/ln_weights.pt')
my_weights_fpath_ = ub.truepath('~/tmp/my_weights.pt')
# Move the weights to the local computer
stamp = nh.util.CacheStamp(
'ln_weights.stamp', product=ln_weights_fpath_, cfgstr='ln',
dpath=ub.truepath('~/tmp'))
if stamp.expired():
shutil.copy2(ln_weights_fpath, ln_weights_fpath_)
stamp.renew()
stamp = nh.util.CacheStamp(
'nh_weights.stamp', product=my_weights_fpath_, cfgstr='nh',
dpath=ub.truepath('~/tmp'))
if stamp.expired():
shutil.copy2(my_weights_fpath, my_weights_fpath_)
stamp.renew()
########
# Create instances of netharn and lightnet YOLOv2 model
########
# Netharn model, postprocess, and lightnet weights
nh_harn = setup_harness(bsize=2)
xpu = nh_harn.hyper.xpu = nh.XPU.cast('auto')
nh_harn.initialize()
my_model = nh_harn.model
my_model.load_state_dict(nh_harn.xpu.load(my_weights_fpath)['model_state_dict'])
# Netharn model, postprocess, and lightnet weights
ln_harn = setup_harness(bsize=2)
ln_harn.initialize()
ln_weights = {'module.' + k: v for k, v in ln_harn.xpu.load(ln_weights_fpath)['weights'].items()}
ln_harn.model.load_state_dict(ln_weights)
# Lightnet model, postprocess, and lightnet weights
ln_model_with_ln_weights = ln.models.Yolo(ln_test.CLASSES,
ln_weights_fpath,
ln_test.CONF_THRESH,
ln_test.NMS_THRESH)
ln_model_with_ln_weights = xpu.move(ln_model_with_ln_weights)
# Lightnet model, postprocess, and netharn weights
import copy
ln_model_with_nh_weights = copy.deepcopy(ln_model_with_ln_weights)
nh_weights = nh_harn.xpu.load(my_weights_fpath)['model_state_dict']
nh_weights = {k.replace('module.', ''): v for k, v in nh_weights.items()}
ln_model_with_nh_weights.load_state_dict(nh_weights)
num = None
num = 50
# Compute brambox-style mAP on ln_model with LN and NH weights
ln_mAP1 = _ln_data_ln_map(ln_model_with_ln_weights, xpu, num=num)
nh_mAP1 = _ln_data_ln_map(ln_model_with_nh_weights, xpu, num=num)
# Compute netharn-style mAP on ln_model with LN and NH weights
ln_mAP2 = _ln_data_nh_map(ln_model_with_ln_weights, xpu, nh_harn, num=num)
nh_mAP2 = _ln_data_nh_map(ln_model_with_nh_weights, xpu, nh_harn, num=num)
print('\n')
print('ln_mAP1 on ln_model with LN_weights = {!r}'.format(ln_mAP1))
print('nh_mAP1 on ln_model with NH_weights = {!r}'.format(nh_mAP1))
print('nh_mAP2 on ln_model with LN_weights = {:.4f}'.format(ln_mAP2))
print('nh_mAP2 on ln_model with NH_weights = {:.4f}'.format(nh_mAP2))
nh_mAP3, nh_aps = _nh_data_nh_map(nh_harn, num=num)
ln_mAP3, ln_aps = _nh_data_nh_map(ln_harn, num=num)
print('\n')
print('nh_mAP3 on nh_model with LN_weights = {:.4f}'.format(nh_mAP3))
print('ln_mAP3 on nh_model with NH_weights = {:.4f}'.format(ln_mAP3))
# Method data mAP aero bike bird boat bottle bus car cat chair cow table dog horse mbike person plant sheep sofa train tv
# YOLOv2 544 07++12 73.4 86.3 82.0 74.8 59.2 51.8 79.8 76.5 90.6 52.1 78.2 58.5 89.3 82.5 83.4 81.3 49.1 77.2 62.4 83.8 68.7
"""
def 导入包_从路径(路径):
return ub.import_module_from_path(路径)
def import_module_from_pyx(fname,
dpath,
error="raise",
autojit=True,
verbose=1):
"""
Attempts to import a module corresponding to a pyx file.
If the corresponding compiled module is not found, this can attempt to
JIT-cythonize the pyx file.
Parameters
----------
fname : str
The basename of the cython pyx file
dpath : str
The directory containing the cython pyx file
error : str
Can be "raise" or "ignore"
autojit : bool
If True, we will cythonize and compile the pyx file if possible.
verbose : int
verbosity level (higher is more verbose)
Returns
-------
ModuleType | None : module
Returns the compiled and imported module if possible, otherwise None
"""
pyx_fpath = join(dpath, fname)
if not exists(pyx_fpath):
raise AssertionError("pyx file {!r} does not exist".format(pyx_fpath))
try:
# This functionality depends on ubelt
# TODO: the required functionality could be moved to nx.utils
import ubelt as ub
except Exception:
if verbose:
print("Autojit requires ubelt, which failed to import")
if error == "ignore":
module = None
elif error == "raise":
raise
else:
raise KeyError(error)
else:
if autojit:
# Try to JIT the cython module if we ship the pyx without the compiled
# library.
NUM_AUTOJIT_TRIES[pyx_fpath] += 1
if NUM_AUTOJIT_TRIES[pyx_fpath] <= MAX_AUTOJIT_TRIES:
try:
_autojit_cython(pyx_fpath, verbose=verbose)
except Exception as ex:
warnings.warn("Cython autojit failed: ex={!r}".format(ex))
if error == "raise":
raise
try:
module = ub.import_module_from_path(pyx_fpath)
except Exception:
if error == "ignore":
module = None
elif error == "raise":
raise
else:
raise KeyError(error)
return module
