def __call__(self, *args, **kw): # newbase = base | (kw & options) # imagine all the syntax if self._magic: kw = self._magic(kw) newbase = ub.dict_union(self._base, ub.dict_isect(kw, self._options)) new = Element(newbase, self._options, self._magic) return new
def json_id(self): children = ub.odict([(key, child.json_id()) for key, child in self.children()]) params = ub.odict([ (key, value.tolist() if isinstance(value, np.ndarray) else value) for key, value in self._params.items()]) return ub.dict_union(ub.odict([('__class__', self.__class__.__name__)]), params, children)
class DetectPredictCLIConfig(scfg.Config): default = ub.dict_union( { 'dataset': scfg.Value(None, help='coco dataset, path to images or folder of images'), 'out_dpath': scfg.Value('./out', help='output directory'), 'draw': scfg.Value(False), 'workdir': scfg.Value('~/work/bioharn', help='work directory for sampler if needed'), }, DetectPredictConfig.default )
def _normalize_attrs(self): tup = self.clf_key.split('-') wrap_type = None if len(tup) == 1 else tup[1] est_type = tup[0] self.est_type = est_type self.wrap_type = wrap_type est_kw1, est_kw2 = self._lookup_params(self.est_type) self.est_kw1 = est_kw1 self.est_kw2 = est_kw2 self.est_kw = ub.dict_union(est_kw1, est_kw2, self.kw)
def __init__(self, base, options={}, _magic=None): """ Args: base (dict): the keys / values this schema must contain options (options): the keys / values this schema may contain _magic (callable): called when creating an instance of this schema element. Allows convinience attributes to be converted to the formal jsonschema specs. TODO: _magic is a terrible name, we need to rename it with something descriptive. """ self._base = base if isinstance(options, (set, list, tuple)): options = {k: None for k in options} self._options = ub.dict_union(options, self.__generics__) self._magic = _magic super().__init__(base)
def likely_duplicates(cls, pfiles, thresh=0.2, verbose=1): final_groups = {} active_groups = [pfiles] mode = 'thread' max_workers = 6 while active_groups: group_sizes = list(map(len, active_groups)) total_active = sum(group_sizes) print('Checking {} active groups with {} items'.format( len(active_groups), total_active)) groups = ub.dict_union( *[ProgressiveFile.group_pfiles(g) for g in active_groups]) # Mark all groups that need refinement refine_items = [] next_groups = [] for key, group in groups.items(): if len(group) > 1 and key[-1] < thresh: next_groups.append(group) refine_items.extend([ item for item in group if item.step_id()[-1] < thresh ]) else: # Any group that doesnt need refinment is added to the # solution and will not appear in the next active group final_groups[key] = group # Refine any item that needs it if len(refine_items): # TODO: if there are few enough items, just refine to the # threshold? ProgressiveFile.parallel_refine(refine_items, mode=mode, step_idx='next', max_workers=max_workers, verbose=verbose) # Continue refinement as long as there are active groups active_groups = next_groups return final_groups
def likely_overlaps(cls, pfiles1, pfiles2, thresh=0.2, verbose=1): """ This is similar to finding duplicates, but between two sets of files Example: >>> fpaths = _demodata_files(num_files=100, rng=0) >>> fpaths1 = fpaths[0::2] >>> fpaths2 = fpaths[1::2] >>> pfiles1 = [ProgressiveFile(f) for f in fpaths1] >>> pfiles2 = [ProgressiveFile(f) for f in fpaths2] >>> overlap, only1, only2 = ProgressiveFile.likely_overlaps(pfiles1, pfiles2) >>> print(len(overlaps)) >>> print(len(only1)) >>> print(len(only2)) """ final_groups = {} # Mark each set of files, so we only refine if a duplicate group # contains elements from multiple sets set1 = {id(p) for p in pfiles1} set2 = {id(p) for p in pfiles2} def _membership(p): partof = [] pid = id(p) if pid in set1: partof.append(1) if pid in set2: partof.append(2) return partof pfiles = pfiles1 + pfiles2 active_groups = [pfiles] mode = 'thread' max_workers = 6 if isinstance(thresh, dict): frac_thresh = thresh.get('frac', None) byte_thresh = thresh.get('byte', None) else: frac_thresh = thresh byte_thresh = thresh while active_groups: group_sizes = list(map(len, active_groups)) total_active = sum(group_sizes) print('Checking {} active groups with {} items'.format( len(active_groups), total_active)) groups = ub.dict_union( *[ProgressiveFile.group_pfiles(g) for g in active_groups]) # Mark all groups that need refinement refine_items = [] next_groups = [] for key, group in groups.items(): membership = {m for p in group for m in _membership(p)} group_frac = key[3] group_byte = key[1] # Check if we have hashed enough of the file by fraction or # number of bytes. terms = [] if frac_thresh is not None: terms.append(group_frac >= frac_thresh) if byte_thresh is not None: terms.append(group_byte >= byte_thresh) good_enough = any(terms) or len(terms) == 0 if not good_enough and len(membership) > 1 and len(group) > 1: next_groups.append(group) needs_refine = [ item for item in group if not item.complete_enough(frac_thresh=frac_thresh, byte_thresh=byte_thresh) ] refine_items.extend(needs_refine) else: # Any group that doesnt need refinment is added to the # solution and will not appear in the next active group final_groups[key] = group # Refine any item that needs it if len(refine_items): # TODO: if there are few enough items, just refine to the # threshold? ProgressiveFile.parallel_refine(refine_items, mode=mode, step_idx='next', max_workers=max_workers, verbose=verbose) # Continue refinement as long as there are active groups active_groups = next_groups only1 = {} only2 = {} overlap = {} for key, group in final_groups.items(): membership = {m for p in group for m in _membership(p)} if len(membership) == 1: if ub.peek(membership) == 1: only1[key] = group else: only2[key] = group else: overlap[key] = group return overlap, only1, only2
def run_benchmark_renormalization(): """ See if we can renormalize probabilities after update with a faster method that maintains memory a bit better Example: >>> import sys, ubelt >>> sys.path.append(ubelt.expandpath('~/misc/tests/python')) >>> from bench_renormalization import * # NOQA >>> run_benchmark_renormalization() """ import ubelt as ub import xdev import pathlib import timerit fpath = pathlib.Path('~/misc/tests/python/renormalize_cython.pyx').expanduser() renormalize_cython = xdev.import_module_from_pyx(fpath, annotate=True, verbose=3, recompile=True) xdev.profile_now(renormalize_demo_v1)(1000, 100) xdev.profile_now(renormalize_demo_v2)(1000, 100) xdev.profile_now(renormalize_demo_v3)(1000, 100) xdev.profile_now(renormalize_demo_v4)(1000, 100) func_list = [ # renormalize_demo_v1, renormalize_demo_v2, # renormalize_demo_v3, # renormalize_demo_v4, renormalize_cython.renormalize_demo_cython_v1, renormalize_cython.renormalize_demo_cython_v2, renormalize_cython.renormalize_demo_cython_v3, ] methods = {f.__name__: f for f in func_list} for key, method in methods.items(): with timerit.Timer(label=key, verbose=0) as t: method(1000, 100) print(f'{key:<30} {t.toc():0.6f}') arg_basis = { 'T': [10, 20, 30, 50], 'D': [10, 50, 100, 300], } args_grid = [] for argkw in list(ub.named_product(arg_basis)): if argkw['T'] <= argkw['D']: arg_basis['size'] = argkw['T'] * argkw['D'] args_grid.append(argkw) ti = timerit.Timerit(100, bestof=10, verbose=2) measures = [] for method_name, method in methods.items(): for argkw in args_grid: row = ub.dict_union({'method': method_name}, argkw) key = ub.repr2(row, compact=1) argkey = ub.repr2(argkw, compact=1) kwargs = ub.dict_subset(argkw, ['T', 'D']) for timer in ti.reset('time'): with timer: method(**kwargs) row['mean'] = ti.mean() row['min'] = ti.min() row['key'] = key row['argkey'] = argkey measures.append(row) import pandas as pd df = pd.DataFrame(measures) import kwplot sns = kwplot.autosns() kwplot.figure(fnum=1, pnum=(1, 2, 1), docla=True) sns.lineplot(data=df, x='D', y='min', hue='method', style='method') kwplot.figure(fnum=1, pnum=(1, 2, 2), docla=True) sns.lineplot(data=df, x='T', y='min', hue='method', style='method') p = (df.pivot(['method'], ['argkey'], ['mean'])) print(p.mean(axis=1).sort_values())
def train(): """ Example: >>> train() """ import random np.random.seed(1031726816 % 4294967295) torch.manual_seed(137852547 % 4294967295) random.seed(2497950049 % 4294967295) xpu = xpu_device.XPU.from_argv() print('Chosen xpu = {!r}'.format(xpu)) cifar_num = 10 if ub.argflag('--lab'): datasets = cifar_training_datasets(output_colorspace='LAB', norm_mode='independent', cifar_num=cifar_num) elif ub.argflag('--rgb'): datasets = cifar_training_datasets(output_colorspace='RGB', norm_mode='independent', cifar_num=cifar_num) elif ub.argflag('--rgb-dep'): datasets = cifar_training_datasets(output_colorspace='RGB', norm_mode='dependant', cifar_num=cifar_num) else: raise AssertionError('specify --rgb / --lab') import netharn.models.densenet # batch_size = (128 // 3) * 3 batch_size = 64 # initializer_ = (initializers.KaimingNormal, { # 'nonlinearity': 'relu', # }) lr = 0.1 initializer_ = (initializers.LSUV, {}) hyper = hyperparams.HyperParams( workdir=ub.ensuredir('train_cifar_work'), model=( netharn.models.densenet.DenseNet, { 'cifar': True, 'block_config': (32, 32, 32), # 100 layer depth 'num_classes': datasets['train'].n_classes, 'drop_rate': float(ub.argval('--drop_rate', default=.2)), 'groups': 1, }), optimizer=( torch.optim.SGD, { # 'weight_decay': .0005, 'weight_decay': float(ub.argval('--weight_decay', default=.0005)), 'momentum': 0.9, 'nesterov': True, 'lr': 0.1, }), scheduler=(nh.schedulers.ListedLR, { 'points': { 0: lr, 150: lr * 0.1, 250: lr * 0.01, }, 'interpolate': False }), monitor=(nh.Monitor, { 'minimize': ['loss'], 'maximize': ['mAP'], 'patience': 314, 'max_epoch': 314, }), initializer=initializer_, criterion=(torch.nn.CrossEntropyLoss, {}), # Specify anything else that is special about your hyperparams here # Especially if you make a custom_batch_runner augment=str(datasets['train'].augmenter), other=ub.dict_union( { # TODO: type of augmentation as a parameter dependency # 'augmenter': str(datasets['train'].augmenter), # 'augment': datasets['train'].augment, 'batch_size': batch_size, 'colorspace': datasets['train'].output_colorspace, 'n_classes': datasets['train'].n_classes, # 'center_inputs': datasets['train'].center_inputs, }, datasets['train'].center_inputs.__dict__), ) # if ub.argflag('--rgb-indie'): # hyper.other['norm'] = 'dependant' hyper.input_ids['train'] = datasets['train'].input_id xpu = xpu_device.XPU.cast('auto') print('xpu = {}'.format(xpu)) data_kw = {'batch_size': batch_size} if xpu.is_gpu(): data_kw.update({'num_workers': 8, 'pin_memory': True}) tags = ['train', 'vali', 'test'] loaders = ub.odict() for tag in tags: dset = datasets[tag] shuffle = tag == 'train' data_kw_ = data_kw.copy() if tag != 'train': data_kw_['batch_size'] = max(batch_size // 4, 1) loader = torch.utils.data.DataLoader(dset, shuffle=shuffle, **data_kw_) loaders[tag] = loader harn = fit_harness.FitHarness( hyper=hyper, datasets=datasets, xpu=xpu, loaders=loaders, ) # harn.monitor = early_stop.EarlyStop(patience=40) harn.monitor = monitor.Monitor(min_keys=['loss'], max_keys=['global_acc', 'class_acc'], patience=40) harn.initialize() harn.run()
def rectify_normalizer(in_channels, key=ub.NoParam, dim=2): """ Allows dictionary based specification of a normalizing layer Example: >>> rectify_normalizer(8) BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) >>> rectify_normalizer(8, 'batch') BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) >>> rectify_normalizer(8, {'type': 'batch'}) BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) >>> rectify_normalizer(8, 'group') GroupNorm(8, 8, eps=1e-05, affine=True) >>> rectify_normalizer(8, {'type': 'group', 'num_groups': 2}) GroupNorm(2, 8, eps=1e-05, affine=True) >>> rectify_normalizer(8, dim=3) BatchNorm3d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) >>> rectify_normalizer(8, None) None """ if key is None: return None if key is ub.NoParam: key = 'batch' if isinstance(key, six.string_types): if key == 'batch': key = {'type': 'batch'} elif key == 'group': key = { 'type': 'group', 'num_groups': ('gcd', min(in_channels, 32)) } elif key == 'batch+group': key = {'type': 'batch+group'} else: raise KeyError(key) elif isinstance(key, dict): key = key.copy() else: raise TypeError(type(key)) norm_type = key.pop('type') if norm_type == 'batch': in_channels_key = 'num_features' if dim == 1: cls = torch.nn.BatchNorm1d elif dim == 2: cls = torch.nn.BatchNorm2d elif dim == 3: cls = torch.nn.BatchNorm3d else: raise ValueError(dim) elif norm_type == 'group': in_channels_key = 'num_channels' if isinstance(key['num_groups'], tuple): if key['num_groups'][0] == 'gcd': key['num_groups'] = gcd(key['num_groups'][1], in_channels) if in_channels % key['num_groups'] != 0: raise AssertionError( 'Cannot divide n_inputs {} by num groups {}'.format( in_channels, key['num_groups'])) cls = torch.nn.GroupNorm elif norm_type == 'batch+group': return torch.nn.Sequential( rectify_normalizer(in_channels, 'batch', dim=dim), rectify_normalizer(in_channels, ub.dict_union({'type': 'group'}, key), dim=dim), ) else: raise KeyError('unknown type: {}'.format(key)) assert in_channels_key not in key key[in_channels_key] = in_channels return cls(**key)
def setup_harness(workers=None): """ CommandLine: python ~/code/clab/examples/yolo_voc.py setup_harness python ~/code/clab/examples/yolo_voc.py setup_harness --profile Example: >>> harn = setup_harness(workers=0) >>> harn.initialize() >>> harn.dry = True >>> # xdoc: +SKIP >>> harn.run() """ workdir = ub.truepath('~/work/VOC2007') devkit_dpath = ub.truepath('~/data/VOC/VOCdevkit') YoloVOCDataset.ensure_voc_data() if ub.argflag('--2007'): dsetkw = {'years': [2007]} elif ub.argflag('--2012'): dsetkw = {'years': [2007, 2012]} else: dsetkw = {'years': [2007]} data_choice = ub.argval('--data', 'normal') if data_choice == 'combined': datasets = { 'test': YoloVOCDataset(devkit_dpath, split='test', **dsetkw), 'train': YoloVOCDataset(devkit_dpath, split='trainval', **dsetkw), } elif data_choice == 'notest': datasets = { 'train': YoloVOCDataset(devkit_dpath, split='train', **dsetkw), 'vali': YoloVOCDataset(devkit_dpath, split='val', **dsetkw), } elif data_choice == 'normal': datasets = { 'train': YoloVOCDataset(devkit_dpath, split='train', **dsetkw), 'vali': YoloVOCDataset(devkit_dpath, split='val', **dsetkw), 'test': YoloVOCDataset(devkit_dpath, split='test', **dsetkw), } else: raise KeyError(data_choice) nice = ub.argval('--nice', default=None) pretrained_fpath = darknet.initial_weights() # NOTE: XPU implicitly supports DataParallel just pass --gpu=0,1,2,3 xpu = xpu_device.XPU.cast('argv') print('xpu = {!r}'.format(xpu)) ensure_ulimit() postproc_params = dict( conf_thresh=0.001, nms_thresh=0.5, ovthresh=0.5, ) max_epoch = 160 lr_step_points = { 0: 0.001, 60: 0.0001, 90: 0.00001, } if ub.argflag('--warmup'): lr_step_points = { # warmup learning rate 0: 0.0001, 1: 0.0001, 2: 0.0002, 3: 0.0003, 4: 0.0004, 5: 0.0005, 6: 0.0006, 7: 0.0007, 8: 0.0008, 9: 0.0009, 10: 0.0010, # cooldown learning rate 60: 0.0001, 90: 0.00001, } batch_size = int(ub.argval('--batch_size', default=16)) n_cpus = psutil.cpu_count(logical=True) workers = int(ub.argval('--workers', default=int(n_cpus / 2))) print('Making loaders') loaders = make_loaders(datasets, batch_size=batch_size, workers=workers if workers is not None else workers) """ Reference: Original YOLO9000 hyperparameters are defined here: https://github.com/pjreddie/darknet/blob/master/cfg/yolo-voc.2.0.cfg https://github.com/longcw/yolo2-pytorch/issues/1#issuecomment-286410772 Notes: jitter is a translation / crop parameter https://groups.google.com/forum/#!topic/darknet/A-JJeXprvJU thresh in 2.0.cfg is iou_thresh here """ print('Making hyperparams') hyper = hyperparams.HyperParams( model=(darknet.Darknet19, { 'num_classes': datasets['train'].num_classes, 'anchors': datasets['train'].anchors }), criterion=(darknet_loss.DarknetLoss, { 'anchors': datasets['train'].anchors, 'object_scale': 5.0, 'noobject_scale': 1.0, 'class_scale': 1.0, 'coord_scale': 1.0, 'iou_thresh': 0.6, 'reproduce_longcw': ub.argflag('--longcw'), 'denom': ub.argval('--denom', default='num_boxes'), }), optimizer=(torch.optim.SGD, dict( lr=lr_step_points[0], momentum=0.9, weight_decay=0.0005 )), # initializer=(nninit.KaimingNormal, {}), initializer=(nninit.Pretrained, { 'fpath': pretrained_fpath, }), scheduler=(ListedLR, dict( step_points=lr_step_points )), other=ub.dict_union({ 'nice': str(nice), 'batch_size': loaders['train'].batch_sampler.batch_size, }, postproc_params), centering=None, # centering=datasets['train'].centering, augment=datasets['train'].augmenter, ) harn = fit_harness.FitHarness( hyper=hyper, xpu=xpu, loaders=loaders, max_iter=max_epoch, workdir=workdir, ) harn.postproc_params = postproc_params harn.nice = nice harn.monitor = monitor.Monitor(min_keys=['loss'], # max_keys=['global_acc', 'class_acc'], patience=max_epoch) @harn.set_batch_runner def batch_runner(harn, inputs, labels): """ Custom function to compute the output of a batch and its loss. Example: >>> import sys >>> sys.path.append('/home/joncrall/code/clab/examples') >>> from yolo_voc import * >>> harn = setup_harness(workers=0) >>> harn.initialize() >>> batch = harn._demo_batch(0, 'train') >>> inputs, labels = batch >>> criterion = harn.criterion >>> weights_fpath = darknet.demo_weights() >>> state_dict = torch.load(weights_fpath)['model_state_dict'] >>> harn.model.module.load_state_dict(state_dict) >>> outputs, loss = harn._custom_run_batch(harn, inputs, labels) """ # hack for data parallel # if harn.current_tag == 'train': outputs = harn.model(*inputs) # else: # # Run test and validation on a single GPU # outputs = harn.model.module(*inputs) # darknet criterion needs to know the input image shape inp_size = tuple(inputs[0].shape[-2:]) aoff_pred, iou_pred, prob_pred = outputs gt_boxes, gt_classes, orig_size, indices, gt_weights = labels loss = harn.criterion(aoff_pred, iou_pred, prob_pred, gt_boxes, gt_classes, gt_weights=gt_weights, inp_size=inp_size, epoch=harn.epoch) return outputs, loss @harn.add_batch_metric_hook def custom_metrics(harn, output, labels): metrics_dict = ub.odict() criterion = harn.criterion metrics_dict['L_bbox'] = float(criterion.bbox_loss.data.cpu().numpy()) metrics_dict['L_iou'] = float(criterion.iou_loss.data.cpu().numpy()) metrics_dict['L_cls'] = float(criterion.cls_loss.data.cpu().numpy()) return metrics_dict # Set as a harness attribute instead of using a closure harn.batch_confusions = [] @harn.add_iter_callback def on_batch(harn, tag, loader, bx, inputs, labels, outputs, loss): """ Custom hook to run on each batch (used to compute mAP on the fly) Example: >>> harn = setup_harness(workers=0) >>> harn.initialize() >>> batch = harn._demo_batch(0, 'train') >>> inputs, labels = batch >>> criterion = harn.criterion >>> loader = harn.loaders['train'] >>> weights_fpath = darknet.demo_weights() >>> state_dict = torch.load(weights_fpath)['model_state_dict'] >>> harn.model.module.load_state_dict(state_dict) >>> outputs, loss = harn._custom_run_batch(harn, inputs, labels) >>> tag = 'train' >>> on_batch(harn, tag, loader, bx, inputs, labels, outputs, loss) """ # Accumulate relevant outputs to measure gt_boxes, gt_classes, orig_size, indices, gt_weights = labels # aoff_pred, iou_pred, prob_pred = outputs im_sizes = orig_size inp_size = inputs[0].shape[-2:][::-1] conf_thresh = harn.postproc_params['conf_thresh'] nms_thresh = harn.postproc_params['nms_thresh'] ovthresh = harn.postproc_params['ovthresh'] postout = harn.model.module.postprocess(outputs, inp_size, im_sizes, conf_thresh, nms_thresh) # batch_pred_boxes, batch_pred_scores, batch_pred_cls_inds = postout # Compute: y_pred, y_true, and y_score for this batch batch_pred_boxes, batch_pred_scores, batch_pred_cls_inds = postout batch_true_boxes, batch_true_cls_inds = labels[0:2] batch_orig_sz, batch_img_inds = labels[2:4] y_batch = [] for bx, index in enumerate(batch_img_inds.data.cpu().numpy().ravel()): pred_boxes = batch_pred_boxes[bx] pred_scores = batch_pred_scores[bx] pred_cxs = batch_pred_cls_inds[bx] # Group groundtruth boxes by class true_boxes_ = batch_true_boxes[bx].data.cpu().numpy() true_cxs = batch_true_cls_inds[bx].data.cpu().numpy() true_weights = gt_weights[bx].data.cpu().numpy() # Unnormalize the true bboxes back to orig coords orig_size = batch_orig_sz[bx] sx, sy = np.array(orig_size) / np.array(inp_size) if len(true_boxes_): true_boxes = np.hstack([true_boxes_, true_weights[:, None]]) true_boxes[:, 0:4:2] *= sx true_boxes[:, 1:4:2] *= sy y = voc.EvaluateVOC.image_confusions(true_boxes, true_cxs, pred_boxes, pred_scores, pred_cxs, ovthresh=ovthresh) y['gx'] = index y_batch.append(y) harn.batch_confusions.extend(y_batch) @harn.add_epoch_callback def on_epoch(harn, tag, loader): y = pd.concat(harn.batch_confusions) num_classes = len(loader.dataset.label_names) mean_ap, ap_list = voc.EvaluateVOC.compute_map(y, num_classes) harn.log_value(tag + ' epoch mAP', mean_ap, harn.epoch) # max_ap = np.nanmax(ap_list) # harn.log_value(tag + ' epoch max-AP', max_ap, harn.epoch) harn.batch_confusions.clear() return harn
def info(self): return ub.dict_union(self._info, { 'unique': self.unique(), 'normed': self.normalize(), })
def 字典_合并(*args): # 字典_取值({'a': 1, 'b': 1}, {'b': 2, 'c': 2}) data = ub.dict_union(*args) return data
def setup_harness(workers=None): """ CommandLine: python ~/code/clab/examples/yolo_voc2.py setup_harness python ~/code/clab/examples/yolo_voc2.py setup_harness --profile python ~/code/clab/examples/yolo_voc2.py setup_harness --flamegraph Example: >>> harn = setup_harness(workers=0) >>> harn.initialize() >>> harn.dry = True >>> harn.run() """ workdir = ub.truepath('~/work/VOC2007') devkit_dpath = ub.truepath('~/data/VOC/VOCdevkit') YoloVOCDataset.ensure_voc_data() if ub.argflag('--2007'): dsetkw = {'years': [2007]} elif ub.argflag('--2012'): dsetkw = {'years': [2007, 2012]} else: dsetkw = {'years': [2007]} data_choice = ub.argval('--data', 'normal') if ub.argflag('--small'): dsetkw['base_wh'] = np.array([7, 7]) * 32 dsetkw['scales'] = [-1, 1] if data_choice == 'combined': datasets = { 'test': YoloVOCDataset(devkit_dpath, split='test', **dsetkw), 'train': YoloVOCDataset(devkit_dpath, split='trainval', **dsetkw), } elif data_choice == 'notest': datasets = { 'train': YoloVOCDataset(devkit_dpath, split='train', **dsetkw), 'vali': YoloVOCDataset(devkit_dpath, split='val', **dsetkw), } elif data_choice == 'normal': datasets = { 'train': YoloVOCDataset(devkit_dpath, split='train', **dsetkw), 'vali': YoloVOCDataset(devkit_dpath, split='val', **dsetkw), 'test': YoloVOCDataset(devkit_dpath, split='test', **dsetkw), } else: raise KeyError(data_choice) nice = ub.argval('--nice', default=None) pretrained_fpath = ensure_lightnet_initial_weights() # NOTE: XPU implicitly supports DataParallel just pass --gpu=0,1,2,3 xpu = xpu_device.XPU.cast('argv') print('xpu = {!r}'.format(xpu)) ensure_ulimit() postproc_params = dict( conf_thresh=0.001, # nms_thresh=0.5, nms_thresh=0.4, ovthresh=0.5, ) max_epoch = 160 lr_step_points = { 0: 0.001, 60: 0.0001, 90: 0.00001, } # if ub.argflag('--warmup'): lr_step_points = { # warmup learning rate 0: 0.0001, 1: 0.0001, 2: 0.0002, 3: 0.0003, 4: 0.0004, 5: 0.0005, 6: 0.0006, 7: 0.0007, 8: 0.0008, 9: 0.0009, 10: 0.0010, # cooldown learning rate 60: 0.0001, 90: 0.00001, } batch_size = int(ub.argval('--batch_size', default=16)) n_cpus = psutil.cpu_count(logical=True) if workers is None: workers = int(ub.argval('--workers', default=int(n_cpus / 2))) print('Making loaders') loaders = make_loaders(datasets, batch_size=batch_size, workers=workers if workers is not None else workers) # anchors = {'num': 5, 'values': list(ub.flatten(datasets['train'].anchors))} anchors = dict(num=5, values=[1.3221, 1.73145, 3.19275, 4.00944, 5.05587, 8.09892, 9.47112, 4.84053, 11.2364, 10.0071]) print('Making hyperparams') hyper = hyperparams.HyperParams( # model=(darknet.Darknet19, { model=(light_yolo.Yolo, { 'num_classes': datasets['train'].num_classes, 'anchors': anchors, 'conf_thresh': postproc_params['conf_thresh'], 'nms_thresh': postproc_params['nms_thresh'], }), criterion=(RegionLoss, { 'num_classes': datasets['train'].num_classes, 'anchors': anchors, # 'object_scale': 5.0, # 'noobject_scale': 1.0, # 'class_scale': 1.0, # 'coord_scale': 1.0, # 'thresh': 0.6, }), optimizer=(torch.optim.SGD, dict( lr=lr_step_points[0], momentum=0.9, weight_decay=0.0005 )), initializer=(nninit.Pretrained, { 'fpath': pretrained_fpath, }), scheduler=(ListedLR, dict( step_points=lr_step_points )), other=ub.dict_union({ 'nice': str(nice), 'batch_size': loaders['train'].batch_sampler.batch_size, }, postproc_params), centering=None, # centering=datasets['train'].centering, augment=datasets['train'].augmenter, ) harn = fit_harness.FitHarness( hyper=hyper, xpu=xpu, loaders=loaders, max_iter=max_epoch, workdir=workdir, ) harn.postproc_params = postproc_params harn.nice = nice harn.monitor = monitor.Monitor(min_keys=['loss'], # max_keys=['global_acc', 'class_acc'], patience=max_epoch) @harn.set_batch_runner @profiler.profile def batch_runner(harn, inputs, labels): """ Custom function to compute the output of a batch and its loss. Example: >>> import sys >>> sys.path.append('/home/joncrall/code/clab/examples') >>> from yolo_voc2 import * >>> harn = setup_harness(workers=0) >>> harn.initialize() >>> batch = harn._demo_batch(0, 'vali') >>> inputs, labels = batch >>> criterion = harn.criterion >>> weights_fpath = light_yolo.demo_weights() >>> state_dict = torch.load(weights_fpath)['weights'] >>> harn.model.module.load_state_dict(state_dict) >>> outputs, loss = harn._custom_run_batch(harn, inputs, labels) """ if harn.dry: shape = harn.model.module.output_shape_for(inputs[0].shape) outputs = torch.rand(*shape) else: outputs = harn.model.forward(*inputs) # darknet criterion needs to know the input image shape # inp_size = tuple(inputs[0].shape[-2:]) target = labels[0] bsize = inputs[0].shape[0] n_items = len(harn.loaders['train']) bx = harn.bxs.get('train', 0) seen = harn.epoch * n_items + (bx * bsize) loss = harn.criterion(outputs, target, seen=seen) return outputs, loss @harn.add_batch_metric_hook @profiler.profile def custom_metrics(harn, output, labels): metrics_dict = ub.odict() criterion = harn.criterion metrics_dict['L_bbox'] = float(criterion.loss_coord.data.cpu().numpy()) metrics_dict['L_iou'] = float(criterion.loss_conf.data.cpu().numpy()) metrics_dict['L_cls'] = float(criterion.loss_cls.data.cpu().numpy()) return metrics_dict # Set as a harness attribute instead of using a closure harn.batch_confusions = [] @harn.add_iter_callback @profiler.profile def on_batch(harn, tag, loader, bx, inputs, labels, outputs, loss): """ Custom hook to run on each batch (used to compute mAP on the fly) Example: >>> harn = setup_harness(workers=0) >>> harn.initialize() >>> batch = harn._demo_batch(0, 'vali') >>> inputs, labels = batch >>> criterion = harn.criterion >>> loader = harn.loaders['train'] >>> weights_fpath = light_yolo.demo_weights() >>> state_dict = torch.load(weights_fpath)['weights'] >>> harn.model.module.load_state_dict(state_dict) >>> outputs, loss = harn._custom_run_batch(harn, inputs, labels) >>> tag = 'train' >>> on_batch(harn, tag, loader, bx, inputs, labels, outputs, loss) Ignore: >>> target, gt_weights, batch_orig_sz, batch_index = labels >>> bx = 0 >>> postout = harn.model.module.postprocess(outputs.clone()) >>> item = postout[bx].cpu().numpy() >>> item = item[item[:, 4] > .6] >>> cxywh = util.Boxes(item[..., 0:4], 'cxywh') >>> orig_size = batch_orig_sz[bx].numpy().ravel() >>> tlbr = cxywh.scale(orig_size).asformat('tlbr').data >>> truth_bx = target[bx] >>> truth_bx = truth_bx[truth_bx[:, 0] != -1] >>> truth_tlbr = util.Boxes(truth_bx[..., 1:5].numpy(), 'cxywh').scale(orig_size).asformat('tlbr').data >>> chw = inputs[0][bx].numpy().transpose(1, 2, 0) >>> rgb255 = cv2.resize(chw * 255, tuple(orig_size)) >>> mplutil.figure(fnum=1, doclf=True) >>> mplutil.imshow(rgb255, colorspace='rgb') >>> mplutil.draw_boxes(tlbr, 'tlbr') >>> mplutil.draw_boxes(truth_tlbr, 'tlbr', color='orange') >>> mplutil.show_if_requested() """ # Accumulate relevant outputs to measure target, gt_weights, batch_orig_sz, batch_index = labels # inp_size = inputs[0].shape[-2:][::-1] conf_thresh = harn.postproc_params['conf_thresh'] nms_thresh = harn.postproc_params['nms_thresh'] ovthresh = harn.postproc_params['ovthresh'] if outputs is None: return get_bboxes = harn.model.module.postprocess get_bboxes.conf_thresh = conf_thresh get_bboxes.nms_thresh = nms_thresh postout = harn.model.module.postprocess(outputs) batch_pred_boxes = [] batch_pred_scores = [] batch_pred_cls_inds = [] for bx, item_ in enumerate(postout): item = item_.cpu().numpy() if len(item): cxywh = util.Boxes(item[..., 0:4], 'cxywh') orig_size = batch_orig_sz[bx].cpu().numpy().ravel() tlbr = cxywh.scale(orig_size).asformat('tlbr').data batch_pred_boxes.append(tlbr) batch_pred_scores.append(item[..., 4]) batch_pred_cls_inds.append(item[..., 5]) else: batch_pred_boxes.append(np.empty((0, 4))) batch_pred_scores.append(np.empty(0)) batch_pred_cls_inds.append(np.empty(0)) batch_true_cls_inds = target[..., 0] batch_true_boxes = target[..., 1:5] batch_img_inds = batch_index y_batch = [] for bx, index in enumerate(batch_img_inds.data.cpu().numpy().ravel()): pred_boxes = batch_pred_boxes[bx] pred_scores = batch_pred_scores[bx] pred_cxs = batch_pred_cls_inds[bx] # Group groundtruth boxes by class true_boxes_ = batch_true_boxes[bx].data.cpu().numpy() true_cxs = batch_true_cls_inds[bx].data.cpu().numpy() true_weights = gt_weights[bx].data.cpu().numpy() # Unnormalize the true bboxes back to orig coords orig_size = batch_orig_sz[bx] if len(true_boxes_): true_boxes = util.Boxes(true_boxes_, 'cxywh').scale( orig_size).asformat('tlbr').data true_boxes = np.hstack([true_boxes, true_weights[:, None]]) else: true_boxes = true_boxes_.reshape(-1, 4) y = voc.EvaluateVOC.image_confusions(true_boxes, true_cxs, pred_boxes, pred_scores, pred_cxs, ovthresh=ovthresh) y['gx'] = index y_batch.append(y) harn.batch_confusions.extend(y_batch) @harn.add_epoch_callback @profiler.profile def on_epoch(harn, tag, loader): y = pd.concat(harn.batch_confusions) num_classes = len(loader.dataset.label_names) mean_ap, ap_list = voc.EvaluateVOC.compute_map(y, num_classes) harn.log_value(tag + ' epoch mAP', mean_ap, harn.epoch) max_ap = np.nanmax(ap_list) harn.log_value(tag + ' epoch max-AP', max_ap, harn.epoch) harn.batch_confusions.clear() return harn
def run_protocol( self, config: Dict[str, Any], extra_plugins: Dict[str, Any] = dict()) -> None: """Run the protocol by printout out the config. Args: Config passed in uses 3 parameters to control the launching of the protocols - protocol: either 'ond' or 'condda' to define which protocol to run - harness: either 'local' or 'par' to define which harness to use - workdir: a directory to save all the information from the run including - Config - Output of algorithm Example: >>> from sailon_tinker_launcher.main import * >>> dpath = ub.ensure_app_cache_dir('tinker/tests') >>> config = get_debug_config() >>> self = LaunchSailonProtocol() >>> self.run_protocol(config) >>> assert(self.working_folder.exists()) >>> ub.delete(str(self.working_folder), verbose=False) """ # Setup working folder and create new config for this run self.working_folder, working_config_fp, privileged_config, config = self.setup_experiment( config) # Now experiment setup, start a new logger for this fh = logging.FileHandler( self.working_folder / f'{datetime.now().strftime("%Y_%m_%d-%I_%M_%S_%p")}.log') fh.setLevel(logging.DEBUG) formatter = logging.Formatter( '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s') fh.setFormatter(formatter) logging.getLogger().addHandler(fh) log.info(f'Config Filepath: {working_config_fp}') log.info(f'Config: \n{json.dumps(config, indent=4)}') # Load the harness # This config is not used but will throw error if not pointed at harnness_config_path = Path(protocol_folder.__file__).parent if privileged_config['harness'] == 'local': log.info('Loading Local Harness') harness = LocalInterface('configuration.json', str(harnness_config_path)) harness.result_directory = config['detectors']['csv_folder'] harness.file_provider.results_folder = config['detectors'][ 'csv_folder'] elif privileged_config['harness'] == 'par': log.info('Loading Par Harness') harness = ParInterface('configuration.json', str(harnness_config_path)) harness.folder = config['detectors']['csv_folder'] else: raise AttributeError( f'Valid harnesses "local" or "par". ' f'Given harness "{privileged_config["harness"]}" ') # Get the plugins plugins = ub.dict_union(discoverable_plugins('tinker'), extra_plugins) log.debug('Plugins found:') log.debug(plugins) # Load the protocol if privileged_config['protocol'] == 'ond': log.info('Running OND Protocol') run_protocol = OND(discovered_plugins=plugins, algorithmsdirectory='', harness=harness, config_file=str(working_config_fp)) elif privileged_config['protocol'] == 'condda': log.info('Running Condda Protocol') run_protocol = Condda(discovered_plugins=plugins, algorithmsdirectory='', harness=harness, config_file=str(working_config_fp)) else: raise AttributeError( f'Please set protocol to either "ond" or "condda". ' f'"{privileged_config["protocol"]}" in the config files') # Run the protocol run_protocol.run_protocol() log.info('Protocol Finished') logging.getLogger().removeHandler(fh)
def rectify_normalizer(in_channels, key=ub.NoParam, dim=2, **kwargs): """ Allows dictionary based specification of a normalizing layer Args: in_channels (int): number of input channels dim (int): dimensionality **kwargs: extra args Example: >>> rectify_normalizer(8) BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) >>> rectify_normalizer(8, 'batch') BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) >>> rectify_normalizer(8, {'type': 'batch'}) BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) >>> rectify_normalizer(8, 'group') GroupNorm(8, 8, eps=1e-05, affine=True) >>> rectify_normalizer(8, {'type': 'group', 'num_groups': 2}) GroupNorm(2, 8, eps=1e-05, affine=True) >>> rectify_normalizer(8, dim=3) BatchNorm3d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) >>> rectify_normalizer(8, None) None >>> rectify_normalizer(8, key={'type': 'syncbatch'}) """ if key is None: return None if key is ub.NoParam: key = 'batch' if isinstance(key, six.string_types): key = {'type': key} elif isinstance(key, dict): key = key.copy() else: raise TypeError(type(key)) norm_type = key.pop('type') if norm_type == 'batch': in_channels_key = 'num_features' if dim == 0: cls = torch.nn.BatchNorm1d elif dim == 1: cls = torch.nn.BatchNorm1d elif dim == 2: cls = torch.nn.BatchNorm2d elif dim == 3: cls = torch.nn.BatchNorm3d else: raise ValueError(dim) elif norm_type == 'syncbatch': in_channels_key = 'num_features' cls = torch.nn.SyncBatchNorm elif norm_type == 'group': in_channels_key = 'num_channels' if key.get('num_groups') is None: key['num_groups'] = ('gcd', min(in_channels, 32)) if isinstance(key['num_groups'], tuple): if key['num_groups'][0] == 'gcd': key['num_groups'] = gcd( key['num_groups'][1], in_channels) if in_channels % key['num_groups'] != 0: raise AssertionError( 'Cannot divide n_inputs {} by num groups {}'.format( in_channels, key['num_groups'])) cls = torch.nn.GroupNorm elif norm_type == 'batch+group': return torch.nn.Sequential( rectify_normalizer(in_channels, 'batch', dim=dim), rectify_normalizer(in_channels, ub.dict_union({'type': 'group'}, key), dim=dim), ) else: raise KeyError('unknown type: {}'.format(key)) assert in_channels_key not in key key[in_channels_key] = in_channels try: import copy kw = copy.copy(key) kw.update(kwargs) return cls(**kw) except Exception: raise # Ignore kwargs import warnings warnings.warn('kwargs ignored in rectify normalizer') return cls(**key)
def train(): """ Example: >>> train() """ import random np.random.seed(1031726816 % 4294967295) torch.manual_seed(137852547 % 4294967295) random.seed(2497950049 % 4294967295) xpu = xpu_device.XPU.from_argv() print('Chosen xpu = {!r}'.format(xpu)) cifar_num = 10 if ub.argflag('--lab'): datasets = cifar_training_datasets(output_colorspace='LAB', norm_mode='independent', cifar_num=cifar_num) elif ub.argflag('--rgb'): datasets = cifar_training_datasets(output_colorspace='RGB', norm_mode='independent', cifar_num=cifar_num) elif ub.argflag('--rgb-dep'): datasets = cifar_training_datasets(output_colorspace='RGB', norm_mode='dependant', cifar_num=cifar_num) else: raise AssertionError('specify --rgb / --lab') import clab.models.densenet # batch_size = (128 // 3) * 3 batch_size = 64 # initializer_ = (nninit.KaimingNormal, { # 'nonlinearity': 'relu', # }) initializer_ = (nninit.LSUV, {}) hyper = hyperparams.HyperParams( model=( clab.models.densenet.DenseNet, { 'cifar': True, 'block_config': (32, 32, 32), # 100 layer depth 'num_classes': datasets['train'].n_classes, 'drop_rate': float(ub.argval('--drop_rate', default=.2)), 'groups': 1, }), optimizer=( torch.optim.SGD, { # 'weight_decay': .0005, 'weight_decay': float(ub.argval('--weight_decay', default=.0005)), 'momentum': 0.9, 'nesterov': True, 'lr': 0.1, }), scheduler=(torch.optim.lr_scheduler.ReduceLROnPlateau, { 'factor': .5, }), initializer=initializer_, criterion=(torch.nn.CrossEntropyLoss, {}), # Specify anything else that is special about your hyperparams here # Especially if you make a custom_batch_runner augment=str(datasets['train'].augmenter), other=ub.dict_union( { # TODO: type of augmentation as a parameter dependency # 'augmenter': str(datasets['train'].augmenter), # 'augment': datasets['train'].augment, 'batch_size': batch_size, 'colorspace': datasets['train'].output_colorspace, 'n_classes': datasets['train'].n_classes, # 'center_inputs': datasets['train'].center_inputs, }, datasets['train'].center_inputs.__dict__), ) # if ub.argflag('--rgb-indie'): # hyper.other['norm'] = 'dependant' hyper.input_ids['train'] = datasets['train'].input_id xpu = xpu_device.XPU.cast('auto') print('xpu = {}'.format(xpu)) data_kw = {'batch_size': batch_size} if xpu.is_gpu(): data_kw.update({'num_workers': 8, 'pin_memory': True}) tags = ['train', 'vali', 'test'] loaders = ub.odict() for tag in tags: dset = datasets[tag] shuffle = tag == 'train' data_kw_ = data_kw.copy() if tag != 'train': data_kw_['batch_size'] = max(batch_size // 4, 1) loader = torch.utils.data.DataLoader(dset, shuffle=shuffle, **data_kw_) loaders[tag] = loader harn = fit_harness.FitHarness( hyper=hyper, datasets=datasets, xpu=xpu, loaders=loaders, ) # harn.monitor = early_stop.EarlyStop(patience=40) harn.monitor = monitor.Monitor(min_keys=['loss'], max_keys=['global_acc', 'class_acc'], patience=40) @harn.set_batch_runner def batch_runner(harn, inputs, labels): """ Custom function to compute the output of a batch and its loss. """ output = harn.model(*inputs) label = labels[0] loss = harn.criterion(output, label) outputs = [output] return outputs, loss task = harn.datasets['train'].task all_labels = task.labels # ignore_label = datasets['train'].ignore_label # from clab import metrics from clab.metrics import (confusion_matrix, pixel_accuracy_from_confusion, perclass_accuracy_from_confusion) @harn.add_batch_metric_hook def custom_metrics(harn, outputs, labels): label = labels[0] output = outputs[0] y_pred = output.data.max(dim=1)[1].cpu().numpy() y_true = label.data.cpu().numpy() cfsn = confusion_matrix(y_pred, y_true, labels=all_labels) global_acc = pixel_accuracy_from_confusion(cfsn) # same as acc perclass_acc = perclass_accuracy_from_confusion(cfsn) # class_accuracy = perclass_acc.fillna(0).mean() class_accuracy = np.nan_to_num(perclass_acc).mean() metrics_dict = ub.odict() metrics_dict['global_acc'] = global_acc metrics_dict['class_acc'] = class_accuracy return metrics_dict workdir = ub.ensuredir('train_cifar_work') harn.setup_dpath(workdir) harn.run()