def dict_performance(self, gt): '''Evaluate against ground truth on a number of criteria.''' res = ezdict( # Accuracy # acc = self.Accuracy(gt), # Error Err=1 - self.Accuracy(gt), # Negative log-likelihood NLL=self.Clip().NLL(gt), # Per-sample NLL. # sNLL= self.Clip().NLL(gt, reduce=False), # Mean Average Precision. # Note that VOC's flawed version is used for better comparison. mAP=self.mAP(gt), # This is the vanilla Brier score. # Bri = self.Bri(gt), # This is our modified Brier loss. Bri=np.sqrt(self.Bri(gt)), # Expected Calibration Error ECE=self.ECE(gt), # Error rate among 99% confident E99=self.E99(gt), # Entropy Ent=self.Ent(gt), ) return res
def __init__(self, paths=None, train=True, transform=None, target_transform=None, split='val', loader=default_loader): paths = paths if paths is not None else default_paths root = paths.LFWproot root = os.path.join(root, '') # add trailing slash imgroot = os.path.join(root, 'lfw+_jpg24') meta = MU.cached(os.path.join(root, '__meta__.pkl'), type(self)._load_meta, root) meta = ezdict(meta) assert split in {'val', 'test'} if split == 'val': trainf = [0, 1] valf = [2] else: trainf = [0, 1, 2] valf = [3, 4] folds = trainf if train else valf names = [xx for x in folds for xx in meta.images[x]] labels = np.concatenate([meta.labels[x] for x in folds], axis=0) # initialize self imgs = [nl for nl in zip(names, labels)] classes = meta.attrs super().__init__(imgroot, imgs, classes, None, transform, target_transform, loader)
def __init__(self, root=None): # decide root based on input and machine name if root is None: hostname = socket.gethostname() if hostname == 'cashew': self.dsetroot = '/home/zhizhong/tmp_dset/' elif hostname.startswith('vision-'): self.dsetroot = '/home/zli115/tmp_dset/' else: raise Exception( 'Please configure your dataset root and paths in this code file' ) else: self.dsetroot = root # dataset root folders self.ImageNetroot = os.path.join(self.dsetroot, 'ILSVRC2012') # self.Places365root = os.path.join(self.dsetroot, 'Places365') self.VOCroot = os.path.join(self.dsetroot, 'VOC2012') self.Cocoroot = os.path.join(self.dsetroot, 'MSCOCO') self.OIIITPetsroot = os.path.join(self.dsetroot, 'Oxford_IIIT_Pets') self.LFWproot = os.path.join(self.dsetroot, 'LFW+_Release') # pretrained models from https://github.com/CSAILVision/places365 for training self.pretrainedroot = os.path.join(self.dsetroot, 'pretrained') self.prePlaces365 = ezdict() for x in ['resnet18', 'resnet50', 'densenet161']: self.prePlaces365[x] = os.path.join( self.pretrainedroot, '%s_places365.pth.tar' % (x.lower()), )
def multiple_runs_evaulate(safe_pred_allruns, labels): '''Evaluate logits from multiple runs and averaging the performance.''' res_allruns = [ x.dict_performance(labels.numpy()) for x in safe_pred_allruns ] res = ezdict() for k in res_allruns[0].keys(): res[k] = np.mean([x[k] for x in res_allruns]) return res
def register_module(self, name, path): _META_.MODULES[name] = ezdict(path=path, classes=ezdict())
class _META_(type): # real argparser instance PARSER = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) # parsed arguments ARGS = ezdict() # monitored modules MODULES = ezdict() def require_args(self): # args for config file _META_.PARSER.add_argument('--cfgs', type=str, nargs='*', default= ['/home/xifeng/Workspace/2020/PICA/configs/base.yaml', '/home/xifeng/Workspace/2020/PICA/configs/cifar100.yaml'], help='config files to load') def register_module(self, name, path): _META_.MODULES[name] = ezdict(path=path, classes=ezdict()) def register_class(self, module, name, obj): assert module in _META_.MODULES.keys(), ('No module ' 'named [%s] has been registered' % module) _META_.MODULES[module].classes[name] = obj def get_class_name(self, module): assert module in _META_.MODULES.keys(), ('No module ' 'named [%s] has been registered' % module) return [name for name, _ in _META_.MODULES[module].classes.items()] def get_class(self, module, name=None): assert module in _META_.MODULES.keys(), ('No module ' 'named [%s] has been registered' % module) if name is None: return [obj for name, obj in _META_.MODULES[module].classes.items()] assert name in _META_.MODULES[module].classes.keys(), ('No class named [%s] ' 'has been registered in module [%s]' % (name, module)) return _META_.MODULES[module].classes[name] def parse(self): # collect self args self.require_args() # load default args from config file self.from_files(self.known_args().cfgs) # collect args for packages for module in _META_.MODULES.keys(): # setup module arguments mod = importlib.import_module(_META_.MODULES[module].path) if hasattr(mod, 'require_args'): mod.require_args() # re-update default value for new args self.from_files(self.known_args().cfgs) for module in _META_.MODULES.keys(): # setup class arguments if hasattr(self.known_args(), module): cls = self.get_class(module, self.known_args().__dict__[module]) if hasattr(cls, 'require_args'): cls.require_args() else: cls_list = self.get_class(module) for cls in cls_list: if hasattr(cls, 'require_args'): cls.require_args() # re-update default value for new args self.from_files(self.known_args().cfgs) # parse args _META_.ARGS = _META_.PARSER.parse_args() def known_args(self): args, _ = _META_.PARSER.parse_known_args() return args def from_files(self, files): # if no config file is provided, skip if files is None or len(files) <= 0: return None for file in files: assert os.path.exists(file), "Config file not found: [%s]" % file configs = yaml.load(open(file, 'r')) _META_.PARSER.set_defaults(**configs) def get(self, attr, default=None): if hasattr(_META_.ARGS, attr): return getattr(_META_.ARGS, attr) return default def set(self, key, val): setattr(_META_.ARGS, key, val) def yaml(self): config = {k: v for k, v in sorted(vars(_META_.ARGS).items())} return yaml.safe_dump(config, default_flow_style=False) def __getattr__(self, attr): try: return _META_.PARSER.__getattribute__(attr) except AttributeError: return _META_.ARGS.__getattribute__(attr) except: traceback.print_exec() exit(-1) def __str__(self): MAX_WIDTH = 20 table = PrettyTable(["#", "Key", "Value", "Default"]) table.align = 'l' for i, (k, v) in enumerate(sorted(vars(_META_.ARGS).items())): v = str(v) default = str(_META_.PARSER.get_default(k)) if default == v: default = '--' table.add_row([i, k, v[:MAX_WIDTH] + ('...' if len(v) > MAX_WIDTH else ''), default]) return table.get_string()
def CocoExistence(paths=None, train=True, split='val', transform=None, target_transform=None, loader=default_loader): '''MSCOCO Dataset, but treat as multi-label problem to classify existence of classes. !!! WARNING: this dataset labels may contain -100 as ignored labels, which pytorch's binary_cross_entropy_with_logits cannot handle.''' paths = paths if paths is not None else default_paths # deal with splits assert split in {'val', 'test'} if split == 'val': phase = 'train2014' if train else 'val2014' annFile = 'instances_%s.json' % phase else: phase = 'trainval' if train else 'test2015' annFile = 'image_info_%s.json' % phase if phase == 'trainval': ret_ = [ CocoExistence(paths, True, 'val', transform, target_transform), CocoExistence(paths, False, 'val', transform, target_transform), ] ret = ret_[0] # Do not use data.ConcatDataset(ret_) ret.extend_imgs(ret_[1]) return ret root = os.path.join(paths.Cocoroot, phase) def __cached_fn(annFile, root): # get dataset object annFile = os.path.join(paths.Cocoroot, 'annotations', annFile) with MU.nostdout(on=True): ret = CocoDetection(root, annFile, transform, target_transform) coco = ret.coco # add label parsing clsCoco = sorted(coco.cats.keys()) ttf = coco_anno2existence(coco, clsCoco) ret.classes = ([ '{:s}:{:2d}'.format(coco.cats[x]['name'], x) for x in clsCoco ]) # also provide labels for convenience # This is redundant code. No idea how to use CocoDetection without loading image... ret.imgs = [(coco.loadImgs(x)[0]['file_name'], ttf(coco.loadAnns(coco.getAnnIds(imgIds=x)))) for x in sorted(ret.ids)] return dict(imgs=ret.imgs, classes=ret.classes) cache_file = os.path.join( paths.Cocoroot, '__cache_{split}split_{train}__.pkl'.format( split=split, train='train' if train else 'eval')) ret = ezdict(MU.cached(cache_file, __cached_fn, annFile, root)) # cast as ImagePathsDataset for uniform interfacing ret = ImagePathsDataset(root, ret.imgs, classes=ret.classes, class_to_idx=None, transform=transform, target_transform=target_transform, loader=loader) return ret
lookup = ezdict( # Number of class num_outputs=dict( lfwp_gender=2, cat_vs_dog=2, imnet_animal=4, voc_to_coco=20, ), # Release model files release_model_file=dict( lfwp_gender='release_models/lfwp_gender/net.pkl', cat_vs_dog='release_models/cat_vs_dog/net.pkl', imnet_animal='release_models/imnet_animal/net.pkl', voc_to_coco='release_models/voc_to_coco/net.pkl', ), # Numerically-safe probability computation class # MC for softmax logit in multi-class, ML for sigmoid logit in multi-label SafeProbClass=dict( lfwp_gender=SafeProbsMC, cat_vs_dog=SafeProbsMC, imnet_animal=SafeProbsMC, voc_to_coco=SafeProbsML, ), # Dataset class / function handles dataset=dict( lfwp_gender=mydatasets.LFWpGenderFamUnfDataset, cat_vs_dog=mydatasets.PetsFamUnfDataset, imnet_animal=mydatasets.ImageNetFamUnfDataset, voc_to_coco=mydatasets.VOCCocoFamUnfDataset, ), # Calibration temperature for baseline ResNet18 models # (obtained on a familiar data validation split) calibrate_T_baseline_resnet18=dict( lfwp_gender=1.60, cat_vs_dog=1.90, imnet_animal=1.68, voc_to_coco=1.02, ))
def load_config(config_path): with open(config_path, 'r', encoding='utf-8') as file: config = ezdict(yaml.load(file, Loader=yaml.SafeLoader)) return config
lookup = ezdict( resnet18=dict( lfwp_gender=dict( # Number of class num_outputs=2, # Dataset class / function handles DS='LFWpGenderFamUnfDataset', # learning rate lr=0.0080, # num of epochs max_epochs=24, # batch size batch_size=128, # label style label_style='MC', ), cat_vs_dog=dict( num_outputs=2, DS='PetsFamUnfDataset', lr=0.0600, max_epochs=128, batch_size=128, label_style='MC', ), imnet_animal=dict( num_outputs=4, DS='ImageNetFamUnfDataset', lr=0.0030, max_epochs=64, batch_size=128, label_style='MC', ), voc_to_coco=dict( num_outputs=20, DS='VOCCocoFamUnfDataset', lr=0.0300, max_epochs=32, batch_size=128, label_style='ML', ), ), densenet161=dict(lfwp_gender=dict( num_outputs=2, DS='LFWpGenderFamUnfDataset', lr=0.0040, max_epochs=96, batch_size=32, label_style='MC', ), ), )