def evaluate(self, scores, keyword, logger=None, topk=(1, 5)): '''results: Tensor (NxC) ''' eval_res = {} target = torch.LongTensor(self.data_source.labels) assert scores.size(0) == target.size(0), \ "Inconsistent length for results and labels, {} vs {}".format( scores.size(0), target.size(0)) num = scores.size(0) _, pred = scores.topk(max(topk), dim=1, largest=True, sorted=True) pred = pred.t() correct = pred.eq(target.view(1, -1).expand_as(pred)) # KxN for k in topk: correct_k = correct[:k].view(-1).float().sum(0).item() acc = correct_k * 100.0 / num eval_res["{}_top{}".format(keyword, k)] = acc if logger is not None and logger != 'silent': print_log("{}_top{}: {:.03f}".format(keyword, k, acc), logger=logger) target_names = self.data_source.classes print(cls_report(target, pred.squeeze(), target_names=target_names)) return eval_res
def evaluate(self, scores, keyword, logger=None, topk=(1, 5)): eval_res = {} target = torch.LongTensor(self.data_source.labels) assert scores.size(0) == target.size(0), \ "Inconsistent length for results and labels, {} vs {}".format( scores.size(0), target.size(0)) num = scores.size(0) _, pred = scores.topk(max(topk), dim=1, largest=True, sorted=True) #################################################################### _pred = torch.squeeze(pred, 1) b_acc = balanced_accuracy_score(target, _pred) * 100.0 ####################################################################### pred = pred.t() correct = pred.eq(target.view(1, -1).expand_as(pred)) # KxN for k in topk: correct_k = correct[:k].contiguous().view(-1).float().sum(0).item() acc = correct_k * 100.0 / num eval_res["{}_top{}".format(keyword, k)] = acc # if logger is not None and logger != 'silent': # print_log( # "{}_top{}: {:.03f}".format(keyword, k, acc), # logger=logger) ######################################## if logger is not None and logger != 'silent': print_log("balanced_acc: {:.03f}".format(b_acc), logger=logger) ####################################### return eval_res
def init_weights(self, pretrained=None): if pretrained is not None: print_log('load model from: {}'.format(pretrained), logger='root') self.backbone.init_weights(pretrained=pretrained) if hasattr(self, 'neck'): self.neck.init_weights() self.head.init_weights()
def init_weights(self, pretrained=None): if pretrained is not None: print_log('load model from: {}'.format(pretrained), logger='root') self.encoder_q[0].init_weights(pretrained=pretrained) self.encoder_q[1].init_weights(init_linear='kaiming') for param_q, param_k in zip(self.encoder_q.parameters(), self.encoder_k.parameters()): param_k.data.copy_(param_q.data)
def evaluate(self, scores, keyword, logger=None, **kwargs): #raise NotImplemented eval_res = {"val_{}".format(keyword): scores} if logger is not None and logger != 'silent': print_log("val_{}: {:.03f}".format(keyword, scores), logger=logger) return eval_res
def _dist_train(model, dataset, cfg, logger=None, timestamp=None, meta=None): # prepare data loaders dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset] data_loaders = [ build_dataloader(ds, cfg.data.imgs_per_gpu, cfg.data.workers_per_gpu, dist=True, shuffle=True, replace=getattr(cfg.data, 'sampling_replace', False), seed=cfg.seed, drop_last=getattr(cfg.data, 'drop_last', False), prefetch=cfg.prefetch, img_norm_cfg=cfg.img_norm_cfg) for ds in dataset ] optimizer = build_optimizer(model, cfg.optimizer) if 'use_fp16' in cfg and cfg.use_fp16: model, optimizer = apex.amp.initialize(model.cuda(), optimizer, opt_level="O1") print_log('**** Initializing mixed precision done. ****') # put model on gpus model = MMDistributedDataParallel( model if next(model.parameters()).is_cuda else model.cuda(), device_ids=[torch.cuda.current_device()], broadcast_buffers=False) # build runner runner = Runner(model, batch_processor, optimizer, cfg.work_dir, logger=logger, meta=meta) # an ugly walkaround to make the .log and .log.json filenames the same runner.timestamp = timestamp optimizer_config = DistOptimizerHook(**cfg.optimizer_config) # register hooks runner.register_training_hooks(cfg.lr_config, optimizer_config, cfg.checkpoint_config, cfg.log_config) runner.register_hook(DistSamplerSeedHook()) # register custom hooks for hook in cfg.get('custom_hooks', ()): if hook.type == 'DeepClusterHook': common_params = dict(dist_mode=True, data_loaders=data_loaders) else: common_params = dict(dist_mode=True) runner.register_hook(build_hook(hook, common_params)) if cfg.resume_from: runner.resume(cfg.resume_from) elif cfg.load_from: runner.load_checkpoint(cfg.load_from) runner.run(data_loaders, cfg.workflow, cfg.total_epochs)
def evaluate(self, runner, new_labels): hist = np.bincount(new_labels, minlength=self.clustering_cfg.k) empty_cls = (hist == 0).sum() minimal_cls_size, maximal_cls_size = hist.min(), hist.max() if runner.rank == 0: print_log("empty_num: {}\tmin_cluster: {}\tmax_cluster:{}".format( empty_cls.item(), minimal_cls_size.item(), maximal_cls_size.item()), logger='root')
def init_weights(self, pretrained=None): if pretrained is not None: print_log(f'load model from: {pretrained}', logger='root') self.online_net[0].init_weights(pretrained=pretrained) # backbone self.online_net[1].init_weights(init_linear='kaiming') # projection for param_ol, param_tgt in zip(self.online_net.parameters(), self.target_net.parameters()): param_tgt.data.copy_(param_ol.data) # init the predictor in the head self.head.init_weights()
def init_weights(self, pretrained=None): """Initialize the weights of model. Args: pretrained (str, optional): Path to pre-trained weights. Default: None. """ if pretrained is not None: print_log("load model from: {}".format(pretrained), logger="root") self.backbone.init_weights(pretrained=pretrained) self.neck.init_weights(init_linear="kaiming")
def init_weights(self, pretrained=None): """Initialize the weights of model. Args: pretrained (str, optional): Path to pre-trained weights. Default: None. """ if pretrained is not None: print_log('load model from: {}'.format(pretrained), logger='root') self.backbone.init_weights(pretrained=pretrained) self.head.init_weights()
def evaluate(self, scores, keyword, logger=None): eval_res = {} target = torch.cat(self.data_source.labels).reshape(scores.shape) aurocs = auroc(scores, target).cpu() for ii in range(len(aurocs)): res = aurocs[ii].item() eval_res["auroc_{}".format(ii)] = res if logger is not None and logger != 'silent': print_log("auroc_{}: {:.03f}".format(ii, res), logger=logger) return eval_res
def init_weights(self, pretrained=None): """Initialize the weights of model. Args: pretrained (str, optional): Path to pre-trained weights. Default: None. """ if pretrained is not None: print_log('load model from: {}'.format(pretrained), logger='root') self.encoder_q[0].init_weights(pretrained=pretrained) self.encoder_q[1].init_weights(init_linear='kaiming') for param_q, param_k in zip(self.encoder_q.parameters(), self.encoder_k.parameters()): param_k.data.copy_(param_q.data)
def init_weights(self, pretrained=None): """Initialize the weights. Args: pretrained (str, optional): Path to pre-trained weights. Default: None. """ if pretrained is not None: # logger = get_root_logger() # print_log(f'load model from: {pretrained}', logger=logger) print_log('load model from: {}'.format(pretrained), logger='root') self.backbone.init_weights(pretrained=pretrained) if self.neck is not None: self.neck.init_weights(init_linear='kaiming')
def init_weights(self, pretrained=None): if pretrained is not None: print_log('load model from: {}'.format(pretrained), logger='root') self.backbone_q.init_weights(pretrained=pretrained) self.backbone_k.init_weights(pretrained=pretrained) for neck in self.encoder_q_necks: neck.init_weights(init_linear='kaiming') for neck in self.encoder_k_necks: neck.init_weights(init_linear='kaiming') for neck in self.encoder_q_patch_necks: neck.init_weights(init_linear='kaiming') for neck in self.encoder_k_patch_necks: neck.init_weights(init_linear='kaiming') """for param_q, param_k in zip(self.encoder_q.parameters(),
def init_weights(self, pretrained=None): """Initialize the weights of model. Args: pretrained (str, optional): Path to pre-trained weights. Default: None. """ if pretrained is not None: print_log('load model from: {}'.format(pretrained), logger='root') self.online_net[0].init_weights(pretrained=pretrained) # backbone self.online_net[1].init_weights(init_linear='kaiming') # projection for param_ol, param_tgt in zip(self.online_net.parameters(), self.target_net.parameters()): param_tgt.data.copy_(param_ol.data) # init the predictor in the head self.head.init_weights()
def after_train_epoch(self, runner): model = runner.model.module if hasattr(runner.model, 'module') else runner.model block_inteval = self.interval[model.start_block] if isinstance( self.interval, list) else self.interval if not self.every_n_epochs(runner, block_inteval): return if len(self.loaded_best_path ) - 1 >= model.start_block: # use loaded best path model.best_paths = self.loaded_best_path[:model.start_block + 1] print_log('loaded best paths: {}'.format(model.best_paths), logger='root') else: self._run_validate(runner) print_log('searched best paths: {}'.format(model.best_paths), logger='root') print_log('best paths from all workers:') print(model.best_paths) torch.cuda.synchronize() # save best path if runner.rank == 0: output_dir = os.path.join(runner.work_dir, 'path_rank') if not os.path.exists(output_dir): os.mkdir(output_dir) time_str = datetime.datetime.strftime(datetime.datetime.now(), '%m-%d-%H') filename = os.path.join( output_dir, "bestpath_" + str(model.start_block) + "_" + str(runner.epoch) + "_" + str(time_str) + ".yml") with open(filename, 'w', encoding='utf8') as f: yaml.dump(model.best_paths, f) # initialize for next stage block_inteval = self.epoch_per_stage[model.start_block] if isinstance( self.epoch_per_stage, list) else self.epoch_per_stage if self.every_n_epochs( runner, block_inteval) and model.start_block < model.num_block - 1: model.start_block += 1 forward_index = model.best_paths[-1][-1] if forward_index < 4: pos = forward_index // 2 else: pos = forward_index - 2 model.target_backbone.stage_depths[model.start_block] = pos + 1 model.online_backbone.stage_depths[model.start_block] = pos + 1 model.set_current_neck_and_head() del model.optimizer del runner.optimizer new_optimizer = build_optimizer(model, self.optimizer_cfg) if model.use_fp16: model, new_optimizer = apex.amp.initialize(model, new_optimizer, opt_level="O1") print_log('**** Initializing mixed precision done. ****') runner.optimizer = new_optimizer model.optimizer = new_optimizer
def evaluate(self, scores, keyword, logger=None, topk=(1, 5)): eval_res = {} target = torch.LongTensor(self.data_source.labels) assert scores.size(0) == target.size(0), \ "Inconsistent length for results and labels, {} vs {}".format( scores.size(0), target.size(0)) num = scores.size(0) _, pred = scores.topk(max(topk), dim=1, largest=True, sorted=True) pred = torch.squeeze(pred, 1) target_names = ['Others', 'Melanoma'] results = (classification_report(target, pred, target_names=target_names, digits=4)) b_acc = balanced_accuracy_score(target, pred) log_str = "\n" + results if logger is not None and logger != 'silent': print_log(log_str + "\nBalanced Accuracy Score: {:.03f}".format(b_acc), logger=logger) eval_res = results return eval_res
def train_model(model, dataset, cfg, distributed=False, timestamp=None, meta=None, debug=False): logger = get_root_logger(cfg.log_level) # multiply the number of iters if we have update intervals set if (cfg.get('by_iter', False) and cfg.get('optimizer_config', False) and cfg.optimizer_config.get('update_interval', 1) > 1): print_log("Iter based runner with update_interval set. Multiplying iters by update_interval") print_log(f"Before iters: {cfg.total_iters}") cfg.total_iters = cfg.optimizer_config.update_interval * cfg.total_iters print_log(f"After iters: {cfg.total_iters}") # start training if distributed and not debug: _dist_train( model, dataset, cfg, logger=logger, timestamp=timestamp, meta=meta) else: _non_dist_train( model, dataset, cfg, logger=logger, timestamp=timestamp, meta=meta)
def init_weights(self, pretrained=None): if pretrained is not None: print_log(f'load model from: {pretrained}', logger='root') self.backbone.init_weights(pretrained=pretrained) self.head.init_weights()
def build_optimizer(model, optimizer_cfg): """Build optimizer from configs. Args: model (:obj:`nn.Module`): The model with parameters to be optimized. optimizer_cfg (dict): The config dict of the optimizer. Positional fields are: - type: class name of the optimizer. - lr: base learning rate. Optional fields are: - any arguments of the corresponding optimizer type, e.g., weight_decay, momentum, etc. - paramwise_options: a dict with regular expression as keys to match parameter names and a dict containing options as values. Options include 6 fields: lr, lr_mult, momentum, momentum_mult, weight_decay, weight_decay_mult. Returns: torch.optim.Optimizer: The initialized optimizer. Example: >>> model = torch.nn.modules.Conv1d(1, 1, 1) >>> paramwise_options = { >>> '(bn|gn)(\d+)?.(weight|bias)': dict(weight_decay_mult=0.1), >>> '\Ahead.': dict(lr_mult=10, momentum=0)} >>> optimizer_cfg = dict(type='SGD', lr=0.01, momentum=0.9, >>> weight_decay=0.0001, >>> paramwise_options=paramwise_options) >>> optimizer = build_optimizer(model, optimizer_cfg) """ if hasattr(model, 'module'): model = model.module optimizer_cfg = optimizer_cfg.copy() paramwise_options = optimizer_cfg.pop('paramwise_options', None) # if no paramwise option is specified, just use the global setting if paramwise_options is None: return obj_from_dict(optimizer_cfg, optimizers, dict(params=model.parameters())) else: assert isinstance(paramwise_options, dict) params = [] for name, param in model.named_parameters(): param_group = {'params': [param]} if not param.requires_grad: params.append(param_group) continue for regexp, options in paramwise_options.items(): if re.search(regexp, name): for key, value in options.items(): if key.endswith('_mult'): # is a multiplier key = key[:-5] assert key in optimizer_cfg, \ "{} not in optimizer_cfg".format(key) value = optimizer_cfg[key] * value param_group[key] = value if not dist.is_initialized() or dist.get_rank() == 0: print_log('paramwise_options -- {}: {}={}'.format( name, key, value)) # otherwise use the global settings params.append(param_group) optimizer_cls = getattr(optimizers, optimizer_cfg.pop('type')) return optimizer_cls(params, **optimizer_cfg)
def init_weights(self, pretrained=None): if pretrained is not None: print_log('load model from: {}'.format(pretrained), logger='root') self.backbone.init_weights(pretrained=pretrained) self.head.init_weights(init_linear='kaiming')
def _non_dist_train(model, dataset, cfg, validate=False, logger=None, timestamp=None, meta=None): # use batch size instead of per-gpu batch size if getattr(cfg.data, 'batch_size', False): cfg.data.imgs_per_gpu = int(cfg.data.batch_size) print_log( f"Using {cfg.data.imgs_per_gpu} per gpu for batch size {cfg.data.batch_size}") # prepare data loaders dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset] data_loaders = [ build_dataloader( ds, cfg.data.imgs_per_gpu, cfg.data.workers_per_gpu, cfg.gpus, dist=False, shuffle=True, replace=getattr(cfg.data, 'sampling_replace', False), seed=cfg.seed, drop_last=getattr(cfg.data, 'drop_last', False), prefetch=cfg.prefetch, img_norm_cfg=cfg.img_norm_cfg) for ds in dataset ] if 'use_fp16' in cfg and cfg.use_fp16 == True: raise NotImplementedError('apex do not support non_dist_train!') # put model on gpus model = MMDataParallel(model, device_ids=range(cfg.gpus)).cuda() # build runner optimizer = build_optimizer(model, cfg.optimizer) optimizer_config = NonDistOptimizerHook(**cfg.optimizer_config) if not cfg.get('by_iter', False): runner = Runner( model, optimizer=optimizer, work_dir=cfg.work_dir, logger=logger, meta=meta) else: runner = IterBasedRunner( model, optimizer=optimizer, work_dir=cfg.work_dir, logger=logger, meta=meta) # an ugly walkaround to make the .log and .log.json filenames the same runner.timestamp = timestamp runner.register_training_hooks(cfg.lr_config, optimizer_config, cfg.checkpoint_config, cfg.log_config) # register custom hooks for hook in cfg.get('custom_hooks', ()): if hook.type == 'DeepClusterHook': common_params = dict(dist_mode=False, data_loaders=data_loaders) else: common_params = dict(dist_mode=False) runner.register_hook(build_hook(hook, common_params)) if cfg.resume_from: runner.resume(cfg.resume_from) elif cfg.load_from: runner.load_checkpoint(cfg.load_from) if not cfg.get('by_iter', False): runner.run(data_loaders, cfg.workflow, cfg.total_epochs) else: runner.run(data_loaders, cfg.workflow, cfg.total_iters)
def init_weights(self, pretrained=None): if pretrained is not None: print_log(f'load model from: {pretrained}', logger='root') self.backbone.init_weights(pretrained=pretrained) self.neck.init_weights(init_linear='kaiming') self.head.init_weights(init_linear='normal')