def evaluate(self, scores, keyword, logger=None, topk=(1, 5)):
'''results: Tensor (NxC)
'''
eval_res = {}
target = torch.LongTensor(self.data_source.labels)
assert scores.size(0) == target.size(0), \
"Inconsistent length for results and labels, {} vs {}".format(
scores.size(0), target.size(0))
num = scores.size(0)
_, pred = scores.topk(max(topk), dim=1, largest=True, sorted=True)
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred)) # KxN
for k in topk:
correct_k = correct[:k].view(-1).float().sum(0).item()
acc = correct_k * 100.0 / num
eval_res["{}_top{}".format(keyword, k)] = acc
if logger is not None and logger != 'silent':
print_log("{}_top{}: {:.03f}".format(keyword, k, acc),
logger=logger)
target_names = self.data_source.classes
print(cls_report(target, pred.squeeze(), target_names=target_names))
return eval_res
def evaluate(self, scores, keyword, logger=None, topk=(1, 5)):
eval_res = {}
target = torch.LongTensor(self.data_source.labels)
assert scores.size(0) == target.size(0), \
"Inconsistent length for results and labels, {} vs {}".format(
scores.size(0), target.size(0))
num = scores.size(0)
_, pred = scores.topk(max(topk), dim=1, largest=True, sorted=True)
####################################################################
_pred = torch.squeeze(pred, 1)
b_acc = balanced_accuracy_score(target, _pred) * 100.0
#######################################################################
pred = pred.t()
correct = pred.eq(target.view(1, -1).expand_as(pred)) # KxN
for k in topk:
correct_k = correct[:k].contiguous().view(-1).float().sum(0).item()
acc = correct_k * 100.0 / num
eval_res["{}_top{}".format(keyword, k)] = acc
# if logger is not None and logger != 'silent':
# print_log(
# "{}_top{}: {:.03f}".format(keyword, k, acc),
# logger=logger)
########################################
if logger is not None and logger != 'silent':
print_log("balanced_acc: {:.03f}".format(b_acc), logger=logger)
#######################################
return eval_res
def init_weights(self, pretrained=None):
if pretrained is not None:
print_log('load model from: {}'.format(pretrained), logger='root')
self.backbone.init_weights(pretrained=pretrained)
if hasattr(self, 'neck'):
self.neck.init_weights()
self.head.init_weights()
def init_weights(self, pretrained=None):
if pretrained is not None:
print_log('load model from: {}'.format(pretrained), logger='root')
self.encoder_q[0].init_weights(pretrained=pretrained)
self.encoder_q[1].init_weights(init_linear='kaiming')
for param_q, param_k in zip(self.encoder_q.parameters(),
self.encoder_k.parameters()):
param_k.data.copy_(param_q.data)
def evaluate(self, scores, keyword, logger=None, **kwargs):
#raise NotImplemented
eval_res = {"val_{}".format(keyword): scores}
if logger is not None and logger != 'silent':
print_log("val_{}: {:.03f}".format(keyword, scores), logger=logger)
return eval_res
def _dist_train(model, dataset, cfg, logger=None, timestamp=None, meta=None):
# prepare data loaders
dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset]
data_loaders = [
build_dataloader(ds,
cfg.data.imgs_per_gpu,
cfg.data.workers_per_gpu,
dist=True,
shuffle=True,
replace=getattr(cfg.data, 'sampling_replace', False),
seed=cfg.seed,
drop_last=getattr(cfg.data, 'drop_last', False),
prefetch=cfg.prefetch,
img_norm_cfg=cfg.img_norm_cfg) for ds in dataset
]
optimizer = build_optimizer(model, cfg.optimizer)
if 'use_fp16' in cfg and cfg.use_fp16:
model, optimizer = apex.amp.initialize(model.cuda(),
optimizer,
opt_level="O1")
print_log('**** Initializing mixed precision done. ****')
# put model on gpus
model = MMDistributedDataParallel(
model if next(model.parameters()).is_cuda else model.cuda(),
device_ids=[torch.cuda.current_device()],
broadcast_buffers=False)
# build runner
runner = Runner(model,
batch_processor,
optimizer,
cfg.work_dir,
logger=logger,
meta=meta)
# an ugly walkaround to make the .log and .log.json filenames the same
runner.timestamp = timestamp
optimizer_config = DistOptimizerHook(**cfg.optimizer_config)
# register hooks
runner.register_training_hooks(cfg.lr_config, optimizer_config,
cfg.checkpoint_config, cfg.log_config)
runner.register_hook(DistSamplerSeedHook())
# register custom hooks
for hook in cfg.get('custom_hooks', ()):
if hook.type == 'DeepClusterHook':
common_params = dict(dist_mode=True, data_loaders=data_loaders)
else:
common_params = dict(dist_mode=True)
runner.register_hook(build_hook(hook, common_params))
if cfg.resume_from:
runner.resume(cfg.resume_from)
elif cfg.load_from:
runner.load_checkpoint(cfg.load_from)
runner.run(data_loaders, cfg.workflow, cfg.total_epochs)
def evaluate(self, runner, new_labels):
hist = np.bincount(new_labels, minlength=self.clustering_cfg.k)
empty_cls = (hist == 0).sum()
minimal_cls_size, maximal_cls_size = hist.min(), hist.max()
if runner.rank == 0:
print_log("empty_num: {}\tmin_cluster: {}\tmax_cluster:{}".format(
empty_cls.item(), minimal_cls_size.item(),
maximal_cls_size.item()),
logger='root')
def init_weights(self, pretrained=None):
if pretrained is not None:
print_log(f'load model from: {pretrained}', logger='root')
self.online_net[0].init_weights(pretrained=pretrained) # backbone
self.online_net[1].init_weights(init_linear='kaiming') # projection
for param_ol, param_tgt in zip(self.online_net.parameters(),
self.target_net.parameters()):
param_tgt.data.copy_(param_ol.data)
# init the predictor in the head
self.head.init_weights()
def init_weights(self, pretrained=None):
"""Initialize the weights of model.
Args:
pretrained (str, optional): Path to pre-trained weights.
Default: None.
"""
if pretrained is not None:
print_log("load model from: {}".format(pretrained), logger="root")
self.backbone.init_weights(pretrained=pretrained)
self.neck.init_weights(init_linear="kaiming")
def init_weights(self, pretrained=None):
"""Initialize the weights of model.
Args:
pretrained (str, optional): Path to pre-trained weights.
Default: None.
"""
if pretrained is not None:
print_log('load model from: {}'.format(pretrained), logger='root')
self.backbone.init_weights(pretrained=pretrained)
self.head.init_weights()
def evaluate(self, scores, keyword, logger=None):
eval_res = {}
target = torch.cat(self.data_source.labels).reshape(scores.shape)
aurocs = auroc(scores, target).cpu()
for ii in range(len(aurocs)):
res = aurocs[ii].item()
eval_res["auroc_{}".format(ii)] = res
if logger is not None and logger != 'silent':
print_log("auroc_{}: {:.03f}".format(ii, res), logger=logger)
return eval_res
def init_weights(self, pretrained=None):
"""Initialize the weights of model.
Args:
pretrained (str, optional): Path to pre-trained weights.
Default: None.
"""
if pretrained is not None:
print_log('load model from: {}'.format(pretrained), logger='root')
self.encoder_q[0].init_weights(pretrained=pretrained)
self.encoder_q[1].init_weights(init_linear='kaiming')
for param_q, param_k in zip(self.encoder_q.parameters(),
self.encoder_k.parameters()):
param_k.data.copy_(param_q.data)
def init_weights(self, pretrained=None):
"""Initialize the weights.
Args:
pretrained (str, optional): Path to pre-trained weights.
Default: None.
"""
if pretrained is not None:
# logger = get_root_logger()
# print_log(f'load model from: {pretrained}', logger=logger)
print_log('load model from: {}'.format(pretrained), logger='root')
self.backbone.init_weights(pretrained=pretrained)
if self.neck is not None:
self.neck.init_weights(init_linear='kaiming')
def init_weights(self, pretrained=None):
if pretrained is not None:
print_log('load model from: {}'.format(pretrained), logger='root')
self.backbone_q.init_weights(pretrained=pretrained)
self.backbone_k.init_weights(pretrained=pretrained)
for neck in self.encoder_q_necks:
neck.init_weights(init_linear='kaiming')
for neck in self.encoder_k_necks:
neck.init_weights(init_linear='kaiming')
for neck in self.encoder_q_patch_necks:
neck.init_weights(init_linear='kaiming')
for neck in self.encoder_k_patch_necks:
neck.init_weights(init_linear='kaiming')
"""for param_q, param_k in zip(self.encoder_q.parameters(),
def init_weights(self, pretrained=None):
"""Initialize the weights of model.
Args:
pretrained (str, optional): Path to pre-trained weights.
Default: None.
"""
if pretrained is not None:
print_log('load model from: {}'.format(pretrained), logger='root')
self.online_net[0].init_weights(pretrained=pretrained) # backbone
self.online_net[1].init_weights(init_linear='kaiming') # projection
for param_ol, param_tgt in zip(self.online_net.parameters(),
self.target_net.parameters()):
param_tgt.data.copy_(param_ol.data)
# init the predictor in the head
self.head.init_weights()
def after_train_epoch(self, runner):
model = runner.model.module if hasattr(runner.model,
'module') else runner.model
block_inteval = self.interval[model.start_block] if isinstance(
self.interval, list) else self.interval
if not self.every_n_epochs(runner, block_inteval):
return
if len(self.loaded_best_path
) - 1 >= model.start_block: # use loaded best path
model.best_paths = self.loaded_best_path[:model.start_block + 1]
print_log('loaded best paths: {}'.format(model.best_paths),
logger='root')
else:
self._run_validate(runner)
print_log('searched best paths: {}'.format(model.best_paths),
logger='root')
print_log('best paths from all workers:')
print(model.best_paths)
torch.cuda.synchronize()
# save best path
if runner.rank == 0:
output_dir = os.path.join(runner.work_dir, 'path_rank')
if not os.path.exists(output_dir):
os.mkdir(output_dir)
time_str = datetime.datetime.strftime(datetime.datetime.now(),
'%m-%d-%H')
filename = os.path.join(
output_dir, "bestpath_" + str(model.start_block) + "_" +
str(runner.epoch) + "_" + str(time_str) + ".yml")
with open(filename, 'w', encoding='utf8') as f:
yaml.dump(model.best_paths, f)
# initialize for next stage
block_inteval = self.epoch_per_stage[model.start_block] if isinstance(
self.epoch_per_stage, list) else self.epoch_per_stage
if self.every_n_epochs(
runner,
block_inteval) and model.start_block < model.num_block - 1:
model.start_block += 1
forward_index = model.best_paths[-1][-1]
if forward_index < 4:
pos = forward_index // 2
else:
pos = forward_index - 2
model.target_backbone.stage_depths[model.start_block] = pos + 1
model.online_backbone.stage_depths[model.start_block] = pos + 1
model.set_current_neck_and_head()
del model.optimizer
del runner.optimizer
new_optimizer = build_optimizer(model, self.optimizer_cfg)
if model.use_fp16:
model, new_optimizer = apex.amp.initialize(model,
new_optimizer,
opt_level="O1")
print_log('**** Initializing mixed precision done. ****')
runner.optimizer = new_optimizer
model.optimizer = new_optimizer
def evaluate(self, scores, keyword, logger=None, topk=(1, 5)):
eval_res = {}
target = torch.LongTensor(self.data_source.labels)
assert scores.size(0) == target.size(0), \
"Inconsistent length for results and labels, {} vs {}".format(
scores.size(0), target.size(0))
num = scores.size(0)
_, pred = scores.topk(max(topk), dim=1, largest=True, sorted=True)
pred = torch.squeeze(pred, 1)
target_names = ['Others', 'Melanoma']
results = (classification_report(target,
pred,
target_names=target_names,
digits=4))
b_acc = balanced_accuracy_score(target, pred)
log_str = "\n" + results
if logger is not None and logger != 'silent':
print_log(log_str +
"\nBalanced Accuracy Score: {:.03f}".format(b_acc),
logger=logger)
eval_res = results
return eval_res
def train_model(model,
dataset,
cfg,
distributed=False,
timestamp=None,
meta=None,
debug=False):
logger = get_root_logger(cfg.log_level)
# multiply the number of iters if we have update intervals set
if (cfg.get('by_iter', False) and cfg.get('optimizer_config', False) and cfg.optimizer_config.get('update_interval', 1) > 1):
print_log("Iter based runner with update_interval set. Multiplying iters by update_interval")
print_log(f"Before iters: {cfg.total_iters}")
cfg.total_iters = cfg.optimizer_config.update_interval * cfg.total_iters
print_log(f"After iters: {cfg.total_iters}")
# start training
if distributed and not debug:
_dist_train(
model, dataset, cfg, logger=logger, timestamp=timestamp, meta=meta)
else:
_non_dist_train(
model, dataset, cfg, logger=logger, timestamp=timestamp, meta=meta)
def init_weights(self, pretrained=None):
if pretrained is not None:
print_log(f'load model from: {pretrained}', logger='root')
self.backbone.init_weights(pretrained=pretrained)
self.head.init_weights()
def build_optimizer(model, optimizer_cfg):
"""Build optimizer from configs.
Args:
model (:obj:`nn.Module`): The model with parameters to be optimized.
optimizer_cfg (dict): The config dict of the optimizer.
Positional fields are:
- type: class name of the optimizer.
- lr: base learning rate.
Optional fields are:
- any arguments of the corresponding optimizer type, e.g.,
weight_decay, momentum, etc.
- paramwise_options: a dict with regular expression as keys
to match parameter names and a dict containing options as
values. Options include 6 fields: lr, lr_mult, momentum,
momentum_mult, weight_decay, weight_decay_mult.
Returns:
torch.optim.Optimizer: The initialized optimizer.
Example:
>>> model = torch.nn.modules.Conv1d(1, 1, 1)
>>> paramwise_options = {
>>> '(bn|gn)(\d+)?.(weight|bias)': dict(weight_decay_mult=0.1),
>>> '\Ahead.': dict(lr_mult=10, momentum=0)}
>>> optimizer_cfg = dict(type='SGD', lr=0.01, momentum=0.9,
>>> weight_decay=0.0001,
>>> paramwise_options=paramwise_options)
>>> optimizer = build_optimizer(model, optimizer_cfg)
"""
if hasattr(model, 'module'):
model = model.module
optimizer_cfg = optimizer_cfg.copy()
paramwise_options = optimizer_cfg.pop('paramwise_options', None)
# if no paramwise option is specified, just use the global setting
if paramwise_options is None:
return obj_from_dict(optimizer_cfg, optimizers,
dict(params=model.parameters()))
else:
assert isinstance(paramwise_options, dict)
params = []
for name, param in model.named_parameters():
param_group = {'params': [param]}
if not param.requires_grad:
params.append(param_group)
continue
for regexp, options in paramwise_options.items():
if re.search(regexp, name):
for key, value in options.items():
if key.endswith('_mult'): # is a multiplier
key = key[:-5]
assert key in optimizer_cfg, \
"{} not in optimizer_cfg".format(key)
value = optimizer_cfg[key] * value
param_group[key] = value
if not dist.is_initialized() or dist.get_rank() == 0:
print_log('paramwise_options -- {}: {}={}'.format(
name, key, value))
# otherwise use the global settings
params.append(param_group)
optimizer_cls = getattr(optimizers, optimizer_cfg.pop('type'))
return optimizer_cls(params, **optimizer_cfg)
def init_weights(self, pretrained=None):
if pretrained is not None:
print_log('load model from: {}'.format(pretrained), logger='root')
self.backbone.init_weights(pretrained=pretrained)
self.head.init_weights(init_linear='kaiming')
def _non_dist_train(model,
dataset,
cfg,
validate=False,
logger=None,
timestamp=None,
meta=None):
# use batch size instead of per-gpu batch size
if getattr(cfg.data, 'batch_size', False):
cfg.data.imgs_per_gpu = int(cfg.data.batch_size)
print_log(
f"Using {cfg.data.imgs_per_gpu} per gpu for batch size {cfg.data.batch_size}")
# prepare data loaders
dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset]
data_loaders = [
build_dataloader(
ds,
cfg.data.imgs_per_gpu,
cfg.data.workers_per_gpu,
cfg.gpus,
dist=False,
shuffle=True,
replace=getattr(cfg.data, 'sampling_replace', False),
seed=cfg.seed,
drop_last=getattr(cfg.data, 'drop_last', False),
prefetch=cfg.prefetch,
img_norm_cfg=cfg.img_norm_cfg) for ds in dataset
]
if 'use_fp16' in cfg and cfg.use_fp16 == True:
raise NotImplementedError('apex do not support non_dist_train!')
# put model on gpus
model = MMDataParallel(model, device_ids=range(cfg.gpus)).cuda()
# build runner
optimizer = build_optimizer(model, cfg.optimizer)
optimizer_config = NonDistOptimizerHook(**cfg.optimizer_config)
if not cfg.get('by_iter', False):
runner = Runner(
model,
optimizer=optimizer,
work_dir=cfg.work_dir,
logger=logger,
meta=meta)
else:
runner = IterBasedRunner(
model,
optimizer=optimizer,
work_dir=cfg.work_dir,
logger=logger,
meta=meta)
# an ugly walkaround to make the .log and .log.json filenames the same
runner.timestamp = timestamp
runner.register_training_hooks(cfg.lr_config, optimizer_config,
cfg.checkpoint_config, cfg.log_config)
# register custom hooks
for hook in cfg.get('custom_hooks', ()):
if hook.type == 'DeepClusterHook':
common_params = dict(dist_mode=False, data_loaders=data_loaders)
else:
common_params = dict(dist_mode=False)
runner.register_hook(build_hook(hook, common_params))
if cfg.resume_from:
runner.resume(cfg.resume_from)
elif cfg.load_from:
runner.load_checkpoint(cfg.load_from)
if not cfg.get('by_iter', False):
runner.run(data_loaders, cfg.workflow, cfg.total_epochs)
else:
runner.run(data_loaders, cfg.workflow, cfg.total_iters)
def init_weights(self, pretrained=None):
if pretrained is not None:
print_log(f'load model from: {pretrained}', logger='root')
self.backbone.init_weights(pretrained=pretrained)
self.neck.init_weights(init_linear='kaiming')
self.head.init_weights(init_linear='normal')
