def _initialize_model(
self,
config_path: str,
model_path: Optional[str],
label_map: Optional[Dict],
extra_config: Optional[Dict],
):
config_path, model_path = self.config_parser(config_path, model_path)
if config_path.startswith("lp://"):
# If it's officially supported by layoutparser
dataset_name, model_name = config_path.lstrip("lp://").split("/")[1:3]
if label_map is None:
label_map = LABEL_MAP_CATALOG[dataset_name]
num_classes = len(label_map)
model_path = PathManager.get_local_path(model_path)
self.model = create_model(
model_name,
num_classes=num_classes,
bench_task="predict",
pretrained=True,
checkpoint_path=model_path,
)
else:
assert (
model_path is not None
), f"When the specified model is not layoutparser-based, you need to specify the model_path"
assert (
label_map is not None or "num_classes" in extra_config
), "When the specified model is not layoutparser-based, you need to specify the label_map or add num_classes in the extra_config"
model_name = config_path
model_path = PathManager.get_local_path(
model_path
) # It might be an https URL
num_classes = len(label_map) if label_map else extra_config["num_classes"]
self.model = create_model(
model_name,
num_classes=num_classes,
bench_task="predict",
pretrained=True,
checkpoint_path=model_path,
)
self.model.to(self.device)
self.model.eval()
self.config = self.model.config
self.label_map = label_map if label_map is not None else {}
def export(args):
# creat output dir
Path(args.out_file).parent.mkdir(parents=True, exist_ok=True)
# create model
bench = create_model(args.model,
bench_task='',
checkpoint_path=args.checkpoint)
bench.eval()
# make dummy run (really required??)
dummy_input = torch.randn([1] + [3, 512, 512])
bench(dummy_input)
# Export ONNX file
input_names = [
"input:0"
] # this are our standardized in/out nameing (required for runtime)
output_names = ["output:0", "output:1"]
print("Exporting ONNX with input resolution of {} to '{}'".format(
args.ONNX_resolution, args.out_file))
torch.onnx._export(bench,
dummy_input,
args.out_file,
opset_version=11,
keep_initializers_as_inputs=True,
output_names=output_names)
#torch.onnx._export(bench, dummy_input, args.out_file, keep_initializers_as_inputs=True, output_names=output_names)
print("Saved to {}".format(args.out_file))
def __init__(self,
backbone,
neck=None,
bbox_head=None,
mask_feat_head=None,
train_cfg=None,
test_cfg=None,
pretrained=None):
super(SingleStageInsDetector, self).__init__()
self.eff_backbone_flag = False if 'efficient' not in backbone[
'type'] else True
if self.eff_backbone_flag == False:
self.backbone = build_backbone(backbone)
else:
'''
self.backbone = effdet.create_model(args.model,
bench_task='train',
num_classes=args.num_classes,
pretrained=args.pretrained,
pretrained_backbone=args.pretrained_backbone,
redundant_bias=args.redundant_bias,
label_smoothing=args.smoothing,
legacy_focal=args.legacy_focal,
jit_loss=args.jit_loss,
soft_nms=args.soft_nms,
bench_labeler=args.bench_labeler,
checkpoint_path=args.initial_checkpoint,
)
'''
self.backbone = effdet.create_model(
'tf_efficientdet_d1',
bench_task='train',
num_classes=num_classes,
pretrained=pretrained,
pretrained_backbone=pretrained_backbone,
redundant_bias=redundant_bias,
label_smoothing=smoothing,
legacy_focal=legacy_focal,
jit_loss=jit_loss,
soft_nms=soft_nms,
bench_labeler=bench_labeler,
checkpoint_path=initial_checkpoint,
)
self.with_neck = False
self.with_neck = False
if neck is not None:
self.neck = build_neck(neck)
self.with_neck = True
if mask_feat_head is not None:
self.mask_feat_head = build_head(mask_feat_head)
self.with_mask_feat_head = True
self.bbox_head = build_head(bbox_head)
self.train_cfg = train_cfg
self.test_cfg = test_cfg
self.init_weights(pretrained=pretrained)
def get_model(self):
model = create_model(
model_name="tf_efficientdet_d0",
bench_task="train",
num_classes=1,
pretrained=False,
checkpoint_path="",
checkpoint_ema=False,
bench_labeler=True,
)
return model
def set_model(model_type, num_classes, checkpoint_path, device):
# create model
model = create_model(model_type,
bench_task='predict',
num_classes=num_classes,
pretrained=False,
redundant_bias=True,
checkpoint_path=checkpoint_path)
param_count = sum([m.numel() for m in model.parameters()])
print('Model %s created, param count: %d' % (model_type, param_count))
model = model.to(device)
return model
def __init__(
self,
num_classes: int,
effdet_backbone: str = "tf_efficientdet_d4",
strides: List[int] = [8, 16, 32, 64, 128],
sizes: List[Tuple[int, int]] = [(-1, 64), (64, 128), (128, 256), (256, 512), (512, 10000000)],
threshold: Optional[float] = None,
nms_threshold: Optional[float] = None,
*args,
**kwargs,
):
super().__init__(*args, **kwargs)
self.save_hyperparameters()
self.num_classes = int(num_classes)
self.strides = [int(x) for x in strides]
self.sizes = [(int(x), int(y)) for x, y in sizes]
# TODO train this from scratch using combustion EfficientDet
# self._model = EffDetFCOS.from_predefined(
# compound_coeff, self.num_classes, fpn_levels=[3, 5, 7, 8, 9], strides=self.strides
# )
self._model = create_model(effdet_backbone, pretrained=True)
del self._model.box_net
del self._model.class_net
fpn_filters = self._model.config.fpn_channels
num_repeats = 4
self.fcos = FCOSDecoder(fpn_filters, self.num_classes, num_repeats, strides)
self.threshold = float(threshold) if threshold is not None else 0.05
self.nms_threshold = float(nms_threshold) if nms_threshold is not None else 0.1
self._criterion = FCOSLoss(self.strides, self.num_classes, radius=1, interest_range=self.sizes)
# metrics
metrics = MetricCollection({
f"ap{thresh}": BoxAveragePrecision(iou_threshold=thresh / 100, compute_on_step=True)
for thresh in (25, 50, 75)
})
self.val_metrics = metrics.clone(prefix="val/")
self.test_metrics = metrics.clone(prefix="test/")
# freeze backbone
for param in self._model.backbone.parameters():
param.requires_grad = False
def validate(args):
setup_default_logging()
if args.amp:
if has_apex:
args.apex_amp = True
elif has_native_amp:
args.native_amp = True
assert not args.apex_amp or not args.native_amp, "Only one AMP mode should be set."
args.pretrained = args.pretrained or not args.checkpoint # might as well try to validate something
args.prefetcher = not args.no_prefetcher
# create model
with set_layer_config(scriptable=args.torchscript):
bench = create_model(
args.model,
bench_task='predict',
num_classes=args.num_classes,
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
soft_nms=args.soft_nms,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
model_config = bench.config
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
amp_autocast = suppress
if args.apex_amp:
bench = amp.initialize(bench, opt_level='O1')
print('Using NVIDIA APEX AMP. Validating in mixed precision.')
elif args.native_amp:
amp_autocast = torch.cuda.amp.autocast
print('Using native Torch AMP. Validating in mixed precision.')
else:
print('AMP not enabled. Validating in float32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
dataset = create_dataset(args.dataset, args.root, args.split)
input_config = resolve_input_config(args, model_config)
loader = create_loader(dataset,
input_size=input_config['input_size'],
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=input_config['interpolation'],
fill_color=input_config['fill_color'],
mean=input_config['mean'],
std=input_config['std'],
num_workers=args.workers,
pin_mem=args.pin_mem)
evaluator = create_evaluator(args.dataset, dataset, pred_yxyx=False)
bench.eval()
batch_time = AverageMeter()
end = time.time()
last_idx = len(loader) - 1
with torch.no_grad():
for i, (input, target) in enumerate(loader):
with amp_autocast():
output = bench(input, img_info=target)
evaluator.add_predictions(output, target)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0 or i == last_idx:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg))
mean_ap = 0.
if dataset.parser.has_labels:
mean_ap = evaluator.evaluate()
else:
evaluator.save(args.results)
return mean_ap
def validate(args):
setup_default_logging()
# might as well try to validate something
args.pretrained = args.pretrained or not args.checkpoint
args.prefetcher = not args.no_prefetcher
if args.no_redundant_bias is None:
args.redundant_bias = None
else:
args.redundant_bias = not args.no_redundant_bias
# create model
bench = create_model(
args.model,
bench_task='predict',
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
if 'test' in args.anno:
annotation_path = os.path.join(args.data, 'annotations',
f'image_info_{args.anno}.json')
image_dir = 'test2017'
else:
annotation_path = os.path.join(args.data, 'annotations',
f'instances_{args.anno}.json')
image_dir = args.anno
dataset = CocoDetection(os.path.join(args.data, image_dir),
annotation_path)
loader = create_loader(dataset,
input_size=input_size,
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
fill_color=args.fill_color,
num_workers=args.workers,
pin_mem=args.pin_mem)
img_ids = []
results = []
bench.eval()
batch_time = AverageMeter()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(loader):
output = bench(input, target['img_scale'], target['img_size'])
output = output.cpu()
sample_ids = target['img_id'].cpu()
for index, sample in enumerate(output):
image_id = int(sample_ids[index])
for det in sample:
score = float(det[4])
if score < .001: # stop when below this threshold, scores in descending order
break
coco_det = dict(image_id=image_id,
bbox=det[0:4].tolist(),
score=score,
category_id=int(det[5]))
img_ids.append(image_id)
results.append(coco_det)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg,
))
json.dump(results, open(args.results, 'w'), indent=4)
if 'test' not in args.anno:
coco_results = dataset.coco.loadRes(args.results)
coco_eval = COCOeval(dataset.coco, coco_results, 'bbox')
coco_eval.params.imgIds = img_ids # score only ids we've used
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return results
def main():
setup_default_logging()
args, args_text = _parse_args()
args.pretrained_backbone = not args.no_pretrained_backbone
args.prefetcher = not args.no_prefetcher
args.distributed = False
if 'WORLD_SIZE' in os.environ:
args.distributed = int(os.environ['WORLD_SIZE']) > 1
args.device = 'cuda:0'
args.world_size = 1
args.rank = 0 # global rank
if args.distributed:
args.device = 'cuda:%d' % args.local_rank
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend='nccl',
init_method='env://')
args.world_size = torch.distributed.get_world_size()
args.rank = torch.distributed.get_rank()
assert args.rank >= 0
if args.distributed:
logging.info(
'Training in distributed mode with multiple processes, 1 GPU per process. Process %d, total %d.'
% (args.rank, args.world_size))
else:
logging.info('Training with a single process on 1 GPU.')
torch.manual_seed(args.seed + args.rank)
model = create_model(
args.model,
bench_task='train',
pretrained=args.pretrained,
pretrained_backbone=args.pretrained_backbone,
redundant_bias=args.redundant_bias,
checkpoint_path=args.initial_checkpoint,
)
# FIXME decide which args to keep and overlay on config / pass to backbone
# num_classes=args.num_classes,
# drop_rate=args.drop,
# drop_path_rate=args.drop_path,
# drop_block_rate=args.drop_block,
input_size = model.config.image_size
if args.local_rank == 0:
logging.info('Model %s created, param count: %d' %
(args.model, sum([m.numel()
for m in model.parameters()])))
model.cuda()
optimizer = create_optimizer(args, model)
use_amp = False
if has_apex and args.amp:
model, optimizer = amp.initialize(model, optimizer, opt_level='O1')
use_amp = True
if args.local_rank == 0:
logging.info('NVIDIA APEX {}. AMP {}.'.format(
'installed' if has_apex else 'not installed',
'on' if use_amp else 'off'))
# optionally resume from a checkpoint
resume_state = {}
resume_epoch = None
if args.resume:
resume_state, resume_epoch = resume_checkpoint(unwrap_bench(model),
args.resume)
if resume_state and not args.no_resume_opt:
if 'optimizer' in resume_state:
if args.local_rank == 0:
logging.info('Restoring Optimizer state from checkpoint')
optimizer.load_state_dict(resume_state['optimizer'])
if use_amp and 'amp' in resume_state and 'load_state_dict' in amp.__dict__:
if args.local_rank == 0:
logging.info('Restoring NVIDIA AMP state from checkpoint')
amp.load_state_dict(resume_state['amp'])
del resume_state
model_ema = None
if args.model_ema:
# Important to create EMA model after cuda(), DP wrapper, and AMP but before SyncBN and DDP wrapper
model_ema = ModelEma(model, decay=args.model_ema_decay)
#resume=args.resume) # FIXME bit of a mess with bench
if args.resume:
load_checkpoint(unwrap_bench(model_ema), args.resume, use_ema=True)
if args.distributed:
if args.sync_bn:
try:
if has_apex:
model = convert_syncbn_model(model)
else:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(
model)
if args.local_rank == 0:
logging.info(
'Converted model to use Synchronized BatchNorm. WARNING: You may have issues if using '
'zero initialized BN layers (enabled by default for ResNets) while sync-bn enabled.'
)
except Exception as e:
logging.error(
'Failed to enable Synchronized BatchNorm. Install Apex or Torch >= 1.1'
)
if has_apex:
model = DDP(model, delay_allreduce=True)
else:
if args.local_rank == 0:
logging.info(
"Using torch DistributedDataParallel. Install NVIDIA Apex for Apex DDP."
)
model = DDP(model,
device_ids=[args.local_rank
]) # can use device str in Torch >= 1.1
# NOTE: EMA model does not need to be wrapped by DDP
lr_scheduler, num_epochs = create_scheduler(args, optimizer)
start_epoch = 0
if args.start_epoch is not None:
# a specified start_epoch will always override the resume epoch
start_epoch = args.start_epoch
elif resume_epoch is not None:
start_epoch = resume_epoch
if lr_scheduler is not None and start_epoch > 0:
lr_scheduler.step(start_epoch)
if args.local_rank == 0:
logging.info('Scheduled epochs: {}'.format(num_epochs))
train_anno_set = 'train2017'
train_annotation_path = os.path.join(args.data, 'annotations',
f'instances_{train_anno_set}.json')
train_image_dir = train_anno_set
dataset_train = CocoDetection(os.path.join(args.data, train_image_dir),
train_annotation_path)
# FIXME cutmix/mixup worth investigating?
# collate_fn = None
# if args.prefetcher and args.mixup > 0:
# collate_fn = FastCollateMixup(args.mixup, args.smoothing, args.num_classes)
loader_train = create_loader(
dataset_train,
input_size=input_size,
batch_size=args.batch_size,
is_training=True,
use_prefetcher=args.prefetcher,
#re_prob=args.reprob, # FIXME add back various augmentations
#re_mode=args.remode,
#re_count=args.recount,
#re_split=args.resplit,
#color_jitter=args.color_jitter,
#auto_augment=args.aa,
interpolation=args.train_interpolation,
#mean=data_config['mean'],
#std=data_config['std'],
num_workers=args.workers,
distributed=args.distributed,
#collate_fn=collate_fn,
pin_mem=args.pin_mem,
)
train_anno_set = 'val2017'
train_annotation_path = os.path.join(args.data, 'annotations',
f'instances_{train_anno_set}.json')
train_image_dir = train_anno_set
dataset_eval = CocoDetection(os.path.join(args.data, train_image_dir),
train_annotation_path)
loader_eval = create_loader(
dataset_eval,
input_size=input_size,
batch_size=args.validation_batch_size_multiplier * args.batch_size,
is_training=False,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
#mean=data_config['mean'],
#std=data_config['std'],
num_workers=args.workers,
#distributed=args.distributed,
pin_mem=args.pin_mem,
)
evaluator = COCOEvaluator(dataset_eval.coco, distributed=args.distributed)
eval_metric = args.eval_metric
best_metric = None
best_epoch = None
saver = None
output_dir = ''
if args.local_rank == 0:
output_base = args.output if args.output else './output'
exp_name = '-'.join(
[datetime.now().strftime("%Y%m%d-%H%M%S"), args.model])
output_dir = get_outdir(output_base, 'train', exp_name)
decreasing = True if eval_metric == 'loss' else False
saver = CheckpointSaver(checkpoint_dir=output_dir,
decreasing=decreasing)
with open(os.path.join(output_dir, 'args.yaml'), 'w') as f:
f.write(args_text)
try:
for epoch in range(start_epoch, num_epochs):
if args.distributed:
loader_train.sampler.set_epoch(epoch)
train_metrics = train_epoch(epoch,
model,
loader_train,
optimizer,
args,
lr_scheduler=lr_scheduler,
saver=saver,
output_dir=output_dir,
use_amp=use_amp,
model_ema=model_ema)
if args.distributed and args.dist_bn in ('broadcast', 'reduce'):
if args.local_rank == 0:
logging.info(
"Distributing BatchNorm running means and vars")
distribute_bn(model, args.world_size, args.dist_bn == 'reduce')
# the overhead of evaluating with coco style datasets is fairly high, so just ema or non, not both
if model_ema is not None:
if args.distributed and args.dist_bn in ('broadcast',
'reduce'):
distribute_bn(model_ema, args.world_size,
args.dist_bn == 'reduce')
eval_metrics = validate(model_ema.ema,
loader_eval,
args,
evaluator,
log_suffix=' (EMA)')
else:
eval_metrics = validate(model, loader_eval, args, evaluator)
if lr_scheduler is not None:
# step LR for next epoch
lr_scheduler.step(epoch + 1, eval_metrics[eval_metric])
if saver is not None:
update_summary(epoch,
train_metrics,
eval_metrics,
os.path.join(output_dir, 'summary.csv'),
write_header=best_metric is None)
# save proper checkpoint with eval metric
save_metric = eval_metrics[eval_metric]
best_metric, best_epoch = saver.save_checkpoint(
unwrap_bench(model),
optimizer,
args,
epoch=epoch,
model_ema=unwrap_bench(model_ema),
metric=save_metric,
use_amp=use_amp)
except KeyboardInterrupt:
pass
if best_metric is not None:
logging.info('*** Best metric: {0} (epoch {1})'.format(
best_metric, best_epoch))
def validate(args):
setup_default_logging()
def setthresh():
if args.checkpoint.split("/")[-1].split(
"_")[0] in getthresholds.keys():
return getthresholds[args.checkpoint.split("/")[-1].split("_")[0]]
else:
a = []
[a.append(args.threshold) for x in range(4)]
return a
threshs = setthresh()
print(threshs)
# might as well try to validate something
args.pretrained = args.pretrained or not args.checkpoint
args.prefetcher = not args.no_prefetcher
# create model
bench = create_model(
args.model,
bench_task='predict',
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
if 'test' in args.anno:
annotation_path = os.path.join(args.data, 'annotations',
f'image_info_{args.anno}.json')
image_dir = args.anno
else:
annotation_path = os.path.join(args.data, 'annotations',
f'instances_{args.anno}.json')
image_dir = args.anno
print(os.path.join(args.data, image_dir), annotation_path)
dataset = CocoDetection(os.path.join(args.data, image_dir),
annotation_path)
loader = create_loader(dataset,
input_size=input_size,
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
fill_color=args.fill_color,
num_workers=args.workers,
pin_mem=args.pin_mem,
mean=args.mean,
std=args.std)
if 'test' in args.anno:
threshold = float(args.threshold)
else:
threshold = .001
img_ids = []
results = []
writetofilearrtay = []
bench.eval()
batch_time = AverageMeter()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(loader):
output = bench(input, target['img_scale'], target['img_size'])
output = output.cpu()
# print(target['img_id'])
sample_ids = target['img_id'].cpu()
for index, sample in enumerate(output):
image_id = int(sample_ids[index])
for det in sample:
score = float(det[4])
if score < threshold: # stop when below this threshold, scores in descending order
coco_det = dict(image_id=image_id, category_id=-1)
img_ids.append(image_id)
results.append(coco_det)
break
coco_det = dict(image_id=image_id,
bbox=det[0:4].tolist(),
score=score,
category_id=int(det[5]),
sizes=target['img_size'].tolist()[0])
img_ids.append(image_id)
results.append(coco_det)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg,
))
if 'test' in args.anno:
from itertools import groupby
results.sort(key=lambda x: x['image_id'])
f = open(
str(args.model) + "-" + str(args.anno) + "-" + str(min(threshs)) +
".txt", "w+")
# for item in tqdm(writetofilearrtay):
xxx = 0
for k, v in tqdm(groupby(results, key=lambda x: x['image_id'])):
xxx += 1
f.write(getimageNamefromid(k) +
",") #print(getimageNamefromid(k),", ")
for i in v:
if i['category_id'] > 0:
if (i['category_id'] ==1 and i['score'] >= threshs[0] ) or (i['category_id'] ==2 and i['score'] >= threshs[1] ) or \
(i['category_id'] ==3 and i['score'] >= threshs[2] ) or (i['category_id'] ==4 and i['score'] >= threshs[3] ) :
f.write(
str(round(i['category_id'])) + " " +
str(round(i['bbox'][0])) + " " +
str(round(i['bbox'][1])) + " " + str(
round(
float(i['bbox'][0]) +
float(i['bbox'][2]))) + " " + str(
round(
float(i['bbox'][1]) +
float(i['bbox'][3]))) + " ")
f.write('\n')
# print(i['category_id']," ",i['bbox'][0]," ",i['bbox'][1]," ",i['bbox'][2]," ",i['bbox'][3]," ")
print("generated lines:", xxx)
f.close()
# f.close()
if 'test' not in args.anno:
array_of_dm = []
array_of_gt = []
i = 0
# if 'test' in args.anno :
for _, item in tqdm(dataset):
# if item["img_id"] == "1000780" :
# print(item)
for i in range(len(item['cls'])):
# print(str(item["img_id"]),)
array_of_gt.append(
BoundingBox(imageName=str(item["img_id"]),
classId=item["cls"][i],
x=item["bbox"][i][1] * item['img_scale'],
y=item["bbox"][i][0] * item['img_scale'],
w=item["bbox"][i][3] * item['img_scale'],
h=item["bbox"][i][2] * item['img_scale'],
typeCoordinates=CoordinatesType.Absolute,
bbType=BBType.GroundTruth,
format=BBFormat.XYX2Y2,
imgSize=(item['img_size'][0],
item['img_size'][1])))
for item in tqdm(results):
if item["category_id"] >= 0:
array_of_dm.append(
BoundingBox(imageName=str(item["image_id"]),
classId=item["category_id"],
classConfidence=item["score"],
x=item['bbox'][0],
y=item['bbox'][1],
w=item['bbox'][2],
h=item['bbox'][3],
typeCoordinates=CoordinatesType.Absolute,
bbType=BBType.Detected,
format=BBFormat.XYWH,
imgSize=(item['sizes'][0], item['sizes'][1])))
myBoundingBoxes = BoundingBoxes()
# # # # Add all bounding boxes to the BoundingBoxes object:
for box in (array_of_gt):
myBoundingBoxes.addBoundingBox(box)
for dm in array_of_dm:
myBoundingBoxes.addBoundingBox(dm)
evaluator = Evaluator()
f1res = []
f1resd0 = []
f1resd10 = []
f1resd20 = []
f1resd40 = []
for conf in tqdm(range(210, 600, 1)):
metricsPerClass = evaluator.GetPascalVOCMetrics(
myBoundingBoxes, IOUThreshold=0.5, ConfThreshold=conf / 1000.0)
totalTP = 0
totalp = 0
totalFP = 0
tp = []
fp = []
ta = []
# print('-------')
for mc in metricsPerClass:
tp.append(mc['total TP'])
fp.append(mc['total FP'])
ta.append(mc['total positives'])
totalFP = totalFP + mc['total FP']
totalTP = totalTP + mc['total TP']
totalp = totalp + (mc['total positives'])
# print(totalTP," ",totalFP," ",totalp)
if totalTP + totalFP == 0:
p = -1
else:
p = totalTP / (totalTP + totalFP)
if totalp == 0:
r = -1
else:
r = totalTP / (totalp)
f1_dict = dict(tp=totalTP,
fp=totalFP,
totalp=totalp,
conf=conf / 1000.0,
prec=p,
rec=r,
f1score=(2 * p * r) / (p + r))
f1res.append(f1_dict)
#must clean these parts
f1resd0.append(
dict(tp=tp[0],
fp=fp[0],
totalp=ta[0],
conf=conf / 1000.0,
prec=tp[0] / (tp[0] + fp[0]),
rec=tp[0] / ta[0],
f1score=(2 * (tp[0] / (tp[0] + fp[0])) *
(tp[0] / ta[0])) / ((tp[0] / (tp[0] + fp[0])) +
(tp[0] / ta[0]))))
f1resd10.append(
dict(tp=tp[1],
fp=fp[1],
totalp=ta[1],
conf=conf / 1000.0,
prec=tp[1] / (tp[1] + fp[1]),
rec=tp[1] / ta[1],
f1score=(2 * (tp[1] / (tp[1] + fp[1])) *
(tp[1] / ta[1])) / ((tp[1] / (tp[1] + fp[1])) +
(tp[1] / ta[1]))))
f1resd20.append(
dict(tp=tp[2],
fp=fp[2],
totalp=ta[2],
conf=conf / 1000.0,
prec=tp[2] / (tp[2] + fp[2]),
rec=tp[2] / ta[2],
f1score=(2 * (tp[2] / (tp[2] + fp[2])) *
(tp[2] / ta[2])) / ((tp[2] / (tp[2] + fp[2])) +
(tp[2] / ta[2]))))
f1resd40.append(
dict(tp=tp[3],
fp=fp[3],
totalp=ta[3],
conf=conf / 1000.0,
prec=tp[3] / (tp[3] + fp[3]),
rec=tp[3] / ta[3],
f1score=(2 * (tp[3] / (tp[3] + fp[3])) *
(tp[3] / ta[3])) / ((tp[3] / (tp[3] + fp[3])) +
(tp[3] / ta[3]))))
sortedf1 = sorted(f1res, key=lambda k: k['f1score'], reverse=True)
f1resd0 = sorted(f1resd0, key=lambda k: k['f1score'], reverse=True)
f1resd10 = sorted(f1resd10, key=lambda k: k['f1score'], reverse=True)
f1resd20 = sorted(f1resd20, key=lambda k: k['f1score'], reverse=True)
f1resd40 = sorted(f1resd40, key=lambda k: k['f1score'], reverse=True)
print(sortedf1[0])
print("\n\n")
print(f1resd0[0])
print(f1resd10[0])
print(f1resd20[0])
print(f1resd40[0])
# sortedf1 = sorted(f1res, key=lambda k: k['f1score'],reverse=True)
# print(sortedf1[0:2])
# json.dump(results, open(args.results, 'w'), indent=4)
json.dump(results, open(args.results, 'w'), indent=4)
# coco_results = dataset.coco.loadRes(args.results)
# coco_eval = COCOeval(dataset.coco, coco_results, 'bbox')
# coco_eval.params.imgIds = img_ids # score only ids we've used
# coco_eval.evaluate()
# coco_eval.accumulate()
# coco_eval.summarize()
# print(coco_eval.eval['params'])
json.dump(results, open(args.results, 'w'), indent=4)
return results
def main():
setup_default_logging()
args, args_text = _parse_args()
args.pretrained_backbone = not args.no_pretrained_backbone
args.prefetcher = not args.no_prefetcher
args.distributed = False
if 'WORLD_SIZE' in os.environ:
args.distributed = int(os.environ['WORLD_SIZE']) > 1
args.device = 'cuda:0'
args.world_size = 1
args.rank = 0 # global rank
if args.distributed:
args.device = 'cuda:%d' % args.local_rank
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend='nccl', init_method='env://')
args.world_size = torch.distributed.get_world_size()
args.rank = torch.distributed.get_rank()
assert args.rank >= 0
if args.distributed:
logging.info('Training in distributed mode with multiple processes, 1 GPU per process. Process %d, total %d.'
% (args.rank, args.world_size))
else:
logging.info('Training with a single process on 1 GPU.')
use_amp = None
if args.amp:
# for backwards compat, `--amp` arg tries apex before native amp
if has_apex:
args.apex_amp = True
elif has_native_amp:
args.native_amp = True
else:
logging.warning("Neither APEX or native Torch AMP is available, using float32. "
"Install NVIDA apex or upgrade to PyTorch 1.6.")
if args.apex_amp:
if has_apex:
use_amp = 'apex'
else:
logging.warning("APEX AMP not available, using float32. Install NVIDA apex")
elif args.native_amp:
if has_native_amp:
use_amp = 'native'
else:
logging.warning("Native AMP not available, using float32. Upgrade to PyTorch 1.6.")
torch.manual_seed(args.seed + args.rank)
model = create_model(
args.model,
bench_task='train',
num_classes=args.num_classes,
pretrained=args.pretrained,
pretrained_backbone=args.pretrained_backbone,
redundant_bias=args.redundant_bias,
label_smoothing=args.smoothing,
new_focal=args.new_focal,
jit_loss=args.jit_loss,
bench_labeler=args.bench_labeler,
checkpoint_path=args.initial_checkpoint,
)
model_config = model.config # grab before we obscure with DP/DDP wrappers
if args.local_rank == 0:
logging.info('Model %s created, param count: %d' % (args.model, sum([m.numel() for m in model.parameters()])))
model.cuda()
if args.channels_last:
model = model.to(memory_format=torch.channels_last)
optimizer = create_optimizer(args, model)
amp_autocast = suppress # do nothing
loss_scaler = None
if use_amp == 'apex':
model, optimizer = amp.initialize(model, optimizer, opt_level='O1')
loss_scaler = ApexScaler()
if args.local_rank == 0:
logging.info('Using NVIDIA APEX AMP. Training in mixed precision.')
elif use_amp == 'native':
amp_autocast = torch.cuda.amp.autocast
loss_scaler = NativeScaler()
if args.local_rank == 0:
logging.info('Using native Torch AMP. Training in mixed precision.')
else:
if args.local_rank == 0:
logging.info('AMP not enabled. Training in float32.')
# optionally resume from a checkpoint
resume_epoch = None
if args.resume:
resume_epoch = resume_checkpoint(
unwrap_bench(model), args.resume,
optimizer=None if args.no_resume_opt else optimizer,
loss_scaler=None if args.no_resume_opt else loss_scaler,
log_info=args.local_rank == 0)
model_ema = None
if args.model_ema:
# Important to create EMA model after cuda(), DP wrapper, and AMP but before SyncBN and DDP wrapper
model_ema = ModelEma(model, decay=args.model_ema_decay)
if args.resume:
# FIXME bit of a mess with bench, cannot use the load in ModelEma
load_checkpoint(unwrap_bench(model_ema), args.resume, use_ema=True)
if args.distributed:
if args.sync_bn:
try:
if has_apex and use_amp != 'native':
model = convert_syncbn_model(model)
else:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
if args.local_rank == 0:
logging.info(
'Converted model to use Synchronized BatchNorm. WARNING: You may have issues if using '
'zero initialized BN layers (enabled by default for ResNets) while sync-bn enabled.')
except Exception as e:
logging.error('Failed to enable Synchronized BatchNorm. Install Apex or Torch >= 1.1')
if has_apex and use_amp != 'native':
if args.local_rank == 0:
logging.info("Using apex DistributedDataParallel.")
model = ApexDDP(model, delay_allreduce=True)
else:
if args.local_rank == 0:
logging.info("Using torch DistributedDataParallel.")
model = NativeDDP(model, device_ids=[args.device])
# NOTE: EMA model does not need to be wrapped by DDP
lr_scheduler, num_epochs = create_scheduler(args, optimizer)
start_epoch = 0
if args.start_epoch is not None:
# a specified start_epoch will always override the resume epoch
start_epoch = args.start_epoch
elif resume_epoch is not None:
start_epoch = resume_epoch
if lr_scheduler is not None and start_epoch > 0:
lr_scheduler.step(start_epoch)
if args.local_rank == 0:
logging.info('Scheduled epochs: {}'.format(num_epochs))
loader_train, loader_eval, evaluator = create_datasets_and_loaders(args, model_config)
if model_config.num_classes < loader_train.dataset.parser.max_label:
logging.error(
f'Model {model_config.num_classes} has fewer classes than dataset {loader_train.dataset.parser.max_label}.')
exit(1)
if model_config.num_classes > loader_train.dataset.parser.max_label:
logging.warning(
f'Model {model_config.num_classes} has more classes than dataset {loader_train.dataset.parser.max_label}.')
eval_metric = args.eval_metric
best_metric = None
best_epoch = None
saver = None
output_dir = ''
if args.local_rank == 0:
output_base = args.output if args.output else './output'
exp_name = '-'.join([
datetime.now().strftime("%Y%m%d-%H%M%S"),
args.model
])
output_dir = get_outdir(output_base, 'train', exp_name)
decreasing = True if eval_metric == 'loss' else False
saver = CheckpointSaver(
model, optimizer, args=args, model_ema=model_ema, amp_scaler=loss_scaler,
checkpoint_dir=output_dir, decreasing=decreasing, unwrap_fn=unwrap_bench)
with open(os.path.join(output_dir, 'args.yaml'), 'w') as f:
f.write(args_text)
try:
for epoch in range(start_epoch, num_epochs):
if args.distributed:
loader_train.sampler.set_epoch(epoch)
train_metrics = train_epoch(
epoch, model, loader_train, optimizer, args,
lr_scheduler=lr_scheduler, saver=saver, output_dir=output_dir,
amp_autocast=amp_autocast, loss_scaler=loss_scaler, model_ema=model_ema)
if args.distributed and args.dist_bn in ('broadcast', 'reduce'):
if args.local_rank == 0:
logging.info("Distributing BatchNorm running means and vars")
distribute_bn(model, args.world_size, args.dist_bn == 'reduce')
# the overhead of evaluating with coco style datasets is fairly high, so just ema or non, not both
if model_ema is not None:
if args.distributed and args.dist_bn in ('broadcast', 'reduce'):
distribute_bn(model_ema, args.world_size, args.dist_bn == 'reduce')
eval_metrics = validate(model_ema.ema, loader_eval, args, evaluator, log_suffix=' (EMA)')
else:
eval_metrics = validate(model, loader_eval, args, evaluator)
if lr_scheduler is not None:
# step LR for next epoch
lr_scheduler.step(epoch + 1, eval_metrics[eval_metric])
if saver is not None:
update_summary(
epoch, train_metrics, eval_metrics, os.path.join(output_dir, 'summary.csv'),
write_header=best_metric is None)
# save proper checkpoint with eval metric
best_metric, best_epoch = saver.save_checkpoint(epoch=epoch, metric=eval_metrics[eval_metric])
except KeyboardInterrupt:
pass
if best_metric is not None:
logging.info('*** Best metric: {0} (epoch {1})'.format(best_metric, best_epoch))
def validate(args):
setup_default_logging()
# might as well try to validate something
args.pretrained = args.pretrained or not args.checkpoint
args.prefetcher = not args.no_prefetcher
# create model
bench = create_model(args.model,
bench_task='predict',
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema)
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
if 'test' in args.anno:
annotation_path = os.path.join(args.data, 'annotations',
f'image_info_{args.anno}.json')
image_dir = args.anno
else:
annotation_path = os.path.join(args.data, 'annotations',
f'instances_{args.anno}.json')
image_dir = args.anno
dataset = CocoDetection(os.path.join(args.data, image_dir),
annotation_path)
loader = create_loader(dataset,
input_size=input_size,
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
fill_color=args.fill_color,
num_workers=args.workers,
mean=args.mean,
std=args.std,
pin_mem=args.pin_mem)
img_ids = []
results = []
bench.eval()
for i, (input, target) in enumerate(loader, 1):
dumm_inp = input
tisc = target['img_scale']
tisz = target['img_size']
break
starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(
enable_timing=True)
# repetitions = 300
# timings=np.zeros((repetitions,1))
#GPU-WARM-UP
# print(enumerate())
for _ in range(10):
_ = bench(dumm_inp, tisc, tisz)
# MEASURE PERFORMANCE
# dummy_input = torch.randn(1, 3,bench.config.image_size,bench.config.image_size,dtype=torch.float).to("cuda")
print("starting")
batch_time = AverageMeter()
# end = time.time()
with torch.no_grad():
for _ in range(2000):
starter.record()
_ = bench(dumm_inp, tisc, tisz)
ender.record()
# measure elapsed time
torch.cuda.synchronize()
curr_time = starter.elapsed_time(ender)
batch_time.update(curr_time)
# print(curr_time)
# end = time.time()
# if i % args.log_freq == 0:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}ms ({batch_time.avg:.3f}ms, {rate_avg:>7.2f}/s) '
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=dumm_inp.size(0) / batch_time.avg,
))
# json.dump(results, open(args.results, 'w'), indent=4)
# if 'test' not in args.anno:
# coco_results = dataset.coco.loadRes(args.results)
# coco_eval = COCOeval(dataset.coco, coco_results, 'bbox')
# coco_eval.params.imgIds = img_ids # score only ids we've used
# coco_eval.evaluate()
# coco_eval.accumulate()
# coco_eval.summarize()
return results
def eval_model(model_name, paper_model_name, paper_arxiv_id, batch_size=64, model_description=''):
# create model
bench = create_model(
model_name,
bench_task='predict',
pretrained=True,
)
bench.eval()
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (model_name, param_count))
bench = bench.cuda()
if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')
annotation_path = os.path.join(DATA_ROOT, 'annotations', f'instances_{ANNO_SET}.json')
evaluator = COCOEvaluator(
root=DATA_ROOT,
model_name=paper_model_name,
model_description=model_description,
paper_arxiv_id=paper_arxiv_id)
dataset = CocoDetection(os.path.join(DATA_ROOT, ANNO_SET), annotation_path)
loader = create_loader(
dataset,
input_size=input_size,
batch_size=batch_size,
use_prefetcher=True,
fill_color='mean',
num_workers=4,
pin_mem=True)
iterator = tqdm.tqdm(loader, desc="Evaluation", mininterval=5)
evaluator.reset_time()
with torch.no_grad():
for i, (input, target) in enumerate(iterator):
output = bench(input, target['img_scale'], target['img_size'])
output = output.cpu()
sample_ids = target['img_id'].cpu()
results = []
for index, sample in enumerate(output):
image_id = int(sample_ids[index])
for det in sample:
score = float(det[4])
if score < .001: # stop when below this threshold, scores in descending order
break
coco_det = dict(
image_id=image_id,
bbox=det[0:4].tolist(),
score=score,
category_id=int(det[5]))
results.append(coco_det)
evaluator.add(results)
if evaluator.cache_exists:
break
evaluator.save()
def __init__(self, context: PyTorchTrialContext) -> None:
self.context = context
self.hparam = self.context.get_hparam
self.args = DotDict(self.context.get_hparams())
# Create a unique download directory for each rank so they don't overwrite each other.
self.download_directory = f"/tmp/data-rank{self.context.distributed.get_rank()}"
self.num_slots = int(self.context.get_experiment_config()['resources']
['slots_per_trial'])
if self.args.sync_bn and self.num_slots == 1:
print(
'Can not use sync_bn with one slot. Either set sync_bn to False or use distributed training.'
)
sys.exit()
self.args.pretrained_backbone = not self.args.no_pretrained_backbone
self.args.prefetcher = not self.args.no_prefetcher
tmp = []
for arg in self.args.lr_noise.split(' '):
tmp.append(float(arg))
self.args.lr_noise = tmp
self.model = create_model(
self.args.model,
bench_task='train',
num_classes=self.args.num_classes,
pretrained=self.args.pretrained,
pretrained_backbone=self.args.pretrained_backbone,
redundant_bias=self.args.redundant_bias,
label_smoothing=self.args.smoothing,
new_focal=self.args.new_focal,
jit_loss=self.args.jit_loss,
bench_labeler=self.args.bench_labeler,
checkpoint_path=self.args.initial_checkpoint,
)
self.model_config = self.model.config
self.input_config = resolve_input_config(
self.args, model_config=self.model_config)
print('h: ', self.args.model,
sum([m.numel() for m in self.model.parameters()]))
if self.args.sync_bn:
print('creating batch sync model')
if self.args.model_ema:
print('creating batch sync ema model')
self.model_ema = self.context.wrap_model(deepcopy(self.model))
self.model = self.convert_syncbn_model(self.model)
self.model = self.context.wrap_model(self.model)
print('Model created, param count:', self.args.model,
sum([m.numel() for m in self.model.parameters()]))
self.optimizer = self.context.wrap_optimizer(
create_optimizer(self.args, self.model))
print('Created optimizer: ', self.optimizer)
if self.args.amp:
print('using amp')
if self.args.sync_bn and self.args.model_ema:
print('using sync_bn and model_ema when creating apex_amp')
(self.model, self.model_ema
), self.optimizer = self.context.configure_apex_amp(
[self.model, self.model_ema],
self.optimizer,
min_loss_scale=self.hparam("min_loss_scale"))
else:
self.model, self.optimizer = self.context.configure_apex_amp(
self.model,
self.optimizer,
min_loss_scale=self.hparam("min_loss_scale"))
if self.args.model_ema:
print('using model ema')
if self.args.sync_bn:
print('using model ema batch syn')
self.model_ema = ModelEma(self.model_ema,
context=self.context,
decay=self.args.model_ema_decay)
else:
# Important to create EMA model after cuda(), DP wrapper, and AMP but before SyncBN and DDP wrapper
self.model_ema = ModelEma(self.model,
context=self.context,
decay=self.args.model_ema_decay)
self.lr_scheduler, self.num_epochs = create_scheduler(
self.args, self.optimizer)
self.lr_scheduler = self.context.wrap_lr_scheduler(
self.lr_scheduler, LRScheduler.StepMode.MANUAL_STEP)
self.cur_epoch = 0
self.num_updates = 0 * self.cur_epoch
if self.args.prefetcher:
self.train_mean, self.train_std, self.train_random_erasing = self.calculate_means(
mean=self.input_config['mean'],
std=self.input_config['std'],
re_prob=self.args.reprob,
re_mode=self.args.remode,
re_count=self.args.recount)
self.val_mean, self.val_std, self.val_random_erasing = self.calculate_means(
self.input_config['mean'], self.input_config['std'])
self.val_reducer = self.context.wrap_reducer(self.validation_reducer,
for_training=False)
def validate(args):
setup_default_logging()
def setthresh():
if args.checkpoint.split("/")[-1].split(
"_")[0] in getthresholds.keys():
return getthresholds[args.checkpoint.split("/")[-1].split("_")[0]]
else:
a = []
[a.append(args.threshold) for x in range(4)]
return a
# might as well try to validate something
args.pretrained = args.pretrained or not args.checkpoint
args.prefetcher = not args.no_prefetcher
# create model
bench = create_model(
args.model,
bench_task='predict',
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
if 'test' in args.anno:
annotation_path = os.path.join(args.data, 'annotations',
f'image_info_{args.anno}.json')
image_dir = args.anno
elif 'val' in args.anno:
annotation_path = os.path.join(args.data, 'annotations',
f'instances_{args.anno}.json')
image_dir = args.anno
# else:
# annotation_path = os.path.join(args.data, f'{args.anno}.json')
# image_dir = args.anno
print(os.path.join(args.data, image_dir), annotation_path)
dataset = CocoDetection(os.path.join(args.data, image_dir),
annotation_path)
loader = create_loader(dataset,
input_size=input_size,
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
fill_color=args.fill_color,
num_workers=args.workers,
pin_mem=args.pin_mem,
mean=args.mean,
std=args.std)
if 'test' in args.anno:
threshold = float(args.threshold)
# elif 'detector' in args.anno:
# threshold = min(getthresholds['d0'])
else:
threshold = .001
img_ids = []
results = []
writetofilearrtay = []
bench.eval()
batch_time = AverageMeter()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(loader):
output = bench(input, target['img_scale'], target['img_size'])
output = output.cpu()
# print(target['img_id'])
sample_ids = target['img_id'].cpu()
for index, sample in enumerate(output):
image_id = int(sample_ids[index])
# if 'test' in args.anno :
# tempWritetoFile = []
# tempWritetoFile.append(getimageNamefromid(image_id))
for det in sample:
score = float(det[4])
if score < threshold: # stop when below this threshold, scores in descending order
coco_det = dict(image_id=image_id, category_id=-1)
img_ids.append(image_id)
results.append(coco_det)
break
coco_det = dict(image_id=image_id,
bbox=det[0:4].tolist(),
score=score,
category_id=int(det[5]),
sizes=target['img_size'].tolist()[0])
img_ids.append(image_id)
results.append(coco_det)
# exit()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg,
))
# if 'test' in args.anno :
if not os.path.exists(args.tosave):
os.makedirs(args.tosave)
from itertools import groupby
results.sort(key=lambda x: x['image_id'])
count = 0
for k, v in tqdm(groupby(results, key=lambda x: x['image_id'])):
# print(args.data +"/" + str(getimageNamefromid(k)))
img = drawonimage(
os.path.join(args.data, image_dir, str(getimageNamefromid(k))), v,
setthresh())
cv2.imwrite(args.tosave + "/" + str(getimageNamefromid(k)), img)
count += 1
# print(i['category_id']," ",i['bbox'][0]," ",i['bbox'][1]," ",i['bbox'][2]," ",i['bbox'][3]," ")
print("generated predictions for ", count, " images.")
return results
def validate(args):
setup_default_logging()
if args.amp:
if has_apex:
args.apex_amp = True
elif has_native_amp:
args.native_amp = True
assert not args.apex_amp or not args.native_amp, "Only one AMP mode should be set."
args.pretrained = args.pretrained or not args.checkpoint # might as well try to validate something
args.prefetcher = not args.no_prefetcher
# create model
with set_layer_config(scriptable=args.torchscript):
bench = create_model(
args.model,
bench_task='predict',
num_classes=args.num_classes,
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
soft_nms=args.soft_nms,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
model_config = bench.config
param_count = sum([m.numel() for m in bench.parameters()])
print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
amp_autocast = suppress
if args.apex_amp:
bench = amp.initialize(bench, opt_level='O1')
print('Using NVIDIA APEX AMP. Validating in mixed precision.')
elif args.native_amp:
amp_autocast = torch.cuda.amp.autocast
print('Using native Torch AMP. Validating in mixed precision.')
else:
print('AMP not enabled. Validating in float32.')
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
dataset = create_dataset(args.dataset, args.root, args.split)
input_config = resolve_input_config(args, model_config)
loader = create_loader(dataset,
input_size=input_config['input_size'],
batch_size=args.batch_size,
use_prefetcher=args.prefetcher,
interpolation=input_config['interpolation'],
fill_color=input_config['fill_color'],
mean=input_config['mean'],
std=input_config['std'],
num_workers=args.workers,
pin_mem=args.pin_mem)
evaluator = create_evaluator(args.dataset, dataset, pred_yxyx=False)
bench.eval()
batch_time = AverageMeter()
end = time.time()
last_idx = len(loader) - 1
imgs = []
with torch.no_grad():
for i, (input, target) in enumerate(loader):
for b in range(input.shape[0]):
imgs.append(input[b].cpu().numpy())
# targets.append(target[b].cpu().numpy())
with amp_autocast():
output = bench(input, img_info=target)
evaluator.add_predictions(output, target)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0 or i == last_idx:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg))
mean_ap = 0.
if dataset.parser.has_labels:
preds = [p[:2, :] for p in evaluator.predictions]
anns = evaluator.coco_api.imgToAnns
targets = [
np.asarray((anns[k][0]['bbox'], anns[k][1]['bbox']))
for k in range(len(imgs))
]
mean_ap = evaluator.evaluate()
if not os.path.exists(args.out_dir):
os.mkdir(args.out_dir)
for i, img in enumerate(imgs):
img = imgs[i]
img_m = np.mean(img, axis=0)
for c in range(3):
img[c] = img_m
img_ = img.transpose(1, 2, 0)
m = img_.min()
M = img_.max()
img_ = ((img_ - m) / (M - m) * 255).astype('uint8').copy()
img_ = draw_bbox(img_, preds[i], targets[i])
cv2.imwrite(os.path.join(args.out_dir, '%d.jpg' % i), img_)
else:
evaluator.save(args.results)
return mean_ap
def validate_det(args):
setup_default_logging()
# might as well try to validate something
args.pretrained = args.checkpoint
args.prefetcher = not args.no_prefetcher
# create model
bench = create_model(
args.model,
bench_task='predict',
num_classes=args.num_classes,
pretrained=args.pretrained,
redundant_bias=args.redundant_bias,
checkpoint_path=args.checkpoint,
checkpoint_ema=args.use_ema,
)
model_config = bench.config
input_size = bench.config.image_size
param_count = sum([m.numel() for m in bench.parameters()])
#print('Model %s created, param count: %d' % (args.model, param_count))
bench = bench.cuda()
'''if has_amp:
print('Using AMP mixed precision.')
bench = amp.initialize(bench, opt_level='O1')
else:
print('AMP not installed, running network in FP32.')'''
if args.num_gpu > 1:
bench = torch.nn.DataParallel(bench,
device_ids=list(range(args.num_gpu)))
'''if 'test' in args.anno:
annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json')
image_dir = 'test2017'''
'''annotation_path = os.path.join(args.data, 'Highway_drop_inlets.v1-vdot_coco.coco/coco_and_dropinlets_annotations/test_annotations', f'{args.anno}.json')
image_dir = os.path.join(args.data, 'Highway_drop_inlets.v1-vdot_coco.coco/coco_and_dropinlets/test/')
dataset = CocoDetection(image_dir, annotation_path)'''
annotation_path = os.path.join(args.data, f'{args.anno}.json')
image_dir = os.path.join(args.data, 'others_mix_set')
dataset = VdotTestDataset(image_dir, annotation_path)
loader = create_loader(dataset,
input_size=input_size,
batch_size=args.batches_size,
use_prefetcher=args.prefetcher,
interpolation=args.interpolation,
fill_color=args.fill_color,
num_workers=args.workers,
pin_mem=args.pin_mem)
img_ids = []
results = []
bench.eval()
#example_input = torch.randn((1, 3, 512, 512), requires_grad=True)
#bench(example_input.cuda())
'''tracingModelInput = torch.ones(1,3,512,512)
torch.onnx._export(bench.model, tracingModelInput.cuda(), './effdet0_checkonly.onnx', opset_version=11, export_params=True)
print('\nDone exporting ONNX model!')
onnx_model='./effdet0_checkonly.onnx'
onnx.checker.check_model(onnx_model)'''
'''dummy_input = torch.randn(1, 3, 512, 512, device='cuda')
input =["input"]
output=["class_1","class_2","class_3","class_4", "class_5", "box_1", "box_2", "box_3","box_4","box_5"]
#dynamic_axes = {"actual_input_1":{input:"batch_size"}, "output1":{output:"batch_size"}}
torch.onnx.export(bench.model, dummy_input, "effdet0_upsamplenormal.onnx", verbose=True, opset_version=11,
input_names=input, output_names=output)'''
#dynamic_axes=dynamic_axes)
batch_time = AverageMeter()
end = time.time()
with torch.no_grad():
for i, (input, target) in enumerate(loader):
output = bench(input, img_info=target)
output = output.cpu()
sample_ids = target['img_id'].cpu()
for index, sample in enumerate(output):
image_id = int(sample_ids[index])
for det in sample:
score = float(det[4])
if score < 0.36: # stop when below this threshold, scores in descending order(perfect 0.5 for 91 classes)
break
coco_det = dict(image_id=image_id,
bbox=det[0:4].tolist(),
score=score,
category_id=int(det[5]))
img_ids.append(image_id)
results.append(coco_det)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % args.log_freq == 0:
print(
'Test: [{0:>4d}/{1}] '
'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) '
.format(
i,
len(loader),
batch_time=batch_time,
rate_avg=input.size(0) / batch_time.avg,
))
json.dump(results, open(args.results, 'w'), indent=4)
'''if 'test' not in args.anno:
coco_results = dataset.coco.loadRes(args.results)
coco_eval = COCOeval(dataset.coco, coco_results, 'bbox')
coco_eval.params.imgIds = img_ids # score only ids we've used
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()'''
return results
