def _initialize_model( self, config_path: str, model_path: Optional[str], label_map: Optional[Dict], extra_config: Optional[Dict], ): config_path, model_path = self.config_parser(config_path, model_path) if config_path.startswith("lp://"): # If it's officially supported by layoutparser dataset_name, model_name = config_path.lstrip("lp://").split("/")[1:3] if label_map is None: label_map = LABEL_MAP_CATALOG[dataset_name] num_classes = len(label_map) model_path = PathManager.get_local_path(model_path) self.model = create_model( model_name, num_classes=num_classes, bench_task="predict", pretrained=True, checkpoint_path=model_path, ) else: assert ( model_path is not None ), f"When the specified model is not layoutparser-based, you need to specify the model_path" assert ( label_map is not None or "num_classes" in extra_config ), "When the specified model is not layoutparser-based, you need to specify the label_map or add num_classes in the extra_config" model_name = config_path model_path = PathManager.get_local_path( model_path ) # It might be an https URL num_classes = len(label_map) if label_map else extra_config["num_classes"] self.model = create_model( model_name, num_classes=num_classes, bench_task="predict", pretrained=True, checkpoint_path=model_path, ) self.model.to(self.device) self.model.eval() self.config = self.model.config self.label_map = label_map if label_map is not None else {}
def export(args): # creat output dir Path(args.out_file).parent.mkdir(parents=True, exist_ok=True) # create model bench = create_model(args.model, bench_task='', checkpoint_path=args.checkpoint) bench.eval() # make dummy run (really required??) dummy_input = torch.randn([1] + [3, 512, 512]) bench(dummy_input) # Export ONNX file input_names = [ "input:0" ] # this are our standardized in/out nameing (required for runtime) output_names = ["output:0", "output:1"] print("Exporting ONNX with input resolution of {} to '{}'".format( args.ONNX_resolution, args.out_file)) torch.onnx._export(bench, dummy_input, args.out_file, opset_version=11, keep_initializers_as_inputs=True, output_names=output_names) #torch.onnx._export(bench, dummy_input, args.out_file, keep_initializers_as_inputs=True, output_names=output_names) print("Saved to {}".format(args.out_file))
def __init__(self, backbone, neck=None, bbox_head=None, mask_feat_head=None, train_cfg=None, test_cfg=None, pretrained=None): super(SingleStageInsDetector, self).__init__() self.eff_backbone_flag = False if 'efficient' not in backbone[ 'type'] else True if self.eff_backbone_flag == False: self.backbone = build_backbone(backbone) else: ''' self.backbone = effdet.create_model(args.model, bench_task='train', num_classes=args.num_classes, pretrained=args.pretrained, pretrained_backbone=args.pretrained_backbone, redundant_bias=args.redundant_bias, label_smoothing=args.smoothing, legacy_focal=args.legacy_focal, jit_loss=args.jit_loss, soft_nms=args.soft_nms, bench_labeler=args.bench_labeler, checkpoint_path=args.initial_checkpoint, ) ''' self.backbone = effdet.create_model( 'tf_efficientdet_d1', bench_task='train', num_classes=num_classes, pretrained=pretrained, pretrained_backbone=pretrained_backbone, redundant_bias=redundant_bias, label_smoothing=smoothing, legacy_focal=legacy_focal, jit_loss=jit_loss, soft_nms=soft_nms, bench_labeler=bench_labeler, checkpoint_path=initial_checkpoint, ) self.with_neck = False self.with_neck = False if neck is not None: self.neck = build_neck(neck) self.with_neck = True if mask_feat_head is not None: self.mask_feat_head = build_head(mask_feat_head) self.with_mask_feat_head = True self.bbox_head = build_head(bbox_head) self.train_cfg = train_cfg self.test_cfg = test_cfg self.init_weights(pretrained=pretrained)
def get_model(self): model = create_model( model_name="tf_efficientdet_d0", bench_task="train", num_classes=1, pretrained=False, checkpoint_path="", checkpoint_ema=False, bench_labeler=True, ) return model
def set_model(model_type, num_classes, checkpoint_path, device): # create model model = create_model(model_type, bench_task='predict', num_classes=num_classes, pretrained=False, redundant_bias=True, checkpoint_path=checkpoint_path) param_count = sum([m.numel() for m in model.parameters()]) print('Model %s created, param count: %d' % (model_type, param_count)) model = model.to(device) return model
def __init__( self, num_classes: int, effdet_backbone: str = "tf_efficientdet_d4", strides: List[int] = [8, 16, 32, 64, 128], sizes: List[Tuple[int, int]] = [(-1, 64), (64, 128), (128, 256), (256, 512), (512, 10000000)], threshold: Optional[float] = None, nms_threshold: Optional[float] = None, *args, **kwargs, ): super().__init__(*args, **kwargs) self.save_hyperparameters() self.num_classes = int(num_classes) self.strides = [int(x) for x in strides] self.sizes = [(int(x), int(y)) for x, y in sizes] # TODO train this from scratch using combustion EfficientDet # self._model = EffDetFCOS.from_predefined( # compound_coeff, self.num_classes, fpn_levels=[3, 5, 7, 8, 9], strides=self.strides # ) self._model = create_model(effdet_backbone, pretrained=True) del self._model.box_net del self._model.class_net fpn_filters = self._model.config.fpn_channels num_repeats = 4 self.fcos = FCOSDecoder(fpn_filters, self.num_classes, num_repeats, strides) self.threshold = float(threshold) if threshold is not None else 0.05 self.nms_threshold = float(nms_threshold) if nms_threshold is not None else 0.1 self._criterion = FCOSLoss(self.strides, self.num_classes, radius=1, interest_range=self.sizes) # metrics metrics = MetricCollection({ f"ap{thresh}": BoxAveragePrecision(iou_threshold=thresh / 100, compute_on_step=True) for thresh in (25, 50, 75) }) self.val_metrics = metrics.clone(prefix="val/") self.test_metrics = metrics.clone(prefix="test/") # freeze backbone for param in self._model.backbone.parameters(): param.requires_grad = False
def validate(args): setup_default_logging() if args.amp: if has_apex: args.apex_amp = True elif has_native_amp: args.native_amp = True assert not args.apex_amp or not args.native_amp, "Only one AMP mode should be set." args.pretrained = args.pretrained or not args.checkpoint # might as well try to validate something args.prefetcher = not args.no_prefetcher # create model with set_layer_config(scriptable=args.torchscript): bench = create_model( args.model, bench_task='predict', num_classes=args.num_classes, pretrained=args.pretrained, redundant_bias=args.redundant_bias, soft_nms=args.soft_nms, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) model_config = bench.config param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() amp_autocast = suppress if args.apex_amp: bench = amp.initialize(bench, opt_level='O1') print('Using NVIDIA APEX AMP. Validating in mixed precision.') elif args.native_amp: amp_autocast = torch.cuda.amp.autocast print('Using native Torch AMP. Validating in mixed precision.') else: print('AMP not enabled. Validating in float32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) dataset = create_dataset(args.dataset, args.root, args.split) input_config = resolve_input_config(args, model_config) loader = create_loader(dataset, input_size=input_config['input_size'], batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=input_config['interpolation'], fill_color=input_config['fill_color'], mean=input_config['mean'], std=input_config['std'], num_workers=args.workers, pin_mem=args.pin_mem) evaluator = create_evaluator(args.dataset, dataset, pred_yxyx=False) bench.eval() batch_time = AverageMeter() end = time.time() last_idx = len(loader) - 1 with torch.no_grad(): for i, (input, target) in enumerate(loader): with amp_autocast(): output = bench(input, img_info=target) evaluator.add_predictions(output, target) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0 or i == last_idx: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format(i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg)) mean_ap = 0. if dataset.parser.has_labels: mean_ap = evaluator.evaluate() else: evaluator.save(args.results) return mean_ap
def validate(args): setup_default_logging() # might as well try to validate something args.pretrained = args.pretrained or not args.checkpoint args.prefetcher = not args.no_prefetcher if args.no_redundant_bias is None: args.redundant_bias = None else: args.redundant_bias = not args.no_redundant_bias # create model bench = create_model( args.model, bench_task='predict', pretrained=args.pretrained, redundant_bias=args.redundant_bias, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) if 'test' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json') image_dir = 'test2017' else: annotation_path = os.path.join(args.data, 'annotations', f'instances_{args.anno}.json') image_dir = args.anno dataset = CocoDetection(os.path.join(args.data, image_dir), annotation_path) loader = create_loader(dataset, input_size=input_size, batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=args.interpolation, fill_color=args.fill_color, num_workers=args.workers, pin_mem=args.pin_mem) img_ids = [] results = [] bench.eval() batch_time = AverageMeter() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(loader): output = bench(input, target['img_scale'], target['img_size']) output = output.cpu() sample_ids = target['img_id'].cpu() for index, sample in enumerate(output): image_id = int(sample_ids[index]) for det in sample: score = float(det[4]) if score < .001: # stop when below this threshold, scores in descending order break coco_det = dict(image_id=image_id, bbox=det[0:4].tolist(), score=score, category_id=int(det[5])) img_ids.append(image_id) results.append(coco_det) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format( i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, )) json.dump(results, open(args.results, 'w'), indent=4) if 'test' not in args.anno: coco_results = dataset.coco.loadRes(args.results) coco_eval = COCOeval(dataset.coco, coco_results, 'bbox') coco_eval.params.imgIds = img_ids # score only ids we've used coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return results
def main(): setup_default_logging() args, args_text = _parse_args() args.pretrained_backbone = not args.no_pretrained_backbone args.prefetcher = not args.no_prefetcher args.distributed = False if 'WORLD_SIZE' in os.environ: args.distributed = int(os.environ['WORLD_SIZE']) > 1 args.device = 'cuda:0' args.world_size = 1 args.rank = 0 # global rank if args.distributed: args.device = 'cuda:%d' % args.local_rank torch.cuda.set_device(args.local_rank) torch.distributed.init_process_group(backend='nccl', init_method='env://') args.world_size = torch.distributed.get_world_size() args.rank = torch.distributed.get_rank() assert args.rank >= 0 if args.distributed: logging.info( 'Training in distributed mode with multiple processes, 1 GPU per process. Process %d, total %d.' % (args.rank, args.world_size)) else: logging.info('Training with a single process on 1 GPU.') torch.manual_seed(args.seed + args.rank) model = create_model( args.model, bench_task='train', pretrained=args.pretrained, pretrained_backbone=args.pretrained_backbone, redundant_bias=args.redundant_bias, checkpoint_path=args.initial_checkpoint, ) # FIXME decide which args to keep and overlay on config / pass to backbone # num_classes=args.num_classes, # drop_rate=args.drop, # drop_path_rate=args.drop_path, # drop_block_rate=args.drop_block, input_size = model.config.image_size if args.local_rank == 0: logging.info('Model %s created, param count: %d' % (args.model, sum([m.numel() for m in model.parameters()]))) model.cuda() optimizer = create_optimizer(args, model) use_amp = False if has_apex and args.amp: model, optimizer = amp.initialize(model, optimizer, opt_level='O1') use_amp = True if args.local_rank == 0: logging.info('NVIDIA APEX {}. AMP {}.'.format( 'installed' if has_apex else 'not installed', 'on' if use_amp else 'off')) # optionally resume from a checkpoint resume_state = {} resume_epoch = None if args.resume: resume_state, resume_epoch = resume_checkpoint(unwrap_bench(model), args.resume) if resume_state and not args.no_resume_opt: if 'optimizer' in resume_state: if args.local_rank == 0: logging.info('Restoring Optimizer state from checkpoint') optimizer.load_state_dict(resume_state['optimizer']) if use_amp and 'amp' in resume_state and 'load_state_dict' in amp.__dict__: if args.local_rank == 0: logging.info('Restoring NVIDIA AMP state from checkpoint') amp.load_state_dict(resume_state['amp']) del resume_state model_ema = None if args.model_ema: # Important to create EMA model after cuda(), DP wrapper, and AMP but before SyncBN and DDP wrapper model_ema = ModelEma(model, decay=args.model_ema_decay) #resume=args.resume) # FIXME bit of a mess with bench if args.resume: load_checkpoint(unwrap_bench(model_ema), args.resume, use_ema=True) if args.distributed: if args.sync_bn: try: if has_apex: model = convert_syncbn_model(model) else: model = torch.nn.SyncBatchNorm.convert_sync_batchnorm( model) if args.local_rank == 0: logging.info( 'Converted model to use Synchronized BatchNorm. WARNING: You may have issues if using ' 'zero initialized BN layers (enabled by default for ResNets) while sync-bn enabled.' ) except Exception as e: logging.error( 'Failed to enable Synchronized BatchNorm. Install Apex or Torch >= 1.1' ) if has_apex: model = DDP(model, delay_allreduce=True) else: if args.local_rank == 0: logging.info( "Using torch DistributedDataParallel. Install NVIDIA Apex for Apex DDP." ) model = DDP(model, device_ids=[args.local_rank ]) # can use device str in Torch >= 1.1 # NOTE: EMA model does not need to be wrapped by DDP lr_scheduler, num_epochs = create_scheduler(args, optimizer) start_epoch = 0 if args.start_epoch is not None: # a specified start_epoch will always override the resume epoch start_epoch = args.start_epoch elif resume_epoch is not None: start_epoch = resume_epoch if lr_scheduler is not None and start_epoch > 0: lr_scheduler.step(start_epoch) if args.local_rank == 0: logging.info('Scheduled epochs: {}'.format(num_epochs)) train_anno_set = 'train2017' train_annotation_path = os.path.join(args.data, 'annotations', f'instances_{train_anno_set}.json') train_image_dir = train_anno_set dataset_train = CocoDetection(os.path.join(args.data, train_image_dir), train_annotation_path) # FIXME cutmix/mixup worth investigating? # collate_fn = None # if args.prefetcher and args.mixup > 0: # collate_fn = FastCollateMixup(args.mixup, args.smoothing, args.num_classes) loader_train = create_loader( dataset_train, input_size=input_size, batch_size=args.batch_size, is_training=True, use_prefetcher=args.prefetcher, #re_prob=args.reprob, # FIXME add back various augmentations #re_mode=args.remode, #re_count=args.recount, #re_split=args.resplit, #color_jitter=args.color_jitter, #auto_augment=args.aa, interpolation=args.train_interpolation, #mean=data_config['mean'], #std=data_config['std'], num_workers=args.workers, distributed=args.distributed, #collate_fn=collate_fn, pin_mem=args.pin_mem, ) train_anno_set = 'val2017' train_annotation_path = os.path.join(args.data, 'annotations', f'instances_{train_anno_set}.json') train_image_dir = train_anno_set dataset_eval = CocoDetection(os.path.join(args.data, train_image_dir), train_annotation_path) loader_eval = create_loader( dataset_eval, input_size=input_size, batch_size=args.validation_batch_size_multiplier * args.batch_size, is_training=False, use_prefetcher=args.prefetcher, interpolation=args.interpolation, #mean=data_config['mean'], #std=data_config['std'], num_workers=args.workers, #distributed=args.distributed, pin_mem=args.pin_mem, ) evaluator = COCOEvaluator(dataset_eval.coco, distributed=args.distributed) eval_metric = args.eval_metric best_metric = None best_epoch = None saver = None output_dir = '' if args.local_rank == 0: output_base = args.output if args.output else './output' exp_name = '-'.join( [datetime.now().strftime("%Y%m%d-%H%M%S"), args.model]) output_dir = get_outdir(output_base, 'train', exp_name) decreasing = True if eval_metric == 'loss' else False saver = CheckpointSaver(checkpoint_dir=output_dir, decreasing=decreasing) with open(os.path.join(output_dir, 'args.yaml'), 'w') as f: f.write(args_text) try: for epoch in range(start_epoch, num_epochs): if args.distributed: loader_train.sampler.set_epoch(epoch) train_metrics = train_epoch(epoch, model, loader_train, optimizer, args, lr_scheduler=lr_scheduler, saver=saver, output_dir=output_dir, use_amp=use_amp, model_ema=model_ema) if args.distributed and args.dist_bn in ('broadcast', 'reduce'): if args.local_rank == 0: logging.info( "Distributing BatchNorm running means and vars") distribute_bn(model, args.world_size, args.dist_bn == 'reduce') # the overhead of evaluating with coco style datasets is fairly high, so just ema or non, not both if model_ema is not None: if args.distributed and args.dist_bn in ('broadcast', 'reduce'): distribute_bn(model_ema, args.world_size, args.dist_bn == 'reduce') eval_metrics = validate(model_ema.ema, loader_eval, args, evaluator, log_suffix=' (EMA)') else: eval_metrics = validate(model, loader_eval, args, evaluator) if lr_scheduler is not None: # step LR for next epoch lr_scheduler.step(epoch + 1, eval_metrics[eval_metric]) if saver is not None: update_summary(epoch, train_metrics, eval_metrics, os.path.join(output_dir, 'summary.csv'), write_header=best_metric is None) # save proper checkpoint with eval metric save_metric = eval_metrics[eval_metric] best_metric, best_epoch = saver.save_checkpoint( unwrap_bench(model), optimizer, args, epoch=epoch, model_ema=unwrap_bench(model_ema), metric=save_metric, use_amp=use_amp) except KeyboardInterrupt: pass if best_metric is not None: logging.info('*** Best metric: {0} (epoch {1})'.format( best_metric, best_epoch))
def validate(args): setup_default_logging() def setthresh(): if args.checkpoint.split("/")[-1].split( "_")[0] in getthresholds.keys(): return getthresholds[args.checkpoint.split("/")[-1].split("_")[0]] else: a = [] [a.append(args.threshold) for x in range(4)] return a threshs = setthresh() print(threshs) # might as well try to validate something args.pretrained = args.pretrained or not args.checkpoint args.prefetcher = not args.no_prefetcher # create model bench = create_model( args.model, bench_task='predict', pretrained=args.pretrained, redundant_bias=args.redundant_bias, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) if 'test' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json') image_dir = args.anno else: annotation_path = os.path.join(args.data, 'annotations', f'instances_{args.anno}.json') image_dir = args.anno print(os.path.join(args.data, image_dir), annotation_path) dataset = CocoDetection(os.path.join(args.data, image_dir), annotation_path) loader = create_loader(dataset, input_size=input_size, batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=args.interpolation, fill_color=args.fill_color, num_workers=args.workers, pin_mem=args.pin_mem, mean=args.mean, std=args.std) if 'test' in args.anno: threshold = float(args.threshold) else: threshold = .001 img_ids = [] results = [] writetofilearrtay = [] bench.eval() batch_time = AverageMeter() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(loader): output = bench(input, target['img_scale'], target['img_size']) output = output.cpu() # print(target['img_id']) sample_ids = target['img_id'].cpu() for index, sample in enumerate(output): image_id = int(sample_ids[index]) for det in sample: score = float(det[4]) if score < threshold: # stop when below this threshold, scores in descending order coco_det = dict(image_id=image_id, category_id=-1) img_ids.append(image_id) results.append(coco_det) break coco_det = dict(image_id=image_id, bbox=det[0:4].tolist(), score=score, category_id=int(det[5]), sizes=target['img_size'].tolist()[0]) img_ids.append(image_id) results.append(coco_det) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format( i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, )) if 'test' in args.anno: from itertools import groupby results.sort(key=lambda x: x['image_id']) f = open( str(args.model) + "-" + str(args.anno) + "-" + str(min(threshs)) + ".txt", "w+") # for item in tqdm(writetofilearrtay): xxx = 0 for k, v in tqdm(groupby(results, key=lambda x: x['image_id'])): xxx += 1 f.write(getimageNamefromid(k) + ",") #print(getimageNamefromid(k),", ") for i in v: if i['category_id'] > 0: if (i['category_id'] ==1 and i['score'] >= threshs[0] ) or (i['category_id'] ==2 and i['score'] >= threshs[1] ) or \ (i['category_id'] ==3 and i['score'] >= threshs[2] ) or (i['category_id'] ==4 and i['score'] >= threshs[3] ) : f.write( str(round(i['category_id'])) + " " + str(round(i['bbox'][0])) + " " + str(round(i['bbox'][1])) + " " + str( round( float(i['bbox'][0]) + float(i['bbox'][2]))) + " " + str( round( float(i['bbox'][1]) + float(i['bbox'][3]))) + " ") f.write('\n') # print(i['category_id']," ",i['bbox'][0]," ",i['bbox'][1]," ",i['bbox'][2]," ",i['bbox'][3]," ") print("generated lines:", xxx) f.close() # f.close() if 'test' not in args.anno: array_of_dm = [] array_of_gt = [] i = 0 # if 'test' in args.anno : for _, item in tqdm(dataset): # if item["img_id"] == "1000780" : # print(item) for i in range(len(item['cls'])): # print(str(item["img_id"]),) array_of_gt.append( BoundingBox(imageName=str(item["img_id"]), classId=item["cls"][i], x=item["bbox"][i][1] * item['img_scale'], y=item["bbox"][i][0] * item['img_scale'], w=item["bbox"][i][3] * item['img_scale'], h=item["bbox"][i][2] * item['img_scale'], typeCoordinates=CoordinatesType.Absolute, bbType=BBType.GroundTruth, format=BBFormat.XYX2Y2, imgSize=(item['img_size'][0], item['img_size'][1]))) for item in tqdm(results): if item["category_id"] >= 0: array_of_dm.append( BoundingBox(imageName=str(item["image_id"]), classId=item["category_id"], classConfidence=item["score"], x=item['bbox'][0], y=item['bbox'][1], w=item['bbox'][2], h=item['bbox'][3], typeCoordinates=CoordinatesType.Absolute, bbType=BBType.Detected, format=BBFormat.XYWH, imgSize=(item['sizes'][0], item['sizes'][1]))) myBoundingBoxes = BoundingBoxes() # # # # Add all bounding boxes to the BoundingBoxes object: for box in (array_of_gt): myBoundingBoxes.addBoundingBox(box) for dm in array_of_dm: myBoundingBoxes.addBoundingBox(dm) evaluator = Evaluator() f1res = [] f1resd0 = [] f1resd10 = [] f1resd20 = [] f1resd40 = [] for conf in tqdm(range(210, 600, 1)): metricsPerClass = evaluator.GetPascalVOCMetrics( myBoundingBoxes, IOUThreshold=0.5, ConfThreshold=conf / 1000.0) totalTP = 0 totalp = 0 totalFP = 0 tp = [] fp = [] ta = [] # print('-------') for mc in metricsPerClass: tp.append(mc['total TP']) fp.append(mc['total FP']) ta.append(mc['total positives']) totalFP = totalFP + mc['total FP'] totalTP = totalTP + mc['total TP'] totalp = totalp + (mc['total positives']) # print(totalTP," ",totalFP," ",totalp) if totalTP + totalFP == 0: p = -1 else: p = totalTP / (totalTP + totalFP) if totalp == 0: r = -1 else: r = totalTP / (totalp) f1_dict = dict(tp=totalTP, fp=totalFP, totalp=totalp, conf=conf / 1000.0, prec=p, rec=r, f1score=(2 * p * r) / (p + r)) f1res.append(f1_dict) #must clean these parts f1resd0.append( dict(tp=tp[0], fp=fp[0], totalp=ta[0], conf=conf / 1000.0, prec=tp[0] / (tp[0] + fp[0]), rec=tp[0] / ta[0], f1score=(2 * (tp[0] / (tp[0] + fp[0])) * (tp[0] / ta[0])) / ((tp[0] / (tp[0] + fp[0])) + (tp[0] / ta[0])))) f1resd10.append( dict(tp=tp[1], fp=fp[1], totalp=ta[1], conf=conf / 1000.0, prec=tp[1] / (tp[1] + fp[1]), rec=tp[1] / ta[1], f1score=(2 * (tp[1] / (tp[1] + fp[1])) * (tp[1] / ta[1])) / ((tp[1] / (tp[1] + fp[1])) + (tp[1] / ta[1])))) f1resd20.append( dict(tp=tp[2], fp=fp[2], totalp=ta[2], conf=conf / 1000.0, prec=tp[2] / (tp[2] + fp[2]), rec=tp[2] / ta[2], f1score=(2 * (tp[2] / (tp[2] + fp[2])) * (tp[2] / ta[2])) / ((tp[2] / (tp[2] + fp[2])) + (tp[2] / ta[2])))) f1resd40.append( dict(tp=tp[3], fp=fp[3], totalp=ta[3], conf=conf / 1000.0, prec=tp[3] / (tp[3] + fp[3]), rec=tp[3] / ta[3], f1score=(2 * (tp[3] / (tp[3] + fp[3])) * (tp[3] / ta[3])) / ((tp[3] / (tp[3] + fp[3])) + (tp[3] / ta[3])))) sortedf1 = sorted(f1res, key=lambda k: k['f1score'], reverse=True) f1resd0 = sorted(f1resd0, key=lambda k: k['f1score'], reverse=True) f1resd10 = sorted(f1resd10, key=lambda k: k['f1score'], reverse=True) f1resd20 = sorted(f1resd20, key=lambda k: k['f1score'], reverse=True) f1resd40 = sorted(f1resd40, key=lambda k: k['f1score'], reverse=True) print(sortedf1[0]) print("\n\n") print(f1resd0[0]) print(f1resd10[0]) print(f1resd20[0]) print(f1resd40[0]) # sortedf1 = sorted(f1res, key=lambda k: k['f1score'],reverse=True) # print(sortedf1[0:2]) # json.dump(results, open(args.results, 'w'), indent=4) json.dump(results, open(args.results, 'w'), indent=4) # coco_results = dataset.coco.loadRes(args.results) # coco_eval = COCOeval(dataset.coco, coco_results, 'bbox') # coco_eval.params.imgIds = img_ids # score only ids we've used # coco_eval.evaluate() # coco_eval.accumulate() # coco_eval.summarize() # print(coco_eval.eval['params']) json.dump(results, open(args.results, 'w'), indent=4) return results
def main(): setup_default_logging() args, args_text = _parse_args() args.pretrained_backbone = not args.no_pretrained_backbone args.prefetcher = not args.no_prefetcher args.distributed = False if 'WORLD_SIZE' in os.environ: args.distributed = int(os.environ['WORLD_SIZE']) > 1 args.device = 'cuda:0' args.world_size = 1 args.rank = 0 # global rank if args.distributed: args.device = 'cuda:%d' % args.local_rank torch.cuda.set_device(args.local_rank) torch.distributed.init_process_group(backend='nccl', init_method='env://') args.world_size = torch.distributed.get_world_size() args.rank = torch.distributed.get_rank() assert args.rank >= 0 if args.distributed: logging.info('Training in distributed mode with multiple processes, 1 GPU per process. Process %d, total %d.' % (args.rank, args.world_size)) else: logging.info('Training with a single process on 1 GPU.') use_amp = None if args.amp: # for backwards compat, `--amp` arg tries apex before native amp if has_apex: args.apex_amp = True elif has_native_amp: args.native_amp = True else: logging.warning("Neither APEX or native Torch AMP is available, using float32. " "Install NVIDA apex or upgrade to PyTorch 1.6.") if args.apex_amp: if has_apex: use_amp = 'apex' else: logging.warning("APEX AMP not available, using float32. Install NVIDA apex") elif args.native_amp: if has_native_amp: use_amp = 'native' else: logging.warning("Native AMP not available, using float32. Upgrade to PyTorch 1.6.") torch.manual_seed(args.seed + args.rank) model = create_model( args.model, bench_task='train', num_classes=args.num_classes, pretrained=args.pretrained, pretrained_backbone=args.pretrained_backbone, redundant_bias=args.redundant_bias, label_smoothing=args.smoothing, new_focal=args.new_focal, jit_loss=args.jit_loss, bench_labeler=args.bench_labeler, checkpoint_path=args.initial_checkpoint, ) model_config = model.config # grab before we obscure with DP/DDP wrappers if args.local_rank == 0: logging.info('Model %s created, param count: %d' % (args.model, sum([m.numel() for m in model.parameters()]))) model.cuda() if args.channels_last: model = model.to(memory_format=torch.channels_last) optimizer = create_optimizer(args, model) amp_autocast = suppress # do nothing loss_scaler = None if use_amp == 'apex': model, optimizer = amp.initialize(model, optimizer, opt_level='O1') loss_scaler = ApexScaler() if args.local_rank == 0: logging.info('Using NVIDIA APEX AMP. Training in mixed precision.') elif use_amp == 'native': amp_autocast = torch.cuda.amp.autocast loss_scaler = NativeScaler() if args.local_rank == 0: logging.info('Using native Torch AMP. Training in mixed precision.') else: if args.local_rank == 0: logging.info('AMP not enabled. Training in float32.') # optionally resume from a checkpoint resume_epoch = None if args.resume: resume_epoch = resume_checkpoint( unwrap_bench(model), args.resume, optimizer=None if args.no_resume_opt else optimizer, loss_scaler=None if args.no_resume_opt else loss_scaler, log_info=args.local_rank == 0) model_ema = None if args.model_ema: # Important to create EMA model after cuda(), DP wrapper, and AMP but before SyncBN and DDP wrapper model_ema = ModelEma(model, decay=args.model_ema_decay) if args.resume: # FIXME bit of a mess with bench, cannot use the load in ModelEma load_checkpoint(unwrap_bench(model_ema), args.resume, use_ema=True) if args.distributed: if args.sync_bn: try: if has_apex and use_amp != 'native': model = convert_syncbn_model(model) else: model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) if args.local_rank == 0: logging.info( 'Converted model to use Synchronized BatchNorm. WARNING: You may have issues if using ' 'zero initialized BN layers (enabled by default for ResNets) while sync-bn enabled.') except Exception as e: logging.error('Failed to enable Synchronized BatchNorm. Install Apex or Torch >= 1.1') if has_apex and use_amp != 'native': if args.local_rank == 0: logging.info("Using apex DistributedDataParallel.") model = ApexDDP(model, delay_allreduce=True) else: if args.local_rank == 0: logging.info("Using torch DistributedDataParallel.") model = NativeDDP(model, device_ids=[args.device]) # NOTE: EMA model does not need to be wrapped by DDP lr_scheduler, num_epochs = create_scheduler(args, optimizer) start_epoch = 0 if args.start_epoch is not None: # a specified start_epoch will always override the resume epoch start_epoch = args.start_epoch elif resume_epoch is not None: start_epoch = resume_epoch if lr_scheduler is not None and start_epoch > 0: lr_scheduler.step(start_epoch) if args.local_rank == 0: logging.info('Scheduled epochs: {}'.format(num_epochs)) loader_train, loader_eval, evaluator = create_datasets_and_loaders(args, model_config) if model_config.num_classes < loader_train.dataset.parser.max_label: logging.error( f'Model {model_config.num_classes} has fewer classes than dataset {loader_train.dataset.parser.max_label}.') exit(1) if model_config.num_classes > loader_train.dataset.parser.max_label: logging.warning( f'Model {model_config.num_classes} has more classes than dataset {loader_train.dataset.parser.max_label}.') eval_metric = args.eval_metric best_metric = None best_epoch = None saver = None output_dir = '' if args.local_rank == 0: output_base = args.output if args.output else './output' exp_name = '-'.join([ datetime.now().strftime("%Y%m%d-%H%M%S"), args.model ]) output_dir = get_outdir(output_base, 'train', exp_name) decreasing = True if eval_metric == 'loss' else False saver = CheckpointSaver( model, optimizer, args=args, model_ema=model_ema, amp_scaler=loss_scaler, checkpoint_dir=output_dir, decreasing=decreasing, unwrap_fn=unwrap_bench) with open(os.path.join(output_dir, 'args.yaml'), 'w') as f: f.write(args_text) try: for epoch in range(start_epoch, num_epochs): if args.distributed: loader_train.sampler.set_epoch(epoch) train_metrics = train_epoch( epoch, model, loader_train, optimizer, args, lr_scheduler=lr_scheduler, saver=saver, output_dir=output_dir, amp_autocast=amp_autocast, loss_scaler=loss_scaler, model_ema=model_ema) if args.distributed and args.dist_bn in ('broadcast', 'reduce'): if args.local_rank == 0: logging.info("Distributing BatchNorm running means and vars") distribute_bn(model, args.world_size, args.dist_bn == 'reduce') # the overhead of evaluating with coco style datasets is fairly high, so just ema or non, not both if model_ema is not None: if args.distributed and args.dist_bn in ('broadcast', 'reduce'): distribute_bn(model_ema, args.world_size, args.dist_bn == 'reduce') eval_metrics = validate(model_ema.ema, loader_eval, args, evaluator, log_suffix=' (EMA)') else: eval_metrics = validate(model, loader_eval, args, evaluator) if lr_scheduler is not None: # step LR for next epoch lr_scheduler.step(epoch + 1, eval_metrics[eval_metric]) if saver is not None: update_summary( epoch, train_metrics, eval_metrics, os.path.join(output_dir, 'summary.csv'), write_header=best_metric is None) # save proper checkpoint with eval metric best_metric, best_epoch = saver.save_checkpoint(epoch=epoch, metric=eval_metrics[eval_metric]) except KeyboardInterrupt: pass if best_metric is not None: logging.info('*** Best metric: {0} (epoch {1})'.format(best_metric, best_epoch))
def validate(args): setup_default_logging() # might as well try to validate something args.pretrained = args.pretrained or not args.checkpoint args.prefetcher = not args.no_prefetcher # create model bench = create_model(args.model, bench_task='predict', pretrained=args.pretrained, redundant_bias=args.redundant_bias, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema) input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) if 'test' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json') image_dir = args.anno else: annotation_path = os.path.join(args.data, 'annotations', f'instances_{args.anno}.json') image_dir = args.anno dataset = CocoDetection(os.path.join(args.data, image_dir), annotation_path) loader = create_loader(dataset, input_size=input_size, batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=args.interpolation, fill_color=args.fill_color, num_workers=args.workers, mean=args.mean, std=args.std, pin_mem=args.pin_mem) img_ids = [] results = [] bench.eval() for i, (input, target) in enumerate(loader, 1): dumm_inp = input tisc = target['img_scale'] tisz = target['img_size'] break starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event( enable_timing=True) # repetitions = 300 # timings=np.zeros((repetitions,1)) #GPU-WARM-UP # print(enumerate()) for _ in range(10): _ = bench(dumm_inp, tisc, tisz) # MEASURE PERFORMANCE # dummy_input = torch.randn(1, 3,bench.config.image_size,bench.config.image_size,dtype=torch.float).to("cuda") print("starting") batch_time = AverageMeter() # end = time.time() with torch.no_grad(): for _ in range(2000): starter.record() _ = bench(dumm_inp, tisc, tisz) ender.record() # measure elapsed time torch.cuda.synchronize() curr_time = starter.elapsed_time(ender) batch_time.update(curr_time) # print(curr_time) # end = time.time() # if i % args.log_freq == 0: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}ms ({batch_time.avg:.3f}ms, {rate_avg:>7.2f}/s) ' .format( i, len(loader), batch_time=batch_time, rate_avg=dumm_inp.size(0) / batch_time.avg, )) # json.dump(results, open(args.results, 'w'), indent=4) # if 'test' not in args.anno: # coco_results = dataset.coco.loadRes(args.results) # coco_eval = COCOeval(dataset.coco, coco_results, 'bbox') # coco_eval.params.imgIds = img_ids # score only ids we've used # coco_eval.evaluate() # coco_eval.accumulate() # coco_eval.summarize() return results
def eval_model(model_name, paper_model_name, paper_arxiv_id, batch_size=64, model_description=''): # create model bench = create_model( model_name, bench_task='predict', pretrained=True, ) bench.eval() input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (model_name, param_count)) bench = bench.cuda() if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.') annotation_path = os.path.join(DATA_ROOT, 'annotations', f'instances_{ANNO_SET}.json') evaluator = COCOEvaluator( root=DATA_ROOT, model_name=paper_model_name, model_description=model_description, paper_arxiv_id=paper_arxiv_id) dataset = CocoDetection(os.path.join(DATA_ROOT, ANNO_SET), annotation_path) loader = create_loader( dataset, input_size=input_size, batch_size=batch_size, use_prefetcher=True, fill_color='mean', num_workers=4, pin_mem=True) iterator = tqdm.tqdm(loader, desc="Evaluation", mininterval=5) evaluator.reset_time() with torch.no_grad(): for i, (input, target) in enumerate(iterator): output = bench(input, target['img_scale'], target['img_size']) output = output.cpu() sample_ids = target['img_id'].cpu() results = [] for index, sample in enumerate(output): image_id = int(sample_ids[index]) for det in sample: score = float(det[4]) if score < .001: # stop when below this threshold, scores in descending order break coco_det = dict( image_id=image_id, bbox=det[0:4].tolist(), score=score, category_id=int(det[5])) results.append(coco_det) evaluator.add(results) if evaluator.cache_exists: break evaluator.save()
def __init__(self, context: PyTorchTrialContext) -> None: self.context = context self.hparam = self.context.get_hparam self.args = DotDict(self.context.get_hparams()) # Create a unique download directory for each rank so they don't overwrite each other. self.download_directory = f"/tmp/data-rank{self.context.distributed.get_rank()}" self.num_slots = int(self.context.get_experiment_config()['resources'] ['slots_per_trial']) if self.args.sync_bn and self.num_slots == 1: print( 'Can not use sync_bn with one slot. Either set sync_bn to False or use distributed training.' ) sys.exit() self.args.pretrained_backbone = not self.args.no_pretrained_backbone self.args.prefetcher = not self.args.no_prefetcher tmp = [] for arg in self.args.lr_noise.split(' '): tmp.append(float(arg)) self.args.lr_noise = tmp self.model = create_model( self.args.model, bench_task='train', num_classes=self.args.num_classes, pretrained=self.args.pretrained, pretrained_backbone=self.args.pretrained_backbone, redundant_bias=self.args.redundant_bias, label_smoothing=self.args.smoothing, new_focal=self.args.new_focal, jit_loss=self.args.jit_loss, bench_labeler=self.args.bench_labeler, checkpoint_path=self.args.initial_checkpoint, ) self.model_config = self.model.config self.input_config = resolve_input_config( self.args, model_config=self.model_config) print('h: ', self.args.model, sum([m.numel() for m in self.model.parameters()])) if self.args.sync_bn: print('creating batch sync model') if self.args.model_ema: print('creating batch sync ema model') self.model_ema = self.context.wrap_model(deepcopy(self.model)) self.model = self.convert_syncbn_model(self.model) self.model = self.context.wrap_model(self.model) print('Model created, param count:', self.args.model, sum([m.numel() for m in self.model.parameters()])) self.optimizer = self.context.wrap_optimizer( create_optimizer(self.args, self.model)) print('Created optimizer: ', self.optimizer) if self.args.amp: print('using amp') if self.args.sync_bn and self.args.model_ema: print('using sync_bn and model_ema when creating apex_amp') (self.model, self.model_ema ), self.optimizer = self.context.configure_apex_amp( [self.model, self.model_ema], self.optimizer, min_loss_scale=self.hparam("min_loss_scale")) else: self.model, self.optimizer = self.context.configure_apex_amp( self.model, self.optimizer, min_loss_scale=self.hparam("min_loss_scale")) if self.args.model_ema: print('using model ema') if self.args.sync_bn: print('using model ema batch syn') self.model_ema = ModelEma(self.model_ema, context=self.context, decay=self.args.model_ema_decay) else: # Important to create EMA model after cuda(), DP wrapper, and AMP but before SyncBN and DDP wrapper self.model_ema = ModelEma(self.model, context=self.context, decay=self.args.model_ema_decay) self.lr_scheduler, self.num_epochs = create_scheduler( self.args, self.optimizer) self.lr_scheduler = self.context.wrap_lr_scheduler( self.lr_scheduler, LRScheduler.StepMode.MANUAL_STEP) self.cur_epoch = 0 self.num_updates = 0 * self.cur_epoch if self.args.prefetcher: self.train_mean, self.train_std, self.train_random_erasing = self.calculate_means( mean=self.input_config['mean'], std=self.input_config['std'], re_prob=self.args.reprob, re_mode=self.args.remode, re_count=self.args.recount) self.val_mean, self.val_std, self.val_random_erasing = self.calculate_means( self.input_config['mean'], self.input_config['std']) self.val_reducer = self.context.wrap_reducer(self.validation_reducer, for_training=False)
def validate(args): setup_default_logging() def setthresh(): if args.checkpoint.split("/")[-1].split( "_")[0] in getthresholds.keys(): return getthresholds[args.checkpoint.split("/")[-1].split("_")[0]] else: a = [] [a.append(args.threshold) for x in range(4)] return a # might as well try to validate something args.pretrained = args.pretrained or not args.checkpoint args.prefetcher = not args.no_prefetcher # create model bench = create_model( args.model, bench_task='predict', pretrained=args.pretrained, redundant_bias=args.redundant_bias, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) if 'test' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json') image_dir = args.anno elif 'val' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'instances_{args.anno}.json') image_dir = args.anno # else: # annotation_path = os.path.join(args.data, f'{args.anno}.json') # image_dir = args.anno print(os.path.join(args.data, image_dir), annotation_path) dataset = CocoDetection(os.path.join(args.data, image_dir), annotation_path) loader = create_loader(dataset, input_size=input_size, batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=args.interpolation, fill_color=args.fill_color, num_workers=args.workers, pin_mem=args.pin_mem, mean=args.mean, std=args.std) if 'test' in args.anno: threshold = float(args.threshold) # elif 'detector' in args.anno: # threshold = min(getthresholds['d0']) else: threshold = .001 img_ids = [] results = [] writetofilearrtay = [] bench.eval() batch_time = AverageMeter() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(loader): output = bench(input, target['img_scale'], target['img_size']) output = output.cpu() # print(target['img_id']) sample_ids = target['img_id'].cpu() for index, sample in enumerate(output): image_id = int(sample_ids[index]) # if 'test' in args.anno : # tempWritetoFile = [] # tempWritetoFile.append(getimageNamefromid(image_id)) for det in sample: score = float(det[4]) if score < threshold: # stop when below this threshold, scores in descending order coco_det = dict(image_id=image_id, category_id=-1) img_ids.append(image_id) results.append(coco_det) break coco_det = dict(image_id=image_id, bbox=det[0:4].tolist(), score=score, category_id=int(det[5]), sizes=target['img_size'].tolist()[0]) img_ids.append(image_id) results.append(coco_det) # exit() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format( i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, )) # if 'test' in args.anno : if not os.path.exists(args.tosave): os.makedirs(args.tosave) from itertools import groupby results.sort(key=lambda x: x['image_id']) count = 0 for k, v in tqdm(groupby(results, key=lambda x: x['image_id'])): # print(args.data +"/" + str(getimageNamefromid(k))) img = drawonimage( os.path.join(args.data, image_dir, str(getimageNamefromid(k))), v, setthresh()) cv2.imwrite(args.tosave + "/" + str(getimageNamefromid(k)), img) count += 1 # print(i['category_id']," ",i['bbox'][0]," ",i['bbox'][1]," ",i['bbox'][2]," ",i['bbox'][3]," ") print("generated predictions for ", count, " images.") return results
def validate(args): setup_default_logging() if args.amp: if has_apex: args.apex_amp = True elif has_native_amp: args.native_amp = True assert not args.apex_amp or not args.native_amp, "Only one AMP mode should be set." args.pretrained = args.pretrained or not args.checkpoint # might as well try to validate something args.prefetcher = not args.no_prefetcher # create model with set_layer_config(scriptable=args.torchscript): bench = create_model( args.model, bench_task='predict', num_classes=args.num_classes, pretrained=args.pretrained, redundant_bias=args.redundant_bias, soft_nms=args.soft_nms, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) model_config = bench.config param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() amp_autocast = suppress if args.apex_amp: bench = amp.initialize(bench, opt_level='O1') print('Using NVIDIA APEX AMP. Validating in mixed precision.') elif args.native_amp: amp_autocast = torch.cuda.amp.autocast print('Using native Torch AMP. Validating in mixed precision.') else: print('AMP not enabled. Validating in float32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) dataset = create_dataset(args.dataset, args.root, args.split) input_config = resolve_input_config(args, model_config) loader = create_loader(dataset, input_size=input_config['input_size'], batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=input_config['interpolation'], fill_color=input_config['fill_color'], mean=input_config['mean'], std=input_config['std'], num_workers=args.workers, pin_mem=args.pin_mem) evaluator = create_evaluator(args.dataset, dataset, pred_yxyx=False) bench.eval() batch_time = AverageMeter() end = time.time() last_idx = len(loader) - 1 imgs = [] with torch.no_grad(): for i, (input, target) in enumerate(loader): for b in range(input.shape[0]): imgs.append(input[b].cpu().numpy()) # targets.append(target[b].cpu().numpy()) with amp_autocast(): output = bench(input, img_info=target) evaluator.add_predictions(output, target) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0 or i == last_idx: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format(i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg)) mean_ap = 0. if dataset.parser.has_labels: preds = [p[:2, :] for p in evaluator.predictions] anns = evaluator.coco_api.imgToAnns targets = [ np.asarray((anns[k][0]['bbox'], anns[k][1]['bbox'])) for k in range(len(imgs)) ] mean_ap = evaluator.evaluate() if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) for i, img in enumerate(imgs): img = imgs[i] img_m = np.mean(img, axis=0) for c in range(3): img[c] = img_m img_ = img.transpose(1, 2, 0) m = img_.min() M = img_.max() img_ = ((img_ - m) / (M - m) * 255).astype('uint8').copy() img_ = draw_bbox(img_, preds[i], targets[i]) cv2.imwrite(os.path.join(args.out_dir, '%d.jpg' % i), img_) else: evaluator.save(args.results) return mean_ap
def validate_det(args): setup_default_logging() # might as well try to validate something args.pretrained = args.checkpoint args.prefetcher = not args.no_prefetcher # create model bench = create_model( args.model, bench_task='predict', num_classes=args.num_classes, pretrained=args.pretrained, redundant_bias=args.redundant_bias, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) model_config = bench.config input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) #print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() '''if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.')''' if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) '''if 'test' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json') image_dir = 'test2017''' '''annotation_path = os.path.join(args.data, 'Highway_drop_inlets.v1-vdot_coco.coco/coco_and_dropinlets_annotations/test_annotations', f'{args.anno}.json') image_dir = os.path.join(args.data, 'Highway_drop_inlets.v1-vdot_coco.coco/coco_and_dropinlets/test/') dataset = CocoDetection(image_dir, annotation_path)''' annotation_path = os.path.join(args.data, f'{args.anno}.json') image_dir = os.path.join(args.data, 'others_mix_set') dataset = VdotTestDataset(image_dir, annotation_path) loader = create_loader(dataset, input_size=input_size, batch_size=args.batches_size, use_prefetcher=args.prefetcher, interpolation=args.interpolation, fill_color=args.fill_color, num_workers=args.workers, pin_mem=args.pin_mem) img_ids = [] results = [] bench.eval() #example_input = torch.randn((1, 3, 512, 512), requires_grad=True) #bench(example_input.cuda()) '''tracingModelInput = torch.ones(1,3,512,512) torch.onnx._export(bench.model, tracingModelInput.cuda(), './effdet0_checkonly.onnx', opset_version=11, export_params=True) print('\nDone exporting ONNX model!') onnx_model='./effdet0_checkonly.onnx' onnx.checker.check_model(onnx_model)''' '''dummy_input = torch.randn(1, 3, 512, 512, device='cuda') input =["input"] output=["class_1","class_2","class_3","class_4", "class_5", "box_1", "box_2", "box_3","box_4","box_5"] #dynamic_axes = {"actual_input_1":{input:"batch_size"}, "output1":{output:"batch_size"}} torch.onnx.export(bench.model, dummy_input, "effdet0_upsamplenormal.onnx", verbose=True, opset_version=11, input_names=input, output_names=output)''' #dynamic_axes=dynamic_axes) batch_time = AverageMeter() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(loader): output = bench(input, img_info=target) output = output.cpu() sample_ids = target['img_id'].cpu() for index, sample in enumerate(output): image_id = int(sample_ids[index]) for det in sample: score = float(det[4]) if score < 0.36: # stop when below this threshold, scores in descending order(perfect 0.5 for 91 classes) break coco_det = dict(image_id=image_id, bbox=det[0:4].tolist(), score=score, category_id=int(det[5])) img_ids.append(image_id) results.append(coco_det) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format( i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, )) json.dump(results, open(args.results, 'w'), indent=4) '''if 'test' not in args.anno: coco_results = dataset.coco.loadRes(args.results) coco_eval = COCOeval(dataset.coco, coco_results, 'bbox') coco_eval.params.imgIds = img_ids # score only ids we've used coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize()''' return results