def main(): setup_default_logging() args, args_text = parse_args() if args.smoke_test: ray.init(num_cpus=int(args.ray_num_workers)) else: ray.init(address=args.ray_address) CustomTrainingOperator = TrainingOperator.from_creators( model_creator=model_creator, optimizer_creator=optimizer_creator, data_creator=data_creator, loss_creator=loss_creator) trainer = TorchTrainer(training_operator_cls=CustomTrainingOperator, use_tqdm=True, use_fp16=args.amp, apex_args={"opt_level": "O1"}, config={ "args": args, BATCH_SIZE: args.batch_size }, num_workers=args.ray_num_workers) if args.smoke_test: args.epochs = 1 pbar = trange(args.epochs, unit="epoch") for i in pbar: trainer.train(num_steps=1 if args.smoke_test else None) val_stats = trainer.validate(num_steps=1 if args.smoke_test else None) pbar.set_postfix(dict(acc=val_stats["val_accuracy"])) trainer.shutdown()
def main(args): setup_default_logging() logging.info('Running validation on {}'.format(args.dataset)) net_config = json.load(open(args.model)) if 'img_size' in net_config: img_size = net_config['img_size'] else: img_size = args.img_size test_loader = get_dataloader( dataset=args.dataset, data=args.data, test_batch_size=args.batch_size, n_worker=args.workers, image_size=img_size).test model = NATNet.build_from_config(net_config, pretrained=args.pretrained) param_count = sum([m.numel() for m in model.parameters()]) logging.info('Model created, param count: %d' % param_count) model = model.cuda() criterion = torch.nn.CrossEntropyLoss().cuda() validate(model, test_loader, criterion) return
def main(args): setup_default_logging() logging.info('Running validation on {}'.format(args.dataset)) net_config = json.load(open(args.model)) if 'img_size' in net_config: img_size = net_config['img_size'] else: img_size = args.img_size run_config = get_run_config(dataset=args.dataset, data_path=args.data, image_size=img_size, n_epochs=0, train_batch_size=args.batch_size, test_batch_size=args.batch_size, n_worker=args.workers, valid_size=None) model = NSGANetV2.build_from_config(net_config) param_count = sum([m.numel() for m in model.parameters()]) logging.info('Model created, param count: %d' % param_count) model = model.cuda() criterion = torch.nn.CrossEntropyLoss().cuda() validate(model, run_config.test_loader, criterion) return
def main(): args = parser.parse_args() model_cfgs = [] model_names = [] if os.path.isdir(args.checkpoint): # validate all checkpoints in a path with same model checkpoints = glob.glob(args.checkpoint + '/*.pth.tar') checkpoints += glob.glob(args.checkpoint + '/*.pth') model_names = list_models(args.model) model_cfgs = [(args.model, c) for c in sorted(checkpoints, key=natural_key)] else: if args.model == 'all': # validate all models in a list of names with pretrained checkpoints args.pretrained = True model_names = list_models(pretrained=True, exclude_filters=['*in21k']) model_cfgs = [(n, '') for n in model_names] elif not is_model(args.model): # model name doesn't exist, try as wildcard filter model_names = list_models(args.model) model_cfgs = [(n, '') for n in model_names] setup_default_logging(log_path=os.path.join(os.path.dirname(args.checkpoint) , "eval_print_log.txt")) if len(model_cfgs): #results_file = args.results_file or './results-all.csv' results_file = args.results_file or os.path.join(os.path.dirname(args.checkpoint) , "eval_results.csv.txt") _logger.info('Running bulk validation on these pretrained models: {}'.format(', '.join(model_names))) results = [] try: start_batch_size = args.batch_size for m, c in model_cfgs: batch_size = start_batch_size args.model = m args.checkpoint = c result = OrderedDict(model=args.model) r = {} while not r and batch_size >= args.num_gpu: torch.cuda.empty_cache() try: args.batch_size = batch_size print('Validating with batch size: %d' % args.batch_size) r = validate(args) except RuntimeError as e: if batch_size <= args.num_gpu: print("Validation failed with no ability to reduce batch size. Exiting.") raise e batch_size = max(batch_size // 2, args.num_gpu) print("Validation failed, reducing batch size by 50%") result.update(r) if args.checkpoint: result['checkpoint'] = args.checkpoint results.append(result) except KeyboardInterrupt as e: pass results = sorted(results, key=lambda x: x['top1'], reverse=True) if len(results): write_results(results_file, results) else: validate(args)
def main(): setup_default_logging() args = parser.parse_args() model_cfgs = [] model_names = [] if os.path.isdir(args.checkpoint): # validate all checkpoints in a path with same model checkpoints = glob.glob(args.checkpoint + '/*.pth.tar') checkpoints += glob.glob(args.checkpoint + '/*.pth') model_names = list_models(args.model) model_cfgs = [(args.model, c) for c in sorted(checkpoints, key=natural_key)] else: if args.model == 'all': # validate all models in a list of names with pretrained checkpoints args.pretrained = True model_names = list_models( pretrained=True, exclude_filters=['*_in21k', '*_in22k', '*_dino']) model_cfgs = [(n, '') for n in model_names] elif not is_model(args.model): # model name doesn't exist, try as wildcard filter model_names = list_models(args.model) model_cfgs = [(n, '') for n in model_names] if not model_cfgs and os.path.isfile(args.model): with open(args.model) as f: model_names = [line.rstrip() for line in f] model_cfgs = [(n, None) for n in model_names if n] if len(model_cfgs): results_file = args.results_file or './results-all.csv' _logger.info( 'Running bulk validation on these pretrained models: {}'.format( ', '.join(model_names))) results = [] try: initial_batch_size = args.batch_size for m, c in model_cfgs: args.model = m args.checkpoint = c r = _try_run(args, initial_batch_size) if 'error' in r: continue if args.checkpoint: r['checkpoint'] = args.checkpoint results.append(r) except KeyboardInterrupt as e: pass results = sorted(results, key=lambda x: x['top1'], reverse=True) if len(results): write_results(results_file, results) else: results = validate(args) # output results in JSON to stdout w/ delimiter for runner script print(f'--result\n{json.dumps(results, indent=4)}')
def main(): setup_default_logging() args = parser.parse_args() model_cfgs = [] model_names = [] if args.model_list: args.model = '' with open(args.model_list) as f: model_names = [line.rstrip() for line in f] model_cfgs = [(n, None) for n in model_names] elif args.model == 'all': # validate all models in a list of names with pretrained checkpoints args.pretrained = True model_names = list_models(pretrained=True, exclude_filters=['*in21k']) model_cfgs = [(n, None) for n in model_names] elif not is_model(args.model): # model name doesn't exist, try as wildcard filter model_names = list_models(args.model) model_cfgs = [(n, None) for n in model_names] if len(model_cfgs): results_file = args.results_file or './benchmark.csv' _logger.info( 'Running bulk validation on these pretrained models: {}'.format( ', '.join(model_names))) results = [] try: for m, _ in model_cfgs: if not m: continue args.model = m r = benchmark(args) if r: results.append(r) time.sleep(10) except KeyboardInterrupt as e: pass sort_key = 'infer_samples_per_sec' if 'train' in args.bench: sort_key = 'train_samples_per_sec' elif 'profile' in args.bench: sort_key = 'infer_gmacs' results = filter(lambda x: sort_key in x, results) results = sorted(results, key=lambda x: x[sort_key], reverse=True) if len(results): write_results(results_file, results) else: results = benchmark(args) # output results in JSON to stdout w/ delimiter for runner script print(f'--result\n{json.dumps(results, indent=4)}')
def main(): setup_default_logging() args = parser.parse_args() model_cfgs = [] model_names = [] if os.path.isdir(args.checkpoint): # validate all checkpoints in a path with same model checkpoints = glob.glob(args.checkpoint + '/*.pth.tar') checkpoints += glob.glob(args.checkpoint + '/*.pth') model_names = list_models(args.model) model_cfgs = [(args.model, c) for c in sorted(checkpoints, key=natural_key)] else: if args.model == 'all': # validate all models in a list of names with pretrained checkpoints args.pretrained = True model_names = list_models(pretrained=True) model_cfgs = [(n, '') for n in model_names] elif not is_model(args.model): # model name doesn't exist, try as wildcard filter model_names = list_models(args.model) model_cfgs = [(n, '') for n in model_names] if len(model_cfgs): results_file = args.results_file or './results-all.csv' logging.info( 'Running bulk validation on these pretrained models: {}'.format( ', '.join(model_names))) results = [] try: for m, c in model_cfgs: args.model = m args.checkpoint = c result = OrderedDict(model=args.model) r = validate(args) result.update(r) if args.checkpoint: result['checkpoint'] = args.checkpoint results.append(result) except KeyboardInterrupt as e: pass results = sorted(results, key=lambda x: x['top1'], reverse=True) if len(results): write_results(results_file, results) else: validate(args)
def main(): setup_default_logging() args = parser.parse_args() model_cfgs = [] model_names = [] if os.path.isdir(args.checkpoint): # validate all checkpoints in a path with same model checkpoints = glob.glob(args.checkpoint + '/*.pth.tar') checkpoints += glob.glob(args.checkpoint + '/*.pth') model_names = list_models(args.model) model_cfgs = [(args.model, c) for c in sorted(checkpoints, key=natural_key)] else: if args.model == 'all': # validate all models in a list of names with pretrained checkpoints args.pretrained = True model_names = list_models(pretrained=True) model_cfgs = [(n, '') for n in model_names] elif not is_model(args.model): # model name doesn't exist, try as wildcard filter model_names = list_models(args.model) model_cfgs = [(n, '') for n in model_names] if len(model_cfgs): logging.info( 'Running bulk validation on these pretrained models: {}'.format( ', '.join(model_names))) header_written = False with open('./results-all.csv', mode='w') as cf: for m, c in model_cfgs: args.model = m args.checkpoint = c result = OrderedDict(model=args.model) r = validate(args) result.update(r) if args.checkpoint: result['checkpoint'] = args.checkpoint dw = csv.DictWriter(cf, fieldnames=result.keys()) if not header_written: dw.writeheader() header_written = True dw.writerow(result) cf.flush() else: validate(args)
def main(): args = parser.parse_args() setup_default_logging(log_path=args.log_path) _logger = logging.getLogger('inference') device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # create model model = create_model(args.model, num_classes=args.num_classes, in_chans=3, checkpoint_path=args.checkpoint) data_config = resolve_data_config(vars(args), model=model) # 默认配置 mean, std img_transform = create_transform(data_config['input_size'], is_training=False, mean=data_config['mean'], std=data_config['std'], crop_pct=data_config['crop_pct']) model = model.to(device) model.eval() # confusion matrix conf_matrix = np.zeros((args.num_classes, args.num_classes)) # predict class_dir = os.listdir(args.data) for cl in class_dir: imgs = os.listdir(os.path.join(args.data, cl)) for img in imgs: im = Image.open(os.path.join(args.data, cl, img)) im1 = img_transform(im).unsqueeze(0) im1 = im1.to(device) outputs = model(im1) idx = torch.max(outputs, 1)[1].tolist()[0] probs = torch.nn.functional.softmax(outputs, dim=1)[0].tolist() _logger.info('%s: %d, %.2f' % (os.path.join(args.data, cl, img), idx, probs[idx])) conf_matrix[int(cl), idx] += 1 _logger.info(conf_matrix)
def main(): setup_default_logging() args = parser.parse_args() model_cfgs = [] model_names = [] if os.path.isdir(args.checkpoint): # validate all checkpoints in a path with same model checkpoints = glob.glob(args.checkpoint + '/*.pth.tar') checkpoints += glob.glob(args.checkpoint + '/*.pth') model_names = list_models(args.model) model_cfgs = [(args.model, c) for c in sorted(checkpoints, key=natural_key)] else: if args.model == 'all': # validate all models in a list of names with pretrained checkpoints args.pretrained = True model_names = list_models(pretrained=True) model_cfgs = [(n, '') for n in model_names] elif not is_model(args.model): # model name doesn't exist, try as wildcard filter model_names = list_models(args.model) model_cfgs = [(n, '') for n in model_names] validate(args)
def validate(args): setup_default_logging() # might as well try to validate something args.pretrained = args.pretrained or not args.checkpoint args.prefetcher = not args.no_prefetcher if args.no_redundant_bias is None: args.redundant_bias = None else: args.redundant_bias = not args.no_redundant_bias # create model bench = create_model( args.model, bench_task='predict', pretrained=args.pretrained, redundant_bias=args.redundant_bias, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) if 'test' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json') image_dir = 'test2017' else: annotation_path = os.path.join(args.data, 'annotations', f'instances_{args.anno}.json') image_dir = args.anno dataset = CocoDetection(os.path.join(args.data, image_dir), annotation_path) loader = create_loader(dataset, input_size=input_size, batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=args.interpolation, fill_color=args.fill_color, num_workers=args.workers, pin_mem=args.pin_mem) img_ids = [] results = [] bench.eval() batch_time = AverageMeter() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(loader): output = bench(input, target['img_scale'], target['img_size']) output = output.cpu() sample_ids = target['img_id'].cpu() for index, sample in enumerate(output): image_id = int(sample_ids[index]) for det in sample: score = float(det[4]) if score < .001: # stop when below this threshold, scores in descending order break coco_det = dict(image_id=image_id, bbox=det[0:4].tolist(), score=score, category_id=int(det[5])) img_ids.append(image_id) results.append(coco_det) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format( i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, )) json.dump(results, open(args.results, 'w'), indent=4) if 'test' not in args.anno: coco_results = dataset.coco.loadRes(args.results) coco_eval = COCOeval(dataset.coco, coco_results, 'bbox') coco_eval.params.imgIds = img_ids # score only ids we've used coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return results
def validate(args): setup_default_logging() def setthresh(): if args.checkpoint.split("/")[-1].split( "_")[0] in getthresholds.keys(): return getthresholds[args.checkpoint.split("/")[-1].split("_")[0]] else: a = [] [a.append(args.threshold) for x in range(4)] return a threshs = setthresh() print(threshs) # might as well try to validate something args.pretrained = args.pretrained or not args.checkpoint args.prefetcher = not args.no_prefetcher # create model bench = create_model( args.model, bench_task='predict', pretrained=args.pretrained, redundant_bias=args.redundant_bias, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) if 'test' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json') image_dir = args.anno else: annotation_path = os.path.join(args.data, 'annotations', f'instances_{args.anno}.json') image_dir = args.anno print(os.path.join(args.data, image_dir), annotation_path) dataset = CocoDetection(os.path.join(args.data, image_dir), annotation_path) loader = create_loader(dataset, input_size=input_size, batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=args.interpolation, fill_color=args.fill_color, num_workers=args.workers, pin_mem=args.pin_mem, mean=args.mean, std=args.std) if 'test' in args.anno: threshold = float(args.threshold) else: threshold = .001 img_ids = [] results = [] writetofilearrtay = [] bench.eval() batch_time = AverageMeter() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(loader): output = bench(input, target['img_scale'], target['img_size']) output = output.cpu() # print(target['img_id']) sample_ids = target['img_id'].cpu() for index, sample in enumerate(output): image_id = int(sample_ids[index]) for det in sample: score = float(det[4]) if score < threshold: # stop when below this threshold, scores in descending order coco_det = dict(image_id=image_id, category_id=-1) img_ids.append(image_id) results.append(coco_det) break coco_det = dict(image_id=image_id, bbox=det[0:4].tolist(), score=score, category_id=int(det[5]), sizes=target['img_size'].tolist()[0]) img_ids.append(image_id) results.append(coco_det) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format( i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, )) if 'test' in args.anno: from itertools import groupby results.sort(key=lambda x: x['image_id']) f = open( str(args.model) + "-" + str(args.anno) + "-" + str(min(threshs)) + ".txt", "w+") # for item in tqdm(writetofilearrtay): xxx = 0 for k, v in tqdm(groupby(results, key=lambda x: x['image_id'])): xxx += 1 f.write(getimageNamefromid(k) + ",") #print(getimageNamefromid(k),", ") for i in v: if i['category_id'] > 0: if (i['category_id'] ==1 and i['score'] >= threshs[0] ) or (i['category_id'] ==2 and i['score'] >= threshs[1] ) or \ (i['category_id'] ==3 and i['score'] >= threshs[2] ) or (i['category_id'] ==4 and i['score'] >= threshs[3] ) : f.write( str(round(i['category_id'])) + " " + str(round(i['bbox'][0])) + " " + str(round(i['bbox'][1])) + " " + str( round( float(i['bbox'][0]) + float(i['bbox'][2]))) + " " + str( round( float(i['bbox'][1]) + float(i['bbox'][3]))) + " ") f.write('\n') # print(i['category_id']," ",i['bbox'][0]," ",i['bbox'][1]," ",i['bbox'][2]," ",i['bbox'][3]," ") print("generated lines:", xxx) f.close() # f.close() if 'test' not in args.anno: array_of_dm = [] array_of_gt = [] i = 0 # if 'test' in args.anno : for _, item in tqdm(dataset): # if item["img_id"] == "1000780" : # print(item) for i in range(len(item['cls'])): # print(str(item["img_id"]),) array_of_gt.append( BoundingBox(imageName=str(item["img_id"]), classId=item["cls"][i], x=item["bbox"][i][1] * item['img_scale'], y=item["bbox"][i][0] * item['img_scale'], w=item["bbox"][i][3] * item['img_scale'], h=item["bbox"][i][2] * item['img_scale'], typeCoordinates=CoordinatesType.Absolute, bbType=BBType.GroundTruth, format=BBFormat.XYX2Y2, imgSize=(item['img_size'][0], item['img_size'][1]))) for item in tqdm(results): if item["category_id"] >= 0: array_of_dm.append( BoundingBox(imageName=str(item["image_id"]), classId=item["category_id"], classConfidence=item["score"], x=item['bbox'][0], y=item['bbox'][1], w=item['bbox'][2], h=item['bbox'][3], typeCoordinates=CoordinatesType.Absolute, bbType=BBType.Detected, format=BBFormat.XYWH, imgSize=(item['sizes'][0], item['sizes'][1]))) myBoundingBoxes = BoundingBoxes() # # # # Add all bounding boxes to the BoundingBoxes object: for box in (array_of_gt): myBoundingBoxes.addBoundingBox(box) for dm in array_of_dm: myBoundingBoxes.addBoundingBox(dm) evaluator = Evaluator() f1res = [] f1resd0 = [] f1resd10 = [] f1resd20 = [] f1resd40 = [] for conf in tqdm(range(210, 600, 1)): metricsPerClass = evaluator.GetPascalVOCMetrics( myBoundingBoxes, IOUThreshold=0.5, ConfThreshold=conf / 1000.0) totalTP = 0 totalp = 0 totalFP = 0 tp = [] fp = [] ta = [] # print('-------') for mc in metricsPerClass: tp.append(mc['total TP']) fp.append(mc['total FP']) ta.append(mc['total positives']) totalFP = totalFP + mc['total FP'] totalTP = totalTP + mc['total TP'] totalp = totalp + (mc['total positives']) # print(totalTP," ",totalFP," ",totalp) if totalTP + totalFP == 0: p = -1 else: p = totalTP / (totalTP + totalFP) if totalp == 0: r = -1 else: r = totalTP / (totalp) f1_dict = dict(tp=totalTP, fp=totalFP, totalp=totalp, conf=conf / 1000.0, prec=p, rec=r, f1score=(2 * p * r) / (p + r)) f1res.append(f1_dict) #must clean these parts f1resd0.append( dict(tp=tp[0], fp=fp[0], totalp=ta[0], conf=conf / 1000.0, prec=tp[0] / (tp[0] + fp[0]), rec=tp[0] / ta[0], f1score=(2 * (tp[0] / (tp[0] + fp[0])) * (tp[0] / ta[0])) / ((tp[0] / (tp[0] + fp[0])) + (tp[0] / ta[0])))) f1resd10.append( dict(tp=tp[1], fp=fp[1], totalp=ta[1], conf=conf / 1000.0, prec=tp[1] / (tp[1] + fp[1]), rec=tp[1] / ta[1], f1score=(2 * (tp[1] / (tp[1] + fp[1])) * (tp[1] / ta[1])) / ((tp[1] / (tp[1] + fp[1])) + (tp[1] / ta[1])))) f1resd20.append( dict(tp=tp[2], fp=fp[2], totalp=ta[2], conf=conf / 1000.0, prec=tp[2] / (tp[2] + fp[2]), rec=tp[2] / ta[2], f1score=(2 * (tp[2] / (tp[2] + fp[2])) * (tp[2] / ta[2])) / ((tp[2] / (tp[2] + fp[2])) + (tp[2] / ta[2])))) f1resd40.append( dict(tp=tp[3], fp=fp[3], totalp=ta[3], conf=conf / 1000.0, prec=tp[3] / (tp[3] + fp[3]), rec=tp[3] / ta[3], f1score=(2 * (tp[3] / (tp[3] + fp[3])) * (tp[3] / ta[3])) / ((tp[3] / (tp[3] + fp[3])) + (tp[3] / ta[3])))) sortedf1 = sorted(f1res, key=lambda k: k['f1score'], reverse=True) f1resd0 = sorted(f1resd0, key=lambda k: k['f1score'], reverse=True) f1resd10 = sorted(f1resd10, key=lambda k: k['f1score'], reverse=True) f1resd20 = sorted(f1resd20, key=lambda k: k['f1score'], reverse=True) f1resd40 = sorted(f1resd40, key=lambda k: k['f1score'], reverse=True) print(sortedf1[0]) print("\n\n") print(f1resd0[0]) print(f1resd10[0]) print(f1resd20[0]) print(f1resd40[0]) # sortedf1 = sorted(f1res, key=lambda k: k['f1score'],reverse=True) # print(sortedf1[0:2]) # json.dump(results, open(args.results, 'w'), indent=4) json.dump(results, open(args.results, 'w'), indent=4) # coco_results = dataset.coco.loadRes(args.results) # coco_eval = COCOeval(dataset.coco, coco_results, 'bbox') # coco_eval.params.imgIds = img_ids # score only ids we've used # coco_eval.evaluate() # coco_eval.accumulate() # coco_eval.summarize() # print(coco_eval.eval['params']) json.dump(results, open(args.results, 'w'), indent=4) return results
def main(): setup_default_logging() args, args_text = _parse_args() if args.log_wandb: if has_wandb: wandb.init(project=args.experiment, config=args) else: _logger.warning( "You've requested to log metrics to wandb but package not found. " "Metrics not being logged to wandb, try `pip install wandb`") args.prefetcher = not args.no_prefetcher args.distributed = False if 'WORLD_SIZE' in os.environ: args.distributed = int(os.environ['WORLD_SIZE']) > 1 args.device = 'cuda:0' args.world_size = 1 args.rank = 0 # global rank if args.distributed: args.device = 'cuda:%d' % args.local_rank torch.cuda.set_device(args.local_rank) torch.distributed.init_process_group(backend='nccl', init_method='env://') args.world_size = torch.distributed.get_world_size() args.rank = torch.distributed.get_rank() _logger.info( 'Training in distributed mode with multiple processes, 1 GPU per process. Process %d, total %d.' % (args.rank, args.world_size)) else: _logger.info('Training with a single process on 1 GPUs.') assert args.rank >= 0 # resolve AMP arguments based on PyTorch / Apex availability use_amp = None if args.amp: # `--amp` chooses native amp before apex (APEX ver not actively maintained) if has_native_amp: args.native_amp = True elif has_apex: args.apex_amp = True if args.apex_amp and has_apex: use_amp = 'apex' elif args.native_amp and has_native_amp: use_amp = 'native' elif args.apex_amp or args.native_amp: _logger.warning( "Neither APEX or native Torch AMP is available, using float32. " "Install NVIDA apex or upgrade to PyTorch 1.6") random_seed(args.seed, args.rank) if args.fuser: set_jit_fuser(args.fuser) model = create_model( args.model, pretrained=args.pretrained, num_classes=args.num_classes, drop_rate=args.drop, drop_connect_rate=args.drop_connect, # DEPRECATED, use drop_path drop_path_rate=args.drop_path, drop_block_rate=args.drop_block, global_pool=args.gp, bn_momentum=args.bn_momentum, bn_eps=args.bn_eps, scriptable=args.torchscript, checkpoint_path=args.initial_checkpoint) if args.num_classes is None: assert hasattr( model, 'num_classes' ), 'Model must have `num_classes` attr if not set on cmd line/config.' args.num_classes = model.num_classes # FIXME handle model default vs config num_classes more elegantly if args.grad_checkpointing: model.set_grad_checkpointing(enable=True) if args.local_rank == 0: _logger.info( f'Model {safe_model_name(args.model)} created, param count:{sum([m.numel() for m in model.parameters()])}' ) data_config = resolve_data_config(vars(args), model=model, verbose=args.local_rank == 0) # setup augmentation batch splits for contrastive loss or split bn num_aug_splits = 0 if args.aug_splits > 0: assert args.aug_splits > 1, 'A split of 1 makes no sense' num_aug_splits = args.aug_splits # enable split bn (separate bn stats per batch-portion) if args.split_bn: assert num_aug_splits > 1 or args.resplit model = convert_splitbn_model(model, max(num_aug_splits, 2)) # move model to GPU, enable channels last layout if set model.cuda() if args.channels_last: model = model.to(memory_format=torch.channels_last) # setup synchronized BatchNorm for distributed training if args.distributed and args.sync_bn: assert not args.split_bn if has_apex and use_amp == 'apex': # Apex SyncBN preferred unless native amp is activated model = convert_syncbn_model(model) else: model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) if args.local_rank == 0: _logger.info( 'Converted model to use Synchronized BatchNorm. WARNING: You may have issues if using ' 'zero initialized BN layers (enabled by default for ResNets) while sync-bn enabled.' ) if args.torchscript: assert not use_amp == 'apex', 'Cannot use APEX AMP with torchscripted model' assert not args.sync_bn, 'Cannot use SyncBatchNorm with torchscripted model' model = torch.jit.script(model) optimizer = create_optimizer_v2(model, **optimizer_kwargs(cfg=args)) # setup automatic mixed-precision (AMP) loss scaling and op casting amp_autocast = suppress # do nothing loss_scaler = None if use_amp == 'apex': model, optimizer = amp.initialize(model, optimizer, opt_level='O1') loss_scaler = ApexScaler() if args.local_rank == 0: _logger.info('Using NVIDIA APEX AMP. Training in mixed precision.') elif use_amp == 'native': amp_autocast = torch.cuda.amp.autocast loss_scaler = NativeScaler() if args.local_rank == 0: _logger.info( 'Using native Torch AMP. Training in mixed precision.') else: if args.local_rank == 0: _logger.info('AMP not enabled. Training in float32.') # optionally resume from a checkpoint resume_epoch = None if args.resume: resume_epoch = resume_checkpoint( model, args.resume, optimizer=None if args.no_resume_opt else optimizer, loss_scaler=None if args.no_resume_opt else loss_scaler, log_info=args.local_rank == 0) # setup exponential moving average of model weights, SWA could be used here too model_ema = None if args.model_ema: # Important to create EMA model after cuda(), DP wrapper, and AMP but before DDP wrapper model_ema = ModelEmaV2( model, decay=args.model_ema_decay, device='cpu' if args.model_ema_force_cpu else None) if args.resume: load_checkpoint(model_ema.module, args.resume, use_ema=True) # setup distributed training if args.distributed: if has_apex and use_amp == 'apex': # Apex DDP preferred unless native amp is activated if args.local_rank == 0: _logger.info("Using NVIDIA APEX DistributedDataParallel.") model = ApexDDP(model, delay_allreduce=True) else: if args.local_rank == 0: _logger.info("Using native Torch DistributedDataParallel.") model = NativeDDP(model, device_ids=[args.local_rank], broadcast_buffers=not args.no_ddp_bb) # NOTE: EMA model does not need to be wrapped by DDP # setup learning rate schedule and starting epoch lr_scheduler, num_epochs = create_scheduler(args, optimizer) start_epoch = 0 if args.start_epoch is not None: # a specified start_epoch will always override the resume epoch start_epoch = args.start_epoch elif resume_epoch is not None: start_epoch = resume_epoch if lr_scheduler is not None and start_epoch > 0: lr_scheduler.step(start_epoch) if args.local_rank == 0: _logger.info('Scheduled epochs: {}'.format(num_epochs)) # create the train and eval datasets dataset_train = create_dataset(args.dataset, root=args.data_dir, split=args.train_split, is_training=True, class_map=args.class_map, download=args.dataset_download, batch_size=args.batch_size, repeats=args.epoch_repeats) dataset_eval = create_dataset(args.dataset, root=args.data_dir, split=args.val_split, is_training=False, class_map=args.class_map, download=args.dataset_download, batch_size=args.batch_size) # setup mixup / cutmix collate_fn = None mixup_fn = None mixup_active = args.mixup > 0 or args.cutmix > 0. or args.cutmix_minmax is not None if mixup_active: mixup_args = dict(mixup_alpha=args.mixup, cutmix_alpha=args.cutmix, cutmix_minmax=args.cutmix_minmax, prob=args.mixup_prob, switch_prob=args.mixup_switch_prob, mode=args.mixup_mode, label_smoothing=args.smoothing, num_classes=args.num_classes) if args.prefetcher: assert not num_aug_splits # collate conflict (need to support deinterleaving in collate mixup) collate_fn = FastCollateMixup(**mixup_args) else: mixup_fn = Mixup(**mixup_args) # wrap dataset in AugMix helper if num_aug_splits > 1: dataset_train = AugMixDataset(dataset_train, num_splits=num_aug_splits) # create data loaders w/ augmentation pipeiine train_interpolation = args.train_interpolation if args.no_aug or not train_interpolation: train_interpolation = data_config['interpolation'] loader_train = create_loader( dataset_train, input_size=data_config['input_size'], batch_size=args.batch_size, is_training=True, use_prefetcher=args.prefetcher, no_aug=args.no_aug, re_prob=args.reprob, re_mode=args.remode, re_count=args.recount, re_split=args.resplit, scale=args.scale, ratio=args.ratio, hflip=args.hflip, vflip=args.vflip, color_jitter=args.color_jitter, auto_augment=args.aa, num_aug_repeats=args.aug_repeats, num_aug_splits=num_aug_splits, interpolation=train_interpolation, mean=data_config['mean'], std=data_config['std'], num_workers=args.workers, distributed=args.distributed, collate_fn=collate_fn, pin_memory=args.pin_mem, use_multi_epochs_loader=args.use_multi_epochs_loader, worker_seeding=args.worker_seeding, ) loader_eval = create_loader( dataset_eval, input_size=data_config['input_size'], batch_size=args.validation_batch_size or args.batch_size, is_training=False, use_prefetcher=args.prefetcher, interpolation=data_config['interpolation'], mean=data_config['mean'], std=data_config['std'], num_workers=args.workers, distributed=args.distributed, crop_pct=data_config['crop_pct'], pin_memory=args.pin_mem, ) # setup loss function if args.jsd_loss: assert num_aug_splits > 1 # JSD only valid with aug splits set train_loss_fn = JsdCrossEntropy(num_splits=num_aug_splits, smoothing=args.smoothing) elif mixup_active: # smoothing is handled with mixup target transform which outputs sparse, soft targets if args.bce_loss: train_loss_fn = BinaryCrossEntropy( target_threshold=args.bce_target_thresh) else: train_loss_fn = SoftTargetCrossEntropy() elif args.smoothing: if args.bce_loss: train_loss_fn = BinaryCrossEntropy( smoothing=args.smoothing, target_threshold=args.bce_target_thresh) else: train_loss_fn = LabelSmoothingCrossEntropy( smoothing=args.smoothing) else: train_loss_fn = nn.CrossEntropyLoss() train_loss_fn = train_loss_fn.cuda() validate_loss_fn = nn.CrossEntropyLoss().cuda() # setup checkpoint saver and eval metric tracking eval_metric = args.eval_metric best_metric = None best_epoch = None saver = None output_dir = None if args.rank == 0: if args.experiment: exp_name = args.experiment else: exp_name = '-'.join([ datetime.now().strftime("%Y%m%d-%H%M%S"), safe_model_name(args.model), str(data_config['input_size'][-1]) ]) output_dir = get_outdir( args.output if args.output else './output/train', exp_name) decreasing = True if eval_metric == 'loss' else False saver = CheckpointSaver(model=model, optimizer=optimizer, args=args, model_ema=model_ema, amp_scaler=loss_scaler, checkpoint_dir=output_dir, recovery_dir=output_dir, decreasing=decreasing, max_history=args.checkpoint_hist) with open(os.path.join(output_dir, 'args.yaml'), 'w') as f: f.write(args_text) try: for epoch in range(start_epoch, num_epochs): if args.distributed and hasattr(loader_train.sampler, 'set_epoch'): loader_train.sampler.set_epoch(epoch) train_metrics = train_one_epoch(epoch, model, loader_train, optimizer, train_loss_fn, args, lr_scheduler=lr_scheduler, saver=saver, output_dir=output_dir, amp_autocast=amp_autocast, loss_scaler=loss_scaler, model_ema=model_ema, mixup_fn=mixup_fn) if args.distributed and args.dist_bn in ('broadcast', 'reduce'): if args.local_rank == 0: _logger.info( "Distributing BatchNorm running means and vars") distribute_bn(model, args.world_size, args.dist_bn == 'reduce') eval_metrics = validate(model, loader_eval, validate_loss_fn, args, amp_autocast=amp_autocast) if model_ema is not None and not args.model_ema_force_cpu: if args.distributed and args.dist_bn in ('broadcast', 'reduce'): distribute_bn(model_ema, args.world_size, args.dist_bn == 'reduce') ema_eval_metrics = validate(model_ema.module, loader_eval, validate_loss_fn, args, amp_autocast=amp_autocast, log_suffix=' (EMA)') eval_metrics = ema_eval_metrics if lr_scheduler is not None: # step LR for next epoch lr_scheduler.step(epoch + 1, eval_metrics[eval_metric]) if output_dir is not None: update_summary(epoch, train_metrics, eval_metrics, os.path.join(output_dir, 'summary.csv'), write_header=best_metric is None, log_wandb=args.log_wandb and has_wandb) if saver is not None: # save proper checkpoint with eval metric save_metric = eval_metrics[eval_metric] best_metric, best_epoch = saver.save_checkpoint( epoch, metric=save_metric) except KeyboardInterrupt: pass if best_metric is not None: _logger.info('*** Best metric: {0} (epoch {1})'.format( best_metric, best_epoch))
def main(): start_endpoint = "http://localhost:3000/start" stop_endpoint = "http://localhost:3000/stop" setup_default_logging() args = parser.parse_args() # might as well try to do something useful... args.pretrained = args.pretrained or not args.checkpoint output_dir = args.checkpoint.split('/') output_dir.pop(-1) output_dir = ('/').join(output_dir) # create model model = create_model( args.model, num_classes=args.num_classes, in_chans=3, pretrained=args.pretrained, checkpoint_path=args.checkpoint) logging.info('Model %s created, param count: %d' % (args.model, sum([m.numel() for m in model.parameters()]))) # config = resolve_data_config(vars(args), model=model) # model, test_time_pool = apply_test_time_pool(model, config, args) if args.num_gpu > 1: model = torch.nn.DataParallel( model, device_ids=list(range(args.num_gpu))).cuda() else: model = model.cuda() dataset_eval = torchvision.datasets.CIFAR100( root='./data', train=False, download=True) data_config = resolve_data_config(vars(args), model=model) # #CIFAR_100_MEAN = (0.5071, 0.4865, 0.4409) # #CIFAR_100_STD = (0.2673, 0.2564, 0.2762) data_config['mean'] = (0.5071, 0.4865, 0.4409) data_config['std'] = (0.2673, 0.2564, 0.2762) loader = create_loader( dataset_eval, input_size=data_config['input_size'], batch_size=args.batch_size, is_training=False, interpolation=data_config['interpolation'], mean=data_config['mean'], std=data_config['std'], num_workers=args.workers, crop_pct=data_config['crop_pct'] ) model.eval() batch_time = AverageMeter() with torch.no_grad(): idle_power = requests.post(url=start_endpoint) idle_json = idle_power.json() for batch_idx, (input, _) in enumerate(loader): input = input.cuda() tstart = time.time() output = model(input) tend = time.time() if batch_idx != 0: batch_time.update(tend - tstart) if batch_idx % args.log_freq == 0: print('Predict: [{0}/{1}] Time {batch_time.val:.6f} ({batch_time.avg:.6f})'.format( batch_idx, len(loader), batch_time=batch_time), end='\r') load_power = requests.post(url=stop_endpoint) load_json = load_power.json() fps = 1 / batch_time.avg inference_power = float(load_json['load']) - float(idle_json['idle']) stats = [{'FPS': [float(fps)]}, {'Total_Power': [float(inference_power)]}] with open(os.path.join(output_dir, '{}_fps_cifar.yaml'.format(args.model)), 'w') as f: yaml.safe_dump(stats, f)
def main(): setup_default_logging() args = parser.parse_args() # might as well try to do something useful... args.pretrained = args.pretrained or not args.checkpoint # create model model = create_model(args.model, num_classes=args.num_classes, in_chans=3, pretrained=args.pretrained, checkpoint_path=args.checkpoint) _logger.info('Model %s created, param count: %d' % (args.model, sum([m.numel() for m in model.parameters()]))) config = resolve_data_config(vars(args), model=model) model, test_time_pool = ( model, False) if args.no_test_pool else apply_test_time_pool(model, config) if args.num_gpu > 1: model = torch.nn.DataParallel(model, device_ids=list(range( args.num_gpu))).cuda() else: model = model.cuda() loader = create_loader( Dataset(args.data, train_mode='test', fold_num=-1), input_size=config['input_size'], batch_size=args.batch_size, use_prefetcher=True, interpolation=config['interpolation'], mean=config['mean'], std=config['std'], num_workers=args.workers, crop_pct=1.0 if test_time_pool else config['crop_pct']) model.eval() # k = min(args.topk, args.num_classes) batch_time = AverageMeter() end = time.time() topk_ids = [] name_list = [] sig_list = [] logits_list = [] m = torch.nn.Sigmoid() with torch.no_grad(): for batch_idx, ( input, _, ) in enumerate(loader): input = input.cuda() labels = model(input) logits_list.append(labels) sigmoided = m(labels) sig_list.append( np.expand_dims(sigmoided[:, 1].cpu().numpy(), axis=1)) # topk = labels.topk(k)[1] # topk_ids.append(topk.cpu().numpy()) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % args.log_freq == 0: _logger.info( 'Predict: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f})' .format(batch_idx, len(loader), batch_time=batch_time)) # topk_ids = np.concatenate(topk_ids, axis=0).squeeze() # logits = torch.cat(logits_list).cuda() # temperature = nn.Parameter(torch.ones(1) * args.te).to(torch.device('cuda') ).detach().requires_grad_(False) # logits = logits/temperature.unsqueeze(1).expand(logits.size(0), logits.size(1)) # temp_sigmoided = m(logits)[:,1].detach().cpu().numpy() sig_list = np.vstack(sig_list) name_list = loader.dataset.filenames(basename=True) real_sigmoid = sig_list.squeeze() # real_sigmoid = temp_sigmoided real_pred = ((sig_list >= args.thresh) * 1).squeeze() name_pred_dict = {} for idx in range(len(name_list)): name_pred_dict[name_list[idx]] = (real_pred[idx], real_sigmoid[idx]) args.output_dir = args.checkpoint.replace( args.checkpoint.split('/')[-1], "") with open(os.path.join(args.output_dir, './prediction.tsv'), 'w') as out_file: # filenames_int = [int(f.split('.')[0]) for f in filenames] # for name, topk in zip(filenames_int, topk_ids): # print(name,topk) # i = i+1 # if i == 10: # break # idx = np.argsort(filenames_int) # topk_ids = topk_ids[idx] for name in name_list: out_file.write('{}\n'.format(str(name_pred_dict[name][0]))) with open(os.path.join(args.output_dir, './probability.tsv'), 'w') as out_file: for name in name_list: out_file.write('{}\n'.format(name_pred_dict[name][1])) copyfile( os.path.join(args.output_dir, './prediction.tsv'), '/home/workspace/user-workspace/prediction/' + 'prediction_153_' + args.checkpoint.split('/')[-2] + '.tsv')
from timm.models import create_model from timm.data import resolve_data_config import torch import torch.nn as nn import argparse from external.utils_pruning import compute_flops, load_module_from_ckpt, measure_time from external.hyperml import HypermlDownloader from timm.utils import setup_default_logging from timm.models import load_checkpoint import logging setup_default_logging() parser = argparse.ArgumentParser(description='PyTorch ImageNet Validation') parser.add_argument( '--model', '-m', metavar='MODEL', default='efficientnet_b2_pruned', help='model architecture (default: efficientnet_b2_pruned)') parser.add_argument('--batch_size', default=512, type=int, help='batch size)') parser.add_argument('--fp16', action='store_true', default=False, help='Use half precision (fp16)') parser.add_argument('--use_eca', action='store_true', default=False, help='Use eca bn for efficientNet')
def validate(args): setup_default_logging() # might as well try to validate something args.pretrained = args.pretrained or not args.checkpoint args.prefetcher = not args.no_prefetcher if args.no_redundant_bias is None: args.redundant_bias = None else: args.redundant_bias = not args.no_redundant_bias # create model bench2 = pt.detection_models.__dict__[args.model]( match_tf_same_padding=True, encoder_norm_act="swish_hard") bench2 = bench2.eval().requires_grad_(False).cuda() input_size2 = bench2.pretrained_settings["input_size"] param_count = sum([m.numel() for m in bench2.parameters()]) print("Model %s created, param count: %d" % (args.model, param_count)) if has_amp: print("Using AMP mixed precision.") bench2 = amp.initialize(bench2, opt_level="O1") else: print("AMP not installed, running network in FP32.") if "test" in args.anno: annotation_path = os.path.join(args.data, "annotations", f"image_info_{args.anno}.json") image_dir = "test2017" else: annotation_path = os.path.join(args.data, "annotations", f"instances_{args.anno}.json") image_dir = args.anno # loader = create_loader( # dataset, # input_size=input_size, # batch_size=args.batch_size, # use_prefetcher=args.prefetcher, # interpolation=args.interpolation, # fill_color=args.fill_color, # num_workers=args.workers, # pin_mem=args.pin_mem, # ) print(f"Input size: {input_size2[0]}") loader = DaliLoader(False, args.batch_size, args.workers, input_size2[0]) img_ids = [] results = [] batch_time = AverageMeter() end = time.time() start_time = time.time() with torch.no_grad(): for i, (input, target) in enumerate(loader): output2 = bench2.predict(input) _, batch_ids, ratios = target target = {"img_scale": ratios, "img_id": batch_ids} # rescale to image size and clip output2[..., :4] *= target["img_scale"].view(-1, 1, 1).to(output2) # works even without clipping # output2[..., :4] = box_utils.clip_bboxes_batch(output2[..., :4], target["img_size"][..., [1, 0]]) # xyxy => xywh output2[..., 2:4] = output2[..., 2:4] - output2[..., :2] output = output2 output = output.cpu() sample_ids = target["img_id"].cpu() for index, sample in enumerate(output): image_id = int(sample_ids[index]) for det in sample: score = float(det[4]) if score < 0.001: # stop when below this threshold, scores in descending order break coco_det = dict(image_id=image_id, bbox=det[0:4].tolist(), score=score, category_id=int(det[5])) img_ids.append(image_id) results.append(coco_det) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0: print( "Test: [{0:>4d}/{1}] " "Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) " .format( i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, )) # if i > 10: # break print(f"Full eval took: {time.time() - start_time:.2f}s") json.dump(results, open(args.results, "w"), indent=4) if "test" not in args.anno: coco_api = COCO("data/annotations/instances_val2017.json") coco_results = coco_api.loadRes(args.results) coco_eval = COCOeval(coco_api, coco_results, "bbox") coco_eval.params.imgIds = img_ids # score only ids we've used coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() return results
def validate(args): setup_default_logging() if args.amp: if has_apex: args.apex_amp = True elif has_native_amp: args.native_amp = True assert not args.apex_amp or not args.native_amp, "Only one AMP mode should be set." args.pretrained = args.pretrained or not args.checkpoint # might as well try to validate something args.prefetcher = not args.no_prefetcher # create model with set_layer_config(scriptable=args.torchscript): bench = create_model( args.model, bench_task='predict', num_classes=args.num_classes, pretrained=args.pretrained, redundant_bias=args.redundant_bias, soft_nms=args.soft_nms, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) model_config = bench.config param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() amp_autocast = suppress if args.apex_amp: bench = amp.initialize(bench, opt_level='O1') print('Using NVIDIA APEX AMP. Validating in mixed precision.') elif args.native_amp: amp_autocast = torch.cuda.amp.autocast print('Using native Torch AMP. Validating in mixed precision.') else: print('AMP not enabled. Validating in float32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) dataset = create_dataset(args.dataset, args.root, args.split) input_config = resolve_input_config(args, model_config) loader = create_loader(dataset, input_size=input_config['input_size'], batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=input_config['interpolation'], fill_color=input_config['fill_color'], mean=input_config['mean'], std=input_config['std'], num_workers=args.workers, pin_mem=args.pin_mem) evaluator = create_evaluator(args.dataset, dataset, pred_yxyx=False) bench.eval() batch_time = AverageMeter() end = time.time() last_idx = len(loader) - 1 imgs = [] with torch.no_grad(): for i, (input, target) in enumerate(loader): for b in range(input.shape[0]): imgs.append(input[b].cpu().numpy()) # targets.append(target[b].cpu().numpy()) with amp_autocast(): output = bench(input, img_info=target) evaluator.add_predictions(output, target) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0 or i == last_idx: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format(i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg)) mean_ap = 0. if dataset.parser.has_labels: preds = [p[:2, :] for p in evaluator.predictions] anns = evaluator.coco_api.imgToAnns targets = [ np.asarray((anns[k][0]['bbox'], anns[k][1]['bbox'])) for k in range(len(imgs)) ] mean_ap = evaluator.evaluate() if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) for i, img in enumerate(imgs): img = imgs[i] img_m = np.mean(img, axis=0) for c in range(3): img[c] = img_m img_ = img.transpose(1, 2, 0) m = img_.min() M = img_.max() img_ = ((img_ - m) / (M - m) * 255).astype('uint8').copy() img_ = draw_bbox(img_, preds[i], targets[i]) cv2.imwrite(os.path.join(args.out_dir, '%d.jpg' % i), img_) else: evaluator.save(args.results) return mean_ap
def main(): setup_default_logging() args = parser.parse_args() # might as well try to do something useful... args.pretrained = args.pretrained or not args.checkpoint # create model model = create_model(args.model, num_classes=args.num_classes, in_chans=3, pretrained=args.pretrained, checkpoint_path=args.checkpoint) logging.info('Model %s created, param count: %d' % (args.model, sum([m.numel() for m in model.parameters()]))) config = resolve_data_config(vars(args), model=model) model, test_time_pool = apply_test_time_pool(model, config, args) if args.num_gpu > 1: model = torch.nn.DataParallel(model, device_ids=list(range( args.num_gpu))).cuda() else: model = model.cuda() loader = create_loader( Dataset(args.data), input_size=config['input_size'], batch_size=args.batch_size, use_prefetcher=True, interpolation=config['interpolation'], mean=config['mean'], std=config['std'], num_workers=args.workers, crop_pct=1.0 if test_time_pool else config['crop_pct']) model.eval() k = min(args.topk, args.num_classes) batch_time = AverageMeter() end = time.time() topk_ids = [] with torch.no_grad(): for batch_idx, (input, _) in enumerate(loader): input = input.cuda() labels = model(input) topk = labels.topk(k)[1] topk_ids.append(topk.cpu().numpy()) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % args.log_freq == 0: logging.info( 'Predict: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f})' .format(batch_idx, len(loader), batch_time=batch_time)) topk_ids = np.concatenate(topk_ids, axis=0).squeeze() savebase = "classification_result/" os.makedirs(savebase, exist_ok=True) classfile = "labels.txt" classpath = os.path.join(os.getcwd(), classfile) classlist = {} with open(classpath) as f: for idx, line in enumerate(f): val = line.split('\n')[0] classlist[idx] = val filenames = loader.dataset.filenames() for filepath, label in zip(filenames, topk_ids): filename = os.path.basename(filepath) prediction = classlist[label[0]] savedir = savebase + prediction savepath = savedir + "/" + filename os.makedirs(savedir, exist_ok=True) copyfile(filepath, savepath) print('{0} : {1}'.format(filename, prediction))
def predict(INP_DIR, BATCH_SIZE, MODEL_PATH): print("[INFO] Predicting") setup_default_logging() args = parser.parse_args() # might as well try to do something useful... args.model = 'tf_efficientnet_b8' args.data = INP_DIR args.num_classes = 8 args.checkpoint = MODEL_PATH args.batch_size = BATCH_SIZE args.pretrained = args.pretrained or not args.checkpoint # create model model = create_model( args.model, num_classes=args.num_classes, in_chans=3, pretrained=args.pretrained, checkpoint_path=args.checkpoint) _logger.info('Model %s created, param count: %d' % (args.model, sum([m.numel() for m in model.parameters()]))) config = resolve_data_config(vars(args), model=model) model, test_time_pool = (model, False) if args.no_test_pool else apply_test_time_pool(model, config) if args.num_gpu > 1: model = torch.nn.DataParallel(model, device_ids=list(range(args.num_gpu))).cuda() else: model = model.cuda() loader = create_loader( Dataset(args.data), input_size=config['input_size'], batch_size=args.batch_size, use_prefetcher=True, interpolation=config['interpolation'], mean=config['mean'], std=config['std'], num_workers=args.workers, crop_pct=1.0 if test_time_pool else config['crop_pct']) model.eval() k = min(args.topk, args.num_classes) batch_time = AverageMeter() end = time.time() topk_ids = [] topk_prob = [] with torch.no_grad(): for batch_idx, (input, _) in enumerate(loader): try: input = input.cuda() labels = model(input) topk = labels.topk(k)[0] topk = topk.cpu().numpy() # print(topk) topk = np.exp(topk) / np.sum(np.exp(topk), axis=-1)[:, np.newaxis] # print(topk) topk_prob.append(topk) topk = labels.topk(k)[1] topk_ids.append(topk.cpu().numpy()) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % args.log_freq == 0: _logger.info('Predict: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f})'.format( batch_idx, len(loader), batch_time=batch_time)) except Exception: pass topk_ids = np.concatenate(topk_ids, axis=0).squeeze() topk_prob = np.concatenate(topk_prob, axis=0).squeeze() print("topk_ids: ", topk_ids) print("topk_prob: ", topk_prob) # out_path = os.path.join(args.output_dir, 'submission_{}.txt'.format(args.model)) out_path = os.path.join('hackathon_test', "output.txt") with open(out_path, 'w') as out_file: filenames = loader.dataset.filenames(basename=True) for filename, label, prob in zip(filenames, topk_ids, topk_prob): # out_file.write(("{}" + "\t{}\t{:.4f}"*5 + "\n").format( # filename, *chain(*zip(label, prob)))) out_file.write(("{}" + " {} {:.4f}"*5 + "\n").format( filename, *chain(*zip(label, prob)))) print("label: {}, prob: {}".format(label, prob)) return out_path
def main(): setup_default_logging() args = parser.parse_args() # might as well try to do something useful... args.pretrained = args.pretrained or not args.checkpoint # create model model = create_model( args.model, num_classes=args.num_classes, in_chans=3, pretrained=args.pretrained, checkpoint_path=args.checkpoint) logging.info('Model %s created, param count: %d' % (args.model, sum([m.numel() for m in model.parameters()]))) config = resolve_data_config(vars(args), model=model) test_time_pool = False if args.num_gpu > 1: model = torch.nn.DataParallel(model, device_ids=list(range(args.num_gpu))).cuda() else: model = model.cuda() test_dataset = Dataset(args.data) class_mapper = {v:k for k,v in test_dataset.class_to_idx.items()} loader = create_loader( test_dataset, input_size=config['input_size'], batch_size=args.batch_size, use_prefetcher=False, interpolation=config['interpolation'], mean=config['mean'], std=config['std'], num_workers=args.workers, crop_pct=1.0 if test_time_pool else config['crop_pct']) model.eval() k = min(args.topk, args.num_classes) batch_time = AverageMeter() end = time.time() topk_ids = [] results = [] with torch.no_grad(): for batch_idx, (input, _, path) in enumerate(loader): input = input.cuda() labels = model(input) topk = labels.topk(k)[1] topk_ids.append(topk.cpu().numpy()) label_ = list(topk.cpu().numpy())[0][0] results.append([path, class_mapper[label_]]) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % args.log_freq == 0: logging.info('Predict: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f})'.format( batch_idx, len(loader), batch_time=batch_time)) topk_ids = np.concatenate(topk_ids, axis=0).squeeze() with open(os.path.join(args.output_dir, './topk_ids.csv'), 'w') as out_file: filenames = loader.dataset.filenames() for filename, label in zip(filenames, topk_ids): filename = os.path.basename(filename) out_file.write('{0},{1},{2},{3}\n'.format( filename, label[0], label[1], label[2])) if args.result_dir: if not os.path.isdir(args.result_dir): os.system("mkdir "+args.result_dir) else: os.system("rm -rf "+args.result_dir) os.system("mkdir "+args.result_dir) for k,v in class_mapper.items(): os.system("mkdir -p "+os.path.join(args.result_dir,v)) for image in results: path = image[0][0] result = image[1] gt = path.split('/')[-2] print("\tPath : %s \tResult : %s \tGT : %s"%(path, result, gt)) os.system("cp "+path+" "+os.path.join(args.result_dir,result))
def validate_det(args): setup_default_logging() # might as well try to validate something args.pretrained = args.checkpoint args.prefetcher = not args.no_prefetcher # create model bench = create_model( args.model, bench_task='predict', num_classes=args.num_classes, pretrained=args.pretrained, redundant_bias=args.redundant_bias, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) model_config = bench.config input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) #print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() '''if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.')''' if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) '''if 'test' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json') image_dir = 'test2017''' '''annotation_path = os.path.join(args.data, 'Highway_drop_inlets.v1-vdot_coco.coco/coco_and_dropinlets_annotations/test_annotations', f'{args.anno}.json') image_dir = os.path.join(args.data, 'Highway_drop_inlets.v1-vdot_coco.coco/coco_and_dropinlets/test/') dataset = CocoDetection(image_dir, annotation_path)''' annotation_path = os.path.join(args.data, f'{args.anno}.json') image_dir = os.path.join(args.data, 'others_mix_set') dataset = VdotTestDataset(image_dir, annotation_path) loader = create_loader(dataset, input_size=input_size, batch_size=args.batches_size, use_prefetcher=args.prefetcher, interpolation=args.interpolation, fill_color=args.fill_color, num_workers=args.workers, pin_mem=args.pin_mem) img_ids = [] results = [] bench.eval() #example_input = torch.randn((1, 3, 512, 512), requires_grad=True) #bench(example_input.cuda()) '''tracingModelInput = torch.ones(1,3,512,512) torch.onnx._export(bench.model, tracingModelInput.cuda(), './effdet0_checkonly.onnx', opset_version=11, export_params=True) print('\nDone exporting ONNX model!') onnx_model='./effdet0_checkonly.onnx' onnx.checker.check_model(onnx_model)''' '''dummy_input = torch.randn(1, 3, 512, 512, device='cuda') input =["input"] output=["class_1","class_2","class_3","class_4", "class_5", "box_1", "box_2", "box_3","box_4","box_5"] #dynamic_axes = {"actual_input_1":{input:"batch_size"}, "output1":{output:"batch_size"}} torch.onnx.export(bench.model, dummy_input, "effdet0_upsamplenormal.onnx", verbose=True, opset_version=11, input_names=input, output_names=output)''' #dynamic_axes=dynamic_axes) batch_time = AverageMeter() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(loader): output = bench(input, img_info=target) output = output.cpu() sample_ids = target['img_id'].cpu() for index, sample in enumerate(output): image_id = int(sample_ids[index]) for det in sample: score = float(det[4]) if score < 0.36: # stop when below this threshold, scores in descending order(perfect 0.5 for 91 classes) break coco_det = dict(image_id=image_id, bbox=det[0:4].tolist(), score=score, category_id=int(det[5])) img_ids.append(image_id) results.append(coco_det) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format( i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, )) json.dump(results, open(args.results, 'w'), indent=4) '''if 'test' not in args.anno: coco_results = dataset.coco.loadRes(args.results) coco_eval = COCOeval(dataset.coco, coco_results, 'bbox') coco_eval.params.imgIds = img_ids # score only ids we've used coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize()''' return results
def validate(args): setup_default_logging() # might as well try to validate something args.pretrained = args.pretrained or not args.checkpoint args.prefetcher = not args.no_prefetcher # create model bench = create_model(args.model, bench_task='predict', pretrained=args.pretrained, redundant_bias=args.redundant_bias, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema) input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) if 'test' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json') image_dir = args.anno else: annotation_path = os.path.join(args.data, 'annotations', f'instances_{args.anno}.json') image_dir = args.anno dataset = CocoDetection(os.path.join(args.data, image_dir), annotation_path) loader = create_loader(dataset, input_size=input_size, batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=args.interpolation, fill_color=args.fill_color, num_workers=args.workers, mean=args.mean, std=args.std, pin_mem=args.pin_mem) img_ids = [] results = [] bench.eval() for i, (input, target) in enumerate(loader, 1): dumm_inp = input tisc = target['img_scale'] tisz = target['img_size'] break starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event( enable_timing=True) # repetitions = 300 # timings=np.zeros((repetitions,1)) #GPU-WARM-UP # print(enumerate()) for _ in range(10): _ = bench(dumm_inp, tisc, tisz) # MEASURE PERFORMANCE # dummy_input = torch.randn(1, 3,bench.config.image_size,bench.config.image_size,dtype=torch.float).to("cuda") print("starting") batch_time = AverageMeter() # end = time.time() with torch.no_grad(): for _ in range(2000): starter.record() _ = bench(dumm_inp, tisc, tisz) ender.record() # measure elapsed time torch.cuda.synchronize() curr_time = starter.elapsed_time(ender) batch_time.update(curr_time) # print(curr_time) # end = time.time() # if i % args.log_freq == 0: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}ms ({batch_time.avg:.3f}ms, {rate_avg:>7.2f}/s) ' .format( i, len(loader), batch_time=batch_time, rate_avg=dumm_inp.size(0) / batch_time.avg, )) # json.dump(results, open(args.results, 'w'), indent=4) # if 'test' not in args.anno: # coco_results = dataset.coco.loadRes(args.results) # coco_eval = COCOeval(dataset.coco, coco_results, 'bbox') # coco_eval.params.imgIds = img_ids # score only ids we've used # coco_eval.evaluate() # coco_eval.accumulate() # coco_eval.summarize() return results
def main(): setup_default_logging() args = parser.parse_args() # might as well try to do something useful... args.pretrained = args.pretrained or not args.checkpoint # create model model = create_model(args.model, num_classes=args.num_classes, in_chans=3, pretrained=args.pretrained, checkpoint_path=args.checkpoint) _logger.info('Model %s created, param count: %d' % (args.model, sum([m.numel() for m in model.parameters()]))) config = resolve_data_config(vars(args), model=model) model, test_time_pool = ( model, False) if args.no_test_pool else apply_test_time_pool(model, config) if args.num_gpu > 1: model = torch.nn.DataParallel(model, device_ids=list(range( args.num_gpu))).cuda() else: model = model.cuda() loader = create_loader( ImageDataset(args.data), input_size=config['input_size'], batch_size=args.batch_size, use_prefetcher=True, interpolation=config['interpolation'], mean=config['mean'], std=config['std'], num_workers=args.workers, crop_pct=1.0 if test_time_pool else config['crop_pct']) model.eval() k = min(args.topk, args.num_classes) batch_time = AverageMeter() end = time.time() topk_ids = [] with torch.no_grad(): for batch_idx, (input, _) in enumerate(loader): input = input.cuda() labels = model(input) topk = labels.topk(k)[1] topk_ids.append(topk.cpu().numpy()) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % args.log_freq == 0: _logger.info( 'Predict: [{0}/{1}] Time {batch_time.val:.3f} ({batch_time.avg:.3f})' .format(batch_idx, len(loader), batch_time=batch_time)) topk_ids = np.concatenate(topk_ids, axis=0).squeeze() with open(os.path.join(args.output_dir, 'topk_ids.csv'), 'w') as out_file: filenames = loader.dataset.filenames(basename=True) for filename, label in zip(filenames, topk_ids): out_file.write('{0},{1},{2},{3},{4},{5}\n'.format( filename, label[0], label[1], label[2], label[3], label[4]))
def main(): setup_default_logging() args = parser.parse_args() validate(args)
def main(): setup_default_logging() args, args_text = _parse_args() # args = parser.parse_args() torch.manual_seed(args.seed) # ! mostly for aug on test random.seed(args.seed) np.random.seed(args.seed) model_cfgs = [] model_names = [] if os.path.isdir( args.checkpoint): # ! can pass in a directory of checkpoints # validate all checkpoints in a path with same model checkpoints = glob.glob(args.checkpoint + '/*.pth.tar') checkpoints += glob.glob(args.checkpoint + '/*.pth') model_names = list_models(args.model) model_cfgs = [(args.model, c) for c in sorted(checkpoints, key=natural_key)] else: if args.model == 'all': # validate all models in a list of names with pretrained checkpoints args.pretrained = True model_names = list_models(pretrained=True) model_cfgs = [(n, '') for n in model_names] elif not is_model(args.model): # model name doesn't exist, try as wildcard filter model_names = list_models(args.model) model_cfgs = [(n, '') for n in model_names] results_file = args.results_file or './results-all.csv' if len(model_cfgs): _logger.info( 'Running bulk validation on these pretrained models: {}'.format( ', '.join(model_names))) results = [] try: start_batch_size = args.batch_size for m, c in model_cfgs: batch_size = start_batch_size args.model = m args.checkpoint = c result = OrderedDict(model=args.model) r = {} while not r and batch_size >= args.num_gpu: torch.cuda.empty_cache() try: args.batch_size = batch_size print('Validating with batch size: %d' % args.batch_size) r, prediction, true_label = validate(args) except RuntimeError as e: if batch_size <= args.num_gpu: print( "Validation failed with no ability to reduce batch size. Exiting." ) raise e batch_size = max(batch_size // 2, args.num_gpu) print("Validation failed, reducing batch size by 50%") result.update(r) if args.checkpoint: result['checkpoint'] = args.checkpoint results.append(result) except KeyboardInterrupt as e: pass results = sorted(results, key=lambda x: x['top1'], reverse=True) if len(results): write_results(results_file, results) else: # ! eval one single model results_file = re.sub(r'\.csv', '', results_file) + '-standard.csv' _, prediction, true_label = validate(args) from HAM10000 import helper prediction = helper.softmax( prediction) # softmax convert to range 0-1, sum to 1 if args.has_eval_label: from sklearn.metrics import accuracy_score, balanced_accuracy_score true_label_onehot = np.identity( args.num_classes)[true_label] # array into 1 hot # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html _logger.info( ' * sklearn multilabel probabilities accuracy_score {:.3f} '. format(accuracy_score(true_label_onehot, np.round(prediction)))) _logger.info( ' * sklearn max probabilities balanced_accuracy_score {:.3f} '. format( balanced_accuracy_score( true_label, helper.convert_max_1_other_0(prediction).argmax( axis=1)))) # output csv, need to reorder columns helper.save_output_csv( prediction, obs_name=[], output_name=results_file) # no name for each observation, use []
def validate(args): setup_default_logging() def setthresh(): if args.checkpoint.split("/")[-1].split( "_")[0] in getthresholds.keys(): return getthresholds[args.checkpoint.split("/")[-1].split("_")[0]] else: a = [] [a.append(args.threshold) for x in range(4)] return a # might as well try to validate something args.pretrained = args.pretrained or not args.checkpoint args.prefetcher = not args.no_prefetcher # create model bench = create_model( args.model, bench_task='predict', pretrained=args.pretrained, redundant_bias=args.redundant_bias, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) input_size = bench.config.image_size param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() if has_amp: print('Using AMP mixed precision.') bench = amp.initialize(bench, opt_level='O1') else: print('AMP not installed, running network in FP32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) if 'test' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'image_info_{args.anno}.json') image_dir = args.anno elif 'val' in args.anno: annotation_path = os.path.join(args.data, 'annotations', f'instances_{args.anno}.json') image_dir = args.anno # else: # annotation_path = os.path.join(args.data, f'{args.anno}.json') # image_dir = args.anno print(os.path.join(args.data, image_dir), annotation_path) dataset = CocoDetection(os.path.join(args.data, image_dir), annotation_path) loader = create_loader(dataset, input_size=input_size, batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=args.interpolation, fill_color=args.fill_color, num_workers=args.workers, pin_mem=args.pin_mem, mean=args.mean, std=args.std) if 'test' in args.anno: threshold = float(args.threshold) # elif 'detector' in args.anno: # threshold = min(getthresholds['d0']) else: threshold = .001 img_ids = [] results = [] writetofilearrtay = [] bench.eval() batch_time = AverageMeter() end = time.time() with torch.no_grad(): for i, (input, target) in enumerate(loader): output = bench(input, target['img_scale'], target['img_size']) output = output.cpu() # print(target['img_id']) sample_ids = target['img_id'].cpu() for index, sample in enumerate(output): image_id = int(sample_ids[index]) # if 'test' in args.anno : # tempWritetoFile = [] # tempWritetoFile.append(getimageNamefromid(image_id)) for det in sample: score = float(det[4]) if score < threshold: # stop when below this threshold, scores in descending order coco_det = dict(image_id=image_id, category_id=-1) img_ids.append(image_id) results.append(coco_det) break coco_det = dict(image_id=image_id, bbox=det[0:4].tolist(), score=score, category_id=int(det[5]), sizes=target['img_size'].tolist()[0]) img_ids.append(image_id) results.append(coco_det) # exit() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format( i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, )) # if 'test' in args.anno : if not os.path.exists(args.tosave): os.makedirs(args.tosave) from itertools import groupby results.sort(key=lambda x: x['image_id']) count = 0 for k, v in tqdm(groupby(results, key=lambda x: x['image_id'])): # print(args.data +"/" + str(getimageNamefromid(k))) img = drawonimage( os.path.join(args.data, image_dir, str(getimageNamefromid(k))), v, setthresh()) cv2.imwrite(args.tosave + "/" + str(getimageNamefromid(k)), img) count += 1 # print(i['category_id']," ",i['bbox'][0]," ",i['bbox'][1]," ",i['bbox'][2]," ",i['bbox'][3]," ") print("generated predictions for ", count, " images.") return results
def validate(args): setup_default_logging() if args.amp: if has_apex: args.apex_amp = True elif has_native_amp: args.native_amp = True assert not args.apex_amp or not args.native_amp, "Only one AMP mode should be set." args.pretrained = args.pretrained or not args.checkpoint # might as well try to validate something args.prefetcher = not args.no_prefetcher # create model with set_layer_config(scriptable=args.torchscript): bench = create_model( args.model, bench_task='predict', num_classes=args.num_classes, pretrained=args.pretrained, redundant_bias=args.redundant_bias, soft_nms=args.soft_nms, checkpoint_path=args.checkpoint, checkpoint_ema=args.use_ema, ) model_config = bench.config param_count = sum([m.numel() for m in bench.parameters()]) print('Model %s created, param count: %d' % (args.model, param_count)) bench = bench.cuda() amp_autocast = suppress if args.apex_amp: bench = amp.initialize(bench, opt_level='O1') print('Using NVIDIA APEX AMP. Validating in mixed precision.') elif args.native_amp: amp_autocast = torch.cuda.amp.autocast print('Using native Torch AMP. Validating in mixed precision.') else: print('AMP not enabled. Validating in float32.') if args.num_gpu > 1: bench = torch.nn.DataParallel(bench, device_ids=list(range(args.num_gpu))) dataset = create_dataset(args.dataset, args.root, args.split) input_config = resolve_input_config(args, model_config) loader = create_loader(dataset, input_size=input_config['input_size'], batch_size=args.batch_size, use_prefetcher=args.prefetcher, interpolation=input_config['interpolation'], fill_color=input_config['fill_color'], mean=input_config['mean'], std=input_config['std'], num_workers=args.workers, pin_mem=args.pin_mem) evaluator = create_evaluator(args.dataset, dataset, pred_yxyx=False) bench.eval() batch_time = AverageMeter() end = time.time() last_idx = len(loader) - 1 with torch.no_grad(): for i, (input, target) in enumerate(loader): with amp_autocast(): output = bench(input, img_info=target) evaluator.add_predictions(output, target) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.log_freq == 0 or i == last_idx: print( 'Test: [{0:>4d}/{1}] ' 'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s, {rate_avg:>7.2f}/s) ' .format(i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg)) mean_ap = 0. if dataset.parser.has_labels: mean_ap = evaluator.evaluate() else: evaluator.save(args.results) return mean_ap
def main(): setup_default_logging() args, args_text = _parse_args() args.device = 'cuda:0' args.world_size = 1 args.rank = 0 # global rank _logger.info('Training with a single process on %d GPUs.' % args.num_gpu) torch.manual_seed(args.seed + args.rank) # prepare model model = create_model(args.model, args.encoder, pretrained=args.pretrained, num_classes=args.num_classes, checkpoint_path=args.initial_checkpoint) # prepare optimizer optimizer = create_optimizer(args, model) # prepare scheduler lr_scheduler, num_epochs = create_scheduler(args, optimizer) _logger.info('Scheduled epochs: {}'.format(num_epochs)) # prepare dataset folder = args.data_folder train_fold = args.train_fold images = np.load(f'{folder}/images/fold{train_fold}_images.npy') masks = np.load(f'{folder}/masks/fold{train_fold}_masks.npy') types = np.load(f'{folder}/types/fold{train_fold}_types.npy') valid_fold = args.valid_fold images_val = np.load(f'{folder}/images/fold{valid_fold}_images.npy') masks_val = np.load(f'{folder}/masks/fold{valid_fold}_masks.npy') types_val = np.load(f'{folder}/types/fold{valid_fold}_types.npy') if args.no_aug: train_dataset = PanNukeDataset(images, masks, types, get_valid_transforms()) else: train_dataset = PanNukeDataset(images, masks, types, get_training_trasnforms(args.aug_type)) val_dataset = PanNukeDataset(images_val, masks_val, types_val, get_valid_transforms()) loaders = { 'train': DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.workers, pin_memory=True, shuffle=True), 'valid': DataLoader(val_dataset, batch_size=args.batch_size * args.validation_batch_size_multiplier, num_workers=args.workers, pin_memory=True, shuffle=False) } # save config output_dir = '' output_base = args.output if args.output else './logs' exp_name = '-'.join([ datetime.now().strftime("%Y%m%d-%H%M%S"), args.model, args.encoder, args.aug_type, args.opt.lower() ]) output_dir = get_outdir(output_base, 'train', exp_name) with open(os.path.join(output_dir, 'args.yaml'), 'w') as f: f.write(args_text) criterion, criterion_names = create_criterion(args) callbacks = create_callbacks(args, criterion_names) eval_metric = args.eval_metric minimize_metric = True if eval_metric == 'loss' else False runner = SupervisedRunner(input_key=args.input_key, input_target_key=args.input_target_key) # set fp16 if args.fp16: fp16_params = dict(opt_level="O1") # params for FP16 _logger.info('Using fp16 O1') else: fp16_params = None _logger.info('Not using fp16 O1') runner.train( model=model, criterion=criterion, optimizer=optimizer, scheduler=lr_scheduler, loaders=loaders, callbacks=callbacks, logdir=output_dir, num_epochs=num_epochs, main_metric=eval_metric, minimize_metric=minimize_metric, verbose=True, fp16=fp16_params, )