def main(): args = parse_args() assert args.show or args.show_dir, ('Please specify at least one ' 'operation (show the results / save )' 'the results with the argument ' '"--show" or "--show-dir".') cfg = Config.fromfile(args.config) # import modules from string list. if cfg.get('custom_imports', None): from mmcv.utils import import_modules_from_strings import_modules_from_strings(**cfg['custom_imports']) # set cudnn_benchmark if cfg.get('cudnn_benchmark', False): torch.backends.cudnn.benchmark = True cfg.model.pretrained = None distributed = False # build the dataloader dataset = build_dataset(cfg.data.test) data_loader = build_dataloader(dataset, samples_per_gpu=1, workers_per_gpu=cfg.data.workers_per_gpu, dist=distributed, shuffle=False) # build the model and load checkpoint cfg.model.train_cfg = None model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg')) load_checkpoint(model, args.checkpoint, map_location='cpu') model = MMDataParallel(model, device_ids=[0]) test(model, data_loader, args.show, args.show_dir)
def main(): parser = argparse.ArgumentParser(description='Benchmark data loading') parser.add_argument('config', help='Train config file path.') args = parser.parse_args() cfg = Config.fromfile(args.config) dataset = build_dataset(cfg.data.train) # prepare data loaders if 'imgs_per_gpu' in cfg.data: cfg.data.samples_per_gpu = cfg.data.imgs_per_gpu data_loader = build_dataloader(dataset, cfg.data.samples_per_gpu, cfg.data.workers_per_gpu, 1, dist=False, seed=None) # Start progress bar after first 5 batches prog_bar = mmcv.ProgressBar(len(dataset) - 5 * cfg.data.samples_per_gpu, start=False) for i, data in enumerate(data_loader): if i == 5: prog_bar.start() for _ in range(len(data['img'])): if i < 5: continue prog_bar.update()
def generate_sample_dataloader(cfg, curr_dir, img_prefix='', ann_file=''): must_keys = ['img_norm_cfg', 'ori_filename', 'img_shape', 'ori_shape'] test_pipeline = cfg.data.test.pipeline for key in must_keys: if test_pipeline[1].type == 'MultiRotateAugOCR': collect_pipeline = test_pipeline[1]['transforms'][-1] else: collect_pipeline = test_pipeline[-1] if 'meta_keys' not in collect_pipeline: continue collect_pipeline['meta_keys'].append(key) img_prefix = osp.join(curr_dir, img_prefix) ann_file = osp.join(curr_dir, ann_file) test = copy.deepcopy(cfg.data.test.datasets[0]) test.img_prefix = img_prefix test.ann_file = ann_file cfg.data.workers_per_gpu = 0 cfg.data.test.datasets = [test] dataset = build_dataset(cfg.data.test) loader_cfg = { **dict((k, cfg.data[k]) for k in [ 'workers_per_gpu', 'samples_per_gpu' ] if k in cfg.data) } test_loader_cfg = { **loader_cfg, **dict(shuffle=False, drop_last=False), **cfg.data.get('test_dataloader', {}) } data_loader = build_dataloader(dataset, **test_loader_cfg) return data_loader
def main(): args = parse_args() # Following strings of text style are from colorama package bright_style, reset_style = '\x1b[1m', '\x1b[0m' red_text, blue_text = '\x1b[31m', '\x1b[34m' white_background = '\x1b[107m' msg = white_background + bright_style + red_text msg += 'DeprecationWarning: This tool will be deprecated in future. ' msg += blue_text + 'Welcome to use the unified model deployment toolbox ' msg += 'MMDeploy: https://github.com/open-mmlab/mmdeploy' msg += reset_style warnings.warn(msg) if args.device == 'cpu': args.device = None cfg = Config.fromfile(args.model_config) # build the model if args.model_type == 'det': if args.backend == 'TensorRT': model = TensorRTDetector(args.model_file, cfg, 0) else: model = ONNXRuntimeDetector(args.model_file, cfg, 0) else: if args.backend == 'TensorRT': model = TensorRTRecognizer(args.model_file, cfg, 0) else: model = ONNXRuntimeRecognizer(args.model_file, cfg, 0) # build the dataloader samples_per_gpu = 1 cfg = disable_text_recog_aug_test(cfg) dataset = build_dataset(cfg.data.test) data_loader = build_dataloader(dataset, samples_per_gpu=samples_per_gpu, workers_per_gpu=cfg.data.workers_per_gpu, dist=False, shuffle=False) model = MMDataParallel(model, device_ids=[0]) outputs = single_gpu_test(model, data_loader) rank, _ = get_dist_info() if rank == 0: kwargs = {} if args.eval: eval_kwargs = cfg.get('evaluation', {}).copy() # hard-code way to remove EvalHook args for key in [ 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', 'rule' ]: eval_kwargs.pop(key, None) eval_kwargs.update(dict(metric=args.eval, **kwargs)) print(dataset.evaluate(outputs, **eval_kwargs))
def gene_sdmgr_model_dataloader(cfg, dirname, curr_dir, empty_img=False): json_obj = { 'file_name': '1.jpg', 'height': 348, 'width': 348, 'annotations': [{ 'box': [114.0, 19.0, 230.0, 19.0, 230.0, 1.0, 114.0, 1.0], 'text': 'CHOEUN', 'label': 1 }] } ann_file = osp.join(dirname, 'test.txt') list_to_file(ann_file, [json.dumps(json_obj, ensure_ascii=False)]) if not empty_img: img = np.ones((348, 348, 3), dtype=np.uint8) img_file = osp.join(dirname, '1.jpg') mmcv.imwrite(img, img_file) test = copy.deepcopy(cfg.data.test) test.ann_file = ann_file test.img_prefix = dirname test.dict_file = osp.join(curr_dir, 'data/kie_toy_dataset/dict.txt') cfg.data.workers_per_gpu = 1 cfg.data.test = test cfg.model.class_list = osp.join(curr_dir, 'data/kie_toy_dataset/class_list.txt') dataset = build_dataset(cfg.data.test) loader_cfg = { **dict((k, cfg.data[k]) for k in [ 'workers_per_gpu', 'samples_per_gpu' ] if k in cfg.data) } test_loader_cfg = { **loader_cfg, **dict(shuffle=False, drop_last=False), **cfg.data.get('test_dataloader', {}) } data_loader = build_dataloader(dataset, **test_loader_cfg) model = build_model(cfg) return model, data_loader
def main(): args = parse_args() if args.device == 'cpu': args.device = None cfg = Config.fromfile(args.model_config) # build the model if args.model_type == 'det': if args.backend == 'TensorRT': model = TensorRTDetector(args.model_file, cfg, 0) else: model = ONNXRuntimeDetector(args.model_file, cfg, 0) else: if args.backend == 'TensorRT': model = TensorRTRecognizer(args.model_file, cfg, 0) else: model = ONNXRuntimeRecognizer(args.model_file, cfg, 0) # build the dataloader samples_per_gpu = 1 dataset = build_dataset(cfg.data.test) data_loader = build_dataloader(dataset, samples_per_gpu=samples_per_gpu, workers_per_gpu=cfg.data.workers_per_gpu, dist=False, shuffle=False) model = MMDataParallel(model, device_ids=[0]) outputs = single_gpu_test(model, data_loader) rank, _ = get_dist_info() if rank == 0: kwargs = {} if args.eval: eval_kwargs = cfg.get('evaluation', {}).copy() # hard-code way to remove EvalHook args for key in [ 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', 'rule' ]: eval_kwargs.pop(key, None) eval_kwargs.update(dict(metric=args.eval, **kwargs)) print(dataset.evaluate(outputs, **eval_kwargs))
def main(): args = parse_args() assert ( args.out or args.eval or args.format_only or args.show or args.show_dir), ( 'Please specify at least one operation (save/eval/format/show the ' 'results / save the results) with the argument "--out", "--eval"' ', "--format-only", "--show" or "--show-dir".') if args.eval and args.format_only: raise ValueError('--eval and --format_only cannot be both specified.') if args.out is not None and not args.out.endswith(('.pkl', '.pickle')): raise ValueError('The output file must be a pkl file.') cfg = Config.fromfile(args.config) if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) # import modules from string list. if cfg.get('custom_imports', None): from mmcv.utils import import_modules_from_strings import_modules_from_strings(**cfg['custom_imports']) # set cudnn_benchmark if cfg.get('cudnn_benchmark', False): torch.backends.cudnn.benchmark = True if cfg.model.get('pretrained'): cfg.model.pretrained = None if cfg.model.get('neck'): if isinstance(cfg.model.neck, list): for neck_cfg in cfg.model.neck: if neck_cfg.get('rfp_backbone'): if neck_cfg.rfp_backbone.get('pretrained'): neck_cfg.rfp_backbone.pretrained = None elif cfg.model.neck.get('rfp_backbone'): if cfg.model.neck.rfp_backbone.get('pretrained'): cfg.model.neck.rfp_backbone.pretrained = None # in case the test dataset is concatenated samples_per_gpu = 1 if isinstance(cfg.data.test, dict): samples_per_gpu = (cfg.data.get('test_dataloader', {})).get( 'samples_per_gpu', cfg.data.get('samples_per_gpu', 1)) if samples_per_gpu > 1: # Support batch_size > 1 in test for text recognition # by disable MultiRotateAugOCR since it is useless for most case cfg = disable_text_recog_aug_test(cfg) if cfg.data.test.get('pipeline', None) is not None: # Replace 'ImageToTensor' to 'DefaultFormatBundle' cfg.data.test.pipeline = replace_ImageToTensor( cfg.data.test.pipeline) elif isinstance(cfg.data.test, list): for ds_cfg in cfg.data.test: ds_cfg.test_mode = True samples_per_gpu = max( [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test]) if samples_per_gpu > 1: for ds_cfg in cfg.data.test: ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) # init distributed env first, since logger depends on the dist info. if args.launcher == 'none': distributed = False else: distributed = True init_dist(args.launcher, **cfg.dist_params) # build the dataloader dataset = build_dataset(cfg.data.test, dict(test_mode=True)) # step 1: give default values and override (if exist) from cfg.data loader_cfg = { **dict(seed=cfg.get('seed'), drop_last=False, dist=distributed), **({} if torch.__version__ != 'parrots' else dict( prefetch_num=2, pin_memory=False, )), **dict((k, cfg.data[k]) for k in [ 'workers_per_gpu', 'seed', 'prefetch_num', 'pin_memory', 'persistent_workers', ] if k in cfg.data) } test_loader_cfg = { **loader_cfg, **dict(shuffle=False, drop_last=False), **cfg.data.get('test_dataloader', {}), **dict(samples_per_gpu=samples_per_gpu) } data_loader = build_dataloader(dataset, **test_loader_cfg) # build the model and load checkpoint cfg.model.train_cfg = None model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg')) model = revert_sync_batchnorm(model) fp16_cfg = cfg.get('fp16', None) if fp16_cfg is not None: wrap_fp16_model(model) load_checkpoint(model, args.checkpoint, map_location='cpu') if args.fuse_conv_bn: model = fuse_conv_bn(model) if not distributed: model = MMDataParallel(model, device_ids=[0]) is_kie = cfg.model.type in ['SDMGR'] outputs = single_gpu_test(model, data_loader, args.show, args.show_dir, is_kie, args.show_score_thr) else: model = MMDistributedDataParallel( model.cuda(), device_ids=[torch.cuda.current_device()], broadcast_buffers=False) outputs = multi_gpu_test(model, data_loader, args.tmpdir, args.gpu_collect) rank, _ = get_dist_info() if rank == 0: if args.out: print(f'\nwriting results to {args.out}') mmcv.dump(outputs, args.out) kwargs = {} if args.eval_options is None else args.eval_options if args.format_only: dataset.format_results(outputs, **kwargs) if args.eval: eval_kwargs = cfg.get('evaluation', {}).copy() # hard-code way to remove EvalHook args for key in [ 'interval', 'tmpdir', 'start', 'gpu_collect', 'save_best', 'rule' ]: eval_kwargs.pop(key, None) eval_kwargs.update(dict(metric=args.eval, **kwargs)) print(dataset.evaluate(outputs, **eval_kwargs))
def main(): args = parse_args() cfg = Config.fromfile(args.config) if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) # update mc config if args.mc_config: mc = Config.fromfile(args.mc_config) if isinstance(cfg.data.train, list): for i in range(len(cfg.data.train)): cfg.data.train[i].pipeline[0].update( file_client_args=mc['mc_file_client_args']) else: cfg.data.train.pipeline[0].update( file_client_args=mc['mc_file_client_args']) # set cudnn_benchmark if cfg.get('cudnn_benchmark', False): torch.backends.cudnn.benchmark = True # work_dir is determined in this priority: CLI > segment in file > filename if args.work_dir is not None: # update configs according to CLI args if args.work_dir is not None cfg.work_dir = args.work_dir elif cfg.get('work_dir', None) is None: # use config filename as default work_dir if cfg.work_dir is None cfg.work_dir = osp.join('./work_dirs', osp.splitext(osp.basename(args.config))[0]) if args.load_from is not None: cfg.load_from = args.load_from if args.resume_from is not None: cfg.resume_from = args.resume_from if args.gpu_ids is not None: cfg.gpu_ids = args.gpu_ids else: cfg.gpu_ids = range(1) if args.gpus is None else range(args.gpus) # init distributed env first, since logger depends on the dist info. if args.launcher == 'none': distributed = False else: distributed = True init_dist(args.launcher, **cfg.dist_params) # re-set gpu_ids with distributed training mode _, world_size = get_dist_info() cfg.gpu_ids = range(world_size) # create work_dir mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir)) # dump config cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config))) # init the logger before other steps timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) log_file = osp.join(cfg.work_dir, f'{timestamp}.log') logger = get_root_logger(log_file=log_file, log_level=cfg.log_level) # init the meta dict to record some important information such as # environment info and seed, which will be logged meta = dict() # log env info env_info_dict = collect_env() env_info = '\n'.join([(f'{k}: {v}') for k, v in env_info_dict.items()]) dash_line = '-' * 60 + '\n' logger.info('Environment info:\n' + dash_line + env_info + '\n' + dash_line) meta['env_info'] = env_info meta['config'] = cfg.pretty_text # log some basic info logger.info(f'Distributed training: {distributed}') logger.info(f'Config:\n{cfg.pretty_text}') # set random seeds seed = init_random_seed(args.seed) logger.info(f'Set random seed to {seed}, ' f'deterministic: {args.deterministic}') set_random_seed(seed, deterministic=args.deterministic) cfg.seed = seed meta['seed'] = seed meta['exp_name'] = osp.basename(args.config) model = build_detector(cfg.model, train_cfg=cfg.get('train_cfg'), test_cfg=cfg.get('test_cfg')) model.init_weights() datasets = [build_dataset(cfg.data.train)] if len(cfg.workflow) == 2: val_dataset = copy.deepcopy(cfg.data.val) if cfg.data.train['type'] == 'ConcatDataset': train_pipeline = cfg.data.train['datasets'][0].pipeline else: train_pipeline = cfg.data.train.pipeline if val_dataset['type'] == 'ConcatDataset': for dataset in val_dataset['datasets']: dataset.pipeline = train_pipeline else: val_dataset.pipeline = train_pipeline datasets.append(build_dataset(val_dataset)) if cfg.checkpoint_config is not None: # save mmdet version, config file content and class names in # checkpoints as meta data cfg.checkpoint_config.meta = dict(mmocr_version=__version__ + get_git_hash()[:7], CLASSES=datasets[0].CLASSES) # add an attribute for visualization convenience model.CLASSES = datasets[0].CLASSES train_detector(model, datasets, cfg, distributed=distributed, validate=(not args.no_validate), timestamp=timestamp, meta=meta)