def main(): """ Main evaluate function """ cfg = load_config(FLAGS.config) if 'architecture' in cfg: main_arch = cfg.architecture else: raise ValueError("'architecture' not specified in config file.") merge_config(FLAGS.opt) # check if set use_gpu=True in paddlepaddle cpu version check_gpu(cfg.use_gpu) if cfg.use_gpu: devices_num = fluid.core.get_cuda_device_count() else: devices_num = int( os.environ.get('CPU_NUM', multiprocessing.cpu_count())) if 'eval_feed' not in cfg: eval_feed = create(main_arch + 'EvalFeed') else: eval_feed = create(cfg.eval_feed) # define executor place = fluid.CUDAPlace(0) if cfg.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) # build program model = create(main_arch) startup_prog = fluid.Program() eval_prog = fluid.Program() with fluid.program_guard(eval_prog, startup_prog): with fluid.unique_name.guard(): pyreader, feed_vars = create_feed(eval_feed) fetches = model.eval(feed_vars) eval_prog = eval_prog.clone(True) reader = create_reader(eval_feed, args_path=FLAGS.dataset_dir) pyreader.decorate_sample_list_generator(reader, place) # eval already exists json file if FLAGS.json_eval: logger.info( "In json_eval mode, PaddleDetection will evaluate json files in " "output_eval directly. And proposal.json, bbox.json and mask.json " "will be detected by default.") json_eval_results(eval_feed, cfg.metric, json_directory=FLAGS.output_eval) return # compile program for multi-devices if devices_num <= 1: compile_program = fluid.compiler.CompiledProgram(eval_prog) else: build_strategy = fluid.BuildStrategy() build_strategy.memory_optimize = False build_strategy.enable_inplace = False compile_program = fluid.compiler.CompiledProgram( eval_prog).with_data_parallel(build_strategy=build_strategy) # load model exe.run(startup_prog) if 'weights' in cfg: checkpoint.load_pretrain(exe, eval_prog, cfg.weights) assert cfg.metric in ['COCO', 'VOC'], \ "unknown metric type {}".format(cfg.metric) extra_keys = [] if cfg.metric == 'COCO': extra_keys = ['im_info', 'im_id', 'im_shape'] if cfg.metric == 'VOC': extra_keys = ['gt_box', 'gt_label', 'is_difficult'] keys, values, cls = parse_fetches(fetches, eval_prog, extra_keys) # whether output bbox is normalized in model output layer is_bbox_normalized = False if hasattr(model, 'is_bbox_normalized') and \ callable(model.is_bbox_normalized): is_bbox_normalized = model.is_bbox_normalized() results = eval_run(exe, compile_program, pyreader, keys, values, cls) # evaluation resolution = None if 'mask' in results[0]: resolution = model.mask_head.resolution eval_results(results, eval_feed, cfg.metric, cfg.num_classes, resolution, is_bbox_normalized, FLAGS.output_eval, cfg.map_type)
def main(): """ Main evaluate function """ cfg = load_config(FLAGS.config) merge_config(FLAGS.opt) check_config(cfg) # check if set use_gpu=True in paddlepaddle cpu version check_gpu(cfg.use_gpu) # check if paddlepaddle version is satisfied check_version() main_arch = cfg.architecture multi_scale_test = getattr(cfg, 'MultiScaleTEST', None) # define executor place = fluid.CUDAPlace(0) if cfg.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) # build program model = create(main_arch) startup_prog = fluid.Program() eval_prog = fluid.Program() with fluid.program_guard(eval_prog, startup_prog): with fluid.unique_name.guard(): inputs_def = cfg['EvalReader']['inputs_def'] feed_vars, loader = model.build_inputs(**inputs_def) if multi_scale_test is None: fetches = model.eval(feed_vars) else: fetches = model.eval(feed_vars, multi_scale_test) eval_prog = eval_prog.clone(True) reader = create_reader(cfg.EvalReader, devices_num=1) loader.set_sample_list_generator(reader, place) dataset = cfg['EvalReader']['dataset'] # eval already exists json file if FLAGS.json_eval: logger.info( "In json_eval mode, PaddleDetection will evaluate json files in " "output_eval directly. And proposal.json, bbox.json and mask.json " "will be detected by default.") json_eval_results(cfg.metric, json_directory=FLAGS.output_eval, dataset=dataset) return compile_program = fluid.compiler.CompiledProgram( eval_prog).with_data_parallel() assert cfg.metric != 'OID', "eval process of OID dataset \ is not supported." if cfg.metric == "WIDERFACE": raise ValueError("metric type {} does not support in tools/eval.py, " "please use tools/face_eval.py".format(cfg.metric)) assert cfg.metric in ['COCO', 'VOC', 'traffic'], \ "unknown metric type {}".format(cfg.metric) extra_keys = [] if cfg.metric == 'COCO': extra_keys = ['im_info', 'im_id', 'im_shape'] elif cfg.metric == 'VOC': extra_keys = ['gt_bbox', 'gt_class', 'is_difficult'] else: extra_keys = ['gt_bbox', 'gt_class', 'im_id'] keys, values, cls = parse_fetches(fetches, eval_prog, extra_keys) # whether output bbox is normalized in model output layer is_bbox_normalized = False if hasattr(model, 'is_bbox_normalized') and \ callable(model.is_bbox_normalized): is_bbox_normalized = model.is_bbox_normalized() sub_eval_prog = None sub_keys = None sub_values = None # build sub-program if 'Mask' in main_arch and multi_scale_test: sub_eval_prog = fluid.Program() with fluid.program_guard(sub_eval_prog, startup_prog): with fluid.unique_name.guard(): inputs_def = cfg['EvalReader']['inputs_def'] inputs_def['mask_branch'] = True feed_vars, eval_loader = model.build_inputs(**inputs_def) sub_fetches = model.eval(feed_vars, multi_scale_test, mask_branch=True) assert cfg.metric == 'COCO' extra_keys = ['im_id', 'im_shape'] sub_keys, sub_values, _ = parse_fetches(sub_fetches, sub_eval_prog, extra_keys) sub_eval_prog = sub_eval_prog.clone(True) #if 'weights' in cfg: # checkpoint.load_params(exe, sub_eval_prog, cfg.weights) # load model exe.run(startup_prog) if 'weights' in cfg: checkpoint.load_params(exe, startup_prog, cfg.weights) resolution = None if 'Mask' in cfg.architecture: resolution = model.mask_head.resolution results = eval_run(exe, compile_program, loader, keys, values, cls, cfg, sub_eval_prog, sub_keys, sub_values, resolution) save_only = getattr(cfg, 'save_prediction_only', False) if cfg.metric == 'traffic': from ppdet.utils.traffic_eval import get_category_info, bbox2out, write_output with_background = dataset.with_background dataset_dir = dataset.dataset_dir im_info_file = os.path.join(dataset_dir, 'data_info.txt') clsid2catid, catid2name = get_category_info( with_background=with_background) xywh_results = bbox2out(results, clsid2catid, is_bbox_normalized=is_bbox_normalized) if save_only: write_output(xywh_results, im_info_file, catid2name, dataset.get_anno(), threshold=0.63, outpath='output/detect') return # evaluation # if map_type not set, use default 11point, only use in VOC eval map_type = cfg.map_type if 'map_type' in cfg else '11point' eval_results(results, cfg.metric, cfg.num_classes, resolution, is_bbox_normalized, FLAGS.output_eval, map_type, dataset=dataset, save_only=save_only)
def main(): """ Main evaluate function """ cfg = load_config(FLAGS.config) merge_config(FLAGS.opt) check_config(cfg) # check if set use_gpu=True in paddlepaddle cpu version check_gpu(cfg.use_gpu) use_xpu = False if hasattr(cfg, 'use_xpu'): check_xpu(cfg.use_xpu) use_xpu = cfg.use_xpu # check if paddlepaddle version is satisfied check_version() assert not (use_xpu and cfg.use_gpu), \ 'Can not run on both XPU and GPU' main_arch = cfg.architecture multi_scale_test = getattr(cfg, 'MultiScaleTEST', None) # define executor if cfg.use_gpu: place = fluid.CUDAPlace(0) elif use_xpu: place = fluid.XPUPlace(0) else: place = fluid.CPUPlace() exe = fluid.Executor(place) # build program model = create(main_arch) startup_prog = fluid.Program() eval_prog = fluid.Program() with fluid.program_guard(eval_prog, startup_prog): with fluid.unique_name.guard(): inputs_def = cfg['EvalReader']['inputs_def'] feed_vars, loader = model.build_inputs(**inputs_def) if multi_scale_test is None: fetches = model.eval(feed_vars) else: fetches = model.eval(feed_vars, multi_scale_test) eval_prog = eval_prog.clone(True) reader = create_reader(cfg.EvalReader, devices_num=1) # When iterable mode, set set_sample_list_generator(reader, place) loader.set_sample_list_generator(reader) dataset = cfg['EvalReader']['dataset'] # eval already exists json file if FLAGS.json_eval: logger.info( "In json_eval mode, PaddleDetection will evaluate json files in " "output_eval directly. And proposal.json, bbox.json and mask.json " "will be detected by default.") json_eval_results( cfg.metric, json_directory=FLAGS.output_eval, dataset=dataset) return compile_program = fluid.CompiledProgram(eval_prog).with_data_parallel() if use_xpu: compile_program = eval_prog assert cfg.metric != 'OID', "eval process of OID dataset \ is not supported." if cfg.metric == "WIDERFACE": raise ValueError("metric type {} does not support in tools/eval.py, " "please use tools/face_eval.py".format(cfg.metric)) assert cfg.metric in ['COCO', 'VOC'], \ "unknown metric type {}".format(cfg.metric) extra_keys = [] if cfg.metric == 'COCO': extra_keys = ['im_info', 'im_id', 'im_shape'] if cfg.metric == 'VOC': extra_keys = ['gt_bbox', 'gt_class', 'is_difficult'] keys, values, cls = parse_fetches(fetches, eval_prog, extra_keys) # whether output bbox is normalized in model output layer is_bbox_normalized = False if hasattr(model, 'is_bbox_normalized') and \ callable(model.is_bbox_normalized): is_bbox_normalized = model.is_bbox_normalized() sub_eval_prog = None sub_keys = None sub_values = None # build sub-program if 'Mask' in main_arch and multi_scale_test: sub_eval_prog = fluid.Program() with fluid.program_guard(sub_eval_prog, startup_prog): with fluid.unique_name.guard(): inputs_def = cfg['EvalReader']['inputs_def'] inputs_def['mask_branch'] = True feed_vars, eval_loader = model.build_inputs(**inputs_def) sub_fetches = model.eval( feed_vars, multi_scale_test, mask_branch=True) assert cfg.metric == 'COCO' extra_keys = ['im_id', 'im_shape'] sub_keys, sub_values, _ = parse_fetches(sub_fetches, sub_eval_prog, extra_keys) sub_eval_prog = sub_eval_prog.clone(True) # load model exe.run(startup_prog) if 'weights' in cfg: checkpoint.load_params(exe, startup_prog, cfg.weights) resolution = None if 'Mask' in cfg.architecture or cfg.architecture == 'HybridTaskCascade': resolution = model.mask_head.resolution results = eval_run(exe, compile_program, loader, keys, values, cls, cfg, sub_eval_prog, sub_keys, sub_values, resolution) # evaluation # if map_type not set, use default 11point, only use in VOC eval map_type = cfg.map_type if 'map_type' in cfg else '11point' save_only = getattr(cfg, 'save_prediction_only', False) eval_results( results, cfg.metric, cfg.num_classes, resolution, is_bbox_normalized, FLAGS.output_eval, map_type, dataset=dataset, save_only=save_only)
def main(): """ Main evaluate function """ cfg = load_config(FLAGS.config) if 'architecture' in cfg: main_arch = cfg.architecture else: raise ValueError("'architecture' not specified in config file.") merge_config(FLAGS.opt) # check if set use_gpu=True in paddlepaddle cpu version check_gpu(cfg.use_gpu) # check if paddlepaddle version is satisfied check_version() if 'eval_feed' not in cfg: eval_feed = create(main_arch + 'EvalFeed') else: eval_feed = create(cfg.eval_feed) multi_scale_test = getattr(cfg, 'MultiScaleTEST', None) # define executor place = fluid.CUDAPlace(0) if cfg.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) # build program model = create(main_arch) startup_prog = fluid.Program() eval_prog = fluid.Program() with fluid.program_guard(eval_prog, startup_prog): with fluid.unique_name.guard(): loader, feed_vars = create_feed(eval_feed) if multi_scale_test is None: fetches = model.eval(feed_vars) else: fetches = model.eval(feed_vars, multi_scale_test) eval_prog = eval_prog.clone(True) reader = create_reader(eval_feed, args_path=FLAGS.dataset_dir) loader.set_sample_list_generator(reader, place) # eval already exists json file if FLAGS.json_eval: logger.info( "In json_eval mode, PaddleDetection will evaluate json files in " "output_eval directly. And proposal.json, bbox.json and mask.json " "will be detected by default.") json_eval_results(eval_feed, cfg.metric, json_directory=FLAGS.output_eval) return compile_program = fluid.compiler.CompiledProgram( eval_prog).with_data_parallel() # load model exe.run(startup_prog) if 'weights' in cfg: checkpoint.load_params(exe, eval_prog, cfg.weights) assert cfg.metric != 'OID', "eval process of OID dataset \ is not supported." if cfg.metric == "WIDERFACE": raise ValueError("metric type {} does not support in tools/eval.py, " "please use tools/face_eval.py".format(cfg.metric)) assert cfg.metric in ['COCO', 'VOC'], \ "unknown metric type {}".format(cfg.metric) extra_keys = [] if cfg.metric == 'COCO': extra_keys = ['im_info', 'im_id', 'im_shape'] if cfg.metric == 'VOC': extra_keys = ['gt_box', 'gt_label', 'is_difficult'] keys, values, cls = parse_fetches(fetches, eval_prog, extra_keys) # whether output bbox is normalized in model output layer is_bbox_normalized = False if hasattr(model, 'is_bbox_normalized') and \ callable(model.is_bbox_normalized): is_bbox_normalized = model.is_bbox_normalized() sub_eval_prog = None sub_keys = None sub_values = None # build sub-program if 'Mask' in main_arch and multi_scale_test: sub_eval_prog = fluid.Program() with fluid.program_guard(sub_eval_prog, startup_prog): with fluid.unique_name.guard(): _, feed_vars = create_feed(eval_feed, False, sub_prog_feed=True) sub_fetches = model.eval(feed_vars, multi_scale_test, mask_branch=True) extra_keys = [] if cfg.metric == 'COCO': extra_keys = ['im_id', 'im_shape'] if cfg.metric == 'VOC': extra_keys = ['gt_box', 'gt_label', 'is_difficult'] sub_keys, sub_values, _ = parse_fetches(sub_fetches, sub_eval_prog, extra_keys) sub_eval_prog = sub_eval_prog.clone(True) if 'weights' in cfg: checkpoint.load_params(exe, sub_eval_prog, cfg.weights) results = eval_run(exe, compile_program, loader, keys, values, cls, cfg, sub_eval_prog, sub_keys, sub_values) # evaluation resolution = None if 'mask' in results[0]: resolution = model.mask_head.resolution # if map_type not set, use default 11point, only use in VOC eval map_type = cfg.map_type if 'map_type' in cfg else '11point' eval_results(results, eval_feed, cfg.metric, cfg.num_classes, resolution, is_bbox_normalized, FLAGS.output_eval, map_type)
def main(): """ Main evaluate function """ cfg = load_config(FLAGS.config) merge_config(FLAGS.opt) check_config(cfg) # check if set use_gpu=True in paddlepaddle cpu version check_gpu(cfg.use_gpu) # check if paddlepaddle version is satisfied check_version() main_arch = cfg.architecture multi_scale_test = getattr(cfg, 'MultiScaleTEST', None) # define executor place = fluid.CUDAPlace(0) if cfg.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) # build program model = create(main_arch) startup_prog = fluid.Program() eval_prog = fluid.Program() with fluid.program_guard(eval_prog, startup_prog): with fluid.unique_name.guard(): inputs_def = cfg['EvalReader']['inputs_def'] feed_vars, loader = model.build_inputs(**inputs_def) if multi_scale_test is None: fetches = model.eval(feed_vars) else: fetches = model.eval(feed_vars, multi_scale_test) eval_prog = eval_prog.clone(True) exe.run(startup_prog) reader = create_reader(cfg.EvalReader) # When iterable mode, set set_sample_list_generator(reader, place) loader.set_sample_list_generator(reader) dataset = cfg['EvalReader']['dataset'] # eval already exists json file if FLAGS.json_eval: logger.info( "In json_eval mode, PaddleDetection will evaluate json files in " "output_eval directly. And proposal.json, bbox.json and mask.json " "will be detected by default.") json_eval_results( cfg.metric, json_directory=FLAGS.output_eval, dataset=dataset) return pruned_params = FLAGS.pruned_params assert ( FLAGS.pruned_params is not None ), "FLAGS.pruned_params is empty!!! Please set it by '--pruned_params' option." pruned_params = FLAGS.pruned_params.strip().split(",") logger.info("pruned params: {}".format(pruned_params)) pruned_ratios = [float(n) for n in FLAGS.pruned_ratios.strip().split(",")] logger.info("pruned ratios: {}".format(pruned_ratios)) assert (len(pruned_params) == len(pruned_ratios) ), "The length of pruned params and pruned ratios should be equal." assert (pruned_ratios > [0] * len(pruned_ratios) and pruned_ratios < [1] * len(pruned_ratios) ), "The elements of pruned ratios should be in range (0, 1)." base_flops = flops(eval_prog) pruner = Pruner() eval_prog, _, _ = pruner.prune( eval_prog, fluid.global_scope(), params=pruned_params, ratios=pruned_ratios, place=place, only_graph=False) pruned_flops = flops(eval_prog) logger.info("pruned FLOPS: {}".format( float(base_flops - pruned_flops) / base_flops)) compile_program = fluid.CompiledProgram(eval_prog).with_data_parallel() assert cfg.metric != 'OID', "eval process of OID dataset \ is not supported." if cfg.metric == "WIDERFACE": raise ValueError("metric type {} does not support in tools/eval.py, " "please use tools/face_eval.py".format(cfg.metric)) assert cfg.metric in ['COCO', 'VOC'], \ "unknown metric type {}".format(cfg.metric) extra_keys = [] if cfg.metric == 'COCO': extra_keys = ['im_info', 'im_id', 'im_shape'] if cfg.metric == 'VOC': extra_keys = ['gt_bbox', 'gt_class', 'is_difficult'] keys, values, cls = parse_fetches(fetches, eval_prog, extra_keys) # whether output bbox is normalized in model output layer is_bbox_normalized = False if hasattr(model, 'is_bbox_normalized') and \ callable(model.is_bbox_normalized): is_bbox_normalized = model.is_bbox_normalized() sub_eval_prog = None sub_keys = None sub_values = None # build sub-program if 'Mask' in main_arch and multi_scale_test: sub_eval_prog = fluid.Program() with fluid.program_guard(sub_eval_prog, startup_prog): with fluid.unique_name.guard(): inputs_def = cfg['EvalReader']['inputs_def'] inputs_def['mask_branch'] = True feed_vars, eval_loader = model.build_inputs(**inputs_def) sub_fetches = model.eval( feed_vars, multi_scale_test, mask_branch=True) assert cfg.metric == 'COCO' extra_keys = ['im_id', 'im_shape'] sub_keys, sub_values, _ = parse_fetches(sub_fetches, sub_eval_prog, extra_keys) sub_eval_prog = sub_eval_prog.clone(True) # load model if 'weights' in cfg: checkpoint.load_checkpoint(exe, eval_prog, cfg.weights) resolution = None if 'Mask' in cfg.architecture: resolution = model.mask_head.resolution results = eval_run( exe, compile_program, loader, keys, values, cls, cfg, sub_eval_prog, sub_keys, sub_values, resolution=resolution) # if map_type not set, use default 11point, only use in VOC eval map_type = cfg.map_type if 'map_type' in cfg else '11point' eval_results( results, cfg.metric, cfg.num_classes, resolution, is_bbox_normalized, FLAGS.output_eval, map_type, dataset=dataset)
def main(): """ Main evaluate function """ cfg = load_config(FLAGS.config) merge_config(FLAGS.opt) check_config(cfg) # check if set use_gpu=True in paddlepaddle cpu version check_gpu(cfg.use_gpu) # check if paddlepaddle version is satisfied check_version() main_arch = cfg.architecture # define executor place = fluid.CUDAPlace(0) if cfg.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) # build program model = create(main_arch) startup_prog = fluid.Program() eval_prog = fluid.Program() with fluid.program_guard(eval_prog, startup_prog): with fluid.unique_name.guard(): inputs_def = cfg['EvalReader']['inputs_def'] test_feed_vars, loader = model.build_inputs(**inputs_def) test_fetches = model.eval(test_feed_vars) eval_prog = eval_prog.clone(True) reader = create_reader(cfg.EvalReader) loader.set_sample_list_generator(reader, place) # eval already exists json file if FLAGS.json_eval: logger.info( "In json_eval mode, PaddleDetection will evaluate json files in " "output_eval directly. And proposal.json, bbox.json and mask.json " "will be detected by default.") json_eval_results(cfg.metric, json_directory=FLAGS.output_eval, dataset=dataset) return assert cfg.metric != 'OID', "eval process of OID dataset \ is not supported." if cfg.metric == "WIDERFACE": raise ValueError("metric type {} does not support in tools/eval.py, " "please use tools/face_eval.py".format(cfg.metric)) assert cfg.metric in ['COCO', 'VOC'], \ "unknown metric type {}".format(cfg.metric) extra_keys = [] if cfg.metric == 'COCO': extra_keys = ['im_info', 'im_id', 'im_shape'] if cfg.metric == 'VOC': extra_keys = ['gt_bbox', 'gt_class', 'is_difficult'] keys, values, cls = parse_fetches(test_fetches, eval_prog, extra_keys) # whether output bbox is normalized in model output layer is_bbox_normalized = False if hasattr(model, 'is_bbox_normalized') and \ callable(model.is_bbox_normalized): is_bbox_normalized = model.is_bbox_normalized() dataset = cfg['EvalReader']['dataset'] sub_eval_prog = None sub_keys = None sub_values = None not_quant_pattern = [] if FLAGS.not_quant_pattern: not_quant_pattern = FLAGS.not_quant_pattern config = { 'weight_quantize_type': 'channel_wise_abs_max', 'activation_quantize_type': 'moving_average_abs_max', 'quantize_op_types': ['depthwise_conv2d', 'mul', 'conv2d'], 'not_quant_pattern': not_quant_pattern } eval_prog = quant_aware(eval_prog, place, config, for_test=True) # load model exe.run(startup_prog) if 'weights' in cfg: checkpoint.load_params(exe, eval_prog, cfg.weights) eval_prog = convert(eval_prog, place, config, save_int8=False) compile_program = fluid.compiler.CompiledProgram( eval_prog).with_data_parallel() results = eval_run(exe, compile_program, loader, keys, values, cls, cfg, sub_eval_prog, sub_keys, sub_values) # evaluation resolution = None if 'mask' in results[0]: resolution = model.mask_head.resolution # if map_type not set, use default 11point, only use in VOC eval map_type = cfg.map_type if 'map_type' in cfg else '11point' eval_results(results, cfg.metric, cfg.num_classes, resolution, is_bbox_normalized, FLAGS.output_eval, map_type, dataset=dataset)