def parse_args(argv):
"""Parse and process arguments for evaluation"""
parser = ArgumentParser(description='Accuracy evaluation tool', allow_abbrev=False)
parser.add_argument(
'-c',
'--config',
help='Path to a config file with model-specific parameters',
required=True)
parser.add_argument(
'-m',
'--compressed_model',
help='Path to a compressed model (.xml)',
required=False)
parser.add_argument(
'-ss',
'--subset_size',
help='Size of subset to make evaluation on',
type=int,
required=False)
args = parser.parse_args(args=argv)
config = Config.read_config(args.config)
subset = range(args.subset_size) if args.subset_size else None
path = None
if args.compressed_model:
model_xml_path = Path(args.compressed_model).resolve()
model_bin_path = model_xml_path.with_suffix('.bin')
path = [{'model': model_xml_path, 'weights': model_bin_path}]
logger.info('Model: %s', model_xml_path)
return config, subset, path
def app(argv):
telemetry = start_session_telemetry()
parser = get_common_argument_parser()
args = parser.parse_args(args=argv)
check_dependencies(args)
if not args.config:
_update_config_path(args)
config = Config.read_config(args.config)
config.configure_params(args.ac_config)
config.update_from_args(args)
if config.engine.type == 'simplified' and args.evaluate:
raise Exception('Can not make evaluation in simplified mode')
log_dir = _create_log_path(config)
init_logger(level=args.log_level,
file_name=os.path.join(log_dir, 'log.txt'),
progress_bar=args.pbar)
logger.info('Output log dir: {}'.format(log_dir))
metrics = optimize(config)
if metrics and logger.progress_bar_disabled:
for name, value in metrics.items():
logger.info('{: <27s}: {}'.format(name, value))
end_session_telemetry(telemetry)
def test_algo_params_validation(algorithm_settings):
tool_config_path = TOOL_CONFIG_PATH.joinpath(
'mobilenet-v2-pytorch_single_dataset.json').as_posix()
config = Config.read_config(tool_config_path)
config['compression']['algorithms'][0] = algorithm_settings[1][0]
config_error = algorithm_settings[1][1]
with pytest.raises(RuntimeError, match=config_error):
config.validate_algo_config()
def test_load_tool_config(config_name, argv):
parser = get_common_argument_parser()
argv = argv.split()
argv[-1] = ENGINE_CONFIG_PATH.joinpath('mobilenet-ssd.json').as_posix()
args = parser.parse_args(args=argv)
tool_config_path = TOOL_CONFIG_PATH.joinpath(config_name).as_posix()
config = Config.read_config(tool_config_path)
config.configure_params()
config.update_from_args(args)
assert config.model.model == argv[5]
assert config.model.weights == argv[3]
def merge_configs(model_conf, engine_conf, algo_conf):
config = Config()
# mo config
config.model = model_conf
# ac config
config.engine = engine_conf
# algo config
opt_config = algo_conf.pop('optimizer', None)
if opt_config is not None:
config.optimizer = opt_config
config.compression = algo_conf
config.add_log_dir(config.model.output_dir, config.model.output_dir)
return config
def test_load_tool_config(config_name, tmp_path, models):
tool_config_path = TOOL_CONFIG_PATH.joinpath(config_name).as_posix()
config = Config.read_config(tool_config_path)
config.configure_params()
config.engine.log_dir = tmp_path.as_posix()
config.engine.evaluate = True
model_name, model_framework = TEST_MODEL
model = models.get(model_name, model_framework, tmp_path)
config.model.model = model.model_params.model
config.model.weights = model.model_params.weights
provide_dataset_path(config.engine)
ConfigReader.convert_paths(config.engine)
pipeline = create_pipeline(config.compression.algorithms, ACEngine(config.engine))
model = load_model(config.model)
assert not isinstance(model, int)
assert pipeline.run(model)
def compress_model():
telemetry.value = set()
tool_config_path = TELEMETRY_CONFIG_PATH.joinpath(
config_name).as_posix()
config = Config.read_config(tool_config_path)
config.configure_params()
config.engine.log_dir = tmp_path.as_posix()
config.engine.evaluate = True
model_name, model_framework = TEST_MODEL
model = models.get(model_name, model_framework, tmp_path)
config.model.model = model.model_params.model
config.model.weights = model.model_params.weights
provide_dataset_path(config.engine)
ConfigReader.convert_paths(config.engine)
pipeline = create_pipeline(config.compression.algorithms,
ACEngine(config.engine), 'CLI')
model = load_model(config.model)
pipeline.run(model)
assert set(telemetry.value) == set(expected[config_name])
def app(argv):
telemetry = start_session_telemetry()
parser = get_common_argument_parser()
args = parser.parse_args(args=argv)
check_dependencies(args)
if not args.config:
_update_config_path(args)
config = Config.read_config(args.config)
if args.engine:
config.engine[
'type'] = args.engine if args.engine else 'accuracy_checker'
if 'data_source' not in config.engine:
if args.data_source is None and config.engine.type == 'data_free':
args.data_source = 'pot_dataset'
config.engine['data_source'] = args.data_source
config.configure_params(args.ac_config)
config.update_from_args(args)
if config.engine.type != 'accuracy_checker' and args.evaluate:
raise Exception(
'Can not make evaluation in simplified or data_free mode')
log_dir = _create_log_path(config)
init_logger(level=args.log_level,
file_name=os.path.join(log_dir, 'log.txt'),
progress_bar=args.pbar)
logger.info('Output log dir: {}'.format(log_dir))
metrics = optimize(config)
if metrics and logger.progress_bar_disabled:
for name, value in metrics.items():
logger.info('{: <27s}: {}'.format(name, value))
end_session_telemetry(telemetry)
def read_config(filename):
tool_config_path = TOOL_CONFIG_PATH.joinpath(filename).as_posix()
config = Config.read_config(tool_config_path)
config.configure_params()
return config['compression']['algorithms'][0]['params'][
'target_device']