def setUpClass(cls):
cls.templates_folder = os.environ['MODEL_TEMPLATES']
cls.template_folder = os.path.join(cls.templates_folder,
domain_name, problem_name,
model_name)
skip_non_instantiated_template_if_its_allowed(
cls.template_folder, problem_name, model_name)
cls.template_file = os.path.join(cls.template_folder,
'template.yaml')
cls.ann_file = ann_file
cls.img_root = img_root
cls.dependencies = get_dependencies(cls.template_file)
cls.epochs_delta = 1
cls.total_epochs = get_epochs(cls.template_file) + cls.epochs_delta
expected_outputs_json = f'{expected_outputs_dir}/{problem_name}/{model_name}.json'
with open(expected_outputs_json) as read_file:
cls.expected_outputs = json.load(read_file)
download_snapshot_if_not_yet(cls.template_file,
cls.template_folder)
run_through_shell(f'cd {cls.template_folder};'
f'pip install -r requirements.txt;')
def export_test_on_cpu(self, thr):
export_folder = 'cpu_export'
if not os.path.exists(export_folder):
self.do_export(export_folder)
export_dir = os.path.join(self.template_folder, export_folder)
run_through_shell(
f'export CUDA_VISIBLE_DEVICES=;'
f'cd {os.path.dirname(self.template_file)};'
f'python eval.py'
f' --test-ann-files {ann_file}'
f' --test-data-roots {img_root}'
f' --load-weights {os.path.join(export_dir, "model.bin")}'
f' --save-metrics-to {os.path.join(export_dir, "metrics.yaml")}'
)
with open(os.path.join(export_dir, "metrics.yaml")) as read_file:
content = yaml.load(read_file, yaml.SafeLoader)
est_accuracy = [
metric['value'] for metric in content['metrics']
if metric['key'] == 'accuracy'
][0]
with open(
f'{os.path.dirname(__file__)}/../expected_outputs/{problem_name}/{model_name}.json'
) as read_file:
content = json.load(read_file)
ref_accuracy = content['accuracy']
self.assertGreater(1e-2 * est_accuracy, ref_accuracy - thr)
def _export_to_openvino(self, args, tools_dir):
run_through_shell(f'python {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{args["save_model_to"]} '
f'openvino '
f'--input_format {args["openvino_input_format"]}')
# FIXME(ikrylov): remove alt_ssd_export block as soon as it becomes useless.
# (LeonidBeynenson): Please, note that alt_ssd_export appoach may be applied only
# to SSD models only that were not compressed by NNCF.
config = Config.fromfile(args["config"])
should_run_alt_ssd_export = (hasattr(config.model, 'bbox_head') and
config.model.bbox_head.type == 'SSDHead'
and not config.get('nncf_config'))
if is_checkpoint_nncf and is_checkpoint_nncf(args['load_weights']):
# If the config does not contain NNCF part,
# but the checkpoint was trained with NNCF compression,
# the NNCF config will be read from checkpoint.
# Since alt_ssd_export is incompatible with NNCF compression,
# alt_ssd_export should not be run in this case.
should_run_alt_ssd_export = False
if should_run_alt_ssd_export:
run_through_shell(
f'python {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{os.path.join(args["save_model_to"], "alt_ssd_export")} '
f'openvino '
f'--input_format {args["openvino_input_format"]} '
f'--alt_ssd_export ')
def _export_to_openvino(self, args, tools_dir):
run_through_shell(f'python {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{args["save_model_to"]} '
f'openvino '
f'--input_format {args["openvino_input_format"]}')
def _export_to_openvino(self, args, tools_dir):
update_config = self._get_update_config(args)
run_through_shell(f'python3 {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{args["save_model_to"]} '
f'{update_config} '
f'--opset={self.opset} '
f'openvino '
f'--input_format {args["openvino_input_format"]}')
# FIXME(ikrylov): remove alt_ssd_export block as soon as it becomes useless.
config = Config.fromfile(args["config"])
should_run_alt_ssd_export = (hasattr(config.model, 'bbox_head') and
config.model.bbox_head.type == 'SSDHead')
if should_run_alt_ssd_export:
run_through_shell(
f'python3 {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{os.path.join(args["save_model_to"], "alt_ssd_export")} '
f'{update_config} '
f'--opset={self.opset} '
f'openvino '
f'--input_format {args["openvino_input_format"]} '
f'--alt_ssd_export ')
def setUpClass(cls):
cls.compress_cmd_line_params = compress_cmd_line_params
cls.test_case_description = cls.generate_test_case_description(
template_update_dict,
compress_cmd_line_params,
compression_cfg_update_dict)
logging.info(f'Begin setting up class for {problem_name}/{model_name}, {cls.test_case_description}')
cls.templates_folder = os.environ['MODEL_TEMPLATES']
cls.src_template_folder = os.path.join(cls.templates_folder,domain_name, problem_name, model_name)
skip_non_instantiated_template_if_its_allowed(cls.src_template_folder, problem_name, model_name)
src_template_file = os.path.join(cls.src_template_folder, 'template.yaml')
download_snapshot_if_not_yet(src_template_file, cls.src_template_folder)
cls.template_folder = cls.generate_template_folder_name(cls.src_template_folder,
cls.test_case_description)
cls.copy_template_folder(cls.src_template_folder, cls.template_folder)
cls.template_file = os.path.join(cls.template_folder, 'template.yaml')
cls.apply_update_dict_params_to_template_file(cls.template_file,
template_update_dict,
compression_cfg_update_dict)
cls.ann_file = ann_file
cls.img_root = img_root
cls.dependencies = get_dependencies(cls.template_file)
download_snapshot_if_not_yet(cls.template_file, cls.template_folder)
run_through_shell(
f'cd {cls.template_folder};'
)
logging.info(f'End setting up class for {problem_name}/{model_name}, {cls.test_case_description}')
def main():
args = parse_args()
log_level = logging.DEBUG if args.verbose else logging.INFO
logging.basicConfig(level=log_level)
with open(args.template) as read_file:
content = yaml.load(read_file, yaml.SafeLoader)
os.makedirs(args.output, exist_ok=True)
run_through_shell(
f'cp -r {os.path.dirname(args.template)}/* --target-directory={args.output}',
verbose=args.verbose)
for dependency in content['dependencies']:
source = dependency['source']
destination = dependency['destination']
if destination != 'snapshot.pth':
rel_source = os.path.join(os.path.dirname(args.template), source)
cur_dst = os.path.join(args.output, destination)
os.makedirs(os.path.dirname(cur_dst), exist_ok=True)
run_through_shell(
f'cp -r --no-target-directory {rel_source} {cur_dst}',
check=True,
verbose=args.verbose)
if not args.do_not_load_snapshot:
download_snapshot_if_not_yet(args.template, args.output)
def test_evaluation_on_cpu(self):
self.skip_if_cpu_is_not_supported()
run_through_shell('export CUDA_VISIBLE_DEVICES=;'
f'cd {self.template_folder};'
f'python eval.py'
f' --test-ann-files {self.ann_file}'
f' --test-data-roots {self.img_root}'
f' --save-metrics-to metrics.yaml'
f' --load-weights snapshot.pth')
with open(os.path.join(self.template_folder,
"metrics.yaml")) as read_file:
content = yaml.load(read_file, yaml.SafeLoader)
est_accuracy = [
metrics['value'] for metrics in content['metrics']
if metrics['key'] == 'accuracy'
][0]
with open(
f'{os.path.dirname(__file__)}/../expected_outputs/{problem_name}/{model_name}.json'
) as read_file:
content = json.load(read_file)
ref_accuracy = content['accuracy']
self.assertLess(abs(ref_accuracy - 1e-2 * est_accuracy), 1e-6)
def test_nncf_compress_and_export(self):
skip_if_cuda_not_available()
log_file = self.do_compress()
latest_file = os.path.join(self.output_folder, 'latest.pth')
self.assertTrue(
os.path.isfile(latest_file),
f'Cannot find the latest.pth in path `{latest_file}`')
run_through_shell(f'cd {os.path.dirname(self.template_file)};'
f'python export.py'
f' --load-weights {latest_file}'
f' --save-model-to {self.output_folder}')
model_bin_paths = list(
glob.glob(os.path.join(self.output_folder, '*.bin')))
assert len(model_bin_paths) == 1, (
f'Wrong result of export.py: globbing "*.bin" in'
f' {self.output_folder} gives {model_bin_paths}')
model_bin_path = model_bin_paths[0]
metrics_path = self.do_eval(model_bin_path)
return log_file, metrics_path
def do_export(self, export_dir, on_gpu):
if not os.path.exists(export_dir):
initial_command = 'export CUDA_VISIBLE_DEVICES=;' if not on_gpu else ''
run_through_shell(f'{initial_command}'
f'cd {os.path.dirname(self.template_file)};'
f'python3 export.py'
f' --load-weights snapshot.pth'
f' --save-model-to {export_dir}')
def download_and_extract_coco_val2017(coco_dir):
val_dir = os.path.join(coco_dir, 'val2017')
zip_file = os.path.join(coco_dir, 'val2017.zip')
link = 'http://images.cocodataset.org/zips/val2017.zip'
if not os.path.exists(val_dir):
if not os.path.exists(zip_file):
run_through_shell(f'wget --no-verbose {link} -P {coco_dir}')
run_through_shell(f'unzip {zip_file} -d {coco_dir}')
def copy_template_folder(src_template_folder, template_folder):
logging.info(f'Copying {src_template_folder} to {template_folder}')
if os.path.isdir(template_folder):
logging.warning('')
logging.warning(f'ATTENTION: the folder that should be created for this test case exists!')
logging.warning(f' It may cause side effects between tests!')
logging.warning(f'The folder is `{template_folder}`.\n')
run_through_shell(f'cp -a --no-target-directory "{src_template_folder}" "{template_folder}"')
assert os.path.isdir(template_folder), f'Cannot create {template_folder}'
def do_export(self, classes, on_gpu):
initial_command = 'export CUDA_VISIBLE_DEVICES=;' if not on_gpu else ''
run_through_shell(
f'{initial_command}'
f'cd {os.path.dirname(self.template_file)};'
f'python3 export.py'
f' --load-weights {os.path.join(self.output_folder, "latest.pth")}'
f' --classes "{classes}"'
f' --save-model-to {self.output_folder}')
def _export_to_onnx(self, args, tools_dir):
update_config = self._get_update_config(args)
run_through_shell(f'python3 {os.path.join(tools_dir, "export.py")} '
f'{args["config"]} '
f'{args["load_weights"]} '
f'{args["save_model_to"]} '
f'{update_config} '
f'--opset={self.opset} '
f'onnx ')
def do_eval(self, file_to_eval):
metrics_path = os.path.join(self.output_folder, 'metrics.yaml')
run_through_shell(f'cd {self.template_folder};'
f'python eval.py'
f' --test-ann-files {self.ann_file}'
f' --test-data-roots {self.img_root}'
f' --save-metrics-to {metrics_path}'
f' --load-weights {file_to_eval}')
return metrics_path
def do_evaluation(self, export_dir):
metrics_path = os.path.join(export_dir, "metrics.yaml")
run_through_shell(
f'cd {os.path.dirname(self.template_file)};'
f'python3 eval.py'
f' --test-ann-files {self.ann_file}'
f' --test-data-roots {self.img_root}'
f' --load-weights {os.path.join(export_dir, "model.bin")}'
f' --save-metrics-to {metrics_path}')
self.assertTrue(os.path.exists(metrics_path))
def do_evaluation(self, on_gpu):
initial_command = 'export CUDA_VISIBLE_DEVICES=;' if not on_gpu else ''
metrics_path = os.path.join(self.output_folder, "metrics.yaml")
run_through_shell(f'{initial_command}'
f'cd {self.template_folder};'
f'python3 eval.py'
f' --test-ann-files {self.ann_file}'
f' --test-data-roots {self.img_root}'
f' --save-metrics-to {metrics_path}'
f' --load-weights snapshot.pth')
self.assertTrue(os.path.exists(metrics_path))
def do_export(self, export_dir, on_gpu):
if not os.path.exists(export_dir):
initial_command = 'export CUDA_VISIBLE_DEVICES=;' if not on_gpu else ''
run_through_shell(f'{initial_command}'
f'cd {os.path.dirname(self.template_file)};'
f'pip install -r requirements.txt;'
f'python3 export.py --openvino'
f' --load-weights snapshot.pth'
f' --save-model-to {export_dir}')
self.assertTrue(
len(list(pathlib.Path(export_dir).rglob('*.onnx'))) > 0,
'Export to onnx failed')
self.assertTrue(
len(list(pathlib.Path(export_dir).rglob('*.bin'))) > 0,
'Export to openvino failed')
def do_compress(self, main_weights_path, aux_weights_path):
log_file = os.path.join(self.output_folder,
f'log__{self.id()}.txt')
run_through_shell(
f'cd {self.template_folder};'
f'python3 compress.py'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {os.path.join(self.img_root, "train")}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {os.path.join(self.img_root, "val")}'
f' --save-checkpoints-to {self.output_folder}'
f' --gpu-num 1'
f' --load-weights {main_weights_path} --load-aux-weights {aux_weights_path}'
+ ' ' + self.compress_cmd_line_params + f' | tee {log_file}')
return log_file
def do_preliminary_finetuning(self, on_gpu):
logging.info(
f'Looking for best models in {self.preliminary_training_folder}'
)
best_models = self._find_best_models(
self.preliminary_training_folder)
if best_models:
logging.info(
f'Found already generated by preliminary finetuning the best models {best_models}'
)
return best_models
logging.info(
f'Begin making preliminary finetuning for the best models {best_models}'
)
self.total_epochs = 5
log_file = os.path.join(self.output_folder, 'test_finetuning.log')
initial_command = 'export CUDA_VISIBLE_DEVICES=;' if not on_gpu else ''
run_through_shell(
f'{initial_command}'
f'cd {self.template_folder};'
f'python3 train.py'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {os.path.join(self.img_root, "train")}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {os.path.join(self.img_root, "val")}'
f' --load-weights snapshot.pth'
f' --save-checkpoints-to {self.preliminary_training_folder}'
f' --gpu-num 1'
f' --batch-size {self.batch_size}'
f' --epochs {self.total_epochs}'
f' | tee {log_file}')
logging.info(
f'End making preliminary finetuning for the best models {best_models}'
)
self.assertTrue(
os.path.exists(
os.path.join(self.preliminary_training_folder,
'latest.pth')))
best_models = self._find_best_models(
self.preliminary_training_folder)
logging.info(
f'Found best models in {self.preliminary_training_folder}\n best_models={best_models}'
)
self.assertGreater(len(best_models), 0)
logging.info(f'Generated best_models = {best_models}')
return best_models
def do_compress(self):
log_file = os.path.join(self.output_folder,
f'log__{self.id()}.txt')
run_through_shell(f'cd {self.template_folder};'
f'python compress.py'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {self.img_root}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {self.img_root}'
f' --load-weights snapshot.pth'
f' --save-checkpoints-to {self.output_folder}'
f' --gpu-num 1'
f' --batch-size 1' + ' ' +
self.compress_cmd_line_params +
f' | tee {log_file}')
return log_file
def setUpClass(cls):
cls.templates_folder = os.environ['MODEL_TEMPLATES']
cls.template_folder = os.path.join(cls.templates_folder,
'action_recognition_2',
problem_name, model_name)
cls.template_file = os.path.join(cls.template_folder,
'template.yaml')
cls.ann_file = ann_file
cls.img_root = img_root
cls.dependencies = get_dependencies(cls.template_file)
download_snapshot_if_not_yet(cls.template_file,
cls.template_folder)
run_through_shell(f'cd {cls.template_folder};'
f'pip install -r requirements.txt;')
def setUpClass(cls):
cls.templates_folder = os.environ['MODEL_TEMPLATES']
cls.template_folder = os.path.join(cls.templates_folder, domain_name, problem_name, model_name)
skip_non_instantiated_template_if_its_allowed(cls.template_folder, problem_name, model_name)
cls.template_file = os.path.join(cls.template_folder, 'template.yaml')
cls.ann_file = ann_file
cls.img_root = img_root
cls.dependencies = get_dependencies(cls.template_file)
cls.total_epochs = 1
download_snapshot_if_not_yet(cls.template_file, cls.template_folder)
run_through_shell(
f'cd {cls.template_folder};'
f'pip install -r requirements.txt;'
)
def do_evaluation(self, export_dir):
metrics_path = os.path.join(export_dir, "metrics.yaml")
run_through_shell(
f'cd {os.path.dirname(self.template_file)};'
f'python3 eval.py'
f' --test-ann-files {ann_file}'
f' --test-data-roots {img_root}'
f' --load-weights {os.path.join(export_dir, "model.bin")}'
f' --save-metrics-to {metrics_path}'
)
with open(metrics_path) as read_file:
content = yaml.safe_load(read_file)
for metric_key in metric_keys:
value = [metrics['value'] for metrics in content['metrics'] if metrics['key'] == metric_key][0]
self.assertGreaterEqual(value, self.expected_outputs[metric_key] - self.test_export_thr)
def do_evaluation_of_exported_model(self, classes):
metrics_path = os.path.join(self.output_folder, "metrics_exported.yaml")
run_through_shell(
f'cd {os.path.dirname(self.template_file)};'
f'python3 eval.py'
f' --test-ann-files {self.ann_file}'
f' --test-data-roots {self.img_root}'
f' --load-weights {os.path.join(self.output_folder, "model.bin")}'
f' --classes "{classes}"'
f' --save-metrics-to {metrics_path}'
)
with open(metrics_path) as read_file:
content = yaml.safe_load(read_file)
for metric_key in self.metrics:
value = [metrics['value'] for metrics in content['metrics'] if metrics['key'] == metric_key][0]
self.assertGreaterEqual(value, 0.0)
def test_finetuning_on_gpu(self):
log_file = os.path.join(self.template_folder,
'test_finetuning.log')
run_through_shell(f'cd {self.template_folder};'
f'python train.py'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {self.img_root}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {self.img_root}'
f' --load-weights snapshot.pth'
f' --save-checkpoints-to {self.template_folder}'
f' --gpu-num 1'
f' --batch-size 2'
f' --epochs 6'
f' | tee {log_file}')
accuracy = collect_accuracy(log_file)
self.assertGreater(accuracy[-1], 0.0)
def do_finetuning(self, on_gpu):
log_file = os.path.join(self.output_folder, 'test_finetuning.log')
initial_command = 'export CUDA_VISIBLE_DEVICES=;' if not on_gpu else ''
run_through_shell(f'{initial_command}'
f'cd {self.template_folder};'
f'python train.py'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {self.img_root}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {self.img_root}'
f' --resume-from snapshot.pth'
f' --save-checkpoints-to {self.output_folder}'
f' --gpu-num 1'
f' --batch-size 1'
f' --epochs {self.total_epochs}'
f' | tee {log_file}')
self.assertTrue(
os.path.exists(os.path.join(self.output_folder, 'latest.pth')))
def do_evaluation(self, on_gpu):
initial_command = 'export CUDA_VISIBLE_DEVICES=;' if not on_gpu else ''
metrics_path = os.path.join(self.output_folder, "metrics.yaml")
run_through_shell(
f'{initial_command}'
f'cd {self.template_folder};'
f'python3 eval.py'
f' --test-ann-files {self.ann_file}'
f' --test-data-roots {self.img_root}'
f' --save-metrics-to {metrics_path}'
f' --load-weights snapshot.pth'
)
with open(metrics_path) as read_file:
content = yaml.safe_load(read_file)
for metric_key in metric_keys:
value = [metrics['value'] for metrics in content['metrics'] if metrics['key'] == metric_key][0]
self.assertLess(abs(self.expected_outputs[metric_key] - value), 1e-3)
def _get_complexity_and_size(self, cfg, config_path, work_dir, update_config):
image_shape = self._get_image_shape(cfg)
tools_dir = self._get_tools_dir()
res_complexity = os.path.join(work_dir, "complexity.json")
update_config = ' '.join([f'{k}={v}' for k, v in update_config.items()])
update_config = f' --update_config {update_config}' if update_config else ''
update_config = update_config.replace('"', '\\"')
run_through_shell(
f'python {tools_dir}/get_flops.py'
f' {config_path}'
f' --shape {image_shape}'
f' --out {res_complexity}'
f'{update_config}')
with open(res_complexity) as read_file:
content = json.load(read_file)
return content
def test_nncf_compress_and_export(self):
skip_if_cuda_not_available()
logging.info('Begin test_nncf_compress_and_export')
best_models = self.do_preliminary_finetuning(True)
self.assertEqual(len(best_models), 2)
self.assertIn('model_0', best_models[0])
self.assertIn('model_1', best_models[1])
log_file = self.do_compress(main_weights_path=best_models[0],
aux_weights_path=best_models[1])
logging.debug('Compression is finished')
latest_compressed_model = self._find_latest_model(
self.output_folder)
logging.debug(
f'Found latest compressed models: {latest_compressed_model}')
logging.info('Exporting the latest compressed model')
export_dir = self.output_folder
run_through_shell(f'cd {os.path.dirname(self.template_file)};'
f'python3 export.py --openvino'
f' --load-weights {latest_compressed_model}'
f' --save-model-to {export_dir}')
onnx_res_files = find_files_by_pattern(export_dir, '*.onnx')
xml_res_files = find_files_by_pattern(export_dir, '*.xml')
bin_res_files = find_files_by_pattern(export_dir, '*.bin')
self.assertTrue(len(onnx_res_files) == 1, 'Export to onnx failed')
self.assertTrue(
len(xml_res_files) == 1, 'Export to openvino failed')
self.assertTrue(
len(bin_res_files) == 1, 'Export to openvino failed')
xml_res_file = xml_res_files[0]
logging.debug(f'Before making evaluation of {xml_res_file}')
metrics_path = self.do_eval(xml_res_file)
logging.debug(f'After making evaluation of {xml_res_file}')
logging.debug(f' metrics are stored to the file {metrics_path}')
logging.info('End test_nncf_compress_and_export')
return log_file, metrics_path
