def test_fine_tuning(self): log_file = os.path.join(self.work_dir, 'test_fine_tuning.log') run_through_shell( f'../../external/mmdetection/tools/dist_train.sh {self.configuration_file} 1 2>&1 |' f' tee {log_file}') ap = collect_ap(log_file) self.assertEqual(len((ap)), 5)
def export_test(self, alt_ssd_export, thr): if alt_ssd_export: export_command_end = '--alt_ssd_export' export_dir = os.path.join(self.work_dir, "alt_ssd_export") log_file = os.path.join(export_dir, 'test_alt_ssd_export.log') else: export_dir = os.path.join(self.work_dir, "export") log_file = os.path.join(export_dir, 'test_export.log') export_command_end = '' os.system( f'/opt/intel/openvino/bin/setupvars.sh;' f'python ../../external/mmdetection/tools/export.py ' f'{self.configuration_file} ' f'{os.path.join(self.work_dir, self.snapshot_name)} ' f'{export_dir} ' f'openvino {export_command_end};' f'python ../../external/mmdetection/tools/test_exported.py ' f'{self.configuration_file} ' f'{os.path.join(export_dir, "config.xml")} ' f'--out res.pkl --eval bbox 2>&1 | tee {log_file}') ap = collect_ap(log_file) with open(f'tests/expected_outputs/{self.problem_name}/{self.model_name}.json') as read_file: content = json.load(read_file) self.assertGreater(ap[0], content['map'] - thr)
def test_fine_tuning(self): log_file = os.path.join(self.work_dir, 'test_fine_tuning.log') os.system( f'../../external/mmdetection/tools/dist_train.sh {self.configuration_file} 1 --validate 2>&1 |' f' tee {log_file}') ap = collect_ap(log_file) self.assertEqual(len((ap)), 5) self.assertLess(ap[0], ap[-1])
def test_nncf_compress_and_export(self): log_file = os.path.join(self.template_folder, f'log__{self.id()}.txt') checkpoints_dir = f'{self.template_folder}/output_{self.id()}' run_through_shell(f'cd {self.template_folder};' f'python compress.py' f' --train-ann-files {self.ann_file}' f' --train-data-roots {self.img_root}' f' --val-ann-files {self.ann_file}' f' --val-data-roots {self.img_root}' f' --load-weights snapshot.pth' f' --save-checkpoints-to {checkpoints_dir}' f' --gpu-num 1' f' --batch-size 1' f' | tee {log_file}') compress_ap = collect_ap(log_file) last_compress_ap = compress_ap[-1] latest_file = f'{checkpoints_dir}/latest.pth' self.assertTrue( os.path.isfile(latest_file), f'Cannot find the latest.pth in path `{latest_file}`') run_through_shell(f'cd {os.path.dirname(self.template_file)};' f'python export.py' f' --load-weights {latest_file}' f' --save-model-to {checkpoints_dir}') model_bin_paths = list( glob.glob(os.path.join(checkpoints_dir, '*.bin'))) assert len(model_bin_paths) == 1, ( f'Wrong result of export.py: globbing "*.bin" in' f' {checkpoints_dir} gives {model_bin_paths}') run_through_shell( f'cd {os.path.dirname(self.template_file)};' f'python eval.py' f' --test-ann-files {ann_file}' f' --test-data-roots {img_root}' f' --load-weights {model_bin_paths[0]}' f' --save-metrics-to {os.path.join(checkpoints_dir, "metrics.yaml")}' ) with open(os.path.join(checkpoints_dir, "metrics.yaml")) as read_file: content = yaml.safe_load(read_file) ap = [ metric for metric in content['metrics'] if metric['key'] == 'ap' ][0]['value'] ap = ap / 100 logging.info(f'From training last_compress_ap={last_compress_ap}') logging.info(f'From evaluation of OpenVINO(TM) model ap={ap}') self.assertGreater(ap, last_compress_ap - self.test_export_thr)
def test_quality_metrics(self): log_file = os.path.join(self.work_dir, 'test_quality_metrics.log') os.system(f'python ../../external/mmdetection/tools/test.py ' f'{self.configuration_file} ' f'{os.path.join(self.work_dir, self.snapshot_name)} ' f'--out res.pkl --eval bbox 2>&1 | tee {log_file}') ap = collect_ap(log_file) with open( f'tests/expected_outputs/{self.model_name}.json') as read_file: content = json.load(read_file) self.assertEqual(content['map'], ap[0])
def test_nncf_compress_on_gpu(self): log_file = os.path.join(self.template_folder, f'log__{self.id()}.txt') run_through_shell( f'cd {self.template_folder};' f'python compress.py' f' --train-ann-files {self.ann_file}' f' --train-data-roots {self.img_root}' f' --val-ann-files {self.ann_file}' f' --val-data-roots {self.img_root}' f' --load-weights snapshot.pth' f' --save-checkpoints-to {self.template_folder}/output_{self.id()}' f' --gpu-num 1' f' --batch-size 1' f' | tee {log_file}') ap = collect_ap(log_file) self.assertGreater(ap[-1], 0)
def test_finetuning_on_gpu(self): log_file = os.path.join(self.template_folder, 'test_finetuning.log') run_through_shell(f'cd {self.template_folder};' f'python train.py' f' --train-ann-files {self.ann_file}' f' --train-data-roots {self.img_root}' f' --val-ann-files {self.ann_file}' f' --val-data-roots {self.img_root}' f' --resume-from snapshot.pth' f' --save-checkpoints-to {self.template_folder}' f' --gpu-num 1' f' --batch-size 1' f' --epochs {self.total_epochs}' f' | tee {log_file}') ap = collect_ap(log_file) self.assertEqual(len((ap)), self.epochs_delta) self.assertGreater(ap[-1], 0)
def test_finetuning(self): log_file = os.path.join(self.work_dir, 'test_finetuning.log') run_through_shell( f'cd {os.path.dirname(self.template_file)};' f'python {self.dependencies["train"]}' f' --train-ann-files {self.ann_file}' f' --train-data-roots {self.img_root}' f' --val-ann-files {self.ann_file}' f' --val-data-roots {self.img_root}' f' --resume-from {os.path.join(self.work_dir, os.path.basename(self.dependencies["snapshot"]))}' f' --save-checkpoints-to {self.work_dir}' f' --gpu-num 1' f' --batch-size 1' f' --epochs {self.total_epochs}' f' | tee {log_file}') ap = collect_ap(log_file) self.assertEqual(len((ap)), self.epochs_delta) self.assertGreater(ap[-1], 0)
def test_nncf_compress_and_eval_on_gpu(self): log_file = os.path.join(self.template_folder, f'log__{self.id()}.txt') checkpoints_dir = f'{self.template_folder}/output_{self.id()}' run_through_shell(f'cd {self.template_folder};' f'python compress.py' f' --train-ann-files {self.ann_file}' f' --train-data-roots {self.img_root}' f' --val-ann-files {self.ann_file}' f' --val-data-roots {self.img_root}' f' --load-weights snapshot.pth' f' --save-checkpoints-to {checkpoints_dir}' f' --gpu-num 1' f' --batch-size 1' f' | tee {log_file}') compress_ap = collect_ap(log_file) last_compress_ap = compress_ap[-1] logging.info(f'From training last_compress_ap={last_compress_ap}') latest_file = f'{checkpoints_dir}/latest.pth' self.assertTrue( os.path.isfile(latest_file), f'Cannot find the latest.pth in path `{latest_file}`') metrics_path = f'{checkpoints_dir}/metrics.yaml' run_through_shell(f'cd {self.template_folder};' f'python eval.py' f' --test-ann-files {self.ann_file}' f' --test-data-roots {self.img_root}' f' --save-metrics-to {metrics_path}' f' --load-weights {latest_file}') with open(metrics_path) as read_file: content = yaml.safe_load(read_file) ap = [ metric['value'] for metric in content['metrics'] if metric['key'] == 'ap' ][0] ap = ap / 100 logging.info(f'Evaluation result ap={ap}') self.assertLess(abs(last_compress_ap - ap), 1e-6)
def test_nncf_finetune_and_compress_on_gpu(self): log_file = os.path.join(self.template_folder, f'log__{self.id()}.txt') total_epochs = get_epochs(self.template_file) total_epochs_with_finetuning = total_epochs + 2 run_through_shell( f'cd {self.template_folder};' f'python train.py' f' --train-ann-files {self.ann_file}' f' --train-data-roots {self.img_root}' f' --val-ann-files {self.ann_file}' f' --val-data-roots {self.img_root}' f' --resume-from snapshot.pth' f' --save-checkpoints-to {self.template_folder}/output_{self.id()}' f' --gpu-num 1' f' --batch-size 1' f' --epochs {total_epochs_with_finetuning}' f' | tee {log_file}') ap = collect_ap(log_file) self.assertGreater(ap[-1], 0)