def test_fine_tuning(self):
log_file = os.path.join(self.work_dir, 'test_fine_tuning.log')
run_through_shell(
f'../../external/mmdetection/tools/dist_train.sh {self.configuration_file} 1 2>&1 |'
f' tee {log_file}')
ap = collect_ap(log_file)
self.assertEqual(len((ap)), 5)
def export_test(self, alt_ssd_export, thr):
if alt_ssd_export:
export_command_end = '--alt_ssd_export'
export_dir = os.path.join(self.work_dir, "alt_ssd_export")
log_file = os.path.join(export_dir, 'test_alt_ssd_export.log')
else:
export_dir = os.path.join(self.work_dir, "export")
log_file = os.path.join(export_dir, 'test_export.log')
export_command_end = ''
os.system(
f'/opt/intel/openvino/bin/setupvars.sh;'
f'python ../../external/mmdetection/tools/export.py '
f'{self.configuration_file} '
f'{os.path.join(self.work_dir, self.snapshot_name)} '
f'{export_dir} '
f'openvino {export_command_end};'
f'python ../../external/mmdetection/tools/test_exported.py '
f'{self.configuration_file} '
f'{os.path.join(export_dir, "config.xml")} '
f'--out res.pkl --eval bbox 2>&1 | tee {log_file}')
ap = collect_ap(log_file)
with open(f'tests/expected_outputs/{self.problem_name}/{self.model_name}.json') as read_file:
content = json.load(read_file)
self.assertGreater(ap[0], content['map'] - thr)
def test_fine_tuning(self):
log_file = os.path.join(self.work_dir, 'test_fine_tuning.log')
os.system(
f'../../external/mmdetection/tools/dist_train.sh {self.configuration_file} 1 --validate 2>&1 |'
f' tee {log_file}')
ap = collect_ap(log_file)
self.assertEqual(len((ap)), 5)
self.assertLess(ap[0], ap[-1])
def test_nncf_compress_and_export(self):
log_file = os.path.join(self.template_folder,
f'log__{self.id()}.txt')
checkpoints_dir = f'{self.template_folder}/output_{self.id()}'
run_through_shell(f'cd {self.template_folder};'
f'python compress.py'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {self.img_root}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {self.img_root}'
f' --load-weights snapshot.pth'
f' --save-checkpoints-to {checkpoints_dir}'
f' --gpu-num 1'
f' --batch-size 1'
f' | tee {log_file}')
compress_ap = collect_ap(log_file)
last_compress_ap = compress_ap[-1]
latest_file = f'{checkpoints_dir}/latest.pth'
self.assertTrue(
os.path.isfile(latest_file),
f'Cannot find the latest.pth in path `{latest_file}`')
run_through_shell(f'cd {os.path.dirname(self.template_file)};'
f'python export.py'
f' --load-weights {latest_file}'
f' --save-model-to {checkpoints_dir}')
model_bin_paths = list(
glob.glob(os.path.join(checkpoints_dir, '*.bin')))
assert len(model_bin_paths) == 1, (
f'Wrong result of export.py: globbing "*.bin" in'
f' {checkpoints_dir} gives {model_bin_paths}')
run_through_shell(
f'cd {os.path.dirname(self.template_file)};'
f'python eval.py'
f' --test-ann-files {ann_file}'
f' --test-data-roots {img_root}'
f' --load-weights {model_bin_paths[0]}'
f' --save-metrics-to {os.path.join(checkpoints_dir, "metrics.yaml")}'
)
with open(os.path.join(checkpoints_dir,
"metrics.yaml")) as read_file:
content = yaml.safe_load(read_file)
ap = [
metric for metric in content['metrics']
if metric['key'] == 'ap'
][0]['value']
ap = ap / 100
logging.info(f'From training last_compress_ap={last_compress_ap}')
logging.info(f'From evaluation of OpenVINO(TM) model ap={ap}')
self.assertGreater(ap, last_compress_ap - self.test_export_thr)
def test_quality_metrics(self):
log_file = os.path.join(self.work_dir, 'test_quality_metrics.log')
os.system(f'python ../../external/mmdetection/tools/test.py '
f'{self.configuration_file} '
f'{os.path.join(self.work_dir, self.snapshot_name)} '
f'--out res.pkl --eval bbox 2>&1 | tee {log_file}')
ap = collect_ap(log_file)
with open(
f'tests/expected_outputs/{self.model_name}.json') as read_file:
content = json.load(read_file)
self.assertEqual(content['map'], ap[0])
def test_nncf_compress_on_gpu(self):
log_file = os.path.join(self.template_folder,
f'log__{self.id()}.txt')
run_through_shell(
f'cd {self.template_folder};'
f'python compress.py'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {self.img_root}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {self.img_root}'
f' --load-weights snapshot.pth'
f' --save-checkpoints-to {self.template_folder}/output_{self.id()}'
f' --gpu-num 1'
f' --batch-size 1'
f' | tee {log_file}')
ap = collect_ap(log_file)
self.assertGreater(ap[-1], 0)
def test_finetuning_on_gpu(self):
log_file = os.path.join(self.template_folder,
'test_finetuning.log')
run_through_shell(f'cd {self.template_folder};'
f'python train.py'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {self.img_root}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {self.img_root}'
f' --resume-from snapshot.pth'
f' --save-checkpoints-to {self.template_folder}'
f' --gpu-num 1'
f' --batch-size 1'
f' --epochs {self.total_epochs}'
f' | tee {log_file}')
ap = collect_ap(log_file)
self.assertEqual(len((ap)), self.epochs_delta)
self.assertGreater(ap[-1], 0)
def test_finetuning(self):
log_file = os.path.join(self.work_dir, 'test_finetuning.log')
run_through_shell(
f'cd {os.path.dirname(self.template_file)};'
f'python {self.dependencies["train"]}'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {self.img_root}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {self.img_root}'
f' --resume-from {os.path.join(self.work_dir, os.path.basename(self.dependencies["snapshot"]))}'
f' --save-checkpoints-to {self.work_dir}'
f' --gpu-num 1'
f' --batch-size 1'
f' --epochs {self.total_epochs}'
f' | tee {log_file}')
ap = collect_ap(log_file)
self.assertEqual(len((ap)), self.epochs_delta)
self.assertGreater(ap[-1], 0)
def test_nncf_compress_and_eval_on_gpu(self):
log_file = os.path.join(self.template_folder,
f'log__{self.id()}.txt')
checkpoints_dir = f'{self.template_folder}/output_{self.id()}'
run_through_shell(f'cd {self.template_folder};'
f'python compress.py'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {self.img_root}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {self.img_root}'
f' --load-weights snapshot.pth'
f' --save-checkpoints-to {checkpoints_dir}'
f' --gpu-num 1'
f' --batch-size 1'
f' | tee {log_file}')
compress_ap = collect_ap(log_file)
last_compress_ap = compress_ap[-1]
logging.info(f'From training last_compress_ap={last_compress_ap}')
latest_file = f'{checkpoints_dir}/latest.pth'
self.assertTrue(
os.path.isfile(latest_file),
f'Cannot find the latest.pth in path `{latest_file}`')
metrics_path = f'{checkpoints_dir}/metrics.yaml'
run_through_shell(f'cd {self.template_folder};'
f'python eval.py'
f' --test-ann-files {self.ann_file}'
f' --test-data-roots {self.img_root}'
f' --save-metrics-to {metrics_path}'
f' --load-weights {latest_file}')
with open(metrics_path) as read_file:
content = yaml.safe_load(read_file)
ap = [
metric['value'] for metric in content['metrics']
if metric['key'] == 'ap'
][0]
ap = ap / 100
logging.info(f'Evaluation result ap={ap}')
self.assertLess(abs(last_compress_ap - ap), 1e-6)
def test_nncf_finetune_and_compress_on_gpu(self):
log_file = os.path.join(self.template_folder,
f'log__{self.id()}.txt')
total_epochs = get_epochs(self.template_file)
total_epochs_with_finetuning = total_epochs + 2
run_through_shell(
f'cd {self.template_folder};'
f'python train.py'
f' --train-ann-files {self.ann_file}'
f' --train-data-roots {self.img_root}'
f' --val-ann-files {self.ann_file}'
f' --val-data-roots {self.img_root}'
f' --resume-from snapshot.pth'
f' --save-checkpoints-to {self.template_folder}/output_{self.id()}'
f' --gpu-num 1'
f' --batch-size 1'
f' --epochs {total_epochs_with_finetuning}'
f' | tee {log_file}')
ap = collect_ap(log_file)
self.assertGreater(ap[-1], 0)
