def create_image_classification_nncf_test_case( problem_name, model_name, ann_file, img_root, compression_cmd_line_parameters, template_update_dict=None, compression_cfg_update_dict=None, field_value_changes_in_nncf_config=None): """ Note the field_value_changes_in_nncf_config should be a dict value that should be applied to make_field_value_changes_in_struct_recursively """ # pylint: disable=too-many-arguments, too-many-statements if template_update_dict is None: template_update_dict = {} NNCFBaseTestCase = create_nncf_test_case( 'image_classification', problem_name, model_name, ann_file, img_root, compression_cmd_line_parameters, template_update_dict=template_update_dict, compression_cfg_update_dict=compression_cfg_update_dict) class ClassificationNNCFTestCase(NNCFBaseTestCase): @classmethod def setUpClass(cls): super(ClassificationNNCFTestCase, cls).setUpClass() if field_value_changes_in_nncf_config: cls._apply_field_value_changes_in_nncf_config_recursively( cls.template_file, field_value_changes_in_nncf_config) @staticmethod def _apply_field_value_changes_in_nncf_config_recursively( template_file, field_value_changes_in_nncf_config): import mmcv from ote.modules.compression import get_optimisation_config_from_template assert isinstance(field_value_changes_in_nncf_config, dict) template_data = mmcv.Config.fromfile(template_file) compression_cfg_rel_path = get_optimisation_config_from_template( template_data) compression_cfg_path = os.path.join(os.path.dirname(template_file), compression_cfg_rel_path) compression_cfg = mmcv.Config.fromfile(compression_cfg_path) compression_cfg = dict(compression_cfg) make_field_value_changes_in_struct_recursively( compression_cfg, field_value_changes_in_nncf_config) mmcv.dump(compression_cfg, compression_cfg_path, file_format='json') logging.debug( f'Made new compression config file:\n{pformat(mmcv.load(compression_cfg_path))}' ) def setUp(self): super().setUp() self.preliminary_training_folder = os.path.join( self.template_folder, 'preliminary_training') os.makedirs(self.preliminary_training_folder, exist_ok=True) self.batch_size = 2 @staticmethod def _find_best_models(folder_path): best_models = find_files_by_pattern(folder_path, '*best*.pth*.tar*') return best_models def _find_latest_model(self, folder_path): latest_models = find_files_by_pattern(folder_path, 'latest.*pth*') self.assertEqual(len(latest_models), 1) return latest_models[0] def do_preliminary_finetuning(self, on_gpu): logging.info( f'Looking for best models in {self.preliminary_training_folder}' ) best_models = self._find_best_models( self.preliminary_training_folder) if best_models: logging.info( f'Found already generated by preliminary finetuning the best models {best_models}' ) return best_models logging.info( f'Begin making preliminary finetuning for the best models {best_models}' ) self.total_epochs = 5 log_file = os.path.join(self.output_folder, 'test_finetuning.log') initial_command = 'export CUDA_VISIBLE_DEVICES=;' if not on_gpu else '' run_through_shell( f'{initial_command}' f'cd {self.template_folder};' f'python3 train.py' f' --train-ann-files {self.ann_file}' f' --train-data-roots {os.path.join(self.img_root, "train")}' f' --val-ann-files {self.ann_file}' f' --val-data-roots {os.path.join(self.img_root, "val")}' f' --load-weights snapshot.pth' f' --save-checkpoints-to {self.preliminary_training_folder}' f' --gpu-num 1' f' --batch-size {self.batch_size}' f' --epochs {self.total_epochs}' f' | tee {log_file}') logging.info( f'End making preliminary finetuning for the best models {best_models}' ) self.assertTrue( os.path.exists( os.path.join(self.preliminary_training_folder, 'latest.pth'))) best_models = self._find_best_models( self.preliminary_training_folder) logging.info( f'Found best models in {self.preliminary_training_folder}\n best_models={best_models}' ) self.assertGreater(len(best_models), 0) logging.info(f'Generated best_models = {best_models}') return best_models def do_compress(self, main_weights_path, aux_weights_path): log_file = os.path.join(self.output_folder, f'log__{self.id()}.txt') run_through_shell( f'cd {self.template_folder};' f'python3 compress.py' f' --train-ann-files {self.ann_file}' f' --train-data-roots {os.path.join(self.img_root, "train")}' f' --val-ann-files {self.ann_file}' f' --val-data-roots {os.path.join(self.img_root, "val")}' f' --save-checkpoints-to {self.output_folder}' f' --gpu-num 1' f' --load-weights {main_weights_path} --load-aux-weights {aux_weights_path}' + ' ' + self.compress_cmd_line_params + f' | tee {log_file}') return log_file def do_eval(self, file_to_eval): metrics_path = os.path.join(self.output_folder, 'metrics.yaml') run_through_shell( f'cd {self.template_folder};' f'python3 eval.py' f' --test-ann-files {self.ann_file}' f' --test-data-roots {os.path.join(self.img_root, "val")}' f' --save-metrics-to {metrics_path}' f' --load-weights {file_to_eval}') return metrics_path def test_nncf_compress_on_gpu(self): skip_if_cuda_not_available() logging.info('Begin test_nncf_compress_on_gpu') best_models = self.do_preliminary_finetuning(True) self.assertEqual(len(best_models), 2) self.assertIn('model_0', best_models[0]) self.assertIn('model_1', best_models[1]) log_file = self.do_compress(main_weights_path=best_models[0], aux_weights_path=best_models[1]) logging.debug('Compression is finished') best_compressed_models = self._find_best_models(self.output_folder) logging.debug( f'Found best compressed models: {best_compressed_models}') self.assertEqual(len(best_compressed_models), 2) self.assertIn('model_0', best_compressed_models[0]) self.assertIn('model_1', best_compressed_models[1]) logging.info('End test_nncf_compress_on_gpu') return log_file def _extract_last_rank1_from_log(self, log_path): re_rank1 = re.compile('^.*Rank-1 *: *') two_last_rank1 = extract_last_lines_by_pattern( log_path, re_rank1, 2) self.assertEqual(len(two_last_rank1), 2) main_rank1_line = two_last_rank1[0] last_rank1 = re.sub(re_rank1, '', main_rank1_line).strip() self.assertEqual(last_rank1[-1], '%', msg=f'Wrong Rank-1 line {main_rank1_line}') last_rank1 = last_rank1.strip('%') logging.debug(f'Found last rank1 in log: rank1 = "{last_rank1}" %') return float(last_rank1) / 100.0 def _extract_accuracy_from_metrics_file(self, metrics_path): with open(metrics_path) as f: metrics = yaml.safe_load(f) metrics = metrics['metrics'] accuracy_metric = [ m for m in metrics if isinstance(m, dict) and m.get('key') == 'accuracy' ] self.assertEqual(len(accuracy_metric), 1, msg=f'Wrong file {metrics_path}') accuracy_metric = accuracy_metric[0] accuracy = accuracy_metric['value'] if accuracy_metric.get('unit') == '%': accuracy = accuracy / 100.0 return accuracy def test_nncf_compress_and_eval_on_gpu(self): skip_if_cuda_not_available() logging.info('Begin test_nncf_compress_and_eval_on_gpu') best_models = self.do_preliminary_finetuning(True) self.assertEqual(len(best_models), 2) self.assertIn('model_0', best_models[0]) self.assertIn('model_1', best_models[1]) log_file = self.do_compress(main_weights_path=best_models[0], aux_weights_path=best_models[1]) logging.debug('Compression is finished') latest_compressed_model = self._find_latest_model( self.output_folder) logging.debug( f'Found latest compressed models: {latest_compressed_model}') last_training_rank1 = self._extract_last_rank1_from_log(log_file) logging.debug( f'Before making evaluation of {latest_compressed_model}') metrics_path = self.do_eval(latest_compressed_model) logging.debug( f'After making evaluation of {latest_compressed_model}') logging.debug(f'Metrics are stored in {metrics_path}') accuracy = self._extract_accuracy_from_metrics_file(metrics_path) self.assertAlmostEqual( last_training_rank1, accuracy, delta=0.001, msg=f'Difference between accuracy from log file {log_file} ' f'and the accuracy from evaluation metrics file {metrics_path}' ) logging.info('End test_nncf_compress_and_eval_on_gpu') return log_file, metrics_path def test_nncf_compress_and_export(self): skip_if_cuda_not_available() logging.info('Begin test_nncf_compress_and_export') best_models = self.do_preliminary_finetuning(True) self.assertEqual(len(best_models), 2) self.assertIn('model_0', best_models[0]) self.assertIn('model_1', best_models[1]) log_file = self.do_compress(main_weights_path=best_models[0], aux_weights_path=best_models[1]) logging.debug('Compression is finished') latest_compressed_model = self._find_latest_model( self.output_folder) logging.debug( f'Found latest compressed models: {latest_compressed_model}') logging.info('Exporting the latest compressed model') export_dir = self.output_folder run_through_shell(f'cd {os.path.dirname(self.template_file)};' f'python3 export.py --openvino' f' --load-weights {latest_compressed_model}' f' --save-model-to {export_dir}') onnx_res_files = find_files_by_pattern(export_dir, '*.onnx') xml_res_files = find_files_by_pattern(export_dir, '*.xml') bin_res_files = find_files_by_pattern(export_dir, '*.bin') self.assertTrue(len(onnx_res_files) == 1, 'Export to onnx failed') self.assertTrue( len(xml_res_files) == 1, 'Export to openvino failed') self.assertTrue( len(bin_res_files) == 1, 'Export to openvino failed') xml_res_file = xml_res_files[0] logging.debug(f'Before making evaluation of {xml_res_file}') metrics_path = self.do_eval(xml_res_file) logging.debug(f'After making evaluation of {xml_res_file}') logging.debug(f' metrics are stored to the file {metrics_path}') logging.info('End test_nncf_compress_and_export') return log_file, metrics_path return ClassificationNNCFTestCase
def create_object_detection_nncf_test_case(problem_name, model_name, ann_file, img_root, compression_cmd_line_parameters, template_update_dict=None, compression_cfg_update_dict=None, test_export_threshold=0.09): # pylint: disable=too-many-arguments if template_update_dict is None: template_update_dict = {} NNCFBaseTestCase = create_nncf_test_case( 'object_detection', problem_name, model_name, ann_file, img_root, compression_cmd_line_parameters, template_update_dict=template_update_dict, compression_cfg_update_dict=compression_cfg_update_dict) class NNCFObjectDetectionTestCase(NNCFBaseTestCase): def setUp(self): super().setUp() # Note that such big threshold is required, since # we have very small dataset for training and evaluation: # if network compression causes other detections # on 2-4 images, the accuracy drop will be significant. self.test_export_thr = test_export_threshold @staticmethod def _get_bbox_metric(metrics_path): with open(metrics_path) as read_file: content = yaml.safe_load(read_file) ap = [ metric['value'] for metric in content['metrics'] if metric['key'] == 'bbox' ][0] ap = ap / 100 return ap def test_nncf_compress_on_gpu(self): skip_if_cuda_not_available() log_file = super().test_nncf_compress_on_gpu() ap = collect_ap(log_file) self.assertGreater(ap[-1], 0) def test_nncf_compress_and_eval_on_gpu(self): skip_if_cuda_not_available() log_file, metrics_path = super( ).test_nncf_compress_and_eval_on_gpu() compress_ap = collect_ap(log_file) last_compress_ap = compress_ap[-1] logging.info(f'From training last_compress_ap={last_compress_ap}') ap = self._get_bbox_metric(metrics_path) logging.info(f'Evaluation result ap={ap}') self.assertLess(abs(last_compress_ap - ap), 1e-6) return log_file, metrics_path def test_nncf_compress_and_export(self): skip_if_cuda_not_available() log_file, metrics_path = super().test_nncf_compress_and_export() compress_ap = collect_ap(log_file) last_compress_ap = compress_ap[-1] logging.info(f'From training last_compress_ap={last_compress_ap}') ap = self._get_bbox_metric(metrics_path) logging.info(f'Evaluation after export result ap={ap}') self.assertGreater(ap, last_compress_ap - self.test_export_thr) return NNCFObjectDetectionTestCase