def test_onnx(self): self.exporter.export_to_onnx_model_if_not_yet(model=self.encoder_name, model_type='encoder') self.exporter.export_to_onnx_model_if_not_yet(model=self.decoder_name, model_type='decoder') evaluator = Evaluator(deepcopy(self.config), RunnerType.ONNX) metric_onnx = evaluator.validate() target_metric = evaluator.expected_outputs.get('target_metric') self.assertGreaterEqual(metric_onnx, target_metric)
def test_run_ir_model(self): if not self.config.get('export_ir'): return self.exporter.export_to_ir_model_if_not_yet(model=self.res_model_name, model_type=None) evaluator = Evaluator(deepcopy(self.config), RunnerType.OpenVINO) ir_metric = evaluator.validate() target_metric = evaluator.expected_outputs.get('target_metric') self.assertGreaterEqual(ir_metric, target_metric)
def setUpClass(cls): test_config = get_config(config_file, section='eval') cls.config = test_config cls.config.update({'expected_outputs': expected_outputs}) if not os.path.exists(cls.config.get("model_path")): download_checkpoint(cls.config.get("model_path"), cls.config.get("model_url")) cls.validator = Evaluator(config=cls.config)
def test_onnx(self): self.exporter.export_complete_model() evaluator = Evaluator(deepcopy(self.config), RunnerType.ONNX) metric_onnx = evaluator.validate() target_metric = evaluator.expected_outputs.get('target_metric') self.assertGreaterEqual(metric_onnx, target_metric)
import os from text_recognition.utils.get_config import get_config from text_recognition.utils.evaluator import Evaluator def parse_args(): args = argparse.ArgumentParser() args.add_argument('--config') return args.parse_args() if __name__ == '__main__': arguments = parse_args() test_config = get_config(arguments.config, section='eval') validator = Evaluator(test_config) if 'model_folder' in test_config.keys(): model_folder = test_config.get('model_folder') best_model, best_result = None, 0 for model in os.listdir(model_folder): validator.runner.reload_model(os.path.join(model_folder, model)) result = validator.validate() if result > best_result: best_result = result best_model = os.path.join(model_folder, model) print('model = {}'.format(best_model)) result = best_result else: result = validator.validate() print('Result metric is: {}'.format(result))