def setup_method(self): self.launcher = MagicMock() infer_request = MagicMock() infer_request.wait = Mock(return_value=0) infer_request.outputs = Mock() self.launcher.infer_requests = [infer_request] data = MagicMock(data=MagicMock(), metadata=MagicMock(), identifier=0) self.preprocessor = Mock() self.preprocessor.process = Mock(return_value=data) self.postprocessor = Mock() self.adapter = MagicMock(return_value=[]) self.input_feeder = Mock() self.data_reader = Mock(return_value=data) self.data_reader.data_source = 'source' annotation_0 = MagicMock() annotation_0.identifier = 0 annotation_0.metadata = {'data_source': MagicMock()} annotation_1 = MagicMock() annotation_1.identifier = 1 annotation_1.metadata = {'data_source': MagicMock()} annotation_container_0 = MagicMock() annotation_container_0.values = MagicMock(return_value=[annotation_0]) annotation_container_1 = MagicMock() annotation_container_1.values = MagicMock(return_value=([annotation_1])) self.annotations = [[annotation_container_0], [annotation_container_1]] self.dataset = MagicMock() self.dataset.__iter__.return_value = self.annotations self.postprocessor.process_batch = Mock(side_effect=[ ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) ]) self.postprocessor.process_dataset = Mock(return_value=( ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) )) self.postprocessor.full_process = Mock(return_value=( ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) )) self.metric = Mock() self.metric.update_metrics_on_batch = Mock() self.evaluator = ModelEvaluator( self.launcher, self.input_feeder, self.adapter, self.data_reader, self.preprocessor, self.postprocessor, self.dataset, self.metric, True ) self.evaluator.store_predictions = Mock() self.evaluator.load = Mock(return_value=( ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) ))
def setup_method(self): self.launcher = MagicMock() self.launcher.get_async_requests = Mock(return_value=[]) data = MagicMock(data=MagicMock(), metadata=MagicMock(), identifier=0) self.preprocessor = Mock() self.preprocessor.process = Mock(return_value=data) self.postprocessor = Mock() self.adapter = MagicMock(return_value=[]) self.input_feeder = MagicMock() self.input_feeder.lstm_inputs = [] annotation_0 = MagicMock() annotation_0.identifier = 0 annotation_0.metadata = {'data_source': MagicMock()} annotation_1 = MagicMock() annotation_1.identifier = 1 annotation_1.metadata = {'data_source': MagicMock()} annotation_container_0 = MagicMock() annotation_container_0.values = MagicMock(return_value=[annotation_0]) annotation_container_1 = MagicMock() annotation_container_1.values = MagicMock(return_value=([annotation_1])) self.annotations = [[annotation_container_0], [annotation_container_1]] self.dataset = MagicMock() self.dataset.__iter__.return_value = [ (range(1), self.annotations[0], data, [0]), (range(1), self.annotations[1], data, [1])] self.dataset.multi_infer = False self.postprocessor.process_batch = Mock(side_effect=[ ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) ]) self.postprocessor.process_dataset = Mock(return_value=( ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) )) self.postprocessor.full_process = Mock(return_value=( ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) )) self.metric = Mock() self.metric.update_metrics_on_batch = Mock(return_value=[{}, {}]) self.evaluator = ModelEvaluator( self.launcher, self.input_feeder, self.adapter, self.preprocessor, self.postprocessor, self.dataset, self.metric, True, {} ) self.evaluator.store_predictions = Mock() self.evaluator.load = Mock(return_value=( ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) ))
def test_preprocessing_config_extra_parameter(self): preprocessing_config = [{ 'type': 'bgr_to_rgb' }, { 'type': 'resize', 'size': 224, 'something_extra': True }] dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'data_source': 'data', 'annotation': 'annotation', 'preprocessing': preprocessing_config } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers' assert config_errors[ 1].message == "datasets.0.preprocessing.1 specifies unknown options: ['something_extra']" assert config_errors[1].entry == preprocessing_config[1] assert config_errors[1].field_uri == 'datasets.0.preprocessing.1'
def test_preprocessing_config_unknown_parameter(self): preprocessing_config = [{ 'type': 'bgr_to_rgb' }, { 'type': 'not_resize', 'size': 224 }] dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'data_source': 'data', 'annotation': 'annotation', 'preprocessing': preprocessing_config } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers' assert config_errors[ 1].message == "preprocessor not_resize unregistered" assert config_errors[1].entry == preprocessing_config[1] assert config_errors[1].field_uri == 'datasets.0.preprocessing.1'
def test_empty_config(self): config_errors = ModelEvaluator.validate_config({'models': [{}]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'models.launchers' assert config_errors[1].message == 'datasets section is not provided' assert not config_errors[1].entry assert config_errors[1].field_uri == 'models.datasets'
def test_unregistered_launcher_config(self): launcher_config = {'framework': 'foo'} config_errors = ModelEvaluator.validate_config( {'models': [{ 'launchers': [launcher_config], 'datasets': [] }]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launcher foo is not unregistered' assert config_errors[0].entry == launcher_config assert config_errors[0].field_uri == 'models.launchers.0' assert config_errors[1].message == 'datasets section is not provided' assert not config_errors[1].entry assert config_errors[1].field_uri == 'models.datasets'
def test_valid_launcher_config(self): launcher_config = { 'model': 'foo', 'framework': 'dlsdk', 'device': 'cpu' } config_errors = ModelEvaluator.validate_config( {'models': [{ 'launchers': [launcher_config], 'datasets': [] }]}) assert len(config_errors) == 1 assert config_errors[0].message == 'datasets section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'models.datasets'
def test_str_data_reader(self): dataset_config = { 'name': 'dataset', 'annotation': 'annotation', 'metrics': [{ 'type': 'accuracy' }], 'reader': 'opencv_imread', 'data_source': 'data' } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 1 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers'
def test_dataset_config_without_metrics(self): dataset_config = { 'name': 'dataset', 'data_source': 'data', 'annotation': 'annotation' } config_errors = ModelEvaluator.validate_config( {'models': [{ 'datasets': [dataset_config] }]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'models.launchers' assert config_errors[1].message == 'Metrics are not provided' assert not config_errors[1].entry assert config_errors[1].field_uri == 'models.datasets.0'
def test_common_validation_scheme(self): validation_scheme = ModelEvaluator.validation_scheme() assert isinstance(validation_scheme, dict) assert len(validation_scheme) == 1 assert 'models' in validation_scheme assert len(validation_scheme['models']) == 1 assert contains_all(validation_scheme['models'][0], ['name', 'launchers', 'datasets']) assert isinstance(validation_scheme['models'][0]['name'], StringField) model_validation_scheme = validation_scheme['models'][0] assert model_validation_scheme[ 'launchers'].__name__ == Launcher.__name__ assert model_validation_scheme['datasets'].__name__ == Dataset.__name__ assert isinstance( model_validation_scheme['launchers'].validation_scheme(), list) assert isinstance( model_validation_scheme['datasets'].validation_scheme(), list)
def test_data_source_does_not_exists(self): dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'data_source': 'data_dir' } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 3 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers' assert config_errors[ -1].message == 'Invalid value "data_dir" for datasets.0.data_source: path does not exist' assert config_errors[-1].entry == 'data_dir' assert config_errors[-1].field_uri == 'datasets.0.data_source'
def test_annotation_conversion_without_converter(self): dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'data_source': 'data', 'annotation_conversion': {} } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers' assert config_errors[1].message == 'converter is not found' assert config_errors[1].entry == {} assert config_errors[1].field_uri == 'datasets.0.annotation_conversion'
def test_postprocessing_config(self): dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'data_source': 'data', 'annotation': 'annotation', 'postprocessing': [{ 'type': 'resize_prediction_boxes' }] } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 1 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers'
def test_annotation_is_not_provided(self): dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'data_source': 'data' } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers' assert config_errors[ 1].message == 'annotation_conversion or annotation field should be provided' assert config_errors[1].entry == dataset_config assert config_errors[1].field_uri == 'datasets.0'
def test_annotation_conversion_config(self): conversion_parameters = { 'converter': 'imagenet', 'annotation_file': 'file' } dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'data_source': 'data', 'annotation_conversion': conversion_parameters } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 1 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers'
def test_data_source_is_file(self): dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'annotation': 'annotation', 'data_source': 'data' } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers' assert config_errors[ 1].message == 'Invalid value "data" for datasets.0.data_source: path is not a directory' assert config_errors[1].entry == 'data' assert config_errors[1].field_uri == 'datasets.0.data_source'
def test_postprocessing_config_unknown_type(self): postprocessing_config = [{'type': 'unknown', 'size': 224}] dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'data_source': 'data', 'annotation': 'annotation', 'postprocessing': postprocessing_config } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers' assert config_errors[1].message == 'postprocessor unknown unregistered' assert config_errors[1].entry == postprocessing_config[0] assert config_errors[1].field_uri == 'datasets.0.postprocessing.0'
def test_input_without_name(self): launcher_config = { 'model': 'foo', 'framework': 'dlsdk', 'device': 'cpu', 'inputs': [{ "type": 'INPUT' }] } config_errors = ModelEvaluator.validate_config( {'models': [{ 'launchers': [launcher_config], 'datasets': [] }]}) assert len(config_errors) == 2 assert config_errors[0].message.endswith('input name is not provided') assert config_errors[0].field_uri == 'models.launchers.0.inputs.0' assert config_errors[1].message == 'datasets section is not provided' assert not config_errors[1].entry assert config_errors[1].field_uri == 'models.datasets'
def test_annotation_conversion_missed_parameter(self): conversion_parameters = {'converter': 'imagenet'} dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'data_source': 'data', 'annotation_conversion': conversion_parameters } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers' assert config_errors[ 1].message == 'Invalid config for datasets.0.annotation_conversion: missing required fields: annotation_file' assert config_errors[1].entry == conversion_parameters assert config_errors[1].field_uri == 'datasets.0.annotation_conversion'
def test_data_reader_without_data_source(self): dataset_config = { 'name': 'dataset', 'annotation': 'annotation', 'metrics': [{ 'type': 'accuracy' }] } config_errors = ModelEvaluator.validate_config( {'models': [{ 'datasets': [dataset_config] }]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'models.launchers' assert config_errors[ 1].message == 'Invalid value "None" for models.datasets.0.data_source: models.datasets.0.data_source is not allowed to be None' assert not config_errors[1].entry assert config_errors[1].field_uri == 'models.datasets.0.data_source'
def test_unregistered_data_reader(self): dataset_config = { 'name': 'dataset', 'annotation': 'annotation', 'metrics': [{ 'type': 'accuracy' }], 'reader': 'unknown' } config_errors = ModelEvaluator.validate_config( {'models': [{ 'datasets': [dataset_config] }]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'models.launchers' assert config_errors[1].message.startswith( 'Invalid value "unknown" for models.datasets.0.reader') assert config_errors[-1].entry == 'unknown' assert config_errors[-1].field_uri == 'models.datasets.0.reader'
def test_unregistered_adapter_config(self): launcher_config = { 'model': 'foo', 'framework': 'dlsdk', 'device': 'cpu', 'adapter': 'not_classification' } config_errors = ModelEvaluator.validate_config( {'models': [{ 'launchers': [launcher_config], 'datasets': [] }]}) assert len(config_errors) == 2 assert config_errors[0].message.startswith( 'Invalid value "not_classification"') assert config_errors[0].entry == 'not_classification' assert config_errors[0].field_uri.startswith( 'models.launchers.0') and config_errors[0].field_uri.endswith( 'adapter') assert config_errors[1].message == 'datasets section is not provided' assert not config_errors[1].entry assert config_errors[1].field_uri == 'models.datasets'
def test_annotation_conversion_extra_parameter(self): conversion_parameters = { 'converter': 'imagenet', 'annotation_file': 'file', 'something_extra': 'extra' } dataset_config = { 'name': 'dataset', 'metrics': [{ 'type': 'accuracy' }], 'data_source': 'data', 'annotation_conversion': conversion_parameters } config_errors = ModelEvaluator.validate_config( {'datasets': [dataset_config]}) assert len(config_errors) == 2 assert config_errors[0].message == 'launchers section is not provided' assert not config_errors[0].entry assert config_errors[0].field_uri == 'launchers' assert config_errors[ 1].message == "datasets.0.annotation_conversion specifies unknown options: ['something_extra']" assert config_errors[1].entry == conversion_parameters assert config_errors[1].field_uri == 'datasets.0.annotation_conversion'
class TestModelEvaluator: def setup_method(self): self.launcher = Mock() self.launcher.predict.return_value = [] data = MagicMock(data=MagicMock(), metadata=MagicMock(), identifier=0) self.preprocessor = Mock() self.preprocessor.process = Mock(return_value=data) self.postprocessor = Mock() self.adapter = MagicMock(return_value=[]) self.input_feeder = Mock() self.data_reader = Mock(return_value=data) self.data_reader.data_source = 'source' annotation_0 = MagicMock() annotation_0.identifier = 0 annotation_0.metadata = {'data_source': MagicMock()} annotation_1 = MagicMock() annotation_1.identifier = 1 annotation_1.metadata = {'data_source': MagicMock()} annotation_container_0 = MagicMock() annotation_container_0.values = MagicMock(return_value=[annotation_0]) annotation_container_1 = MagicMock() annotation_container_1.values = MagicMock(return_value=([annotation_1])) self.annotations = [[annotation_container_0], [annotation_container_1]] self.dataset = MagicMock() self.dataset.__iter__.return_value = [(range(1), self.annotations[0]), (range(1), self.annotations[1])] self.postprocessor.process_batch = Mock(side_effect=[ ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) ]) self.postprocessor.process_dataset = Mock(return_value=( ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) )) self.postprocessor.full_process = Mock(return_value=( ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) )) self.metric = Mock() self.metric.update_metrics_on_batch = Mock() self.evaluator = ModelEvaluator( self.launcher, self.input_feeder, self.adapter, self.data_reader, self.preprocessor, self.postprocessor, self.dataset, self.metric, False ) self.evaluator.store_predictions = Mock() self.evaluator.load = Mock(return_value=( ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]) )) def test_process_dataset_without_storing_predictions_and_dataset_processors(self): self.postprocessor.has_dataset_processors = False self.evaluator.process_dataset(None, None) assert not self.evaluator.store_predictions.called assert not self.evaluator.load.called assert self.launcher.predict.called assert self.postprocessor.process_batch.called assert self.metric.update_metrics_on_batch.call_count == len(self.annotations) assert not self.postprocessor.process_dataset.called assert not self.postprocessor.full_process.called def test_process_dataset_with_storing_predictions_and_without_dataset_processors(self): self.postprocessor.has_dataset_processors = False self.evaluator.process_dataset('path', None) assert self.evaluator.store_predictions.called assert not self.evaluator.load.called assert self.launcher.predict.called assert self.postprocessor.process_batch.called assert self.metric.update_metrics_on_batch.call_count == len(self.annotations) assert not self.postprocessor.process_dataset.called assert not self.postprocessor.full_process.called def test_process_dataset_with_loading_predictions_and_without_dataset_processors(self, mocker): mocker.patch('accuracy_checker.evaluators.model_evaluator.get_path') self.postprocessor.has_dataset_processors = False self.evaluator.process_dataset('path', None) assert self.evaluator.load.called assert not self.launcher.predict.called assert not self.postprocessor.process_batch.called assert self.metric.update_metrics_on_batch.call_count == 1 assert not self.postprocessor.process_dataset.called assert self.postprocessor.full_process.called def test_process_dataset_with_loading_predictions_and_with_dataset_processors(self, mocker): mocker.patch('accuracy_checker.evaluators.model_evaluator.get_path') self.postprocessor.has_dataset_processors = True self.evaluator.process_dataset('path', None) assert not self.evaluator.store_predictions.called assert self.evaluator.load.called assert not self.launcher.predict.called assert not self.postprocessor.process_batch.called assert self.metric.update_metrics_on_batch.call_count == 1 assert not self.postprocessor.process_dataset.called assert self.postprocessor.full_process.called
class TestModelEvaluatorAsync: def setup_method(self): self.launcher = MagicMock() self.launcher.get_async_requests = Mock(return_value=[]) data = MagicMock(data=MagicMock(), metadata=MagicMock(), identifier=0) self.preprocessor = Mock() self.preprocessor.process = Mock(return_value=data) self.postprocessor = Mock() self.adapter = MagicMock(return_value=[]) self.input_feeder = MagicMock() self.input_feeder.lstm_inputs = [] annotation_0 = MagicMock() annotation_0.identifier = 0 annotation_0.metadata = {'data_source': MagicMock()} annotation_1 = MagicMock() annotation_1.identifier = 1 annotation_1.metadata = {'data_source': MagicMock()} annotation_container_0 = MagicMock() annotation_container_0.values = MagicMock(return_value=[annotation_0]) annotation_container_1 = MagicMock() annotation_container_1.values = MagicMock( return_value=([annotation_1])) self.annotations = [[annotation_container_0], [annotation_container_1]] self.dataset = MagicMock() self.dataset.__iter__.return_value = [ (range(1), self.annotations[0], data, [0]), (range(1), self.annotations[1], data, [1]) ] self.dataset.multi_infer = False self.postprocessor.process_batch = Mock( side_effect=[([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1])]) self.postprocessor.process_dataset = Mock( return_value=(([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]))) self.postprocessor.full_process = Mock( return_value=(([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]))) self.metric = Mock() self.metric.update_metrics_on_batch = Mock(return_value=[{}, {}]) self.evaluator = ModelEvaluator(self.launcher, self.input_feeder, self.adapter, self.preprocessor, self.postprocessor, self.dataset, self.metric, True) self.evaluator.store_predictions = Mock() self.evaluator.load = Mock(return_value=(([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1]))) def test_process_dataset_without_storing_predictions_and_dataset_processors( self): self.postprocessor.has_dataset_processors = False self.launcher.allow_reshape_input = False self.preprocessor.has_multi_infer_transformations = False self.evaluator.process_dataset(None, None) assert not self.evaluator.store_predictions.called assert not self.evaluator.load.called assert not self.launcher.predict.called assert self.launcher.get_async_requests.called def test_process_dataset_with_storing_predictions_and_without_dataset_processors( self): self.postprocessor.has_dataset_processors = False self.launcher.allow_reshape_input = False self.preprocessor.has_multi_infer_transformations = False self.dataset.multi_infer = False self.evaluator.process_dataset('path', None) assert not self.evaluator.load.called assert not self.launcher.predict.called assert self.launcher.get_async_requests.called def test_process_dataset_with_loading_predictions_and_without_dataset_processors( self, mocker): mocker.patch('accuracy_checker.evaluators.model_evaluator.get_path') self.postprocessor.has_dataset_processors = False self.evaluator.process_dataset('path', None) assert self.evaluator.load.called assert not self.launcher.predict.called assert not self.launcher.predict_async.called assert not self.postprocessor.process_batch.called assert self.metric.update_metrics_on_batch.call_count == 1 assert not self.postprocessor.process_dataset.called assert self.postprocessor.full_process.called def test_switch_to_sync_predict_if_need_reshaping(self): self.postprocessor.has_dataset_processors = False self.launcher.allow_reshape_input = True self.preprocessor.has_multi_infer_transformations = False self.evaluator.process_dataset(None, None) assert not self.evaluator.store_predictions.called assert not self.evaluator.load.called assert self.launcher.predict.called assert not self.launcher.predict_async.called assert self.metric.update_metrics_on_batch.call_count == len( self.annotations) def test_switch_to_sync_predict_if_need_multi_infer_after_preprocessing( self): self.postprocessor.has_dataset_processors = False self.launcher.allow_reshape_input = False self.preprocessor.has_multi_infer_transformations = True self.evaluator.process_dataset(None, None) assert not self.evaluator.store_predictions.called assert not self.evaluator.load.called assert self.launcher.predict.called assert not self.launcher.predict_async.called assert self.metric.update_metrics_on_batch.call_count == len( self.annotations) def test_switch_to_sync_predict_if_need_multi_infer(self): self.postprocessor.has_dataset_processors = False self.launcher.allow_reshape_input = False self.preprocessor.has_multi_infer_transformations = False self.dataset.multi_infer = True self.evaluator.process_dataset(None, None) assert not self.evaluator.store_predictions.called assert not self.evaluator.load.called assert self.launcher.predict.called assert not self.launcher.predict_async.called assert self.metric.update_metrics_on_batch.call_count == len( self.annotations)