def load_image_meta(annotation, image_root): print_info("Loading image level metadata") for ann in tqdm(annotation): path = os.path.join(image_root, ann.identifier) image = cv2.imread(path) if image is None: raise FileNotFoundError('Image file not found: {}'.format(path)) Dataset.set_image_metadata(ann, image) return annotation
def test_annotation_conversion_converter_without_required_options_raise_config_error( self): addition_options = {'annotation_conversion': {'converter': 'wider'}} config = copy_dataset_config(self.dataset_config) config.update(addition_options) with pytest.raises(ConfigError): Dataset(config, MockPreprocessor())
def from_configs(cls, config): dataset_config = config['datasets'][0] dataset = Dataset(dataset_config) data_reader_config = dataset_config.get('reader', 'opencv_imread') data_source = dataset_config['data_source'] if isinstance(data_reader_config, str): data_reader_type = data_reader_config data_reader_config = None elif isinstance(data_reader_config, dict): data_reader_type = data_reader_config['type'] else: raise ConfigError('reader should be dict or string') if data_reader_type in REQUIRES_ANNOTATIONS: data_source = dataset.annotation data_reader = BaseReader.provide(data_reader_type, data_source, data_reader_config) models_info = config['network_info'] launcher_config = config['launchers'][0] launcher = create_launcher(launcher_config, delayed_model_loading=True) preprocessors_config = dataset_config.get('preprocessing', []) stages = build_stages(models_info, preprocessors_config, launcher) metrics_executor = MetricsExecutor(dataset_config['metrics'], dataset) postprocessing = PostprocessingExecutor( dataset_config['postprocessing']) return cls(dataset, data_reader, stages, postprocessing, metrics_executor)
def test_setting_custom_dataset_with_missed_data_source_raises_config_error_exception(self, mocker): local_dataset = { 'name': 'custom', 'annotation': 'custom', 'metrics': [{'type': 'map'}] } with pytest.raises(ConfigError): Dataset(local_dataset, MockPreprocessor())
def test_annotation_conversion_closer_to_zero_subset_ratio(self, mocker): addition_options = { 'annotation_conversion': {'converter': 'wider', 'annotation_file': Path('file')}, 'subsample_size': '0.001%' } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation(['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=ConverterReturn(converted_annotation, None, None) ) subset_maker_mock = mocker.patch( 'accuracy_checker.dataset.make_subset' ) Dataset.load_annotation(config) subset_maker_mock.assert_called_once_with(converted_annotation, 1, 666, True)
def test_successful_annotation_conversion(self, mocker): addition_options = {'annotation_conversion': {'converter': 'wider', 'annotation_file': Path('file')}} config = copy_dataset_config(self.dataset_config) config.update(addition_options) annotation_converter_mock = mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=ConverterReturn(make_representation("0 0 0 5 5", True), None, None) ) Dataset(config) annotation_converter_mock.assert_called_once_with()
def test_create_data_provider_with_subset_file(self, mocker): mocker.patch('pathlib.Path.exists', return_value=True) mocker.patch('accuracy_checker.dataset.read_yaml', return_value=['1']) addition_options = { 'annotation_conversion': {'converter': 'wider', 'annotation_file': Path('file')}, "subset_file": "subset.yml" } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation(['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=ConverterReturn(converted_annotation, None, None) ) dataset = Dataset(config) assert len(dataset.data_provider) == 1 assert dataset.identifiers() == ['1'] assert dataset.data_provider.full_size == 2
def test_annotation_conversion_raise_config_error_on_extra_args(self): addition_options = { 'annotation_conversion': { 'converter': 'wider', 'annotation_file': 'file', 'something_extra': 'extra' } } config = copy_dataset_config(self.dataset_config) config.update(addition_options) with pytest.raises(ConfigError): Dataset(config, MockPreprocessor())
def test_missed_name_raises_config_error_exception(self): local_dataset = { 'annotation': 'custom', 'data_source': 'custom', 'metrics': [{ 'type': 'fppi', 'mr_rates': [0.0, 0.1] }], } with pytest.raises(ConfigError): Dataset(local_dataset, MockPreprocessor())
def test_ignore_subset_parameters_if_file_provided(self, mocker): mocker.patch('pathlib.Path.exists', return_value=True) mocker.patch('accuracy_checker.utils.read_yaml', return_value=['1']) subset_maker_mock = mocker.patch( 'accuracy_checker.dataset.make_subset' ) addition_options = { 'annotation_conversion': {'converter': 'wider', 'annotation_file': Path('file')}, 'subsample_size': 1, 'shuffle': False, "subset_file": "subset.yml" } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation(['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=ConverterReturn(converted_annotation, None, None) ) Dataset.load_annotation(config) assert not subset_maker_mock.called
def test_annotation_conversion_subset_size(self, mocker): addition_options = { 'annotation_conversion': {'converter': 'wider', 'annotation_file': Path('file')}, 'subsample_size': 1 } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation(['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=ConverterReturn(converted_annotation, None, None) ) dataset = Dataset(config) assert dataset.data_provider.annotation_provider._data_buffer == {converted_annotation[1].identifier: converted_annotation[1]}
def test_annotation_conversion_invalid_subset_size_raise_config_error(self, mocker): addition_options = { 'annotation_conversion': {'converter': 'wider', 'annotation_file': Path('file')}, 'subsample_size': 'aaa' } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation(['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=ConverterReturn(converted_annotation, None, None) ) with pytest.raises(ConfigError): Dataset(config)
def test_dataset_validation_scheme(self): dataset_validation_scheme = Dataset.validation_scheme() assert isinstance(dataset_validation_scheme, list) assert len(dataset_validation_scheme) == 1 dataset_params = [ key for key in Dataset.parameters() if not key.startswith('_') ] assert isinstance(dataset_validation_scheme[0], dict) assert contains_all(dataset_validation_scheme[0], dataset_params) assert len(dataset_validation_scheme[0]) == len(dataset_params) assert isinstance(dataset_validation_scheme[0]['name'], StringField) assert isinstance(dataset_validation_scheme[0]['annotation'], PathField) assert isinstance(dataset_validation_scheme[0]['data_source'], PathField) assert isinstance(dataset_validation_scheme[0]['dataset_meta'], PathField) assert isinstance(dataset_validation_scheme[0]['subsample_size'], BaseField) assert isinstance(dataset_validation_scheme[0]['shuffle'], BoolField) assert isinstance(dataset_validation_scheme[0]['subsample_seed'], NumberField) assert isinstance(dataset_validation_scheme[0]['analyze_dataset'], BoolField) assert isinstance( dataset_validation_scheme[0]['segmentation_masks_source'], PathField) assert isinstance( dataset_validation_scheme[0]['additional_data_source'], PathField) assert isinstance(dataset_validation_scheme[0]['batch'], NumberField) assert dataset_validation_scheme[0]['reader'] == BaseReader assert dataset_validation_scheme[0]['preprocessing'] == Preprocessor assert dataset_validation_scheme[0]['postprocessing'] == Postprocessor assert dataset_validation_scheme[0]['metrics'] == Metric assert dataset_validation_scheme[0][ 'annotation_conversion'] == BaseFormatConverter
def test_annotation_conversion_not_convert_twice(self, mocker): addition_options = { 'annotation_conversion': {'converter': 'wider', 'annotation_file': Path('file')}, 'annotation': Path('custom') } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation('0 0 0 5 5', True) annotation_reader_mock = mocker.patch( 'accuracy_checker.dataset.read_annotation', return_value=converted_annotation ) Dataset(config) annotation_reader_mock.assert_called_once_with(Path('custom'))
def test_annotation_conversion_subset_with_disabled_shuffle(self, mocker): addition_options = { 'annotation_conversion': {'converter': 'wider', 'annotation_file': Path('file')}, 'subsample_size': 1, 'shuffle': False } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation(['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=ConverterReturn(converted_annotation, None, None) ) annotation, _ = Dataset.load_annotation(config) assert annotation == [converted_annotation[0]]
def test_annotation_conversion_subset_more_than_dataset_size(self, mocker): addition_options = { 'annotation_conversion': {'converter': 'wider', 'annotation_file': Path('file')}, 'subsample_size': 3, 'subsample_seed': 1 } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation(['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=ConverterReturn(converted_annotation, None, None) ) with pytest.warns(UserWarning): annotation, _ = Dataset.load_annotation(config) assert annotation == converted_annotation
def test_annotation_conversion_subset_size(self, mocker): addition_options = { 'annotation_conversion': { 'converter': 'wider', 'annotation_file': 'file' }, 'subsample_size': 1 } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation( ['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=(converted_annotation, None)) dataset = Dataset(config, MockPreprocessor()) assert dataset.annotation == [converted_annotation[1]]
def from_configs(cls, config): dataset_config = config['datasets'][0] dataset = Dataset(dataset_config) data_reader_config = dataset_config.get('reader', 'opencv_imread') data_source = dataset_config['data_source'] if isinstance(data_reader_config, str): reader = BaseReader.provide(data_reader_config, data_source) elif isinstance(data_reader_config, dict): reader = BaseReader.provide(data_reader_config['type'], data_source, data_reader_config) else: raise ConfigError('reader should be dict or string') preprocessing = PreprocessingExecutor(dataset_config.get('preprocessing', []), dataset.name) metrics_executor = MetricsExecutor(dataset_config['metrics'], dataset) launcher = create_launcher(config['launchers'][0], delayed_model_loading=True) model = SequentialModel( config.get('network_info', {}), launcher, config.get('_models', []), config.get('_model_is_blob') ) return cls(dataset, reader, preprocessing, metrics_executor, launcher, model)
def test_annotation_conversion_subset_with_seed(self, mocker): addition_options = { 'annotation_conversion': { 'converter': 'wider', 'annotation_file': Path('file') }, 'subsample_size': 1, 'subsample_seed': 1 } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation( ['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=converted_annotation) dataset = Dataset(config) annotation = dataset.annotation assert annotation == [converted_annotation[0]]
def test_annotation_conversion_subset_ratio(self, mocker): addition_options = { 'annotation_conversion': { 'converter': 'wider', 'annotation_file': 'file' }, 'subsample_size': '50%' } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation( ['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=(converted_annotation, None)) subset_maker_mock = mocker.patch( 'accuracy_checker.dataset.make_subset') Dataset(config, MockPreprocessor()) subset_maker_mock.assert_called_once_with(converted_annotation, 1, 666)
def test_annotation_conversion_save_subset(self, mocker): addition_options = { 'annotation_conversion': {'converter': 'wider', 'annotation_file': Path('file')}, 'annotation': Path('custom'), 'subsample_size': 1, } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation(['0 0 0 5 5', '0 1 1 10 10'], True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=ConverterReturn(converted_annotation, None, None) ) annotation_saver_mock = mocker.patch( 'accuracy_checker.dataset.save_annotation' ) mocker.patch('pathlib.Path.exists', return_value=False) Dataset(config) annotation_saver_mock.assert_called_once_with([converted_annotation[1]], None, Path('custom'), None, config)
def test_annotation_conversion_with_store_annotation(self, mocker): addition_options = { 'annotation_conversion': { 'converter': 'wider', 'annotation_file': 'file' }, 'annotation': 'custom' } config = copy_dataset_config(self.dataset_config) config.update(addition_options) converted_annotation = make_representation('0 0 0 5 5', True) mocker.patch( 'accuracy_checker.annotation_converters.WiderFormatConverter.convert', return_value=(converted_annotation, None)) annotation_saver_mock = mocker.patch( 'accuracy_checker.dataset.save_annotation') Dataset(config, MockPreprocessor()) annotation_saver_mock.assert_called_once_with(converted_annotation, None, Path('custom'), None)
def from_configs(cls, config): dataset_config = config['datasets'][0] dataset = Dataset(dataset_config) data_reader_config = dataset_config.get('reader', 'opencv_imread') data_source = dataset_config['data_source'] if isinstance(data_reader_config, str): reader = BaseReader.provide(data_reader_config, data_source) elif isinstance(data_reader_config, dict): reader = BaseReader.provide(data_reader_config['type'], data_source, data_reader_config) else: raise ConfigError('reader should be dict or string') preprocessing = PreprocessingExecutor(dataset_config.get('preprocessing', []), dataset.name) metrics_executor = MetricsExecutor(dataset_config['metrics'], dataset) launcher_settings = config['launchers'][0] supported_frameworks = ['dlsdk'] if not launcher_settings['framework'] in supported_frameworks: raise ConfigError('{} framework not supported'.format(launcher_settings['framework'])) launcher = create_launcher(launcher_settings, delayed_model_loading=True) network_info = config.get('network_info', {}) colorization_network = network_info.get('colorization_network', {}) verification_network = network_info.get('verification_network', {}) model_args = config.get('_models', []) models_is_blob = config.get('_model_is_blob') if 'model' not in colorization_network and model_args: colorization_network['model'] = model_args[0] colorization_network['_model_is_blob'] = models_is_blob if 'model' not in verification_network and model_args: verification_network['model'] = model_args[1 if len(model_args) > 1 else 0] verification_network['_model_is_blob'] = models_is_blob network_info.update({ 'colorization_network': colorization_network, 'verification_network': verification_network }) if not contains_all(network_info, ['colorization_network', 'verification_network']): raise ConfigError('configuration for colorization_network/verification_network does not exist') test_model = ColorizationTestModel(network_info['colorization_network'], launcher) check_model = ColorizationCheckModel(network_info['verification_network'], launcher) return cls(dataset, reader, preprocessing, metrics_executor, launcher, test_model, check_model)
def test_annotation_conversion_unknown_converter_raise_config_error(self): addition_options = {'annotation_conversion': {'converter': 'unknown'}} config = copy_dataset_config(self.dataset_config) config.update(addition_options) with pytest.raises(ValueError): Dataset(config, MockPreprocessor())
def test_setting_custom_dataset_with_missed_data_source_raises_config_error_exception( self): local_dataset = copy_dataset_config(self.dataset_config) local_dataset.pop('data_source') with pytest.raises(ConfigError): Dataset(local_dataset, MockPreprocessor())
def test_missed_name_raises_config_error_exception(self): local_dataset = copy_dataset_config(self.dataset_config) local_dataset.pop('name') with pytest.raises(ConfigError): Dataset(local_dataset, MockPreprocessor())
def test_setting_custom_dataset_with_missed_annotation_raises_config_error_exception( self): local_dataset = copy_dataset_config(self.dataset_config) local_dataset.pop('annotation') with pytest.raises(ConfigError): Dataset(local_dataset)