def test_adapter_dict_config(self):
launcher_config = {'model': 'foo', 'framework': 'dlsdk', 'device': 'cpu', 'adapter': {'type': 'classification'}}
config_errors = ModelEvaluator.validate_config({'models': [{'launchers': [launcher_config], 'datasets': []}]})
assert len(config_errors) == 1
assert config_errors[0].message == 'datasets section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'models.datasets'
def test_input_without_name(self):
launcher_config = {'model': 'foo', 'framework': 'dlsdk', 'device': 'cpu', 'inputs': [{"type": 'INPUT'}]}
config_errors = ModelEvaluator.validate_config({'models': [{'launchers': [launcher_config], 'datasets': []}]})
assert len(config_errors) == 2
assert config_errors[0].message.endswith('input name is not provided')
assert config_errors[0].field_uri == 'models.launchers.0.inputs.0'
assert config_errors[1].message == 'datasets section is not provided'
assert not config_errors[1].entry
assert config_errors[1].field_uri == 'models.datasets'
def test_dataset_config_ignore_without_metrics(self):
dataset_config = {'name': 'dataset', 'data_source': 'data', 'annotation': 'annotation'}
config_errors = ModelEvaluator.validate_config(
{'models': [{'datasets': [dataset_config]}]}, delayed_annotation_loading=True
)
assert len(config_errors) == 1
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'models.launchers'
def test_empty_launchers_and_datasets_config(self):
config_errors = ModelEvaluator.validate_config({'models': [{'launchers': [], 'datasets': []}]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'models.launchers'
assert config_errors[1].message == 'datasets section is not provided'
assert not config_errors[1].entry
assert config_errors[1].field_uri == 'models.datasets'
def test_unregistered_adapter_config(self):
launcher_config = {'model': 'foo', 'framework': 'dlsdk', 'device': 'cpu', 'adapter': 'not_classification'}
config_errors = ModelEvaluator.validate_config({'models': [{'launchers': [launcher_config], 'datasets': []}]})
assert len(config_errors) == 2
assert config_errors[0].message.startswith('Invalid value "not_classification"')
assert config_errors[0].entry == 'not_classification'
assert config_errors[0].field_uri.startswith('models.launchers.0') and config_errors[0].field_uri.endswith('adapter')
assert config_errors[1].message == 'datasets section is not provided'
assert not config_errors[1].entry
assert config_errors[1].field_uri == 'models.datasets'
def test_dataset_config_without_metrics(self):
dataset_config = {'name': 'dataset', 'data_source': 'data', 'annotation': 'annotation'}
config_errors = ModelEvaluator.validate_config({'models': [{'datasets': [dataset_config]}]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'models.launchers'
assert config_errors[1].message == 'Metrics are not provided'
assert not config_errors[1].entry
assert config_errors[1].field_uri == 'models.datasets.0'
def test_data_source_does_not_exists(self):
dataset_config = {'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'data_source': 'data_dir'}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 3
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
assert config_errors[-1].message == 'Invalid value "data_dir" for datasets.0.data_source: path does not exist'
assert config_errors[-1].entry == 'data_dir'
assert config_errors[-1].field_uri == 'datasets.0.data_source'
def test_unregistered_launcher_config(self):
launcher_config = {'framework': 'foo'}
config_errors = ModelEvaluator.validate_config({'models': [{'launchers': [launcher_config], 'datasets': []}]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launcher foo is not unregistered'
assert config_errors[0].entry == launcher_config
assert config_errors[0].field_uri == 'models.launchers.0'
assert config_errors[1].message == 'datasets section is not provided'
assert not config_errors[1].entry
assert config_errors[1].field_uri == 'models.datasets'
def test_postprocessing_config(self):
dataset_config = {
'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'data_source': 'data',
'annotation': 'annotation', 'postprocessing': [{'type': 'resize_prediction_boxes'}]
}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 1
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
def test_dict_data_reader(self):
dataset_config = {
'name': 'dataset', 'annotation': 'annotation', 'metrics': [{'type': 'accuracy'}],
'reader': {'type': 'opencv_imread'}, 'data_source': 'data'
}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 1
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
def test_data_reader_ignore_without_data_source(self):
dataset_config = {'name': 'dataset', 'annotation': 'annotation', 'metrics': [{'type': 'accuracy'}]}
config_errors = ModelEvaluator.validate_config({'models': [{'datasets': [dataset_config]}]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'models.launchers'
assert config_errors[1].message == 'Invalid value "None" for models.datasets.0.data_source: models.datasets.0.data_source is not allowed to be None'
assert not config_errors[1].entry
assert config_errors[1].field_uri == 'models.datasets.0.data_source'
def test_annotation_conversion_config(self):
conversion_parameters = {'converter': 'imagenet', 'annotation_file': 'file'}
dataset_config = {
'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'data_source': 'data',
'annotation_conversion': conversion_parameters
}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 1
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
def test_annotation_is_not_provided(self):
dataset_config = {
'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'data_source': 'data'
}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
assert config_errors[1].message == 'annotation_conversion or annotation field should be provided'
assert config_errors[1].entry == dataset_config
assert config_errors[1].field_uri == 'datasets.0'
def test_data_source_is_file(self):
dataset_config = {
'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'annotation': 'annotation', 'data_source': 'data'
}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
assert config_errors[1].message == 'Invalid value "data" for datasets.0.data_source: path is not a directory'
assert config_errors[1].entry == 'data'
assert config_errors[1].field_uri == 'datasets.0.data_source'
def test_unregistered_data_reader(self):
dataset_config = {
'name': 'dataset', 'annotation': 'annotation', 'metrics': [{'type': 'accuracy'}], 'reader': 'unknown'
}
config_errors = ModelEvaluator.validate_config({'models': [{'datasets': [dataset_config]}]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'models.launchers'
assert config_errors[1].message.startswith('Invalid value "unknown" for models.datasets.0.reader')
assert config_errors[-1].entry == 'unknown'
assert config_errors[-1].field_uri == 'models.datasets.0.reader'
def test_annotation_conversion_without_converter(self):
dataset_config = {
'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'data_source': 'data',
'annotation_conversion': {}
}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
assert config_errors[1].message == 'converter is not found'
assert config_errors[1].entry == {}
assert config_errors[1].field_uri == 'datasets.0.annotation_conversion'
def test_postprocessing_config_unknown_parameter(self):
postprocessing_config = [{'type': 'bgr_to_rgb'}]
dataset_config = {
'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'data_source': 'data',
'annotation': 'annotation', 'postprocessing': postprocessing_config
}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
assert config_errors[1].message == "postprocessor bgr_to_rgb unregistered"
assert config_errors[1].entry == postprocessing_config[0]
assert config_errors[1].field_uri == 'datasets.0.postprocessing.0'
def test_postprocessing_config_extra_parameter(self):
postprocessing_config = [{'type': 'resize_prediction_boxes', 'something_extra': True}]
dataset_config = {
'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'data_source': 'data',
'annotation': 'annotation', 'postprocessing': postprocessing_config
}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
assert config_errors[1].message == "datasets.0.postprocessing.0 specifies unknown options: ['something_extra']"
assert config_errors[1].entry == postprocessing_config[0]
assert config_errors[1].field_uri == 'datasets.0.postprocessing.0'
def test_annotation_conversion_extra_parameter(self):
conversion_parameters = {'converter': 'imagenet', 'annotation_file': 'file', 'something_extra': 'extra'}
dataset_config = {
'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'data_source': 'data',
'annotation_conversion': conversion_parameters
}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
assert config_errors[1].message == "datasets.0.annotation_conversion specifies unknown options: ['something_extra']"
assert config_errors[1].entry == conversion_parameters
assert config_errors[1].field_uri == 'datasets.0.annotation_conversion'
def test_annotation_conversion_missed_parameter(self):
conversion_parameters = {'converter': 'imagenet'}
dataset_config = {
'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'data_source': 'data',
'annotation_conversion': conversion_parameters
}
config_errors = ModelEvaluator.validate_config({'datasets': [dataset_config]})
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
assert config_errors[1].message == 'Invalid config for datasets.0.annotation_conversion: missing required fields: annotation_file'
assert config_errors[1].entry == conversion_parameters
assert config_errors[1].field_uri == 'datasets.0.annotation_conversion'
def test_preprocessing_config_unknown_type_with_postponed_annotation(self):
preprocessing_config = [{'type': 'bgr_to_rgb'}, {'type': 'unknown', 'size': 224}]
dataset_config = {
'name': 'dataset', 'metrics': [{'type': 'accuracy'}], 'data_source': 'data',
'annotation': 'annotation', 'preprocessing': preprocessing_config
}
config_errors = ModelEvaluator.validate_config(
{'datasets': [dataset_config]}, delayed_annotation_loading=True
)
assert len(config_errors) == 2
assert config_errors[0].message == 'launchers section is not provided'
assert not config_errors[0].entry
assert config_errors[0].field_uri == 'launchers'
assert config_errors[1].message == 'preprocessor unknown unregistered'
assert config_errors[1].entry == preprocessing_config[1]
assert config_errors[1].field_uri == 'datasets.0.preprocessing.1'
def test_ignore_dataset_config(self):
launcher_config = {'model': 'foo', 'framework': 'dlsdk', 'device': 'cpu'}
config_errors = ModelEvaluator.validate_config(
{'models': [{'launchers': [launcher_config], 'datasets': []}]}, delayed_annotation_loading=True
)
assert not config_errors