def __init__(self, spec):
spec = self._decode_spec(spec)
if spec and not isinstance(spec, dict):
spec = {'name': spec}
self.spec = ModelSpec.get(spec['name'], **spec)
self.serving_timeout = 30
self.serving_client = TensorflowServingClient('localhost', 9000)
def test_augmented_image_data_generator_wrong_crop_original(
test_catdog_dataset_path):
model_spec = ModelSpec.get('test',
preprocess_func='mean_subtraction',
preprocess_args=[141., 130., 123.],
target_size=(224, 224, 3))
with pytest.raises(ValueError) as exception:
test_data_generator = AugmentedImageDataGenerator(
preprocessing_function=model_spec.preprocess_input,
data_augmentation={
'crop_original': 'fail',
'scale_sizes': [256]
})
datagen = test_data_generator.flow_from_directory(
directory=test_catdog_dataset_path,
batch_size=1,
target_size=model_spec.target_size[:2],
class_mode='categorical',
shuffle=False)
datagen.next()
expected = 'crop_original mode entered not supported, only `center_crop` is being supported now'
actual = str(exception).split('ValueError: ')[1]
assert actual == expected
def test_augmented_image_data_generator_wrong_arguments(
test_catdog_dataset_path):
model_spec = ModelSpec.get('test',
preprocess_func='mean_subtraction',
preprocess_args=[141., 130., 123.],
target_size=(224, 224, 3))
with pytest.raises(ValueError) as exception:
test_data_generator = AugmentedImageDataGenerator(
preprocessing_function=model_spec.preprocess_input,
data_augmentation={'error': 123})
datagen = test_data_generator.flow_from_directory(
directory=test_catdog_dataset_path,
batch_size=1,
target_size=model_spec.target_size[:2],
class_mode='categorical',
shuffle=False)
datagen.next()
expected = 'data_augmentation dictionary should contain `crop_original`, `scale_sizes` or `transforms` as keys'
actual = str(exception).split('ValueError: ')[1]
assert actual == expected
model_spec = ModelSpec.get('test',
preprocess_func='mean_subtraction',
preprocess_args=[141., 130., 123.],
target_size=(224, 224, 3))
with pytest.raises(ValueError) as exception:
test_data_generator = AugmentedImageDataGenerator(
preprocessing_function=model_spec.preprocess_input,
data_augmentation=[1, 2, 3])
datagen = test_data_generator.flow_from_directory(
directory=test_catdog_dataset_path,
batch_size=1,
target_size=model_spec.target_size[:2],
class_mode='categorical',
shuffle=False)
datagen.next()
expected = '`data_augmentation` is a %s and it should be a dictionary' % type(
[1, 2, 3])
actual = str(exception).split('ValueError: ')[1]
assert actual == expected
def test_as_json_mobilenet_v1():
spec = ModelSpec.get('mobilenet_v1')
expected = {
'name': 'mobilenet_v1',
'model': 'keras_applications.mobilenet.MobileNet',
'preprocess_args': None,
'preprocess_func': 'between_plus_minus_1',
'target_size': [224, 224, 3]
}
assert spec.as_json() == expected
def assert_model_serving(model_name, expected_scores, check_features=False):
model_spec = ModelSpec.get(model_name)
client = TensorflowServingClient('localhost', MODEL_SERVING_PORT)
result = client.make_prediction(cat_image(model_spec), 'image')
assert 'class_probabilities' in result
assert len(result['class_probabilities']) == 1
scores = result['class_probabilities'][0]
np.testing.assert_array_almost_equal(np.array(scores),
np.array(expected_scores).flatten())
if check_features:
assert 'image_features' in result
assert result['image_features'].shape == (1, 1024)
def test_augmented_image_data_generator_wrong_transforms(
test_catdog_dataset_path):
model_spec = ModelSpec.get('test',
preprocess_func='mean_subtraction',
preprocess_args=[141., 130., 123.],
target_size=(224, 224, 3))
with pytest.raises(ValueError) as exception:
test_data_generator = AugmentedImageDataGenerator(
preprocessing_function=model_spec.preprocess_input,
data_augmentation={
'scale_sizes': [256],
'transforms': ['failure']
})
datagen = test_data_generator.flow_from_directory(
directory=test_catdog_dataset_path,
batch_size=1,
target_size=model_spec.target_size[:2],
class_mode='categorical',
shuffle=False)
datagen.next()
expected = 'Wrong transform: failure . Check documentation to see the supported ones'
actual = str(exception).split('ValueError: ')[1]
assert actual == expected
with pytest.raises(ValueError) as exception:
test_data_generator = AugmentedImageDataGenerator(
preprocessing_function=model_spec.preprocess_input,
data_augmentation={
'scale_sizes': [256],
'transforms': 'blah'
})
datagen = test_data_generator.flow_from_directory(
directory=test_catdog_dataset_path,
batch_size=1,
target_size=model_spec.target_size[:2],
class_mode='categorical',
shuffle=False)
datagen.next()
expected = 'Incorrect format for `transforms`, a list of transforms is expected'
actual = str(exception).split('ValueError: ')[1]
assert actual == expected
def test_returns_existing_with_overrides():
spec = ModelSpec.get('mobilenet_v1',
model='keras_applications.mobilenet.MobileNet',
target_size=[512, 512, 3],
preprocess_func='mean_subtraction',
preprocess_args=[1, 2, 3])
assert spec is not None
assert spec.model == MobileNet
assert spec.target_size == [512, 512, 3]
assert spec.preprocess_func == 'mean_subtraction'
assert spec.preprocess_args == [1, 2, 3]
assert spec.preprocess_input is not None
assert spec.keras_kwargs == {
'backend': keras.backend,
'layers': keras.layers,
'models': keras.models,
'utils': keras.utils
}
def test_has_all_base_specs():
assert_lists_same_items(model_spec.BASE_SPEC_NAMES,
model_spec.BASE_SPECS.keys())
for name in model_spec.BASE_SPEC_NAMES:
spec = ModelSpec.get(name)
assert spec is not None
assert spec.name == name
assert spec.model is not None
assert spec.target_size is not None
assert spec.preprocess_func is not None
assert spec.preprocess_input is not None
assert spec.keras_kwargs == {
'backend': keras.backend,
'layers': keras.layers,
'models': keras.models,
'utils': keras.utils
}
def test_augmented_image_data_generator_wrong_scale_size(
test_catdog_dataset_path):
model_spec = ModelSpec.get('test',
preprocess_func='mean_subtraction',
preprocess_args=[141., 130., 123.],
target_size=(224, 224, 3))
with pytest.raises(ValueError) as exception:
test_data_generator = AugmentedImageDataGenerator(
preprocessing_function=model_spec.preprocess_input,
data_augmentation={'scale_sizes': 'asd'})
test_data_generator.flow_from_directory(
directory=test_catdog_dataset_path,
batch_size=1,
target_size=model_spec.target_size[:2],
class_mode='categorical',
shuffle=False)
expected = 'Incorrect format for `scale_sizes`, list of ints or `= default` is expected'
actual = str(exception).split('ValueError: ')[1]
assert actual == expected
def setup_model(name, model_path):
tf_model_dir = '.cache/model'
model_spec = ModelSpec.get(name)
model = model_spec.klass(weights='imagenet',
input_shape=tuple(model_spec.target_size))
model_dir = os.path.dirname(model_path)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
model.save(model_path)
image_data = cat_image(model_spec)
expected_scores = model.predict(image_data)
if os.path.exists(tf_model_dir):
shutil.rmtree(tf_model_dir)
tf_model_dir_dir = os.path.dirname(tf_model_dir)
if not os.path.exists(tf_model_dir_dir):
os.makedirs(tf_model_dir_dir)
return tf_model_dir, expected_scores
def get_model_spec(base_spec_name, **overrides):
return ModelSpec.get(base_spec_name, **overrides)
def test_augmented_image_data_generator(test_catdog_dataset_path):
model_spec = ModelSpec.get('test',
preprocess_func='mean_subtraction',
preprocess_args=[141., 130., 123.],
target_size=(224, 224, 3))
test_data_generator = AugmentedImageDataGenerator(
preprocessing_function=model_spec.preprocess_input,
data_augmentation={
'scale_sizes': 'default',
'transforms': ['horizontal_flip']
})
datagen = test_data_generator.flow_from_directory(
directory=test_catdog_dataset_path,
batch_size=1,
target_size=model_spec.target_size[:2],
class_mode='categorical',
shuffle=False)
batch_x, batch_y = datagen.next()
assert batch_x.shape == (1, 144, 224, 224, 3)
assert len(batch_y) == 144
test_data_generator = AugmentedImageDataGenerator(
preprocessing_function=model_spec.preprocess_input,
data_augmentation={
'scale_sizes': [256],
'transforms': [
'horizontal_flip', 'vertical_flip', 'rotate_90', 'rotate_180',
'rotate_270'
]
})
datagen = test_data_generator.flow_from_directory(
directory=test_catdog_dataset_path,
batch_size=1,
target_size=model_spec.target_size[:2],
class_mode='categorical',
shuffle=False)
batch_x, batch_y = datagen.next()
assert batch_x.shape == (1, 108, 224, 224, 3)
assert len(batch_y) == 108
test_data_generator = AugmentedImageDataGenerator(
preprocessing_function=model_spec.preprocess_input,
data_augmentation={
'transforms': [
'horizontal_flip', 'vertical_flip', 'rotate_90', 'rotate_180',
'rotate_270'
]
})
datagen = test_data_generator.flow_from_directory(
directory=test_catdog_dataset_path,
batch_size=1,
target_size=model_spec.target_size[:2],
class_mode='categorical',
shuffle=False)
batch_x, batch_y = datagen.next()
assert batch_x.shape == (1, 6, 224, 224, 3)
assert len(batch_y) == 6
test_data_generator = AugmentedImageDataGenerator(
preprocessing_function=model_spec.preprocess_input,
data_augmentation={
'scale_sizes': [256],
'crop_original':
'center_crop',
'transforms': [
'horizontal_flip', 'vertical_flip', 'rotate_90', 'rotate_180',
'rotate_270'
]
})
datagen = test_data_generator.flow_from_directory(
directory=test_catdog_dataset_path,
batch_size=1,
target_size=model_spec.target_size[:2],
class_mode='categorical',
shuffle=False)
batch_x, batch_y = datagen.next()
assert batch_x.shape == (1, 108, 224, 224, 3)
assert len(batch_y) == 108
def test_load_image_for_all_base_specs():
for name in model_spec.BASE_SPEC_NAMES:
spec = ModelSpec.get(name)
image_data = spec.load_image('tests/files/cat.jpg')
assert image_data.any()
def test_returns_none_for_nonexistent_and_spec():
spec = ModelSpec.get('nonexistent_v1')
assert spec is None
def model_spec_mobilenet():
dataset_mean = [142.69182214, 119.05833338, 106.89884415]
return ModelSpec.get('mobilenet_v1', preprocess_func='mean_subtraction', preprocess_args=dataset_mean)
def query_model(model_spec_name):
model_spec = ModelSpec.get(model_spec_name)
client = TensorflowServingClient('localhost',
MODEL_SERVING_PORTS[model_spec_name])
image = model_spec.load_image('tests/fixtures/files/cat.jpg')
return client.make_prediction(image, 'image')