def test_sparse_recall():
config_reset()
test_str = '''
dataset:
classes:
- 0:
name: class_0
- 1:
name: class_1
'''
config.load(yaml_str=test_str)
r0 = ext.metric('SparseRecall')(0)
r1 = ext.metric('SparseRecall')(1)
p0 = ext.metric('SparseRecall')(0)
p1 = ext.metric('SparseRecall')(1)
z = tf.zeros((3, 3, 3, 3), dtype=tf.int32)
o = tf.ones((3, 3, 3, 3), dtype=tf.int32)
metrics = [r0, r1, p0, p1]
for m in metrics:
m.reset_state()
m.update_state(z, z, None)
assert r0.result() == 1.0
assert r1.result() == 0.0
assert p0.result() == 1.0
assert p1.result() == 0.0
for m in metrics:
m.reset_state()
m.update_state(o, z, None)
assert r0.result() == 0.0
assert r1.result() == 0.0
assert p0.result() == 0.0
assert p1.result() == 0.0
def test_train_main(identity_config, tmp_path):
config_reset()
train_config = tmp_path / 'config.yaml'
with open(train_config, 'w') as f:
f.write('''
train:
steps: 5
epochs: 3
network:
layers:
- Input:
shape: [1, 1, num_bands]
- Conv2D:
filters: 2
kernel_size: [1, 1]
activation: relu
padding: same
batch_size: 1
validation:
steps: 2
callbacks:
- ExponentialLRScheduler:
start_epoch: 2
''')
args = 'delta train --config %s --config %s' % (identity_config,
train_config)
main(args.split())
def test_optimizer():
config_reset()
test_str = '''
train:
optimizer:
Adam:
learning_rate: 0.05
'''
config.load(yaml_str=test_str)
opt = config_parser.optimizer_from_dict(config.train.spec().optimizer)
assert opt.lr.numpy() == pytest.approx(0.05)
config_reset()
test_str = '''
train:
optimizer:
Adam:
learning_rate:
PolynomialDecay:
initial_learning_rate: 0.0001
decay_steps: 100000
end_learning_rate: 0.0000001
power: 0.9
cycle: false
epsilon: 0.0001
'''
config.load(yaml_str=test_str)
opt = config_parser.optimizer_from_dict(config.train.spec().optimizer)
assert isinstance(opt.lr, tf.keras.optimizers.schedules.PolynomialDecay)
assert opt.lr(0).numpy() == pytest.approx(0.0001)
assert opt.lr(100000).numpy() == pytest.approx(0.0000001)
def test_train():
config_reset()
test_str = '''
train:
stride: 2
batch_size: 5
steps: 10
epochs: 3
loss: SparseCategoricalCrossentropy
metrics: [metric]
optimizer: opt
validation:
steps: 20
from_training: true
'''
config.load(yaml_str=test_str)
tc = config.train.spec()
assert tc.stride == (2, 2)
assert tc.batch_size == 5
assert tc.steps == 10
assert tc.epochs == 3
assert isinstance(config_parser.loss_from_dict(tc.loss),
tf.keras.losses.SparseCategoricalCrossentropy)
assert tc.metrics == ['metric']
assert tc.optimizer == 'opt'
assert tc.validation.steps == 20
assert tc.validation.from_training
def test_train_validate(identity_config, binary_identity_tiff_filenames, tmp_path):
config_reset()
train_config = tmp_path / 'config.yaml'
with open(train_config, 'w') as f:
f.write('''
train:
steps: 5
epochs: 3
network:
layers:
- Input:
shape: [~, ~, num_bands]
- Conv2D:
filters: 2
kernel_size: [1, 1]
activation: relu
padding: same
batch_size: 1
validation:
from_training: false
images:
nodata_value: ~
files: [%s]
labels:
nodata_value: ~
files: [%s]
steps: 2
callbacks:
- ExponentialLRScheduler:
start_epoch: 2
''' % (binary_identity_tiff_filenames[0][0], binary_identity_tiff_filenames[1][0]))
args = 'delta train --config %s --config %s' % (identity_config, train_config)
main(args.split())
def test_mapped():
mcce = ext.loss('MappedCategoricalCrossentropy')
z = tf.zeros((3, 3, 3, 3), dtype=tf.int32)
o = tf.ones((3, 3, 3, 3), dtype=tf.float32)
assert tf.reduce_sum(mcce([0, 0]).call(z, o)) == 0.0
assert tf.reduce_sum(mcce([1, 0]).call(z, o)) > 10.0
oo = tf.ones((3, 3, 3, 3, 2), dtype=tf.float32)
assert tf.reduce_sum(mcce([[0, 0], [1, 1]]).call(z, oo)) == 0.0
assert tf.reduce_sum(mcce([[1, 1], [0, 0]]).call(z, oo)) > 10.0
config_reset()
test_str = '''
dataset:
classes:
- 0:
name: class_0
- 1:
name: class_1
'''
config.load(yaml_str=test_str)
assert tf.reduce_sum(mcce({0: 0, 1: 0}).call(z, o)) == 0.0
assert tf.reduce_sum(mcce({'class_0': 0, 'class_1': 0}).call(z, o)) == 0.0
assert tf.reduce_sum(mcce({0: 1, 1: 0}).call(z, o)) > 10.0
assert tf.reduce_sum(mcce({'class_0': 1, 'class_1': 0}).call(z, o)) > 10.0
def test_network_inline():
config_reset()
test_str = '''
dataset:
classes: 3
train:
network:
params:
v1 : 10
layers:
- Input:
shape: [5, 5, num_bands]
- Flatten:
- Dense:
units: v1
activation : relu
- Dense:
units: 3
activation : softmax
'''
config.load(yaml_str=test_str)
assert len(config.dataset.classes) == 3
model = config_parser.config_model(2)()
assert model.input_shape == (None, 5, 5, 2)
assert model.output_shape == (None, len(config.dataset.classes))
def test_general():
config_reset()
assert config.general.gpus() == -1
test_str = '''
general:
gpus: 3
io:
threads: 5
tile_size: [5, 5]
interleave_images: 3
cache:
dir: nonsense
limit: 2
'''
config.load(yaml_str=test_str)
assert config.general.gpus() == 3
assert config.io.threads() == 5
assert config.io.tile_size()[0] == 5
assert config.io.tile_size()[1] == 5
assert config.io.interleave_images() == 3
cache = config.io.cache.manager()
assert cache.folder() == 'nonsense'
assert cache.limit() == 2
os.rmdir('nonsense')
def test_model_from_dict():
config_reset()
test_str = '''
params:
v1 : 10
layers:
- Input:
shape: in_shape
- Flatten:
- Dense:
units: v1
activation : relu
- Dense:
units: out_shape
activation : softmax
'''
input_shape = (17, 17, 8)
output_shape = 3
params_exposed = {'out_shape': output_shape, 'in_shape': input_shape}
model = config_parser.model_from_dict(yaml.safe_load(test_str),
params_exposed)()
model.compile(optimizer='adam', loss='mse')
assert model.input_shape[1:] == input_shape
assert model.output_shape[1] == output_shape
assert len(model.layers) == 4 # Input layer is added behind the scenes
def test_missing_file():
config_reset()
parser = argparse.ArgumentParser()
config.setup_arg_parser(parser)
options = parser.parse_args(['--config', 'garbage.yaml'])
with pytest.raises(FileNotFoundError):
config.initialize(options, [])
def test_predict_main(identity_config, tmp_path):
config_reset()
model_path = tmp_path / 'model.h5'
inputs = tf.keras.layers.Input((32, 32, 2))
tf.keras.Model(inputs, inputs).save(model_path)
args = 'delta classify --config %s %s' % (identity_config, model_path)
old = os.getcwd()
os.chdir(tmp_path) # put temporary outputs here
main(args.split())
os.chdir(old)
def test_network_file():
config_reset()
test_str = '''
dataset:
classes: 3
train:
network:
yaml_file: networks/convpool.yaml
'''
config.load(yaml_str=test_str)
model = config_parser.config_model(2)()
assert model.input_shape == (None, 5, 5, 2)
assert model.output_shape == (None, 3, 3, 3)
def test_tensorboard():
config_reset()
assert not config.tensorboard.enabled()
test_str = '''
tensorboard:
enabled: false
dir: nonsense
'''
config.load(yaml_str=test_str)
assert not config.tensorboard.enabled()
assert config.tensorboard.dir() == 'nonsense'
def test_validate():
config_reset()
test_str = '''
train:
stride: -1
'''
with pytest.raises(AssertionError):
config.load(yaml_str=test_str)
config_reset()
test_str = '''
train:
stride: 0.5
'''
with pytest.raises(TypeError):
config.load(yaml_str=test_str)
def test_images_files():
config_reset()
file_path = os.path.join(os.path.dirname(__file__), 'data', 'landsat.tiff')
test_str = '''
dataset:
images:
type: tiff
preprocess: ~
files: [%s]
''' % (file_path)
config.load(yaml_str=test_str)
im = config.dataset.images()
assert im.type() == 'tiff'
assert len(im) == 1
assert im[0] == file_path
def test_images_dir():
config_reset()
dir_path = os.path.join(os.path.dirname(__file__), 'data')
test_str = '''
dataset:
images:
type: tiff
preprocess: ~
directory: %s/
extension: .tiff
''' % (dir_path)
config.load(yaml_str=test_str)
im = config.dataset.images()
assert im.type() == 'tiff'
assert len(im) == 1
assert im[0].endswith('landsat.tiff') and os.path.exists(im[0])
def test_callbacks():
config_reset()
test_str = '''
train:
callbacks:
- EarlyStopping:
verbose: true
- ReduceLROnPlateau:
factor: 0.5
'''
config.load(yaml_str=test_str)
cbs = config_parser.config_callbacks()
assert len(cbs) == 2
assert isinstance(cbs[0], tf.keras.callbacks.EarlyStopping)
assert cbs[0].verbose
assert isinstance(cbs[1], tf.keras.callbacks.ReduceLROnPlateau)
assert cbs[1].factor == 0.5
def test_argparser_config_file(tmp_path):
config_reset()
test_str = '''
tensorboard:
enabled: false
dir: nonsense
'''
p = tmp_path / "temp.yaml"
p.write_text(test_str)
parser = argparse.ArgumentParser()
config.setup_arg_parser(parser)
options = parser.parse_args(['--config', str(p)])
config.initialize(options, [])
assert not config.tensorboard.enabled()
assert config.tensorboard.dir() == 'nonsense'
def test_mlflow():
config_reset()
test_str = '''
mlflow:
enabled: false
uri: nonsense
experiment_name: name
frequency: 5
checkpoints:
frequency: 10
'''
config.load(yaml_str=test_str)
assert not config.mlflow.enabled()
assert config.mlflow.uri() == 'nonsense'
assert config.mlflow.frequency() == 5
assert config.mlflow.experiment() == 'name'
assert config.mlflow.checkpoints.frequency() == 10
def test_preprocess():
config_reset()
test_str = '''
dataset:
images:
preprocess:
- scale:
factor: 2.0
- offset:
factor: 1.0
- clip:
bounds: [0, 5]
'''
config.load(yaml_str=test_str)
f = config.dataset.images().preprocess()
assert f(np.asarray([0.0]), None, None) == 1.0
assert f(np.asarray([2.0]), None, None) == 2.0
assert f(np.asarray([-5.0]), None, None) == 0.0
assert f(np.asarray([20.0]), None, None) == 5.0
def autoencoder(all_sources):
source = all_sources[0]
conftest.config_reset()
(image_path, _) = source[0]
config.load(yaml_str='''
io:
cache:
dir: %s
dataset:
images:
type: %s
directory: %s
extension: %s
preprocess: ~''' %
(os.path.dirname(image_path), source[2],
os.path.dirname(image_path), source[1]))
dataset = imagery_dataset.AutoencoderDataset(
config.dataset.images(), (3, 3), stride=config.train.spec().stride)
return dataset
def test_fcn(dataset):
conftest.config_reset()
assert config.general.gpus() == -1
def model_fn():
inputs = keras.layers.Input((None, None, 1))
conv = keras.layers.Conv2D(filters=9,
kernel_size=2,
padding='same',
strides=1)(inputs)
upscore = keras.layers.Conv2D(filters=2,
kernel_size=1,
padding='same',
strides=1)(conv)
l = keras.layers.MaxPooling2D(pool_size=(2, 2), strides=2)(conv)
l = keras.layers.Conv2D(filters=2, kernel_size=1, strides=1)(l)
l = keras.layers.Conv2DTranspose(filters=2,
padding='same',
kernel_size=2,
strides=2)(l)
l = keras.layers.Add()([upscore, upscore])
#l = keras.layers.Softmax(axis=3)(l)
m = keras.Model(inputs=inputs, outputs=l)
return m
dataset.set_chunk_output_shapes(None, (32, 32))
dataset.set_tile_shape((32, 32))
count = 0
for d in dataset.dataset():
count += 1
assert len(d) == 2
assert d[0].shape == (32, 32, 1)
assert d[1].shape == (32, 32, 1)
assert count == 2
# don't actually test correctness, this is not enough data for this size network
evaluate_model(model_fn,
dataset,
threshold=0.0,
max_wrong=10000,
batch_size=1)
def test_augmentations():
config_reset()
test_str = '''
train:
augmentations:
- random_flip_left_right:
probability: 1.0
- random_flip_up_down:
probability: 1.0
'''
config.load(yaml_str=test_str)
aug = config_parser.config_augmentation()
a = tf.constant(np.expand_dims(np.array([[0, 1], [2, 3]]), (0, 3)))
o = tf.constant(np.expand_dims(np.array([[3, 2], [1, 0]]), (0, 3)))
(b, c) = aug(a, a)
assert (b.numpy() == o.numpy()).all()
assert (c.numpy() == o.numpy()).all()
config_reset()
test_str = '''
train:
augmentations:
- random_flip_left_right:
probability: 1.0
- random_flip_up_down:
probability: 1.0
- random_rotate:
probability: 1.0
max_angle: 0.5
- random_translate:
probability: 1.0
max_pixels: 10
'''
config.load(yaml_str=test_str)
aug = config_parser.config_augmentation()
a = tf.constant(np.expand_dims(np.array([[0, 1], [2, 3]]), (0, 3)))
o = tf.constant(np.expand_dims(np.array([[3, 2], [1, 0]]), (0, 3)))
(b, c) = aug(a, a)
# just make sure it doesn't crash
assert b.numpy().shape == o.numpy().shape
assert c.numpy().shape == o.numpy().shape
def test_argparser():
config_reset()
parser = argparse.ArgumentParser()
config.setup_arg_parser(parser)
file_path = os.path.join(os.path.dirname(__file__), 'data', 'landsat.tiff')
options = parser.parse_args(
('--image-type tiff --image %s' % (file_path) +
' --label-type tiff --label %s' % (file_path)).split())
config.parse_args(options)
im = config.dataset.images()
assert im.preprocess() is not None
assert im.type() == 'tiff'
assert len(im) == 1
assert im[0].endswith('landsat.tiff') and os.path.exists(im[0])
im = config.dataset.labels()
assert im.type() == 'tiff'
assert len(im) == 1
assert im[0].endswith('landsat.tiff') and os.path.exists(im[0])
def test_pretrained_layer():
config_reset()
base_model = '''
params:
v1 : 10
layers:
- Input:
shape: in_shape
- Flatten:
- Dense:
units: v1
activation : relu
name: encoding
- Dense:
units: out_shape
activation : softmax
'''
input_shape = (17, 17, 8)
output_shape = 3
params_exposed = {'out_shape': output_shape, 'in_shape': input_shape}
m1 = config_parser.model_from_dict(yaml.safe_load(base_model),
params_exposed)()
m1.compile(optimizer='adam', loss='mse')
_, tmp_filename = tempfile.mkstemp(suffix='.h5')
tf.keras.models.save_model(m1, tmp_filename)
pretrained_model = '''
params:
v1 : 10
layers:
- Input:
shape: in_shape
- Pretrained:
filename: %s
encoding_layer: encoding
- Dense:
units: 100
activation: relu
- Dense:
units: out_shape
activation: softmax
''' % tmp_filename
m2 = config_parser.model_from_dict(yaml.safe_load(pretrained_model),
params_exposed)()
m2.compile(optimizer='adam', loss='mse')
assert len(m2.layers[1].layers) == 3
for i in range(1, len(m1.layers)):
assert isinstance(m1.layers[i], type(m2.layers[1].layers[i]))
if m1.layers[i].name == 'encoding':
break
# test using internal layer of pretrained as input
pretrained_model = '''
params:
v1 : 10
layers:
- Input:
shape: in_shape
- Pretrained:
filename: %s
encoding_layer: encoding
name: pretrained
outputs: [encoding]
- Dense:
units: 100
activation: relu
inputs: pretrained/encoding
- Dense:
units: out_shape
activation: softmax
''' % tmp_filename
m2 = config_parser.model_from_dict(yaml.safe_load(pretrained_model),
params_exposed)()
m2.compile(optimizer='adam', loss='mse')
assert len(m2.layers[1].layers) == (len(m1.layers) - 1
) # also don't take the input layer
for i in range(1, len(m1.layers)):
assert isinstance(m1.layers[i], type(m2.layers[1].layers[i]))
if m1.layers[i].name == 'encoding':
break
os.remove(tmp_filename)
def test_validate_main(identity_config):
config_reset()
args = 'delta validate --config %s' % (identity_config, )
main(args.split())
def test_dump():
config_reset()
assert config.to_dict() == yaml.load(config.export())
def test_classes():
config_reset()
test_str = '''
dataset:
classes: 2
'''
config.load(yaml_str=test_str)
assert len(config.dataset.classes) == 2
for (i, c) in enumerate(config.dataset.classes):
assert c.value == i
assert config.dataset.classes.weights() is None
def assert_classes(classes):
assert classes
values = [1, 2, 5]
for (i, c) in enumerate(classes):
e = values[i]
assert c.value == e
assert c.name == str(e)
assert c.color == e
assert classes.weights() == [1.0, 5.0, 2.0]
arr = np.array(values)
ind = classes.classes_to_indices_func()(arr)
assert np.max(ind) == 2
assert (classes.indices_to_classes_func()(ind) == values).all()
config_reset()
test_str = '''
dataset:
classes:
- 2:
name: 2
color: 2
weight: 5.0
- 1:
name: 1
color: 1
weight: 1.0
- 5:
name: 5
color: 5
weight: 2.0
'''
config.load(yaml_str=test_str)
assert_classes(config.dataset.classes)
config_reset()
test_str = '''
dataset:
classes:
2:
name: 2
color: 2
weight: 5.0
1:
name: 1
color: 1
weight: 1.0
5:
name: 5
color: 5
weight: 2.0
'''
config.load(yaml_str=test_str)
assert_classes(config.dataset.classes)
