def test_sparse_recall(): config_reset() test_str = ''' dataset: classes: - 0: name: class_0 - 1: name: class_1 ''' config.load(yaml_str=test_str) r0 = ext.metric('SparseRecall')(0) r1 = ext.metric('SparseRecall')(1) p0 = ext.metric('SparseRecall')(0) p1 = ext.metric('SparseRecall')(1) z = tf.zeros((3, 3, 3, 3), dtype=tf.int32) o = tf.ones((3, 3, 3, 3), dtype=tf.int32) metrics = [r0, r1, p0, p1] for m in metrics: m.reset_state() m.update_state(z, z, None) assert r0.result() == 1.0 assert r1.result() == 0.0 assert p0.result() == 1.0 assert p1.result() == 0.0 for m in metrics: m.reset_state() m.update_state(o, z, None) assert r0.result() == 0.0 assert r1.result() == 0.0 assert p0.result() == 0.0 assert p1.result() == 0.0
def test_train_main(identity_config, tmp_path): config_reset() train_config = tmp_path / 'config.yaml' with open(train_config, 'w') as f: f.write(''' train: steps: 5 epochs: 3 network: layers: - Input: shape: [1, 1, num_bands] - Conv2D: filters: 2 kernel_size: [1, 1] activation: relu padding: same batch_size: 1 validation: steps: 2 callbacks: - ExponentialLRScheduler: start_epoch: 2 ''') args = 'delta train --config %s --config %s' % (identity_config, train_config) main(args.split())
def test_optimizer(): config_reset() test_str = ''' train: optimizer: Adam: learning_rate: 0.05 ''' config.load(yaml_str=test_str) opt = config_parser.optimizer_from_dict(config.train.spec().optimizer) assert opt.lr.numpy() == pytest.approx(0.05) config_reset() test_str = ''' train: optimizer: Adam: learning_rate: PolynomialDecay: initial_learning_rate: 0.0001 decay_steps: 100000 end_learning_rate: 0.0000001 power: 0.9 cycle: false epsilon: 0.0001 ''' config.load(yaml_str=test_str) opt = config_parser.optimizer_from_dict(config.train.spec().optimizer) assert isinstance(opt.lr, tf.keras.optimizers.schedules.PolynomialDecay) assert opt.lr(0).numpy() == pytest.approx(0.0001) assert opt.lr(100000).numpy() == pytest.approx(0.0000001)
def test_train(): config_reset() test_str = ''' train: stride: 2 batch_size: 5 steps: 10 epochs: 3 loss: SparseCategoricalCrossentropy metrics: [metric] optimizer: opt validation: steps: 20 from_training: true ''' config.load(yaml_str=test_str) tc = config.train.spec() assert tc.stride == (2, 2) assert tc.batch_size == 5 assert tc.steps == 10 assert tc.epochs == 3 assert isinstance(config_parser.loss_from_dict(tc.loss), tf.keras.losses.SparseCategoricalCrossentropy) assert tc.metrics == ['metric'] assert tc.optimizer == 'opt' assert tc.validation.steps == 20 assert tc.validation.from_training
def test_train_validate(identity_config, binary_identity_tiff_filenames, tmp_path): config_reset() train_config = tmp_path / 'config.yaml' with open(train_config, 'w') as f: f.write(''' train: steps: 5 epochs: 3 network: layers: - Input: shape: [~, ~, num_bands] - Conv2D: filters: 2 kernel_size: [1, 1] activation: relu padding: same batch_size: 1 validation: from_training: false images: nodata_value: ~ files: [%s] labels: nodata_value: ~ files: [%s] steps: 2 callbacks: - ExponentialLRScheduler: start_epoch: 2 ''' % (binary_identity_tiff_filenames[0][0], binary_identity_tiff_filenames[1][0])) args = 'delta train --config %s --config %s' % (identity_config, train_config) main(args.split())
def test_mapped(): mcce = ext.loss('MappedCategoricalCrossentropy') z = tf.zeros((3, 3, 3, 3), dtype=tf.int32) o = tf.ones((3, 3, 3, 3), dtype=tf.float32) assert tf.reduce_sum(mcce([0, 0]).call(z, o)) == 0.0 assert tf.reduce_sum(mcce([1, 0]).call(z, o)) > 10.0 oo = tf.ones((3, 3, 3, 3, 2), dtype=tf.float32) assert tf.reduce_sum(mcce([[0, 0], [1, 1]]).call(z, oo)) == 0.0 assert tf.reduce_sum(mcce([[1, 1], [0, 0]]).call(z, oo)) > 10.0 config_reset() test_str = ''' dataset: classes: - 0: name: class_0 - 1: name: class_1 ''' config.load(yaml_str=test_str) assert tf.reduce_sum(mcce({0: 0, 1: 0}).call(z, o)) == 0.0 assert tf.reduce_sum(mcce({'class_0': 0, 'class_1': 0}).call(z, o)) == 0.0 assert tf.reduce_sum(mcce({0: 1, 1: 0}).call(z, o)) > 10.0 assert tf.reduce_sum(mcce({'class_0': 1, 'class_1': 0}).call(z, o)) > 10.0
def test_network_inline(): config_reset() test_str = ''' dataset: classes: 3 train: network: params: v1 : 10 layers: - Input: shape: [5, 5, num_bands] - Flatten: - Dense: units: v1 activation : relu - Dense: units: 3 activation : softmax ''' config.load(yaml_str=test_str) assert len(config.dataset.classes) == 3 model = config_parser.config_model(2)() assert model.input_shape == (None, 5, 5, 2) assert model.output_shape == (None, len(config.dataset.classes))
def test_general(): config_reset() assert config.general.gpus() == -1 test_str = ''' general: gpus: 3 io: threads: 5 tile_size: [5, 5] interleave_images: 3 cache: dir: nonsense limit: 2 ''' config.load(yaml_str=test_str) assert config.general.gpus() == 3 assert config.io.threads() == 5 assert config.io.tile_size()[0] == 5 assert config.io.tile_size()[1] == 5 assert config.io.interleave_images() == 3 cache = config.io.cache.manager() assert cache.folder() == 'nonsense' assert cache.limit() == 2 os.rmdir('nonsense')
def test_model_from_dict(): config_reset() test_str = ''' params: v1 : 10 layers: - Input: shape: in_shape - Flatten: - Dense: units: v1 activation : relu - Dense: units: out_shape activation : softmax ''' input_shape = (17, 17, 8) output_shape = 3 params_exposed = {'out_shape': output_shape, 'in_shape': input_shape} model = config_parser.model_from_dict(yaml.safe_load(test_str), params_exposed)() model.compile(optimizer='adam', loss='mse') assert model.input_shape[1:] == input_shape assert model.output_shape[1] == output_shape assert len(model.layers) == 4 # Input layer is added behind the scenes
def test_missing_file(): config_reset() parser = argparse.ArgumentParser() config.setup_arg_parser(parser) options = parser.parse_args(['--config', 'garbage.yaml']) with pytest.raises(FileNotFoundError): config.initialize(options, [])
def test_predict_main(identity_config, tmp_path): config_reset() model_path = tmp_path / 'model.h5' inputs = tf.keras.layers.Input((32, 32, 2)) tf.keras.Model(inputs, inputs).save(model_path) args = 'delta classify --config %s %s' % (identity_config, model_path) old = os.getcwd() os.chdir(tmp_path) # put temporary outputs here main(args.split()) os.chdir(old)
def test_network_file(): config_reset() test_str = ''' dataset: classes: 3 train: network: yaml_file: networks/convpool.yaml ''' config.load(yaml_str=test_str) model = config_parser.config_model(2)() assert model.input_shape == (None, 5, 5, 2) assert model.output_shape == (None, 3, 3, 3)
def test_tensorboard(): config_reset() assert not config.tensorboard.enabled() test_str = ''' tensorboard: enabled: false dir: nonsense ''' config.load(yaml_str=test_str) assert not config.tensorboard.enabled() assert config.tensorboard.dir() == 'nonsense'
def test_validate(): config_reset() test_str = ''' train: stride: -1 ''' with pytest.raises(AssertionError): config.load(yaml_str=test_str) config_reset() test_str = ''' train: stride: 0.5 ''' with pytest.raises(TypeError): config.load(yaml_str=test_str)
def test_images_files(): config_reset() file_path = os.path.join(os.path.dirname(__file__), 'data', 'landsat.tiff') test_str = ''' dataset: images: type: tiff preprocess: ~ files: [%s] ''' % (file_path) config.load(yaml_str=test_str) im = config.dataset.images() assert im.type() == 'tiff' assert len(im) == 1 assert im[0] == file_path
def test_images_dir(): config_reset() dir_path = os.path.join(os.path.dirname(__file__), 'data') test_str = ''' dataset: images: type: tiff preprocess: ~ directory: %s/ extension: .tiff ''' % (dir_path) config.load(yaml_str=test_str) im = config.dataset.images() assert im.type() == 'tiff' assert len(im) == 1 assert im[0].endswith('landsat.tiff') and os.path.exists(im[0])
def test_callbacks(): config_reset() test_str = ''' train: callbacks: - EarlyStopping: verbose: true - ReduceLROnPlateau: factor: 0.5 ''' config.load(yaml_str=test_str) cbs = config_parser.config_callbacks() assert len(cbs) == 2 assert isinstance(cbs[0], tf.keras.callbacks.EarlyStopping) assert cbs[0].verbose assert isinstance(cbs[1], tf.keras.callbacks.ReduceLROnPlateau) assert cbs[1].factor == 0.5
def test_argparser_config_file(tmp_path): config_reset() test_str = ''' tensorboard: enabled: false dir: nonsense ''' p = tmp_path / "temp.yaml" p.write_text(test_str) parser = argparse.ArgumentParser() config.setup_arg_parser(parser) options = parser.parse_args(['--config', str(p)]) config.initialize(options, []) assert not config.tensorboard.enabled() assert config.tensorboard.dir() == 'nonsense'
def test_mlflow(): config_reset() test_str = ''' mlflow: enabled: false uri: nonsense experiment_name: name frequency: 5 checkpoints: frequency: 10 ''' config.load(yaml_str=test_str) assert not config.mlflow.enabled() assert config.mlflow.uri() == 'nonsense' assert config.mlflow.frequency() == 5 assert config.mlflow.experiment() == 'name' assert config.mlflow.checkpoints.frequency() == 10
def test_preprocess(): config_reset() test_str = ''' dataset: images: preprocess: - scale: factor: 2.0 - offset: factor: 1.0 - clip: bounds: [0, 5] ''' config.load(yaml_str=test_str) f = config.dataset.images().preprocess() assert f(np.asarray([0.0]), None, None) == 1.0 assert f(np.asarray([2.0]), None, None) == 2.0 assert f(np.asarray([-5.0]), None, None) == 0.0 assert f(np.asarray([20.0]), None, None) == 5.0
def autoencoder(all_sources): source = all_sources[0] conftest.config_reset() (image_path, _) = source[0] config.load(yaml_str=''' io: cache: dir: %s dataset: images: type: %s directory: %s extension: %s preprocess: ~''' % (os.path.dirname(image_path), source[2], os.path.dirname(image_path), source[1])) dataset = imagery_dataset.AutoencoderDataset( config.dataset.images(), (3, 3), stride=config.train.spec().stride) return dataset
def test_fcn(dataset): conftest.config_reset() assert config.general.gpus() == -1 def model_fn(): inputs = keras.layers.Input((None, None, 1)) conv = keras.layers.Conv2D(filters=9, kernel_size=2, padding='same', strides=1)(inputs) upscore = keras.layers.Conv2D(filters=2, kernel_size=1, padding='same', strides=1)(conv) l = keras.layers.MaxPooling2D(pool_size=(2, 2), strides=2)(conv) l = keras.layers.Conv2D(filters=2, kernel_size=1, strides=1)(l) l = keras.layers.Conv2DTranspose(filters=2, padding='same', kernel_size=2, strides=2)(l) l = keras.layers.Add()([upscore, upscore]) #l = keras.layers.Softmax(axis=3)(l) m = keras.Model(inputs=inputs, outputs=l) return m dataset.set_chunk_output_shapes(None, (32, 32)) dataset.set_tile_shape((32, 32)) count = 0 for d in dataset.dataset(): count += 1 assert len(d) == 2 assert d[0].shape == (32, 32, 1) assert d[1].shape == (32, 32, 1) assert count == 2 # don't actually test correctness, this is not enough data for this size network evaluate_model(model_fn, dataset, threshold=0.0, max_wrong=10000, batch_size=1)
def test_augmentations(): config_reset() test_str = ''' train: augmentations: - random_flip_left_right: probability: 1.0 - random_flip_up_down: probability: 1.0 ''' config.load(yaml_str=test_str) aug = config_parser.config_augmentation() a = tf.constant(np.expand_dims(np.array([[0, 1], [2, 3]]), (0, 3))) o = tf.constant(np.expand_dims(np.array([[3, 2], [1, 0]]), (0, 3))) (b, c) = aug(a, a) assert (b.numpy() == o.numpy()).all() assert (c.numpy() == o.numpy()).all() config_reset() test_str = ''' train: augmentations: - random_flip_left_right: probability: 1.0 - random_flip_up_down: probability: 1.0 - random_rotate: probability: 1.0 max_angle: 0.5 - random_translate: probability: 1.0 max_pixels: 10 ''' config.load(yaml_str=test_str) aug = config_parser.config_augmentation() a = tf.constant(np.expand_dims(np.array([[0, 1], [2, 3]]), (0, 3))) o = tf.constant(np.expand_dims(np.array([[3, 2], [1, 0]]), (0, 3))) (b, c) = aug(a, a) # just make sure it doesn't crash assert b.numpy().shape == o.numpy().shape assert c.numpy().shape == o.numpy().shape
def test_argparser(): config_reset() parser = argparse.ArgumentParser() config.setup_arg_parser(parser) file_path = os.path.join(os.path.dirname(__file__), 'data', 'landsat.tiff') options = parser.parse_args( ('--image-type tiff --image %s' % (file_path) + ' --label-type tiff --label %s' % (file_path)).split()) config.parse_args(options) im = config.dataset.images() assert im.preprocess() is not None assert im.type() == 'tiff' assert len(im) == 1 assert im[0].endswith('landsat.tiff') and os.path.exists(im[0]) im = config.dataset.labels() assert im.type() == 'tiff' assert len(im) == 1 assert im[0].endswith('landsat.tiff') and os.path.exists(im[0])
def test_pretrained_layer(): config_reset() base_model = ''' params: v1 : 10 layers: - Input: shape: in_shape - Flatten: - Dense: units: v1 activation : relu name: encoding - Dense: units: out_shape activation : softmax ''' input_shape = (17, 17, 8) output_shape = 3 params_exposed = {'out_shape': output_shape, 'in_shape': input_shape} m1 = config_parser.model_from_dict(yaml.safe_load(base_model), params_exposed)() m1.compile(optimizer='adam', loss='mse') _, tmp_filename = tempfile.mkstemp(suffix='.h5') tf.keras.models.save_model(m1, tmp_filename) pretrained_model = ''' params: v1 : 10 layers: - Input: shape: in_shape - Pretrained: filename: %s encoding_layer: encoding - Dense: units: 100 activation: relu - Dense: units: out_shape activation: softmax ''' % tmp_filename m2 = config_parser.model_from_dict(yaml.safe_load(pretrained_model), params_exposed)() m2.compile(optimizer='adam', loss='mse') assert len(m2.layers[1].layers) == 3 for i in range(1, len(m1.layers)): assert isinstance(m1.layers[i], type(m2.layers[1].layers[i])) if m1.layers[i].name == 'encoding': break # test using internal layer of pretrained as input pretrained_model = ''' params: v1 : 10 layers: - Input: shape: in_shape - Pretrained: filename: %s encoding_layer: encoding name: pretrained outputs: [encoding] - Dense: units: 100 activation: relu inputs: pretrained/encoding - Dense: units: out_shape activation: softmax ''' % tmp_filename m2 = config_parser.model_from_dict(yaml.safe_load(pretrained_model), params_exposed)() m2.compile(optimizer='adam', loss='mse') assert len(m2.layers[1].layers) == (len(m1.layers) - 1 ) # also don't take the input layer for i in range(1, len(m1.layers)): assert isinstance(m1.layers[i], type(m2.layers[1].layers[i])) if m1.layers[i].name == 'encoding': break os.remove(tmp_filename)
def test_validate_main(identity_config): config_reset() args = 'delta validate --config %s' % (identity_config, ) main(args.split())
def test_dump(): config_reset() assert config.to_dict() == yaml.load(config.export())
def test_classes(): config_reset() test_str = ''' dataset: classes: 2 ''' config.load(yaml_str=test_str) assert len(config.dataset.classes) == 2 for (i, c) in enumerate(config.dataset.classes): assert c.value == i assert config.dataset.classes.weights() is None def assert_classes(classes): assert classes values = [1, 2, 5] for (i, c) in enumerate(classes): e = values[i] assert c.value == e assert c.name == str(e) assert c.color == e assert classes.weights() == [1.0, 5.0, 2.0] arr = np.array(values) ind = classes.classes_to_indices_func()(arr) assert np.max(ind) == 2 assert (classes.indices_to_classes_func()(ind) == values).all() config_reset() test_str = ''' dataset: classes: - 2: name: 2 color: 2 weight: 5.0 - 1: name: 1 color: 1 weight: 1.0 - 5: name: 5 color: 5 weight: 2.0 ''' config.load(yaml_str=test_str) assert_classes(config.dataset.classes) config_reset() test_str = ''' dataset: classes: 2: name: 2 color: 2 weight: 5.0 1: name: 1 color: 1 weight: 1.0 5: name: 5 color: 5 weight: 2.0 ''' config.load(yaml_str=test_str) assert_classes(config.dataset.classes)