def test(DATASET="Texas", CONFIG=None): """ 1. Fetch data (x, y, change_map) 2. Compute/estimate A_x and A_y (for patches) 3. Compute change_prior 4. Define dataset with (x, A_x, y, A_y, p). Choose patch size compatible with affinity computations. 5. Train CrossCyclicImageTransformer unsupervised a. Evaluate the image transformations in some way? 6. Evaluate the change detection scheme a. change_map = threshold [(x - f_y(y))/2 + (y - f_x(x))/2] """ if CONFIG is None: CONFIG = get_config_kACE(DATASET) print(f"Loading {DATASET} data") x_im, y_im, EVALUATE, (C_X, C_Y) = datasets.fetch(DATASET, **CONFIG) if tf.test.is_gpu_available() and not CONFIG["debug"]: C_CODE = 3 TRANSLATION_SPEC = { "enc_X": { "input_chs": C_X, "filter_spec": [50, 50, C_CODE] }, "enc_Y": { "input_chs": C_Y, "filter_spec": [50, 50, C_CODE] }, "dec_X": { "input_chs": C_CODE, "filter_spec": [50, 50, C_X] }, "dec_Y": { "input_chs": C_CODE, "filter_spec": [50, 50, C_Y] }, } else: C_CODE = 1 TRANSLATION_SPEC = { "enc_X": { "input_chs": C_X, "filter_spec": [C_CODE] }, "enc_Y": { "input_chs": C_Y, "filter_spec": [C_CODE] }, "dec_X": { "input_chs": C_CODE, "filter_spec": [C_X] }, "dec_Y": { "input_chs": C_CODE, "filter_spec": [C_Y] }, } print("Change Detector Init") cd = Kern_AceNet(TRANSLATION_SPEC, **CONFIG) print("Training") training_time = 0 cross_loss_weight = tf.expand_dims( tf.zeros(x_im.shape[:-1], dtype=tf.float32), -1) for epochs in CONFIG["list_epochs"]: CONFIG.update(epochs=epochs) tr_gen, dtypes, shapes = datasets._training_data_generator( x_im[0], y_im[0], cross_loss_weight[0], CONFIG["patch_size"]) TRAIN = tf.data.Dataset.from_generator(tr_gen, dtypes, shapes) TRAIN = TRAIN.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) tr_time, _ = cd.train(TRAIN, evaluation_dataset=EVALUATE, **CONFIG) for x, y, _ in EVALUATE.batch(1): alpha = cd([x, y]) cross_loss_weight = 1.0 - alpha training_time += tr_time cd.final_evaluate(EVALUATE, **CONFIG) final_kappa = cd.metrics_history["cohens kappa"][-1] final_acc = cd.metrics_history["ACC"][-1] performance = (final_kappa, final_acc) timestamp = cd.timestamp epoch = cd.epoch.numpy() speed = (epoch, training_time, timestamp) del cd gc.collect() return performance, speed
def test(DATASET="Texas", CONFIG=None): """ 1. Fetch data (x, y, change_map) 2. Compute/estimate A_x and A_y (for patches) 3. Compute change_prior 4. Define dataset with (x, A_x, y, A_y, p). Choose patch size compatible with affinity computations. 5. Train CrossCyclicImageTransformer unsupervised a. Evaluate the image transformations in some way? 6. Evaluate the change detection scheme a. change_map = threshold [(x - f_y(y))/2 + (y - f_x(x))/2] """ if CONFIG is None: CONFIG = get_config_kACE(DATASET) print(f"Loading {DATASET} data") x_im, y_im, EVALUATE, (C_X, C_Y) = datasets.fetch(DATASET, **CONFIG) if tf.test.is_gpu_available() and not CONFIG["debug"]: C_CODE = 3 print("here") TRANSLATION_SPEC = { "enc_X": { "input_chs": C_X, "filter_spec": [50, 50, C_CODE] }, "enc_Y": { "input_chs": C_Y, "filter_spec": [50, 50, C_CODE] }, "dec_X": { "input_chs": C_CODE, "filter_spec": [50, 50, C_X] }, "dec_Y": { "input_chs": C_CODE, "filter_spec": [50, 50, C_Y] }, } else: print("why here?") C_CODE = 1 TRANSLATION_SPEC = { "enc_X": { "input_chs": C_X, "filter_spec": [C_CODE] }, "enc_Y": { "input_chs": C_Y, "filter_spec": [C_CODE] }, "dec_X": { "input_chs": C_CODE, "filter_spec": [C_X] }, "dec_Y": { "input_chs": C_CODE, "filter_spec": [C_Y] }, } print("Change Detector Init") cd = Kern_AceNet(TRANSLATION_SPEC, **CONFIG) print("Training") training_time = 0 cross_loss_weight = tf.expand_dims( tf.zeros(x_im.shape[:-1], dtype=tf.float32), -1) for epochs in CONFIG["list_epochs"]: CONFIG.update(epochs=epochs) tr_gen, dtypes, shapes = datasets._training_data_generator( x_im[0], y_im[0], cross_loss_weight[0], CONFIG["patch_size"]) TRAIN = tf.data.Dataset.from_generator(tr_gen, dtypes, shapes) TRAIN = TRAIN.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) tr_time, _ = cd.train(TRAIN, evaluation_dataset=EVALUATE, **CONFIG) for x, y, _ in EVALUATE.batch(1): alpha = cd([x, y]) cross_loss_weight = 1.0 - alpha training_time += tr_time cd.save_all_weights() cd.final_evaluate(EVALUATE, **CONFIG) # Alternate evaluation cd.alt_evaluate(EVALUATE, **CONFIG) final_kappa = cd.metrics_history["cohens kappa"][-1] final_acc = cd.metrics_history["ACC"][-1] performance = (final_kappa, final_acc) timestamp = cd.timestamp epoch = cd.epoch.numpy() speed = (epoch, training_time, timestamp) # Save config parameters with open(os.path.join(CONFIG["logdir"], "config.txt"), "w+") as f: print(CONFIG, file=f) # Save full translated images train_flag_tmp = False x_code, y_code = cd._enc_x(x, train_flag_tmp), cd._enc_y(y, train_flag_tmp) x_hat, y_hat = cd._dec_x(y_code, train_flag_tmp), cd._dec_y( x_code, train_flag_tmp) im_dir = CONFIG["logdir"] #print("SHAPE OF X_HAT: ", x_hat.shape) #print("SHAPE OF Y_HAT: ", y_hat.shape) x_hat_out = np.squeeze(np.array(x_hat)) y_hat_out = np.squeeze(np.array(y_hat)) #print("SHAPE OF X_HAT_OUT: ", x_hat_out.shape) #print("SHAPE OF Y_HAT_OUT: ", y_hat_out.shape) tifffile.imsave(os.path.join(im_dir, "x_hat_full.tif"), x_hat_out, planarconfig="contig") tifffile.imsave(os.path.join(im_dir, "y_hat_full.tif"), y_hat_out, planarconfig="contig") del x_hat, x_code, y_hat, y_code del cd gc.collect() return performance, speed
#!/usr/bin/env python import os import os.path import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, 'sparat')) import argparse from datasets import datasets, get_dataset_path, fetch parser = argparse.ArgumentParser(description="Fetch raw data files.", epilog="available datasets: " + ", ".join(datasets.keys())) parser.add_argument( 'tofetch', nargs='*', help="Names of datasets to fetch. If not given, all will be fetched.") args = parser.parse_args() if len(args.tofetch) <= 0: args.tofetch = datasets.keys() for dataset_name in args.tofetch: fetch(dataset_name, get_dataset_path(dataset_name, datasets[dataset_name]))
def test(DATASET="Texas", CONFIG=None): if CONFIG is None: CONFIG = get_config(DATASET) _, _, EVALUATE, _ = datasets.fetch(DATASET, **CONFIG) cd = ChangePrior(**CONFIG) cd.print_all_input_images(EVALUATE)
def test(DATASET="Texas"): CONFIG = get_config(DATASET) _, _, EVALUATE, _ = datasets.fetch(DATASET, **CONFIG) cd = ChangeDetector(**CONFIG) cd.print_all_input_images(EVALUATE)
#!/usr/bin/env python import os import os.path import sys sys.path.insert( 0, os.path.join(os.path.dirname(__file__), os.pardir, 'sparat')) import argparse from datasets import datasets, get_dataset_path, fetch parser = argparse.ArgumentParser( description="Fetch raw data files.", epilog="available datasets: " + ", ".join(datasets.keys())) parser.add_argument( 'tofetch', nargs='*', help="Names of datasets to fetch. If not given, all will be fetched.") args = parser.parse_args() if len(args.tofetch) <= 0: args.tofetch = datasets.keys() for dataset_name in args.tofetch: fetch(dataset_name, get_dataset_path(dataset_name, datasets[dataset_name]))
def test(DATASET="Texas", CONFIG=None): """ 1. Fetch data (x, y, change_map) 2. Compute/estimate A_x and A_y (for patches) 3. Compute change_prior 4. Define dataset with (x, A_x, y, A_y, p). Choose patch size compatible with affinity computations. 5. Train CrossCyclicImageTransformer unsupervised a. Evaluate the image transformations in some way? 6. Evaluate the change detection scheme a. change_map = threshold [(x - f_y(y))/2 + (y - f_x(x))/2] """ if CONFIG is None: CONFIG = get_config_kACE(DATASET) print(f"Loading {DATASET} data") x_im, y_im, EVALUATE, (C_X, C_Y) = datasets.fetch(DATASET, **CONFIG) if tf.config.list_physical_devices("GPU") and not CONFIG["debug"]: C_CODE = 3 print("here") TRANSLATION_SPEC = { "enc_X": {"input_chs": C_X, "filter_spec": [50, 50, C_CODE]}, "enc_Y": {"input_chs": C_Y, "filter_spec": [50, 50, C_CODE]}, "dec_X": {"input_chs": C_CODE, "filter_spec": [50, 50, C_X]}, "dec_Y": {"input_chs": C_CODE, "filter_spec": [50, 50, C_Y]}, } else: print("why here?") C_CODE = 1 TRANSLATION_SPEC = { "enc_X": {"input_chs": C_X, "filter_spec": [C_CODE]}, "enc_Y": {"input_chs": C_Y, "filter_spec": [C_CODE]}, "dec_X": {"input_chs": C_CODE, "filter_spec": [C_X]}, "dec_Y": {"input_chs": C_CODE, "filter_spec": [C_Y]}, } print("Change Detector Init") cd = Kern_AceNet(TRANSLATION_SPEC, **CONFIG) print("Training") training_time = 0 cross_loss_weight = tf.expand_dims(tf.zeros(x_im.shape[:-1], dtype=tf.float32), -1) for epochs in CONFIG["list_epochs"]: CONFIG.update(epochs=epochs) tr_gen, dtypes, shapes = datasets._training_data_generator( x_im[0], y_im[0], cross_loss_weight[0], CONFIG["patch_size"] ) TRAIN = tf.data.Dataset.from_generator(tr_gen, dtypes, shapes) TRAIN = TRAIN.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) tr_time, _ = cd.train(TRAIN, evaluation_dataset=EVALUATE, **CONFIG) for x, y, _ in EVALUATE.batch(1): alpha = cd([x, y]) cross_loss_weight = 1.0 - alpha training_time += tr_time cd.load_all_weights(cd.log_path) cd.final_evaluate(EVALUATE, **CONFIG) metrics = {} for key in list(cd.difference_img_metrics.keys()) + list( cd.change_map_metrics.keys() ): metrics[key] = cd.metrics_history[key][-1] metrics["F1"] = metrics["TP"] / ( metrics["TP"] + 0.5 * (metrics["FP"] + metrics["FN"]) ) timestamp = cd.timestamp epoch = cd.epoch.numpy() speed = (epoch, training_time, timestamp) del cd gc.collect() return metrics, speed
def test(DATASET="Texas", CONFIG=None): """ 1. Fetch data (x, y, change_map) 2. Compute/estimate A_x and A_y (for patches) 3. Compute change_prior 4. Define dataset with (x, A_x, y, A_y, p). Choose patch size compatible with affinity computations. 5. Train CrossCyclicImageTransformer unsupervised a. Evaluate the image transformations in some way? 6. Evaluate the change detection scheme a. change_map = threshold [(x - f_y(y))/2 + (y - f_x(x))/2] """ if CONFIG is None: CONFIG = get_config_SCCN(DATASET) x_im, y_im, EVALUATE, (C_X, C_Y) = datasets.fetch(DATASET, **CONFIG) if tf.test.is_gpu_available() and not CONFIG["debug"]: C_CODE = CONFIG["C_CODE"] TRANSLATION_SPEC = { "enc_X": { "input_chs": C_X, "filter_spec": [C_CODE, C_CODE, C_CODE, C_CODE], }, "enc_Y": { "input_chs": C_Y, "filter_spec": [C_CODE, C_CODE, C_CODE, C_CODE], }, "dec_X": {"input_chs": C_CODE, "filter_spec": [C_X]}, "dec_Y": {"input_chs": C_CODE, "filter_spec": [C_Y]}, } else: C_CODE = 1 TRANSLATION_SPEC = { "enc_X": {"input_chs": C_X, "filter_spec": [C_CODE]}, "enc_Y": {"input_chs": C_Y, "filter_spec": [C_CODE]}, "dec_X": {"input_chs": C_CODE, "filter_spec": [C_X]}, "dec_Y": {"input_chs": C_CODE, "filter_spec": [C_Y]}, } cd = SCCN(TRANSLATION_SPEC, **CONFIG) training_time = 0 Pu = tf.expand_dims(tf.ones(x_im.shape[:-1], dtype=tf.float32), -1) TRAIN = tf.data.Dataset.from_tensor_slices((x_im, y_im, Pu)) TRAIN = TRAIN.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) training_time, _ = cd.pretrain(EVALUATE, evaluation_dataset=EVALUATE, **CONFIG) epochs = CONFIG["epochs"] CONFIG.update(epochs=1) for epoch in trange(epochs): tr_time, _ = cd.train(TRAIN, evaluation_dataset=EVALUATE, **CONFIG) training_time += tr_time if epoch > 10: for x, y, _ in EVALUATE.batch(1): Pu = 1.0 - tf.cast(cd._change_map(cd([x, y])), dtype=tf.float32) del TRAIN gc.collect() TRAIN = tf.data.Dataset.from_tensor_slices((x_im, y_im, Pu)) TRAIN = TRAIN.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) cd.final_evaluate(EVALUATE, **CONFIG) final_kappa = cd.metrics_history["cohens kappa"][-1] timestamp = cd.timestamp epoch = cd.epoch.numpy() return final_kappa, epoch, training_time, timestamp
"""Command Line Interface for the collections service. Usage: cli.py [fetch] [decrypt] NAME Options: -h --help Show this screen. --version Show version. """ import sys import iorw import datasets import encryption from docopt import docopt # Load info about all appropriate datasets into # memory dataRepo = datasets.Repository() dataRepo.loadDatasets('./fixtures/datasets.yaml') if __name__ == '__main__': arguments = docopt(__doc__, version='Collections CLI 0.1') if arguments['fetch']: datasets.fetch(arguments['NAME'], dataRepo) if arguments['decrypt']: datasets.decrypt(arguments['NAME'], dataRepo)