def experiment(p: Parameters, o: Options): assert (len(p.transformations) > 0) use_cuda = torch.cuda.is_available() model, training_parameters, training_options, scores = training.load_model( p.model_path, use_cuda) if o.verbose: print("### ", model) print("### Scores obtained:") training.print_scores(scores) dataset = datasets.get_classification(p.dataset.name) dataset.normalize_features() dataset = dataset.reduce_size_stratified(p.dataset.percentage) if o.verbose: print(dataset.summary()) if o.verbose: print( f"Measuring accuracy with transformations {p.transformations} on dataset {p.dataset} of size {dataset.size(p.dataset.subset)}..." ) result: float = measure(model, dataset, p.transformations, o, p.dataset.subset) del model del dataset torch.cuda.empty_cache() return AccuracyExperimentResult(p, result)
def prepare_dataset(p: TrainParameters): tc, transformations, dataset_name, task = p.tc, p.transformations, p.dataset_name, p.task strategy = TransformationStrategy.random_sample if p.task == Task.Regression: dataset = datasets.get_regression(dataset_name) dim_output = len(transformations) dataset.normalize_features() train_dataset = ImageTransformRegressionDataset( NumpyDataset(dataset.x_train), p.transformations, strategy) test_dataset = ImageTransformRegressionDataset( NumpyDataset(dataset.x_test), p.transformations, strategy) elif task == Task.Classification: dataset = datasets.get_classification(dataset_name) dim_output = dataset.num_classes dataset.normalize_features() train_dataset = ImageClassificationDataset( NumpyDataset(dataset.x_train, dataset.y_train), p.transformations, strategy) test_dataset = ImageClassificationDataset( NumpyDataset(dataset.x_test, dataset.y_test), p.transformations, strategy) else: raise ValueError(task) return train_dataset, test_dataset, dataset.input_shape, dim_output
def run(self): measures = normalized_measures # conv_model_names = [m for m in common_model_names if (not "FFNet" in m)] conv_model_names = simple_models_generators # [models.SimpleConv.__name__] transformations = common_transformations combinations = itertools.product(conv_model_names, dataset_names, transformations, measures) for (model_config_generator, dataset_name, transformation, measure) in combinations: model_config = model_config_generator.for_dataset(dataset_name) # train epochs = config.get_epochs(model_config, dataset_name, transformation) p_training = training.Parameters(model_config, dataset_name, transformation, epochs) experiment_name = f"{model_config.name}_{dataset_name}_{transformation.id()}_{measure.id()}" plot_folderpath = self.folderpath / experiment_name finished = Path(plot_folderpath) / "finished" if finished.exists(): continue # train self.experiment_training(p_training) p = config.dataset_size_for_measure(measure) p_dataset = measure_package.DatasetParameters( dataset_name, datasets.DatasetSubset.test, p) p_variance = measure_package.Parameters(p_training.id(), p_dataset, transformation, measure) model_path = self.model_path(p_training) self.experiment_measure(p_variance) model_filepath = self.model_path(p_training) model, p_model, o, scores = training.load_model( model_filepath, use_cuda=torch.cuda.is_available()) result_filepath = self.results_path(p_variance) result = self.load_experiment_result( result_filepath).measure_result dataset = datasets.get_classification(dataset_name) plot_folderpath.mkdir(parents=True, exist_ok=True) self.plot(plot_folderpath, model, dataset, transformation, result, images=2, most_invariant_k=4, least_invariant_k=4, conv_aggregation=ca_mean) finished.touch()
def run(self): dataset_names = ["mnist", "cifar10"] cudas = [False, True] r, s, t, combined = config.common_transformations_combined transformation_sets = [r, s, t, combined] n_images = 4 p = profiler.Profiler() p.event("start") for transformations, dataset_name, use_cuda in itertools.product( transformation_sets, dataset_names, cudas): print(f"### Loading dataset {dataset_name} ....") folderpath = self.folderpath / f"{dataset_name}" folderpath.mkdir(exist_ok=True, parents=True) dataset = datasets.get_classification(dataset_name) dataset.normalize_features() adapter = tm.NumpyPytorchImageTransformationAdapter( use_cuda=use_cuda) numpy_dataset = NumpyDataset(dataset.x_test, dataset.y_test) x, y = numpy_dataset.get_batch(range(n_images)) if use_cuda: x = x.cuda() n_t = len(transformations) print( f"Dataset {dataset_name}, Transformations: {n_t} ({transformations})" ) for i in range(n_images): print(f"Generating plots for image {i}") original_torch = x[i, :] # transformed_images = [] transformed_torch = torch.zeros((n_t, *original_torch.shape)) original_torch = original_torch.unsqueeze(0) for j, t in enumerate(transformations): transformed_torch[j, :] = t(original_torch) transformed_numpy = adapter.post_adapt(transformed_torch) cuda_str = "_cuda" if use_cuda else "" filepath = folderpath / f"samples_first_{i}_{transformations}{cuda_str}.png" util.plot_image_grid(transformed_numpy, samples=n_t, grid_cols=16, show=False, save=filepath) p.event("end") print(p.summary(human=True))
def adapt_dataset(dataset:datasets.ClassificationDataset, dataset_template:str): dataset_template = datasets.get_classification(dataset_template) h,w,c= dataset_template.input_shape del dataset_template oh,ow,oc=dataset.input_shape # fix channels if c !=oc and oc==1: expand_channels(dataset,c) elif c != oc and c ==1: collapse_channels(dataset) else: raise ValueError(f"Cannot transform image with {oc} channels into image with {c} channels.") #fix size if h!=oh or w!=ow: resize(dataset,h,w,c) dataset.input_shape=(h,w,c)
def main(experiment: TMExperiment, p: training.Parameters, o: training.Options, min_accuracy: float): dataset = datasets.get_classification(p.dataset) dataset.normalize_features() if o.verbose_general: print_summary(dataset, p, o, min_accuracy) def do_train(): model, optimizer = p.model.make_model_and_optimizer( dataset.input_shape, dataset.num_classes, o.use_cuda) def generate_epochs_callbacks(): epochs_callbacks = [] for epoch in p.savepoints: def callback(epoch=epoch): scores = training.eval_scores( model, dataset, p.transformations, TransformationStrategy.random_sample, o.get_eval_options()) if o.verbose_general: print( f"Saving model {model.name} at epoch {epoch}/{p.epochs}." ) training.save_model(p, o, model, scores, experiment.model_path(p, epoch)) epochs_callbacks.append((epoch, callback)) return dict(epochs_callbacks) epochs_callbacks = generate_epochs_callbacks() # TRAINING if 0 in p.savepoints: scores = training.eval_scores(model, dataset, p.transformations, TransformationStrategy.random_sample, o.get_eval_options()) print(f"Saving model {model.name} at epoch {0} (before training).") training.save_model(p, o, model, scores, experiment.model_path(p, 0)) pr = Profiler() pr.event("start") scores, history = training.run(p, o, model, optimizer, dataset, epochs_callbacks=epochs_callbacks) pr.event("end") print(pr.summary(human=True)) training.print_scores(scores) return model, history, scores converged = False restarts = 0 test_accuracy = 0 model, history, scores = None, None, None while not converged and restarts <= o.max_restarts: if restarts > 0: message = f"""Model did not converge since it did not reach minimum accuracy ({test_accuracy}<{min_accuracy}). Restarting.. {restarts}/{o.max_restarts}""" print(message) model, history, scores = do_train() training.plot_history(history, p, experiment.training_plots_path()) test_accuracy = scores["test"][1] converged = test_accuracy > min_accuracy restarts += 1 # SAVING if o.save_model: if converged: path = experiment.model_path(p) training.save_model(p, o, model, scores, path) print(f"Model saved to {path}") else: print( f"Model was not saved since it did not reach minimum accuracy. Accuracy={test_accuracy}<{min_accuracy}." ) # delete model and empty cuda cache del model del dataset torch.cuda.empty_cache()
import skimage.io import datasets d = datasets.get_classification("cifar10") imgs = d.x_train.transpose((0, 2, 3, 1)) print(imgs.shape) for i in range(10): skimage.io.imsave(f"testing/cifar{i}.png", imgs[i, :])
import datasets from testing.util import plot_image_grid from pytorch.numpy_dataset import NumpyDataset from transformational_measures.pytorch import ImageDataset import transformational_measures as tm dataformat = "NCHW" dataset = datasets.get_classification("cifar10", dataformat=dataformat) print(dataset.summary()) numpy_dataset = NumpyDataset(dataset.x_test, dataset.y_test) transformations = tm.SimpleAffineTransformationGenerator(r=360, s=5, t=3) image_dataset = ImageDataset(numpy_dataset, transformations, dataformat=dataformat) x, y = image_dataset.get_batch(list(range(128))) x = x.permute(0, 2, 3, 1).numpy() print("pytorch_iterators", x.shape, x.dtype, x.min(axis=(0, 1, 2)), x.max(axis=(0, 1, 2))) filepath = f"testing/{dataset.name}_samples.png" print(f"Saving transformed image batch to {filepath}") plot_image_grid(x, y, show=False, save=filepath)
from pytorch.numpy_dataset import NumpyDataset from testing import util import transformational_measures as tm import matplotlib from pytorch.pytorch_image_dataset import ImageClassificationDataset matplotlib.use('Agg') import config model_config = config.SimpleConvConfig() dataset_name = "mnist" print(f"### Loading dataset {dataset_name} and model {model_config.name}....") use_cuda = True dataset = datasets.get_classification(dataset_name) numpy_dataset = NumpyDataset(dataset.x_test, dataset.y_test) image_dataset = ImageClassificationDataset(numpy_dataset) model, optimizer = model_config.make_model_and_optimizer( dataset.input_shape, dataset.num_classes, use_cuda) p = profiler.Profiler() p.event("start") #transformations=tm.SimpleAffineTransformationGenerator(r=360, s=4, t=3) transformations = tm.SimpleAffineTransformationGenerator(r=360, n_rotations=4) transformations.set_input_shape(dataset.input_shape) transformations.set_pytorch(True) transformations.set_cuda(use_cuda)
from testing.util import plot_image_grid import datasets import numpy as np dataset="lsa16" d=datasets.get_classification(dataset) print(d.summary()) (unique, counts) = np.unique(d.y_test, return_counts=True) print("Test") for i,c in zip(unique,counts): print(f"Class {i:02}, {c:005} samples ") (unique, counts) = np.unique(d.y_train, return_counts=True) print("train") for i,c in zip(unique,counts): print(f"Class {i:02}, {c:005} samples ") #plot_image_grid(d.x_train,d.y_train,samples=32)
def run(self): random_models_folderpath = self.models_folder() / "random" random_models_folderpath.mkdir(exist_ok=True, parents=True) o = training.Options(False, 32, 4, torch.cuda.is_available(), False, 0) measures = normalized_measures # number of random models to generate random_model_n = 10 combinations = itertools.product(simple_models_generators, dataset_names, common_transformations, measures) for model_config_generator, dataset_name, transformation, measure in combinations: model_config = model_config_generator.for_dataset(dataset_name) p = config.dataset_size_for_measure(measure) # generate `random_model_n` models and save them without training models_paths = [] p_training = training.Parameters(model_config, dataset_name, transformation, 0) dataset = datasets.get_classification(dataset_name) for i in range(random_model_n): model_path = self.model_path( p_training, custom_models_folderpath=random_models_folderpath) # append index to model name name, ext = os.path.splitext(str(model_path)) name += f"_random{i:03}" model_path = Path(f"{name}{ext}") if not model_path.exists(): model, optimizer = model_config.make_model_and_optimizer( dataset.input_shape, dataset.num_classes, o.use_cuda) scores = training.eval_scores( model, dataset, p_training.transformations, TransformationStrategy.random_sample, o.get_eval_options()) training.save_model(p_training, o, model, scores, model_path) del model models_paths.append(model_path) # generate variance params variance_parameters = [] p_dataset = measure_package.DatasetParameters( dataset_name, datasets.DatasetSubset.test, p) for model_path in models_paths: model_id, ext = os.path.splitext(os.path.basename(model_path)) p_variance = measure_package.Parameters( model_id, p_dataset, transformation, measure) self.experiment_measure(p_variance, model_path=model_path) variance_parameters.append(p_variance) # plot results experiment_name = f"{model_config.name}_{dataset_name}_{transformation.id()}_{measure}" plot_filepath = self.folderpath / f"{experiment_name}.jpg" results = self.load_measure_results( self.results_paths(variance_parameters)) n = len(results) labels = [f"{l.random_models} ({n} {l.samples})."] + ([None] * (n - 1)) # get alpha colors import matplotlib.pyplot as plt color = plt.cm.hsv(np.linspace(0.1, 0.9, n)) color[:, 3] = 0.5 visualization.plot_collapsing_layers_same_model(results, plot_filepath, plot_mean=True, labels=labels, colors=color)