def experiment(p: Parameters, o: Options):
assert (len(p.transformations) > 0)
use_cuda = torch.cuda.is_available()
model, training_parameters, training_options, scores = training.load_model(
p.model_path, use_cuda)
if o.verbose:
print("### ", model)
print("### Scores obtained:")
training.print_scores(scores)
dataset = datasets.get_classification(p.dataset.name)
dataset.normalize_features()
dataset = dataset.reduce_size_stratified(p.dataset.percentage)
if o.verbose:
print(dataset.summary())
if o.verbose:
print(
f"Measuring accuracy with transformations {p.transformations} on dataset {p.dataset} of size {dataset.size(p.dataset.subset)}..."
)
result: float = measure(model, dataset, p.transformations, o,
p.dataset.subset)
del model
del dataset
torch.cuda.empty_cache()
return AccuracyExperimentResult(p, result)
def prepare_dataset(p: TrainParameters):
tc, transformations, dataset_name, task = p.tc, p.transformations, p.dataset_name, p.task
strategy = TransformationStrategy.random_sample
if p.task == Task.Regression:
dataset = datasets.get_regression(dataset_name)
dim_output = len(transformations)
dataset.normalize_features()
train_dataset = ImageTransformRegressionDataset(
NumpyDataset(dataset.x_train), p.transformations, strategy)
test_dataset = ImageTransformRegressionDataset(
NumpyDataset(dataset.x_test), p.transformations, strategy)
elif task == Task.Classification:
dataset = datasets.get_classification(dataset_name)
dim_output = dataset.num_classes
dataset.normalize_features()
train_dataset = ImageClassificationDataset(
NumpyDataset(dataset.x_train, dataset.y_train), p.transformations,
strategy)
test_dataset = ImageClassificationDataset(
NumpyDataset(dataset.x_test, dataset.y_test), p.transformations,
strategy)
else:
raise ValueError(task)
return train_dataset, test_dataset, dataset.input_shape, dim_output
def run(self):
measures = normalized_measures
# conv_model_names = [m for m in common_model_names if (not "FFNet" in m)]
conv_model_names = simple_models_generators # [models.SimpleConv.__name__]
transformations = common_transformations
combinations = itertools.product(conv_model_names, dataset_names,
transformations, measures)
for (model_config_generator, dataset_name, transformation,
measure) in combinations:
model_config = model_config_generator.for_dataset(dataset_name)
# train
epochs = config.get_epochs(model_config, dataset_name,
transformation)
p_training = training.Parameters(model_config, dataset_name,
transformation, epochs)
experiment_name = f"{model_config.name}_{dataset_name}_{transformation.id()}_{measure.id()}"
plot_folderpath = self.folderpath / experiment_name
finished = Path(plot_folderpath) / "finished"
if finished.exists():
continue
# train
self.experiment_training(p_training)
p = config.dataset_size_for_measure(measure)
p_dataset = measure_package.DatasetParameters(
dataset_name, datasets.DatasetSubset.test, p)
p_variance = measure_package.Parameters(p_training.id(), p_dataset,
transformation, measure)
model_path = self.model_path(p_training)
self.experiment_measure(p_variance)
model_filepath = self.model_path(p_training)
model, p_model, o, scores = training.load_model(
model_filepath, use_cuda=torch.cuda.is_available())
result_filepath = self.results_path(p_variance)
result = self.load_experiment_result(
result_filepath).measure_result
dataset = datasets.get_classification(dataset_name)
plot_folderpath.mkdir(parents=True, exist_ok=True)
self.plot(plot_folderpath,
model,
dataset,
transformation,
result,
images=2,
most_invariant_k=4,
least_invariant_k=4,
conv_aggregation=ca_mean)
finished.touch()
def run(self):
dataset_names = ["mnist", "cifar10"]
cudas = [False, True]
r, s, t, combined = config.common_transformations_combined
transformation_sets = [r, s, t, combined]
n_images = 4
p = profiler.Profiler()
p.event("start")
for transformations, dataset_name, use_cuda in itertools.product(
transformation_sets, dataset_names, cudas):
print(f"### Loading dataset {dataset_name} ....")
folderpath = self.folderpath / f"{dataset_name}"
folderpath.mkdir(exist_ok=True, parents=True)
dataset = datasets.get_classification(dataset_name)
dataset.normalize_features()
adapter = tm.NumpyPytorchImageTransformationAdapter(
use_cuda=use_cuda)
numpy_dataset = NumpyDataset(dataset.x_test, dataset.y_test)
x, y = numpy_dataset.get_batch(range(n_images))
if use_cuda:
x = x.cuda()
n_t = len(transformations)
print(
f"Dataset {dataset_name}, Transformations: {n_t} ({transformations})"
)
for i in range(n_images):
print(f"Generating plots for image {i}")
original_torch = x[i, :]
# transformed_images = []
transformed_torch = torch.zeros((n_t, *original_torch.shape))
original_torch = original_torch.unsqueeze(0)
for j, t in enumerate(transformations):
transformed_torch[j, :] = t(original_torch)
transformed_numpy = adapter.post_adapt(transformed_torch)
cuda_str = "_cuda" if use_cuda else ""
filepath = folderpath / f"samples_first_{i}_{transformations}{cuda_str}.png"
util.plot_image_grid(transformed_numpy,
samples=n_t,
grid_cols=16,
show=False,
save=filepath)
p.event("end")
print(p.summary(human=True))
def adapt_dataset(dataset:datasets.ClassificationDataset, dataset_template:str):
dataset_template = datasets.get_classification(dataset_template)
h,w,c= dataset_template.input_shape
del dataset_template
oh,ow,oc=dataset.input_shape
# fix channels
if c !=oc and oc==1:
expand_channels(dataset,c)
elif c != oc and c ==1:
collapse_channels(dataset)
else:
raise ValueError(f"Cannot transform image with {oc} channels into image with {c} channels.")
#fix size
if h!=oh or w!=ow:
resize(dataset,h,w,c)
dataset.input_shape=(h,w,c)
def main(experiment: TMExperiment, p: training.Parameters, o: training.Options,
min_accuracy: float):
dataset = datasets.get_classification(p.dataset)
dataset.normalize_features()
if o.verbose_general:
print_summary(dataset, p, o, min_accuracy)
def do_train():
model, optimizer = p.model.make_model_and_optimizer(
dataset.input_shape, dataset.num_classes, o.use_cuda)
def generate_epochs_callbacks():
epochs_callbacks = []
for epoch in p.savepoints:
def callback(epoch=epoch):
scores = training.eval_scores(
model, dataset, p.transformations,
TransformationStrategy.random_sample,
o.get_eval_options())
if o.verbose_general:
print(
f"Saving model {model.name} at epoch {epoch}/{p.epochs}."
)
training.save_model(p, o, model, scores,
experiment.model_path(p, epoch))
epochs_callbacks.append((epoch, callback))
return dict(epochs_callbacks)
epochs_callbacks = generate_epochs_callbacks()
# TRAINING
if 0 in p.savepoints:
scores = training.eval_scores(model, dataset, p.transformations,
TransformationStrategy.random_sample,
o.get_eval_options())
print(f"Saving model {model.name} at epoch {0} (before training).")
training.save_model(p, o, model, scores,
experiment.model_path(p, 0))
pr = Profiler()
pr.event("start")
scores, history = training.run(p,
o,
model,
optimizer,
dataset,
epochs_callbacks=epochs_callbacks)
pr.event("end")
print(pr.summary(human=True))
training.print_scores(scores)
return model, history, scores
converged = False
restarts = 0
test_accuracy = 0
model, history, scores = None, None, None
while not converged and restarts <= o.max_restarts:
if restarts > 0:
message = f"""Model did not converge since it did not reach minimum accuracy ({test_accuracy}<{min_accuracy}). Restarting.. {restarts}/{o.max_restarts}"""
print(message)
model, history, scores = do_train()
training.plot_history(history, p, experiment.training_plots_path())
test_accuracy = scores["test"][1]
converged = test_accuracy > min_accuracy
restarts += 1
# SAVING
if o.save_model:
if converged:
path = experiment.model_path(p)
training.save_model(p, o, model, scores, path)
print(f"Model saved to {path}")
else:
print(
f"Model was not saved since it did not reach minimum accuracy. Accuracy={test_accuracy}<{min_accuracy}."
)
# delete model and empty cuda cache
del model
del dataset
torch.cuda.empty_cache()
import skimage.io
import datasets
d = datasets.get_classification("cifar10")
imgs = d.x_train.transpose((0, 2, 3, 1))
print(imgs.shape)
for i in range(10):
skimage.io.imsave(f"testing/cifar{i}.png", imgs[i, :])
import datasets
from testing.util import plot_image_grid
from pytorch.numpy_dataset import NumpyDataset
from transformational_measures.pytorch import ImageDataset
import transformational_measures as tm
dataformat = "NCHW"
dataset = datasets.get_classification("cifar10", dataformat=dataformat)
print(dataset.summary())
numpy_dataset = NumpyDataset(dataset.x_test, dataset.y_test)
transformations = tm.SimpleAffineTransformationGenerator(r=360, s=5, t=3)
image_dataset = ImageDataset(numpy_dataset,
transformations,
dataformat=dataformat)
x, y = image_dataset.get_batch(list(range(128)))
x = x.permute(0, 2, 3, 1).numpy()
print("pytorch_iterators", x.shape, x.dtype, x.min(axis=(0, 1, 2)),
x.max(axis=(0, 1, 2)))
filepath = f"testing/{dataset.name}_samples.png"
print(f"Saving transformed image batch to {filepath}")
plot_image_grid(x, y, show=False, save=filepath)
from pytorch.numpy_dataset import NumpyDataset
from testing import util
import transformational_measures as tm
import matplotlib
from pytorch.pytorch_image_dataset import ImageClassificationDataset
matplotlib.use('Agg')
import config
model_config = config.SimpleConvConfig()
dataset_name = "mnist"
print(f"### Loading dataset {dataset_name} and model {model_config.name}....")
use_cuda = True
dataset = datasets.get_classification(dataset_name)
numpy_dataset = NumpyDataset(dataset.x_test, dataset.y_test)
image_dataset = ImageClassificationDataset(numpy_dataset)
model, optimizer = model_config.make_model_and_optimizer(
dataset.input_shape, dataset.num_classes, use_cuda)
p = profiler.Profiler()
p.event("start")
#transformations=tm.SimpleAffineTransformationGenerator(r=360, s=4, t=3)
transformations = tm.SimpleAffineTransformationGenerator(r=360, n_rotations=4)
transformations.set_input_shape(dataset.input_shape)
transformations.set_pytorch(True)
transformations.set_cuda(use_cuda)
from testing.util import plot_image_grid
import datasets
import numpy as np
dataset="lsa16"
d=datasets.get_classification(dataset)
print(d.summary())
(unique, counts) = np.unique(d.y_test, return_counts=True)
print("Test")
for i,c in zip(unique,counts):
print(f"Class {i:02}, {c:005} samples ")
(unique, counts) = np.unique(d.y_train, return_counts=True)
print("train")
for i,c in zip(unique,counts):
print(f"Class {i:02}, {c:005} samples ")
#plot_image_grid(d.x_train,d.y_train,samples=32)
def run(self):
random_models_folderpath = self.models_folder() / "random"
random_models_folderpath.mkdir(exist_ok=True, parents=True)
o = training.Options(False, 32, 4, torch.cuda.is_available(), False, 0)
measures = normalized_measures
# number of random models to generate
random_model_n = 10
combinations = itertools.product(simple_models_generators,
dataset_names, common_transformations,
measures)
for model_config_generator, dataset_name, transformation, measure in combinations:
model_config = model_config_generator.for_dataset(dataset_name)
p = config.dataset_size_for_measure(measure)
# generate `random_model_n` models and save them without training
models_paths = []
p_training = training.Parameters(model_config, dataset_name,
transformation, 0)
dataset = datasets.get_classification(dataset_name)
for i in range(random_model_n):
model_path = self.model_path(
p_training,
custom_models_folderpath=random_models_folderpath)
# append index to model name
name, ext = os.path.splitext(str(model_path))
name += f"_random{i:03}"
model_path = Path(f"{name}{ext}")
if not model_path.exists():
model, optimizer = model_config.make_model_and_optimizer(
dataset.input_shape, dataset.num_classes, o.use_cuda)
scores = training.eval_scores(
model, dataset, p_training.transformations,
TransformationStrategy.random_sample,
o.get_eval_options())
training.save_model(p_training, o, model, scores,
model_path)
del model
models_paths.append(model_path)
# generate variance params
variance_parameters = []
p_dataset = measure_package.DatasetParameters(
dataset_name, datasets.DatasetSubset.test, p)
for model_path in models_paths:
model_id, ext = os.path.splitext(os.path.basename(model_path))
p_variance = measure_package.Parameters(
model_id, p_dataset, transformation, measure)
self.experiment_measure(p_variance, model_path=model_path)
variance_parameters.append(p_variance)
# plot results
experiment_name = f"{model_config.name}_{dataset_name}_{transformation.id()}_{measure}"
plot_filepath = self.folderpath / f"{experiment_name}.jpg"
results = self.load_measure_results(
self.results_paths(variance_parameters))
n = len(results)
labels = [f"{l.random_models} ({n} {l.samples})."] + ([None] *
(n - 1))
# get alpha colors
import matplotlib.pyplot as plt
color = plt.cm.hsv(np.linspace(0.1, 0.9, n))
color[:, 3] = 0.5
visualization.plot_collapsing_layers_same_model(results,
plot_filepath,
plot_mean=True,
labels=labels,
colors=color)
