def run_experiment(self, raise_error=False, expected_msg=None):
dummy_exp = DummyExperiment()
dummy_exp = self.messenger_cls(dummy_exp, **self.messenger_kwargs)
dset_train = DummyDataset(10)
dset_test = DummyDataset(10)
dmgr_train = BaseDataManager(dset_train, 2, 1, None)
dmgr_test = BaseDataManager(dset_test, 2, 1, None)
with self.assertLogs(logger, level='INFO') as cm:
if raise_error:
with self.assertRaises(RuntimeError):
dummy_exp.run(dmgr_train,
dmgr_test,
raise_error=True,
**self.run_kwargs)
else:
dummy_exp.run(
dmgr_train,
dmgr_test,
raise_error=False,
**self.run_kwargs,
)
if expected_msg is None or not expected_msg:
logger.info("NoExpectedMessage")
if expected_msg is None or not expected_msg:
self.assertEqual(cm.output,
["INFO:UnitTestMessenger:NoExpectedMessage"])
else:
self.assertEqual(cm.output, expected_msg)
def test_experiment_kfold_tf(self):
from delira.training import TfExperiment
from delira.data_loading import BaseDataManager
# all test cases
for case in self._test_cases_tf:
with self.subTest(case=case):
# both split_types
for split_type in ["random", "stratified", "error"]:
with self.subTest(split_type=split_type):
if split_type == "error":
# must raise ValueError
with self.assertRaises(ValueError):
(params, dataset_length_train,
dataset_length_test, network_cls) = case
exp = TfExperiment(
params, network_cls,
key_mapping={"images": "data"})
dset = DummyDataset(
dataset_length_test + dataset_length_train)
dmgr = BaseDataManager(dset, 16, 1, None)
exp.kfold(
dmgr,
params.nested_get("val_metrics"),
shuffle=True,
split_type=split_type,
num_splits=2)
continue
# check all types of validation data
for val_split in [0.2, None]:
with self.subTest(val_split=val_split):
(params, dataset_length_train,
dataset_length_test, network_cls) = case
exp = TfExperiment(
params, network_cls,
key_mapping={"images": "data"})
dset = DummyDataset(
dataset_length_test + dataset_length_train)
dmgr = BaseDataManager(dset, 16, 1, None)
exp.kfold(
dmgr,
params.nested_get("val_metrics"),
shuffle=True,
split_type=split_type,
val_split=val_split,
num_splits=2,
)
def run_experiment(experiment_cls, config, network_cls, len_train, len_test,
**kwargs):
assert issubclass(experiment_cls, BaseExperiment)
exp = experiment_cls(config, network_cls, **kwargs)
dset_train = DummyDataset(len_train)
dset_test = DummyDataset(len_test)
dmgr_train = BaseDataManager(dset_train, 16, 4, None)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
return exp.run(dmgr_train, dmgr_test)
def setup_data(img_path,
batchsize,
num_processes,
transforms_train,
transforms_val,
dset_type="mnist"):
print("Setting Up Data")
img_path = os.path.expanduser(img_path)
data = {}
for train, key, trafo, sampler_cls in zip(
[True, False], ["train", "val"], [transforms_train, transforms_val],
[RandomSampler, SequentialSampler]):
dset = TorchvisionClassificationDataset(dset_type,
img_path,
train=train,
download=True,
img_shape=(28, 28))
dmgr = BaseDataManager(dset,
batchsize,
num_processes,
trafo,
sampler_cls=sampler_cls)
data[key] = dmgr
return data
def run(self, train_paths, val_paths=None):
dataset_train = self.dataset_cls(path_list=train_paths,
**self.dataset_train_kwargs)
mgr_train = BaseDataManager(dataset_train, **self.datamgr_train_kwargs)
if val_paths is not None:
dataset_val = self.dataset_cls(path_list=val_paths,
**self.dataset_val_kwargs)
mgr_val = BaseDataManager(dataset_val, **self.datamgr_train_kwargs)
if val_paths is not None:
super().run(mgr_train, mgr_val)
else:
super().run(mgr_train, None)
def test_experiment_test(self):
from delira.data_loading import BaseDataManager
# iterate over test cases
for case in self._test_cases:
with self.subTest(case=case):
# pop arguments (to use remaining case as kwargs later)
_ = case.pop("len_train")
config = case.pop("config")
metric_keys = case.pop("metric_keys")
network_cls = case.pop("network_cls")
len_test = case.pop("len_test")
exp = self._experiment_cls(config, network_cls, **case)
# create data
dset_test = DummyDataset(len_test)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
model = network_cls()
# must fit on 2 samples to initialize coefficients
model.fit(np.random.rand(2, 32), np.array([[0], [1]]))
exp.test(model, dmgr_test,
config.nested_get("metrics", {}),
metric_keys)
def test_experiment_test_torch(self):
from delira.training import PyTorchExperiment
from delira.data_loading import BaseDataManager
for case in self._test_cases_torch:
with self.subTest(case=case):
(params, dataset_length_train, dataset_length_test,
val_score_key, val_score_mode, network_cls) = case
exp = PyTorchExperiment(params, network_cls,
key_mapping={"x": "data"},
val_score_key=val_score_key,
val_score_mode=val_score_mode)
model = network_cls()
dset_test = DummyDataset(dataset_length_test)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
prepare_batch = partial(
model.prepare_batch,
output_device="cpu",
input_device="cpu")
exp.test(model, dmgr_test,
params.nested_get("val_metrics"),
prepare_batch=prepare_batch)
def kfold_experiment(self, raise_error=False, expected_msg=None):
kfold_kwargs = copy.deepcopy(self.run_kwargs)
kfold_kwargs.pop("fold")
dummy_exp = DummyExperiment()
dummy_exp = self.messenger_cls(dummy_exp, **self.messenger_kwargs)
dset = DummyDataset(10)
dmgr = BaseDataManager(dset, 2, 1, None)
with self.assertLogs(logger, level='INFO') as cm:
if raise_error:
with self.assertRaises(RuntimeError):
dummy_exp.kfold(data=dmgr,
metrics={},
num_splits=2,
raise_error=True,
**kfold_kwargs)
else:
dummy_exp.kfold(data=dmgr,
metrics={},
num_splits=2,
raise_error=False,
**kfold_kwargs)
if expected_msg is None:
logger.info("NoExpectedMessage")
if expected_msg is None:
self.assertEqual(cm.output,
["INFO:UnitTestMessenger:NoExpectedMessage"])
else:
self.assertEqual(cm.output, expected_msg)
def kfold_experiment(experiment_cls,
config,
network_cls,
len_data,
shuffle=True,
split_type="random",
num_splits=2,
val_split=None,
**kwargs):
assert issubclass(experiment_cls, BaseExperiment)
metric_keys = kwargs.pop("metric_keys", None)
exp = experiment_cls(config, network_cls, **kwargs)
dset = DummyDataset(len_data)
dmgr = BaseDataManager(dset, 16, 1, None)
return exp.kfold(data=dmgr,
metrics=config.nested_get("metrics"),
shuffle=shuffle,
split_type=split_type,
num_splits=num_splits,
val_split=val_split,
metric_keys=metric_keys)
def test_base_datamanager():
batch_size = 16
np.random.seed(1)
dset = DummyDataset(600, [0.5, 0.3, 0.2])
manager = BaseDataManager(dset,
batch_size,
n_process_augmentation=1,
transforms=None)
assert isinstance(manager.get_batchgen(), MultiThreadedAugmenter)
# create batch manually
data, labels = [], []
for i in range(batch_size):
data.append(dset[i]["data"])
labels.append(dset[i]["label"])
batch_dict = {"data": np.asarray(data), "label": np.asarray(labels)}
for key, val in next(manager.get_batchgen()).items():
assert (val == batch_dict[key]).all()
for key, val in next(manager.get_batchgen()).items():
assert len(val) == batch_size
def test_trainer(criterions, optimizer_cls, optimizer_params,
dataset_length_train, dataset_length_test):
class DummyNetwork(ClassificationNetworkBasePyTorch):
def __init__(self):
super().__init__(32, 1)
def forward(self, x):
return self.module(x)
@staticmethod
def _build_model(in_channels, n_outputs):
return torch.nn.Sequential(torch.nn.Linear(in_channels, 64),
torch.nn.ReLU(),
torch.nn.Linear(64, n_outputs))
class DummyDataset(AbstractDataset):
def __init__(self, length):
super().__init__(None, None, None, None)
self.length = length
def __getitem__(self, index):
return {
"data": np.random.rand(1, 32),
"label": np.random.rand(1, 1)
}
def __len__(self):
return self.length
network = DummyNetwork()
trainer = PyTorchNetworkTrainer(network=network,
save_path="/tmp/delira_trainer_test",
criterions=criterions,
optimizer_cls=optimizer_cls,
optimizer_params=optimizer_params)
dset_train = DummyDataset(dataset_length_train)
dset_test = DummyDataset(dataset_length_test)
dmgr_train = BaseDataManager(dset_train, 16, 4, None)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
trainer.train(2, dmgr_train, dmgr_test)
def test_experiment_run_tf(self):
from delira.training import TfExperiment
from delira.data_loading import BaseDataManager
for case in self._test_cases_tf:
with self.subTest(case=case):
(params, dataset_length_train, dataset_length_test,
network_cls) = case
exp = TfExperiment(params, network_cls,
key_mapping={"images": "data"})
dset_train = DummyDataset(dataset_length_train)
dset_test = DummyDataset(dataset_length_test)
dmgr_train = BaseDataManager(dset_train, 16, 2, None)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
exp.run(dmgr_train, dmgr_test)
def test_experiment(params, dataset_length_train, dataset_length_test):
class DummyNetwork(ClassificationNetworkBasePyTorch):
def __init__(self):
super().__init__(32, 1)
def forward(self, x):
return self.module(x)
@staticmethod
def _build_model(in_channels, n_outputs):
return torch.nn.Sequential(torch.nn.Linear(in_channels, 64),
torch.nn.ReLU(),
torch.nn.Linear(64, n_outputs))
class DummyDataset(AbstractDataset):
def __init__(self, length):
super().__init__(None, None, None, None)
self.length = length
def __getitem__(self, index):
return {
"data": np.random.rand(1, 32),
"label": np.random.rand(1, 1)
}
def __len__(self):
return self.length
exp = PyTorchExperiment(params, DummyNetwork)
dset_train = DummyDataset(dataset_length_train)
dset_test = DummyDataset(dataset_length_test)
dmgr_train = BaseDataManager(dset_train, 16, 4, None)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
net = exp.run(dmgr_train, dmgr_test)
exp.test(
params=params,
network=net,
datamgr_test=dmgr_test,
)
def test_experiment(experiment_cls, config, network_cls, len_test, **kwargs):
assert issubclass(experiment_cls, BaseExperiment)
exp = experiment_cls(config, network_cls, **kwargs)
dset_test = DummyDataset(len_test)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
model = network_cls()
return exp.test(model, dmgr_test, config.nested_get("metrics", {}),
kwargs.get("metric_keys", None))
def test_experiment_run_torch(self):
from delira.training import PyTorchExperiment
from delira.data_loading import BaseDataManager
for case in self._test_cases_torch:
with self.subTest(case=case):
(params, dataset_length_train, dataset_length_test,
val_score_key, val_score_mode, network_cls) = case
exp = PyTorchExperiment(params, network_cls,
key_mapping={"x": "data"},
val_score_key=val_score_key,
val_score_mode=val_score_mode)
dset_train = DummyDataset(dataset_length_train)
dset_test = DummyDataset(dataset_length_test)
dmgr_train = BaseDataManager(dset_train, 16, 2, None)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
exp.run(dmgr_train, dmgr_test)
def test_experiment_test_tf(self):
from delira.training import TfExperiment
from delira.data_loading import BaseDataManager
for case in self._test_cases_tf:
with self.subTest(case=case):
(params, dataset_length_train, dataset_length_test,
network_cls) = case
exp = TfExperiment(params, network_cls,
key_mapping={"images": "data"},
)
model = network_cls()
dset_test = DummyDataset(dataset_length_test)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
exp.test(model, dmgr_test, params.nested_get("val_metrics"))
def test_experiment_kfold_torch(self):
from delira.training import PyTorchExperiment
from delira.data_loading import BaseDataManager
from copy import deepcopy
# all test cases
for case in self._test_cases_torch:
with self.subTest(case=case):
(params, dataset_length_train,
dataset_length_test, val_score_key,
val_score_mode, network_cls) = case
# both split_types
for split_type in ["random", "stratified", "error"]:
with self.subTest(split_type=split_type):
if split_type == "error":
# must raise ValueError
with self.assertRaises(ValueError):
exp = PyTorchExperiment(
params, network_cls,
key_mapping={"x": "data"},
val_score_key=val_score_key,
val_score_mode=val_score_mode)
dset = DummyDataset(
dataset_length_test + dataset_length_train)
dmgr = BaseDataManager(dset, 16, 1, None)
exp.kfold(
dmgr,
params.nested_get("val_metrics"),
shuffle=True,
split_type=split_type,
num_splits=2)
continue
# check all types of validation data
for val_split in [0.2, None]:
with self.subTest(val_split=val_split):
# disable lr scheduling if no validation data
# is present
_params = deepcopy(params)
if val_split is None:
_params["fixed"]["training"
]["lr_sched_cls"] = None
exp = PyTorchExperiment(
_params, network_cls,
key_mapping={"x": "data"},
val_score_key=val_score_key,
val_score_mode=val_score_mode)
dset = DummyDataset(
dataset_length_test + dataset_length_train)
dmgr = BaseDataManager(dset, 16, 1, None)
exp.kfold(
dmgr,
params.nested_get("val_metrics"),
shuffle=True,
split_type=split_type,
val_split=val_split,
num_splits=2)
def kfold(self,
paths,
num_splits=5,
shuffle=True,
random_seed=None,
valid_size=0.1,
**kwargs):
"""
Runs K-Fold Crossvalidation
Parameters
----------
num_epochs: int
number of epochs to train the model
data: str
path to root dir
num_splits: None or int
number of splits for kfold
if None: len(data) splits will be validated
shuffle: bool
whether or not to shuffle indices for kfold
random_seed: None or int
random seed used to seed the kfold (if shuffle is true),
pytorch and numpy
valid_size : float, default: 0.1
relative size of validation dataset in relation to training set
"""
if random_seed is not None:
torch.manual_seed(random_seed)
np.random.seed(random_seed)
if "dataset_type" in kwargs and kwargs["dataset_type"] is not None:
dataset_type = kwargs["dataset_type"]
if dataset_type != "INbreast" and dataset_type != "DDSM":
raise ValueError("Unknown dataset!")
else:
raise ValueError("No dataset type!")
train_splits, _ = utils.kfold_patientwise(paths,
dataset_type=dataset_type,
num_splits=num_splits,
shuffle=shuffle,
random_state=random_seed)
for i in range(len(train_splits)):
train_paths, val_paths, _ = \
utils.split_paths_patientwise(train_splits[i],
dataset_type=dataset_type,
train_size= 1 - valid_size)
dataset_train = self.dataset_cls(path_list=train_paths,
**self.dataset_train_kwargs)
dataset_valid = self.dataset_cls(path_list=val_paths,
**self.dataset_val_kwargs)
mgr_train = BaseDataManager(dataset_train,
**self.datamgr_train_kwargs)
mgr_valid = BaseDataManager(dataset_valid,
**self.datamgr_val_kwargs)
super().run(mgr_train, mgr_valid, fold=i, **kwargs)
from batchgenerators.transforms import RandomCropTransform, Compose
from batchgenerators.transforms.spatial_transforms import ResizeTransform,SpatialTransform
transforms = Compose([
#SpatialTransform(patch_size=(1024, 1024), do_rotation=True, patch_center_dist_from_border=1024, border_mode_data='reflect',
# border_mode_seg='reflect', angle_x=(args.rot_angle, args.rot_angle), angle_y=(0, 0), angle_z=(0, 0),
# do_elastic_deform=False, order_data=1, order_seg=1)
ResizeTransform((int(args.resize_size), int(args.resize_size)), order=1),
RandomCropTransform((params.nested_get("image_size"), params.nested_get("image_size"))),
])
from delira.data_loading import BaseDataManager, SequentialSampler, RandomSampler
manager_test = BaseDataManager(dataset_test, params.nested_get("batch_size"),
transforms=transforms,
sampler_cls=SequentialSampler,
n_process_augmentation=1)
import warnings
warnings.simplefilter("ignore", UserWarning) # ignore UserWarnings raised by dependency code
warnings.simplefilter("ignore", FutureWarning) # ignore FutureWarnings raised by dependency code
### Setting path for loading best checkpoint
from delira.training.tf_trainer import tf_load_checkpoint
test_path = '/../checkpoint_best'
def train_shapenet():
"""
Trains a single shapenet with config file from comandline arguments
See Also
--------
:class:`delira.training.PyTorchNetworkTrainer`
"""
import logging
import numpy as np
import torch
from shapedata.single_shape import SingleShapeDataset
from delira.training import PyTorchNetworkTrainer
from ..utils import Config
from ..layer import HomogeneousShapeLayer
from ..networks import SingleShapeNetwork
from delira.logging import TrixiHandler
from trixi.logger import PytorchVisdomLogger
from delira.training.callbacks import ReduceLROnPlateauCallbackPyTorch
from delira.data_loading import BaseDataManager, RandomSampler, \
SequentialSampler
import os
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-c",
"--config",
type=str,
help="Path to configuration file")
parser.add_argument("-v", "--verbose", action="store_true")
args = parser.parse_args()
config = Config()
config_dict = config(os.path.abspath(args.config))
shapes = np.load(os.path.abspath(config_dict["layer"].pop(
"pca_path")))["shapes"][:config_dict["layer"].pop("num_shape_params") +
1]
# layer_cls = HomogeneousShapeLayer
net = SingleShapeNetwork(HomogeneousShapeLayer, {
"shapes": shapes,
**config_dict["layer"]
},
img_size=config_dict["data"]["img_size"],
**config_dict["network"])
num_params = 0
for param in net.parameters():
num_params += param.numel()
if args.verbose:
print("Number of Parameters: %d" % num_params)
criterions = {"L1": torch.nn.L1Loss()}
metrics = {"MSE": torch.nn.MSELoss()}
mixed_prec = config_dict["training"].pop("mixed_prec", False)
config_dict["training"]["save_path"] = os.path.abspath(
config_dict["training"]["save_path"])
trainer = PyTorchNetworkTrainer(
net,
criterions=criterions,
metrics=metrics,
lr_scheduler_cls=ReduceLROnPlateauCallbackPyTorch,
lr_scheduler_params=config_dict["scheduler"],
optimizer_cls=torch.optim.Adam,
optimizer_params=config_dict["optimizer"],
mixed_precision=mixed_prec,
**config_dict["training"])
if args.verbose:
print(trainer.input_device)
print("Load Data")
dset_train = SingleShapeDataset(
os.path.abspath(config_dict["data"]["train_path"]),
config_dict["data"]["img_size"],
config_dict["data"]["crop"],
config_dict["data"]["landmark_extension_train"],
cached=config_dict["data"]["cached"],
rotate=config_dict["data"]["rotate_train"],
random_offset=config_dict["data"]["offset_train"])
if config_dict["data"]["test_path"]:
dset_val = SingleShapeDataset(
os.path.abspath(config_dict["data"]["test_path"]),
config_dict["data"]["img_size"],
config_dict["data"]["crop"],
config_dict["data"]["landmark_extension_test"],
cached=config_dict["data"]["cached"],
rotate=config_dict["data"]["rotate_test"],
random_offset=config_dict["data"]["offset_test"])
else:
dset_val = None
mgr_train = BaseDataManager(
dset_train,
batch_size=config_dict["data"]["batch_size"],
n_process_augmentation=config_dict["data"]["num_workers"],
transforms=None,
sampler_cls=RandomSampler)
mgr_val = BaseDataManager(
dset_val,
batch_size=config_dict["data"]["batch_size"],
n_process_augmentation=config_dict["data"]["num_workers"],
transforms=None,
sampler_cls=SequentialSampler)
if args.verbose:
print("Data loaded")
if config_dict["logging"].pop("enable", False):
logger_cls = PytorchVisdomLogger
logging.basicConfig(
level=logging.INFO,
handlers=[TrixiHandler(logger_cls, **config_dict["logging"])])
else:
logging.basicConfig(level=logging.INFO,
handlers=[logging.NullHandler()])
logger = logging.getLogger("Test Logger")
logger.info("Start Training")
if args.verbose:
print("Start Training")
trainer.train(config_dict["training"]["num_epochs"],
mgr_train,
mgr_val,
config_dict["training"]["val_score_key"],
val_score_mode='lowest')
#####################
# Augmentation #
#####################
base_transforms = [ZeroMeanUnitVarianceTransform(),
]
train_transforms = [SpatialTransform(patch_size=(200, 200),
random_crop=False,
),
]
#####################
# Datamanagers #
#####################
manager_train = BaseDataManager(dataset_train, params.nested_get("batch_size"),
transforms=Compose(
base_transforms + train_transforms),
sampler_cls=RandomSampler,
n_process_augmentation=n_process_augmentation)
manager_val = BaseDataManager(dataset_val, params.nested_get("batch_size"),
transforms=Compose(base_transforms),
sampler_cls=SequentialSampler,
n_process_augmentation=n_process_augmentation)
manager_test = BaseDataManager(dataset_test, 1,
transforms=Compose(base_transforms),
sampler_cls=SequentialSampler,
n_process_augmentation=n_process_augmentation)
logger.info("Init Experiment")
experiment = PyTorchExperiment(params,
def test_experiment(self):
from delira.training import TfExperiment, Parameters
from delira.models.classification import ClassificationNetworkBaseTf
from delira.data_loading import AbstractDataset, BaseDataManager
import tensorflow as tf
test_cases = [(Parameters(
fixed_params={
"model": {
'in_channels': 32,
'n_outputs': 1
},
"training": {
"criterions": {
"CE": tf.losses.softmax_cross_entropy
},
"optimizer_cls": tf.train.AdamOptimizer,
"optimizer_params": {
"learning_rate": 1e-3
},
"num_epochs": 2,
"metrics": {},
"lr_sched_cls": None,
"lr_sched_params": {}
}
}), 500, 50)]
class DummyNetwork(ClassificationNetworkBaseTf):
def __init__(self):
super().__init__(32, 1)
self.model = self._build_model(1)
images = tf.placeholder(shape=[None, 32], dtype=tf.float32)
labels = tf.placeholder(shape=[None, 1], dtype=tf.float32)
preds_train = self.model(images, training=True)
preds_eval = self.model(images, training=False)
self.inputs = [images, labels]
self.outputs_train = [preds_train]
self.outputs_eval = [preds_eval]
@staticmethod
def _build_model(n_outputs):
return tf.keras.models.Sequential(layers=[
tf.keras.layers.Dense(64,
input_shape=(32, ),
bias_initializer='glorot_uniform'),
tf.keras.layers.ReLU(),
tf.keras.layers.Dense(n_outputs,
bias_initializer='glorot_uniform')
])
class DummyDataset(AbstractDataset):
def __init__(self, length):
super().__init__(None, None, None, None)
self.length = length
def __getitem__(self, index):
return {
"data": np.random.rand(32),
"label": np.random.randint(0, 1, 1)
}
def __len__(self):
return self.length
def get_sample_from_index(self, index):
return self.__getitem__(index)
for case in test_cases:
with self.subTest(case=case):
params, dataset_length_train, dataset_length_test = case
exp = TfExperiment(params, DummyNetwork)
dset_train = DummyDataset(dataset_length_train)
dset_test = DummyDataset(dataset_length_test)
dmgr_train = BaseDataManager(dset_train, 16, 4, None)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
net = exp.run(dmgr_train, dmgr_test)
exp.test(
params=params,
network=net,
datamgr_test=dmgr_test,
)
exp.kfold(2, dmgr_train, num_splits=2)
exp.stratified_kfold(2, dmgr_train, num_splits=2)
exp.stratified_kfold_predict(2, dmgr_train, num_splits=2)
transforms = Compose([
#SpatialTransform(patch_size=(1024, 1024), do_rotation=True, patch_center_dist_from_border=1024, border_mode_data='reflect',
# border_mode_seg='reflect', angle_x=(args.rot_angle, args.rot_angle), angle_y=(0, 0), angle_z=(0, 0),
# do_elastic_deform=False, order_data=1, order_seg=1)
ResizeTransform((int(args.resize_size), int(args.resize_size)), order=1),
RandomCropTransform(
(params.nested_get("image_size"), params.nested_get("image_size"))),
])
### Data manager
from delira.data_loading import BaseDataManager, SequentialSampler, RandomSampler
manager_train = BaseDataManager(dataset_train,
params.nested_get("batch_size"),
transforms=transforms,
sampler_cls=RandomSampler,
n_process_augmentation=4)
manager_val = BaseDataManager(dataset_val,
params.nested_get("val_batch_size"),
transforms=transforms,
sampler_cls=SequentialSampler,
n_process_augmentation=4)
import warnings
warnings.simplefilter(
"ignore", UserWarning) # ignore UserWarnings raised by dependency code
warnings.simplefilter(
"ignore", FutureWarning) # ignore FutureWarnings raised by dependency code
def test_experiment(self):
from delira.training import PyTorchExperiment, Parameters
from delira.training.callbacks import ReduceLROnPlateauCallbackPyTorch
from delira.models.classification import ClassificationNetworkBasePyTorch
from delira.data_loading import AbstractDataset, BaseDataManager
import torch
test_cases = [(Parameters(
fixed_params={
"model": {},
"training": {
"criterions": {
"CE": torch.nn.CrossEntropyLoss()
},
"optimizer_cls": torch.optim.Adam,
"optimizer_params": {
"lr": 1e-3
},
"num_epochs": 2,
"metrics": {},
"lr_sched_cls": ReduceLROnPlateauCallbackPyTorch,
"lr_sched_params": {}
}
}), 500, 50)]
class DummyNetwork(ClassificationNetworkBasePyTorch):
def __init__(self):
super().__init__(32, 1)
def forward(self, x):
return self.module(x)
@staticmethod
def _build_model(in_channels, n_outputs):
return torch.nn.Sequential(torch.nn.Linear(in_channels, 64),
torch.nn.ReLU(),
torch.nn.Linear(64, n_outputs))
@staticmethod
def prepare_batch(batch_dict, input_device, output_device):
return {
"data":
torch.from_numpy(batch_dict["data"]).to(
input_device, torch.float),
"label":
torch.from_numpy(batch_dict["label"]).to(
output_device, torch.long)
}
class DummyDataset(AbstractDataset):
def __init__(self, length):
super().__init__(None, None, None, None)
self.length = length
def __getitem__(self, index):
return {
"data": np.random.rand(32),
"label": np.random.randint(0, 1, 1)
}
def __len__(self):
return self.length
def get_sample_from_index(self, index):
return self.__getitem__(index)
for case in test_cases:
with self.subTest(case=case):
params, dataset_length_train, dataset_length_test = case
exp = PyTorchExperiment(params, DummyNetwork)
dset_train = DummyDataset(dataset_length_train)
dset_test = DummyDataset(dataset_length_test)
dmgr_train = BaseDataManager(dset_train, 16, 4, None)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
net = exp.run(dmgr_train, dmgr_test)
exp.test(
params=params,
network=net,
datamgr_test=dmgr_test,
)
exp.kfold(2, dmgr_train, num_splits=2)
exp.stratified_kfold(2, dmgr_train, num_splits=2)
exp.stratified_kfold_predict(2, dmgr_train, num_splits=2)
def setUp(self):
self.dset = DummyDataset(20)
self.dmgr = BaseDataManager(self.dset, 4, 1, transforms=None)
def run_experiment(cp: str, test=True) -> str:
"""
Run classification experiment on patches
Imports moved inside because of logging setups
Parameters
----------
ch : str
path to config file
test : bool
test best model on test set
Returns
-------
str
path to experiment folder
"""
# setup config
ch = ConfigHandlerPyTorchDelira(cp)
ch = feature_map_params(ch)
if 'mixed_precision' not in ch or ch['mixed_precision'] is None:
ch['mixed_precision'] = True
if 'debug_delira' in ch and ch['debug_delira'] is not None:
delira.set_debug_mode(ch['debug_delira'])
print("Debug mode active: settings n_process_augmentation to 1!")
ch['augment.n_process'] = 1
dset_keys = ['train', 'val', 'test']
losses = {'class_ce': torch.nn.CrossEntropyLoss()}
train_metrics = {}
val_metrics = {'CE': metric_wrapper_pytorch(torch.nn.CrossEntropyLoss())}
test_metrics = {'CE': metric_wrapper_pytorch(torch.nn.CrossEntropyLoss())}
#########################
# Setup Parameters #
#########################
params_dict = ch.get_params(losses=losses,
train_metrics=train_metrics,
val_metrics=val_metrics,
add_self=ch['add_config_to_params'])
params = Parameters(**params_dict)
#################
# Setup IO #
#################
# setup io
load_sample = load_pickle
load_fn = LoadPatches(load_fn=load_sample,
patch_size=ch['patch_size'],
**ch['data.load_patch'])
datasets = {}
for key in dset_keys:
p = os.path.join(ch["data.path"], str(key))
datasets[key] = BaseExtendCacheDataset(p,
load_fn=load_fn,
**ch['data.kwargs'])
#############################
# Setup Transformations #
#############################
base_transforms = []
base_transforms.append(PopKeys("mapping"))
train_transforms = []
if ch['augment.mode']:
logger.info("Training augmentation enabled.")
train_transforms.append(
SpatialTransform(patch_size=ch['patch_size'],
**ch['augment.kwargs']))
train_transforms.append(MirrorTransform(axes=(0, 1)))
process = ch['augment.n_process'] if 'augment.n_process' in ch else 1
#########################
# Setup Datamanagers #
#########################
datamanagers = {}
for key in dset_keys:
if key == 'train':
trafos = base_transforms + train_transforms
sampler = WeightedPrevalenceRandomSampler
else:
trafos = base_transforms
sampler = SequentialSampler
datamanagers[key] = BaseDataManager(
data=datasets[key],
batch_size=params.nested_get('batch_size'),
n_process_augmentation=process,
transforms=Compose(trafos),
sampler_cls=sampler,
)
#############################
# Initialize Experiment #
#############################
experiment = \
PyTorchExperiment(
params=params,
model_cls=ClassNetwork,
name=ch['exp.name'],
save_path=ch['exp.dir'],
optim_builder=create_optims_default_pytorch,
trainer_cls=PyTorchNetworkTrainer,
mixed_precision=ch['mixed_precision'],
mixed_precision_kwargs={'verbose': False},
key_mapping={"input_batch": "data"},
**ch['exp.kwargs'],
)
# save configurations
ch.dump(os.path.join(experiment.save_path, 'config.json'))
#################
# Training #
#################
model = experiment.run(datamanagers['train'],
datamanagers['val'],
save_path_exp=experiment.save_path,
ch=ch,
metric_keys={'val_CE': ['pred', 'label']},
val_freq=1,
verbose=True)
################
# Testing #
################
if test and datamanagers['test'] is not None:
# metrics and metric_keys are used differently than in original
# Delira implementation in order to support Evaluator
# see mscl.training.predictor
preds = experiment.test(
network=model,
test_data=datamanagers['test'],
metrics=test_metrics,
metric_keys={'CE': ['pred', 'label']},
verbose=True,
)
softmax_fn = metric_wrapper_pytorch(
partial(torch.nn.functional.softmax, dim=1))
preds = softmax_fn(preds[0]['pred'])
labels = [d['label'] for d in datasets['test']]
fpr, tpr, thresholds = roc_curve(labels, preds[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, label='ROC (AUC = %0.2f)' % roc_auc)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.savefig(os.path.join(experiment.save_path, 'test_roc.pdf'))
plt.close()
preds = experiment.test(
network=model,
test_data=datamanagers['val'],
metrics=test_metrics,
metric_keys={'CE': ['pred', 'label']},
verbose=True,
)
preds = softmax_fn(preds[0]['pred'])
labels = [d['label'] for d in datasets['val']]
fpr, tpr, thresholds = roc_curve(labels, preds[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, label='ROC (AUC = %0.2f)' % roc_auc)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.savefig(os.path.join(experiment.save_path, 'best_val_roc.pdf'))
plt.close()
return experiment.save_path