from delira.training import PyTorchExperiment, Parameters
from delira.models.classification import ClassificationNetworkBasePyTorch
from delira.data_loading import AbstractDataset, BaseDataManager
import torch
@pytest.mark.parametrize("params,dataset_length_train,dataset_length_test",
[(Parameters(
fixed_params={
"model": {},
"training": {
"criterions": {
"CE": torch.nn.CrossEntropyLoss()
},
"optimizer_cls": torch.optim.Adam,
"optimizer_params": {
"lr": 1e-3
},
"num_epochs": 2,
"metrics": {},
"lr_sched_cls": None,
"lr_sched_params": {}
}
}), 500, 50)])
def test_experiment(params, dataset_length_train, dataset_length_test):
class DummyNetwork(ClassificationNetworkBasePyTorch):
def __init__(self):
super().__init__(32, 1)
def forward(self, x):
return self.module(x)
def setUp(self) -> None:
test_cases_torch, test_cases_tf = [], []
from sklearn.metrics import mean_absolute_error
# setup torch testcases
if "TORCH" in get_backends():
import torch
from delira.models.classification import \
ClassificationNetworkBasePyTorch
from delira.training.callbacks import \
ReduceLROnPlateauCallbackPyTorch
class DummyNetworkTorch(ClassificationNetworkBasePyTorch):
def __init__(self):
super().__init__(32, 1)
def forward(self, x):
return {"pred": self.module(x)}
@staticmethod
def _build_model(in_channels, n_outputs):
return torch.nn.Sequential(
torch.nn.Linear(in_channels, 64),
torch.nn.ReLU(),
torch.nn.Linear(64, n_outputs)
)
@staticmethod
def prepare_batch(batch_dict, input_device, output_device):
return {"data": torch.from_numpy(batch_dict["data"]
).to(input_device,
torch.float),
"label": torch.from_numpy(batch_dict["label"]
).to(output_device,
torch.float)}
test_cases_torch.append((
Parameters(fixed_params={
"model": {},
"training": {
"losses": {"CE":
torch.nn.BCEWithLogitsLoss()},
"optimizer_cls": torch.optim.Adam,
"optimizer_params": {"lr": 1e-3},
"num_epochs": 2,
"val_metrics": {"val_mae": mean_absolute_error},
"lr_sched_cls": ReduceLROnPlateauCallbackPyTorch,
"lr_sched_params": {"mode": "min"}
}
}
),
500,
50,
"mae",
"lowest",
DummyNetworkTorch))
self._test_cases_torch = test_cases_torch
# setup tf tescases
if "TF" in get_backends():
from delira.models import ClassificationNetworkBaseTf, \
AbstractTfNetwork
import tensorflow as tf
class DummyNetworkTf(ClassificationNetworkBaseTf):
def __init__(self):
AbstractTfNetwork.__init__(self)
self.model = self._build_model(1)
images = tf.placeholder(shape=[None, 32],
dtype=tf.float32)
labels = tf.placeholder_with_default(
tf.zeros([tf.shape(images)[0], 1]), shape=[None, 1])
preds_train = self.model(images, training=True)
preds_eval = self.model(images, training=False)
self.inputs["images"] = images
self.inputs["labels"] = labels
self.outputs_train["pred"] = preds_train
self.outputs_eval["pred"] = preds_eval
@staticmethod
def _build_model(n_outputs):
return tf.keras.models.Sequential(
layers=[
tf.keras.layers.Dense(64, input_shape=(
32,), bias_initializer='glorot_uniform'),
tf.keras.layers.ReLU(),
tf.keras.layers.Dense(
n_outputs,
bias_initializer='glorot_uniform')]
)
test_cases_tf.append(
(
Parameters(fixed_params={
"model": {},
"training": {
"losses": {"CE":
tf.losses.softmax_cross_entropy},
"optimizer_cls": tf.train.AdamOptimizer,
"optimizer_params": {"learning_rate": 1e-3},
"num_epochs": 2,
"val_metrics": {"val_mae": mean_absolute_error},
"lr_sched_cls": None,
"lr_sched_params": {}}
}
),
500,
50,
DummyNetworkTf)
)
self._test_cases_tf = test_cases_tf
logger.info(self._testMethodName)
def run_experiment(cp: str, test=True) -> str:
"""
Run classification experiment on patches
Imports moved inside because of logging setups
Parameters
----------
ch : str
path to config file
test : bool
test best model on test set
Returns
-------
str
path to experiment folder
"""
# setup config
ch = ConfigHandlerPyTorchDelira(cp)
ch = feature_map_params(ch)
if 'mixed_precision' not in ch or ch['mixed_precision'] is None:
ch['mixed_precision'] = True
if 'debug_delira' in ch and ch['debug_delira'] is not None:
delira.set_debug_mode(ch['debug_delira'])
print("Debug mode active: settings n_process_augmentation to 1!")
ch['augment.n_process'] = 1
dset_keys = ['train', 'val', 'test']
losses = {'class_ce': torch.nn.CrossEntropyLoss()}
train_metrics = {}
val_metrics = {'CE': metric_wrapper_pytorch(torch.nn.CrossEntropyLoss())}
test_metrics = {'CE': metric_wrapper_pytorch(torch.nn.CrossEntropyLoss())}
#########################
# Setup Parameters #
#########################
params_dict = ch.get_params(losses=losses,
train_metrics=train_metrics,
val_metrics=val_metrics,
add_self=ch['add_config_to_params'])
params = Parameters(**params_dict)
#################
# Setup IO #
#################
# setup io
load_sample = load_pickle
load_fn = LoadPatches(load_fn=load_sample,
patch_size=ch['patch_size'],
**ch['data.load_patch'])
datasets = {}
for key in dset_keys:
p = os.path.join(ch["data.path"], str(key))
datasets[key] = BaseExtendCacheDataset(p,
load_fn=load_fn,
**ch['data.kwargs'])
#############################
# Setup Transformations #
#############################
base_transforms = []
base_transforms.append(PopKeys("mapping"))
train_transforms = []
if ch['augment.mode']:
logger.info("Training augmentation enabled.")
train_transforms.append(
SpatialTransform(patch_size=ch['patch_size'],
**ch['augment.kwargs']))
train_transforms.append(MirrorTransform(axes=(0, 1)))
process = ch['augment.n_process'] if 'augment.n_process' in ch else 1
#########################
# Setup Datamanagers #
#########################
datamanagers = {}
for key in dset_keys:
if key == 'train':
trafos = base_transforms + train_transforms
sampler = WeightedPrevalenceRandomSampler
else:
trafos = base_transforms
sampler = SequentialSampler
datamanagers[key] = BaseDataManager(
data=datasets[key],
batch_size=params.nested_get('batch_size'),
n_process_augmentation=process,
transforms=Compose(trafos),
sampler_cls=sampler,
)
#############################
# Initialize Experiment #
#############################
experiment = \
PyTorchExperiment(
params=params,
model_cls=ClassNetwork,
name=ch['exp.name'],
save_path=ch['exp.dir'],
optim_builder=create_optims_default_pytorch,
trainer_cls=PyTorchNetworkTrainer,
mixed_precision=ch['mixed_precision'],
mixed_precision_kwargs={'verbose': False},
key_mapping={"input_batch": "data"},
**ch['exp.kwargs'],
)
# save configurations
ch.dump(os.path.join(experiment.save_path, 'config.json'))
#################
# Training #
#################
model = experiment.run(datamanagers['train'],
datamanagers['val'],
save_path_exp=experiment.save_path,
ch=ch,
metric_keys={'val_CE': ['pred', 'label']},
val_freq=1,
verbose=True)
################
# Testing #
################
if test and datamanagers['test'] is not None:
# metrics and metric_keys are used differently than in original
# Delira implementation in order to support Evaluator
# see mscl.training.predictor
preds = experiment.test(
network=model,
test_data=datamanagers['test'],
metrics=test_metrics,
metric_keys={'CE': ['pred', 'label']},
verbose=True,
)
softmax_fn = metric_wrapper_pytorch(
partial(torch.nn.functional.softmax, dim=1))
preds = softmax_fn(preds[0]['pred'])
labels = [d['label'] for d in datasets['test']]
fpr, tpr, thresholds = roc_curve(labels, preds[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, label='ROC (AUC = %0.2f)' % roc_auc)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.savefig(os.path.join(experiment.save_path, 'test_roc.pdf'))
plt.close()
preds = experiment.test(
network=model,
test_data=datamanagers['val'],
metrics=test_metrics,
metric_keys={'CE': ['pred', 'label']},
verbose=True,
)
preds = softmax_fn(preds[0]['pred'])
labels = [d['label'] for d in datasets['val']]
fpr, tpr, thresholds = roc_curve(labels, preds[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, label='ROC (AUC = %0.2f)' % roc_auc)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.savefig(os.path.join(experiment.save_path, 'best_val_roc.pdf'))
plt.close()
return experiment.save_path
params = Parameters(fixed_params={
"model": {
"image_size": args.image_size,
"output_size": args.output_size,
"input_c_dim": args.input_c_dim,
"output_c_dim": args.output_c_dim,
"df_dim": args.df_dim,
"gf_dim": args.gf_dim,
"f_size": args.f_size,
"strides": args.strides,
"l1_lambda": args.l1_lambda
},
"training": {
"batch_size": args.batch_size,
"num_epochs": args.num_epochs,
"optimizer_cls": tf.train.AdamOptimizer, # optimization algorithm to use
"optimizer_params": {'learning_rate': args.learning_rate,
#'momentum': 0.5
'beta1': 0.5,
#'beta2': 0.8
},
"criterions": {'CE': tf.losses.sigmoid_cross_entropy}, # the original loss :tf.nn.sigmoid_cross_entropy_with_logits
"lr_sched_cls": None, # the learning rate scheduling algorithm to use
"lr_sched_params": {}, # the corresponding initialization parameters
"metrics": {'LPIPS': lpips_score([None, 3, args.image_size, args.image_size])}, # and some evaluation metrics
"dataset_name": args.dataset_name,
"val_dataset": args.val_dataset,
"exp_name": args.exp_name
}
})
def test_parameters(self):
def to_lookup_config(dictionary):
tmp = LookupConfig()
tmp.update(dictionary)
return tmp
test_cases = [({
"a": 1,
"b": [1, 2],
"c": {
"d": 3,
"e": 56
}
}, {
"f": 1,
"g": {
"h": {
"i": {
"a": 3
}
}
}
}, {
"j": 1,
"k": 2
}, {}, "e", 56, "a", "q")]
for case in test_cases:
with self.subTest(case=case):
fixed_model_params, variable_model_params, \
fixed_training_params, variable_training_params, \
valid_nested_key, valid_nested_value, doubled_key, \
invalid_key = case
fixed_model_params = to_lookup_config(fixed_model_params)
variable_model_params = to_lookup_config(variable_model_params)
fixed_training_params = to_lookup_config(fixed_training_params)
variable_training_params = to_lookup_config(
variable_training_params)
params = Parameters(fixed_params={
"model": fixed_model_params,
"training": fixed_training_params
},
variable_params={
"model": variable_model_params,
"training": variable_training_params
})
self.assertFalse(params.training_on_top)
self.assertTrue(params.variability_on_top)
self.assertEqual(
params.fixed,
to_lookup_config({
"model": fixed_model_params,
"training": fixed_training_params
}))
self.assertEqual(
params.variable,
to_lookup_config({
"model": variable_model_params,
"training": variable_training_params
}))
params = params.permute_training_on_top()
self.assertFalse(params.variability_on_top)
self.assertTrue(params.training_on_top)
self.assertEqual(
params.model,
to_lookup_config({
"fixed": fixed_model_params,
"variable": variable_model_params
}))
self.assertEqual(
params.training,
to_lookup_config({
"fixed": fixed_training_params,
"variable": variable_training_params
}))
params_copy = params.deepcopy()
params = params.permute_variability_on_top(
).permute_training_on_top()
self.assertEqual(params_copy, params)
self.assertEqual(params.nested_get(valid_nested_key),
valid_nested_value)
with self.assertRaises(KeyError):
params.nested_get(doubled_key)
with self.assertRaises(KeyError):
params.nested_get(invalid_key)
self.assertEqual("default",
params.nested_get(invalid_key, "default"))
self.assertEqual(
"default", params.nested_get(invalid_key,
default="default"))
params = Parameters(
fixed_params={
"model": {
"image_size": int(args.image_size),
"input_c_dim": args.input_c_dim,
"output_c_dim": args.output_c_dim,
"df_dim": args.df_dim,
"gf_dim": args.gf_dim,
"f_size": args.f_size,
"strides": args.strides,
"l1_lambda": args.l1_lambda
},
"training": {
"batch_size": args.batch_size,
"val_batch_size": args.val_batch_size,
"num_epochs": args.num_epochs,
"optimizer_cls":
tf.train.AdamOptimizer, # optimization algorithm to use
"optimizer_params": {
'learning_rate': args.learning_rate,
#'momentum': 0.5
'beta1': 0.5,
#'beta2': 0.8
},
"criterions": {
'CE': tf.losses.sigmoid_cross_entropy
},
"lr_sched_cls":
None, # the learning rate scheduling algorithm to use
"lr_sched_params":
{}, # the corresponding initialization parameters
"metrics": {
'LPIPS': lpips_score(
[None, 3, args.image_size, args.image_size])
}, # and some evaluation metrics
"dataset_name": args.dataset_name,
"val_dataset": args.val_dataset,
"exp_name": args.exp_name
}
})
#please check logger status
params = Parameters(
fixed_params={
"model": {
"in_channels": 1,
"num_classes": 26
},
"training": {
"batch_size": 64, # batchsize to use
"num_epochs": 10, # number of epochs to train
"optimizer_cls": torch.optim.Adam, # optimization algorithm to use
"optimizer_params": {
'lr': 1e-3
}, # initialization parameters for this algorithm
"losses": {
"CE": torch.nn.CrossEntropyLoss()
}, # the loss function
"lr_sched_cls":
None, # the learning rate scheduling algorithm to use
"lr_sched_params":
{}, # the corresponding initialization parameters
"criterions": accuracy_score,
"metrics": {
'img': {
'l1': accuracy_score
}
} # and some evaluation metrics
}
})
import SimpleITK as sitk
seed = 0
val_score_key = "val_score_key"
val_score_mode = "val_score_mode"
n_process_augmentation = ch['n_process_augmentation']
# Parameters
losses = {"CE": torch.nn.CrossEntropyLoss()}
params = Parameters(fixed_params={
"model": {
"in_channels": 1,
"n_outputs": 10
},
"training": {
"batch_size": 64, # batchsize to use
"num_epochs": 10, # number of epochs to train
"optimizer_cls": torch.optim.Adam, # optimization algorithm to use
# initialization parameters for this algorithm
"optimizer_params": {'lr': 1e-3},
"losses": losses, # the loss function
"lr_sched_cls": None, # the learning rate scheduling algorithm to use
"lr_sched_params": {}, # the corresponding initialization parameters
"metrics": {} # and some evaluation metrics
}
})
#################
# Datasets #
#################
dset = TorchvisionClassificationDataset("mnist", # which dataset to use
train=True, # use trainset
# resample to 224 x 224 pixels
params = Parameters(
fixed_params={
"model": {
"image_size": args.image_size,
"input_c_dim": args.input_c_dim,
"output_c_dim": args.output_c_dim,
"df_dim": args.df_dim,
"gf_dim": args.gf_dim,
"f_size": args.f_size,
"latent_dim": int(args.latent_dim),
"n_latent": int(args.n_latent),
"coeff_reconstruct": float(args.coeff_reconstruct),
"coeff_ds": int(args.coeff_ds),
"is_batchnorm": args.is_batchnorm,
#"d_layers": args.d_layers
},
"training": {
"batch_size": args.batch_size,
"val_batch_size": args.val_batch_size,
"num_epochs": args.num_epochs,
"optimizer_cls":
tf.train.AdamOptimizer, # optimization algorithm to use
"optimizer_params": {
'learning_rate': args.learning_rate,
'beta1': 0.5,
'beta2': 0.999
},
"criterions": {
'L1': tf.losses.absolute_difference,
'L2': tf.losses.mean_squared_error,
},
"lr_sched_cls":
None, # the learning rate scheduling algorithm to use
"lr_sched_params":
{}, # the corresponding initialization parameters
"metrics": {
'LPIPS': lpips_score(
[None, 3, args.image_size, args.image_size])
}, # evaluation metrics
"dataset_name": args.dataset_name,
"val_dataset": args.val_dataset,
"exp_name": args.exp_name
}
})
params = Parameters(fixed_params={
"model": {
"image_size": args.image_size,
"input_c_dim": args.input_c_dim,
"output_c_dim": args.output_c_dim,
"df_dim": args.df_dim,
"gf_dim": args.gf_dim,
"f_size": args.f_size,
"latent_dim": int(args.latent_dim),
"n_latent": int(args.n_latent),
"coeff_gan": float(args.coeff_gan),
"coeff_vae": float(args.coeff_vae),
"coeff_reconstruct": float(args.coeff_reconstruct),
"coeff_kl": float(args.coeff_kl),
"coeff_latent": float(args.coeff_latent),
"coeff_ds": int(args.coeff_ds),
"is_batchnorm": args.is_batchnorm,
"with_DS": args.with_DS,
"is_condD": args.is_condD
#"d_layers": args.d_layers # Could be refactored later for number of downsampling layers in discriminator
},
"training": {
"batch_size": args.batch_size, # batchsize to use
"val_batch_size": args.val_batch_size, # batchsize to use for validation
"num_epochs": args.num_epochs, # number of epochs to train
"optimizer_cls": tf.train.AdamOptimizer, # optimization algorithm to use
"optimizer_params": {'learning_rate': args.learning_rate,
'beta1': 0.5,
'beta2': 0.999
},
"criterions": {'L1': tf.losses.absolute_difference,
'L2': tf.losses.mean_squared_error,
},
"lr_sched_cls": None, # the learning rate scheduling algorithm to use
"lr_sched_params": {}, # the corresponding initialization parameters
"metrics": {'LPIPS': lpips_score([None, 3, args.image_size, args.image_size])}, # evaluation metrics
"dataset_name": args.dataset_name,
"val_dataset": args.val_dataset,
"exp_name": args.exp_name
}
})
def test_experiment(self):
from delira.training import PyTorchExperiment, Parameters
from delira.training.callbacks import ReduceLROnPlateauCallbackPyTorch
from delira.models.classification import ClassificationNetworkBasePyTorch
from delira.data_loading import AbstractDataset, BaseDataManager
import torch
test_cases = [(Parameters(
fixed_params={
"model": {},
"training": {
"criterions": {
"CE": torch.nn.CrossEntropyLoss()
},
"optimizer_cls": torch.optim.Adam,
"optimizer_params": {
"lr": 1e-3
},
"num_epochs": 2,
"metrics": {},
"lr_sched_cls": ReduceLROnPlateauCallbackPyTorch,
"lr_sched_params": {}
}
}), 500, 50)]
class DummyNetwork(ClassificationNetworkBasePyTorch):
def __init__(self):
super().__init__(32, 1)
def forward(self, x):
return self.module(x)
@staticmethod
def _build_model(in_channels, n_outputs):
return torch.nn.Sequential(torch.nn.Linear(in_channels, 64),
torch.nn.ReLU(),
torch.nn.Linear(64, n_outputs))
@staticmethod
def prepare_batch(batch_dict, input_device, output_device):
return {
"data":
torch.from_numpy(batch_dict["data"]).to(
input_device, torch.float),
"label":
torch.from_numpy(batch_dict["label"]).to(
output_device, torch.long)
}
class DummyDataset(AbstractDataset):
def __init__(self, length):
super().__init__(None, None, None, None)
self.length = length
def __getitem__(self, index):
return {
"data": np.random.rand(32),
"label": np.random.randint(0, 1, 1)
}
def __len__(self):
return self.length
def get_sample_from_index(self, index):
return self.__getitem__(index)
for case in test_cases:
with self.subTest(case=case):
params, dataset_length_train, dataset_length_test = case
exp = PyTorchExperiment(params, DummyNetwork)
dset_train = DummyDataset(dataset_length_train)
dset_test = DummyDataset(dataset_length_test)
dmgr_train = BaseDataManager(dset_train, 16, 4, None)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
net = exp.run(dmgr_train, dmgr_test)
exp.test(
params=params,
network=net,
datamgr_test=dmgr_test,
)
exp.kfold(2, dmgr_train, num_splits=2)
exp.stratified_kfold(2, dmgr_train, num_splits=2)
exp.stratified_kfold_predict(2, dmgr_train, num_splits=2)
def test_experiment(self):
from delira.training import TfExperiment, Parameters
from delira.models.classification import ClassificationNetworkBaseTf
from delira.data_loading import AbstractDataset, BaseDataManager
import tensorflow as tf
test_cases = [(Parameters(
fixed_params={
"model": {
'in_channels': 32,
'n_outputs': 1
},
"training": {
"criterions": {
"CE": tf.losses.softmax_cross_entropy
},
"optimizer_cls": tf.train.AdamOptimizer,
"optimizer_params": {
"learning_rate": 1e-3
},
"num_epochs": 2,
"metrics": {},
"lr_sched_cls": None,
"lr_sched_params": {}
}
}), 500, 50)]
class DummyNetwork(ClassificationNetworkBaseTf):
def __init__(self):
super().__init__(32, 1)
self.model = self._build_model(1)
images = tf.placeholder(shape=[None, 32], dtype=tf.float32)
labels = tf.placeholder(shape=[None, 1], dtype=tf.float32)
preds_train = self.model(images, training=True)
preds_eval = self.model(images, training=False)
self.inputs = [images, labels]
self.outputs_train = [preds_train]
self.outputs_eval = [preds_eval]
@staticmethod
def _build_model(n_outputs):
return tf.keras.models.Sequential(layers=[
tf.keras.layers.Dense(64,
input_shape=(32, ),
bias_initializer='glorot_uniform'),
tf.keras.layers.ReLU(),
tf.keras.layers.Dense(n_outputs,
bias_initializer='glorot_uniform')
])
class DummyDataset(AbstractDataset):
def __init__(self, length):
super().__init__(None, None, None, None)
self.length = length
def __getitem__(self, index):
return {
"data": np.random.rand(32),
"label": np.random.randint(0, 1, 1)
}
def __len__(self):
return self.length
def get_sample_from_index(self, index):
return self.__getitem__(index)
for case in test_cases:
with self.subTest(case=case):
params, dataset_length_train, dataset_length_test = case
exp = TfExperiment(params, DummyNetwork)
dset_train = DummyDataset(dataset_length_train)
dset_test = DummyDataset(dataset_length_test)
dmgr_train = BaseDataManager(dset_train, 16, 4, None)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
net = exp.run(dmgr_train, dmgr_test)
exp.test(
params=params,
network=net,
datamgr_test=dmgr_test,
)
exp.kfold(2, dmgr_train, num_splits=2)
exp.stratified_kfold(2, dmgr_train, num_splits=2)
exp.stratified_kfold_predict(2, dmgr_train, num_splits=2)
def test_parameters(fixed_model_params, variable_model_params,
fixed_training_params, variable_training_params,
valid_nested_key, valid_nested_value, doubled_key,
invalid_key):
def to_lookup_config(dictionary):
tmp = LookupConfig()
tmp.update(dictionary)
return tmp
fixed_model_params = to_lookup_config(fixed_model_params)
variable_model_params = to_lookup_config(variable_model_params)
fixed_training_params = to_lookup_config(fixed_training_params)
variable_training_params = to_lookup_config(variable_training_params)
params = Parameters(fixed_params={
"model": fixed_model_params,
"training": fixed_training_params
},
variable_params={
"model": variable_model_params,
"training": variable_training_params
})
assert params.training_on_top == False
assert params.variability_on_top
assert params.fixed == to_lookup_config({
"model": fixed_model_params,
"training": fixed_training_params
})
assert params.variable == to_lookup_config({
"model":
variable_model_params,
"training":
variable_training_params
})
params = params.permute_training_on_top()
assert params.variability_on_top == False
assert params.training_on_top
print(
params.model.difference_config(
to_lookup_config({
"fixed": fixed_model_params,
"variable": variable_model_params
})))
assert params.model == to_lookup_config({
"fixed": fixed_model_params,
"variable": variable_model_params
})
assert params.training == to_lookup_config({
"fixed":
fixed_training_params,
"variable":
variable_training_params
})
params_copy = params.deepcopy()
params = params.permute_variability_on_top().permute_training_on_top()
assert params_copy == params
assert params.nested_get(valid_nested_key) == valid_nested_value
try:
params.nested_get(doubled_key)
assert False
except KeyError:
assert True
try:
params.nested_get(invalid_key)
assert False
except KeyError:
assert True
assert "default" == params.nested_get(invalid_key, "default")
assert "default" == params.nested_get(invalid_key, default="default")
def main(args):
########################################
# #
# DEFINE THE HYPERPARAMETERS #
# #
########################################
# load settings from config file
config_file = args.get("config_file")
config_handler = ConfigHandler()
config_dict = config_handler(config_file)
# some are changed rarely and given manually if required
train_size = args.get("train_size")
val_size = args.get("val_size")
margin = args.get("margin")
optimizer = args.get("optimizer")
if optimizer != "SGD" and optimizer != "Adam":
ValueError("Invalid optimizer")
elif optimizer == "Adam":
optimizer_cls = torch.optim.Adam
else:
optimizer_cls = torch.optim.SGD
params = Parameters(
fixed_params={
"model": config_dict["model"],
"training": {
**config_dict["training"],
"optimizer_cls": optimizer_cls,
**config_dict["optimizer"],
"criterions": {
"FocalLoss": losses.FocalLoss(),
"SmoothL1Loss": losses.SmoothL1Loss()
},
# "criterions": {"FocalMSELoss": losses.FocalMSELoss(),
# "SmoothL1Loss": losses.SmoothL1Loss()},
# "lr_sched_cls": ReduceLROnPlateauCallbackPyTorch,
# "lr_sched_params": {"verbose": True},
"lr_sched_cls": None,
"lr_sched_params": {},
"metrics": {}
}
})
########################################
# #
# DEFINE THE AUGMENTATIONS #
# #
########################################
my_transforms = []
mirror_transform = MirrorTransform(axes=(1, 2))
my_transforms.append(mirror_transform)
crop_size = config_dict["data"]["crop_size"]
img_shape = config_dict["data"]["img_shape"]
shape_limit = config_dict["data"]["shape_limit"]
if (crop_size is not None and crop_size[0] == crop_size[1]) or \
(img_shape is not None and len(img_shape) > 1
and img_shape[0] == img_shape[1]):
rot_transform = Rot90Transform(axes=(0, 1), p_per_sample=0.5)
my_transforms.append(rot_transform)
else:
rot_transform = Rot90Transform(axes=(0, 1),
num_rot=(0, 2),
p_per_sample=0.5)
my_transforms.append(rot_transform)
# apply a more extended augmentation (if desiered)
if "ext_aug" in config_dict["data"].keys() and \
config_dict["data"]["ext_aug"] is not None and \
config_dict["data"]["ext_aug"]:
if crop_size is not None:
size = [crop_size[0] + 25, crop_size[1] + 25]
elif img_shape is not None:
size = [img_shape[0] + 5, img_shape[1] + 5]
elif shape_limit is not None:
size = [shape_limit[0] + 5, shape_limit[1] + 5]
else:
raise KeyError("Crop size or image shape requried!")
if crop_size is not None:
spatial_transforms = SpatialTransform([size[0] - 25, size[1] - 25],
np.asarray(size) // 2,
do_elastic_deform=False,
do_rotation=True,
angle_x=(0, 0.01 * np.pi),
do_scale=True,
scale=(0.9, 1.1),
random_crop=True,
border_mode_data="mirror",
border_mode_seg="mirror")
my_transforms.append(spatial_transforms)
elif img_shape is not None or shape_limit is not None:
spatial_transforms = SpatialTransform(
[size[0] - 5, size[1] - 5],
np.asarray(size) // 2,
do_elastic_deform=False,
do_rotation=False,
#angle_x=(0, 0.01 * np.pi),
do_scale=True,
scale=(0.9, 1.1),
random_crop=True,
border_mode_data="constant",
border_mode_seg="nearest")
my_transforms.append(spatial_transforms)
# bbox generation
bb_transform = ConvertSegToBB(dim=2, margin=margin)
my_transforms.append(bb_transform)
transforms = Compose(my_transforms)
########################################
# #
# DEFINE THE DATASETS and MANAGER #
# #
########################################
# paths to csv files containing labels (and other information)
csv_calc_train = '/home/temp/moriz/data/' \
'calc_case_description_train_set.csv'
csv_mass_train = '/home/temp/moriz/data/' \
'mass_case_description_train_set.csv'
# path to data directory
ddsm_dir = '/home/temp/moriz/data/CBIS-DDSM/'
# path to data directory
inbreast_dir = '/images/Mammography/INbreast/AllDICOMs/'
# paths to csv files containing labels (and other information)
xls_file = '/images/Mammography/INbreast/INbreast.xls'
# determine class and load function
if config_dict["data"]["dataset_type"] == "INbreast":
dataset_cls = CacheINbreastDataset
data_dir = inbreast_dir
csv_file = None
if config_dict["data"]["level"] == "crops":
load_fn = inbreast_utils.load_pos_crops
elif config_dict["data"]["level"] == "images":
load_fn = inbreast_utils.load_sample
elif config_dict["data"]["level"] == "both":
#TODO: fix
load_fn = inbreast_utils.load_sample_and_crops
else:
raise TypeError("Level required!")
elif config_dict["data"]["dataset_type"] == "DDSM":
data_dir = ddsm_dir
if config_dict["data"]["level"] == "crops":
load_fn = ddsm_utils.load_pos_crops
elif config_dict["data"]["level"] == "images":
load_fn = ddsm_utils.load_sample
elif config_dict["data"]["level"] == "images+":
load_fn = ddsm_utils.load_sample_with_crops
else:
raise TypeError("Level required!")
if config_dict["data"]["type"] == "mass":
csv_file = csv_mass_train
elif config_dict["data"]["type"] == "calc":
csv_file = csv_calc_train
elif config_dict["data"]["type"] == "both":
raise NotImplementedError("Todo")
else:
raise TypeError("Unknown lesion type!")
if "mode" in config_dict["data"].keys():
if config_dict["data"]["mode"] == "lazy":
dataset_cls = LazyDDSMDataset
if config_dict["data"]["level"] == "crops":
load_fn = ddsm_utils.load_single_pos_crops
elif config_dict["data"]["mode"] == "cache":
dataset_cls = CacheDDSMDataset
else:
raise TypeError("Unsupported loading mode!")
else:
dataset_cls = CacheDDSMDataset
else:
raise TypeError("Dataset is not supported!")
dataset_train_dict = {
'data_path': data_dir,
'xls_file': xls_file,
'csv_file': csv_file,
'load_fn': load_fn,
'num_elements': config_dict["debug"]["n_train"],
**config_dict["data"]
}
dataset_val_dict = {
'data_path': data_dir,
'xls_file': xls_file,
'csv_file': csv_file,
'load_fn': load_fn,
'num_elements': config_dict["debug"]["n_val"],
**config_dict["data"]
}
datamgr_train_dict = {
'batch_size': params.nested_get("batch_size"),
'n_process_augmentation': 4,
'transforms': transforms,
'sampler_cls': RandomSampler,
'data_loader_cls': BaseDataLoader
}
datamgr_val_dict = {
'batch_size': params.nested_get("batch_size"),
'n_process_augmentation': 4,
'transforms': transforms,
'sampler_cls': SequentialSampler,
'data_loader_cls': BaseDataLoader
}
########################################
# #
# INITIALIZE THE ACTUAL EXPERIMENT #
# #
########################################
checkpoint_path = config_dict["checkpoint_path"]["path"]
# if "checkpoint_path" in args and args["checkpoint_path"] is not None:
# checkpoint_path = args.get("checkpoint_path")
experiment = \
RetinaNetExperiment(params,
RetinaNet,
name = config_dict["logging"]["name"],
save_path = checkpoint_path,
dataset_cls=dataset_cls,
dataset_train_kwargs=dataset_train_dict,
datamgr_train_kwargs=datamgr_train_dict,
dataset_val_kwargs=dataset_val_dict,
datamgr_val_kwargs=datamgr_val_dict,
optim_builder=create_optims_default_pytorch,
gpu_ids=list(range(args.get('gpus'))),
val_score_key="val_FocalLoss",
val_score_mode="lowest",
checkpoint_freq=2)
########################################
# #
# LOGGING DEFINITION AND CONFIGURATION #
# #
########################################
logger_kwargs = config_dict["logging"]
# setup initial logging
log_file = os.path.join(experiment.save_path, 'logger.log')
logging.basicConfig(level=logging.INFO,
handlers=[
TrixiHandler(PytorchVisdomLogger,
**config_dict["logging"]),
logging.StreamHandler(),
logging.FileHandler(log_file)
])
logger = logging.getLogger("RetinaNet Logger")
with open(experiment.save_path + "/config.yml", 'w') as file:
yaml.dump(config_dict, file)
########################################
# #
# LOAD PATHS AND EXECUTE MODEL #
# #
########################################
seed = config_dict["data"]["seed"]
if "train_size" in config_dict["data"].keys():
train_size = config_dict["data"]["train_size"]
if "val_size" in config_dict["data"].keys():
val_size = config_dict["data"]["val_size"]
if config_dict["data"]["dataset_type"] == "INbreast":
if not config_dict["kfold"]["enable"]:
train_paths, _, val_paths = \
inbreast_utils.load_single_set(inbreast_dir,
xls_file=xls_file,
train_size=train_size,
val_size=val_size,
type=config_dict["data"]["type"],
random_state=seed)
if img_shape is not None or crop_size is not None:
experiment.run(train_paths, val_paths)
else:
experiment.run(train_paths, None)
else:
paths = inbreast_utils.get_paths(inbreast_dir,
xls_file=xls_file,
type=config_dict["data"]["type"])
if "splits" in config_dict["kfold"].keys():
num_splits = config_dict["kfold"]["splits"]
else:
num_splits = 5
experiment.kfold(paths,
num_splits=num_splits,
random_seed=seed,
dataset_type="INbreast")
else:
train_paths, val_paths, _ = \
ddsm_utils.load_single_set(ddsm_dir,
csv_file=csv_file,
train_size=train_size,
val_size=None,
random_state=seed)
if img_shape is not None or crop_size is not None:
experiment.run(train_paths, val_paths)
else:
experiment.run(train_paths, None)
def train(model_cls,
model_kwargs: dict,
outpath: str,
data_path,
exp_name=None,
batchsize=64,
num_epochs=1500,
checkpoint_freq=10,
additional_losses: dict = None,
dset_type="mnist",
key_mapping=None,
create_optim_fn=None):
if exp_name is None:
exp_name = model_cls.__name__
if additional_losses is None:
additional_losses = {}
if create_optim_fn is None:
create_optim_fn = create_optims
outpath = os.path.expanduser(outpath)
losses = {"adversarial": AdversarialLoss()}
losses.update(additional_losses)
params = Parameters(
fixed_params={
"model": {
**model_kwargs
},
"training": {
"num_epochs": num_epochs,
"batchsize": batchsize,
"losses": losses,
"val_metrics": {},
"optimizer_cls": torch.optim.Adam,
"optimizer_params": {
"lr": 0.001,
"betas": (0.5, 0.9995)
},
"scheduler_cls": None,
"scheduler_params": {}
}
})
data = setup_data(data_path, params.nested_get("batchsize"), 4,
RangeTransform(), RangeTransform(), dset_type)
exp = PyTorchExperiment(params,
model_cls,
params.nested_get("num_epochs"),
name=exp_name,
save_path=outpath,
key_mapping=key_mapping,
optim_builder=create_optim_fn,
checkpoint_freq=checkpoint_freq,
gpu_ids=[0])
model = exp.run(data["train"], data["val"])
weight_dir = os.path.join(exp.save_path, "checkpoints", "run_00")
return model, weight_dir