class TfExperimentTest(unittest.TestCase):
@unittest.skipIf("TF" not in delira.get_backends(),
reason="No TF Backend installed")
def test_experiment(self):
from delira.training import TfExperiment, Parameters
from delira.models.classification import ClassificationNetworkBaseTf
from delira.data_loading import AbstractDataset, BaseDataManager
import tensorflow as tf
test_cases = [(Parameters(
fixed_params={
"model": {
'in_channels': 32,
'n_outputs': 1
},
"training": {
"criterions": {
"CE": tf.losses.softmax_cross_entropy
},
"optimizer_cls": tf.train.AdamOptimizer,
"optimizer_params": {
"learning_rate": 1e-3
},
"num_epochs": 2,
"metrics": {},
"lr_sched_cls": None,
"lr_sched_params": {}
}
}), 500, 50)]
class DummyNetwork(ClassificationNetworkBaseTf):
def __init__(self):
super().__init__(32, 1)
self.model = self._build_model(1)
images = tf.placeholder(shape=[None, 32], dtype=tf.float32)
labels = tf.placeholder(shape=[None, 1], dtype=tf.float32)
preds_train = self.model(images, training=True)
preds_eval = self.model(images, training=False)
self.inputs = [images, labels]
self.outputs_train = [preds_train]
self.outputs_eval = [preds_eval]
@staticmethod
def _build_model(n_outputs):
return tf.keras.models.Sequential(layers=[
tf.keras.layers.Dense(64,
input_shape=(32, ),
bias_initializer='glorot_uniform'),
tf.keras.layers.ReLU(),
tf.keras.layers.Dense(n_outputs,
bias_initializer='glorot_uniform')
])
class DummyDataset(AbstractDataset):
def __init__(self, length):
super().__init__(None, None, None, None)
self.length = length
def __getitem__(self, index):
return {
"data": np.random.rand(32),
"label": np.random.randint(0, 1, 1)
}
def __len__(self):
return self.length
def get_sample_from_index(self, index):
return self.__getitem__(index)
for case in test_cases:
with self.subTest(case=case):
params, dataset_length_train, dataset_length_test = case
exp = TfExperiment(params, DummyNetwork)
dset_train = DummyDataset(dataset_length_train)
dset_test = DummyDataset(dataset_length_test)
dmgr_train = BaseDataManager(dset_train, 16, 4, None)
dmgr_test = BaseDataManager(dset_test, 16, 1, None)
net = exp.run(dmgr_train, dmgr_test)
exp.test(
params=params,
network=net,
datamgr_test=dmgr_test,
)
exp.kfold(2, dmgr_train, num_splits=2)
exp.stratified_kfold(2, dmgr_train, num_splits=2)
exp.stratified_kfold_predict(2, dmgr_train, num_splits=2)
from delira import get_backends
if "TORCH" in get_backends():
import torch
import torch.nn.functional as F
class BCEFocalLossPyTorch(torch.nn.Module):
"""
Focal loss for binary case without(!) logit
"""
def __init__(self, alpha=None, gamma=2, reduction='elementwise_mean'):
"""
Implements Focal Loss for binary class case
Parameters
----------
alpha : float
alpha has to be in range [0,1], assigns class weight
gamma : float
focusing parameter
reduction : str
Specifies the reduction to apply to the output: ‘none’ |
‘elementwise_mean’ | ‘sum’.
‘none’: no reduction will be
applied,
‘elementwise_mean’: the sum of the output will be divided
by the number of elements in the output,
‘sum’: the output will be summed
(further information about parameters above can be found in pytorch
documentation)
from delira import get_backends
from delira.training.callbacks.abstract_callback import AbstractCallback
if 'TORCH' in get_backends():
from torch.optim.lr_scheduler import ReduceLROnPlateau, \
CosineAnnealingLR, ExponentialLR, LambdaLR, MultiStepLR, StepLR
class DefaultPyTorchSchedulerCallback(AbstractCallback):
"""
Implements a Callback, which `at_epoch_end` function is suitable for
most schedulers
"""
def __init__(self, *args, **kwargs):
"""
Parameters
----------
*args :
Arbitrary Positional Arguments
**kwargs :
Arbitrary Keyword Arguments
"""
super().__init__()
self.scheduler = None
def at_epoch_end(self, trainer, **kwargs):
"""
Executes a single scheduling step
class TorchModelTest(unittest.TestCase):
@unittest.skipIf((virtual_memory().total / 1024.**3) < 20,
reason="Less than 20GB of memory")
@unittest.skipIf("TORCH" not in get_backends(),
reason="torch backend not installed")
def test_pytorch_model_default(self):
from delira.models import UNet2dPyTorch, \
UNet3dPyTorch, ClassificationNetworkBasePyTorch, \
VGG3DClassificationNetworkPyTorch, \
GenerativeAdversarialNetworkBasePyTorch
from delira.training.train_utils import \
create_optims_default_pytorch, create_optims_gan_default_pytorch
from delira.utils.context_managers import DefaultOptimWrapperTorch
import torch
test_cases = [
# UNet 2D
(
UNet2dPyTorch(5, in_channels=1), # model
(1, 32, 32), # input shape
(32, 32), # output shape
{
"loss_fn": torch.nn.CrossEntropyLoss()
}, # loss function
create_optims_default_pytorch, # optim_fn
4, # output range (num_classes -1)
True # half precision
),
# UNet 3D
(
UNet3dPyTorch(5, in_channels=1), # model
(1, 32, 32, 32), # input shape
(32, 32, 32), # output shape
{
"loss_fn": torch.nn.CrossEntropyLoss()
}, # loss function
create_optims_default_pytorch, # optim_fn
4, # output range (num_classes) - 1
True # half precision
),
# Base Classifier (Resnet 18)
(
ClassificationNetworkBasePyTorch(1, 10),
# model
(1, 224, 224), # input shape
9, # output shape (num_classes - 1)
{
"loss_fn": torch.nn.CrossEntropyLoss()
}, # loss function
create_optims_default_pytorch, # optim_fn
None, # no max_range needed
True # half precision
),
# 3D VGG
(
VGG3DClassificationNetworkPyTorch(1, 10),
# model
(1, 32, 224, 224), # input shape
9, # output shape (num_classes - 1)
{
"loss_fn": torch.nn.CrossEntropyLoss()
}, # loss function
create_optims_default_pytorch, # optim fn
None, # no max_range needed
True # half precision
),
# DCGAN
(
GenerativeAdversarialNetworkBasePyTorch(1, 100),
# model
(1, 64, 64), # input shape
(1, 1), # arbitrary shape (not needed)
{
"loss_fn": torch.nn.MSELoss()
}, # loss
create_optims_gan_default_pytorch,
# optimizer function
1, # standard max range
True # half precision
)
]
for case in test_cases:
with self.subTest(case=case):
model, input_shape, output_shape, loss_fn, optim_fn, \
max_range, half_precision = case
try:
from apex import amp
amp_handle = amp.init(half_precision)
wrapper_fn = amp_handle.wrap_optimizer
except ImportError:
wrapper_fn = DefaultOptimWrapperTorch
start_time = time.time()
# test backward if optimizer fn is not None
if optim_fn is not None:
optim = {
k: wrapper_fn(v, num_loss=len(loss_fn))
for k, v in optim_fn(model, torch.optim.Adam).items()
}
else:
optim = {}
closure = model.closure
device = torch.device("cpu")
model = model.to(device)
prepare_batch = model.prepare_batch
# classification label: target_shape specifies max label
if isinstance(output_shape, int):
label = np.asarray(
[np.random.randint(output_shape) for i in range(10)])
else:
label = np.random.rand(10, *output_shape) * max_range
data_dict = {
"data": np.random.rand(10, *input_shape),
"label": label
}
try:
data_dict = prepare_batch(data_dict, device, device)
closure(model, data_dict, optim, loss_fn, {})
except Exception as e:
assert False, "Test for %s not passed: Error: %s" \
% (model.__class__.__name__, e)
end_time = time.time()
print("Time needed for %s: %.3f" %
(model.__class__.__name__, end_time - start_time))
del device
del optim
del closure
del prepare_batch
del model
try:
del amp_handle
except NameError:
pass
gc.collect()
class TfModelTest(unittest.TestCase):
@unittest.skipIf((virtual_memory().total / 1024.**3) < 20,
reason="Less than 20GB of memory")
@unittest.skipIf("TF" not in get_backends(),
reason="No TF Backend Installed")
def test_tf_model_default(self):
from delira.models import ClassificationNetworkBaseTf
from delira.training.train_utils import create_optims_default_tf
import tensorflow as tf
test_cases = [(
ClassificationNetworkBaseTf(1, 10),
# model
(1, 224, 224), # input shape
9, # output shape (num_classes - 1)
{
"loss_fn": tf.losses.softmax_cross_entropy
}, # loss function
create_optims_default_tf, # optim_fn
4)]
for case in test_cases:
with self.subTest(case=case):
model, input_shape, output_shape, loss_fns, optim_fn, \
max_range = case
start_time = time.time()
# test backward if optimizer fn is not None
if optim_fn is not None:
optim = optim_fn(tf.train.AdamOptimizer)
else:
optim = {}
model._add_losses(loss_fns)
model._add_optims(optim)
model._sess.run(tf.initializers.global_variables())
closure = model.closure
# classification label: target_shape specifies max label
if isinstance(output_shape, int):
label = np.asarray([
np.random.randint(output_shape) for i in range(10)
])[np.newaxis, :]
else:
label = np.random.rand(10, *output_shape) * max_range
data_dict = {
"data": np.random.rand(10, *input_shape),
"label": label
}
try:
closure(model, data_dict)
except Exception as e:
assert False, "Test for %s not passed: Error: %s" \
% (model.__class__.__name__, e)
end_time = time.time()
print("Time needed for %s: %.3f" %
(model.__class__.__name__, end_time - start_time))
del optim
del closure
del model
gc.collect()