def model_initializer(hparams, continue_training, base_path, logger=None):
logger = logger or ScreenLogger()
# Init model
model = init_model(hparams["build"], logger)
if continue_training:
from MultiPlanarUNet.utils import get_last_model, get_lr_at_epoch, \
clear_csv_after_epoch
model_path, epoch = get_last_model(os.path.join(base_path, "model"))
model.load_weights(model_path, by_name=True)
hparams["fit"]["init_epoch"] = epoch + 1
# Get the LR at the continued epoch
lr, name = get_lr_at_epoch(epoch, os.path.join(base_path, "logs"))
hparams["fit"]["optimizer_kwargs"][name] = lr
# Remove entries in training.csv file that occurred after the
# continued epoch
clear_csv_after_epoch(epoch,
os.path.join(base_path, "logs", "training.csv"))
logger("[NOTICE] Training continues from:\n"
"Model: %s\n"
"Epoch: %i\n"
"LR: %s" % (os.path.split(model_path)[-1], epoch, lr))
else:
hparams["fit"]["init_epoch"] = 0
return model
def plot_all_training_curves(glob_path,
out_path,
raise_error=False,
logger=None,
**kwargs):
logger = logger or ScreenLogger()
try:
from glob import glob
paths = glob(glob_path)
if not paths:
raise OSError("File pattern {} gave none or too many matches " \
"({})".format(glob_path, paths))
out_folder = os.path.split(out_path)[0]
for p in paths:
if len(paths) > 1:
# Set unique names
uniq = os.path.splitext(os.path.split(p)[-1])[0]
f_name = uniq + "_" + os.path.split(out_path)[-1]
save_path = os.path.join(out_folder, f_name)
else:
save_path = out_path
plot_training_curves(p, save_path, **kwargs)
except Exception as e:
s = "Could not plot training curves. ({})".format(e)
if raise_error:
raise RuntimeError(s) from e
else:
logger.warn(s)
def init_model(build_hparams, logger=None, clear_previous=True):
"""
From a set of hyperparameters 'build_hparams' (dict) initializes the
model specified under build_hparams['model_class_name'].
Typically, this function is not called directly, but used by the
higher-level 'initialize_model' function.
Args:
build_hparams: A dictionary of model build hyperparameters
logger: A Logger instance
clear_previous: Clear previous tf sessions
Returns:
A tf.keras Model instance
"""
from utime import models
logger = logger or ScreenLogger()
if clear_previous:
import tensorflow as tf
tf.keras.backend.clear_session()
# Build new model of the specified type
cls_name = build_hparams["model_class_name"]
logger("Creating new model of type '%s'" % cls_name)
return models.__dict__[cls_name](logger=logger, **build_hparams)
def __init__(self,
val_sequence,
steps,
logger=None,
verbose=True,
ignore_class_zero=True):
"""
Args:
val_sequence: A MultiPlanarUNet.sequence object from which validation
batches can be sampled via its __getitem__ method.
steps: Numer of batches to sample from val_sequences in each
validation epoch
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
verbose: Print progress to screen - OBS does not use Logger
"""
super().__init__()
self.logger = logger or ScreenLogger()
self.data = val_sequence
self.steps = steps
self.verbose = verbose
self.ignore_bg = ignore_class_zero
self.n_classes = self.data.n_classes
if isinstance(self.n_classes, int):
self.task_names = [""]
self.n_classes = [self.n_classes]
else:
self.task_names = self.data.task_names
def __init__(self,
batch_shape,
n_classes,
padding="valid",
activation="relu",
use_dropout=True,
use_bn=True,
classify=True,
flatten=True,
l2_reg=0.0,
logger=None,
log=True,
build=True,
**unused):
super(DeepFeatureNet, self).__init__()
self.logger = logger or ScreenLogger()
self.batch_shape = standardize_batch_shape(batch_shape)
self.n_classes = n_classes
self.use_dropout = use_dropout
self.padding = padding
self.activation = activation
self.use_bn = use_bn
self.classify = classify
self.flatten = flatten
self.l2_reg = l2_reg
self.reg = None
self.model_name = "DeepFeatureNet"
# Build model and init base keras Model class
if build:
with tf.name_scope(self.model_name):
super(DeepFeatureNet, self).__init__(*self.init_model())
if log:
self.log()
def __init__(self,
layer,
every=10,
first=10,
per_epoch=False,
logger=None):
"""
Args:
layer: A tf.keras layer
every: Print the weights every 'every' batch or epoch if
per_epoch=True
first: Print the first 'first' elements of each weight matrix
per_epoch: Print after 'every' epoch instead of batch
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
"""
super().__init__()
if isinstance(layer, int):
self.layer = self.model.layers[layer]
else:
self.layer = layer
self.first = first
self.every = every
self.logger = logger or ScreenLogger()
self.per_epoch = per_epoch
if per_epoch:
# Apply on every epoch instead of per batches
self.on_epoch_begin = self.on_batch_begin
self.on_batch_begin = lambda x, y: None
self.log()
def __init__(self, image, labels, affine,
bg_value=0, bg_class=0, logger=None):
# Ensure 4D
if not image.ndim == 4:
raise ValueError("Input img of dim %i must be dim 4."
"If image has only 1 channel, use "
"np.expand_dims(img, -1)." % image.ndim)
# Set logger
self.logger = logger if logger is not None else ScreenLogger()
# Number of channels in the input image
self.im_shape = image.shape
self.n_channels = self.im_shape[-1]
self.im_dtype = image.dtype
# Store potential transformation to regular grid
self.rot_mat = None
# Define interpolators
self.im_intrps, self.lab_intrp = self._init_interpolators(image,
labels,
bg_value,
bg_class,
affine)
def __init__(self, logger=None, verbose=1):
super().__init__()
self.logger = logger or ScreenLogger()
self.verbose = bool(verbose)
# Timing attributes
self.train_begin_time = None
self.prev_epoch_time = None
def __init__(self, n_classes, logger=None, **kwargs):
self.logger = logger if logger is not None else ScreenLogger()
self.__name__ = "BatchWeightedCrossEntropyWithLogits"
# if not class_weights
self.n_classes = n_classes
self._log()
def __init__(self, logger=None):
import MultiPlanarUNet
from MultiPlanarUNet.logging import ScreenLogger
code_path = MultiPlanarUNet.__path__
assert len(code_path) == 1
self.logger = logger or ScreenLogger()
self.git_path = os.path.split(os.path.abspath(code_path[0]))[0]
self._mem_path = None
def __init__(self, sequences, no_log=False, logger=None):
_assert_comparable_sequencers(sequences)
self.sequences = sequences
self.IDs = [s.identifier.split("/")[0] for s in self.sequences]
self.n_classes = self.sequences[0].n_classes
self.logger = logger or ScreenLogger()
if not no_log:
self.log()
def __init__(self, logger=None):
"""
Args:
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
"""
super().__init__()
self.logger = logger or ScreenLogger()
def __init__(self, model, logger=None):
self.model = model
self.logger = logger if logger is not None else ScreenLogger()
self.target_tensor = None
# Extra reference to original (non multiple-GPU) model
# Is set from train.py as needed
self.org_model = None
def __init__(self, sequencers, task_names, logger=None):
super().__init__()
self.logger = logger or ScreenLogger()
self.task_names = task_names
self.sequencers = sequencers
self.log()
# Redirect setattrs to the sub-sequences
self.redirect = True
def __init__(self,
image_pair_loader,
dim,
batch_size,
n_classes,
real_space_span=None,
noise_sd=0.,
force_all_fg="auto",
fg_batch_fraction=0.50,
label_crop=None,
logger=None,
is_validation=False,
list_of_augmenters=None,
sparse=True,
**kwargs):
super().__init__()
# Validation or training batch generator?
self.is_validation = is_validation
# Set logger or default print
self.logger = logger or ScreenLogger()
# Set views and attributes for plane sample generation
self.sample_dim = dim
self.real_space_span = real_space_span
self.noise_sd = noise_sd if not self.is_validation else 0.
# Set data
self.image_pair_loader = image_pair_loader
self.images = image_pair_loader.images
# Augmenter, applied to batch at creation time
# Do not augment validation data
self.list_of_augmenters = list_of_augmenters if not self.is_validation else None
# Batch creation options
self.batch_size = batch_size
self.n_classes = n_classes
self.sparse = sparse
# Minimum fraction of slices in each batch with FG
self.force_all_fg_switch = force_all_fg
self.fg_batch_fraction = fg_batch_fraction
# Store labels?
self.store_y = False
self.stored_y = []
# Foreground label settings
self.fg_classes = np.arange(1, self.n_classes)
if self.fg_classes.shape[0] == 0:
self.fg_classes = [1]
# Set potential label label_crop
self.label_crop = np.array([[0, 0], [0, 0]
]) if label_crop is None else label_crop
def init_model(build_hparams, logger=None):
from MultiPlanarUNet import models
logger = logger or ScreenLogger()
# Build new model of the specified type
cls_name = build_hparams["model_class_name"]
logger("Creating new model of type '%s'" % cls_name)
return models.__dict__[cls_name](logger=logger, **build_hparams)
def prepare_for_continued_training(hparams, project_dir, logger=None):
"""
Prepares the hyperparameter set and project directory for continued
training.
Will find the latest model (highest epoch number) of parameter files in
the 'model' subdir of 'project_dir' and base the continued training on this
file. If no file is found, training will start from scratch as
normally (note: no error is raised, but None is returned instead of a path
to a parameter file).
The hparams['fit']['init_epoch'] parameter will be set to match the found
parameter file or to 0 if no file was found. Note that if init_epoch is set
to 0 all rows in the training.csv file will be deleted.
The hparams['fit']['optimizer_kwargs']['learning_rate'] parameter will
be set according to the value stored in the project_dir/logs/training.csv
file at the corresponding epoch (left default if no init_epoch was found)
Args:
hparams: (YAMLHParams) The hyperparameters to use for training
project_dir: (string) The path to the current project directory
logger: (Logger) An optional Logger instance
Returns:
A path to the model weight files to use for continued training.
Will be None if no model files were found
"""
from MultiPlanarUNet.utils import (get_last_model, get_lr_at_epoch,
get_last_epoch, clear_csv_after_epoch)
model_path, epoch = get_last_model(os.path.join(project_dir, "model"))
if model_path:
model_name = os.path.split(model_path)[-1]
else:
model_name = None
csv_path = os.path.join(project_dir, "logs", "training.csv")
if epoch == 0:
epoch = get_last_epoch(csv_path)
else:
if epoch is None:
epoch = 0
clear_csv_after_epoch(epoch, csv_path)
hparams["fit"]["init_epoch"] = epoch + 1
# Get the LR at the continued epoch
lr, name = get_lr_at_epoch(epoch, os.path.join(project_dir, "logs"))
if lr:
hparams["fit"]["optimizer_kwargs"][name] = lr
logger = logger or ScreenLogger()
logger("[NOTICE] Training continues from:\n"
"Model: {}\n"
"Epoch: {}\n"
"LR: {}".format(
model_name or "<No model found - "
"Starting for scratch!>", epoch, lr))
return model_path
def __init__(self, callback, start_from=0, logger=None):
"""
Args:
callback: A tf.keras callback
start_from: Delay the activity of 'callback' until this epoch
'start_from'
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
"""
self.logger = logger or ScreenLogger()
self.callback = callback
self.start_from = start_from
def __init__(self,
n_classes,
type_weight="Square",
sparse=True,
logger=None,
**kwargs):
self.type_weight = type_weight
self.n_classes = n_classes
self.sparse = sparse
self.logger = logger or ScreenLogger()
self.__name__ = "GeneralizedDiceLoss"
self.log()
def load_from_file(model, file_path, logger=None, by_name=True):
"""
Load parameters from file 'file_path' into model 'model'.
Args:
model: A tf.keras Model instance
file_path: A path to a parameter file (h5 format typically)
logger: An optional Logger instance
by_name: Load parameters by layer names instead of order (default).
"""
model.load_weights(file_path, by_name=by_name)
logger = logger or ScreenLogger()
logger("Loading parameters from:\n{}".format(file_path))
def __init__(self, logger=None):
self.logger = logger or ScreenLogger()
# Prepare signal
self.stop_signal = Event()
self.run_signal = Event()
self.set_signal = Event()
# Stores list of available GPUs
self._free_GPUs = Queue()
super(GPUMonitor, self).__init__(target=self._monitor)
self.start()
def __init__(self, sequencers, batch_size, no_log=False, logger=None):
# Make sure we can use the 0th sequencer as a reference that respects
# all the sequences (same batch-size, margins etc.)
_assert_comparable_sequencers(sequencers)
super().__init__()
self.logger = logger or ScreenLogger()
self.sequences = sequencers
self.batch_size = batch_size
self.margin = sequencers[0].margin
self.n_classes = sequencers[0].n_classes
for s in self.sequences:
s.batch_size = 1
if not no_log:
self.log()
def __init__(self, train_data, val_data=None, logger=None):
"""
Args:
train_data: A MultiPlanarUNet.sequence object representing the
training data
val_data: A MultiPlanarUNet.sequence object representing the
validation data
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
"""
super().__init__()
self.data = (("train", train_data), ("val", val_data))
self.logger = logger or ScreenLogger()
self.active = True
def __init__(self,
log_dir="logs",
out_dir="logs",
fname="curve.png",
csv_regex="*training.csv",
logger=None):
"""
"""
super().__init__()
out_dir = os.path.abspath(out_dir)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
self.csv_regex = os.path.join(os.path.abspath(log_dir), csv_regex)
self.save_path = os.path.join(out_dir, fname)
self.logger = logger or ScreenLogger()
def save_images(train, val, out_dir, logger):
logger = logger or ScreenLogger()
# Write a few images to disk
im_path = out_dir
if not os.path.exists(im_path):
os.mkdir(im_path)
training = train[0]
if val is not None and len(val) != 0:
validation = val[0]
v_len = len(validation[0])
else:
validation = None
v_len = 0
logger("Saving %i sample images in '<project_dir>/images' folder" %
((len(training[0]) + v_len) * 2))
for rr in range(2):
for k, temp in enumerate((training, validation)):
if temp is None:
# No validation data
continue
X, Y, W = temp
for i, (xx, yy, ww) in enumerate(zip(X, Y, W)):
# Make figure
fig = plt.figure(figsize=(10, 4))
ax1 = fig.add_subplot(121)
ax2 = fig.add_subplot(122)
# Plot image and overlayed labels
chnl, view, _ = imshow_with_label_overlay(ax1, xx, yy)
# Plot histogram
ax2.hist(xx.flatten(), bins=200)
# Set labels
ax1.set_title("Channel %i - Axis %i - "
"Weight %.3f" % (chnl, view, ww),
size=18)
# Get path
out_path = im_path + "/%s%i.png" % ("train" if k == 0 else
"val", len(X) * rr + i)
with np.testing.suppress_warnings() as sup:
sup.filter(UserWarning)
fig.savefig(out_path)
plt.close(fig)
def model_initializer(hparams,
continue_training,
project_dir,
initialize_from=None,
logger=None):
logger = logger or ScreenLogger()
# Init model
model = init_model(hparams["build"], logger)
if continue_training:
if initialize_from:
raise ValueError("Failed to initialize model with both "
"continue_training and initialize_from set.")
from MultiPlanarUNet.utils import get_last_model, get_lr_at_epoch, \
clear_csv_after_epoch, get_last_epoch
model_path, epoch = get_last_model(os.path.join(project_dir, "model"))
if model_path:
model.load_weights(model_path, by_name=True)
model_name = os.path.split(model_path)[-1]
else:
model_name = "<No model found>"
csv_path = os.path.join(project_dir, "logs", "training.csv")
if epoch == 0:
epoch = get_last_epoch(csv_path)
else:
if epoch is None:
epoch = 0
clear_csv_after_epoch(epoch, csv_path)
hparams["fit"]["init_epoch"] = epoch + 1
# Get the LR at the continued epoch
lr, name = get_lr_at_epoch(epoch, os.path.join(project_dir, "logs"))
if lr:
hparams["fit"]["optimizer_kwargs"][name] = lr
logger("[NOTICE] Training continues from:\n"
"Model: %s\n"
"Epoch: %i\n"
"LR: %s" % (model_name, epoch, lr))
else:
hparams["fit"]["init_epoch"] = 0
if initialize_from:
model.load_weights(initialize_from, by_name=True)
logger("[NOTICE] Initializing parameters from:\n"
"{}".format(initialize_from))
return model
def __init__(self,
yaml_path,
logger=None,
no_log=False,
no_version_control=False,
**kwargs):
dict.__init__(self, **kwargs)
# Set logger or default print
self.logger = logger or ScreenLogger()
# Set YAML path
self.yaml_path = os.path.abspath(yaml_path)
self.string_rep = ""
self.project_path = os.path.split(self.yaml_path)[0]
if not os.path.exists(self.yaml_path):
raise OSError("YAML path '%s' does not exist" % self.yaml_path)
else:
with open(self.yaml_path, "r") as yaml_file:
for line in yaml_file:
self.string_rep += line
hparams = YAML(typ="safe").load(self.string_rep)
# Set dict elements
self.update({k: hparams[k] for k in hparams if k[:4] != "__CB"})
if self.get('fit') and self["fit"].get("callbacks"):
# Convert potential callback paths to absolute paths
cb = _cb_paths_to_abs_paths(callbacks=self["fit"]["callbacks"],
patterns=("log.?dir", "file.?name",
"file.?path"),
project_dir=self.project_path)
self["fit"]["callbacks"] = cb
# Log basic information here...
self.no_log = no_log
if not self.no_log:
self.logger("YAML path: %s" % self.yaml_path)
# Version controlling
_check_deprecated_params(self, self.logger)
if not no_version_control:
package = kwargs.get('package') or "MultiPlanarUNet"
has_git = _check_version(self, self.logger, package)
if has_git:
_set_version(self, self.logger if not no_log else None,
package)
def __init__(self, class_weights, logger=None, *args, **kwargs):
"""
weights: A Nx1 matrix of class weights for N classes.
"""
if class_weights is None or class_weights is False:
raise ValueError("No class weights passed.")
self.logger = logger if logger is not None else ScreenLogger()
self.__name__ = "WeightedSemanticCCE"
self.weights = _to_tensor(class_weights, dtype=tf.float32)
self.n_classes = self.weights.get_shape()[0]
with print_options_context(precision=3, suppress=True):
logger("Class weights:\n%s" % class_weights)
# Log to console/file
self._log()
def __init__(self, validation_data, n_classes, batch_size=8, logger=None):
"""
Args:
validation_data: A tuple (X, y) of two ndarrays of validation data
and corresponding labels.
Any shape accepted by the model.
Labels must be integer targets (not one-hot)
n_classes: Number of classes, including background
batch_size: Batch size used for prediction
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
"""
super().__init__()
self.logger = logger or ScreenLogger()
self.X_val, self.y_val = validation_data
self.n_classes = n_classes
self.batch_size = batch_size
self.scores = []
def __init__(self,
n_classes,
gamma_dice=0.3,
gamma_cross=0.3,
weight_dice=1,
weight_cross=1,
logger=None,
int_targets=True,
**kwargs):
self.gamma_dice = gamma_dice
self.gamma_cross = gamma_cross
self.weight_dice = weight_dice
self.weight_cross = weight_cross
self.n_classes = n_classes
self.int_targets = int_targets
self.logger = logger or ScreenLogger()
self.__name__ = "ExponentialLogarithmicLoss"
self.log()
