def __init__(self, layer, every=10, first=10, per_epoch=False, logger=None):
"""
Args:
layer: A tf.keras layer
every: Print the weights every 'every' batch or epoch if
per_epoch=True
first: Print the first 'first' elements of each weight matrix
per_epoch: Print after 'every' epoch instead of batch
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
"""
super().__init__()
if isinstance(layer, int):
self.layer = self.model.layers[layer]
else:
self.layer = layer
self.first = first
self.every = every
self.logger = logger or ScreenLogger()
self.per_epoch = per_epoch
if per_epoch:
# Apply on every epoch instead of per batches
self.on_epoch_begin = self.on_batch_begin
self.on_batch_begin = lambda *args, **kwargs: None
self.log()
def __init__(self,
val_sequence,
steps,
logger=None,
verbose=True,
ignore_class_zero=True):
"""
Args:
val_sequence: A mpunet.sequence object from which validation
batches can be sampled via its __getitem__ method.
steps: Numer of batches to sample from val_sequences in each
validation epoch
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
verbose: Print progress to screen - OBS does not use Logger
ignore_class_zero: TODO
"""
super().__init__()
self.logger = logger or ScreenLogger()
self.data = val_sequence
self.steps = steps
self.verbose = verbose
self.ignore_bg = ignore_class_zero
self.print_round = 3
self.log_round = 4
self.n_classes = self.data.n_classes
if isinstance(self.n_classes, int):
self.task_names = [""]
self.n_classes = [self.n_classes]
else:
self.task_names = self.data.task_names
def __init__(self,
annotation_dict,
period_length_sec,
no_hypnogram,
logger=None):
"""
TODO
Args:
annotation_dict:
period_length_sec:
no_hypnogram:
logger:
"""
self.logger = logger or ScreenLogger()
self.annotation_dict = annotation_dict
self.no_hypnogram = no_hypnogram
self.period_length_sec = period_length_sec or \
Defaults.get_default_period_length(self.logger)
# Hidden attributes controlled in property functions to limit setting
# of these values to the load() function
self._psg = None
self._hypnogram = None
self._select_channels = None
self._alternative_select_channels = None
def plot_all_training_curves(glob_path,
out_path,
raise_error=False,
logger=None,
**kwargs):
logger = logger or ScreenLogger()
try:
from glob import glob
paths = glob(glob_path)
if not paths:
raise OSError("File pattern {} gave none or too many matches " \
"({})".format(glob_path, paths))
out_folder = os.path.split(out_path)[0]
for p in paths:
if len(paths) > 1:
# Set unique names
uniq = os.path.splitext(os.path.split(p)[-1])[0]
f_name = uniq + "_" + os.path.split(out_path)[-1]
save_path = os.path.join(out_folder, f_name)
else:
save_path = out_path
plot_training_curves(p, save_path, **kwargs)
except Exception as e:
s = "Could not plot training curves. ({})".format(e)
if raise_error:
raise RuntimeError(s) from e
else:
logger.warn(s)
def __init__(self,
batch_shape,
n_classes,
padding="valid",
activation="relu",
use_dropout=True,
use_bn=True,
classify=True,
flatten=True,
l2_reg=0.0,
logger=None,
log=True,
build=True,
**unused):
super(DeepFeatureNet, self).__init__()
self.logger = logger or ScreenLogger()
self.batch_shape = standardize_batch_shape(batch_shape)
self.n_classes = n_classes
self.use_dropout = use_dropout
self.padding = padding
self.activation = activation
self.use_bn = use_bn
self.classify = classify
self.flatten = flatten
self.l2_reg = l2_reg
self.reg = None
self.model_name = "DeepFeatureNet"
# Build model and init base keras Model class
if build:
with tf.name_scope(self.model_name):
super(DeepFeatureNet, self).__init__(*self.init_model())
if log:
self.log()
def __init__(self, image_pair_loader, dim, n_classes, batch_size, is_validation=False,
label_crop=None, fg_batch_fraction=0.33, logger=None, bg_val=0.,
no_log=False, **kwargs):
super().__init__()
# Set logger or default print
self.logger = logger or ScreenLogger()
self.image_pair_loader = image_pair_loader
# Box dimension and image dim
self.dim = dim
# Various attributes
self.n_classes = n_classes
self.label_crop = label_crop
self.is_validation = is_validation
self.batch_size = batch_size
self.bg_value = bg_val
# How many foreground slices should be in each batch?
self.fg_batch_fraction = fg_batch_fraction
# Foreground label settings
self.fg_classes = np.arange(1, self.n_classes)
if self.fg_classes.shape[0] == 0:
self.fg_classes = 1
if not is_validation and not no_log:
self.log()
def sample_random_views_with_angle_restriction(views,
min_angle_deg,
auditor=None,
logger=None):
from itertools import combinations
from mpunet.logging import ScreenLogger
logger = logger or ScreenLogger()
logger("Generating %i random views..." % views)
# Weight by median sample resolution along each axis
if auditor is not None:
res = np.median(auditor.info["pixdims"], axis=0)
logger("[OBS] Weighting random views by median res: %s" % res)
else:
res = None
N = views
found = False
tries = 0
while not found:
tries += 1
views = get_random_views(N, dim=3, pos_z=True, weights=res)
angles = [get_angle(v1, v2) for v1, v2 in combinations(views, 2)]
found = np.all(np.asarray(angles) > min_angle_deg)
min_angle_deg -= 1
return views
def init_model(build_hparams, logger=None, clear_previous=True):
"""
From a set of hyperparameters 'build_hparams' (dict) initializes the
model specified under build_hparams['model_class_name'].
Typically, this function is not called directly, but used by the
higher-level 'initialize_model' function.
Args:
build_hparams: A dictionary of model build hyperparameters
logger: A Logger instance
clear_previous: Clear previous tf sessions
Returns:
A tf.keras Model instance
"""
from utime import models
logger = logger or ScreenLogger()
if clear_previous:
import tensorflow as tf
tf.keras.backend.clear_session()
# Build new model of the specified type
cls_name = build_hparams["model_class_name"]
logger("Creating new model of type '%s'" % cls_name)
return models.__dict__[cls_name](logger=logger, **build_hparams)
def __init__(self, sequences, no_log=False, logger=None):
_assert_comparable_sequencers(sequences)
self.sequences = sequences
self.IDs = [s.identifier.split("/")[0] for s in self.sequences]
self.n_classes = self.sequences[0].n_classes
self.logger = logger or ScreenLogger()
if not no_log:
self.log()
def __init__(self, logger=None):
"""
Args:
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
"""
super().__init__()
self.logger = logger or ScreenLogger()
def __init__(self, sequencers, task_names, logger=None):
super().__init__()
self.logger = logger or ScreenLogger()
self.task_names = task_names
self.sequencers = sequencers
self.log()
# Redirect setattrs to the sub-sequences
self.redirect = True
def __init__(self, max_minutes, log_name='train_time_total', logger=None):
"""
TODO
Args:
"""
super().__init__()
self.max_minutes = int(max_minutes)
self.log_name = log_name
self.logger = logger or ScreenLogger()
def __init__(self, logger=None, max_minutes=None, verbose=1):
super().__init__()
self.logger = logger or ScreenLogger()
self.max_minutes = int(max_minutes) if max_minutes else None
self.verbose = bool(verbose)
# Timing attributes
self.train_begin_time = None
self.prev_epoch_time = None
def prepare_for_continued_training(hparams, project_dir, logger=None):
"""
Prepares the hyperparameter set and project directory for continued
training.
Will find the latest model (highest epoch number) of parameter files in
the 'model' subdir of 'project_dir' and base the continued training on this
file. If no file is found, training will start from scratch as
normally (note: no error is raised, but None is returned instead of a path
to a parameter file).
The hparams['fit']['init_epoch'] parameter will be set to match the found
parameter file or to 0 if no file was found. Note that if init_epoch is set
to 0 all rows in the training.csv file will be deleted.
The hparams['fit']['optimizer_kwargs']['learning_rate'] parameter will
be set according to the value stored in the project_dir/logs/training.csv
file at the corresponding epoch (left default if no init_epoch was found)
Args:
hparams: (YAMLHParams) The hyperparameters to use for training
project_dir: (string) The path to the current project directory
logger: (Logger) An optional Logger instance
Returns:
A path to the model weight files to use for continued training.
Will be None if no model files were found
"""
from mpunet.utils import (get_last_model, get_lr_at_epoch, get_last_epoch,
clear_csv_after_epoch)
model_path, epoch = get_last_model(os.path.join(project_dir, "model"))
if model_path:
model_name = os.path.split(model_path)[-1]
else:
model_name = None
csv_path = os.path.join(project_dir, "logs", "training.csv")
if epoch == 0:
epoch = get_last_epoch(csv_path)
else:
if epoch is None:
epoch = 0
clear_csv_after_epoch(epoch, csv_path)
hparams["fit"]["init_epoch"] = epoch + 1
# Get the LR at the continued epoch
lr, name = get_lr_at_epoch(epoch, os.path.join(project_dir, "logs"))
if lr:
hparams["fit"]["optimizer_kwargs"][name] = lr
logger = logger or ScreenLogger()
logger("[NOTICE] Training continues from:\n"
"Model: {}\n"
"Epoch: {}\n"
"LR: {}".format(
model_name or "<No model found - "
"Starting for scratch!>", epoch, lr))
return model_path
def __init__(self,
image_pair_loader,
dim,
batch_size,
n_classes,
real_space_span=None,
noise_sd=0.,
force_all_fg="auto",
fg_batch_fraction=0.50,
label_crop=None,
logger=None,
is_validation=False,
list_of_augmenters=None,
flatten_y=False,
**kwargs):
super().__init__()
# Validation or training batch generator?
self.is_validation = is_validation
# Set logger or default print
self.logger = logger or ScreenLogger()
# Set views and attributes for plane sample generation
self.sample_dim = dim
self.real_space_span = real_space_span
self.noise_sd = noise_sd if not self.is_validation else 0.
# Set data
self.image_pair_loader = image_pair_loader
self.images = image_pair_loader.images
# Augmenter, applied to batch at creation time
# Do not augment validation data
self.list_of_augmenters = list_of_augmenters if not self.is_validation else None
# Batch creation options
self.batch_size = batch_size
self.n_classes = n_classes
self.flatten_y = flatten_y
# Minimum fraction of slices in each batch with FG
self.force_all_fg_switch = force_all_fg
self.fg_batch_fraction = fg_batch_fraction
# Foreground label settings
self.fg_classes = np.arange(1, self.n_classes)
if self.fg_classes.shape[0] == 0:
self.fg_classes = [1]
# Set potential label label_crop
self.label_crop = np.array([[0, 0], [0, 0]
]) if label_crop is None else label_crop
def __init__(self, callback, start_from=0, logger=None):
"""
Args:
callback: A tf.keras callback
start_from: Delay the activity of 'callback' until this epoch
'start_from'
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
"""
self.logger = logger or ScreenLogger()
self.callback = callback
self.start_from = start_from
def load_from_file(model, file_path, logger=None, by_name=True):
"""
Load parameters from file 'file_path' into model 'model'.
Args:
model: A tf.keras Model instance
file_path: A path to a parameter file (h5 format typically)
logger: An optional Logger instance
by_name: Load parameters by layer names instead of order (default).
"""
model.load_weights(file_path, by_name=by_name)
logger = logger or ScreenLogger()
logger("Loading parameters from:\n{}".format(file_path))
def __init__(self, logger=None):
self.logger = logger or ScreenLogger()
# Prepare signal
self.stop_signal = Event()
self.run_signal = Event()
self.set_signal = Event()
# Stores list of available GPUs
self._free_GPUs = Queue()
super(GPUMonitor, self).__init__(target=self._monitor)
self.start()
def save_images(train, val, out_dir, logger):
logger = logger or ScreenLogger()
# Write a few images to disk
im_path = out_dir
if not os.path.exists(im_path):
os.mkdir(im_path)
# Sample two batches of training and (if passed) validation data
training = [train[0], train[1]]
if val is not None and len(val) != 0:
validation = [val[0], val[1]]
val_bs = val.batch_size
else:
validation = None
val_bs = 0
logger("Saving %i sample images in '<project_dir>/images' folder" %
((train.batch_size + val_bs) * 2))
for rr, (train_batch, val_batch) in enumerate(zip(training, validation)):
for k, temp in enumerate((train_batch, val_batch)):
if temp is None:
# No validation data
continue
X, Y, W = temp
for i, (xx, yy, ww) in enumerate(zip(X, Y, W)):
yy = yy.reshape(xx.shape[:-1] + (yy.shape[-1], ))
# Make figure
fig = plt.figure(figsize=(10, 4))
ax1 = fig.add_subplot(121)
ax2 = fig.add_subplot(122)
# Plot image and overlayed labels
chnl, view, _ = imshow_with_label_overlay(ax1, xx, yy)
# Plot histogram
ax2.hist(xx.flatten(), bins=200)
# Set labels
ax1.set_title("Channel %i - Axis %i - "
"Weight %.3f" % (chnl, view, ww),
size=18)
# Get path
out_path = im_path + "/%s%i.png" % ("train" if k == 0 else
"val", len(X) * rr + i)
with np.testing.suppress_warnings() as sup:
sup.filter(UserWarning)
fig.savefig(out_path)
plt.close(fig)
def __init__(self, train_data, val_data=None, logger=None):
"""
Args:
train_data: A mpunet.sequence object representing the
training data
val_data: A mpunet.sequence object representing the
validation data
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
"""
super().__init__()
self.data = (("train", train_data), ("val", val_data))
self.logger = logger or ScreenLogger()
self.active = True
def __init__(self, sequencers, batch_size, no_log=False, logger=None):
# Make sure we can use the 0th sequencer as a reference that respects
# all the sequences (same batch-size, margins etc.)
_assert_comparable_sequencers(sequencers)
super().__init__()
self.logger = logger or ScreenLogger()
self.sequences = sequencers
self.batch_size = batch_size
self.margin = sequencers[0].margin
self.n_classes = sequencers[0].n_classes
for s in self.sequences:
s.batch_size = 1
if not no_log:
self.log()
def __init__(self, n_inputs, n_classes, weight="Simple", logger=None,
verbose=True):
self.n_inputs = n_inputs
self.n_classes = n_classes
# Set Logger object
self.logger = logger or ScreenLogger()
# Set loss
self.loss = SparseGeneralizedDiceLoss(tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE,
type_weight=weight)
# Init model
super().__init__(*self.init_model(n_inputs, n_classes))
if verbose:
self._log()
def __init__(self,
dataset_queue,
n_classes,
n_channels,
batch_size,
augmenters,
batch_scaler,
logger=None,
require_all_loaded=False,
identifier=""):
"""
Args:
dataset_queue: (queue) TODO
n_classes: (int) Number of classes (sleep stages)
n_channels: (int) The number of PSG channels to expect in
data extracted from a SleepStudy object
batch_size: (int) The size of the generated batch
augmenters: (list) List of utime.augmentation.augmenters
batch_scaler: (string) The name of a sklearn.preprocessing
Scaler object to apply to each sampled
batch (optional)
logger: (Logger) A Logger object
identifier: (string) A string identifier name
"""
super().__init__()
self._all_loaded = assert_all_loaded(dataset_queue.dataset.pairs,
raise_=require_all_loaded)
self.identifier = identifier
self.dataset_queue = dataset_queue
self.n_classes = int(n_classes)
self.n_channels = int(n_channels)
self.logger = logger or ScreenLogger()
self.augmenters = augmenters or []
self.augmentation_enabled = bool(augmenters)
self.batch_size = batch_size
if self.all_loaded:
self._periods_per_pair = np.array(
[ss.n_periods for ss in self.dataset_queue])
self._cum_periods_per_pair = np.cumsum(self.periods_per_pair)
if batch_scaler not in (None, False):
if not assert_scaler(batch_scaler):
raise ValueError(
"Invalid batch scaler {}".format(batch_scaler))
self.batch_scaler = batch_scaler
else:
self.batch_scaler = None
def __init__(self, log_dir="logs", out_dir="logs", fname="curve.png",
csv_regex="*training.csv", logger=None):
"""
Args:
log_dir: Relative path from the
out_dir:
fname:
csv_regex:
logger:
"""
super().__init__()
out_dir = os.path.abspath(out_dir)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
self.csv_regex = os.path.join(os.path.abspath(log_dir), csv_regex)
self.save_path = os.path.join(out_dir, fname)
self.logger = logger or ScreenLogger()
def __init__(self,
sequencers,
batch_size,
dataset_sample_alpha=0.5,
no_log=False,
logger=None):
"""
TODO
Args:
sequencers:
batch_size:
dataset_sample_alpha: TODO
no_log:
logger:
"""
# Make sure we can use the 0th sequencer as a reference that respects
# all the sequences (same batch-size, margins etc.)
_assert_comparable_sequencers(sequencers)
super().__init__()
self.logger = logger or ScreenLogger()
self.sequences = sequencers
self.sequences_idxs = np.arange(len(self.sequences))
self.batch_size = batch_size
self.margin = sequencers[0].margin
self.n_classes = sequencers[0].n_classes
# Compute probability of sampling a given dataset
# We sample a given dataset either:
# 1) Uniformly across datasets, independent of dataset size
# 2) Unifomrly across records, sample a dataset according to size
# The 'dataset_sample_alpha' parameter in [0...1] determines the
# degree to which strategy 1 or 2 is followed:
# P = (1-alpha)*P_r + alpha*P_d
# If alpha = 0, sample only according to
n_samples = [len(s.dataset_queue) for s in sequencers]
linear = n_samples / np.sum(n_samples)
uniform = np.array([1 / len(self.sequences)] * len(self.sequences))
self.alpha = dataset_sample_alpha
self.sample_prob = (1 - self.alpha) * linear + self.alpha * uniform
for s in self.sequences:
s.batch_size = 1
if not no_log:
self.log()
def __init__(self, validation_data, n_classes, batch_size=8, logger=None):
"""
Args:
validation_data: A tuple (X, y) of two ndarrays of validation data
and corresponding labels.
Any shape accepted by the model.
Labels must be integer targets (not one-hot)
n_classes: Number of classes, including background
batch_size: Batch size used for prediction
logger: An instance of a MultiPlanar Logger that prints to screen
and/or file
"""
super().__init__()
self.logger = logger or ScreenLogger()
self.X_val, self.y_val = validation_data
self.n_classes = n_classes
self.batch_size = batch_size
self.scores = []
def __init__(self, model, logger=None):
"""
Init. simply accepts a model and stores it.
Optionally, an 'org_model' (original model) may be passed and stored
as well. This is for training multi-GPU models prepared by the
tf.keras.utils.multi_gpu_model utility, which returns a new, split
model for training (passed as 'model' parameter here). For properly
saving the model parameter, however, it is recommended to use the
original, non-split model (here passed as 'org_model').
Args:
model: (tf.keras Model) Initialized model to train
org_model: (tf.keras Model) Optional single-GPU version for the
passed 'model' parameter.
logger: (Logger) Optional Logger instance
"""
self.model = model
self.logger = logger if logger is not None else ScreenLogger()
def __init__(self,
sleep_study_pairs,
n_classes,
n_channels,
batch_size,
batch_scaler,
logger=None,
require_all_loaded=True,
identifier=""):
"""
Args:
sleep_study_pairs: (list) A list of SleepStudy objects
n_classes: (int) Number of classes (sleep stages)
n_channels: (int) The number of PSG channels to expect in
data extracted from a SleepStudy object
batch_size: (int) The size of the generated batch
batch_scaler: (string) The name of a sklearn.preprocessing
Scaler object to apply to each sampled
batch (optional)
logger: (Logger) A Logger object
identifier: (string) A string identifier name
"""
super().__init__()
self._all_loaded = assert_all_loaded(sleep_study_pairs,
raise_=require_all_loaded)
self.identifier = identifier
self.pairs = sleep_study_pairs
self.id_to_pair = {pair.identifier: pair for pair in self.pairs}
self.n_classes = int(n_classes)
self.n_channels = int(n_channels)
self.logger = logger or ScreenLogger()
self.batch_size = batch_size
if self.all_loaded:
self._periods_per_pair = np.array(
[ss.n_periods for ss in self.pairs])
self._cum_periods_per_pair = np.cumsum(self.periods_per_pair)
if batch_scaler not in (None, False):
if not assert_scaler(batch_scaler):
raise ValueError(
"Invalid batch scaler {}".format(batch_scaler))
self.batch_scaler = batch_scaler
else:
self.batch_scaler = None
def __init__(self, val_sequence, steps, logger=None, verbose=True):
"""
Args:
val_sequence: A deepsleep ValidationMultiSequence object
steps: Numer of batches to sample from val_sequences in each
validation epoch for each validation set
logger: An instance of a MultiPlanar Logger that prints to
screen and/or file
verbose: Print progress to screen - OBS does not use Logger
"""
super().__init__()
self.logger = logger or ScreenLogger()
self.sequences = val_sequence.sequences
self.steps = steps
self.verbose = verbose
self.n_classes = val_sequence.n_classes
self.IDs = val_sequence.IDs
self.print_round = 3
self.log_round = 4
def __init__(self,
val_sequence,
max_val_studies_per_dataset=20,
logger=None,
verbose=True):
"""
Args:
val_sequence: A deepsleep ValidationMultiSequence object
logger: An instance of a MultiPlanar Logger that prints to
screen and/or file
verbose: Print progress to screen - OBS does not use Logger
"""
super().__init__()
self.logger = logger or ScreenLogger()
self.sequences = val_sequence.sequences
self.verbose = verbose
self.max_studies = max_val_studies_per_dataset
self.n_classes = val_sequence.n_classes
self.IDs = val_sequence.IDs
self.print_round = 3
self.log_round = 4
