def __init__(self, layer, every=10, first=10, per_epoch=False, logger=None): """ Args: layer: A tf.keras layer every: Print the weights every 'every' batch or epoch if per_epoch=True first: Print the first 'first' elements of each weight matrix per_epoch: Print after 'every' epoch instead of batch logger: An instance of a MultiPlanar Logger that prints to screen and/or file """ super().__init__() if isinstance(layer, int): self.layer = self.model.layers[layer] else: self.layer = layer self.first = first self.every = every self.logger = logger or ScreenLogger() self.per_epoch = per_epoch if per_epoch: # Apply on every epoch instead of per batches self.on_epoch_begin = self.on_batch_begin self.on_batch_begin = lambda *args, **kwargs: None self.log()
def __init__(self, val_sequence, steps, logger=None, verbose=True, ignore_class_zero=True): """ Args: val_sequence: A mpunet.sequence object from which validation batches can be sampled via its __getitem__ method. steps: Numer of batches to sample from val_sequences in each validation epoch logger: An instance of a MultiPlanar Logger that prints to screen and/or file verbose: Print progress to screen - OBS does not use Logger ignore_class_zero: TODO """ super().__init__() self.logger = logger or ScreenLogger() self.data = val_sequence self.steps = steps self.verbose = verbose self.ignore_bg = ignore_class_zero self.print_round = 3 self.log_round = 4 self.n_classes = self.data.n_classes if isinstance(self.n_classes, int): self.task_names = [""] self.n_classes = [self.n_classes] else: self.task_names = self.data.task_names
def __init__(self, annotation_dict, period_length_sec, no_hypnogram, logger=None): """ TODO Args: annotation_dict: period_length_sec: no_hypnogram: logger: """ self.logger = logger or ScreenLogger() self.annotation_dict = annotation_dict self.no_hypnogram = no_hypnogram self.period_length_sec = period_length_sec or \ Defaults.get_default_period_length(self.logger) # Hidden attributes controlled in property functions to limit setting # of these values to the load() function self._psg = None self._hypnogram = None self._select_channels = None self._alternative_select_channels = None
def plot_all_training_curves(glob_path, out_path, raise_error=False, logger=None, **kwargs): logger = logger or ScreenLogger() try: from glob import glob paths = glob(glob_path) if not paths: raise OSError("File pattern {} gave none or too many matches " \ "({})".format(glob_path, paths)) out_folder = os.path.split(out_path)[0] for p in paths: if len(paths) > 1: # Set unique names uniq = os.path.splitext(os.path.split(p)[-1])[0] f_name = uniq + "_" + os.path.split(out_path)[-1] save_path = os.path.join(out_folder, f_name) else: save_path = out_path plot_training_curves(p, save_path, **kwargs) except Exception as e: s = "Could not plot training curves. ({})".format(e) if raise_error: raise RuntimeError(s) from e else: logger.warn(s)
def __init__(self, batch_shape, n_classes, padding="valid", activation="relu", use_dropout=True, use_bn=True, classify=True, flatten=True, l2_reg=0.0, logger=None, log=True, build=True, **unused): super(DeepFeatureNet, self).__init__() self.logger = logger or ScreenLogger() self.batch_shape = standardize_batch_shape(batch_shape) self.n_classes = n_classes self.use_dropout = use_dropout self.padding = padding self.activation = activation self.use_bn = use_bn self.classify = classify self.flatten = flatten self.l2_reg = l2_reg self.reg = None self.model_name = "DeepFeatureNet" # Build model and init base keras Model class if build: with tf.name_scope(self.model_name): super(DeepFeatureNet, self).__init__(*self.init_model()) if log: self.log()
def __init__(self, image_pair_loader, dim, n_classes, batch_size, is_validation=False, label_crop=None, fg_batch_fraction=0.33, logger=None, bg_val=0., no_log=False, **kwargs): super().__init__() # Set logger or default print self.logger = logger or ScreenLogger() self.image_pair_loader = image_pair_loader # Box dimension and image dim self.dim = dim # Various attributes self.n_classes = n_classes self.label_crop = label_crop self.is_validation = is_validation self.batch_size = batch_size self.bg_value = bg_val # How many foreground slices should be in each batch? self.fg_batch_fraction = fg_batch_fraction # Foreground label settings self.fg_classes = np.arange(1, self.n_classes) if self.fg_classes.shape[0] == 0: self.fg_classes = 1 if not is_validation and not no_log: self.log()
def sample_random_views_with_angle_restriction(views, min_angle_deg, auditor=None, logger=None): from itertools import combinations from mpunet.logging import ScreenLogger logger = logger or ScreenLogger() logger("Generating %i random views..." % views) # Weight by median sample resolution along each axis if auditor is not None: res = np.median(auditor.info["pixdims"], axis=0) logger("[OBS] Weighting random views by median res: %s" % res) else: res = None N = views found = False tries = 0 while not found: tries += 1 views = get_random_views(N, dim=3, pos_z=True, weights=res) angles = [get_angle(v1, v2) for v1, v2 in combinations(views, 2)] found = np.all(np.asarray(angles) > min_angle_deg) min_angle_deg -= 1 return views
def init_model(build_hparams, logger=None, clear_previous=True): """ From a set of hyperparameters 'build_hparams' (dict) initializes the model specified under build_hparams['model_class_name']. Typically, this function is not called directly, but used by the higher-level 'initialize_model' function. Args: build_hparams: A dictionary of model build hyperparameters logger: A Logger instance clear_previous: Clear previous tf sessions Returns: A tf.keras Model instance """ from utime import models logger = logger or ScreenLogger() if clear_previous: import tensorflow as tf tf.keras.backend.clear_session() # Build new model of the specified type cls_name = build_hparams["model_class_name"] logger("Creating new model of type '%s'" % cls_name) return models.__dict__[cls_name](logger=logger, **build_hparams)
def __init__(self, sequences, no_log=False, logger=None): _assert_comparable_sequencers(sequences) self.sequences = sequences self.IDs = [s.identifier.split("/")[0] for s in self.sequences] self.n_classes = self.sequences[0].n_classes self.logger = logger or ScreenLogger() if not no_log: self.log()
def __init__(self, logger=None): """ Args: logger: An instance of a MultiPlanar Logger that prints to screen and/or file """ super().__init__() self.logger = logger or ScreenLogger()
def __init__(self, sequencers, task_names, logger=None): super().__init__() self.logger = logger or ScreenLogger() self.task_names = task_names self.sequencers = sequencers self.log() # Redirect setattrs to the sub-sequences self.redirect = True
def __init__(self, max_minutes, log_name='train_time_total', logger=None): """ TODO Args: """ super().__init__() self.max_minutes = int(max_minutes) self.log_name = log_name self.logger = logger or ScreenLogger()
def __init__(self, logger=None, max_minutes=None, verbose=1): super().__init__() self.logger = logger or ScreenLogger() self.max_minutes = int(max_minutes) if max_minutes else None self.verbose = bool(verbose) # Timing attributes self.train_begin_time = None self.prev_epoch_time = None
def prepare_for_continued_training(hparams, project_dir, logger=None): """ Prepares the hyperparameter set and project directory for continued training. Will find the latest model (highest epoch number) of parameter files in the 'model' subdir of 'project_dir' and base the continued training on this file. If no file is found, training will start from scratch as normally (note: no error is raised, but None is returned instead of a path to a parameter file). The hparams['fit']['init_epoch'] parameter will be set to match the found parameter file or to 0 if no file was found. Note that if init_epoch is set to 0 all rows in the training.csv file will be deleted. The hparams['fit']['optimizer_kwargs']['learning_rate'] parameter will be set according to the value stored in the project_dir/logs/training.csv file at the corresponding epoch (left default if no init_epoch was found) Args: hparams: (YAMLHParams) The hyperparameters to use for training project_dir: (string) The path to the current project directory logger: (Logger) An optional Logger instance Returns: A path to the model weight files to use for continued training. Will be None if no model files were found """ from mpunet.utils import (get_last_model, get_lr_at_epoch, get_last_epoch, clear_csv_after_epoch) model_path, epoch = get_last_model(os.path.join(project_dir, "model")) if model_path: model_name = os.path.split(model_path)[-1] else: model_name = None csv_path = os.path.join(project_dir, "logs", "training.csv") if epoch == 0: epoch = get_last_epoch(csv_path) else: if epoch is None: epoch = 0 clear_csv_after_epoch(epoch, csv_path) hparams["fit"]["init_epoch"] = epoch + 1 # Get the LR at the continued epoch lr, name = get_lr_at_epoch(epoch, os.path.join(project_dir, "logs")) if lr: hparams["fit"]["optimizer_kwargs"][name] = lr logger = logger or ScreenLogger() logger("[NOTICE] Training continues from:\n" "Model: {}\n" "Epoch: {}\n" "LR: {}".format( model_name or "<No model found - " "Starting for scratch!>", epoch, lr)) return model_path
def __init__(self, image_pair_loader, dim, batch_size, n_classes, real_space_span=None, noise_sd=0., force_all_fg="auto", fg_batch_fraction=0.50, label_crop=None, logger=None, is_validation=False, list_of_augmenters=None, flatten_y=False, **kwargs): super().__init__() # Validation or training batch generator? self.is_validation = is_validation # Set logger or default print self.logger = logger or ScreenLogger() # Set views and attributes for plane sample generation self.sample_dim = dim self.real_space_span = real_space_span self.noise_sd = noise_sd if not self.is_validation else 0. # Set data self.image_pair_loader = image_pair_loader self.images = image_pair_loader.images # Augmenter, applied to batch at creation time # Do not augment validation data self.list_of_augmenters = list_of_augmenters if not self.is_validation else None # Batch creation options self.batch_size = batch_size self.n_classes = n_classes self.flatten_y = flatten_y # Minimum fraction of slices in each batch with FG self.force_all_fg_switch = force_all_fg self.fg_batch_fraction = fg_batch_fraction # Foreground label settings self.fg_classes = np.arange(1, self.n_classes) if self.fg_classes.shape[0] == 0: self.fg_classes = [1] # Set potential label label_crop self.label_crop = np.array([[0, 0], [0, 0] ]) if label_crop is None else label_crop
def __init__(self, callback, start_from=0, logger=None): """ Args: callback: A tf.keras callback start_from: Delay the activity of 'callback' until this epoch 'start_from' logger: An instance of a MultiPlanar Logger that prints to screen and/or file """ self.logger = logger or ScreenLogger() self.callback = callback self.start_from = start_from
def load_from_file(model, file_path, logger=None, by_name=True): """ Load parameters from file 'file_path' into model 'model'. Args: model: A tf.keras Model instance file_path: A path to a parameter file (h5 format typically) logger: An optional Logger instance by_name: Load parameters by layer names instead of order (default). """ model.load_weights(file_path, by_name=by_name) logger = logger or ScreenLogger() logger("Loading parameters from:\n{}".format(file_path))
def __init__(self, logger=None): self.logger = logger or ScreenLogger() # Prepare signal self.stop_signal = Event() self.run_signal = Event() self.set_signal = Event() # Stores list of available GPUs self._free_GPUs = Queue() super(GPUMonitor, self).__init__(target=self._monitor) self.start()
def save_images(train, val, out_dir, logger): logger = logger or ScreenLogger() # Write a few images to disk im_path = out_dir if not os.path.exists(im_path): os.mkdir(im_path) # Sample two batches of training and (if passed) validation data training = [train[0], train[1]] if val is not None and len(val) != 0: validation = [val[0], val[1]] val_bs = val.batch_size else: validation = None val_bs = 0 logger("Saving %i sample images in '<project_dir>/images' folder" % ((train.batch_size + val_bs) * 2)) for rr, (train_batch, val_batch) in enumerate(zip(training, validation)): for k, temp in enumerate((train_batch, val_batch)): if temp is None: # No validation data continue X, Y, W = temp for i, (xx, yy, ww) in enumerate(zip(X, Y, W)): yy = yy.reshape(xx.shape[:-1] + (yy.shape[-1], )) # Make figure fig = plt.figure(figsize=(10, 4)) ax1 = fig.add_subplot(121) ax2 = fig.add_subplot(122) # Plot image and overlayed labels chnl, view, _ = imshow_with_label_overlay(ax1, xx, yy) # Plot histogram ax2.hist(xx.flatten(), bins=200) # Set labels ax1.set_title("Channel %i - Axis %i - " "Weight %.3f" % (chnl, view, ww), size=18) # Get path out_path = im_path + "/%s%i.png" % ("train" if k == 0 else "val", len(X) * rr + i) with np.testing.suppress_warnings() as sup: sup.filter(UserWarning) fig.savefig(out_path) plt.close(fig)
def __init__(self, train_data, val_data=None, logger=None): """ Args: train_data: A mpunet.sequence object representing the training data val_data: A mpunet.sequence object representing the validation data logger: An instance of a MultiPlanar Logger that prints to screen and/or file """ super().__init__() self.data = (("train", train_data), ("val", val_data)) self.logger = logger or ScreenLogger() self.active = True
def __init__(self, sequencers, batch_size, no_log=False, logger=None): # Make sure we can use the 0th sequencer as a reference that respects # all the sequences (same batch-size, margins etc.) _assert_comparable_sequencers(sequencers) super().__init__() self.logger = logger or ScreenLogger() self.sequences = sequencers self.batch_size = batch_size self.margin = sequencers[0].margin self.n_classes = sequencers[0].n_classes for s in self.sequences: s.batch_size = 1 if not no_log: self.log()
def __init__(self, n_inputs, n_classes, weight="Simple", logger=None, verbose=True): self.n_inputs = n_inputs self.n_classes = n_classes # Set Logger object self.logger = logger or ScreenLogger() # Set loss self.loss = SparseGeneralizedDiceLoss(tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE, type_weight=weight) # Init model super().__init__(*self.init_model(n_inputs, n_classes)) if verbose: self._log()
def __init__(self, dataset_queue, n_classes, n_channels, batch_size, augmenters, batch_scaler, logger=None, require_all_loaded=False, identifier=""): """ Args: dataset_queue: (queue) TODO n_classes: (int) Number of classes (sleep stages) n_channels: (int) The number of PSG channels to expect in data extracted from a SleepStudy object batch_size: (int) The size of the generated batch augmenters: (list) List of utime.augmentation.augmenters batch_scaler: (string) The name of a sklearn.preprocessing Scaler object to apply to each sampled batch (optional) logger: (Logger) A Logger object identifier: (string) A string identifier name """ super().__init__() self._all_loaded = assert_all_loaded(dataset_queue.dataset.pairs, raise_=require_all_loaded) self.identifier = identifier self.dataset_queue = dataset_queue self.n_classes = int(n_classes) self.n_channels = int(n_channels) self.logger = logger or ScreenLogger() self.augmenters = augmenters or [] self.augmentation_enabled = bool(augmenters) self.batch_size = batch_size if self.all_loaded: self._periods_per_pair = np.array( [ss.n_periods for ss in self.dataset_queue]) self._cum_periods_per_pair = np.cumsum(self.periods_per_pair) if batch_scaler not in (None, False): if not assert_scaler(batch_scaler): raise ValueError( "Invalid batch scaler {}".format(batch_scaler)) self.batch_scaler = batch_scaler else: self.batch_scaler = None
def __init__(self, log_dir="logs", out_dir="logs", fname="curve.png", csv_regex="*training.csv", logger=None): """ Args: log_dir: Relative path from the out_dir: fname: csv_regex: logger: """ super().__init__() out_dir = os.path.abspath(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) self.csv_regex = os.path.join(os.path.abspath(log_dir), csv_regex) self.save_path = os.path.join(out_dir, fname) self.logger = logger or ScreenLogger()
def __init__(self, sequencers, batch_size, dataset_sample_alpha=0.5, no_log=False, logger=None): """ TODO Args: sequencers: batch_size: dataset_sample_alpha: TODO no_log: logger: """ # Make sure we can use the 0th sequencer as a reference that respects # all the sequences (same batch-size, margins etc.) _assert_comparable_sequencers(sequencers) super().__init__() self.logger = logger or ScreenLogger() self.sequences = sequencers self.sequences_idxs = np.arange(len(self.sequences)) self.batch_size = batch_size self.margin = sequencers[0].margin self.n_classes = sequencers[0].n_classes # Compute probability of sampling a given dataset # We sample a given dataset either: # 1) Uniformly across datasets, independent of dataset size # 2) Unifomrly across records, sample a dataset according to size # The 'dataset_sample_alpha' parameter in [0...1] determines the # degree to which strategy 1 or 2 is followed: # P = (1-alpha)*P_r + alpha*P_d # If alpha = 0, sample only according to n_samples = [len(s.dataset_queue) for s in sequencers] linear = n_samples / np.sum(n_samples) uniform = np.array([1 / len(self.sequences)] * len(self.sequences)) self.alpha = dataset_sample_alpha self.sample_prob = (1 - self.alpha) * linear + self.alpha * uniform for s in self.sequences: s.batch_size = 1 if not no_log: self.log()
def __init__(self, validation_data, n_classes, batch_size=8, logger=None): """ Args: validation_data: A tuple (X, y) of two ndarrays of validation data and corresponding labels. Any shape accepted by the model. Labels must be integer targets (not one-hot) n_classes: Number of classes, including background batch_size: Batch size used for prediction logger: An instance of a MultiPlanar Logger that prints to screen and/or file """ super().__init__() self.logger = logger or ScreenLogger() self.X_val, self.y_val = validation_data self.n_classes = n_classes self.batch_size = batch_size self.scores = []
def __init__(self, model, logger=None): """ Init. simply accepts a model and stores it. Optionally, an 'org_model' (original model) may be passed and stored as well. This is for training multi-GPU models prepared by the tf.keras.utils.multi_gpu_model utility, which returns a new, split model for training (passed as 'model' parameter here). For properly saving the model parameter, however, it is recommended to use the original, non-split model (here passed as 'org_model'). Args: model: (tf.keras Model) Initialized model to train org_model: (tf.keras Model) Optional single-GPU version for the passed 'model' parameter. logger: (Logger) Optional Logger instance """ self.model = model self.logger = logger if logger is not None else ScreenLogger()
def __init__(self, sleep_study_pairs, n_classes, n_channels, batch_size, batch_scaler, logger=None, require_all_loaded=True, identifier=""): """ Args: sleep_study_pairs: (list) A list of SleepStudy objects n_classes: (int) Number of classes (sleep stages) n_channels: (int) The number of PSG channels to expect in data extracted from a SleepStudy object batch_size: (int) The size of the generated batch batch_scaler: (string) The name of a sklearn.preprocessing Scaler object to apply to each sampled batch (optional) logger: (Logger) A Logger object identifier: (string) A string identifier name """ super().__init__() self._all_loaded = assert_all_loaded(sleep_study_pairs, raise_=require_all_loaded) self.identifier = identifier self.pairs = sleep_study_pairs self.id_to_pair = {pair.identifier: pair for pair in self.pairs} self.n_classes = int(n_classes) self.n_channels = int(n_channels) self.logger = logger or ScreenLogger() self.batch_size = batch_size if self.all_loaded: self._periods_per_pair = np.array( [ss.n_periods for ss in self.pairs]) self._cum_periods_per_pair = np.cumsum(self.periods_per_pair) if batch_scaler not in (None, False): if not assert_scaler(batch_scaler): raise ValueError( "Invalid batch scaler {}".format(batch_scaler)) self.batch_scaler = batch_scaler else: self.batch_scaler = None
def __init__(self, val_sequence, steps, logger=None, verbose=True): """ Args: val_sequence: A deepsleep ValidationMultiSequence object steps: Numer of batches to sample from val_sequences in each validation epoch for each validation set logger: An instance of a MultiPlanar Logger that prints to screen and/or file verbose: Print progress to screen - OBS does not use Logger """ super().__init__() self.logger = logger or ScreenLogger() self.sequences = val_sequence.sequences self.steps = steps self.verbose = verbose self.n_classes = val_sequence.n_classes self.IDs = val_sequence.IDs self.print_round = 3 self.log_round = 4
def __init__(self, val_sequence, max_val_studies_per_dataset=20, logger=None, verbose=True): """ Args: val_sequence: A deepsleep ValidationMultiSequence object logger: An instance of a MultiPlanar Logger that prints to screen and/or file verbose: Print progress to screen - OBS does not use Logger """ super().__init__() self.logger = logger or ScreenLogger() self.sequences = val_sequence.sequences self.verbose = verbose self.max_studies = max_val_studies_per_dataset self.n_classes = val_sequence.n_classes self.IDs = val_sequence.IDs self.print_round = 3 self.log_round = 4