def _fit(self, train, val, batch_size, n_epochs, callbacks, train_samples_per_epoch, verbose=1, init_epoch=0, **unused): """ Args: train: (Sequence) The training Sequence object val (Sequence, None) The validation Sequence object or None if no validation is to be performed batch_size: (int) The batch size to use for training n_epochs: (int) Number of epochs to train for callbacks: (list) List of uninitialized callback kwargs. train_samples_per_epoch: (int) Number of training samples to sample before an epoch is determined over. verbose: (int/bool) Verbosity level passed to keras.fit_generator init_epoch: (int) The initial epoch use_multiprocessing: (bool) Whether to use multiprocessing instead of multithreading. """ train.batch_size = batch_size train_steps = get_steps(train_samples_per_epoch, train) self.logger("Using {} steps per train epoch".format(train_steps)) if val is None: # No validation to be performed, remove callbacks that might need # validation data to function properly remove_validation_callbacks(callbacks, self.logger) else: val.batch_size = batch_size # Add validation callback # Important: Should be first in callbacks list as other CBs may # depend on the validation metrics/loss validation = Validation(val, logger=self.logger, verbose=verbose) callbacks = [validation] + callbacks # Add various callbacks for plotting learning curves etc. callbacks.append(MemoryConsumption(max_gib=45, logger=self.logger)) callbacks.append(LearningCurve(logger=self.logger)) # callbacks.append(CarbonUsageTracking(epochs=n_epochs, add_to_logs=False)) callbacks.append(DividerLine(self.logger)) # Get initialized callback objects callbacks, cb_dict = init_callback_objects(callbacks, self.logger) # If ModelCheckPointClean is used, set the original model to store # the correct weights when using multi-GPU models cb = cb_dict.get("ModelCheckPointClean") if cb: cb.org_model = self.org_model # Temporary memory leak fix import tensorflow as tf dtypes, shapes = list( zip(*map(lambda x: (x.dtype, x.shape), train[0]))) train = tf.data.Dataset.from_generator(train, dtypes, shapes) # Fit the model self.logger.active_log_file = "training" self.logger.print_calling_method = False self.model.fit( train, steps_per_epoch=train_steps, epochs=n_epochs, callbacks=callbacks, initial_epoch=init_epoch, use_multiprocessing=False, workers=3, max_queue_size=10, shuffle=False, # Determined by the chosen Sequence class verbose=verbose)
def _fit(self, train, val, batch_size, n_epochs, callbacks, train_im_per_epoch, val_im_per_epoch, val_ignore_class_zero=True, no_im=False, verbose=1, init_epoch=0, use_multiprocessing=False, **unused): train.batch_size = batch_size # Get number of steps per train epoch train_steps = get_steps(train, train_im_per_epoch) self.logger("Using %i steps per train epoch (total batches=%i)" % (train_steps, len(train))) if val is None: # No validation to be performed, remove callbacks that might need # validation data to function properly remove_validation_callbacks(callbacks, self.logger) else: val.batch_size = batch_size val_steps = get_steps(val, val_im_per_epoch) self.logger("Using %i steps per val epoch (total batches=%i)" % (val_steps, len(val))) # Add validation callback # Important: Should be first in callbacks list as other CBs may # depend on the validation metrics/loss validation = Validation(val, steps=val_steps, ignore_class_zero=val_ignore_class_zero, logger=self.logger, verbose=verbose) callbacks = [validation] + callbacks # Add various callbacks for plotting learning curves etc. # Get FGBatchBalancer callbacks, etc. if hasattr(train, "n_fg_slices"): callbacks.append(FGBatchBalancer(train, logger=self.logger)) if not no_im: # Add save images cb callbacks.append(SavePredictionImages(train, val)) callbacks.insert(1, MeanReduceLogArrays()) # callbacks.insert(1, MemoryConsumption(logger=self.logger)) callbacks.append(LearningCurve(logger=self.logger)) callbacks.append(DividerLine(self.logger)) # Get initialized callback objects callbacks, cb_dict = init_callback_objects(callbacks, self.logger) # If ModelCheckPointClean is used, set the original model to store # the correct weights when using multi-GPU models cb = cb_dict.get("ModelCheckPointClean") if cb: cb.org_model = self.model # TEMP TODO # Init TF dataset with DATA autosharding dtypes, shapes = list( zip(*map(lambda x: (x.dtype, x.shape), train[0]))) train = tf.data.Dataset.from_generator(train, dtypes, shapes) # Fit the model # is_queued = bool(train.image_pair_loader.queue) self.logger.active_log_file = "training" self.logger.print_calling_method = False self.model.fit( train, steps_per_epoch=train_steps, epochs=n_epochs, callbacks=callbacks, initial_epoch=init_epoch, use_multiprocessing=use_multiprocessing, workers=5, max_queue_size=5, shuffle=False, # Determined by the chosen Sequence class verbose=verbose)
def _fit(self, train, val, batch_size, n_epochs, callbacks, train_samples_per_epoch, val_samples_per_epoch, verbose=1, init_epoch=0, use_multiprocessing=False, **unused): """ Args: train: (Sequence) The training Sequence object val (Sequence, None) The validation Sequence object or None if no validation is to be performed batch_size: (int) The batch size to use for training n_epochs: (int) Number of epochs to train for callbacks: (list) List of uninitialized callback kwargs. train_samples_per_epoch: (int) Number of training samples to sample before an epoch is determined over. val_samples_per_epoch: (int) Same as 'train_samples_per_epoch' verbose: (int/bool) Verbosity level passed to keras.fit_generator init_epoch: (int) The initial epoch use_multiprocessing: (bool) Whether to use multiprocessing instead of multithreading. """ train.batch_size = batch_size train_steps = get_steps(train_samples_per_epoch, train) self.logger("Using %i steps per train epoch (total batches=%i)" % (train_steps, len(train))) if val is None or len(val) == 0: # No validation to be performed, remove callbacks that might need # validation data to function properly remove_validation_callbacks(callbacks, self.logger) else: val.batch_size = batch_size val_steps = get_steps(val_samples_per_epoch, val) self.logger("Using %i steps per validation epoch " "(total batches=%i)" % (val_steps, len(val))) # Add validation callback # Important: Should be first in callbacks list as other CBs may # depend on the validation metrics/loss validation = Validation(val, steps=val_steps, logger=self.logger, verbose=verbose) callbacks = [validation] + callbacks # Callback for plotting learning curves callbacks.append(LearningCurve(logger=self.logger)) callbacks = callbacks + [DividerLine(self.logger)] # Get initialized callback objects callbacks, cb_dict = init_callback_objects(callbacks, self.logger) # If ModelCheckPointClean is used, set the original model to store # the correct weights when using multi-GPU models cb = cb_dict.get("ModelCheckPointClean") if cb: cb.org_model = self.org_model # Fit the model self.logger.active_log_file = "training" self.logger.print_calling_method = False self.model.fit_generator( generator=train, steps_per_epoch=train_steps, epochs=n_epochs, callbacks=callbacks, initial_epoch=init_epoch, use_multiprocessing=use_multiprocessing, # Normally False workers=min(7, cpu_count() - 1), max_queue_size=25, shuffle=False, # Determined by the chosen Sequence class verbose=verbose)