def _fit(self,
train,
val,
batch_size,
n_epochs,
callbacks,
train_samples_per_epoch,
verbose=1,
init_epoch=0,
**unused):
"""
Args:
train: (Sequence) The training Sequence object
val (Sequence, None) The validation Sequence object or None if no
validation is to be performed
batch_size: (int) The batch size to use for training
n_epochs: (int) Number of epochs to train for
callbacks: (list) List of uninitialized callback kwargs.
train_samples_per_epoch: (int) Number of training samples to sample
before an epoch is determined over.
verbose: (int/bool) Verbosity level passed to keras.fit_generator
init_epoch: (int) The initial epoch
use_multiprocessing: (bool) Whether to use multiprocessing instead
of multithreading.
"""
train.batch_size = batch_size
train_steps = get_steps(train_samples_per_epoch, train)
self.logger("Using {} steps per train epoch".format(train_steps))
if val is None:
# No validation to be performed, remove callbacks that might need
# validation data to function properly
remove_validation_callbacks(callbacks, self.logger)
else:
val.batch_size = batch_size
# Add validation callback
# Important: Should be first in callbacks list as other CBs may
# depend on the validation metrics/loss
validation = Validation(val, logger=self.logger, verbose=verbose)
callbacks = [validation] + callbacks
# Add various callbacks for plotting learning curves etc.
callbacks.append(MemoryConsumption(max_gib=45, logger=self.logger))
callbacks.append(LearningCurve(logger=self.logger))
# callbacks.append(CarbonUsageTracking(epochs=n_epochs, add_to_logs=False))
callbacks.append(DividerLine(self.logger))
# Get initialized callback objects
callbacks, cb_dict = init_callback_objects(callbacks, self.logger)
# If ModelCheckPointClean is used, set the original model to store
# the correct weights when using multi-GPU models
cb = cb_dict.get("ModelCheckPointClean")
if cb:
cb.org_model = self.org_model
# Temporary memory leak fix
import tensorflow as tf
dtypes, shapes = list(
zip(*map(lambda x: (x.dtype, x.shape), train[0])))
train = tf.data.Dataset.from_generator(train, dtypes, shapes)
# Fit the model
self.logger.active_log_file = "training"
self.logger.print_calling_method = False
self.model.fit(
train,
steps_per_epoch=train_steps,
epochs=n_epochs,
callbacks=callbacks,
initial_epoch=init_epoch,
use_multiprocessing=False,
workers=3,
max_queue_size=10,
shuffle=False, # Determined by the chosen Sequence class
verbose=verbose)
def _fit(self,
train,
val,
batch_size,
n_epochs,
callbacks,
train_im_per_epoch,
val_im_per_epoch,
val_ignore_class_zero=True,
no_im=False,
verbose=1,
init_epoch=0,
use_multiprocessing=False,
**unused):
train.batch_size = batch_size
# Get number of steps per train epoch
train_steps = get_steps(train, train_im_per_epoch)
self.logger("Using %i steps per train epoch (total batches=%i)" %
(train_steps, len(train)))
if val is None:
# No validation to be performed, remove callbacks that might need
# validation data to function properly
remove_validation_callbacks(callbacks, self.logger)
else:
val.batch_size = batch_size
val_steps = get_steps(val, val_im_per_epoch)
self.logger("Using %i steps per val epoch (total batches=%i)" %
(val_steps, len(val)))
# Add validation callback
# Important: Should be first in callbacks list as other CBs may
# depend on the validation metrics/loss
validation = Validation(val,
steps=val_steps,
ignore_class_zero=val_ignore_class_zero,
logger=self.logger,
verbose=verbose)
callbacks = [validation] + callbacks
# Add various callbacks for plotting learning curves etc.
# Get FGBatchBalancer callbacks, etc.
if hasattr(train, "n_fg_slices"):
callbacks.append(FGBatchBalancer(train, logger=self.logger))
if not no_im:
# Add save images cb
callbacks.append(SavePredictionImages(train, val))
callbacks.insert(1, MeanReduceLogArrays())
# callbacks.insert(1, MemoryConsumption(logger=self.logger))
callbacks.append(LearningCurve(logger=self.logger))
callbacks.append(DividerLine(self.logger))
# Get initialized callback objects
callbacks, cb_dict = init_callback_objects(callbacks, self.logger)
# If ModelCheckPointClean is used, set the original model to store
# the correct weights when using multi-GPU models
cb = cb_dict.get("ModelCheckPointClean")
if cb:
cb.org_model = self.model # TEMP TODO
# Init TF dataset with DATA autosharding
dtypes, shapes = list(
zip(*map(lambda x: (x.dtype, x.shape), train[0])))
train = tf.data.Dataset.from_generator(train, dtypes, shapes)
# Fit the model
# is_queued = bool(train.image_pair_loader.queue)
self.logger.active_log_file = "training"
self.logger.print_calling_method = False
self.model.fit(
train,
steps_per_epoch=train_steps,
epochs=n_epochs,
callbacks=callbacks,
initial_epoch=init_epoch,
use_multiprocessing=use_multiprocessing,
workers=5,
max_queue_size=5,
shuffle=False, # Determined by the chosen Sequence class
verbose=verbose)
def _fit(self,
train,
val,
batch_size,
n_epochs,
callbacks,
train_samples_per_epoch,
val_samples_per_epoch,
verbose=1,
init_epoch=0,
use_multiprocessing=False,
**unused):
"""
Args:
train: (Sequence) The training Sequence object
val (Sequence, None) The validation Sequence object or None if no
validation is to be performed
batch_size: (int) The batch size to use for training
n_epochs: (int) Number of epochs to train for
callbacks: (list) List of uninitialized callback kwargs.
train_samples_per_epoch: (int) Number of training samples to sample
before an epoch is determined over.
val_samples_per_epoch: (int) Same as 'train_samples_per_epoch'
verbose: (int/bool) Verbosity level passed to keras.fit_generator
init_epoch: (int) The initial epoch
use_multiprocessing: (bool) Whether to use multiprocessing instead
of multithreading.
"""
train.batch_size = batch_size
train_steps = get_steps(train_samples_per_epoch, train)
self.logger("Using %i steps per train epoch (total batches=%i)" %
(train_steps, len(train)))
if val is None or len(val) == 0:
# No validation to be performed, remove callbacks that might need
# validation data to function properly
remove_validation_callbacks(callbacks, self.logger)
else:
val.batch_size = batch_size
val_steps = get_steps(val_samples_per_epoch, val)
self.logger("Using %i steps per validation epoch "
"(total batches=%i)" % (val_steps, len(val)))
# Add validation callback
# Important: Should be first in callbacks list as other CBs may
# depend on the validation metrics/loss
validation = Validation(val,
steps=val_steps,
logger=self.logger,
verbose=verbose)
callbacks = [validation] + callbacks
# Callback for plotting learning curves
callbacks.append(LearningCurve(logger=self.logger))
callbacks = callbacks + [DividerLine(self.logger)]
# Get initialized callback objects
callbacks, cb_dict = init_callback_objects(callbacks, self.logger)
# If ModelCheckPointClean is used, set the original model to store
# the correct weights when using multi-GPU models
cb = cb_dict.get("ModelCheckPointClean")
if cb:
cb.org_model = self.org_model
# Fit the model
self.logger.active_log_file = "training"
self.logger.print_calling_method = False
self.model.fit_generator(
generator=train,
steps_per_epoch=train_steps,
epochs=n_epochs,
callbacks=callbacks,
initial_epoch=init_epoch,
use_multiprocessing=use_multiprocessing, # Normally False
workers=min(7,
cpu_count() - 1),
max_queue_size=25,
shuffle=False, # Determined by the chosen Sequence class
verbose=verbose)