def setup_callback_list(self, model_name):
if model_name in self.callback_lists:
return self.callback_lists[model_name]
model = self.models[model_name]
callbacks = self.callbacks[model_name] \
if model_name in self.callbacks else []
# Prepare callbacks for autoencoder model
all_callbacks = [cbks.BaseLogger()] + callbacks + [cbks.History()]
all_callbacks = cbks.CallbackList(all_callbacks)
out_labels = model.metrics_names
if self.do_validation:
callback_metrics = copy.copy(out_labels) + \
["val_" + l for l in out_labels]
else:
callback_metrics = copy.copy(out_labels)
callback_list = cbks.CallbackList(all_callbacks)
callback_list.set_params({
'batch_size': self.batch_size,
'epochs': self.epochs,
'verbose': 2,
'do_validation': model_name in self.do_validation,
'metrics': callback_metrics or [],
})
callback_list.set_model(model)
return callback_list
def test_callback_list_methods(self):
counter = Counter()
callback_list = callbacks.CallbackList([counter])
batch = 0
callback_list.on_test_batch_begin(batch)
callback_list.on_test_batch_end(batch)
callback_list.on_predict_batch_begin(batch)
callback_list.on_predict_batch_end(batch)
self._check_counts(
counter, {
'on_test_batch_begin': 1,
'on_test_batch_end': 1,
'on_predict_batch_begin': 1,
'on_predict_batch_end': 1,
'on_predict_begin': 0,
'on_predict_end': 0,
'on_batch_begin': 0,
'on_batch_end': 0,
'on_epoch_begin': 0,
'on_epoch_end': 0,
'on_test_begin': 0,
'on_test_end': 0,
'on_train_batch_begin': 0,
'on_train_batch_end': 0,
'on_train_begin': 0,
'on_train_end': 0,
})
def build_callbacks(conf):
'''
The purpose of the method is to set up logging and history. It is based on
Keras Callbacks
https://github.com/fchollet/keras/blob/fbc9a18f0abc5784607cd4a2a3886558efa3f794/keras/callbacks.py
Currently used callbacks include: BaseLogger, CSVLogger, EarlyStopping.
Other possible callbacks to add in future:
RemoteMonitor, LearningRateScheduler
Argument list:
- conf: There is a "callbacks" section in conf.yaml file.
Relevant parameters are:
list: Parameter specifying additional callbacks, read in the driver
script and passed as an argument of type list (see next arg)
metrics: List of quantities monitored during training and
validation
mode: one of {auto, min, max}. The decision to overwrite the
current save file is made based on either the maximization or the
minimization of the monitored quantity. For val_acc, this should be max,
for val_loss this should be min, etc. In auto mode, the direction is
automatically inferred from the name of the monitored quantity.
monitor: Quantity used for early stopping, has to be from the list
of metrics
patience: Number of epochs used to decide on whether to apply early
stopping or continue training
- callbacks_list: uses callbacks.list configuration parameter,
specifies the list of additional callbacks
Returns:
modified list of callbacks
'''
# mode = conf['callbacks']['mode']
# monitor = conf['callbacks']['monitor']
# patience = conf['callbacks']['patience']
csvlog_save_path = conf['paths']['csvlog_save_path']
# CSV callback is on by default
if not os.path.exists(csvlog_save_path):
os.makedirs(csvlog_save_path)
# callbacks_list = conf['callbacks']['list']
callbacks = [cbks.BaseLogger()]
callbacks += [
cbks.CSVLogger("{}callbacks-{}.log".format(
csvlog_save_path,
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")))
]
return cbks.CallbackList(callbacks)
def _fit(self,
f,
nb_train_sample,
nb_batches,
batch_size=128,
nb_epoch=100,
verbose=1,
callbacks=[],
shuffle=True,
metrics=[]):
"""
Abstract fit function for f(*ins). Assume that f returns a list,
labelled by out_labels. """
history = cbks.History()
callbacks = [cbks.BaseLogger()] + callbacks + [history]
if verbose:
callbacks = callbacks + [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
callbacks._set_model(self)
callbacks._set_params({
'batch_size': nb_train_sample // nb_batches,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': verbose,
'do_validation': False,
'metrics': metrics,
})
callbacks.on_train_begin()
self.stop_training = False
for epoch in range(nb_epoch):
callbacks.on_epoch_begin(epoch)
for batch_index in range(nb_batches):
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
f(self, batch_index, batch_logs)
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
callbacks.on_epoch_end(epoch, epoch_logs)
if self.stop_training:
break
callbacks.on_train_end()
return history
def fit_generator(self,
generator,
nb_epoch,
nb_batches_per_epoch,
callbacks=[],
batch_size=None,
verbose=False):
if batch_size is None:
batch_size = 2 * len(next(generator)[0])
out_labels = ['g', 'd', 'm']
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + callbacks + [self.history]
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
callbacks.set_model(self)
callbacks.set_params({
'nb_epoch': nb_epoch,
'nb_sample': nb_batches_per_epoch * batch_size,
'verbose': verbose,
'metrics': out_labels,
})
callbacks.on_train_begin()
for e in range(nb_epoch):
callbacks.on_epoch_begin(e)
for batch_index, (seq_input, real) in enumerate(generator):
callbacks.on_batch_begin(batch_index)
batch_logs = dict()
batch_logs['batch'] = batch_index
batch_logs['size'] = len(real) + len(seq_input)
outs = self.train_on_batch(seq_input, real)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index + 1 == nb_batches_per_epoch:
break
callbacks.on_epoch_end(e)
callbacks.on_train_end()
def callbacks(model, callbacks, params):
model.history = cbks.History()
_callbacks = [
cbks.BaseLogger(stateful_metrics=model.stateful_metric_names)
]
_callbacks.append(
cbks.ProgbarLogger(count_mode='steps',
stateful_metrics=model.stateful_metric_names))
_callbacks += (callbacks or []) + [model.history]
callbacks = cbks.CallbackList(_callbacks)
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
out_labels = model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
callbacks.set_params({
**params,
'metrics': callback_metrics,
})
return callbacks
def init_callbacks(self, for_worker=False):
"""Prepares all keras callbacks to be used in training.
Automatically attaches a History callback to the end of the callback list.
If for_worker is True, leaves out callbacks that only make sense
with validation enabled."""
import keras.callbacks as cbks
remove_for_worker = [cbks.EarlyStopping, cbks.ModelCheckpoint]
if for_worker:
for obj in remove_for_worker:
self.callbacks_list = [
c for c in self.callbacks_list if not isinstance(c, obj)
]
self.model.history = cbks.History()
self.callbacks = cbks.CallbackList(self.callbacks_list +
[self.model.history])
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self.model, 'callback_model') and self.model.callback_model:
self.callback_model = self.model.callback_model
else:
self.callback_model = self.model
self.callbacks.set_model(self.callback_model)
self.callback_model.stop_training = False
def fit_generator_feed(self,
generator,
steps_per_epoch=None,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
class_weight=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
shuffle=True,
initial_epoch=0,
check_array_lengths=True):
"""Train the model on data generated batch-by-batch by a Python generator
or an instance of `Sequence`.
See `Model.fit_generator()` for the full documentation.
The only difference here is that the generator must also generate data for
any native placeholders of the model.
Only use this if you know what you are doing (especially with the `shuffle`
and `check_array_lengths` parameters). If not, prefer `self.fit_fullbatches()`
or `self.fit_minibatches()`.
"""
# Disable validation, as we haven't converted the code for this yet.
# All related code is commented with a `disabled:` prefix.
if validation_data is not None:
raise ValueError(
'Validation with a feeding generator is not yet supported')
# The original (feed-modified) method starts here.
wait_time = 0.01 # in seconds
epoch = initial_epoch
# disable: do_validation = bool(validation_data)
self._make_train_function()
# disable: if do_validation:
# disable: self._make_test_function()
is_sequence = isinstance(generator, Sequence)
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
if steps_per_epoch is None:
if is_sequence:
steps_per_epoch = len(generator)
else:
raise ValueError(
'`steps_per_epoch=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `steps_per_epoch` or use'
' the `keras.utils.Sequence` class.')
# disable: # python 2 has 'next', 3 has '__next__'
# disable: # avoid any explicit version checks
# disable: val_gen = (hasattr(validation_data, 'next') or
# disable: hasattr(validation_data, '__next__') or
# disable: isinstance(validation_data, Sequence))
# disable: if (val_gen and not isinstance(validation_data, Sequence) and
# disable: not validation_steps):
# disable: raise ValueError('`validation_steps=None` is only valid for a'
# disable: ' generator based on the `keras.utils.Sequence`'
# disable: ' class. Please specify `validation_steps` or use'
# disable: ' the `keras.utils.Sequence` class.')
# Prepare display labels.
out_labels = self.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
self.history = cbks.History()
_callbacks = [
cbks.BaseLogger(stateful_metrics=self.stateful_metric_names)
]
if verbose:
_callbacks.append(
cbks.ProgbarLogger(
count_mode='steps',
stateful_metrics=self.stateful_metric_names))
_callbacks += (callbacks or []) + [self.history]
callbacks = cbks.CallbackList(_callbacks)
# it's possible to callback a different model than self:
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
# disable: 'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
enqueuer = None
# disable: val_enqueuer = None
try:
# disable: if do_validation and not val_gen:
# disable: # Prepare data for validation
# disable: if len(validation_data) == 2:
# disable: val_x, val_y = validation_data
# disable: val_sample_weight = None
# disable: elif len(validation_data) == 3:
# disable: val_x, val_y, val_sample_weight = validation_data
# disable: else:
# disable: raise ValueError('`validation_data` should be a tuple '
# disable: '`(val_x, val_y, val_sample_weight)` '
# disable: 'or `(val_x, val_y)`. Found: ' +
# disable: str(validation_data))
# disable: val_x, val_y, val_sample_weights = self._standardize_user_data(
# disable: val_x, val_y, val_sample_weight)
# disable: val_data = val_x + val_y + val_sample_weights
# disable: if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
# disable: val_data += [0.]
# disable: for cbk in callbacks:
# disable: cbk.validation_data = val_data
if workers > 0:
if is_sequence:
enqueuer = OrderedEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
else:
if is_sequence:
output_generator = iter(generator)
else:
output_generator = generator
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
for m in self.metrics:
if isinstance(m, Layer) and m.stateful:
m.reset_states()
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
generator_output = next(output_generator)
if not hasattr(generator_output, '__len__'):
raise ValueError(
'Output of generator should be '
'a tuple `(x, y, feeds, sample_weight)` '
'or `(x, y, feeds)`. Found: ' +
str(generator_output))
if len(generator_output) == 3:
x, y, feeds = generator_output
sample_weight = None
elif len(generator_output) == 4:
x, y, feeds, sample_weight = generator_output
else:
raise ValueError(
'Output of generator should be '
'a tuple `(x, y, feeds, sample_weight)` '
'or `(x, y, feeds)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if x is None or len(x) == 0:
# Handle data tensors support when no input given
# step-size = 1 for data tensors
batch_size = 1
elif isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = self.train_on_fed_batch(
x,
y,
feeds=feeds,
sample_weight=sample_weight,
class_weight=class_weight,
check_array_lengths=check_array_lengths)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
batch_index += 1
steps_done += 1
# Epoch finished.
# disable: if steps_done >= steps_per_epoch and do_validation:
# disable: if val_gen:
# disable: val_outs = self.evaluate_generator(
# disable: validation_data,
# disable: validation_steps,
# disable: workers=workers,
# disable: use_multiprocessing=use_multiprocessing,
# disable: max_queue_size=max_queue_size)
# disable: else:
# disable: # No need for try/except because
# disable: # data has already been validated.
# disable: val_outs = self.evaluate(
# disable: val_x, val_y,
# disable: batch_size=batch_size,
# disable: sample_weight=val_sample_weights,
# disable: verbose=0)
# disable: if not isinstance(val_outs, list):
# disable: val_outs = [val_outs]
# disable: # Same labels assumed.
# disable: for l, o in zip(out_labels, val_outs):
# disable: epoch_logs['val_' + l] = o
if callback_model.stop_training:
break
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
pass
# disable: if val_enqueuer is not None:
# disable: val_enqueuer.stop()
callbacks.on_train_end()
return self.history
async def fit_generator(model,
generator,
steps_per_epoch=None,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
class_weight=None,
shuffle=True,
initial_epoch=0):
"""See docstring for `Model.fit_generator`."""
epoch = initial_epoch
do_validation = bool(validation_data)
model._make_train_function()
if do_validation:
model._make_test_function()
if steps_per_epoch is None:
steps_per_epoch = len(generator)
# Prepare display labels.
out_labels = model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
model.history = cbks.History()
_callbacks = [
cbks.BaseLogger(stateful_metrics=model.stateful_metric_names)
]
if verbose:
_callbacks.append(
cbks.ProgbarLogger(count_mode='steps',
stateful_metrics=model.stateful_metric_names))
_callbacks += (callbacks or []) + [model.history]
callbacks = cbks.CallbackList(_callbacks)
# it's possible to callback a different model than self:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
output_generator = generator.async_next
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
for m in model.stateful_metric_functions:
m.reset_states()
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
generator_output = await output_generator()
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if x is None or len(x) == 0:
# Handle data tensors support when no input given
# step-size = 1 for data tensors
batch_size = 1
elif isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = model.train_on_batch(x,
y,
sample_weight=sample_weight,
class_weight=class_weight)
outs = to_list(outs)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
val_outs = await evaluate_generator(model, validation_data,
validation_steps)
val_outs = to_list(val_outs)
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
if callback_model.stop_training:
break
generator.on_epoch_end()
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
callbacks.on_train_end()
return model.history
def fit_models(callback_model,
models,
generators,
metrics_names,
batch_size,
steps_per_epoch=None,
epochs=1,
verbose=1,
callbacks=None,
initial_epoch=0):
epoch = initial_epoch
# Prepare display labels.
callback_metrics = [n for m in metrics_names for n in m.keys()]
# prepare callbacks
stateful_metric_names = []
for model in models:
model.history = cbks.History()
try:
stateful_metric_names.extend(model.stateful_metric_names)
except AttributeError:
stateful_metric_names.extend(model.model.stateful_metric_names)
_callbacks = [cbks.BaseLogger(stateful_metrics=stateful_metric_names)]
if verbose:
_callbacks.append(
cbks.ProgbarLogger(count_mode='steps',
stateful_metrics=stateful_metric_names))
_callbacks += (callbacks or []) + [model.history for model in models]
callbacks = cbks.CallbackList(_callbacks)
# it's possible to callback a different model than self:
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': False,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
try:
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
for model in models:
try:
stateful_metric_functions = model.stateful_metric_functions
except AttributeError:
stateful_metric_functions = model.model.stateful_metric_functions
for m in stateful_metric_functions:
m.reset_states()
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
# build batch logs
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
for model, output_generator, metrics in zip(
models, generators, metrics_names):
generator_output = next(output_generator)
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
outs = model.train_on_batch(x,
y,
sample_weight=sample_weight)
if not isinstance(outs, list):
outs = [outs]
for name, i in metrics.items():
batch_logs[name] = outs[i]
callbacks.on_batch_end(batch_index, batch_logs)
batch_index += 1
steps_done += 1
# Epoch finished.
if callback_model.stop_training:
break
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
pass
callbacks.on_train_end()
return [model.history for model in models]
def _fit_loop(self,
f,
ins,
out_labels=None,
batch_size=32,
epochs=100,
verbose=1,
callbacks=None,
val_f=None,
val_ins=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0,
steps_per_epoch=None):
"""Abstract fit function for `f(ins)`.
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: list of tensors to be fed to `f`
out_labels: list of strings, display names of
the outputs of `f`
batch_size: integer batch size
epochs: number of times to iterate over the data
verbose: verbosity mode, 0, 1 or 2
callbacks: list of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: list of tensors to be fed to `val_f`
shuffle: whether to shuffle the data at the beginning of each epoch
callback_metrics: list of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. The default `None` is equal to the number
of unique samples in your dataset divided by the batch
size, or 1 if that cannot be determined.
# Returns
`History` object.
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose and ins and hasattr(ins[0], 'shape'):
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if steps_per_epoch is not None:
num_train_samples = steps_per_epoch
else:
if ins and hasattr(ins[0], 'shape'):
num_train_samples = ins[0].shape[0]
else:
# May happen if we are running `fit` without Numpy input data,
# i.e. if all inputs to the models are data tensors
# instead of placeholders.
# In that case we will run `fit` over a single batch.
num_train_samples = batch_size
verbose = 2
index_array = np.arange(num_train_samples)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
# callbacks += [cbks.ProgbarLogger()]
callbacks += [ProgbarLogger_TFRecord()]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
for cbk in callbacks:
cbk.validation_data = val_ins
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = _batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = \
_slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if batch_index == len(batches) - 1: # Last batch.
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def fit_dataflow(self,
dflow,
steps_per_epoch,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
class_weight=None,
max_q_size=10,
workers=1,
pickle_safe=False,
initial_epoch=0):
"""Fits the model on data yielded batch-by-batch by a Python generator.
The generator is run in parallel to the model, for efficiency.
For instance, this allows you to do real-time data augmentation
on images on CPU in parallel to training your model on GPU.
# Arguments
dflow: a dataflow object a-la-carte Tensorpack.
The output of the generator must be either
- a tuple (inputs, targets)
- a tuple (inputs, targets, sample_weights).
All arrays should contain the same number of samples.
The generator is expected to loop over its data
indefinitely. An epoch finishes when `steps_per_epoch`
samples have been seen by the model.
steps_per_epoch: Total number of steps (batches of samples)
to yield from `generator` before declaring one epoch
finished and starting the next epoch. It should typically
be equal to the number of unique samples if your dataset
divided by the batch size.
epochs: integer, total number of iterations on the data.
verbose: verbosity mode, 0, 1, or 2.
callbacks: list of callbacks to be called during training.
validation_data: this can be either
- a generator for the validation data
- a tuple (inputs, targets)
- a tuple (inputs, targets, sample_weights).
validation_steps: Only relevant if `validation_data`
is a generator. Total number of steps (batches of samples)
to yield from `generator` before stopping.
class_weight: dictionary mapping class indices to a weight
for the class.
max_q_size: maximum size for the generator queue
workers: maximum number of processes to spin up
when using process based threading
pickle_safe: if True, use process based threading.
Note that because
this implementation relies on multiprocessing,
you should not pass
non picklable arguments to the generator
as they can't be passed
easily to children processes.
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
# Returns
A `History` object.
# Example
```python
def generate_arrays_from_file(path):
while 1:
f = open(path)
for line in f:
# create numpy arrays of input data
# and labels, from each line in the file
x1, x2, y = process_line(line)
yield ({'input_1': x1, 'input_2': x2}, {'output': y})
f.close()
model.fit_generator(generate_arrays_from_file('/my_file.txt'),
steps_per_epoch=10000, epochs=10)
```
# Raises
ValueError: In case the generator yields
data in an invalid format.
"""
# wait_time = 0.01 # in seconds
epoch = initial_epoch
do_validation = bool(validation_data)
self._make_train_function()
if do_validation:
self._make_test_function()
# python 2 has 'next', 3 has '__next__'
# avoid any explicit version checks
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__'))
if val_gen and not validation_steps:
raise ValueError('When using a generator for validation data, '
'you must specify a value for '
'`validation_steps`.')
out_labels = self.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
callbacks += [cbks.ProgbarLogger(count_mode='steps')]
callbacks = cbks.CallbackList(callbacks)
# it's possible to callback a different model than self:
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
if do_validation and not val_gen:
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise ValueError('validation_data should be a tuple '
'`(val_x, val_y, val_sample_weight)` '
'or `(val_x, val_y)`. Found: ' +
str(validation_data))
val_x, val_y, val_sample_weights = self._standardize_user_data(
val_x, val_y, val_sample_weight)
for cbk in callbacks:
cbk.validation_data = val_x + [val_y, val_sample_weights]
# enqueuer = None
# TODO: Tensorpack does some kind of acceleratn using
# QueueInputTrainer, QueueInput, and EnqueueThread. The
# implementation below corresponds to SimpleTrainer which
# Tensorpack notes as being slow. I still cannot decipher what
# exactly is going on in Tensorpack. For the same per-GPU batchsize
# the runtime per epoch seems on par. Perhaps with Tensorpack
# implementation using Queue+Thread for datafalow the feed_dict
# would be faster. The keras fit_generator does use an enqueuer,
# but I did not notice performance difference between using
# fit_generator or this mixed-in fit_dataflow method.
try:
# enqueuer = GeneratorEnqueuer(generator, pickle_safe=pickle_safe)
# enqueuer.start(max_q_size=max_q_size, workers=workers)
dflow.reset_state()
_generator = dflow.get_data()
callback_model.stop_training = False
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
# generator_output = None
generator_output = next(_generator)
# while enqueuer.is_running():
# if not enqueuer.queue.empty():
# generator_output = enqueuer.queue.get()
# break
# else:
# time.sleep(wait_time)
if not hasattr(generator_output, '__len__'):
raise ValueError('output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = self.train_on_batch(x,
y,
sample_weight=sample_weight,
class_weight=class_weight)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
# Construct epoch logs.
epoch_logs = {}
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
if val_gen:
val_outs = self.evaluate_generator(
validation_data,
validation_steps,
max_q_size=max_q_size,
workers=workers,
pickle_safe=pickle_safe)
else:
# No need for try/except because
# data has already been validated.
val_outs = self.evaluate(
val_x,
val_y,
batch_size=batch_size,
sample_weight=val_sample_weights,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
# if enqueuer is not None:
# enqueuer.stop()
pass
callbacks.on_train_end()
return self.history
def train_model(name,
ftrain,
generator,
samples_per_epoch,
nb_epoch,
verbose=1,
callbacks=[],
ftest=None,
validation_data=None,
nb_val_samples=None,
saver=None):
"""
Main training loop.
modified from Keras fit_generator
"""
gif = True
if gif:
plt.subplot(121)
IM = plt.imshow(np.random.randn(ims, ims, 3), interpolation="none")
plt.subplot(122)
IM2 = plt.imshow(np.random.randn(ims, ims, 3), interpolation="none")
plt.draw()
plt.pause(.001)
epoch = 0
counter = 0
out_labels = ['loss', 'time'] # self.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
history = cbks.History()
callbacks = [cbks.BaseLogger()] + callbacks + [history]
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
callbacks._set_params({
'nb_epoch': nb_epoch,
'nb_sample': samples_per_epoch,
'verbose': verbose,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
while epoch < nb_epoch:
callbacks.on_epoch_begin(epoch)
samples_seen = 0
batch_index = 0
while samples_seen < samples_per_epoch:
x, y = next(generator)
# build batch logs
batch_logs = {}
if type(x) is list:
batch_size = len(x[0])
elif type(x) is dict:
batch_size = len(list(x.values())[0])
else:
batch_size = len(x)
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
t1 = time.time()
samples, losses = ftrain(x, y, counter)
outs = (losses, ) + (time.time() - t1, )
counter += 1
if (counter % 100 == 0) and gif:
for v, u in zip(samples[0], y[0]):
IM.set_data(v.reshape(ims, ims, 3))
IM2.set_data(u.reshape(ims, ims, 3))
plt.draw()
plt.pause(.01)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
# construct epoch logs
epoch_logs = {}
batch_index += 1
samples_seen += batch_size
if validation_data is not None:
valid_cost = 0
valid_samples_seen = 0
while valid_samples_seen < nb_val_samples:
x, y = next(validation_data)
valid_cost += ftest(x, y)[1]
valid_samples_seen += 1
valid_cost /= float(nb_val_samples)
print "\nValidation: ", valid_cost
if saver is not None:
saver(epoch)
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
# _stop.set()
callbacks.on_train_end()
def fit(
self,
x: Optional[
Union[np.ndarray, tf.Tensor, tf.data.Dataset, tf.keras.utils.Sequence]
] = None,
y: Optional[
Union[np.ndarray, tf.Tensor, tf.data.Dataset, tf.keras.utils.Sequence]
] = None,
batch_size: Optional[int] = None,
epochs: int = 1,
verbose: int = 1,
callbacks: Optional[List[Callback]] = None,
validation_split: float = 0.0,
validation_data: Optional[Any] = None,
shuffle: bool = True,
class_weight: Optional[Dict[int, float]] = None,
sample_weight: Optional[np.ndarray] = None,
initial_epoch: int = 0,
steps_per_epoch: Optional[int] = None,
validation_steps: Optional[int] = None,
validation_batch_size: Optional[int] = None,
validation_freq: int = 1,
max_queue_size: int = 10,
workers: int = 1,
use_multiprocessing: bool = False,
) -> History:
"""Trains the model for a fixed number of epochs (iterations on a dataset).
Args:
x: Input data. It could be:
- A Numpy array (or array-like), or a list of arrays
(in case the model has multiple inputs).
- A TensorFlow tensor, or a list of tensors
(in case the model has multiple inputs).
- A dict mapping input names to the corresponding array/tensors,
if the model has named inputs.
- A `tf.data` dataset. Should return a tuple
of either `(inputs, targets)` or
`(inputs, targets, sample_weights)`.
- A generator or `keras.utils.Sequence` returning `(inputs, targets)`
or `(inputs, targets, sample_weights)`.
- A `tf.keras.utils.experimental.DatasetCreator`, which wraps a
callable that takes a single argument of type
`tf.distribute.InputContext`, and returns a `tf.data.Dataset`.
`DatasetCreator` should be used when users prefer to specify the
per-replica batching and sharding logic for the `Dataset`.
See `tf.keras.utils.experimental.DatasetCreator` doc for more
information.
A more detailed description of unpacking behavior for iterator types
(Dataset, generator, Sequence) is given below. If using
`tf.distribute.experimental.ParameterServerStrategy`, only
`DatasetCreator` type is supported for `x`.
y: Target data. Like the input data `x`,
it could be either Numpy array(s) or TensorFlow tensor(s).
It should be consistent with `x` (you cannot have Numpy inputs and
tensor targets, or inversely). If `x` is a dataset, generator,
or `keras.utils.Sequence` instance, `y` should
not be specified (since targets will be obtained from `x`).
batch_size: Integer or `None`.
Number of samples per gradient update.
If unspecified, `batch_size` will default to 32.
Do not specify the `batch_size` if your data is in the
form of datasets, generators, or `keras.utils.Sequence` instances
(since they generate batches).
epochs: Integer. Number of epochs to train the model.
An epoch is an iteration over the entire `x` and `y`
data provided
(unless the `steps_per_epoch` flag is set to
something other than None).
Note that in conjunction with `initial_epoch`,
`epochs` is to be understood as "final epoch".
The model is not trained for a number of iterations
given by `epochs`, but merely until the epoch
of index `epochs` is reached.
verbose: 'auto', 0, 1, or 2. Verbosity mode.
0 = silent, 1 = progress bar, 2 = one line per epoch.
'auto' defaults to 1 for most cases, but 2 when used with
`ParameterServerStrategy`. Note that the progress bar is not
particularly useful when logged to a file, so verbose=2 is
recommended when not running interactively (eg, in a production
environment).
callbacks: List of `keras.callbacks.Callback` instances.
List of callbacks to apply during training.
See `tf.keras.callbacks`. Note `tf.keras.callbacks.ProgbarLogger`
and `tf.keras.callbacks.History` callbacks are created automatically
and need not be passed into `model.fit`.
`tf.keras.callbacks.ProgbarLogger` is created or not based on
`verbose` argument to `model.fit`.
Callbacks with batch-level calls are currently unsupported with
`tf.distribute.experimental.ParameterServerStrategy`, and users are
advised to implement epoch-level calls instead with an appropriate
`steps_per_epoch` value.
validation_split: Float between 0 and 1.
Fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate
the loss and any model metrics
on this data at the end of each epoch.
The validation data is selected from the last samples
in the `x` and `y` data provided, before shuffling. This argument is
not supported when `x` is a dataset, generator or
`keras.utils.Sequence` instance.
`validation_split` is not yet supported with
`tf.distribute.experimental.ParameterServerStrategy`.
validation_data: Data on which to evaluate
the loss and any model metrics at the end of each epoch.
The model will not be trained on this data. Thus, note the fact
that the validation loss of data provided using `validation_split`
or `validation_data` is not affected by regularization layers like
noise and dropout.
`validation_data` will override `validation_split`.
`validation_data` could be:
- A tuple `(x_val, y_val)` of Numpy arrays or tensors.
- A tuple `(x_val, y_val, val_sample_weights)` of NumPy arrays.
- A `tf.data.Dataset`.
- A Python generator or `keras.utils.Sequence` returning
`(inputs, targets)` or `(inputs, targets, sample_weights)`.
`validation_data` is not yet supported with
`tf.distribute.experimental.ParameterServerStrategy`.
shuffle: Boolean (whether to shuffle the training data
before each epoch) or str (for 'batch'). This argument is ignored
when `x` is a generator or an object of tf.data.Dataset.
'batch' is a special option for dealing
with the limitations of HDF5 data; it shuffles in batch-sized
chunks. Has no effect when `steps_per_epoch` is not `None`.
class_weight: Optional dictionary mapping class indices (integers)
to a weight (float) value, used for weighting the loss function
(during training only).
This can be useful to tell the model to
"pay more attention" to samples from
an under-represented class.
sample_weight: Optional Numpy array of weights for
the training samples, used for weighting the loss function
(during training only). You can either pass a flat (1D)
Numpy array with the same length as the input samples
(1:1 mapping between weights and samples),
or in the case of temporal data,
you can pass a 2D array with shape
`(samples, sequence_length)`,
to apply a different weight to every timestep of every sample. This
argument is not supported when `x` is a dataset, generator, or
`keras.utils.Sequence` instance, instead provide the sample_weights
as the third element of `x`.
initial_epoch: Integer.
Epoch at which to start training
(useful for resuming a previous training run).
steps_per_epoch: Integer or `None`.
Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. When training with input tensors such as
TensorFlow data tensors, the default `None` is equal to
the number of samples in your dataset divided by
the batch size, or 1 if that cannot be determined. If x is a
`tf.data` dataset, and 'steps_per_epoch'
is None, the epoch will run until the input dataset is exhausted.
When passing an infinitely repeating dataset, you must specify the
`steps_per_epoch` argument. If `steps_per_epoch=-1` the training
will run indefinitely with an infinitely repeating dataset.
This argument is not supported with array inputs.
When using `tf.distribute.experimental.ParameterServerStrategy`:
* `steps_per_epoch=None` is not supported.
validation_steps: Only relevant if `validation_data` is provided and
is a `tf.data` dataset. Total number of steps (batches of
samples) to draw before stopping when performing validation
at the end of every epoch. If 'validation_steps' is None, validation
will run until the `validation_data` dataset is exhausted. In the
case of an infinitely repeated dataset, it will run into an
infinite loop. If 'validation_steps' is specified and only part of
the dataset will be consumed, the evaluation will start from the
beginning of the dataset at each epoch. This ensures that the same
validation samples are used every time.
validation_batch_size: Integer or `None`.
Number of samples per validation batch.
If unspecified, will default to `batch_size`.
Do not specify the `validation_batch_size` if your data is in the
form of datasets, generators, or `keras.utils.Sequence` instances
(since they generate batches).
validation_freq: Only relevant if validation data is provided. Integer
or `collections.abc.Container` instance (e.g. list, tuple, etc.).
If an integer, specifies how many training epochs to run before a
new validation run is performed, e.g. `validation_freq=2` runs
validation every 2 epochs. If a Container, specifies the epochs on
which to run validation, e.g. `validation_freq=[1, 2, 10]` runs
validation at the end of the 1st, 2nd, and 10th epochs.
max_queue_size: Integer. Used for generator or `keras.utils.Sequence`
input only. Maximum size for the generator queue.
If unspecified, `max_queue_size` will default to 10.
workers: Integer. Used for generator or `keras.utils.Sequence` input
only. Maximum number of processes to spin up
when using process-based threading. If unspecified, `workers`
will default to 1.
use_multiprocessing: Boolean. Used for generator or
`keras.utils.Sequence` input only. If `True`, use process-based
threading. If unspecified, `use_multiprocessing` will default to
`False`. Note that because this implementation relies on
multiprocessing, you should not pass non-picklable arguments to
the generator as they can't be passed easily to children processes.
Unpacking behavior for iterator-like inputs:
A common pattern is to pass a tf.data.Dataset, generator, or
tf.keras.utils.Sequence to the `x` argument of fit, which will in fact
yield not only features (x) but optionally targets (y) and sample weights.
Keras requires that the output of such iterator-likes be unambiguous. The
iterator should return a tuple of length 1, 2, or 3, where the optional
second and third elements will be used for y and sample_weight
respectively. Any other type provided will be wrapped in a length one
tuple, effectively treating everything as 'x'. When yielding dicts, they
should still adhere to the top-level tuple structure.
e.g. `({"x0": x0, "x1": x1}, y)`. Keras will not attempt to separate
features, targets, and weights from the keys of a single dict.
A notable unsupported data type is the namedtuple. The reason is that
it behaves like both an ordered datatype (tuple) and a mapping
datatype (dict). So given a namedtuple of the form:
`namedtuple("example_tuple", ["y", "x"])`
it is ambiguous whether to reverse the order of the elements when
interpreting the value. Even worse is a tuple of the form:
`namedtuple("other_tuple", ["x", "y", "z"])`
where it is unclear if the tuple was intended to be unpacked into x, y,
and sample_weight or passed through as a single element to `x`. As a
result the data processing code will simply raise a ValueError if it
encounters a namedtuple. (Along with instructions to remedy the issue.)
Returns:
A `History` object. Its `History.history` attribute is
a record of training loss values and metrics values
at successive epochs, as well as validation loss values
and validation metrics values (if applicable).
Raises:
RuntimeError: 1. If the model was never compiled or,
2. If `model.fit` is wrapped in `tf.function`.
ValueError: In case of mismatch between the provided input data
and what the model expects or when the input data is empty.
"""
base_layer.keras_api_gauge.get_cell("fit").set(True)
# Legacy graph support is contained in `training_v1.Model`.
version_utils.disallow_legacy_graph("Model", "fit")
self._assert_compile_was_called()
self._check_call_args("fit")
_disallow_inside_tf_function("fit")
if verbose == "auto":
if (
self.distribute_strategy._should_use_with_coordinator
): # pylint: disable=protected-access
verbose = 2 # Default to epoch-level logging for PSStrategy.
else:
verbose = 1 # Default to batch-level logging otherwise.
elif (
verbose == 1 and self.distribute_strategy._should_use_with_coordinator
): # pylint: disable=protected-access
raise ValueError(
"`verbose=1` is not allowed with `ParameterServerStrategy` for "
f"performance reasons. Received: `verbose`={verbose}"
)
if validation_split:
# Create the validation data using the training data. Only supported for
# `Tensor` and `NumPy` input.
(
(x, y, sample_weight),
validation_data,
) = data_adapter.train_validation_split(
(x, y, sample_weight), validation_split=validation_split
)
if validation_data:
val_x, val_y, val_sample_weight = data_adapter.unpack_x_y_sample_weight(
validation_data
)
if (
self.distribute_strategy._should_use_with_coordinator
): # pylint: disable=protected-access
self._cluster_coordinator = (
tf.distribute.experimental.coordinator.ClusterCoordinator(
self.distribute_strategy
)
)
with self.distribute_strategy.scope(), training_utils.RespectCompiledTrainableState( # noqa: E501
self
):
# Creates a `tf.data.Dataset` and handles batch and epoch iteration.
# Adaption: Use our own custom data handler to handle increasing batch size
data_handler = CustomDataHandler(
x=x,
y=y,
sample_weight=sample_weight,
batch_size=batch_size,
steps_per_epoch=steps_per_epoch,
initial_epoch=initial_epoch,
epochs=epochs,
shuffle=shuffle,
class_weight=class_weight,
max_queue_size=max_queue_size,
workers=workers,
use_multiprocessing=use_multiprocessing,
model=self,
steps_per_execution=self._steps_per_execution,
)
# Container that configures and calls `tf.keras.Callback`s.
if not isinstance(callbacks, callbacks_module.CallbackList):
callbacks = callbacks_module.CallbackList(
callbacks,
add_history=True,
add_progbar=verbose != 0,
model=self,
verbose=verbose,
epochs=epochs,
steps=data_handler.inferred_steps,
)
callbacks_list = cast(callbacks_module.CallbackList, callbacks)
self.stop_training = False
self.train_function = self.make_train_function()
self._train_counter.assign(0)
callbacks_list.on_train_begin()
training_logs = None
# Handle fault-tolerance for multi-worker.
# TODO(omalleyt): Fix the ordering issues that mean this has to
# happen after `callbacks.on_train_begin`.
data_handler._initial_epoch = self._maybe_load_initial_epoch_from_ckpt( # pylint: disable=protected-access # noqa: E501
initial_epoch
)
logs = None
for epoch, iterator in data_handler.enumerate_epochs():
self.reset_metrics()
callbacks_list.on_epoch_begin(epoch)
with data_handler.catch_stop_iteration():
for step in data_handler.steps():
with tf.profiler.experimental.Trace(
"train",
epoch_num=epoch,
step_num=step,
batch_size=batch_size,
_r=1,
):
callbacks_list.on_train_batch_begin(step)
tmp_logs = self.train_function(iterator)
if data_handler.should_sync:
context.async_wait()
logs = tmp_logs # No error, now safe to assign to logs.
end_step = step + data_handler.step_increment
callbacks_list.on_train_batch_end(end_step, logs)
if self.stop_training:
break
logs = tf_utils.sync_to_numpy_or_python_type(logs)
if logs is None:
raise ValueError(
"Unexpected result of `train_function` "
"(Empty logs). Please use "
"`Model.compile(..., run_eagerly=True)`, or "
"`tf.config.run_functions_eagerly(True)` for more "
"information of where went wrong, or file a "
"issue/bug to `tf.keras`."
)
epoch_logs = copy.copy(logs)
# Run validation.
if validation_data and self._should_eval(epoch, validation_freq):
# Create data_handler for evaluation and cache it.
if getattr(self, "_eval_data_handler", None) is None:
self._eval_data_handler = data_adapter.get_data_handler(
x=val_x,
y=val_y,
sample_weight=val_sample_weight,
batch_size=validation_batch_size or batch_size,
steps_per_epoch=validation_steps,
initial_epoch=0,
epochs=1,
max_queue_size=max_queue_size,
workers=workers,
use_multiprocessing=use_multiprocessing,
model=self,
steps_per_execution=self._steps_per_execution,
)
val_logs = self.evaluate(
x=val_x,
y=val_y,
sample_weight=val_sample_weight,
batch_size=validation_batch_size or batch_size,
steps=validation_steps,
callbacks=callbacks_list,
max_queue_size=max_queue_size,
workers=workers,
use_multiprocessing=use_multiprocessing,
return_dict=True,
_use_cached_eval_dataset=True,
)
val_logs = {"val_" + name: val for name, val in val_logs.items()}
epoch_logs.update(val_logs)
callbacks_list.on_epoch_end(epoch, epoch_logs)
training_logs = epoch_logs
if self.stop_training:
break
# If eval_data_handler exists, delete it after all epochs are done.
if getattr(self, "_eval_data_handler", None) is not None:
del self._eval_data_handler
callbacks_list.on_train_end(logs=training_logs)
return self.history
def train_model(workspaceDir, modelName, devFileSuffix, testFileSuffix,
saveModel, batchSize, epochs, max_len, num_buckets, vocab_size,
training_mode, early_stop, predictor_model, predictor_data,
**kwargs):
logger.info("initializing TQE training")
predictorModelFile = None
if predictor_model:
predictorModelFile = os.path.join(
workspaceDir, ".".join(["tqe", predictor_model,
"predictor.model"]))
srcVocabTransformer = WordIndexTransformer(vocab_size=vocab_size)
refVocabTransformer = WordIndexTransformer(vocab_size=vocab_size)
X_train, y_train, X_dev, y_dev, X_test, y_test, pred_train = _prepareInput(
workspaceDir,
modelName,
srcVocabTransformer,
refVocabTransformer,
max_len=max_len,
num_buckets=num_buckets,
devFileSuffix=devFileSuffix,
testFileSuffix=testFileSuffix,
predictorDataModel=predictor_data)
model_multitask, model_predictor, model_estimator = \
getEnsembledModel(srcVocabTransformer=srcVocabTransformer,
refVocabTransformer=refVocabTransformer,
keep_trainable=(training_mode == "stack-prop"),
**kwargs)
if predictorModelFile and not pred_train:
logger.info("Loading weights for predictor")
model_predictor.load_weights(predictorModelFile)
logger.info("Training")
if early_stop < 0:
early_stop = epochs
def reshapeRef(ref):
return np.array(map(lambda r: r.reshape((-1, 1)), ref))
if pred_train:
logger.info("Training predictor on predictor data")
callbacks = None
if predictorModelFile:
callbacks = [
ModelCheckpoint(filepath=(predictorModelFile + ".{epoch:02d}"),
save_weights_only=True)
]
model_predictor.fit_generator(
getBatchGenerator([pred_train['src'], pred_train['ref']],
[reshapeRef(pred_train['ref'])],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize),
epochs=epochs,
verbose=2,
callbacks=callbacks)
if predictorModelFile:
logger.info("Saving weights for predictor")
model_predictor.save_weights(predictorModelFile)
if training_mode == "multitask":
logger.info("Training multitask model")
model_multitask.fit_generator(
getBatchGenerator(
[X_train['src'], X_train['mt']],
[reshapeRef(X_train["ref"]), y_train],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
),
epochs=epochs,
validation_data=getBatchGenerator(
[X_dev['src'], X_dev['mt']],
[reshapeRef(X_dev["ref"]), y_dev],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
),
callbacks=[
EarlyStopping(monitor="val_quality_pearsonr",
patience=early_stop,
mode="max"),
],
verbose=2)
elif training_mode == "two-step":
logger.info("Training predictor")
model_predictor.fit_generator(
getBatchGenerator(
[X_train['src'], X_train['ref']],
[
reshapeRef(X_train["ref"]),
],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
),
epochs=epochs,
validation_data=getBatchGenerator(
[X_dev['src'], X_dev['mt']],
[
reshapeRef(X_dev["ref"]),
],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
),
callbacks=[
EarlyStopping(monitor="val_sparse_categorical_accuracy",
patience=early_stop,
mode="max"),
],
verbose=2)
logger.info("Training estimator")
model_estimator.fit_generator(
getBatchGenerator(
[X_train['src'], X_train['mt']],
[y_train],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
),
epochs=epochs,
validation_data=getBatchGenerator(
[X_dev['src'], X_dev['mt']],
[y_dev],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
),
callbacks=[
EarlyStopping(monitor="val_pearsonr",
patience=early_stop,
mode="max"),
],
verbose=2)
elif training_mode == "stack-prop":
logger.info("Training with stack propogation")
# Set parameters
models = [model_predictor, model_estimator]
train_data = [
getBatchGenerator(
[X_train['src'], X_train['ref']],
[
reshapeRef(X_train["ref"]),
],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
),
getBatchGenerator(
[X_train['src'], X_train['mt']],
[
y_train,
],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
)
]
validation_data = [
getBatchGenerator(
[X_dev['src'], X_dev['mt']],
[
reshapeRef(X_dev["ref"]),
],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
),
getBatchGenerator(
[X_dev['src'], X_dev['mt']],
[
y_dev,
],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
)
]
callbacks = [
EarlyStopping(monitor="val_pearsonr",
patience=early_stop,
mode="max"),
]
verbose = 2
# Done with setting parameters
# Assume num_batches in all generator are equal
steps_per_epoch = len(train_data[0])
do_validation = bool(validation_data)
# Prepare display labels.
out_labels = sum(map(lambda m: m.metrics_names, models), [])
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
history = cbks.History()
_callbacks = [cbks.BaseLogger()]
if verbose:
_callbacks.append(cbks.ProgbarLogger(count_mode='steps', ))
_callbacks += (callbacks or []) + [history]
callbacks = cbks.CallbackList(_callbacks)
callback_model = model_estimator
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
# Prepare for training
callback_model.stop_training = False
epoch_logs = {}
# Start training
for epoch in range(0, epochs):
callbacks.on_epoch_begin(epoch)
for batch_index in range(0, steps_per_epoch):
# build batch logs
# Get size of the batch
x, y = train_data[0][batch_index]
if isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = []
for i, model in enumerate(models):
x, y = train_data[i][batch_index]
model_outs = model.train_on_batch(x, y)
if not isinstance(model_outs, list):
model_outs = [model_outs]
outs.extend(model_outs)
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if do_validation:
val_outs = []
for i, model in enumerate(models):
outs = model.evaluate_generator(validation_data[i])
if not isinstance(outs, list):
outs = [outs]
val_outs.extend(outs)
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
else:
raise ValueError("Training mode not recognized")
# logger.info("Saving model")
# model.save(fileBasename + "neural.model.h5")
if saveModel:
logger.info("Saving model")
shelf = shelve.open(os.path.join(workspaceDir, "model." + saveModel))
models = [model_multitask, model_predictor, model_estimator]
shelf['config'] = [model.get_config() for model in models]
shelf['weights'] = [model.get_weights() for model in models]
shelf['params'] = {
'srcVocabTransformer': srcVocabTransformer,
'refVocabTransformer': refVocabTransformer,
}
shelf.close()
logger.info("Evaluating on development data of size %d" % len(y_dev))
dev_batches = getBatchGenerator(
[X_dev['src'], X_dev['mt']],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
)
y_dev = dev_batches.align(y_dev)
evaluate(
model_estimator.predict_generator(dev_batches).reshape((-1, )), y_dev)
logger.info("Evaluating on test data of size %d" % len(y_test))
test_batches = getBatchGenerator(
[X_test['src'], X_test['mt']],
key=lambda x: "_".join(map(str, map(len, x))),
batch_size=batchSize,
)
y_test = test_batches.align(y_test)
evaluate(
model_estimator.predict_generator(test_batches).reshape((-1, )),
y_test)
def train(
model,
image_training,
image_validation,
label_training,
label_validation,
epochs,
batch_size,
class_weight=None,
callbacks=None,
save_weight=False,
seed=None):
# # メモリ確保の方法を変更
from keras import callbacks as cbks
# from keras import backend as K
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
# sess = tf.Session(config=config)
# K.set_session(sess)
# コールバックの準備
callback_metrics = None # 内容が不明なので空とする
callbacks = cbks.CallbackList(callbacks or [])
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
# 'steps': steps_per_epoch,
# 'samples': num_train_samples,
# 'verbose': verbose,
'do_validation': True,
'metrics': callback_metrics or [],
})
# コールバック実行
callback_model.stop_training = False
callbacks.on_train_begin()
# import pandas as pd
for i in range(epochs):
callback_logs = {}
callbacks.on_epoch_begin(epoch=i, logs=callback_logs)
# print_epoch_start_time_training_validation(i + 1, epochs)
train_batch_returns = []
p = np.random.permutation(image_training.shape[0])
image_training_shuffle = image_training[p]
label_training_shuffle = label_training[p]
# load numpy cache and train on batch
train_generator = tqdm(
iterable=iter_train_batch(image_training_shuffle, label_training_shuffle, batch_size),
total=math.ceil(image_training_shuffle.shape[0] / batch_size))
for j, [image_batch, label_batch] in enumerate(train_generator):
# TODO: 2nd arguments 'batch_logs' must be implemented
callbacks.on_batch_begin(j, {})
# main process
batch_return = model.train_on_batch(
x=image_batch,
y=label_batch,
class_weight=class_weight)
# result
# num_image = image_batch.shape[0]
train_batch_returns.append(batch_return)
# loss_sum += loss * num_image
# TODO: 2nd arguments 'batch_logs' must be implemented
callbacks.on_batch_end(j, {})
# # 進捗を出力
# # 累積のloss, accを出力
# print_batch_remain_time_training_validation(time_start_batch=time_start_batch,
# current_batch=j + 1,
# batches=len(cache_image_training),
# loss_sum=loss_sum,
# acc_sum=acc_sum,
# num_image_total=num_image_total,
# )
# num_train_image_total = image_training_shuffle.shape[0]
# train_loss = loss_sum / num_image_total
# Validation after batch loop
validate_batch_returns = []
validate_generator = tqdm(
iterable=iter_train_batch(image_validation, label_validation, batch_size),
total=math.ceil(image_validation.shape[0] / batch_size))
for j, [image_batch, label_batch] in enumerate(validate_generator):
# main process
batch_return = model.test_on_batch(x=image_batch, y=label_batch)
# result
# num_image = image_batch.shape[0]
# loss_sum += loss * num_image
validate_batch_returns.append(batch_return)
# num_val_image_total = image_validation.shape[0]
# val_loss = loss_sum / num_image_total
# print_batch_end_time_training_validation(val_loss=val_loss, val_acc=val_acc)
# all batches end
# print_all_batch_end_time_training_validation(time_start_batch=time_start_batch)
import json
from utility.save import check_dir
weight_dump = model.get_weights()
for widx, weight in enumerate(weight_dump):
model_path = 'work/output/{:04d}/weight_{:02d}.json'.format(i, widx)
check_dir(model_path)
with open(model_path, 'w') as f:
f.write(json.dumps(weight.tolist()))
epoch_logs = {
'batch_train_history': train_batch_returns,
'batch_val_history': validate_batch_returns,
}
callbacks.on_epoch_end(i, epoch_logs)
if callback_model.stop_training:
break
# # all epochs end
# print_all_epoch_end_time_training_validation(epochs=epochs,
# time_start_epoch=time_start_epoch)
callbacks.on_train_end()
print('training finished')
def _fit_loop(self,
f,
ins,
out_labels=None,
batch_size=32,
epochs=100,
verbose=1,
callbacks=None,
val_f=None,
val_ins=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None):
"""Abstract fit function for f(ins).
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: List of tensors to be fed to `f`
out_labels: List of strings, display names of
the outputs of `f`
batch_size: Integer batch size or None if unknown.
epochs: Number of times to iterate over the data
verbose: Verbosity mode, 0, 1 or 2
callbacks: List of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: List of tensors to be fed to `val_f`
shuffle: Whether to shuffle the data at the beginning of each epoch
callback_metrics: List of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: Epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. Ignored with the default value of `None`.
validation_steps: Number of steps to run validation for
(only if doing validation from data tensors).
Ignored with the default value of `None`.
# Returns
`History` object.
[A tweaked version.]
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose and ins and hasattr(ins[0], 'shape') and hasattr(
val_ins[0], 'shape'):
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if validation_steps:
do_validation = True
if steps_per_epoch is None:
raise ValueError('Can only use `validation_steps` '
'when doing step-wise '
'training, i.e. `steps_per_epoch` '
'must be set.')
num_train_samples = self._check_num_samples(ins, batch_size,
steps_per_epoch,
'steps_per_epoch')
if num_train_samples is not None:
index_array = np.arange(num_train_samples)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
if steps_per_epoch is not None:
count_mode = 'steps'
else:
count_mode = 'samples'
callbacks += [cbks.ProgbarLogger(count_mode)]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
# for cbk in callbacks:
# cbk.validation_data = val_ins
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
if steps_per_epoch is not None:
for step_index in range(steps_per_epoch):
batch_logs = {}
batch_logs['batch'] = step_index
batch_logs['size'] = 1
callbacks.on_batch_begin(step_index, batch_logs)
outs = f(ins)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(step_index, batch_logs)
if callback_model.stop_training:
break
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
steps=validation_steps,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
else:
if shuffle == 'batch':
index_array = _batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = _slice_arrays(ins[:-1],
batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['ids'] = batch_ids
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if batch_index == len(batches) - 1: # last batch.
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def fit_generator_autosized(
model,
generator,
epochs=1,
#steps_per_epoch=None,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
validation_callbacks=None,
class_weight=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
shuffle=True,
initial_epoch=0):
"""See docstring for `Model.fit_generator`."""
wait_time = 0.01 # in seconds
epoch = initial_epoch
do_validation = bool(validation_data)
model._make_train_function()
if do_validation:
model._make_test_function()
is_sequence = isinstance(generator, Sequence)
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
# if steps_per_epoch is None:
# if is_sequence:
# steps_per_epoch = len(generator)
# else:
# raise ValueError('`steps_per_epoch=None` is only valid for a'
# ' generator based on the '
# '`keras.utils.Sequence`'
# ' class. Please specify `steps_per_epoch` '
# 'or use the `keras.utils.Sequence` class.')
# python 2 has 'next', 3 has '__next__'
# avoid any explicit version checks
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__')
or isinstance(validation_data, Sequence))
# if (val_gen and not isinstance(validation_data, Sequence) and
# not validation_steps):
# raise ValueError('`validation_steps=None` is only valid for a'
# ' generator based on the `keras.utils.Sequence`'
# ' class. Please specify `validation_steps` or use'
# ' the `keras.utils.Sequence` class.')
# Prepare display labels.
out_labels = model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
model.history = cbks.History()
_callbacks = [
cbks.BaseLogger(stateful_metrics=model.stateful_metric_names)
]
# instead of ProgbarLogger (but only for first epoch):
if verbose:
print('Epoch 1/%d' % epochs)
progbar = Progbar(target=None,
verbose=1,
stateful_metrics=model.stateful_metric_names)
_callbacks += (callbacks or []) + [model.history]
callbacks = cbks.CallbackList(_callbacks)
# it's possible to callback a different model than self:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': None, # will be refined during first epoch
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
enqueuer = None
val_enqueuer = None
try:
if do_validation and not val_gen:
# Prepare data for validation
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise ValueError('`validation_data` should be a tuple '
'`(val_x, val_y, val_sample_weight)` '
'or `(val_x, val_y)`. Found: ' +
str(validation_data))
val_x, val_y, val_sample_weights = model._standardize_user_data(
val_x, val_y, val_sample_weight)
val_data = val_x + val_y + val_sample_weights
if model.uses_learning_phase and not isinstance(
K.learning_phase(), int):
val_data += [0.]
for cbk in callbacks:
cbk.validation_data = val_data
if workers > 0:
if is_sequence:
enqueuer = OrderedEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
else:
if is_sequence:
output_generator = iter(generator)
else:
output_generator = generator
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
for m in model.stateful_metric_functions:
m.reset_states()
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
for generator_output in output_generator:
if not generator_output: # end of epoch?
break
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if not x:
# Handle data tensors support when no input given
# step-size = 1 for data tensors
batch_size = 1
elif isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = model.train_on_batch(x,
y,
sample_weight=sample_weight,
class_weight=class_weight)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if epoch == initial_epoch and verbose:
log_values = []
for k in callback_metrics:
if k in batch_logs:
log_values.append((k, batch_logs[k]))
progbar.update(steps_done, log_values)
batch_index += 1
steps_done += 1
if callback_model.stop_training:
break
if epoch == initial_epoch:
if verbose:
log_values = []
for k in callback_metrics:
if k in batch_logs:
log_values.append((k, batch_logs[k]))
progbar.update(steps_done, log_values)
# Epoch finished.
if do_validation:
if val_gen:
val_outs, validation_steps = evaluate_generator_autosized(
model,
validation_data,
steps=validation_steps,
callbacks=validation_callbacks,
workers=workers,
use_multiprocessing=use_multiprocessing,
max_queue_size=max_queue_size,
verbose=1)
else:
# No need for try/except because
# data has already been validated.
val_outs = model.evaluate(val_x,
val_y,
batch_size=batch_size,
sample_weight=val_sample_weights,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
if callback_model.stop_training:
break
callbacks.on_epoch_end(epoch, epoch_logs)
if epoch == initial_epoch:
if verbose:
print()
progbar = cbks.ProgbarLogger(
count_mode='steps',
stateful_metrics=model.stateful_metric_names)
progbar.set_model(callback_model)
callbacks.append(progbar)
callbacks.set_params({
'epochs': epochs,
'steps': steps_done, # refine
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
if verbose:
progbar.on_train_begin()
epoch += 1
if callback_model.stop_training:
break
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
if val_enqueuer is not None:
val_enqueuer.stop()
callbacks.on_train_end()
return model.history
def gan_fit_generator(
self,
generator,
datacollection,
steps_per_epoch=None,
epochs=1,
verbose=1,
callbacks_discriminator=None,
callbacks_gan=None,
validation_data=None,
validation_steps=None,
class_weight=None,
gan_skipping_factor=1,
discr_skipping_factor=1,
validation_freq=1, ###TBI FIXME
max_queue_size=10,
initial_epoch=0,
recover_discriminator=True):
"""See docstring for `Model.fit_generator`."""
import keras
from sklearn.utils import shuffle
import keras.callbacks as cbks
#from keras.training_utils import should_run_validation
from keras.utils.generic_utils import to_list
import numpy as np
epoch = initial_epoch
do_validation = bool(validation_data)
#DEBUG self.discriminator._make_train_function()
#DEBUG self.gan._make_train_function()
if do_validation and False: #DEBUG
self.discriminator._make_test_function()
self.gan._make_test_function()
d_out_labels = ['dis_' + n for n in self.discriminator.metrics_names]
g_out_labels = ['gan_' + n for n in self.gan.metrics_names]
d_callback_metrics = d_out_labels + ['val_' + n for n in d_out_labels]
g_callback_metrics = g_out_labels + ['val_' + n for n in g_out_labels]
# prepare callbacks
self.discriminator.history = cbks.History()
self.gan.history = cbks.History()
_callbacks = [
cbks.BaseLogger(
stateful_metrics=self.discriminator.stateful_metric_names)
]
_callbacks += [
cbks.BaseLogger(stateful_metrics=self.gan.stateful_metric_names)
]
if verbose:
_callbacks.append(
cbks.ProgbarLogger(
count_mode='steps',
stateful_metrics=self.gan.stateful_metric_names)
) #one model is enough here!#use only gan here
callbacks_gan = callbacks_gan or []
callbacks_discriminator = callbacks_discriminator or []
for c in callbacks_gan:
c.set_model(self.gan)
for c in callbacks_discriminator:
c.set_model(self.discriminator)
_callbacks += (callbacks_gan) + (callbacks_discriminator) + [
self.discriminator.history
] + [self.gan.history]
callbacks = cbks.CallbackList(_callbacks)
callbacks.set_params({
'epochs':
epochs,
'steps':
steps_per_epoch,
'verbose':
verbose,
'do_validation':
do_validation,
'metrics':
d_callback_metrics + g_callback_metrics,
})
#newer keras callbacks._call_begin_hook('train')
callbacks.on_train_begin()
enqueuer = None
val_enqueuer = None
try:
if do_validation:
val_data = validation_data
val_enqueuer_gen = val_data
output_generator = generator
## callbacks.model.stop_training = False ##FIXME TBI
# Construct epoch logs.
epoch_logs = {}
skip_gan_training = False
while epoch < epochs:
for m in self.discriminator.stateful_metric_functions:
m.reset_states()
for m in self.gan.stateful_metric_functions:
m.reset_states()
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
generator_output = next(output_generator)
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if x is None or len(x) == 0:
# Handle data tensors support when no input given
# step-size = 1 for data tensors
batch_size = 1
elif isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
# build batch logs
batch_logs = {'batch': batch_index, 'size': batch_size}
callbacks.on_batch_begin(batch_index, batch_logs)
#GAN training here
x_gen = self.generator.predict(x)
#DEBUG - NEEDS CALLBACK
# REMOVE IN FULL VERSION
if False and steps_done % 50:
forplots = np.concatenate([x_gen[0][:4], x[0][:4]],
axis=0)
from tools import quickplot, plotgrid
plotgrid(forplots,
nplotsx=4,
nplotsy=2,
outname="merged.pdf")
quickplot(x_gen[0][0], "gen.pdf")
quickplot(x[0][0], "data.pdf")
#this needs to be more generic and actually done for every list item
#replaceTruthForGAN gives a list
adapted_truth_data = datacollection.replaceTruthForGAN(
generated_array=np.zeros(batch_size, dtype='float32') +
1,
original_truth=y)
adapted_truth_generated = datacollection.replaceTruthForGAN(
generated_array=np.zeros(batch_size, dtype='float32'),
original_truth=y)
y_dis = [np.concatenate([adapted_truth_data[i],adapted_truth_generated[i]],axis=0) \
for i in range(len(adapted_truth_data))]
x_dis = [
np.concatenate([x[i], x_gen[i]], axis=0)
for i in range(len(x))
]
y_dis_new = [
shuffle(n, random_state=steps_done) for n in y_dis
]
x_dis_new = [
shuffle(n, random_state=steps_done) for n in x_dis
]
y_dis_b1 = [
y_dis_new[i][:batch_size, ...]
for i in range(len(y_dis_new))
]
y_dis_b2 = [
y_dis_new[i][batch_size:, ...]
for i in range(len(y_dis_new))
]
x_dis_b1 = [
x_dis_new[i][:batch_size, ...]
for i in range(len(x_dis_new))
]
x_dis_b2 = [
x_dis_new[i][batch_size:, ...]
for i in range(len(x_dis_new))
]
#add [:batch_size,...]
#to the above for cut-off
# TBI TBI FIXME
## FIXME: cut in half to have same batch size everywhere
# also here would be the place to implement weighting of discr versus gen
if (not batch_index % discr_skipping_factor):
self.discriminator.trainable = True
outs = self.discriminator.train_on_batch(
x_dis_b1,
y_dis_b1,
sample_weight=sample_weight,
class_weight=class_weight)
outs = self.discriminator.train_on_batch(
x_dis_b2,
y_dis_b2,
sample_weight=sample_weight,
class_weight=class_weight)
outs = to_list(outs)
if recover_discriminator:
if outs[1] < 0.5:
skip_gan_training = True
else:
skip_gan_training = False
for l, o in zip(d_out_labels, outs):
batch_logs[l] = o
if (not skip_gan_training) and (
not batch_index % gan_skipping_factor):
self.discriminator.trainable = False
y_gen = np.zeros(batch_size, dtype='float32') + 1.
outs = self.gan.train_on_batch(
x,
y_gen,
sample_weight=sample_weight,
class_weight=class_weight)
outs = to_list(outs)
for l, o in zip(g_out_labels, outs):
batch_logs[l] = o
#callbacks._call_batch_hook('train', 'end', batch_index, batch_logs)
callbacks.on_batch_end(batch_index, batch_logs)
batch_index += 1
steps_done += 1
# Epoch finished.
if (steps_done >= steps_per_epoch and do_validation):
# Note that `callbacks` here is an instance of
# `keras.callbacks.CallbackList`
## this evaluate will get problems with the truth definition
## needs to be fixed in the generator? Or just make traindata do it?
val_outs = self.discriminator.evaluate_generator(
val_enqueuer_gen,
validation_steps,
#callbacks=callbacks,
workers=0)
val_outs = to_list(val_outs)
# Same labels assumed.
for l, o in zip(d_out_labels, val_outs):
epoch_logs['val_' + l] = o
val_outs = self.gan.evaluate_generator(
val_enqueuer_gen,
validation_steps,
#callbacks=callbacks,
workers=0)
val_outs = to_list(val_outs)
# Same labels assumed.
for l, o in zip(g_out_labels, val_outs):
epoch_logs['val_' + l] = o
#if callbacks.model.stop_training: ##FIXME TBI
# break
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
#if callbacks.model.stop_training: ##FIXME TBI
# break
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
if val_enqueuer is not None:
val_enqueuer.stop()
#callbacks._call_end_hook('train')
callbacks.on_train_end()
return self.gan.history, self.discriminator.history
def custom_fit_generator(model,
generator,
steps_per_epoch=None,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
class_weight=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
shuffle=True,
initial_epoch=0):
"""
Same function fit_generator as Keras but with only a subset of the variables displayed
"""
wait_time = 0.01 # in seconds
epoch = initial_epoch
do_validation = bool(validation_data)
model._make_train_function()
if do_validation:
model._make_test_function()
is_sequence = isinstance(generator, Sequence)
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
if steps_per_epoch is None:
if is_sequence:
steps_per_epoch = len(generator)
else:
raise ValueError('`steps_per_epoch=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `steps_per_epoch` or use'
' the `keras.utils.Sequence` class.')
# python 2 has 'next', 3 has '__next__'
# avoid any explicit version checks
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__')
or isinstance(validation_data, Sequence))
if (val_gen and not isinstance(validation_data, Sequence)
and not validation_steps):
raise ValueError('`validation_steps=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `validation_steps` or use'
' the `keras.utils.Sequence` class.')
# Prepare display labels.
out_labels = model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
callback_metrics = [
'loss', 'acc', 'case_loss', 'case_acc', 'val_loss', 'val_acc',
'val_case_loss', 'val_case_acc'
]
# prepare callbacks
model.history = cbks.History()
_callbacks = [
cbks.BaseLogger(stateful_metrics=model.stateful_metric_names)
]
if verbose:
_callbacks.append(
cbks.ProgbarLogger(count_mode='steps',
stateful_metrics=model.stateful_metric_names))
_callbacks += (callbacks or []) + [model.history]
callbacks = cbks.CallbackList(_callbacks)
# it's possible to callback a different model than model:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
enqueuer = None
val_enqueuer = None
try:
if do_validation and not val_gen:
# Prepare data for validation
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise ValueError('`validation_data` should be a tuple '
'`(val_x, val_y, val_sample_weight)` '
'or `(val_x, val_y)`. Found: ' +
str(validation_data))
val_x, val_y, val_sample_weights = model._standardize_user_data(
val_x, val_y, val_sample_weight)
val_data = val_x + val_y + val_sample_weights
if model.uses_learning_phase and not isinstance(
K.learning_phase(), int):
val_data += [0.]
for cbk in callbacks:
cbk.validation_data = val_data
if workers > 0:
if is_sequence:
enqueuer = OrderedEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
else:
if is_sequence:
output_generator = iter(generator)
else:
output_generator = generator
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
generator_output = next(output_generator)
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = model.train_on_batch(x,
y,
sample_weight=sample_weight,
class_weight=class_weight)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
if val_gen:
val_outs = model.evaluate_generator(
validation_data,
validation_steps,
workers=workers,
use_multiprocessing=use_multiprocessing,
max_queue_size=max_queue_size)
else:
# No need for try/except because
# data has already been validated.
val_outs = model.evaluate(
val_x,
val_y,
batch_size=batch_size,
sample_weight=val_sample_weights,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
if callback_model.stop_training:
break
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
if val_enqueuer is not None:
val_enqueuer.stop()
callbacks.on_train_end()
return model.history
def main():
encoder, decoder, discriminator, vae, vae_loss = create_models()
#
# encoder.compile('rmsprop', 'mse')
#
# x = np.random.uniform(-1.0, 1.0, size=[1, 64, 64, 1])
# y1 = np.random.uniform(-1.0, 1.0, size=[1, 128])
# y2 = np.random.uniform(-1.0, 1.0, size=[1, 128])
#
# encoder.fit(x, [y1, y2], callbacks=[TensorBoard()])
#
# return
batch_size = 32
(x_train, y_train), (x_test, y_test) = mnist.load_data()
# Resize to 64x64
x_train_new = np.zeros((x_train.shape[0], 64, 64), dtype='int32')
for i, img in enumerate(x_train):
x_train_new[i] = cv2.resize(img, (64, 64),
interpolation=cv2.INTER_CUBIC)
x_train = x_train_new
del x_train_new
# Normalize to [-1, 1]
#x_train = np.pad(x_train, ((0, 0), (18, 18), (18, 18)), mode='constant', constant_values=0)
x_train = np.expand_dims(x_train, -1)
x_train = (x_train.astype('float32') - 127.5) / 127.5
x_train = np.clip(x_train, -1., 1.)
# Assume images in x_train
# x_train = np.zeros((100, 64, 64, 3))
discriminator.compile('rmsprop', 'binary_crossentropy', ['accuracy'])
discriminator.trainable = False
model = Model(vae.inputs, discriminator(vae.outputs), name='vaegan')
model.add_loss(vae_loss)
model.compile('rmsprop', 'binary_crossentropy', ['accuracy'])
import keras.callbacks as cbks
import os.path
verbose = True
checkpoint = cbks.ModelCheckpoint(os.path.join('.',
'model.{epoch:02d}.h5'),
save_weights_only=True)
callbacks = [TensorBoard(batch_size=batch_size), checkpoint]
epochs = 100
steps_per_epoch = x_train.shape[0] // batch_size
do_validation = False
callback_metrics = [
'disc_loss', 'disc_accuracy', 'vaegan_loss', 'vaegan_accuracy'
]
model.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [model.history]
if verbose:
callbacks += [cbks.ProgbarLogger(count_mode='steps')]
callbacks = cbks.CallbackList(callbacks)
# it's possible to callback a different model than self:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
epoch_logs = {}
for epoch in range(epochs):
callbacks.on_epoch_begin(epoch)
for batch_index in range(steps_per_epoch):
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
rand_indexes = np.random.randint(0,
x_train.shape[0],
size=batch_size)
real_images = x_train[rand_indexes]
fake_images = vae.predict(real_images)
# print(fake_images.shape)
half_batch = batch_size // 2
inputs = np.concatenate(
[real_images[:half_batch], fake_images[:half_batch]])
# Label real and fake images
y = np.ones([batch_size, 1], dtype='float32')
y[half_batch:, :] = 0
# Train the Discriminator network
metrics = discriminator.train_on_batch(inputs, y)
# print('discriminator', metrics)
y = np.ones([batch_size, 1], dtype='float32')
vg_metrics = model.train_on_batch(fake_images, y)
# print('full', metrics)
batch_logs['disc_loss'] = metrics[0]
batch_logs['disc_accuracy'] = metrics[1]
batch_logs['vaegan_loss'] = vg_metrics[0]
batch_logs['vaegan_accuracy'] = vg_metrics[1]
callbacks.on_batch_end(batch_index, batch_logs)
callbacks.on_epoch_end(epoch, epoch_logs)
rand_indexes = np.random.randint(0, x_train.shape[0], size=1)
real_images = x_train[rand_indexes]
model.save_weights('trained.h5')
a = encoder.predict(real_images)
print(a)
def fit_with_pseudo_label(self,
steps_per_epoch,
use_checkpoints=False,
class_labels=None,
verbose=1,
use_multiprocessing=False,
shuffle=False,
workers=1,
max_queue_size=10):
wait_time = 0.01 # in seconds
self.model._make_train_function()
# Create a checkpoint callback
checkpoint = ModelCheckpoint("../models_checkpoints/" +
str(self.h5_filename) + ".h5",
monitor='val_acc',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1)
# Generate callbacks
callback_list = []
if use_checkpoints:
callback_list.extend(checkpoint)
# Init train counters
epoch = 0
# Prepare display labels.
out_labels = self.model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# Prepare train callbacks
self.model.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callback_list or []) + \
[self.model.history]
if verbose:
callbacks += [cbks.ProgbarLogger(count_mode='steps')]
callbacks = cbks.CallbackList(callbacks)
# it's possible to callback a different model than self:
if hasattr(self.model, 'callback_model') and self.model.callback_model:
callback_model = self.model.callback_model
else:
callback_model = self.model
callbacks.set_model(callback_model)
is_sequence = isinstance(self.train_generator, Sequence)
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
if is_sequence:
steps_per_epoch = len(self.train_generator)
enqueuer = None
callbacks.set_params({
'epochs': self.epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': True,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
try:
if is_sequence:
enqueuer = OrderedEnqueuer(
self.train_generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
self.train_generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
# Train the model
# Epochs
while epoch < self.epochs:
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
# Steps per epoch
while steps_done < steps_per_epoch:
generator_output = next(output_generator)
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
#==========================
# Mini-batch
#==========================
print ''
print 'Generating pseudo-labels...'
no_label_output = self.model.predict_generator(
self.no_label_generator,
None, # because the model is instance of sequence
verbose=1)
# One-hot encoded
self.no_label_generator.classes = np.argmax(
no_label_output, axis=1)
# Concat Pseudo labels with true labels
x_pseudo, y_pseudo = next(self.no_label_generator)
x, y = np.concatenate(
(x, x_pseudo), axis=0), np.concatenate((y, y_pseudo),
axis=0)
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
# Runs a single gradient update on a single batch of data
scalar_training_loss = self.model.train_on_batch(x=x, y=y)
if not isinstance(scalar_training_loss, list):
scalar_training_loss = [scalar_training_loss]
for l, o in zip(out_labels, scalar_training_loss):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
#==========================
# end Mini-batch
#==========================
batch_index += 1
steps_done += 1
# Epoch finished.
epoch += 1
finally:
if enqueuer is not None:
enqueuer.stop()
callbacks.on_train_end()
return self.model.history
def evaluate_generator_autosized(model,
generator,
steps=None,
callbacks=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
verbose=0):
"""See docstring for `Model.evaluate_generator`."""
model._make_test_function()
stateful_metric_indices = []
if hasattr(model, 'metrics'):
for m in model.stateful_metric_functions:
m.reset_states()
stateful_metric_indices = [
i for i, name in enumerate(model.metrics_names)
if str(name) in model.stateful_metric_names
]
else:
stateful_metric_indices = []
callbacks = cbks.CallbackList(callbacks or [])
# it's possible to callback a different model than self:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': 1,
'steps': steps, # if None, will be refined during first epoch
'verbose': verbose,
'do_validation': False,
'metrics': model.metrics_names,
})
steps_done = 0
wait_time = 0.01
outs_per_batch = []
batch_sizes = []
is_sequence = isinstance(generator, Sequence)
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
# if steps is None:
# if is_sequence:
# steps = len(generator)
# else:
# raise ValueError('`steps=None` is only valid for a generator'
# ' based on the `keras.utils.Sequence` class.'
# ' Please specify `steps` or use the'
# ' `keras.utils.Sequence` class.')
enqueuer = None
try:
if workers > 0:
if is_sequence:
enqueuer = OrderedEnqueuer(
generator, use_multiprocessing=use_multiprocessing)
else:
enqueuer = GeneratorEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
else:
if is_sequence:
output_generator = iter(generator)
else:
output_generator = generator
if verbose == 1:
progbar = Progbar(target=steps)
callbacks.on_epoch_begin(0)
for generator_output in output_generator:
if not generator_output: # end of epoch?
break
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be a tuple '
'(x, y, sample_weight) '
'or (x, y). Found: ' + str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be a tuple '
'(x, y, sample_weight) '
'or (x, y). Found: ' + str(generator_output))
# build batch logs
batch_logs = {}
if not x:
# Handle data tensors support when no input given
# step-size = 1 for data tensors
batch_size = 1
elif isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
if batch_size == 0:
raise ValueError('Received an empty batch. '
'Batches should contain '
'at least one item.')
batch_logs['batch'] = steps_done
batch_logs['size'] = batch_size
callbacks.on_batch_begin(steps_done, batch_logs)
outs = model.test_on_batch(x, y, sample_weight=sample_weight)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(model.metrics_names, outs):
batch_logs[l] = o
outs_per_batch.append(outs)
callbacks.on_batch_end(steps_done, batch_logs)
steps_done += 1
batch_sizes.append(batch_size)
if verbose == 1:
log_values = []
for k in model.metrics_names:
if k in batch_logs:
log_values.append(('val_' + k, batch_logs[k]))
progbar.update(steps_done, log_values)
callbacks.on_epoch_end(1, {})
finally:
if enqueuer is not None:
enqueuer.stop()
averages = []
for i in range(len(model.metrics_names)):
if i not in stateful_metric_indices:
averages.append(
np.average([out[i] for out in outs_per_batch],
weights=batch_sizes))
else:
averages.append(float(outs_per_batch[-1][i]))
if len(averages) == 1:
return averages[0], steps_done
return averages, steps_done
def fit_tfrecord(train_model,
nb_train_sample,
batch_size,
nb_epoch=10,
verbose=1,
callbacks=[],
initial_epoch=0):
def _make_train_function(model):
if not hasattr(model, 'train_function'):
raise RuntimeError('You must compile your model before using it.')
if model.train_function is None:
inputs = [K.learning_phase()]
training_updates = model.optimizer.get_updates(
model._collected_trainable_weights, model.constraints,
model.total_loss)
updates = model.updates + training_updates
# returns loss and metrics. Updates weights at each call.
model.train_function = K.function(inputs, [model.total_loss] +
model.metrics_tensors,
updates=updates)
ins = [1.]
_make_train_function(train_model)
f = train_model.train_function
# prepare display labels
out_labels = train_model.metrics_names
# rename duplicated metrics name
# (can happen with an output layer shared among multiple dataflows)
deduped_out_labels = []
for i, label in enumerate(out_labels):
new_label = label
if out_labels.count(label) > 1:
dup_idx = out_labels[:i].count(label)
new_label += '_' + str(dup_idx + 1)
deduped_out_labels.append(new_label)
out_labels = deduped_out_labels
callback_metrics = copy.copy(out_labels)
train_model.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks) + [train_model.history]
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
callback_model = train_model
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': nb_epoch,
'samples': nb_train_sample,
'verbose': verbose,
'do_validation': False,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
sess = K.get_session()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for epoch in range(initial_epoch, nb_epoch):
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
for batch_index in range(0, nb_train_sample // batch_size):
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
coord.request_stop()
coord.join(threads)
# sess.close()
return train_model.history
def fit_generator_Ndiff(model,
generator,
steps_per_epoch=None,
batch_size=1,
N_diff=5,
margin=0.5,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
class_weight=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
shuffle=True,
initial_epoch=0):
"""Trains the model on data yielded batch-by-batch by a Python generator.
The generator is run in parallel to the model, for efficiency.
For instance, this allows you to do real-time data augmentation
on images on CPU in parallel to training your model on GPU.
The use of `keras.utils.Sequence` guarantees the ordering
and guarantees the single use of every input per epoch when
using `use_multiprocessing=True`.
# Arguments
generator: A generator or an instance of `Sequence`
(`keras.utils.Sequence`) object in order to avoid
duplicate data when using multiprocessing.
The output of the generator must be either
- a tuple `(inputs, targets)`
- a tuple `(inputs, targets, sample_weights)`.
This tuple (a single output of the generator) makes a single
batch. Therefore, all arrays in this tuple must have the same
length (equal to the size of this batch). Different batches
may have different sizes. For example, the last batch of the
epoch is commonly smaller than the others, if the size of the
dataset is not divisible by the batch size.
The generator is expected to loop over its data
indefinitely. An epoch finishes when `steps_per_epoch`
batches have been seen by the model.
steps_per_epoch: Integer.
Total number of steps (batches of samples)
to yield from `generator` before declaring one epoch
finished and starting the next epoch. It should typically
be equal to the number of samples of your dataset
divided by the batch size.
Optional for `Sequence`: if unspecified, will use
the `len(generator)` as a number of steps.
epochs: Integer. Number of epochs to train the model.
An epoch is an iteration over the entire data provided,
as defined by `steps_per_epoch`.
Note that in conjunction with `initial_epoch`,
`epochs` is to be understood as "final epoch".
The model is not trained for a number of iterations
given by `epochs`, but merely until the epoch
of index `epochs` is reached.
verbose: Integer. 0, 1, or 2. Verbosity mode.
0 = silent, 1 = progress bar, 2 = one line per epoch.
callbacks: List of `keras.callbacks.Callback` instances.
List of callbacks to apply during training.
See [callbacks](/callbacks).
validation_data: This can be either
- a generator for the validation data
- tuple `(x_val, y_val)`
- tuple `(x_val, y_val, val_sample_weights)`
on which to evaluate
the loss and any model metrics at the end of each epoch.
The model will not be trained on this data.
validation_steps: Only relevant if `validation_data`
is a generator. Total number of steps (batches of samples)
to yield from `validation_data` generator before stopping.
Optional for `Sequence`: if unspecified, will use
the `len(validation_data)` as a number of steps.
class_weight: Optional dictionary mapping class indices (integers)
to a weight (float) value, used for weighting the loss function
(during training only).
This can be useful to tell the model to
"pay more attention" to samples from
an under-represented class.
max_queue_size: Integer. Maximum size for the generator queue.
If unspecified, `max_queue_size` will default to 10.
workers: Integer. Maximum number of processes to spin up
when using process based threading.
If unspecified, `workers` will default to 1. If 0, will
execute the generator on the main thread.
use_multiprocessing: Boolean. If True, use process based threading.
If unspecified, `use_multiprocessing` will default to False.
Note that because
this implementation relies on multiprocessing,
you should not pass
non picklable arguments to the generator
as they can't be passed
easily to children processes.
shuffle: Boolean. Whether to shuffle the training data
in batch-sized chunks before each epoch.
Only used with instances of `Sequence` (`keras.utils.Sequence`).
initial_epoch: Integer.
Epoch at which to start training
(useful for resuming a previous training run).
# Returns
A `History` object. Its `History.history` attribute is
a record of training loss values and metrics values
at successive epochs, as well as validation loss values
and validation metrics values (if applicable).
# Example
```python
def generate_arrays_from_file(path):
while 1:
with open(path) as f:
for line in f:
# create numpy arrays of input data
# and labels, from each line in the file
x1, x2, y = process_line(line)
yield ({'input_1': x1, 'input_2': x2}, {'output': y})
model.fit_generator(generate_arrays_from_file('/my_file.txt'),
steps_per_epoch=10000, epochs=10)
```
# Raises
ValueError: In case the generator yields
data in an invalid format.
"""
wait_time = 0.01 # in seconds
epoch = initial_epoch
do_validation = bool(validation_data)
# self._make_train_function()
# if do_validation:
# self._make_test_function()
is_sequence = isinstance(generator, Sequence)
# do_validation = True if is_sequence else False
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
if steps_per_epoch is None:
if is_sequence:
steps_per_epoch = len(generator)
else:
raise ValueError('`steps_per_epoch=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `steps_per_epoch` or use'
' the `keras.utils.Sequence` class.')
# python 2 has 'next', 3 has '__next__'
# avoid any explicit version checks
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__')
or isinstance(validation_data, Sequence))
if (val_gen and not isinstance(validation_data, Sequence)
and not validation_steps):
raise ValueError('`validation_steps=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `validation_steps` or use'
' the `keras.utils.Sequence` class.')
# Prepare display labels.
out_labels = model._get_deduped_metrics_names()
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [history]
if verbose:
callbacks += [cbks.ProgbarLogger(count_mode='steps')]
callbacks = cbks.CallbackList(callbacks)
# # it's possible to callback a different model than self:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
enqueuer = None
val_enqueuer = None
try:
if do_validation:
if val_gen:
if workers > 0:
if isinstance(validation_data, Sequence):
val_enqueuer = OrderedEnqueuer(
validation_data,
use_multiprocessing=use_multiprocessing)
if validation_steps is None:
validation_steps = len(validation_data)
else:
val_enqueuer = GeneratorEnqueuer(
validation_data,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
val_enqueuer.start(workers=workers,
max_queue_size=max_queue_size)
validation_generator = val_enqueuer.get()
else:
validation_generator = validation_data
else:
pass
# if len(validation_data) == 2:
# val_x, val_y = validation_data
# val_sample_weights = None
# elif len(validation_data) == 3:
# val_x, val_y, val_sample_weights = validation_data
# else:
# raise ValueError('`validation_data` should be a tuple '
# '`(val_x, val_y, val_sample_weight)` '
# 'or `(val_x, val_y)`. Found: ' +
# str(validation_data))
# val_x, val_y, val_sample_weights = _standardize_user_data(
# val_x, val_y, val_sample_weight)
# val_data = val_x + val_y + val_sample_weights
# if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
# val_data += [0.]
# for cbk in callbacks:
# cbk.validation_data = val_data
if workers > 0:
if is_sequence:
enqueuer = OrderedEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
else:
output_generator = generator
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
generator_output = next(output_generator)
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'batch_size lists ' +
str(generator_output))
if len(generator_output) == batch_size:
# ii_ndiff: the index of the negative sample
gen_out = generator_output
sample_weight = None
else:
raise ValueError('Output of generator should be '
'batch_size lists ' +
str(generator_output))
# build batch logs
batch_logs = {}
# if isinstance(x, list):
# batch_size = x[0].shape[0]
# elif isinstance(x, dict):
# batch_size = list(x.values())[0].shape[0]
# else:
# batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
# aggregate the losses by inner index n_diff
loss_mat = np.zeros((batch_size, N_diff))
for ii_ndiff in range(N_diff):
# get the maximum sequence length
len_anchor_max, len_same_max, len_diff_max = \
get_maximum_length(batch_size=batch_size,
generator_output=gen_out,
index=[ii_ndiff]*batch_size)
print(len_anchor_max, len_same_max, len_diff_max)
# organize the input for the prediction
input_anchor, input_same, input_diff = \
make_same_length_batch(batch_size=batch_size,
len_anchor_max=len_anchor_max,
len_same_max=len_same_max,
len_diff_max=len_diff_max,
generator_output=gen_out,
index=[ii_ndiff]*batch_size)
output_batch_pred = model.predict_on_batch(
[input_anchor, input_same, input_diff])
loss = K.eval(
triplet_loss_no_mean(output_batch_pred, margin))
loss_mat[:, ii_ndiff] = loss
# this the index of the input which has the maximum loss for each N_diff pairs
index_max_loss = np.argmax(loss_mat, axis=-1)
len_anchor_max, len_same_max, len_diff_max = get_maximum_length(
batch_size=batch_size,
generator_output=gen_out,
index=index_max_loss)
input_anchor, input_same, input_diff = \
make_same_length_batch(batch_size=batch_size,
len_anchor_max=len_anchor_max,
len_same_max=len_same_max,
len_diff_max=len_diff_max,
generator_output=gen_out,
index=index_max_loss)
outs = model.train_on_batch(
[input_anchor, input_same, input_diff],
None,
sample_weight=sample_weight,
class_weight=class_weight)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
if val_gen:
val_outs = evaluate_generator(
model=model,
generator=validation_generator,
steps=validation_steps,
batch_size=batch_size,
margin=margin,
N_diff=N_diff,
workers=0)
else:
pass
# # No need for try/except because
# # data has already been validated.
# val_outs = model.evaluate(
# val_x, val_y,
# batch_size=batch_size,
# sample_weight=val_sample_weights,
# verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
if callback_model.stop_training:
break
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
if val_enqueuer is not None:
val_enqueuer.stop()
callbacks.on_train_end()
return history
def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
nb_epoch=100, verbose=1, callbacks=None,
val_f=None, val_ins=None, shuffle=True,
callback_metrics=None, initial_epoch=0):
"""Abstract fit function for f(ins).
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: list of tensors to be fed to `f`
out_labels: list of strings, display names of
the outputs of `f`
batch_size: integer batch size
nb_epoch: number of times to iterate over the data
verbose: verbosity mode, 0, 1 or 2
callbacks: list of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: list of tensors to be fed to `val_f`
shuffle: whether to shuffle the data at the beginning of each epoch
callback_metrics: list of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
# Returns
`History` object.
[A tweaked version.]
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
nb_train_sample = ins[0].shape[0]
index_array = np.arange(nb_train_sample)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
self.validation_data = val_ins
for epoch in range(initial_epoch, nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['ids'] = batch_ids
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index == len(batches) - 1: # last batch
# validation
if do_validation:
# replace with self._evaluate
val_outs = self._test_loop(val_f, val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def _fit(self,
f,
ins,
out_labels=[],
batch_size=128,
nb_epoch=100,
verbose=1,
callbacks=[],
val_f=None,
val_ins=None,
shuffle=True,
metrics=[]):
'''
Abstract fit function for f(*ins). Assume that f returns a list, labelled by out_labels.
'''
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print("Train on %d samples, validate on %d samples" %
(len(ins[0]), len(val_ins[0])))
nb_train_sample = len(ins[0])
index_array = np.arange(nb_train_sample)
history = cbks.History()
if verbose:
callbacks = [history, cbks.BaseLogger()] + callbacks
else:
callbacks = [history] + callbacks
callbacks = cbks.CallbackList(callbacks)
callbacks._set_model(self)
callbacks._set_params({
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': verbose,
'do_validation': do_validation,
'metrics': metrics,
})
callbacks.on_train_begin()
self.stop_training = False
for epoch in range(nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
ins_batch = slice_X(ins, batch_ids)
except TypeError as err:
raise Exception('TypeError while preparing batch. \
If using HDF5 input data, pass shuffle="batch".\n')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(*ins_batch)
if type(outs) != list:
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
epoch_logs = {}
if batch_index == len(batches) - 1: # last batch
# validation
if do_validation:
# replace with self._evaluate
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if type(val_outs) != list:
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if self.stop_training:
break
callbacks.on_train_end()
return history
def fit_with_pseudo_label(self,
steps_per_epoch,
validation_steps=None,
use_checkpoints=True,
class_labels=None,
verbose=1,
use_multiprocessing=False,
shuffle=False,
workers=1,
max_queue_size=10):
# Default value if validation steps is none
if (validation_steps == None):
validation_steps = self.validation_generator.samples // self.batch_size
wait_time = 0.01 # in seconds
self.model._make_train_function()
# Create a checkpoint callback
checkpoint = ModelCheckpoint("../models_checkpoints/" +
str(self.h5_filename) + ".h5",
monitor='val_acc',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1)
# Generate callbacks
callback_list = []
if use_checkpoints:
callback_list.append(checkpoint)
# Init train counters
epoch = 0
validation_data = self.validation_generator
do_validation = bool(validation_data)
self.model._make_train_function()
if do_validation:
self.model._make_test_function()
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__')
or isinstance(validation_data, Sequence))
if (val_gen and not isinstance(validation_data, Sequence)
and not validation_steps):
raise ValueError('`validation_steps=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `validation_steps` or use'
' the `keras.utils.Sequence` class.')
# Prepare display labels.
out_labels = self.model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# Prepare train callbacks
self.model.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callback_list or []) + \
[self.model.history]
if verbose:
callbacks += [cbks.ProgbarLogger(count_mode='steps')]
callbacks = cbks.CallbackList(callbacks)
# it's possible to callback a different model than self:
if hasattr(self.model, 'callback_model') and self.model.callback_model:
callback_model = self.model.callback_model
else:
callback_model = self.model
callbacks.set_model(callback_model)
is_sequence = isinstance(self.train_generator, Sequence)
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
if is_sequence:
steps_per_epoch = len(self.train_generator)
enqueuer = None
val_enqueuer = None
callbacks.set_params({
'epochs': self.epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
try:
if do_validation and not val_gen:
# Prepare data for validation
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise ValueError('`validation_data` should be a tuple '
'`(val_x, val_y, val_sample_weight)` '
'or `(val_x, val_y)`. Found: ' +
str(validation_data))
val_x, val_y, val_sample_weights = self.model._standardize_user_data(
val_x, val_y, val_sample_weight)
val_data = val_x + val_y + val_sample_weights
if self.model.uses_learning_phase and not isinstance(
K.learning_phase(), int):
val_data += [0.]
for cbk in callbacks:
cbk.validation_data = val_data
if is_sequence:
enqueuer = OrderedEnqueuer(
self.train_generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
self.train_generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
# Train the model
# Construct epoch logs.
epoch_logs = {}
# Epochs
while epoch < self.epochs:
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
# Steps per epoch
while steps_done < steps_per_epoch:
generator_output = next(output_generator)
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
#==========================
# Mini-batch
#==========================
if (self.print_pseudo_generate):
print ''
print 'Generating pseudo-labels...'
verbose = 1
else:
verbose = 0
if self.no_label_generator.samples > 0:
no_label_output = self.model.predict_generator(
self.no_label_generator,
self.no_label_generator.samples,
verbose=verbose)
# One-hot encoded
self.no_label_generator.classes = np.argmax(
no_label_output, axis=1)
# Concat Pseudo labels with true labels
x_pseudo, y_pseudo = next(self.no_label_generator)
x, y = np.concatenate((x, x_pseudo),
axis=0), np.concatenate(
(y, y_pseudo), axis=0)
# build batch logs
batch_logs = {}
if isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
callbacks.on_batch_begin(batch_index, batch_logs)
# Runs a single gradient update on a single batch of data
scalar_training_loss = self.model.train_on_batch(x=x, y=y)
if not isinstance(scalar_training_loss, list):
scalar_training_loss = [scalar_training_loss]
for l, o in zip(out_labels, scalar_training_loss):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
#==========================
# end Mini-batch
#==========================
batch_index += 1
steps_done += 1
if steps_done >= steps_per_epoch and do_validation:
if val_gen:
val_outs = self.model.evaluate_generator(
validation_data,
validation_steps,
workers=workers,
use_multiprocessing=use_multiprocessing,
max_queue_size=max_queue_size)
else:
# No need for try/except because
# data has already been validated.
val_outs = self.model.evaluate(
val_x,
val_y,
batch_size=batch_size,
sample_weight=val_sample_weights,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
# Epoch finished.
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
if val_enqueuer is not None:
val_enqueuer.stop()
callbacks.on_train_end()
return self.model.history
def fit_tfrecord(self,
steps_per_epoch,
epochs=1,
verbose=1,
callbacks=None,
validation_steps=None,
initial_epoch=0):
epoch = initial_epoch
self._make_tfrecord_train_function()
do_validation = bool(len(self.val_inputs) > 0)
if do_validation and not validation_steps:
raise ValueError('When using a validation batch, '
'you must specify a value for '
'`validation_steps`.')
# Prepare display labels.
out_labels = self._get_deduped_metrics_names()
if do_validation:
callback_metrics = copy.copy(out_labels) + [
'val_' + n for n in out_labels
]
else:
callback_metrics = copy.copy(out_labels)
# prepare callbacks
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
callbacks += [cbks.ProgbarLogger(count_mode='steps')]
callbacks = cbks.CallbackList(callbacks)
# it's possible to callback a different model than self:
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
if do_validation:
val_sample_weight = None
for cbk in callbacks:
cbk.validation_data = [
self.val_inputs, self.y_val, val_sample_weight
]
try:
sess = K.get_session()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
callback_model.stop_training = False
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
while steps_done < steps_per_epoch:
# build batch logs
batch_logs = {
'batch': batch_index,
'size': self.inputs[0].shape[0].value
}
callbacks.on_batch_begin(batch_index, batch_logs)
if self.uses_learning_phase and not isinstance(
K.learning_phase(), int):
ins = [1.]
else:
ins = []
outs = self.train_function(ins)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
# Construct epoch logs.
epoch_logs = {}
batch_index += 1
steps_done += 1
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
val_outs = self._validate_tfrecord(
steps=validation_steps)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
# TODO: If you close the queue, you can't open it again..
# coord.request_stop()
# coord.join(threads)
pass
callbacks.on_train_end()
return self.history
def fit_and_predict_generator_with_sceneinst_metrics(
model,
generator,
params,
multithreading_metrics=False,
steps_per_epoch=None,
epochs=1,
verbose=1,
callbacks=None,
validation_data=None,
validation_steps=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False,
shuffle=True,
initial_epoch=0):
"""See docstring for `Model.fit_generator`."""
wait_time = 0.01 # in seconds
epoch = initial_epoch
do_validation = bool(validation_data)
model._make_train_function()
if do_validation:
model._make_test_function()
is_sequence = isinstance(generator, Sequence)
if not is_sequence and use_multiprocessing and workers > 1:
warnings.warn(
UserWarning('Using a generator with `use_multiprocessing=True`'
' and multiple workers may duplicate your data.'
' Please consider using the`keras.utils.Sequence'
' class.'))
if steps_per_epoch is None:
if is_sequence:
steps_per_epoch = len(generator)
else:
raise ValueError('`steps_per_epoch=None` is only valid for a'
' generator based on the '
'`keras.utils.Sequence`'
' class. Please specify `steps_per_epoch` '
'or use the `keras.utils.Sequence` class.')
# python 2 has 'next', 3 has '__next__'
# avoid any explicit version checks
val_gen = (hasattr(validation_data, 'next')
or hasattr(validation_data, '__next__')
or isinstance(validation_data, Sequence))
if (val_gen and not isinstance(validation_data, Sequence)
and not validation_steps):
raise ValueError('`validation_steps=None` is only valid for a'
' generator based on the `keras.utils.Sequence`'
' class. Please specify `validation_steps` or use'
' the `keras.utils.Sequence` class.')
# Prepare display labels.
out_labels = model.metrics_names
callback_metrics = out_labels + ['val_' + n for n in out_labels]
# prepare callbacks
model.history = cbks.History()
_callbacks = [
cbks.BaseLogger(stateful_metrics=model.stateful_metric_names)
]
if verbose:
_callbacks.append(
cbks.ProgbarLogger(count_mode='steps',
stateful_metrics=model.stateful_metric_names))
_callbacks += (callbacks or []) + [model.history]
callbacks = cbks.CallbackList(_callbacks)
# it's possible to callback a different model than self:
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': epochs,
'steps': steps_per_epoch,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics,
})
callbacks.on_train_begin()
enqueuer = None
val_enqueuer = None
try:
if do_validation:
if val_gen and workers > 0:
# Create an Enqueuer that can be reused
val_data = validation_data
if isinstance(val_data, Sequence):
val_enqueuer = OrderedEnqueuer(
val_data, use_multiprocessing=use_multiprocessing)
validation_steps = len(val_data)
else:
val_enqueuer = GeneratorEnqueuer(
val_data, use_multiprocessing=use_multiprocessing)
val_enqueuer.start(workers=workers,
max_queue_size=max_queue_size)
val_enqueuer_gen = val_enqueuer.get()
elif val_gen:
val_data = validation_data
if isinstance(val_data, Sequence):
val_enqueuer_gen = iter(val_data)
else:
val_enqueuer_gen = val_data
else:
# Prepare data for validation
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise ValueError('`validation_data` should be a tuple '
'`(val_x, val_y, val_sample_weight)` '
'or `(val_x, val_y)`. Found: ' +
str(validation_data))
val_x, val_y, val_sample_weights = model._standardize_user_data(
val_x, val_y, val_sample_weight)
val_data = val_x + val_y + val_sample_weights
if model.uses_learning_phase and not isinstance(
K.learning_phase(), int):
val_data += [0.]
for cbk in callbacks:
cbk.validation_data = val_data
if workers > 0:
if is_sequence:
enqueuer = OrderedEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
shuffle=shuffle)
else:
enqueuer = GeneratorEnqueuer(
generator,
use_multiprocessing=use_multiprocessing,
wait_time=wait_time)
enqueuer.start(workers=workers, max_queue_size=max_queue_size)
output_generator = enqueuer.get()
else:
if is_sequence:
output_generator = iter(generator)
else:
output_generator = generator
callback_model.stop_training = False
# Construct epoch logs.
epoch_logs = {}
while epoch < epochs:
# setup scene instance dictionary
model.scene_instance_id_metrics_dict_train = {}
# create thread for asynchronous batch metrics calculation (one thread per epoch, joined before final metrics calculation)
if multithreading_metrics:
label_queue = queue.Queue(
) # threadsafe queue into which we will push (y_pred, y) tuples
trainmetrics_thread = threading.Thread(
target=metrics_per_batch_thread_handler,
args=(label_queue,
model.scene_instance_id_metrics_dict_train,
params['mask_value'], steps_per_epoch))
trainmetrics_thread.start()
#print('thread for calculating the batch train metrics has been started')
for m in model.stateful_metric_functions:
m.reset_states()
callbacks.on_epoch_begin(epoch)
steps_done = 0
batch_index = 0
runtime_generator_cumulated = 0.
runtime_train_and_predict_on_batch_cumulated = 0.
runtime_class_accuracies_cumulated = 0.
skip_runtime_avg = 5 # skipping the first few batches to reduce bias due to inital extra time
while steps_done < steps_per_epoch:
t_start_batch = time()
t_start = time()
generator_output = next(output_generator)
runtime_generator_next = time() - t_start
if batch_index >= skip_runtime_avg:
runtime_generator_cumulated += runtime_generator_next
if not hasattr(generator_output, '__len__'):
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
if len(generator_output) == 2:
x, y = generator_output
sample_weight = None
elif len(generator_output) == 3:
x, y, sample_weight = generator_output
else:
raise ValueError('Output of generator should be '
'a tuple `(x, y, sample_weight)` '
'or `(x, y)`. Found: ' +
str(generator_output))
# build batch logs
batch_logs = {}
if x is None or len(x) == 0:
# Handle data tensors support when no input given
# step-size = 1 for data tensors
batch_size = 1
elif isinstance(x, list):
batch_size = x[0].shape[0]
elif isinstance(x, dict):
batch_size = list(x.values())[0].shape[0]
else:
batch_size = x.shape[0]
batch_logs['batch'] = batch_index
batch_logs['size'] = batch_size
t_start = time()
callbacks.on_batch_begin(batch_index, batch_logs)
runtime_callbacks_on_batch_begin = time() - t_start
# remark on label shape: last (fourth) dimension contains in 0 the true labels, in 1 the corresponding sceneinstid (millioncode)
t_start = time()
# set sample weights
if params['nosceneinstweights']:
sample_weight = None
else:
sample_weight = heiner_calculate_sample_weights_batch(
y[:, :, 0, 1], generator.length_dict,
generator.scene_instance_ids_dict, 'train')
# run forward and backward pass and do the gradient descent step
batch_loss, y_pred_logits, gradient_norm = heiner_train_and_predict_on_batch(
model,
x,
y[:, :, :, 0],
sample_weight=sample_weight,
calc_global_gradient_norm=not params['nocalcgradientnorm'])
runtime_train_and_predict_on_batch = time() - t_start
if batch_index >= skip_runtime_avg:
runtime_train_and_predict_on_batch_cumulated += runtime_train_and_predict_on_batch
batch_logs['loss'] = batch_loss
model.gradient_norm = gradient_norm
t_start = time()
# from logits to predicted class probabilities
y_pred_probs = sigmoid(y_pred_logits,
out=y_pred_logits) # last arg: inplace
# from probabilities to hard class decisions
y_pred = np.greater_equal(
y_pred_probs, params['outputthreshold'],
out=y_pred_probs) # last arg: inplace
# increment metrics for scene instances in batch
if multithreading_metrics:
# the following two arrays need to be unchanged in order for being thread-safe
# assumption 1: batchloader yields array copies (true for moritz loader)
# assumption 2: *_and_predict_on_batch return newly allocated arrays
label_queue.put((y_pred, y))
else:
heiner_calculate_class_accuracies_metrics_per_scene_instance_in_batch(
model.scene_instance_id_metrics_dict_train, y_pred, y,
params['mask_value'])
runtime_class_accuracies = time() - t_start
if batch_index >= skip_runtime_avg:
runtime_class_accuracies_cumulated += runtime_class_accuracies
t_start = time()
callbacks.on_batch_end(batch_index, batch_logs)
runtime_callbacks_on_batch_end = time() - t_start
runtime_batch = time() - t_start_batch
# print((' ----> batch {} in epoch {} took in total {:.2f} sec => generator {:.2f} ' +
# 'train_and_predict {:.2f}, metrics {:.2f}')
# .format(batch_index + 1, epoch + 1, runtime_batch, runtime_generator_next,
# runtime_train_and_predict_on_batch,
# runtime_class_accuracies))
batch_index += 1
steps_done += 1
if steps_done > skip_runtime_avg and steps_done == steps_per_epoch - 1:
print(
' --> batch {} we have average runtimes: generator {:.2f}, train_predict {:.2f}, metrics {:.2f}'
.format(
batch_index, runtime_generator_cumulated /
(steps_done - skip_runtime_avg),
runtime_train_and_predict_on_batch_cumulated /
(steps_done - skip_runtime_avg),
runtime_class_accuracies_cumulated /
(steps_done - skip_runtime_avg)))
# Epoch finished.
if steps_done >= steps_per_epoch and do_validation:
if val_gen:
val_outs = evaluate_and_predict_generator_with_sceneinst_metrics(
model,
val_enqueuer_gen,
params,
multithreading_metrics,
validation_steps,
workers=0,
verbose=1)
else:
# No need for try/except because
# data has already been validated.
val_outs = model.evaluate(
val_x,
val_y,
batch_size=batch_size,
sample_weight=val_sample_weights,
verbose=0)
val_outs = to_list(val_outs)
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
if callback_model.stop_training:
break
if multithreading_metrics:
trainmetrics_thread.join()
print(
' --> both threads for calculating the batch metrics -- training and validation -- finished all their work'
)
callbacks.on_epoch_end(epoch, epoch_logs)
epoch += 1
if callback_model.stop_training:
break
finally:
try:
if enqueuer is not None:
enqueuer.stop()
finally:
if val_enqueuer is not None:
val_enqueuer.stop()
if multithreading_metrics:
trainmetrics_thread.join() # joined again (harmless)
callbacks.on_train_end()
return model.history
