def fit_generator(self, generator, epochs=1,
validation_data=None,
callbacks=None,
verbose=True):
method = self._model.optimizer.method
x0 = self._collect_weights()
history = History()
_callbacks = [BaseLogger(stateful_metrics=self._model.metrics_names)]
_callbacks += (callbacks or []) + [history]
callback_list = CallbackList(_callbacks)
callback_list.set_model(self._model)
callback_list.set_params({
'epochs': epochs,
'verbose': False,
'metrics': list(self._model.metrics_names),
})
state = {
'epoch': 0,
'verbose': verbose,
'callbacks': callback_list,
'in_epoch': False,
'epoch_logs': {},
}
min_options = {
'maxiter': epochs,
'maxfun': epochs*10,
'ftol': 1e-10,
'gtol': 1e-10,
'eps': 1e-8,
}
val_generator = None
if validation_data is not None:
if isinstance(validation_data, keras.utils.Sequence):
val_generator = validation_data
elif isinstance(validation_data, tuple) and len(validation_data) == 2:
val_generator = GeneratorWrapper(*validation_data)
def on_iteration_end(xk):
cb = state['callbacks']
if val_generator is not None:
self._validate(xk, val_generator, state)
cb.on_epoch_end(state['epoch'], state['epoch_logs'])
# if state['verbose']:
# epoch_logs = state['epoch_logs']
# print('epoch: ', state['epoch'],
# ', '.join([' {0}: {1:.3e}'.format(k, v) for k, v in epoch_logs.items()]))
state['epoch'] += 1
state['in_epoch'] = False
state['epoch_logs'] = {}
callback_list.on_train_begin()
result = minimize(
self._fun_generator, x0, method=method, jac=True, options=min_options,
callback=on_iteration_end, args=(generator, state))
self._update_weights(result['x'])
callback_list.on_train_end()
return history
def prepare_callbacks(g,
d,
callbacks,
n_epochs=1,
n_batches=1,
include_d_metrics=False):
# all the callback stuff is from https://github.com/keras-team/keras/blob/master/keras/engine/training_generator.py
# NOTE: see if saving the weights of d_on_g is enough
# Prepare display labels.
out_labels = g.metrics_names
out_labels = [_replace_label_first_underscore(l) for l in out_labels]
# we only want to validate on the output of g
val_out_labels = ['val_' + n for n in out_labels if g.name in n]
callback_metrics = out_labels + val_out_labels
if include_d_metrics:
d_metrics_names = d.metrics_names
d_metrics_fake = ['d_training/' + l + '_fake' for l in d_metrics_names]
d_metrics_real = ['d_training/' + l + '_real' for l in d_metrics_names]
d_metrics_names = d_metrics_fake + d_metrics_real
callback_metrics += d_metrics_names
# prepare callbacks
g.history = cbks.History()
_callbacks = [cbks.BaseLogger(stateful_metrics=g.metrics_names[1:])]
_callbacks += (callbacks or []) + [g.history]
callbacks = CallbackList(_callbacks)
# it's possible to callback a different model than self:
callback_model = g._get_callback_model()
callbacks.set_model(callback_model)
callbacks.set_params({
'epochs': n_epochs,
'steps': n_batches,
'verbose': 0,
# 'do_validation': do_validation, to set when using validation data
'metrics': callback_metrics,
})
if not include_d_metrics:
d_metrics_fake, d_metrics_real = None, None
return callbacks, out_labels, val_out_labels, d_metrics_fake, d_metrics_real
def fit_dataset(self,
dataset,
steps_per_epoch=None,
batch_size=32,
epochs=1,
verbose=1,
callbacks=None,
on_sample=None,
on_scores=None):
"""Train the model on the given dataset for a given number of epochs.
Arguments
---------
dataset: Instance of `BaseDataset` that provides the data
to train on.
steps_per_epoch: int or None, number of gradient updates before
considering an epoch has passed. If None it is set
to be `len(dataset.train_data) / batch_size`.
batch_size: int, number of samples per gradient update
epochs: int, number of times to iterate `steps_per_epoch` times
verbose: {0, >0}, whether to employ the progress bar Keras
callback or not
callbacks: list of Keras callbacks to be called during training
on_sample: callable that accepts the sampler, idxs, w, scores
on_scores: callable that accepts the sampler and scores
"""
try:
if len(dataset.train_data) < batch_size:
raise ValueError(("The model cannot be trained with "
"batch_size > training set"))
except RuntimeError as e:
assert "no size" in str(e)
# Set steps_per_epoch properly
if steps_per_epoch is None:
steps_per_epoch = len(dataset.train_data) // batch_size
# Create the callbacks list
self.history = History()
callbacks = [BaseLogger()] + (callbacks or []) + [self.history]
if verbose > 0:
callbacks += [ProgbarLogger(count_mode="steps")]
callbacks = CallbackList(callbacks)
callbacks.set_model(self.original_model)
callbacks.set_params({
"epochs":
epochs,
"steps":
steps_per_epoch,
"verbose":
verbose,
"do_validation":
len(dataset.test_data) > 0,
"metrics":
self.metrics_names + ["val_" + n for n in self.metrics_names]
})
# Create the sampler
sampler = self.sampler(dataset, batch_size, steps_per_epoch, epochs)
# Start the training loop
epoch = 0
self.original_model.stop_training = False
callbacks.on_train_begin()
while epoch < epochs:
callbacks.on_epoch_begin(epoch)
for step in range(steps_per_epoch):
batch_logs = {"batch": step, "size": batch_size}
callbacks.on_batch_begin(step, batch_logs)
# Importance sampling is done here
idxs, (x, y), w = sampler.sample(batch_size)
# Train on the sampled data
loss, metrics, scores = self.model.train_batch(x, y, w)
# Update the sampler
sampler.update(idxs, scores)
values = map(lambda x: x.mean(), [loss] + metrics)
for l, o in zip(self.metrics_names, values):
batch_logs[l] = o
callbacks.on_batch_end(step, batch_logs)
if on_scores is not None and hasattr(self, "_latest_scores"):
on_scores(sampler, self._latest_scores)
if on_sample is not None:
on_sample(sampler, self._latest_sample_event["idxs"],
self._latest_sample_event["w"],
self._latest_sample_event["predicted_scores"])
if self.original_model.stop_training:
break
# Evaluate now that an epoch passed
epoch_logs = {}
if len(dataset.test_data) > 0:
val = self.model.evaluate(*dataset.test_data[:],
batch_size=batch_size)
epoch_logs = {
"val_" + l: o
for l, o in zip(self.metrics_names, val)
}
callbacks.on_epoch_end(epoch, epoch_logs)
if self.original_model.stop_training:
break
epoch += 1
callbacks.on_train_end()
return self.history
def fit_generator(self,
generator,
n_steps_per_epoch,
n_epochs=1,
validation_data=None,
n_validation_steps=None):
"""Train the network on batches of data generated from `generator`
:param generator: a generator yielding batches indefinitely, where each
batch is a tuple of (inputs, targets)
:type generator: generator
:param n_steps_per_epoch: number of batches to train on in one epoch
:type n_steps_per_epoch: int
:param n_epochs: number of epochs to train the model
:type n_epochs: int
:param validation_data: generator yielding batches to evaluate the loss
on at the end of each epoch, where each batch is a tuple of (inputs,
targets)
:type validation_data: generator
:param n_validation_steps: number of batches to evaluate on from
`validation_data`
:raises RuntimeError: if only one of `validation_data` and
`n_validation_steps` are passed in
"""
default_callbacks = self._default_callbacks()
callbacks = CallbackList(default_callbacks)
self._assert_compiled()
invalid_inputs = (
(validation_data is not None and n_validation_steps is None)
or (n_validation_steps is not None and validation_data is None))
if invalid_inputs:
msg = ('`validation_data` and `n_validation_steps` must both be '
'passed, or neither.')
raise RuntimeError(msg)
if self.device:
self.network.to(self.device)
callbacks.set_params({
'epochs': n_epochs,
'metrics': ['loss', 'val_loss'],
'steps': n_steps_per_epoch,
'verbose': True
})
callbacks.set_model(self)
callbacks.on_train_begin()
for idx_epoch in range(n_epochs):
if self.stop_training:
break
epoch_logs = {}
callbacks.on_epoch_begin(idx_epoch)
for idx_batch in range(n_steps_per_epoch):
batch_logs = {'batch': idx_batch, 'size': 1}
callbacks.on_batch_begin(idx_batch, batch_logs)
inputs, targets = next(generator)
loss = self.train_on_batch(inputs, targets)
batch_logs['loss'] = loss
callbacks.on_batch_end(idx_batch, batch_logs)
if self.stop_training:
break
if validation_data:
val_loss = self.evaluate_generator(validation_data,
n_validation_steps)
epoch_logs['val_loss'] = val_loss
callbacks.on_epoch_end(idx_epoch, epoch_logs)
callbacks.on_train_end()