def split_data(training_data):
X_q, X_p, X_n, Y = training_data
X = [np.asarray(X_q), np.asarray(X_p), np.asarray(X_n)]
Y = np.array(Y)
split_at = int(len(Y) * (1.0 - 0.1))
train_X, val_X = training._slice_arrays(X, 0,
split_at), training._slice_arrays(
X, split_at)
train_Y = Y[0:split_at]
val_Y = Y[split_at:]
#train_Y, val_Y = training._slice_arrays(Y, 0, split_at), training._slice_arrays(Y, split_at)
return train_X, val_X, train_Y, val_Y
def split_train_and_valid(cls,
data,
mask,
validation_split,
shuffle=False):
print('Shuffle & split...')
if shuffle:
data, mask = cls.shuffle_train(data, mask)
split_at = int(len(data) * (1. - validation_split))
x_train, x_valid = (_slice_arrays(data, 0, split_at),
_slice_arrays(data, split_at))
y_train, y_valid = (_slice_arrays(mask, 0, split_at),
_slice_arrays(mask, split_at))
cls.save_valid_idx(range(len(data))[split_at:])
# print('x_train像素点之和：{}'.format(x_train.sum()))
# print('y_train_mask像素点之和：{}'.format(y_train.sum()))
return (x_train, y_train), (x_valid, y_valid)
def test_slice_arrays():
input_a = np.random.random((10, 3))
_slice_arrays(None)
_slice_arrays(input_a, 0)
_slice_arrays(input_a, 0, 1)
_slice_arrays(input_a, stop=2)
input_a = [None, [1, 1], None, [1, 1]]
_slice_arrays(input_a, 0)
_slice_arrays(input_a, 0, 1)
_slice_arrays(input_a, stop=2)
input_a = [None]
_slice_arrays(input_a, 0)
_slice_arrays(input_a, 0, 1)
_slice_arrays(input_a, stop=2)
input_a = None
_slice_arrays(input_a, 0)
_slice_arrays(input_a, 0, 1)
_slice_arrays(input_a, stop=2)
X = np.zeros((len(questions), MAXLEN, len(chars)), dtype=np.bool)
y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool)
for i, sentence in enumerate(questions):
X[i] = ctable.encode(sentence, maxlen=MAXLEN)
for i, sentence in enumerate(expected):
y[i] = ctable.encode(sentence, maxlen=DIGITS + 1)
# Shuffle (X, y) in unison as the later parts of X will almost all be larger digits
indices = np.arange(len(y))
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
# Explicitly set apart 20% for validation data that we never train over
split_at = int(len(X) - len(X) / 20)
(X_train, X_val) = (_slice_arrays(X, 0, split_at), _slice_arrays(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
print(X_train.shape)
print(y_train.shape)
print('Build model...')
# from https://gist.github.com/rouseguy/1122811f2375064d009dac797d59bae9
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
# note: in a situation where your input sequences have a variable length,
# use input_shape=(None, nb_feature).
model.add(RNN(HIDDEN_SIZE, return_sequences=True, input_shape=(MAXLEN, len(chars))))
# todo : for trying multi-layer
def _fit_loop(self,
f,
ins,
out_labels=None,
batch_size=32,
epochs=100,
verbose=1,
callbacks=None,
val_f=None,
val_ins=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None):
"""Abstract fit function for f(ins).
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: List of tensors to be fed to `f`
out_labels: List of strings, display names of
the outputs of `f`
batch_size: Integer batch size or None if unknown.
epochs: Number of times to iterate over the data
verbose: Verbosity mode, 0, 1 or 2
callbacks: List of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: List of tensors to be fed to `val_f`
shuffle: Whether to shuffle the data at the beginning of each epoch
callback_metrics: List of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: Epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. Ignored with the default value of `None`.
validation_steps: Number of steps to run validation for
(only if doing validation from data tensors).
Ignored with the default value of `None`.
# Returns
`History` object.
[A tweaked version.]
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose and ins and hasattr(ins[0], 'shape') and hasattr(
val_ins[0], 'shape'):
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if validation_steps:
do_validation = True
if steps_per_epoch is None:
raise ValueError('Can only use `validation_steps` '
'when doing step-wise '
'training, i.e. `steps_per_epoch` '
'must be set.')
num_train_samples = self._check_num_samples(ins, batch_size,
steps_per_epoch,
'steps_per_epoch')
if num_train_samples is not None:
index_array = np.arange(num_train_samples)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
if steps_per_epoch is not None:
count_mode = 'steps'
else:
count_mode = 'samples'
callbacks += [cbks.ProgbarLogger(count_mode)]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
# for cbk in callbacks:
# cbk.validation_data = val_ins
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
if steps_per_epoch is not None:
for step_index in range(steps_per_epoch):
batch_logs = {}
batch_logs['batch'] = step_index
batch_logs['size'] = 1
callbacks.on_batch_begin(step_index, batch_logs)
outs = f(ins)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(step_index, batch_logs)
if callback_model.stop_training:
break
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
steps=validation_steps,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
else:
if shuffle == 'batch':
index_array = _batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = _slice_arrays(ins[:-1],
batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['ids'] = batch_ids
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if batch_index == len(batches) - 1: # last batch.
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def fit(model,
x,
y,
batch_size=32,
nb_epoch=10,
verbose=1,
callbacks=[],
validation_split=0.,
validation_data=None,
shuffle=True,
class_weight=None,
sample_weight=None):
'''Trains the model for a fixed number of epochs (iterations on a dataset).
# Arguments
x: Numpy array of training data,
or list of Numpy arrays if the model has multiple inputs.
If all inputs in the model are named, you can also pass a dictionary
mapping input names to Numpy arrays.
y: Numpy array of target data,
or list of Numpy arrays if the model has multiple outputs.
If all outputs in the model are named, you can also pass a dictionary
mapping output names to Numpy arrays.
batch_size: integer. Number of samples per gradient update.
nb_epoch: integer, the number of times to iterate over the training data arrays.
verbose: 0, 1, or 2. Verbosity mode. 0 = silent, 1 = verbose, 2 = one log line per epoch.
callbacks: list of callbacks to be called during training.
See [callbacks](/callbacks).
validation_split: float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data: data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (x_val, y_val) or a tuple (val_x, val_y, val_sample_weights).
shuffle: boolean, whether to shuffle the training data before each epoch.
class_weight: optional dictionary mapping class indices (integers) to
a weight (float) to apply to the model's loss for the samples
from this class during training.
This can be useful to tell the model to "pay more attention" to
samples from an under-represented class.
sample_weight: optional array of the same length as x, containing
weights to apply to the model's loss for each sample.
In the case of temporal data, you can pass a 2D array
with shape (samples, sequence_length),
to apply a different weight to every timestep of every sample.
In this case you should make sure to specify sample_weight_mode="temporal" in compile().
# Returns
A `History` instance. Its `history` attribute contains
all information collected during training.
'''
# validate user data
'''
We need to use the custom standardize_user_data function defined above
to skip checking for the array_length. Everything else is the same as the original fit code
Note: We did not use model._standardize_user_data(...) but instead used the above
standardize_user_data(...) method to bypass the original standardize code in keras.
'''
x, y, sample_weights = _standardize_user_data(model,
x,
y,
sample_weight=sample_weight,
class_weight=class_weight,
check_batch_dim=False,
batch_size=batch_size)
# prepare validation data
if validation_data:
do_validation = True
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise
val_x, val_y, val_sample_weights = model._standardize_user_data(
val_x,
val_y,
sample_weight=val_sample_weight,
check_batch_dim=False,
batch_size=batch_size)
model._make_test_function()
val_f = model.test_function
if model.uses_learning_phase and type(K.learning_phase()) is not int:
val_ins = val_x + val_y + val_sample_weights + [0.]
else:
val_ins = val_x + val_y + val_sample_weights
elif validation_split and 0. < validation_split < 1.:
do_validation = True
split_at = int(len(x[0]) * (1. - validation_split))
x, val_x = (_slice_arrays(x, 0, split_at), _slice_arrays(x, split_at))
y, val_y = (_slice_arrays(y, 0, split_at), _slice_arrays(y, split_at))
sample_weights, val_sample_weights = (_slice_arrays(
sample_weights, 0,
split_at), _slice_arrays(sample_weights, split_at))
model._make_test_function()
val_f = model.test_function
if model.uses_learning_phase and type(K.learning_phase()) is not int:
val_ins = val_x + val_y + val_sample_weights + [0.]
else:
val_ins = val_x + val_y + val_sample_weights
else:
do_validation = False
val_f = None
val_ins = None
# prepare input arrays and training function
if model.uses_learning_phase and type(K.learning_phase()) is not int:
ins = x + y + sample_weights + [1.]
else:
ins = x + y + sample_weights
model._make_train_function()
f = model.train_function
# prepare display labels
out_labels = model.metrics_names
# rename duplicated metrics name
# (can happen with an output layer shared among multiple dataflows)
deduped_out_labels = []
for i, label in enumerate(out_labels):
new_label = label
if out_labels.count(label) > 1:
dup_idx = out_labels[:i].count(label)
new_label += '_' + str(dup_idx + 1)
deduped_out_labels.append(new_label)
out_labels = deduped_out_labels
if do_validation:
callback_metrics = copy.copy(out_labels) + [
'val_' + n for n in out_labels
]
else:
callback_metrics = copy.copy(out_labels)
# delegate logic to _fit_loop
return model._fit_loop(f,
ins,
out_labels=out_labels,
batch_size=batch_size,
nb_epoch=nb_epoch,
verbose=verbose,
callbacks=callbacks,
val_f=val_f,
val_ins=val_ins,
shuffle=shuffle,
callback_metrics=callback_metrics)
def test_slice_arrays():
input_a = np.random.random((10, 3))
_slice_arrays(None)
_slice_arrays(input_a, 0)
_slice_arrays(input_a, 0, 1)
_slice_arrays(input_a, stop=2)
input_a = [None, [1, 1], None, [1, 1]]
_slice_arrays(input_a, 0)
_slice_arrays(input_a, 0, 1)
_slice_arrays(input_a, stop=2)
input_a = [None]
_slice_arrays(input_a, 0)
_slice_arrays(input_a, 0, 1)
_slice_arrays(input_a, stop=2)
input_a = None
_slice_arrays(input_a, 0)
_slice_arrays(input_a, 0, 1)
_slice_arrays(input_a, stop=2)
def fit_tfrecord(self,
x=None,
y=None,
batch_size=32,
epochs=1,
verbose=1,
callbacks=None,
validation_split=0.,
validation_data=None,
shuffle=True,
class_weight=None,
sample_weight=None,
initial_epoch=0,
steps_per_epoch=None,
**kwargs):
"""Trains the model for a fixed number of epochs (iterations on a dataset).
# Arguments
x: Numpy array of training data,
or list of Numpy arrays if the model has multiple inputs.
If all inputs in the model are named,
you can also pass a dictionary
mapping input names to Numpy arrays.
y: Numpy array of target data,
or list of Numpy arrays if the model has multiple outputs.
If all outputs in the model are named,
you can also pass a dictionary
mapping output names to Numpy arrays.
batch_size: integer. Number of samples per gradient update.
epochs: integer, the number of times to iterate
over the training data arrays.
verbose: 0, 1, or 2. Verbosity mode.
0 = silent, 1 = verbose, 2 = one log line per epoch.
callbacks: list of callbacks to be called during training.
See [callbacks](/callbacks).
validation_split: float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate
the loss and any model metrics
on this data at the end of each epoch.
validation_data: data on which to evaluate
the loss and any model metrics
at the end of each epoch. The model will not
be trained on this data.
This could be a tuple (x_val, y_val)
or a tuple (x_val, y_val, val_sample_weights).
shuffle: boolean, whether to shuffle the training data
before each epoch.
class_weight: optional dictionary mapping
class indices (integers) to
a weight (float) to apply to the model's loss for the samples
from this class during training.
This can be useful to tell the model to "pay more attention" to
samples from an under-represented class.
sample_weight: optional array of the same length as x, containing
weights to apply to the model's loss for each sample.
In the case of temporal data, you can pass a 2D array
with shape (samples, sequence_length),
to apply a different weight to every timestep of every sample.
In this case you should make sure to specify
sample_weight_mode="temporal" in compile().
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. The default `None` is equal to the number
of unique samples in your dataset divided by the batch
size, or 1 if that cannot be determined.
# Returns
A `History` instance. Its `history` attribute contains
all information collected during training.
# Raises
ValueError: In case of mismatch between the provided input data
and what the model expects.
"""
# Legacy support
if 'nb_epoch' in kwargs:
warnings.warn(
'The `nb_epoch` argument in `fit` '
'has been renamed `epochs`.',
stacklevel=2)
epochs = kwargs.pop('nb_epoch')
if kwargs:
raise TypeError('Unrecognized keyword arguments: ' + str(kwargs))
# Validate user data.
x, y, sample_weights = self._standardize_user_data(
x,
y,
sample_weight=sample_weight,
class_weight=class_weight,
check_batch_axis=False,
batch_size=batch_size)
# Prepare validation data.
if validation_data:
do_validation = True
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise ValueError('When passing validation_data, '
'it must contain 2 (x_val, y_val) '
'or 3 (x_val, y_val, val_sample_weights) '
'items, however it contains %d items' %
len(validation_data))
val_x, val_y, val_sample_weights = self._standardize_user_data(
val_x,
val_y,
sample_weight=val_sample_weight,
check_batch_axis=False,
batch_size=batch_size)
self._make_test_function()
val_f = self.test_function
if self.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
val_ins = val_x + val_y + val_sample_weights + [0.]
else:
val_ins = val_x + val_y + val_sample_weights
elif validation_split and 0. < validation_split < 1.:
do_validation = True
if hasattr(x[0], 'shape'):
split_at = int(x[0].shape[0] * (1. - validation_split))
else:
split_at = int(len(x[0]) * (1. - validation_split))
x, val_x = (_slice_arrays(x, 0,
split_at), _slice_arrays(x, split_at))
y, val_y = (_slice_arrays(y, 0,
split_at), _slice_arrays(y, split_at))
sample_weights, val_sample_weights = (_slice_arrays(
sample_weights, 0,
split_at), _slice_arrays(sample_weights, split_at))
self._make_test_function()
val_f = self.test_function
if self.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
val_ins = val_x + val_y + val_sample_weights + [0.]
else:
val_ins = val_x + val_y + val_sample_weights
else:
do_validation = False
val_f = None
val_ins = None
# Prepare input arrays and training function.
if self.uses_learning_phase and \
not isinstance(K.learning_phase(), int):
ins = x + y + sample_weights + [1.]
else:
ins = x + y + sample_weights
self._make_train_function()
f = self.train_function
# Prepare display labels.
out_labels = self._get_deduped_metrics_names()
if do_validation:
callback_metrics = \
copy.copy(out_labels) + ['val_' + n for n in out_labels]
else:
callback_metrics = copy.copy(out_labels)
# Delegate logic to `_fit_loop`.
return self._fit_loop(f,
ins,
out_labels=out_labels,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
callbacks=callbacks,
val_f=val_f,
val_ins=val_ins,
shuffle=shuffle,
callback_metrics=callback_metrics,
initial_epoch=initial_epoch,
steps_per_epoch=steps_per_epoch)
def _fit_loop(self,
f,
ins,
out_labels=None,
batch_size=32,
epochs=100,
verbose=1,
callbacks=None,
val_f=None,
val_ins=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0,
steps_per_epoch=None):
"""Abstract fit function for `f(ins)`.
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: list of tensors to be fed to `f`
out_labels: list of strings, display names of
the outputs of `f`
batch_size: integer batch size
epochs: number of times to iterate over the data
verbose: verbosity mode, 0, 1 or 2
callbacks: list of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: list of tensors to be fed to `val_f`
shuffle: whether to shuffle the data at the beginning of each epoch
callback_metrics: list of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. The default `None` is equal to the number
of unique samples in your dataset divided by the batch
size, or 1 if that cannot be determined.
# Returns
`History` object.
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose and ins and hasattr(ins[0], 'shape'):
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if steps_per_epoch is not None:
num_train_samples = steps_per_epoch
else:
if ins and hasattr(ins[0], 'shape'):
num_train_samples = ins[0].shape[0]
else:
# May happen if we are running `fit` without Numpy input data,
# i.e. if all inputs to the models are data tensors
# instead of placeholders.
# In that case we will run `fit` over a single batch.
num_train_samples = batch_size
verbose = 2
index_array = np.arange(num_train_samples)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
# callbacks += [cbks.ProgbarLogger()]
callbacks += [ProgbarLogger_TFRecord()]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
for cbk in callbacks:
cbk.validation_data = val_ins
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = _batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = \
_slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if batch_index == len(batches) - 1: # Last batch.
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def batch_data(self, model, batch_size):
if self.batched == False:
x, y, sample_weights = model._standardize_user_data(
self.X_train,
self.Y_train,
sample_weight=None,
class_weight=None,
check_batch_axis=False,
batch_size=batch_size)
val_x, val_y, val_sample_weights = model._standardize_user_data(
self.X_test,
self.Y_test,
sample_weight=None,
check_batch_axis=False,
batch_size=batch_size)
ins = x + y + sample_weights
val_ins = val_x + val_y + val_sample_weights
if model.uses_learning_phase and not isinstance(
K.learning_phase(), int):
ins += [1.]
val_ins += [0.]
from keras.engine.training import _slice_arrays
self.n_batch_train = ins[0].shape[0] // batch_size
self.train_batches = []
index_arr = range(ins[0].shape[0])
for batch_index in range(self.n_batch_train):
batch_ids = index_arr[batch_index *
batch_size:(batch_index + 1) *
batch_size]
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
self.train_batches.append(ins_batch)
self.n_batch_val = val_ins[0].shape[0] // batch_size
self.val_batches = []
index_arr = range(val_ins[0].shape[0])
for batch_index in range(self.n_batch_val):
batch_ids = index_arr[batch_index *
batch_size:(batch_index + 1) *
batch_size]
if isinstance(val_ins[-1], float):
# do not slice the training phase flag
ins_batch = _slice_arrays(val_ins[:-1],
batch_ids) + [val_ins[-1]]
else:
ins_batch = _slice_arrays(val_ins, batch_ids)
self.val_batches.append(ins_batch)
self.batched = True
def batch_data(self, model, batch_size):
if self.batched==False:
x, y, sample_weights = model._standardize_user_data(
self.X_train, self.Y_train,
sample_weight=None,
class_weight=None,
check_batch_axis=False,
batch_size=batch_size)
val_x, val_y, val_sample_weights = model._standardize_user_data(
self.X_test, self.Y_test,
sample_weight=None,
check_batch_axis=False,
batch_size=batch_size)
ins = x + y + sample_weights
val_ins = val_x + val_y + val_sample_weights
if model.uses_learning_phase and not isinstance(K.learning_phase(), int):
ins+=[1.]
val_ins+=[0.]
from keras.engine.training import _slice_arrays
self.n_batch_train = ins[0].shape[0]//batch_size
self.train_batches = []
index_arr = range(ins[0].shape[0])
for batch_index in range(self.n_batch_train):
batch_ids = index_arr[batch_index * batch_size:
(batch_index+1)*batch_size]
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
self.train_batches.append(ins_batch)
self.n_batch_val = val_ins[0].shape[0]//batch_size
self.val_batches = []
index_arr = range(val_ins[0].shape[0])
for batch_index in range(self.n_batch_val):
batch_ids = index_arr[batch_index * batch_size:
(batch_index+1)*batch_size]
if isinstance(val_ins[-1], float):
# do not slice the training phase flag
ins_batch = _slice_arrays(val_ins[:-1], batch_ids) + [val_ins[-1]]
else:
ins_batch = _slice_arrays(val_ins, batch_ids)
self.val_batches.append(ins_batch)
self.batched=True
X_new, y_new = np.array(X_new), np.array(y_new)
# Shuffle (X_new, y_new) in unison as the later parts of X_new
# will almost all be larger digits
indices = np.arange(len(y_new))
np.random.shuffle(indices)
X_new = X_new[indices]
y_new = y_new[indices]
# Save memory!
del indices
# Explicitly set apart 10% for test data that we never train over
split_at = int(len(X_new) - len(X_new) / 10)
(X_train_new, X_test_new) = (_slice_arrays(X_new, 0, split_at),
_slice_arrays(X_new, split_at))
(y_train_new, y_test_new) = (y_new[:split_at], y_new[split_at:])
# Save memory!!
del X_new, y_new
print(X_train_new.shape, y_train_new.shape)
# (You don't need to change this part of the code)
from __future__ import print_function
import numpy as np
np.random.seed(1234)
import keras
def _fit_loop(self,
f: callable,
ins: List[numpy.array],
out_labels: List[str] = None,
batch_size: int = 32,
epochs: int = 100,
verbose: int = 1,
callbacks: List[Callback] = None,
val_f: callable = None,
val_ins: List[numpy.array] = None,
shuffle: bool = True,
callback_metrics: List[str] = None,
initial_epoch: int = 0):
"""
Abstract fit function which preprocesses and batches
data before training a model. We override this keras backend
function to support multi-gpu training via splitting a large
batch size across multiple gpus. This function is broadly the
same as the Keras backend version aside from this - changed elements
have corresponding comments attached.
Note that this should not be called directly - it is used by calling
model.fit().
Assume that step_function returns a list, labeled by out_labels.
Parameters
----------
f: A callable ``Step`` or a Keras ``Function``, required.
A DeepQA Step or Keras Function returning a list of tensors.
ins: List[numpy.array], required.
The list of tensors to be fed to ``step_function``.
out_labels: List[str], optional (default = None).
The display names of the outputs of ``step_function``.
batch_size: int, optional (default = 32).
The integer batch size.
epochs: int, optional (default = 100).
Number of times to iterate over the data.
verbose: int, optional, (default = 1)
Verbosity mode, 0, 1 or 2.
callbacks: List[Callback], optional (default = None).
A list of Keras callbacks to be called during training.
val_f: A callable ``Step`` or a Keras ``Function``, optional (default = None).
The Keras function to call for validation.
val_ins: List[numpy.array], optional (default)
A list of tensors to be fed to ``val_f``.
shuffle: bool, optional (default = True).
whether to shuffle the data at the beginning of each epoch
callback_metrics: List[str], optional, (default = None).
A list of strings, the display names of the validation metrics.
passed to the callbacks. They should be the concatenation of list the display
names of the outputs of ``f`` and the list of display names of the outputs of ``f_val``.
initial_epoch: int, optional (default = 0).
The epoch at which to start training (useful for resuming a previous training run).
Returns
-------
A Keras ``History`` object.
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if ins and hasattr(ins[0], 'shape'):
num_train_samples = ins[0].shape[0]
else:
# May happen if we are running `fit` without Numpy input data,
# i.e. if all inputs to the models are data tensors
# instead of placeholders.
# In that case we will run `fit` over a single batch.
num_train_samples = batch_size
verbose = 2
index_array = numpy.arange(num_train_samples)
out_labels = out_labels or []
callbacks, callback_model = self._prepare_callbacks(
callbacks, val_ins, epochs, batch_size, num_train_samples,
callback_metrics, do_validation, verbose)
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = _batch_shuffle(index_array, batch_size)
elif shuffle:
numpy.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = _slice_arrays(ins[:-1],
batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
# Here is the main difference between a single gpu model and one split
# across multiple gpus. In our multiple gpu model, all of the inputs
# are replicated num_gpus times, so we need to split our large batch
# into the corresponding sets of smaller batches for each model.
if self.num_gpus > 1:
# The Keras learning phase is a global variable used across model towers.
# If it is present, we remove it before splitting up the inputs
# and add it back on afterwards.
if isinstance(ins_batch[-1], float):
model_inputs = self._multi_gpu_batch(ins_batch[:-1])
model_inputs.append(ins_batch[-1])
else:
model_inputs = self._multi_gpu_batch(ins_batch)
ins_batch = model_inputs
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for label, output in zip(out_labels, outs):
batch_logs[label] = output
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index == len(batches) - 1: # Last batch.
if do_validation:
# If we are using multiple gpus, our batch size will be
# scaled up accordingly. However, validation will run
# on a single gpu, so we divide by the number of gpus
# to avoid OOM errors.
if self.num_gpus > 1:
val_batch_size = int(batch_size / self.num_gpus) # pylint: disable=no-member
else:
val_batch_size = batch_size
val_outs = self._test_loop(val_f,
val_ins,
batch_size=val_batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for label, output in zip(out_labels, val_outs):
epoch_logs['val_' + label] = output
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training: # pylint: disable=no-member
break
callbacks.on_train_end()
return self.history
