def _clone_and_build_model(mode,
keras_model,
custom_objects,
features=None,
labels=None):
"""Clone and build the given keras_model.
Args:
mode: training mode.
keras_model: an instance of compiled keras model.
custom_objects: Dictionary for custom objects.
features:
labels:
Returns:
The newly built model.
"""
# Set to True during training, False for inference.
K.set_learning_phase(mode == model_fn_lib.ModeKeys.TRAIN)
# Clone keras model.
input_tensors = None if features is None else _create_ordered_io(
keras_model, features)
if custom_objects:
with CustomObjectScope(custom_objects):
model = models.clone_model(keras_model, input_tensors=input_tensors)
else:
model = models.clone_model(keras_model, input_tensors=input_tensors)
# Compile/Build model
if mode is model_fn_lib.ModeKeys.PREDICT and not model.built:
model.build()
else:
optimizer_config = keras_model.optimizer.get_config()
optimizer = keras_model.optimizer.__class__.from_config(optimizer_config)
optimizer.iterations = training_util.get_or_create_global_step()
# Get list of outputs.
if labels is None:
target_tensors = None
elif isinstance(labels, dict):
target_tensors = _create_ordered_io(keras_model, labels, is_input=False)
else:
target_tensors = [
_cast_tensor_to_floatx(
sparse_tensor_lib.convert_to_tensor_or_sparse_tensor(labels))
]
model.compile(
optimizer,
keras_model.loss,
metrics=keras_model.metrics,
loss_weights=keras_model.loss_weights,
sample_weight_mode=keras_model.sample_weight_mode,
weighted_metrics=keras_model.weighted_metrics,
target_tensors=target_tensors)
if isinstance(model, models.Sequential):
model = model.model
return model
def _call_graph_fn(self, features, labels=None):
"""Calls graph function.
Args:
features: `Tensor` or `dict` of tensors
labels: `Tensor` or `dict` of tensors
"""
set_learning_phase(Modes.is_train(self.mode))
kwargs = {}
if 'labels' in get_arguments(self._graph_fn):
kwargs['labels'] = labels
return self._graph_fn(mode=self.mode, features=features, **kwargs)
def _process_single_batch(model,
inputs,
targets,
sample_weights=None,
training=False):
"""Calculate the loss and gradient for one input batch.
The model weights are updated if training is set to True.
Arguments:
model: Model whose loss has to be calculated.
inputs: List of input arrays.
targets: List of target arrays.
sample_weights: Optional list of sample weight arrays.
training: The boolean represents if the weights of the model are updated.
'fit' methods will set this to True while 'evaluate' methods will
set this to False.
Returns:
output of the model, total loss and the loss associated with each output.
Raises:
ValueError: If the model has no loss to optimize.
"""
K.set_learning_phase(training)
with GradientTape() as tape:
outs, loss, loss_metrics = _model_loss(model,
inputs,
targets,
sample_weights=sample_weights,
training=training)
if loss is None:
raise ValueError('The model cannot be run '
'because it has no loss to optimize.')
if training:
if not model._collected_trainable_weights:
logging.warning(
'The list of trainable weights is empty. Make sure that '
'you are not setting model.trainable to False before '
'compiling the model.')
else:
grads = tape.gradient(loss, model._collected_trainable_weights)
model.optimizer.apply_gradients(
zip(grads, model._collected_trainable_weights))
return outs, loss, loss_metrics
def _call_graph_fn(self, features, labels=None):
"""Calls graph function.
Creates first one or two graph, i.e. train and target graphs.
Return the optimal action given an exploration policy.
If `is_dueling` is set to `True`,
then another layer is added that represents the state value.
Args:
inputs: `Tensor` or `dict` of tensors
"""
set_learning_phase(Modes.is_train(self.mode))
graph_fn = self._build_graph_fn()
self._graph_results = graph_fn(mode=self.mode,
features=features,
labels=labels)
return self._build_actions()
def _process_single_batch(eager_model_inputs,
eager_model_outputs,
model,
training=False):
"""Calculate the loss and gradient for one input batch.
The model weights are updated if training is set to True.
Arguments:
eager_model_inputs: Input batch data.
eager_model_outputs: Output batch data.
model: Model whose loss has to be calculated.
training: The boolean represents if the weights of the model are updated.
'fit' methods will set this to True while 'evaluate' methods will
set this to False.
Returns:
output of the model, total loss and the loss associated with each output.
Raises:
ValueError: If the model loss is 0 or if the trainable weights list is
empty when the trainable parameter is set to True.
"""
K.set_learning_phase(training)
with GradientTape() as tape:
outs, loss, loss_metrics = _model_loss(model,
eager_model_inputs,
eager_model_outputs,
training=training)
if loss is None:
raise ValueError('The model cannot be run '
'because it has no loss to optimize.')
if training:
if not model._collected_trainable_weights:
raise ValueError(
'The list of trainable weights is empty. Make sure that '
'you are not setting model.trainable to False before '
'compiling the model.')
grads = tape.gradient(loss, model._collected_trainable_weights)
model.optimizer.apply_gradients(
zip(grads, model._collected_trainable_weights))
return outs, loss, loss_metrics
def _process_single_batch(eager_model_inputs, eager_model_outputs, model,
training=False):
"""Calculate the loss and gradient for one input batch.
The model weights are updated if training is set to True.
Arguments:
eager_model_inputs: Input batch data.
eager_model_outputs: Output batch data.
model: Model whose loss has to be calculated.
training: The boolean represents if the weights of the model are updated.
'fit' methods will set this to True while 'evaluate' methods will
set this to False.
Returns:
output of the model, total loss and the loss associated with each output.
Raises:
ValueError: If the model has no loss to optimize.
"""
K.set_learning_phase(training)
with GradientTape() as tape:
outs, loss, loss_metrics = _model_loss(model, eager_model_inputs,
eager_model_outputs,
training=training)
if loss is None:
raise ValueError('The model cannot be run '
'because it has no loss to optimize.')
if training:
if not model._collected_trainable_weights:
logging.warning('The list of trainable weights is empty. Make sure that '
'you are not setting model.trainable to False before '
'compiling the model.')
else:
grads = tape.gradient(loss, model._collected_trainable_weights)
model.optimizer.apply_gradients(zip(grads,
model._collected_trainable_weights))
return outs, loss, loss_metrics
def _call_graph_fn(self, features, labels=None):
"""Calls graph function.
Creates first one or two graph, i.e. train and target graphs.
Return the optimal action given an exploration policy.
If `is_dueling` is set to `True`,
then another layer is added that represents the state value.
Args:
features: `Tensor` or `dict` of tensors
labels: `Tensor` or `dict` of tensors
"""
set_learning_phase(Modes.is_train(self.mode))
graph_fn = self._build_graph_fn()
if self.use_target_graph:
# We create 2 graphs: a training graph and a target graph,
# so that we can copy one graph to another given a frequency.
self._train_graph = FunctionModule(mode=self.mode,
build_fn=graph_fn,
name='train')
self._train_results = self._train_graph(features=features,
labels=labels)
self._target_graph = FunctionModule(mode=self.mode,
build_fn=graph_fn,
name='target')
self._target_results = self._target_graph(features=features,
labels=labels)
return self._build_actions()
else:
self._train_results = graph_fn(mode=self.mode,
features=features,
labels=labels)
self._target_results = self._train_results
return self._build_actions()
def _clone_and_build_model(mode,
keras_model,
custom_objects,
features=None,
labels=None):
"""Clone and build the given keras_model.
Args:
mode: training mode.
keras_model: an instance of compiled keras model.
custom_objects: Dictionary for custom objects.
features: Dict of tensors.
labels: Dict of tensors, or single tensor instance.
Returns:
The newly built model.
"""
# Set to True during training, False for inference.
K.set_learning_phase(mode == model_fn_lib.ModeKeys.TRAIN)
# Get list of inputs.
if features is None:
input_tensors = None
else:
input_tensors = _create_ordered_io(keras_model,
estimator_io=features,
is_input=True)
# Get list of outputs.
if labels is None:
target_tensors = None
elif isinstance(labels, dict):
target_tensors = _create_ordered_io(keras_model,
estimator_io=labels,
is_input=False)
else:
target_tensors = [_convert_tensor(labels)]
if keras_model._is_graph_network:
if custom_objects:
with CustomObjectScope(custom_objects):
model = models.clone_model(keras_model,
input_tensors=input_tensors)
else:
model = models.clone_model(keras_model,
input_tensors=input_tensors)
else:
model = keras_model
_in_place_subclassed_model_reset(model)
if input_tensors is not None:
model._set_inputs(input_tensors)
# Compile/Build model
if mode is model_fn_lib.ModeKeys.PREDICT:
if isinstance(model, models.Sequential):
model.build()
else:
if isinstance(keras_model.optimizer, optimizers.TFOptimizer):
optimizer = keras_model.optimizer
else:
optimizer_config = keras_model.optimizer.get_config()
optimizer = keras_model.optimizer.__class__.from_config(
optimizer_config)
optimizer.iterations = training_util.get_or_create_global_step()
model.compile(optimizer,
keras_model.loss,
metrics=keras_model.metrics,
loss_weights=keras_model.loss_weights,
sample_weight_mode=keras_model.sample_weight_mode,
weighted_metrics=keras_model.weighted_metrics,
target_tensors=target_tensors)
return model
def predict_loop(model, ins, batch_size=32, verbose=0, steps=None):
"""Abstract method to loop over some data in batches.
Arguments:
model:
ins: list of tensors to be fed to `f`.
batch_size: integer batch size.
verbose: verbosity mode.
steps: Total number of steps (batches of samples)
before declaring `_predict_loop` finished.
Ignored with the default value of `None`.
Returns:
Array of predictions (if the model has a single output)
or list of arrays of predictions
(if the model has multiple outputs).
"""
K.set_learning_phase(False)
num_samples = model._check_num_samples(ins, batch_size, steps, 'steps')
if verbose == 1:
if steps is not None:
progbar = Progbar(target=steps)
else:
progbar = Progbar(target=num_samples)
outs = []
batches = _make_batches(num_samples, batch_size)
index_array = np.arange(num_samples)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if ins and isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
ins_batch_converted = []
for ib in ins_batch:
ins_batch_converted.append(
ops.convert_to_tensor(ib, dtype=K.floatx()))
eager_model_inputs = []
for i in range(len(model.inputs)):
eager_model_inputs.append(ins_batch_converted[i])
batch_outs = model(eager_model_inputs)
if not isinstance(batch_outs, list):
batch_outs = [batch_outs]
if batch_index == 0:
# Pre-allocate the results arrays.
for batch_out in batch_outs:
dims = batch_out.shape[1:].dims
dims_list = [d.value for d in dims]
shape = (num_samples, ) + tuple(dims_list)
outs.append(
np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype))
for i, batch_out in enumerate(batch_outs):
outs[i][batch_start:batch_end] = batch_out
if verbose == 1:
progbar.update(batch_end)
if len(outs) == 1:
return outs[0]
return outs
def test_loop(model, ins, batch_size=None, verbose=0, steps=None):
"""Abstract method to loop over some data in batches.
Arguments:
model: Model instance that is being evaluated in Eager mode.
ins: list of tensors to be fed to `f`.
batch_size: integer batch size or `None`.
verbose: verbosity mode.
steps: Total number of steps (batches of samples)
before declaring predictions finished.
Ignored with the default value of `None`.
Returns:
Scalar loss (if the model has a single output and no metrics)
or list of scalars (if the model has multiple outputs
and/or metrics). The attribute `model.metrics_names` will give you
the display labels for the scalar outputs.
"""
K.set_learning_phase(False)
num_samples = model._check_num_samples(ins, batch_size, steps, 'steps')
outs = []
if verbose == 1:
progbar = Progbar(target=num_samples)
batches = _make_batches(num_samples, batch_size)
index_array = np.arange(num_samples)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
ins_batch_converted = []
for ib in ins_batch:
ins_batch_converted.append(
ops.convert_to_tensor(ib, dtype=K.floatx()))
eager_model_inputs = []
eager_model_outputs = []
for i in range(len(model.inputs)):
eager_model_inputs.append(ins_batch_converted[i])
for i in range(len(model.inputs), len(ins_batch_converted)):
eager_model_outputs.append(ins_batch_converted[i])
loss_outs, loss, loss_metrics = _model_loss(model, eager_model_inputs,
eager_model_outputs)
_, metrics_results = _eager_metrics_fn(model, loss_outs,
eager_model_outputs)
batch_outs = []
for _, v in zip(model.metrics_names,
[K.mean(loss)] + loss_metrics + metrics_results):
batch_outs.append(tensor_util.constant_value(v))
if isinstance(batch_outs, list):
if batch_index == 0:
for batch_out in enumerate(batch_outs):
outs.append(0.)
for i, batch_out in enumerate(batch_outs):
outs[i] += batch_out * len(batch_ids)
else:
if batch_index == 0:
outs.append(0.)
outs[0] += batch_outs * len(batch_ids)
if verbose == 1:
progbar.update(batch_end)
for i in range(len(outs)):
outs[i] /= num_samples
if len(outs) == 1:
return outs[0]
return outs
def fit_loop(model,
ins,
out_labels=None,
batch_size=None,
epochs=100,
verbose=1,
callbacks=None,
val_ins=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None):
"""Abstract fit function for `f(ins)`.
Assume that f returns a list, labeled by out_labels.
Arguments:
model: Instance of the model that is being executed in Eager mode.
ins: List of tensors to be fed to `f`
out_labels: List of strings, display names of
the outputs of `f`
batch_size: Integer batch size or None if unknown.
epochs: Number of times to iterate over the data
verbose: Verbosity mode, 0, 1 or 2
callbacks: List of callbacks to be called during training
val_ins: List of tensors to be fed to `val_f`
shuffle: Whether to shuffle the data at the beginning of each epoch
callback_metrics: List of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: Epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. Ignored with the default value of `None`.
validation_steps: Number of steps to run validation for (only if doing
validation from data tensors). Ignored with default value of `None`.
Returns:
`History` object.
Raises:
ValueError: In case of invalid argument values.
"""
# Required for Eager mode
K.set_learning_phase(True)
do_validation = False
if val_ins:
do_validation = True
if (verbose and ins and hasattr(ins[0], 'shape')
and hasattr(val_ins[0], 'shape')):
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if validation_steps:
if steps_per_epoch is None:
raise ValueError(
'Can only use `validation_steps` when doing step-wise '
'training, i.e. `steps_per_epoch` must be set.')
do_validation = True
num_train_samples = model._check_num_samples(ins, batch_size,
steps_per_epoch,
'steps_per_epoch')
if num_train_samples is not None:
index_array = np.arange(num_train_samples)
model.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [model.history]
if verbose:
if steps_per_epoch is not None:
count_mode = 'steps'
else:
count_mode = 'samples'
callbacks += [cbks.ProgbarLogger(count_mode)]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
for cbk in callbacks:
cbk.validation_data = val_ins
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
if shuffle == 'batch':
index_array = model._batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
ins_batch_converted = []
for ib in ins_batch:
ins_batch_converted.append(
ops.convert_to_tensor(ib, dtype=K.floatx()))
eager_model_inputs = []
eager_model_outputs = []
for i in range(len(model.inputs)):
eager_model_inputs.append(ins_batch_converted[i])
for i in range(len(model.inputs), len(ins_batch_converted)):
eager_model_outputs.append(ins_batch_converted[i])
outs, loss, loss_metrics = _process_single_batch(
eager_model_inputs, eager_model_outputs, model)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
# Required for Eager mode
metrics_names, metrics_results = _eager_metrics_fn(
model, outs, eager_model_outputs)
batch_logs['loss'] = tensor_util.constant_value(K.mean(loss))
# TODO(anjalisridhar): Move this to compile to avoid duplicate code.
# In graph mode we set the metric names in compile. However in
# Eager mode we calculate the metrics for each batch in fit_loop.
# We could calculate the metric names and functions in compile.
# This would avoid setting the callback parameters separately.
# We need to do this for the first iteration alone
for m in metrics_names:
if m not in callback_metrics:
callback_metrics.append(m)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
for k, v in zip(model.metrics_names,
[K.mean(loss)] + loss_metrics + metrics_results):
batch_logs[k] = tensor_util.constant_value(v)
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if batch_index == len(batches) - 1: # Last batch.
if do_validation:
val_outs = test_loop(model,
val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return model.history
def _clone_and_build_model(mode,
keras_model,
custom_objects,
features=None,
labels=None):
"""Clone and build the given keras_model.
Args:
mode: training mode.
keras_model: an instance of compiled keras model.
custom_objects: Dictionary for custom objects.
features: Dict of tensors.
labels: Dict of tensors, or single tensor instance.
Returns:
The newly built model.
"""
# Set to True during training, False for inference.
K.set_learning_phase(mode == model_fn_lib.ModeKeys.TRAIN)
# Get list of inputs.
if features is None:
input_tensors = None
else:
input_tensors = _create_ordered_io(keras_model,
estimator_io=features,
is_input=True)
# Get list of outputs.
if labels is None:
target_tensors = None
elif isinstance(labels, dict):
target_tensors = _create_ordered_io(keras_model,
estimator_io=labels,
is_input=False)
else:
target_tensors = [
_convert_tensor(labels)
]
if keras_model._is_graph_network:
if custom_objects:
with CustomObjectScope(custom_objects):
model = models.clone_model(keras_model, input_tensors=input_tensors)
else:
model = models.clone_model(keras_model, input_tensors=input_tensors)
else:
model = keras_model
_in_place_subclassed_model_reset(model)
if input_tensors is not None:
model._set_inputs(input_tensors)
# Compile/Build model
if mode is model_fn_lib.ModeKeys.PREDICT:
if isinstance(model, models.Sequential):
model.build()
else:
if isinstance(keras_model.optimizer, optimizers.TFOptimizer):
optimizer = keras_model.optimizer
else:
optimizer_config = keras_model.optimizer.get_config()
optimizer = keras_model.optimizer.__class__.from_config(optimizer_config)
optimizer.iterations = training_util.get_or_create_global_step()
model.compile(
optimizer,
keras_model.loss,
metrics=keras_model.metrics,
loss_weights=keras_model.loss_weights,
sample_weight_mode=keras_model.sample_weight_mode,
weighted_metrics=keras_model.weighted_metrics,
target_tensors=target_tensors)
return model
def _specialize_model(self, input_specs):
"""Specialize `self.model` (a Keras model) for the given input shapes."""
# Re-create our input and output layers inside our subgraph. They will be
# attached to the true computation when we clone our model in `tpu_fn`.
K.set_learning_phase(
self.execution_mode == model_fn_lib.ModeKeys.TRAIN
)
# functools.partial and callable objects are not supported by tpu.rewrite
def _model_fn():
"""Compute fit/eval/predict for the TPU."""
is_training = self.execution_mode == model_fn_lib.ModeKeys.TRAIN
is_test = self.execution_mode == model_fn_lib.ModeKeys.EVAL
is_predict = self.execution_mode == model_fn_lib.ModeKeys.PREDICT
# During train/eval, we infeed our features as well as labels.
if is_training or is_test:
infeed_layers = self.model._input_layers + self.model._output_layers
else:
infeed_layers = self.model._input_layers
# Generate our infeed operation to read features & labels.
infeed_tensors = tpu_ops.infeed_dequeue_tuple(
dtypes=[spec.dtype for spec in input_specs],
shapes=[spec.shape for spec in input_specs],
name='infeed-%s' % self.execution_mode)
assert len(infeed_tensors) == len(infeed_layers), (
'Infeed inputs did not match model: %s vs %s', (infeed_layers,
infeed_tensors))
tpu_targets = []
tpu_inputs = []
# Sort infeed outputs into inputs and labels for calling our Keras model.
for tensor, layer in zip(infeed_tensors, infeed_layers):
if layer in self.model._input_layers:
tpu_inputs.append(layers.Input(name=layer.name, tensor=tensor))
if layer in self.model._output_layers:
tpu_targets.append(tensor)
# Call our model with our infeed inputs (re-using the weights).
model_outputs = self.model(tpu_inputs)
child_model = models.Model(inputs=tpu_inputs, outputs=model_outputs)
if is_training or is_test:
child_model.compile(
optimizer=self.model.optimizer,
loss=self.model.loss,
loss_weights=self.model.loss_weights,
metrics=self.model.metrics,
weighted_metrics=self.model.weighted_metrics,
target_tensors=tpu_targets,
)
# Compute our outfeed depending on the execution mode
if is_training:
child_model._make_train_function()
self._outfeed_spec = [
tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name)
for tensor in child_model.train_function.outputs
]
return [
child_model.train_function.updates_op,
tpu_ops.outfeed_enqueue_tuple(
child_model.train_function.outputs, name='oufeed-enqueue-train')
]
elif is_test:
child_model._make_test_function()
self._outfeed_spec = [
tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name)
for tensor in child_model.test_function.outputs
]
return [
tpu_ops.outfeed_enqueue_tuple(
child_model.test_function.outputs, name='outfeed-enqueue-test')
]
elif is_predict:
child_model._make_predict_function()
self._outfeed_spec = [
tensor_spec.TensorSpec(tensor.shape, tensor.dtype, tensor.name)
for tensor in child_model.predict_function.outputs
]
return [
tpu_ops.outfeed_enqueue_tuple(
child_model.predict_function.outputs,
name='outfeed-enqueue-predict',
)
]
else:
assert False, 'Unexpected execution mode: %s' % self.execution_mode
# Capture outfeed metadata computed during the rewrite.
self._outfeed_spec = None
tpu_execute_op = tpu.rewrite(_model_fn)
# Generate CPU side operations to enqueue features/labels and dequeue
# outputs from the model call.
with ops.device('/device:TPU:0'):
infeed_tensors = []
for spec in input_specs:
infeed_tensors.append(
array_ops.placeholder(
dtype=spec.dtype,
shape=spec.shape,
name='infeed-enqueue-%s' % spec.name))
infeed_op = tpu_ops.infeed_enqueue_tuple(
infeed_tensors, [spec.shape for spec in input_specs],
name='infeed-enqueue-%s' % self.execution_mode)
outfeed_op = tpu_ops.outfeed_dequeue_tuple(
dtypes=[spec.dtype for spec in self._outfeed_spec],
shapes=[spec.shape for spec in self._outfeed_spec],
name='outfeed-dequeue-%s' % self.execution_mode)
return CompiledTPUOp(tpu_execute_op, infeed_tensors, infeed_op, outfeed_op)
def predict_loop(model, ins, batch_size=32, verbose=0, steps=None):
"""Abstract method to loop over some data in batches.
Arguments:
model:
ins: list of tensors to be fed to `f`.
batch_size: integer batch size.
verbose: verbosity mode.
steps: Total number of steps (batches of samples)
before declaring `_predict_loop` finished.
Ignored with the default value of `None`.
Returns:
Array of predictions (if the model has a single output)
or list of arrays of predictions
(if the model has multiple outputs).
"""
K.set_learning_phase(False)
num_samples = model._check_num_samples(ins, batch_size, steps, 'steps')
if verbose == 1:
if steps is not None:
progbar = Progbar(target=steps)
else:
progbar = Progbar(target=num_samples)
outs = []
batches = make_batches(num_samples, batch_size)
index_array = np.arange(num_samples)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if ins and isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_arrays(ins, batch_ids)
ins_batch_converted = []
for ib in ins_batch:
ins_batch_converted.append(ops.convert_to_tensor(ib, dtype=K.floatx()))
eager_model_inputs = []
for i in range(len(model.inputs)):
eager_model_inputs.append(ins_batch_converted[i])
if len(eager_model_inputs) == 1:
batch_outs = model.call(eager_model_inputs[0])
else:
batch_outs = model.call(eager_model_inputs)
if not isinstance(batch_outs, list):
batch_outs = [batch_outs]
if batch_index == 0:
# Pre-allocate the results arrays.
for batch_out in batch_outs:
dims = batch_out.shape[1:].dims
dims_list = [d.value for d in dims]
shape = (num_samples,) + tuple(dims_list)
outs.append(np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype))
for i, batch_out in enumerate(batch_outs):
outs[i][batch_start:batch_end] = batch_out
if verbose == 1:
progbar.update(batch_end)
if len(outs) == 1:
return outs[0]
return outs
def test_loop(model,
inputs,
targets,
sample_weights=None,
batch_size=None,
verbose=0,
steps=None):
"""Abstract method to loop over some data in batches.
Arguments:
model: Model instance that is being evaluated in Eager mode.
inputs: List of input arrays.
targets: List of target arrays.
sample_weights: Optional list of sample weight arrays.
batch_size: integer batch size or `None`.
verbose: verbosity mode.
steps: Total number of steps (batches of samples)
before declaring predictions finished.
Ignored with the default value of `None`.
Returns:
Scalar loss (if the model has a single output and no metrics)
or list of scalars (if the model has multiple outputs
and/or metrics). The attribute `model.metrics_names` will give you
the display labels for the scalar outputs.
"""
K.set_learning_phase(False)
feed_data = inputs + targets
if sample_weights:
feed_data += sample_weights
num_samples = training_utils.check_num_samples(feed_data,
batch_size=batch_size,
steps=steps,
steps_name='steps')
outs = []
if verbose == 1:
progbar = Progbar(target=num_samples)
batches = make_batches(num_samples, batch_size)
index_array = np.arange(num_samples)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
inputs_batch = slice_arrays(inputs, batch_ids)
targets_batch = slice_arrays(targets, batch_ids)
if sample_weights:
sample_weights_batch = slice_arrays(sample_weights, batch_ids)
else:
sample_weights_batch = None
inputs_batch = [
ops.convert_to_tensor(val, dtype=K.floatx())
for val in inputs_batch
]
targets_batch = [
ops.convert_to_tensor(val, dtype=K.floatx())
for val in targets_batch
]
if sample_weights:
sample_weights_batch = [
ops.convert_to_tensor(val, dtype=K.floatx())
if val is not None else None for val in sample_weights_batch
]
loss_outs, loss, loss_metrics = _model_loss(
model,
inputs_batch,
targets_batch,
sample_weights=sample_weights_batch,
training=False)
_, metrics_results = _eager_metrics_fn(model, loss_outs, targets_batch)
batch_outs = []
for _, v in zip(model.metrics_names,
[K.mean(loss)] + loss_metrics + metrics_results):
batch_outs.append(tensor_util.constant_value(v))
if isinstance(batch_outs, list):
if batch_index == 0:
for batch_out in enumerate(batch_outs):
outs.append(0.)
for i, batch_out in enumerate(batch_outs):
outs[i] += batch_out * len(batch_ids)
else:
if batch_index == 0:
outs.append(0.)
outs[0] += batch_outs * len(batch_ids)
if verbose == 1:
progbar.update(batch_end)
for i in range(len(outs)):
outs[i] /= num_samples
if len(outs) == 1:
return outs[0]
return outs
def fit_loop(
model,
ins,
out_labels=None,
batch_size=None,
epochs=100,
verbose=1,
callbacks=None,
val_ins=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None):
"""Abstract fit function for `f(ins)`.
Assume that f returns a list, labeled by out_labels.
Arguments:
model: Instance of the model that is being executed in Eager mode.
ins: List of tensors to be fed to `f`
out_labels: List of strings, display names of
the outputs of `f`
batch_size: Integer batch size or None if unknown.
epochs: Number of times to iterate over the data
verbose: Verbosity mode, 0, 1 or 2
callbacks: List of callbacks to be called during training
val_ins: List of tensors to be fed to `val_f`
shuffle: Whether to shuffle the data at the beginning of each epoch
callback_metrics: List of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: Epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. Ignored with the default value of `None`.
validation_steps: Number of steps to run validation for (only if doing
validation from data tensors). Ignored with default value of `None`.
Returns:
`History` object.
Raises:
ValueError: In case of invalid argument values.
"""
# Required for Eager mode
K.set_learning_phase(True)
do_validation = False
if val_ins:
do_validation = True
if (verbose and ins and hasattr(ins[0], 'shape') and
hasattr(val_ins[0], 'shape')):
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if validation_steps:
if steps_per_epoch is None:
raise ValueError('Can only use `validation_steps` when doing step-wise '
'training, i.e. `steps_per_epoch` must be set.')
do_validation = True
num_train_samples = model._check_num_samples(
ins, batch_size, steps_per_epoch, 'steps_per_epoch')
if num_train_samples is not None:
index_array = np.arange(num_train_samples)
model.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [model.history]
if verbose:
if steps_per_epoch is not None:
count_mode = 'steps'
else:
count_mode = 'samples'
callbacks += [cbks.ProgbarLogger(count_mode)]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(model, 'callback_model') and model.callback_model:
callback_model = model.callback_model
else:
callback_model = model
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
for cbk in callbacks:
cbk.validation_data = val_ins
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
if shuffle == 'batch':
index_array = model._batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
ins_batch_converted = []
for ib in ins_batch:
ins_batch_converted.append(ops.convert_to_tensor(ib, dtype=K.floatx()))
eager_model_inputs = []
eager_model_outputs = []
for i in range(len(model.inputs)):
eager_model_inputs.append(ins_batch_converted[i])
for i in range(len(model.inputs), len(ins_batch_converted)):
eager_model_outputs.append(ins_batch_converted[i])
outs, loss, loss_metrics = _process_single_batch(eager_model_inputs,
eager_model_outputs,
model)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
# Required for Eager mode
metrics_names, metrics_results = _eager_metrics_fn(model, outs,
eager_model_outputs)
batch_logs['loss'] = tensor_util.constant_value(K.mean(loss))
# TODO(anjalisridhar): Move this to compile to avoid duplicate code.
# In graph mode we set the metric names in compile. However in
# Eager mode we calculate the metrics for each batch in fit_loop.
# We could calculate the metric names and functions in compile.
# This would avoid setting the callback parameters separately.
# We need to do this for the first iteration alone
for m in metrics_names:
if m not in callback_metrics:
callback_metrics.append(m)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
for k, v in zip(model.metrics_names,
[K.mean(loss)] + loss_metrics + metrics_results):
batch_logs[k] = tensor_util.constant_value(v)
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if batch_index == len(batches) - 1: # Last batch.
if do_validation:
val_outs = test_loop(
model, val_ins, batch_size=batch_size, verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return model.history
def _specialize_model(self, input_specs):
"""Specialize `self.model` (a Keras model) for the given input shapes."""
# Re-create our input and output layers inside our subgraph. They will be
# attached to the true computation when we clone our model in `tpu_fn`.
K.set_learning_phase(
self.execution_mode == model_fn_lib.ModeKeys.TRAIN)
# functools.partial and callable objects are not supported by tpu.rewrite
def _model_fn():
"""Compute fit/eval/predict for the TPU."""
is_training = self.execution_mode == model_fn_lib.ModeKeys.TRAIN
is_test = self.execution_mode == model_fn_lib.ModeKeys.EVAL
is_predict = self.execution_mode == model_fn_lib.ModeKeys.PREDICT
# During train/eval, we infeed our features as well as labels.
if is_training or is_test:
infeed_layers = self.model._input_layers + self.model._output_layers
else:
infeed_layers = self.model._input_layers
# Generate our infeed operation to read features & labels.
infeed_tensors = tpu_ops.infeed_dequeue_tuple(
dtypes=[spec.dtype for spec in input_specs],
shapes=[spec.shape for spec in input_specs],
name='infeed-%s' % self.execution_mode)
assert len(infeed_tensors) == len(infeed_layers), (
'Infeed inputs did not match model: %s vs %s',
(infeed_layers, infeed_tensors))
tpu_targets = []
tpu_inputs = []
# Sort infeed outputs into inputs and labels for calling our Keras model.
for tensor, layer in zip(infeed_tensors, infeed_layers):
if layer in self.model._input_layers:
tpu_inputs.append(
layers.Input(name=layer.name, tensor=tensor))
if layer in self.model._output_layers:
tpu_targets.append(tensor)
# Call our model with our infeed inputs (re-using the weights).
model_outputs = self.model(tpu_inputs)
child_model = models.Model(inputs=tpu_inputs,
outputs=model_outputs)
if is_training or is_test:
child_model.compile(
optimizer=_replicated_optimizer(self.model.optimizer,
self.num_replicas),
loss=self.model.loss,
loss_weights=self.model.loss_weights,
metrics=self.model.metrics,
weighted_metrics=self.model.weighted_metrics,
target_tensors=tpu_targets,
)
# Compute our outfeed depending on the execution mode
if is_training:
child_model._make_train_function()
self._outfeed_spec = [
tensor_spec.TensorSpec(tensor.shape, tensor.dtype,
tensor.name)
for tensor in child_model.train_function.outputs
]
return [
child_model.train_function.updates_op,
tpu_ops.outfeed_enqueue_tuple(
child_model.train_function.outputs,
name='outfeed-enqueue-train')
]
elif is_test:
child_model._make_test_function()
self._outfeed_spec = [
tensor_spec.TensorSpec(tensor.shape, tensor.dtype,
tensor.name)
for tensor in child_model.test_function.outputs
]
return [
tpu_ops.outfeed_enqueue_tuple(
child_model.test_function.outputs,
name='outfeed-enqueue-test')
]
elif is_predict:
child_model._make_predict_function()
self._outfeed_spec = [
tensor_spec.TensorSpec(tensor.shape, tensor.dtype,
tensor.name)
for tensor in child_model.predict_function.outputs
]
return [
tpu_ops.outfeed_enqueue_tuple(
child_model.predict_function.outputs,
name='outfeed-enqueue-predict',
)
]
else:
assert False, 'Unexpected execution mode: %s' % self.execution_mode
# Capture outfeed metadata computed during the rewrite.
self._outfeed_spec = None
# Generate out TPU operations using `tpu.split_compile_and_replicate`.
# `compile_op` can be used to test the TPU model compiles before execution.
# `execute op` replicates `_model_fn` `num_replicas` times, with each shard
# running on a different logical core.
compile_op, execute_op = tpu.split_compile_and_replicate(
_model_fn, inputs=[[]] * self.num_replicas)
# Generate CPU side operations to enqueue features/labels and dequeue
# outputs from the model call.
infeed_op = []
outfeed_op = []
shard_infeed_tensors = []
for shard_id in range(self.num_replicas):
with ops.device('/device:TPU:%d' % shard_id):
infeed_tensors = []
for spec in input_specs:
infeed_tensors.append(
array_ops.placeholder(dtype=spec.dtype,
shape=spec.shape,
name='infeed-enqueue-%s-%d' %
(spec.name, shard_id)))
shard_infeed_tensors.append(infeed_tensors)
infeed_op.append(
tpu_ops.infeed_enqueue_tuple(
infeed_tensors, [spec.shape for spec in input_specs],
name='infeed-enqueue-%s-%d' %
(self.execution_mode, shard_id)))
outfeed_op.extend(
tpu_ops.outfeed_dequeue_tuple(
dtypes=[spec.dtype for spec in self._outfeed_spec],
shapes=[spec.shape for spec in self._outfeed_spec],
name='outfeed-dequeue-%s-%d' %
(self.execution_mode, shard_id)))
return TPUModelOp(compile_op,
execute_op,
infeed_tensors=shard_infeed_tensors,
infeed_op=infeed_op,
outfeed_op=outfeed_op)
def _specialize_model(self, input_specs):
"""Specialize `self.model` (a Keras model) for the given input shapes."""
# Re-create our input and output layers inside our subgraph. They will be
# attached to the true computation when we clone our model in `tpu_fn`.
K.set_learning_phase(
self.execution_mode == model_fn_lib.ModeKeys.TRAIN)
# functools.partial and callable objects are not supported by tpu.rewrite
def _model_fn():
"""Compute fit/eval/predict for the TPU."""
is_training = self.execution_mode == model_fn_lib.ModeKeys.TRAIN
is_test = self.execution_mode == model_fn_lib.ModeKeys.EVAL
is_predict = self.execution_mode == model_fn_lib.ModeKeys.PREDICT
# During train/eval, we infeed our features as well as labels.
if is_training or is_test:
infeed_layers = self.model._input_layers + self.model._output_layers
else:
infeed_layers = self.model._input_layers
# Generate our infeed operation to read features & labels.
infeed_tensors = tpu_ops.infeed_dequeue_tuple(
dtypes=[spec.dtype for spec in input_specs],
shapes=[spec.shape for spec in input_specs],
name='infeed-%s' % self.execution_mode)
assert len(infeed_tensors) == len(infeed_layers), (
'Infeed inputs did not match model: %s vs %s',
(infeed_layers, infeed_tensors))
tpu_targets = []
tpu_inputs = []
# Sort infeed outputs into inputs and labels for calling our Keras model.
for tensor, layer in zip(infeed_tensors, infeed_layers):
if layer in self.model._input_layers:
tpu_inputs.append(
layers.Input(name=layer.name, tensor=tensor))
if layer in self.model._output_layers:
tpu_targets.append(tensor)
optimizer = self.model.optimizer
optimizer.iterations = training_util.get_or_create_global_step()
# Call our model with our infeed inputs (re-using the weights).
model_outputs = self.model(tpu_inputs)
child_model = models.Model(inputs=tpu_inputs,
outputs=model_outputs)
if is_training or is_test:
child_model.compile(
optimizer=self.model.optimizer,
loss=self.model.loss,
loss_weights=self.model.loss_weights,
metrics=self.model.metrics,
weighted_metrics=self.model.weighted_metrics,
target_tensors=tpu_targets,
)
# Compute our outfeed depending on the execution mode
if is_training:
child_model._make_train_function()
self._outfeed_spec = [
tensor_spec.TensorSpec(tensor.shape, tensor.dtype,
tensor.name)
for tensor in child_model.train_function.outputs
]
return [
child_model.train_function.updates_op,
tpu_ops.outfeed_enqueue_tuple(
child_model.train_function.outputs,
name='oufeed-enqueue-train')
]
elif is_test:
child_model._make_test_function()
self._outfeed_spec = [
tensor_spec.TensorSpec(tensor.shape, tensor.dtype,
tensor.name)
for tensor in child_model.test_function.outputs
]
return [
tpu_ops.outfeed_enqueue_tuple(
child_model.test_function.outputs,
name='outfeed-enqueue-test')
]
elif is_predict:
child_model._make_predict_function()
self._outfeed_spec = [
tensor_spec.TensorSpec(tensor.shape, tensor.dtype,
tensor.name)
for tensor in child_model.predict_function.outputs
]
return [
tpu_ops.outfeed_enqueue_tuple(
child_model.predict_function.outputs,
name='outfeed-enqueue-predict',
)
]
else:
assert False, 'Unexpected execution mode: %s' % self.execution_mode
# Capture outfeed metadata computed during the rewrite.
self._outfeed_spec = None
tpu_execute_op = tpu.rewrite(_model_fn)
K._initialize_variables(
K.get_session()) # pylint-disable: protected-access
# Generate CPU side operations to enqueue features/labels and dequeue
# outputs from the model call.
with ops.device('/device:TPU:0'):
infeed_tensors = []
for spec in input_specs:
infeed_tensors.append(
array_ops.placeholder(dtype=spec.dtype,
shape=spec.shape,
name='infeed-enqueue-%s' %
spec.name))
infeed_op = tpu_ops.infeed_enqueue_tuple(
infeed_tensors, [spec.shape for spec in input_specs],
name='infeed-enqueue-%s' % self.execution_mode)
outfeed_op = tpu_ops.outfeed_dequeue_tuple(
dtypes=[spec.dtype for spec in self._outfeed_spec],
shapes=[spec.shape for spec in self._outfeed_spec],
name='outfeed-dequeue-%s' % self.execution_mode)
return CompiledTPUOp(tpu_execute_op, infeed_tensors, infeed_op,
outfeed_op)
def predict_loop(model, inputs, batch_size=32, verbose=0, steps=None):
"""Abstract method to loop over some data in batches.
Arguments:
model:
inputs: List of input arrays.
batch_size: integer batch size.
verbose: verbosity mode.
steps: Total number of steps (batches of samples)
before declaring `_predict_loop` finished.
Ignored with the default value of `None`.
Returns:
Array of predictions (if the model has a single output)
or list of arrays of predictions
(if the model has multiple outputs).
"""
K.set_learning_phase(False)
num_samples = training_utils.check_num_samples(inputs, batch_size, steps,
'steps')
if verbose == 1:
if steps is not None:
progbar = Progbar(target=steps)
else:
progbar = Progbar(target=num_samples)
outs = []
batches = make_batches(num_samples, batch_size)
index_array = np.arange(num_samples)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
inputs_batch = slice_arrays(inputs, batch_ids)
inputs_batch = [
ops.convert_to_tensor(val, dtype=K.floatx())
for val in inputs_batch
]
if len(inputs_batch) == 1:
if model._expects_training_arg:
batch_outs = model.call(inputs_batch[0], training=False)
else:
batch_outs = model.call(inputs_batch[0])
else:
if model._expects_training_arg:
batch_outs = model.call(inputs_batch, training=False)
else:
batch_outs = model.call(inputs_batch)
if not isinstance(batch_outs, list):
batch_outs = [batch_outs]
if batch_index == 0:
# Pre-allocate the results arrays.
for batch_out in batch_outs:
dims = batch_out.shape[1:].dims
dims_list = [d.value for d in dims]
shape = (num_samples, ) + tuple(dims_list)
outs.append(
np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype))
for i, batch_out in enumerate(batch_outs):
outs[i][batch_start:batch_end] = batch_out
if verbose == 1:
progbar.update(batch_end)
if len(outs) == 1:
return outs[0]
return outs
def test_loop(model, ins, batch_size=None, verbose=0, steps=None):
"""Abstract method to loop over some data in batches.
Arguments:
model: Model instance that is being evaluated in Eager mode.
ins: list of tensors to be fed to `f`.
batch_size: integer batch size or `None`.
verbose: verbosity mode.
steps: Total number of steps (batches of samples)
before declaring predictions finished.
Ignored with the default value of `None`.
Returns:
Scalar loss (if the model has a single output and no metrics)
or list of scalars (if the model has multiple outputs
and/or metrics). The attribute `model.metrics_names` will give you
the display labels for the scalar outputs.
"""
K.set_learning_phase(False)
num_samples = model._check_num_samples(ins, batch_size, steps, 'steps')
outs = []
if verbose == 1:
progbar = Progbar(target=num_samples)
batches = make_batches(num_samples, batch_size)
index_array = np.arange(num_samples)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_arrays(ins, batch_ids)
ins_batch_converted = []
for ib in ins_batch:
ins_batch_converted.append(ops.convert_to_tensor(ib, dtype=K.floatx()))
eager_model_inputs = []
eager_model_outputs = []
for i in range(len(model.inputs)):
eager_model_inputs.append(ins_batch_converted[i])
for i in range(len(model.inputs), len(ins_batch_converted)):
eager_model_outputs.append(ins_batch_converted[i])
loss_outs, loss, loss_metrics = _model_loss(model, eager_model_inputs,
eager_model_outputs)
_, metrics_results = _eager_metrics_fn(model, loss_outs,
eager_model_outputs)
batch_outs = []
for _, v in zip(model.metrics_names,
[K.mean(loss)] + loss_metrics + metrics_results):
batch_outs.append(tensor_util.constant_value(v))
if isinstance(batch_outs, list):
if batch_index == 0:
for batch_out in enumerate(batch_outs):
outs.append(0.)
for i, batch_out in enumerate(batch_outs):
outs[i] += batch_out * len(batch_ids)
else:
if batch_index == 0:
outs.append(0.)
outs[0] += batch_outs * len(batch_ids)
if verbose == 1:
progbar.update(batch_end)
for i in range(len(outs)):
outs[i] /= num_samples
if len(outs) == 1:
return outs[0]
return outs
def from_config(cls, mode, features, labels, config): # pylint: disable=arguments-differ
"""Instantiates a Graph container from its config (output of `get_config()`).
Arguments:
mode:
features:
labels:
config: Model config dictionary.
Returns:
A model instance.
Raises:
ValueError: In case of improperly formatted config dict.
"""
# set the training mode
set_learning_phase(Modes.is_train(mode))
if not isinstance(config, GraphConfig):
config = GraphConfig.from_dict(config)
# layer instances created during
# the graph reconstruction process
created_layers = {}
# Create an input layer based on the defined inputs and features
for layer in config.input_layers:
layer_name, node_index, tensor_index = cls.get_node_data(layer)
if layer_name in features:
created_layers[layer_name] = InputLayer(
input_tensor=features[layer_name], name=layer_name)
elif isinstance(labels, Mapping) and layer_name in labels:
created_layers[layer_name] = InputLayer(
input_tensor=labels[layer_name], name=layer_name)
else:
raise ConfigurationError("Input `{}`is not found".format(layer_name))
def process_layer(layer):
"""Deserialize a layer, then call it on appropriate inputs.
Arguments:
layer_data: layer config dict.
Raises:
ValueError: In case of improperly formatted `layer_data` dict.
"""
layer_class = layer.IDENTIFIER
layer_name = layer.name
# Instantiate layer.
if layer_class in LAYERS:
created_layer = LAYERS[layer_class].from_config(layer)
elif layer_class in IMAGE_PROCESSORS:
created_layer = IMAGE_PROCESSORS[layer_class].from_config(layer)
else:
raise ValueError("The layer `{}` is not supported.".format(layer_class))
created_layers[layer_name] = created_layer
# Gather layer inputs.
inbound_nodes_data = layer.inbound_nodes
input_tensors = []
for input_data in inbound_nodes_data:
in_layer_name, in_node_index, in_tensor_index = cls.get_node_data(input_data)
if len(input_data) == 3:
kwargs = {}
elif len(input_data) == 4:
kwargs = input_data[3]
else:
raise ValueError('Improperly formatted model config.')
if in_layer_name not in created_layers:
raise ValueError('Missing layer: ' + in_layer_name)
inbound_layer = created_layers[in_layer_name]
inbound_node = inbound_layer.inbound_nodes[in_node_index]
input_tensors.append(inbound_node.output_tensors[in_tensor_index])
# Call layer on its inputs, thus creating the node
# and building the layer if needed.
if input_tensors:
if len(input_tensors) == 1:
created_layer(input_tensors[0], **kwargs)
else:
created_layer(input_tensors, **kwargs)
for layer in config.layers:
process_layer(layer)
name = config.name
input_tensors = []
output_tensors = []
for layer_data in config.input_layers:
layer_name, node_index, tensor_index = cls.get_node_data(layer_data)
assert layer_name in created_layers, "Layer `{}` not found".format(layer_name)
layer = created_layers[layer_name]
layer_output_tensors = layer.inbound_nodes[node_index].output_tensors
input_tensors.append(layer_output_tensors[tensor_index])
for layer_data in config.output_layers:
layer_name, node_index, tensor_index = cls.get_node_data(layer_data)
assert layer_name in created_layers
layer = created_layers[layer_name]
layer_output_tensors = layer.inbound_nodes[node_index].output_tensors
output_tensors.append(layer_output_tensors[tensor_index])
return cls(inputs=input_tensors, outputs=output_tensors, name=name)
