def test_ask_to_proceed_with_overwrite():
with patch('six.moves.input') as mock:
mock.return_value = 'y'
assert ask_to_proceed_with_overwrite('/tmp/not_exists')
mock.return_value = 'n'
assert not ask_to_proceed_with_overwrite('/tmp/not_exists')
def test_ask_to_proceed_with_overwrite():
with patch('six.moves.input') as mock:
mock.return_value = 'y'
assert ask_to_proceed_with_overwrite('/tmp/not_exists')
mock.return_value = 'n'
assert not ask_to_proceed_with_overwrite('/tmp/not_exists')
def test_ask_to_proceed_with_overwrite(self):
with tf.compat.v1.test.mock.patch.object(builtins, 'input') as mock_log:
mock_log.return_value = 'y'
self.assertTrue(io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists'))
mock_log.return_value = 'n'
self.assertFalse(
io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists'))
mock_log.side_effect = ['m', 'y']
self.assertTrue(io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists'))
mock_log.side_effect = ['m', 'n']
self.assertFalse(
io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists'))
def test_ask_to_proceed_with_overwrite():
if sys.version_info[:2] <= (2, 7):
with patch('__builtin__.raw_input') as mock:
mock.return_value = 'y'
assert ask_to_proceed_with_overwrite('/tmp/not_exists')
mock.return_value = 'n'
assert not ask_to_proceed_with_overwrite('/tmp/not_exists')
else:
with patch('builtins.input') as mock:
mock.return_value = 'y'
assert ask_to_proceed_with_overwrite('/tmp/not_exists')
mock.return_value = 'n'
assert not ask_to_proceed_with_overwrite('/tmp/not_exists')
def save(model, filepath, overwrite, include_optimizer, signatures=None,
options=None, save_traces=True):
"""Saves a model as a SavedModel to the filepath.
Args:
model: Keras model instance to be saved.
filepath: String path to save the model.
overwrite: whether to overwrite the existing filepath.
include_optimizer: If True, save the model's optimizer state.
signatures: Signatures to save with the SavedModel. Applicable to the 'tf'
format only. Please see the `signatures` argument in `tf.saved_model.save`
for details.
options: (only applies to SavedModel format) `tf.saved_model.SaveOptions`
object that specifies options for saving to SavedModel.
save_traces: (only applies to SavedModel format) When enabled, the
SavedModel will store the function traces for each layer. This
can be disabled, so that only the configs of each layer are stored.
Defaults to `True`. Disabling this will decrease serialization time
and reduce file size, but it requires that all custom layers/models
implement a `get_config()` method.
Raises:
ValueError: if the model's inputs have not been defined.
"""
# If file exists and should not be overwritten.
if not overwrite and os.path.exists(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
if save_traces:
if save_impl.should_skip_serialization(model):
saving_utils.raise_model_input_error(model)
if not include_optimizer:
orig_optimizer = model.optimizer
model.optimizer = None
# TODO(b/180760306) Change to del model.optimizer if Layer's __delattr__
# calls AutoTrackable's __delattr__.
model._delete_tracking("optimizer") # pylint: disable=protected-access
# Trace all functions and signatures with `training=0` instead of using an
# already-set learning phase placeholder.
# This is needed for compatibility reasons until learning phase setting
# is removed from the public apis.
with K.deprecated_internal_learning_phase_scope(0):
with utils.keras_option_scope(save_traces):
saved_nodes, node_paths = save_lib.save_and_return_nodes(
model, filepath, signatures, options)
# Save all metadata to a separate file in the SavedModel directory.
metadata = generate_keras_metadata(saved_nodes, node_paths)
with tf.io.gfile.GFile(
os.path.join(filepath, constants.SAVED_METADATA_PATH), "wb") as w:
w.write(metadata.SerializeToString(deterministic=True))
if not include_optimizer:
model.optimizer = orig_optimizer
def test_ask_to_proceed_with_overwrite(self):
with tf.compat.v1.test.mock.patch.object(builtins,
"input") as mock_log:
mock_log.return_value = "y"
self.assertTrue(
io_utils.ask_to_proceed_with_overwrite("/tmp/not_exists"))
mock_log.return_value = "n"
self.assertFalse(
io_utils.ask_to_proceed_with_overwrite("/tmp/not_exists"))
mock_log.side_effect = ["m", "y"]
self.assertTrue(
io_utils.ask_to_proceed_with_overwrite("/tmp/not_exists"))
mock_log.side_effect = ["m", "n"]
self.assertFalse(
io_utils.ask_to_proceed_with_overwrite("/tmp/not_exists"))
def save_weights(_self, fname, overwrite=True):
if not overwrite and os.path.isfile(fname):
proceed = ask_to_proceed_with_overwrite(fname)
if not proceed:
return
outf = h5py.File(fname, 'w')
weight = _self.get_weights()
for i in range(len(weight)):
outf.create_dataset('weight' + str(i), data=weight[i])
outf.close()
def save_model(model, filepath, overwrite=True, include_optimizer=True):
"""Save a model to a HDF5 file.
Note: Please also see
[How can I install HDF5 or h5py to save my models in Keras?](
/getting-started/faq/
#how-can-i-install-HDF5-or-h5py-to-save-my-models-in-Keras)
in the FAQ for instructions on how to install `h5py`.
The saved model contains:
- the model's configuration (topology)
- the model's weights
- the model's optimizer's state (if any)
Thus the saved model can be reinstantiated in
the exact same state, without any of the code
used for model definition or training.
# Arguments
model: Keras model instance to be saved.
filepath: one of the following:
- string, path where to save the model, or
- h5py.File or h5py.Group object where to save the model
overwrite: Whether we should overwrite any existing
model at the target location, or instead
ask the user with a manual prompt.
include_optimizer: If True, save optimizer's state together.
# Raises
ImportError: if h5py is not available.
"""
if h5py is None:
raise ImportError('`save_model` requires h5py.')
if not isinstance(filepath, h5py.Group):
# If file exists and should not be overwritten.
if not overwrite and os.path.isfile(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
opened_new_file = True
else:
opened_new_file = False
f = h5dict(filepath, mode='w')
try:
_serialize_model(model, f, include_optimizer)
finally:
if opened_new_file:
f.close()
def should_overwrite(filepath, overwrite):
"""Returns whether the filepath should be overwritten."""
# If file exists and should not be overwritten.
if not overwrite and os.path.isfile(filepath):
return ask_to_proceed_with_overwrite(filepath)
return True
def save_model_to_hdf5(model, filepath, overwrite=True, include_optimizer=True):
"""Saves a model to a HDF5 file.
The saved model contains:
- the model's configuration (topology)
- the model's weights
- the model's optimizer's state (if any)
Thus the saved model can be reinstantiated in
the exact same state, without any of the code
used for model definition or training.
Args:
model: Keras model instance to be saved.
filepath: One of the following:
- String, path where to save the model
- `h5py.File` object where to save the model
overwrite: Whether we should overwrite any existing
model at the target location, or instead
ask the user with a manual prompt.
include_optimizer: If True, save optimizer's state together.
Raises:
ImportError: if h5py is not available.
"""
if h5py is None:
raise ImportError('`save_model()` using h5 format requires h5py. Could not '
'import h5py.')
# TODO(psv) Add warning when we save models that contain non-serializable
# entities like metrics added using `add_metric` and losses added using
# `add_loss.`
if len(model.weights) != len(model._undeduplicated_weights):
logging.warning('Found duplicated `Variable`s in Model\'s `weights`. '
'This is usually caused by `Variable`s being shared by '
'Layers in the Model. These `Variable`s will be treated '
'as separate `Variable`s when the Model is restored. To '
'avoid this, please save with `save_format="tf"`.')
if not isinstance(filepath, h5py.File):
# If file exists and should not be overwritten.
if not overwrite and os.path.isfile(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
# Try creating dir if not exist
dirpath = os.path.dirname(filepath)
if not os.path.exists(dirpath):
tf.io.gfile.makedirs(dirpath)
f = h5py.File(filepath, mode='w')
opened_new_file = True
else:
f = filepath
opened_new_file = False
try:
model_metadata = saving_utils.model_metadata(model, include_optimizer)
for k, v in model_metadata.items():
if isinstance(v, (dict, list, tuple)):
f.attrs[k] = json.dumps(
v, default=json_utils.get_json_type).encode('utf8')
else:
f.attrs[k] = v
model_weights_group = f.create_group('model_weights')
save_weights_to_hdf5_group(model_weights_group, model)
# TODO(b/128683857): Add integration tests between tf.keras and external
# Keras, to avoid breaking TF.js users.
if (include_optimizer and model.optimizer and
not isinstance(model.optimizer, optimizer_v1.TFOptimizer)):
save_optimizer_weights_to_hdf5_group(f, model.optimizer)
f.flush()
finally:
if opened_new_file:
f.close()
def save_model(model, filepath, overwrite=True, include_optimizer=True):
"""Save a model to a HDF5 file.
The saved model contains:
- the model's configuration (topology)
- the model's weights
- the model's optimizer's state (if any)
- the model's metadata
Thus the saved model can be reinstantiated in
the exact same state, without any of the code
used for model definition or training.
# Arguments
model: Keras model instance to be saved.
filepath: String, path where to save the model.
overwrite: Whether we should overwrite any existing
model at the target location, or instead
ask the user with a manual prompt.
include_optimizer: If True, save optimizer's state together.
# Raises
ImportError: if h5py is not available.
"""
if h5py is None:
raise ImportError('`save_model` requires h5py.')
def get_json_type(obj):
"""Serialize any object to a JSON-serializable structure.
# Arguments
obj: the object to serialize
# Returns
JSON-serializable structure representing `obj`.
# Raises
TypeError: if `obj` cannot be serialized.
"""
# if obj is a serializable Keras class instance
# e.g. optimizer, layer
if hasattr(obj, 'get_config'):
return {
'class_name': obj.__class__.__name__,
'config': obj.get_config(),
}
# if obj is any numpy type
if type(obj).__module__ == np.__name__:
if isinstance(obj, np.ndarray):
return {
'type': type(obj),
'value': obj.tolist(),
}
else:
return obj.item()
# misc functions (e.g. loss function)
if callable(obj):
return obj.__name__
# if obj is a python 'type'
if type(obj).__name__ == type.__name__:
return obj.__name__
raise TypeError('Not JSON Serializable:', obj)
from keras import __version__ as keras_version
# If file exists and should not be overwritten.
if not overwrite and os.path.isfile(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
f = h5py.File(filepath, 'w')
f.attrs['keras_version'] = str(keras_version).encode('utf8')
f.attrs['backend'] = K.backend().encode('utf8')
f.attrs['model_config'] = json.dumps(
{
'class_name': model.__class__.__name__,
'config': model.get_config()
},
default=get_json_type).encode('utf8')
if hasattr(model, 'metadata'):
f.attrs['metadata'] = json.dumps(model.metadata).encode('utf8')
model_weights_group = f.create_group('model_weights')
if legacy_models.needs_legacy_support(model):
model_layers = legacy_models.legacy_sequential_layers(model)
else:
model_layers = model.layers
topology.save_weights_to_hdf5_group(model_weights_group, model_layers)
if include_optimizer and hasattr(model, 'optimizer'):
if isinstance(model.optimizer, optimizers.TFOptimizer):
warnings.warn(
'TensorFlow optimizers do not '
'make it possible to access '
'optimizer attributes or optimizer state '
'after instantiation. '
'As a result, we cannot save the optimizer '
'as part of the model save file.'
'You will have to compile your model again after loading it. '
'Prefer using a Keras optimizer instead '
'(see keras.io/optimizers).')
else:
f.attrs['training_config'] = json.dumps(
{
'optimizer_config': {
'class_name': model.optimizer.__class__.__name__,
'config': model.optimizer.get_config()
},
'loss': model.loss,
'metrics': model.metrics,
'sample_weight_mode': model.sample_weight_mode,
'loss_weights': model.loss_weights,
},
default=get_json_type).encode('utf8')
# Save optimizer weights.
symbolic_weights = getattr(model.optimizer, 'weights')
if symbolic_weights:
optimizer_weights_group = f.create_group('optimizer_weights')
weight_values = K.batch_get_value(symbolic_weights)
weight_names = []
for i, (w,
val) in enumerate(zip(symbolic_weights,
weight_values)):
# Default values of symbolic_weights is /variable for theano and cntk
if K.backend() == 'theano' or K.backend() == 'cntk':
if hasattr(w, 'name') and w.name != "/variable":
name = str(w.name)
else:
name = 'param_' + str(i)
else:
if hasattr(w, 'name') and w.name:
name = str(w.name)
else:
name = 'param_' + str(i)
weight_names.append(name.encode('utf8'))
optimizer_weights_group.attrs['weight_names'] = weight_names
for name, val in zip(weight_names, weight_values):
param_dset = optimizer_weights_group.create_dataset(
name, val.shape, dtype=val.dtype)
if not val.shape:
# scalar
param_dset[()] = val
else:
param_dset[:] = val
f.flush()
f.close()
