def test_ask_to_proceed_with_overwrite(self):
with test.mock.patch.object(builtins, 'input') as mock_log:
mock_log.return_value = 'y'
self.assertTrue(io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists'))
mock_log.return_value = 'n'
self.assertFalse(
io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists'))
mock_log.side_effect = ['m', 'y']
self.assertTrue(io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists'))
mock_log.side_effect = ['m', 'n']
self.assertFalse(
io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists'))
def save(model, filepath, overwrite, include_optimizer):
"""Saves a model as a SavedModel to the filepath.
Args:
model: Keras model instance to be saved.
filepath: String path to save the model.
overwrite: whether to overwrite the existing filepath.
include_optimizer: If True, save the model's optimizer state.
Raises:
ValueError: if the model's inputs have not been defined.
"""
# If file exists and should not be overwritten.
if not overwrite and os.path.exists(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
if _should_skip_serialization(model):
saving_utils.raise_model_input_error(model)
if not include_optimizer:
orig_optimizer = model.optimizer
model.optimizer = None
save_lib.save(model, filepath)
if not include_optimizer:
model.optimizer = orig_optimizer
def save(model, filepath, overwrite, include_optimizer, signatures=None):
"""Saves a model as a SavedModel to the filepath.
Args:
model: Keras model instance to be saved.
filepath: String path to save the model.
overwrite: whether to overwrite the existing filepath.
include_optimizer: If True, save the model's optimizer state.
signatures: Signatures to save with the SavedModel. Applicable to the 'tf'
format only. Please see the `signatures` argument in `tf.saved_model.save`
for details.
Raises:
ValueError: if the model's inputs have not been defined.
"""
# If file exists and should not be overwritten.
if not overwrite and os.path.exists(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
if _should_skip_serialization(model):
saving_utils.raise_model_input_error(model)
if not include_optimizer:
orig_optimizer = model.optimizer
model.optimizer = None
# Trace all functions and signatures with `training=0` instead of using the
# default learning phase placeholder.
with K.learning_phase_scope(0):
save_lib.save(model, filepath, signatures)
if not include_optimizer:
model.optimizer = orig_optimizer
def save(model, filepath, overwrite, include_optimizer, signatures=None,
options=None, save_traces=True):
"""Saves a model as a SavedModel to the filepath.
Args:
model: Keras model instance to be saved.
filepath: String path to save the model.
overwrite: whether to overwrite the existing filepath.
include_optimizer: If True, save the model's optimizer state.
signatures: Signatures to save with the SavedModel. Applicable to the 'tf'
format only. Please see the `signatures` argument in `tf.saved_model.save`
for details.
options: (only applies to SavedModel format) `tf.saved_model.SaveOptions`
object that specifies options for saving to SavedModel.
save_traces: (only applies to SavedModel format) When enabled, the
SavedModel will store the function traces for each layer. This
can be disabled, so that only the configs of each layer are stored.
Defaults to `True`. Disabling this will decrease serialization time
and reduce file size, but it requires that all custom layers/models
implement a `get_config()` method.
Raises:
ValueError: if the model's inputs have not been defined.
"""
# If file exists and should not be overwritten.
if not overwrite and os.path.exists(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
if save_traces:
if save_impl.should_skip_serialization(model):
saving_utils.raise_model_input_error(model)
if not include_optimizer:
orig_optimizer = model.optimizer
model.optimizer = None
# Trace all functions and signatures with `training=0` instead of using an
# already-set learning phase placeholder.
# This is needed for compatibility reasons until learning phase setting
# is removed from the public apis.
with K.deprecated_internal_learning_phase_scope(0):
# When saving a model involving batch norm layer within a strategy scope,
# the replica context is not available when calling `add_update()`, and thus
# we use the default replica context here.
with distribution_strategy_context._get_default_replica_context(): # pylint: disable=protected-access
with utils.keras_option_scope(save_traces):
save_lib.save(model, filepath, signatures, options)
if not include_optimizer:
model.optimizer = orig_optimizer
def save(model,
filepath,
overwrite,
include_optimizer,
signatures=None,
options=None):
"""Saves a model as a SavedModel to the filepath.
Args:
model: Keras model instance to be saved.
filepath: String path to save the model.
overwrite: whether to overwrite the existing filepath.
include_optimizer: If True, save the model's optimizer state.
signatures: Signatures to save with the SavedModel. Applicable to the 'tf'
format only. Please see the `signatures` argument in `tf.saved_model.save`
for details.
options: Optional`tf.saved_model.SaveOptions` object that specifies
options for saving to SavedModel.
Raises:
ValueError: if the model's inputs have not been defined.
"""
# If file exists and should not be overwritten.
if not overwrite and os.path.exists(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
if save_impl.should_skip_serialization(model):
saving_utils.raise_model_input_error(model)
if not include_optimizer:
orig_optimizer = model.optimizer
model.optimizer = None
# Trace all functions and signatures with `training=0` instead of using the
# default learning phase placeholder.
with K.learning_phase_scope(0):
# When saving a model involving batch norm layer within a strategy scope,
# the replica context is not available when calling `add_update()`, and thus
# we use the default replica context here.
with distribution_strategy_context._get_default_replica_context(): # pylint: disable=protected-access
save_lib.save(model, filepath, signatures, options)
if not include_optimizer:
model.optimizer = orig_optimizer
def save_model(model, filepath, overwrite=True, include_optimizer=True):
"""Saves a model to a HDF5 file.
The saved model contains:
- the model's configuration (topology)
- the model's weights
- the model's optimizer's state (if any)
Thus the saved model can be reinstantiated in
the exact same state, without any of the code
used for model definition or training.
Arguments:
model: Keras model instance to be saved.
filepath: One of the following:
- String, path where to save the model
- `h5py.File` object where to save the model
overwrite: Whether we should overwrite any existing
model at the target location, or instead
ask the user with a manual prompt.
include_optimizer: If True, save optimizer's state together.
Raises:
ImportError: if h5py is not available.
"""
if h5py is None:
raise ImportError('`save_model` requires h5py.')
from tensorflow.python.keras import __version__ as keras_version # pylint: disable=g-import-not-at-top
# TODO(psv) Add warning when we save models that contain non-serializable
# entities like metrics added using `add_metric` and losses added using
# `add_loss.`
if not isinstance(filepath, h5py.File):
# If file exists and should not be overwritten.
if not overwrite and os.path.isfile(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
f = h5py.File(filepath, mode='w')
opened_new_file = True
else:
f = filepath
opened_new_file = False
try:
f.attrs['keras_version'] = str(keras_version).encode('utf8')
f.attrs['backend'] = K.backend().encode('utf8')
f.attrs['model_config'] = json.dumps(
{
'class_name': model.__class__.__name__,
'config': model.get_config()
},
default=serialization.get_json_type).encode('utf8')
model_weights_group = f.create_group('model_weights')
model_layers = model.layers
save_weights_to_hdf5_group(model_weights_group, model_layers)
if include_optimizer and model.optimizer:
if isinstance(model.optimizer, optimizers.TFOptimizer):
logging.warning(
'TensorFlow optimizers do not '
'make it possible to access '
'optimizer attributes or optimizer state '
'after instantiation. '
'As a result, we cannot save the optimizer '
'as part of the model save file.'
'You will have to compile your model again after loading it. '
'Prefer using a Keras optimizer instead '
'(see keras.io/optimizers).')
else:
f.attrs['training_config'] = json.dumps(
{
'optimizer_config': {
'class_name': model.optimizer.__class__.__name__,
'config': model.optimizer.get_config()
},
'loss': model.loss,
'metrics': model._compile_metrics,
'weighted_metrics': model._compile_weighted_metrics,
'sample_weight_mode': model.sample_weight_mode,
'loss_weights': model.loss_weights,
},
default=serialization.get_json_type).encode('utf8')
# Save optimizer weights.
save_optimizer_weights_to_hdf5_group(f, model.optimizer)
f.flush()
finally:
if opened_new_file:
f.close()
def save_model_to_hdf5(model,
filepath,
overwrite=True,
include_optimizer=True):
"""Saves a model to a HDF5 file.
The saved model contains:
- the model's configuration (topology)
- the model's weights
- the model's optimizer's state (if any)
Thus the saved model can be reinstantiated in
the exact same state, without any of the code
used for model definition or training.
Arguments:
model: Keras model instance to be saved.
filepath: One of the following:
- String, path where to save the model
- `h5py.File` object where to save the model
overwrite: Whether we should overwrite any existing
model at the target location, or instead
ask the user with a manual prompt.
include_optimizer: If True, save optimizer's state together.
Raises:
ImportError: if h5py is not available.
"""
if h5py is None:
raise ImportError('`save_model` requires h5py.')
# TODO(psv) Add warning when we save models that contain non-serializable
# entities like metrics added using `add_metric` and losses added using
# `add_loss.`
if len(model.weights) != len(model._undeduplicated_weights):
logging.warning(
'Found duplicated `Variable`s in Model\'s `weights`. '
'This is usually caused by `Variable`s being shared by '
'Layers in the Model. These `Variable`s will be treated '
'as separate `Variable`s when the Model is restored. To '
'avoid this, please save with `save_format="tf"`.')
if not isinstance(filepath, h5py.File):
# If file exists and should not be overwritten.
if not overwrite and os.path.isfile(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
f = h5py.File(filepath, mode='w')
opened_new_file = True
else:
f = filepath
opened_new_file = False
try:
model_metadata = saving_utils.model_metadata(model, include_optimizer)
for k, v in model_metadata.items():
if isinstance(v, (dict, list, tuple)):
f.attrs[k] = json.dumps(
v, default=serialization.get_json_type).encode('utf8')
else:
f.attrs[k] = v
model_weights_group = f.create_group('model_weights')
model_layers = model.layers
save_weights_to_hdf5_group(model_weights_group, model_layers)
# TODO(b/128683857): Add integration tests between tf.keras and external
# Keras, to avoid breaking TF.js users.
if (include_optimizer and model.optimizer
and not isinstance(model.optimizer, optimizers.TFOptimizer)):
save_optimizer_weights_to_hdf5_group(f, model.optimizer)
f.flush()
finally:
if opened_new_file:
f.close()
def save_model(model, filepath, overwrite=True, include_optimizer=True):
"""Saves a model to a HDF5 file.
The saved model contains:
- the model's configuration (topology)
- the model's weights
- the model's optimizer's state (if any)
Thus the saved model can be reinstantiated in
the exact same state, without any of the code
used for model definition or training.
Arguments:
model: Keras model instance to be saved.
filepath: One of the following:
- String, path where to save the model
- `h5py.File` object where to save the model
overwrite: Whether we should overwrite any existing
model at the target location, or instead
ask the user with a manual prompt.
include_optimizer: If True, save optimizer's state together.
Raises:
ImportError: if h5py is not available.
"""
if h5py is None:
raise ImportError('`save_model` requires h5py.')
from tensorflow.python.keras import __version__ as keras_version # pylint: disable=g-import-not-at-top
if not isinstance(filepath, h5py.File):
# If file exists and should not be overwritten.
if not overwrite and os.path.isfile(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
f = h5py.File(filepath, mode='w')
opened_new_file = True
else:
f = filepath
opened_new_file = False
try:
f.attrs['keras_version'] = str(keras_version).encode('utf8')
f.attrs['backend'] = K.backend().encode('utf8')
f.attrs['model_config'] = json.dumps(
{
'class_name': model.__class__.__name__,
'config': model.get_config()
},
default=serialization.get_json_type).encode('utf8')
model_weights_group = f.create_group('model_weights')
model_layers = model.layers
save_weights_to_hdf5_group(model_weights_group, model_layers)
if include_optimizer and hasattr(model, 'optimizer'):
if isinstance(model.optimizer, optimizers.TFOptimizer):
logging.warning(
'TensorFlow optimizers do not '
'make it possible to access '
'optimizer attributes or optimizer state '
'after instantiation. '
'As a result, we cannot save the optimizer '
'as part of the model save file.'
'You will have to compile your model again after loading it. '
'Prefer using a Keras optimizer instead '
'(see keras.io/optimizers).')
else:
f.attrs['training_config'] = json.dumps(
{
'optimizer_config': {
'class_name': model.optimizer.__class__.__name__,
'config': model.optimizer.get_config()
},
'loss': model.loss,
'metrics': model.metrics,
'sample_weight_mode': model.sample_weight_mode,
'loss_weights': model.loss_weights,
},
default=serialization.get_json_type).encode('utf8')
# Save optimizer weights.
symbolic_weights = getattr(model.optimizer, 'weights')
if symbolic_weights:
optimizer_weights_group = f.create_group(
'optimizer_weights')
weight_values = K.batch_get_value(symbolic_weights)
weight_names = []
for w, val in zip(symbolic_weights, weight_values):
name = str(w.name)
weight_names.append(name.encode('utf8'))
optimizer_weights_group.attrs[
'weight_names'] = weight_names
for name, val in zip(weight_names, weight_values):
param_dset = optimizer_weights_group.create_dataset(
name, val.shape, dtype=val.dtype)
if not val.shape:
# scalar
param_dset[()] = val
else:
param_dset[:] = val
f.flush()
finally:
if opened_new_file:
f.close()
def should_overwrite(filepath, overwrite):
"""Returns whether the filepath should be overwritten."""
# If file exists and should not be overwritten.
if not overwrite and os.path.isfile(filepath):
return ask_to_proceed_with_overwrite(filepath)
return True
def save(model,
filepath,
overwrite,
include_optimizer,
signatures=None,
options=None,
save_traces=True):
"""Saves a model as a SavedModel to the filepath.
Args:
model: Keras model instance to be saved.
filepath: String path to save the model.
overwrite: whether to overwrite the existing filepath.
include_optimizer: If True, save the model's optimizer state.
signatures: Signatures to save with the SavedModel. Applicable to the 'tf'
format only. Please see the `signatures` argument in `tf.saved_model.save`
for details.
options: (only applies to SavedModel format) `tf.saved_model.SaveOptions`
object that specifies options for saving to SavedModel.
save_traces: (only applies to SavedModel format) When enabled, the
SavedModel will store the function traces for each layer. This
can be disabled, so that only the configs of each layer are stored.
Defaults to `True`. Disabling this will decrease serialization time
and reduce file size, but it requires that all custom layers/models
implement a `get_config()` method.
Raises:
ValueError: if the model's inputs have not been defined.
"""
# If file exists and should not be overwritten.
if not overwrite and os.path.exists(filepath):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
if save_traces:
if save_impl.should_skip_serialization(model):
saving_utils.raise_model_input_error(model)
if not include_optimizer:
orig_optimizer = model.optimizer
model.optimizer = None
# TODO(b/180760306) Change to del model.optimizer if Layer's __delattr__
# calls AutoTrackable's __delattr__.
model._delete_tracking("optimizer") # pylint: disable=protected-access
# Trace all functions and signatures with `training=0` instead of using an
# already-set learning phase placeholder.
# This is needed for compatibility reasons until learning phase setting
# is removed from the public apis.
with K.deprecated_internal_learning_phase_scope(0):
with utils.keras_option_scope(save_traces):
saved_nodes, node_paths = save_lib.save_and_return_nodes(
model, filepath, signatures, options)
# Save all metadata to a separate file in the SavedModel directory.
metadata = generate_keras_metadata(saved_nodes, node_paths)
with gfile.GFile(os.path.join(filepath, constants.SAVED_METADATA_PATH),
"wb") as w:
w.write(metadata.SerializeToString(deterministic=True))
if not include_optimizer:
model.optimizer = orig_optimizer
def save_model(model,
filepath,
headpath=None,
optmpath=None,
overwrite=True,
include_optimizer=True,
compress=True):
"""Saves a model to two JSON files and a HDF5 file
The saved model would be divided into three parts. In the first part,
i.e. the JSON file, there are:
- the model's configuration (topology)
- the model's parameter list
- the model's optimizer's parameter list (if any)
The second file is also a JSON file which contains
- the configurations for current optimizer state
It exists only when `overwrite` is `True`.
The third file, i.e. HDF5 file, contains:
- the model's weights
Thus the saved model can be reinstantiated in the exact same state,
without any of the code used for model definition or training.
This scheme is used to replace the original strategy in tf-Keras,
because in tf.keras.Model.save, the model's (including the
optimizer's) configurations are saved in the heading of HDF5 file.
However, the HDF5 file requires the heading to be smaller than
64KB. Hence, a too complicated model may generates a too large
configuration file. The refined version in MDNT avoid this situation
by splitting the configurations into the JSON file, but it requires
users to use mdnt.save_model and mdnt.load_model together.
Arguments:
model: Keras model instance to be saved.
filepath: One of the following:
- String, path where to save the model weights
- `h5py.File` object where to save the model weights
headpath: One of the following:
- String, path where to save the model configurations
- `File` object where to save the model configurations
- If set None, would get deduced from `filepath`
optmpath: One of the following:
- String, path where to save the model configurations
- `File` object where to save the model configurations
- If set None, would get deduced from `filepath`
- In most cases, this variable could be left `None`
overwrite: Whether we should overwrite any existing
model at the target location, or instead
ask the user with a manual prompt.
include_optimizer: If True, save optimizer's state together.
Raises:
ImportError: if h5py is not available.
"""
if h5py is None:
raise ImportError('`save_model` requires h5py.')
from tensorflow.python.keras import __version__ as keras_version # pylint: disable=g-import-not-at-top
# TODO(psv) Add warning when we save models that contain non-serializable
# entities like metrics added using `add_metric` and losses added using
# `add_loss.`
# Examine the file existence and open the HDF5 file.
if not isinstance(filepath, h5py.File):
# If file exists and should not be overwritten.
filepath_nopsfx = os.path.splitext(filepath)[0]
psfx = os.path.splitext(filepath)[1].casefold()
if (psfx != '.h5') and (psfx != '.hdf5'):
filepath = filepath_nopsfx + '.h5'
if (not overwrite) and (os.path.isfile(filepath)):
proceed = ask_to_proceed_with_overwrite(filepath)
if not proceed:
return
f = h5py.File(filepath, mode='w')
opened_new_file = True
else:
f = filepath
opened_new_file = False
# Infer the headpath if set None.
if headpath is None:
headpath = f.filename
if hasattr(headpath, 'decode'):
headpath = headpath.decode('utf-8')
headpath = os.path.splitext(headpath)[0]
ind = headpath.rfind('-')
if ind != -1:
headpath = headpath[:ind]
ind = headpath.find('_loss')
if ind != -1:
headpath = headpath[:ind]
ind = headpath.find('_acc')
if ind != -1:
headpath = headpath[:ind]
headpath = headpath + '.json'
# Examine the file existence and open the JSON file.
if not isinstance(headpath, io.IOBase):
# If file exists and should not be overwritten.
if (not overwrite) and os.path.isfile(headpath):
proceed = ask_to_proceed_with_overwrite(headpath)
if not proceed:
return
psfx = os.path.splitext(headpath)[1].casefold()
if (psfx != '.json'):
headpath = headpath + '.json'
fh = open(headpath, 'w')
opened_new_head = True
else:
fh = headpath
opened_new_head = False
if include_optimizer and model.optimizer:
if optmpath is None:
optmpath = f.filename
if hasattr(optmpath, 'decode'):
optmpath = optmpath.decode('utf-8')
optmpath = os.path.splitext(optmpath)[0] + '.json'
if isinstance(optmpath, str) and optmpath == headpath:
optmpath = os.path.splitext(optmpath)[0] + '_opt.json'
# Examine the file existence and open the JSON file.
if not isinstance(optmpath, io.IOBase):
# If file exists and should not be overwritten.
if (not overwrite) and os.path.isfile(optmpath):
proceed = ask_to_proceed_with_overwrite(optmpath)
if not proceed:
return
psfx = os.path.splitext(optmpath)[1].casefold()
if (psfx != '.json'):
optmpath = optmpath + '.json'
fo = open(optmpath, 'w')
opened_new_optm = True
else:
fo = optmpath
opened_new_optm = False
else:
fo = None
try:
json_dict = {
'keras_version': str(keras_version),
'backend': K.backend(),
'model_config': {
'class_name': model.__class__.__name__,
'config': model.get_config()
}
}
model_weights_group = f.create_group('model_weights')
model_layers = model.layers
save_weights_to_hdf5_group(model_weights_group,
json_dict,
model_layers,
compress=compress)
if include_optimizer and model.optimizer:
if isinstance(model.optimizer, optimizers.TFOptimizer):
logging.warning(
'TensorFlow optimizers do not '
'make it possible to access '
'optimizer attributes or optimizer state '
'after instantiation. '
'As a result, we cannot save the optimizer '
'as part of the model save file.'
'You will have to compile your model again after loading it. '
'Prefer using a Keras optimizer instead '
'(see keras.io/optimizers).')
else:
json_optm_dict = {
'training_config': {
'optimizer_config': {
'class_name': model.optimizer.__class__.__name__,
'config': model.optimizer.get_config()
},
'loss': model.loss,
'metrics': model._compile_metrics,
'weighted_metrics': model._compile_weighted_metrics,
'sample_weight_mode': model.sample_weight_mode
#'loss_weights': loss_weights
}
}
# Save loss weights
loss_weights_group = f.create_group('loss_weights')
save_loss_weights_to_hdf5_group(loss_weights_group,
json_optm_dict,
model.loss_weights,
compress=compress)
# Save optimizer weights.
symbolic_weights = getattr(model.optimizer, 'weights')
compression = 'gzip' if compress else None
if symbolic_weights:
optimizer_weights_group = f.create_group(
'optimizer_weights')
weight_values = K.batch_get_value(symbolic_weights)
weight_names = []
for w, val in zip(symbolic_weights, weight_values):
name = str(w.name)
weight_names.append(name)
save_attributes_to_hdf5_group(json_dict,
optimizer_weights_group.name,
'weight_names', weight_names)
for name, val in zip(weight_names, weight_values):
param_dset = optimizer_weights_group.create_dataset(
name,
val.shape,
dtype=val.dtype,
compression=(compression if val.shape else None))
if not val.shape:
# scalar
param_dset[()] = val
else:
param_dset[:] = val
fo.write(
json.dumps(json_optm_dict,
default=serialization.get_json_type,
indent=4))
# Save the JSON file.
fh.write(
json.dumps(json_dict,
default=serialization.get_json_type,
indent=4))
f.flush()
fh.flush()
if fo is not None:
fo.flush()
finally:
if opened_new_file:
f.close()
if opened_new_head:
fh.close()
if fo is not None:
if opened_new_optm:
fo.close()
