def test_ask_to_proceed_with_overwrite(self): with test.mock.patch.object(builtins, 'input') as mock_log: mock_log.return_value = 'y' self.assertTrue(io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists')) mock_log.return_value = 'n' self.assertFalse( io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists')) mock_log.side_effect = ['m', 'y'] self.assertTrue(io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists')) mock_log.side_effect = ['m', 'n'] self.assertFalse( io_utils.ask_to_proceed_with_overwrite('/tmp/not_exists'))
def save(model, filepath, overwrite, include_optimizer): """Saves a model as a SavedModel to the filepath. Args: model: Keras model instance to be saved. filepath: String path to save the model. overwrite: whether to overwrite the existing filepath. include_optimizer: If True, save the model's optimizer state. Raises: ValueError: if the model's inputs have not been defined. """ # If file exists and should not be overwritten. if not overwrite and os.path.exists(filepath): proceed = ask_to_proceed_with_overwrite(filepath) if not proceed: return if _should_skip_serialization(model): saving_utils.raise_model_input_error(model) if not include_optimizer: orig_optimizer = model.optimizer model.optimizer = None save_lib.save(model, filepath) if not include_optimizer: model.optimizer = orig_optimizer
def save(model, filepath, overwrite, include_optimizer, signatures=None): """Saves a model as a SavedModel to the filepath. Args: model: Keras model instance to be saved. filepath: String path to save the model. overwrite: whether to overwrite the existing filepath. include_optimizer: If True, save the model's optimizer state. signatures: Signatures to save with the SavedModel. Applicable to the 'tf' format only. Please see the `signatures` argument in `tf.saved_model.save` for details. Raises: ValueError: if the model's inputs have not been defined. """ # If file exists and should not be overwritten. if not overwrite and os.path.exists(filepath): proceed = ask_to_proceed_with_overwrite(filepath) if not proceed: return if _should_skip_serialization(model): saving_utils.raise_model_input_error(model) if not include_optimizer: orig_optimizer = model.optimizer model.optimizer = None # Trace all functions and signatures with `training=0` instead of using the # default learning phase placeholder. with K.learning_phase_scope(0): save_lib.save(model, filepath, signatures) if not include_optimizer: model.optimizer = orig_optimizer
def save(model, filepath, overwrite, include_optimizer, signatures=None, options=None, save_traces=True): """Saves a model as a SavedModel to the filepath. Args: model: Keras model instance to be saved. filepath: String path to save the model. overwrite: whether to overwrite the existing filepath. include_optimizer: If True, save the model's optimizer state. signatures: Signatures to save with the SavedModel. Applicable to the 'tf' format only. Please see the `signatures` argument in `tf.saved_model.save` for details. options: (only applies to SavedModel format) `tf.saved_model.SaveOptions` object that specifies options for saving to SavedModel. save_traces: (only applies to SavedModel format) When enabled, the SavedModel will store the function traces for each layer. This can be disabled, so that only the configs of each layer are stored. Defaults to `True`. Disabling this will decrease serialization time and reduce file size, but it requires that all custom layers/models implement a `get_config()` method. Raises: ValueError: if the model's inputs have not been defined. """ # If file exists and should not be overwritten. if not overwrite and os.path.exists(filepath): proceed = ask_to_proceed_with_overwrite(filepath) if not proceed: return if save_traces: if save_impl.should_skip_serialization(model): saving_utils.raise_model_input_error(model) if not include_optimizer: orig_optimizer = model.optimizer model.optimizer = None # Trace all functions and signatures with `training=0` instead of using an # already-set learning phase placeholder. # This is needed for compatibility reasons until learning phase setting # is removed from the public apis. with K.deprecated_internal_learning_phase_scope(0): # When saving a model involving batch norm layer within a strategy scope, # the replica context is not available when calling `add_update()`, and thus # we use the default replica context here. with distribution_strategy_context._get_default_replica_context(): # pylint: disable=protected-access with utils.keras_option_scope(save_traces): save_lib.save(model, filepath, signatures, options) if not include_optimizer: model.optimizer = orig_optimizer
def save(model, filepath, overwrite, include_optimizer, signatures=None, options=None): """Saves a model as a SavedModel to the filepath. Args: model: Keras model instance to be saved. filepath: String path to save the model. overwrite: whether to overwrite the existing filepath. include_optimizer: If True, save the model's optimizer state. signatures: Signatures to save with the SavedModel. Applicable to the 'tf' format only. Please see the `signatures` argument in `tf.saved_model.save` for details. options: Optional`tf.saved_model.SaveOptions` object that specifies options for saving to SavedModel. Raises: ValueError: if the model's inputs have not been defined. """ # If file exists and should not be overwritten. if not overwrite and os.path.exists(filepath): proceed = ask_to_proceed_with_overwrite(filepath) if not proceed: return if save_impl.should_skip_serialization(model): saving_utils.raise_model_input_error(model) if not include_optimizer: orig_optimizer = model.optimizer model.optimizer = None # Trace all functions and signatures with `training=0` instead of using the # default learning phase placeholder. with K.learning_phase_scope(0): # When saving a model involving batch norm layer within a strategy scope, # the replica context is not available when calling `add_update()`, and thus # we use the default replica context here. with distribution_strategy_context._get_default_replica_context(): # pylint: disable=protected-access save_lib.save(model, filepath, signatures, options) if not include_optimizer: model.optimizer = orig_optimizer
def save_model(model, filepath, overwrite=True, include_optimizer=True): """Saves a model to a HDF5 file. The saved model contains: - the model's configuration (topology) - the model's weights - the model's optimizer's state (if any) Thus the saved model can be reinstantiated in the exact same state, without any of the code used for model definition or training. Arguments: model: Keras model instance to be saved. filepath: One of the following: - String, path where to save the model - `h5py.File` object where to save the model overwrite: Whether we should overwrite any existing model at the target location, or instead ask the user with a manual prompt. include_optimizer: If True, save optimizer's state together. Raises: ImportError: if h5py is not available. """ if h5py is None: raise ImportError('`save_model` requires h5py.') from tensorflow.python.keras import __version__ as keras_version # pylint: disable=g-import-not-at-top # TODO(psv) Add warning when we save models that contain non-serializable # entities like metrics added using `add_metric` and losses added using # `add_loss.` if not isinstance(filepath, h5py.File): # If file exists and should not be overwritten. if not overwrite and os.path.isfile(filepath): proceed = ask_to_proceed_with_overwrite(filepath) if not proceed: return f = h5py.File(filepath, mode='w') opened_new_file = True else: f = filepath opened_new_file = False try: f.attrs['keras_version'] = str(keras_version).encode('utf8') f.attrs['backend'] = K.backend().encode('utf8') f.attrs['model_config'] = json.dumps( { 'class_name': model.__class__.__name__, 'config': model.get_config() }, default=serialization.get_json_type).encode('utf8') model_weights_group = f.create_group('model_weights') model_layers = model.layers save_weights_to_hdf5_group(model_weights_group, model_layers) if include_optimizer and model.optimizer: if isinstance(model.optimizer, optimizers.TFOptimizer): logging.warning( 'TensorFlow optimizers do not ' 'make it possible to access ' 'optimizer attributes or optimizer state ' 'after instantiation. ' 'As a result, we cannot save the optimizer ' 'as part of the model save file.' 'You will have to compile your model again after loading it. ' 'Prefer using a Keras optimizer instead ' '(see keras.io/optimizers).') else: f.attrs['training_config'] = json.dumps( { 'optimizer_config': { 'class_name': model.optimizer.__class__.__name__, 'config': model.optimizer.get_config() }, 'loss': model.loss, 'metrics': model._compile_metrics, 'weighted_metrics': model._compile_weighted_metrics, 'sample_weight_mode': model.sample_weight_mode, 'loss_weights': model.loss_weights, }, default=serialization.get_json_type).encode('utf8') # Save optimizer weights. save_optimizer_weights_to_hdf5_group(f, model.optimizer) f.flush() finally: if opened_new_file: f.close()
def save_model_to_hdf5(model, filepath, overwrite=True, include_optimizer=True): """Saves a model to a HDF5 file. The saved model contains: - the model's configuration (topology) - the model's weights - the model's optimizer's state (if any) Thus the saved model can be reinstantiated in the exact same state, without any of the code used for model definition or training. Arguments: model: Keras model instance to be saved. filepath: One of the following: - String, path where to save the model - `h5py.File` object where to save the model overwrite: Whether we should overwrite any existing model at the target location, or instead ask the user with a manual prompt. include_optimizer: If True, save optimizer's state together. Raises: ImportError: if h5py is not available. """ if h5py is None: raise ImportError('`save_model` requires h5py.') # TODO(psv) Add warning when we save models that contain non-serializable # entities like metrics added using `add_metric` and losses added using # `add_loss.` if len(model.weights) != len(model._undeduplicated_weights): logging.warning( 'Found duplicated `Variable`s in Model\'s `weights`. ' 'This is usually caused by `Variable`s being shared by ' 'Layers in the Model. These `Variable`s will be treated ' 'as separate `Variable`s when the Model is restored. To ' 'avoid this, please save with `save_format="tf"`.') if not isinstance(filepath, h5py.File): # If file exists and should not be overwritten. if not overwrite and os.path.isfile(filepath): proceed = ask_to_proceed_with_overwrite(filepath) if not proceed: return f = h5py.File(filepath, mode='w') opened_new_file = True else: f = filepath opened_new_file = False try: model_metadata = saving_utils.model_metadata(model, include_optimizer) for k, v in model_metadata.items(): if isinstance(v, (dict, list, tuple)): f.attrs[k] = json.dumps( v, default=serialization.get_json_type).encode('utf8') else: f.attrs[k] = v model_weights_group = f.create_group('model_weights') model_layers = model.layers save_weights_to_hdf5_group(model_weights_group, model_layers) # TODO(b/128683857): Add integration tests between tf.keras and external # Keras, to avoid breaking TF.js users. if (include_optimizer and model.optimizer and not isinstance(model.optimizer, optimizers.TFOptimizer)): save_optimizer_weights_to_hdf5_group(f, model.optimizer) f.flush() finally: if opened_new_file: f.close()
def save_model(model, filepath, overwrite=True, include_optimizer=True): """Saves a model to a HDF5 file. The saved model contains: - the model's configuration (topology) - the model's weights - the model's optimizer's state (if any) Thus the saved model can be reinstantiated in the exact same state, without any of the code used for model definition or training. Arguments: model: Keras model instance to be saved. filepath: One of the following: - String, path where to save the model - `h5py.File` object where to save the model overwrite: Whether we should overwrite any existing model at the target location, or instead ask the user with a manual prompt. include_optimizer: If True, save optimizer's state together. Raises: ImportError: if h5py is not available. """ if h5py is None: raise ImportError('`save_model` requires h5py.') from tensorflow.python.keras import __version__ as keras_version # pylint: disable=g-import-not-at-top if not isinstance(filepath, h5py.File): # If file exists and should not be overwritten. if not overwrite and os.path.isfile(filepath): proceed = ask_to_proceed_with_overwrite(filepath) if not proceed: return f = h5py.File(filepath, mode='w') opened_new_file = True else: f = filepath opened_new_file = False try: f.attrs['keras_version'] = str(keras_version).encode('utf8') f.attrs['backend'] = K.backend().encode('utf8') f.attrs['model_config'] = json.dumps( { 'class_name': model.__class__.__name__, 'config': model.get_config() }, default=serialization.get_json_type).encode('utf8') model_weights_group = f.create_group('model_weights') model_layers = model.layers save_weights_to_hdf5_group(model_weights_group, model_layers) if include_optimizer and hasattr(model, 'optimizer'): if isinstance(model.optimizer, optimizers.TFOptimizer): logging.warning( 'TensorFlow optimizers do not ' 'make it possible to access ' 'optimizer attributes or optimizer state ' 'after instantiation. ' 'As a result, we cannot save the optimizer ' 'as part of the model save file.' 'You will have to compile your model again after loading it. ' 'Prefer using a Keras optimizer instead ' '(see keras.io/optimizers).') else: f.attrs['training_config'] = json.dumps( { 'optimizer_config': { 'class_name': model.optimizer.__class__.__name__, 'config': model.optimizer.get_config() }, 'loss': model.loss, 'metrics': model.metrics, 'sample_weight_mode': model.sample_weight_mode, 'loss_weights': model.loss_weights, }, default=serialization.get_json_type).encode('utf8') # Save optimizer weights. symbolic_weights = getattr(model.optimizer, 'weights') if symbolic_weights: optimizer_weights_group = f.create_group( 'optimizer_weights') weight_values = K.batch_get_value(symbolic_weights) weight_names = [] for w, val in zip(symbolic_weights, weight_values): name = str(w.name) weight_names.append(name.encode('utf8')) optimizer_weights_group.attrs[ 'weight_names'] = weight_names for name, val in zip(weight_names, weight_values): param_dset = optimizer_weights_group.create_dataset( name, val.shape, dtype=val.dtype) if not val.shape: # scalar param_dset[()] = val else: param_dset[:] = val f.flush() finally: if opened_new_file: f.close()
def should_overwrite(filepath, overwrite): """Returns whether the filepath should be overwritten.""" # If file exists and should not be overwritten. if not overwrite and os.path.isfile(filepath): return ask_to_proceed_with_overwrite(filepath) return True
def save(model, filepath, overwrite, include_optimizer, signatures=None, options=None, save_traces=True): """Saves a model as a SavedModel to the filepath. Args: model: Keras model instance to be saved. filepath: String path to save the model. overwrite: whether to overwrite the existing filepath. include_optimizer: If True, save the model's optimizer state. signatures: Signatures to save with the SavedModel. Applicable to the 'tf' format only. Please see the `signatures` argument in `tf.saved_model.save` for details. options: (only applies to SavedModel format) `tf.saved_model.SaveOptions` object that specifies options for saving to SavedModel. save_traces: (only applies to SavedModel format) When enabled, the SavedModel will store the function traces for each layer. This can be disabled, so that only the configs of each layer are stored. Defaults to `True`. Disabling this will decrease serialization time and reduce file size, but it requires that all custom layers/models implement a `get_config()` method. Raises: ValueError: if the model's inputs have not been defined. """ # If file exists and should not be overwritten. if not overwrite and os.path.exists(filepath): proceed = ask_to_proceed_with_overwrite(filepath) if not proceed: return if save_traces: if save_impl.should_skip_serialization(model): saving_utils.raise_model_input_error(model) if not include_optimizer: orig_optimizer = model.optimizer model.optimizer = None # TODO(b/180760306) Change to del model.optimizer if Layer's __delattr__ # calls AutoTrackable's __delattr__. model._delete_tracking("optimizer") # pylint: disable=protected-access # Trace all functions and signatures with `training=0` instead of using an # already-set learning phase placeholder. # This is needed for compatibility reasons until learning phase setting # is removed from the public apis. with K.deprecated_internal_learning_phase_scope(0): with utils.keras_option_scope(save_traces): saved_nodes, node_paths = save_lib.save_and_return_nodes( model, filepath, signatures, options) # Save all metadata to a separate file in the SavedModel directory. metadata = generate_keras_metadata(saved_nodes, node_paths) with gfile.GFile(os.path.join(filepath, constants.SAVED_METADATA_PATH), "wb") as w: w.write(metadata.SerializeToString(deterministic=True)) if not include_optimizer: model.optimizer = orig_optimizer
def save_model(model, filepath, headpath=None, optmpath=None, overwrite=True, include_optimizer=True, compress=True): """Saves a model to two JSON files and a HDF5 file The saved model would be divided into three parts. In the first part, i.e. the JSON file, there are: - the model's configuration (topology) - the model's parameter list - the model's optimizer's parameter list (if any) The second file is also a JSON file which contains - the configurations for current optimizer state It exists only when `overwrite` is `True`. The third file, i.e. HDF5 file, contains: - the model's weights Thus the saved model can be reinstantiated in the exact same state, without any of the code used for model definition or training. This scheme is used to replace the original strategy in tf-Keras, because in tf.keras.Model.save, the model's (including the optimizer's) configurations are saved in the heading of HDF5 file. However, the HDF5 file requires the heading to be smaller than 64KB. Hence, a too complicated model may generates a too large configuration file. The refined version in MDNT avoid this situation by splitting the configurations into the JSON file, but it requires users to use mdnt.save_model and mdnt.load_model together. Arguments: model: Keras model instance to be saved. filepath: One of the following: - String, path where to save the model weights - `h5py.File` object where to save the model weights headpath: One of the following: - String, path where to save the model configurations - `File` object where to save the model configurations - If set None, would get deduced from `filepath` optmpath: One of the following: - String, path where to save the model configurations - `File` object where to save the model configurations - If set None, would get deduced from `filepath` - In most cases, this variable could be left `None` overwrite: Whether we should overwrite any existing model at the target location, or instead ask the user with a manual prompt. include_optimizer: If True, save optimizer's state together. Raises: ImportError: if h5py is not available. """ if h5py is None: raise ImportError('`save_model` requires h5py.') from tensorflow.python.keras import __version__ as keras_version # pylint: disable=g-import-not-at-top # TODO(psv) Add warning when we save models that contain non-serializable # entities like metrics added using `add_metric` and losses added using # `add_loss.` # Examine the file existence and open the HDF5 file. if not isinstance(filepath, h5py.File): # If file exists and should not be overwritten. filepath_nopsfx = os.path.splitext(filepath)[0] psfx = os.path.splitext(filepath)[1].casefold() if (psfx != '.h5') and (psfx != '.hdf5'): filepath = filepath_nopsfx + '.h5' if (not overwrite) and (os.path.isfile(filepath)): proceed = ask_to_proceed_with_overwrite(filepath) if not proceed: return f = h5py.File(filepath, mode='w') opened_new_file = True else: f = filepath opened_new_file = False # Infer the headpath if set None. if headpath is None: headpath = f.filename if hasattr(headpath, 'decode'): headpath = headpath.decode('utf-8') headpath = os.path.splitext(headpath)[0] ind = headpath.rfind('-') if ind != -1: headpath = headpath[:ind] ind = headpath.find('_loss') if ind != -1: headpath = headpath[:ind] ind = headpath.find('_acc') if ind != -1: headpath = headpath[:ind] headpath = headpath + '.json' # Examine the file existence and open the JSON file. if not isinstance(headpath, io.IOBase): # If file exists and should not be overwritten. if (not overwrite) and os.path.isfile(headpath): proceed = ask_to_proceed_with_overwrite(headpath) if not proceed: return psfx = os.path.splitext(headpath)[1].casefold() if (psfx != '.json'): headpath = headpath + '.json' fh = open(headpath, 'w') opened_new_head = True else: fh = headpath opened_new_head = False if include_optimizer and model.optimizer: if optmpath is None: optmpath = f.filename if hasattr(optmpath, 'decode'): optmpath = optmpath.decode('utf-8') optmpath = os.path.splitext(optmpath)[0] + '.json' if isinstance(optmpath, str) and optmpath == headpath: optmpath = os.path.splitext(optmpath)[0] + '_opt.json' # Examine the file existence and open the JSON file. if not isinstance(optmpath, io.IOBase): # If file exists and should not be overwritten. if (not overwrite) and os.path.isfile(optmpath): proceed = ask_to_proceed_with_overwrite(optmpath) if not proceed: return psfx = os.path.splitext(optmpath)[1].casefold() if (psfx != '.json'): optmpath = optmpath + '.json' fo = open(optmpath, 'w') opened_new_optm = True else: fo = optmpath opened_new_optm = False else: fo = None try: json_dict = { 'keras_version': str(keras_version), 'backend': K.backend(), 'model_config': { 'class_name': model.__class__.__name__, 'config': model.get_config() } } model_weights_group = f.create_group('model_weights') model_layers = model.layers save_weights_to_hdf5_group(model_weights_group, json_dict, model_layers, compress=compress) if include_optimizer and model.optimizer: if isinstance(model.optimizer, optimizers.TFOptimizer): logging.warning( 'TensorFlow optimizers do not ' 'make it possible to access ' 'optimizer attributes or optimizer state ' 'after instantiation. ' 'As a result, we cannot save the optimizer ' 'as part of the model save file.' 'You will have to compile your model again after loading it. ' 'Prefer using a Keras optimizer instead ' '(see keras.io/optimizers).') else: json_optm_dict = { 'training_config': { 'optimizer_config': { 'class_name': model.optimizer.__class__.__name__, 'config': model.optimizer.get_config() }, 'loss': model.loss, 'metrics': model._compile_metrics, 'weighted_metrics': model._compile_weighted_metrics, 'sample_weight_mode': model.sample_weight_mode #'loss_weights': loss_weights } } # Save loss weights loss_weights_group = f.create_group('loss_weights') save_loss_weights_to_hdf5_group(loss_weights_group, json_optm_dict, model.loss_weights, compress=compress) # Save optimizer weights. symbolic_weights = getattr(model.optimizer, 'weights') compression = 'gzip' if compress else None if symbolic_weights: optimizer_weights_group = f.create_group( 'optimizer_weights') weight_values = K.batch_get_value(symbolic_weights) weight_names = [] for w, val in zip(symbolic_weights, weight_values): name = str(w.name) weight_names.append(name) save_attributes_to_hdf5_group(json_dict, optimizer_weights_group.name, 'weight_names', weight_names) for name, val in zip(weight_names, weight_values): param_dset = optimizer_weights_group.create_dataset( name, val.shape, dtype=val.dtype, compression=(compression if val.shape else None)) if not val.shape: # scalar param_dset[()] = val else: param_dset[:] = val fo.write( json.dumps(json_optm_dict, default=serialization.get_json_type, indent=4)) # Save the JSON file. fh.write( json.dumps(json_dict, default=serialization.get_json_type, indent=4)) f.flush() fh.flush() if fo is not None: fo.flush() finally: if opened_new_file: f.close() if opened_new_head: fh.close() if fo is not None: if opened_new_optm: fo.close()