def load_model(filepath,
inserted_layers=None,
custom_objects=None,
initial_inputs=None,
new_output_layers=None):
"""loads model like keras load_model, updates the input layer,
as well inserts extra layers after the input
Args:
filepath: path to the keras model
inserted_layers: list of list of layers to insert. layers can either be
layer config objects, or the layers themselves.
custom_objects: dict, with key,val : (class_name,class)
initial_inputs: initial input for the model, can be tf.Tensor or np.array
new_output_layers: New output layers for the model, any layers occuring after
these layers are removed. Note: buggy.
Returns:
A keras model with inserted layers and layers removed
"""
if new_output_layers is not None:
warn('using new_output_layers is buggy')
K.set_learning_phase(False)
#Make sure inserted_layers is a list of lists (added layers for each input)
if inserted_layers is not None:
if not isinstance(inserted_layers[0], list):
inserted_layers = list(inserted_layers)
added_objects = convert_layer_to_config(inserted_layers)
else:
added_objects = {}
#Make sure initial_inputs is a list of inputs
if (initial_inputs is not None and not isinstance(initial_inputs, list)):
initial_inputs = [initial_inputs]
#Make sure new_output_layers is a list of outputs
if (new_output_layers is not None
and not isinstance(new_output_layers, list)):
new_output_layers = [new_output_layers]
if not custom_objects:
custom_objects = {}
custom_objects = {**base_layers_dict, **custom_objects, **added_objects}
with h5py.File(filepath, mode='r') as f:
# instantiate model
model_config = f.attrs.get('model_config')
if model_config is None:
raise ValueError('No model found in config file.')
model_config = json.loads(model_config.decode('utf-8'))
update_config(model_config['config'], inserted_layers,
new_output_layers, initial_inputs)
model = model_from_config(model_config, custom_objects=custom_objects)
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
saving.load_weights_from_hdf5_group_by_name(f, model.layers)
return model
def load_weights(self,
filepath,
by_name=True,
skip_mismatch=False,
reshape=False,
exclude=None,
verbose=False):
import h5py
import re
from keras.engine import saving
if h5py is None:
raise ImportError('`load_weights` requires h5py.')
keras_model = self.keras_model
layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model")\
else keras_model.layers
if exclude != None:
by_name = True
layers = filter(lambda x: not re.match(exclude, x.name), layers)
if verbose:
print('[INFO] Loading following layers: ')
for layer in layers:
print('Layer: ', layer.name)
with h5py.File(filepath, mode='r') as f:
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
if by_name:
saving.load_weights_from_hdf5_group_by_name(
f, layers, skip_mismatch=skip_mismatch, reshape=reshape)
else:
saving.load_weights_from_hdf5_group(f, layers, reshape=reshape)
def load_weights(file_path, model, by_name=False, exclude=None):
import h5py
try:
from keras.engine import saving
except ImportError:
# keras before 2.2 used the 'topology'
from keras.engine import topology as saving
if exclude:
by_name = True
if h5py is None:
raise ImportError("requires h5py.")
f = h5py.File(file_path, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
# In multi-GPU training, we wrap the model. Get layers
# of the inner model because they have the weights.
keras_model = model
layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model") \
else keras_model.layers
# Exclude some layers
if exclude:
layers = filter(lambda l: l.name not in exclude, layers)
if by_name:
saving.load_weights_from_hdf5_group_by_name(f, layers)
else:
saving.load_weights_from_hdf5_group(f, layers)
if hasattr(f, 'close'):
f.close()
def load_weights(self, model_path, by_name=True, exclude=None):
'''Modified version of the corresponding Keras function with
the addition of multi-GPU support and the ability to exclude
some layers from loading.
exclude: list of layer names to exclude
'''
import h5py
from keras.engine import saving
if exclude:
by_name = True
if h5py is None:
raise ImportError('`load_weights` requires h5py.')
f = h5py.File(model_path, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
# In multi-GPU training, we wrap the model. Get layers
# of the inner model because they have the weights.
layers = self.model.inner_model.layers if hasattr(self.model, 'inner_model') \
else self.model.layers
# Exclude some layers
if exclude:
layers = filter(lambda l: l.name not in exclude, layers)
if by_name:
saving.load_weights_from_hdf5_group_by_name(f, layers)
else:
saving.load_weights_from_hdf5_group(f, layers)
if hasattr(f, 'close'):
f.close()
def load_weights(self, filepath, by_name=False, exclude=None):
import h5py
from keras.engine import saving
if exclude:
by_name = True
if h5py is None:
raise ImportError('`load_weights` requires h5py.')
f = h5py.File(filepath, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
# In multi-GPU training, we wrap the model. Get layers
# of the inner model because they have the weights.
keras_model = self.keras_model
layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model")\
else keras_model.layers
# Exclude some layers
if exclude:
layers = filter(lambda l: l.name not in exclude, layers)
if by_name:
saving.load_weights_from_hdf5_group_by_name(f, layers)
else:
saving.load_weights_from_hdf5_group(f, layers)
if hasattr(f, 'close'):
f.close()
# Update the log directory
self.set_log_dir(filepath)
def load_weights(self, model_path, by_name=True):
'''
load weights from trained model for detection
inputs:
path to model
'''
import h5py
from keras.engine import saving
with h5py.File(model_path, 'r') as f:
saving.load_weights_from_hdf5_group_by_name(f, self.model.layers)
def load_weights(self, filepath, by_name=False, exclude=None):
'''
Modified version of the corresponding Keras function with
the addition of multi-GPU support and the ability to exclude
some layers from loading.
:param filepath: path of load weights file, e.g., 'trained_models/mask_rcnn_coco.h5'
:param by_name: Boolean
:param exclude: list of layer names to exclude
:return:
'''
import h5py
# Conditional import to support versions of Keras before 2.2
# TODO: remove in about 6 months (end of 2018)
try:
from keras.engine import saving
except ImportError:
# Keras before 2.2 used the 'topology' namespace.
from keras.engine import topology as saving
if exclude:
by_name = True
if h5py is None:
raise ImportError('`load_weights` requires h5py.')
f = h5py.File(filepath, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
# In multi-GPU training, we wrap the model. Get layers
# of the inner model because they have the weights.
keras_model = self.keras_model
layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model")\
else keras_model.layers
# Exclude some layers
if exclude:
layers = filter(lambda l: l.name not in exclude, layers)
if by_name:
saving.load_weights_from_hdf5_group_by_name(f, layers)
else:
saving.load_weights_from_hdf5_group(f, layers)
if hasattr(f, 'close'):
f.close()
# Update the log directory
self.set_log_dir(filepath)
def load_weights_new(self, filepath, by_name=False,
skip_mismatch=False, reshape=False):
"""Loads all layer weights from a HDF5 save file.
If `by_name` is False (default) weights are loaded
based on the network's topology, meaning the architecture
should be the same as when the weights were saved.
Note that layers that don't have weights are not taken
into account in the topological ordering, so adding or
removing layers is fine as long as they don't have weights.
If `by_name` is True, weights are loaded into layers
only if they share the same name. This is useful
for fine-tuning or transfer-learning models where
some of the layers have changed.
# Arguments
filepath: String, path to the weights file to load.
by_name: Boolean, whether to load weights by name
or by topological order.
skip_mismatch: Boolean, whether to skip loading of layers
where there is a mismatch in the number of weights,
or a mismatch in the shape of the weight
(only valid when `by_name`=True).
reshape: Reshape weights to fit the layer when the correct number
of weight arrays is present but their shape does not match.
# Raises
ImportError: If h5py is not available.
"""
with h5py.File(filepath, mode='r') as f:
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
if by_name:
saving.load_weights_from_hdf5_group_by_name(
f, self.layers, skip_mismatch=skip_mismatch,
reshape=reshape)
else:
saving.load_weights_from_hdf5_group(
f, self.layers, reshape=reshape)
if hasattr(f, 'close'):
f.close()
elif hasattr(f.file, 'close'):
f.file.close()
def load_weights(self, pth, by_name=True, exclude=None):
import h5py
from keras.engine import saving
if exclude:
by_name = True
f = h5py.File(pth, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
# print('\n', 'Pretrained layers')
pretrained_layers = list(
[n.decode('utf8') for n in f.attrs['layer_names']])
# layers = self.model.inner_model.layers if hasattr(model, "inner_model") else self.model.layers
ls = []
for layer in self.model.layers:
if hasattr(layer, 'layers'):
for l in layer.layers:
if l.name in pretrained_layers:
# print(l.name)
ls.append(l)
else:
if layer.name in pretrained_layers:
# print(layer.name)
ls.append(layer)
if exclude:
ls = filter(lambda l: l.name not in exclude, ls)
if by_name:
saving.load_weights_from_hdf5_group_by_name(f, ls)
else:
saving.load_weights_from_hdf5_group(f, ls)
for l in ls:
l.trainable = False
if hasattr(f, 'close'):
f.close()
# print('Trainable layers')
# self.get_trainable_layers()
self.set_log_dir(pth)
def load_weights(self, filepath, by_name=False, exclude=None):
"""Modified version of the corresponding Keras function with
the addition of multi-GPU support and the ability to exclude
some layers from loading.
exclude: list of layer names to exclude
"""
self._logger.info('Loading {} model weights from {}...\n'.format(
'TRAINING' if self.env and self.env.purpose == SuiteActionEnum.TRAINING else 'INFERENCE',
filepath))
import h5py
# Conditional import to support versions of Keras before 2.2
# TODO: remove in about 6 months (end of 2018)
try:
from keras.engine import saving
except ImportError:
# Keras before 2.2 used the 'topology' namespace.
from keras.engine import topology as saving
if exclude:
by_name = True
if h5py is None:
raise ImportError('`load_weights` requires h5py.')
f = h5py.File(filepath, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
# In multi-GPU training, we wrap the model. Get layers
# of the inner model because they have the weights.
keras_model = self.train_model if self.env and self.env.purpose == SuiteActionEnum.TRAINING else self.inference_model
layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model") \
else keras_model.layers
# Exclude some layers
if exclude:
layers = filter(lambda l: l.name not in exclude, layers)
if by_name:
saving.load_weights_from_hdf5_group_by_name(f, layers)
else:
saving.load_weights_from_hdf5_group(f, layers)
if hasattr(f, 'close'):
f.close()
def load_weights(self, model_path, by_name=False, exclude=None):
"""
Modified version of the corresponding Keras function
with the addition of multi-GPU support and the ability
to exclude some layers from loading.
:param model_path:
:param by_name:
:param exclude: list of layer names to exclude
:return:
"""
if exclude:
by_name = True
pass
if h5py is None:
raise ImportError('`load_weights` requires h5py.')
pass
model_file = h5py.File(model_path, mode='r')
if 'layer_names' not in model_file.attrs and 'model_weights' in model_file:
model_file = model_file['model_weights']
# In multi-GPU training, we wrap the model. Get layers
# of the inner model because they have the weights.
keras_model = self.keras_model
layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model") else keras_model.layers
print("layers: {}".format(layers))
# Exclude some layers
if exclude:
layers = filter(lambda l: l.name not in exclude, layers)
if by_name:
# TODO ! 测试这里出错
saving.load_weights_from_hdf5_group_by_name(model_file, layers)
else:
saving.load_weights_from_hdf5_group(model_file, layers)
if hasattr(model_file, 'close'):
model_file.close()
pass
def load_weights(keras_model, filepath, by_name=True, exclude=None):
"""Modified version of the corresponding Keras function with
the addition of multi-GPU support and the ability to exclude
some layers from loading.
exclude: list of layer names to exclude
"""
import h5py
# Conditional import to support versions of Keras before 2.2
# TODO: remove in about 6 months (end of 2018)
try:
from keras.engine import saving
except ImportError:
# Keras before 2.2 used the 'topology' namespace.
from keras.engine import topology as saving
if exclude:
by_name = True
if h5py is None:
raise ImportError('`load_weights` requires h5py.')
f = h5py.File(filepath, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
# In multi-GPU training, we wrap the model. Get layers
# of the inner model because they have the weights.
#keras_model = self.keras_model
layers = keras_model.layers
#layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model")\
# else keras_model.layers
# Exclude some layers
if exclude:
layers = filter(lambda l: l.name not in exclude, layers)
if by_name:
saving.load_weights_from_hdf5_group_by_name(f,
layers,
skip_mismatch=True)
else:
saving.load_weights_from_hdf5_group(f, layers)
if hasattr(f, 'close'):
f.close()
def load_weights(self, filepath, by_name=False, exclude=None):
"""Modified version of the correspoding Keras function with
the addition of multi-GPU support and the ability to exclude
some layers from loading.
exlude: list of layer names to excluce
"""
import h5py
# Keras 2.2 use saving
#try:
# from keras.engine import saving
#except ImportError:
# # Keras before 2.2 used the 'topology' namespace.
# from keras.engine import topology as saving"""
if exclude:
by_name = True
if h5py is None:
raise ImportError('`load_weights` requires h5py.')
f = h5py.File(filepath, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
# In multi-GPU training, we wrap the model. Get layers
# of the inner model because they have the weights.
keras_model = self.keras_model
layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model")\
else keras_model.layers
# Exclude some layers
if exclude:
layers = filter(lambda l: l.name not in exclude, layers)
if by_name:
saving.load_weights_from_hdf5_group_by_name(f, layers)
else:
saving.load_weights_from_hdf5_group(f, layers)
if hasattr(f, 'close'):
f.close()
# Update the log directory
self.set_log_dir(filepath)
def load_weights(file_path, model, by_name=False, exclude=None):
import h5py
from keras.engine import saving
if exclude:
by_name = True
if h5py is None:
raise ImportError("requires h5py.")
f = h5py.File(file_path, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
layers = model.layers
if exclude:
layers = filter(lambda l: l.name not in exclude, layers)
if by_name:
saving.load_weights_from_hdf5_group_by_name(f, layers)
else:
saving.load_weights_from_hdf5_group(f, layers)
if hasattr(f, 'close'):
f.close()
def load_weights(self, filepath, by_name=False, exclude=None):
"""Modified version of the corresponding Keras function with
the addition of multi-GPU support and the ability to exclude
some layers from loading.
exclude: list of layer names to exclude
"""
import h5py
# Conditional import to support versions of Keras before 2.2
# TODO: remove in about 6 months (end of 2018)
try:
from keras.engine import saving
except ImportError:
# Keras before 2.2 used the 'topology' namespace.
from keras.engine import topology as saving
if exclude:
by_name = True
if h5py is None:
raise ImportError('`load_weights` requires h5py.')
f = h5py.File(filepath, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
# In multi-GPU training, we wrap the model. Get layers
# of the inner model because they have the weights.
keras_model = self.keras_model
layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model")\
else keras_model.layers
# Pre-defined layer regular expressions
layer_regex = {
# All layers
"all":
".*",
"mask":
r"(conv\_.*)|(mask\_.*)",
# all layers but the backbone
"heads":
r"(rpn\_.*)|(fpn\_.*)|(mask\_.*)",
# From a specific Resnet stage and up
"2+":
r"(conv2.*)|(bn_conv2.*)|(conv3.*)|(bn_conv3.*)|(conv4.*)|(bn_conv4.*)|(conv5.*)|(bn_conv5.*)|(conv\_.*)|(block2.*)|(bn_block2.*)|(res2.*)|(bn2.*)|(block3.*)|(bn_block3.*)|(res3.*)|(bn3.*)|(block4.*)|(bn_block4.*)|(res4.*)|(bn4.*)|(block5.*)|(bn_block5.*)|(res5.*)|(bn5.*)|(dw\_.*)|(rpn\_.*)|(fpn\_.*)|(mask\_.*)",
"3+":
r"(conv3.*)|(bn_conv3.*)|(conv4.*)|(bn_conv4.*)|(conv5.*)|(bn_conv5.*)|(conv\_.*)|(block3.*)|(bn_block3.*)|(res3.*)|(bn3.*)|(block4.*)|(bn_block4.*)|(res4.*)|(bn4.*)|(block5.*)|(bn_block5.*)|(res5.*)|(bn5.*)|(dw\_.*)|(rpn\_.*)|(fpn\_.*)|(mask\_.*)",
"4+":
r"(conv4.*)|(bn_conv4.*)|(conv5.*)|(bn_conv5.*)|(conv\_.*)|(block4.*)|(bn_block4.*)|(res4.*)|(bn4.*)|(block5.*)|(bn_block5.*)|(res5.*)|(bn5.*)|(dw\_.*)|(rpn\_.*)|(fpn\_.*)|(mask\_.*)",
"5+":
r"(conv5.*)|(bn_conv5.*)|(conv\_.*)|(res5.*)|(bn5.*)|(dw\_.*)|(rpn\_.*)|(fpn\_.*)|(mask\_.*)",
}
# Exclude some layers
layers_show = layers
if exclude:
layers_show = filter(
lambda l: not bool(re.fullmatch(layer_regex[exclude], l.name)),
layers_show)
layers = filter(
lambda l: not bool(re.fullmatch(layer_regex[exclude], l.name)),
layers)
indent = 0
print("LAYERS LOADED: ")
for layer in layers_show:
# Is the layer a model?
if layer.__class__.__name__ == 'Model':
print("In model: ", layer.name)
indent = indent + 4
continue
if not layer.weights:
continue
log("{}{:20} ({})".format(" " * indent, layer.name,
layer.__class__.__name__))
if by_name:
saving.load_weights_from_hdf5_group_by_name(f, layers)
else:
saving.load_weights_from_hdf5_group(f, layers)
if hasattr(f, 'close'):
f.close()
# Update the log directory
self.set_log_dir(filepath)
f = h5py.File(filepath, mode='r')
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
# In multi-GPU training, we wrap the model. Get layers
# of the inner model because they have the weights.
keras_model = self.keras_model
layers = keras_model.inner_model.layers if hasattr(keras_model, "inner_model")\
else keras_model.layers
# Exclude some layers
if exclude:
layers = filter(lambda l: l.name not in exclude, layers)
if by_name:
saving.load_weights_from_hdf5_group_by_name(f, layers)
else:
saving.load_weights_from_hdf5_group(f, layers)
if hasattr(f, 'close'):
f.close()
# Update the log directory
self.set_log_dir(filepath)
def get_imagenet_weights(self):
"""Downloads ImageNet trained weights from Keras.
Returns path to weights file.
"""
from keras.utils.data_utils import get_file
TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/'\
'releases/download/v0.2/'\
def load_pretrained_weights(self, weights_path):
f = h5py.File(self.weights_path, 'r')
load_weights_from_hdf5_group_by_name(f,
self.layers,
skip_mismatch=True)
