def test_H5Dict_attrs():
_, h5_path = tempfile.mkstemp('.h5')
# test both HDF5 and dict implementations
paths = [h5_path, dict()]
for path in paths:
f = H5Dict(path, mode='w')
# str
f['x'] = 'abcd'
f['x2'] = u'abcd'
# list<bytes>
f['y'] = [b'efg', b'hij', b'klmn']
f['y2'] = (b'asd', b'sdf', b'dfg')
# ndarray
array = np.random.random((4, 5, 512))
f['z'] = array
f.close()
del f
f = H5Dict(path, mode='r')
assert f['x'] == 'abcd'
assert f['x2'] == u'abcd'
assert f['y'] == [b'efg', b'hij', b'klmn']
assert list(f['y2']) == [b'asd', b'sdf', b'dfg']
assert_allclose(f['z'], array)
f.close()
os.remove(h5_path)
def test_H5Dict_accepts_pathlib_Path():
"""GitHub issue: 11459"""
_, h5_path = tempfile.mkstemp('.h5')
f = H5Dict(Path(h5_path), mode='w')
f['x'] = 'abcd'
f.close()
del f
f = H5Dict(Path(h5_path), mode='r')
assert f['x'] == 'abcd'
f.close()
os.remove(h5_path)
def test_H5Dict_groups():
_, h5_path = tempfile.mkstemp('.h5')
# test both HDF5 and dict implementations
paths = [h5_path, dict()]
for path in paths:
f = H5Dict(path, mode='w')
group1 = f['group1']
group2 = group1['group2']
group2['x'] = 'abcd'
group3 = group2['group3']
group3['y'] = [b'efg', b'hij', b'klmn']
group4 = group3['group4']
array = np.random.random((4, 5, 512))
group4['z'] = array
f.close()
f = H5Dict(path, mode='r')
assert 'group1' in f
group1 = f['group1']
assert 'group2' in group1
group2 = group1['group2']
assert group2['x'] == 'abcd'
assert 'group3' in group2
group3 = group2['group3']
assert group3['y'] == [b'efg', b'hij', b'klmn']
assert 'group4' in group3
group4 = group3['group4']
assert_allclose(group4['z'], array)
f.close()
os.remove(h5_path)
def probe_model_shape(path):
h5dict = H5Dict(path, mode='r')
layers = json.loads(h5dict['model_config'])['config']['layers']
input_layer = next(l for l in layers if l['name'] == 'input')
B, T, F = input_layer['config']['batch_input_shape']
mask_layer = next(l for l in layers if l['name'] == 'mask_reshape')
T_, F_, C = mask_layer['config']['target_shape']
embedding_layer = next(l for l in layers
if l['name'] == 'embedding_reshape')
T__, F__, D = embedding_layer['config']['target_shape']
h5dict.close()
return T, F, C, D
def load_model(filepath, custom_objects=None, compile=True, **kwargs):
"""Loads a model saved via `save_model`.
This method can be used to resume a training as the optimization states are also restored.
Example: Restore a PIB model for training resume. Note that for PIB, always explicitly specify
`target_tensors` in `load_model` as the output of PIB is not the same shape as its input.
from keras.layers import Input
from pib.generic_utils import parse_best_keras_checkpoint
from pib.saving import load_model
from pib.layers import BinaryStochasticDense
from pib.losses import weighted_vcr, _update_var
from pib.metrics import stochastic_categorical_error
pib_model_path = parse_best_keras_checkpoint(checkpoint_path, mode='min', ckpt_type='model')
custom_objects = {
'BinaryStochasticDense':BinaryStochasticDense,
'stochastic_categorical_error':stochastic_categorical_error,
'weighted_vcr': weighted_vcr}
y_true = Input(shape=(config.layer_sizes[-1],), dtype='float32', name='targets')
restored_model = load_model(pib_model_path, custom_objects=custom_objects, compile=True, target_tensors=[y_true])
# Arguments
filepath: one of the following:
- string, path to the saved model, or
- h5py.File or h5py.Group object from which to load the model
custom_objects: Optional dictionary mapping names
(strings) to custom classes or functions to be
considered during deserialization.
compile: Boolean, whether to compile the model
after loading.
kwargs: Custom arguments. __allowed_kwargs = ['target_tensors', 'config_modifier', 'loss', 'metrics', 'sample_weight_mode', 'loss_weights' ]
target_tensors: Tensor. It is critical to explicitly provide `target_tensors` for a stochastic model.
config_modifier: A function instance that modifies the model config.
loss:
metrics:
sample_weight_mode:
loss_weights:
# Returns
A Keras model instance. If an optimizer was found
as part of the saved model, the model is already
compiled. Otherwise, the model is uncompiled and
a warning will be displayed. When `compile` is set
to False, the compilation is omitted without any
warning.
# Raises
ImportError: if h5py is not available.
ValueError: In case of an invalid savefile.
"""
if h5py is None:
raise ImportError('`load_model` requires h5py.')
model = None
opened_new_file = not isinstance(filepath, h5py.Group)
h5dict = H5Dict(filepath, 'r')
try:
model = _deserialize_model(h5dict, custom_objects, compile, **kwargs)
finally:
if opened_new_file:
h5dict.close()
return model
def build_internal_rep(inputFile):
h5dict = H5Dict(inputFile, mode='r')
model_config = get_json_key(h5dict, 'model_config')
# Dlib requires an add_loss_layer at the end of the network (I think)
training_config = get_json_key(h5dict, 'training_config')
model_layers = model_config['config']['layers']
optimizer_config = training_config['optimizer_config']
loss = training_config['loss']
# This list stores all the Dlib_Layer objects used
# This prevents problems with python shallow copying if you would pass the
# layer objects themselves while building the subnetworks
layers = []
for layer in model_config['config']['layers']:
layerClass = layer['class_name']
print('-- Converting Keras.layers.{}'.format(layerClass))
layer = layer['config'] # save some verbosity
# Create additional dlib::input<> for the first Keras layer (input layer)
isInputLayer = 'batch_input_shape' in layer
if isInputLayer:
if len(layers) > 0:
raise SerializationError(
'Unexpected input layer at index {}'.format(len(layers)))
elif len(layers) == 0:
raise SerializationError('Expected an input layer as first layer')
# If we leave out the input matrix size calculation (which is probably a good idea)
# we can refactor adding the input layer out to here, instead of duplicating it
# to every layer which could serve as the first layer
if isInputLayer:
layers.append(Dlib_matrix_input(layer['dtype']))
########################################################################
# keras.layers.Dense
########################################################################
if layerClass == 'Dense':
num_outputs = layer['units']
activation = layer['activation']
use_bias = layer['use_bias']
params = load_layer_params_Dense(h5dict, layer['name'], use_bias)
num_inputs = params.shape[0]
if use_bias: # subtract the additional space biases occupy in layer parameters
num_inputs -= 1
layers.append(
Dlib_add_layer(
Dlib_fc(num_inputs, num_outputs, params, use_bias),
layers[-1], isInputLayer))
if activation != 'linear':
layers.append(
Dlib_add_layer(Dlib_activation(activation), layers[-1]))
########################################################################
# keras.layers.Conv2d
########################################################################
elif layerClass == 'Conv2D':
# "data_format": "channels_last"
# "dilation_rate": [1, 1]
num_filters = layer['filters']
nr, nc = layer['kernel_size']
stride_y, stride_x = layer['strides']
padding = layer['padding']
activation = layer['activation']
use_bias = layer['use_bias']
# TODO: Figure out how to properly handle valid/same padding to dlib
# until then we use the same as the default template parameters
padding_y = 0 if stride_y != 1 else nr // 2
padding_x = 0 if stride_x != 1 else nc // 2
if padding == 'valid':
pass
elif padding == 'same':
pass
else:
raise SerializationError(
'Unsupported layer padding: {}'.format(padding))
params = load_layer_params_Conv2D(h5dict, layer['name'])
layers.append(
Dlib_add_layer(
Dlib_con(num_filters, nr, nc, stride_y, stride_x,
padding_y, padding_x, params), layers[-1],
isInputLayer))
if activation != 'linear':
layers.append(
Dlib_add_layer(Dlib_activation(activation), layers[-1]))
########################################################################
# keras.layers.MaxPooling2D
########################################################################
elif layerClass == 'MaxPooling2D':
nr, nc = layer['pool_size']
stride_y, stride_x = layer['strides']
padding = layer['padding']
# TODO: Figure out how to properly handle valid/same padding to dlib
# until then we use the same as the default template parameters
padding_y = 0 if stride_y != 1 else nr // 2
padding_x = 0 if stride_x != 1 else nc // 2
if padding == 'valid':
pass
elif padding == 'same':
pass
else:
raise SerializationError(
'Unsupported layer padding: {}'.format(padding))
layers.append(
Dlib_add_layer(
Dlib_max_pool(nr, nc, stride_y, stride_x, padding_y,
padding_x), layers[-1]))
########################################################################
# keras.layers.Flatten
########################################################################
elif layerClass == 'Flatten':
# For now we ignore Flatten() Layers, I think dlib works without them
pass
########################################################################
# keras.layers.Dropout
########################################################################
elif layerClass == 'Dropout':
layers.append(
Dlib_add_layer(Dlib_dropout(layer['rate']), layers[-1],
isInputLayer))
########################################################################
# keras.layers.LeakyReLU
########################################################################
elif layerClass == 'LeakyReLU':
layers.append(
Dlib_add_layer(Dlib_prelu(layer['alpha']), layers[-1]))
########################################################################
# Other layers
########################################################################
else:
raise SerializationError(
'Unsupported layer class {}'.format(layerClass))
layers.append(Dlib_add_loss_layer(Dlib_loss(loss), layers[-1]))
return layers