def _load_keras(self, json_path, hdf5_path):
with open(json_path, "r") as jp:
kmodel = model_from_json(jp.read())
kmodel.load_weights_from_hdf5(hdf5_path)
bmodel = DefinitionLoader.from_json_path(json_path)
WeightLoader.load_weights_from_hdf5(bmodel, kmodel, hdf5_path)
return kmodel, bmodel
def test_load_definition(self):
K.set_image_dim_ordering("th")
kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist()
keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True)
bmodel = DefinitionLoader.from_json_path(keras_model_json_path)
WeightLoader.load_weights_from_kmodel(bmodel, kmodel)
self.assert_allclose(bmodel.forward(input_data), kmodel.predict(input_data))
def _load_keras(self, json_path, hdf5_path):
with open(json_path, "r") as jp:
kmodel = model_from_json(jp.read())
kmodel.load_weights_from_hdf5(hdf5_path)
bmodel = DefinitionLoader.from_json_path(json_path)
WeightLoader.load_weights_from_hdf5(bmodel, kmodel, hdf5_path)
return kmodel, bmodel
def modelTest(self,
input_data,
keras_model,
dump_weights=False,
is_training=False,
rtol=1e-7,
atol=1e-7):
# weight_converter is a function keras [ndarray]-> bigdl [ndarray]
keras_model_json_path, keras_model_hdf5_path = self._dump_keras(keras_model, dump_weights)
bigdl_model = DefinitionLoader.from_json_path(keras_model_json_path)
bigdl_model.training(is_training)
bigdl_output = bigdl_model.forward(input_data)
keras_output = keras_model.predict(input_data)
# TODO: we should verify bigdl_output and keras_output here
# init result is not the same, so we disable the verification for now
# self.assert_allclose(bigdl_output,
# keras_output,
# rtol=rtol,
# atol=atol)
if dump_weights: # load weights if possible
WeightLoader.load_weights_from_hdf5(bigdl_model, keras_model, keras_model_hdf5_path)
bweights = bigdl_model.get_weights()
bweights_from_keras = WeightsConverter.get_bigdl_weigths_from_keras(keras_model)
# bweights and bweights_from_keras are all list
assert isinstance(bweights, list)
assert len(bweights) == len(bweights_from_keras)
for i in range(len(bweights)):
self.assert_allclose(bweights[i], bweights_from_keras[i], rtol, atol)
bigdl_output2 = bigdl_model.forward(input_data)
self.assert_allclose(bigdl_output2,
keras_output,
rtol=rtol,
atol=atol)
def test_load_definition(self):
K.set_image_dim_ordering("th")
kmodel, input_data, output_data = TestModels.kmodel_seq_lenet_mnist()
keras_model_json_path, keras_model_hdf5_path = dump_keras(kmodel, dump_weights=True)
bmodel = DefinitionLoader.from_json_path(keras_model_json_path)
WeightLoader.load_weights_from_kmodel(bmodel, kmodel)
self.assert_allclose(bmodel.forward(input_data), kmodel.predict(input_data))
def modelTest(self,
input_data,
keras_model,
random_weights=True,
dump_weights=False,
is_training=False,
rtol=1e-6,
atol=1e-6):
if random_weights:
# Randomly generate weights instead of using initial weights
kweights = keras_model.get_weights()
new_kweights = self.__generate_random_weights(kweights)
keras_model.set_weights(new_kweights)
# weight_converter is a function keras [ndarray]-> bigdl [ndarray]
keras_model_json_path, keras_model_hdf5_path = self._dump_keras(
keras_model, dump_weights)
bigdl_model = DefinitionLoader.from_json_path(keras_model_json_path)
bigdl_model.training(is_training)
bigdl_output = bigdl_model.forward(input_data)
keras_output = keras_model.predict(input_data)
# TODO: we should verify bigdl_output and keras_output here
# init result is not the same, so we disable the verification for now
# self.assert_allclose(bigdl_output,
# keras_output,
# rtol=rtol,
# atol=atol)
if dump_weights: # load weights if possible
WeightLoader.load_weights_from_hdf5(bigdl_model, keras_model,
keras_model_hdf5_path)
bigdl_output2 = bigdl_model.forward(input_data)
self.assert_allclose(bigdl_output2, keras_output, rtol=rtol, atol=atol)
def test_load_weights(self):
K.set_image_dim_ordering("th")
kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer()
keras_model_json_path, keras_model_hdf5_path = self._dump_keras(kmodel, dump_weights=True)
bmodel = DefinitionLoader.from_json_path(keras_model_json_path)
kmodel.set_weights([kmodel.get_weights()[0] + 100, kmodel.get_weights()[1]])
WeightLoader.load_weights_from_hdf5(bmodel, kmodel, filepath=keras_model_hdf5_path)
self.assert_allclose(bmodel.forward(input_data), kmodel.predict(input_data))
def test_load_weights(self):
K.set_image_dim_ordering("th")
kmodel, input_data, output_data = TestModels.kmodel_graph_1_layer()
keras_model_json_path, keras_model_hdf5_path = dump_keras(kmodel, dump_weights=True)
bmodel = DefinitionLoader.from_json_path(keras_model_json_path)
kmodel.set_weights([kmodel.get_weights()[0] + 100, kmodel.get_weights()[1]])
WeightLoader.load_weights_from_hdf5(bmodel, kmodel, filepath=keras_model_hdf5_path)
self.assert_allclose(bmodel.forward(input_data), kmodel.predict(input_data))
def compare_model(self, zmodel, kmodel, input_data, rtol=1e-5, atol=1e-5):
"""
Compare forward results for Keras model against Zoo Keras API model.
"""
WeightLoader.load_weights_from_kmodel(zmodel, kmodel)
zmodel.training(is_training=False)
bigdl_output = zmodel.forward(input_data)
keras_output = kmodel.predict(input_data)
self.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol)
def compare_model(self, zmodel, kmodel, input_data, rtol=1e-5, atol=1e-5):
"""
Compare forward results for Keras model against Zoo Keras API model.
"""
WeightLoader.load_weights_from_kmodel(zmodel, kmodel)
zmodel.training(is_training=False)
bigdl_output = zmodel.forward(input_data)
keras_output = kmodel.predict(input_data)
self.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol)
def __kmodel_load_def_weight_test(self, kmodel, input_data):
keras_model_path_json, keras_model_path_hdf5 = self._dump_keras(
kmodel, dump_weights=True)
bmodel = DefinitionLoader.from_json_path(keras_model_path_json)
WeightLoader.load_weights_from_hdf5(bmodel, kmodel,
keras_model_path_hdf5)
bmodel.training(False)
boutput = bmodel.forward(input_data)
koutput = kmodel.predict(input_data)
assert_allclose(boutput, koutput, rtol=1e-5)
def __kmodel_load_def_weight_test(self, kmodel, input_data):
keras_model_path_json, keras_model_path_hdf5 = dump_keras(kmodel, dump_weights=True)
bmodel = DefinitionLoader.from_json_path(keras_model_path_json)
WeightLoader.load_weights_from_hdf5(bmodel,
kmodel,
keras_model_path_hdf5)
bmodel.training(False)
boutput = bmodel.forward(input_data)
koutput = kmodel.predict(input_data)
assert_allclose(boutput, koutput, rtol=1e-5)
def modelTest(self,
input_data,
keras_model,
random_weights=True,
dump_weights=False,
is_training=False,
rtol=1e-6,
atol=1e-6):
if random_weights:
# Randomly generate weights instead of using initial weights
kweights = keras_model.get_weights()
new_kweights = self.__generate_random_weights(kweights)
keras_model.set_weights(new_kweights)
# weight_converter is a function keras [ndarray]-> bigdl [ndarray]
keras_model_json_path, keras_model_hdf5_path = dump_keras(keras_model,
dump_weights=dump_weights)
# Use Theano backend to load as a bigdl model
self.__set_keras_backend("theano")
bigdl_model = DefinitionLoader.from_json_path(keras_model_json_path)
bigdl_model.training(is_training)
bigdl_output = bigdl_model.forward(input_data)
# Use TensorFlow backend to compare results
self.__set_keras_backend("tensorflow")
keras_output = keras_model.predict(input_data)
# TODO: we should verify bigdl_output and keras_output here
# init result is not the same, so we disable the verification for now
# self.assert_allclose(bigdl_output,
# keras_output,
# rtol=rtol,
# atol=atol)
if dump_weights: # load weights if possible
WeightLoader.load_weights_from_hdf5(bigdl_model, keras_model, keras_model_hdf5_path)
bigdl_output2 = bigdl_model.forward(input_data)
self.assert_allclose(bigdl_output2,
keras_output,
rtol=rtol,
atol=atol)
def compare_model(self, bmodel, kmodel, input_data, rtol=1e-5, atol=1e-5):
WeightLoader.load_weights_from_kmodel(bmodel, kmodel)
bmodel.training(is_training=False)
bigdl_output = bmodel.forward(input_data)
keras_output = kmodel.predict(input_data)
self.assert_allclose(bigdl_output, keras_output, rtol=rtol, atol=atol)
