def test_load_keras(self):
model = KSequential()
model.add(KLayer.Dense(32, activation='relu', input_dim=100))
tmp_path_json = create_tmp_path() + ".json"
model_json = model.to_json()
with open(tmp_path_json, "w") as json_file:
json_file.write(model_json)
reloaded_json_model = Net.load_keras(json_path=tmp_path_json)
tmp_path_hdf5 = create_tmp_path() + ".h5"
model.save(tmp_path_hdf5)
reloaded_hdf5_model = Net.load_keras(hdf5_path=tmp_path_hdf5)
def test_load_tf_from_folder(self):
resource_path = os.path.join(
os.path.split(__file__)[0], "../../../resources")
tfnet_path = os.path.join(resource_path, "tf")
net = Net.load_tf(tfnet_path)
output = net.forward(np.random.rand(4, 1, 28, 28))
assert output.shape == (4, 10)
def test_layers_method(self):
resource_path = os.path.join(
os.path.split(__file__)[0], "../../../resources")
model_path = os.path.join(resource_path,
"models/bigdl/bigdl_lenet.model")
model = Net.load_bigdl(model_path)
assert len(model.layers) == 12
def load(self, checkpoint, optimizer=None, loss=None, feature_preprocessing=None,
label_preprocessing=None, model_dir=None, is_checkpoint=False):
if loss is not None:
self.loss = loss
if optimizer is not None:
self.optimizer = optimizer
if feature_preprocessing is not None:
self.feature_preprocessing = feature_preprocessing
if label_preprocessing is not None:
self.label_preprocessing = label_preprocessing
if model_dir is not None:
self.model_dir = model_dir
if is_checkpoint:
self.load_latest_orca_checkpoint(checkpoint)
else:
from zoo.pipeline.api.net import Net
self.model = Net.load_bigdl(checkpoint + ".bigdl", checkpoint + ".bin")
self.nn_estimator = NNEstimator(self.model, self.loss, self.feature_preprocessing,
self.label_preprocessing)
if self.optimizer is None:
from bigdl.optim.optimizer import SGD
self.optimizer = SGD()
self.nn_estimator.setOptimMethod(self.optimizer)
self.estimator = SparkEstimator(self.model, self.optimizer, self.model_dir)
self.nn_model = NNModel(self.model, feature_preprocessing=self.feature_preprocessing)
return self
def test_load_caffe_model(self):
resource_path = os.path.join(os.path.split(__file__)[0], "../../../resources")
model_path = os.path.join(resource_path, "models/caffe/test_persist.caffemodel")
def_path = os.path.join(resource_path, "models/caffe/test_persist.prototxt")
model = Net.load_caffe(def_path, model_path)
model2 = model.new_graph(["ip"])
model2.freeze_up_to(["conv2"])
model2.unfreeze()
def test_flatten_layers_method(self):
resource_path = os.path.join(
os.path.split(__file__)[0], "../../../resources")
model_path = os.path.join(resource_path,
"models/bigdl/bigdl_lenet.model")
model = Net.load_bigdl(model_path)
assert len(Sequential().add(model).flattened_layers()) == 12
def test_load(self):
input = ZLayer.Input(shape=(5, ))
output = ZLayer.Dense(10)(input)
zmodel = ZModel(input, output, name="graph1")
tmp_path = create_tmp_path()
zmodel.saveModel(tmp_path, None, True)
model_reloaded = Net.load(tmp_path)
input_data = np.random.random([3, 5])
self.compare_output_and_grad_input(zmodel, model_reloaded, input_data)
def test_load_bigdl_model(self):
resource_path = os.path.join(os.path.split(__file__)[0], "../../../resources")
model_path = os.path.join(resource_path, "models/bigdl/bigdl_lenet.model")
model = Net.load_bigdl(model_path)
model2 = model.new_graph(["reshape2"])
model2.freeze_up_to(["pool3"])
model2.unfreeze()
import numpy as np
data = np.zeros([1, 1, 28, 28])
output = model2.forward(data)
assert output.shape == (1, 192)
def test_save_load_Sequential(self):
zmodel = ZSequential()
dense = ZLayer.Dense(10, input_dim=5)
zmodel.add(dense)
tmp_path = create_tmp_path()
zmodel.saveModel(tmp_path, None, True)
model_reloaded = Net.load(tmp_path)
input_data = np.random.random([10, 5])
y = np.random.random([10, 10])
model_reloaded.compile(optimizer="adam", loss="mse")
model_reloaded.fit(x=input_data, y=y, batch_size=8, nb_epoch=1)
def test_save_load_Model(self):
input = ZLayer.Input(shape=(5, ))
output = ZLayer.Dense(10)(input)
zmodel = ZModel(input, output, name="graph1")
tmp_path = create_tmp_path()
zmodel.saveModel(tmp_path, None, True)
model_reloaded = Net.load(tmp_path)
input_data = np.random.random([10, 5])
y = np.random.random([10, 10])
model_reloaded.compile(optimizer="adam", loss="mse")
model_reloaded.fit(x=input_data, y=y, batch_size=8, nb_epoch=2)
def test_tf_load(self):
linear = Linear(10, 2)()
sigmoid = Sigmoid()(linear)
softmax = SoftMax().set_name("output")(sigmoid)
model = BModel(linear, softmax)
input = np.random.random((4, 10))
tmp_path = create_tmp_path() + "/model.pb"
model.save_tensorflow([("input", [4, 10])], tmp_path)
model_reloaded = Net.load_tf(tmp_path, ["input"], ["output"])
expected_output = model.forward(input)
output = model_reloaded.forward(input)
self.assert_allclose(output, expected_output)
def load(self,
checkpoint,
optimizer=None,
loss=None,
feature_preprocessing=None,
label_preprocessing=None,
model_dir=None,
is_checkpoint=False):
"""
Load existing BigDL model or checkpoint
:param checkpoint: Path to the existing model or checkpoint.
:param optimizer: BigDL optimizer.
:param loss: BigDL criterion.
:param feature_preprocessing: Used when data in `fit` and `predict` is a Spark DataFrame.
The param converts the data in feature column to a Tensor or to a Sample directly.
It expects a List of Int as the size of the converted Tensor, or a Preprocessing[F,
Tensor[T]]
If a List of Int is set as feature_preprocessing, it can only handle the case that
feature column contains the following data types:
Float, Double, Int, Array[Float], Array[Double], Array[Int] and MLlib Vector. The
feature data are converted to Tensors with the specified sizes before
sending to the model. Internally, a SeqToTensor is generated according to the
size, and used as the feature_preprocessing.
Alternatively, user can set feature_preprocessing as Preprocessing[F, Tensor[T]]
that transforms the feature data to a Tensor[T]. Some pre-defined Preprocessing are
provided in package zoo.feature. Multiple Preprocessing can be combined as a
ChainedPreprocessing.
The feature_preprocessing will also be copied to the generated NNModel and applied
to feature column during transform.
:param label_preprocessing: Used when data in `fit` and `predict` is a Spark DataFrame.
similar to feature_preprocessing, but applies to Label data.
:param model_dir: The path to save model. During the training, if checkpoint_trigger is
defined and triggered, the model will be saved to model_dir.
:param is_checkpoint: Whether the path is a checkpoint or a saved BigDL model.
Default: False.
:return: The loaded estimator object.
"""
if loss is not None:
self.loss = loss
if optimizer is not None:
self.optimizer = optimizer
if feature_preprocessing is not None:
self.feature_preprocessing = feature_preprocessing
if label_preprocessing is not None:
self.label_preprocessing = label_preprocessing
if model_dir is not None:
self.model_dir = model_dir
if is_checkpoint:
self.load_latest_orca_checkpoint(checkpoint)
else:
from zoo.pipeline.api.net import Net
self.model = Net.load_bigdl(checkpoint + ".bigdl",
checkpoint + ".bin")
self.nn_estimator = NNEstimator(self.model, self.loss,
self.feature_preprocessing,
self.label_preprocessing)
if self.optimizer is None:
from bigdl.optim.optimizer import SGD
self.optimizer = SGD()
self.nn_estimator.setOptimMethod(self.optimizer)
self.estimator = SparkEstimator(self.model, self.optimizer,
self.model_dir)
self.nn_model = NNModel(
self.model, feature_preprocessing=self.feature_preprocessing)
return self
def load(self,
checkpoint,
optimizer=None,
loss=None,
feature_preprocessing=None,
label_preprocessing=None,
model_dir=None,
is_checkpoint=False):
if loss is not None:
self.loss = loss
if optimizer is not None:
self.optimizer = optimizer
if feature_preprocessing is not None:
self.feature_preprocessing = feature_preprocessing
if label_preprocessing is not None:
self.label_preprocessing = label_preprocessing
if model_dir is not None:
self.model_dir = model_dir
if is_checkpoint:
from zoo.orca.learn.utils import find_latest_checkpoint
from zoo.pipeline.api.net import Net
from bigdl.nn.layer import Model, Container
from bigdl.optim.optimizer import OptimMethod
import os
path, prefix, version = find_latest_checkpoint(checkpoint,
model_type="bigdl")
if path is None:
raise ValueError(
"Cannot find BigDL checkpoint, please check your checkpoint path."
)
try:
self.model = Model.load(
os.path.join(path, "model.{}".format(version)))
assert isinstance(self.model, Container), \
"The loaded model should be a Container, please check your checkpoint type."
self.optimizer = OptimMethod.load(
os.path.join(path, "{}.{}".format(prefix, version)))
except Exception:
raise ValueError(
"Cannot load BigDL checkpoint, please check your checkpoint path "
"and checkpoint type.")
self.estimator = SparkEstimator(self.model, self.optimizer,
self.model_dir)
self.nn_estimator = NNEstimator(self.model, self.loss,
self.feature_preprocessing,
self.label_preprocessing)
if self.optimizer is not None:
self.nn_estimator.setOptimMethod(self.optimizer)
self.nn_model = NNModel(
self.model, feature_preprocessing=self.feature_preprocessing)
else:
from zoo.pipeline.api.net import Net
self.model = Net.load_bigdl(checkpoint + ".bigdl",
checkpoint + ".bin")
self.nn_estimator = NNEstimator(self.model, self.loss,
self.feature_preprocessing,
self.label_preprocessing)
if self.optimizer is None:
from bigdl.optim.optimizer import SGD
self.optimizer = SGD()
self.nn_estimator.setOptimMethod(self.optimizer)
self.estimator = SparkEstimator(self.model, self.optimizer,
self.model_dir)
self.nn_model = NNModel(
self.model, feature_preprocessing=self.feature_preprocessing)
return self
def load_model(model_path=MODEL_PATH, model_weights_path=MODEL_WEIGHTS_PATH):
model = Net.load(model_path, model_weights_path)
return model