def test_pipeline(self):
""" Pipeline should provide correct function composition """
img_fpaths = glob(os.path.join(_getSampleJPEGDir(), '*.jpg'))
xcpt_model = Xception(weights="imagenet")
stages = [('spimage', gfac.buildSpImageConverter(SparkMode.RGB_FLOAT32)),
('xception', GraphFunction.fromKeras(xcpt_model))]
piped_model = GraphFunction.fromList(stages)
for fpath in img_fpaths:
target_size = tuple(xcpt_model.input.shape.as_list()[1:-1])
img = load_img(fpath, target_size=target_size)
img_arr = np.expand_dims(img_to_array(img), axis=0)
img_input = xcpt.preprocess_input(img_arr)
preds_ref = xcpt_model.predict(img_input)
spimg_input_dict = imageArrayToStruct(img_input).asDict()
spimg_input_dict['data'] = bytes(spimg_input_dict['data'])
with IsolatedSession() as issn:
# Need blank import scope name so that spimg fields match the input names
feeds, fetches = issn.importGraphFunction(piped_model, prefix="")
feed_dict = dict((tnsr, spimg_input_dict[tfx.op_name(issn.graph, tnsr)]) for tnsr in feeds)
preds_tgt = issn.run(fetches[0], feed_dict=feed_dict)
# Uncomment the line below to see the graph
# tfx.write_visualization_html(issn.graph,
# NamedTemporaryFile(prefix="gdef", suffix=".html").name)
self.assertTrue(np.all(preds_tgt == preds_ref))
def test_pipeline(self):
""" Pipeline should provide correct function composition """
img_fpaths = glob(os.path.join(_getSampleJPEGDir(), '*.jpg'))
xcpt_model = Xception(weights="imagenet")
stages = [('spimage',
gfac.buildSpImageConverter(SparkMode.RGB_FLOAT32)),
('xception', GraphFunction.fromKeras(xcpt_model))]
piped_model = GraphFunction.fromList(stages)
for fpath in img_fpaths:
target_size = tuple(xcpt_model.input.shape.as_list()[1:-1])
img = load_img(fpath, target_size=target_size)
img_arr = np.expand_dims(img_to_array(img), axis=0)
img_input = xcpt.preprocess_input(img_arr)
preds_ref = xcpt_model.predict(img_input)
spimg_input_dict = imageArrayToStruct(img_input).asDict()
spimg_input_dict['data'] = bytes(spimg_input_dict['data'])
with IsolatedSession() as issn:
# Need blank import scope name so that spimg fields match the input names
feeds, fetches = issn.importGraphFunction(piped_model,
prefix="")
feed_dict = dict(
(tnsr, spimg_input_dict[tfx.op_name(tnsr, issn.graph)])
for tnsr in feeds)
preds_tgt = issn.run(fetches[0], feed_dict=feed_dict)
# Uncomment the line below to see the graph
# tfx.write_visualization_html(issn.graph,
# NamedTemporaryFile(prefix="gdef", suffix=".html").name)
self.assertTrue(np.all(preds_tgt == preds_ref))
def test_bare_keras_module(self):
""" Keras GraphFunctions should give the same result as standard Keras models """
img_fpaths = glob(os.path.join(_getSampleJPEGDir(), '*.jpg'))
for model_gen, preproc_fn in [(InceptionV3, iv3.preprocess_input),
(Xception, xcpt.preprocess_input),
(ResNet50, rsnt.preprocess_input)]:
keras_model = model_gen(weights="imagenet")
target_size = tuple(keras_model.input.shape.as_list()[1:-1])
_preproc_img_list = []
for fpath in img_fpaths:
img = load_img(fpath, target_size=target_size)
# WARNING: must apply expand dimensions first, or ResNet50 preprocessor fails
img_arr = np.expand_dims(img_to_array(img), axis=0)
_preproc_img_list.append(preproc_fn(img_arr))
imgs_input = np.vstack(_preproc_img_list)
preds_ref = keras_model.predict(imgs_input)
gfn_bare_keras = GraphFunction.fromKeras(keras_model)
with IsolatedSession(using_keras=True) as issn:
K.set_learning_phase(0)
feeds, fetches = issn.importGraphFunction(gfn_bare_keras)
preds_tgt = issn.run(fetches[0], {feeds[0]: imgs_input})
self.assertTrue(np.all(preds_tgt == preds_ref))
def test_bare_keras_module(self):
""" Keras GraphFunctions should give the same result as standard Keras models """
img_fpaths = glob(os.path.join(_getSampleJPEGDir(), '*.jpg'))
for model_gen, preproc_fn in [(InceptionV3, iv3.preprocess_input),
(Xception, xcpt.preprocess_input),
(ResNet50, rsnt.preprocess_input)]:
keras_model = model_gen(weights="imagenet")
target_size = tuple(keras_model.input.shape.as_list()[1:-1])
_preproc_img_list = []
for fpath in img_fpaths:
img = load_img(fpath, target_size=target_size)
# WARNING: must apply expand dimensions first, or ResNet50 preprocessor fails
img_arr = np.expand_dims(img_to_array(img), axis=0)
_preproc_img_list.append(preproc_fn(img_arr))
imgs_input = np.vstack(_preproc_img_list)
preds_ref = keras_model.predict(imgs_input)
gfn_bare_keras = GraphFunction.fromKeras(keras_model)
with IsolatedSession(using_keras=True) as issn:
K.set_learning_phase(0)
feeds, fetches = issn.importGraphFunction(gfn_bare_keras)
preds_tgt = issn.run(fetches[0], {feeds[0]: imgs_input})
self.assertTrue(np.all(preds_tgt == preds_ref))
def registerKerasImageUDF(udf_name,
keras_model_or_file_path,
preprocessor=None):
"""
Create a Keras image model as a Spark SQL UDF.
The UDF takes a column (formatted in :py:const:`sparkdl.image.imageIO.imageSchema`)
and produces the output of the given Keras model (e.g.
for `Inception V3 <https://keras.io/applications/#inceptionv3]>`_
it produces a real valued score vector over the ImageNet object categories).
For other models, the output could have different meanings.
Please consult the actual models specification.
The user can provide an existing model in Keras as follows.
.. code-block:: python
from keras.applications import InceptionV3
registerKerasImageUDF("udf_name", InceptionV3(weights="imagenet"))
To use a customized Keras model, we can save it and pass the file path as parameter.
.. code-block:: python
# Assume we have a compiled and trained Keras model
model.save('path/to/my/model.h5')
registerKerasImageUDF("my_custom_keras_model_udf", "path/to/my/model.h5")
If there are further preprocessing steps are required to prepare the images,
the user has the option to provide a preprocessing function :py:obj:`preprocessor`.
The :py:obj:`preprocessor` converts a file path into a image array.
This function is usually introduced in Keras workflow, as in the following example.
.. warning:: There is a performance penalty to use a :py:obj:`preprocessor` as it will
first convert the image into a file buffer and reloaded back.
This provides compatibility with the usual way Keras model input are preprocessed.
Please consider directly using Keras/TensorFlow layers for this purpose.
.. code-block:: python
def keras_load_img(fpath):
from keras.preprocessing.image import load_img, img_to_array
import numpy as np
from pyspark.sql import Row
img = load_img(fpath, target_size=(299, 299))
return img_to_array(img).astype(np.uint8)
registerKerasImageUDF("my_inception_udf", InceptionV3(weights="imagenet"), keras_load_img)
If the `preprocessor` is not provided, we assume the function will be applied to
a (struct) column encoded in [sparkdl.image.imageIO.imageSchema].
The output will be a single (struct) column containing the resulting tensor data.
:param udf_name: str, name of the UserDefinedFunction. If the name exists, it will be
overwritten.
:param keras_model_or_file_path: str or KerasModel,
either a path to the HDF5 Keras model file
or an actual loaded Keras model
:param preprocessor: function, optional, a function that
converts image file path to image tensor/ndarray
in the correct shape to be served as input to the Keras model
:return: :py:class:`GraphFunction`, the graph function for the Keras image model
"""
warnings.warn(
"registerKerasImageUDF() will be removed in the next release of sparkdl. "
"Please use Pandas UDF for distributed model inference.",
DeprecationWarning)
ordered_udf_names = []
keras_udf_name = udf_name
if preprocessor is not None:
# Spill the image structure to file and reload it
# with the user provided preprocessing funcition
preproc_udf_name = '{}__preprocess'.format(udf_name)
ordered_udf_names.append(preproc_udf_name)
JVMAPI.registerUDF(preproc_udf_name,
_serialize_and_reload_with(preprocessor),
ImageSchema.imageSchema['image'].dataType)
keras_udf_name = '{}__model_predict'.format(udf_name)
stages = [('spimg', buildSpImageConverter('RGB', "uint8")),
('model', GraphFunction.fromKeras(keras_model_or_file_path)),
('final', buildFlattener())]
gfn = GraphFunction.fromList(stages)
with IsolatedSession() as issn:
_, fetches = issn.importGraphFunction(gfn, prefix='')
makeGraphUDF(issn.graph, keras_udf_name, fetches)
ordered_udf_names.append(keras_udf_name)
if len(ordered_udf_names) > 1:
msg = "registering pipelined UDF {udf} with stages {udfs}"
msg = msg.format(udf=udf_name, udfs=ordered_udf_names)
logger.info(msg)
JVMAPI.registerPipeline(udf_name, ordered_udf_names)
return gfn
def registerKerasImageUDF(udf_name, keras_model_or_file_path, preprocessor=None):
"""
Create a Keras image model as a Spark SQL UDF.
The UDF takes a column (formatted in :py:const:`sparkdl.image.imageIO.imageSchema`)
and produces the output of the given Keras model (e.g.
for `Inception V3 <https://keras.io/applications/#inceptionv3]>`_
it produces a real valued score vector over the ImageNet object categories).
For other models, the output could have different meanings.
Please consult the actual models specification.
The user can provide an existing model in Keras as follows.
.. code-block:: python
from keras.applications import InceptionV3
registerKerasImageUDF("udf_name", InceptionV3(weights="imagenet"))
To use a customized Keras model, we can save it and pass the file path as parameter.
.. code-block:: python
# Assume we have a compiled and trained Keras model
model.save('path/to/my/model.h5')
registerKerasImageUDF("my_custom_keras_model_udf", "path/to/my/model.h5")
If there are further preprocessing steps are required to prepare the images,
the user has the option to provide a preprocessing function :py:obj:`preprocessor`.
The :py:obj:`preprocessor` converts a file path into a image array.
This function is usually introduced in Keras workflow, as in the following example.
.. warning:: There is a performance penalty to use a :py:obj:`preprocessor` as it will
first convert the image into a file buffer and reloaded back.
This provides compatibility with the usual way Keras model input are preprocessed.
Please consider directly using Keras/TensorFlow layers for this purpose.
.. code-block:: python
def keras_load_img(fpath):
from keras.preprocessing.image import load_img, img_to_array
import numpy as np
from pyspark.sql import Row
img = load_img(fpath, target_size=(299, 299))
return img_to_array(img).astype(np.uint8)
registerKerasImageUDF("my_inception_udf", InceptionV3(weights="imagenet"), keras_load_img)
If the `preprocessor` is not provided, we assume the function will be applied to
a (struct) column encoded in [sparkdl.image.imageIO.imageSchema].
The output will be a single (struct) column containing the resulting tensor data.
:param udf_name: str, name of the UserDefinedFunction. If the name exists, it will be
overwritten.
:param keras_model_or_file_path: str or KerasModel,
either a path to the HDF5 Keras model file
or an actual loaded Keras model
:param preprocessor: function, optional, a function that
converts image file path to image tensor/ndarray
in the correct shape to be served as input to the Keras model
:return: :py:class:`GraphFunction`, the graph function for the Keras image model
"""
ordered_udf_names = []
keras_udf_name = udf_name
if preprocessor is not None:
# Spill the image structure to file and reload it
# with the user provided preprocessing funcition
preproc_udf_name = '{}__preprocess'.format(udf_name)
ordered_udf_names.append(preproc_udf_name)
JVMAPI.registerUDF(
preproc_udf_name,
_serialize_and_reload_with(preprocessor),
ImageSchema.imageSchema['image'].dataType)
keras_udf_name = '{}__model_predict'.format(udf_name)
stages = [('spimg', buildSpImageConverter('RGB', "uint8")),
('model', GraphFunction.fromKeras(keras_model_or_file_path)),
('final', buildFlattener())]
gfn = GraphFunction.fromList(stages)
with IsolatedSession() as issn:
_, fetches = issn.importGraphFunction(gfn, prefix='')
makeGraphUDF(issn.graph, keras_udf_name, fetches)
ordered_udf_names.append(keras_udf_name)
if len(ordered_udf_names) > 1:
msg = "registering pipelined UDF {udf} with stages {udfs}"
msg = msg.format(udf=udf_name, udfs=ordered_udf_names)
logger.info(msg)
JVMAPI.registerPipeline(udf_name, ordered_udf_names)
return gfn