def test_dead_layer_partial_branch(self):
convergence_tolerance = 1e-8
input_features = [("input", datatypes.Array(*(2,)))]
output_features = [("out", None)]
builder = neural_network.NeuralNetworkBuilder(
input_features, output_features, disable_rank5_shape_mapping=True
)
# add condition to break from the loop, if convergence criterion is met
builder.add_less_than("cond", ["input"], "cond", alpha=convergence_tolerance)
branch_layer = builder.add_branch("branch_layer", "cond")
builder_ifbranch = neural_network.NeuralNetworkBuilder(
nn_spec=branch_layer.branch.ifBranch
)
builder_ifbranch.add_activation("relu1", "RELU", "input", "relu1_out")
builder_ifbranch.add_activation("relu2_out", "RELU", "relu1_out", "relu2_out")
builder_elsebranch = neural_network.NeuralNetworkBuilder(
nn_spec=branch_layer.branch.elseBranch
)
builder_elsebranch.add_activation("linear1", "LINEAR", "input", "linear1_out")
builder_elsebranch.add_activation(
"linear_red_1", "LINEAR", "input", "linear_red1_out"
)
builder_elsebranch.add_activation(
"linear_red_2", "LINEAR", "linear_red1_out", "linear_red2_out"
)
builder_elsebranch.add_activation(
"linear2", "LINEAR", "linear1_out", "relu2_out"
)
builder.add_squeeze("out", "relu2_out", "out", squeeze_all=True)
mlmodel = MLModel(builder.spec, compute_units=ComputeUnit.CPU_ONLY)
if not _IS_MACOS:
# Can not get predictions unless on macOS.
return
data = np.random.rand(2,)
data_dict = {"input": data}
before_pass_out = mlmodel.predict(data_dict)["out"]
if DEBUG:
print("\n mlmodel description before remove disconnected layers pass: \n")
print_network_spec(builder.spec, style="coding")
old_spec = copy.copy(builder.spec)
remove_disconnected_layers(builder.spec)
if DEBUG:
print("\n mlmodel description after remove disconnected layers pass: \n")
print_network_spec(builder.spec, style="coding")
mlmodel = MLModel(builder.spec, compute_units=ComputeUnit.CPU_ONLY)
after_pass_out = mlmodel.predict(data_dict)["out"]
np.testing.assert_almost_equal(before_pass_out, after_pass_out, decimal=2)
np.testing.assert_equal(
len(old_spec.neuralNetwork.layers[1].branch.ifBranch.layers),
len(builder.spec.neuralNetwork.layers[1].branch.ifBranch.layers),
)
np.testing.assert_equal(
len(builder.spec.neuralNetwork.layers[1].branch.elseBranch.layers), 2
)
def assert_model_is_valid(
program, inputs, backend="nn_proto", verbose=True, expected_output_shapes=None
):
"""
Assert Core ML model is valid.
Inputs:
- input: str -> shape tuple. All program input names need to appear in str.
shape tuple can only contain positive integers.
"""
input_dict = dict()
for name, shape in inputs.items():
input_dict[name] = np.random.rand(*shape)
# Avoid circular import
from coremltools.converters.mil.testing_reqs import ct
mlmodel = ct.convert(program, source="mil", convert_to=backend)
assert mlmodel is not None
if verbose:
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(mlmodel.get_spec(), style="coding")
if _IS_MACOS:
prediction = mlmodel.predict(input_dict, useCPUOnly=True)
assert prediction is not None
if expected_output_shapes is not None:
for out_name, out_shape in expected_output_shapes.items():
assert out_name in prediction
assert out_shape == prediction[out_name].shape
def test_dead_layer_remove_branch(self):
convergence_tolerance = 1e-8
input_features = [('input', datatypes.Array(*(2,)))]
output_features = [('out', None)]
builder = neural_network.NeuralNetworkBuilder(input_features, output_features, disable_rank5_shape_mapping=True)
# add condition to break from the loop, if convergence criterion is met
builder.add_less_than('cond', ['input'], 'cond', alpha=convergence_tolerance)
branch_layer = builder.add_branch('branch_layer', 'cond')
builder_ifbranch = neural_network.NeuralNetworkBuilder(nn_spec=branch_layer.branch.ifBranch)
builder_ifbranch.add_activation('relu1', 'RELU', 'input', 'relu1_out')
builder_ifbranch.add_activation('relu2_out', 'RELU', 'relu1_out', 'relu2_out')
builder_elsebranch = neural_network.NeuralNetworkBuilder(nn_spec=branch_layer.branch.elseBranch)
builder_elsebranch.add_activation('linear1', 'LINEAR', 'input', 'linear1_out')
builder_elsebranch.add_activation('linear2', 'LINEAR', 'linear1_out', 'relu2_out')
builder.add_squeeze('out', 'input', 'out', squeeze_all=True)
mlmodel = MLModel(builder.spec)
data = np.random.rand(2,)
data_dict = {'input': data}
before_pass_out = mlmodel.predict(data_dict)['out']
if DEBUG:
print('\n mlmodel description before remove disconnected layers pass: \n')
print_network_spec(builder.spec, style='coding')
remove_disconnected_layers(builder.spec)
if DEBUG:
print('\n mlmodel description after remove disconnected layers pass: \n')
print_network_spec(builder.spec, style='coding')
mlmodel = MLModel(builder.spec)
after_pass_out = mlmodel.predict(data_dict)['out']
np.testing.assert_almost_equal(before_pass_out, after_pass_out, decimal=4)
np.testing.assert_equal(len(builder.spec.neuralNetwork.layers), 1)
def _test_keras_model_tf2(self, model, data_mode, decimal, use_cpu_only, has_variables, verbose):
core_ml_model_file = self.model_file.rsplit('.')[0] + '.mlmodel'
input_dict = {inp.op.name: inp.shape.as_list() for inp in model.inputs}
for name, shape in input_dict.items():
input_dict[name] = [dim if dim is not None else 1 for dim in shape]
output_list = ['Identity']
model.save(self.model_file)
# convert Keras model into Core ML model format
core_ml_model = coremltools.converters.tensorflow.convert(
filename=self.model_file,
inputs=input_dict,
outputs=output_list,
use_cpu_only=use_cpu_only)
if verbose:
print('\nKeras model saved at {}'.format(self.model_file))
print('\nCore ML model description:')
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(core_ml_model.get_spec(), style='coding')
core_ml_model.save(core_ml_model_file)
print('\nCore ML model saved at {}'.format(core_ml_model_file))
core_ml_inputs = {
name: generate_data(shape, data_mode) for name, shape in input_dict.items()
}
# run prediction and compare results
keras_output = model.predict(list(core_ml_inputs.values())[0])
core_ml_output = core_ml_model.predict(
core_ml_inputs, useCPUOnly=use_cpu_only)[output_list[0]]
if verbose:
print('\nPredictions', keras_output.shape, ' vs.', core_ml_output.shape)
print(keras_output.flatten()[:6])
print(core_ml_output.flatten()[:6])
np.testing.assert_array_equal(
keras_output.shape, core_ml_output.shape)
np.testing.assert_almost_equal(
keras_output.flatten(), core_ml_output.flatten(), decimal=decimal)
def assert_model_is_valid(program,
inputs,
backend=("neuralnetwork", "fp32"),
verbose=True,
expected_output_shapes=None):
"""
Assert Core ML model is valid.
Inputs:
- input: str -> shape tuple. All program input names need to appear in str.
shape tuple can only contain positive integers.
"""
# Avoid circular import
from coremltools.converters.mil.testing_reqs import ct
input_dict = dict()
for name, shape in inputs.items():
input_dict[name] = np.random.rand(*shape)
mlmodel = ct_convert(program,
source="milinternal",
convert_to=backend,
compute_units=ct.ComputeUnit.CPU_ONLY)
assert mlmodel is not None
if verbose:
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(mlmodel.get_spec(), style="coding")
if _IS_MACOS and (not mlmodel.is_package
or coremltoolsutils._macos_version() >= (12, 0)):
prediction = mlmodel.predict(input_dict)
assert prediction is not None
if expected_output_shapes is not None:
for out_name, out_shape in expected_output_shapes.items():
assert out_name in prediction
assert out_shape == prediction[out_name].shape, \
"{} != {}".format(out_shape, prediction[out_name].shape)
def _test_keras_model(self,
model,
data_mode='random',
delta=1e-2,
use_cpu_only=True,
has_variables=True):
"""Saves out the backend TF graph from the Keras model and tests it
"""
model_dir = tempfile.mkdtemp()
graph_def_file = os.path.join(model_dir, 'tf_graph.pb')
checkpoint_file = os.path.join(model_dir, 'tf_model.ckpt')
frozen_model_file = os.path.join(model_dir, 'tf_frozen.pb')
coreml_model_file = os.path.join(model_dir, 'coreml_model.mlmodel')
input_shapes = {
inp.op.name: inp.shape.as_list()
for inp in model.inputs
}
for name, shape in input_shapes.items():
input_shapes[name] = [
dim if dim is not None else 1 for dim in shape
]
output_node_names = [output.op.name for output in model.outputs]
tf_graph = K.get_session().graph
tf.reset_default_graph()
if has_variables:
with tf_graph.as_default() as g:
saver = tf.train.Saver()
# TODO - if Keras backend has_variable is False, we're not making variables constant
with tf.Session(graph=tf_graph) as sess:
sess.run(tf.global_variables_initializer())
feed_dict = {}
for name, shape in input_shapes.items():
tensor_name = tf_graph.get_operation_by_name(
name).outputs[0].name
feed_dict[tensor_name] = generate_data(shape, data_mode)
# run the result
fetches = [
tf_graph.get_operation_by_name(name).outputs[0]
for name in output_node_names
]
result = sess.run(fetches, feed_dict=feed_dict)
# save graph definition somewhere
tf.train.write_graph(sess.graph,
model_dir,
graph_def_file,
as_text=False)
# freeze_graph() has been raising error with tf.keras models since no
# later than TensorFlow 1.6, so we're not using freeze_graph() here.
# See: https://github.com/tensorflow/models/issues/5387
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, # The session is used to retrieve the weights
tf_graph.as_graph_def(
), # The graph_def is used to retrieve the nodes
output_node_names # The output node names are used to select the useful nodes
)
with tf.gfile.GFile(frozen_model_file, "wb") as f:
f.write(output_graph_def.SerializeToString())
K.clear_session()
# convert to CoreML
mlmodel = coremltools.converters.tensorflow.convert(
frozen_model_file,
inputs=input_shapes,
outputs=output_node_names,
use_cpu_only=use_cpu_only)
if DEBUG:
print('\n mlmodel description: \n')
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(mlmodel.get_spec(), style='coding')
mlmodel.save(coreml_model_file)
print('\n mlmodel saved at %s' % (coreml_model_file))
# Transpose input data as CoreML requires
coreml_inputs = {
name:
tf_transpose(feed_dict[self._get_tf_tensor_name(tf_graph, name)])
for name in input_shapes
}
# Run predict in CoreML
coreml_output = mlmodel.predict(coreml_inputs, useCPUOnly=use_cpu_only)
for idx, out_name in enumerate(output_node_names):
tf_out = result[idx]
if len(tf_out.shape) == 0:
tf_out = np.array([tf_out])
tp = tf_out.flatten()
coreml_out = coreml_output[out_name]
cp = coreml_out.flatten()
self.assertTrue(tf_out.shape == coreml_out.shape)
for i in range(len(tp)):
max_den = max(1.0, tp[i], cp[i])
self.assertAlmostEqual(tp[i] / max_den,
cp[i] / max_den,
delta=delta)
# Cleanup files - models on disk no longer useful
if os.path.exists(model_dir):
shutil.rmtree(model_dir)
def _test_tf_model(self,
graph,
input_shapes,
output_node_names,
data_mode='random',
input_refs=None,
delta=1e-2,
use_cpu_only=False,
use_freeze=True,
quantize_tf_model=False):
"""
Common entry to testing routine.
graph - defined TensorFlow graph.
input_shapes - dict str:shape for each input op (placeholder)
output_node_names - output_node_names, a list of strings
data_mode - auto-generated input vectors, can be 'random', 'zeros', 'ones', 'linear', etc.
input_refs - a dictionary of reference input in tensorFlow axis order, each entry is str:shape.
When using auto-generated input vectors, set input_refs to None.
delta - maximum difference of normalized TensorFlow and CoreML outputs
use_cpu_only - If True, instantiate and run CoreML model with CPU only
use_freeze - If True, force TensorFlow graph to be frozen before converting.
quantize_tf_model - If True, try to quantize TensorFlow model before converting
"""
# Some file processing
model_dir = tempfile.mkdtemp()
graph_def_file = os.path.join(model_dir, 'tf_graph.pbtxt')
checkpoint_file = os.path.join(model_dir, 'tf_model.ckpt')
static_model_file = os.path.join(model_dir, 'tf_static.pb')
coreml_model_file = os.path.join(model_dir, 'coreml_model.mlmodel')
# add a saver
tf.reset_default_graph()
if use_freeze:
with graph.as_default() as g:
saver = tf.train.Saver()
if input_refs is None:
feed_dict = {
self._get_tf_tensor_name(graph, name):
generate_data(input_shapes[name], data_mode)
for name in input_shapes
}
else:
feed_dict = {
self._get_tf_tensor_name(graph, name): input_refs[name]
for name in list(input_refs.keys())
}
with tf.Session(graph=graph) as sess:
# initialize
sess.run(tf.global_variables_initializer())
# run the result
fetches = [
graph.get_operation_by_name(name).outputs[0]
for name in output_node_names
]
result = sess.run(fetches, feed_dict=feed_dict)
# save graph definition somewhere
tf.train.write_graph(sess.graph,
model_dir,
graph_def_file,
as_text=False)
# save the weights if freezing is needed
if use_freeze:
saver.save(sess, checkpoint_file)
else:
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, graph.as_graph_def(), output_node_names)
with tf.gfile.GFile(static_model_file, "wb") as f:
f.write(output_graph_def.SerializeToString())
# freeze the graph
if use_freeze:
self._simple_freeze(input_graph=graph_def_file,
input_checkpoint=checkpoint_file,
output_graph=static_model_file,
output_node_names=",".join(output_node_names))
if DEBUG:
self._simple_freeze(
input_graph=graph_def_file,
input_checkpoint=checkpoint_file,
output_graph='/tmp/model.pb',
output_node_names=",".join(output_node_names))
# if TF needs to be quantized, quantize the graph
if quantize_tf_model:
static_model_file = self._quantize_static_tf_model(
model_dir, static_model_file, output_node_names)
# convert to CoreML
mlmodel = coremltools.converters.tensorflow.convert(
static_model_file,
inputs=input_shapes,
outputs=output_node_names,
use_cpu_only=use_cpu_only)
if DEBUG:
print('\n mlmodel description: \n')
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(mlmodel.get_spec(), style='coding')
mlmodel.save(coreml_model_file)
print('\n mlmodel saved at %s' % coreml_model_file)
# Transpose input data as CoreML requires
coreml_inputs = {
name: tf_transpose(feed_dict[self._get_tf_tensor_name(graph,
name)])
for name in input_shapes
}
# Run predict in CoreML
coreml_output = mlmodel.predict(coreml_inputs, useCPUOnly=use_cpu_only)
for idx, out_name in enumerate(output_node_names):
tf_out = result[idx]
if len(tf_out.shape) == 0:
tf_out = np.array([tf_out])
tp = tf_out.flatten()
coreml_out = coreml_output[out_name]
cp = coreml_out.flatten()
self.assertTrue(tf_out.shape == coreml_out.shape)
for i in range(len(tp)):
max_den = max(1.0, tp[i], cp[i])
self.assertAlmostEqual(tp[i] / max_den,
cp[i] / max_den,
delta=delta)
# Cleanup files - models on disk no longer useful
if os.path.exists(model_dir):
shutil.rmtree(model_dir)
def _test_tf_model_constant(self,
graph,
input_shapes,
output_node_names,
data_mode='random',
delta=1e-2,
use_cpu_only=False,
one_dim_seq_flags=None):
"""
Common entry to testing routine for graphs that have no variables.
graph - defined TensorFlow graph.
input_tensor_shapes - dict str:shape for each input (placeholder)
output_node_names - output_node_names, a list of strings
output_tensor_names - output tensor names, a list of strings, usually
just output_node_names each appended with ':0'
"""
model_dir = tempfile.mkdtemp()
frozen_model_file = os.path.join(model_dir, 'tf_frozen.pb')
coreml_model_file = os.path.join(model_dir, 'coreml_model.mlmodel')
feed_dict = {
self._get_tf_tensor_name(graph, name):
generate_data(input_shapes[name], data_mode)
for name in input_shapes
}
with tf.Session(graph=graph) as sess:
# initialize
sess.run(tf.global_variables_initializer())
# run the result
fetches = [
graph.get_operation_by_name(name).outputs[0]
for name in output_node_names
]
result = sess.run(fetches, feed_dict=feed_dict)
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, # The session is used to retrieve the weights
tf.get_default_graph().as_graph_def(
), # The graph_def is used to retrieve the nodes
output_node_names # The output node names are used to select the useful nodes
)
with tf.gfile.GFile(frozen_model_file, 'wb') as f:
f.write(output_graph_def.SerializeToString())
# convert to CoreML
mlmodel = coremltools.converters.tensorflow.convert(
frozen_model_file,
inputs=input_shapes,
outputs=output_node_names,
use_cpu_only=use_cpu_only)
if DEBUG:
print('\n mlmodel description: \n')
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(mlmodel.get_spec(), style='coding')
mlmodel.save(coreml_model_file)
print('\n mlmodel saved at %s' % coreml_model_file)
# Transpose input data as CoreML requires
coreml_inputs = {
name: tf_transpose(feed_dict[self._get_tf_tensor_name(graph,
name)])
for name in input_shapes
}
# Run predict in CoreML
coreml_output = mlmodel.predict(coreml_inputs, useCPUOnly=use_cpu_only)
for idx, out_name in enumerate(output_node_names):
tf_out = result[idx]
if len(tf_out.shape) == 0:
tf_out = np.array([tf_out])
tp = tf_out.flatten()
coreml_out = coreml_output[out_name]
cp = coreml_out.flatten()
self.assertTrue(tf_out.shape == coreml_out.shape)
for i in range(len(tp)):
max_den = max(1.0, tp[i], cp[i])
self.assertAlmostEqual(tp[i] / max_den,
cp[i] / max_den,
delta=delta)
# Cleanup files - models on disk no longer useful
if os.path.exists(model_dir):
shutil.rmtree(model_dir)
def convert(
model, # type: Union[onnx.ModelProto, Text]
mode=None, # type: Optional[Text]
image_input_names=[], # type: Sequence[Text]
preprocessing_args={}, # type: Dict[Text, Any]
image_output_names=[], # type: Sequence[Text]
deprocessing_args={}, # type: Dict[Text, Any]
class_labels=None, # type: Union[Text, Iterable[Text], None]
predicted_feature_name='classLabel', # type: Text
add_custom_layers=False, # type: bool
custom_conversion_functions={}, #type: Dict[Text, Any]
onnx_coreml_input_shape_map={}, # type: Dict[Text, List[int,...]]
disable_coreml_rank5_mapping=False):
# type: (...) -> MLModel
"""
Convert ONNX model to CoreML.
Parameters
----------
model:
An ONNX model with parameters loaded in onnx package or path to file
with models.
mode: 'classifier', 'regressor' or None
Mode of the converted coreml model:
'classifier', a NeuralNetworkClassifier spec will be constructed.
'regressor', a NeuralNetworkRegressor spec will be constructed.
preprocessing_args:
'is_bgr', 'red_bias', 'green_bias', 'blue_bias', 'gray_bias',
'image_scale' keys with the same meaning as
https://apple.github.io/coremltools/generated/coremltools.models.neural_network.html#coremltools.models.neural_network.NeuralNetworkBuilder.set_pre_processing_parameters
deprocessing_args:
Same as 'preprocessing_args' but for deprocessing.
class_labels:
As a string it represents the name of the file which contains
the classification labels (one per line).
As a list of strings it represents a list of categories that map
the index of the output of a neural network to labels in a classifier.
predicted_feature_name:
Name of the output feature for the class labels exposed in the Core ML
model (applies to classifiers only). Defaults to 'classLabel'
add_custom_layers: bool
Flag to turn on addition of custom CoreML layers for unsupported ONNX ops or attributes within
a supported op.
custom_conversion_functions: dict()
A dictionary with keys corresponding to the names/types of onnx ops and values as functions taking
an object of class coreml-tools's 'NeuralNetworkBuilder', Graph' (see onnx-coreml/_graph.Graph),
'Node' (see onnx-coreml/_graph.Node), ErrorHandling (see onnx-coreml/_error_utils.ErrorHandling).
This custom conversion function gets full control and responsibility for converting given onnx op.
This function returns nothing and is responsible for adding a equivalent CoreML layer via 'NeuralNetworkBuilder'
onnx_coreml_input_shape_map: dict()
(Optional) A dictionary with keys corresponding to the model input names. Values are a list of integers that specify
how the shape of the input is mapped to CoreML. Convention used for CoreML shapes is
0: Sequence, 1: Batch, 2: channel, 3: height, 4: width.
For example, an input of rank 2 could be mapped as [3,4] (i.e. H,W) or [1,2] (i.e. B,C) etc.
This is ignored if "disable_coreml_rank5_mapping" is set to True.
disable_coreml_rank5_mapping: bool
If True, then it disables the "RANK5_ARRAY_MAPPING" or enables the "EXACT_ARRAY_MAPPING"
option in CoreML (https://github.com/apple/coremltools/blob/655b3be5cc0d42c3c4fa49f0f0e4a93a26b3e492/mlmodel/format/NeuralNetwork.proto#L67)
Thus, no longer, onnx tensors are forced to map to rank 5 CoreML tensors.
With this flag on, a rank r ONNX tensor, (1<=r<=5), will map to a rank r tensor in CoreML as well.
This flag must be on to utilize any of the new layers added in CoreML 3 (i.e. specification version 4, iOS13)
Returns
-------
model: A coreml model.
"""
if isinstance(model, Text):
onnx_model = onnx.load(model)
elif isinstance(model, onnx.ModelProto):
onnx_model = model
else:
raise TypeError(
"Model must be file path to .onnx file or onnx loaded model")
global USE_SHAPE_MAPPING
if disable_coreml_rank5_mapping:
USE_SHAPE_MAPPING = False
'''
First, apply a few optimizations to the ONNX graph,
in preparation for conversion to CoreML.
'''
# Using Dummy transformation to conditionally disable certain transformation
class DummyTransformation(object):
def __call__(self, graph):
return graph
transformers = [
ConstantsToInitializers(),
ShapeOpRemover(),
ReshapeInitTensorFuser(),
DropoutRemover(),
UnsqueezeConstantRemover(),
TransposeConstantRemover(),
SliceConstantRemover(),
ConcatConstantRemover(),
ConvAddFuser(),
BNBroadcastedMulFuser(),
BNBroadcastedAddFuser(),
ReshapeTransposeReshape_pattern1(),
PixelShuffleFuser(),
AddModelInputsOutputs()
if not disable_coreml_rank5_mapping else DummyTransformation(),
DivMulConstantRemover(),
GatherConstantRemover(),
ConstantFillToInitializers(),
] # type: Iterable[Transformer]
onnx_model = onnx.shape_inference.infer_shapes(onnx_model)
graph = _prepare_onnx_graph(onnx_model.graph, transformers)
'''
Check for ImageScalar nodes in ONNX, this will indicate whether input image preprocessing needs
to be added to the CoreML graph or not.
'''
# are there ImageScaler nodes in the Graph?
# If yes then add the info from it to the "preprocessing_args" dictionary, if the dictionary is not
# already provided by the user
if not bool(preprocessing_args):
for node in graph.nodes:
if node.op_type == 'ImageScaler':
inp_name = node.inputs[0]
scale = node.attrs.get('scale', 1.0)
bias = node.attrs.get('bias', [0, 0, 0])
if not (len(bias) == 1 or len(bias) == 3):
continue
if 'image_scale' in preprocessing_args:
preprocessing_args['image_scale'][inp_name] = scale
else:
preprocessing_args['image_scale'] = {inp_name: scale}
if len(bias) == 3:
for i, color in enumerate(['red', 'green', 'blue']):
if color + '_bias' in preprocessing_args:
preprocessing_args[color +
'_bias'][inp_name] = bias[i]
else:
preprocessing_args[color + '_bias'] = {
inp_name: bias[i]
}
else:
if 'gray_bias' in preprocessing_args:
preprocessing_args['gray_bias'][inp_name] = bias[0]
else:
preprocessing_args['gray_bias'] = {inp_name: bias[0]}
if inp_name not in image_input_names:
image_input_names.append(inp_name) # type: ignore
# remove all ImageScaler ops
graph = graph.transformed([ImageScalerRemover()])
'''
Gather information (name, shape) for model inputs and outputs
This information is then used to initialize the neural network builder object of coremltools.
The builder object is later used to add layers to the CoreML model.
'''
#Make CoreML input and output features by gathering shape info and
#interpreting it for CoreML
input_features = _make_coreml_input_features(graph,
onnx_coreml_input_shape_map,
disable_coreml_rank5_mapping)
if len(image_output_names) > 0:
output_features = _make_coreml_output_features(
graph,
forceShape=True,
disable_coreml_rank5_mapping=disable_coreml_rank5_mapping)
else:
output_features = _make_coreml_output_features(
graph, disable_coreml_rank5_mapping=disable_coreml_rank5_mapping)
builder = NeuralNetworkBuilder(
input_features,
output_features,
mode=mode,
disable_rank5_shape_mapping=disable_coreml_rank5_mapping)
'''
Set CoreML input,output types (float, double, int) same as onnx types, if supported
'''
_transform_coreml_dtypes(builder, graph.inputs, graph.outputs)
'''what follows is some book-keeping to support outputs of type image.
'''
is_deprocess_bgr_only = (len(deprocessing_args) == 1) and \
("is_bgr" in deprocessing_args)
add_deprocess = (len(image_output_names) > 0) and \
(len(deprocessing_args) > 0) and \
(not is_deprocess_bgr_only)
if add_deprocess:
mapping = {}
for f in output_features:
output_name = f[0]
mapping[output_name] = graph.get_unique_edge_name(output_name)
graph = OutputRenamer(mapping)(graph)
if len(image_input_names) > 0:
builder.set_pre_processing_parameters(
image_input_names=image_input_names,
is_bgr=preprocessing_args.get('is_bgr', False),
red_bias=preprocessing_args.get('red_bias', 0.0),
green_bias=preprocessing_args.get('green_bias', 0.0),
blue_bias=preprocessing_args.get('blue_bias', 0.0),
gray_bias=preprocessing_args.get('gray_bias', 0.0),
image_scale=preprocessing_args.get('image_scale', 1.0))
preprocessing_args.clear()
if len(image_output_names) > 0:
for f in output_features:
f_name = f[0]
if f_name in image_output_names:
is_bgr = deprocessing_args.get('is_bgr', False)
_convert_multiarray_output_to_image(builder.spec,
f_name,
is_bgr=is_bgr)
'''
Iterate through all the ONNX ops and translate them to CoreML layers, one by one.
'''
'''
before proceeding to start the layer translation process,
check whether there is an op in the ONNX graph, whose translation function is not yet
implemented in the converter or which is not supported in the CoreML framework. If so,
raise an error before starting the process.
(if the user desires to add a custom layer then this check is not required)
'''
if not add_custom_layers:
_check_unsupported_ops(graph.nodes, disable_coreml_rank5_mapping)
'''
ErrorHandling is a generic class, useful to store a variety of parameters during the conversion process
'''
err = ErrorHandling(
add_custom_layers,
custom_conversion_functions,
disable_coreml_rank5_mapping=disable_coreml_rank5_mapping)
for i, node in enumerate(graph.nodes):
print("%d/%d: Converting Node Type %s" %
(i + 1, len(graph.nodes), node.op_type))
if disable_coreml_rank5_mapping:
_convert_node_nd(builder, node, graph, err)
else:
_add_const_inputs_if_required(builder, node, graph, err)
_convert_node(builder, node, graph, err)
if DEBUG:
plot_graph(graph,
graph_img_path='/tmp/after_conversion.pdf',
show_coreml_mapped_shapes=not disable_coreml_rank5_mapping)
if add_deprocess:
for f in output_features:
output_name = f[0]
if output_name not in image_output_names:
continue
output_shape = f[1].dimensions
if len(output_shape) == 2 or output_shape[0] == 1:
is_grayscale = True
elif output_shape[0] == 3:
is_grayscale = False
else:
raise ValueError('Output must be RGB image or Grayscale')
_set_deprocessing(is_grayscale, builder, deprocessing_args,
mapping[output_name], output_name)
if class_labels is not None:
if isinstance(class_labels, Text):
labels = [l.strip() for l in open(class_labels).readlines()
] # type: Sequence[Text]
elif isinstance(class_labels, list):
labels = class_labels
else:
raise TypeError("synset variable of unknown type. Type found: {}. \
Expected either string or list of strings.".format(
type(class_labels), ))
builder.set_class_labels(class_labels=labels,
predicted_feature_name=predicted_feature_name)
def _add_informative_description(feature, raise_error=True):
if feature.type.WhichOneof('Type') == 'multiArrayType':
if feature.name in graph.onnx_coreml_shape_mapping and feature.name in graph.shape_dict:
mapp = graph.onnx_coreml_shape_mapping[feature.name]
onnx_shape = graph.shape_dict[feature.name]
if raise_error:
assert len(mapp) == len(
onnx_shape), "Something wrong in shape"
if len(mapp) == len(onnx_shape):
shape = []
for i in range(5):
if i in mapp:
shape += [int(onnx_shape[mapp.index(i)])]
else:
shape += [1]
msg = 'MultiArray of shape {}. The first and second dimensions correspond to sequence and batch size, respectively'.format(
str(tuple(shape)))
feature.shortDescription += msg
optional_input_names = []
for tup in graph.optional_inputs:
optional_input_names.append(tup[0])
optional_output_names = []
for tup in graph.optional_outputs:
optional_output_names.append(tup[0])
# add description for inputs and outputs shapes
remove_input_id = []
for i, input_ in enumerate(builder.spec.description.input):
if input_.name not in optional_input_names:
if not disable_coreml_rank5_mapping:
_add_informative_description(input_)
else:
remove_input_id.append(i)
remove_output_id = []
for i, output_ in enumerate(builder.spec.description.output):
if output_.name not in optional_output_names:
if not disable_coreml_rank5_mapping:
_add_informative_description(output_, raise_error=False)
else:
remove_output_id.append(i)
for index in sorted(remove_input_id, reverse=True):
del builder.spec.description.input[index]
for index in sorted(remove_output_id, reverse=True):
del builder.spec.description.output[index]
if len(graph.optional_inputs) > 0 or len(graph.optional_outputs):
builder.add_optionals(graph.optional_inputs, graph.optional_outputs)
print(
"Translation to CoreML spec completed. Now compiling the CoreML model."
)
try:
if DEBUG:
import coremltools
coremltools.models.utils.save_spec(
builder.spec, '/tmp/node_model_raw_spec.mlmodel')
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(builder.spec, style='coding')
mlmodel = MLModel(builder.spec)
except RuntimeError as e:
raise ValueError('Compilation failed: {}'.format(str(e)))
print('Model Compilation done.')
# print information about all ops for which custom layers have been added
if len(err.custom_layer_nodes) > 0:
print('\n')
print("Custom layers have been added to the CoreML model "
"corresponding to the following ops in the onnx model: ")
for i, node in enumerate(err.custom_layer_nodes):
input_info = []
for input_ in node.inputs:
input_info.append(
(str(input_),
graph.shape_dict.get(input_, str("Shape not available"))))
output_info = []
for output_ in node.outputs:
output_info.append(
(str(output_),
graph.shape_dict.get(output_,
str("Shape not available"))))
print(
"{}/{}: op type: {}, op input names and shapes: {}, op output names and shapes: {}"
.format(i + 1, len(err.custom_layer_nodes), node.op_type,
str(input_info), str(output_info)))
return mlmodel
def _test_tf_model_constant(self,
graph,
input_shapes,
output_node_names,
data_mode='random_zero_mean',
delta=1e-2,
use_cpu_only=False,
validate_bool_only=False):
"""
Common entry to testing routine for graphs that have no variables.
Parameters
----------
graph: tf.Graph()
TensorFlow graph.
input_shapes: dict [str : shape]
Shapes for each input (placeholder).
output_node_names: list of str
Output tensor names.
data_mode: str
Data mode for the placeholder data generation.
input_refs: a dictionary of reference input in tensorFlow axis order.
Each entry is str:shape. When using auto-generated input vectors,
set input_refs to None.
delta: float
Delta for error checking, default 1e-2.
use_cpu_only: bool
If true, force use CPU only, default False.
validate_bool_only: bool
If true, only validate it's zero or non-zero, otherwise, validate
float values, default False.
"""
model_dir = tempfile.mkdtemp()
frozen_model_file = os.path.join(model_dir, 'tf_frozen.pb')
coreml_model_file = os.path.join(model_dir, 'coreml_model.mlmodel')
feed_dict = {
self._get_tf_tensor_name(graph, name):
generate_data(input_shapes[name], data_mode)
for name in input_shapes
}
with tf.Session(graph=graph) as sess:
# initialize
sess.run(tf.global_variables_initializer())
# run the result
fetches = [
graph.get_operation_by_name(name).outputs[0]
for name in output_node_names
]
result = sess.run(fetches, feed_dict=feed_dict)
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, # The session is used to retrieve the weights
tf.get_default_graph().as_graph_def(
), # The graph_def is used to retrieve the nodes
output_node_names # The output node names are used to select the useful nodes
)
with tf.gfile.GFile(frozen_model_file, 'wb') as f:
f.write(output_graph_def.SerializeToString())
# convert to CoreML
mlmodel = coremltools.converters.tensorflow.convert(
frozen_model_file,
inputs=input_shapes,
outputs=output_node_names,
use_cpu_only=use_cpu_only)
if DEBUG:
print('\n mlmodel description: \n')
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(mlmodel.get_spec(), style='coding')
mlmodel.save(coreml_model_file)
print('\n mlmodel saved at %s' % coreml_model_file)
# Transpose input data as CoreML requires
coreml_inputs = {
name: tf_transpose(feed_dict[self._get_tf_tensor_name(graph,
name)])
for name in input_shapes
}
# Run predict in CoreML
coreml_output = mlmodel.predict(coreml_inputs, useCPUOnly=use_cpu_only)
for idx, out_name in enumerate(output_node_names):
tf_out = result[idx]
if len(tf_out.shape) == 0:
tf_out = np.array([tf_out])
tp = tf_out.flatten()
coreml_out = coreml_output[out_name]
cp = coreml_out.flatten()
self.assertTrue(tf_out.shape == coreml_out.shape,
msg=(tf_out.shape, 'vs.', coreml_out.shape))
if validate_bool_only:
cp = np.logical_and(cp, cp)
for i in range(len(tp)):
max_den = max(1.0, tp[i], cp[i])
self.assertAlmostEqual(tp[i] / max_den,
cp[i] / max_den,
delta=delta)
# Cleanup files - models on disk no longer useful
if os.path.exists(model_dir):
shutil.rmtree(model_dir)
def convert(
model, # type: Union[onnx.ModelProto, Text]
mode=None, # type: Optional[Text]
image_input_names=[], # type: Sequence[Text]
preprocessing_args={}, # type: Dict[Text, Any]
image_output_names=[], # type: Sequence[Text]
deprocessing_args={}, # type: Dict[Text, Any]
class_labels=None, # type: Union[Text, Iterable[Text], None]
predicted_feature_name="classLabel", # type: Text
add_custom_layers=False, # type: bool
custom_conversion_functions={}, # type: Dict[Text, Any]
onnx_coreml_input_shape_map={}, # type: Dict[Text, List[int,...]]
minimum_ios_deployment_target="12",
):
# type: (...) -> MLModel
"""
WARNING: This function is deprecated. It will be removed in the 6.0.
Convert ONNX model to CoreML.
Parameters
----------
model:
An ONNX model with parameters loaded in the ONNX package, or path to file
with models.
mode: 'classifier', 'regressor' or None
Mode of the converted coreml model:
* ``'classifier'``: a NeuralNetworkClassifier spec will be constructed.
* ``'regressor'``: a NeuralNetworkRegressor spec will be constructed.
preprocessing_args:
The ``'is_bgr'``, ``'red_bias'``, ``'green_bias'``, ``'blue_bias'``, ``'gray_bias'``,
and ``'image_scale'`` keys have the same meaning as the pre-processing arguments for
`NeuralNetworkBuilder <https://coremltools.readme.io/reference/modelsneural_network>`_.
deprocessing_args:
Same as ``'preprocessing_args'`` but for de-processing.
class_labels:
* As a string, it represents the name of the file which contains
the classification labels (one per line).
* As a list of strings, it represents a list of categories that map
the index of the output of a neural network to labels in a classifier.
predicted_feature_name:
Name of the output feature for the class labels exposed in the Core ML
model (applies to classifiers only). Defaults to ``'classLabel'``.
add_custom_layers: bool
Flag to turn on additional custom CoreML layers for unsupported ONNX ops or
attributes within a supported op.
custom_conversion_functions: dict()
* A dictionary with keys corresponding to the names/types of ONNX ops and values as
functions taking an object of the ``coreml-tools`` class:
``'NeuralNetworkBuilder'``, ``'Graph'`` (see ``onnx-coreml/_graph.Graph``),
``'Node'`` (see ``onnx-coreml/_graph.Node``), and
``'ErrorHandling'`` (see ``onnx-coreml/_error_utils.ErrorHandling``).
* This custom conversion function gets full control and responsibility for
converting a given ONNX op.
* The function returns nothing and is responsible for adding an equivalent CoreML
layer via ``'NeuralNetworkBuilder'``.
onnx_coreml_input_shape_map: dict() (Optional)
* A dictionary with keys corresponding to the model input names.
* Values are a list of integers that specify how the shape of the input is mapped
to CoreML.
* Convention used for CoreML shapes is ``0: Sequence``, ``1: Batch``,
``2: channel``, ``3: height``, ``4: width``. For example, an input of rank 2
could be mapped as ``[3,4]`` (H,W) or ``[1,2]`` (B,C), and so on. This is
ignored if ``minimum_ios_deployment_target`` is set to ``13``.
minimum_ios_deployment_target: str
Target Deployment iOS Version (default: ``'12'``). Supported iOS version options:
``'11.2'``, ``'12'``, ``'13'``. CoreML model produced by the converter will be
compatible with the iOS version specified in this argument. For example, if
``minimum_ios_deployment_target = '12'``, the converter would utilize only CoreML
features released up to version iOS12 (equivalent to macOS 10.14, watchOS 5, and
so on). iOS 11.2 (CoreML 0.8) does not support ``resize_bilinear`` and
``crop_resize`` layers. See `supported v0.8 features <https://github.com/apple/coremltools/releases/tag/v0.8>`_.
iOS 12 (CoreML 2.0), see `supported v2.0 features <https://github.com/apple/coremltools/releases/tag/v2.0>`_.
iSO 13 (CoreML 3.0), see `supported v3.0 features <https://github.com/apple/coremltools/releases/tag/3.0-beta6>`_.
Returns
-------
model: A coreml model.
"""
if not _HAS_ONNX:
raise ModuleNotFoundError("Missing ONNX package.")
if isinstance(model, Text):
onnx_model = onnx.load(model)
elif isinstance(model, onnx.ModelProto):
onnx_model = model
else:
raise TypeError(
"Model must be file path to .onnx file or onnx loaded model")
if not SupportedVersion.ios_support_check(minimum_ios_deployment_target):
raise TypeError(
"{} not supported. Please provide one of target iOS: {}",
minimum_ios_deployment_target,
SupportedVersion.get_supported_ios(),
)
global USE_SHAPE_MAPPING
disable_coreml_rank5_mapping = False
if SupportedVersion.is_nd_array_supported(minimum_ios_deployment_target):
disable_coreml_rank5_mapping = True
if disable_coreml_rank5_mapping:
USE_SHAPE_MAPPING = False
else:
USE_SHAPE_MAPPING = True
"""
First, apply a few optimizations to the ONNX graph,
in preparation for conversion to CoreML.
"""
# Using Dummy transformation to conditionally disable certain transformation
class DummyTransformation(object):
def __call__(self, graph):
return graph
transformers = [
ConstantsToInitializers(),
ShapeOpRemover(),
ConstantRemover(),
CastOpRemover(),
PaddingOpRemover(),
ReshapeInitTensorFuser(),
DropoutRemover(),
DeadCodeElimination(),
ConvAddFuser(),
BNBroadcastedMulFuser(),
BNBroadcastedAddFuser(),
ReshapeTransposeReshape_pattern1(),
PixelShuffleFuser(),
AddModelInputsOutputs()
if not disable_coreml_rank5_mapping else DummyTransformation(),
ConstantFillToInitializers(),
] # type: Iterable[Transformer]
onnx_model = onnx.shape_inference.infer_shapes(onnx_model)
graph = _prepare_onnx_graph(onnx_model.graph, transformers,
onnx_model.ir_version)
"""
Check for ImageScalar nodes in ONNX, this will indicate whether input image preprocessing needs
to be added to the CoreML graph or not.
"""
# are there ImageScaler nodes in the Graph?
# If yes then add the info from it to the "preprocessing_args" dictionary, if the dictionary is not
# already provided by the user
if not bool(preprocessing_args):
for node in graph.nodes:
if node.op_type == "ImageScaler":
inp_name = node.inputs[0]
scale = node.attrs.get("scale", 1.0)
bias = node.attrs.get("bias", [0, 0, 0])
if not (len(bias) == 1 or len(bias) == 3):
continue
if "image_scale" in preprocessing_args:
preprocessing_args["image_scale"][inp_name] = scale
else:
preprocessing_args["image_scale"] = {inp_name: scale}
if len(bias) == 3:
for i, color in enumerate(["red", "green", "blue"]):
if color + "_bias" in preprocessing_args:
preprocessing_args[color +
"_bias"][inp_name] = bias[i]
else:
preprocessing_args[color + "_bias"] = {
inp_name: bias[i]
}
else:
if "gray_bias" in preprocessing_args:
preprocessing_args["gray_bias"][inp_name] = bias[0]
else:
preprocessing_args["gray_bias"] = {inp_name: bias[0]}
if inp_name not in image_input_names:
image_input_names.append(inp_name) # type: ignore
# remove all ImageScaler ops
graph = graph.transformed([ImageScalerRemover()])
"""
Gather information (name, shape) for model inputs and outputs
This information is then used to initialize the neural network builder object of coremltools.
The builder object is later used to add layers to the CoreML model.
"""
# Make CoreML input and output features by gathering shape info and
# interpreting it for CoreML
input_features = _make_coreml_input_features(graph,
onnx_coreml_input_shape_map,
disable_coreml_rank5_mapping)
if len(image_output_names) > 0:
output_features = _make_coreml_output_features(
graph,
forceShape=True,
disable_coreml_rank5_mapping=disable_coreml_rank5_mapping,
)
else:
output_features = _make_coreml_output_features(
graph, disable_coreml_rank5_mapping=disable_coreml_rank5_mapping)
builder = NeuralNetworkBuilder(
input_features,
output_features,
mode=mode,
disable_rank5_shape_mapping=disable_coreml_rank5_mapping,
)
# TODO: To be removed once, auto-downgrading of spec version is enabled
builder.spec.specificationVersion = SupportedVersion.get_specification_version(
minimum_ios_deployment_target)
"""
Set CoreML input,output types (float, double, int) same as onnx types, if supported
"""
_transform_coreml_dtypes(builder, graph.inputs, graph.outputs)
"""what follows is some book-keeping to support outputs of type image.
"""
is_deprocess_bgr_only = (len(deprocessing_args)
== 1) and ("is_bgr" in deprocessing_args)
add_deprocess = ((len(image_output_names) > 0)
and (len(deprocessing_args) > 0)
and (not is_deprocess_bgr_only))
if add_deprocess:
mapping = {}
for f in output_features:
output_name = f[0]
mapping[output_name] = graph.get_unique_edge_name(output_name)
graph = OutputRenamer(mapping)(graph)
if len(image_input_names) > 0:
builder.set_pre_processing_parameters(
image_input_names=image_input_names,
is_bgr=preprocessing_args.get("is_bgr", False),
red_bias=preprocessing_args.get("red_bias", 0.0),
green_bias=preprocessing_args.get("green_bias", 0.0),
blue_bias=preprocessing_args.get("blue_bias", 0.0),
gray_bias=preprocessing_args.get("gray_bias", 0.0),
image_scale=preprocessing_args.get("image_scale", 1.0),
)
preprocessing_args.clear()
if len(image_output_names) > 0:
for f in output_features:
f_name = f[0]
if f_name in image_output_names:
is_bgr = deprocessing_args.get("is_bgr", False)
_convert_multiarray_output_to_image(builder.spec,
f_name,
is_bgr=is_bgr)
"""
Iterate through all the ONNX ops and translate them to CoreML layers, one by one.
"""
"""
before proceeding to start the layer translation process,
check whether there is an op in the ONNX graph, whose translation function is not yet
implemented in the converter or which is not supported in the CoreML framework. If so,
raise an error before starting the process.
(if the user desires to add a custom layer then this check is not required)
"""
if not add_custom_layers:
_check_unsupported_ops(graph.nodes, disable_coreml_rank5_mapping)
"""
ErrorHandling is a generic class, useful to store a variety of parameters during the conversion process
"""
err = ErrorHandling(add_custom_layers, custom_conversion_functions)
for i, node in enumerate(graph.nodes):
print("%d/%d: Converting Node Type %s" %
(i + 1, len(graph.nodes), node.op_type))
if disable_coreml_rank5_mapping:
_convert_node_nd(builder, node, graph, err)
else:
_add_const_inputs_if_required(builder, node, graph, err)
_convert_node(builder, node, graph, err)
if DEBUG:
plot_graph(
graph,
graph_img_path="/tmp/after_conversion.pdf",
show_coreml_mapped_shapes=not disable_coreml_rank5_mapping,
)
if add_deprocess:
for f in output_features:
output_name = f[0]
if output_name not in image_output_names:
continue
output_shape = f[1].dimensions
if len(output_shape) == 2 or output_shape[0] == 1:
is_grayscale = True
elif output_shape[0] == 3:
is_grayscale = False
else:
raise ValueError("Output must be RGB image or Grayscale")
_set_deprocessing(
is_grayscale,
builder,
deprocessing_args,
mapping[output_name],
output_name,
)
if class_labels is not None:
if isinstance(class_labels, Text):
labels = [l.strip() for l in open(class_labels).readlines()
] # type: Sequence[Text]
elif isinstance(class_labels, list):
labels = class_labels
else:
raise TypeError("synset variable of unknown type. Type found: {}. \
Expected either string or list of strings.".format(
type(class_labels), ))
builder.set_class_labels(class_labels=labels,
predicted_feature_name=predicted_feature_name)
def _add_informative_description(feature, raise_error=True):
if feature.type.WhichOneof("Type") == "multiArrayType":
if (feature.name in graph.onnx_coreml_shape_mapping
and feature.name in graph.shape_dict):
mapp = graph.onnx_coreml_shape_mapping[feature.name]
onnx_shape = graph.shape_dict[feature.name]
if raise_error:
assert len(mapp) == len(
onnx_shape), "Something wrong in shape"
if len(mapp) == len(onnx_shape):
shape = []
for i in range(5):
if i in mapp:
shape += [int(onnx_shape[mapp.index(i)])]
else:
shape += [1]
msg = "MultiArray of shape {}. The first and second dimensions correspond to sequence and batch size, respectively".format(
str(tuple(shape)))
feature.shortDescription += msg
optional_input_names = []
for tup in graph.optional_inputs:
optional_input_names.append(tup[0])
optional_output_names = []
for tup in graph.optional_outputs:
optional_output_names.append(tup[0])
# add description for inputs and outputs shapes
remove_input_id = []
for i, input_ in enumerate(builder.spec.description.input):
if input_.name not in optional_input_names:
if not disable_coreml_rank5_mapping:
_add_informative_description(input_)
else:
remove_input_id.append(i)
remove_output_id = []
for i, output_ in enumerate(builder.spec.description.output):
if output_.name not in optional_output_names:
if not disable_coreml_rank5_mapping:
_add_informative_description(output_, raise_error=False)
else:
remove_output_id.append(i)
for index in sorted(remove_input_id, reverse=True):
del builder.spec.description.input[index]
for index in sorted(remove_output_id, reverse=True):
del builder.spec.description.output[index]
if len(graph.optional_inputs) > 0 or len(graph.optional_outputs):
builder.add_optionals(graph.optional_inputs, graph.optional_outputs)
# Check for specification version and target ios compatibility
if (minimum_ios_deployment_target == "11.2"
and builder.spec.WhichOneof("Type") == "neuralNetwork"):
nn_spec = builder.spec.neuralNetwork
for layer in nn_spec.layers:
if (layer.WhichOneof("layer") == "resizeBilinear"
or layer.WhichOneof("layer") == "cropResize"):
raise TypeError(
"{} not supported with target iOS 11.2 please provide higher target iOS"
.format(layer.WhichOneof("layer")))
# Optimize ML Model Spec
ml_model_passes = [remove_disconnected_layers, transform_conv_crop]
for opt in ml_model_passes:
opt(builder.spec)
print(
"Translation to CoreML spec completed. Now compiling the CoreML model."
)
try:
if DEBUG:
import coremltools
coremltools.models.utils.save_spec(
builder.spec, "/tmp/node_model_raw_spec.mlmodel")
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(builder.spec, style="coding")
mlmodel = MLModel(builder.spec)
except RuntimeError as e:
raise ValueError("Compilation failed: {}".format(str(e)))
print("Model Compilation done.")
# print information about all ops for which custom layers have been added
if len(err.custom_layer_nodes) > 0:
print("\n")
print("Custom layers have been added to the CoreML model "
"corresponding to the following ops in the onnx model: ")
for i, node in enumerate(err.custom_layer_nodes):
input_info = []
for input_ in node.inputs:
input_info.append((
str(input_),
graph.shape_dict.get(input_, str("Shape not available")),
))
output_info = []
for output_ in node.outputs:
output_info.append((
str(output_),
graph.shape_dict.get(output_, str("Shape not available")),
))
print(
"{}/{}: op type: {}, op input names and shapes: {}, op output names and shapes: {}"
.format(
i + 1,
len(err.custom_layer_nodes),
node.op_type,
str(input_info),
str(output_info),
))
mlmodel.user_defined_metadata[_METADATA_VERSION] = ct_version
mlmodel.user_defined_metadata[_METADATA_SOURCE] = "onnx=={0}".format(
onnx.__version__)
return mlmodel
def _test_keras_model_tf1(self, model, data_mode, decimal, use_cpu_only, has_variables, verbose):
graph_def_file = os.path.join(self.saved_model_dir, 'graph.pb')
frozen_model_file = os.path.join(self.saved_model_dir, 'frozen.pb')
core_ml_model_file = os.path.join(self.saved_model_dir, 'model.mlmodel')
input_shapes = {inp.op.name: inp.shape.as_list() for inp in model.inputs}
for name, shape in input_shapes.items():
input_shapes[name] = [dim if dim is not None else 1 for dim in shape]
output_node_names = [output.op.name for output in model.outputs]
tf_graph = _keras.get_session().graph
tf.reset_default_graph()
if has_variables:
with tf_graph.as_default():
saver = tf.train.Saver()
# note: if Keras backend has_variable is False, we're not making variables constant
with tf.Session(graph=tf_graph) as sess:
sess.run(tf.global_variables_initializer())
feed_dict = {}
for name, shape in input_shapes.items():
tensor_name = tf_graph.get_operation_by_name(name).outputs[0].name
feed_dict[tensor_name] = generate_data(shape, data_mode)
# run the result
fetches = [
tf_graph.get_operation_by_name(name).outputs[0] for name in output_node_names
]
result = sess.run(fetches, feed_dict=feed_dict)
# save graph definition somewhere
tf.train.write_graph(sess.graph, self.saved_model_dir, graph_def_file, as_text=False)
# freeze_graph() has been raising error with tf.keras models since no
# later than TensorFlow 1.6, so we're not using freeze_graph() here.
# See: https://github.com/tensorflow/models/issues/5387
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, # The session is used to retrieve the weights
tf_graph.as_graph_def(), # The graph_def is used to retrieve the nodes
output_node_names # The output node names are used to select the useful nodes
)
with tf.gfile.GFile(frozen_model_file, 'wb') as f:
f.write(output_graph_def.SerializeToString())
_keras.clear_session()
# convert to Core ML model format
core_ml_model = coremltools.converters.tensorflow.convert(
frozen_model_file,
inputs=input_shapes,
outputs=output_node_names,
use_cpu_only=use_cpu_only)
if verbose:
print('\nFrozen model saved at {}'.format(frozen_model_file))
print('\nCore ML model description:')
from coremltools.models.neural_network.printer import print_network_spec
print_network_spec(core_ml_model.get_spec(), style='coding')
core_ml_model.save(core_ml_model_file)
print('\nCore ML model saved at {}'.format(core_ml_model_file))
# transpose input data as Core ML requires
core_ml_inputs = {
name: tf_transpose(feed_dict[self._get_tf_tensor_name(tf_graph, name)])
for name in input_shapes
}
# run prediction in Core ML
core_ml_output = core_ml_model.predict(core_ml_inputs, useCPUOnly=use_cpu_only)
for idx, out_name in enumerate(output_node_names):
tf_out = result[idx]
if len(tf_out.shape) == 0:
tf_out = np.array([tf_out])
tp = tf_out.flatten()
coreml_out = core_ml_output[out_name]
cp = coreml_out.flatten()
self.assertTrue(tf_out.shape == coreml_out.shape)
for i in range(len(tp)):
max_den = max(1.0, tp[i], cp[i])
self.assertAlmostEqual(tp[i] / max_den, cp[i] / max_den, delta=10 ** -decimal)
