def test_nn_classifier_util_file(self):
input_features = [("data", datatypes.Array(3))]
output_features = [("out", datatypes.Array(3))]
builder = NeuralNetworkBuilder(
input_features, output_features, disable_rank5_shape_mapping=True
)
builder.add_activation("linear", "LINEAR", "data", "out")
spec = builder.spec
mlmodel = MLModel(spec)
class_labels = ["a", "b", "c"]
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt") as f:
f.write("\n".join(class_labels))
f.flush()
mlmodel = make_nn_classifier(
mlmodel,
class_labels=f.name,
predicted_feature_name="out_confidence",
predicted_probabilities_output="out",
)
out_dict = mlmodel.predict({"data": np.array([4.0, 5.5, 6.0])}, useCPUOnly=True)
self.assertEqual(out_dict["out_confidence"], "c")
self.assertEqual(
mlmodel.get_spec().WhichOneof("Type"), "neuralNetworkClassifier"
)
def test_downgrade_specification_version(self):
# manually set a invalid specification version
self.spec.specificationVersion = -1
model = MLModel(self.spec)
assert model.get_spec().specificationVersion == 1
# manually set a high specification version
self.spec.specificationVersion = 4
filename = tempfile.mktemp(suffix='.mlmodel')
save_spec(self.spec, filename, auto_set_specification_version=True)
model = MLModel(filename)
assert model.get_spec().specificationVersion == 1
# simple neural network with only spec 1 layer
input_features = [('data', datatypes.Array(3))]
output_features = [('out', datatypes.Array(3))]
builder = NeuralNetworkBuilder(input_features, output_features)
builder.add_activation('relu', 'RELU', 'data', 'out')
# set a high specification version
builder.spec.specificationVersion = 3
model = MLModel(builder.spec)
filename = tempfile.mktemp(suffix='.mlmodel')
model.save(filename)
# load the model back
model = MLModel(filename)
assert model.get_spec().specificationVersion == 1
# test save without automatic set specification version
self.spec.specificationVersion = 3
filename = tempfile.mktemp(suffix='.mlmodel')
save_spec(self.spec, filename, auto_set_specification_version=False)
model = MLModel(filename)
# the specification version should be original
assert model.get_spec().specificationVersion == 3
def test_multiarray_to_image_input_util_HWC_format(self):
H, W, C = 1, 1, 3
input_features = [("data", datatypes.Array(H, W, C))]
output_features = [("out", datatypes.Array(H, W, C))]
builder = NeuralNetworkBuilder(
input_features, output_features, disable_rank5_shape_mapping=True
)
builder.add_activation("linear", "LINEAR", "data", "out")
spec = builder.spec
mlmodel = MLModel(spec)
mlmodel = make_image_input(
mlmodel,
"data",
red_bias=-5,
green_bias=-6,
blue_bias=-2.5,
scale=10.0,
image_format="NHWC",
)
x = np.array([4, 2, 5], dtype=np.uint8)
x = np.reshape(x, (H, W, C))
pil_img = PIL.Image.fromarray(x)
y = mlmodel.predict({"data": pil_img}, useCPUOnly=True)["out"]
self.assertEqual(y.shape, (H, W, C))
np.testing.assert_almost_equal(y.flatten(), [35.0, 14.0, 47.5])
def get_custom_model_spec():
from coremltools.models.neural_network import NeuralNetworkBuilder
from coremltools.models.datatypes import Array, Dictionary, String
input_name = 'output1'
input_length = self._feature_extractor.output_length
builder = NeuralNetworkBuilder(
[(input_name, Array(input_length, ))],
[(prob_name, Dictionary(String))], 'classifier')
ctx = _mxnet_utils.get_mxnet_context()[0]
input_name, output_name = input_name, 0
import mxnet as _mx
for i, cur_layer in enumerate(self._custom_classifier):
output_name = str(i)
if type(cur_layer) == _mx.gluon.nn.basic_layers.Dense:
W = cur_layer.weight.data(ctx).asnumpy()
nC, nB = W.shape
Wb = cur_layer.bias.data(ctx).asnumpy()
builder.add_inner_product(name='inner_product_' + str(i),
W=W,
b=Wb,
input_channels=nB,
output_channels=nC,
has_bias=True,
input_name=input_name,
output_name='inner_product_' +
output_name)
if cur_layer.act:
builder.add_activation("activation" + str(i), 'RELU',
'inner_product_' + output_name,
output_name)
elif type(cur_layer) == _mx.gluon.nn.basic_layers.BatchNorm:
zeros = _np.zeros(nC)
ones = _np.ones(nC)
builder.add_batchnorm(name='bn_layer_' + str(i),
channels=nC,
gamma=ones,
beta=zeros,
mean=zeros,
variance=ones,
input_name=input_name,
output_name=output_name)
elif type(cur_layer) == _mx.gluon.nn.basic_layers.Dropout:
continue
input_name = output_name
last_output = builder.spec.neuralNetworkClassifier.layers[
-1].output[0]
builder.add_softmax('softmax', last_output, self.target)
builder.set_class_labels(self.classes)
builder.set_input([input_name], [(input_length, )])
builder.set_output([self.target], [(self.num_classes, )])
return builder.spec
def test_activation_converter(self):
input_dim = (3,)
output_dim = (3,)
input = [('input', datatypes.Array(*input_dim))]
output = [('output', datatypes.Array(*output_dim))]
builder = NeuralNetworkBuilder(input, output)
builder.add_activation(name='Activation', non_linearity='RELU', input_name='input', output_name='output')
model_onnx = convert_coreml(builder.spec)
self.assertTrue(model_onnx is not None)
def get_custom_model_spec():
from coremltools.models.neural_network import NeuralNetworkBuilder
from coremltools.models.datatypes import Array
input_name = "output1"
input_length = self._feature_extractor.output_length
builder = NeuralNetworkBuilder(
[(input_name, Array(input_length,))],
[(prob_name, Array(self.num_classes,))],
"classifier",
)
layer_counter = [0]
builder.set_input([input_name], [(input_length,)])
def next_layer_name():
layer_counter[0] += 1
return "layer_%d" % layer_counter[0]
for i, cur_layer in enumerate(self._custom_classifier.export_weights()):
W = cur_layer["weight"]
nC, nB = W.shape
Wb = cur_layer["bias"]
output_name = next_layer_name()
builder.add_inner_product(
name="inner_product_" + str(i),
W=W,
b=Wb,
input_channels=nB,
output_channels=nC,
has_bias=True,
input_name=input_name,
output_name=output_name,
)
input_name = output_name
if cur_layer["act"]:
output_name = next_layer_name()
builder.add_activation(
"activation" + str(i), "RELU", input_name, output_name
)
input_name = output_name
builder.add_softmax("softmax", input_name, prob_name)
builder.set_class_labels(
self.classes,
predicted_feature_name=self.target,
prediction_blob=prob_name,
)
return builder.spec
def test_activation_converter(self):
input_dim = (3, )
output_dim = (3, )
input = [('input', datatypes.Array(*input_dim))]
output = [('output', datatypes.Array(*output_dim))]
builder = NeuralNetworkBuilder(input, output)
builder.add_activation(name='Activation',
non_linearity='RELU',
input_name='input',
output_name='output')
context = ConvertContext()
node = ActivationConverter.convert(
context, builder.spec.neuralNetwork.layers[0], ['input'],
['output'])
self.assertTrue(node is not None)
def get_custom_model_spec():
from coremltools.models.neural_network import NeuralNetworkBuilder
from coremltools.models.datatypes import Array, Dictionary, String
input_name = 'output1'
input_length = self._feature_extractor.output_length
builder = NeuralNetworkBuilder(
[(input_name, Array(input_length, ))],
[(prob_name, Dictionary(String))], 'classifier')
input_name, output_name = input_name, 0
for i, cur_layer in enumerate(
self._custom_classifier.export_weights()):
W = cur_layer['weight']
nC, nB = W.shape
Wb = cur_layer['bias']
builder.add_inner_product(name="inner_product_" + str(i),
W=W,
b=Wb,
input_channels=nB,
output_channels=nC,
has_bias=True,
input_name=str(input_name),
output_name='inner_product_' +
str(output_name))
if cur_layer['act']:
builder.add_activation("activation" + str(i), 'RELU',
'inner_product_' + str(output_name),
str(output_name))
input_name = i
output_name = i + 1
last_output = builder.spec.neuralNetworkClassifier.layers[
-1].output[0]
builder.add_softmax('softmax', last_output, self.target)
builder.set_class_labels(self.classes,
predicted_feature_name=self.target)
builder.set_input([input_name], [(input_length, )])
builder.set_output([self.target], [(self.num_classes, )])
return builder.spec
def test_rename_image_input(self):
input_features = [("data", datatypes.Array(3, 1, 1))]
output_features = [("out", datatypes.Array(3, 1, 1))]
builder = NeuralNetworkBuilder(
input_features, output_features, disable_rank5_shape_mapping=True
)
builder.add_activation("linear", "LINEAR", "data", "out")
spec = builder.spec
# make an image input
mlmodel = make_image_input(MLModel(spec), "data", image_format="NCHW", scale=2.0)
# rename the input
spec = mlmodel.get_spec()
rename_feature(spec, "data", "new_input_name")
mlmodel = MLModel(spec)
# test
x = np.array([4, 5, 6], dtype=np.uint8).reshape(1, 1, 3)
pil_img = PIL.Image.fromarray(x)
out = mlmodel.predict({"new_input_name": pil_img}, useCPUOnly=True)['out']
np.testing.assert_equal(out, np.array([8.0, 10.0, 12.0]).reshape(3, 1, 1))
def test_nn_classifier_util(self):
input_features = [("data", datatypes.Array(3))]
output_features = [("out", datatypes.Array(3))]
builder = NeuralNetworkBuilder(input_features,
output_features,
disable_rank5_shape_mapping=True)
builder.add_activation("linear", "LINEAR", "data", "out")
spec = builder.spec
mlmodel = MLModel(spec, compute_units=ComputeUnit.CPU_ONLY)
mlmodel = make_nn_classifier(
mlmodel,
class_labels=["a", "b", "c"],
predicted_feature_name="out_confidence",
predicted_probabilities_output="out",
)
out_dict = mlmodel.predict({"data": np.array([4.0, 5.5, 6.0])})
self.assertEqual(out_dict["out_confidence"], "c")
self.assertEqual(mlmodel.get_spec().WhichOneof("Type"),
"neuralNetworkClassifier")
def test_rename_output_nn_classifier(self):
input_features = [("data", datatypes.Array(3))]
output_features = [("out", datatypes.Array(3))]
builder = NeuralNetworkBuilder(
input_features, output_features, disable_rank5_shape_mapping=True
)
builder.add_activation("linear", "LINEAR", "data", "out")
spec = builder.spec
mlmodel = MLModel(spec)
class_labels = ["a", "b", "c"]
mlmodel = make_nn_classifier(mlmodel, class_labels=["a", "b", "c"])
# rename output
spec = mlmodel.get_spec()
rename_feature(spec, "out", "new_out_name")
mlmodel = MLModel(spec)
out_dict = mlmodel.predict({"data": np.array([4.0, 5.5, 6.0])}, useCPUOnly=True)
self.assertEqual(out_dict["classLabel"], "c")
self.assertTrue("new_out_name" in out_dict)
self.assertTrue(isinstance(out_dict["new_out_name"], dict))
def test_downgrade_specification_version(self):
# manually set a invalid specification version
self.spec.specificationVersion = -1
model = MLModel(self.spec)
if model.get_spec().specificationVersion != 1:
raise AssertionError
# manually set a high specification version
self.spec.specificationVersion = 4
filename = tempfile.mktemp(suffix=".mlmodel")
save_spec(self.spec, filename, auto_set_specification_version=True)
model = MLModel(filename)
if model.get_spec().specificationVersion != 1:
raise AssertionError
# simple neural network with only spec 1 layer
input_features = [("data", datatypes.Array(3))]
output_features = [("out", datatypes.Array(3))]
builder = NeuralNetworkBuilder(input_features, output_features)
builder.add_activation("relu", "RELU", "data", "out")
# set a high specification version
builder.spec.specificationVersion = 3
model = MLModel(builder.spec)
filename = tempfile.mktemp(suffix=".mlmodel")
model.save(filename)
# load the model back
model = MLModel(filename)
if model.get_spec().specificationVersion != 1:
raise AssertionError
# test save without automatic set specification version
self.spec.specificationVersion = 3
filename = tempfile.mktemp(suffix=".mlmodel")
save_spec(self.spec, filename, auto_set_specification_version=False)
model = MLModel(filename)
# the specification version should be original
if model.get_spec().specificationVersion != 3:
raise AssertionError
def make_mlmodel(variables):
# Specify the inputs and outputs (there can be multiple).
# Each name corresponds to the input_name/output_name of a layer in the network so
# that Core ML knows where to insert and extract data.
input_features = [('image', datatypes.Array(1, IMAGE_HEIGHT, IMAGE_WIDTH))]
output_features = [('labelValues', datatypes.Array(NUM_LABEL_INDEXES))]
builder = NeuralNetworkBuilder(input_features, output_features, mode=None)
# The "name" parameter has no effect on the function of the network. As far as I know
# it's only used when Xcode fails to load your mlmodel and gives you an error telling
# you what the problem is.
# The input_names and output_name are used to link layers to each other and to the
# inputs and outputs of the model. When adding or removing layers, or renaming their
# outputs, always make sure you correct the input and output names of the layers
# before and after them.
builder.add_elementwise(name='add_layer',
input_names=['image'],
output_name='add_layer',
mode='ADD',
alpha=-0.5)
# Although Core ML internally uses weight matrices of shape
# (outputChannels, inputChannels, height, width) (as can be found by looking at the
# protobuf specification comments), add_convolution takes the shape
# (height, width, inputChannels, outputChannels) (as can be found in the coremltools
# documentation). The latter shape matches what TensorFlow uses so we don't need to
# reorder the matrix axes ourselves.
builder.add_convolution(name='conv2d_1',
kernel_channels=1,
output_channels=32,
height=3,
width=3,
stride_height=1,
stride_width=1,
border_mode='same',
groups=0,
W=variables['W_conv1'].eval(),
b=variables['b_conv1'].eval(),
has_bias=True,
is_deconv=False,
output_shape=None,
input_name='add_layer',
output_name='conv2d_1')
builder.add_activation(name='relu_1',
non_linearity='RELU',
input_name='conv2d_1',
output_name='relu_1',
params=None)
builder.add_pooling(name='maxpool_1',
height=2,
width=2,
stride_height=2,
stride_width=2,
layer_type='MAX',
padding_type='SAME',
input_name='relu_1',
output_name='maxpool_1')
# ...
builder.add_flatten(name='maxpool_3_flat',
mode=1,
input_name='maxpool_3',
output_name='maxpool_3_flat')
# We must swap the axes of the weight matrix because add_inner_product takes the shape
# (outputChannels, inputChannels) whereas TensorFlow uses
# (inputChannels, outputChannels). Unlike with add_convolution (see the comment
# above), the shape add_inner_product expects matches what the protobuf specification
# requires for inner products.
builder.add_inner_product(name='fc1',
W=tf_fc_weights_order_to_mlmodel(
variables['W_fc1'].eval()).flatten(),
b=variables['b_fc1'].eval().flatten(),
input_channels=6 * 6 * 64,
output_channels=1024,
has_bias=True,
input_name='maxpool_3_flat',
output_name='fc1')
# ...
builder.add_softmax(name='softmax',
input_name='fc2',
output_name='labelValues')
model = MLModel(builder.spec)
model.short_description = 'Model for recognizing a variety of images drawn on screen with one\'s finger'
model.input_description['image'] = 'A gesture image to classify'
model.output_description[
'labelValues'] = 'The "probability" of each label, in a dense array'
return model
