def test_linear_quant_inner_product_3bit(self):
W = np.reshape(np.arange(6), (2, 3)).astype(np.uint8)
input_features = [("data", datatypes.Array(3))]
output_features = [("probs", None)]
builder = NeuralNetworkBuilder(input_features, output_features)
builder.add_inner_product(
name="ip1",
W=_convert_array_to_nbit_quantized_bytes(W.flatten(), 3).tobytes(),
b=None,
input_channels=3,
output_channels=2,
has_bias=False,
input_name="data",
output_name="probs",
quantization_type="linear",
nbits=3,
quant_scale=[11.0, 2.0],
quant_bias=[-2.0, 10.0],
)
mlmodel = MLModel(builder.spec)
data = np.array([1.0, 3.0, 5.0])
data_dict = {"data": data}
probs = mlmodel.predict(data_dict)["probs"]
expected_out = np.array([125, 170])
self.assertTrue(np.allclose(probs.flatten(), expected_out.flatten()))
def test_lut_quant_inner_product_1bit(self):
W = np.zeros((2, 3), dtype=np.uint8)
W[0, :] = [0, 1, 1]
W[1, :] = [1, 0, 0]
input_features = [("data", datatypes.Array(3))]
output_features = [("probs", None)]
builder = NeuralNetworkBuilder(input_features, output_features)
builder.add_inner_product(
name="ip1",
W=_convert_array_to_nbit_quantized_bytes(W.flatten(), 1).tobytes(),
b=None,
input_channels=3,
output_channels=2,
has_bias=False,
input_name="data",
output_name="probs",
quantization_type="lut",
nbits=1,
quant_lut=[5.0, -3.0],
)
mlmodel = MLModel(builder.spec)
data = np.array([1.0, 3.0, 5.0])
data_dict = {"data": data}
probs = mlmodel.predict(data_dict)["probs"]
expected_out = np.array([-19, 37])
self.assertTrue(np.allclose(probs.flatten(), expected_out.flatten()))
def create_base_builder(self):
self.input_features = [("input", datatypes.Array(3))]
self.output_features = [("output", None)]
self.output_names = ["output"]
builder = NeuralNetworkBuilder(self.input_features,
self.output_features)
W1 = _np.random.uniform(-0.5, 0.5, (3, 3))
W2 = _np.random.uniform(-0.5, 0.5, (3, 3))
builder.add_inner_product(
name="ip1",
W=W1,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name="input",
output_name="hidden",
)
builder.add_inner_product(
name="ip2",
W=W2,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name="hidden",
output_name="output",
)
builder.make_updatable(["ip1", "ip2"]) # or a dict for weightParams
return builder
def create_base_builder(self):
self.input_features = [('input', datatypes.Array(3))]
self.output_features = [('output', None)]
self.output_names = ["output"]
builder = NeuralNetworkBuilder(self.input_features,
self.output_features,
disable_rank5_shape_mapping=True)
W1 = _np.random.uniform(-0.5, 0.5, (3, 3))
W2 = _np.random.uniform(-0.5, 0.5, (3, 3))
builder.add_inner_product(name='ip1',
W=W1,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name='input',
output_name='hidden')
builder.add_inner_product(name='ip2',
W=W2,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name='hidden',
output_name='output')
builder.make_updatable(['ip1', 'ip2']) # or a dict for weightParams
return builder
def test_convert_nn_spec_to_half_precision(self):
# simple network with quantization layer
input_features = [('data', datatypes.Array(3))]
output_features = [('out', datatypes.Array(3))]
builder = NeuralNetworkBuilder(input_features, output_features)
weights = np.random.uniform(-0.5, 0.5, (3, 3))
builder.add_inner_product(name='inner_product',
W=weights,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name='data',
output_name='out')
model = MLModel(builder.spec)
spec = convert_neural_network_spec_weights_to_fp16(model.get_spec())
self.assertIsNotNone(spec)
# simple network without quantization layer
input_features = [('data', datatypes.Array(3))]
output_features = [('out', datatypes.Array(3))]
builder = NeuralNetworkBuilder(input_features, output_features)
builder.add_lrn(name='lrn',
input_name='data',
output_name='out',
alpha=2,
beta=3,
local_size=1,
k=8)
model = MLModel(builder.spec)
spec = convert_neural_network_spec_weights_to_fp16(model.get_spec())
self.assertIsNotNone(spec)
def test_nn_builder_with_training_features(self):
input_features = [('input', datatypes.Array(3))]
output_features = [('output', None)]
training_features = [('input', datatypes.Array(3)),
('target', datatypes.Double)]
builder = NeuralNetworkBuilder(input_features,
output_features,
disable_rank5_shape_mapping=True,
training_features=training_features)
W1 = _np.random.uniform(-0.5, 0.5, (3, 3))
W2 = _np.random.uniform(-0.5, 0.5, (3, 3))
builder.add_inner_product(name='ip1',
W=W1,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name='input',
output_name='hidden')
builder.add_inner_product(name='ip2',
W=W2,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name='hidden',
output_name='output')
builder.make_updatable(['ip1', 'ip2']) # or a dict for weightParams
builder.set_mean_squared_error_loss(name='mse',
input='output',
target='target')
builder.set_adam_optimizer(
AdamParams(lr=1e-2, batch=10, beta1=0.9, beta2=0.999, eps=1e-8))
builder.set_epochs(20, allowed_set=[10, 20, 30])
builder.set_training_input([('input', datatypes.Array(3)),
('target', 'Double')])
model_path = os.path.join(self.model_dir, 'updatable_creation.mlmodel')
print(model_path)
save_spec(builder.spec, model_path)
mlmodel = MLModel(model_path)
self.assertTrue(mlmodel is not None)
spec = mlmodel.get_spec()
self.assertEqual(spec.description.trainingInput[0].name, 'input')
self.assertEqual(
spec.description.trainingInput[0].type.WhichOneof('Type'),
'multiArrayType')
self.assertEqual(spec.description.trainingInput[1].name, 'target')
self.assertEqual(
spec.description.trainingInput[1].type.WhichOneof('Type'),
'doubleType')
def get_custom_model_spec():
from coremltools.models.neural_network import NeuralNetworkBuilder
from coremltools.models.datatypes import Array, Dictionary, String
input_name = 'output1'
input_length = self._feature_extractor.output_length
builder = NeuralNetworkBuilder(
[(input_name, Array(input_length, ))],
[(prob_name, Dictionary(String))], 'classifier')
ctx = _mxnet_utils.get_mxnet_context()[0]
input_name, output_name = input_name, 0
import mxnet as _mx
for i, cur_layer in enumerate(self._custom_classifier):
output_name = str(i)
if type(cur_layer) == _mx.gluon.nn.basic_layers.Dense:
W = cur_layer.weight.data(ctx).asnumpy()
nC, nB = W.shape
Wb = cur_layer.bias.data(ctx).asnumpy()
builder.add_inner_product(name='inner_product_' + str(i),
W=W,
b=Wb,
input_channels=nB,
output_channels=nC,
has_bias=True,
input_name=input_name,
output_name='inner_product_' +
output_name)
if cur_layer.act:
builder.add_activation("activation" + str(i), 'RELU',
'inner_product_' + output_name,
output_name)
elif type(cur_layer) == _mx.gluon.nn.basic_layers.BatchNorm:
zeros = _np.zeros(nC)
ones = _np.ones(nC)
builder.add_batchnorm(name='bn_layer_' + str(i),
channels=nC,
gamma=ones,
beta=zeros,
mean=zeros,
variance=ones,
input_name=input_name,
output_name=output_name)
elif type(cur_layer) == _mx.gluon.nn.basic_layers.Dropout:
continue
input_name = output_name
last_output = builder.spec.neuralNetworkClassifier.layers[
-1].output[0]
builder.add_softmax('softmax', last_output, self.target)
builder.set_class_labels(self.classes)
builder.set_input([input_name], [(input_length, )])
builder.set_output([self.target], [(self.num_classes, )])
return builder.spec
def test_nn_builder_with_training_features(self):
input_features = [("input", datatypes.Array(3))]
output_features = [("output", datatypes.Array(3))]
builder = NeuralNetworkBuilder(input_features, output_features)
W1 = _np.random.uniform(-0.5, 0.5, (3, 3))
W2 = _np.random.uniform(-0.5, 0.5, (3, 3))
builder.add_inner_product(
name="ip1",
W=W1,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name="input",
output_name="hidden",
)
builder.add_inner_product(
name="ip2",
W=W2,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name="hidden",
output_name="output",
)
builder.make_updatable(["ip1", "ip2"]) # or a dict for weightParams
builder.set_mean_squared_error_loss(name="mse",
input_feature=("output",
datatypes.Array(3)))
builder.set_adam_optimizer(
AdamParams(lr=1e-2, batch=10, beta1=0.9, beta2=0.999, eps=1e-8))
builder.set_epochs(20, allowed_set=[10, 20, 30])
model_path = os.path.join(self.model_dir, "updatable_creation.mlmodel")
print(model_path)
save_spec(builder.spec, model_path)
mlmodel = MLModel(model_path)
self.assertTrue(mlmodel is not None)
spec = mlmodel.get_spec()
self.assertEqual(spec.description.trainingInput[0].name, "input")
self.assertEqual(
spec.description.trainingInput[0].type.WhichOneof("Type"),
"multiArrayType")
self.assertEqual(spec.description.trainingInput[1].name, "output_true")
self.assertEqual(
spec.description.trainingInput[1].type.WhichOneof("Type"),
"multiArrayType")
def get_custom_model_spec():
from coremltools.models.neural_network import NeuralNetworkBuilder
from coremltools.models.datatypes import Array
input_name = "output1"
input_length = self._feature_extractor.output_length
builder = NeuralNetworkBuilder(
[(input_name, Array(input_length,))],
[(prob_name, Array(self.num_classes,))],
"classifier",
)
layer_counter = [0]
builder.set_input([input_name], [(input_length,)])
def next_layer_name():
layer_counter[0] += 1
return "layer_%d" % layer_counter[0]
for i, cur_layer in enumerate(self._custom_classifier.export_weights()):
W = cur_layer["weight"]
nC, nB = W.shape
Wb = cur_layer["bias"]
output_name = next_layer_name()
builder.add_inner_product(
name="inner_product_" + str(i),
W=W,
b=Wb,
input_channels=nB,
output_channels=nC,
has_bias=True,
input_name=input_name,
output_name=output_name,
)
input_name = output_name
if cur_layer["act"]:
output_name = next_layer_name()
builder.add_activation(
"activation" + str(i), "RELU", input_name, output_name
)
input_name = output_name
builder.add_softmax("softmax", input_name, prob_name)
builder.set_class_labels(
self.classes,
predicted_feature_name=self.target,
prediction_blob=prob_name,
)
return builder.spec
def test_inner_product_converter(self):
input_dim = (3,)
output_dim = (2,)
input = [('input', datatypes.Array(*input_dim))]
output = [('output', datatypes.Array(*output_dim))]
weights = numpy.zeros(shape=(3, 2))
weights[:] = [[1, 2], [3, 4], [5, 6]]
bias = numpy.zeros(shape=(2))
bias[:] = [-100, 100]
builder = NeuralNetworkBuilder(input, output)
builder.add_inner_product(name='FC', W=weights, b=bias, input_channels=3, output_channels=2, has_bias=True,
input_name='input', output_name='output')
model_onnx = convert_coreml(builder.spec)
self.assertTrue(model_onnx is not None)
def _build_nn_with_one_ip_layer(self):
input_features = [('data', datatypes.Array(3))]
output_features = [('out', None)]
builder = NeuralNetworkBuilder(input_features,
output_features,
disable_rank5_shape_mapping=True)
w = np.random.uniform(-0.5, 0.5, (3, 3))
builder.add_inner_product(name='ip1',
W=w,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name='input',
output_name='hidden')
return builder
def get_custom_model_spec():
from coremltools.models.neural_network import NeuralNetworkBuilder
from coremltools.models.datatypes import Array, Dictionary, String
input_name = 'output1'
input_length = self._feature_extractor.output_length
builder = NeuralNetworkBuilder(
[(input_name, Array(input_length, ))],
[(prob_name, Dictionary(String))], 'classifier')
input_name, output_name = input_name, 0
for i, cur_layer in enumerate(
self._custom_classifier.export_weights()):
W = cur_layer['weight']
nC, nB = W.shape
Wb = cur_layer['bias']
builder.add_inner_product(name="inner_product_" + str(i),
W=W,
b=Wb,
input_channels=nB,
output_channels=nC,
has_bias=True,
input_name=str(input_name),
output_name='inner_product_' +
str(output_name))
if cur_layer['act']:
builder.add_activation("activation" + str(i), 'RELU',
'inner_product_' + str(output_name),
str(output_name))
input_name = i
output_name = i + 1
last_output = builder.spec.neuralNetworkClassifier.layers[
-1].output[0]
builder.add_softmax('softmax', last_output, self.target)
builder.set_class_labels(self.classes,
predicted_feature_name=self.target)
builder.set_input([input_name], [(input_length, )])
builder.set_output([self.target], [(self.num_classes, )])
return builder.spec
def test_undefined_shape_single_output(self):
W = np.ones((3, 3))
input_features = [('data', datatypes.Array(3))]
output_features = [('probs', None)]
builder = NeuralNetworkBuilder(input_features, output_features)
builder.add_inner_product(name='ip1',
W=W,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name='data',
output_name='probs')
mlmodel = MLModel(builder.spec)
data = np.ones((3, ))
data_dict = {'data': data}
probs = mlmodel.predict(data_dict)['probs']
self.assertTrue(np.allclose(probs, np.ones(3) * 3))
def _test_use_float_array_helper(self, use_float_arraytype):
input_features = [("data", datatypes.Array(3))]
output_features = [("probs", None)]
builder = NeuralNetworkBuilder(
input_features=input_features,
output_features=output_features,
use_float_arraytype=use_float_arraytype,
)
weights = np.ones((3, 3))
builder.add_inner_product(
name="ip1",
W=weights,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name="data",
output_name="probs",
)
spec = builder.spec
array_feature_type = (
coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.FLOAT32
if use_float_arraytype else
coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.DOUBLE)
for input in spec.description.input:
self.assertEqual(input.type.multiArrayType.dataType,
array_feature_type)
for output in spec.description.input:
self.assertEqual(output.type.multiArrayType.dataType,
array_feature_type)
# Assert that the generated spec is functional
mlmodel = MLModel(spec)
data = np.ones((3, ))
data_dict = {"data": data}
try:
predictions = mlmodel.predict(data_dict)
except Exception as e:
self.fail(e)
self.assertTrue(np.allclose(predictions["probs"], np.ones(3) * 3))
def test_inner_product_converter(self):
input_dim = (3, )
output_dim = (2, )
input = [('input', datatypes.Array(*input_dim))]
output = [('output', datatypes.Array(*output_dim))]
weights = numpy.zeros(shape=(3, 2))
weights[:] = [[1, 2], [3, 4], [5, 6]]
bias = numpy.zeros(shape=(2))
bias[:] = [-100, 100]
builder = NeuralNetworkBuilder(input, output)
builder.add_inner_product(name='FC',
W=weights,
b=bias,
input_channels=3,
output_channels=2,
has_bias=True,
input_name='input',
output_name='output')
context = ConvertContext()
node = InnerProductLayerConverter.convert(
context, builder.spec.neuralNetwork.layers[0], ['input'],
['output'])
self.assertTrue(node is not None)
#!/usr/bin/env python3
import numpy as np
import coremltools as ct
from coremltools.models.neural_network import datatypes, NeuralNetworkBuilder
input_features = [('image', datatypes.Array(3))]
output_features = [('probs', datatypes.Array(2))]
weights = np.zeros((3, 2)) + 3
bias = np.ones(2)
builder = NeuralNetworkBuilder(input_features, output_features)
builder.add_inner_product(name='ip_layer', W=weights, b=bias, input_channels=3, output_channels=2, has_bias=True, input_name='image', output_name='probs')
# compile the spec
mlmodel = ct.models.MLModel(builder.spec)
# trigger the ANE!
out = mlmodel.predict({"image": np.array([1337,0,0], dtype=np.float32)})
print(out)
mlmodel.save('test.mlmodel')
#!/usr/bin/env python3
import numpy as np
import coremltools as ct
from coremltools.models.neural_network import datatypes, NeuralNetworkBuilder
input_features = [('image', datatypes.Array(3))]
output_features = [('probs', datatypes.Array(3))]
weights = np.zeros((3, 3)) + 3
bias = np.ones(3)
builder = NeuralNetworkBuilder(input_features, output_features)
builder.add_inner_product(name='ip_layer',
W=weights,
b=None,
input_channels=3,
output_channels=3,
has_bias=False,
input_name='image',
output_name='med')
#builder.add_inner_product(name='ip_layer_2', W=weights, b=None, input_channels=3, output_channels=3, has_bias=False, input_name='med', output_name='probs')
#builder.add_elementwise(name='element', input_names=['med', 'med'], output_name='probs', mode='ADD')
builder.add_bias(name='bias',
b=bias,
input_name='med',
output_name='probs',
shape_bias=(3, ))
#builder.add_activation(name='act_layer', non_linearity='SIGMOID', input_name='med', output_name='probs')
# compile the spec
mlmodel = ct.models.MLModel(builder.spec)
def make_mlmodel(variables):
# Specify the inputs and outputs (there can be multiple).
# Each name corresponds to the input_name/output_name of a layer in the network so
# that Core ML knows where to insert and extract data.
input_features = [('image', datatypes.Array(1, IMAGE_HEIGHT, IMAGE_WIDTH))]
output_features = [('labelValues', datatypes.Array(NUM_LABEL_INDEXES))]
builder = NeuralNetworkBuilder(input_features, output_features, mode=None)
# The "name" parameter has no effect on the function of the network. As far as I know
# it's only used when Xcode fails to load your mlmodel and gives you an error telling
# you what the problem is.
# The input_names and output_name are used to link layers to each other and to the
# inputs and outputs of the model. When adding or removing layers, or renaming their
# outputs, always make sure you correct the input and output names of the layers
# before and after them.
builder.add_elementwise(name='add_layer',
input_names=['image'],
output_name='add_layer',
mode='ADD',
alpha=-0.5)
# Although Core ML internally uses weight matrices of shape
# (outputChannels, inputChannels, height, width) (as can be found by looking at the
# protobuf specification comments), add_convolution takes the shape
# (height, width, inputChannels, outputChannels) (as can be found in the coremltools
# documentation). The latter shape matches what TensorFlow uses so we don't need to
# reorder the matrix axes ourselves.
builder.add_convolution(name='conv2d_1',
kernel_channels=1,
output_channels=32,
height=3,
width=3,
stride_height=1,
stride_width=1,
border_mode='same',
groups=0,
W=variables['W_conv1'].eval(),
b=variables['b_conv1'].eval(),
has_bias=True,
is_deconv=False,
output_shape=None,
input_name='add_layer',
output_name='conv2d_1')
builder.add_activation(name='relu_1',
non_linearity='RELU',
input_name='conv2d_1',
output_name='relu_1',
params=None)
builder.add_pooling(name='maxpool_1',
height=2,
width=2,
stride_height=2,
stride_width=2,
layer_type='MAX',
padding_type='SAME',
input_name='relu_1',
output_name='maxpool_1')
# ...
builder.add_flatten(name='maxpool_3_flat',
mode=1,
input_name='maxpool_3',
output_name='maxpool_3_flat')
# We must swap the axes of the weight matrix because add_inner_product takes the shape
# (outputChannels, inputChannels) whereas TensorFlow uses
# (inputChannels, outputChannels). Unlike with add_convolution (see the comment
# above), the shape add_inner_product expects matches what the protobuf specification
# requires for inner products.
builder.add_inner_product(name='fc1',
W=tf_fc_weights_order_to_mlmodel(
variables['W_fc1'].eval()).flatten(),
b=variables['b_fc1'].eval().flatten(),
input_channels=6 * 6 * 64,
output_channels=1024,
has_bias=True,
input_name='maxpool_3_flat',
output_name='fc1')
# ...
builder.add_softmax(name='softmax',
input_name='fc2',
output_name='labelValues')
model = MLModel(builder.spec)
model.short_description = 'Model for recognizing a variety of images drawn on screen with one\'s finger'
model.input_description['image'] = 'A gesture image to classify'
model.output_description[
'labelValues'] = 'The "probability" of each label, in a dense array'
return model
input_max_size = 128
input_dim = (input_max_size, )
output_dim = (1, )
input_features = [('data', datatypes.Array(*input_dim))]
output_features = [('result', datatypes.Array(*output_dim))]
weights = np.random.rand(1, input_max_size)
bias = np.random.rand(1)
builder = NeuralNetworkBuilder(input_features, output_features)
builder.add_inner_product(name='ip_layer',
W=weights,
b=bias,
input_channels=input_max_size,
output_channels=1,
has_bias=True,
input_name='data',
output_name='result')
builder.make_updatable(['ip_layer'])
builder.set_mean_squared_error_loss(name='lossLayer',
input_feature=output_features[0])
optimizerParams = SgdParams(lr=0.01, batch=1)
optimizerParams.set_batch(1, allowed_set=[1, 2, 4, 8, 16, 32])
builder.set_sgd_optimizer(optimizerParams)
builder.set_epochs(16, allowed_set=[2, 4, 8, 16, 32, 64, 128, 256])
#builder.spec = convert_neural_network_spec_weights_to_fp16(builder.spec)
