def export_coreml(self, filename):
"""
Export the model in Core ML format.
Parameters
----------
filename: str
A valid filename where the model can be saved.
Examples
--------
>>> model.export_coreml("MyModel.mlmodel")
"""
from turicreate.toolkits import _coreml_utils
display_name = "random forest classifier"
short_description = _coreml_utils._mlmodel_short_description(
display_name)
context = {
"mode": "classification",
"model_type": "random_forest",
"version": _turicreate.__version__,
"class": self.__class__.__name__,
"short_description": short_description
}
self._export_coreml_impl(filename, context)
def export_coreml(self, filename):
"""
Export the model in Core ML format.
Parameters
----------
filename: str
A valid filename where the model can be saved.
Examples
--------
>>> model.export_coreml("MyModel.mlmodel")
"""
from turicreate.extensions import _linear_svm_export_as_model_asset
from turicreate.toolkits import _coreml_utils
display_name = "svm classifier"
short_description = _coreml_utils._mlmodel_short_description(display_name)
context = {
"class": self.__class__.__name__,
"short_description": short_description,
}
context["user_defined"] = _coreml_utils._get_model_metadata(
self.__class__.__name__, None
)
_linear_svm_export_as_model_asset(self.__proxy__, filename, context)
def export_coreml(self, filename):
"""
Export the model in Core ML format.
Parameters
----------
filename: str
A valid filename where the model can be saved.
Examples
--------
>>> model.export_coreml("MyModel.mlmodel")
"""
from turicreate.extensions import _logistic_classifier_export_as_model_asset
from turicreate.toolkits import _coreml_utils
display_name = 'text classifier'
short_description = _coreml_utils._mlmodel_short_description(
display_name)
context = {
'class': self.__class__.__name__,
'version': _tc.__version__,
'short_description': short_description,
'user_defined': {
'turicreate_version': _tc.__version__
}
}
model = self.__proxy__['classifier'].__proxy__
_logistic_classifier_export_as_model_asset(model, filename, context)
def export_coreml(self, filename):
"""
Export the model in Core ML format.
Parameters
----------
filename: str
A valid filename where the model can be saved.
Examples
--------
>>> model.export_coreml("MyModel.mlmodel")
"""
from turicreate.extensions import _linear_regression_export_as_model_asset
from turicreate.toolkits import _coreml_utils
display_name = "linear regression"
short_description = _coreml_utils._mlmodel_short_description(
display_name)
context = {
"class": self.__class__.__name__,
"version": _turicreate.__version__,
"short_description": short_description
}
_linear_regression_export_as_model_asset(self.__proxy__, filename,
context)
def export_coreml(self, filename):
"""
Export the model in Core ML format.
Parameters
----------
filename: str
A valid filename where the model can be saved.
Examples
--------
>>> model.export_coreml("MyModel.mlmodel")
"""
from turicreate.toolkits import _coreml_utils
display_name = "boosted trees regression"
short_description = _coreml_utils._mlmodel_short_description(
display_name)
context = {
"mode": "regression",
"model_type": "boosted_trees",
"class": self.__class__.__name__,
"short_description": short_description,
}
self._export_coreml_impl(filename, context)
def _set_inputs_outputs_and_metadata(spec, nn_spec):
# Replace the classifier with the new classes
class_labels = self.classifier.classes
probOutput = spec.description.output[0]
classLabel = spec.description.output[1]
probOutput.type.dictionaryType.MergeFromString(b'')
if type(class_labels[0]) == int:
nn_spec.ClearField('int64ClassLabels')
probOutput.type.dictionaryType.int64KeyType.MergeFromString(
b'')
classLabel.type.int64Type.MergeFromString(b'')
del nn_spec.int64ClassLabels.vector[:]
for c in class_labels:
nn_spec.int64ClassLabels.vector.append(c)
else:
nn_spec.ClearField('stringClassLabels')
probOutput.type.dictionaryType.stringKeyType.MergeFromString(
b'')
classLabel.type.stringType.MergeFromString(b'')
del nn_spec.stringClassLabels.vector[:]
for c in class_labels:
nn_spec.stringClassLabels.vector.append(c)
prob_name = self.target + 'Probability'
label_name = self.target
old_output_name = nn_spec.layers[-1].name
coremltools.models.utils.rename_feature(spec, 'classLabel',
label_name)
coremltools.models.utils.rename_feature(spec, old_output_name,
prob_name)
if nn_spec.layers[-1].name == old_output_name:
nn_spec.layers[-1].name = prob_name
if nn_spec.labelProbabilityLayerName == old_output_name:
nn_spec.labelProbabilityLayerName = prob_name
coremltools.models.utils.rename_feature(spec, 'data', self.feature)
if len(nn_spec.preprocessing) > 0:
nn_spec.preprocessing[0].featureName = self.feature
mlmodel = coremltools.models.MLModel(spec)
model_type = 'image classifier (%s)' % self.model
mlmodel.short_description = _coreml_utils._mlmodel_short_description(
model_type)
mlmodel.input_description[self.feature] = u'Input image'
mlmodel.output_description[prob_name] = 'Prediction probabilities'
mlmodel.output_description[
label_name] = 'Class label of top prediction'
_coreml_utils._set_model_metadata(
mlmodel,
self.__class__.__name__, {
'model': self.model,
'target': self.target,
'features': self.feature,
'max_iterations': str(self.max_iterations),
},
version=ImageClassifier._PYTHON_IMAGE_CLASSIFIER_VERSION)
return mlmodel
def export_coreml(self, filename):
"""
Export the model in Core ML format.
Parameters
----------
filename: str
A valid filename where the model can be saved.
Examples
--------
>>> model.export_coreml("MyModel.mlmodel")
"""
additional_user_defined_metadata = _coreml_utils._get_tc_version_info()
short_description = _coreml_utils._mlmodel_short_description(
"Drawing Classifier")
self.__proxy__.export_to_coreml(filename, short_description,
additional_user_defined_metadata)
def export_coreml(self,
filename,
image_shape=(256, 256),
include_flexible_shape=True):
"""
Save the model in Core ML format. The Core ML model takes an image of
fixed size, and a style index inputs and produces an output
of an image of fixed size
Parameters
----------
path : string
A string to the path for saving the Core ML model.
image_shape: tuple
A tuple (defaults to (256, 256)) will bind the coreml model to a fixed shape.
include_flexible_shape: bool
Allows the size of the input image to be flexible. Any input image were the
height and width are at least 64 will be accepted by the Core ML Model.
See Also
--------
save
Examples
--------
>>> model.export_coreml('StyleTransfer.mlmodel')
"""
options = {}
options['image_width'] = image_shape[1]
options['image_height'] = image_shape[0]
options['include_flexible_shape'] = include_flexible_shape
additional_user_defined_metadata = _coreml_utils._get_tc_version_info()
short_description = _coreml_utils._mlmodel_short_description(
'Style Transfer')
self.__proxy__.export_to_coreml(filename, short_description,
additional_user_defined_metadata,
options)
def export_coreml(self, filename):
"""
Save the model in Core ML format.
See Also
--------
save
Examples
--------
>>> model.export_coreml('myModel.mlmodel')
"""
import coremltools as _cmt
import mxnet as _mx
from ._model_architecture import _define_model, _fit_model, _net_params
prob_name = self.target + 'Probability'
label_name = self.target
input_features = [
('features', _cmt.models.datatypes.Array(*(1, self.prediction_window, self.num_features)))
]
output_features = [
(prob_name, _cmt.models.datatypes.Array(*(self.num_classes,)))
]
model_params = self._pred_model.get_params()
weights = {k: v.asnumpy() for k, v in model_params[0].items()}
weights = _mx.rnn.LSTMCell(num_hidden=_net_params['lstm_h']).unpack_weights(weights)
moving_weights = {k: v.asnumpy() for k, v in model_params[1].items()}
builder = _cmt.models.neural_network.NeuralNetworkBuilder(
input_features,
output_features,
mode='classifier'
)
# Conv
# (1,1,W,C) -> (1,C,1,W)
builder.add_permute(name='permute_layer', dim=(0, 3, 1, 2),
input_name='features', output_name='conv_in')
W = _np.expand_dims(weights['conv_weight'], axis=0).transpose((2, 3, 1, 0))
builder.add_convolution(name='conv_layer',
kernel_channels=self.num_features,
output_channels=_net_params['conv_h'],
height=1, width=self.prediction_window,
stride_height=1, stride_width=self.prediction_window,
border_mode='valid', groups=1,
W=W, b=weights['conv_bias'], has_bias=True,
input_name='conv_in', output_name='relu0_in')
builder.add_activation(name='relu_layer0', non_linearity='RELU',
input_name='relu0_in', output_name='lstm_in')
# LSTM
builder.add_optionals([('lstm_h_in', _net_params['lstm_h']),
('lstm_c_in', _net_params['lstm_h'])],
[('lstm_h_out', _net_params['lstm_h']),
('lstm_c_out', _net_params['lstm_h'])])
W_x = [weights['lstm_i2h_i_weight'], weights['lstm_i2h_f_weight'],
weights['lstm_i2h_o_weight'], weights['lstm_i2h_c_weight']]
W_h = [weights['lstm_h2h_i_weight'], weights['lstm_h2h_f_weight'],
weights['lstm_h2h_o_weight'], weights['lstm_h2h_c_weight']]
bias = [weights['lstm_h2h_i_bias'], weights['lstm_h2h_f_bias'],
weights['lstm_h2h_o_bias'], weights['lstm_h2h_c_bias']]
builder.add_unilstm(name='lstm_layer',
W_h=W_h, W_x=W_x, b=bias,
input_size=_net_params['conv_h'],
hidden_size=_net_params['lstm_h'],
input_names=['lstm_in', 'lstm_h_in', 'lstm_c_in'],
output_names=['dense0_in', 'lstm_h_out', 'lstm_c_out'],
inner_activation='SIGMOID')
# Dense
builder.add_inner_product(name='dense_layer',
W=weights['dense0_weight'], b=weights['dense0_bias'],
input_channels=_net_params['lstm_h'],
output_channels=_net_params['dense_h'],
has_bias=True,
input_name='dense0_in',
output_name='bn_in')
builder.add_batchnorm(name='bn_layer',
channels=_net_params['dense_h'],
gamma=weights['bn_gamma'], beta=weights['bn_beta'],
mean=moving_weights['bn_moving_mean'],
variance=moving_weights['bn_moving_var'],
input_name='bn_in', output_name='relu1_in',
epsilon=0.001)
builder.add_activation(name='relu_layer1', non_linearity='RELU',
input_name='relu1_in', output_name='dense1_in')
# Softmax
builder.add_inner_product(name='dense_layer1',
W=weights['dense1_weight'], b=weights['dense1_bias'],
has_bias=True,
input_channels=_net_params['dense_h'],
output_channels=self.num_classes,
input_name='dense1_in', output_name='softmax_in')
builder.add_softmax(name=prob_name,
input_name='softmax_in',
output_name=prob_name)
labels = list(map(str, sorted(self._target_id_map.keys())))
builder.set_class_labels(labels)
mlmodel = _cmt.models.MLModel(builder.spec)
model_type = 'activity classifier'
mlmodel.short_description = _coreml_utils._mlmodel_short_description(model_type)
# Add useful information to the mlmodel
features_str = ', '.join(self.features)
mlmodel.input_description['features'] = u'Window \xd7 [%s]' % features_str
mlmodel.input_description['lstm_h_in'] = 'LSTM hidden state input'
mlmodel.input_description['lstm_c_in'] = 'LSTM cell state input'
mlmodel.output_description[prob_name] = 'Activity prediction probabilities'
mlmodel.output_description['classLabel'] = 'Class label of top prediction'
mlmodel.output_description['lstm_h_out'] = 'LSTM hidden state output'
mlmodel.output_description['lstm_c_out'] = 'LSTM cell state output'
_coreml_utils._set_model_metadata(mlmodel, self.__class__.__name__, {
'prediction_window': str(self.prediction_window),
'session_id': self.session_id,
'target': self.target,
'features': ','.join(self.features),
'max_iterations': str(self.max_iterations),
}, version=ActivityClassifier._PYTHON_ACTIVITY_CLASSIFIER_VERSION)
spec = mlmodel.get_spec()
_cmt.models.utils.rename_feature(spec, 'classLabel', label_name)
_cmt.models.utils.rename_feature(spec, 'lstm_h_in', 'hiddenIn')
_cmt.models.utils.rename_feature(spec, 'lstm_c_in', 'cellIn')
_cmt.models.utils.rename_feature(spec, 'lstm_h_out', 'hiddenOut')
_cmt.models.utils.rename_feature(spec, 'lstm_c_out', 'cellOut')
_cmt.utils.save_spec(spec, filename)
def _set_inputs_outputs_and_metadata(spec, nn_spec):
# Replace the classifier with the new classes
class_labels = self.classifier.classes
probOutput = spec.description.output[0]
classLabel = spec.description.output[1]
probOutput.type.dictionaryType.MergeFromString(b"")
if type(class_labels[0]) == int:
nn_spec.ClearField("int64ClassLabels")
probOutput.type.dictionaryType.int64KeyType.MergeFromString(
b"")
classLabel.type.int64Type.MergeFromString(b"")
del nn_spec.int64ClassLabels.vector[:]
for c in class_labels:
nn_spec.int64ClassLabels.vector.append(c)
else:
nn_spec.ClearField("stringClassLabels")
probOutput.type.dictionaryType.stringKeyType.MergeFromString(
b"")
classLabel.type.stringType.MergeFromString(b"")
del nn_spec.stringClassLabels.vector[:]
for c in class_labels:
nn_spec.stringClassLabels.vector.append(c)
prob_name = self.target + "Probability"
label_name = self.target
old_output_name = nn_spec.layers[-1].name
coremltools.models.utils.rename_feature(spec, "classLabel",
label_name)
coremltools.models.utils.rename_feature(spec, old_output_name,
prob_name)
if nn_spec.layers[-1].name == old_output_name:
nn_spec.layers[-1].name = prob_name
if nn_spec.labelProbabilityLayerName == old_output_name:
nn_spec.labelProbabilityLayerName = prob_name
coremltools.models.utils.rename_feature(spec, "data", self.feature)
if len(nn_spec.preprocessing) > 0:
nn_spec.preprocessing[0].featureName = self.feature
mlmodel = coremltools.models.MLModel(spec)
model_type = "image classifier (%s)" % self.model
mlmodel.short_description = _coreml_utils._mlmodel_short_description(
model_type)
mlmodel.input_description[self.feature] = u"Input image"
mlmodel.output_description[prob_name] = "Prediction probabilities"
mlmodel.output_description[
label_name] = "Class label of top prediction"
model_metadata = {
"model": self.model,
"target": self.target,
"features": self.feature,
"max_iterations": str(self.max_iterations),
}
user_defined_metadata = model_metadata.update(
_coreml_utils._get_tc_version_info())
_coreml_utils._set_model_metadata(
mlmodel,
self.__class__.__name__,
user_defined_metadata,
version=ImageClassifier._PYTHON_IMAGE_CLASSIFIER_VERSION,
)
return mlmodel
def export_coreml(self, filename):
"""
Save the model in Core ML format.
See Also
--------
save
Examples
--------
>>> model.export_coreml('myModel.mlmodel')
"""
coreml_model = self.feature_extractor.get_coreml_model()
spec = coreml_model.get_spec()
nn_spec = spec.neuralNetworkClassifier
num_classes = self.num_classes
# Replace the softmax layer with new coeffients
fc_layer = nn_spec.layers[-2]
fc_layer_params = fc_layer.innerProduct
fc_layer_params.outputChannels = self.classifier.num_classes
inputChannels = fc_layer_params.inputChannels
fc_layer_params.hasBias = True
coefs = self.classifier.coefficients
weights = fc_layer_params.weights
bias = fc_layer_params.bias
del weights.floatValue[:]
del bias.floatValue[:]
import numpy as np
W = np.array(coefs[coefs['index'] != None]['value'],
ndmin=2).reshape(inputChannels,
num_classes - 1,
order='F')
b = coefs[coefs['index'] == None]['value']
Wa = np.hstack((np.zeros((inputChannels, 1)), W))
weights.floatValue.extend(Wa.flatten(order='F'))
bias.floatValue.extend([0.0] + list(b))
# Replace the classifier with the new classes
class_labels = self.classifier.classes
probOutput = spec.description.output[0]
classLabel = spec.description.output[1]
probOutput.type.dictionaryType.MergeFromString(b'')
if type(class_labels[0]) == int:
nn_spec.ClearField('int64ClassLabels')
probOutput.type.dictionaryType.int64KeyType.MergeFromString(b'')
classLabel.type.int64Type.MergeFromString(b'')
del nn_spec.int64ClassLabels.vector[:]
for c in class_labels:
nn_spec.int64ClassLabels.vector.append(c)
else:
nn_spec.ClearField('stringClassLabels')
probOutput.type.dictionaryType.stringKeyType.MergeFromString(b'')
classLabel.type.stringType.MergeFromString(b'')
del nn_spec.stringClassLabels.vector[:]
for c in class_labels:
nn_spec.stringClassLabels.vector.append(c)
import coremltools
prob_name = self.target + 'Probability'
label_name = self.target
old_output_name = spec.neuralNetworkClassifier.layers[-1].name
coremltools.models.utils.rename_feature(spec, 'classLabel', label_name)
coremltools.models.utils.rename_feature(spec, old_output_name,
prob_name)
if spec.neuralNetworkClassifier.layers[-1].name == old_output_name:
spec.neuralNetworkClassifier.layers[-1].name = prob_name
if spec.neuralNetworkClassifier.labelProbabilityLayerName == old_output_name:
spec.neuralNetworkClassifier.labelProbabilityLayerName = prob_name
coremltools.models.utils.rename_feature(spec, 'data', self.feature)
spec.neuralNetworkClassifier.preprocessing[
0].featureName = self.feature
mlmodel = coremltools.models.MLModel(spec)
model_type = 'image classifier (%s)' % self.model
mlmodel.short_description = _coreml_utils._mlmodel_short_description(
model_type)
mlmodel.input_description[self.feature] = u'Input image'
mlmodel.output_description[prob_name] = 'Prediction probabilities'
mlmodel.output_description[
label_name] = 'Class label of top prediction'
_coreml_utils._set_model_metadata(
mlmodel,
self.__class__.__name__, {
'model': self.model,
'target': self.target,
'features': self.feature,
'max_iterations': str(self.max_iterations),
},
version=ImageClassifier._PYTHON_IMAGE_CLASSIFIER_VERSION)
mlmodel.save(filename)
def export_coreml(self, filename):
"""
Save the model in Core ML format.
See Also
--------
save
Examples
--------
>>> model.export_coreml('./myModel.mlmodel')
"""
import coremltools
from coremltools.proto.FeatureTypes_pb2 import ArrayFeatureType
prob_name = self.target + "Probability"
def get_custom_model_spec():
from coremltools.models.neural_network import NeuralNetworkBuilder
from coremltools.models.datatypes import Array
input_name = "output1"
input_length = self._feature_extractor.output_length
builder = NeuralNetworkBuilder(
[(input_name, Array(input_length,))],
[(prob_name, Array(self.num_classes,))],
"classifier",
)
layer_counter = [0]
builder.set_input([input_name], [(input_length,)])
def next_layer_name():
layer_counter[0] += 1
return "layer_%d" % layer_counter[0]
for i, cur_layer in enumerate(self._custom_classifier.export_weights()):
W = cur_layer["weight"]
nC, nB = W.shape
Wb = cur_layer["bias"]
output_name = next_layer_name()
builder.add_inner_product(
name="inner_product_" + str(i),
W=W,
b=Wb,
input_channels=nB,
output_channels=nC,
has_bias=True,
input_name=input_name,
output_name=output_name,
)
input_name = output_name
if cur_layer["act"]:
output_name = next_layer_name()
builder.add_activation(
"activation" + str(i), "RELU", input_name, output_name
)
input_name = output_name
builder.add_softmax("softmax", input_name, prob_name)
builder.set_class_labels(
self.classes,
predicted_feature_name=self.target,
prediction_blob=prob_name,
)
return builder.spec
top_level_spec = coremltools.proto.Model_pb2.Model()
top_level_spec.specificationVersion = 3
# Set input
desc = top_level_spec.description
input = desc.input.add()
input.name = self.feature
assert type(self.feature) is str
input.type.multiArrayType.dataType = ArrayFeatureType.ArrayDataType.Value(
"FLOAT32"
)
input.type.multiArrayType.shape.append(15600)
# Set outputs
prob_output = desc.output.add()
prob_output.name = prob_name
label_output = desc.output.add()
label_output.name = self.target
desc.predictedFeatureName = self.target
desc.predictedProbabilitiesName = prob_name
if type(self.classes[0]) == int:
# Class labels are ints
prob_output.type.dictionaryType.int64KeyType.MergeFromString(b"")
label_output.type.int64Type.MergeFromString(b"")
else: # Class are strings
prob_output.type.dictionaryType.stringKeyType.MergeFromString(b"")
label_output.type.stringType.MergeFromString(b"")
# Set metadata
user_metadata = desc.metadata.userDefined
user_metadata["sampleRate"] = str(self._feature_extractor.input_sample_rate)
pipeline = top_level_spec.pipelineClassifier.pipeline
# Add the preprocessing model
preprocessing_model = pipeline.models.add()
preprocessing_model.customModel.className = "TCSoundClassifierPreprocessing"
preprocessing_model.specificationVersion = 3
preprocessing_input = preprocessing_model.description.input.add()
preprocessing_input.CopyFrom(input)
preprocessed_output = preprocessing_model.description.output.add()
preprocessed_output.name = "preprocessed_data"
preprocessed_output.type.multiArrayType.dataType = ArrayFeatureType.ArrayDataType.Value(
"DOUBLE"
)
preprocessed_output.type.multiArrayType.shape.append(1)
preprocessed_output.type.multiArrayType.shape.append(96)
preprocessed_output.type.multiArrayType.shape.append(64)
# Add the feature extractor, updating its input name
feature_extractor_spec = self._feature_extractor.get_spec()
pipeline.models.add().CopyFrom(feature_extractor_spec)
pipeline.models[-1].description.input[0].name = preprocessed_output.name
pipeline.models[-1].neuralNetwork.layers[0].input[0] = preprocessed_output.name
# Add the custom neural network
pipeline.models.add().CopyFrom(get_custom_model_spec())
# Set key type for the probability dictionary
prob_output_type = pipeline.models[-1].description.output[0].type.dictionaryType
if type(self.classes[0]) == int:
prob_output_type.int64KeyType.MergeFromString(b"")
else: # String labels
prob_output_type.stringKeyType.MergeFromString(b"")
mlmodel = coremltools.models.MLModel(top_level_spec)
model_type = "sound classifier"
mlmodel.short_description = _coreml_utils._mlmodel_short_description(model_type)
mlmodel.input_description[self.feature] = u"Input audio features"
mlmodel.output_description[prob_name] = "Prediction probabilities"
mlmodel.output_description[self.target] = "Class label of top prediction"
model_metadata = {
"target": self.target,
"feature": self.feature,
}
user_defined_metadata = model_metadata.update(
_coreml_utils._get_tc_version_info()
)
_coreml_utils._set_model_metadata(
mlmodel,
self.__class__.__name__,
user_defined_metadata,
version=SoundClassifier._PYTHON_SOUND_CLASSIFIER_VERSION,
)
mlmodel.save(filename)
def export_coreml(self, filename, verbose=False):
"""
Save the model in Core ML format. The Core ML model takes a grayscale
drawing of fixed size as input and produces two outputs:
`classLabel` and `labelProbabilities`.
The first one, `classLabel` is an integer or string (depending on the
classes the model was trained on) to store the label of the top
prediction by the model.
The second one, `labelProbabilities`, is a dictionary with all the
class labels in the dataset as the keys, and their respective
probabilities as the values.
See Also
--------
save
Parameters
----------
filename : string
The path of the file where we want to save the Core ML model.
verbose : bool optional
If True, prints export progress.
Examples
--------
>>> model.export_coreml('drawing_classifier.mlmodel')
"""
import mxnet as _mx
from .._mxnet._mxnet_to_coreml import _mxnet_converter
import coremltools as _coremltools
batch_size = 1
image_shape = (batch_size,) + (1, BITMAP_WIDTH, BITMAP_HEIGHT)
s_image = _mx.sym.Variable(self.feature,
shape=image_shape, dtype=_np.float32)
from copy import copy as _copy
net = _copy(self._model)
s_ymap = net(s_image)
mod = _mx.mod.Module(symbol=s_ymap, label_names=None, data_names=[self.feature])
mod.bind(for_training=False, data_shapes=[(self.feature, image_shape)])
mod.init_params()
arg_params, aux_params = mod.get_params()
net_params = net.collect_params()
new_arg_params = {}
for k, param in arg_params.items():
new_arg_params[k] = net_params[k].data(net_params[k].list_ctx()[0])
new_aux_params = {}
for k, param in aux_params.items():
new_aux_params[k] = net_params[k].data(net_params[k].list_ctx()[0])
mod.set_params(new_arg_params, new_aux_params)
coreml_model = _mxnet_converter.convert(mod, mode='classifier',
class_labels=self.classes,
input_shape=[(self.feature, image_shape)],
builder=None, verbose=verbose,
preprocessor_args={
'image_input_names': [self.feature],
'image_scale': 1.0/255
})
DESIRED_OUTPUT_NAME = self.target + "Probabilities"
spec = coreml_model._spec
class_label_output_index = 0 if spec.description.output[0].name == "classLabel" else 1
probabilities_output_index = 1-class_label_output_index
spec.neuralNetworkClassifier.labelProbabilityLayerName = DESIRED_OUTPUT_NAME
spec.neuralNetworkClassifier.layers[-1].name = DESIRED_OUTPUT_NAME
spec.neuralNetworkClassifier.layers[-1].output[0] = DESIRED_OUTPUT_NAME
spec.description.predictedProbabilitiesName = DESIRED_OUTPUT_NAME
spec.description.output[probabilities_output_index].name = DESIRED_OUTPUT_NAME
from turicreate.toolkits import _coreml_utils
model_type = "drawing classifier"
spec.description.metadata.shortDescription = _coreml_utils._mlmodel_short_description(model_type)
spec.description.input[0].shortDescription = self.feature
spec.description.output[probabilities_output_index].shortDescription = 'Prediction probabilities'
spec.description.output[class_label_output_index].shortDescription = 'Class Label of Top Prediction'
from coremltools.models.utils import save_spec as _save_spec
_save_spec(spec, filename)
def export_coreml(self, path, image_shape=(256, 256),
include_flexible_shape=True):
"""
Save the model in Core ML format. The Core ML model takes an image of
fixed size, and a style index inputs and produces an output
of an image of fixed size
Parameters
----------
path : string
A string to the path for saving the Core ML model.
image_shape: tuple
A tuple (defaults to (256, 256)) will bind the coreml model to a fixed shape.
include_flexible_shape: bool
A boolean value indicating whether flexible_shape should be included or not.
See Also
--------
save
Examples
--------
>>> model.export_coreml('StyleTransfer.mlmodel')
"""
import mxnet as _mx
from .._mxnet_to_coreml import _mxnet_converter
import coremltools
transformer = self._model
index = _mx.sym.Variable("index", shape=(1,), dtype=_np.int32)
# append batch size and channels
image_shape = (1, 3) + image_shape
c_image = _mx.sym.Variable(self.content_feature, shape=image_shape,
dtype=_np.float32)
# signal that we want the transformer to prepare for coreml export
# using a zero batch size
transformer.batch_size = 0
transformer.scale255 = True
sym_out = transformer(c_image, index)
mod = _mx.mod.Module(symbol=sym_out, data_names=[self.content_feature, "index"],
label_names=None)
mod.bind(data_shapes=zip([self.content_feature, "index"], [image_shape, (1,)]), for_training=False,
inputs_need_grad=False)
gluon_weights = transformer.collect_params()
gluon_layers = []
for layer in transformer.collect_params()._params:
gluon_layers.append(layer)
sym_layers = mod._param_names
sym_weight_dict = {}
for gluon_layer, sym_layer in zip(gluon_layers, sym_layers):
sym_weight_dict[sym_layer] = gluon_weights[gluon_layer]._data[0]
mod.set_params(sym_weight_dict, sym_weight_dict)
index_dim = (1, self.num_styles)
coreml_model = _mxnet_converter.convert(mod, input_shape=[(self.content_feature, image_shape), ('index', index_dim)],
mode=None, preprocessor_args=None, builder=None, verbose=False)
transformer.scale255 = False
spec = coreml_model.get_spec()
image_input = spec.description.input[0]
image_output = spec.description.output[0]
input_array_shape = tuple(image_input.type.multiArrayType.shape)
output_array_shape = tuple(image_output.type.multiArrayType.shape)
self._export_coreml_image(image_input, input_array_shape)
self._export_coreml_image(image_output, output_array_shape)
stylized_image = 'stylized%s' % self.content_feature.capitalize()
coremltools.utils.rename_feature(spec,
'transformer__mulscalar0_output', stylized_image, True, True)
if include_flexible_shape:
# Support flexible shape
flexible_shape_utils = _mxnet_converter._coremltools.models.neural_network.flexible_shape_utils
img_size_ranges = flexible_shape_utils.NeuralNetworkImageSizeRange()
img_size_ranges.add_height_range((64, -1))
img_size_ranges.add_width_range((64, -1))
flexible_shape_utils.update_image_size_range(spec, feature_name=self.content_feature, size_range=img_size_ranges)
flexible_shape_utils.update_image_size_range(spec, feature_name=stylized_image, size_range=img_size_ranges)
model_type = 'style transfer (%s)' % self.model
spec.description.metadata.shortDescription = _coreml_utils._mlmodel_short_description(
model_type)
spec.description.input[0].shortDescription = 'Input image'
spec.description.input[1].shortDescription = u'Style index array (set index I to 1.0 to enable Ith style)'
spec.description.output[0].shortDescription = 'Stylized image'
user_defined_metadata = _coreml_utils._get_model_metadata(
self.__class__.__name__, {
'model': self.model,
'num_styles': str(self.num_styles),
'content_feature': self.content_feature,
'style_feature': self.style_feature,
'max_iterations': str(self.max_iterations),
'training_iterations': str(self.training_iterations),
}, version=StyleTransfer._PYTHON_STYLE_TRANSFER_VERSION)
spec.description.metadata.userDefined.update(user_defined_metadata)
from coremltools.models.utils import save_spec as _save_spec
_save_spec(spec, path)
def export_coreml(self, filename):
"""
Save the model in Core ML format.
The exported model calculates the distance between a query image and
each row of the model's stored data. It does not sort and retrieve
the k nearest neighbors of the query image.
See Also
--------
save
Examples
--------
# Train an image similarity model
>>> model = turicreate.image_similarity.create(data)
# Query the model for similar images
>>> similar_images = model.query(data)
+-------------+-----------------+---------------+------+
| query_label | reference_label | distance | rank |
+-------------+-----------------+---------------+------+
| 0 | 0 | 0.0 | 1 |
| 0 | 2 | 24.9664942809 | 2 |
| 0 | 1 | 28.4416069428 | 3 |
| 1 | 1 | 0.0 | 1 |
| 1 | 2 | 21.8715131191 | 2 |
| 1 | 0 | 28.4416069428 | 3 |
| 2 | 2 | 0.0 | 1 |
| 2 | 1 | 21.8715131191 | 2 |
| 2 | 0 | 24.9664942809 | 3 |
+-------------+-----------------+---------------+------+
[9 rows x 4 columns]
# Export the model to Core ML format
>>> model.export_coreml('myModel.mlmodel')
# Load the Core ML model
>>> import coremltools
>>> ml_model = coremltools.models.MLModel('myModel.mlmodel')
# Prepare the first image of reference data for consumption
# by the Core ML model
>>> import PIL
>>> image = tc.image_analysis.resize(
data['image'][0], *reversed(model.input_image_shape))
>>> image = PIL.Image.fromarray(image.pixel_data)
# Calculate distances using the Core ML model
>>> ml_model.predict(data={'image': image})
{'distance': array([ 0. , 28.453125, 24.96875 ])}
"""
import numpy as _np
import coremltools as _cmt
from coremltools.models import datatypes as _datatypes, neural_network as _neural_network
from .._mxnet_to_coreml import _mxnet_converter
from turicreate.toolkits import _coreml_utils
# Get the reference data from the model
proxy = self.similarity_model.__proxy__
reference_data = _np.array(
_tc.extensions._nearest_neighbors._nn_get_reference_data(proxy))
num_examples, embedding_size = reference_data.shape
# Get the input and output names
input_name = self.feature_extractor.data_layer
output_name = 'distance'
input_features = [(input_name,
_datatypes.Array(*(self.input_image_shape)))]
output_features = [(output_name, _datatypes.Array(num_examples))]
# Create a neural network
builder = _neural_network.NeuralNetworkBuilder(input_features,
output_features,
mode=None)
# Convert the feature extraction network
mx_feature_extractor = self.feature_extractor._get_mx_module(
self.feature_extractor.ptModel.mxmodel,
self.feature_extractor.data_layer,
self.feature_extractor.feature_layer,
self.feature_extractor.context, self.input_image_shape)
batch_input_shape = (1, ) + self.input_image_shape
_mxnet_converter.convert(mx_feature_extractor,
mode=None,
input_shape={input_name: batch_input_shape},
builder=builder,
verbose=False)
# To add the nearest neighbors model we add calculation of the euclidean
# distance between the newly extracted query features (denoted by the vector u)
# and each extracted reference feature (denoted by the rows of matrix V).
# Calculation of sqrt((v_i-u)^2) = sqrt(v_i^2 - 2v_i*u + u^2) ensues.
V = reference_data
v_squared = (V * V).sum(axis=1)
feature_layer = self.feature_extractor.feature_layer
builder.add_inner_product('v^2-2vu',
W=-2 * V,
b=v_squared,
has_bias=True,
input_channels=embedding_size,
output_channels=num_examples,
input_name=feature_layer,
output_name='v^2-2vu')
builder.add_unary('element_wise-u^2',
mode='power',
alpha=2,
input_name=feature_layer,
output_name='element_wise-u^2')
# Produce a vector of length num_examples with all values equal to u^2
builder.add_inner_product('u^2',
W=_np.ones((embedding_size, num_examples)),
b=None,
has_bias=False,
input_channels=embedding_size,
output_channels=num_examples,
input_name='element_wise-u^2',
output_name='u^2')
builder.add_elementwise('v^2-2vu+u^2',
mode='ADD',
input_names=['v^2-2vu', 'u^2'],
output_name='v^2-2vu+u^2')
# v^2-2vu+u^2=(v-u)^2 is non-negative but some computations on GPU may result in
# small negative values. Apply RELU so we don't take the square root of negative values.
builder.add_activation('relu',
non_linearity='RELU',
input_name='v^2-2vu+u^2',
output_name='relu')
builder.add_unary('sqrt',
mode='sqrt',
input_name='relu',
output_name=output_name)
# Finalize model
_mxnet_converter._set_input_output_layers(builder, [input_name],
[output_name])
builder.set_input([input_name], [self.input_image_shape])
builder.set_output([output_name], [(num_examples, )])
_cmt.models.utils.rename_feature(builder.spec, input_name,
self.feature)
builder.set_pre_processing_parameters(image_input_names=self.feature)
# Add metadata
mlmodel = _cmt.models.MLModel(builder.spec)
model_type = 'image similarity'
mlmodel.short_description = _coreml_utils._mlmodel_short_description(
model_type)
mlmodel.input_description[self.feature] = u'Input image'
mlmodel.output_description[
output_name] = u'Distances between the input and reference images'
mlmodel.save(filename)
def export_coreml(self, filename):
"""
Save the model in Core ML format.
The exported model calculates the distance between a query image and
each row of the model's stored data. It does not sort and retrieve
the k nearest neighbors of the query image.
See Also
--------
save
Examples
--------
>>> # Train an image similarity model
>>> model = turicreate.image_similarity.create(data)
>>>
>>> # Query the model for similar images
>>> similar_images = model.query(data)
+-------------+-----------------+---------------+------+
| query_label | reference_label | distance | rank |
+-------------+-----------------+---------------+------+
| 0 | 0 | 0.0 | 1 |
| 0 | 2 | 24.9664942809 | 2 |
| 0 | 1 | 28.4416069428 | 3 |
| 1 | 1 | 0.0 | 1 |
| 1 | 2 | 21.8715131191 | 2 |
| 1 | 0 | 28.4416069428 | 3 |
| 2 | 2 | 0.0 | 1 |
| 2 | 1 | 21.8715131191 | 2 |
| 2 | 0 | 24.9664942809 | 3 |
+-------------+-----------------+---------------+------+
[9 rows x 4 columns]
>>>
>>> # Export the model to Core ML format
>>> model.export_coreml('myModel.mlmodel')
>>>
>>> # Load the Core ML model
>>> import coremltools
>>> ml_model = coremltools.models.MLModel('myModel.mlmodel')
>>>
>>> # Prepare the first image of reference data for consumption
>>> # by the Core ML model
>>> import PIL
>>> image = tc.image_analysis.resize(data['image'][0], *reversed(model.input_image_shape))
>>> image = PIL.Image.fromarray(image.pixel_data)
>>>
>>> # Calculate distances using the Core ML model
>>> ml_model.predict(data={'image': image})
{'distance': array([ 0., 28.453125, 24.96875 ])}
"""
import numpy as _np
import coremltools as _cmt
from coremltools.models import datatypes as _datatypes, neural_network as _neural_network
from .._mxnet._mxnet_to_coreml import _mxnet_converter
from turicreate.toolkits import _coreml_utils
# Get the reference data from the model
proxy = self.similarity_model.__proxy__
reference_data = _np.array(_tc.extensions._nearest_neighbors._nn_get_reference_data(proxy))
num_examples, embedding_size = reference_data.shape
output_name = 'distance'
output_features = [(output_name, _datatypes.Array(num_examples))]
if self.model != 'VisionFeaturePrint_Scene':
# Convert the MxNet model to Core ML
ptModel = _pre_trained_models.MODELS[self.model]()
feature_extractor = _image_feature_extractor.MXFeatureExtractor(ptModel)
input_name = feature_extractor.data_layer
input_features = [(input_name, _datatypes.Array(*(self.input_image_shape)))]
# Create a neural network
builder = _neural_network.NeuralNetworkBuilder(
input_features, output_features, mode=None)
# Convert the feature extraction network
mx_feature_extractor = feature_extractor._get_mx_module(
feature_extractor.ptModel.mxmodel,
feature_extractor.data_layer,
feature_extractor.feature_layer,
feature_extractor.context,
self.input_image_shape
)
batch_input_shape = (1, ) + self.input_image_shape
_mxnet_converter.convert(mx_feature_extractor, mode=None,
input_shape=[(input_name, batch_input_shape)],
builder=builder, verbose=False)
feature_layer = feature_extractor.feature_layer
else: # self.model == VisionFeaturePrint_Scene
# Create a pipleline that contains a VisionFeaturePrint followed by a
# neural network.
BGR_VALUE = _cmt.proto.FeatureTypes_pb2.ImageFeatureType.ColorSpace.Value('BGR')
DOUBLE_ARRAY_VALUE = _cmt.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value('DOUBLE')
INPUT_IMAGE_SHAPE = 299
top_spec = _cmt.proto.Model_pb2.Model()
top_spec.specificationVersion = 3
desc = top_spec.description
input = desc.input.add()
input.name = self.feature
input.type.imageType.width = INPUT_IMAGE_SHAPE
input.type.imageType.height = INPUT_IMAGE_SHAPE
input.type.imageType.colorSpace = BGR_VALUE
output = desc.output.add()
output.name = output_name
output.type.multiArrayType.shape.append(num_examples)
output.type.multiArrayType.dataType = DOUBLE_ARRAY_VALUE
# VisionFeaturePrint extractor
pipeline = top_spec.pipeline
scene_print = pipeline.models.add()
scene_print.specificationVersion = 3
scene_print.visionFeaturePrint.scene.version = 1
input = scene_print.description.input.add()
input.name = self.feature
input.type.imageType.width = 299
input.type.imageType.height = 299
input.type.imageType.colorSpace = BGR_VALUE
feature_layer = 'VisionFeaturePrint_Scene_output'
output = scene_print.description.output.add()
output.name = feature_layer
output.type.multiArrayType.dataType = DOUBLE_ARRAY_VALUE
output.type.multiArrayType.shape.append(2048)
# Neural network builder
input_features = [(feature_layer, _datatypes.Array(2048))]
builder = _neural_network.NeuralNetworkBuilder(input_features, output_features)
# To add the nearest neighbors model we add calculation of the euclidean
# distance between the newly extracted query features (denoted by the vector u)
# and each extracted reference feature (denoted by the rows of matrix V).
# Calculation of sqrt((v_i-u)^2) = sqrt(v_i^2 - 2v_i*u + u^2) ensues.
V = reference_data
v_squared = (V * V).sum(axis=1)
builder.add_inner_product('v^2-2vu', W=-2 * V, b=v_squared, has_bias=True,
input_channels=embedding_size, output_channels=num_examples,
input_name=feature_layer, output_name='v^2-2vu')
builder.add_unary('element_wise-u^2', mode='power', alpha=2,
input_name=feature_layer, output_name='element_wise-u^2')
# Produce a vector of length num_examples with all values equal to u^2
builder.add_inner_product('u^2', W=_np.ones((embedding_size, num_examples)),
b=None, has_bias=False,
input_channels=embedding_size, output_channels=num_examples,
input_name='element_wise-u^2', output_name='u^2')
builder.add_elementwise('v^2-2vu+u^2', mode='ADD',
input_names=['v^2-2vu', 'u^2'],
output_name='v^2-2vu+u^2')
# v^2-2vu+u^2=(v-u)^2 is non-negative but some computations on GPU may result in
# small negative values. Apply RELU so we don't take the square root of negative values.
builder.add_activation('relu', non_linearity='RELU',
input_name='v^2-2vu+u^2', output_name='relu')
builder.add_unary('sqrt', mode='sqrt', input_name='relu', output_name=output_name)
# Finalize model
if self.model != 'VisionFeaturePrint_Scene':
_mxnet_converter._set_input_output_layers(builder, [input_name], [output_name])
builder.set_input([input_name], [self.input_image_shape])
builder.set_output([output_name], [(num_examples,)])
_cmt.models.utils.rename_feature(builder.spec, input_name, self.feature)
builder.set_pre_processing_parameters(image_input_names=self.feature)
mlmodel = _cmt.models.MLModel(builder.spec)
else:
top_spec.pipeline.models.extend([builder.spec])
mlmodel = _cmt.models.MLModel(top_spec)
# Add metadata
model_type = 'image similarity'
mlmodel.short_description = _coreml_utils._mlmodel_short_description(model_type)
mlmodel.input_description[self.feature] = u'Input image'
mlmodel.output_description[output_name] = u'Distances between the input and reference images'
_coreml_utils._set_model_metadata(mlmodel, self.__class__.__name__, {
'model': self.model,
'num_examples': str(self.num_examples)
}, version=ImageSimilarityModel._PYTHON_IMAGE_SIMILARITY_VERSION)
mlmodel.save(filename)
def export_coreml(self, filename):
"""
Save the model in Core ML format.
The exported model calculates the distance between a query image and
each row of the model's stored data. It does not sort and retrieve
the k nearest neighbors of the query image.
See Also
--------
save
Examples
--------
>>> # Train an image similarity model
>>> model = turicreate.image_similarity.create(data)
>>>
>>> # Query the model for similar images
>>> similar_images = model.query(data)
+-------------+-----------------+---------------+------+
| query_label | reference_label | distance | rank |
+-------------+-----------------+---------------+------+
| 0 | 0 | 0.0 | 1 |
| 0 | 2 | 24.9664942809 | 2 |
| 0 | 1 | 28.4416069428 | 3 |
| 1 | 1 | 0.0 | 1 |
| 1 | 2 | 21.8715131191 | 2 |
| 1 | 0 | 28.4416069428 | 3 |
| 2 | 2 | 0.0 | 1 |
| 2 | 1 | 21.8715131191 | 2 |
| 2 | 0 | 24.9664942809 | 3 |
+-------------+-----------------+---------------+------+
[9 rows x 4 columns]
>>>
>>> # Export the model to Core ML format
>>> model.export_coreml('myModel.mlmodel')
>>>
>>> # Load the Core ML model
>>> import coremltools
>>> ml_model = coremltools.models.MLModel('myModel.mlmodel')
>>>
>>> # Prepare the first image of reference data for consumption
>>> # by the Core ML model
>>> import PIL
>>> image = tc.image_analysis.resize(data['image'][0], *reversed(model.input_image_shape))
>>> image = PIL.Image.fromarray(image.pixel_data)
>>>
>>> # Calculate distances using the Core ML model
>>> ml_model.predict(data={'image': image})
{'distance': array([ 0., 28.453125, 24.96875 ])}
"""
import numpy as _np
from copy import deepcopy
from turicreate._deps.minimal_package import _minimal_package_import_check
_cmt = _minimal_package_import_check("coremltools")
from coremltools.models import (
datatypes as _datatypes,
neural_network as _neural_network,
)
from turicreate.toolkits import _coreml_utils
# Get the reference data from the model
proxy = self.similarity_model.__proxy__
reference_data = _np.array(
_tc.extensions._nearest_neighbors._nn_get_reference_data(proxy))
num_examples, embedding_size = reference_data.shape
output_name = "distance"
output_features = [(output_name, _datatypes.Array(num_examples))]
if self.model != "VisionFeaturePrint_Scene":
# Get the Core ML spec for the feature extractor
ptModel = _pre_trained_models.IMAGE_MODELS[self.model]()
feature_extractor = _image_feature_extractor.TensorFlowFeatureExtractor(
ptModel)
feature_extractor_spec = feature_extractor.get_coreml_model(
).get_spec()
input_name = feature_extractor.coreml_data_layer
input_features = [(input_name,
_datatypes.Array(*(self.input_image_shape)))]
# Convert the neuralNetworkClassifier to a neuralNetwork
layers = deepcopy(
feature_extractor_spec.neuralNetworkClassifier.layers)
for l in layers:
feature_extractor_spec.neuralNetwork.layers.append(l)
builder = _neural_network.NeuralNetworkBuilder(
input_features, output_features, spec=feature_extractor_spec)
feature_layer = feature_extractor.coreml_feature_layer
else: # self.model == VisionFeaturePrint_Scene
# Create a pipleline that contains a VisionFeaturePrint followed by a
# neural network.
BGR_VALUE = _cmt.proto.FeatureTypes_pb2.ImageFeatureType.ColorSpace.Value(
"BGR")
DOUBLE_ARRAY_VALUE = _cmt.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value(
"DOUBLE")
INPUT_IMAGE_SHAPE = 299
top_spec = _cmt.proto.Model_pb2.Model()
top_spec.specificationVersion = 3
desc = top_spec.description
input = desc.input.add()
input.name = self.feature
input.type.imageType.width = INPUT_IMAGE_SHAPE
input.type.imageType.height = INPUT_IMAGE_SHAPE
input.type.imageType.colorSpace = BGR_VALUE
output = desc.output.add()
output.name = output_name
output.type.multiArrayType.shape.append(num_examples)
output.type.multiArrayType.dataType = DOUBLE_ARRAY_VALUE
# VisionFeaturePrint extractor
pipeline = top_spec.pipeline
scene_print = pipeline.models.add()
scene_print.specificationVersion = 3
scene_print.visionFeaturePrint.scene.version = 1
input = scene_print.description.input.add()
input.name = self.feature
input.type.imageType.width = 299
input.type.imageType.height = 299
input.type.imageType.colorSpace = BGR_VALUE
feature_layer = "VisionFeaturePrint_Scene_output"
output = scene_print.description.output.add()
output.name = feature_layer
output.type.multiArrayType.dataType = DOUBLE_ARRAY_VALUE
output.type.multiArrayType.shape.append(2048)
# Neural network builder
input_features = [(feature_layer, _datatypes.Array(2048))]
builder = _neural_network.NeuralNetworkBuilder(
input_features, output_features)
# To add the nearest neighbors model we add calculation of the euclidean
# distance between the newly extracted query features (denoted by the vector u)
# and each extracted reference feature (denoted by the rows of matrix V).
# Calculation of sqrt((v_i-u)^2) = sqrt(v_i^2 - 2v_i*u + u^2) ensues.
V = reference_data
v_squared = (V * V).sum(axis=1)
builder.add_inner_product(
"v^2-2vu",
W=-2 * V,
b=v_squared,
has_bias=True,
input_channels=embedding_size,
output_channels=num_examples,
input_name=feature_layer,
output_name="v^2-2vu",
)
builder.add_unary(
"element_wise-u^2",
mode="power",
alpha=2,
input_name=feature_layer,
output_name="element_wise-u^2",
)
# Produce a vector of length num_examples with all values equal to u^2
builder.add_inner_product(
"u^2",
W=_np.ones((embedding_size, num_examples)),
b=None,
has_bias=False,
input_channels=embedding_size,
output_channels=num_examples,
input_name="element_wise-u^2",
output_name="u^2",
)
builder.add_elementwise(
"v^2-2vu+u^2",
mode="ADD",
input_names=["v^2-2vu", "u^2"],
output_name="v^2-2vu+u^2",
)
# v^2-2vu+u^2=(v-u)^2 is non-negative but some computations on GPU may result in
# small negative values. Apply RELU so we don't take the square root of negative values.
builder.add_activation("relu",
non_linearity="RELU",
input_name="v^2-2vu+u^2",
output_name="relu")
builder.add_unary("sqrt",
mode="sqrt",
input_name="relu",
output_name=output_name)
# Finalize model
if self.model != "VisionFeaturePrint_Scene":
builder.set_input([input_name], [self.input_image_shape])
builder.set_output([output_name], [(num_examples, )])
_cmt.models.utils.rename_feature(builder.spec, input_name,
self.feature)
builder.set_pre_processing_parameters(
image_input_names=self.feature)
mlmodel = _cmt.models.MLModel(builder.spec)
else:
top_spec.pipeline.models.extend([builder.spec])
mlmodel = _cmt.models.MLModel(top_spec)
# Add metadata
model_type = "image similarity"
mlmodel.short_description = _coreml_utils._mlmodel_short_description(
model_type)
mlmodel.input_description[self.feature] = u"Input image"
mlmodel.output_description[
output_name] = u"Distances between the input and reference images"
model_metadata = {
"model": self.model,
"num_examples": str(self.num_examples),
}
user_defined_metadata = model_metadata.update(
_coreml_utils._get_tc_version_info())
_coreml_utils._set_model_metadata(
mlmodel,
self.__class__.__name__,
user_defined_metadata,
version=ImageSimilarityModel._PYTHON_IMAGE_SIMILARITY_VERSION,
)
mlmodel.save(filename)
def export_coreml(
self,
filename,
include_non_maximum_suppression=True,
iou_threshold=None,
confidence_threshold=None,
):
"""
Save the model in Core ML format. The Core ML model takes an image of
fixed size as input and produces two output arrays: `confidence` and
`coordinates`.
The first one, `confidence` is an `N`-by-`C` array, where `N` is the
number of instances predicted and `C` is the number of classes. The
number `N` is fixed and will include many low-confidence predictions.
The instances are not sorted by confidence, so the first one will
generally not have the highest confidence (unlike in `predict`). Also
unlike the `predict` function, the instances have not undergone
what is called `non-maximum suppression`, which means there could be
several instances close in location and size that have all discovered
the same object instance. Confidences do not need to sum to 1 over the
classes; any remaining probability is implied as confidence there is no
object instance present at all at the given coordinates. The classes
appear in the array alphabetically sorted.
The second array `coordinates` is of size `N`-by-4, where the first
dimension `N` again represents instances and corresponds to the
`confidence` array. The second dimension represents `x`, `y`, `width`,
`height`, in that order. The values are represented in relative
coordinates, so (0.5, 0.5) represents the center of the image and (1,
1) the bottom right corner. You will need to multiply the relative
values with the original image size before you resized it to the fixed
input size to get pixel-value coordinates similar to `predict`.
See Also
--------
save
Parameters
----------
filename : string
The path of the file where we want to save the Core ML model.
include_non_maximum_suppression : bool
Non-maximum suppression is only available in iOS 12+.
A boolean parameter to indicate whether the Core ML model should be
saved with built-in non-maximum suppression or not.
This parameter is set to True by default.
iou_threshold : float
Threshold value for non-maximum suppression. Non-maximum suppression
prevents multiple bounding boxes appearing over a single object.
This threshold, set between 0 and 1, controls how aggressive this
suppression is. A value of 1 means no maximum suppression will
occur, while a value of 0 will maximally suppress neighboring
boxes around a prediction.
confidence_threshold : float
Only return predictions above this level of confidence. The
threshold can range from 0 to 1.
Examples
--------
>>> model.export_coreml('one_shot.mlmodel')
"""
from turicreate.toolkits import _coreml_utils
additional_user_defined_metadata = _coreml_utils._get_tc_version_info()
short_description = _coreml_utils._mlmodel_short_description(
"Object Detector")
options = {
"include_non_maximum_suppression": include_non_maximum_suppression,
}
options["version"] = self._PYTHON_ONE_SHOT_OBJECT_DETECTOR_VERSION
if confidence_threshold is not None:
options["confidence_threshold"] = confidence_threshold
if iou_threshold is not None:
options["iou_threshold"] = iou_threshold
additional_user_defined_metadata = _coreml_utils._get_tc_version_info()
short_description = _coreml_utils._mlmodel_short_description(
"One Shot Object Detector")
self.__proxy__["detector"].__proxy__.export_to_coreml(
filename, short_description, additional_user_defined_metadata,
options)
def export_coreml(self, filename):
"""
Save the model in Core ML format. The Core ML model takes an image of
fixed size as input and produces two output arrays: `confidence` and
`coordinates`.
The first one, `confidence` is an `N`-by-`C` array, where `N` is the
number of instances predicted and `C` is the number of classes. The
number `N` is fixed and will include many low-confidence predictions.
The instances are not sorted by confidence, so the first one will
generally not have the highest confidence (unlike in `predict`). Also
unlike the `predict` function, the instances have not undergone
what is called `non-maximum suppression`, which means there could be
several instances close in location and size that have all discovered
the same object instance. Confidences do not need to sum to 1 over the
classes; any remaining probability is implied as confidence there is no
object instance present at all at the given coordinates. The classes
appear in the array alphabetically sorted.
The second array `coordinates` is of size `N`-by-4, where the first
dimension `N` again represents instances and corresponds to the
`confidence` array. The second dimension represents `x`, `y`, `width`,
`height`, in that order. The values are represented in relative
coordinates, so (0.5, 0.5) represents the center of the image and (1,
1) the bottom right corner. You will need to multiply the relative
values with the original image size before you resized it to the fixed
input size to get pixel-value coordinates similar to `predict`.
See Also
--------
save
Examples
--------
>>> model.export_coreml('detector.mlmodel')
"""
import mxnet as _mx
from .._mxnet_to_coreml import _mxnet_converter
import coremltools
from coremltools.models import datatypes, neural_network
preds_per_box = 5 + self.num_classes
num_anchors = len(self.anchors)
num_classes = self.num_classes
batch_size = 1
image_shape = (batch_size,) + tuple(self.input_image_shape)
s_image_uint8 = _mx.sym.Variable(self.feature, shape=image_shape, dtype=_np.float32)
s_image = s_image_uint8 / 255
# Swap a maxpool+slice in mxnet to a coreml natively supported layer
from copy import copy
net = copy(self._model)
net._children = copy(self._model._children)
from ._model import _SpecialDarknetMaxpoolBlock
op = _SpecialDarknetMaxpoolBlock(name='pool5')
# Make sure we are removing the right layers
assert (self._model[23].name == 'pool5' and
self._model[24].name == 'specialcrop5')
del net._children[24]
net._children[23] = op
s_ymap = net(s_image)
mod = _mx.mod.Module(symbol=s_ymap, label_names=None, data_names=[self.feature])
mod.bind(for_training=False, data_shapes=[(self.feature, image_shape)])
# Copy over params from net
mod.init_params()
arg_params, aux_params = mod.get_params()
net_params = net.collect_params()
new_arg_params = {}
for k, param in arg_params.items():
new_arg_params[k] = net_params[k].data(net_params[k].list_ctx()[0])
new_aux_params = {}
for k, param in aux_params.items():
new_aux_params[k] = net_params[k].data(net_params[k].list_ctx()[0])
mod.set_params(new_arg_params, new_aux_params)
input_names = [self.feature]
input_dims = [list(self.input_image_shape)]
input_types = [datatypes.Array(*dim) for dim in input_dims]
input_features = zip(input_names, input_types)
num_spatial = self._grid_shape[0] * self._grid_shape[1]
output_names = [
'confidence',
'coordinates',
]
output_dims = [
(num_anchors * num_spatial, num_classes),
(num_anchors * num_spatial, 4),
]
output_types = [datatypes.Array(*dim) for dim in output_dims]
output_features = zip(output_names, output_types)
mode = None
builder = neural_network.NeuralNetworkBuilder(input_features, output_features, mode)
_mxnet_converter.convert(mod, mode=None,
input_shape={self.feature: image_shape},
builder=builder, verbose=False)
prefix = '__tc__internal__'
# (1, B, C+5, S*S)
builder.add_reshape(name=prefix + 'ymap_sp_pre',
target_shape=[batch_size, num_anchors, preds_per_box, num_spatial],
mode=0,
input_name='conv8_fwd_output',
output_name=prefix + 'ymap_sp_pre')
# (1, C+5, B, S*S)
builder.add_permute(name=prefix + 'ymap_sp',
dim=[0, 2, 1, 3],
input_name=prefix + 'ymap_sp_pre',
output_name=prefix + 'ymap_sp')
# POSITION: X/Y
# (1, 2, B, S*S)
builder.add_slice(name=prefix + 'raw_rel_xy_sp',
axis='channel',
start_index=0,
end_index=2,
stride=1,
input_name=prefix + 'ymap_sp',
output_name=prefix + 'raw_rel_xy_sp')
# (1, 2, B, S*S)
builder.add_activation(name=prefix + 'rel_xy_sp',
non_linearity='SIGMOID',
input_name=prefix + 'raw_rel_xy_sp',
output_name=prefix + 'rel_xy_sp')
# (1, 2, B*H*W, 1)
builder.add_reshape(name=prefix + 'rel_xy',
target_shape=[batch_size, 2, num_anchors * num_spatial, 1],
mode=0,
input_name=prefix + 'rel_xy_sp',
output_name=prefix + 'rel_xy')
c_xy = _np.array(_np.meshgrid(_np.arange(self._grid_shape[1]),
_np.arange(self._grid_shape[0])), dtype=_np.float32)
c_xy_reshaped = (_np.tile(c_xy[:, _np.newaxis], (num_anchors, 1, 1))
.reshape(2, -1))[_np.newaxis, ..., _np.newaxis]
# (1, 2, B*H*W, 1)
builder.add_load_constant(prefix + 'constant_xy',
constant_value=c_xy_reshaped,
shape=c_xy_reshaped.shape[1:],
output_name=prefix + 'constant_xy')
# (1, 2, B*H*W, 1)
builder.add_elementwise(name=prefix + 'xy',
mode='ADD',
input_names=[prefix + 'constant_xy', prefix + 'rel_xy'],
output_name=prefix + 'xy')
# SHAPE: WIDTH/HEIGHT
# (1, 2, B, S*S)
builder.add_slice(name=prefix + 'raw_rel_wh_sp',
axis='channel',
start_index=2,
end_index=4,
stride=1,
input_name=prefix + 'ymap_sp',
output_name=prefix + 'raw_rel_wh_sp')
# (1, 2, B, S*S)
builder.add_unary(name=prefix + 'rel_wh_sp',
mode='exp',
input_name=prefix + 'raw_rel_wh_sp',
output_name=prefix + 'rel_wh_sp')
# (1, 2*B, S, S)
builder.add_reshape(name=prefix + 'rel_wh',
target_shape=[batch_size, 2 * num_anchors] + list(self._grid_shape),
mode=0,
input_name=prefix + 'rel_wh_sp',
output_name=prefix + 'rel_wh')
np_anchors = _np.asarray(self.anchors, dtype=_np.float32).T
anchors_0 = _np.tile(np_anchors.reshape([2 * num_anchors, 1, 1]), self._grid_shape)
# (1, 2*B, S, S)
builder.add_load_constant(name=prefix + 'c_anchors',
constant_value=anchors_0,
shape=anchors_0.shape,
output_name=prefix + 'c_anchors')
# (1, 2*B, S, S)
builder.add_elementwise(name=prefix + 'wh_pre',
mode='MULTIPLY',
input_names=[prefix + 'c_anchors', prefix + 'rel_wh'],
output_name=prefix + 'wh_pre')
# (1, 2, B*H*W, 1)
builder.add_reshape(name=prefix + 'wh',
target_shape=[1, 2, num_anchors * num_spatial, 1],
mode=0,
input_name=prefix + 'wh_pre',
output_name=prefix + 'wh')
# (1, 4, B*H*W, 1)
builder.add_elementwise(name=prefix + 'boxes_out_transposed',
mode='CONCAT',
input_names=[prefix + 'xy', prefix + 'wh'],
output_name=prefix + 'boxes_out_transposed')
# (1, B*H*W, 4, 1)
builder.add_permute(name=prefix + 'boxes_out',
dim=[0, 2, 1, 3],
input_name=prefix + 'boxes_out_transposed',
output_name=prefix + 'boxes_out')
scale = _np.zeros((num_anchors * num_spatial, 4, 1))
scale[:, 0::2] = 1.0 / self._grid_shape[1]
scale[:, 1::2] = 1.0 / self._grid_shape[0]
# (1, B*H*W, 4, 1)
builder.add_scale(name='coordinates',
W=scale,
b=0,
has_bias=False,
shape_scale=(num_anchors * num_spatial, 4, 1),
input_name=prefix + 'boxes_out',
output_name='coordinates')
# CLASS PROBABILITIES AND OBJECT CONFIDENCE
# (1, C, B, H*W)
builder.add_slice(name=prefix + 'scores_sp',
axis='channel',
start_index=5,
end_index=preds_per_box,
stride=1,
input_name=prefix + 'ymap_sp',
output_name=prefix + 'scores_sp')
# (1, C, B, H*W)
builder.add_softmax(name=prefix + 'probs_sp',
input_name=prefix + 'scores_sp',
output_name=prefix + 'probs_sp')
# (1, 1, B, H*W)
builder.add_slice(name=prefix + 'logit_conf_sp',
axis='channel',
start_index=4,
end_index=5,
stride=1,
input_name=prefix + 'ymap_sp',
output_name=prefix + 'logit_conf_sp')
# (1, 1, B, H*W)
builder.add_activation(name=prefix + 'conf_sp',
non_linearity='SIGMOID',
input_name=prefix + 'logit_conf_sp',
output_name=prefix + 'conf_sp')
# (1, C, B, H*W)
if num_classes > 1:
conf = prefix + 'conf_tiled_sp'
builder.add_elementwise(name=prefix + 'conf_tiled_sp',
mode='CONCAT',
input_names=[prefix + 'conf_sp'] * num_classes,
output_name=conf)
else:
conf = prefix + 'conf_sp'
# (1, C, B, H*W)
builder.add_elementwise(name=prefix + 'confprobs_sp',
mode='MULTIPLY',
input_names=[conf, prefix + 'probs_sp'],
output_name=prefix + 'confprobs_sp')
# (1, C, B*H*W, 1)
builder.add_reshape(name=prefix + 'confprobs_transposed',
target_shape=[1, num_classes, num_anchors * num_spatial, 1],
mode=0,
input_name=prefix + 'confprobs_sp',
output_name=prefix + 'confprobs_transposed')
# (1, B*H*W, C, 1)
builder.add_permute(name='confidence',
dim=[0, 2, 1, 3],
input_name=prefix + 'confprobs_transposed',
output_name='confidence')
_mxnet_converter._set_input_output_layers(builder, input_names, output_names)
builder.set_input(input_names, input_dims)
builder.set_output(output_names, output_dims)
builder.set_pre_processing_parameters(image_input_names=self.feature)
mlmodel = coremltools.models.MLModel(builder.spec)
model_type = 'object detector (%s)' % self.model
mlmodel.short_description = _coreml_utils._mlmodel_short_description(model_type)
mlmodel.input_description[self.feature] = 'Input image'
mlmodel.output_description['confidence'] = \
u'Boxes \xd7 Class confidence (see user-defined metadata "classes")'
mlmodel.output_description['coordinates'] = \
u'Boxes \xd7 [x, y, width, height] (relative to image size)'
_coreml_utils._set_model_metadata(mlmodel, self.__class__.__name__, {
'model': self.model,
'max_iterations': str(self.max_iterations),
'training_iterations': str(self.training_iterations),
'non_maximum_suppression_threshold': str(self.non_maximum_suppression_threshold),
'feature': self.feature,
'annotations': self.annotations,
'classes': ','.join(self.classes),
}, version=ObjectDetector._PYTHON_OBJECT_DETECTOR_VERSION)
mlmodel.save(filename)
def export_coreml(self, filename):
"""
Save the model in Core ML format.
See Also
--------
save
Examples
--------
>>> model.export_coreml('./myModel.mlmodel')
"""
import coremltools
from coremltools.proto.FeatureTypes_pb2 import ArrayFeatureType
prob_name = self.target + 'Probability'
def get_custom_model_spec():
from coremltools.models.neural_network import NeuralNetworkBuilder
from coremltools.models.datatypes import Array, Dictionary, String
input_name = 'output1'
input_length = self._feature_extractor.output_length
builder = NeuralNetworkBuilder(
[(input_name, Array(input_length, ))],
[(prob_name, Dictionary(String))], 'classifier')
input_name, output_name = input_name, 0
for i, cur_layer in enumerate(
self._custom_classifier.export_weights()):
W = cur_layer['weight']
nC, nB = W.shape
Wb = cur_layer['bias']
builder.add_inner_product(name="inner_product_" + str(i),
W=W,
b=Wb,
input_channels=nB,
output_channels=nC,
has_bias=True,
input_name=str(input_name),
output_name='inner_product_' +
str(output_name))
if cur_layer['act']:
builder.add_activation("activation" + str(i), 'RELU',
'inner_product_' + str(output_name),
str(output_name))
input_name = i
output_name = i + 1
last_output = builder.spec.neuralNetworkClassifier.layers[
-1].output[0]
builder.add_softmax('softmax', last_output, self.target)
builder.set_class_labels(self.classes,
predicted_feature_name=self.target)
builder.set_input([input_name], [(input_length, )])
builder.set_output([self.target], [(self.num_classes, )])
return builder.spec
top_level_spec = coremltools.proto.Model_pb2.Model()
top_level_spec.specificationVersion = 3
# Set input
desc = top_level_spec.description
input = desc.input.add()
input.name = self.feature
input.type.multiArrayType.dataType = ArrayFeatureType.ArrayDataType.Value(
'FLOAT32')
input.type.multiArrayType.shape.append(15600)
# Set outputs
prob_output = desc.output.add()
prob_output.name = prob_name
label_output = desc.output.add()
label_output.name = self.target
desc.predictedFeatureName = self.target
desc.predictedProbabilitiesName = prob_name
if type(self.classes[0]) == int:
# Class labels are ints
prob_output.type.dictionaryType.int64KeyType.MergeFromString(b'')
label_output.type.int64Type.MergeFromString(b'')
else: # Class are strings
prob_output.type.dictionaryType.stringKeyType.MergeFromString(b'')
label_output.type.stringType.MergeFromString(b'')
# Set metadata
user_metadata = desc.metadata.userDefined
user_metadata['sampleRate'] = str(
self._feature_extractor.input_sample_rate)
pipeline = top_level_spec.pipelineClassifier.pipeline
# Add the preprocessing model
preprocessing_model = pipeline.models.add()
preprocessing_model.customModel.className = 'TCSoundClassifierPreprocessing'
preprocessing_model.specificationVersion = 3
preprocessing_input = preprocessing_model.description.input.add()
preprocessing_input.CopyFrom(input)
preprocessed_output = preprocessing_model.description.output.add()
preprocessed_output.name = 'preprocessed_data'
preprocessed_output.type.multiArrayType.dataType = ArrayFeatureType.ArrayDataType.Value(
'DOUBLE')
preprocessed_output.type.multiArrayType.shape.append(1)
preprocessed_output.type.multiArrayType.shape.append(96)
preprocessed_output.type.multiArrayType.shape.append(64)
# Add the feature extractor, updating its input name
feature_extractor_spec = self._feature_extractor.get_spec()
pipeline.models.add().CopyFrom(feature_extractor_spec)
pipeline.models[-1].description.input[
0].name = preprocessed_output.name
pipeline.models[-1].neuralNetwork.layers[0].input[
0] = preprocessed_output.name
# Add the custom neural network
pipeline.models.add().CopyFrom(get_custom_model_spec())
# Set key type for the probability dictionary
prob_output_type = pipeline.models[-1].description.output[
0].type.dictionaryType
if type(self.classes[0]) == int:
prob_output_type.int64KeyType.MergeFromString(b'')
else: # String labels
prob_output_type.stringKeyType.MergeFromString(b'')
mlmodel = coremltools.models.MLModel(top_level_spec)
model_type = 'sound classifier'
mlmodel.short_description = _coreml_utils._mlmodel_short_description(
model_type)
mlmodel.input_description[self.feature] = u'Input audio features'
mlmodel.output_description[prob_name] = 'Prediction probabilities'
mlmodel.output_description[
self.target] = 'Class label of top prediction'
model_metadata = {
'target': self.target,
'feature': self.feature,
}
user_defined_metadata = model_metadata.update(
_coreml_utils._get_tc_version_info())
_coreml_utils._set_model_metadata(
mlmodel,
self.__class__.__name__,
user_defined_metadata,
version=SoundClassifier._PYTHON_SOUND_CLASSIFIER_VERSION)
mlmodel.save(filename)
