def export_coreml(self, filename): """ Export the model in Core ML format. Parameters ---------- filename: str A valid filename where the model can be saved. Examples -------- >>> model.export_coreml("MyModel.mlmodel") """ from turicreate.toolkits import _coreml_utils display_name = "random forest classifier" short_description = _coreml_utils._mlmodel_short_description( display_name) context = { "mode": "classification", "model_type": "random_forest", "version": _turicreate.__version__, "class": self.__class__.__name__, "short_description": short_description } self._export_coreml_impl(filename, context)
def export_coreml(self, filename): """ Export the model in Core ML format. Parameters ---------- filename: str A valid filename where the model can be saved. Examples -------- >>> model.export_coreml("MyModel.mlmodel") """ from turicreate.extensions import _linear_svm_export_as_model_asset from turicreate.toolkits import _coreml_utils display_name = "svm classifier" short_description = _coreml_utils._mlmodel_short_description(display_name) context = { "class": self.__class__.__name__, "short_description": short_description, } context["user_defined"] = _coreml_utils._get_model_metadata( self.__class__.__name__, None ) _linear_svm_export_as_model_asset(self.__proxy__, filename, context)
def export_coreml(self, filename): """ Export the model in Core ML format. Parameters ---------- filename: str A valid filename where the model can be saved. Examples -------- >>> model.export_coreml("MyModel.mlmodel") """ from turicreate.extensions import _logistic_classifier_export_as_model_asset from turicreate.toolkits import _coreml_utils display_name = 'text classifier' short_description = _coreml_utils._mlmodel_short_description( display_name) context = { 'class': self.__class__.__name__, 'version': _tc.__version__, 'short_description': short_description, 'user_defined': { 'turicreate_version': _tc.__version__ } } model = self.__proxy__['classifier'].__proxy__ _logistic_classifier_export_as_model_asset(model, filename, context)
def export_coreml(self, filename): """ Export the model in Core ML format. Parameters ---------- filename: str A valid filename where the model can be saved. Examples -------- >>> model.export_coreml("MyModel.mlmodel") """ from turicreate.extensions import _linear_regression_export_as_model_asset from turicreate.toolkits import _coreml_utils display_name = "linear regression" short_description = _coreml_utils._mlmodel_short_description( display_name) context = { "class": self.__class__.__name__, "version": _turicreate.__version__, "short_description": short_description } _linear_regression_export_as_model_asset(self.__proxy__, filename, context)
def export_coreml(self, filename): """ Export the model in Core ML format. Parameters ---------- filename: str A valid filename where the model can be saved. Examples -------- >>> model.export_coreml("MyModel.mlmodel") """ from turicreate.toolkits import _coreml_utils display_name = "boosted trees regression" short_description = _coreml_utils._mlmodel_short_description( display_name) context = { "mode": "regression", "model_type": "boosted_trees", "class": self.__class__.__name__, "short_description": short_description, } self._export_coreml_impl(filename, context)
def _set_inputs_outputs_and_metadata(spec, nn_spec): # Replace the classifier with the new classes class_labels = self.classifier.classes probOutput = spec.description.output[0] classLabel = spec.description.output[1] probOutput.type.dictionaryType.MergeFromString(b'') if type(class_labels[0]) == int: nn_spec.ClearField('int64ClassLabels') probOutput.type.dictionaryType.int64KeyType.MergeFromString( b'') classLabel.type.int64Type.MergeFromString(b'') del nn_spec.int64ClassLabels.vector[:] for c in class_labels: nn_spec.int64ClassLabels.vector.append(c) else: nn_spec.ClearField('stringClassLabels') probOutput.type.dictionaryType.stringKeyType.MergeFromString( b'') classLabel.type.stringType.MergeFromString(b'') del nn_spec.stringClassLabels.vector[:] for c in class_labels: nn_spec.stringClassLabels.vector.append(c) prob_name = self.target + 'Probability' label_name = self.target old_output_name = nn_spec.layers[-1].name coremltools.models.utils.rename_feature(spec, 'classLabel', label_name) coremltools.models.utils.rename_feature(spec, old_output_name, prob_name) if nn_spec.layers[-1].name == old_output_name: nn_spec.layers[-1].name = prob_name if nn_spec.labelProbabilityLayerName == old_output_name: nn_spec.labelProbabilityLayerName = prob_name coremltools.models.utils.rename_feature(spec, 'data', self.feature) if len(nn_spec.preprocessing) > 0: nn_spec.preprocessing[0].featureName = self.feature mlmodel = coremltools.models.MLModel(spec) model_type = 'image classifier (%s)' % self.model mlmodel.short_description = _coreml_utils._mlmodel_short_description( model_type) mlmodel.input_description[self.feature] = u'Input image' mlmodel.output_description[prob_name] = 'Prediction probabilities' mlmodel.output_description[ label_name] = 'Class label of top prediction' _coreml_utils._set_model_metadata( mlmodel, self.__class__.__name__, { 'model': self.model, 'target': self.target, 'features': self.feature, 'max_iterations': str(self.max_iterations), }, version=ImageClassifier._PYTHON_IMAGE_CLASSIFIER_VERSION) return mlmodel
def export_coreml(self, filename): """ Export the model in Core ML format. Parameters ---------- filename: str A valid filename where the model can be saved. Examples -------- >>> model.export_coreml("MyModel.mlmodel") """ additional_user_defined_metadata = _coreml_utils._get_tc_version_info() short_description = _coreml_utils._mlmodel_short_description( "Drawing Classifier") self.__proxy__.export_to_coreml(filename, short_description, additional_user_defined_metadata)
def export_coreml(self, filename, image_shape=(256, 256), include_flexible_shape=True): """ Save the model in Core ML format. The Core ML model takes an image of fixed size, and a style index inputs and produces an output of an image of fixed size Parameters ---------- path : string A string to the path for saving the Core ML model. image_shape: tuple A tuple (defaults to (256, 256)) will bind the coreml model to a fixed shape. include_flexible_shape: bool Allows the size of the input image to be flexible. Any input image were the height and width are at least 64 will be accepted by the Core ML Model. See Also -------- save Examples -------- >>> model.export_coreml('StyleTransfer.mlmodel') """ options = {} options['image_width'] = image_shape[1] options['image_height'] = image_shape[0] options['include_flexible_shape'] = include_flexible_shape additional_user_defined_metadata = _coreml_utils._get_tc_version_info() short_description = _coreml_utils._mlmodel_short_description( 'Style Transfer') self.__proxy__.export_to_coreml(filename, short_description, additional_user_defined_metadata, options)
def export_coreml(self, filename): """ Save the model in Core ML format. See Also -------- save Examples -------- >>> model.export_coreml('myModel.mlmodel') """ import coremltools as _cmt import mxnet as _mx from ._model_architecture import _define_model, _fit_model, _net_params prob_name = self.target + 'Probability' label_name = self.target input_features = [ ('features', _cmt.models.datatypes.Array(*(1, self.prediction_window, self.num_features))) ] output_features = [ (prob_name, _cmt.models.datatypes.Array(*(self.num_classes,))) ] model_params = self._pred_model.get_params() weights = {k: v.asnumpy() for k, v in model_params[0].items()} weights = _mx.rnn.LSTMCell(num_hidden=_net_params['lstm_h']).unpack_weights(weights) moving_weights = {k: v.asnumpy() for k, v in model_params[1].items()} builder = _cmt.models.neural_network.NeuralNetworkBuilder( input_features, output_features, mode='classifier' ) # Conv # (1,1,W,C) -> (1,C,1,W) builder.add_permute(name='permute_layer', dim=(0, 3, 1, 2), input_name='features', output_name='conv_in') W = _np.expand_dims(weights['conv_weight'], axis=0).transpose((2, 3, 1, 0)) builder.add_convolution(name='conv_layer', kernel_channels=self.num_features, output_channels=_net_params['conv_h'], height=1, width=self.prediction_window, stride_height=1, stride_width=self.prediction_window, border_mode='valid', groups=1, W=W, b=weights['conv_bias'], has_bias=True, input_name='conv_in', output_name='relu0_in') builder.add_activation(name='relu_layer0', non_linearity='RELU', input_name='relu0_in', output_name='lstm_in') # LSTM builder.add_optionals([('lstm_h_in', _net_params['lstm_h']), ('lstm_c_in', _net_params['lstm_h'])], [('lstm_h_out', _net_params['lstm_h']), ('lstm_c_out', _net_params['lstm_h'])]) W_x = [weights['lstm_i2h_i_weight'], weights['lstm_i2h_f_weight'], weights['lstm_i2h_o_weight'], weights['lstm_i2h_c_weight']] W_h = [weights['lstm_h2h_i_weight'], weights['lstm_h2h_f_weight'], weights['lstm_h2h_o_weight'], weights['lstm_h2h_c_weight']] bias = [weights['lstm_h2h_i_bias'], weights['lstm_h2h_f_bias'], weights['lstm_h2h_o_bias'], weights['lstm_h2h_c_bias']] builder.add_unilstm(name='lstm_layer', W_h=W_h, W_x=W_x, b=bias, input_size=_net_params['conv_h'], hidden_size=_net_params['lstm_h'], input_names=['lstm_in', 'lstm_h_in', 'lstm_c_in'], output_names=['dense0_in', 'lstm_h_out', 'lstm_c_out'], inner_activation='SIGMOID') # Dense builder.add_inner_product(name='dense_layer', W=weights['dense0_weight'], b=weights['dense0_bias'], input_channels=_net_params['lstm_h'], output_channels=_net_params['dense_h'], has_bias=True, input_name='dense0_in', output_name='bn_in') builder.add_batchnorm(name='bn_layer', channels=_net_params['dense_h'], gamma=weights['bn_gamma'], beta=weights['bn_beta'], mean=moving_weights['bn_moving_mean'], variance=moving_weights['bn_moving_var'], input_name='bn_in', output_name='relu1_in', epsilon=0.001) builder.add_activation(name='relu_layer1', non_linearity='RELU', input_name='relu1_in', output_name='dense1_in') # Softmax builder.add_inner_product(name='dense_layer1', W=weights['dense1_weight'], b=weights['dense1_bias'], has_bias=True, input_channels=_net_params['dense_h'], output_channels=self.num_classes, input_name='dense1_in', output_name='softmax_in') builder.add_softmax(name=prob_name, input_name='softmax_in', output_name=prob_name) labels = list(map(str, sorted(self._target_id_map.keys()))) builder.set_class_labels(labels) mlmodel = _cmt.models.MLModel(builder.spec) model_type = 'activity classifier' mlmodel.short_description = _coreml_utils._mlmodel_short_description(model_type) # Add useful information to the mlmodel features_str = ', '.join(self.features) mlmodel.input_description['features'] = u'Window \xd7 [%s]' % features_str mlmodel.input_description['lstm_h_in'] = 'LSTM hidden state input' mlmodel.input_description['lstm_c_in'] = 'LSTM cell state input' mlmodel.output_description[prob_name] = 'Activity prediction probabilities' mlmodel.output_description['classLabel'] = 'Class label of top prediction' mlmodel.output_description['lstm_h_out'] = 'LSTM hidden state output' mlmodel.output_description['lstm_c_out'] = 'LSTM cell state output' _coreml_utils._set_model_metadata(mlmodel, self.__class__.__name__, { 'prediction_window': str(self.prediction_window), 'session_id': self.session_id, 'target': self.target, 'features': ','.join(self.features), 'max_iterations': str(self.max_iterations), }, version=ActivityClassifier._PYTHON_ACTIVITY_CLASSIFIER_VERSION) spec = mlmodel.get_spec() _cmt.models.utils.rename_feature(spec, 'classLabel', label_name) _cmt.models.utils.rename_feature(spec, 'lstm_h_in', 'hiddenIn') _cmt.models.utils.rename_feature(spec, 'lstm_c_in', 'cellIn') _cmt.models.utils.rename_feature(spec, 'lstm_h_out', 'hiddenOut') _cmt.models.utils.rename_feature(spec, 'lstm_c_out', 'cellOut') _cmt.utils.save_spec(spec, filename)
def _set_inputs_outputs_and_metadata(spec, nn_spec): # Replace the classifier with the new classes class_labels = self.classifier.classes probOutput = spec.description.output[0] classLabel = spec.description.output[1] probOutput.type.dictionaryType.MergeFromString(b"") if type(class_labels[0]) == int: nn_spec.ClearField("int64ClassLabels") probOutput.type.dictionaryType.int64KeyType.MergeFromString( b"") classLabel.type.int64Type.MergeFromString(b"") del nn_spec.int64ClassLabels.vector[:] for c in class_labels: nn_spec.int64ClassLabels.vector.append(c) else: nn_spec.ClearField("stringClassLabels") probOutput.type.dictionaryType.stringKeyType.MergeFromString( b"") classLabel.type.stringType.MergeFromString(b"") del nn_spec.stringClassLabels.vector[:] for c in class_labels: nn_spec.stringClassLabels.vector.append(c) prob_name = self.target + "Probability" label_name = self.target old_output_name = nn_spec.layers[-1].name coremltools.models.utils.rename_feature(spec, "classLabel", label_name) coremltools.models.utils.rename_feature(spec, old_output_name, prob_name) if nn_spec.layers[-1].name == old_output_name: nn_spec.layers[-1].name = prob_name if nn_spec.labelProbabilityLayerName == old_output_name: nn_spec.labelProbabilityLayerName = prob_name coremltools.models.utils.rename_feature(spec, "data", self.feature) if len(nn_spec.preprocessing) > 0: nn_spec.preprocessing[0].featureName = self.feature mlmodel = coremltools.models.MLModel(spec) model_type = "image classifier (%s)" % self.model mlmodel.short_description = _coreml_utils._mlmodel_short_description( model_type) mlmodel.input_description[self.feature] = u"Input image" mlmodel.output_description[prob_name] = "Prediction probabilities" mlmodel.output_description[ label_name] = "Class label of top prediction" model_metadata = { "model": self.model, "target": self.target, "features": self.feature, "max_iterations": str(self.max_iterations), } user_defined_metadata = model_metadata.update( _coreml_utils._get_tc_version_info()) _coreml_utils._set_model_metadata( mlmodel, self.__class__.__name__, user_defined_metadata, version=ImageClassifier._PYTHON_IMAGE_CLASSIFIER_VERSION, ) return mlmodel
def export_coreml(self, filename): """ Save the model in Core ML format. See Also -------- save Examples -------- >>> model.export_coreml('myModel.mlmodel') """ coreml_model = self.feature_extractor.get_coreml_model() spec = coreml_model.get_spec() nn_spec = spec.neuralNetworkClassifier num_classes = self.num_classes # Replace the softmax layer with new coeffients fc_layer = nn_spec.layers[-2] fc_layer_params = fc_layer.innerProduct fc_layer_params.outputChannels = self.classifier.num_classes inputChannels = fc_layer_params.inputChannels fc_layer_params.hasBias = True coefs = self.classifier.coefficients weights = fc_layer_params.weights bias = fc_layer_params.bias del weights.floatValue[:] del bias.floatValue[:] import numpy as np W = np.array(coefs[coefs['index'] != None]['value'], ndmin=2).reshape(inputChannels, num_classes - 1, order='F') b = coefs[coefs['index'] == None]['value'] Wa = np.hstack((np.zeros((inputChannels, 1)), W)) weights.floatValue.extend(Wa.flatten(order='F')) bias.floatValue.extend([0.0] + list(b)) # Replace the classifier with the new classes class_labels = self.classifier.classes probOutput = spec.description.output[0] classLabel = spec.description.output[1] probOutput.type.dictionaryType.MergeFromString(b'') if type(class_labels[0]) == int: nn_spec.ClearField('int64ClassLabels') probOutput.type.dictionaryType.int64KeyType.MergeFromString(b'') classLabel.type.int64Type.MergeFromString(b'') del nn_spec.int64ClassLabels.vector[:] for c in class_labels: nn_spec.int64ClassLabels.vector.append(c) else: nn_spec.ClearField('stringClassLabels') probOutput.type.dictionaryType.stringKeyType.MergeFromString(b'') classLabel.type.stringType.MergeFromString(b'') del nn_spec.stringClassLabels.vector[:] for c in class_labels: nn_spec.stringClassLabels.vector.append(c) import coremltools prob_name = self.target + 'Probability' label_name = self.target old_output_name = spec.neuralNetworkClassifier.layers[-1].name coremltools.models.utils.rename_feature(spec, 'classLabel', label_name) coremltools.models.utils.rename_feature(spec, old_output_name, prob_name) if spec.neuralNetworkClassifier.layers[-1].name == old_output_name: spec.neuralNetworkClassifier.layers[-1].name = prob_name if spec.neuralNetworkClassifier.labelProbabilityLayerName == old_output_name: spec.neuralNetworkClassifier.labelProbabilityLayerName = prob_name coremltools.models.utils.rename_feature(spec, 'data', self.feature) spec.neuralNetworkClassifier.preprocessing[ 0].featureName = self.feature mlmodel = coremltools.models.MLModel(spec) model_type = 'image classifier (%s)' % self.model mlmodel.short_description = _coreml_utils._mlmodel_short_description( model_type) mlmodel.input_description[self.feature] = u'Input image' mlmodel.output_description[prob_name] = 'Prediction probabilities' mlmodel.output_description[ label_name] = 'Class label of top prediction' _coreml_utils._set_model_metadata( mlmodel, self.__class__.__name__, { 'model': self.model, 'target': self.target, 'features': self.feature, 'max_iterations': str(self.max_iterations), }, version=ImageClassifier._PYTHON_IMAGE_CLASSIFIER_VERSION) mlmodel.save(filename)
def export_coreml(self, filename): """ Save the model in Core ML format. See Also -------- save Examples -------- >>> model.export_coreml('./myModel.mlmodel') """ import coremltools from coremltools.proto.FeatureTypes_pb2 import ArrayFeatureType prob_name = self.target + "Probability" def get_custom_model_spec(): from coremltools.models.neural_network import NeuralNetworkBuilder from coremltools.models.datatypes import Array input_name = "output1" input_length = self._feature_extractor.output_length builder = NeuralNetworkBuilder( [(input_name, Array(input_length,))], [(prob_name, Array(self.num_classes,))], "classifier", ) layer_counter = [0] builder.set_input([input_name], [(input_length,)]) def next_layer_name(): layer_counter[0] += 1 return "layer_%d" % layer_counter[0] for i, cur_layer in enumerate(self._custom_classifier.export_weights()): W = cur_layer["weight"] nC, nB = W.shape Wb = cur_layer["bias"] output_name = next_layer_name() builder.add_inner_product( name="inner_product_" + str(i), W=W, b=Wb, input_channels=nB, output_channels=nC, has_bias=True, input_name=input_name, output_name=output_name, ) input_name = output_name if cur_layer["act"]: output_name = next_layer_name() builder.add_activation( "activation" + str(i), "RELU", input_name, output_name ) input_name = output_name builder.add_softmax("softmax", input_name, prob_name) builder.set_class_labels( self.classes, predicted_feature_name=self.target, prediction_blob=prob_name, ) return builder.spec top_level_spec = coremltools.proto.Model_pb2.Model() top_level_spec.specificationVersion = 3 # Set input desc = top_level_spec.description input = desc.input.add() input.name = self.feature assert type(self.feature) is str input.type.multiArrayType.dataType = ArrayFeatureType.ArrayDataType.Value( "FLOAT32" ) input.type.multiArrayType.shape.append(15600) # Set outputs prob_output = desc.output.add() prob_output.name = prob_name label_output = desc.output.add() label_output.name = self.target desc.predictedFeatureName = self.target desc.predictedProbabilitiesName = prob_name if type(self.classes[0]) == int: # Class labels are ints prob_output.type.dictionaryType.int64KeyType.MergeFromString(b"") label_output.type.int64Type.MergeFromString(b"") else: # Class are strings prob_output.type.dictionaryType.stringKeyType.MergeFromString(b"") label_output.type.stringType.MergeFromString(b"") # Set metadata user_metadata = desc.metadata.userDefined user_metadata["sampleRate"] = str(self._feature_extractor.input_sample_rate) pipeline = top_level_spec.pipelineClassifier.pipeline # Add the preprocessing model preprocessing_model = pipeline.models.add() preprocessing_model.customModel.className = "TCSoundClassifierPreprocessing" preprocessing_model.specificationVersion = 3 preprocessing_input = preprocessing_model.description.input.add() preprocessing_input.CopyFrom(input) preprocessed_output = preprocessing_model.description.output.add() preprocessed_output.name = "preprocessed_data" preprocessed_output.type.multiArrayType.dataType = ArrayFeatureType.ArrayDataType.Value( "DOUBLE" ) preprocessed_output.type.multiArrayType.shape.append(1) preprocessed_output.type.multiArrayType.shape.append(96) preprocessed_output.type.multiArrayType.shape.append(64) # Add the feature extractor, updating its input name feature_extractor_spec = self._feature_extractor.get_spec() pipeline.models.add().CopyFrom(feature_extractor_spec) pipeline.models[-1].description.input[0].name = preprocessed_output.name pipeline.models[-1].neuralNetwork.layers[0].input[0] = preprocessed_output.name # Add the custom neural network pipeline.models.add().CopyFrom(get_custom_model_spec()) # Set key type for the probability dictionary prob_output_type = pipeline.models[-1].description.output[0].type.dictionaryType if type(self.classes[0]) == int: prob_output_type.int64KeyType.MergeFromString(b"") else: # String labels prob_output_type.stringKeyType.MergeFromString(b"") mlmodel = coremltools.models.MLModel(top_level_spec) model_type = "sound classifier" mlmodel.short_description = _coreml_utils._mlmodel_short_description(model_type) mlmodel.input_description[self.feature] = u"Input audio features" mlmodel.output_description[prob_name] = "Prediction probabilities" mlmodel.output_description[self.target] = "Class label of top prediction" model_metadata = { "target": self.target, "feature": self.feature, } user_defined_metadata = model_metadata.update( _coreml_utils._get_tc_version_info() ) _coreml_utils._set_model_metadata( mlmodel, self.__class__.__name__, user_defined_metadata, version=SoundClassifier._PYTHON_SOUND_CLASSIFIER_VERSION, ) mlmodel.save(filename)
def export_coreml(self, filename, verbose=False): """ Save the model in Core ML format. The Core ML model takes a grayscale drawing of fixed size as input and produces two outputs: `classLabel` and `labelProbabilities`. The first one, `classLabel` is an integer or string (depending on the classes the model was trained on) to store the label of the top prediction by the model. The second one, `labelProbabilities`, is a dictionary with all the class labels in the dataset as the keys, and their respective probabilities as the values. See Also -------- save Parameters ---------- filename : string The path of the file where we want to save the Core ML model. verbose : bool optional If True, prints export progress. Examples -------- >>> model.export_coreml('drawing_classifier.mlmodel') """ import mxnet as _mx from .._mxnet._mxnet_to_coreml import _mxnet_converter import coremltools as _coremltools batch_size = 1 image_shape = (batch_size,) + (1, BITMAP_WIDTH, BITMAP_HEIGHT) s_image = _mx.sym.Variable(self.feature, shape=image_shape, dtype=_np.float32) from copy import copy as _copy net = _copy(self._model) s_ymap = net(s_image) mod = _mx.mod.Module(symbol=s_ymap, label_names=None, data_names=[self.feature]) mod.bind(for_training=False, data_shapes=[(self.feature, image_shape)]) mod.init_params() arg_params, aux_params = mod.get_params() net_params = net.collect_params() new_arg_params = {} for k, param in arg_params.items(): new_arg_params[k] = net_params[k].data(net_params[k].list_ctx()[0]) new_aux_params = {} for k, param in aux_params.items(): new_aux_params[k] = net_params[k].data(net_params[k].list_ctx()[0]) mod.set_params(new_arg_params, new_aux_params) coreml_model = _mxnet_converter.convert(mod, mode='classifier', class_labels=self.classes, input_shape=[(self.feature, image_shape)], builder=None, verbose=verbose, preprocessor_args={ 'image_input_names': [self.feature], 'image_scale': 1.0/255 }) DESIRED_OUTPUT_NAME = self.target + "Probabilities" spec = coreml_model._spec class_label_output_index = 0 if spec.description.output[0].name == "classLabel" else 1 probabilities_output_index = 1-class_label_output_index spec.neuralNetworkClassifier.labelProbabilityLayerName = DESIRED_OUTPUT_NAME spec.neuralNetworkClassifier.layers[-1].name = DESIRED_OUTPUT_NAME spec.neuralNetworkClassifier.layers[-1].output[0] = DESIRED_OUTPUT_NAME spec.description.predictedProbabilitiesName = DESIRED_OUTPUT_NAME spec.description.output[probabilities_output_index].name = DESIRED_OUTPUT_NAME from turicreate.toolkits import _coreml_utils model_type = "drawing classifier" spec.description.metadata.shortDescription = _coreml_utils._mlmodel_short_description(model_type) spec.description.input[0].shortDescription = self.feature spec.description.output[probabilities_output_index].shortDescription = 'Prediction probabilities' spec.description.output[class_label_output_index].shortDescription = 'Class Label of Top Prediction' from coremltools.models.utils import save_spec as _save_spec _save_spec(spec, filename)
def export_coreml(self, path, image_shape=(256, 256), include_flexible_shape=True): """ Save the model in Core ML format. The Core ML model takes an image of fixed size, and a style index inputs and produces an output of an image of fixed size Parameters ---------- path : string A string to the path for saving the Core ML model. image_shape: tuple A tuple (defaults to (256, 256)) will bind the coreml model to a fixed shape. include_flexible_shape: bool A boolean value indicating whether flexible_shape should be included or not. See Also -------- save Examples -------- >>> model.export_coreml('StyleTransfer.mlmodel') """ import mxnet as _mx from .._mxnet_to_coreml import _mxnet_converter import coremltools transformer = self._model index = _mx.sym.Variable("index", shape=(1,), dtype=_np.int32) # append batch size and channels image_shape = (1, 3) + image_shape c_image = _mx.sym.Variable(self.content_feature, shape=image_shape, dtype=_np.float32) # signal that we want the transformer to prepare for coreml export # using a zero batch size transformer.batch_size = 0 transformer.scale255 = True sym_out = transformer(c_image, index) mod = _mx.mod.Module(symbol=sym_out, data_names=[self.content_feature, "index"], label_names=None) mod.bind(data_shapes=zip([self.content_feature, "index"], [image_shape, (1,)]), for_training=False, inputs_need_grad=False) gluon_weights = transformer.collect_params() gluon_layers = [] for layer in transformer.collect_params()._params: gluon_layers.append(layer) sym_layers = mod._param_names sym_weight_dict = {} for gluon_layer, sym_layer in zip(gluon_layers, sym_layers): sym_weight_dict[sym_layer] = gluon_weights[gluon_layer]._data[0] mod.set_params(sym_weight_dict, sym_weight_dict) index_dim = (1, self.num_styles) coreml_model = _mxnet_converter.convert(mod, input_shape=[(self.content_feature, image_shape), ('index', index_dim)], mode=None, preprocessor_args=None, builder=None, verbose=False) transformer.scale255 = False spec = coreml_model.get_spec() image_input = spec.description.input[0] image_output = spec.description.output[0] input_array_shape = tuple(image_input.type.multiArrayType.shape) output_array_shape = tuple(image_output.type.multiArrayType.shape) self._export_coreml_image(image_input, input_array_shape) self._export_coreml_image(image_output, output_array_shape) stylized_image = 'stylized%s' % self.content_feature.capitalize() coremltools.utils.rename_feature(spec, 'transformer__mulscalar0_output', stylized_image, True, True) if include_flexible_shape: # Support flexible shape flexible_shape_utils = _mxnet_converter._coremltools.models.neural_network.flexible_shape_utils img_size_ranges = flexible_shape_utils.NeuralNetworkImageSizeRange() img_size_ranges.add_height_range((64, -1)) img_size_ranges.add_width_range((64, -1)) flexible_shape_utils.update_image_size_range(spec, feature_name=self.content_feature, size_range=img_size_ranges) flexible_shape_utils.update_image_size_range(spec, feature_name=stylized_image, size_range=img_size_ranges) model_type = 'style transfer (%s)' % self.model spec.description.metadata.shortDescription = _coreml_utils._mlmodel_short_description( model_type) spec.description.input[0].shortDescription = 'Input image' spec.description.input[1].shortDescription = u'Style index array (set index I to 1.0 to enable Ith style)' spec.description.output[0].shortDescription = 'Stylized image' user_defined_metadata = _coreml_utils._get_model_metadata( self.__class__.__name__, { 'model': self.model, 'num_styles': str(self.num_styles), 'content_feature': self.content_feature, 'style_feature': self.style_feature, 'max_iterations': str(self.max_iterations), 'training_iterations': str(self.training_iterations), }, version=StyleTransfer._PYTHON_STYLE_TRANSFER_VERSION) spec.description.metadata.userDefined.update(user_defined_metadata) from coremltools.models.utils import save_spec as _save_spec _save_spec(spec, path)
def export_coreml(self, filename): """ Save the model in Core ML format. The exported model calculates the distance between a query image and each row of the model's stored data. It does not sort and retrieve the k nearest neighbors of the query image. See Also -------- save Examples -------- # Train an image similarity model >>> model = turicreate.image_similarity.create(data) # Query the model for similar images >>> similar_images = model.query(data) +-------------+-----------------+---------------+------+ | query_label | reference_label | distance | rank | +-------------+-----------------+---------------+------+ | 0 | 0 | 0.0 | 1 | | 0 | 2 | 24.9664942809 | 2 | | 0 | 1 | 28.4416069428 | 3 | | 1 | 1 | 0.0 | 1 | | 1 | 2 | 21.8715131191 | 2 | | 1 | 0 | 28.4416069428 | 3 | | 2 | 2 | 0.0 | 1 | | 2 | 1 | 21.8715131191 | 2 | | 2 | 0 | 24.9664942809 | 3 | +-------------+-----------------+---------------+------+ [9 rows x 4 columns] # Export the model to Core ML format >>> model.export_coreml('myModel.mlmodel') # Load the Core ML model >>> import coremltools >>> ml_model = coremltools.models.MLModel('myModel.mlmodel') # Prepare the first image of reference data for consumption # by the Core ML model >>> import PIL >>> image = tc.image_analysis.resize( data['image'][0], *reversed(model.input_image_shape)) >>> image = PIL.Image.fromarray(image.pixel_data) # Calculate distances using the Core ML model >>> ml_model.predict(data={'image': image}) {'distance': array([ 0. , 28.453125, 24.96875 ])} """ import numpy as _np import coremltools as _cmt from coremltools.models import datatypes as _datatypes, neural_network as _neural_network from .._mxnet_to_coreml import _mxnet_converter from turicreate.toolkits import _coreml_utils # Get the reference data from the model proxy = self.similarity_model.__proxy__ reference_data = _np.array( _tc.extensions._nearest_neighbors._nn_get_reference_data(proxy)) num_examples, embedding_size = reference_data.shape # Get the input and output names input_name = self.feature_extractor.data_layer output_name = 'distance' input_features = [(input_name, _datatypes.Array(*(self.input_image_shape)))] output_features = [(output_name, _datatypes.Array(num_examples))] # Create a neural network builder = _neural_network.NeuralNetworkBuilder(input_features, output_features, mode=None) # Convert the feature extraction network mx_feature_extractor = self.feature_extractor._get_mx_module( self.feature_extractor.ptModel.mxmodel, self.feature_extractor.data_layer, self.feature_extractor.feature_layer, self.feature_extractor.context, self.input_image_shape) batch_input_shape = (1, ) + self.input_image_shape _mxnet_converter.convert(mx_feature_extractor, mode=None, input_shape={input_name: batch_input_shape}, builder=builder, verbose=False) # To add the nearest neighbors model we add calculation of the euclidean # distance between the newly extracted query features (denoted by the vector u) # and each extracted reference feature (denoted by the rows of matrix V). # Calculation of sqrt((v_i-u)^2) = sqrt(v_i^2 - 2v_i*u + u^2) ensues. V = reference_data v_squared = (V * V).sum(axis=1) feature_layer = self.feature_extractor.feature_layer builder.add_inner_product('v^2-2vu', W=-2 * V, b=v_squared, has_bias=True, input_channels=embedding_size, output_channels=num_examples, input_name=feature_layer, output_name='v^2-2vu') builder.add_unary('element_wise-u^2', mode='power', alpha=2, input_name=feature_layer, output_name='element_wise-u^2') # Produce a vector of length num_examples with all values equal to u^2 builder.add_inner_product('u^2', W=_np.ones((embedding_size, num_examples)), b=None, has_bias=False, input_channels=embedding_size, output_channels=num_examples, input_name='element_wise-u^2', output_name='u^2') builder.add_elementwise('v^2-2vu+u^2', mode='ADD', input_names=['v^2-2vu', 'u^2'], output_name='v^2-2vu+u^2') # v^2-2vu+u^2=(v-u)^2 is non-negative but some computations on GPU may result in # small negative values. Apply RELU so we don't take the square root of negative values. builder.add_activation('relu', non_linearity='RELU', input_name='v^2-2vu+u^2', output_name='relu') builder.add_unary('sqrt', mode='sqrt', input_name='relu', output_name=output_name) # Finalize model _mxnet_converter._set_input_output_layers(builder, [input_name], [output_name]) builder.set_input([input_name], [self.input_image_shape]) builder.set_output([output_name], [(num_examples, )]) _cmt.models.utils.rename_feature(builder.spec, input_name, self.feature) builder.set_pre_processing_parameters(image_input_names=self.feature) # Add metadata mlmodel = _cmt.models.MLModel(builder.spec) model_type = 'image similarity' mlmodel.short_description = _coreml_utils._mlmodel_short_description( model_type) mlmodel.input_description[self.feature] = u'Input image' mlmodel.output_description[ output_name] = u'Distances between the input and reference images' mlmodel.save(filename)
def export_coreml(self, filename): """ Save the model in Core ML format. The exported model calculates the distance between a query image and each row of the model's stored data. It does not sort and retrieve the k nearest neighbors of the query image. See Also -------- save Examples -------- >>> # Train an image similarity model >>> model = turicreate.image_similarity.create(data) >>> >>> # Query the model for similar images >>> similar_images = model.query(data) +-------------+-----------------+---------------+------+ | query_label | reference_label | distance | rank | +-------------+-----------------+---------------+------+ | 0 | 0 | 0.0 | 1 | | 0 | 2 | 24.9664942809 | 2 | | 0 | 1 | 28.4416069428 | 3 | | 1 | 1 | 0.0 | 1 | | 1 | 2 | 21.8715131191 | 2 | | 1 | 0 | 28.4416069428 | 3 | | 2 | 2 | 0.0 | 1 | | 2 | 1 | 21.8715131191 | 2 | | 2 | 0 | 24.9664942809 | 3 | +-------------+-----------------+---------------+------+ [9 rows x 4 columns] >>> >>> # Export the model to Core ML format >>> model.export_coreml('myModel.mlmodel') >>> >>> # Load the Core ML model >>> import coremltools >>> ml_model = coremltools.models.MLModel('myModel.mlmodel') >>> >>> # Prepare the first image of reference data for consumption >>> # by the Core ML model >>> import PIL >>> image = tc.image_analysis.resize(data['image'][0], *reversed(model.input_image_shape)) >>> image = PIL.Image.fromarray(image.pixel_data) >>> >>> # Calculate distances using the Core ML model >>> ml_model.predict(data={'image': image}) {'distance': array([ 0., 28.453125, 24.96875 ])} """ import numpy as _np import coremltools as _cmt from coremltools.models import datatypes as _datatypes, neural_network as _neural_network from .._mxnet._mxnet_to_coreml import _mxnet_converter from turicreate.toolkits import _coreml_utils # Get the reference data from the model proxy = self.similarity_model.__proxy__ reference_data = _np.array(_tc.extensions._nearest_neighbors._nn_get_reference_data(proxy)) num_examples, embedding_size = reference_data.shape output_name = 'distance' output_features = [(output_name, _datatypes.Array(num_examples))] if self.model != 'VisionFeaturePrint_Scene': # Convert the MxNet model to Core ML ptModel = _pre_trained_models.MODELS[self.model]() feature_extractor = _image_feature_extractor.MXFeatureExtractor(ptModel) input_name = feature_extractor.data_layer input_features = [(input_name, _datatypes.Array(*(self.input_image_shape)))] # Create a neural network builder = _neural_network.NeuralNetworkBuilder( input_features, output_features, mode=None) # Convert the feature extraction network mx_feature_extractor = feature_extractor._get_mx_module( feature_extractor.ptModel.mxmodel, feature_extractor.data_layer, feature_extractor.feature_layer, feature_extractor.context, self.input_image_shape ) batch_input_shape = (1, ) + self.input_image_shape _mxnet_converter.convert(mx_feature_extractor, mode=None, input_shape=[(input_name, batch_input_shape)], builder=builder, verbose=False) feature_layer = feature_extractor.feature_layer else: # self.model == VisionFeaturePrint_Scene # Create a pipleline that contains a VisionFeaturePrint followed by a # neural network. BGR_VALUE = _cmt.proto.FeatureTypes_pb2.ImageFeatureType.ColorSpace.Value('BGR') DOUBLE_ARRAY_VALUE = _cmt.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value('DOUBLE') INPUT_IMAGE_SHAPE = 299 top_spec = _cmt.proto.Model_pb2.Model() top_spec.specificationVersion = 3 desc = top_spec.description input = desc.input.add() input.name = self.feature input.type.imageType.width = INPUT_IMAGE_SHAPE input.type.imageType.height = INPUT_IMAGE_SHAPE input.type.imageType.colorSpace = BGR_VALUE output = desc.output.add() output.name = output_name output.type.multiArrayType.shape.append(num_examples) output.type.multiArrayType.dataType = DOUBLE_ARRAY_VALUE # VisionFeaturePrint extractor pipeline = top_spec.pipeline scene_print = pipeline.models.add() scene_print.specificationVersion = 3 scene_print.visionFeaturePrint.scene.version = 1 input = scene_print.description.input.add() input.name = self.feature input.type.imageType.width = 299 input.type.imageType.height = 299 input.type.imageType.colorSpace = BGR_VALUE feature_layer = 'VisionFeaturePrint_Scene_output' output = scene_print.description.output.add() output.name = feature_layer output.type.multiArrayType.dataType = DOUBLE_ARRAY_VALUE output.type.multiArrayType.shape.append(2048) # Neural network builder input_features = [(feature_layer, _datatypes.Array(2048))] builder = _neural_network.NeuralNetworkBuilder(input_features, output_features) # To add the nearest neighbors model we add calculation of the euclidean # distance between the newly extracted query features (denoted by the vector u) # and each extracted reference feature (denoted by the rows of matrix V). # Calculation of sqrt((v_i-u)^2) = sqrt(v_i^2 - 2v_i*u + u^2) ensues. V = reference_data v_squared = (V * V).sum(axis=1) builder.add_inner_product('v^2-2vu', W=-2 * V, b=v_squared, has_bias=True, input_channels=embedding_size, output_channels=num_examples, input_name=feature_layer, output_name='v^2-2vu') builder.add_unary('element_wise-u^2', mode='power', alpha=2, input_name=feature_layer, output_name='element_wise-u^2') # Produce a vector of length num_examples with all values equal to u^2 builder.add_inner_product('u^2', W=_np.ones((embedding_size, num_examples)), b=None, has_bias=False, input_channels=embedding_size, output_channels=num_examples, input_name='element_wise-u^2', output_name='u^2') builder.add_elementwise('v^2-2vu+u^2', mode='ADD', input_names=['v^2-2vu', 'u^2'], output_name='v^2-2vu+u^2') # v^2-2vu+u^2=(v-u)^2 is non-negative but some computations on GPU may result in # small negative values. Apply RELU so we don't take the square root of negative values. builder.add_activation('relu', non_linearity='RELU', input_name='v^2-2vu+u^2', output_name='relu') builder.add_unary('sqrt', mode='sqrt', input_name='relu', output_name=output_name) # Finalize model if self.model != 'VisionFeaturePrint_Scene': _mxnet_converter._set_input_output_layers(builder, [input_name], [output_name]) builder.set_input([input_name], [self.input_image_shape]) builder.set_output([output_name], [(num_examples,)]) _cmt.models.utils.rename_feature(builder.spec, input_name, self.feature) builder.set_pre_processing_parameters(image_input_names=self.feature) mlmodel = _cmt.models.MLModel(builder.spec) else: top_spec.pipeline.models.extend([builder.spec]) mlmodel = _cmt.models.MLModel(top_spec) # Add metadata model_type = 'image similarity' mlmodel.short_description = _coreml_utils._mlmodel_short_description(model_type) mlmodel.input_description[self.feature] = u'Input image' mlmodel.output_description[output_name] = u'Distances between the input and reference images' _coreml_utils._set_model_metadata(mlmodel, self.__class__.__name__, { 'model': self.model, 'num_examples': str(self.num_examples) }, version=ImageSimilarityModel._PYTHON_IMAGE_SIMILARITY_VERSION) mlmodel.save(filename)
def export_coreml(self, filename): """ Save the model in Core ML format. The exported model calculates the distance between a query image and each row of the model's stored data. It does not sort and retrieve the k nearest neighbors of the query image. See Also -------- save Examples -------- >>> # Train an image similarity model >>> model = turicreate.image_similarity.create(data) >>> >>> # Query the model for similar images >>> similar_images = model.query(data) +-------------+-----------------+---------------+------+ | query_label | reference_label | distance | rank | +-------------+-----------------+---------------+------+ | 0 | 0 | 0.0 | 1 | | 0 | 2 | 24.9664942809 | 2 | | 0 | 1 | 28.4416069428 | 3 | | 1 | 1 | 0.0 | 1 | | 1 | 2 | 21.8715131191 | 2 | | 1 | 0 | 28.4416069428 | 3 | | 2 | 2 | 0.0 | 1 | | 2 | 1 | 21.8715131191 | 2 | | 2 | 0 | 24.9664942809 | 3 | +-------------+-----------------+---------------+------+ [9 rows x 4 columns] >>> >>> # Export the model to Core ML format >>> model.export_coreml('myModel.mlmodel') >>> >>> # Load the Core ML model >>> import coremltools >>> ml_model = coremltools.models.MLModel('myModel.mlmodel') >>> >>> # Prepare the first image of reference data for consumption >>> # by the Core ML model >>> import PIL >>> image = tc.image_analysis.resize(data['image'][0], *reversed(model.input_image_shape)) >>> image = PIL.Image.fromarray(image.pixel_data) >>> >>> # Calculate distances using the Core ML model >>> ml_model.predict(data={'image': image}) {'distance': array([ 0., 28.453125, 24.96875 ])} """ import numpy as _np from copy import deepcopy from turicreate._deps.minimal_package import _minimal_package_import_check _cmt = _minimal_package_import_check("coremltools") from coremltools.models import ( datatypes as _datatypes, neural_network as _neural_network, ) from turicreate.toolkits import _coreml_utils # Get the reference data from the model proxy = self.similarity_model.__proxy__ reference_data = _np.array( _tc.extensions._nearest_neighbors._nn_get_reference_data(proxy)) num_examples, embedding_size = reference_data.shape output_name = "distance" output_features = [(output_name, _datatypes.Array(num_examples))] if self.model != "VisionFeaturePrint_Scene": # Get the Core ML spec for the feature extractor ptModel = _pre_trained_models.IMAGE_MODELS[self.model]() feature_extractor = _image_feature_extractor.TensorFlowFeatureExtractor( ptModel) feature_extractor_spec = feature_extractor.get_coreml_model( ).get_spec() input_name = feature_extractor.coreml_data_layer input_features = [(input_name, _datatypes.Array(*(self.input_image_shape)))] # Convert the neuralNetworkClassifier to a neuralNetwork layers = deepcopy( feature_extractor_spec.neuralNetworkClassifier.layers) for l in layers: feature_extractor_spec.neuralNetwork.layers.append(l) builder = _neural_network.NeuralNetworkBuilder( input_features, output_features, spec=feature_extractor_spec) feature_layer = feature_extractor.coreml_feature_layer else: # self.model == VisionFeaturePrint_Scene # Create a pipleline that contains a VisionFeaturePrint followed by a # neural network. BGR_VALUE = _cmt.proto.FeatureTypes_pb2.ImageFeatureType.ColorSpace.Value( "BGR") DOUBLE_ARRAY_VALUE = _cmt.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value( "DOUBLE") INPUT_IMAGE_SHAPE = 299 top_spec = _cmt.proto.Model_pb2.Model() top_spec.specificationVersion = 3 desc = top_spec.description input = desc.input.add() input.name = self.feature input.type.imageType.width = INPUT_IMAGE_SHAPE input.type.imageType.height = INPUT_IMAGE_SHAPE input.type.imageType.colorSpace = BGR_VALUE output = desc.output.add() output.name = output_name output.type.multiArrayType.shape.append(num_examples) output.type.multiArrayType.dataType = DOUBLE_ARRAY_VALUE # VisionFeaturePrint extractor pipeline = top_spec.pipeline scene_print = pipeline.models.add() scene_print.specificationVersion = 3 scene_print.visionFeaturePrint.scene.version = 1 input = scene_print.description.input.add() input.name = self.feature input.type.imageType.width = 299 input.type.imageType.height = 299 input.type.imageType.colorSpace = BGR_VALUE feature_layer = "VisionFeaturePrint_Scene_output" output = scene_print.description.output.add() output.name = feature_layer output.type.multiArrayType.dataType = DOUBLE_ARRAY_VALUE output.type.multiArrayType.shape.append(2048) # Neural network builder input_features = [(feature_layer, _datatypes.Array(2048))] builder = _neural_network.NeuralNetworkBuilder( input_features, output_features) # To add the nearest neighbors model we add calculation of the euclidean # distance between the newly extracted query features (denoted by the vector u) # and each extracted reference feature (denoted by the rows of matrix V). # Calculation of sqrt((v_i-u)^2) = sqrt(v_i^2 - 2v_i*u + u^2) ensues. V = reference_data v_squared = (V * V).sum(axis=1) builder.add_inner_product( "v^2-2vu", W=-2 * V, b=v_squared, has_bias=True, input_channels=embedding_size, output_channels=num_examples, input_name=feature_layer, output_name="v^2-2vu", ) builder.add_unary( "element_wise-u^2", mode="power", alpha=2, input_name=feature_layer, output_name="element_wise-u^2", ) # Produce a vector of length num_examples with all values equal to u^2 builder.add_inner_product( "u^2", W=_np.ones((embedding_size, num_examples)), b=None, has_bias=False, input_channels=embedding_size, output_channels=num_examples, input_name="element_wise-u^2", output_name="u^2", ) builder.add_elementwise( "v^2-2vu+u^2", mode="ADD", input_names=["v^2-2vu", "u^2"], output_name="v^2-2vu+u^2", ) # v^2-2vu+u^2=(v-u)^2 is non-negative but some computations on GPU may result in # small negative values. Apply RELU so we don't take the square root of negative values. builder.add_activation("relu", non_linearity="RELU", input_name="v^2-2vu+u^2", output_name="relu") builder.add_unary("sqrt", mode="sqrt", input_name="relu", output_name=output_name) # Finalize model if self.model != "VisionFeaturePrint_Scene": builder.set_input([input_name], [self.input_image_shape]) builder.set_output([output_name], [(num_examples, )]) _cmt.models.utils.rename_feature(builder.spec, input_name, self.feature) builder.set_pre_processing_parameters( image_input_names=self.feature) mlmodel = _cmt.models.MLModel(builder.spec) else: top_spec.pipeline.models.extend([builder.spec]) mlmodel = _cmt.models.MLModel(top_spec) # Add metadata model_type = "image similarity" mlmodel.short_description = _coreml_utils._mlmodel_short_description( model_type) mlmodel.input_description[self.feature] = u"Input image" mlmodel.output_description[ output_name] = u"Distances between the input and reference images" model_metadata = { "model": self.model, "num_examples": str(self.num_examples), } user_defined_metadata = model_metadata.update( _coreml_utils._get_tc_version_info()) _coreml_utils._set_model_metadata( mlmodel, self.__class__.__name__, user_defined_metadata, version=ImageSimilarityModel._PYTHON_IMAGE_SIMILARITY_VERSION, ) mlmodel.save(filename)
def export_coreml( self, filename, include_non_maximum_suppression=True, iou_threshold=None, confidence_threshold=None, ): """ Save the model in Core ML format. The Core ML model takes an image of fixed size as input and produces two output arrays: `confidence` and `coordinates`. The first one, `confidence` is an `N`-by-`C` array, where `N` is the number of instances predicted and `C` is the number of classes. The number `N` is fixed and will include many low-confidence predictions. The instances are not sorted by confidence, so the first one will generally not have the highest confidence (unlike in `predict`). Also unlike the `predict` function, the instances have not undergone what is called `non-maximum suppression`, which means there could be several instances close in location and size that have all discovered the same object instance. Confidences do not need to sum to 1 over the classes; any remaining probability is implied as confidence there is no object instance present at all at the given coordinates. The classes appear in the array alphabetically sorted. The second array `coordinates` is of size `N`-by-4, where the first dimension `N` again represents instances and corresponds to the `confidence` array. The second dimension represents `x`, `y`, `width`, `height`, in that order. The values are represented in relative coordinates, so (0.5, 0.5) represents the center of the image and (1, 1) the bottom right corner. You will need to multiply the relative values with the original image size before you resized it to the fixed input size to get pixel-value coordinates similar to `predict`. See Also -------- save Parameters ---------- filename : string The path of the file where we want to save the Core ML model. include_non_maximum_suppression : bool Non-maximum suppression is only available in iOS 12+. A boolean parameter to indicate whether the Core ML model should be saved with built-in non-maximum suppression or not. This parameter is set to True by default. iou_threshold : float Threshold value for non-maximum suppression. Non-maximum suppression prevents multiple bounding boxes appearing over a single object. This threshold, set between 0 and 1, controls how aggressive this suppression is. A value of 1 means no maximum suppression will occur, while a value of 0 will maximally suppress neighboring boxes around a prediction. confidence_threshold : float Only return predictions above this level of confidence. The threshold can range from 0 to 1. Examples -------- >>> model.export_coreml('one_shot.mlmodel') """ from turicreate.toolkits import _coreml_utils additional_user_defined_metadata = _coreml_utils._get_tc_version_info() short_description = _coreml_utils._mlmodel_short_description( "Object Detector") options = { "include_non_maximum_suppression": include_non_maximum_suppression, } options["version"] = self._PYTHON_ONE_SHOT_OBJECT_DETECTOR_VERSION if confidence_threshold is not None: options["confidence_threshold"] = confidence_threshold if iou_threshold is not None: options["iou_threshold"] = iou_threshold additional_user_defined_metadata = _coreml_utils._get_tc_version_info() short_description = _coreml_utils._mlmodel_short_description( "One Shot Object Detector") self.__proxy__["detector"].__proxy__.export_to_coreml( filename, short_description, additional_user_defined_metadata, options)
def export_coreml(self, filename): """ Save the model in Core ML format. The Core ML model takes an image of fixed size as input and produces two output arrays: `confidence` and `coordinates`. The first one, `confidence` is an `N`-by-`C` array, where `N` is the number of instances predicted and `C` is the number of classes. The number `N` is fixed and will include many low-confidence predictions. The instances are not sorted by confidence, so the first one will generally not have the highest confidence (unlike in `predict`). Also unlike the `predict` function, the instances have not undergone what is called `non-maximum suppression`, which means there could be several instances close in location and size that have all discovered the same object instance. Confidences do not need to sum to 1 over the classes; any remaining probability is implied as confidence there is no object instance present at all at the given coordinates. The classes appear in the array alphabetically sorted. The second array `coordinates` is of size `N`-by-4, where the first dimension `N` again represents instances and corresponds to the `confidence` array. The second dimension represents `x`, `y`, `width`, `height`, in that order. The values are represented in relative coordinates, so (0.5, 0.5) represents the center of the image and (1, 1) the bottom right corner. You will need to multiply the relative values with the original image size before you resized it to the fixed input size to get pixel-value coordinates similar to `predict`. See Also -------- save Examples -------- >>> model.export_coreml('detector.mlmodel') """ import mxnet as _mx from .._mxnet_to_coreml import _mxnet_converter import coremltools from coremltools.models import datatypes, neural_network preds_per_box = 5 + self.num_classes num_anchors = len(self.anchors) num_classes = self.num_classes batch_size = 1 image_shape = (batch_size,) + tuple(self.input_image_shape) s_image_uint8 = _mx.sym.Variable(self.feature, shape=image_shape, dtype=_np.float32) s_image = s_image_uint8 / 255 # Swap a maxpool+slice in mxnet to a coreml natively supported layer from copy import copy net = copy(self._model) net._children = copy(self._model._children) from ._model import _SpecialDarknetMaxpoolBlock op = _SpecialDarknetMaxpoolBlock(name='pool5') # Make sure we are removing the right layers assert (self._model[23].name == 'pool5' and self._model[24].name == 'specialcrop5') del net._children[24] net._children[23] = op s_ymap = net(s_image) mod = _mx.mod.Module(symbol=s_ymap, label_names=None, data_names=[self.feature]) mod.bind(for_training=False, data_shapes=[(self.feature, image_shape)]) # Copy over params from net mod.init_params() arg_params, aux_params = mod.get_params() net_params = net.collect_params() new_arg_params = {} for k, param in arg_params.items(): new_arg_params[k] = net_params[k].data(net_params[k].list_ctx()[0]) new_aux_params = {} for k, param in aux_params.items(): new_aux_params[k] = net_params[k].data(net_params[k].list_ctx()[0]) mod.set_params(new_arg_params, new_aux_params) input_names = [self.feature] input_dims = [list(self.input_image_shape)] input_types = [datatypes.Array(*dim) for dim in input_dims] input_features = zip(input_names, input_types) num_spatial = self._grid_shape[0] * self._grid_shape[1] output_names = [ 'confidence', 'coordinates', ] output_dims = [ (num_anchors * num_spatial, num_classes), (num_anchors * num_spatial, 4), ] output_types = [datatypes.Array(*dim) for dim in output_dims] output_features = zip(output_names, output_types) mode = None builder = neural_network.NeuralNetworkBuilder(input_features, output_features, mode) _mxnet_converter.convert(mod, mode=None, input_shape={self.feature: image_shape}, builder=builder, verbose=False) prefix = '__tc__internal__' # (1, B, C+5, S*S) builder.add_reshape(name=prefix + 'ymap_sp_pre', target_shape=[batch_size, num_anchors, preds_per_box, num_spatial], mode=0, input_name='conv8_fwd_output', output_name=prefix + 'ymap_sp_pre') # (1, C+5, B, S*S) builder.add_permute(name=prefix + 'ymap_sp', dim=[0, 2, 1, 3], input_name=prefix + 'ymap_sp_pre', output_name=prefix + 'ymap_sp') # POSITION: X/Y # (1, 2, B, S*S) builder.add_slice(name=prefix + 'raw_rel_xy_sp', axis='channel', start_index=0, end_index=2, stride=1, input_name=prefix + 'ymap_sp', output_name=prefix + 'raw_rel_xy_sp') # (1, 2, B, S*S) builder.add_activation(name=prefix + 'rel_xy_sp', non_linearity='SIGMOID', input_name=prefix + 'raw_rel_xy_sp', output_name=prefix + 'rel_xy_sp') # (1, 2, B*H*W, 1) builder.add_reshape(name=prefix + 'rel_xy', target_shape=[batch_size, 2, num_anchors * num_spatial, 1], mode=0, input_name=prefix + 'rel_xy_sp', output_name=prefix + 'rel_xy') c_xy = _np.array(_np.meshgrid(_np.arange(self._grid_shape[1]), _np.arange(self._grid_shape[0])), dtype=_np.float32) c_xy_reshaped = (_np.tile(c_xy[:, _np.newaxis], (num_anchors, 1, 1)) .reshape(2, -1))[_np.newaxis, ..., _np.newaxis] # (1, 2, B*H*W, 1) builder.add_load_constant(prefix + 'constant_xy', constant_value=c_xy_reshaped, shape=c_xy_reshaped.shape[1:], output_name=prefix + 'constant_xy') # (1, 2, B*H*W, 1) builder.add_elementwise(name=prefix + 'xy', mode='ADD', input_names=[prefix + 'constant_xy', prefix + 'rel_xy'], output_name=prefix + 'xy') # SHAPE: WIDTH/HEIGHT # (1, 2, B, S*S) builder.add_slice(name=prefix + 'raw_rel_wh_sp', axis='channel', start_index=2, end_index=4, stride=1, input_name=prefix + 'ymap_sp', output_name=prefix + 'raw_rel_wh_sp') # (1, 2, B, S*S) builder.add_unary(name=prefix + 'rel_wh_sp', mode='exp', input_name=prefix + 'raw_rel_wh_sp', output_name=prefix + 'rel_wh_sp') # (1, 2*B, S, S) builder.add_reshape(name=prefix + 'rel_wh', target_shape=[batch_size, 2 * num_anchors] + list(self._grid_shape), mode=0, input_name=prefix + 'rel_wh_sp', output_name=prefix + 'rel_wh') np_anchors = _np.asarray(self.anchors, dtype=_np.float32).T anchors_0 = _np.tile(np_anchors.reshape([2 * num_anchors, 1, 1]), self._grid_shape) # (1, 2*B, S, S) builder.add_load_constant(name=prefix + 'c_anchors', constant_value=anchors_0, shape=anchors_0.shape, output_name=prefix + 'c_anchors') # (1, 2*B, S, S) builder.add_elementwise(name=prefix + 'wh_pre', mode='MULTIPLY', input_names=[prefix + 'c_anchors', prefix + 'rel_wh'], output_name=prefix + 'wh_pre') # (1, 2, B*H*W, 1) builder.add_reshape(name=prefix + 'wh', target_shape=[1, 2, num_anchors * num_spatial, 1], mode=0, input_name=prefix + 'wh_pre', output_name=prefix + 'wh') # (1, 4, B*H*W, 1) builder.add_elementwise(name=prefix + 'boxes_out_transposed', mode='CONCAT', input_names=[prefix + 'xy', prefix + 'wh'], output_name=prefix + 'boxes_out_transposed') # (1, B*H*W, 4, 1) builder.add_permute(name=prefix + 'boxes_out', dim=[0, 2, 1, 3], input_name=prefix + 'boxes_out_transposed', output_name=prefix + 'boxes_out') scale = _np.zeros((num_anchors * num_spatial, 4, 1)) scale[:, 0::2] = 1.0 / self._grid_shape[1] scale[:, 1::2] = 1.0 / self._grid_shape[0] # (1, B*H*W, 4, 1) builder.add_scale(name='coordinates', W=scale, b=0, has_bias=False, shape_scale=(num_anchors * num_spatial, 4, 1), input_name=prefix + 'boxes_out', output_name='coordinates') # CLASS PROBABILITIES AND OBJECT CONFIDENCE # (1, C, B, H*W) builder.add_slice(name=prefix + 'scores_sp', axis='channel', start_index=5, end_index=preds_per_box, stride=1, input_name=prefix + 'ymap_sp', output_name=prefix + 'scores_sp') # (1, C, B, H*W) builder.add_softmax(name=prefix + 'probs_sp', input_name=prefix + 'scores_sp', output_name=prefix + 'probs_sp') # (1, 1, B, H*W) builder.add_slice(name=prefix + 'logit_conf_sp', axis='channel', start_index=4, end_index=5, stride=1, input_name=prefix + 'ymap_sp', output_name=prefix + 'logit_conf_sp') # (1, 1, B, H*W) builder.add_activation(name=prefix + 'conf_sp', non_linearity='SIGMOID', input_name=prefix + 'logit_conf_sp', output_name=prefix + 'conf_sp') # (1, C, B, H*W) if num_classes > 1: conf = prefix + 'conf_tiled_sp' builder.add_elementwise(name=prefix + 'conf_tiled_sp', mode='CONCAT', input_names=[prefix + 'conf_sp'] * num_classes, output_name=conf) else: conf = prefix + 'conf_sp' # (1, C, B, H*W) builder.add_elementwise(name=prefix + 'confprobs_sp', mode='MULTIPLY', input_names=[conf, prefix + 'probs_sp'], output_name=prefix + 'confprobs_sp') # (1, C, B*H*W, 1) builder.add_reshape(name=prefix + 'confprobs_transposed', target_shape=[1, num_classes, num_anchors * num_spatial, 1], mode=0, input_name=prefix + 'confprobs_sp', output_name=prefix + 'confprobs_transposed') # (1, B*H*W, C, 1) builder.add_permute(name='confidence', dim=[0, 2, 1, 3], input_name=prefix + 'confprobs_transposed', output_name='confidence') _mxnet_converter._set_input_output_layers(builder, input_names, output_names) builder.set_input(input_names, input_dims) builder.set_output(output_names, output_dims) builder.set_pre_processing_parameters(image_input_names=self.feature) mlmodel = coremltools.models.MLModel(builder.spec) model_type = 'object detector (%s)' % self.model mlmodel.short_description = _coreml_utils._mlmodel_short_description(model_type) mlmodel.input_description[self.feature] = 'Input image' mlmodel.output_description['confidence'] = \ u'Boxes \xd7 Class confidence (see user-defined metadata "classes")' mlmodel.output_description['coordinates'] = \ u'Boxes \xd7 [x, y, width, height] (relative to image size)' _coreml_utils._set_model_metadata(mlmodel, self.__class__.__name__, { 'model': self.model, 'max_iterations': str(self.max_iterations), 'training_iterations': str(self.training_iterations), 'non_maximum_suppression_threshold': str(self.non_maximum_suppression_threshold), 'feature': self.feature, 'annotations': self.annotations, 'classes': ','.join(self.classes), }, version=ObjectDetector._PYTHON_OBJECT_DETECTOR_VERSION) mlmodel.save(filename)
def export_coreml(self, filename): """ Save the model in Core ML format. See Also -------- save Examples -------- >>> model.export_coreml('./myModel.mlmodel') """ import coremltools from coremltools.proto.FeatureTypes_pb2 import ArrayFeatureType prob_name = self.target + 'Probability' def get_custom_model_spec(): from coremltools.models.neural_network import NeuralNetworkBuilder from coremltools.models.datatypes import Array, Dictionary, String input_name = 'output1' input_length = self._feature_extractor.output_length builder = NeuralNetworkBuilder( [(input_name, Array(input_length, ))], [(prob_name, Dictionary(String))], 'classifier') input_name, output_name = input_name, 0 for i, cur_layer in enumerate( self._custom_classifier.export_weights()): W = cur_layer['weight'] nC, nB = W.shape Wb = cur_layer['bias'] builder.add_inner_product(name="inner_product_" + str(i), W=W, b=Wb, input_channels=nB, output_channels=nC, has_bias=True, input_name=str(input_name), output_name='inner_product_' + str(output_name)) if cur_layer['act']: builder.add_activation("activation" + str(i), 'RELU', 'inner_product_' + str(output_name), str(output_name)) input_name = i output_name = i + 1 last_output = builder.spec.neuralNetworkClassifier.layers[ -1].output[0] builder.add_softmax('softmax', last_output, self.target) builder.set_class_labels(self.classes, predicted_feature_name=self.target) builder.set_input([input_name], [(input_length, )]) builder.set_output([self.target], [(self.num_classes, )]) return builder.spec top_level_spec = coremltools.proto.Model_pb2.Model() top_level_spec.specificationVersion = 3 # Set input desc = top_level_spec.description input = desc.input.add() input.name = self.feature input.type.multiArrayType.dataType = ArrayFeatureType.ArrayDataType.Value( 'FLOAT32') input.type.multiArrayType.shape.append(15600) # Set outputs prob_output = desc.output.add() prob_output.name = prob_name label_output = desc.output.add() label_output.name = self.target desc.predictedFeatureName = self.target desc.predictedProbabilitiesName = prob_name if type(self.classes[0]) == int: # Class labels are ints prob_output.type.dictionaryType.int64KeyType.MergeFromString(b'') label_output.type.int64Type.MergeFromString(b'') else: # Class are strings prob_output.type.dictionaryType.stringKeyType.MergeFromString(b'') label_output.type.stringType.MergeFromString(b'') # Set metadata user_metadata = desc.metadata.userDefined user_metadata['sampleRate'] = str( self._feature_extractor.input_sample_rate) pipeline = top_level_spec.pipelineClassifier.pipeline # Add the preprocessing model preprocessing_model = pipeline.models.add() preprocessing_model.customModel.className = 'TCSoundClassifierPreprocessing' preprocessing_model.specificationVersion = 3 preprocessing_input = preprocessing_model.description.input.add() preprocessing_input.CopyFrom(input) preprocessed_output = preprocessing_model.description.output.add() preprocessed_output.name = 'preprocessed_data' preprocessed_output.type.multiArrayType.dataType = ArrayFeatureType.ArrayDataType.Value( 'DOUBLE') preprocessed_output.type.multiArrayType.shape.append(1) preprocessed_output.type.multiArrayType.shape.append(96) preprocessed_output.type.multiArrayType.shape.append(64) # Add the feature extractor, updating its input name feature_extractor_spec = self._feature_extractor.get_spec() pipeline.models.add().CopyFrom(feature_extractor_spec) pipeline.models[-1].description.input[ 0].name = preprocessed_output.name pipeline.models[-1].neuralNetwork.layers[0].input[ 0] = preprocessed_output.name # Add the custom neural network pipeline.models.add().CopyFrom(get_custom_model_spec()) # Set key type for the probability dictionary prob_output_type = pipeline.models[-1].description.output[ 0].type.dictionaryType if type(self.classes[0]) == int: prob_output_type.int64KeyType.MergeFromString(b'') else: # String labels prob_output_type.stringKeyType.MergeFromString(b'') mlmodel = coremltools.models.MLModel(top_level_spec) model_type = 'sound classifier' mlmodel.short_description = _coreml_utils._mlmodel_short_description( model_type) mlmodel.input_description[self.feature] = u'Input audio features' mlmodel.output_description[prob_name] = 'Prediction probabilities' mlmodel.output_description[ self.target] = 'Class label of top prediction' model_metadata = { 'target': self.target, 'feature': self.feature, } user_defined_metadata = model_metadata.update( _coreml_utils._get_tc_version_info()) _coreml_utils._set_model_metadata( mlmodel, self.__class__.__name__, user_defined_metadata, version=SoundClassifier._PYTHON_SOUND_CLASSIFIER_VERSION) mlmodel.save(filename)