def test_convert_nn_spec_to_half_precision(self): # simple network with quantization layer input_features = [('data', datatypes.Array(3))] output_features = [('out', datatypes.Array(3))] builder = NeuralNetworkBuilder(input_features, output_features) weights = np.random.uniform(-0.5, 0.5, (3, 3)) builder.add_inner_product(name='inner_product', W=weights, b=None, input_channels=3, output_channels=3, has_bias=False, input_name='data', output_name='out') model = MLModel(builder.spec) spec = convert_neural_network_spec_weights_to_fp16(model.get_spec()) self.assertIsNotNone(spec) # simple network without quantization layer input_features = [('data', datatypes.Array(3))] output_features = [('out', datatypes.Array(3))] builder = NeuralNetworkBuilder(input_features, output_features) builder.add_lrn(name='lrn', input_name='data', output_name='out', alpha=2, beta=3, local_size=1, k=8) model = MLModel(builder.spec) spec = convert_neural_network_spec_weights_to_fp16(model.get_spec()) self.assertIsNotNone(spec)
def test_model_api(self): model = MLModel(self.spec) self.assertIsNotNone(model) model.author = 'Test author' self.assertEquals(model.author, 'Test author') self.assertEquals(model.get_spec().description.metadata.author, 'Test author') model.license = 'Test license' self.assertEquals(model.license, 'Test license') self.assertEquals(model.get_spec().description.metadata.license, 'Test license') model.short_description = 'Test model' self.assertEquals(model.short_description, 'Test model') self.assertEquals(model.get_spec().description.metadata.shortDescription, 'Test model') model.input_description['feature_1'] = 'This is feature 1' self.assertEquals(model.input_description['feature_1'], 'This is feature 1') model.output_description['output'] = 'This is output' self.assertEquals(model.output_description['output'], 'This is output') filename = tempfile.mktemp(suffix = '.mlmodel') model.save(filename) loaded_model = MLModel(filename) self.assertEquals(model.author, 'Test author') self.assertEquals(model.license, 'Test license') # self.assertEquals(model.short_description, 'Test model') self.assertEquals(model.input_description['feature_1'], 'This is feature 1') self.assertEquals(model.output_description['output'], 'This is output')
def test_downgrade_specification_version(self): # manually set a invalid specification version self.spec.specificationVersion = -1 model = MLModel(self.spec) assert model.get_spec().specificationVersion == 1 # manually set a high specification version self.spec.specificationVersion = 4 filename = tempfile.mktemp(suffix='.mlmodel') save_spec(self.spec, filename, auto_set_specification_version=True) model = MLModel(filename) assert model.get_spec().specificationVersion == 1 # simple neural network with only spec 1 layer input_features = [('data', datatypes.Array(3))] output_features = [('out', datatypes.Array(3))] builder = NeuralNetworkBuilder(input_features, output_features) builder.add_activation('relu', 'RELU', 'data', 'out') # set a high specification version builder.spec.specificationVersion = 3 model = MLModel(builder.spec) filename = tempfile.mktemp(suffix='.mlmodel') model.save(filename) # load the model back model = MLModel(filename) assert model.get_spec().specificationVersion == 1 # test save without automatic set specification version self.spec.specificationVersion = 3 filename = tempfile.mktemp(suffix='.mlmodel') save_spec(self.spec, filename, auto_set_specification_version=False) model = MLModel(filename) # the specification version should be original assert model.get_spec().specificationVersion == 3
def _convert_to_image_input(proto, inputs): tmp_model = MLModel(proto) for input_type in inputs: if isinstance(input_type, ImageType): if input_type.color_layout == "G": gray_bias = input_type.bias red_bias, green_bias, blue_bias = 0.0, 0.0, 0.0 elif input_type.color_layout == "RGB": gray_bias = 0.0 red_bias, green_bias, blue_bias = input_type.bias elif input_type.color_layout == "BGR": gray_bias = 0.0 blue_bias, green_bias, red_bias = input_type.bias tmp_model = neural_network.utils.make_image_input( tmp_model, input_type.name, is_bgr=input_type.color_layout == "BGR", image_format="NCHW" if input_type.channel_first else "NHWC", red_bias=red_bias, green_bias=green_bias, blue_bias=blue_bias, gray_bias=gray_bias, scale=input_type.scale, ) return tmp_model.get_spec()
def inference(cls, architecture, model_path, image_path): # TODO import numpy as np from coremltools.models._infer_shapes_nn_mlmodel import infer_shapes if cls.sanity_check(architecture): func = TestKit.preprocess_func['coreml'][architecture] img = func(image_path) # load model model = MLModel(model_path) spec = model.get_spec() # TODO: Multiple inputs input_name = spec.description.input[0].name # TODO: Multiple outputs output_name = spec.description.output[0].name # inference input_data = img coreml_input = {input_name: img} coreml_output = model.predict(coreml_input) prob = coreml_output[output_name].values() prob = np.array(prob).squeeze() return prob else: return None
def test_nn_classifier_util_file(self): input_features = [("data", datatypes.Array(3))] output_features = [("out", datatypes.Array(3))] builder = NeuralNetworkBuilder( input_features, output_features, disable_rank5_shape_mapping=True ) builder.add_activation("linear", "LINEAR", "data", "out") spec = builder.spec mlmodel = MLModel(spec) class_labels = ["a", "b", "c"] with tempfile.NamedTemporaryFile(mode="w", suffix=".txt") as f: f.write("\n".join(class_labels)) f.flush() mlmodel = make_nn_classifier( mlmodel, class_labels=f.name, predicted_feature_name="out_confidence", predicted_probabilities_output="out", ) out_dict = mlmodel.predict({"data": np.array([4.0, 5.5, 6.0])}, useCPUOnly=True) self.assertEqual(out_dict["out_confidence"], "c") self.assertEqual( mlmodel.get_spec().WhichOneof("Type"), "neuralNetworkClassifier" )
def _convert_to_image_input(proto, inputs, skip_model_load=False): tmp_model = MLModel(proto, skip_model_load=skip_model_load) for input_type in inputs: if isinstance(input_type, ImageType): if input_type.color_layout in (ColorLayout.GRAYSCALE, ColorLayout.GRAYSCALE_FLOAT16): gray_bias = input_type.bias red_bias, green_bias, blue_bias = 0.0, 0.0, 0.0 elif input_type.color_layout == ColorLayout.RGB: gray_bias = 0.0 red_bias, green_bias, blue_bias = input_type.bias elif input_type.color_layout == ColorLayout.BGR: gray_bias = 0.0 blue_bias, green_bias, red_bias = input_type.bias tmp_model = neural_network.utils.make_image_input( tmp_model, input_type.name, is_bgr=input_type.color_layout == ColorLayout.BGR, image_format="NCHW" if input_type.channel_first else "NHWC", red_bias=red_bias, green_bias=green_bias, blue_bias=blue_bias, gray_bias=gray_bias, scale=input_type.scale, ) return tmp_model.get_spec()
def _load_model(self, model_network_path): """Load a Coreml model from disk Parameters ---------- model_network_path: str Path where the model network path is (mlmodel file) Returns ------- model: A coreml model """ from coremltools.models import MLModel if os.path.isfile(model_network_path): # load the model network loaded_model_ml = MLModel(model_network_path) # convert to Model_pb2.Model loaded_model_pb = loaded_model_ml.get_spec() self.weight_loaded = True print("Network file [{}] is loaded successfully.".format( model_network_path)) else: print("Warning: Weights File [{}] is not found.".format( model_network_path)) return loaded_model_pb
def load_model(cls, path: str): """ Deserializes a VGSL model from a CoreML file. Args: path (str): CoreML file Returns: A TorchVGSLModel instance. Raises: KrakenInvalidModelException if the model data is invalid (not a string, protobuf file, or without appropriate metadata). FileNotFoundError if the path doesn't point to a file. """ try: mlmodel = MLModel(path) except TypeError as e: raise KrakenInvalidModelException(str(e)) except DecodeError as e: raise KrakenInvalidModelException('Failure parsing model protobuf: {}'.format(str(e))) if 'vgsl' not in mlmodel.user_defined_metadata: raise KrakenInvalidModelException('No VGSL spec in model metadata') vgsl_spec = mlmodel.user_defined_metadata['vgsl'] nn = cls(vgsl_spec) for name, layer in nn.nn.named_children(): layer.deserialize(name, mlmodel.get_spec()) if 'codec' in mlmodel.user_defined_metadata: nn.add_codec(PytorchCodec(json.loads(mlmodel.user_defined_metadata['codec']))) nn.user_metadata = {'accuracy': [], 'seg_type': 'bbox', 'one_channel_mode': '1', 'model_type': None, 'hyper_params': {}} # type: dict[str, str] if 'kraken_meta' in mlmodel.user_defined_metadata: nn.user_metadata.update(json.loads(mlmodel.user_defined_metadata['kraken_meta'])) return nn
def test_nn_set_training_input(self): builder = self.create_base_builder() builder.set_mean_squared_error_loss(name="mse", input_feature=("output", datatypes.Array(3))) builder.set_adam_optimizer( AdamParams(lr=1e-2, batch=10, beta1=0.9, beta2=0.999, eps=1e-8)) builder.set_epochs(20, allowed_set=[10, 20, 30]) model_path = os.path.join(self.model_dir, "updatable_creation.mlmodel") print(model_path) save_spec(builder.spec, model_path) mlmodel = MLModel(model_path) self.assertTrue(mlmodel is not None) spec = mlmodel.get_spec() self.assertEqual(spec.description.trainingInput[0].name, "input") self.assertEqual( spec.description.trainingInput[0].type.WhichOneof("Type"), "multiArrayType") self.assertEqual(spec.description.trainingInput[1].name, "output_true") self.assertEqual( spec.description.trainingInput[1].type.WhichOneof("Type"), "multiArrayType")
def test_nn_set_training_input(self): builder = self.create_base_builder() builder.set_mean_squared_error_loss(name='mse', input='output', target='target') builder.set_adam_optimizer( AdamParams(lr=1e-2, batch=10, beta1=0.9, beta2=0.999, eps=1e-8)) builder.set_epochs(20, allowed_set=[10, 20, 30]) builder.set_training_input([('input', datatypes.Array(3)), ('target', 'Double')]) model_path = os.path.join(self.model_dir, 'updatable_creation.mlmodel') print(model_path) save_spec(builder.spec, model_path) mlmodel = MLModel(model_path) self.assertTrue(mlmodel is not None) spec = mlmodel.get_spec() self.assertEqual(spec.description.trainingInput[0].name, 'input') self.assertEqual( spec.description.trainingInput[0].type.WhichOneof('Type'), 'multiArrayType') self.assertEqual(spec.description.trainingInput[1].name, 'target') self.assertEqual( spec.description.trainingInput[1].type.WhichOneof('Type'), 'doubleType')
def test_model_api(self): model = MLModel(self.spec) self.assertIsNotNone(model) model.author = "Test author" self.assertEqual(model.author, "Test author") self.assertEqual(model.get_spec().description.metadata.author, "Test author") model.license = "Test license" self.assertEqual(model.license, "Test license") self.assertEqual(model.get_spec().description.metadata.license, "Test license") model.short_description = "Test model" self.assertEqual(model.short_description, "Test model") self.assertEqual( model.get_spec().description.metadata.shortDescription, "Test model") model.version = "1.3" self.assertEqual(model.version, "1.3") self.assertEqual(model.get_spec().description.metadata.versionString, "1.3") model.input_description["feature_1"] = "This is feature 1" self.assertEqual(model.input_description["feature_1"], "This is feature 1") model.output_description["output"] = "This is output" self.assertEqual(model.output_description["output"], "This is output") package = tempfile.TemporaryDirectory(suffix=".mlpackage") package.cleanup() model.save(package.name) loaded_model = MLModel(package.name) self.assertEqual(model.author, "Test author") self.assertEqual(model.license, "Test license") self.assertEqual(model.short_description, "Test model") self.assertEqual(model.input_description["feature_1"], "This is feature 1") self.assertEqual(model.output_description["output"], "This is output") # cleanup MLModelTest._remove_path(package.name)
def test_nn_builder_with_training_features(self): input_features = [('input', datatypes.Array(3))] output_features = [('output', None)] training_features = [('input', datatypes.Array(3)), ('target', datatypes.Double)] builder = NeuralNetworkBuilder(input_features, output_features, disable_rank5_shape_mapping=True, training_features=training_features) W1 = _np.random.uniform(-0.5, 0.5, (3, 3)) W2 = _np.random.uniform(-0.5, 0.5, (3, 3)) builder.add_inner_product(name='ip1', W=W1, b=None, input_channels=3, output_channels=3, has_bias=False, input_name='input', output_name='hidden') builder.add_inner_product(name='ip2', W=W2, b=None, input_channels=3, output_channels=3, has_bias=False, input_name='hidden', output_name='output') builder.make_updatable(['ip1', 'ip2']) # or a dict for weightParams builder.set_mean_squared_error_loss(name='mse', input='output', target='target') builder.set_adam_optimizer( AdamParams(lr=1e-2, batch=10, beta1=0.9, beta2=0.999, eps=1e-8)) builder.set_epochs(20, allowed_set=[10, 20, 30]) builder.set_training_input([('input', datatypes.Array(3)), ('target', 'Double')]) model_path = os.path.join(self.model_dir, 'updatable_creation.mlmodel') print(model_path) save_spec(builder.spec, model_path) mlmodel = MLModel(model_path) self.assertTrue(mlmodel is not None) spec = mlmodel.get_spec() self.assertEqual(spec.description.trainingInput[0].name, 'input') self.assertEqual( spec.description.trainingInput[0].type.WhichOneof('Type'), 'multiArrayType') self.assertEqual(spec.description.trainingInput[1].name, 'target') self.assertEqual( spec.description.trainingInput[1].type.WhichOneof('Type'), 'doubleType')
def test_path_to_mlmodel_to_spec_to_mlmodel(self): """ load an mlmodel from disk, convert it to spec, and then convert the spec back to mlmodel """ mlmodel_from_disk = MLModel(self.mlpackage_path) spec = mlmodel_from_disk.get_spec() mlmodel_from_spec = MLModel(spec, weights_dir=mlmodel_from_disk.weights_dir) self._test_mlmodel_correctness(mlmodel_from_spec)
def _convert_to_classifier(proto, classifier_config, skip_model_load=False): tmp_model = MLModel(proto, skip_model_load=skip_model_load) tmp_model = neural_network.utils.make_nn_classifier( tmp_model, classifier_config.class_labels, classifier_config.predicted_feature_name, classifier_config.predicted_probabilities_output, ) return tmp_model.get_spec()
def test_model_api(self): model = MLModel(self.spec) self.assertIsNotNone(model) model.author = "Test author" self.assertEqual(model.author, "Test author") self.assertEqual(model.get_spec().description.metadata.author, "Test author") model.license = "Test license" self.assertEqual(model.license, "Test license") self.assertEqual(model.get_spec().description.metadata.license, "Test license") model.short_description = "Test model" self.assertEqual(model.short_description, "Test model") self.assertEqual( model.get_spec().description.metadata.shortDescription, "Test model") model.version = "1.3" self.assertEqual(model.version, "1.3") self.assertEqual(model.get_spec().description.metadata.versionString, "1.3") model.input_description["feature_1"] = "This is feature 1" self.assertEqual(model.input_description["feature_1"], "This is feature 1") model.output_description["output"] = "This is output" self.assertEqual(model.output_description["output"], "This is output") filename = tempfile.mktemp(suffix=".mlmodel") model.save(filename) loaded_model = MLModel(filename) self.assertEqual(model.author, "Test author") self.assertEqual(model.license, "Test license") # self.assertEqual(model.short_description, 'Test model') self.assertEqual(model.input_description["feature_1"], "This is feature 1") self.assertEqual(model.output_description["output"], "This is output")
def test_model_api(self): model = MLModel(self.spec) assert model is not None model.author = "Test author" assert model.author == "Test author" assert model.get_spec().description.metadata.author == "Test author" model.license = "Test license" assert model.license == "Test license" assert model.get_spec().description.metadata.license == "Test license" model.short_description = "Test model" assert model.short_description == "Test model" assert model.get_spec( ).description.metadata.shortDescription == "Test model" model.version = "1.3" assert model.version == "1.3" assert model.get_spec().description.metadata.versionString == "1.3" model.input_description["feature_1"] = "This is feature 1" assert model.input_description["feature_1"] == "This is feature 1" model.output_description["output"] = "This is output" assert model.output_description["output"] == "This is output" package = tempfile.TemporaryDirectory(suffix=".mlpackage") package.cleanup() model.save(package.name) loaded_model = MLModel(package.name) assert model.author == "Test author" assert model.license == "Test license" assert model.short_description == "Test model" assert model.input_description["feature_1"] == "This is feature 1" assert model.output_description["output"] == "This is output" # cleanup _remove_path(package.name)
def test_nn_builder_with_training_features(self): input_features = [("input", datatypes.Array(3))] output_features = [("output", datatypes.Array(3))] builder = NeuralNetworkBuilder(input_features, output_features) W1 = _np.random.uniform(-0.5, 0.5, (3, 3)) W2 = _np.random.uniform(-0.5, 0.5, (3, 3)) builder.add_inner_product( name="ip1", W=W1, b=None, input_channels=3, output_channels=3, has_bias=False, input_name="input", output_name="hidden", ) builder.add_inner_product( name="ip2", W=W2, b=None, input_channels=3, output_channels=3, has_bias=False, input_name="hidden", output_name="output", ) builder.make_updatable(["ip1", "ip2"]) # or a dict for weightParams builder.set_mean_squared_error_loss(name="mse", input_feature=("output", datatypes.Array(3))) builder.set_adam_optimizer( AdamParams(lr=1e-2, batch=10, beta1=0.9, beta2=0.999, eps=1e-8)) builder.set_epochs(20, allowed_set=[10, 20, 30]) model_path = os.path.join(self.model_dir, "updatable_creation.mlmodel") print(model_path) save_spec(builder.spec, model_path) mlmodel = MLModel(model_path) self.assertTrue(mlmodel is not None) spec = mlmodel.get_spec() self.assertEqual(spec.description.trainingInput[0].name, "input") self.assertEqual( spec.description.trainingInput[0].type.WhichOneof("Type"), "multiArrayType") self.assertEqual(spec.description.trainingInput[1].name, "output_true") self.assertEqual( spec.description.trainingInput[1].type.WhichOneof("Type"), "multiArrayType")
def test_downgrade_specification_version(self): # manually set a invalid specification version self.spec.specificationVersion = -1 model = MLModel(self.spec) if model.get_spec().specificationVersion != 1: raise AssertionError # manually set a high specification version self.spec.specificationVersion = 4 filename = tempfile.mktemp(suffix=".mlmodel") save_spec(self.spec, filename, auto_set_specification_version=True) model = MLModel(filename) if model.get_spec().specificationVersion != 1: raise AssertionError # simple neural network with only spec 1 layer input_features = [("data", datatypes.Array(3))] output_features = [("out", datatypes.Array(3))] builder = NeuralNetworkBuilder(input_features, output_features) builder.add_activation("relu", "RELU", "data", "out") # set a high specification version builder.spec.specificationVersion = 3 model = MLModel(builder.spec) filename = tempfile.mktemp(suffix=".mlmodel") model.save(filename) # load the model back model = MLModel(filename) if model.get_spec().specificationVersion != 1: raise AssertionError # test save without automatic set specification version self.spec.specificationVersion = 3 filename = tempfile.mktemp(suffix=".mlmodel") save_spec(self.spec, filename, auto_set_specification_version=False) model = MLModel(filename) # the specification version should be original if model.get_spec().specificationVersion != 3: raise AssertionError
def inference(cls, architecture, model_path, image_path): # TODO from PIL import Image import numpy as np from coremltools.models._infer_shapes_nn_mlmodel import infer_shapes if cls.sanity_check(architecture): func = TestKit.preprocess_func['coreml'][architecture] import inspect funcstr = inspect.getsource(func) if len(funcstr.split(',')) == 3: size = int(funcstr.split('path,')[1].split(')')[0]) else: size = int(funcstr.split('path,')[1].split(',')[0]) img = Image.open(image_path) img = img.resize((size, size)) # load model model = MLModel(model_path) spec = model.get_spec() # TODO: Multiple inputs input_name = spec.description.input[0].name # TODO: Multiple outputs output_name = spec.description.output[0].name # inference input_data = img coreml_input = {input_name: img} coreml_output = model.predict(coreml_input) prob = coreml_output[output_name] if isinstance(prob, dict): prob = coreml_output[output_name].values() prob = np.array(prob).squeeze() return prob else: return None
def test_nn_classifier_util(self): input_features = [("data", datatypes.Array(3))] output_features = [("out", datatypes.Array(3))] builder = NeuralNetworkBuilder(input_features, output_features, disable_rank5_shape_mapping=True) builder.add_activation("linear", "LINEAR", "data", "out") spec = builder.spec mlmodel = MLModel(spec, compute_units=ComputeUnit.CPU_ONLY) mlmodel = make_nn_classifier( mlmodel, class_labels=["a", "b", "c"], predicted_feature_name="out_confidence", predicted_probabilities_output="out", ) out_dict = mlmodel.predict({"data": np.array([4.0, 5.5, 6.0])}) self.assertEqual(out_dict["out_confidence"], "c") self.assertEqual(mlmodel.get_spec().WhichOneof("Type"), "neuralNetworkClassifier")
def load_model(cls, path: str): """ Deserializes a VGSL model from a CoreML file. Args: path (str): CoreML file """ mlmodel = MLModel(path) if 'vgsl' not in mlmodel.user_defined_metadata: raise ValueError('No VGSL spec in model metadata') vgsl_spec = mlmodel.user_defined_metadata['vgsl'] nn = cls(vgsl_spec) for name, layer in nn.nn.named_children(): layer.deserialize(name, mlmodel.get_spec()) if 'codec' in mlmodel.user_defined_metadata: nn.add_codec( PytorchCodec(json.loads( mlmodel.user_defined_metadata['codec']))) return nn
def load_model(cls, path: str): """ Deserializes a VGSL model from a CoreML file. Args: path (str): CoreML file """ mlmodel = MLModel(path) if 'vgsl' not in mlmodel.user_defined_metadata: raise ValueError('No VGSL spec in model metadata') vgsl_spec = mlmodel.user_defined_metadata['vgsl'] nn = cls(vgsl_spec) for name, layer in nn.nn.named_children(): layer.deserialize(name, mlmodel.get_spec()) if 'codec' in mlmodel.user_defined_metadata: nn.add_codec(PytorchCodec(json.loads(mlmodel.user_defined_metadata['codec']))) if 'kraken_meta' in mlmodel.user_defined_metadata: nn.user_metadata = json.loads(mlmodel.user_defined_metadata['kraken_meta']) return nn
def test_rename_output_nn_classifier(self): input_features = [("data", datatypes.Array(3))] output_features = [("out", datatypes.Array(3))] builder = NeuralNetworkBuilder( input_features, output_features, disable_rank5_shape_mapping=True ) builder.add_activation("linear", "LINEAR", "data", "out") spec = builder.spec mlmodel = MLModel(spec) class_labels = ["a", "b", "c"] mlmodel = make_nn_classifier(mlmodel, class_labels=["a", "b", "c"]) # rename output spec = mlmodel.get_spec() rename_feature(spec, "out", "new_out_name") mlmodel = MLModel(spec) out_dict = mlmodel.predict({"data": np.array([4.0, 5.5, 6.0])}, useCPUOnly=True) self.assertEqual(out_dict["classLabel"], "c") self.assertTrue("new_out_name" in out_dict) self.assertTrue(isinstance(out_dict["new_out_name"], dict))
def test_updatable_model_creation_mse_sgd(self): builder = self.create_base_builder() builder.set_mean_squared_error_loss(name='mse', input='output', target='target') builder.set_sgd_optimizer(SgdParams(lr=1e-2, batch=10, momentum=0.0)) builder.set_epochs(20) model_path = os.path.join(self.model_dir, 'updatable_creation.mlmodel') print(model_path) save_spec(builder.spec, model_path) mlmodel = MLModel(model_path) self.assertTrue(mlmodel is not None) spec = mlmodel.get_spec() self.assertTrue(spec.isUpdatable) self.assertTrue(spec.neuralNetwork.layers[0].isUpdatable) self.assertTrue( spec.neuralNetwork.layers[0].innerProduct.weights.isUpdatable) self.assertTrue(spec.neuralNetwork.layers[1].isUpdatable) self.assertTrue( spec.neuralNetwork.layers[1].innerProduct.weights.isUpdatable) self.assertTrue(spec.neuralNetwork.updateParams.lossLayers[0]. categoricalCrossEntropyLossLayer is not None) self.assertTrue( spec.neuralNetwork.updateParams.optimizer.sgdOptimizer is not None) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. learningRate.defaultValue, 1e-2, atol=1e-4)) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. miniBatchSize.defaultValue, 10, atol=1e-4)) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. momentum.defaultValue, 0, atol=1e-8)) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.epochs.defaultValue, 20, atol=1e-4)) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. learningRate.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. learningRate.range.maxValue == 1) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. miniBatchSize.set.values == [10]) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. momentum.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. momentum.range.maxValue == 1)
from coremltools.models import MLModel m = MLModel("openface.mlmodel") print(m.get_spec())
def test_updatable_model_creation_ce_sgd(self): builder = self.create_base_builder() builder.add_softmax(name="softmax", input_name="output", output_name="softmax_output") builder.set_categorical_cross_entropy_loss(name="cross_entropy", input="softmax_output") builder.set_sgd_optimizer(SgdParams(lr=1e-2, batch=10, momentum=0.0)) builder.set_epochs(20, allowed_set=[10, 20, 30, 40]) model_path = os.path.join(self.model_dir, "updatable_creation.mlmodel") print(model_path) save_spec(builder.spec, model_path) mlmodel = MLModel(model_path) self.assertTrue(mlmodel is not None) spec = mlmodel.get_spec() self.assertTrue(spec.isUpdatable) self.assertTrue(spec.neuralNetwork.layers[0].isUpdatable) self.assertTrue( spec.neuralNetwork.layers[0].innerProduct.weights.isUpdatable) self.assertTrue(spec.neuralNetwork.layers[1].isUpdatable) self.assertTrue( spec.neuralNetwork.layers[1].innerProduct.weights.isUpdatable) self.assertTrue(spec.neuralNetwork.updateParams.lossLayers[0]. categoricalCrossEntropyLossLayer is not None) self.assertTrue( spec.neuralNetwork.updateParams.optimizer.sgdOptimizer is not None) self.assertTrue( _np.isclose( spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. learningRate.defaultValue, 1e-2, atol=1e-4, )) self.assertTrue( _np.isclose( spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. miniBatchSize.defaultValue, 10, atol=1e-4, )) self.assertTrue( _np.isclose( spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. momentum.defaultValue, 0, atol=1e-8, )) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.epochs.defaultValue, 20, atol=1e-4)) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. learningRate.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. learningRate.range.maxValue == 1) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. miniBatchSize.set.values == [10]) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. momentum.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.sgdOptimizer. momentum.range.maxValue == 1)
class VGGishFeatureExtractor(object): name = 'VGGish' output_length = 12288 input_sample_rate = SAMPLE_RATE @staticmethod def _preprocess_data(audio_data, verbose=True): ''' Preprocess each example, breaking it up into frames. Returns two numpy arrays: preprocessed frame and their indexes ''' from .vggish_input import waveform_to_examples last_progress_update = _time.time() progress_header_printed = False # Can't run as a ".apply(...)" due to numba.jit decorator issue: # https://github.com/apple/turicreate/issues/1216 preprocessed_data, audio_data_index = [], [] for i, audio_dict in enumerate(audio_data): scaled_data = audio_dict['data'] / 32768.0 data = waveform_to_examples(scaled_data, audio_dict['sample_rate']) for j in data: preprocessed_data.append([j]) audio_data_index.append(i) # If `verbose` is set, print an progress update about every 20s if verbose and _time.time() - last_progress_update >= 20: if not progress_header_printed: print("Preprocessing audio data -") progress_header_printed = True print("Preprocessed {} of {} examples".format( i, len(audio_data))) last_progress_update = _time.time() if progress_header_printed: print("Preprocessed {} of {} examples\n".format( len(audio_data), len(audio_data))) return _np.asarray(preprocessed_data), audio_data_index def __init__(self): vggish_model_file = VGGish() self.mac_ver = _mac_ver() if self.mac_ver < (10, 14): # Use TensorFlow/Keras import turicreate.toolkits._tf_utils as _utils self.gpu_policy = _utils.TensorFlowGPUPolicy() self.gpu_policy.start() model_path = vggish_model_file.get_model_path(format='tensorflow') self.vggish_model = _keras.models.load_model(model_path) else: # Use Core ML model_path = vggish_model_file.get_model_path(format='coreml') self.vggish_model = MLModel(model_path) def __del__(self): if self.mac_ver < (10, 14): self.gpu_policy.stop() def _extract_features(self, preprocessed_data, verbose=True): """ Parameters ---------- preprocessed_data : SArray Returns ------- numpy array containing the deep features """ last_progress_update = _time.time() progress_header_printed = False deep_features = _tc.SArrayBuilder(_np.ndarray) if _mac_ver() < (10, 14): # Use TensorFlow/Keras # Transpose data from channel first to channel last preprocessed_data = _np.transpose(preprocessed_data, (0, 2, 3, 1)) for i, cur_example in enumerate(preprocessed_data): y = self.vggish_model.predict([[cur_example]]) deep_features.append(y[0]) # If `verbose` is set, print an progress update about every 20s if verbose and _time.time() - last_progress_update >= 20: if not progress_header_printed: print("Extracting deep features -") progress_header_printed = True print("Extracted {} of {}".format(i, len(preprocessed_data))) last_progress_update = _time.time() if progress_header_printed: print("Extracted {} of {}\n".format(len(preprocessed_data), len(preprocessed_data))) else: # Use Core ML for i, cur_example in enumerate(preprocessed_data): for cur_frame in cur_example: x = {'input1': _np.asarray([cur_frame])} y = self.vggish_model.predict(x) deep_features.append(y['output1']) # If `verbose` is set, print an progress update about every 20s if verbose and _time.time() - last_progress_update >= 20: if not progress_header_printed: print("Extracting deep features -") progress_header_printed = True print("Extracted {} of {}".format(i, len(preprocessed_data))) last_progress_update = _time.time() if progress_header_printed: print("Extracted {} of {}\n".format(len(preprocessed_data), len(preprocessed_data))) return deep_features.close() def get_deep_features(self, audio_data, verbose): ''' Performs both audio preprocessing and VGGish deep feature extraction. ''' preprocessed_data, row_ids = self._preprocess_data(audio_data, verbose) deep_features = self._extract_features(preprocessed_data, verbose) output = _tc.SFrame({ 'deep features': deep_features, 'row id': row_ids }) output = output.unstack('deep features') max_row_id = len(audio_data) missing_ids = set(range(max_row_id)) - set(output['row id'].unique()) if len(missing_ids) != 0: empty_rows = _tc.SFrame({ 'List of deep features': [[] for _ in range(len(missing_ids))], 'row id': missing_ids }) output = output.append(empty_rows) output = output.sort('row id') return output['List of deep features'] def get_spec(self): """ Return the Core ML spec """ if _mac_ver() >= (10, 14): return self.vggish_model.get_spec() else: vggish_model_file = VGGish() coreml_model_path = vggish_model_file.get_model_path( format='coreml') return MLModel(coreml_model_path).get_spec()
def test_updatable_model_creation_mse_adam(self): builder = self.create_base_builder() builder.set_mean_squared_error_loss(name="mse", input_feature=("output", datatypes.Array(3))) builder.set_adam_optimizer( AdamParams(lr=1e-2, batch=10, beta1=0.9, beta2=0.999, eps=1e-8)) builder.set_epochs(20, allowed_set=[10, 20, 30]) model_path = os.path.join(self.model_dir, "updatable_creation.mlmodel") print(model_path) save_spec(builder.spec, model_path) mlmodel = MLModel(model_path) self.assertTrue(mlmodel is not None) spec = mlmodel.get_spec() self.assertTrue(spec.isUpdatable) self.assertTrue(spec.neuralNetwork.layers[0].isUpdatable) self.assertTrue( spec.neuralNetwork.layers[0].innerProduct.weights.isUpdatable) self.assertTrue(spec.neuralNetwork.layers[1].isUpdatable) self.assertTrue( spec.neuralNetwork.layers[1].innerProduct.weights.isUpdatable) self.assertTrue(spec.neuralNetwork.updateParams.lossLayers[0]. categoricalCrossEntropyLossLayer is not None) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.adamOptimizer is not None) self.assertTrue( _np.isclose( spec.neuralNetwork.updateParams.optimizer.adamOptimizer. learningRate.defaultValue, 1e-2, atol=1e-4, )) self.assertTrue( _np.isclose( spec.neuralNetwork.updateParams.optimizer.adamOptimizer. miniBatchSize.defaultValue, 10, atol=1e-4, )) self.assertTrue( _np.isclose( spec.neuralNetwork.updateParams.optimizer.adamOptimizer.beta1. defaultValue, 0.9, atol=1e-4, )) self.assertTrue( _np.isclose( spec.neuralNetwork.updateParams.optimizer.adamOptimizer.beta2. defaultValue, 0.999, atol=1e-4, )) self.assertTrue( _np.isclose( spec.neuralNetwork.updateParams.optimizer.adamOptimizer.eps. defaultValue, 1e-8, atol=1e-8, )) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.epochs.defaultValue, 20, atol=1e-4)) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.learningRate.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.learningRate.range.maxValue == 1) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.miniBatchSize.set.values == [10]) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.beta1.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.beta1.range.maxValue == 1) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.beta2.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.beta2.range.maxValue == 1) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.eps.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.eps.range.maxValue == 1) self.assertTrue( spec.neuralNetwork.updateParams.epochs.set.values == [10, 20, 30])
class VGGishFeatureExtractor(object): name = 'VGGish' output_length = 12288 @staticmethod def _preprocess_data(audio_data, verbose=True): ''' Preprocess each example, breaking it up into frames. Returns two numpy arrays: preprocessed frame and their indexes ''' from .vggish_input import waveform_to_examples last_progress_update = _time.time() progress_header_printed = False # Can't run as a ".apply(...)" due to numba.jit decorator issue: # https://github.com/apple/turicreate/issues/1216 preprocessed_data, audio_data_index = [], [] for i, audio_dict in enumerate(audio_data): scaled_data = audio_dict['data'] / 32768.0 data = waveform_to_examples(scaled_data, audio_dict['sample_rate']) for j in data: preprocessed_data.append([j]) audio_data_index.append(i) # If `verbose` is set, print an progress update about every 20s if verbose and _time.time() - last_progress_update >= 20: if not progress_header_printed: print("Preprocessing audio data -") progress_header_printed = True print("Preprocessed {} of {} examples".format( i, len(audio_data))) last_progress_update = _time.time() if progress_header_printed: print("Preprocessed {} of {} examples\n".format( len(audio_data), len(audio_data))) return _np.asarray(preprocessed_data), audio_data_index @staticmethod def _build_net(): net = nn.HybridSequential() net.add( nn.Conv2D(channels=64, kernel_size=(3, 3), in_channels=1, padding=(1, 1), prefix='vggish_conv0_')) net.add(nn.Activation('relu')) net.add(nn.MaxPool2D()) net.add( nn.Conv2D(channels=128, kernel_size=(3, 3), in_channels=64, padding=(1, 1), prefix='vggish_conv1_')) net.add(nn.Activation('relu')) net.add(nn.MaxPool2D()) net.add( nn.Conv2D(channels=256, kernel_size=(3, 3), in_channels=128, padding=(1, 1), prefix='vggish_conv2_')) net.add(nn.Activation('relu')) net.add( nn.Conv2D(channels=256, kernel_size=(3, 3), in_channels=256, padding=(1, 1), prefix='vggish_conv3_')) net.add(nn.Activation('relu')) net.add(nn.MaxPool2D()) net.add( nn.Conv2D(channels=512, kernel_size=(3, 3), in_channels=256, padding=(1, 1), prefix='vggish_conv4_')) net.add(nn.Activation('relu')) net.add( nn.Conv2D(channels=512, kernel_size=(3, 3), in_channels=512, padding=(1, 1), prefix='vggish_conv5_')) net.add(nn.Activation('relu')) net.add(nn.MaxPool2D()) net.add(Flatten_channel_last()) return net def __init__(self): vggish_model_file = VGGish() if _mac_ver() < (10, 14): # Use MXNet model_path = vggish_model_file.get_model_path(format='mxnet') self.vggish_model = VGGishFeatureExtractor._build_net() net_params = self.vggish_model.collect_params() self.ctx = _mxnet_utils.get_mxnet_context() net_params.load(model_path, ctx=self.ctx) else: # Use Core ML model_path = vggish_model_file.get_model_path(format='coreml') self.vggish_model = MLModel(model_path) def _extract_features(self, preprocessed_data, verbose=True): """ Parameters ---------- preprocessed_data : SArray Returns ------- numpy array containing the deep features """ last_progress_update = _time.time() progress_header_printed = False deep_features = _tc.SArrayBuilder(_np.ndarray) if _mac_ver() < (10, 14): # Use MXNet preprocessed_data = mx.nd.array(preprocessed_data) ctx_list = self.ctx if len(preprocessed_data) < len(ctx_list): ctx_list = ctx_list[:len(preprocessed_data)] batches = utils.split_and_load(preprocessed_data, ctx_list=ctx_list, even_split=False) for i, cur_batch in enumerate(batches): y = self.vggish_model.forward(cur_batch).asnumpy() for j in y: deep_features.append(j) # If `verbose` is set, print an progress update about every 20s if verbose and _time.time() - last_progress_update >= 20: if not progress_header_printed: print("Extracting deep features -") progress_header_printed = True print("Extracted {} of {} batches".format(i, len(batches))) last_progress_update = _time.time() if progress_header_printed: print("Extracted {} of {} batches\n".format( len(batches), len(batches))) else: # Use Core ML for i, cur_example in enumerate(preprocessed_data): for cur_frame in cur_example: x = {'input1': [cur_frame]} y = self.vggish_model.predict(x) deep_features.append(y['output1']) # If `verbose` is set, print an progress update about every 20s if verbose and _time.time() - last_progress_update >= 20: if not progress_header_printed: print("Extracting deep features -") progress_header_printed = True print("Extracted {} of {}".format(i, len(preprocessed_data))) last_progress_update = _time.time() if progress_header_printed: print("Extracted {} of {}\n".format(len(preprocessed_data), len(preprocessed_data))) return deep_features.close() def get_deep_features(self, audio_data, verbose): ''' Performs both audio preprocessing and VGGish deep feature extraction. ''' preprocessed_data, row_ids = self._preprocess_data(audio_data, verbose) deep_features = self._extract_features(preprocessed_data, verbose) output = _tc.SFrame({ 'deep features': deep_features, 'row id': row_ids }) output = output.unstack('deep features') output = output.sort('row id') return output['List of deep features'] def get_spec(self): """ Return the Core ML spec """ if _mac_ver() >= (10, 14): return self.vggish_model.get_spec() else: vggish_model_file = VGGish() coreml_model_path = vggish_model_file.get_model_path( format='coreml') return MLModel(coreml_model_path).get_spec()
def test_updatable_model_creation_ce_adam(self): builder = self.create_base_builder() builder.add_softmax(name='softmax', input_name='output', output_name='softmax_output') builder.set_categorical_cross_entropy_loss(name='cross_entropy', input='softmax_output', target='target') adam_params = AdamParams() adam_params.set_batch(value=10, allowed_set=[10, 20]) builder.set_adam_optimizer(adam_params) builder.set_epochs(20) model_path = os.path.join(self.model_dir, 'updatable_creation.mlmodel') print(model_path) save_spec(builder.spec, model_path) mlmodel = MLModel(model_path) self.assertTrue(mlmodel is not None) spec = mlmodel.get_spec() self.assertTrue(spec.isUpdatable) self.assertTrue(spec.neuralNetwork.layers[0].isUpdatable) self.assertTrue( spec.neuralNetwork.layers[0].innerProduct.weights.isUpdatable) self.assertTrue(spec.neuralNetwork.layers[1].isUpdatable) self.assertTrue( spec.neuralNetwork.layers[1].innerProduct.weights.isUpdatable) self.assertTrue(spec.neuralNetwork.updateParams.lossLayers[0]. categoricalCrossEntropyLossLayer is not None) self.assertTrue(spec.neuralNetwork.updateParams.optimizer.adamOptimizer is not None) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.learningRate.defaultValue, 1e-2, atol=1e-4)) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.miniBatchSize.defaultValue, 10, atol=1e-4)) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.beta1.defaultValue, 0.9, atol=1e-4)) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.beta2.defaultValue, 0.999, atol=1e-4)) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.eps.defaultValue, 1e-8, atol=1e-8)) self.assertTrue( _np.isclose(spec.neuralNetwork.updateParams.epochs.defaultValue, 20, atol=1e-4)) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.learningRate.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.learningRate.range.maxValue == 1) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.miniBatchSize.set.values == [10, 20]) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.beta1.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.beta1.range.maxValue == 1) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.beta2.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.beta2.range.maxValue == 1) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.eps.range.minValue == 0) self.assertTrue(spec.neuralNetwork.updateParams.optimizer. adamOptimizer.eps.range.maxValue == 1) self.assertTrue( spec.neuralNetwork.updateParams.epochs.set.values == [20])