def get_gelu_pattern1():
"""
y = x * (0.5 * (tanh(((.0447)x^3 + x ) * sqrt(2/pi)) + 1))
[...] -----> pow (3) ----> mul (.044715) ---> add -----> mul (sqrt(2/pi)) ---> tanh ----> add (1) ----> mul (0.5) -----> mul ---> [...]
| ^ ^
| | |
|------------------------------------------------------------------------------------------------------------------------
"""
@mb.program(input_specs=[mb.TensorSpec(shape=([get_new_symbol(), get_new_symbol(), get_new_symbol()])), ])
def gelu_to_detect_1(x):
# MIL operation takes named inputs (instead of positional inputs).
# Here `name` argument is MANDATORY.
pow = mb.pow(x=x, y=3.0, name="pow")
mul_1 = mb.mul(x=0.044714998453855515, y=pow, name="mul_1")
add = mb.add(x=x, y=mul_1, name="add")
mul_2 = mb.mul(x=0.7978845834732056, y=add, name="mul_2")
tanh = mb.tanh(x=mul_2, name="tanh")
add_1 = mb.add(x=1.0, y=tanh, name="add_1")
mul = mb.mul(x=0.5, y=add_1, name="mul")
mul_3 = mb.mul(x=mul, y=x, name="mul_3")
return mul_3
return gelu_to_detect_1
def get_gelu_pattern2():
"""
y = (0.5 * x) * (tanh(((.0447)x^3 + x ) * sqrt(2/pi)) + 1)
---------------------------------------------------------------------------------------------------------
^ |
| V
[...] -----> mul(0.5) pow (3) ----> mul (.044715) ---> add -----> mul (sqrt(2/pi)) ---> tanh ----> add (1) -----> mul ---> [...]
| ^ ^
| | |
|------------------------------------------------------------
"""
@mb.program(input_specs=[mb.TensorSpec(shape=([get_new_symbol(), get_new_symbol(), get_new_symbol()])), ])
def gelu_to_detect_2(x):
pow = mb.pow(x=x, y=3.0, name="pow")
mul_1 = mb.mul(x=0.044714998453855515, y=pow, name="mul_1")
add = mb.add(x=x, y=mul_1, name="add")
mul_2 = mb.mul(x=0.7978845834732056, y=add, name="mul_2")
tanh = mb.tanh(x=mul_2, name="tanh")
add_1 = mb.add(x=1.0, y=tanh, name="add_1")
mul = mb.mul(x=0.5, y=x, name="mul")
mul_3 = mb.mul(x=mul, y=add_1, name="mul_3")
return mul_3
return gelu_to_detect_2
def test_tutorial():
from coremltools.converters.mil import Builder as mb
@mb.program(
input_specs=[mb.TensorSpec(shape=(1, 100, 100, 3)),]
)
def prog(x):
x = mb.relu(x=x, name="relu")
x = mb.transpose(x=x, perm=[0, 3, 1, 2], name="transpose")
x = mb.reduce_mean(x=x, axes=[2, 3], keep_dims=False, name="reduce")
x = mb.log(x=x, name="log")
y = mb.add(x=1, y=2)
return x
print("prog:\n", prog)
# Convert and verify
from coremltools.converters.mil.converter import _convert
from coremltools import models
proto = _convert(prog, convert_from="mil")
model = models.MLModel(proto)
# running predict() is only supported on macOS
if ct.utils._is_macos():
prediction = model.predict(
{"x": np.random.rand(1, 100, 100, 3).astype(np.float32),}
)
assert len(prediction) == 1
def test_rename_feature_mlprogram(self):
@mb.program(input_specs=[mb.TensorSpec(shape=(3, ))])
def linear_prog(input):
W = np.ones((10, 3), dtype=np.float)
out = mb.linear(x=input, weight=W, name="output")
return out
model = coremltools.convert(linear_prog, convert_to='mlprogram')
spec = model.get_spec()
input_name = spec.description.input[0].name
output_name = spec.description.output[0].name
# rename input
rename_feature(spec, input_name, "new_input_name")
self.assertEqual(spec.description.input[0].name, "new_input_name")
model = coremltools.models.MLModel(spec, weights_dir=model.weights_dir)
out = model.predict({"new_input_name": np.array([1.0, 2.0,
3.0])})[output_name]
self.assertEqual(out.shape, (10, ))
self.assertEqual(out[0], 6.0)
# rename output
rename_feature(spec, output_name, "new_output_name")
self.assertEqual(spec.description.output[0].name, "new_output_name")
model = coremltools.models.MLModel(spec, weights_dir=model.weights_dir)
out = model.predict({"new_input_name":
np.array([1.0, 2.0, 3.0])})["new_output_name"]
self.assertEqual(out.shape, (10, ))
self.assertEqual(out[1], 6.0)
def pattern_to_detect(conv_transpose, transpose, sub):
"""
Wrapper to create 8 patterns to detect for conciseness.
"""
@mb.program(input_specs=[mb.TensorSpec(shape=arbitrary_input)])
def conv_bias_pattern(x):
if not conv_transpose:
conv = mb.conv(x=x,
weight=arbitrary_weight,
pad_type="valid",
name="conv")
else:
conv = mb.conv_transpose(x=x,
weight=arbitrary_weight,
pad_type="valid",
name="conv")
if transpose:
transpose_layer = mb.transpose(x=conv,
perm=arbitrary_perm,
name="transpose")
if sub:
add_or_sub = mb.sub(x=transpose_layer if transpose else conv,
y=arbitrary_scalar,
name="add_or_sub")
else:
add_or_sub = mb.add(x=transpose_layer if transpose else conv,
y=arbitrary_scalar,
name="add_or_sub")
return add_or_sub
return conv_bias_pattern
def setup_class(self):
# define an mlprogram, which has weights
@mb.program(input_specs=[mb.TensorSpec(shape=(4, 5000))])
def linear_prog(input):
W = mb.const(val=np.random.rand(100, 5000), name="const_W")
out = mb.linear(x=input, weight=W, name="output")
return out
# define another mlprogram, which does not have weights
@mb.program(input_specs=[mb.TensorSpec(shape=(4, 5, 2))])
def relu_prog(input):
out = mb.relu(x=input, name="output")
return out
# convert and save model on disk
self.mlmodel = coremltools.convert(linear_prog, convert_to="mlprogram")
self.mlpackage_path = tempfile.mkdtemp(suffix=utils._MLPACKAGE_EXTENSION)
self.mlmodel.save(self.mlpackage_path)
self.mlmodel_no_weights = coremltools.convert(relu_prog, convert_to="mlprogram")
def get_prelu_pattern():
"""
y = a * relu(-1 * x) + relu(x)
when x is rank 4, and "a" is of shape (1, C, 1, 1) or (C, 1, 1),
this is equivalent to prelu with alpha = -a.flatten(),
"""
@mb.program(input_specs=[mb.TensorSpec(shape=([get_new_symbol(), get_new_symbol(),
get_new_symbol(), get_new_symbol()])), ])
def prelu_pattern(x):
return _prelu_pattern(x)
return prelu_pattern
def get_prelu_pattern():
"""
x1 = transpose(perm=(0,2,3,1))(x)
y = a * relu(-1 * x1) + relu(x1)
when x is rank 4, and "a" is of shape (C,) or (1, C) or (1,1,C) or (1,1,1,C),
this is equivalent to prelu with alpha = -a.flatten(), followed by a transpose
with perm (0,2,3,1)
"""
@mb.program(input_specs=[mb.TensorSpec(shape=([get_new_symbol(), get_new_symbol(),
get_new_symbol(), get_new_symbol()])), ])
def prelu_pattern(x):
# perm value can be anything, it will be checked in "is_var_constraint_satisifed" method
x = mb.transpose(x=x, perm=[0,1,2,3], name="transpose")
return _prelu_pattern(x)
return prelu_pattern
...
Result:
%3 = conv(%1)
...
"""
arbitrary_cin = 5
arbitrary_cout = 8
arbitrary_scalar = 5
np.random.seed()
arbitrary_input = (3, arbitrary_cin, 224, 224)
arbitrary_weight = np.random.rand(arbitrary_cout, arbitrary_cin, 10, 10)
@mb.program(input_specs=[mb.TensorSpec(shape=arbitrary_input)])
def conv_scale_mul(x):
conv = mb.conv(x=x, weight=arbitrary_weight, pad_type="valid", name="conv")
mul = mb.mul(x=conv, y=arbitrary_scalar, name="scale")
return mul
@mb.program(input_specs=[mb.TensorSpec(shape=arbitrary_input)])
def conv_transpose_scale_mul(x):
conv = mb.conv_transpose(x=x,
weight=arbitrary_weight,
pad_type="valid",
name="conv")
mul = mb.mul(x=conv, y=arbitrary_scalar, name="scale")
return mul
:param mode: reduce mode
"""
if reduce_op is None:
return False
if reduce_op.op_type != mode:
return False
if reduce_op.keep_dims.val is False:
return False
if reduce_op.axes is None or reduce_op.axes.val is None:
return False
return True
if os.getenv("ENABLE_EXPERIMENTAL_PASSES") == "1":
@mb.program(input_specs=[mb.TensorSpec(shape=shape)])
def instancenorm_or_layernorm(x):
"""
Identify the pattern:
y = gamma * (x - mean) / sqrt(variance + epsilon) + beta
y = x * [gamma * rsqrt(variance + eps)] + (beta - mean * [gamma * rsqrt(variance + eps)])
x --> main_reduce --> sub --> square --> reduce_mean_2 --> add(epsilon) --> rsqrt
| | ^ |
| | | V
|----------------------- mul (gamma)
| | |
| | --------|---------
| | | |
# -*- coding: utf-8 -*-
# Copyright (c) 2020, Apple Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-3-clause license that can be
# found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
import os
from coremltools.converters.mil import Builder as mb
from coremltools.converters.mil.mil.passes.helper import _check_var_scalar_value
from coremltools.converters.mil.experimental.passes.generic_pass_infrastructure import register_generic_pass
@mb.program(input_specs=[
mb.TensorSpec(shape=([1, 1024, 4096])),
])
def gelu_to_detect_1(x):
# MIL operation takes named inputs (instead of positional inputs).
# Here `name` argument is MANDATORY.
pow = mb.pow(x=x, y=3.0, name="pow")
mul_1 = mb.mul(x=0.044714998453855515, y=pow, name="mul_1")
add = mb.add(x=x, y=mul_1, name="add")
mul_2 = mb.mul(x=0.7978845834732056, y=add, name="mul_2")
tanh = mb.tanh(x=mul_2, name="tanh")
add_1 = mb.add(x=1.0, y=tanh, name="add_1")
mul = mb.mul(x=0.5, y=add_1, name="mul")
mul_3 = mb.mul(x=mul, y=x, name="mul_3")
return mul_3
