def instancenorm_2(x):
"""
Identify the pattern:
y = (x - mean) / pow(variance + epsilon) * gamma + beta
This pattern corresponds to, should be fused as instance_norm.
All of the following must be satisty:
1) Input is rank 4 tensor
2) Reduce operates on spatial dimensions axes=[-2, -1], or axes=[-3, -2] (a
channel first to channel last transpose would be inserted in such case)
3) Gamma and beta are both shape (C,) after squeeze, where C is number of channels
|----> sub0 ----------| const (0.5)
| ^ | |
| | V V
x ---> main_reduce square --> mean1 --> add_eps ---> pow const_gamma const_beta
| | | | |
| V V V V
|----> sub1 --------------------------------------> real_div --> mul_gamma --> add_beta --> ...
"""
main_reduce = mb.reduce_mean(x=x, axes=[2, 3], keep_dims=True, name="main_reduce")
sub0 = mb.sub(x=x, y=main_reduce, name="sub0")
sub1 = mb.sub(x=x, y=main_reduce, name="sub1")
square = mb.square(x=sub0, name="square")
mean1 = mb.reduce_mean(x=square, axes=[2, 3], keep_dims=True, name="mean1")
add_epsilon = mb.add(x=mean1, y=1e-5, name="add_epsilon")
pow = mb.pow(x=add_epsilon, y=0.5, name="pow")
real_div = mb.real_div(x=sub1, y=pow, name="real_div")
mul_gamma = mb.mul(x=np.random.rand(1, 5, 1, 1), y=real_div, name="mul_gamma")
add_beta = mb.add(x=np.random.rand(1, 5, 1, 1), y=mul_gamma, name="add_beta")
return add_beta
def gelu_to_detect_2(x):
pow = mb.pow(x=x, y=3.0, name="pow")
mul_1 = mb.mul(x=0.044714998453855515, y=pow, name="mul_1")
add = mb.add(x=x, y=mul_1, name="add")
mul_2 = mb.mul(x=0.7978845834732056, y=add, name="mul_2")
tanh = mb.tanh(x=mul_2, name="tanh")
add_1 = mb.add(x=1.0, y=tanh, name="add_1")
mul = mb.mul(x=0.5, y=x, name="mul")
mul_3 = mb.mul(x=mul, y=add_1, name="mul_3")
return mul_3
def gelu_to_detect_1(x):
# MIL operation takes named inputs (instead of positional inputs).
# Here `name` argument is MANDATORY.
pow = mb.pow(x=x, y=3.0, name="pow")
mul_1 = mb.mul(x=0.044714998453855515, y=pow, name="mul_1")
add = mb.add(x=x, y=mul_1, name="add")
mul_2 = mb.mul(x=0.7978845834732056, y=add, name="mul_2")
tanh = mb.tanh(x=mul_2, name="tanh")
add_1 = mb.add(x=1.0, y=tanh, name="add_1")
mul = mb.mul(x=0.5, y=add_1, name="mul")
mul_3 = mb.mul(x=mul, y=x, name="mul_3")
return mul_3
